[ { "path": ".circleci/config.yml", "content": "version: 2.1\norbs:\n aws-s3: circleci/aws-s3@1.0.4\njobs:\n build-bionic:\n docker:\n - image: circleci/buildpack-deps:bionic\n steps:\n - run:\n name: Update & Upgrade\n command: sudo apt update && sudo apt upgrade -y\n - checkout\n - run:\n name: Install deps\n command: >\n sudo apt install -y gcc git python-minimal python2.7-dev\n libffi-dev libssl-dev make g++ libleveldb-dev librrd-dev\n libxslt1-dev libc-ares-dev libsnappy-dev python-pip\n - run:\n name: Install Python dev requirements\n command: sudo -H pip install -r requirements-dev.txt\n - run:\n name: Show Python Version\n command: /usr/bin/env python -V\n - run:\n name: Build package\n command: >\n platter build --virtualenv-version 16.7.9 -r requirements-web.txt\n --prebuild-script scripts/prebuild-script.sh\n - run:\n name: Rename files\n command: for file in dist/*; do mv \"$file\" \"${file%.tar.gz}.$CIRCLE_SHA1.bionic\"; done\n - persist_to_workspace:\n root: dist\n paths:\n - .\n\n build-xenial:\n docker:\n - image: circleci/buildpack-deps:xenial\n steps:\n - run:\n name: Update & Upgrade\n command: sudo apt update && sudo apt upgrade -y\n - checkout\n - run:\n name: Install deps\n command: >\n sudo apt install -y gcc git python-minimal python2.7-dev\n libffi-dev libssl-dev make g++ libleveldb-dev librrd-dev\n libxslt1-dev libc-ares-dev libsnappy-dev python-pip\n - run:\n name: Install Python dev requirements\n command: sudo -H pip install -r requirements-dev.txt\n - run:\n name: Show Python Version\n command: /usr/bin/env python -V\n - run:\n name: Build package\n command: >\n platter build --virtualenv-version 16.7.9 -r requirements-web.txt\n --prebuild-script scripts/prebuild-script.sh\n - run:\n name: Rename files\n command: for file in dist/*; do mv \"$file\" \"${file%.tar.gz}.$CIRCLE_SHA1.xenial\"; done\n - persist_to_workspace:\n root: dist\n paths:\n - .\n\n deploy:\n docker:\n - image: circleci/python:2.7\n steps:\n - run:\n name: Create workspace dir\n command: mkdir ~/workspace\n - attach_workspace:\n at: ~/workspace\n - run:\n name: Workspace contents\n command: find ~/workspace\n - aws-s3/sync:\n from: ~/workspace\n to: 's3://minemeld/'\n arguments: |\n --acl public-read\n\nworkflows:\n version: 2\n xenial:\n jobs:\n - build-xenial:\n filters:\n tags:\n only: '/.*/'\n - deploy:\n requires:\n - build-xenial\n filters:\n branches:\n ignore: '/.*/'\n tags:\n only: '/.*/'\n\n bionic:\n jobs:\n - build-bionic:\n filters:\n tags:\n only: '/.*/'\n - deploy:\n requires:\n - build-bionic\n filters:\n branches:\n ignore: '/.*/'\n tags:\n only: '/.*/'\n" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nenv/\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\n*.egg-info/\n.installed.cfg\n*.egg\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*,cover\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# PyCharm\n.idea/\n\n.vscode/\n\n_ares.c\n_ares.h" }, { "path": "AUTHORS", "content": "Luigi Mori \nKevin Steves \nPhil Da Silva \nKevin Stilwell \niDev \nJonas Eichinger \nEgon Kidmose \nDan James \nBen Carroll \nJohn Marion \n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n" }, { "path": "NOTICE", "content": "MineMeld Core\nCopyright 2016 Palo Alto Networks\n\nThis product includes software developed at\nPalo Alto Networks Inc. (http://www.paloaltonetworks.com/).\n" }, { "path": "README.md", "content": "# minemeld-core\n\nThis repo contains the code for the engine and the API of MineMeld, an extensible Threat Intelligence processing framework.\n\nFor details check the MineMeld [Wiki](https://github.com/PaloAltoNetworks/minemeld/wiki)\n" }, { "path": "docs/Makefile", "content": "# Makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line.\nSPHINXOPTS =\nSPHINXBUILD = sphinx-build\nPAPER =\nBUILDDIR = _build\n\n# User-friendly check for sphinx-build\nifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)\n$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)\nendif\n\n# Internal variables.\nPAPEROPT_a4 = -D latex_paper_size=a4\nPAPEROPT_letter = -D latex_paper_size=letter\nALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n# the i18n builder cannot share the environment and doctrees with the others\nI18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .\n\n.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext\n\nhelp:\n\t@echo \"Please use \\`make ' where is one of\"\n\t@echo \" html to make standalone HTML files\"\n\t@echo \" dirhtml to make HTML files named index.html in directories\"\n\t@echo \" singlehtml to make a single large HTML file\"\n\t@echo \" pickle to make pickle files\"\n\t@echo \" json to make JSON files\"\n\t@echo \" htmlhelp to make HTML files and a HTML help project\"\n\t@echo \" qthelp to make HTML files and a qthelp project\"\n\t@echo \" applehelp to make an Apple Help Book\"\n\t@echo \" devhelp to make HTML files and a Devhelp project\"\n\t@echo \" epub to make an epub\"\n\t@echo \" latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter\"\n\t@echo \" latexpdf to make LaTeX files and run them through pdflatex\"\n\t@echo \" latexpdfja to make LaTeX files and run them through platex/dvipdfmx\"\n\t@echo \" text to make text files\"\n\t@echo \" man to make manual pages\"\n\t@echo \" texinfo to make Texinfo files\"\n\t@echo \" info to make Texinfo files and run them through makeinfo\"\n\t@echo \" gettext to make PO message catalogs\"\n\t@echo \" changes to make an overview of all changed/added/deprecated items\"\n\t@echo \" xml to make Docutils-native XML files\"\n\t@echo \" pseudoxml to make pseudoxml-XML files for display purposes\"\n\t@echo \" linkcheck to check all external links for integrity\"\n\t@echo \" doctest to run all doctests embedded in the documentation (if enabled)\"\n\t@echo \" coverage to run coverage check of the documentation (if enabled)\"\n\nclean:\n\trm -rf $(BUILDDIR)/*\n\nhtml:\n\t$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/html.\"\n\ndirhtml:\n\t$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml\n\t@echo\n\t@echo \"Build finished. The HTML pages are in $(BUILDDIR)/dirhtml.\"\n\nsinglehtml:\n\t$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml\n\t@echo\n\t@echo \"Build finished. The HTML page is in $(BUILDDIR)/singlehtml.\"\n\npickle:\n\t$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle\n\t@echo\n\t@echo \"Build finished; now you can process the pickle files.\"\n\njson:\n\t$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json\n\t@echo\n\t@echo \"Build finished; now you can process the JSON files.\"\n\nhtmlhelp:\n\t$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp\n\t@echo\n\t@echo \"Build finished; now you can run HTML Help Workshop with the\" \\\n\t \".hhp project file in $(BUILDDIR)/htmlhelp.\"\n\nqthelp:\n\t$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp\n\t@echo\n\t@echo \"Build finished; now you can run \"qcollectiongenerator\" with the\" \\\n\t \".qhcp project file in $(BUILDDIR)/qthelp, like this:\"\n\t@echo \"# qcollectiongenerator $(BUILDDIR)/qthelp/minemeld.qhcp\"\n\t@echo \"To view the help file:\"\n\t@echo \"# assistant -collectionFile $(BUILDDIR)/qthelp/minemeld.qhc\"\n\napplehelp:\n\t$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp\n\t@echo\n\t@echo \"Build finished. The help book is in $(BUILDDIR)/applehelp.\"\n\t@echo \"N.B. You won't be able to view it unless you put it in\" \\\n\t \"~/Library/Documentation/Help or install it in your application\" \\\n\t \"bundle.\"\n\ndevhelp:\n\t$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp\n\t@echo\n\t@echo \"Build finished.\"\n\t@echo \"To view the help file:\"\n\t@echo \"# mkdir -p $$HOME/.local/share/devhelp/minemeld\"\n\t@echo \"# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/minemeld\"\n\t@echo \"# devhelp\"\n\nepub:\n\t$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub\n\t@echo\n\t@echo \"Build finished. The epub file is in $(BUILDDIR)/epub.\"\n\nlatex:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo\n\t@echo \"Build finished; the LaTeX files are in $(BUILDDIR)/latex.\"\n\t@echo \"Run \\`make' in that directory to run these through (pdf)latex\" \\\n\t \"(use \\`make latexpdf' here to do that automatically).\"\n\nlatexpdf:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through pdflatex...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\nlatexpdfja:\n\t$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex\n\t@echo \"Running LaTeX files through platex and dvipdfmx...\"\n\t$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja\n\t@echo \"pdflatex finished; the PDF files are in $(BUILDDIR)/latex.\"\n\ntext:\n\t$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text\n\t@echo\n\t@echo \"Build finished. The text files are in $(BUILDDIR)/text.\"\n\nman:\n\t$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man\n\t@echo\n\t@echo \"Build finished. The manual pages are in $(BUILDDIR)/man.\"\n\ntexinfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo\n\t@echo \"Build finished. The Texinfo files are in $(BUILDDIR)/texinfo.\"\n\t@echo \"Run \\`make' in that directory to run these through makeinfo\" \\\n\t \"(use \\`make info' here to do that automatically).\"\n\ninfo:\n\t$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo\n\t@echo \"Running Texinfo files through makeinfo...\"\n\tmake -C $(BUILDDIR)/texinfo info\n\t@echo \"makeinfo finished; the Info files are in $(BUILDDIR)/texinfo.\"\n\ngettext:\n\t$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale\n\t@echo\n\t@echo \"Build finished. The message catalogs are in $(BUILDDIR)/locale.\"\n\nchanges:\n\t$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes\n\t@echo\n\t@echo \"The overview file is in $(BUILDDIR)/changes.\"\n\nlinkcheck:\n\t$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck\n\t@echo\n\t@echo \"Link check complete; look for any errors in the above output \" \\\n\t \"or in $(BUILDDIR)/linkcheck/output.txt.\"\n\ndoctest:\n\t$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest\n\t@echo \"Testing of doctests in the sources finished, look at the \" \\\n\t \"results in $(BUILDDIR)/doctest/output.txt.\"\n\ncoverage:\n\t$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage\n\t@echo \"Testing of coverage in the sources finished, look at the \" \\\n\t \"results in $(BUILDDIR)/coverage/python.txt.\"\n\nxml:\n\t$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml\n\t@echo\n\t@echo \"Build finished. The XML files are in $(BUILDDIR)/xml.\"\n\npseudoxml:\n\t$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml\n\t@echo\n\t@echo \"Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml.\"\n" }, { "path": "docs/architecture.rst", "content": "Architecture\n============\n\nProcessing graph\n----------------\n\nThe core processing engine of MineMeld is based on a Direct Acyclic Graph\nof nodes. Indicators are retrieved or received by Miner nodes and then pushed to\ndownstream nodes via *update* messages. Nodes can also signal to downstream\nnodes the removal of an indicator (in case of expiration, ...) by sending\na *withdraw* message.\n\nEach node in the graph is independent from other nodes, there is no global\nstate or global clock inside the engine. All the nodes are independent and\nasynchrnous. Each node is also responsabile for maintaining its own state.\nThis architecture trades memory and disk for flexibility.\n\n.. image:: images/updates.png\n\n.. image:: images/withdraws.png\n\nNode\n----\n\nEach node may have 0 or more inputs and 0 or more output. Rather obviuosly\nif a node has 0 inputs it is considered a Miner node, if a node has 0 ouput\nit is considered an Output node.\n\nEach node also offers a RPC interface, for direct out of band requests, and\na connection to a *management bus* for status checks and management commands\ncoming from the *management bus master*.\n\n.. image:: images/nodes.png\n\nThe connections between nodes are implemented with a pubsub mechanism over\nthe *fabric*. Each node sends its downstream message to a *topic* named as\nthe node, and all the downstream nodes are subscribers of this topic.\n\n.. image:: images/topics.png\n\nRuntime architecture\n--------------------\n\nTo take advantage of all the cores available on the system, the engine\nby default automatically splits the nodes of the graph into multiple\nprocesses called *chassis*. Each *chassis* has a dedicated connection\nto the *fabric* and to the *management bus*.\n\nThe *master* process monitors the health and the metrics of each *chassis*\nusing the *management bus*. The *master* process is also responsible for\nsynchronzing the nodes when the engine starts and stops, to ensure that\nthe state of the graph is consistent. At shutdown this is achieved\nusing a super simplified version of the Chandy-Lamport checkpoint\nalgorithm.\n\nBoth the *management bus* and the *fabric* are implemented using an\nexternal message broker, RabbitMQ.\n\n.. image:: images/chassis-architecture.png\n" }, { "path": "docs/conf.py", "content": "# -*- coding: utf-8 -*-\n#\n# minemeld-core documentation build configuration file, created by\n# sphinx-quickstart on Thu Aug 6 14:14:33 2015.\n#\n# This file is execfile()d with the current directory set to its\n# containing dir.\n#\n# Note that not all possible configuration values are present in this\n# autogenerated file.\n#\n# All configuration values have a default; values that are commented out\n# serve to show the default.\n\n# flake8: noqa\n\nimport sys\nimport os\nimport shlex\n\n# from http://blog.rtwilson.com/how-to-make-your-sphinx-documentation-compile-with-readthedocs-when-youre-using-numpy-and-scipy/\nimport mock\n \nMOCK_MODULES = ['plyvel']\nfor mod_name in MOCK_MODULES:\n sys.modules[mod_name] = mock.Mock()\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\nsys.path.insert(0, os.path.abspath('..'))\n\n# -- General configuration ------------------------------------------------\n\n# If your documentation needs a minimal Sphinx version, state it here.\n#needs_sphinx = '1.0'\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\n 'sphinx.ext.autodoc',\n 'sphinx.ext.napoleon'\n]\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# The suffix(es) of source filenames.\n# You can specify multiple suffix as a list of string:\n# source_suffix = ['.rst', '.md']\nsource_suffix = '.rst'\n\n# The encoding of source files.\n#source_encoding = 'utf-8-sig'\n\n# The master toctree document.\nmaster_doc = 'index'\n\n# General information about the project.\nproject = u'minemeld-core'\ncopyright = u'2015, Palo Alto Networks'\nauthor = u'Palo Alto Networks'\n\n# The version info for the project you're documenting, acts as replacement for\n# |version| and |release|, also used in various other places throughout the\n# built documents.\n#\n# The short X.Y version.\nversion = '0.9'\n# The full version, including alpha/beta/rc tags.\nrelease = '0.9.70.post5'\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = None\n\n# There are two options for replacing |today|: either, you set today to some\n# non-false value, then it is used:\n#today = ''\n# Else, today_fmt is used as the format for a strftime call.\n#today_fmt = '%B %d, %Y'\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\nexclude_patterns = ['_build']\n\n# The reST default role (used for this markup: `text`) to use for all\n# documents.\n#default_role = None\n\n# If true, '()' will be appended to :func: etc. cross-reference text.\n#add_function_parentheses = True\n\n# If true, the current module name will be prepended to all description\n# unit titles (such as .. function::).\n#add_module_names = True\n\n# If true, sectionauthor and moduleauthor directives will be shown in the\n# output. They are ignored by default.\n#show_authors = False\n\n# The name of the Pygments (syntax highlighting) style to use.\npygments_style = 'sphinx'\n\n# A list of ignored prefixes for module index sorting.\n#modindex_common_prefix = []\n\n# If true, keep warnings as \"system message\" paragraphs in the built documents.\n#keep_warnings = False\n\n# If true, `todo` and `todoList` produce output, else they produce nothing.\ntodo_include_todos = False\n\n\n# -- Options for HTML output ----------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\nhtml_theme = 'alabaster'\n\n# Theme options are theme-specific and customize the look and feel of a theme\n# further. For a list of options available for each theme, see the\n# documentation.\n#html_theme_options = {}\n\n# Add any paths that contain custom themes here, relative to this directory.\n#html_theme_path = []\n\n# The name for this set of Sphinx documents. If None, it defaults to\n# \" v documentation\".\n#html_title = None\n\n# A shorter title for the navigation bar. Default is the same as html_title.\n#html_short_title = None\n\n# The name of an image file (relative to this directory) to place at the top\n# of the sidebar.\n#html_logo = None\n\n# The name of an image file (within the static path) to use as favicon of the\n# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32\n# pixels large.\n#html_favicon = None\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = ['_static']\n\n# Add any extra paths that contain custom files (such as robots.txt or\n# .htaccess) here, relative to this directory. These files are copied\n# directly to the root of the documentation.\n#html_extra_path = []\n\n# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,\n# using the given strftime format.\n#html_last_updated_fmt = '%b %d, %Y'\n\n# If true, SmartyPants will be used to convert quotes and dashes to\n# typographically correct entities.\n#html_use_smartypants = True\n\n# Custom sidebar templates, maps document names to template names.\n#html_sidebars = {}\n\n# Additional templates that should be rendered to pages, maps page names to\n# template names.\n#html_additional_pages = {}\n\n# If false, no module index is generated.\n#html_domain_indices = True\n\n# If false, no index is generated.\n#html_use_index = True\n\n# If true, the index is split into individual pages for each letter.\n#html_split_index = False\n\n# If true, links to the reST sources are added to the pages.\n#html_show_sourcelink = True\n\n# If true, \"Created using Sphinx\" is shown in the HTML footer. Default is True.\n#html_show_sphinx = True\n\n# If true, \"(C) Copyright ...\" is shown in the HTML footer. Default is True.\n#html_show_copyright = True\n\n# If true, an OpenSearch description file will be output, and all pages will\n# contain a tag referring to it. The value of this option must be the\n# base URL from which the finished HTML is served.\n#html_use_opensearch = ''\n\n# This is the file name suffix for HTML files (e.g. \".xhtml\").\n#html_file_suffix = None\n\n# Language to be used for generating the HTML full-text search index.\n# Sphinx supports the following languages:\n# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'\n# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'\n#html_search_language = 'en'\n\n# A dictionary with options for the search language support, empty by default.\n# Now only 'ja' uses this config value\n#html_search_options = {'type': 'default'}\n\n# The name of a javascript file (relative to the configuration directory) that\n# implements a search results scorer. If empty, the default will be used.\n#html_search_scorer = 'scorer.js'\n\n# Output file base name for HTML help builder.\nhtmlhelp_basename = 'minemeld-coredoc'\n\n# -- Options for LaTeX output ---------------------------------------------\n\nlatex_elements = {\n# The paper size ('letterpaper' or 'a4paper').\n#'papersize': 'letterpaper',\n\n# The font size ('10pt', '11pt' or '12pt').\n#'pointsize': '10pt',\n\n# Additional stuff for the LaTeX preamble.\n#'preamble': '',\n\n# Latex figure (float) alignment\n#'figure_align': 'htbp',\n}\n\n# Grouping the document tree into LaTeX files. List of tuples\n# (source start file, target name, title,\n# author, documentclass [howto, manual, or own class]).\nlatex_documents = [\n (master_doc, 'minemeld-core.tex', u'modindex_common_prefixer-wagon Documentation',\n u'Palo Alto Networks', 'manual'),\n]\n\n# The name of an image file (relative to this directory) to place at the top of\n# the title page.\n#latex_logo = None\n\n# For \"manual\" documents, if this is true, then toplevel headings are parts,\n# not chapters.\n#latex_use_parts = False\n\n# If true, show page references after internal links.\n#latex_show_pagerefs = False\n\n# If true, show URL addresses after external links.\n#latex_show_urls = False\n\n# Documents to append as an appendix to all manuals.\n#latex_appendices = []\n\n# If false, no module index is generated.\n#latex_domain_indices = True\n\n\n# -- Options for manual page output ---------------------------------------\n\n# One entry per manual page. List of tuples\n# (source start file, name, description, authors, manual section).\nman_pages = [\n (master_doc, 'minemeld-core', u'minemeld-core Documentation',\n [author], 1)\n]\n\n# If true, show URL addresses after external links.\n#man_show_urls = False\n\n\n# -- Options for Texinfo output -------------------------------------------\n\n# Grouping the document tree into Texinfo files. List of tuples\n# (source start file, target name, title, author,\n# dir menu entry, description, category)\ntexinfo_documents = [\n (master_doc, 'minemeld-core', u'minemeld-core Documentation',\n author, 'minemeld-core', 'One line description of project.',\n 'Miscellaneous'),\n]\n\n# Documents to append as an appendix to all manuals.\n#texinfo_appendices = []\n\n# If false, no module index is generated.\n#texinfo_domain_indices = True\n\n# How to display URL addresses: 'footnote', 'no', or 'inline'.\n#texinfo_show_urls = 'footnote'\n\n# If true, do not generate a @detailmenu in the \"Top\" node's menu.\n#texinfo_no_detailmenu = False\n" }, { "path": "docs/configapi.rst", "content": "Configuration API\n=================\n\nConfiguration API can be used to change a temporary configuration. The temp\nconfiguration is applied when the /config/commit API is called.\n\nThe temp configuration has a *version* attribute. The version changes\nevery time the temp configuration is reinitialized or reloaded from\nthe current MineMeld running configuration. Some API calls (commit,\ncreate node, ...) require a version parameter, if the supplied version\nis different from the current temp config version a 409 error is returned.\n\nEach node has a *version* attribute. The node version changes every time\nthe node config is changed. Some API calls (set node, ...) require a version\nparameter. If the supplied version is different from the current node version\na 409 error is returned. \n\nAuthentication\n--------------\n\nAuthentication is performed via basic authentication. A 401 error is returned\nin case of invalid credentials:\n\n::\n\n $ curl -u 'admin:goodpassword' -i http://127.0.0.1/config/info\n HTTP/1.1 200 OK\n [...]\n\n $ curl -u 'admin:baspassword' -i http://127.0.0.1/config/info\n HTTP/1.1 401 UNAUTHORIZED\n [...]\n\n.. warning:: Use HTTPS with a trusted certificate in production !\n\nConfiguration information\n-------------------------\n\n::\n\n $ curl -u 'admin:admin' -i http://127.0.0.1/config/info\n HTTP/1.1 200 OK\n Server: nginx/1.4.6 (Ubuntu)\n Date: Sun, 06 Sep 2015 14:51:41 GMT\n Content-Type: application/json\n Content-Length: 141\n Connection: keep-alive\n \n {\n \"result\": {\n \"fabric\": false, \n \"mgmtbus\": false, \n \"num_nodes\": 8, \n \"version\": \"58102565-1b93-4095-9130-84556496b84b\"\n }\n }\n\nReload configuration\n--------------------\n\nReload current running configuration as temporary configuration. Config version\nis changed. Returns new config version.\n\n::\n\n $ curl -u 'admin:admin' -i http://127.0.0.1/config/reload\n HTTP/1.1 200 OK\n Server: nginx/1.4.6 (Ubuntu)\n Date: Sun, 06 Sep 2015 14:52:46 GMT\n Content-Type: application/json\n Content-Length: 54\n Connection: keep-alive\n \n {\n \"result\": \"b2482473-1e9f-4a24-a8b7-7296f1dfb856\"\n }\n\nCreate node\n-----------\n\nCreate a new node. *version* attribute is required, and should be the config\nversion. Returns the new node id and version.\n\n::\n\n $ curl -XPOST -H 'Content-Type: application/json' -u 'admin:admin' -i http://127.0.0.1/config/node -d '{\n \"name\": \"spamhaus_EDROP2\", \n \"properties\": {\n \"class\": \"HTTP\", \n \"config\": {\n \"attributes\": {\n \"direction\": \"inbound\", \n \"type\": \"IPv4\"\n }, \n \"cchar\": \";\", \n \"source_name\": \"http://www.spamhaus.org/drop/edrop.txt\", \n \"split_char\": \";\", \n \"url\": \"http://www.spamhaus.org/drop/edrop.txt\"\n }, \n \"output\": true\n }, \n \"version\": \"b2482473-1e9f-4a24-a8b7-7296f1dfb856\"\n }'\n HTTP/1.1 200 OK\n Server: nginx/1.4.6 (Ubuntu)\n Date: Sun, 06 Sep 2015 14:59:28 GMT\n Content-Type: application/json\n Content-Length: 91\n Connection: keep-alive\n \n {\n \"result\": {\n \"id\": 9, \n \"version\": \"b2482473-1e9f-4a24-a8b7-7296f1dfb856+0\"\n }\n }\n\nGet node configuration\n----------------------\n\n::\n\n $ curl -u 'admin:admin' -i http://127.0.0.1/config/node/9\n HTTP/1.1 200 OK\n Server: nginx/1.4.6 (Ubuntu)\n Date: Sun, 06 Sep 2015 15:01:00 GMT\n Content-Type: application/json\n Content-Length: 479\n Connection: keep-alive\n \n {\n \"result\": {\n \"name\": \"spamhaus_EDROP2\", \n \"properties\": {\n \"class\": \"HTTP\", \n \"config\": {\n \"attributes\": {\n \"direction\": \"inbound\", \n \"type\": \"IPv4\"\n }, \n \"cchar\": \";\", \n \"source_name\": \"http://www.spamhaus.org/drop/edrop.txt\", \n \"split_char\": \";\", \n \"url\": \"http://www.spamhaus.org/drop/edrop.txt\"\n }, \n \"output\": true\n }, \n \"version\": \"b2482473-1e9f-4a24-a8b7-7296f1dfb856+0\"\n }\n }\n\nChange node configuration\n-------------------------\n\n*version* is the current node version.\n\n::\n\n $ curl -XPUT -u 'admin:admin' -H 'Content-Type: application/json' -i http://127.0.0.1/config/node/8 -d '{\n \"name\": \"spamhaus_EDROP2\", \n \"properties\": {\n \"class\": \"HTTP\", \n \"config\": {\n \"attributes\": {\n \"direction\": \"inbound\", \n \"type\": \"IPv4\"\n }, \n \"cchar\": \";\", \n \"source_name\": \"http://www.spamhaus.org/drop/edrop2.txt\", \n \"split_char\": \";\", \n \"url\": \"http://www.spamhaus.org/drop/edrop.txt\"\n }, \n \"output\": true\n }, \n \"version\": \"b2482473-1e9f-4a24-a8b7-7296f1dfb856+0\"\n }'\n HTTP/1.1 200 OK\n Server: nginx/1.4.6 (Ubuntu)\n Date: Sun, 06 Sep 2015 15:24:25 GMT\n Content-Type: application/json\n Content-Length: 56\n Connection: keep-alive\n \n {\n \"result\": \"b2482473-1e9f-4a24-a8b7-7296f1dfb856+1\"\n }\n\nDelete node\n-----------\n\nDelete a node. *version* is the current node version.\n\n::\n\n $ curl -XDELETE -H 'Content-type: application/json' -u 'admin:admin' -i http://127.0.0.1/config/node/9 -d '{\"version\": \"b2482473-1e9f-4a24-a8b7-7296f1dfb856+0\"}'\n HTTP/1.1 200 OK\n Server: nginx/1.4.6 (Ubuntu)\n Date: Sun, 06 Sep 2015 15:17:42 GMT\n Content-Type: application/json\n Content-Length: 20\n Connection: keep-alive\n \n {\n \"result\": \"OK\"\n }\n\nCommit configuration\n--------------------\n\n*version* is the current configuration version.\n\n::\n\n $ curl -XPOST -H 'Content-Type: application/json' -u 'admin:admin' -i http://127.0.0.1/config/commit -d '{\"version\": \"b2482473-1e9f-4a24-a8b7- 7296f1dfb856\"}'\n HTTP/1.1 200 OK\n Server: nginx/1.4.6 (Ubuntu)\n Date: Sun, 06 Sep 2015 15:31:26 GMT\n Content-Type: application/json\n Content-Length: 20\n Connection: keep-alive\n \n {\n \"result\": \"OK\"\n }\n" }, { "path": "docs/index.rst", "content": ".. MineMeld documentation master file, created by\n sphinx-quickstart on Thu Aug 6 14:14:33 2015.\n You can adapt this file completely to your liking, but it should at least\n contain the root `toctree` directive.\n\nWelcome to MineMeld's documentation!\n=======================================\n\nContents:\n\n.. toctree::\n :maxdepth: 1\n\n architecture\n messages \n scripts\n schema\n statusapi\n configapi\n metricsapi\n internals\n license\n\n\nIndices and tables\n==================\n\n* :ref:`genindex`\n* :ref:`modindex`\n* :ref:`search`\n" }, { "path": "docs/internals.rst", "content": "Internals\n=========\n\nminemeld.ft.table\n-------------------\n\n.. automodule:: minemeld.ft.table\n\n\nminemeld.ft.st\n----------------\n\n.. automodule:: minemeld.ft.st\n" }, { "path": "docs/license.rst", "content": "LICENSE\n=======\n\nCopyright (c) 2015, Palo Alto Networks \n\nPermission to use, copy, modify, and/or distribute this software for any\npurpose with or without fee is hereby granted, provided that the above\ncopyright notice and this permission notice appear in all copies.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES\nWITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF\nMERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR\nANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES\nWHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN\nACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF\nOR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\n\n" }, { "path": "docs/messages.rst", "content": "Messages\n========\n\nIntra-node protocol\n-------------------\n\nThe protocol used between nodes is super simple. There are 3 messages:\n\nupdate\n******\n\nupdate(indicator, value)\n\n:indicator: string\n:value: a dictionary of attributes for the indicator\n\nNotifies a new indicator or an update of the attributes associated to an\nindicator.\n\nwithdraw\n********\n\nwithdraw(indicator[, value])\n\n:indicator: string\n:value: (optional) a dictionary of attributes for the indicator\n\nNotifies a withdraw of the indicator\n\ncheckpoint\n**********\n\ncheckpoint(id)\n\nUsed as a processing barrier for the graph when the graph is being stopped.\n" }, { "path": "docs/metricsapi.rst", "content": "Metrics API\n===========\n\nMetrics API can be used to retrieve historical statistics of the system and\nMineMeld nodes.\n\nMetrics list\n------------\n\n::\n\n $ curl -i -u 'admin:admin' http://127.0.0.1/metrics\n HTTP/1.1 200 OK\n Server: nginx/1.4.6 (Ubuntu)\n Date: Tue, 08 Sep 2015 21:41:07 GMT\n Content-Type: application/json\n Content-Length: 1040\n Connection: keep-alive\n \n {\n \"result\": [\n \"spamhaus_EDROP.update.tx\", \n \"df-run\", \n \"inboundaggregator.update.rx\", \n \"dshield_blocklist.length\", \n \"df-run-user\", \n \"zeustracker_badips.length\", \n \"spamhaus_DROP.length\", \n \"df-root\", \n \"df-sys-fs-cgroup\", \n \"spamhaus_EDROP.length\", \n [...]\n ]\n }\n\nMetric history\n--------------\n\nMetric history can be retrieved using /metrics/ endpoint.\n\n::\n\n $ curl -i -u 'admin:admin' http://127.0.0.1/metrics/outboundaggregator.length\n HTTP/1.1 200 OK\n Server: nginx/1.4.6 (Ubuntu)\n Date: Tue, 08 Sep 2015 21:42:52 GMT\n Content-Type: application/json\n Content-Length: 1779\n Connection: keep-alive\n \n {\n \"result\": [\n [\n 1441661340, \n null\n ], \n [...]\n ]\n }\n" }, { "path": "docs/nodeconfig.rst", "content": "Node config\n===========\n\nThe set of config parameters supported by a node depends on the node class.\n\n.. note::\n This document has been extracted from the docstrings of the python code.\n For the most updated documentation check the original source code.\n\nBase class\n----------\n\nAll nodes support these parameters.\n\nParameters\n+++++++++++++++++\n\n:infilters: inbound *filter set*. Filters to apply to received indicators.\n:outfilters: outbound *filter set*. Filters to apply to transmitted\n indicators.\n\nFilter set\n++++++++++\n\nEach filter set is a list of filters. Filters are checked from top\nto bottom, the first matching filter is applied and following filters are not\nchecked. Default action is **accept**.\nEach filter is a dictionary with 3 keys:\n\n:name: name of the filter.\n:conditions: list of boolean expressions to match on the\n indicator and indicator value.\n:actions: list of actions to apply to the indicator.\n Currently the only supported actions are **accept** and **drop**\n\nIn addition to the atttributes in the indicator value, filters can\nmatch on 3 special attributes:\n\n:__indicator: the indicator itself.\n:__method: the method of the message, **update** or **withdraw**.\n:__origin: the name of the node who sent the indicator.\n\nCondition\n+++++++++\n\nA condition in the filter is a boolean expression composed by: a JMESPath\nexpression, an operator (<, <=, ==, >=, >, !=) and a value.\n\nExample\n+++++++\n\nExample config in YAML::\n\n infilters:\n - name: accept withdraws\n conditions:\n - __method == 'withdraw'\n actions:\n - accept\n - name: accept URL\n conditions:\n - type == 'URL'\n actions:\n - accept\n - name: drop all\n actions:\n - drop\n outfilters:\n - name: accept all (default)\n actions:\n - accept\n\nBase poller class\n-----------------\n\nIn addition to `Base class` config parameters, the base poller class support\nthe following parameters.\n\nConfig parameters\n+++++++++++++++++\n\n:source_name: name of the source. This is added to the\n *sources* attribute of the generated indicators. Default: name\n of the node.\n:attributes: dictionary of attributes for the generated indicators.\n This dictionary is used as template for the value of the generated\n indicators. Default: empty\n:interval: polling interval in seconds. Default: 3600.\n:num_retries: how many times the miner should try to reach the source in case\n of failure. If this number is exceeded, the miner\n waits until the next polling time to try again. Default: 2\n:age_out: age out policies to apply to the indicators.\n Default: age out check interval 3600 seconds, sudden death enabled,\n default age out interval 30 days.\n\nAge out policy\n++++++++++++++\n\nAge out policy is described by a dictionary with at least 3 keys:\n\n:interval: number of seconds between successive age out checks.\n:sudden_death: boolean, if *true* indicators are immediately aged out\n when they disappear from the feed.\n:default: age out interval. After this interval an indicator is aged\n out even if it is still present in the feed. If *null*, no age out\n interval is applied.\n\nAdditional keys can be used to specify age out interval per indicator\n*type*.\n\nAge out interval\n++++++++++++++++\n\nAge out intervals have the following format::\n\n +\n\n*base attribute* can be *last_seen*, if the age out interval should be\ncalculated based on the last time the indicator was found in the feed,\nor *first_seen*, if instead the age out interval should be based on the\ntime the indicator was first seen in the feed. If not specified\n*first_seen* is used.\n\n*interval* is the length of the interval expressed in seconds. Suffixes\n*d*, *h* and *m* can be used to specify days, hours or minutes.\n\nExample\n+++++++\n\nExample config in YAML for a feed where indicators should be aged out\nonly when they are removed from the feed::\n\n source_name: example.persistent_feed\n interval: 600\n age_out:\n default: null\n sudden_death: true\n interval: 300\n attributes:\n type: IPv4\n confidence: 100\n share_level: green\n direction: inbound\n\nExample config in YAML for a feed where indicators are aged out when\nthey disappear from the feed and 30 days after they have seen for the\nfirst time in the feed::\n\n source_name: example.long_running_feed\n interval: 3600\n age_out:\n default: first_seen+30d\n sudden_death: true\n interval: 1800\n attributes:\n type: URL\n confidence: 50\n share_level: green\n\nExample config in YAML for a feed where indicators are aged 30 days\nafter they have seen for the last time in the feed::\n\n source_name: example.delta_feed\n interval: 3600\n age_out:\n default: last_seen+30d\n sudden_death: false\n interval: 1800\n attributes:\n type: URL\n confidence: 50\n share_level: green\n\nminemeld.ft.http.HttpFT\n-----------------------\n\nIn addition to `Base poller class` config parameters, the base poller class\nsupport the following parameters.\n\nParameters\n+++++++++++++++++\n\n:url: URL of the feed.\n:polling_timeout: timeout of the polling request in seconds.\n Default: 20\n:verify_cert: boolean, if *true* feed HTTPS server certificate is\n verified. Default: *true*\n:ignore_regex: Python regular expression for lines that should be\n ignored. Default: *null*\n:indicator: an *extraction dictionary* to extract the indicator from\n the line. If *null*, the text until the first whitespace or newline\n character is used as indicator. Default: *null*\n:fields: a dicionary of *extraction dictionaries* to extract\n additional attributes from each line. Default: {}\n\nExtraction dictionary\n+++++++++++++++++++++\n\nExtraction dictionaries contain the following keys:\n\n:regex: Python regular expression for searching the text.\n:transform: template to generate the final value from the result\n of the regular expression. Default: the entire match of the regex\n is used as extracted value.\n\nSee Python `re `_ module for\ndetails about Python regular expressions and templates.\n\nExample\n+++++++\n\nExample config in YAML where extraction dictionaries are used to\nextract the indicator and additional fields::\n\n url: https://www.dshield.org/block.txt\n ignore_regex: \"[#S].*\"\n indicator:\n regex: '^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})\\t([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})'\n transform: '\\1-\\2'\n fields:\n dshield_nattacks:\n regex: '^.*\\t.*\\t[0-9]+\\t([0-9]+)'\n transform: '\\1'\n dshield_name:\n regex: '^.*\\t.*\\t[0-9]+\\t[0-9]+\\t([^\\t]+)'\n transform: '\\1'\n dshield_country:\n regex: '^.*\\t.*\\t[0-9]+\\t[0-9]+\\t[^\\t]+\\t([A-Z]+)'\n transform: '\\1'\n dshield_email:\n regex: '^.*\\t.*\\t[0-9]+\\t[0-9]+\\t[^\\t]+\\t[A-Z]+\\t(\\S+)'\n transform: '\\1'\n\nExample config in YAML where the text in each line until the first\nwhitespace is used as indicator::\n\n url: https://ransomwaretracker.abuse.ch/downloads/CW_C2_URLBL.txt\n ignore_regex: '^#'\n\nFor a complete config example check **dshield.block** prototype.\n\nminemeld.ft.csv.CSVFT\n---------------------\n\nIn addition to `Base poller class` config parameters, the base poller class\nsupport the following parameters.\n\nParameters\n++++++++++\n\n:url: URL of the feed.\n:polling_timeout: timeout of the polling request in seconds.\n Default: 20\n:verify_cert: boolean, if *true* feed HTTPS server certificate is\n verified. Default: *true*\n:ignore_regex: Python regular expression for lines that should be\n ignored. Default: *null*\n:fieldnames: list of field names in the file. If *null* the values\n in the first row of the file are used as names. Default: *null*\n:delimiter: see `csv Python module `_.\n Default: ,\n:doublequote: see `csv Python module `_.\n Default: true\n:escapechar: see `csv Python module `_.\n Default: null\n:quotechar: see `csv Python module `_.\n Default: \"\n:skipinitialspace: see `csv Python module `_.\n Default: false\n\nExample\n+++++++\n\nExample config in YAML::\n\n url: https://sslbl.abuse.ch/blacklist/sslipblacklist.csv\n ignore_regex: '^#'\n fieldnames:\n - indicator\n - port\n - sslblabusech_type\n\nFor a complete config example check **sslabusech.ipblacklist** prototype.\n\nminemeld.ft.json.SimpleJSON\n---------------------------\n\nIn addition to `Base poller class` config parameters, the base poller class\nsupport the following parameters.\n\nParameters\n++++++++++\n\n:url: URL of the feed.\n:polling_timeout: timeout of the polling request in seconds.\n Default: 20\n:verify_cert: boolean, if *true* feed HTTPS server certificate is\n verified. Default: *true*\n:extractor: JMESPath expression for extracting the indicators from\n the JSON document. Default: @\n:indicator: the JSON attribute to use as indicator. Default: indicator\n:fields: list of JSON attributes to include in the indicator value.\n If *null* no additional attributes are extracted. Default: *null*\n:prefix: prefix to add to field names. Default: json\n\nExample\n+++++++\n\nExample config in YAML::\n\n url: https://ip-ranges.amazonaws.com/ip-ranges.json\n extractor: \"prefixes[?service=='AMAZON']\"\n prefix: aws\n indicator: ip_prefix\n fields:\n - region\n - service\n\nFor a complete config example check **aws.AMAZON** prototype.\n" }, { "path": "docs/schema-indicator-0-1.json", "content": "{\n \"id\": \"https://www.paloaltonetworks.com/minemeld-indicator-schema-0-1#\",\n \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n \"description\": \"schema for minemeld attributes\",\n \"type\": \"object\",\n \"required\": [ \"type\" ],\n \"properties\": {\n \"type\": {\n \"description\": \"type of the indicator\",\n \"type\": \"string\",\n \"enum\": [\n \"IPv4\",\n \"IPv6\",\n \"domain\",\n \"URL\",\n \"sha512\",\n \"sha256\",\n \"sha1\",\n \"md5\",\n \"ssdeep\",\n \"mutex\",\n \"windows-registry-value\",\n \"user-agent.fragment\",\n \"file.name\",\n \"process.command_line\",\n \"email-addr\",\n \"autonomous-system\"\n ]\n },\n \"direction\": {\n \"description\": \"direction of the session, applies to IPv4\",\n \"type\": \"string\",\n \"enum\": [\"inbound\", \"outbound\"]\n },\n \"first_seen\": {\n \"type\": \"integer\",\n \"format\": \"utc-millisec\",\n \"description\": \"time the indicator has been seen for the first time. <\"\n },\n \"last_seen\": {\n \"type\": \"integer\",\n \"format\": \"utc-millisec\",\n \"description\": \"time the indicator has been seen for the last time. >\"\n },\n \"sources\": {\n \"description\": \"list of sources for this indicator\",\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\",\n \"format\": \"uri\"\n }\n },\n \"confidence\": {\n \"type\": \"integer\",\n \"description\": \"confidence in the indicator 0-100\",\n \"minimum\": 0,\n \"maximum\": 100\n },\n \"share_level\": {\n \"description\": \"share level of indicator\",\n \"type\": \"string\",\n \"enum\": [\"white\", \"green\", \"amber\", \"red\"]\n },\n \"country\": {\n \"type\": \"string\",\n \"description\": \"ISO country code (IPv4 and IPv6 only)\",\n \"minLength\": 2,\n \"maxLength\": 2\n },\n \"AS\": {\n \"type\": \"string\",\n \"description\": \"Autonmous system (IPv4 and IPv6 only)\"\n }\n },\n \"patternProperties\": {\n \"^_[a-zA-Z0-9$_]*$\": {\n \"description\": \"private properties\"\n },\n \"^$[a-zA-Z0-9$_]*$\": {\n \"description\": \"reserved, temporary properties\"\n }\n }\n}\n" }, { "path": "docs/schema.rst", "content": "Indicator value schema\n======================\n\n.. literalinclude:: schema-indicator-0-1.json\n :language: json\n :linenos:\n" }, { "path": "docs/scripts.rst", "content": "mm-run\n=========\n\nmm-run is a simple script that can be used as frontend to the minemeld library.\n\nUsage\n-----\n\n::\n\n $ mm-run --help\n usage: mm-run.py [-h] [--version] [--multiprocessing NP] [--verbose] CONFIG\n \n Low-latency threat indicators processor\n \n positional arguments:\n CONFIG path of the config file or of the config directory\n \n optional arguments:\n -h, --help show this help message and exit\n --version show program's version number and exit\n --multiprocessing NP enable multiprocessing. NP is the number of processes,\n 0 to use a process per machine core\n --verbose verbose\n\nConfiguration\n-------------\n\nCONFIG parameter on the command line can point to a configuration file or to a \nconfiguration directory. If CONIG is a directory, mm-run will check for \ncommited-config.yml and running-config.yml files. If only running-config.yml exists,\nmm-run assumes that the config has not been changed and the processing continues\nwhere it was stopped. If only canidate-config.yml exists, mm-run copies the\nfile to running-config.yml and reinitializes the processing. If both files exist,\ncandidate-config.yml is copied over running-config.yml and the processing is\nreinitialized if the 2 files are different. If CONFIG instead is a path to a\nconfiguration file, the configuration will be considered as new and the processing\nis reinitialized.\n\nnodes\n~~~~~\n\nThe **nodes** section contains the desription of the processing DAG. It is composed\nby a list of descriptions of nodes.\n\nEach node config has the following general format:\n\n::\n\n nodename: # name of the node\n config:\n # list of parameters for the node, depends on the node class\n class: nodeclass # class of the node\n inputs:\n # list of upstream nodes\n - node1\n - node2\n output: true|false # if the node should generate updates & withdraws\n\nNode can also be based on prototypes, in that case the *config* and *class*\nsections are omitted as they are specified inside the prototype.\n\n::\n\n nodename: # name of the node\n prototype: prototype1 # name of the prototype to be used\n inputs:\n # list of upstream nodes\n - node1\n - node2\n output: true|false # if the node should generate updates & withdraws \n\nExample 1\n^^^^^^^^^\n\n::\n\n spamhaus_DROP:\n config:\n source_name: http://www.spamhaus.org/drop/drop.txt\n attributes:\n type: IPv4\n direction: inbound\n cchar: ;\n split_char: ;\n url: http://www.spamhaus.org/drop/drop.txt\n class: HTTP\n output: true\n\nThis describes a node with the following properties:\n\n:name: spamhaus_DROP\n:class: HTTP\n:inputs: *none*, this is a Miner\n:output: enabled, this node will emit indicators\n:config: specific configuration for this node class\n\nExample 2\n^^^^^^^^^\n\n::\n\n inboundaggregator:\n config:\n infilters:\n - name: accept inbound IPv4\n conditions:\n - type == 'IPv4'\n - direction == 'inbound'\n actions:\n - accept\n - name: drop all\n actions:\n - drop\n class: AggregatorIPv4\n output: true\n inputs:\n - spamhaus_DROP\n - spamhaus_EDROP\n - dshield_blocklist\n\n:name: inboundaggregator\n:class: AggregatorIPv4\n:inputs: this node will receive indicators from spamhaus_DROP, spamhaus_EDROP, ...\n:output: enabled, this node will emit indicators\n:config: specific configuration for this node class\n\nExample 3\n^^^^^^^^^\n\n::\n\n spamhaus_DROP:\n output: true\n prototype: spamhaus.DROP\n\n:name: spamhaus_DROP\n:inputs: *none*, this is a Miner\n:output: enabled, this node will emit indicators\n:prototype: *config* and *class* of this node will be loaded from the spamhaus.DROP prototype\n" }, { "path": "docs/statusapi.rst", "content": "Status API\n==========\n\nStatus API can be used to retrieve status information about minemeld core\nengine and the system.\n\nMineMeld status\n---------------\n\n::\n\n $ curl -i -u 'admin:admin' http://127.0.0.1/status/minemeld\n HTTP/1.1 200 OK\n Server: nginx/1.4.6 (Ubuntu)\n Date: Tue, 08 Sep 2015 21:34:47 GMT\n Content-Type: application/json\n Content-Length: 1878\n Connection: keep-alive\n \n {\n \"result\": {\n \"dshield_blocklist\": {\n \"inputs\": [], \n \"length\": 20, \n \"output\": true, \n \"state\": 5, \n \"statistics\": {\n \"update.tx\": 20\n }\n }, \n [...]\n }\n\nSystem status\n-------------\n\nReports percent usage of CPUs, disk, memory and swap.\n\n::\n\n $curl -i -u 'admin:admin' http://127.0.0.1/status/system\n HTTP/1.1 200 OK\n Server: nginx/1.4.6 (Ubuntu)\n Date: Tue, 08 Sep 2015 21:37:31 GMT\n Content-Type: application/json\n Content-Length: 108\n Connection: keep-alive\n \n {\n \"result\": {\n \"cpu\": [\n 0.0\n ], \n \"disk\": 17.7, \n \"memory\": 25.1, \n \"swap\": 0.0\n }\n }\n" }, { "path": "minemeld/__init__.py", "content": "\"\"\"\nminemeld\n========\n\nMineMeld core engine\n\"\"\"\n\n__version__ = '0.9.70.post5'\n" }, { "path": "minemeld/chassis.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nminemeld.chassis\n\nA chassis instance contains a list of nodes and a fabric.\nNodes communicate using the fabric.\n\"\"\"\n\nimport os\nimport logging\n\nimport gevent\nimport gevent.queue\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport minemeld.mgmtbus\nimport minemeld.ft\nimport minemeld.fabric\n\nLOG = logging.getLogger(__name__)\nSTATE_REPORT_INTERVAL = 10\n\n\nclass Chassis(object):\n \"\"\"Chassis class\n\n Args:\n fabricclass (str): class for the fabric\n fabricconfig (dict): config dictionary for fabric,\n class specific\n mgmtbusconfig (dict): config dictionary for mgmt bus\n \"\"\"\n def __init__(self, fabricclass, fabricconfig, mgmtbusconfig):\n self.chassis_id = os.getpid()\n\n self.fts = {}\n self.poweroff = gevent.event.AsyncResult()\n\n self.fabric_class = fabricclass\n self.fabric_config = fabricconfig\n self.fabric = minemeld.fabric.factory(\n self.fabric_class,\n self,\n self.fabric_config\n )\n\n self.mgmtbus = minemeld.mgmtbus.slave_hub_factory(\n mgmtbusconfig['slave'],\n mgmtbusconfig['transport']['class'],\n mgmtbusconfig['transport']['config']\n )\n self.mgmtbus.add_failure_listener(self.mgmtbus_failed)\n self.mgmtbus.request_chassis_rpc_channel(self)\n\n self.log_channel_queue = gevent.queue.Queue(maxsize=128)\n self.log_channel = self.mgmtbus.request_log_channel()\n self.log_glet = None\n\n self.status_channel_queue = gevent.queue.Queue(maxsize=128)\n self.status_glet = None\n\n def _dynamic_load(self, classname):\n modname, classname = classname.rsplit('.', 1)\n imodule = __import__(modname, globals(), locals(), [classname])\n cls = getattr(imodule, classname)\n return cls\n\n def get_ft(self, ftname):\n return self.fts.get(ftname, None)\n\n def configure(self, config):\n \"\"\"configures the chassis instance\n\n Args:\n config (list): list of FTs\n \"\"\"\n newfts = {}\n for ft in config:\n ftconfig = config[ft]\n LOG.debug(ftconfig)\n\n # new FT\n newfts[ft] = minemeld.ft.factory(\n ftconfig['class'],\n name=ft,\n chassis=self,\n config=ftconfig.get('config', {})\n )\n newfts[ft].connect(\n ftconfig.get('inputs', []),\n ftconfig.get('output', False)\n )\n\n self.fts = newfts\n\n # XXX should be moved to constructor\n self.mgmtbus.start()\n self.fabric.start()\n\n self.mgmtbus.send_master_rpc(\n 'chassis_ready',\n params={'chassis_id': self.chassis_id},\n timeout=10\n )\n\n def request_mgmtbus_channel(self, ft):\n self.mgmtbus.request_channel(ft)\n\n def request_rpc_channel(self, ftname, ft, allowed_methods=None):\n if allowed_methods is None:\n allowed_methods = []\n self.fabric.request_rpc_channel(ftname, ft, allowed_methods)\n\n def request_pub_channel(self, ftname):\n return self.fabric.request_pub_channel(ftname)\n\n def request_sub_channel(self, ftname, ft, subname, allowed_methods=None):\n if allowed_methods is None:\n allowed_methods = []\n self.fabric.request_sub_channel(ftname, ft, subname, allowed_methods)\n\n def send_rpc(self, sftname, dftname, method, params, block, timeout):\n return self.fabric.send_rpc(sftname, dftname, method, params,\n block=block, timeout=timeout)\n\n def _log_actor(self):\n while True:\n try:\n params = self.log_channel_queue.get()\n self.log_channel.publish(\n method='log',\n params=params\n )\n\n except Exception:\n LOG.exception('Error sending log')\n\n def log(self, timestamp, nodename, log_type, value):\n self.log_channel_queue.put({\n 'timestamp': timestamp,\n 'source': nodename,\n 'log_type': log_type,\n 'log': value\n })\n\n def _status_actor(self):\n while True:\n try:\n params = self.status_channel_queue.get()\n self.mgmtbus.send_status(\n params=params\n )\n\n except Exception:\n LOG.exception('Error publishing status')\n\n def publish_status(self, timestamp, nodename, status):\n self.status_channel_queue.put({\n 'timestamp': timestamp,\n 'source': nodename,\n 'status': status\n })\n\n def fabric_failed(self):\n self.stop()\n\n def mgmtbus_failed(self):\n LOG.critical('chassis - mgmtbus failed')\n self.stop()\n\n def mgmtbus_start(self):\n LOG.info('chassis - start received from mgmtbus')\n self.start()\n return 'ok'\n\n def fts_init(self):\n for ft in self.fts.values():\n if ft.get_state() < minemeld.ft.ft_states.INIT:\n return False\n return True\n\n def stop(self):\n LOG.info(\"chassis stop called\")\n\n if self.log_glet is not None:\n self.log_glet.kill()\n\n if self.status_glet is not None:\n self.status_glet.kill()\n\n if self.fabric is None:\n return\n\n for ftname, ft in self.fts.iteritems():\n try:\n ft.stop()\n except:\n LOG.exception('Error stopping {}'.format(ftname))\n\n LOG.info('Stopping fabric')\n self.fabric.stop()\n\n LOG.info('Stopping mgmtbus')\n self.mgmtbus.stop()\n\n LOG.info('chassis - stopped')\n self.poweroff.set(value='stop')\n\n def start(self):\n LOG.info(\"chassis start called\")\n\n self.log_glet = gevent.spawn(self._log_actor)\n self.status_glet = gevent.spawn(self._status_actor)\n\n for ftname, ft in self.fts.iteritems():\n LOG.debug(\"starting %s\", ftname)\n ft.start()\n\n self.fabric.start_dispatching()\n" }, { "path": "minemeld/collectd.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nminemeld.collectd\n\nProvides a client to collectd for storing metrics.\n\"\"\"\n\nimport socket\nimport logging\n\nLOG = logging.getLogger(__name__)\n\n\nclass CollectdClient(object):\n \"\"\"Collectd client.\n\n Args:\n path (str): path to the collectd unix socket\n \"\"\"\n def __init__(self, path):\n self.path = path\n self.socket = None\n\n def _open_socket(self):\n if self.socket is not None:\n return\n\n _socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n _socket.connect(self.path)\n\n self.socket = _socket\n\n def _readline(self):\n result = ''\n\n data = None\n while data != '\\n':\n data = self.socket.recv(1)\n if data == '\\n' or data is None:\n return result\n result += data\n\n def _send_cmd(self, command):\n self._open_socket()\n self.socket.send(command+'\\n')\n\n ans = self._readline()\n status, message = ans.split(None, 1)\n\n status = int(status)\n if status < 0:\n raise RuntimeError('Error communicating with collectd %s' %\n message)\n message = [message]\n for _ in range(status):\n message.append(self._readline())\n\n return status, '\\n'.join(message)\n\n def flush(self, identifier=None, timeout=None):\n cmd = 'FLUSH'\n if timeout is not None:\n cmd += ' timeout=%d' % timeout\n if identifier is not None:\n cmd += ' identifier=%s' % identifier\n\n self._send_cmd(\n cmd\n )\n\n def putval(self, identifier, value, timestamp='N',\n type_='minemeld_counter', hostname='minemeld', interval=None):\n if isinstance(timestamp, int):\n timestamp = '%d' % timestamp\n\n identifier = '/'.join([hostname, identifier, type_])\n command = 'PUTVAL %s' % identifier\n if interval is not None:\n command += ' interval=%d' % interval\n\n command += ' %s:%d' % (timestamp, value)\n\n self._send_cmd(command)\n" }, { "path": "minemeld/comm/__init__.py", "content": "from __future__ import absolute_import\n\nfrom .zmqredis import ZMQRedis\n\n\ndef factory(commclass, config):\n if commclass == 'ZMQRedis':\n return ZMQRedis(config)\n\n return ZMQRedis(config)\n\n\ndef cleanup(commclass, config):\n return ZMQRedis.cleanup(config)\n" }, { "path": "minemeld/comm/zmqredis.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# disable import error\n# pylint:disable=E1101\n\n\"\"\"\nThis module implements ZMQ and Redis communication class for mgmtbus and fabric.\n\"\"\"\n\nfrom __future__ import absolute_import\n\nimport logging\nimport uuid\nimport os\nimport time\n\nimport gevent\nimport gevent.event\nimport ujson as json\nfrom errno import EAGAIN\n\nimport redis\nimport zmq.green as zmq\n\nLOG = logging.getLogger(__name__)\n\n\nclass RedisPubChannel(object):\n def __init__(self, topic, connection_pool):\n self.topic = topic\n self.prefix = 'mm:topic:{}'.format(self.topic)\n\n self.connection_pool = connection_pool\n self.SR = None\n\n self.num_publish = 0\n\n def connect(self):\n if self.SR is not None:\n return\n\n self.SR = redis.StrictRedis(\n connection_pool=self.connection_pool\n )\n\n def disconnect(self):\n if self.SR is None:\n return\n\n self.SR = None\n\n def lagger(self):\n # get status of subscribers\n subscribersc = self.SR.lrange(\n '{}:subscribers'.format(self.prefix),\n 0, -1\n )\n subscribersc = [int(sc) for sc in subscribersc]\n\n # check the lagger\n minsubc = self.num_publish\n if len(subscribersc) != 0:\n minsubc = min(subscribersc)\n\n return minsubc\n\n def gc(self, lagger):\n minhighbits = lagger >> 12\n\n minqname = '{}:queue:{:013X}'.format(\n self.prefix,\n minhighbits\n )\n\n # delete all the lists before the lagger\n queues = self.SR.keys('{}:queue:*'.format(self.prefix))\n LOG.debug('topic {} - queues: {!r}'.format(self.topic, queues))\n queues = [q for q in queues if q < minqname]\n LOG.debug('topic {} - queues to be deleted: {!r}'.format(self.topic, queues))\n if len(queues) != 0:\n LOG.debug('topic {} - deleting {!r}'.format(\n self.topic,\n queues\n ))\n self.SR.delete(*queues)\n\n def publish(self, method, params=None):\n high_bits = self.num_publish >> 12\n low_bits = self.num_publish & 0xfff\n\n if (low_bits % 128) == 127:\n lagger = self.lagger()\n LOG.debug('topic {} - sent {} lagger {}'.format(\n self.topic,\n self.num_publish,\n lagger\n ))\n\n while (self.num_publish - lagger) > 1024:\n LOG.debug('topic {} - waiting lagger delta: {}'.format(\n self.topic,\n self.num_publish - lagger\n ))\n gevent.sleep(0.1)\n lagger = self.lagger()\n\n if low_bits == 0xfff:\n # we are switching to a new list, gc\n self.gc(lagger)\n\n msg = {\n 'method': method,\n 'params': params\n }\n\n qname = '{}:queue:{:013X}'.format(\n self.prefix,\n high_bits\n )\n\n self.SR.rpush(qname, json.dumps(msg))\n self.num_publish += 1\n\n\nclass ZMQRpcFanoutClientChannel(object):\n def __init__(self, fanout):\n self.socket = None\n self.reply_socket = None\n self.context = None\n\n self.fanout = fanout\n self.active_rpcs = {}\n\n def run(self):\n while True:\n LOG.debug('RPC Fanout reply recving from {}:reply'.format(self.fanout))\n body = self.reply_socket.recv_json()\n LOG.debug('RPC Fanout reply from {}:reply recvd: {!r}'.format(self.fanout, body))\n self.reply_socket.send('OK')\n LOG.debug('RPC Fanout reply from {}:reply recvd: {!r} - ok'.format(self.fanout, body))\n\n source = body.get('source', None)\n if source is None:\n LOG.error('No source in reply in ZMQRpcFanoutClientChannel {}'.format(self.fanout))\n continue\n\n id_ = body.get('id', None)\n if id_ is None:\n LOG.error('No id in reply in ZMQRpcFanoutClientChannel {} from {}'.format(self.fanout, source))\n continue\n actreq = self.active_rpcs.get(id_, None)\n if actreq is None:\n LOG.error('Unknown id {} in reply in ZMQRpcFanoutClientChannel {} from {}'.format(id_, self.fanout, source))\n continue\n\n result = body.get('result', None)\n if result is None:\n actreq['errors'] += 1\n errmsg = body.get('error', 'no error in reply')\n LOG.error('Error in RPC reply from {}: {}'.format(source, errmsg))\n\n else:\n actreq['answers'][source] = result\n LOG.debug('RPC Fanout state: {!r}'.format(actreq))\n\n if len(actreq['answers'])+actreq['errors'] >= actreq['num_results']:\n actreq['event'].set({\n 'answers': actreq['answers'],\n 'errors': actreq['errors']\n })\n self.active_rpcs.pop(id_)\n\n gevent.sleep(0)\n\n def send_rpc(self, method, params=None, num_results=0, and_discard=False):\n if self.socket is None:\n raise RuntimeError('Not connected')\n\n if params is None:\n params = {}\n\n id_ = str(uuid.uuid1())\n\n body = {\n 'reply_to': '{}:reply'.format(self.fanout),\n 'method': method,\n 'id': id_,\n 'params': params\n }\n\n event = gevent.event.AsyncResult()\n\n if num_results == 0:\n event.set({\n 'answers': {},\n 'errors': 0\n })\n return event\n\n self.active_rpcs[id_] = {\n 'cmd': method,\n 'answers': {},\n 'num_results': num_results,\n 'event': event,\n 'errors': 0,\n 'discard': and_discard\n }\n\n LOG.debug('RPC Fanout Client: send multipart to {}: {!r}'.format(self.fanout, json.dumps(body)))\n self.socket.send_multipart([\n '{}'.format(self.fanout),\n json.dumps(body)\n ])\n LOG.debug('RPC Fanout Client: send multipart to {}: {!r} - done'.format(self.fanout, json.dumps(body)))\n\n gevent.sleep(0)\n\n return event\n\n def connect(self, context):\n if self.socket is not None:\n return\n\n self.context = context\n\n self.socket = context.socket(zmq.PUB)\n self.socket.bind('ipc:///var/run/minemeld/{}'.format(self.fanout))\n\n self.reply_socket = context.socket(zmq.REP)\n self.reply_socket.bind('ipc:///var/run/minemeld/{}:reply'.format(self.fanout))\n\n def disconnect(self):\n if self.socket is None:\n return\n\n self.socket.close(linger=0)\n self.reply_socket.close(linger=0)\n\n self.socket = None\n self.reply_socket = None\n\n\nclass ZMQRpcServerChannel(object):\n def __init__(self, name, obj, allowed_methods=None,\n method_prefix='', fanout=None):\n if allowed_methods is None:\n allowed_methods = []\n\n self.name = name\n self.obj = obj\n\n self.allowed_methods = allowed_methods\n self.method_prefix = method_prefix\n\n self.fanout = fanout\n self.context = None\n self.socket = None\n\n def _send_result(self, reply_to, id_, result=None, error=None):\n ans = {\n 'source': self.name,\n 'id': id_,\n 'result': result,\n 'error': error\n }\n\n if self.fanout is not None:\n reply_socket = self.context.socket(zmq.REQ)\n reply_socket.connect('ipc:///var/run/minemeld/{}'.format(reply_to))\n LOG.debug('RPC Server {} result to {}'.format(self.name, reply_to))\n reply_socket.send_json(ans)\n reply_socket.recv()\n LOG.debug('RPC Server {} result to {} - done'.format(self.name, reply_to))\n reply_socket.close(linger=0)\n LOG.debug('RPC Server {} result to {} - closed'.format(self.name, reply_to))\n reply_socket = None\n\n else:\n self.socket.send_multipart([reply_to, '', json.dumps(ans)])\n\n def run(self):\n if self.socket is None:\n LOG.error('Run called with invalid socket in RPC server channel: {}'.format(self.name))\n\n while True:\n LOG.debug('RPC Server receiving from {} - {}'.format(self.name, self.fanout))\n toks = self.socket.recv_multipart()\n LOG.debug('RPC Server recvd from {} - {}: {!r}'.format(self.name, self.fanout, toks))\n\n if self.fanout is not None:\n reply_to, body = toks\n reply_to = reply_to+':reply'\n else:\n reply_to, _, body = toks\n\n body = json.loads(body)\n LOG.debug('RPC command to {}: {!r}'.format(self.name, body))\n\n method = body.get('method', None)\n id_ = body.get('id', None)\n params = body.get('params', {})\n\n if method is None:\n LOG.error('No method in msg body')\n return\n if id_ is None:\n LOG.error('No id in msg body')\n return\n\n method = self.method_prefix+method\n\n if method not in self.allowed_methods:\n LOG.error('Method not allowed in RPC server channel {}: {}'.format(self.name, method))\n self._send_result(reply_to, id_, error='Method not allowed')\n\n m = getattr(self.obj, method, None)\n if m is None:\n LOG.error('Method {} not defined in RPC server channel {}'.format(method, self.name))\n self._send_result(reply_to, id_, error='Method not defined')\n\n try:\n result = m(**params)\n\n except gevent.GreenletExit:\n raise\n\n except Exception as e:\n self._send_result(reply_to, id_, error=str(e))\n\n else:\n self._send_result(reply_to, id_, result=result)\n\n def connect(self, context):\n if self.socket is not None:\n return\n\n self.context = context\n\n if self.fanout is not None:\n # we are subscribers\n self.socket = self.context.socket(zmq.SUB)\n self.socket.connect('ipc:///var/run/minemeld/{}'.format(self.fanout))\n self.socket.setsockopt(zmq.SUBSCRIBE, b'') # set the filter to empty to recv all messages\n\n else:\n # we are a router\n self.socket = self.context.socket(zmq.ROUTER)\n\n if self.name[0] == '@':\n address = 'ipc://@/var/run/minemeld/{}:rpc'.format(\n self.name[1:]\n )\n else:\n address = 'ipc:///var/run/minemeld/{}:rpc'.format(\n self.name\n )\n self.socket.bind(address)\n\n def disconnect(self):\n if self.socket is not None:\n self.socket.close(linger=0)\n self.socket = None\n\n\nclass ZMQPubChannel(object):\n def __init__(self, topic):\n self.socket = None\n self.reply_socket = None\n self.context = None\n self.topic = topic\n\n def publish(self, method, params=None):\n if self.socket is None:\n raise RuntimeError('Not connected')\n\n if params is None:\n params = {}\n\n id_ = str(uuid.uuid1())\n\n body = {\n 'method': method,\n 'id': id_,\n 'params': params\n }\n\n try:\n self.socket.send_json(\n obj=body,\n flags=zmq.NOBLOCK\n )\n except zmq.ZMQError:\n LOG.error('Topic {} queue full - dropping message'.format(self.topic))\n\n gevent.sleep(0)\n\n def connect(self, context):\n if self.socket is not None:\n return\n\n self.context = context\n\n self.socket = context.socket(zmq.PUB)\n self.socket.bind('ipc:///var/run/minemeld/{}'.format(self.topic))\n\n def disconnect(self):\n if self.socket is None:\n return\n\n self.socket.close(linger=0)\n self.socket = None\n\n\nclass ZMQSubChannel(object):\n def __init__(self, name, obj, allowed_methods=None,\n method_prefix='', topic=None):\n if allowed_methods is None:\n allowed_methods = []\n\n self.name = name\n self.obj = obj\n\n self.allowed_methods = allowed_methods\n self.method_prefix = method_prefix\n self.topic = topic\n\n self.context = None\n self.socket = None\n\n def run(self):\n if self.socket is None:\n LOG.error('Run called with invalid socket in ZMQ Pub channel: {}'.format(self.name))\n\n while True:\n LOG.debug('ZMQPub {} receiving'.format(self.name))\n body = self.socket.recv_json()\n LOG.debug('ZMQPub {} recvd: {!r}'.format(self.name, body))\n\n method = body.get('method', None)\n id_ = body.get('id', None)\n params = body.get('params', {})\n\n if method is None:\n LOG.error('No method in msg body')\n return\n if id_ is None:\n LOG.error('No id in msg body')\n return\n\n method = self.method_prefix+method\n\n if method not in self.allowed_methods:\n LOG.error('Method not allowed in RPC server channel {}: {}'.format(self.name, method))\n continue\n\n m = getattr(self.obj, method, None)\n if m is None:\n LOG.error('Method {} not defined in RPC server channel {}'.format(method, self.name))\n continue\n\n try:\n m(**params)\n\n except gevent.GreenletExit:\n raise\n\n except Exception:\n LOG.exception('Exception in ZMQPub {}'.format(self.name))\n\n def connect(self, context):\n if self.socket is not None:\n return\n\n self.context = context\n\n self.socket = self.context.socket(zmq.SUB)\n self.socket.connect('ipc:///var/run/minemeld/{}'.format(self.topic))\n self.socket.setsockopt(zmq.SUBSCRIBE, b'') # set the filter to empty to recv all messages\n\n def disconnect(self):\n if self.socket is not None:\n self.socket.close(linger=0)\n self.socket = None\n\n\nclass RedisSubChannel(object):\n def __init__(self, topic, connection_pool, object_,\n allowed_methods, name=None):\n self.topic = topic\n self.prefix = 'mm:topic:{}'.format(self.topic)\n self.channel = None\n self.name = name\n self.object = object_\n self.allowed_methods = allowed_methods\n self.connection_pool = connection_pool\n\n self.num_callbacks = 0\n\n self.sub_number = None\n\n def _callback(self, msg):\n try:\n msg = json.loads(msg)\n except ValueError:\n LOG.error(\"invalid message received\")\n return\n\n method = msg.get('method', None)\n params = msg.get('params', {})\n if method is None:\n LOG.error(\"Message without method field\")\n return\n\n if method not in self.allowed_methods:\n LOG.error(\"Method not allowed: %s\", method)\n return\n\n m = getattr(self.object, method, None)\n if m is None:\n LOG.error('Method %s not defined', method)\n return\n\n try:\n m(**params)\n\n except gevent.GreenletExit:\n raise\n\n except:\n LOG.exception('Exception in handling %s on topic %s '\n 'with params %s', method, self.topic, params)\n\n self.num_callbacks += 1\n\n def connect(self):\n subscribers_key = '{}:subscribers'.format(self.prefix)\n\n SR = redis.StrictRedis(\n connection_pool=self.connection_pool\n )\n\n self.sub_number = SR.rpush(\n subscribers_key,\n 0\n )\n self.sub_number -= 1\n LOG.debug('Sub Number {} on {}'.format(self.sub_number, subscribers_key))\n\n def disconnect(self):\n pass\n\n\nclass ZMQRedis(object):\n def __init__(self, config):\n self.context = None\n self.rpc_server_channels = {}\n self.pub_channels = []\n self.mw_pub_channels = []\n self.sub_channels = []\n self.mw_sub_channels = []\n self.rpc_fanout_clients_channels = []\n\n self.active_rpcs = {}\n\n self.ioloops = []\n\n self.failure_listeners = []\n\n self.redis_config = {\n 'url': os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock')\n }\n self.redis_cp = redis.ConnectionPool.from_url(\n self.redis_config['url']\n )\n\n def add_failure_listener(self, listener):\n self.failure_listeners.append(listener)\n\n def request_rpc_server_channel(self, name, obj=None, allowed_methods=None,\n method_prefix='', fanout=None):\n if allowed_methods is None:\n allowed_methods = []\n\n if name in self.rpc_server_channels:\n return\n\n self.rpc_server_channels[name] = ZMQRpcServerChannel(\n name,\n obj,\n method_prefix=method_prefix,\n allowed_methods=allowed_methods,\n fanout=fanout\n )\n\n def request_rpc_fanout_client_channel(self, topic):\n c = ZMQRpcFanoutClientChannel(topic)\n self.rpc_fanout_clients_channels.append(c)\n return c\n\n def request_pub_channel(self, topic, multi_write=False):\n if not multi_write:\n redis_pub_channel = RedisPubChannel(\n topic=topic,\n connection_pool=self.redis_cp\n )\n self.pub_channels.append(redis_pub_channel)\n\n return redis_pub_channel\n\n zmq_pub_channel = ZMQPubChannel(topic=topic)\n self.mw_pub_channels.append(zmq_pub_channel)\n \n return zmq_pub_channel\n\n def request_sub_channel(self, topic, obj=None, allowed_methods=None,\n name=None, max_length=None, multi_write=False):\n if allowed_methods is None:\n allowed_methods = []\n\n if not multi_write:\n subchannel = RedisSubChannel(\n topic=topic,\n connection_pool=self.redis_cp,\n object_=obj,\n allowed_methods=allowed_methods,\n name=name\n )\n self.sub_channels.append(subchannel)\n\n return\n\n subchannel = ZMQSubChannel(\n name=name,\n obj=obj,\n allowed_methods=allowed_methods,\n topic=topic\n )\n self.mw_sub_channels.append(subchannel)\n\n def send_rpc(self, dest, method, params,\n block=True, timeout=None):\n if self.context is None:\n LOG.error('send_rpc to {} when not connected'.format(dest))\n return\n\n id_ = str(uuid.uuid1())\n\n body = {\n 'method': method,\n 'id': id_,\n 'params': params\n }\n\n socket = self.context.socket(zmq.REQ)\n\n if dest[0] == '@':\n address = 'ipc://@/var/run/minemeld/{}:rpc'.format(\n dest[1:]\n )\n else:\n address = 'ipc:///var/run/minemeld/{}:rpc'.format(\n dest\n )\n\n socket.connect(address)\n socket.setsockopt(zmq.LINGER, 0)\n socket.send_json(body)\n LOG.debug('RPC sent to {}:rpc for method {}'.format(dest, method))\n\n if not block:\n socket.close(linger=0)\n return\n\n if timeout is not None:\n # zmq green does not support RCVTIMEO\n if socket.poll(flags=zmq.POLLIN, timeout=int(timeout*1000)) != 0:\n result = socket.recv_json(flags=zmq.NOBLOCK)\n\n else:\n socket.close(linger=0)\n raise RuntimeError('Timeout in RPC')\n\n else:\n result = socket.recv_json()\n\n socket.close(linger=0)\n\n return result\n\n def _ioloop(self, executor):\n executor.run()\n\n def _sub_ioloop(self, schannel):\n LOG.debug('start draining messages on topic {}'.format(schannel.topic))\n\n counter = 0\n SR = redis.StrictRedis(connection_pool=self.redis_cp)\n subscribers_key = '{}:subscribers'.format(schannel.prefix)\n\n while True:\n base = counter & 0xfff\n top = min(base + 127, 0xfff)\n\n msgs = SR.lrange(\n '{}:queue:{:013X}'.format(schannel.prefix, counter >> 12),\n base,\n top\n )\n\n for m in msgs:\n LOG.debug('topic {} - {!r}'.format(\n schannel.topic,\n m\n ))\n schannel._callback(m)\n\n counter += len(msgs)\n\n if len(msgs) > 0:\n SR.lset(\n subscribers_key,\n schannel.sub_number,\n counter\n )\n\n if len(msgs) < (top - base + 1):\n gevent.sleep(1.0)\n else:\n gevent.sleep(0)\n\n def _ioloop_failure(self, g):\n LOG.error('_ioloop_failure')\n\n try:\n g.get()\n\n except gevent.GreenletExit:\n return\n\n except:\n LOG.exception(\"_ioloop_failure: exception in ioloop\")\n for l in self.failure_listeners:\n l()\n\n def start(self, start_dispatching=True):\n self.context = zmq.Context()\n\n for rfcc in self.rpc_fanout_clients_channels:\n rfcc.connect(self.context)\n\n for rpcc in self.rpc_server_channels.values():\n rpcc.connect(self.context)\n\n for sc in self.sub_channels:\n sc.connect()\n\n for mwsc in self.mw_sub_channels:\n mwsc.connect(self.context)\n\n for pc in self.pub_channels:\n pc.connect()\n\n for mwpc in self.mw_pub_channels:\n mwpc.connect(self.context)\n\n if start_dispatching:\n self.start_dispatching()\n\n def start_dispatching(self):\n for rfcc in self.rpc_fanout_clients_channels:\n g = gevent.spawn(self._ioloop, rfcc)\n self.ioloops.append(g)\n g.link_exception(self._ioloop_failure)\n\n for rpcc in self.rpc_server_channels.values():\n g = gevent.spawn(self._ioloop, rpcc)\n self.ioloops.append(g)\n g.link_exception(self._ioloop_failure)\n\n for schannel in self.sub_channels:\n g = gevent.spawn(self._sub_ioloop, schannel)\n self.ioloops.append(g)\n g.link_exception(self._ioloop_failure)\n\n for mwschannel in self.mw_sub_channels:\n g = gevent.spawn(self._ioloop, mwschannel)\n self.ioloops.append(g)\n g.link_exception(self._ioloop_failure)\n\n def stop(self):\n # kill ioloops\n for j in xrange(len(self.ioloops)):\n self.ioloops[j].unlink(self._ioloop_failure)\n self.ioloops[j].kill()\n self.ioloops[j] = None\n self.ioloops = None\n\n # close channels\n for rpcc in self.rpc_server_channels.values():\n try:\n rpcc.disconnect()\n except Exception:\n LOG.debug(\"exception in disconnect: \", exc_info=True)\n\n for pc in self.pub_channels:\n try:\n pc.disconnect()\n except Exception:\n LOG.debug(\"exception in disconnect: \", exc_info=True)\n\n for mwpc in self.mw_pub_channels:\n try:\n mwpc.disconnect()\n except Exception:\n LOG.debug(\"exception in disconnect: \", exc_info=True)\n\n for sc in self.sub_channels:\n try:\n sc.disconnect()\n except Exception:\n LOG.debug(\"exception in disconnect: \", exc_info=True)\n\n for mwsc in self.mw_sub_channels:\n try:\n mwsc.disconnect()\n except Exception:\n LOG.debug(\"exception in disconnect: \", exc_info=True)\n\n for rfc in self.rpc_fanout_clients_channels:\n try:\n rfc.disconnect()\n except Exception:\n LOG.debug(\"exception in disconnect: \", exc_info=True)\n\n self.context.destroy()\n\n @staticmethod\n def cleanup(config):\n redis_cp = redis.ConnectionPool.from_url(\n os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock')\n )\n SR = redis.StrictRedis(connection_pool=redis_cp)\n tkeys = SR.keys(pattern='mm:topic:*')\n if len(tkeys) > 0:\n LOG.info('Deleting old keys: {}'.format(len(tkeys)))\n SR.delete(*tkeys)\n\n SR = None\n redis_cp = None\n" }, { "path": "minemeld/extensions/__init__.py", "content": "from .manager import * # noqa\n" }, { "path": "minemeld/extensions/manager.py", "content": "import sys\nimport os\nimport os.path\nimport json\nimport logging\nfrom email.parser import Parser\nfrom collections import namedtuple\nfrom zipfile import ZipFile\n\nfrom pkg_resources import EntryPoint, parse_version\n\nimport minemeld.loader\n\nLOG = logging.getLogger(__name__)\n\n\n__all__ = [\n 'get_metadata_from_wheel',\n 'activated_extensions',\n 'installed_extensions',\n 'extensions',\n 'freeze',\n 'load_frozen_paths'\n]\n\n\nMETADATA_MAP = {\n 'name': 'Name',\n 'version': 'Version',\n 'author': 'Author',\n 'author_email': 'Author-email',\n 'description': 'Summary',\n 'url': 'Home-page'\n}\n\n\nInstalledExtension = namedtuple(\n 'InstalledExtension',\n [\n 'name', 'version', 'author', 'author_email',\n 'description', 'url', 'path', 'entry_points'\n ]\n)\n\n\nActivatedExtension = namedtuple(\n 'ActivatedExtension',\n [\n 'name', 'version', 'author', 'author_email',\n 'description', 'url', 'location', 'entry_points'\n ]\n)\n\n\nExternalExtension = namedtuple(\n 'ExternalExtension',\n [\n 'name', 'version', 'author', 'author_email',\n 'description', 'url', 'path', 'activated',\n 'installed', 'entry_points'\n ]\n)\n\n\ndef _egg_link_path(dist):\n for path_item in sys.path:\n egg_link = os.path.join(path_item, dist.project_name + '.egg-link')\n if os.path.isfile(egg_link):\n return egg_link\n return None\n\n\ndef _read_metadata(metadata_str):\n return Parser().parsestr(metadata_str)\n\n\ndef _read_entry_points(ep_contents):\n ep_map = EntryPoint.parse_map(ep_contents)\n\n for _, epgroup in ep_map.iteritems():\n for epname, ep in epgroup.iteritems():\n epgroup[epname] = str(ep)\n\n return ep_map\n\n\ndef _activated_extensions():\n epgroups = (\n minemeld.loader.MM_NODES_ENTRYPOINT,\n minemeld.loader.MM_NODES_GCS_ENTRYPOINT,\n minemeld.loader.MM_NODES_VALIDATORS_ENTRYPOINT,\n minemeld.loader.MM_PROTOTYPES_ENTRYPOINT,\n minemeld.loader.MM_API_ENTRYPOINT,\n minemeld.loader.MM_WEBUI_ENTRYPOINT\n )\n\n activated_extensions = {}\n\n for epgroup in epgroups:\n for _, epvalue in minemeld.loader.map(epgroup).iteritems():\n if epvalue.ep.dist.project_name == 'minemeld-core':\n continue\n\n location = 'site-packages'\n egg_link = _egg_link_path(epvalue.ep.dist)\n if egg_link is not None:\n with open(egg_link, 'r') as f:\n location = f.readline().strip()\n\n metadata = {\n 'name': epvalue.ep.dist.project_name,\n 'version': epvalue.ep.dist.version,\n 'author': None,\n 'author_email': None,\n 'description': None,\n 'url': None,\n 'entry_points': None\n }\n if egg_link:\n try:\n with open(os.path.join(location, 'minemeld.json'), 'r') as f:\n dist_metadata = json.load(f)\n for k in metadata.keys():\n metadata[k] = dist_metadata.get(k, None)\n\n except (IOError, OSError) as excpt:\n LOG.error('Error loading metatdata from {}: {}'.format(location, str(excpt)))\n\n elif epvalue.ep.dist.has_metadata('METADATA'):\n dist_metadata = _read_metadata(\n epvalue.ep.dist.get_metadata('METADATA')\n )\n for k in metadata.keys():\n if k in METADATA_MAP and METADATA_MAP[k] in dist_metadata:\n metadata[k] = dist_metadata[METADATA_MAP[k]]\n\n if epvalue.ep.dist.has_metadata('entry_points.txt'):\n metadata['entry_points'] = _read_entry_points(\n epvalue.ep.dist.get_metadata('entry_points.txt')\n )\n\n activated_extensions[epvalue.ep.dist.project_name] = ActivatedExtension(\n location=location,\n **metadata\n )\n\n return activated_extensions\n\n\ndef _load_metadata_from_wheel(extpath, extname=None):\n wheel_name = extname\n if extname is None:\n wheel_name = os.path.basename(extpath)\n\n project_name, version, _ = wheel_name.split('-', 2)\n metadata_path = '{}-{}.dist-info/METADATA'.format(project_name, version)\n\n with ZipFile(extpath, 'r') as wheel_file:\n metadata_file = wheel_file.open(metadata_path, 'r')\n metadata_lines = metadata_file.read()\n\n metadata = _read_metadata(metadata_lines)\n\n # classifier framework :: minemeld should be in METADATA\n # for this to be an extension\n classifiers = metadata.get_all('Classifier')\n if classifiers is None:\n return None\n\n for c in classifiers:\n if c.lower() == 'framework :: minemeld':\n break\n else:\n return None\n\n ie_metadata = {}\n for field in InstalledExtension._fields:\n if field == 'path' or field == 'entry_points':\n continue\n ie_metadata[field] = metadata.get(METADATA_MAP[field], None)\n\n entry_points = None\n\n try:\n ep_path = '{}-{}.dist-info/entry_points.txt'.format(project_name, version)\n with ZipFile(extpath, 'r') as wheel_file:\n ep_file = wheel_file.open(ep_path, 'r')\n ep_contents = ep_file.read()\n\n entry_points = _read_entry_points(ep_contents)\n\n except (IOError, OSError):\n pass\n\n ie_metadata['entry_points'] = entry_points\n\n return InstalledExtension(\n path=extpath,\n **ie_metadata\n )\n\n\ndef _load_metadata_from_dir(extpath):\n with open(os.path.join(extpath, 'minemeld.json'), 'r') as f:\n metadata = json.load(f)\n\n return InstalledExtension(\n name=metadata['name'],\n version=metadata['version'],\n author=metadata['author'],\n author_email=metadata.get('author_email', None),\n description=metadata.get('description', None),\n url=metadata.get('url', None),\n entry_points=metadata.get('entry_points', None),\n path=extpath\n )\n\n\ndef _is_activated(installed_extension, activated):\n activated_extension = activated.get(installed_extension.name, None)\n if activated_extension is None:\n return False\n\n if activated_extension.version != installed_extension.version:\n return False\n\n if installed_extension.path == activated_extension.location:\n return True\n\n if activated_extension.location == 'site-packages' and \\\n installed_extension.path.endswith('.whl'):\n return True\n\n return False\n\n\ndef get_metadata_from_wheel(wheelpath, wheelname=None):\n return _load_metadata_from_wheel(wheelpath, wheelname)\n\n\ndef installed_extensions(installation_dir):\n _installed_extensions = []\n\n entries = os.listdir(installation_dir)\n\n for e in entries:\n epath = os.path.join(installation_dir, e)\n\n # check if this is a wheel\n if e.endswith('.whl'):\n try:\n installed_extension = _load_metadata_from_wheel(epath)\n if installed_extension is None:\n continue\n\n _installed_extensions.append(installed_extension)\n\n except (ValueError, IOError, KeyError, OSError) as excpt:\n LOG.error(u'Error extracting metadata from {}: {}'.format(e, str(excpt)))\n\n # check if it is a directory\n elif os.path.isdir(epath):\n try:\n installed_extension = _load_metadata_from_dir(epath)\n if installed_extension is None:\n continue\n\n _installed_extensions.append(installed_extension)\n\n except (IOError, OSError, KeyError) as excpt:\n LOG.error(u'Error extracting metadata from {}: {}'.format(e, str(excpt)))\n\n return _installed_extensions\n\n\ndef activated_extensions():\n return _activated_extensions()\n\n\ndef extensions(installation_dir):\n _extensions = []\n\n _installed = installed_extensions(installation_dir)\n _activated = activated_extensions()\n\n for installed_extension in _installed:\n _extension_activated = _is_activated(installed_extension, _activated)\n _extensions.append(ExternalExtension(\n installed=True,\n activated=_extension_activated,\n **installed_extension._asdict()\n ))\n if _extension_activated:\n _activated.pop(installed_extension.name)\n\n for _activated_extension in _activated.values():\n _adict = _activated_extension._asdict()\n _adict.pop('location')\n _extensions.append(ExternalExtension(\n installed=False,\n activated=True,\n path=None,\n **_adict\n ))\n\n return _extensions\n\n\ndef freeze(installation_dir):\n _freeze = []\n\n _extensions = extensions(installation_dir)\n\n for e in _extensions:\n if not e.activated:\n continue\n\n if not e.installed:\n continue\n\n if e.path.endswith('.whl'):\n _freeze.append(e.path)\n else:\n _freeze.append('-e {}'.format(e.path))\n\n return _freeze\n\n\ndef load_frozen_paths(freeze_file):\n for l in freeze_file:\n l = l.strip()\n if not l.startswith('-e '):\n continue\n\n _, epath = l.split(' ', 1)\n if epath not in sys.path:\n LOG.info('Extension path {!r} not in sys.path, adding'.format(epath))\n sys.path.append(epath)\n" }, { "path": "minemeld/fabric.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nminemeld.fabric\n\nThis module implements fabric abstraction over communication backend class.\nEach chassis has an instance of Fabric and nodes request connections to the\nfabric using this instance.\n\"\"\"\n\nfrom __future__ import absolute_import\n\nimport logging\n\nimport minemeld.comm\n\nLOG = logging.getLogger(__name__)\n\n\nclass Fabric(object):\n \"\"\"MineMeld chassis fabric class\n\n Args:\n chassis: MineMeld chassis instance\n config (dict): communication backend config\n comm_class (string): communication backend to be used\n \"\"\"\n def __init__(self, chassis, config, comm_class):\n self.chassis = chassis\n\n self.comm_config = config\n self.comm_class = comm_class\n\n self.comm = minemeld.comm.factory(self.comm_class, self.comm_config)\n\n def request_rpc_channel(self, ftname, node, allowed_methods):\n \"\"\"Creates a new RPC channel on the communication backend.\n\n Args:\n ftname (str): node name\n node: node instance\n allowed_methods (list): list of allowed methods\n \"\"\"\n self.comm.request_rpc_server_channel(ftname, node, allowed_methods)\n\n def request_pub_channel(self, ftname):\n \"\"\"Creates a new channel for publishing to a topic with name ftname.\n\n Args:\n ftname (str): node name\n \"\"\"\n return self.comm.request_pub_channel(ftname)\n\n def request_sub_channel(self, ftname, node, subname, allowed_methods):\n \"\"\"Creates a subscription channel to topic subname.\n\n Args:\n ftname (str): name of the node\n node: node instance\n subname (str): name of the topic to subscribe to\n allowed_methods (list): list of allowed methods\n \"\"\"\n _ = ftname # noqa\n self.comm.request_sub_channel(subname, node, allowed_methods)\n\n def send_rpc(self, sftname, dftname, method, params,\n block=True, timeout=None):\n \"\"\"Sends a RPC command to a specific node.\n\n Args:\n sftname (str): source node name\n dftname (str): destination node name\n method (str): method name\n params (dict): parameters\n block (bool): if call should block\n timeout (int): timeout in seconds\n \"\"\"\n params['source'] = sftname\n self.comm.send_rpc(\n dftname,\n method,\n params,\n block=block,\n timeout=timeout\n )\n\n def _comm_failure(self):\n self.chassis.fabric_failed()\n\n def start(self):\n LOG.debug(\"fabric start called\")\n self.comm.add_failure_listener(self._comm_failure)\n self.comm.start(start_dispatching=False)\n\n def start_dispatching(self):\n self.comm.start_dispatching()\n\n def stop(self):\n LOG.debug(\"fabric stop called\")\n self.comm.stop()\n\n\ndef factory(classname, chassis, config):\n \"\"\"Factory for Fabric class.\n\n Args:\n classname (str): communication backend name\n chassis: chassis instance\n config (dict): communication backend config\n \"\"\"\n return Fabric(\n chassis=chassis,\n config=config,\n comm_class=classname\n )\n" }, { "path": "minemeld/flask/__init__.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport logging\n\nimport yaml\nfrom flask import Flask\n\nimport minemeld.loader\nfrom .logger import LOG\n\nREDIS_URL = os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock')\n\n\ndef create_app():\n yaml.SafeLoader.add_constructor(\n u'tag:yaml.org,2002:timestamp',\n yaml.SafeLoader.construct_yaml_str\n )\n\n app = Flask(__name__)\n\n app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024 # max 5MB for uploads\n\n LOG.init_app(app)\n\n # extension code\n from . import config\n from . import aaa\n from . import session\n from . import mmrpc\n from . import redisclient\n from . import supervisorclient\n from . import jobs\n from . import sns\n from . import events\n\n session.init_app(app, REDIS_URL)\n aaa.init_app(app)\n\n config.init()\n if config.get('DEBUG', False):\n logging.getLogger().setLevel(logging.DEBUG)\n else:\n logging.getLogger().setLevel(logging.INFO)\n\n mmrpc.init_app(app)\n redisclient.init_app(app)\n supervisorclient.init_app(app)\n jobs.init_app(app)\n sns.init_app()\n events.init_app(app, REDIS_URL)\n\n # entrypoints\n from . import metricsapi # noqa\n from . import feedredis # noqa\n from . import configapi # noqa\n from . import configdataapi # noqa\n from . import taxiidiscovery # noqa\n from . import taxiicollmgmt # noqa\n from . import taxiipoll # noqa\n from . import supervisorapi # noqa\n from . import loginapi # noqa\n from . import prototypeapi # noqa\n from . import validateapi # noqa\n from . import aaaapi # noqa\n from . import statusapi # noqa\n from . import tracedapi # noqa\n from . import logsapi # noqa\n from . import extensionsapi # noqa\n from . import jobsapi # noqa\n\n configapi.init_app(app)\n extensionsapi.init_app(app)\n\n app.register_blueprint(metricsapi.BLUEPRINT)\n app.register_blueprint(statusapi.BLUEPRINT)\n app.register_blueprint(feedredis.BLUEPRINT)\n app.register_blueprint(configapi.BLUEPRINT)\n app.register_blueprint(configdataapi.BLUEPRINT)\n app.register_blueprint(taxiidiscovery.BLUEPRINT)\n app.register_blueprint(taxiicollmgmt.BLUEPRINT)\n app.register_blueprint(taxiipoll.BLUEPRINT)\n app.register_blueprint(supervisorapi.BLUEPRINT)\n app.register_blueprint(loginapi.BLUEPRINT)\n app.register_blueprint(prototypeapi.BLUEPRINT)\n app.register_blueprint(validateapi.BLUEPRINT)\n app.register_blueprint(aaaapi.BLUEPRINT)\n app.register_blueprint(tracedapi.BLUEPRINT)\n app.register_blueprint(logsapi.BLUEPRINT)\n app.register_blueprint(extensionsapi.BLUEPRINT)\n app.register_blueprint(jobsapi.BLUEPRINT)\n\n # install blueprints from extensions\n for apiname, apimmep in minemeld.loader.map(minemeld.loader.MM_API_ENTRYPOINT).iteritems():\n LOG.info('Loading blueprint from {}'.format(apiname))\n if not apimmep.loadable:\n LOG.info('API entrypoint {} not loadable, ignored'.format(apiname))\n continue\n\n try:\n bprint = apimmep.ep.load()\n app.register_blueprint(bprint())\n\n except (ImportError, RuntimeError):\n LOG.exception('Error loading API entry point {}'.format(apiname))\n\n # install webui blueprints from extensions\n for webuiname, webuimmep in minemeld.loader.map(minemeld.loader.MM_WEBUI_ENTRYPOINT).iteritems():\n LOG.info('Loading blueprint from {}'.format(webuiname))\n if not webuimmep.loadable:\n LOG.info('API entrypoint {} not loadable, ignored'.format(webuiname))\n continue\n\n try:\n bprint = webuimmep.ep.load()\n app.register_blueprint(\n bprint(),\n url_prefix='/extensions/webui/{}'.format(webuiname)\n )\n\n except (ImportError, RuntimeError):\n LOG.exception('Error loading WebUI entry point {}'.format(webuiname))\n\n for r in app.url_map.iter_rules():\n LOG.debug('app rule: {!r}'.format(r))\n\n return app\n" }, { "path": "minemeld/flask/aaa.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport json\nimport base64\nfrom functools import wraps\n\nimport gevent\nimport gevent.lock\nimport flask.ext.login\nfrom flask import current_app, Blueprint, request\n\nfrom . import config\nfrom .logger import LOG\n\n\nANONYMOUS = 'mm-anonymous'\nPREVENT_WRITE_GUARD = None\nPREVENT_WRITE = None\n\n\ndef disable_prevent_write(locker):\n global PREVENT_WRITE\n\n with PREVENT_WRITE_GUARD:\n if PREVENT_WRITE == locker:\n LOG.info('Disabled prevent write from locker {}'.format(locker))\n PREVENT_WRITE = None\n\n\ndef enable_prevent_write(locker, timeout=900):\n global PREVENT_WRITE\n\n def _cleanup_prevent_write():\n gevent.sleep(timeout)\n LOG.info('Checking if prevent write still enabled by locker {}'.format(locker))\n disable_prevent_write(locker)\n\n with PREVENT_WRITE_GUARD:\n if PREVENT_WRITE is None:\n PREVENT_WRITE = locker\n gevent.spawn(_cleanup_prevent_write)\n\n return False\n\n\nclass MMBlueprint(Blueprint):\n def __init__(self, *args, **kwargs):\n super(MMBlueprint, self).__init__(*args, **kwargs)\n\n self.send_static_file = self._login_required(\n super(MMBlueprint, self).send_static_file,\n login_required=True,\n read_write=False,\n feeds=False\n )\n\n def _audit(self, f, audit_required):\n if not audit_required:\n return f\n\n @wraps(f)\n def audited_view(*args, **kwargs):\n if request and flask.ext.login.current_user:\n params = []\n\n for key, values in request.values.iterlists():\n if key == '_':\n continue\n\n params.append(('value:{}'.format(key), values))\n\n for filename, files in request.files.iterlists():\n params.append(('file:{}'.format(filename), [file.filename for file in files]))\n\n body = request.get_json(silent=True)\n if body is not None:\n params.append(['jsonbody', json.dumps(body)[:1024]])\n\n LOG.audit(\n user_id=flask.ext.login.current_user.get_id(),\n action_name='{} {}'.format(request.method, request.path),\n params=params\n )\n\n else:\n LOG.critical('no request or current_user in audited_view')\n\n return f(*args, **kwargs)\n\n return audited_view\n\n def _login_required(self, f, login_required, read_write, feeds):\n @wraps(f)\n def decorated_view(*args, **kwargs):\n if not login_required:\n return f(*args, **kwargs)\n\n if not config.get('API_AUTH_ENABLED', True) and not feeds:\n return f(*args, **kwargs)\n\n if not config.get('FEEDS_AUTH_ENABLED', False) and feeds:\n return f(*args, **kwargs)\n\n if not feeds:\n if not flask.ext.login.current_user.is_authenticated():\n return current_app.login_manager.unauthorized()\n if flask.ext.login.current_user.get_id().startswith('feeds/'):\n return current_app.login_manager.unauthorized()\n\n if read_write and not flask.ext.login.current_user.is_read_write():\n return 'Forbidden', 403\n\n return f(*args, **kwargs)\n\n return decorated_view\n\n def _write_prevented(self, f, read_write):\n @wraps(f)\n def decorated_view(*args, **kwargs):\n if read_write and PREVENT_WRITE is not None:\n return 'Changes disabled by {}'.format(PREVENT_WRITE), 403\n\n return f(*args, **kwargs)\n\n return decorated_view\n\n def route(self, rule, **options):\n def decorator(f):\n login_required = options.pop('login_required', True)\n read_write = options.pop('read_write', True)\n feeds = options.pop(\"feeds\", False)\n\n super_decorator = super(MMBlueprint, self).route(rule, **options)\n\n _wp_f = self._write_prevented(f, read_write)\n _lr_f = self._login_required(_wp_f, login_required, read_write, feeds)\n _audit_f = self._audit(_lr_f, read_write)\n\n return super_decorator(_audit_f)\n\n return decorator\n\n\nclass MMAnonynmousUser(object):\n def __init__(self):\n self._id = ANONYMOUS\n\n def get_id(self):\n return self._id\n\n def is_authenticated(self):\n return False\n\n def is_active(self):\n return True\n\n def is_anonymous(self):\n return True\n\n def is_read_write(self):\n return False\n\n def check_feed(self, feedname):\n if not config.get('FEEDS_AUTH_ENABLED', False):\n return True\n\n fattributes = config.get('FEEDS_ATTRS', None)\n if fattributes is None or feedname not in fattributes:\n return False\n ftags = set(fattributes[feedname].get('tags', []))\n\n if 'anonymous' in ftags:\n return True\n\n return False\n\nclass MMAuthenticatedUser(object):\n def __init__(self, _id=None):\n self._id = unicode(_id)\n\n def get_id(self):\n return self._id\n\n def is_authenticated(self):\n return True\n\n def is_active(self):\n return True\n\n def is_anonymous(self):\n return False\n\n\nclass MMAuthenticatedAdminUser(MMAuthenticatedUser):\n def __init__(self, _id):\n super(MMAuthenticatedAdminUser, self).__init__(_id=u'admin/{}'.format(_id))\n\n def is_read_write(self):\n read_write = config.get('READ_WRITE', None)\n if read_write is None:\n return True\n\n if isinstance(read_write, str) or isinstance(read_write, unicode):\n read_write = read_write.split(',')\n elif not isinstance(read_write, list):\n LOG.error('Unknown READ_WRITE format')\n return False\n\n if self._id[6:] in read_write:\n return True\n\n return False\n\n def check_feed(self, feedname):\n return True\n\n\nclass MMAuthenticatedFeedUser(MMAuthenticatedUser):\n def __init__(self, _id):\n super(MMAuthenticatedFeedUser, self).__init__(_id=u'feeds/{}'.format(_id))\n\n def is_read_write(self):\n # this should never be called\n return False\n\n def check_feed(self, feedname):\n if not config.get('FEEDS_AUTH_ENABLED', False):\n return True\n\n fattributes = config.get('FEEDS_ATTRS', None)\n if fattributes is None or feedname not in fattributes:\n return False\n ftags = set(fattributes[feedname].get('tags', []))\n\n # if 'any' is present, any authenticated user can access\n # the feed\n if 'any' in ftags:\n return True\n\n uattributes = config.get('FEEDS_USERS_ATTRS', None)\n if uattributes is None or self._id[6:] not in uattributes:\n return False\n tags = set(uattributes[self._id[6:]].get('tags', []))\n\n return len(tags.intersection(ftags)) != 0\n\n\ndef authenticated_user_factory(_id):\n if _id.startswith('feeds/'):\n return MMAuthenticatedFeedUser(_id=_id[6:])\n\n if _id.startswith('admin/'):\n return MMAuthenticatedAdminUser(_id=_id[6:])\n\n if _id == ANONYMOUS:\n return MMAnonynmousUser()\n\n raise RuntimeError('Unknown user_id prefix: {}'.format(_id))\n\n\nLOGIN_MANAGER = flask.ext.login.LoginManager()\nLOGIN_MANAGER.session_protection = None\nLOGIN_MANAGER.anonymous_user = MMAnonynmousUser\n\n\n@LOGIN_MANAGER.request_loader\ndef request_loader(request):\n api_key = request.headers.get('Authorization')\n if api_key is None:\n return None\n\n api_key = api_key.replace('Basic', '', 1)\n\n try:\n api_key = base64.b64decode(api_key)\n except TypeError:\n return None\n\n try:\n user, password = api_key.split(':', 1)\n except ValueError:\n return None\n\n auth_user = check_feeds_user(user, password)\n if auth_user is not None:\n return auth_user\n\n auth_user = check_admin_user(user, password)\n if auth_user is not None:\n return auth_user\n\n return None\n\n\n@LOGIN_MANAGER.user_loader\ndef user_loader(_id):\n return authenticated_user_factory(_id)\n\n\ndef check_feeds_user(username, password):\n if not config.get('FEEDS_USERS_DB').check_password(username, password):\n return None\n\n return MMAuthenticatedFeedUser(_id=username)\n\n\ndef check_admin_user(username, password):\n if not config.get('USERS_DB').check_password(username, password):\n return None\n\n return MMAuthenticatedAdminUser(_id=username)\n\n\n@LOGIN_MANAGER.unauthorized_handler\ndef unauthorized():\n return 'Unauthorized', 401\n\n\ndef init_app(app):\n global PREVENT_WRITE\n global PREVENT_WRITE_GUARD\n\n app.config['REMEMBER_COOKIE_NAME'] = None # to block remember cookie\n LOGIN_MANAGER.init_app(app)\n\n # initialize PREVENT_WRITE\n PREVENT_WRITE_GUARD = gevent.lock.BoundedSemaphore()\n PREVENT_WRITE = None\n" }, { "path": "minemeld/flask/aaaapi.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport collections\n\nfrom flask import request, jsonify\nfrom flask.ext.login import current_user\n\nfrom . import config\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\n\n\n__all__ = ['BLUEPRINT']\n\n\nBLUEPRINT = MMBlueprint('aaa', __name__, url_prefix='/aaa')\n\n# if you change things here change also backup/import API\nAPI_USERS_ATTRS_ATTR = 'API_USERS_ATTRS'\nFEEDS_USERS_ATTRS_ATTR = 'FEEDS_USERS_ATTRS'\nFEEDS_ATTRS_ATTR = 'FEEDS_ATTRS'\n\n\nSubsystem = collections.namedtuple(\n 'Subsystem',\n ['authdb', 'attrs', 'enabled', 'enabled_default'],\n verbose=True\n)\n\n\n_SUBSYSTEM_MAP = {\n 'api': Subsystem(\n authdb='USERS_DB',\n enabled='API_AUTH_ENABLED',\n enabled_default=True,\n attrs=config.APIConfigDict(attribute=API_USERS_ATTRS_ATTR, level=50)\n ),\n 'feeds': Subsystem(\n authdb='FEEDS_USERS_DB',\n enabled='FEEDS_AUTH_ENABLED',\n enabled_default=False,\n attrs=config.APIConfigDict(attribute=FEEDS_USERS_ATTRS_ATTR, level=50)\n )\n}\n\n\n_FEEDS_ATTRS = config.APIConfigDict(attribute=FEEDS_ATTRS_ATTR, level=50)\n\n\n@BLUEPRINT.route('/users/current', methods=['GET'], read_write=False)\ndef get_current_user():\n return jsonify(result={\n 'id': current_user.get_id(),\n 'read_write': current_user.is_read_write()\n })\n\n\n@BLUEPRINT.route('/users/', methods=['GET'], read_write=False)\ndef get_users(subsystem):\n subsystem = _SUBSYSTEM_MAP.get(subsystem, None)\n if subsystem is None:\n return jsonify(error='Invalid subsystem'), 400\n\n result = {\n 'enabled': config.get(subsystem.enabled, subsystem.enabled_default),\n 'users': {}\n }\n users = config.get(subsystem.authdb).users()\n users_attrs = subsystem.attrs.value()\n for u in users:\n attrs = {}\n if u in users_attrs:\n attrs = users_attrs[u]\n result['users'][u] = attrs\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/users//', methods=['PUT'], read_write=True)\ndef set_user_password(subsystem, username):\n subsystem = _SUBSYSTEM_MAP.get(subsystem, None)\n if subsystem is None:\n return jsonify(error='Invalid subsystem'), 400\n\n with config.lock():\n users_db = config.get(subsystem.authdb)\n if not users_db.path:\n return jsonify(error='Users database not available'), 500\n\n try:\n password = request.get_json()['password']\n except Exception:\n return jsonify(error='Invalid request'), 400\n\n users_db.set_password(username, password)\n users_db.save()\n\n return jsonify(result='ok')\n\n\n@BLUEPRINT.route('/users///attributes', methods=['POST'], read_write=True)\ndef set_user_attributes(subsystem, username):\n subsystem = _SUBSYSTEM_MAP.get(subsystem, None)\n if subsystem is None:\n return jsonify(error='Invalid subsystem'), 400\n\n with config.lock():\n users_db = config.get(subsystem.authdb)\n if not users_db.path:\n return jsonify(error='Users database not available'), 500\n\n if username not in users_db.users():\n return jsonify(error='Unknown user'), 400\n\n try:\n attributes = request.get_json()\n except Exception:\n return jsonify(error='Invalid request'), 400\n\n if not isinstance(attributes, dict):\n return jsonify(error='Attributes should be a dict'), 400\n\n subsystem.attrs.set(username, attributes)\n\n return jsonify(result='ok')\n\n\n@BLUEPRINT.route('/users//', methods=['DELETE'], read_write=True)\ndef delete_user(subsystem, username):\n subsystem = _SUBSYSTEM_MAP.get(subsystem, None)\n if subsystem is None:\n return jsonify(error='Invalid subsystem'), 400\n\n with config.lock():\n users_db = config.get(subsystem.authdb)\n if not users_db.path:\n return jsonify(error='Users database not available'), 500\n\n # delete user from database and tags\n if users_db.delete(username):\n users_db.save()\n\n subsystem.attrs.delete(username)\n\n return jsonify(result='ok')\n\n\n@BLUEPRINT.route('/feeds', methods=['GET'], read_write=False)\ndef get_feeds():\n result = {\n 'enabled': config.get(\n _SUBSYSTEM_MAP['feeds'].enabled,\n _SUBSYSTEM_MAP['feeds'].enabled_default\n ),\n 'feeds': _FEEDS_ATTRS.value()\n }\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/feeds//attributes', methods=['PUT', 'POST'], read_write=True)\ndef set_feed_attributes(feedname):\n with config.lock():\n try:\n attributes = request.get_json()\n except Exception:\n return jsonify(error='Invalid request'), 400\n\n if not isinstance(attributes, dict):\n return jsonify(error='Attributes should be a dict'), 400\n\n _FEEDS_ATTRS.set(feedname, attributes)\n\n return jsonify(result='ok')\n\n\n@BLUEPRINT.route('/feeds/', methods=['DELETE'], read_write=True)\ndef delete_feed(feedname):\n with config.lock():\n _FEEDS_ATTRS.delete(feedname)\n\n return jsonify(result='ok')\n\n\n@BLUEPRINT.route('/tags', methods=['GET'], read_write=False)\ndef get_tags():\n tags = set()\n\n for _, subsystem in _SUBSYSTEM_MAP.iteritems():\n for _, attributes in subsystem.attrs.value().iteritems():\n if 'tags' in attributes:\n for t in attributes['tags']:\n tags.add(t)\n for _, attributes in _FEEDS_ATTRS.value().iteritems():\n if 'tags' in attributes:\n for t in attributes['tags']:\n tags.add(t)\n\n return jsonify(result=list(tags - set(['any', 'anonymous'])))\n" }, { "path": "minemeld/flask/cbfeed.py", "content": "import json\nimport time\n\n\nclass CbFeed(object):\n def __init__(self, feedinfo, reports):\n self.data = {'feedinfo': feedinfo,\n 'reports': reports}\n\n def dump(self):\n return json.dumps(self.data, indent=2)\n\n\nclass CbFeedInfo(object):\n def __init__(self, **kwargs):\n self.yieldable_atts = (\"category\", \"icon\", \"icon_small\", \"version\", \"provider_url\",\n \"display_name\", \"summary\", \"tech_data\", \"name\")\n self.data = kwargs\n self.data[\"category\"] = self.data.get(\"category\", \"MineMeld\")\n self.data[\"icon\"] = self.data.get(\"icon\", MinemeldIcon.MM_icon_png)\n self.data[\"icon_small\"] = self.data.get(\"icon_small\", MinemeldIcon.MM_icon_small_png)\n self.data[\"version\"] = self.data.get(\"version\", \"0.1\")\n self.data[\"provider_url\"] = self.data.get(\"provider_url\",\n \"https://live.paloaltonetworks.com/t5/MineMeld/ct-p/MineMeld\")\n self.data[\"display_name\"] = self.data.get(\"display_name\", \"MineMeld Feed\")\n self.data[\"summary\"] = self.data.get(\"summary\", \"Indicators routed through MineMeld\")\n self.data[\"tech_data\"] = self.data.get(\"tech_data\", \"Indicators routed through MineMeld\")\n if \"name\" not in kwargs:\n raise ValueError(\"Mandatory 'name' attribute not provided\")\n\n def dump(self):\n return self.data\n\n def iterate(self):\n last_element = len(self.yieldable_atts) - 1\n for idx, id in enumerate(self.yieldable_atts):\n final_comma = \",\" if idx < last_element else \"\"\n if isinstance(self.data[id], int):\n yield \"\\\"{}\\\": {}{}\".format(id, self.data[id], final_comma)\n else:\n yield \"\\\"{}\\\": \\\"{}\\\"{}\".format(id, self.data[id], final_comma)\n\n\nclass CbReport(object):\n def __init__(self, **kwargs):\n\n # these fields are optional\n self.optional = (\"tags\", \"description\")\n self.yieldable_atts = (\"timestamp\", \"id\", \"link\", \"score\", \"description\", \"title\")\n\n if \"timestamp\" not in kwargs:\n kwargs[\"timestamp\"] = int(time.mktime(time.gmtime()))\n\n if \"id\" not in kwargs:\n raise ValueError(\"Mandatory 'id' attribute not provided\")\n\n self.data = kwargs\n self.data[\"link\"] = self.data.get(\"link\", \"https://live.paloaltonetworks.com/t5/MineMeld/ct-p/MineMeld\")\n self.data[\"score\"] = self.data.get(\"score\", 100)\n if not isinstance(self.data[\"score\"], int):\n self.data[\"score\"] = 100\n self.data[\"description\"] = self.data.get(\"description\", \"MineMeld Generated Report\")\n self.data[\"iocs\"] = self.data.get(\"iocs\", None)\n self.data[\"title\"] = self.data.get(\"title\", \"MineMeld Generated Report\")\n\n def dump(self):\n return self.data\n\n def iterate(self):\n last_element = len(self.yieldable_atts) - 1\n for idx, id in enumerate(self.yieldable_atts):\n final_comma = \",\" if idx < last_element else \"\"\n if isinstance(self.data[id], int):\n yield \"\\\"{}\\\": {}{}\".format(id, self.data[id], final_comma)\n else:\n yield \"\\\"{}\\\": \\\"{}\\\"{}\".format(id, self.data[id], final_comma)\n\n\nclass MinemeldIcon(object):\n MM_icon_small_png = (\"iVBORw0KGgoAAAANSUhEUgAAADIAAAAyCAYAAAAeP4ixAAAMFmlDQ1BJQ0MgUHJvZmlsZ\"\n \"QAASImVVwdYU8kWnltSCAktEAEpoTdBehUIHQQB6WAjJAFCCZAQVOzIooJrQcWCFV0Bsa\"\n \"0FkLUiioVFwF4XRFRW1sWCDZU3KaDP1753vm/u/Dlzzpn/zD13MgOAsi07NzcLVQEgW5A\"\n \"vjAryZSYkJjFJPUABUAEFGACczRHl+kRGhgEoo/0/y7tbAJH0160lsf51/L+KKpcn4gCA\"\n \"REKcwhVxsiE+BgCuyckV5gNAaIN6o9n5uRI8CLG6EBIEgIhLcJoMa0pwigxPkNrERPlBz\"\n \"AKATGWzhWkAKEl4Mws4aTCOkoSjrYDLF0C8FWIvTjqbC/EDiCdkZ+dArEyG2Dzluzhp/x\"\n \"QzZSwmm502hmW5SIXszxflZrHn/p/L8b8lO0s8OochbNR0YXCUJGe4bjWZOaESTIX4pCA\"\n \"lPAJiNYgv8blSewm+ly4OjpXbD3BEfnDNAAMAFHDZ/qEQ60DMEGfG+sixPVso9YX2aDg/\"\n \"PyRGjlOEOVHy+GiBICs8TB5neTovZBRv54kCokdtUvmBIRDDSkOPFabHxMt4oi0F/Lhwi\"\n \"JUg7hBlRofKfR8VpvuFj9oIxVESzsYQv00VBkbJbDDNbNFoXpgNhy2dC9YCxspPjwmW+W\"\n \"IJPFFC2CgHLs8/QMYB4/IEsXJuGKwu3yi5b0luVqTcHtvOywqKkq0zdlhUED3q25UPC0y\"\n \"2DtjjDPbkSPlc73LzI2Nk3HAUhAE/4A+YQAxbCsgBGYDfPtAwAH/JRgIBGwhBGuABa7lm\"\n \"1CNeOiKAz2hQCP6CiAdEY36+0lEeKID6L2Na2dMapEpHC6QemeApxNm4Nu6Fe+Bh8MmCz\"\n \"R53xd1G/ZjKo7MSA4j+xGBiINFijAcHss6CTQj4/0YXCnsezE7CRTCaw7d4hKeETsJjwk\"\n \"1CN+EuiANPpFHkVrP4RcIfmDPBFNANowXKs0v5PjvcFLJ2wn1xT8gfcscZuDawxh1hJj6\"\n \"4N8zNCWq/Zyge4/ZtLX+cT8L6+3zkeiVLJSc5i5SxN+M3ZvVjFL/v1ogL+9AfLbHl2FGs\"\n \"FTuHXcZOYg2AiZ3BGrE27JQEj1XCE2kljM4WJeWWCePwR21s62z7bT//MDdbPr9kvUT5v\"\n \"Dn5ko/BLyd3rpCflp7P9IG7MY8ZIuDYTGDa29q5ACDZ22VbxxuGdM9GGFe+6fLOAuBWCp\"\n \"Vp33RsIwBOPAWA/u6bzug1LPc1AJzq4IiFBTKdZDsGBPiPoQy/Ci2gB4yAOczHHjgDD8A\"\n \"CAWAyiAAxIBHMhCueDrIh59lgPlgCSkAZWAM2gC1gB9gNasABcAQ0gJPgHLgIroIOcBPc\"\n \"h3XRB16AQfAODCMIQkJoCB3RQvQRE8QKsUdcES8kAAlDopBEJBlJQwSIGJmPLEXKkHJkC\"\n \"7ILqUV+RU4g55DLSCdyF+lB+pHXyCcUQ6moOqqLmqITUVfUBw1FY9AZaBqahxaixegqdB\"\n \"Nahe5H69Fz6FX0JtqNvkCHMIApYgzMALPGXDE/LAJLwlIxIbYQK8UqsCrsINYE3/N1rBs\"\n \"bwD7iRJyOM3FrWJvBeCzOwfPwhfhKfAteg9fjLfh1vAcfxL8SaAQdghXBnRBCSCCkEWYT\"\n \"SggVhL2E44QL8LvpI7wjEokMohnRBX6XicQM4jziSuI24iHiWWInsZc4RCKRtEhWJE9SB\"\n \"IlNyieVkDaT9pPOkLpIfaQPZEWyPtmeHEhOIgvIReQK8j7yaXIX+Rl5WEFFwUTBXSFCga\"\n \"swV2G1wh6FJoVrCn0KwxRVihnFkxJDyaAsoWyiHKRcoDygvFFUVDRUdFOcqshXXKy4SfG\"\n \"w4iXFHsWPVDWqJdWPOp0qpq6iVlPPUu9S39BoNFMai5ZEy6etotXSztMe0T4o0ZVslEKU\"\n \"uEqLlCqV6pW6lF4qKyibKPsoz1QuVK5QPqp8TXlARUHFVMVPha2yUKVS5YTKbZUhVbqqn\"\n \"WqEarbqStV9qpdVn6uR1EzVAtS4asVqu9XOq/XSMboR3Y/OoS+l76FfoPepE9XN1EPUM9\"\n \"TL1A+ot6sPaqhpOGrEaczRqNQ4pdHNwBimjBBGFmM14wjjFuPTON1xPuN441aMOziua9x\"\n \"7zfGaLE2eZqnmIc2bmp+0mFoBWplaa7UatB5q49qW2lO1Z2tv176gPTBefbzHeM740vFH\"\n \"xt/TQXUsdaJ05uns1mnTGdLV0w3SzdXdrHted0CPocfSy9Bbr3dar1+fru+lz9dfr39G/\"\n \"0+mBtOHmcXcxGxhDhroGAQbiA12GbQbDBuaGcYaFhkeMnxoRDFyNUo1Wm/UbDRorG88xX\"\n \"i+cZ3xPRMFE1eTdJONJq0m703NTONNl5k2mD430zQLMSs0qzN7YE4z9zbPM68yv2FBtHC\"\n \"1yLTYZtFhiVo6WaZbVlpes0KtnK34VtusOicQJrhNEEyomnDbmmrtY11gXWfdY8OwCbMp\"\n \"smmweTnReGLSxLUTWyd+tXWyzbLdY3vfTs1usl2RXZPda3tLe459pf0NB5pDoMMih0aHV\"\n \"45WjjzH7Y53nOhOU5yWOTU7fXF2cRY6H3TudzF2SXbZ6nLbVd010nWl6yU3gpuv2yK3k2\"\n \"4f3Z3d892PuP/tYe2R6bHP4/kks0m8SXsm9XoaerI9d3l2ezG9kr12enV7G3izvau8H7O\"\n \"MWFzWXtYzHwufDJ/9Pi99bX2Fvsd93/u5+y3wO+uP+Qf5l/q3B6gFxAZsCXgUaBiYFlgX\"\n \"OBjkFDQv6GwwITg0eG3w7RDdEE5IbcjgZJfJCya3hFJDo0O3hD4OswwThjVNQadMnrJuy\"\n \"oNwk3BBeEMEiAiJWBfxMNIsMi/yt6nEqZFTK6c+jbKLmh/VGk2PnhW9L/pdjG/M6pj7se\"\n \"ax4tjmOOW46XG1ce/j/ePL47sTJiYsSLiaqJ3IT2xMIiXFJe1NGpoWMG3DtL7pTtNLpt+\"\n \"aYTZjzozLM7VnZs08NUt5FnvW0WRCcnzyvuTP7Ah2FXsoJSRla8ogx4+zkfOCy+Ku5/bz\"\n \"PHnlvGepnqnlqc/TPNPWpfWne6dXpA/w/fhb+K8ygjN2ZLzPjMiszhzJis86lE3OTs4+I\"\n \"VATZApacvRy5uR05lrlluR257nnbcgbFIYK94oQ0QxRY746POa0ic3FP4l7CrwKKgs+zI\"\n \"6bfXSO6hzBnLa5lnNXzH1WGFj4yzx8Hmde83yD+Uvm9yzwWbBrIbIwZWHzIqNFxYv6Fgc\"\n \"trllCWZK55Pci26LyordL45c2FesWLy7u/Snop7oSpRJhye1lHst2LMeX85e3r3BYsXnF\"\n \"11Ju6ZUy27KKss8rOSuv/Gz386afR1alrmpf7bx6+xriGsGaW2u919aUq5YXlveum7Kuf\"\n \"j1zfen6txtmbbhc4VixYyNlo3hj96awTY2bjTev2fx5S/qWm5W+lYe26mxdsfX9Nu62ru\"\n \"2s7Qd36O4o2/FpJ3/nnV1Bu+qrTKsqdhN3F+x+uiduT+svrr/U7tXeW7b3S7Wgursmqqa\"\n \"l1qW2dp/OvtV1aJ24rn//9P0dB/wPNB60PrjrEONQ2WFwWHz4z1+Tf711JPRI81HXoweP\"\n \"mRzbepx+vLQeqZ9bP9iQ3tDdmNjYeWLyieYmj6bjv9n8Vn3S4GTlKY1Tq09TThefHjlTe\"\n \"GbobO7ZgXNp53qbZzXfP59w/kbL1Jb2C6EXLl0MvHi+1af1zCXPSycvu18+ccX1SsNV56\"\n \"v1bU5tx393+v14u3N7/TWXa40dbh1NnZM6T3d5d5277n/94o2QG1dvht/svBV7687t6be\"\n \"773DvPL+bdffVvYJ7w/cXPyA8KH2o8rDikc6jqj8s/jjU7dx9qse/p+1x9OP7vZzeF09E\"\n \"Tz73FT+lPa14pv+s9rn985P9gf0df077s+9F7ovhgZK/VP/a+tL85bG/WX+3DSYM9r0Sv\"\n \"hp5vfKN1pvqt45vm4cihx69y343/L70g9aHmo+uH1s/xX96Njz7M+nzpi8WX5q+hn59MJ\"\n \"I9MpLLFrKlRwEMNjQ1FYDX1QDQEuHZoQMAipLs7iUVRHZflCLwn7DsfiYVZwCqWQDELgY\"\n \"gDJ5RtsNmAjEV9pKjdwwLoA4OY00uolQHe1ksKrzBED6MjLzRBYDUBMAX4cjI8LaRkS97\"\n \"INm7AJzNk935JEKE5/udEyWoo++PQfCD/AMf7G3o0obnYAAAAAlwSFlzAAAWJQAAFiUBS\"\n \"VIk8AAAAgVpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD\"\n \"0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjp\"\n \"SREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50\"\n \"YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgI\"\n \"CAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iCiAgIC\"\n \"AgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4\"\n \"KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjEwMjI8L2V4aWY6UGl4ZWxZRGlt\"\n \"ZW5zaW9uPgogICAgICAgICA8ZXhpZjpQaXhlbFhEaW1lbnNpb24+OTE2PC9leGlmOlBpe\"\n \"GVsWERpbWVuc2lvbj4KICAgICAgICAgPHRpZmY6T3JpZW50YXRpb24+MTwvdGlmZjpPcm\"\n \"llbnRhdGlvbj4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC9\"\n \"4OnhtcG1ldGE+Cj6cgV0AAA+rSURBVGgFzVprcFXHfd/dc859IfGSBAaHGlkgg4TAiZKm\"\n \"7bSxnDpN/MhrYhHbsfOa1E0m0/RbP3Xq637px2YmM52J+6GeTp2HNAmNXRtjZxKRh5PJo\"\n \"EkISIDAwjgEMEQCrMe9uuec3f5+/z1HlpBkE7dOvXDv2bOP//u1e6XV26g5p/Sg6jeXh9\"\n \"eZzqkx19c3lGqt3NuIxDcmxbmqWW5Vtbr8+HJr/9/HBgb6g5yIp0fue++zo/d/4cCxT96\"\n \"Tj71lzNAEMuA6R/ZmnzkTT4/037B/9BPPPz92rxs6s9f9+Own3f6RTzx74MJDqwj7jZhZ\"\n \"Vp0rEUVgA64/oN2ib8GQWskkVoKxcJwC2bt3MD3kHo6MtvvWtkR3xHGa1qbjeOpKI17bW\"\n \"vhgOln7a+7ZdM9T81pbCCPvXzcjlByJ36sH04GRauH5F/vXkCGtq7a6gn3nSFZ6PjbcG3\"\n \"Lud6OTd5Ur5k8u/26uobTWkE+Eh6rXElUx7k+55nzvcMrnSu26GCETlNyzI/3rnxnt/2q\"\n \"TOjIS19Kx/aP3/veB4/3tVTDzZjTT2dssEcmq9F3awEqdMlCx9BxkZK1TrUW3g7CrWlmY\"\n \"wYr0rjiRc05p50xYlQytXR/8XVRQ20ygNqxrje62afotrqVmaCr5vut46j41lElZ714YZ\"\n \"AkHHGobW7UqtFtV7eRmgffIyIrw35CRLuU3W23/cX1bqefyxFwtbtjUpi6ZuFS3kOQf/3\"\n \"DsgVuJiDngOhiQJRAQrceNjPQXYEw7k8SCdEyRA+oFojFQSXPFrJ54Nb2Fm4aGLr05RsQ\"\n \"Z4ROAoZ2zfz47HVNUEd7p8AFwurVNgWovq24iansdRJxf2HIBjWt7E6zophjSBzwRBKkF\"\n \"D6ZgVApGVGLj3Qv3Ltd/AwnCMtGeO37fJnRuSmJarvLWDOZSq2xTUaly2Hgn1/X1tYGW6\"\n \"2s505EynYWSKUHDFjgEXw6hHCjvMNoJI5curQxfoka+8drn0NAQGbU2adwSRLoV6gcfYr\"\n \"8ZRq0iZ8FZKozoTHvYc90MwbH2FIqBatTTFJsQrWhWaEBUCZ2mppSzu5xDgNQ6FZvDGi5\"\n \"Z2F5XI2PNU15CRu8ulYXnlAP8EBLgGQO1wMZ3nD376RYCRoTxe/jyOu1g3xAoBAzt9oDI\"\n \"RZRZgGTSKAXazM0Rvr55cvLBGz245eG/LiPrxm8WZIiCu4W6BXLAuwtgZiZNKUJElbntR\"\n \"JRp0eNc4ZtSBTn20KGHI/R3JpS6l4//hoQKMKuCgUYS54JQrW80YoEPDMvSvOwgoRIZw6\"\n \"7H4HaJ4vmSNaKOIDZYRbIKDm9MQyJXc67FfOEyz8FBH92uFKa3ANNWREFqJPc9JAzANYg\"\n \"qoA7dtAJnCXTaQ1DDw5mVXAN3RUZyZN8/cv9G7OkQW9WMXrAoqIPISkAWos9kBkC9hN3b\"\n \"67V4DZ5Fr239PozGodpRKJhmCEkcHSDnG2jPONPi8HAO8cPxzErmF2adFRlR/X4FDKfDB\"\n \"LolVz+R0cz4JDKYl0oSvBknEqPDC7N++7LfY5lUtUp2RUUhgYx4uBQUGl0Sjk/T0An9UL\"\n \"kujtNKloO/IiN5eHSh7S5V6HoqAVwKH3B8KwWCVM/BNMDZ9osX+2/gzOhof8SyhghZZF5\"\n \"biz2c1U0IgT4/ZACFQMoEiMrIUtC6ZPhGA8HAuY4LFx7aQPi5tbCftxXDbx6zAfxW460X\"\n \"shEhSYgRZKHwpKEtBxNZ34gTSu1Cd/dgI0cAtPCzQRBSNVJgol7y5YzTB0b3dqfUpuiAU\"\n \"lKOwkcihI8IBAn3NOsoMi2plYBysa1taYZfnhFIglUuQUESu4hM6gmInWjxcRGQExmkZj\"\n \"AZr1oVRsmVmBn+B8+d2HtXmqT3gcAtyKAnQFsVxF/IC8tqVannT/ztJuTTrUyyaJrSF23\"\n \"jlY4eArbMeDaTVauC8NWplPB/2tfHLYub53vxmKqqqsjoqRMPtyLOb/NRxclaSVgZsiyq\"\n \"yG7mAlRH73nq6N5/KJXV05Xm4KFCUfeta4v+BpH2uWdOfmo1NTE8fE7sVLmr28NIr/U1F\"\n \"nMdiwYQj2fue6JvzwizIaW2x5O6NMMvq5HbsoxeSGodqbKbYy81b2BEhk+ODOkYPLhw6m\"\n \"pDTTSC+4tFF85MJxhUDSw09Xpqm5qinpmZ+P3Y9l/jjcvEGVub9lQQtqcbCReH0J7QTXn\"\n \"QP/g+rxIwIQwrX6rkAYXMA5a0ZTUy1twpGnFB0lVeJbx6M8s2cXeJw1jFviyGxV1tuLBW\"\n \"Zxmr+ClggkGtMDsTI6jZCYypprZpzmGf7YG20cMXFgkMvGKDKwUBEgvnME4GcUZpSK6x2\"\n \"86f72+TicxqfB8BIu8sfPIqRt6t3RPQECAbIuKHEuOzDDtmH10ZoBvVIVsQwlwj2gMQFz\"\n \"LRODMZhZVxrv3FE/tjPsFrN8O2mAzfuBaVllPBK0UT/SjE4Ze88IMmjGDtDTDfDg5cW0E\"\n \"sYYTE3X77UMLFTrseVKXsSkNPHB3lgy6Fws88gzSxOWQDjpJ0rsViGzIEKfPS+ztvPk8g\"\n \"1aqyPz31xQ2Ygu8hSyHfcZxrWTyiBH0xNG5fAG8H4fPIAZcOj6iQSMi+toJYwghQiZR/c\"\n \"vzvm+Fc27MDDyGSSmsA36hwIjLBBPtCAr6gDU1XwkLZT+LQg0aAwphjdPQBHKI4/Go82w\"\n \"EaN0rEYtTjUgxkxI9XIv0TmhLkIXMULiUT0FC1FUaurSCWMDKYnQhr7uJWbIKjSx0kxIH\"\n \"ItAKfiV34I8B8gX0oQrJyHXUxlcdcQML4zU00HZB4RMauKolYxsH3fJIV38M6uIGAwWlN\"\n \"j61p3TCSxOoKcodAEt2jx3oPYHdxMHd49tmWMJJndOtiImN2ZukujBBMhEoxToOfQf0/j\"\n \"2hj3oxUTUjyCylB/MPDhQ2U4aEpHCay3NFTl96KMz8XCfXCOdIk1wZBCO19bQ42+yKSLL\"\n \"dJcMBTZ1rqPHPmgXWcyK2HvSWMHOQoGqNKII4qgMgI8Rn6TKyjX4c6PSJZmWOguA7FySK\"\n \"hCiuRPExg9NycqylTHsOImv5lk/geproIB2rwRSh0hr1BvWYbFV06ybUAcDRi1vV4ZUAK\"\n \"V+U2RVG9nQMqsx72ljDyCC6OOeFc2sPrmLyhh6SkgpmpVCX16BSEeXJqKmF4ChLQU08QT\"\n \"4gXH6yj8hwJwb+XN8UbzxIOC77vHf887oDstsz3ZD2xRZA+aujfNBebXuZaqPMwPmyEyh\"\n \"6ZTpubeWVgpYBU6rVSZREjlBCJOHSuWgEyHHiEJ1kDaBYhEedede7iml1n67Z0FuHzYhE\"\n \"ENFCP0jeJkSjxn6woOjqC0rHu7mrj64d6SYEq2fgmDN6Y+V6O39JMgf/YO9u/eoXrEEgO\"\n \"17y9wn4lehG8ZTgHjXL2wXUHGZSWA5KXQbVX3iemzv0RKNnC0gQ0CVFCGBgBp2Nf2f6Vu\"\n \"Q0bBqcB8iTtGPicODpQEXL2kOIJFP2awDtbmsXREzt7S7mMjIdCC+tInOccPWOc+BKH0r\"\n \"R4Ej4xBcKB3+sGPXF4ECkOj2I095/FpsXfJQjEuPotvNngLQmwiJY4zvAHA/IRSBaqoxy\"\n \"rscBf0MgMjN740iYQRpTaKisCHe6m72EH/YWi5mFDGhL4L3xH6c2bd/wWyF7KHF4kD3aQ\"\n \"GCX3dJ6U2k3xDtrT7EH4785en9FTF++JCkAG+4KIRa9Api0tzYa/yveAw18mUAUtINMbp\"\n \"UuyXGCAtGbrQaJHub6WvCLEwKnO0ozQWORABE6vay1FM6+mP7yze/BJTjhc9DHvoDsqeU\"\n \"jGOKMQPChbtWX16tl2DjyS3T4KNxxgO/jokKgKPtWTOZqf8N9BbQYUR2YkH4xUcvTqdKq\"\n \"Q0RnffEMHFMN5Jb6eqc+Vz3Dizm3vldLkQzu/9W+4rfwPzBtkcjBjLC6vnwhLLR/nOn8I\"\n \"y5zY6iM0JzYPFkJlLquEOAn5E+Pw8PhijVBFLB8GBgYCZ92ORVEF8RJ1EMFdKOu14wIZX\"\n \"wjmp+qJe0VB/MDkJe6RSmAAnLEPv/uxWSCH8qrzd8N3dX33MzbR727U3Ud0EnTf1f3dBz\"\n \"/Q8dhVf1nNUn/+GuqIaADc5jjxlGiIiJpleH8RTvVK6+ryd7wtuw68o+GyAw9NGeQhytt\"\n \"CIQhQhpx83/avXcq2aL3xyVeOnP74CdRTG01sYxCMe0ex+QQRjj8NvMC1Q0N9UM8QArS4\"\n \"rcgYmXkYU/x4LTwKPN6csguMYTi/PlavJ7PwwwovKLBUGGJaQKj3x2RchFNQ84zkx8dY1\"\n \"bbDP5rgVGRBNjJosA4CGWJWPI9zPYtLBLbHipXgfQiVRayDJ6pg7fpi+epkfBqR5+sk9G\"\n \"BfH4gYYnfel3iWZxXRh7mcAVkgXz4azcw0/6ZcunIGDr+T5xqQAd/BYQZFHZ6dzn22pPX\"\n \"jdWpyocoyOHF3AbcK8MH8VpEmI0UbJCwRKGOCrq97O/Y9MXVl7p9gRq4QSdGfgolnorB4\"\n \"xz27v3FZfiDKJJ0hkAeP0hTEUiY8s5Rye/vjddjECUYuvOemK9eoCN9bLl2a2uJhjniJ8\"\n \"yW/bOBdbFY9Yy8jFv+rsD6bKhMUXwu91E8G+q927nsENVJH2tB/meryzju7vnP3HZ3fGM\"\n \"9/IPLIfr9vb45Eog5DgPOb8Y7LTWdR0hdtHO/kBH9uENPCJNb6ywZs7ZJ7KjDhd+McCkd\"\n \"PYzdZUuYUx3KmX7P5KvZXT2OKHyLXvLLJbyo59mabMeYwcwcaYzZFR76SIqJePbC8jJCQ\"\n \"nflIFURX3ffH798Y1+baJaOz9PESByP4oSJRp2/vevwCNqr+/gHYvOeTzHBv7jdkEkLBf\"\n \"C4Y7vj9W/4TBaR/fLaGDIaaDpjo8IKYDoMkJw5PPxNGsmOjbcym22BsG7JbxfxkgYxOV7\"\n \"JHSQ4JhkhERHzP2/+F9HNY/ukdfvPmfcfOn/voC62thb+YmGjMQXAR/CNB5MIvGvodXEs\"\n \"/E2fPLxsQDxYdeEQn4B/Mgw9fB12+2Scgj+yt+/Zm6/+QIAjd5ycnG6daWopl3J+Fa9ZE\"\n \"ZSlIjf53UuAQAUUj63ovU2VkrUdqcUQIAJIGHrTcazktoTf/JdbPvrXf9FtaAArUU6dPf\"\n \"+w9brLxJWD8M1COFO3+84Ybn8x+iB1MhZHRRwcpc5rhVj7RkeM3ng0cZ8u1aXu6bDZKcj\"\n \"v46Gs5Qda+xV9yaU1zbh9kef/PC9H5IOU9WUzrttv65Inj9zc5XGkKy0VcLq1eUyjPTqf\"\n \"1wJjP3d79r9NyIY0/GlgI7A/R13IDXzXO9fG2GTmNff8XGDn+zIDklX23/+i996EQ+DL4\"\n \"KYGpQ84E/3L3zm+P/W9yQo7sD/Ykt8shoyaWG39bj2VEC0PsUxNva4Iz4v4H8vqTm++oi\"\n \"2AAAAAASUVORK5C\")\n\n MM_icon_png = (\"iVBORw0KGgoAAAANSUhEUgAAASwAAAFLCAYAAABsjLGXAAAMFmlDQ1BJQ0MgUHJvZmlsZ\"\n \"QAASImVVwdYU8kWnltSCAktEAEpoTdBehUIHQQB6WAjJAFCCZAQVOzIooJrQcWCFV0Bsa\"\n \"0FkLUiioVFwF4XRFRW1sWCDZU3KaDP1753vm/u/Dlzzpn/zD13MgOAsi07NzcLVQEgW5A\"\n \"vjAryZSYkJjFJPUABUAEFGACczRHl+kRGhgEoo/0/y7tbAJH0160lsf51/L+KKpcn4gCA\"\n \"REKcwhVxsiE+BgCuyckV5gNAaIN6o9n5uRI8CLG6EBIEgIhLcJoMa0pwigxPkNrERPlBz\"\n \"AKATGWzhWkAKEl4Mws4aTCOkoSjrYDLF0C8FWIvTjqbC/EDiCdkZ+dArEyG2Dzluzhp/x\"\n \"QzZSwmm502hmW5SIXszxflZrHn/p/L8b8lO0s8OochbNR0YXCUJGe4bjWZOaESTIX4pCA\"\n \"lPAJiNYgv8blSewm+ly4OjpXbD3BEfnDNAAMAFHDZ/qEQ60DMEGfG+sixPVso9YX2aDg/\"\n \"PyRGjlOEOVHy+GiBICs8TB5neTovZBRv54kCokdtUvmBIRDDSkOPFabHxMt4oi0F/Lhwi\"\n \"JUg7hBlRofKfR8VpvuFj9oIxVESzsYQv00VBkbJbDDNbNFoXpgNhy2dC9YCxspPjwmW+W\"\n \"IJPFFC2CgHLs8/QMYB4/IEsXJuGKwu3yi5b0luVqTcHtvOywqKkq0zdlhUED3q25UPC0y\"\n \"2DtjjDPbkSPlc73LzI2Nk3HAUhAE/4A+YQAxbCsgBGYDfPtAwAH/JRgIBGwhBGuABa7lm\"\n \"1CNeOiKAz2hQCP6CiAdEY36+0lEeKID6L2Na2dMapEpHC6QemeApxNm4Nu6Fe+Bh8MmCz\"\n \"R53xd1G/ZjKo7MSA4j+xGBiINFijAcHss6CTQj4/0YXCnsezE7CRTCaw7d4hKeETsJjwk\"\n \"1CN+EuiANPpFHkVrP4RcIfmDPBFNANowXKs0v5PjvcFLJ2wn1xT8gfcscZuDawxh1hJj6\"\n \"4N8zNCWq/Zyge4/ZtLX+cT8L6+3zkeiVLJSc5i5SxN+M3ZvVjFL/v1ogL+9AfLbHl2FGs\"\n \"FTuHXcZOYg2AiZ3BGrE27JQEj1XCE2kljM4WJeWWCePwR21s62z7bT//MDdbPr9kvUT5v\"\n \"Dn5ko/BLyd3rpCflp7P9IG7MY8ZIuDYTGDa29q5ACDZ22VbxxuGdM9GGFe+6fLOAuBWCp\"\n \"Vp33RsIwBOPAWA/u6bzug1LPc1AJzq4IiFBTKdZDsGBPiPoQy/Ci2gB4yAOczHHjgDD8A\"\n \"CAWAyiAAxIBHMhCueDrIh59lgPlgCSkAZWAM2gC1gB9gNasABcAQ0gJPgHLgIroIOcBPc\"\n \"h3XRB16AQfAODCMIQkJoCB3RQvQRE8QKsUdcES8kAAlDopBEJBlJQwSIGJmPLEXKkHJkC\"\n \"7ILqUV+RU4g55DLSCdyF+lB+pHXyCcUQ6moOqqLmqITUVfUBw1FY9AZaBqahxaixegqdB\"\n \"Nahe5H69Fz6FX0JtqNvkCHMIApYgzMALPGXDE/LAJLwlIxIbYQK8UqsCrsINYE3/N1rBs\"\n \"bwD7iRJyOM3FrWJvBeCzOwfPwhfhKfAteg9fjLfh1vAcfxL8SaAQdghXBnRBCSCCkEWYT\"\n \"SggVhL2E44QL8LvpI7wjEokMohnRBX6XicQM4jziSuI24iHiWWInsZc4RCKRtEhWJE9SB\"\n \"IlNyieVkDaT9pPOkLpIfaQPZEWyPtmeHEhOIgvIReQK8j7yaXIX+Rl5WEFFwUTBXSFCga\"\n \"swV2G1wh6FJoVrCn0KwxRVihnFkxJDyaAsoWyiHKRcoDygvFFUVDRUdFOcqshXXKy4SfG\"\n \"w4iXFHsWPVDWqJdWPOp0qpq6iVlPPUu9S39BoNFMai5ZEy6etotXSztMe0T4o0ZVslEKU\"\n \"uEqLlCqV6pW6lF4qKyibKPsoz1QuVK5QPqp8TXlARUHFVMVPha2yUKVS5YTKbZUhVbqqn\"\n \"WqEarbqStV9qpdVn6uR1EzVAtS4asVqu9XOq/XSMboR3Y/OoS+l76FfoPepE9XN1EPUM9\"\n \"TL1A+ot6sPaqhpOGrEaczRqNQ4pdHNwBimjBBGFmM14wjjFuPTON1xPuN441aMOziua9x\"\n \"7zfGaLE2eZqnmIc2bmp+0mFoBWplaa7UatB5q49qW2lO1Z2tv176gPTBefbzHeM740vFH\"\n \"xt/TQXUsdaJ05uns1mnTGdLV0w3SzdXdrHted0CPocfSy9Bbr3dar1+fru+lz9dfr39G/\"\n \"0+mBtOHmcXcxGxhDhroGAQbiA12GbQbDBuaGcYaFhkeMnxoRDFyNUo1Wm/UbDRorG88xX\"\n \"i+cZ3xPRMFE1eTdJONJq0m703NTONNl5k2mD430zQLMSs0qzN7YE4z9zbPM68yv2FBtHC\"\n \"1yLTYZtFhiVo6WaZbVlpes0KtnK34VtusOicQJrhNEEyomnDbmmrtY11gXWfdY8OwCbMp\"\n \"smmweTnReGLSxLUTWyd+tXWyzbLdY3vfTs1usl2RXZPda3tLe459pf0NB5pDoMMih0aHV\"\n \"45WjjzH7Y53nOhOU5yWOTU7fXF2cRY6H3TudzF2SXbZ6nLbVd010nWl6yU3gpuv2yK3k2\"\n \"4f3Z3d892PuP/tYe2R6bHP4/kks0m8SXsm9XoaerI9d3l2ezG9kr12enV7G3izvau8H7O\"\n \"MWFzWXtYzHwufDJ/9Pi99bX2Fvsd93/u5+y3wO+uP+Qf5l/q3B6gFxAZsCXgUaBiYFlgX\"\n \"OBjkFDQv6GwwITg0eG3w7RDdEE5IbcjgZJfJCya3hFJDo0O3hD4OswwThjVNQadMnrJuy\"\n \"oNwk3BBeEMEiAiJWBfxMNIsMi/yt6nEqZFTK6c+jbKLmh/VGk2PnhW9L/pdjG/M6pj7se\"\n \"ax4tjmOOW46XG1ce/j/ePL47sTJiYsSLiaqJ3IT2xMIiXFJe1NGpoWMG3DtL7pTtNLpt+\"\n \"aYTZjzozLM7VnZs08NUt5FnvW0WRCcnzyvuTP7Ah2FXsoJSRla8ogx4+zkfOCy+Ku5/bz\"\n \"PHnlvGepnqnlqc/TPNPWpfWne6dXpA/w/fhb+K8ygjN2ZLzPjMiszhzJis86lE3OTs4+I\"\n \"VATZApacvRy5uR05lrlluR257nnbcgbFIYK94oQ0QxRY746POa0ic3FP4l7CrwKKgs+zI\"\n \"6bfXSO6hzBnLa5lnNXzH1WGFj4yzx8Hmde83yD+Uvm9yzwWbBrIbIwZWHzIqNFxYv6Fgc\"\n \"trllCWZK55Pci26LyordL45c2FesWLy7u/Snop7oSpRJhye1lHst2LMeX85e3r3BYsXnF\"\n \"11Ju6ZUy27KKss8rOSuv/Gz386afR1alrmpf7bx6+xriGsGaW2u919aUq5YXlveum7Kuf\"\n \"j1zfen6txtmbbhc4VixYyNlo3hj96awTY2bjTev2fx5S/qWm5W+lYe26mxdsfX9Nu62ru\"\n \"2s7Qd36O4o2/FpJ3/nnV1Bu+qrTKsqdhN3F+x+uiduT+svrr/U7tXeW7b3S7Wgursmqqa\"\n \"l1qW2dp/OvtV1aJ24rn//9P0dB/wPNB60PrjrEONQ2WFwWHz4z1+Tf711JPRI81HXoweP\"\n \"mRzbepx+vLQeqZ9bP9iQ3tDdmNjYeWLyieYmj6bjv9n8Vn3S4GTlKY1Tq09TThefHjlTe\"\n \"GbobO7ZgXNp53qbZzXfP59w/kbL1Jb2C6EXLl0MvHi+1af1zCXPSycvu18+ccX1SsNV56\"\n \"v1bU5tx393+v14u3N7/TWXa40dbh1NnZM6T3d5d5277n/94o2QG1dvht/svBV7687t6be\"\n \"773DvPL+bdffVvYJ7w/cXPyA8KH2o8rDikc6jqj8s/jjU7dx9qse/p+1x9OP7vZzeF09E\"\n \"Tz73FT+lPa14pv+s9rn985P9gf0df077s+9F7ovhgZK/VP/a+tL85bG/WX+3DSYM9r0Sv\"\n \"hp5vfKN1pvqt45vm4cihx69y343/L70g9aHmo+uH1s/xX96Njz7M+nzpi8WX5q+hn59MJ\"\n \"I9MpLLFrKlRwEMNjQ1FYDX1QDQEuHZoQMAipLs7iUVRHZflCLwn7DsfiYVZwCqWQDELgY\"\n \"gDJ5RtsNmAjEV9pKjdwwLoA4OY00uolQHe1ksKrzBED6MjLzRBYDUBMAX4cjI8LaRkS97\"\n \"INm7AJzNk935JEKE5/udEyWoo++PQfCD/AMf7G3o0obnYAAAAAlwSFlzAAAWJQAAFiUBS\"\n \"VIk8AAAAgRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD\"\n \"0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjp\"\n \"SREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50\"\n \"YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgI\"\n \"CAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iCiAgIC\"\n \"AgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4\"\n \"KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjk5NjwvZXhpZjpQaXhlbFlEaW1l\"\n \"bnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj45MDI8L2V4aWY6UGl4Z\"\n \"WxYRGltZW5zaW9uPgogICAgICAgICA8dGlmZjpPcmllbnRhdGlvbj4xPC90aWZmOk9yaW\"\n \"VudGF0aW9uPgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g\"\n \"6eG1wbWV0YT4KTlGaRAAAQABJREFUeAHsvQmYXNlVJvi2WHPRmirtS2qXajNZA8ZuqJDB\"\n \"XxsGA20mhdtuFg9QYMyOh6U/BoWmjcHdA3xA21DFzLDaMMpv+gNc2GCXrZRtDIbKqrJdU\"\n \"qm0lfaUlJJyz4h46/z/vfFSKeUWEflexI1U3CplbO/dd+6555577lk1rdVaGGhhoIWBFg\"\n \"ZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIW\"\n \"BFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgYeCgzoD8UoW4NsYUARDARBgDV3VO/r\"\n \"O6l3dR2cXn9DQyeD3t6DgablA13X8NpqLQy0MNDCQJ0xEASafizoNY8fz1n5QDMqeTyv5\"\n \"z/eW8n1D9M1LYQ8TLPdGmvdMJAP8saGgU+aP/7UgDPzoZCwrK9c+4P1CTfTYSTc1UEwld\"\n \"b1dKFgF+52ZFcMP/bIj92ccb1+/HjePHQo78747qF+22JYD/X0twYfNQbIqJ7u7zcOHeq\"\n \"fZjJfPpff5enFb/W94rd6vn3QDSZ3ekExbeiJFI5/hh8E+N8pWXp2zDCSr+pG6gupoOPv\"\n \"3rrnw68QvnxeM44cyWu6nvejhrfZ+msxrGabsRa8SmKAx7d+LWce0iWjOnvnLzuv3vnKu\"\n \"xxv/D1+UHxrpk3P6oanua6tObamBWA9vu8LZRWYlmbopmaYgZZMmZppWtrkuOaZWuazyc\"\n \"Ta//b0rv/z8xz08SBnhf0riYQ6ANViWHVAcusRyxsD1Dcd1vs8jvLkrWPtg8P9P+75Yz+\"\n \"ZyjrdWuBpxaKn+R7eaLqvQ06CRt0Q2imhgJ+JG/wEaQvsTNMNzcq2W5pdMPEp/acr/L0f\"\n \"/Kb9//kOdWEzpbeZdz8M71sM62GY5dYYY8PAsy/2JEI91Qun3/+fvGD0aLbD7y4Vbc0u+\"\n \"WRS4E9gP1pQkcJ9GlAdzC3QPPxvda5M6ZOj+uWEseo9b9v3R/+UP65Z+UPa9JFz+p6H4E\"\n \"2LYT0Ek9waYjwYOA7GcQiM4+zg73ddHHv5jxKpqXe5rqPZRd8Fj8LaCswInkwXByeVMZJ\"\n \"2UfdSxqp3vW3fH//dwypptRhWBBTV6uLhwgD1VX041h3WNe9L537y0KQz/PFMu7thYtzx\"\n \"tIASlWZFjpFAd6yklvCh2Uqaa7/rbXv+8O8fRqZVnZga+Sy0OmxhoLkwQGZ19Kimk1mdO\"\n \"PPMeybsW5+zkqUNE2OurQc8+sXArIgiPUh4juYYpqeV3Nt9J87/8mPUZZFpNRcGlwZti2\"\n \"EtDX+tux8yDBzt10y4GfjHz/zAT3jm8Mc1w9PtokZfqySVVXGiAxr5hOsEdqZNyxRKV/7\"\n \"sxeDFBJlWPp9/aNZxrAiOc/JafbcwUG8MhDqrL5372e8vaYN/bTsO3ROo/K63lON0rEgm\"\n \"7ImV//u37Xv2Q8cCDVZKTVgp642Tej/voeHM9UZs63nLCwO0BlLB/qULv/otE/aNv3KgX\"\n \"Ncaw6wgx+nG1KStuf74L754/fe2klkdO9YbhYJf+UlrMSzlp6gFYKMxQGZA14WvXfr4qq\"\n \"nStT9Pph098HQHR8B6S1YSFUFgeq7mZDv9lePjZ9/HL7t6hx6K01JjEN5oCmw9v4WBSjE\"\n \"AJfthrU+ExAwVjv/3dHth+8RYAF91LVlpFzFdZ9i2rcFBtRfxiR/WdTBQOKKi0Uq5bFtL\"\n \"wlq2U9saWBQYeHbgGQuq9OALZ3/+e4zkxHvGxxwyr0QUfS+xDwP+XhrCfg7864Vfe4J99\"\n \"fcfWvbHwpaEtUSqad2+fDFQllgouVifOf1D/0fCcsC7dDKshq8bxh9C+POybZo5XrzxjY\"\n \"DpxeU7E/dG1pKw7uGi9a6FgfswEEos/Wd/5j+lsqXHi1OeBz6hhBQjcmUhdlp4fvnuXgJ\"\n \"+Itcvjq73DWKZfWj4TrHM8NkazjLBQFm6cvFqfvb0D33AQrQNvTfj9rWqFn0IqoaxsiCO\"\n \"hHkdJ0SIXcs5Y2lLwqqWQlrXPxQY6O8/KiSpL1745ZxhlZ4qTrnMbayEdDU9AdS0u65m6\"\n \"enNQXBshfwebGsZtxbDWsaT2xpa7RgYyuWFtc12br8rmYbYopmuatIVbAFGqRRoCb24bX\"\n \"j4lS1ytCdbDKv2aW/d2cJA82GAx6rQc9z1x3o8HLvwn3KMgAzUx4kwm9EtvNklMX2vsEX\"\n \"zYX5xiFsS1uI4al3xkGGgr69XrIvPvd67KQi8XY4tMoMqt1bIQeF65XW24dUfOsBpGhj4\"\n \"pHKMNUryaSnda8SmsNJg3w1LNrEbWabpCLZi0EyrVFONmG38bV1d0mvccYNupLVa47m+n\"\n \"NHGgzYXBHoS4Yy6Ye3jjz09A8s6prDFsOYigTm+Y0T8009rxpmOT+qDIApYYmBCFpuZ0H\"\n \"XIW5AlScvzrX480KychrBU7bC/nK02HOxybZCrnmhrY3515LlSTeEOgEh4TBPIJMqePy4\"\n \"YFumybOGcQZfLZ4ZaDGuRuQyQr7uvr087fDhPor3Pz4Um76/cPdo2OvzVtGms0dfoB0tP\"\n \"bv+5cRhvvEM6o/gPi94fxkRri6BV6Z9P5HKY537CeAAMgJyBWnflWplhGboL0tT1Lbduf\"\n \"XT9unUfuNHXd5jH12UpaSk4DWrQBerBWYdO5H0ITIJJXbny5cxVv/8bbGf0m31/6smSO7\"\n \"QDkKZBzSuCwM4iw4dhGtYUnKDHTCM7lDRWfDGZ7Pj7N2/79QGOKJ/XjAMHevXDh2WxAjV\"\n \"G2YLiQQzM9GP61Ml3fTHbZv67qQn4DuiN925/EFYo3DUU2fEPrvQMCwdD3dz/VFfXfxlY\"\n \"zhtkS8J6gAoYmX/qVF8QFq986epvPzE2dek9r008+92BVtqX7QCP9x0tDU2VTxMNLUiiv\"\n \"gA99myUanI0wyiCeMa+fWRcP/LC6R/5x2RixW9/687f+Zym9WnLmZgeQGVTfjyq5THB+e\"\n \"BTZ9/Xpduj2x2GDrLGjWIHLEoaBCkF2CBO+ZmsaRSKE4/iK7FBNiXyKwC6xbBCJIHfPDv\"\n \"QYx1+qk9U6v3nSx9+61TxygeHxgfemWn3TNN2tBLKNY2P6DjqgVxIMfwrX8NeQEWCssHO\"\n \"PLzzE4kO5ztKxYnv+MxrP/DX7cbWD75l729cI9PK5fqpB1NsGdwbxsP67oAm/ZgMR9uCy\"\n \"dnkkmFVWGK+3jgjqaWNQDdBbFbCM7Spyd2EAbS1bOlKOVNtvSedz2O1Xs4wcx69dPkjGz\"\n \"//+o/+6UTx1S9ZmaHv9bVJc2K05JaKyICk4TJdZ/Q+GT3/Ud95/z+pnLXQHyL6jWBq3Hd\"\n \"su6RlOkvvHvPODrxw5offzrS2zAtO5Sj6aDWFMDA8sEqsCRz7H01nTcbiUBek5DyRZtNl\"\n \"kYOSPtI1P05UYiPkbqkkzIRvKe2hl7DEEU3Pu3lg8bOv/+gP3pp4+XezHd7qwhgyOo6bD\"\n \"vYvE6xF4IkEUmVDcqIgwRxF4yOOg8Lkj/jexGc+//qPfOBte//vj4G0DDAtENjyzmFUJc\"\n \"4aevmenjNimk3D2GMluOrpmxlJua5Ix0UgaSFMcasFh3I9FOzxi9tBT0nQE2tL44rlx7M\"\n \"eaglLKNYh7ZCSPnPqB/4wkRr7MzNhr0YFFIfSFJlNWUHFS5bSSDlJpxi4PmprGqmJj/af\"\n \"+emfyUOhD0Fr+VHVUjDV4HtzWr+wrgWa8wQqyWPZqykFkx1ZZFiIboSgbqBoK77xd9y69\"\n \"aEtRGHZUsi3y6o9tAzreAAr4KG8++L531rxmdPvfSG7wv6JqSnbc2yWFBcJ2qJnJDhKYh\"\n \"F4U5NFzTdv/96Jsx88nIfu/niQe+glXRVWFY/o1CuePHkM2USN7XQYRYueDpY4WAKE3VR\"\n \"LYPVa+Eem6iLS0TTddsNwtrL7rq7lGaLzUDKs8Bj4tUu/ueqO/crnM+3Ot42NQKsuswvF\"\n \"HZGP/g3X84ta0b3+xy8P/u72Q3q/ewz+Xkuk49btS8QAvO3EehhM/AOyH3g7yiE5yjEsD\"\n \"pMSFo+DlLKkVkF329oMvL/1GH/P5a4rCTdhW0p76BgW3Rao9L5y5VjmxtSrn850+N8wPu\"\n \"qUgESmva3XJFuOrdlglJ13Rk/9N07gYR15w5epopTja4bW1S9DcowgscO09DbPUzckhxJ\"\n \"WmltfmWJRSAfiIbQbvr5L4vo5IR42A96rgfGhYlgU+UPHzdMTn/rLTIfzTROjYB3YrKpB\"\n \"WkTXJibGbLhwjf8vXzj7q0+jz+B4f74lZUWE3KV0Y3uTsBByaejKWgi5tZJhkV9R2sJfn\"\n \"VklNG1yWVsKHyqG1d+vC4bwudefOZLuKLwLkhWU6w0rKAB1iekm055mezc/QJI7kYNnfa\"\n \"s1DAMnTvQL/MMXcx8lGDT5t2EQzf1gAsWFmzahvSpfQvWbg5BHfLP5zp1Pdcqv82X5q3z\"\n \"RMnh5aBiW0FuhEOYXzv1vb3X9kfzEGE+BOqa8bsfAWeSCZ5vFgouCmIV3/PMbv7kd5OU/\"\n \"LAUxZyGjwV9QD0SrLcHwA/sgpHFF2ZUEi7qr5P2rFwyL7Ku0NQi+vInj6O/vv/8Kftnkb\"\n \"dkNaK754FGQeqsgOG4VnGu/l4BUg2SN8GgXMTVz3VKX72CQgnVH89NZv6PkXD7Eh4apTe\"\n \"oCQOsh0xiQITkoPfN6fi1MhdscGzRiqOlyQl5KZkUrId9LMUoeCZNJLYHvhMf79OCW0Zu\"\n \"HgmE9N/CUcBv4/Nm/fn+23espTnowAsPHqsGNtAZXat+AnOdpzrcQnPBY0mDQHrrHhyE5\"\n \"U8HwRi9wN7uUVmSQqFK4IHOiGJgCzSAkZ/pIiK/AZ3U3kyHYQ/sl0P3yZRn9XfYMK0DYD\"\n \"UNuzp79y07XG//5UgmCFb1tlGmIo4CXsueVRMZIHksoESoD3kMCSJcm/ZZsvXQgnTENTA\"\n \"L5gprzAF4qFO73MywwKgbki9ODoKVcbvmF6Cx7hhWe4y+5X/qP6ay3Ax7BmFHE/6nSYI/\"\n \"m8cPQjS0n3/joegnWUTUXiio4iwGOM2Fq4cDenUgy2tkgnajZQB2hhfBBAH14Jvv+WJjM\"\n \"j2Lig5c09WdxVGrqESwAPCUVism8xA1Gf8gUvEq1uD2hx4IDYHG9b42sA6g3+soZAxYYW\"\n \"uuniDEw+LxMLYxAhMeZNgj/KblpkP1wt6XT6GxWpKOKjocIVWPr5OSzG9rafnyQKbxx+e\"\n \"xLI8Zfvbpb1gyr7Lnsff619/cUg6FvKhbEpqmOdCVmGelJfc/vbE8abVaGO+PXesXxhOm\"\n \"WW60eGChvbNJC6LvdmA5xFlRxlVPJznAcKt35/v4mNz9NK6yfmrpMSyEY1vIq+7Wsj4Td\"\n \"5VQhdnDnze0dhgGtBKUtpXZOpsSCP5aXTTsgwuui5PjAQF4pGO9fFMvvUxiS8y9X85uh0\"\n \"hYhOWBZyq0NEgW5apIhOWRYeP8AoYh0ONksEozoUwc5UwPlNcD3y6EpNylRIvVCz7DcNQ\"\n \"NXTB6Oh7P2pCifV2tf2CmhLXXhweoKL+Xnn1+e+bhrxU/c94UhOa7jgGG5q+gxrpJZZub\"\n \"4ScBprFpaCOduhp9MetBjTe3i7z09zylJ83PDvvi3y/ZISCag6zJ/OqSYx0QqY0X1EuCq\"\n \"yApIxfvozvB4Un5dVsS2ODk29oqSfeexTJupTU46SNYYMNOUeg1AhRZC5nSfzbfgj8Ua1\"\n \"b77RBl4dY0HNWB3GUtY8lj1wmv/YQ2Y13am38Dszp7fGpAW+S1grcjfgDyBXvfw8HNbZP\"\n \"8yc0Dkz2p1OAsDQ0P9IA6hE9qlG9g+AiaDVY9WCCQlP/pgLUDIiClkiE5pGwZh4nq8qGl\"\n \"AmDURFXyxbCWssjuDb/vGViSNlLm559qQKkBSnJeQCJEURMQIGUZhhavd2YavLveXMwfE\"\n \"+exW34JJYbuQR3A/KD4RQGzBvgZ+JXiYUigStAJORQlrfuh0vYRK1dTFjYz8Fje/i8gFQ\"\n \"sFkWUhay1jCwhShwUx9EI6AOvxTSIMLbEzy+nr/FUQIRorqJx5KSkGFckvoseoNx8P6vK\"\n \"NH84ImguDFLISRLS5ipTAnSq4LcFEtAWjDkJy554whOhiAYXdoTmkrr1lOm5+SEzP3RFT\"\n \"37dBQl9iEzMDaa4koUTVThQgixCwkDRoE4IXvu8LpD5VPuE22WswYOHBAmv2/cuFvNnlB\"\n \"qRsZZ/HExsaYzjVkclVCJpL2gV74fp7dV1gK27D5edqQSOY3V3/N+t2yPBJS/xAq3APdf\"\n \"8JHBaSFZrhRkzdNhBCsTIDoUQjUx8N8RogO4TgE5I0Ccdk/V6YS7tN8w+g2LS2NjAdQ+K\"\n \"iJdG5uzOFOCyFTX83DsDBnukjmh0uEpXA5bX5YJsuxcf/RtE+d/ekUXnbQEXCB2RXXNuq\"\n \"PIELYo6Adhe4BbmKBv31s7BNrJTz5+WmyUQAvs+d2dHxS4Lho3z6QxqEQxUekeUbBcZKq\"\n \"p0NyJInPC6Uo8htMJ/MTm9+8FzfRD8uSYYUVQwz7xhZQ4w6bSkhQoorzco8IA8Nmcaags\"\n \"KVYPAN/ILbl5aUsx6TW3wsXBuTRWzf2+fCFo1SrFoQSGtIJ0yEzad/iTSbz0wJvaxAsr2\"\n \"R+y/JIOJ1TSte3myZyc7s+7brKeY0+QIQ8h3jQPZjF0jgdXV8peykvC+vO4ous/leQOYU\"\n \"WQs+f2M/CbmqyK6nR4FEQxhmpI8D7+Rs3P1KXve3u3S9y8zu1XDY/JaWO+Seisl86OsbF\"\n \"dGL9P4bqvbhJV1LMJ0ndR4Sa9FIOggmheF9uXsqVzV49r8oLOrk08vwq0MhWh0dysjAFG\"\n \"3kprYMLWwinAdcRhgaP9yDh+6bQY2nl9DnTVzTpm2XJsC70dAsxH+rTPSrPy2wiLBcS8O\"\n \"0ny3C3pKtYJ1AeuW8OvbTR86c2eeBXEF+UY1gEiJtbBRbCe9hCTH02i/vKyfwGwvQ5965\"\n \"oynfLjmFRzEfJrPJC95UNyZmLCEFBSJmMODDN7g6CsylaCPP5/LKbI1VWSuif5CfcfcmU\"\n \"iSK33ELU9AonZFS4V0oMXAczk/n19Mj0OargvlY4Kh1/rf034D4p5v/DyR9ZDel+s8vqv\"\n \"YqG5MwmQl2n7kHXnO2jo38tFO9HjrQU73ER0VCun4KLViiN7k6kuMep6asXjj+MIRRAh1\"\n \"/O88qDLYO4A21cnDLwGexYTWY8zxDm/HrZMawww2jgO5sxQ1tEbm4oJuYcvQJf3k+EsuR\"\n \"4IhFkPW9MFBIIpQAFQF1uIOiHGQiBBl89JO3D20o4QQOwQLBoIZw7ad98AHHzc3m+3TY+\"\n \"/udMDKmF1vP57miG75cdwwqRrhvFfekUcnOLkBz1dpZ5iJCM1WMhAd8dFilxwvG0XqPFA\"\n \"KQNSt6CRbneRLdgWFzeCjYCKYwztB9V2Dg6RBlBrCqsLxTOMpnfsqjItOwY1lBOhuQg/9\"\n \"reZJoEqKaYfx8RimUzTYlYS6xa7wuPd3gpi7PK9K+tNxFh4KjgTq9c+cNNkL+77ZKwECq\"\n \"5HshbZ5f1WgQNOP4xvXtb1kLG5MKy2fyUnKBFpmL+n6lw1/rKFkIXCndcej8zmP/eOv8i\"\n \"iTCQZur7ns3KJ6C0oIjqw8xJL9KD3HdF68PSMRDmzbftSVoI1woLoYIKd3JVEjSzjFLKq\"\n \"oKcxa2JBFLNeDKZHza/Km5fOo7j6GFZMSyREkQucFQb13cKMV/so3GgrvY+CRIph7smK/\"\n \"jy/T0wWUiA+hR/5927R4UoXy4kUPsDW3fOwkCYPtvR7jyaaUvQqVhdSRbACQthmVZmDWa\"\n \"BL6gRQVC9SOaHzY920HuktsB9qv60rBhWuLA/9dp7t+BYJUNyKrcE13WOKGHdr3APHx+I\"\n \"kuOG4axx3clt8tuWpTDETlSv4+MydbDrFbsN04a7gCkk86j6j7ofQSvslPyn4oZEX3STC\"\n \"QrbIa2zcDDurqqDip9UrwuXGcOSCzthZDYbprZKpIqdKbzUC6uVPAf73DxEyB3QbW+nhn\"\n \"W4nBtr1TKbp0oQFO81h3IyoR183p5UOX022UsYQyhYTVXyEUxPkNZ13e8eHf2QcJNpdkv\"\n \"hsloIz5UrhPje5GOZjDCpQJOqnggsiBCAMdXtXESIhMmBAVsBCE2E6Gjac0rv/vGyluh7\"\n \"p24QVIGXwPL84hZPKrCqYgXRQzV3j6QPqg1ESjdBLHNfN/e3MnICDg6dtu1t4TUync7cV\"\n \"zfDt8uKYe0ZPyOm1DDMXZCwQJPKxTsLmuBxkETIQNa5GwgNugffL4S6B6/ZdQ9zj7Mx3/\"\n \"ZpfQLzLw9+dFOgueWkfdDwKNYIEGklCVoWMYT4XCWQuFyHtM4cBzKZXy53vcou1ELKvEt\"\n \"GLTArgkY/JF0AcGa3H5devur5XwkixHBIhKK2HAjyQQoic3IcUcxz+8jIxxCYy5Z/8DL5\"\n \"detv1Rjo6v+YwGXg2lv9oNTBzWHWJFTdazw3ULSmw2iVFsJpYCitI9kMTI2+CILu75e6u\"\n \"+kLmuzNsmFYiLkTYv6L15/NYr1vRUoZToWSizwkwtBCOJtmZIgO0oNs9v1LQvcQevDPvr\"\n \"b1Ta0YmCzdfDQLXSE2CNj+1UuLLMYFXjq3cabSUctjYaDLZH6HDiFZaRNbCpcNwzpQjrk\"\n \"bGX0RC9zbJkNy1GRYVFwtTIR0+guCTMYwPXvk0UpJs3VdZRg4Uc6Xj7q6OwPNBpEoqjrA\"\n \"cHhQncc4U9FgyZwYUA+V3eYg+GobbgL15ZXcyCsZ0LJJ4BdW7zV1o9s3jTSq+JanuxI01\"\n \"O8aUIuQ+5g5cgGqwU+6i3xGVmHK2c9bloPTH8fR6MYFTH8kwuF4Y0+wDCEZlpiXRgM3x/\"\n \"MpUaRxJAzpZo5LFvmK0jockf3S9tu3+yitv97MyfyWjYQVzlrJm0BZL+wniobkEE7aL6c\"\n \"zR86zUrCqAhEGqRdFbiwssqYW5cP5afRrWNZraOh0B3z1NruOTVpRch2Qi1LPmQDB1M5R\"\n \"5ZEwmYTa1NR2Ev/NHFCv5ETVQtQnTvSLXdMwdYS0YNcsb0q19BXnPSERUukuiHBeMYsOp\"\n \"CK2aOf1659EKja4xZZr6MUJ33LvOyzrda30wgYvKO4QgriiZb1CXSdrEXJfm5dUFp407H\"\n \"26y2R+jjN0YOFL1f91WTAsLHw9n5divufZ+1m9V8VGgiMRMi5sMSLEmMxybqwdicRxoXg\"\n \"/Uq6hp+LYmgWm0A/JcUu7DMtLgVZgRVMTelIxJXE6ji6lcWNkMj8wrlC9wJ2wKduyYFih\"\n \"EvGlwV/tQmj6NiGZIJpQxRkJiZBm6kUaKvj6AUT5jOY5e8W1vQcXv2uRTh/2n6fLejmD+\"\n \"9KwJ4MduNzwVMQLGQ0V7mRYhLT2xoB6HDnKtQLAoOk0q+SYFxvjsmBYYeT95NjExsC3Nw\"\n \"kLYVBxNtnFcBTt76A8OuFzV1+MCOFD42aojwvGhSi/XPJyR4vQ6nr7SLmsV6Cb+zwPhhl\"\n \"l5SvJqFg4VbTFiGVBNOioosNkfvq2mzf/+BF56dGmZFjLwkrYJSqC9Gm2XjiQyiT0QqHk\"\n \"Y3KUY8akOTKq+4hwAbLhzs+ME6xeTSJbLnm55YKp/19KFWjiOGS7w/utBGZkAfzXH8J7T\"\n \"ySt1B5DeK8f+Y76UL4rbNSNyxvx5mazWgqVW9QSwdX9PVOuCOIFzu5EkvK9oewZnUQ4Xw\"\n \"zhHKPmsRBSgL2s8nLPMc46fSWlipHga4weQFkvrGJwsDo9vKrHUA1LPWeFZb0W65tjRCZ\"\n \"b1hefKkvrzRlQvywY1uDzsiJI4HvIzc29Sc3zOXUSJEIGsvJ9BU340MBDY+edO78kFO+a\"\n \"dnhZzFkFY4/hEpnN48LlL2/w/MmNTCFMn4YYHrSkLgkQyaOqsl6LPtHwUyy04U+KWgEIq\"\n \"F/0DhUvaPojYVnMFy4NQeDuYKoQNCWJkGKfIEJARygrABJeylDG6f5K6Ft24JarzexDA/\"\n \"gb2kLcBcbEvkRKt4pFbhuKbm7AFFUH3J0ERS8Rc1wnMr7WFSmLnn9eptdZYrd1v73pGVa\"\n \"flDi8E+d+YUvBvrbDgVcvJlhJOZ+EJ4iwzLAqmG24YOteW4dhToyNktC+WME9rUvmwUBY\"\n \"1muieGt3IuNqxaJQHai5BkAsNM5QhRBR2XIwLIZMFneAeRk4G8KhQ+j0ouCH82A8+q+b/\"\n \"ngxHZITWEgV4q3iLqKoWkJslZVaCGdMNagKNGX4QveAEB0hTc74vfW2MgxMl/WC3fUxWW\"\n \"S0shvrfRU5iDTOROn9LEN0wKe2F+7+ziY5puZTLzQ9wwqJqejefjzbZiK1jOZTLAm/V+V\"\n \"1JhFWC5OwFPrFx3gfCLnp83JXO/4oroc0wTM4p0Gz/THk++cBXTkyEUMlkPTTmz9fmris\"\n \"qj9kgEJnp9krC1phC28Oj8hVddTgi5ueYQ0N9QsixEruRuofkiDV7spRItdLLUQIaVEvl\"\n \"bC49KD7xo28KIgZOso2mHaa7PFHBU1cHPr/NuBcVC7rZShJ/6SVmWW9okA0j4Ho12vDpu\"\n \"6618XmF0W/9e5DzfN7hVjABPD0x60SXKr4pLAQkn0px66k4rQWIiShOQ69lJ1NpjW6FaO\"\n \"71aw+NJynRrU+7YCgijtTN1HWa6KLCVdANcpRCgEiCTN8SyR4xPvogNQDw3A1sOld6JYZ\"\n \"QJpOvaDkDkNkVtLCYGAGB0Os2sIKIRR9VWshEdZopmbuE49FKQJ3XOyMA+Xc9aqNU2V4u\"\n \"gdeELTuasMH020mcBooeyakhBVNSM79M4LdHRlAwBB9mcwPa6Xp1AtNzbDCyPvzBQYHOy\"\n \"I3N+Y6DGa4f7Ya/GkpRIgQHd+yIE4avgheDUtUNXhITfX4EGcle2ynYZbgLtAkZb0ixDJ\"\n \"o0HCpyAqCrUFwvF12nVdwi59/0E3NsMLIe983t5uWlsaRUOHIMCZiK4v3lPmraJAGhA8N\"\n \"jr0iRCcnS1Q1FaFVMdxYLj10QmbzCFBUVAQCR3nSihBikoaIhgiT9kXYNzgVivTCKgXXh\"\n \"tu3P1e2FDZXzcumZlhh5L3j3nk0nWV6Bl3JyHsSIY+qDMkRrXpWo7siN5bfHQTH2tEXeF\"\n \"hz7YyRrrsqOyPD1/I8/kAf6E5uVbqsF0hEGGdiOCeQBrmpIwNICu+EHksTcbhVIrSBlzc\"\n \"1w7pQjrzX9cRuqiQg8lbPCuqAfLIpUdYLRFhmWVU+FT40UvG+9c6dr5RDdJprZ6xywJFe\"\n \"Hpb1Onf34xsR5gTVAZ2LuXzVa6SPmcaZKIHk+mAGECbz0/y75ZqXn4zyEbEjtGmthBL50\"\n \"kKIuLBHTYbkKIh6gkQ/i+nacjVxLHEkDDJpI1m0Jw+iy9PN6EMTOzXP84CwrNd4YWKLF0\"\n \"ytYDy5YFc1zcU8D4nga9IKQVpKWa/FwOC6EZu7rglH5L6+gaayFDaxhCWPRG8M/8lKKK4\"\n \"2O7AQgmEpyLJmE2ENQGJkupdKk9pKYmdsFaVYbGnO/r1k3zhI52IsWmXLegG2WCyEITbI\"\n \"qEVMYTAhkkIePtxctQKalmH19/cL2K/dvrAJSsQtzMWmajK2SIgQhCyUxYH9DWXigzZCT\"\n \"QYdLg5VXj9Wdi52PXtXoJe4rykmW0lMCaDAUBapqLREtKKsKvWhgb59bOzZtbKzfA176B\"\n \"LBqPH2pj0STo/XKO1Noh5IqWRjvmlfUavdI8KlnVi5M7osDotUM0FwMqnrB22hTG7xrAU\"\n \"nnEwduBPOxZ42+Tid2siwxLwseGdjfuQuzAD5uODj8OmIjGR+61335ga8uV3e/JviaNi0\"\n \"ElYYeW+7o3uTOCpxJfMP/inXJBEuNZBVh0kayuLA3XXr1p9u4SD7+poveLXekxM6F4O5w\"\n \"7nY3uKoXtYLFFxFvrQa0BkW6UXRL2+M+tCmas3KsKYj78GiDjI4OLYtaYnTya2cFsKlEi\"\n \"GlKegegkRCQ/VeW5ike1tFKRadndC5+NUrz23wvKJwLqaIteiNdb6AAFHEScL1RWQZxfu\"\n \"YgES3updMekhbM7WTw2wmfWhTMiwsXs4mxSrN8ad2C4YV1/TyIUtokgjLqW7RzxKIEEcb\"\n \"3W1rQwJobVyUr28VpVh8YkLnYjdwujXDzjJJAw6ES5iGxZ9Z6xUkaGZoqKCiUq2PmL6P+\"\n \"lCk3hZFevFl0+hDm5JhoaSoILiXLv/BRrzZbotsBuoVnSCQkghRZDCCJUL9A6oVo09XxB\"\n \"S2ilJMr79533R0XBeYn3Ku78uA2YNdRZQPb95H1vwD9+E4YghnAwSlHpL5aUFpOzZ/JHo\"\n \"nmZJS1W9NybDCsl4FZ3iT7xfXeczNrWiqW9IBiZBK86WSBPugSRq6GBFTiM/03hYLUn1S\"\n \"awyEFy48J5TJwNJez6eFUF2S5/ySVuJvZX2o5u0aHv5IOUTnaFPQkbqzt8CsdZezFTje8\"\n \"KOZtgQZgbIKd54BBcPieJbIsbADo74c9XV+99DQf6aFB62leJd4mP2XEin8jATDctyRA2\"\n \"VVwuwLFfiGpEFeRafRJZJJBaMJUCuAl9krHafYVJETTcmwwsh7Tysh8h75fZA/uIJZqvs\"\n \"lIRGmQyJc4h5G+7zj+Mja4K01DHs7B9TyeF9oWvPEOPjUUIev+VsdFucjEhVsPA4y/xUj\"\n \"Ivg+5kYcuB0dJnJj3Son82uOsl9NybByMlsBJtZ+glVy8J/SRJiIiAix8pA1Uvc62g3Nd\"\n \"+4KxXvMhN3k3ct4y1euPr/eCwqbheqAbliKNQLEHZdJ+1gGjvwqbiARUyiS+SGH4U6io7\"\n \"//ufjZJB+0xNZ0DIs6G+yReAksz3cQec/TYOzzWzWaQyKssqxXJc8RRSnApAXDasaskZU\"\n \"MMoprQulT1yf3mpaf8qhuV3VzA2Shwj2KsS/WB9cRl44fTAhLYVkIiJtPLgbWor83HcMq\"\n \"l/XS/uXib+Hs7e3gEUmqtBcda90v4PKg13IUFsIQeHBrZI2kpbAgGBaYd9OYpMMx1Ov1h\"\n \"NYvHjVRGtyVSjMbrU7NjZqLEsQi8qUBujqJOtBjuUCGv/nKlS9npBCQVxM3Mwim6RjWvb\"\n \"Je3lYEGXQ0jYUwOiqUinct2DU29uE1ci7VJ7QZNFe/t+WkfXjgo5DGVRWu5BEQrCLeGMI\"\n \"H0c6YQkpY9o5U6jOb5K/qpyxq2ljCKe/2wUzW1MbHNQ9zDTlGrUb+RPUuFe5RblsQ5cGw\"\n \"OFZnk+eN0sJzp1WUgvi4v/HIgyaMMSV3eDeOhJgPFo6Jbue4/4m1fyJEYdK++kFHS6GvJ\"\n \"RJBWvPcboBwLjxC1z6S+O9sOgnrxHSlj2A3JCwyg/rNcRXzMYsIo+NacB4NvGzWNG37ti\"\n \"g73ipKMdfEHBUYHxz/QhfkCJT1EhZCJemdVkEq20VITv2oGU9EVXEEevnBLZEbay4sqvZ\"\n \"dU0lYmFhsmtKvxvUnH8OmCYal6K5JIsTyiIMIWZQCO6OpT/kip1FPT3NYeOpJ/KFz8a3h\"\n \"M4+43vh6yFew1PjKMSxyVfIoYSEEdHwvOG19kMW9D4ZCK8yxJiTS+jy6tqcoN4ELDeOol\"\n \"hdzefbOpzqhed7q4hCOCVZuDCER0kIYRyArjzsyN1YhzI0lTKUL4e5h+y10Lna10YOpjA\"\n \"mLMmKaxCFdPUxQwgothGRY9WvY/RlT6I8JCYvCAIWC+j2/+icpt9gXGsKBPqkUHBn9+gY\"\n \"vKG6n0hB7kpJjIBHSQsgMXTEQIRSmtBRqO65cORZaeJTEw0LzGedvA9pzovvJ0lC3mbCh\"\n \"IzLB1GOYiQgGQaiiioaoBhwq+WwbhlNd39YsyfyaisjDyHsnsHdbiSAZYD9QdTsgEWbAs\"\n \"AR8ka8TEhqld2d7KvVvVLyjqW/hkXDW5+/g8zJpn6b7KOuFjMh1PWlVPkaSBjc1Wgjr3S\"\n \"Cplw04hU2FwuBG+Xy16aipdFhhWa+SN7Qvjcoftm3QFTBR74le7HkhEU6X9Vrshqp/p4U\"\n \"n0NJpI+16U9RjnW0GC0/Vw6zxBh6Z0YQ+xvbGtplgWNw46s8SFh8AYaKFUORL4+Vih1v8\"\n \"voiuYOpVP5NBxl57jAH1XysbcJRVMTQVw/pIWNZLM/Z7PkRZnrfrO8EV0UlIhMxtJBZJ9\"\n \"DDqhqG7mUxgjY2IKjrPR5mELZ/PG0x819uLkA3UrTszIFO07Bk/EwwNdQWneg+iKmKeQx\"\n \"PDqwgpdbyor6+PJwfvjVt/v/7C3b/dxWNPoCONIs/pijWpOijrOhsAHgw4XjrlGaViYbd\"\n \"EjTxKK4amaXCahmGVd03B+R1/bG+CysLoGcE0YpbyhkS4tLJeiz8d+EDSBnAM3RXVoHGH\"\n \"kCgWv3P2FcRtGEHQq/X5up6f0Vff7Bs0fIfN4nh/ziIDO3y4T6kdubdXwjw6NbjBCSZXB\"\n \"/TUU5BaCBV5FKUrBj7zfb1JmnMPgwQe7AoXmZ6e8lEasKjYmoZhlZP2BVfHXlhz6vontj\"\n \"kOlYU4/Su2a4ZEKGIIYyVCXXNFEjZ3TxAct3T9EKpei6MQ6b6ihurRRr/Wb+D4BGTeI9S\"\n \"Tt4613x15aZ1hJtbY7s1NU/btjkQiXUyYnUP4N2gkskNv0fN3D2n9vE87hvyEWl+vpgrj\"\n \"Co/HRf3Wo/BXM8Yn6M8QKFkqh+RLXSf1WOQb9W50bvZYqDEo7OCzpaWwOjqqJ8xNw7DCy\"\n \"h6DIy9v8IOpDQGXiqImWNJdaKaOjwhlEjbsjDuHh/+BCtPLIVNfjIDyYFRPo0waJCliUU\"\n \"hTJ87//GOOM/W2QHO+5dqdvz/o+VNb4JebSqQMK4s0JAHqqBVLt7WifbuoF81bnz39Qy9\"\n \"ZZvsLmczev32z/jNXURJDO348Z+Fo6oHoG7D07o16qFzWy7Ynd2hpkbSP06AcrQskgWHQ\"\n \"moyXBklYugj18gOve3LyIxvb2n75ejnHmlJSczi7yk1iCNh8r65bOJBKmWahIORYzrOSL\"\n \"bQQkijjAVIq3k0zWOl5hZ14zOX+fuGTNuM49wBqeIzT8uahMqP68pXfWV2cev3drjf5Hw\"\n \"ulq2/OthsWrGowZkDnwzgzoLhUwjqydeGfAz5kQHeWRpjL1mTK36rrpe8dHfuXD33+9fd\"\n \"/fHX28d98csv7r3EToeR2/7HyATji/IjnHy4zYdQgfJLnZtXLetFC2CgOTwmLBhxYnFcX\"\n \"Cs5mvLkeSqhxTlOtfePQ0hwtLOuFtMi7rCQFA4M7QDy8YAko4dQTqXXIHMmxu+3IjRV4Y\"\n \"+VUM5+cFx/HjvViH9c0MqsbN/687XOvP/Mr4+MDL5uZkY8mssV/B0nNmhi3vckxz7GLyD\"\n \"zik0npcBtBlzIlPYbFTBHgZY7mTY17zvhY0dPMqZWJtrsfuDnxxZePv/5zP4rLcZ7I+8c\"\n \"C+Tw+s55NLH0qCpB+yPFQmh6SIXjYvHipJ2wPPovHwdBC+OBv9fpMNQLg8JnMz/cHlc+x\"\n \"1hwMa8auaWge/Gp45q7XlFb3nGkiFOyhunurv5rrkEvUL2eNHJhTujoe5C2hX8JC/vxrz\"\n \"7znlbuf+mqybew3jcTU1snxkluY9FwfDAqdca+Hm0jAeu6kDep9Zi52vic7ENeBmZmua/\"\n \"jjI7ajm1NdZvbmH79w+sf+MgjeSB/W+1CUrBFM66iA99TgX+CY7O62UcsRECtJ5yTh6ZC\"\n \"cxtEz8KX7BvZ/IGkXQNLCIzXfq9aUnMgHkTRj10TJk8ntPpXNam6ago/S6hPGEM5c7Q+O\"\n \"a6mfiQLJvKeLUoBJ3MdghF6JUtXXLn181WdP/8jHjczox62Us3N8pIjIJhaf1akWwL/aI\"\n \"gbAvQw8MeE6hjc+VvAynWPvfeH1D30aOzeqU/d5oWS31LFWfv8BgXI7KG72vKkVLOsFBh\"\n \"vnNFQO2owrCRB3CLq+sG4l+VWjgAQdUSIFLU0KizPy4M+ioxmgN/RtUzCssKzX1679X5u\"\n \"A1p22LU6DysE+TYSQP0IijHl2dVaDZvn6wcFfRVYCVoPuncYLleCHDvW7Xzn/oaduTH76\"\n \"5XT7+HsmJ0u+XYQTmw5GBUkpOvgQjA2z7cjdkp3pHM999vQPC98CSnY8dkT3nIV7Ghh4Q\"\n \"Yy/YF9/lMYCcALqD+r2/IWhu/9XSuNpbBXceBonYAl+btDqjrW1LQhehEs2VtzRvJI4my\"\n \"bu+1Gp2qfyrmmPb/T8whrlk/YBq/UgQvAB4fGO8vXrLauwhbNGZ0826pDIrE6c/YXvHrF\"\n \"PfdFKT20bH3VsiEMkREpVcTQ4muiJsWHHyXQWv/szr73vKB8S+njF8cAH+3x+XGaugGNx\"\n \"d6AV8XOEPPnBhy3xMydCZBllP43kWJCSbZsAFHeMjv6NqMYUVswmaCq1pmBY4a5Z0u4+l\"\n \"m2HYCB9hpTcATi59QtkZdYGzW9vt6A4HRV6LFp4jgc5izqkz51+f68dXPtb3SikiwWNoe\"\n \"JJ4C5WvMn+dRN6LWjfx3/9M6d/5K2EhdJeHQhfz+ekP5nrj0HXiSfGOtraR0T2EMYQ8n2\"\n \"j4US9y8A0tSy0LTsJTm/vQSUx1xQMKyzr5XqFbniNQDlokhSVayERMoawTkRIovItPA9e\"\n \"hyKn0ebNGbgt9Lv9Z37x7Y5/5xhS4Gquw2MRlen1agHWou6kMtSxTf5XPpXSXtxHQ/iX0\"\n \"aiJxwQp37NRJYc8WkiU9Rp4xc/hcZBqA+o6GytdCZABie61t0MScIZEkV5Nm9/iXPEgY7\"\n \"iwKRjWofKuiSrPT7LyMf5TkvsLIgRGqXSvFxHCS51JK7BKp0T1k927P136yvkP7yl51/v\"\n \"MhKP5rkjLWg/p5j7yBESJqQnPS2f9t3zutR/7Yf7Yrx2K9Xx2QJOqg1du/OkGXyuhQEmd\"\n \"to37Rr74B7HL4LIkNhoyLDUEQXJ6ZLXQ9TLDmtvivPjo4r1CeYYldmW5ayJ8sATLD3Woj\"\n \"RagZ0/KNBHCEUBYCOsEJPAD/YMgtF03bnywjZCN2Wf+LN3mrHBsDZqJekpWs/ACZurhjG\"\n \"b/FH+h5Cfmc9Zl0XzR1f8xuZG5xe2BbrexZiV2DjU3N0BG/RX9sFRpMpnfuGBYkEuVtBQ\"\n \"qz7D6NBF5r71y449gIXS6HZEHSlExn0QIGYJEKPb2+lCiVJgGpe2PPPJI22dP/+yvZVcU\"\n \"3zwJx048PlkfEOZ9ilmYRGVuy+75l3NHD/Gq/v54pSw+Y7x4eX+mDbscjjn4qBBLIHTlB\"\n \"gIRus760kr49NmvENWF9R3J/EZGPr5KXoAjtmJNeYYV7pqBU9qKGJF24Vej6K5JLtUIIu\"\n \"RR1IDf56XBwm/Y3t1fmhwvgMz0WI9fFdIxloEBXRbCfYKJd/KeMGKhwvuruiwsUIL4IQS\"\n \"EI4ZQzXhnsZlxy6WuUxmOAFBQphDHwuIm172wXiJevWR+dddtVEWBMy6edG48mmkztLEx\"\n \"tct6hYGsM0CP+y1iaEQ4kDXinftRX5tE1SY6qNfmCBo1sNCCI50zmYfzFh5fwcGEXxZ1b\"\n \"1E+C0wbXcpAbscbPsjSJCqX9aKkwM0tUiQsCaGsBO0HaUSKejLH+2sqxhQqL2GFuyaUIb\"\n \"sgYWFHipbQlzTHD9wsiRBaI35fJ0qkdMUj6ISra9fGRnxEhIml+gBoDfsI8Awa6zxvbPu\"\n \"ZwedWS0CORi5YhAVK7gRnO6GL2WbbiCFUNSQHSKGeU+g660QnFRAApWEvnfY1zx7bxeuj\"\n \"TApZwfMrukRpCWvmrun6o48bdOoGw1Jnju/hmFLOfUQY+ZK896yZ70Jt3p0itKS+aSRQq\"\n \"Ucl/BAaHadT27+rnb/xLwIrR2cOIKL3YYGSKxf+cb0XTG4VGjwF9VdEAOdHxBDiA98LpE\"\n \"SEh6V0AwlYhHpB1pqZFDIEeSldR3av0hJWuGsGwekOVGna4sDsBbOrcjCHMyrKetWRCEN\"\n \"iL0G1POzokLTUYlb3USlOg15bMTZe2lV2dHRNZ7dh+WlIWUxDoAovuB8VwEKMFZXue1Z1\"\n \"H1j2i9WYit28D5sheFhsU1YdaOWrlVv8M0cR7ppfu/4PG32tuE34AcosAjMvU+I95zUkw\"\n \"roBhGfSW3oczIohhXyvWiP/Rg4tLILgjtYxMUX4jmj5yFfBmQHp6Fj0bu2mkl8PDDo1KI\"\n \"gRKVVRfyXmK3JMLIUCmBSSBTv8nbdvf2iT7Cn64/tSIFSaYYW7JhwAd5mml6KHZHgEWsq\"\n \"g47iXdBcSYb1okLjgUXTUjmNEEfUJTmUhYTkW56V3bvykYFhxcJEXygVKQCIHfR9KM0UJ\"\n \"hbRBRpXG0V20OJBR89QxKSRvLq1JaD6z2KKpZSlUmmF1lHfNgju4L5UVUw3HQ/V2zVlEK\"\n \"Gc61r98JmmdySKnUGSBE8nvlGsQrgzpHTlG2JiNFIBHCiqOLXofUqKwf8cd2a3aMYZwzW\"\n \"xEB6XxSJEw8wG1v6eCGCE6ll7yb4rY1HLZr9p7jPhOpRlWWNYLiqv9zBxJBVbE44+sO+6\"\n \"a9SZCYoP6K5FhRlHMyOA+Cjz6dSK7L5Yd+6gY/XDw8koIdDscWAj5wMgmN8KOhHEGkClm\"\n \"IZwxQh1B0BSz/J38sqdHZr+YcUFD3yprJeSuiSZ2TThD7jMYeq8kCcpjWb2S9s2kFqKDD\"\n \"IvKGinEzPxVjfdgIHCJglhspF8lRF3IJhE1ZCETPH/xX2EhnFgfwMUD4kvkz1kq3ASIUh\"\n \"VppVFlvSoZg8hy4U+K2FRcL9ZgJffV4xqFJSy5a44GX14d6MZWWdZLvV0zJMKZZb3qMXH\"\n \"hM4o4DnIRKLc6JUwCNBHT57t3Qpijfp1mgubIvmTKSMLSBfOguhbCtOUw26GqDXosxKZq\"\n \"/pYgOImssVrAwrqqAKsMILMRIpV9Z658FUn7JjaImGcFd03CzVUpFO6zBxHbN6R3Hi+KD\"\n \"PVXtGG6oL/SjFJBKyYSm84SzKFcF9EVaQvDfSbtOzutJI6DKFCChyjHEiBtCqg603txeE\"\n \"hB8KTwohqYjCnkFDndIyN9QvF+5Ig6indlGVYYFuAH4/uxa1rCr0bRXZN0WG8LIVe8YFi\"\n \"gedVInrCJBhWuCeUe0DPRlknfEN/JoszlC6J5OXVU7BkwPASPq1qghA7POuI9TT0RJMwd\"\n \"x7HNFUwhZtEhRqXGEJ1AMwy/zXX9HYQsXIsqQKkswwp3zZJ9G2W9HCxKVXdNaaaeTtpXh\"\n \"1klhZNJMd2T7StsIQR8JrLUmUb2wjdu+vAwUdPbeyxSmZC6znxexhCWvLEdvoiOV4+FU7\"\n \"pCUDZcGkw30BOfwBY3nEgQTuVCzcqWQmw07o0DnDOVmqoMa7oYJs7QjwkiVGwfCieR+6O\"\n \"wEBKTdYSR2jwH0pVDW4R663MaPSa0y0gxeAVnIB+MhcHPEWPpqBj9ufH/sQ4+E7sYQwiE\"\n \"KEjXGDckrITRMbGq7al/Qm2MG8kkbV6qSVhi6jBHvmaY5l5+QkxhpJuMeEKNfxScWE4h8\"\n \"AUGz93T9ka6Vd01iXOuvmkLYY2TUMttXKV0ZyAlqcqv5CyiBpi15gzHeOBI9KD2lbOMjo\"\n \"3e2uD6413IsIonKalwDxIJFNbWjRtrO3peC7QV1xgBQGD5R6UGGANWzPX88YOECxuiiKh\"\n \"XAUYlGVZY1uvMVSj9Am2nXYJfiA55WrEmlgZgYiBrI8zUwkKoHLnfmyShvYJmCQvgNL8d\"\n \"HuiJfA67y2W9iqVbB9NZy4JeT0VNNodPAwTSIq+4yA9AxGkTVR/QlJtByAtGSZTS07aPj\"\n \"HxMqWR+kRMQZ2DpTebmngqGNrrBBMp6kTVAWaNgozBYb4W7wAaei0Lx0Yss0eEYEPpmcd\"\n \"LUOpIbL7LbPePvjHxxhgVKnGBih2Ei/ZCiSfs0xJVZVprs+xRx4RtrztsousZ3/KxWC1C\"\n \"nEFo3FqMt3BJlv1QJ0VEQWZo2XdbLu/toJosSVhrtv+pqasiwBDeNfDnOT8a029OlQV2s\"\n \"wGOaYqemD2czm85xJLmcVI7PP6rqfzlxQvYZoEAJzjHKIgR8FHnBdC1prXuNo9SDxKmiT\"\n \"F5BMauOlFMRjkX5OCTzSwbGuKjGpGlqlP1SkmFN75reVLduFrErWaBE1eZUQkQ1BANZBX\"\n \"R1kAH5HE4ale2OyhZCoMeETtkyO6/sXfvuu3KZHIl0EmdaCF1/aquHonp1mIKKVvzMi0T\"\n \"SycA3XdtCBVtTMCzL6r7sealbCRRfo0Fi5vVqvNd9JvPz9YndhGdgIK8EapVkWIfKZb08\"\n \"v/QkvaTxnxLIepCQuProSpOs9x6JZ7IWh+IWQj+ZSlACPIcFWYzDQtjXd1jQ78lbf4Ic5\"\n \"P5OBwhBYhnlaJoOWBZdGILkzbXtj14iHa1a9QPXQNjXaCmkdelB2lLhs8iNFXiPE5aeHj\"\n \"VCdNSbXFgGsU1yDi3Xn9ziChd39Q4+ID8Aeb+FkN/Vo/E51F8pSeUhAgKIf4GJgPAucRw\"\n \"8cCAG4adXPsx1Jze6fmEVNzf1KAUwQn9FCyGcRq9vXvO2QUINJu4ZRscp6Twqx6HSX+JR\"\n \"OuEWuiW8tBQ2XnBQjmGFZb1ODf4FwgLcbu6aaPXiBVXRDBkGHUYpZdWTefBZJcYQ4o2Si\"\n \"AF80NmA5qncM2TQc1cuclDDGMKJycGD2TaT8fHkkpE/pyqimOdigw60esdJHv+Ow6jMy+\"\n \"A4cNYw8DZq17R5YKjua4bocO0F3Tdv/sYjvDeUaKvrJ9qrlWNYYVkvxyls9bRCh6KOy2I\"\n \"WyDBYDJN6rHoxLK5GhuQUlDZDgFvRQjhlae1paSHM5X4ychQNDfWLPl3EvWmGKFCijL/Q\"\n \"/csUITlGGv9MIW1uPvcdVCJoprn264UChoAIJnyMHD/3w1DdJ9A2gqAJkr02mfRETGFvW\"\n \"aKtrqdor1aOYYXDm/KvcdekFIHQcTVKVoWw8VVQF7jHtIVw5o8xveczybBoIRQ5sGJ6zp\"\n \"K7JbeysAYDYzSRWC0WaV9f5EGE+uFeaSH0g8knOCFCub1k4CPvgNNm2piwhNn1dfZ+1Xq\"\n \"E08gj14WSbfhwIGXjdQq1QFgK2yC52vbNRwmYCjGFyjGssKyX73ko61VUlQgFYXFbrGcM\"\n \"IR9K3UJoISTVK0blBFEAZVo49+grLu9f+y6RVuZU78FIQRV6aqnrTHi+A10nszQoqMEiI\"\n \"4LA6TspL6mZbxA966a2Cz1HMvmWS7pmXU4mBf+KFD98zhIbgIKxwPJglQ52si8Vyn4pxb\"\n \"AohuaxOxM5CLN4nEcuRXdNoT8ygb0U/hHOurTyc6haoLSu4PKUaAA3SSSSDE44D8kBpY4\"\n \"4r/mIsXRUrPJXr39sva+Vdtsl4VqiFD2XaSJIMHYrMK9uW/HON/jdyZMHcKDXtJUrv2tY\"\n \"hyLeojRaPyrioytqoC9E6CAzljYVJvNr+JFbqQk+ehTsCi0IbrV7gbPFsQWtKwVjONPkq\"\n \"knD9S2DGqX6NSIoDMkRyKrfoyt+EhXuOA5qKWvN67zpmHQdixRP/f3CHU0ruvZWlKXvFE\"\n \"kCFVS40+vZws5mmdnrq1btGCE+ent7ufDJpcDL01+3oJDH9hMpftj30psukvkhbcO2IDi\"\n \"bIgMjxI1sSjGDAwdkorBXrv6PDX5Q3MHwACBIKRg5WUzo4geFYHX7k0Y6uQ2l4qewQusA\"\n \"JuiaGFHepQF7sqGnNCMwRRhK98AzkSOno+O64NcF+/r+TJtILOWKqeEEKdS4wq0Eshhqq\"\n \"a8RrGM0KuOYOFDGiaGZ5+Fei01avdAzMFWU/SLF2TtHRj4uQnQabSmMnJCWQitdXdL9H0\"\n \"LoLt1w0oi0oG1cwcZE/ZOoZ7wLZunNI3wPIYIzG2sjKijPkWHJQP9YH1dj50K8MqcmDb+\"\n \"zbZtQuI+Pb4wcN8+Py+IIEGCg67RV3NcE/pBVFCSc0pKJzAV+ETLvECeGtuJrhQLldfUs\"\n \"hYAJyfzgiqsHHVBCbCf84Rrl+0Y0pRhWuGtO2df2prNAF/AFLq8cy/KhYetoWw9v86ljh\"\n \"ua92ta2BnOHRNgxNq54IsIFbTNpH99HzgWigR8hOVAl66nhtN4pFmkuJ/WS0XQve8mXoy\"\n \"Fcf1SUo1KPSiSc4FbCQgjG9FV+EzKqXO6kmD6kZL1cKpkl5CJR0FJIrOpuR4euwVIokvn\"\n \"lcrJgrRxd/f8qxbA+cuE5bjVUP+5HWA6wpRR407NDBY3vwXPZ2vBFzci8ZllJQj39e1xv\"\n \"KKirA7YAAEAASURBVG1KCyHIiBxLwQawEIbCqjDt57d3fedtCeKRSJEjiiJgEtCS+LfFd\"\n \"XgaVJBl6QbEE1/3nHSpPb36CoEcKjMqBBMLnKxZ872XYJy4lGKNOOi4eY1ajYSGGE1dJv\"\n \"NDVGFDYVSGI4Dw7hXD9Ib34bOKJIjJo5nah5k6g2wJyZPQ05zW9TZ8z3mMn4vQ/0pYCPE\"\n \"0FRuOaEEymYGF0HwDIoObhx6EOpsoYT1QLoowcOH310OXuIMOjpiU+JFf5SCQaxjxlPBk\"\n \"D7QrB9a/4yJvP1VmVJoGzKDp+lNTWpC+YkIxT5JXrnEHAs0FiiTzU4ZhhUn7hoM3VgI9W\"\n \"xUuhulLM7V1+c2bv/OqZ7S/ViySzKiDiJfkuCJVLuslF5uPdapraWutCMnBOSJyRhI6XP\"\n \"tGaQc8nNo9MCwIXJE/R46n9r9ko2REycSKS7q+cYo9HdGktAn2Ghw7Ji2Fmm5+TVoKa39\"\n \"WbHcCtSyxh01n29DQ30KXhUFp+YbhWiGGJS2EVwb71yPoeZNLKZ90qFqD6Gchb4pltl8l\"\n \"aAhgvVQspqGD4IKJVpKYOXQiIlS4z/xepfeUpGCvhz0gQ1OYSIvc1Z+LfA5Dj+sJ5+reL\"\n \"IRbkAn1h5E/Z8m4xf5lWSJsUDBvMqiZ0mZXOb7SNLPnXFccCaHJIkNQqelgWJSyiqhT+G\"\n \"/IisHWuLJfyjCskAg9d2KflQwSwmNNwV2Ty4OJ6Qwj8TKnrlDIXcb2czlF0V/sPnyJp/H\"\n \"QqXKWUTAO6JADozChuR2ZXYJhDcUQQxhGQxi6tZeRW9gmFFvkPOpB3EbSPt+1IG2uEdbS\"\n \"kEGF1JHLyXe+n361iGyMuAfnwvBXVV4D+GL5WiplpCy9JIpShGu1ERAqw7DCsl6T9o1dy\"\n \"RTFK1PJXVM6RaYQyNomiHDdunUTutZx3hLZNeMhN9IwxQf1LYSI4gUeLDNzN5veKJTMCP\"\n \"GPlq7BFfNlKcTxRg5Q18miCdE+ZOm9UTmAOROphv3AFBLW7F5lUdlkcu0Vz0uOSGEsPil\"\n \"99vMr+gY2Ad3NZLg1jItkfogqrOjGOC5ShmEdllprrHj/cc8Hw1KOBAX6ee4zSxBzDL1N\"\n \"mKnFt7r/KkNRAHRsUFOlHFoIVfXBgvIKTpJwudBTZzd3fpOIIYy6DmFeRkMEt4Lj7WAJ2\"\n \"xycV4B0Zej43iJl1WuIfl5mdG12h2Des6VNWaNxxYoP4vfgagIOpnHS0D3YqntHqubGEP\"\n \"iOSOYXpqWurpdorlZiooEMOhaJxY5dU+GyXpIIdT87viaz6XI4BYax6lTcBQUoYZUgc8b\"\n \"q7BUOqMZXnswsK4Xae22X2EU+DgthORri7Ll/e8TzJ7e5IuZZCKA1Qh3bbYghpJpAv75r\"\n \"3fcLfPRqvTzVTzfqs8AH8KK7ht5+rqx4j23Tm35w1W8Q4Is8T4Ff2MVb83moUwF31d1Ec\"\n \"IMSDCu0EL5x6+/XAxEo68VimOqV9eI+QyIElV0BEV4GZZUnTYelULwVW2QE8zJnF8rHEA\"\n \"If3HsSescrHMDT/dFLPqGntWUVdiFbZ1ZYCPlQ9RqiITQw7xVMEU0dBxnTLGY0MKAJ5ac\"\n \"XeKdlELTcuNUajo4QHfBaXdsZJvNrlKVQCYYVFsMcnRrc4Abj6zxk08QJUVEiROUTcwV9\"\n \"jJw+SBAkrFTqTRehg7iZxKkQ39+3i0ZBeAIbIHVRJSeKDmPog4sRWg5d87JaKtF+no/o6\"\n \"Hgm8jk80yE9rafs27vTWW7zuppuadjVEokU0SBqMj77omRM/GJm6+mBHIpmWStfK5WILv\"\n \"ViZwFTOZlfaV0y6TS07JcSDCsshunpwwfTGdiZqN/D0p85sUq8x5nHohu3rkkl6qmDlKi\"\n \"0trbvvAlwrzFvN+h01i4aBeyhS4OCWBHDw6BhIcROXAym2qxNYpFe6Pn2yJn3qgvS0xoy\"\n \"1QFf6bJevuHYupZNyLJee8qMaTYtyBAdqOFep6UQLVYpffbzK/qGyfy8bNYyvXIyv4GBV\"\n \"Q3hHQ156IMoGu+RgawFd6TbsFjWy8TMxbLuH3x0VZ/hpGh4DgorGGuFyb678FayJwgXDP\"\n \"jNfD2OggLEAjk3E1cwhpATph5mCBSC5pC0L2l13t61/onr+AZe3aciBZW4PnxYxiV6/tg\"\n \"eUYeQD1KsYb6wtQUmaQWHZJFiJ6eFjOlBYGWIThBsu+L7yZsKl/2CxAg9VjmZX095zT44\"\n \"mrg/K8GwDh0tE2Fgo6wXUjTINRr32KvqXxChSHUL6AxLpE3pERJEr8ChoZtnobFAnxHrU\"\n \"4gMPJyVw+nWoKqEBZuJl0xy/AxVknmfQq/uqhC94MV58u5Aelzr212Hgrjg5wveVe8fwV\"\n \"dhLYVSw0veXt2x74p8/nwZV48Ipr527U8N4rbrapf9gg0/mBTJ/I6W12y9cdtwhgURRde\"\n \"E1YHn5ImtwqVBQYYFToF0sdw400NrEruuyomiBDHERQQ21fHVEsx4+CJakR698wGskhP5\"\n \"+UoOIpK/zEtpmkk4Sa56gx0+6NUdxUP6yh7Wlwqvr/P8QjkaQkUWHvhUD2ATu75z9fcIa\"\n \"RNBOYIxzcYDPUyllG5o7Sj7NfsKRb5BqhlsEBAFsWYNaSmMeHOuYKANZ1hhWa8zd/o2wm\"\n \"jf7UCU4PxVAHtdL8EkwUKY4IZ+bdemd00TYX9/TsBhmqmLTBMist1iq4wSOHZG9QYIWz1\"\n \"xojxQsHJ4aTOyJCkshKdiLOtVdIb3Wgkj7VHkjFqijWjiDOQUtIyO16AuwCqff9pA6Zxe\"\n \"sQ6hJjpnGCaoP1LyiWhEGsp+kQDt7rt3j2KtsslCtvJ9ff42nGH1atITuliY2Oz6EyiGC\"\n \"ZagHLuSkwGPX5ipBRFypVB3pYVOdLb9zbQcXkkyf7dgLdFMIFFBRiVCchTFC2YMEHqGa6\"\n \"e1tpSUsJ7WctEgYEYvJ8rvS/bIzlSGoSzSXWDGJYq8ZdAzMq7qsqwXsowumLECm56YWUt\"\n \"b83UG0mO+o5XSI8AKaV2W/fJXQKO6NYIua+qi4QwrjEsquTcOZNstzYPpB5uOgkuTRJgg\"\n \"ZELhTiIExoMjR/JiO1y//t9PoujXZYbolHfNmiZk5k3smIhgWS+1YwhhIYTFAU6cE6a+R\"\n \"SiZ7+V9mjmiJb7P9XOjQHID7VEIvMD+EvuL4XbMFwEzHaQWRlkvYU2uNAAcdsU3SiUNZb\"\n \"/ml8hiALmiLnkMBMq9jg6kB3eGRDK/cO1W1EFEFzWcYYXFMG2vsCvQYSFE0jMgRjWGJYi\"\n \"QlVkyyXX3ESGZE+AVzn+6oX89kWBeo+hkeiJC+bJeXKHQ7yXMzsF9699xi7TZO533KRpK\"\n \"JU3ky2o8x5vYxTLqSsYQcrgwGrt2Av56UsLSFpE2c7mcYMSJxP/0BjyyIKXzlBG9P18EM\"\n \"4GU5QBV10UQNMp+Cbgj6LfiLhrKsEiEoZka1geU9eKWGd1irxgLi11ItwXQkGubqHqkn5\"\n \"eX52bclS+/zyBNCCWsCJ1ewbGUL+sFFp1kWa9AP4uFxrxPgPoIJzOydrScg+nkld9ZDQr\"\n \"ZKXSdKqakxeQnEE+p6dbV9Su/9SIRkMvBIrFgk1L6ypXvHUaEx6C6Zb8gZGGN+v6EKKxa\"\n \"3qy5p9atNZRhlWV6IiHt+MWtrmNj4HqDYZoT94HFXU9LXN2y6ulLvGImEfb394ubEPQLH\"\n \"QTcMiLUQZAaGJJDx9G6UoYYUaV/CBz80xJr4dohLIQL6mwq7XXmdQf6ZA6mscLkI55WXO\"\n \"8x2EXBiuA8ELLqdUJPD27sfKqiFNHlhS/sg4aWKSfzU2/jBl3DGdalWmR7EBxD8LkWhKX\"\n \"5Zs5VnO8byhzCkkGv3/yLDUFgw0IoVqVy61JkIQARggwHt6x4y105IXJX5PuhIZkmJJFY\"\n \"ccVxrFGkCcEYoiE4YETor/iqaoOFEGsOFlRdF6XYh7t7Iqerrl5ZUck17u5LJs2U0HUqR\"\n \"yliU4H7C6RsIy1w0ftA0r755jAs+wWm8EbUnjHzPbPa7zHJus1g86C0bWzoJZHMLyzNV2\"\n \"1ftV4fOWFVA0gYyFpymeq22E43DxE9W00ndbmWZb0wW0ZW1pYTRMi9VLZTp6RT4IoV33U\"\n \"Fs3kNCwqLd+kFBbgew5AcVVPKSMbsmcUpS8umtojj8p7xd07jJsTRUl/PDMgYwpJX6E6k\"\n \"oL9C2AEWt3Isi/5orMlomSmBi2/vrqwm4/j4GYkzY8VXpZQeUC8aOR6XMg84CbHsFytZZ\"\n \"2y9KPRYveWNZCn9VnNvQxlWWNZr0r6CYpikPTUDWSURJpG0z7pA5HY/QIShpRAFBbD/ZM\"\n \"6SuWExLbkRI7QQql7WizoXU7OG015WJDXMLaqzqR41z/QMYDtjs6HrxItyrAogyWk3bSg\"\n \"dE0GH2Nz2VFiTMZeTUrrl61dKJcNR0VIosW54WSbz8yZ2czYGyhsJ39ejNZRhXQjLemnB\"\n \"Hs8vggYbCs6c+CYRYqOzStBNJfXV99WWC28QhCr1VrjUe52FB5baxJrEoqTCnf/wDFUbw\"\n \"lAweKPjIhxqhYUwaoU7d3aMXyiuPd/eTguhio1BG9C5G14p5ZjJjkuEsXL3Dimlu8b/fB\"\n \"F+pJelP99iyvrGYAHzgZUaiHqQF8rB6PWCZOkrq0ZIKc6HFkLblaluVdw1uZtjsaC2XML\"\n \"OJMzLHO5cRBg6/+lm5ylbpgmJxPmPMYSUshTmV6gilAaOzAvgKjZCNiJXuIf50mghRBa5\"\n \"XS79PDAnNZJenLf5VAdguq5uS7/9Ih90r6zXYo89wj1KW7fuEFJuJ67EmXJ7MUgW+p0bJ\"\n \"4POA10m88Marmux44YxrHICMDDrW+2+74pUt6B6BYkQZuoU0WRc2ph55yVO5lxEmMvxF6\"\n \"hLTaQJQbIzDGXJY2EHwkKI1yV3RuDiaBQr8H/SWiX80w4ciR7UMIZwzLv7iK85az1ycOr\"\n \"5lWtI0QDpOmG2X+rq2jdO8CoNAAe5SMEF9+BA+XVly36BFHnkxcaxc3QUG4ho+brNRcMY\"\n \"Vpi079VbnwIRFrZIM3X0xC4RuoS/ICN4cSMkJ3t13bqDE+xpbiLMiXOK72++jMzmN5gmB\"\n \"HMrdk2+q7aRAngzPdzVUr3eG4lYZPA5QxVCHBEM4eFeqVf3vV4Wf9fVLwPMC4XbB1JpM+\"\n \"lzTyeXVK0BLLo0YLcSzDu/SEjOg+BDShfrERWZznueMNxQjqyZhh7sP4rPPJ3LEB1nneu\"\n \"OCUthPct+NYxhdfV/TBBcYE/tMUxP6UBWKtF9FLvkhM9PhPgFbe3aXxrEYgLDEifCJRGb\"\n \"sBDy9KPe0uRQuZJQhDAwC5OB15naKkKWhsrKY3FBRH/CPn3D324luC8sHFAc0WOr6gZzR\"\n \"FnT8MFokJFW4OLpMgOqtKNQSkcl8a8VsVOR/6nFrjgSmGcRPpfJIE7Nu3mQ39QzRKdhDI\"\n \"sDZRstDe5mICt8BihLqLc0USAVyUK0pJG9QHjnI0JBsNM7avYURXpIIFxdVTdyOSJClPV\"\n \"iMDjeL4nzVQ1BxTeIsl6mnhm29FWXxV0RV/Vin6eO9onh+15R5GICPpRDB2U+wAXHSnil\"\n \"+YaQsCrG4vSF0lIYBNmrrmuNq1r2CyOFrg65JTS/m6AjRKdu89EwhnXiRL9YzDDfHoTlh\"\n \"341dRv0NH0s/gbUp5k28lyZZvu9sl7z3Dcw0EOxCvMZiDQhVErU2ihVUVXATKOq+mAJh1\"\n \"qEoRh6+uzuDd85xLFGXdYLKNSZe4l9wwtoKy2E/I6f1WoQNZkvzU2OdCa2XyJss8t6LQa\"\n \"xLPvV1fVfLmKMV6nAx6hrJ6LFHlf772XSnnpTuYuaNuZaHt8QhoXRThNhyb4rUt2qGMhK\"\n \"ziNqy/mpsZS15hoRvBAR9vR0COLS9ZWvlhAoDWKTDKyGmeGKDC2ENdxel1sEN0fSPstIi\"\n \"wWan5Ywo3x8XjCnfzr3wXVB4Oy2YdCAGqUhdLvwqFiTESEOun79sa3vu8xrHyzrtfD9oD\"\n \"apeMeL7hh65g2qItCUY1jcMJjMD9XZt2EtwyoMrsoDcR1agyb+qBjc2NhX1sAps1uW9VK\"\n \"TCMUu5wdXv2HzT17kfCxMhFKkB91dKBQwq9KxrHqCK99RRAy12jGExAglrPbYynqFFkIY\"\n \"ZVACzhcWQmGW5KNVauDeVAOgIjhSRCNPF6cfhFADiGKTA+2cZOYPtFr6qOGx1dwiy35hA\"\n \"9lz586vbZB3yjVdTS+1XNsQhhUS4YXhr673/IkNnksW7TcElkWQBgmLpdc73hBEKDbBhY\"\n \"hQOv+lUgcvwcEBITrky9XrsUCs0yE5grMvAmQjfuZi9JG6SfMycGnInCcMcZT1Gi5XZyn\"\n \"4sBBmjAROhDx+qIgWWJNRNUnzhcL92YEekXKo+rnJy1uMjrOOzSWBNK7KNVn2CyralaDU\"\n \"zRI8GZweN6gNQUZ3mQhdbepAIq0nEBQGIuQyVayJXdMk8zhJyI7TarNgOyJ2w87OH78NV\"\n \"fn1paQJ4apUOcsojgA4Cmoo6+UXssmtoihHHGW99vTIGDtsHLAQgkoYJ6VggwrWZLKRpL\"\n \"WyHJJTazzlSUFDvm+eoj8fmlBkKTZkqC81r70dyevcmyLVjKbVp+xXQxjWgPacwH/JGdp\"\n \"hJR1qijAz6km+LOvlolRTOrla7JpwaliQbiB1sIy3wKmvZV6txfmPWCDnFhZCHAn5Xj3M\"\n \"EA2YNcgQkD6HNmU2wJUj+rJe7POEJo0zvl94AotEIoc/qNWowDHsIkNWNBFPOVc0RCUg9\"\n \"5WtrJ2J9cz8cVdRSyGGgvk3OCHGLo6rv/+5upBpQxjWYA8LpdKj13sSXu4Yu3rCFcATOm\"\n \"WHXuueLiSI+WvLcTSiBU8/LZ3/8OmCTNdUvTKS6FDeQoiIoWQKHEszTq9c+S3DHP0Reql\"\n \"F2KjIzZdjCMGshIUwwu6j64pJ+8Tx37q2NvP4G+y41oyroZU1veqXr+HULTJ/gBTVlCox\"\n \"KUEw8QTHm8vJNc33cba6M6yZRGh7o9t8X/CuOMdYY9+ytpzmJ293ZndclZ1IHdVCHWLiR\"\n \"DPNlSj7JeiMOK5qIZN9i7JeVd0ln1uvv+Tmhs40wB1igcqyXtWNczFYw3xpXzj7011Q8O\"\n \"5y4eMEkVO93Q1ELTzcA+Pa3o3vqShp3/xjl8ODtO7BgRSB9OoNtww7cvhj7er6liD4VAq\"\n \"bLLCQj52fxP6AByemr69PPPPG+D+ugzpil40MhrSnPHhd4z/L2nKQj649uvF9wqWhkiwE\"\n \"5eSj8NvyLyFNiM00IWhVsR5eTP0Vj0AKIkZMDRgW3MM4lbKsV1cMZb0gpshnGRZCQILVy\"\n \"MWEFwVpBfMkHIUNS0ZDQC1Q7ZzLkXIpcOGXdaWBf4ZhYVXud2FXcb8imR8p1dkxMvJPdQ\"\n \"vRqTvD0nrlIX1o5AYthGtY1gtNyXUpasvpHa+D+CgqAcbFwcyVCwqsWvW2N3A9RHoWpSD\"\n \"rqazxCbxaxBAu/rjKOo38KjDgwENlmKTWnll9QXafi/wpYQyh7QztT2dNixZCJV0aIPXR\"\n \"yI0XoeucoRaoCSfTmT/0ladQRYdNRa4lYgqxIbd53uROAlmPEJ26M6yQCEve4KMgQlZ18\"\n \"EH+Ci5NbHMwU4MURVCvrC1XyZEnL5iTrn/POHPAV1P2izcSEaKsV1nhTkJQrQFOOEmaGt\"\n \"K8TCS09acJX61K5oXGFsYQYia2cyrQVHRLIzpk0j4jI/zRFhpTJb/lcvIq3QzOMPsopC4\"\n \"F1wdh0mkpRDK/UZF9tJKxLfWaujOssKxXwZ3YYRgljBplvSDnL3UgEd8viRC7W8pYJxTu\"\n \"lWYhoEh//Lgs+6UZ1qusoALdMXlRRY2IoIWQKZ84ORXfWFHv0VwEGOFzhG3faL92YEOvD\"\n \"MmJuKwXIQ1jCFGWXtkYQu5oJhxGUZNxXDfNi4Q7ZLR8X1vLUaLX0ulHL+PIfb3sz6cgKf\"\n \"AsAGkj0ESdwnqU/ao3w9IP90o/miAovolJuTnhtU1qvHdR8eSAYZmaJyrBaFqu4geGDpS\"\n \"GkTwHfxrcB3Gp0oYrhYUQJKugtqY8igB5vekUpbGsV4E0K/JYVDrGCq4DaUyHb6H/LT70\"\n \"V/yuglvrfAlwAWkTsbBXv7X7By/y4b2ajAmsHZC8WBPt7T92EyNWvuxXoBX3cayg19jz7\"\n \"NeVYQlVjlAqwsnOn9zieSzBoeCyhM6KTopgOFfXtD2GXQ7sKicZLd8v1sKCAoaR/XrZUl\"\n \"ixDoIrUvmkfQIBSNpnrJLHZaCKkuVieKnu97xgTi98/UcfCTR3t80sCFLLX103cV8NsCh\"\n \"h6YaJkJynHDJVtCXhgrhEP2Jt6lrbSYQoslMhdcU9nGr6J4wlJAaAlNU9PPy78Hpnk/Mm\"\n \"30f/t64MK0x1e+7uxzcgrB2BrMKloc4wLI5EkYVA1OoyB3dvOCyOPNVIEGFBAYTDXikWj\"\n \"QKOT1h8lRExKT20EC4OaWOuIOtgWS9MnEij0j3wTORzGIZv4TGPYLGuooRFJDZmxAs8Fc\"\n \"yF4VsI1hCqg+dqDsm5/xlh5g84lJ/X4aFbhd3m/o5i/RQYDiyFKNG3ySsMbpCPijdEJ3J\"\n \"CWxg/BwTBTRUKm11/cqV0wVJOf4UhUC9BHU2bWJCytlw1EoT011qz5vsuYWVXXPaLyMGu\"\n \"pUHPqqLcGU4tJsxH0j7NzyY2iOPyeIWVYcIOKnkdHrggaNPxR5Bl1EiCX1HCUI5hYboQk\"\n \"oPSXkZSpB/aMy4zdlQyxoWuCTN/mNoaFOflNlZ75o+FnrOk3zAbNJqlUmZKM4fFsVDTZP\"\n \"3IJfW7wM11ZVgDAy+I5xXd6wez7ZCjoV/GPwWJUBAgATtP3FVaW47XyiZ1ELr+lgLU9iJ\"\n \"NCBnRYk0gBNdR8CSiKrhlsS5j+J3MnCZ8c9gLkhf4gFwVx+VKAdoTpurRElstGC6AG+WO\"\n \"RBiLcEYrTfmuoaXE8XjpCvcQQzLzh6/7FwvQEmLjU48khE5R9zNpyIGat4uQx132SzCQE\"\n \"EVxvz4/LuONSo6zE4VTldw0iAMwCrOEc1lCXyuq91ZaWy7EH3UQ9Pwu9/VauQJK+POcr2\"\n \"ROuE9YBx3IMGReKjYyDoahmEbmXM+2Z2Ip68Vx57ScYFB+UJIxhCoio4wLXU9e2rD6TYJ\"\n \"5z1WgpDbQpZSeTPZcBDleqTXzR23Pru4u+ppogS1CdHqm60dW10elV9eVYeXL8UZ+MPZE\"\n \"oKJHjcQadrPAcG3LQz2Bi/yqFh+j0PNb17OvO45gP8T1okITk/bRrYHMS8VGjUUCFTbAU\"\n \"lnWy4MsWbNX93zjgzSKrvOCYZl6ehMSxc13aWO/BzISyAqaMNv+9eC6D0xgdvXo4imPCF\"\n \"pZufK9w1DoX1tK5o84kUQ6lXUi3R18Dj7DUhifmqduDCufz/McgbEECc93NruqWghFvmp\"\n \"x5LnctfKtlzgJteyauRzvpKUxcUqURaogrxF5lEjah1e+V7PBqQz0mLA6hc4GDjgxgJoX\"\n \"fQ4HL69MWCt22A6syWBhiuGDByLDw2HQ1Ns/SdiO9+dMQLnoplTJODBc6tnl+gxSNWX+q\"\n \"OQ5EVwjyn4FgbtrcPBXu9hfGAMaQd+zuqgbwzpwRFoPTg3+xQZfK3YzkBUzqxoRQgaSZb\"\n \"0MI3Vt5+q3jxJjc5f1moXLB76QOohE4pFLrmMN04EUw12QmPmjCMlZ8KoHHlPHjxgBVXG\"\n \"oDINUwJomFO6VOtRWB+YRcbk5nkbsbxYmMgUlLGZoSGmGW0pdXpF4+tMEOFcOy6purPNf\"\n \"jRAdsT4hYZ0HVeLC+CSX+aFY+BdsXiJEB+hYk077G3l1bzkGdOE7a/u1bgwrHIOrlbYFm\"\n \"t3OYpjYjNRjWMCj8KvRLaG/qv3II50HV6z4jqtgzSj7RVRzvc/diAieksmwlJMlyiBjxg\"\n \"JY8I3ClGdnrc3lkBzJmOceVc3fCjx1du6/4/nFK8yVvgDqan7Ikm6EH0M6k0S2ilX/75t\"\n \"2/IcRRjeEx9gl9Tvj5lwu/NApyn7hU8X+fOGd8b+y7Jfmt7WZZqk0eJDPizOmsG4MKxzE\"\n \"ZOny/kwb5Wb1asuJyWX5nsBiYQVhIZyvrNdihFAW6fHylGMYmdPlZH5zMix+SYY1M2nfY\"\n \"v035HdghszcNFK3VnSsFxksas37tBD8xF0+L49Dum5eFzGdYFkL3VPP30C7vmH6iamx1P\"\n \"iKxL6P8dlDQ70xwCc3A9OzLtu2VRSJGxaR0uuJh/KzxF5rmYj41K2d/C7Osl91Y1gncv1\"\n \"Crg+8YA8kLKB94eNRAxDPR5LoTBtijml0iFQhS4MjJy2FvndOKk3nX3SUqhg/yLAcvo+B\"\n \"+pc2FN5NJTMkRfx3Zufqw+K4XEnKnVoe/PQReRwyjew/g0WKh9fSTxz3YLv12trTkK46f\"\n \"/9NO95/EYHx5uHDfZCNo21Hj0pL4er1vW+AJlD2a2EpPdqnV96b2Jxxua9NCkuheFv57V\"\n \"VdWReGhYPQdObIkj/8KLWJHGRVkNblYpgHsVP4bmoqrXde4SNzuZ9cApw5CbXR+apd4tu\"\n \"FCwow3x8zNXDLUrVBnwJDQlqkAZZJ++KZx5wm8Z4I0p+bmoC/HvIpAidLmItoMArrqJPJ\"\n \"GpCurJPbE9/3IfZ6KuJMqyGkR47kxXghpU/Bn+9yNZk/wj7q88qiFODXvrE9CF5MgLlii\"\n \"cej7qkLwzp6NC/WYBDcasfRf6tj26A8Nct6iUBW7GY927/vopzsXiEZ1jbxJ8MFdm6xmE\"\n \"IiiFlGqeVSmWEROrB1IX0Od/fESD+HBd6/eXf+JUhYX85mEdxJ/8RGNmRSweATpSLiKFN\"\n \"rf2LHjkPF40HeypddMKIGTS58mfkDJT8QU8gQoHg2iKXADpgQU0jCLe28fv0T5WR+R2Mh\"\n \"4xgJ7h4KDhyQFsKXrv7dej+Y3I5UHILu712hzDvhxY2QnEu6vgOerQATVFN7kyJ9e/vuN\"\n \"xzXvEXnP/Q3JwMkZxMWwtofFved2DR9q1SALd/ICgvhnp5aK8MsDioX6/HjOUpVwFnyj2\"\n \"Cc5P+cjXATWLyTSK/QRSaCbFsCB9TsTx/a+XtfInyH9DyjNWJsedE3rNZnXY/LtYrMHzF\"\n \"CdX/XlLBENo2ViYSznb/19/fHwlti6fT+wWhaV28YXzS5C4WBMog/ks48D17Y6M/Q0Qiv\"\n \"9EA/SVDKea1qXiBHj8oBZbM/dxOM6sZ8zn9kiTwKqmwhBKtA0j5D871gzDJXiDCUHNUWM\"\n \"bZc7riQqN6+7//568Kk/nIGCR/BvmJmEHMOCLMT+CtWJWEhtX7n7fv//L/jsGZAuRy7xI\"\n \"eFLwDS9QyqiQt0q6XQk+gCeetuZyd2FH9ExBTmcshfGUOrC8PqGPikAH6qdGNPKgukB4a\"\n \"aqhrdRwlui4UVhASh5fJLQjl1EFJdR6lKOv/xyDez8SORM9NC+MAlMy9v3Hsyc8b06amL\"\n \"G3e89aYE5EisoGIRCCmLrwk988vMTwYNI+vBx8oo70OyfJbf3pmwJkeNZ9+x/xO/KH/PU\"\n \"9yLdfx8zr3MH51XbDsxXk3mj/vGEfOHkK59XS8HQQ/EMkd1YVgXLkwDf8D3cR5c0jErHs\"\n \"yDaZD4kKccoq2WEhIW5NolPUwSdK/AMdQfCGWRZ5oHOyU6aCFkTn9UdlC1+Qn4QyHy4vw\"\n \"O/VAxn48+JGeugR861O9Suf/tB/70s76f+O0Vq1IM/ohdshGwIPQIQmTQuSppTo2bf/zv\"\n \"9//VT/B7VoeJS281GwdSrdDV9Y1XIOQh+yhyFbHotpKNS6ggso/ijThCRw1m7AyL1oLDh\"\n \"6WytOTe3UuRQ8UGIwAshJBo3dToys4NwkI4NBSFU+SQYEGBvuoVivQY/yyRnhcwQwPFTn\"\n \"UbdSf0wWoXCvcDR3rrxlpPncoLzLxj/199cGJE+0wHNCX4wo4XVzoiQD2zvT1lFkaTv/u\"\n \"OA3/1DJ937Fgvwm9knGO8zw97l2PX9e8s6Wb2LP35FF1CusNd19d3XL/+bFZu1vnIaSR2\"\n \"hhUm7bt798UVgH67YwsJK/KBhNNb+yvKenH30vxrj617+ir7OXVK7m6193nvTsPwLhaLM\"\n \"PwyN+ccjVlGSYhz/jjH9fX8isRHqy5jkD3fFfqrsJhIPeBAHKp/LOglo9ey7o53T47pL3\"\n \"WuSCTxkUwrajaPrSNw4dycsMx2zSus++Db9//FL/DZlKzi8Ldi3/M1gXv4efF3GD1Qp7A\"\n \"OS3Y+YBb+HgwLegPN2ZLJvLZOXhp9Mr/YRx9mjrw89dIjKLKy0WNZL1D/wmNvzK8kBsvo\"\n \"RJ7yg2L3PnLkSASLISfE97Vrv/E8Dnxw/qMe6J6ymogIQ3LUxAoXSjkkZ8J3E+aq1zg70\"\n \"eV9qmyuD+t9Hq1y3/L4bw2vTb3pbYWx1InOlckklghXyVIV8Zxn9BG4yZRutnWkLLfY8W\"\n \"8rUo+99dDeP/ht0msezKq+ktU9vIRlvyy9E4H0Yulw3UZAm/eeEcE7YSmEY3HWtmUVnTC\"\n \"6JYK+p7uInWGFO7HjD+9LpowkUt0C0fE4lU2PqqY32Mag0Qx0T0gQ0TlF5gVh6fr7RhBm\"\n \"gjQhAuX3EZtgWGBrSnJx4rIckgP47mQ9SxyXNVlesiZM13oT9VlkWk/t/JXR1ft/9u2ly\"\n \"dV/1JZtM9Jt8FXShV4LDqaomLzIYpa/w8IimJRgVDqcQa32zrTl2ZlzfmHtT3373j/55m\"\n \"/c/itfziNGkByxfjqr2djJ5eR3umW+TrUCdFixr9vZUCz6DdCpe9ksXjRv56JX13hB7AM\"\n \"fyvWLxVko3d6ZSOE4iIwc+MNRKdMADE9jyIGlo6xXl5AgwnxWSwVypkiP+sBfZ2gLJRb2\"\n \"yz9EBJXtNpYPJ0P8gFelGlYsXRp0PfH6N+3/zTuErbd3qZVhahuhUMLjePgUYjS/bc8fv\"\n \"j9p7H5n4Kx7KZPJmNl2FIw3A8Qq8D+hN3XwhkQ3/YpfRP0dHNGNZEYHk0pYlpXUnGLmK/\"\n \"5U1zNrVr77iUN7P/pRWCaFRJc/pLm0UtYGbTR39fVJXWoQbLjk+8lblWT+iObJ1fbCZYQ\"\n \"WOI/zJY6yX8Ixj53H1U4dDddg8BjO4PdWaVwPrKFfYhmMw3BwEEymNSFh1dDNArfk0H0/\"\n \"HmKg7BdZlFRg8wYwNM0BCxd1CPmTos0wRKGFCwSvLH1y42lI4/GQ+iREUGhv2ZZ/HoaMT\"\n \"//zhV///oJ9472BN/HN6UywyrSQWAKpqsSOwHVERKOxjgCrNJcKZtErmecC2/x80lr7N0\"\n \"/v+Ui/ZEx/oB0PclZOO+7h81KPmpHgR24OurZ69S9dv3nz3Tfh8b4OOdYaykTnGhis4Jg\"\n \"KbAnIjcXfqfrAR6Ax5AFz3VXdd7EyLAAPYKXPjO2O7bRE1Qn1ViVwiqBeHFO91M215q7L\"\n \"RGE0FkI5Gf398hWlMb9aKk3ygwksTC8jFp3A/4qF+IYw33s1jeRL/CRDcgYaxrAIQ6hP4\"\n \"hGxzFg+ga8/8cqVP9w0MnXycd/xdxa8oQNYPrBYmYHvuTrSOk+lrDWn4TV+ecOKnacf2/\"\n \"AM6yqWx/Ff4Sics04gp5X0XleHTgEj+QAzu/q3bv2vJ62E8xjf40uhjCc+1GgBQnR4ZNV\"\n \"3jY7+/OoVK373LqpNAZH5yJhrrAyrbCEMzg4e67o0+tmddgkbFshGillqoJhQ0AgmSq/7\"\n \"+o09m3/wuqb9UPnIEw3Rhs5/UJFdLpW0KWalg/GB4qY4a4QWQiENqIOWEBIQW2CWCijMo\"\n \"ftn+WVYICK8oJGvPCJyY+zXDpmH9H7vyS3vZ9obkfpmcbh+XGOmha7+vJ7LHSlLVP2L39\"\n \"aAK2TZL/gz+t55ptsJAnjRKtbIVOnagNf1sBg+AvDAsKK1FMbKsEILYcEdXu/5k+tkam7\"\n \"1FO6w2/lgIkhD0HEKu4PIxYRXCj0RNbpH9Glr1nzzxVs3n0fZL2N3sQhsYIIpZikdkgPp\"\n \"EzmwDCRcHEvo7SJpX04UiOiPCDdL74Zzhl7AuDS9r6/X7OqSvm/Tlsw+WAjKaTBDIxB/O\"\n \"4X0LYf1PCSsPG7nP3Vbj4jbHIDiYjWS+V0FoJSuotlQIxy1SOaXThupQvHWo+j3tYGBVV\"\n \"TNRiaNx8qwusvATro3DqQyZmJySihvOAClGuVt00zgWOafJ2AbvqvH1PLT3vkRwCpFYlg\"\n \"Ki7duHIbzn/b/t3ctwHFd5fme+9inVi9bluTYsWVLduxAEqLwKBC8TlKYSR/DlK5oM9BC\"\n \"mCYlbaADocx0OmjVTttpIRAIBUISkiYpBYlJQ2GghQySeTUhUbETbMd2/I5tET+l1Ur7u\"\n \"I9+/zl7beNIiXb37u6Rfc6Mdle7d+/9z7f3fvc//7OP7Hl0ulGwKAWN0mu66uQbohU7ZD\"\n \"ygRbomhHzBqfhBzlfYSuarSzWXW3Ou94KUKMh9icofuu7uF30Kz/poZGMt9ClE9Yac0UO\"\n \"z7+/fHehpXVPyyPSLtl5FO7tGN3nRvgC1luBOBgozKqI8RsTsKLX1CrYKAV1IxIkkMQhq\"\n \"J4JHufD0SMb2ApTOmv4Q/GiVPjCXPIRYde3Z3JMupeRIyq2VTnFRfE8EMYdCG/fDgHEEp\"\n \"ACpg1wFBAcCrRo8bfYNYo9jgV7zNb1OtghbMoSn3nKYBa5cCQfoAi23C6hE4Dn7Sb5K2n\"\n \"q99rySfPK6bj5HEcEYFPPuUYVRmdt64dQTcjMmuuTUMSXntTG9lLYY5CdNc/PtJ2DCODZ\"\n \"f5Y9GI0KXOLX9wjL9cpIF/5NzILALv2aERUKmSxHdBTu7Cl4avuxpNKCvOD5sVSaizxFp\"\n \"dGRZ7LcO0Oe1qFPuHxe/5QszMzbcp6RUebBfnRf27m8kzzNdJLpt44mhsCaGbwOSR8RLQ\"\n \"xIQgJZOC0Xc1SLPl3oEyDh5tP2i0Aanb2Liz3mKTpBtv2pGWL6QL048sgwBsL3ce0Coyz\"\n \"bArHS30pl1rK/75uNCPHE3C1bUJFeNbTv6IjhA1OcGWZL9ihZY8gHDZ4+EcKbn0CXHsMO\"\n \"csM4asoMFR+3ttRHwNm1K8usVD/s8HuwenOby2odf2BbQU3iKDk7qdgTnLqdvBdn2q2aE\"\n \"BTWFj9O5M5c53mybIy5MKa9LeAhRRTPK7VcgD9Bq8GIiboir9CtXPngKAaS7qaEAQv88h\"\n \"K1IulLmPx/IHAsQzXppWcdVh+idWmqf/IjqYV4EkknxETOaeeUP/EfXLz+vxCcyPIq2X0\"\n \"1NuKrcDC81E2ROYc0Iy1865ItHN0ZjpoEqo6jtJeEdATYapJzgl9f308+dRvdePNXiJIA\"\n \"ulyx5ZUPPUE4hzHpoIiTDSTaPDJDPArGG0OX56q5PZGlJ4gdszvMN9XYNEfADkGEjOpzP\"\n \"6wVE8tPNVbYziO72CBPCA9PXEhx+HGIQ0NSMsI4fH+NAup69hunwEMKjgQs0eNWlOhSIQ\"\n \"fUC1BxEQG+lXW2qbn+v+m3/hGOsaWx2hsKwPCTW1oQcX1WOBX+IG4yLOuKm1vxt+o7fem\"\n \"vB31cbBopAElH4tMNly67fCy39pXCYbnq4qiQbPom6bv4aIdoIlzsIMWtGWFi3CpvN2V5\"\n \"l0t0JaC2Gfh+4KItWETerQwLQZBC4zrkP/4Rj7LKf5QtsXzRCGToCpzm/0NA3mWOh1Eph\"\n \"1jp0+bLXccLawgNGGyrUJX7wNCcnxgamdc06TOWQYL2QjrBwD+Z2LCgpl3vP3ldq+xWMs\"\n \"lITwgLpg2Q5+ZsoiXy5LdrkyKZd0cnPlzz41Q+v6RnYR29sSdaOQGg5hfuhsWxZelpjoc\"\n \"cjYUNDmXT5bpHAAWK5kUgIAbWJB3vaPngmjTy7RpZYod/mUh90TY2i3A3hgDTCX51f+UM\"\n \"mbHBCl3IK7d5TPc9Qig5GOpDrvyaE5VcZffrIvWjrVegtws2JUaNj0a4rHGBWXrSPRY92\"\n \"sCsytJfBABM155ZKeCOKWuSxzJRto90eloWSBQAiIRiB/1Y2Y57oaLkKLbaAS2k5Mvec1\"\n \"Lv1QiCZTPNDwav9oiNt2y/EBXDjLGtxnMJqEjiotl81IpGNnE1DmrMShNXqUBxPKQCRhJ\"\n \"dnoHUVL6hn8jrlVKyt1io246VRNH1F59e2wTr0eGebSUxOPlR5BjwkpF1FjPZ/ubrrwy+\"\n \"LigjpwOwQ8kx08Unit/1ytPDzeR4TI2WRD7r+7UQC5e/dfKBtv2pCWOPjT/L9ZmYPb4g1\"\n \"8TwUqisUiEoY6CnmUSF1qBJGE18Odiduq4uMY2NJjo9pRj/rzDpaSPcsCnEIdG4V7gx2h\"\n \"2IUhfDy2dj/ber7wudoN1u2JBVZVYhn0F/zPW6GEUbbL2MawQM4Z+WzD/t2DthA1gsM1g\"\n \"RyDtWEsL6bETmE0Kr6RLntmhymqnMBvzL5LI0C7lLMjXANa11md11IQ5RE0fSWpY8+DSv\"\n \"RIyuWmCg1wOy6sOWroUa1oZhr5WdMLx5e9WEYInlJYmoC8WpfU5/VEwGRU7h06U0HYchC\"\n \"2y8yvMvc9su5UqAzEkiUQE2YZDApljhFN4MSE3XhgLLPGLgF4CHEWrsYmm1uaj5MO6hvF\"\n \"Ldok9URafmUmbPPRMKMWlc1cmkIi56roa65Fja6PvHWNelfkIGXyLVscNUXaohAml9QvO\"\n \"0Xi7xIQc++NlPDg1ay61IXHXcVqsNGhKmlesN74ISFuzFP6sXZH8IfPIT8fG+48nAh4qj\"\n \"h6FJTSsj4Ulv3ew/S5zu04Np6XXi8C/8nW9azz/ZbrO3Bg60R85PdCZ2SoMGjDWF4XASe\"\n \"3dIaNvLZpq9uXv/5uyEvitrxxg4Xiq7+byACdH6AoBCWSZU/nBdKTU0aKNG8hxZ9CjVv1\"\n \"bFjh0uewuqL+QVOWBsHhVDPHbl3mevleiiigbJd5p1Wgz6g2xTy5DTLTOxfyVbOkhiD2i\"\n \"C/e9VLpP7+cc7mnZ0jX7Xy7uNL2y0Ttiw08KzroDnbzW1hKz/d/sRN6x+4nY6OuyJWhPL\"\n \"ZRuqKjKQHgw2UnyIIQN5Z4A3puAe+rufuAqBBLJanIbUrruvZXto+iBSdwAmrlEIIbWG6\"\n \"B/eAhAO1ATeFOl+DC4ATtylKesavzJN6g2vrtYBjlzahuyV1EqZ/r0gs/6Ax42yPxk302\"\n \"qt1V2NfRrJZeS51Ui5klnzzhnVfeg99Qk1LVQqOj5F8z8mkkElH5Q9RQ13CkCHhZHPicf\"\n \"QBYR4nrCCQDJywfBadmp1YH2sinuJF/qUiLK5WI9/ZsQ247lt3E5BBtfUq90ehTsKUY8i\"\n \"W3Du1pjX+B2beOYXqrCHsR9w7y93hArbHjwFOZEXDdNGKPWbkMy1fumH9l/4IGhXvsExd\"\n \"aRawG7VJgxDw234Z4eUHXNdA2y8SREptmGt9nlfkhnc/06Ma2AInLD+HUGf6eo9syJK46\"\n \"88HiYyUuGj1Ii+VoG8//7NGvGZszCZ7VkfHI7tXxiM3GgX3eDRucE1LkEtgUtEJBEXdc+\"\n \"MJ3dKcWNbNLb/1pivu/ws6AmlWiqwCw7pmO0qlRG5ee/vgUfyWE5ZFSjqd1XINMimQwQ1\"\n \"Uuo4kK2V6VKW8BE1YbKCUQ5h3z2wkYZGkKR2QdDdC4xFctuFT7dHeQwTm8eN3NFTO664b\"\n \"LxJp9az4+ta+ROv1Rt7Y1YJW7OQLBulTyEMV8iGSnuJL4AWMxnUzFo8axdm27y9pvfa65\"\n \"LrPPITPeCt2RVZ0Jsg/SoZ33vaLsch2XgII2rFskuP616mYH0Jl1p46lWoR8lXnKQyUsO\"\n \"AhpOJJ3oS3LY6ggdXFIrcpB3qMYH4U0VgBUB67ousDPKQhlUo1/Acn0roPpHXZZV/bdf3\"\n \"St71Jn2GPNsUtFolj8QbCoqBO4EvLNSIv+ptr8M/AQSWS8mz0vNFjCcOMRCOsOBt9Riuu\"\n \"GLhp/f03X7f8rhfoePQFlSc4F5TyvifafuEk8Ny9osntfKdD4+ZAXptS26/lth0vVR+tz\"\n \"lMIPSO4sXGjEObgwf9ahqJ9q3iTcE5hwR0jkD1Bj0brKiyOmnYBU4c0DAwpfvHbQVo8FW\"\n \"bJR6Yw1z/55f73PX4mn/+bWNR8o2GhUA8CXYsoBA9hSfmiFtLnyU0vefsnFgppzAoZOvW\"\n \"wy07pBScXe9LSWx66YcPd3/IxLC0Bi9wt6L+pnhcFAv39Cf67W3oCzXlPkszceSOT8NCw\"\n \"KKfQQ/hQzClmKEVnTyolWrBVKmeghNXRcSWunxGsPPJ9uqHFiV3RvhrvSTeQ9EzR5S43u\"\n \"N833m/ero0jAEOOQcGa5D0c2DHivaHnsScg1RNP7b3l5uxMMYUGrJuga/VEooZuotQDBY\"\n \"zwlTehDK4izIsw17u2ebTohJ93WfQHrfGe/37Lqru4N5RmOOqlzaTGG4cq4zoBsihHByc\"\n \"slzn7Z1BbDecB9bik92S63iALs6NRw5wssjUE8/h4pir5AiWs3YnvcGFmiqd6IzEYTaYY\"\n \"rQkDPQZNuuqho0sO6ClidXCD+7oMtfVCk0qJBnkPSRzStojA3rL269/Dv9/bs+fO5sPe9\"\n \"AZWYL2zucwGx9ORrSnshNFQZzZiLN0esjKHY7H1u67tvrVUo15MjPaVTI6WOhynxZvqcZ\"\n \"EiIJrzUtuvnP1z9AhwV+TzPEVHQk0LEHv21QT0vn2UU1j5tRYombTtO9t89EpucJeK7M+\"\n \"elxCN+hAiaFTPv0jvJpOiSeXZLSR6Uco7ZAgWNI4f7/D6+u6lpeLTpb9XlxRL3VEN39M6\"\n \"vJQ2ghLVlGZT1Q3u1Y+nPq0jAoO4yaa1lpaPnZo49odHEPEOwpKv4LZYAXBlsIfAEWE8Z\"\n \"II535SxcNgCIyyooyQEN1wXncw63oN94XLUb0t4UyyTGZ5jTVzWdt0BcWBxt6qfEOUdSf\"\n \"y4IqePUp82bRrjjgye+zhC+8LDb7RiT4KEYd9CwcDNmsoFLA/txbE12VxxzVEanKuzENp\"\n \"+5d8so+TECwVuc/V6T5/+QGtb28NnRDG/NGexcmUOjLBKQrgve8NNW1/47mq7iBWNlAZ3\"\n \"0dYLtqBjqztSvxaAibtVueA1YvtS5YQ5PJqcuUoijTVCNHXMOiMArRs3rjG0uvT2uuR+k\"\n \"VN95ik6kK7DtrOUUwjCqtxTyO/UQeA8UhLi4KG98BDmVlDRPqqUHMS+A90HeJ08hDqLbe\"\n \"eeQdwB5PydA5212tlFiAC0aD50L7qNtBgYiuh6rkhzEXuqySP3FEajehjl0jfQEfxsmEq\"\n \"OFhhhnWvrdWI9ak1HbElzCD0EiDEGxZKxvQTYsJbi1SUqAU99RyHQWASEpxBxOQfzeW1G\"\n \"0mJ+ZCpywiF0+HFYD+HlFyGsBLvACMs/eM7JrAlFQPKIVsR7smlYvK0XpeSEjLZtJLNPt\"\n \"L786lkhsHgQGOamga6u1+/HDZgX84OCJZuGBThhb6NH5nFPoVZFGafACGtLckzYVTzv9a\"\n \"SVcpOgbL88EKOifTYarxuefkCIl5RNSiWPQmBBCJDqQoOxj80yT99bqo0lHWHBO8DQJoD\"\n \"WqquEvLx7VEXKTCCERZ6AdMlD6Lgza6lyJQJGpQMOYHkW9dXS9COdS96xj8DbUsO2XrR/\"\n \"NRQCNUQAXJDkjjNcbzvp3MaQ8brjbb+gxaybmvrjpQIPMEYFIxDCGir1HHvu4D+1gfXXU\"\n \"idlrL0C2XcFc5r/KxSAhRpYhh5+aW37b0/ShrVv6zW/OOoThUBQCHjM2EnB0BjyXXdQAn\"\n \"nbL09bMjOT6xJzrsxTGMjkNo6Ig085WXgI8928rRcytYVgMj1SWy9iUuN5kqoebb1kmr2\"\n \"S5eJDYGxMzIk0LCrmh0HXnWxaFhGWixSdMHSGjSTk+Pi+ivihoi/RAc8fHSnKIdS0ontm\"\n \"AxIdI1ivImjx/C0keQ2xEIWl6XqYewjr1dZLktkrMS5CBCj7gaZlGM0HUC75JJaFdOXJR\"\n \"lgkohuydLApW0X/+Mnb9LqcEQhh7R4XOYS2U+ixwsTyLJCWPuVMZEHboq1XseAgBita17\"\n \"ZeC5JNbaQQqACBHTtE45T29uRR8NQxal8PyhIOsAr2V8uvEIvC/H6NOEayIhkDIazb+sc\"\n \"phAFVGgpXy+hUFQAxV3gIzdmQFeY1sPy7k/hcPSoEFh8Cg4NprLLAUWwA16DJi/nJeA2S\"\n \"3keeQhRxWkEoV1p9tGrCEmAJRve8wmryEMo44BtEWy8Er3nakRWrbj5AMu5I1a+tl4yYK\"\n \"JkWPwIgAiguKXEdM28PhToQOcg2iCd4HTem9U1OptpJvpGRktxlCFs1YYkcQk0jDyE4tJ\"\n \"eEQriodJCROqobumbqkb0r2Vsb0tarjN9FbaoQKAMBURQPaf1byfBOznB8mU55mQazqZi\"\n \"Ep7VnszlefbSUr1+WjFUTlp9DKDyEdodwX8rI8ZRDCA1L03YSQo1o61XWL6M2VggsEIGz\"\n \"nkLT3gvCKqJkMhQtOSronjeFUk6hEYbqUHFOYdWE5ae25Iu/3ogqmCEXJVHJtHaeoA1/C\"\n \"WEglKc7aFEa0hMvkECNauvVcDCUABcdAn77LMPo2oegaBTzwzqRLFtyDeIEbpYxGbusUt\"\n \"GqJixek4kk0dzVJm9nIF8fQvrlgJZOy1VUsOMaVqWAqe8pBGRDwDdgL0FvS5zme8hWiyE\"\n \"bYZ2VCcFFV9E/PtHS64WOqglrhzbCgXGdWe6uJG1moQev33YIGEXagutaJ1qaug/QcZWH\"\n \"sH7oqyPVA4EkL42s66EfURcdYYyvx3EXfgxfJgRDck8h7N/cw7nwPYCOy9n4wm3J8o+MI\"\n \"O4WRNmWy10qJYb3Ltyu8f+Lon3oyj7R3/3JIyRPKiUy3Rsvm5JAIRAEAiKAVNPi/zM5id\"\n \"pOmkb9oKVSHsAN6FNIaXv6yomJd8Z9Aitn9lURlp9D+OM9d3ag2M26ktuyqn2WI/xCt4X\"\n \"T0qWifYYe2wljpJNOU2lZ6YySC52O2k4hMAcCohv0smX3b3U9/WfxOFe4eHzkHBs36i1e\"\n \"fRRaTZeumwkhBFSeMkZV5LKxVGXUYeFOZOMspRxCHL0sAcqQteJNqbCFgZAG17P30E66f\"\n \"7ef/5oV71B9USEgGQKkrZyr3BB5mMcWyRdehPZ6JBQ7OTnZNkMQDg2VB2RVhHV6vI1/37\"\n \"Yn4CHUQw4s7zi8jISFlBxNC1ttpZQc0YSyPKjU1goBuREYGkpy84zr9n0jO63tikTQHVT\"\n \"jncJlEVyUd2LW0b6+f4eDANVSEKlfjnBVEda6/t38YNBf4CHkTMUBK0eAOmzLq4zmc1g7\"\n \"O8V9dDzfs1mHY6tDKATqhgA1KCEta/ny9AzyZT+DQgR07LIIoZbCkhnGQF9lqIKHuGCi6\"\n \"09Z8lVFWFvQsYMO7Lqz10gX9XEOedeCmxcWrCNNiTX76e1UFSVaz+1WvVIIyIjAKLdbLe\"\n \"189MHJSe3ppiZYbxkTlbIaLC6ISscqTDNY9Cckyhjv+lOeUBUT1vkeQjgbV5KHUNKBon2\"\n \"YpseOXLf8rhNCxsGyWF3SeSmxFAKvQIC0GOrwTc+mlfhoNku9qzzyGDb6AnVRi87IZNwZ\"\n \"j7WOkeCVNKOomLD8xMXRXbctRVP6XrsIDpDPyMcVYjL06Zr1KwJJeQgJBTUuZgREt/Cku\"\n \"XTpQ097WvhvW1pCFJdFmlcjb9RuLIYlKjOe7Oy8dy8pPEh/LptEKyYsrKv48GyrE4vSJQ\"\n \"6FfsjYh5BIFMVPUVHwRRJ406Zk5XMWU1aPCoFFgIBYGnZ1feMfz5z2Hm9ttUjLQqJHQwa\"\n \"I0jNyOfAVi39RSJA06hqH5ecQFo2TG8Ix3cKKkDNWQ+CY/6DUzRUeQlez9Cbe1mv+TdUn\"\n \"CoGLBwFaEg4Pp7jVvWC/9X2nT3tPEWlBs4G/vO6jCC0P5WXMxzs7H/6h0K7GKooRq1jb8\"\n \"D1tsPuvojrpGKTeSRbSgOZHkA3F+WdN3T5AQvpy02s1FAIXMwIDAyMoRZA0V6782Gw01v\"\n \"o7MMJvbWuzQpgzGeHrtTy0kdsYmpx0Mq7W9nGBd+XNiysmLD+HEE0nSjmEMkaOe4j7IA+\"\n \"hdqSn98b9BFZKUyk54qRRj5cCAoyN2URaLS0PngpHOm6YmmSj7e0h3whfkZZTBm60f5OC\"\n \"tnUj/Gfd3V85QLIwNlLxcXlPszIE4JuSSof1Z8lgxuAhpJUXwp0k068gLPc5wq7KAAAJP\"\n \"0lEQVQQQhVF0b4BXrSPBFdDIXApIeCTFmNfPo3r9MZfT7z3gaYm69ZCwdYKBY+0LeKBoC\"\n \"8MG5ea2dxsaVOT+l2d3d/4Ji1RQVZV2dEq0rD8HMLv7flghydxDiGUXrT1gobliZIy5O7\"\n \"FD1MvVfhSuibUXCVHgEjLD3fo6h7+UGY6cmuxaJxpaaGQb3jLeKwWKykhVU2Grq8CanKZ\"\n \"4bAJsmJ/3dk9fDcpOalU+V7BCyWpiLD8PoRG0e7EDttlzSHEPUO3wec6M3fwiScvnL76X\"\n \"yFw6SAgwh3SOpFHd/djD5nWqqumJqOPGEZIa242LMPgvURJA6K/cm/scMTzAFXW1mqFig\"\n \"XjdC4ff3dn98inhZGdPIRl7/MVP05FhHV6jWiC6OqzG8IRPewg8xl7DlqlfIWwZb7BU3I\"\n \"KeSyXdUsU7Rsrcw9qc4XARYYAFfujKT37bL+1ZMlnD3d2P/anurf8bVOZ2IjjmPmWFstE\"\n \"pQeTl9RiHpZ1rAiiIQIju5OD1/yZvwaxlUjKgeNNb242LROd1U9PGt8MRTqu7ep65NueJ\"\n \"zyVQZAVjsnXrvRc1ljXL5KHmaNfbsU0LZ+jgHvEZso0YGNDUoLh2OxEyAhxg7uocDgmk5\"\n \"RKFoVA3REQ5DFeJDIZGhrxlnR+4ecQ4ucnTvzVhqnMSURY5n4fq8SrEgkdbVk93p6L7NT\"\n \"UrwHaEjdgGwiYQOl4VEEBmyFoPJt1Tk9Oed/X9cR9XV3/9mOaVMnATmQX2KjI6J7UkiCo\"\n \"Meh39tVeZZwX2ATm3VHJfuUU3SMnt90woWlfKbuUxbz7Vh8oBC4CBHxvHRELrmcs6e6hl\"\n \"cjfed7wP7z88k9el53OvtnVnGs0d/pKpptxGOxb0Xs0ojE9j1XlJJStSbx+HtmBT7le/K\"\n \"fLux85RLCA1KC8pKB8VWdgnwvispdxtB711bvvb3/Pj6JN+uaZabIUScdcdjxhmrlp87F\"\n \"3bfyP93O5CYEA1tFzAaneUwgsdgTOLd9eGXZAicunTg015fPjoUiku9jWdtO0aN56btZ0\"\n \"jSGh2SBb2bl3g31VgYaVBsmlvf/85btbocSsPZtDWK6JLth5vGJvpLnSKtXQ47vpwyEAq\"\n \"dUQyFcIoN5QCCwyBM5pXHTtpPTx8X16f/84NC+yXXEPIq9hJab1Vf5EJCe2+z1sk8ZlVz\"\n \"uyogOWTVh+H8L26Iq2rHOsg3IIz2ovYiYyPMLg7hqFnK41Wav+lwTaJINUSgaFwCJAQKy\"\n \"guJbFAzzp+hZOwyE8+2OQ6wQlksN24/4HNX0um7D8HEJPN3ot3YwWi3nMBlOUaeBugHQA\"\n \"w7PDe9f2vf0pEm1LUtjdZBJTyaIQWAwICALjlzhIyh9p/0Vdn8smLF+6kBHpMJCWVCjmX\"\n \"BCWXB5Cz3PDkZBhzyaeWMY2T4+OauZmlq7ZutrHRD0rBBQCtUWgbKLxi25Z+tKjszOOMB\"\n \"QJfbG2ki5077TWZq41kzGdFmP1o/S148nUeXeGhe5IbacQUAjIhkDZhDVSmkHRMV6CIjO\"\n \"rIw4DS1xpCAFNGp14IqRZRvvwtWs+vo2EG2DVpwTI9sMpeRQClyICZROWXw99zdobjxl6\"\n \"9CiVlgFlSUJY0K5018plQsXWWN8/0w/qacM0R0nkuxRPMTVnhUBwCJRNWBTan0ZgWBe7O\"\n \"su08C+sEMxgiG8ITqQq9gTtKtEc1cJm5+euXfGRbWnYri6MFali7+qrCgGFQIMRKJuwSN\"\n \"5NWpp/L2QmvuM6VKeZ0vYaO6DnFcMxZs1mYjs39d0yRNIMJkXeVGMlU0dXCCgEgkKgIsL\"\n \"aMqTxBMr2xBu/U8yF9oTCjW7YyBxmuFYxH3JaQ2s+wNh1M6OjaWhXirCCOlHUfhQCMiBQ\"\n \"sWZEtXUoBH909x0fNaMn78lM5m2EY1UcJlEpGLCpI6zCYbF4DHnlnR/a1HfP10aRG7UZ9\"\n \"X8q3af6nkJAISAnAhVpWDSVLVsoEFPTkn3/+sVsRn8uGufm9/o2bGTQrEBWTYkoc3Mtny\"\n \"KyIvvaZq2yAvc0HzUUAgoBeRGoWMOiKfla1tiuO6/PORM/9pitoTIWhfPzbh01nTZKRWi\"\n \"6ZzYlIloh2/T3N13xwKdg+kd8BR6oJoYaCgGFwEWHQMUaFiFBS8I0lobJ9ff+xNBid8Tj\"\n \"1JADVQuh9tQKKTAskVEhFNFRK8zSijPNd3KywpvpoTS4SpFVrbBX+1UINBqBqjWh0YcPU\"\n \"PtU/UOpbc/ccls/iyW0ZCFvgwihbgVY1E/kM2koIaaxRLNl5mbZcUtvTd24/v6vp9Oank\"\n \"ymWTqtjOyNPqHU8RUCtUSgqiWhLxh1zPE1mx/sfP/HzXDuM47jaMWCR00bqaVQxcchjQo\"\n \"kBY3NY+EoGgYxQ4NncqQ10v2RN/V8emLY0wyUSKQSGGoZ6P8g6lkhcJEiUNWS0MeEyAra\"\n \"DS9u/84Nj97tOi0pp2ieTLRQ00ZSwBiKqJI3b8GkQuTjYGubbFKRGModJiKGnY/+ys23D\"\n \"rxzw2MDRFb3oS71AK/Vs+D9+iKrZ4WAQmARIlCx5jPXXKluzsiIpg8MaM5PD6WXZ7N7P6\"\n \"1ps7dE4kzLzdqabXsuLEzQhnQiJAo3ReofbOTEarxzNNETvfTMUNjQQmgTVMjhAzv8TNh\"\n \"c8pV39L7vUcRYkSeSDXspHTmCNbOVzTU/9Z5CQCHQWAQCJSx/KufHQY3uuuPtBW/6LzXX\"\n \"eVck7rUyHZ5EuBJtFP5zqag9vmSAvwx029ARfwpzmDabBUm57KCuh34YMVq/dX3vPU9Ci\"\n \"+PkxPdNYQtqCejDrZ4VApcMAjUhLEKPurxS40TftrT18JcvyxReekfePvF218326iy8xP\"\n \"PsDjQNhL4VOm17uRNho3lCN5qeDZmRZxLRjVuvXDYw7f8SFEJBsV/KsO4jop4VAgqBwBE\"\n \"goknzLhq/uWvP2xPePvlA+86XvgjiOhz9zU/Ff8PDmjHqpU0y6s/1uXpPIaAQUAjUBAHS\"\n \"uIi8YLOa19BP4Qnk9RPbpefdriYCqp0qBBQC0iPQEM2FDO3kMBzShvjxB7VBmLJggVd2K\"\n \"elPGCWgQkAhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQk\"\n \"AhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQkAhoBCoOQL/D8pAub21P/M1AAAAAEl\"\n \"FTkSuQmCC\")\n" }, { "path": "minemeld/flask/config.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\n\nimport gevent\n\nimport yaml\nimport filelock\nimport passlib.apache\n\nfrom . import utils\nfrom .logger import LOG\n\n\nCONFIG = {}\nAPI_CONFIG_PATH = None\nAPI_CONFIG_LOCK = None\n\nCONFIG_FILES_RE = '^(?:(?:[0-9]+.*\\.yml)|(?:.*\\.htpasswd))$'\n\n# if you change things here change also backup/import API\n_AUTH_DBS = {\n 'USERS_DB': 'wsgi.htpasswd',\n 'FEEDS_USERS_DB': 'feeds.htpasswd'\n}\n\n\ndef get(key, default=None):\n try:\n result = CONFIG[key]\n except KeyError:\n pass\n else:\n return result\n\n try:\n result = os.environ[key]\n except KeyError:\n pass\n else:\n if result == 'False':\n result = False\n if result == 'True':\n result = True\n\n return result\n\n return default\n\n\ndef store(file, value):\n with API_CONFIG_LOCK.acquire():\n with open(os.path.join(API_CONFIG_PATH, file), 'w+') as f:\n yaml.safe_dump(value, stream=f)\n\n\ndef lock():\n return API_CONFIG_LOCK.acquire()\n\n\nclass APIConfigDict(object):\n def __init__(self, attribute, level=50):\n self.attribute = attribute\n self.filename = '%d-%s.yml' % (level, attribute.lower().replace('_', '-'))\n\n def set(self, key, value):\n curvalues = get(self.attribute, {})\n curvalues[key] = value\n store(self.filename, {self.attribute: curvalues})\n\n def delete(self, key):\n curvalues = get(self.attribute, {})\n curvalues.pop(key, None)\n store(self.filename, {self.attribute: curvalues})\n\n def value(self):\n return get(self.attribute, {})\n\n\ndef _load_config(config_path):\n global CONFIG\n\n new_config = {}\n\n # comptaibilty early releases where all the config\n # was store in a single file\n old_config_file = os.path.join(config_path, 'wsgi.yml')\n if os.path.exists(old_config_file):\n try:\n with open(old_config_file, 'r') as f:\n add_config = yaml.safe_load(f)\n\n if add_config is not None:\n new_config.update(add_config)\n except OSError:\n pass\n\n with API_CONFIG_LOCK.acquire():\n api_config_path = os.path.join(config_path, 'api')\n if os.path.exists(api_config_path):\n config_files = sorted(os.listdir(api_config_path))\n\n for cf in config_files:\n if not cf.endswith('.yml'):\n continue\n\n try:\n with open(os.path.join(api_config_path, cf), 'r') as f:\n add_config = yaml.safe_load(f)\n\n if add_config is not None:\n new_config.update(add_config)\n\n except (OSError, IOError, ValueError):\n LOG.exception('Error loading config file %s' % cf)\n\n CONFIG = new_config\n LOG.info('Config loaded: %r', new_config)\n\n\ndef _load_auth_dbs(config_path):\n with API_CONFIG_LOCK.acquire():\n api_config_path = os.path.join(config_path, 'api')\n for env, default in _AUTH_DBS.iteritems():\n dbname = get(env, default)\n new_db = False\n\n dbpath = os.path.join(\n api_config_path,\n dbname\n )\n\n # for compatibility with old releases\n if not os.path.exists(dbpath):\n old_dbpath = os.path.join(\n config_path,\n dbname\n )\n if os.path.exists(old_dbpath):\n dbpath = old_dbpath\n else:\n new_db = True\n\n CONFIG[env] = passlib.apache.HtpasswdFile(\n path=dbpath,\n new=new_db\n )\n\n LOG.info('%s loaded from %s', env, dbpath)\n\n\ndef _config_monitor(config_path):\n api_config_path = os.path.join(config_path, 'api')\n dirsnapshot = utils.DirSnapshot(api_config_path, CONFIG_FILES_RE)\n while True:\n try:\n with API_CONFIG_LOCK.acquire(timeout=600):\n new_snapshot = utils.DirSnapshot(api_config_path, CONFIG_FILES_RE)\n\n if new_snapshot != dirsnapshot:\n try:\n _load_config(config_path)\n _load_auth_dbs(config_path)\n\n except gevent.GreenletExit:\n break\n\n except:\n LOG.exception('Error loading config')\n\n dirsnapshot = new_snapshot\n\n except filelock.Timeout:\n LOG.error('Timeout locking config in config monitor')\n\n gevent.sleep(1)\n\n\n# initialization\ndef init():\n global API_CONFIG_PATH\n global API_CONFIG_LOCK\n\n config_path = os.environ.get('MM_CONFIG', None)\n if config_path is None:\n LOG.critical('MM_CONFIG environment variable not set')\n raise RuntimeError('MM_CONFIG environment variable not set')\n\n if not os.path.isdir(config_path):\n config_path = os.path.dirname(config_path)\n\n # init global vars\n API_CONFIG_PATH = os.path.join(config_path, 'api')\n API_CONFIG_LOCK = filelock.FileLock(\n os.environ.get('API_CONFIG_LOCK', '/var/run/minemeld/api-config.lock')\n )\n\n _load_config(config_path)\n _load_auth_dbs(config_path)\n if config_path is not None:\n gevent.spawn(_config_monitor, config_path)\n" }, { "path": "minemeld/flask/configapi.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport yaml\nimport uuid\nimport time\nimport json\nimport copy\n\nimport minemeld.run.config\n\nfrom flask import request, jsonify\n\nfrom .redisclient import SR\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\nfrom . import utils\n\n\n__all__ = ['BLUEPRINT']\n\n\nFEED_INTERVAL = 100\n\n# these should be in sync with restore.py\nREDIS_KEY_PREFIX = 'mm:config:'\nREDIS_KEY_CONFIG = REDIS_KEY_PREFIX+'candidate'\n\nREDIS_NODES_LIST = 'nodes'\nLOCK_TIMEOUT = 3000\n\n\nBLUEPRINT = MMBlueprint('config', __name__, url_prefix='/config')\n\n\nclass VersionMismatchError(Exception):\n pass\n\n\nclass MMConfigVersion(object):\n def __init__(self, version=None):\n if version is None:\n self.config = str(uuid.uuid4())\n self.counter = 0\n return\n\n LOG.error('version: %s', version)\n\n self.config, self.counter = version.split('+', 1)\n self.counter = int(self.counter)\n\n def __str__(self):\n return '%s+%d' % (self.config, self.counter)\n\n def __repr__(self):\n return 'MMConfigVersion(%s+%d)' % (self.config, self.counter)\n\n def __eq__(self, other):\n return self.config == other.config and self.counter == other.counter\n\n def __ne__(self, other):\n return not self.__eq__(other)\n\n def __iadd__(self, y):\n self.counter += y\n return self\n\n\ndef _lock(resource):\n resname = resource+':lock'\n value = str(uuid.uuid4())\n result = SR.set(resname, value,\n nx=True, px=LOCK_TIMEOUT)\n\n if result is None:\n return None\n\n return value\n\n\ndef _lock_timeout(resource, timeout=30):\n t1 = time.time()\n tt = t1+timeout\n\n while t1 < tt:\n result = _lock(resource)\n if result is not None:\n return result\n\n t1 = time.sleep(0.01)\n\n return None\n\n\ndef _unlock(resource, value):\n resname = resource+':lock'\n result = SR.get(resname)\n\n if result == value:\n SR.delete(resname)\n return True\n\n LOG.error('lost lock %s - %s', value, result)\n\n return False\n\n\ndef _redlock(f):\n def _redlocked(*args, **kwargs):\n lock = kwargs.pop('lock', False)\n timeout = kwargs.pop('timeout', 30)\n\n if lock:\n clock = _lock_timeout(REDIS_KEY_CONFIG, timeout=timeout)\n if clock is None:\n raise ValueError('Unable to lock config')\n LOG.info('lock set %s', clock)\n\n result = f(*args, **kwargs)\n\n if lock:\n _unlock(REDIS_KEY_CONFIG, clock)\n LOG.info('lock cleared %s', clock)\n\n return result\n\n return _redlocked\n\n\ndef _set_stanza(stanza, value, version, config_key=REDIS_KEY_CONFIG):\n version_key = stanza+':version'\n cversion = SR.hget(config_key, version_key)\n if cversion is not None:\n if version != MMConfigVersion(version=cversion):\n raise VersionMismatchError('version mismatch, current version %s' %\n cversion)\n version += 1\n\n SR.hset(config_key, version_key, str(version))\n SR.hset(config_key, stanza, json.dumps(value))\n\n return version\n\n\n@_redlock\ndef _get_stanza(stanza, config_key=REDIS_KEY_CONFIG):\n version_key = stanza+':version'\n\n version = SR.hget(config_key, version_key)\n if version is None:\n return None\n\n value = SR.hget(config_key, stanza)\n if value is None:\n return None\n\n value = json.loads(value)\n value['version'] = version\n\n return value\n\n\ndef _load_running_config():\n return _load_config_from_file(utils.running_config_path())\n\n\ndef _load_committed_config():\n return _load_config_from_file(utils.committed_config_path())\n\n\ndef _load_config_from_file(rcpath):\n with open(rcpath, 'r') as f:\n rcconfig = yaml.safe_load(f)\n\n if rcconfig is None:\n rcconfig = {}\n\n version = MMConfigVersion()\n tempconfigkey = REDIS_KEY_PREFIX+str(version)\n\n SR.hset(tempconfigkey, 'version', version.config)\n SR.hset(tempconfigkey, 'changed', 0)\n\n if 'fabric' in rcconfig:\n _set_stanza(\n 'fabric',\n {'name': 'fabric', 'properties': rcconfig['fabric']},\n config_key=tempconfigkey,\n version=version\n )\n\n if 'mgmtbus' in rcconfig:\n _set_stanza(\n 'mgmtbus',\n {'name': 'mgmtbus', 'properties': rcconfig['mgmtbus']},\n config_key=tempconfigkey,\n version=version\n )\n\n nodes = rcconfig.get('nodes', {})\n for idx, (nodename, nodevalue) in enumerate(nodes.iteritems()):\n _set_stanza(\n 'node%d' % idx,\n {'name': nodename, 'properties': nodevalue},\n config_key=tempconfigkey,\n version=version\n )\n\n SR.hset(tempconfigkey, 'next_node_id', len(nodes))\n\n clock = _lock_timeout(REDIS_KEY_CONFIG)\n if clock is None:\n SR.delete(tempconfigkey)\n raise ValueError('Unable to lock config')\n\n SR.delete(REDIS_KEY_CONFIG)\n SR.rename(tempconfigkey, REDIS_KEY_CONFIG)\n\n _unlock(REDIS_KEY_CONFIG, clock)\n\n return version.config\n\n\ndef _commit_config(version):\n ccpath = utils.committed_config_path()\n\n clock = _lock_timeout(REDIS_KEY_CONFIG)\n if clock is None:\n raise ValueError('Unable to lock config')\n\n config_info = _config_info()\n\n if version != config_info['version']:\n raise VersionMismatchError('Versions mismatch')\n\n newconfig = {}\n\n fabric = _get_stanza('fabric')\n if fabric is not None:\n newconfig['fabric'] = json.loads(fabric)['properties']\n\n mgmtbus = _get_stanza('mgmtbus')\n if mgmtbus is not None:\n newconfig['mgmtbus'] = json.loads(mgmtbus)['properties']\n\n newconfig['nodes'] = {}\n for n in range(config_info['next_node_id']):\n node = _get_stanza('node%d' % n)\n if node is None:\n continue\n\n if node['name'] in newconfig:\n raise ValueError('Error in config: duplicate node name - %s' %\n node['name'])\n if 'properties' not in node:\n raise ValueError('Error in config: no properties for node %s' %\n node['name'])\n newconfig['nodes'][node['name']] = node['properties']\n\n _unlock(REDIS_KEY_CONFIG, clock)\n\n # we build a copy of the config for validation\n # original config is not used because it could be modified\n # during validation\n temp_config = minemeld.run.config.MineMeldConfig.from_dict(copy.deepcopy(newconfig))\n valid = minemeld.run.config.resolve_prototypes(temp_config)\n if not valid:\n raise ValueError('Error resolving prototypes')\n messages = minemeld.run.config.validate_config(temp_config)\n if len(messages) != 0:\n return messages\n\n with open(ccpath, 'w') as f:\n yaml.safe_dump(\n newconfig,\n f,\n encoding='utf-8',\n default_flow_style=False\n )\n\n SR.hset(REDIS_KEY_CONFIG, 'changed', 0)\n\n return 'OK'\n\n\n@_redlock\ndef _config_full():\n cinfo = _config_info(lock=False)\n\n cinfo['nodes'] = []\n nnid = cinfo['next_node_id']\n for n in range(nnid):\n nc = _get_stanza('node%d' % n, lock=False)\n cinfo['nodes'].append(nc)\n\n return cinfo\n\n\n@_redlock\ndef _config_info():\n version = SR.hget(REDIS_KEY_CONFIG, 'version')\n if version is None:\n raise ValueError('candidate config not initialized')\n\n fabric = SR.hget(REDIS_KEY_CONFIG, 'fabric') is not None\n mgmtbus = SR.hget(REDIS_KEY_CONFIG, 'mgmtbus') is not None\n changed = SR.hget(REDIS_KEY_CONFIG, 'changed') == \"1\"\n next_node_id = int(SR.hget(REDIS_KEY_CONFIG, 'next_node_id'))\n\n return {\n 'fabric': fabric,\n 'mgmtbus': mgmtbus,\n 'version': version,\n 'next_node_id': next_node_id,\n 'changed': changed\n }\n\n\n@_redlock\ndef _create_node(nodebody):\n info = _config_info()\n\n version = nodebody.pop('version', None)\n if version != info['version']:\n raise ValueError('version mismatch')\n\n cversion = MMConfigVersion(version=info['version']+'+0')\n\n _set_stanza(\n 'node%d' % info['next_node_id'],\n nodebody,\n cversion\n )\n\n SR.hset(REDIS_KEY_CONFIG, 'changed', 1)\n SR.hset(REDIS_KEY_CONFIG, 'next_node_id', info['next_node_id']+1)\n\n return {\n 'version': str(cversion),\n 'id': info['next_node_id']\n }\n\n\n@_redlock\ndef _delete_node(nodenum, version):\n node = _get_stanza('node%d' % nodenum)\n if node is None:\n raise ValueError('node %d does not exist' % nodenum)\n\n if MMConfigVersion(version=version) != MMConfigVersion(node['version']):\n raise VersionMismatchError('version mismatch')\n\n SR.hdel(REDIS_KEY_CONFIG, 'node%d' % nodenum)\n SR.hdel(REDIS_KEY_CONFIG, 'node%d:version' % nodenum)\n\n SR.hset(REDIS_KEY_CONFIG, 'changed', 1)\n\n return 'OK'\n\n\n@_redlock\ndef _set_node(nodenum, nodebody):\n if 'version' not in nodebody:\n raise ValueError('version is required')\n version = MMConfigVersion(version=nodebody.pop('version'))\n\n result = _set_stanza(\n 'node%d' % nodenum,\n nodebody,\n version,\n )\n\n SR.hset(REDIS_KEY_CONFIG, 'changed', 1)\n\n return str(result)\n\n\n@BLUEPRINT.route('/running', methods=['GET'], read_write=False)\ndef get_running_config():\n return jsonify(result=utils.running_config())\n\n\n@BLUEPRINT.route('/committed', methods=['GET'], read_write=False)\ndef get_committed_config():\n return jsonify(result=utils.committed_config())\n\n\n# API for manipulating candidate config\n@BLUEPRINT.route('/reload', methods=['GET'], read_write=False)\ndef reload_running_config():\n cname = request.args.get('c', 'running')\n\n try:\n if cname == 'running':\n version = _load_running_config()\n elif cname == 'committed':\n version = _load_committed_config()\n else:\n return jsonify(error={'message': 'Unknown config'}), 400\n\n except Exception as e:\n LOG.exception('Error in loading config')\n return jsonify(error={'message': str(e)}), 500\n\n return jsonify(result=str(version))\n\n\n@BLUEPRINT.route('/commit', methods=['POST'], read_write=True)\ndef commit():\n try:\n body = request.get_json()\n except Exception as e:\n return jsonify(error={'message': str(e)}), 400\n\n version = body.get('version', None)\n if version is None:\n return jsonify(error={'message': 'version required'}), 400\n\n try:\n result = _commit_config(version)\n except VersionMismatchError:\n return jsonify(error={'message': 'version mismatch'}), 409\n except Exception as e:\n LOG.exception('exception in commit')\n return jsonify(error={'message': str(e)}), 400\n\n if result != 'OK':\n return jsonify(error={'message': result}), 402\n\n return jsonify(result='OK')\n\n\n@BLUEPRINT.route('/info', methods=['GET'], read_write=False)\ndef get_config_info():\n try:\n result = _config_info(lock=True)\n except Exception as e:\n return jsonify(error={'message': str(e)}), 500\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/full', methods=['GET'], read_write=False)\ndef get_config_full():\n try:\n result = _config_full(lock=True)\n\n except Exception as e:\n return jsonify(error={'message': str(e)}), 500\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/fabric', methods=['GET'], read_write=False)\ndef get_fabric():\n try:\n result = _get_stanza('fabric', lock=True)\n except Exception as e:\n return jsonify(error={'message': str(e)}), 500\n\n if result is None:\n return jsonify(error={'message': 'Not Found'}), 404\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/mgmtbus', methods=['GET'], read_write=False)\ndef get_mgmtbus():\n try:\n result = _get_stanza('mgmtbus', lock=True)\n except Exception as e:\n return jsonify(error={'message': str(e)}), 500\n\n if result is None:\n return jsonify(error={'message': 'Not Found'}), 404\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/node', methods=['POST'], read_write=False)\ndef create_node():\n try:\n body = request.get_json()\n except Exception as e:\n return jsonify(error={'message': str(e)}), 400\n\n try:\n result = _create_node(body, lock=True)\n except VersionMismatchError:\n return jsonify(error={'message': 'version mismatch'}), 409\n except Exception as e:\n return jsonify(error={'message': str(e)}), 500\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/node/', methods=['GET'], read_write=False)\ndef get_node(nodenum):\n try:\n nodenum = int(nodenum)\n except ValueError:\n return jsonify(error='invalid node number'), 400\n\n try:\n result = _get_stanza('node%d' % nodenum, lock=True)\n except Exception as e:\n LOG.exception('error in get_node')\n return jsonify(error={'message': str(e)}), 500\n\n if result is None:\n return jsonify(error={'message': 'Not Found'}), 404\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/node/', methods=['PUT'], read_write=False)\ndef set_node(nodenum):\n try:\n nodenum = int(nodenum)\n except ValueError:\n return jsonify(error='invalid node number'), 400\n\n try:\n body = request.get_json()\n except Exception as e:\n return jsonify(error={'message': str(e)}), 400\n\n try:\n result = _set_node(nodenum, body, lock=True)\n except VersionMismatchError:\n return jsonify(error={'message': 'version mismatch'}), 409\n except Exception as e:\n LOG.exception('exception is _set_node')\n return jsonify(error={'message': str(e)}), 500\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/node/', methods=['DELETE'], read_write=False)\ndef delete_node(nodenum):\n try:\n nodenum = int(nodenum)\n except ValueError:\n return jsonify(error='invalid node number'), 400\n\n version = request.args.get('version', None)\n if version is None:\n return jsonify(error={'message': 'version required'}), 400\n\n try:\n result = _delete_node(nodenum, version, lock=True)\n except VersionMismatchError:\n return jsonify(error={'message': 'version mismatch'}), 409\n except Exception as e:\n return jsonify(error={'message': str(e)}), 500\n\n return jsonify(result=result)\n\n\n@_redlock\ndef _init_config():\n try:\n _config_info(lock=False)\n\n except ValueError:\n LOG.info('Loading running config in memory')\n\n try:\n _load_running_config()\n\n except OSError:\n LOG.exception('Error loading running config')\n\n\ndef init_app(app):\n app.before_first_request(_init_config)\n" }, { "path": "minemeld/flask/configdataapi.py", "content": "# Copyright 2015-present Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os.path\nimport os\nimport shutil\nimport sqlite3\nimport time\nfrom tempfile import NamedTemporaryFile\n\nimport yaml\nimport filelock\nimport ujson as json\nfrom gevent import sleep\n\nfrom flask import request, jsonify\n\nfrom .mmrpc import MMRpcClient\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\n\n\n__all__ = ['BLUEPRINT']\n\n\nLOCK_TIMEOUT = 3000\n\n\nBLUEPRINT = MMBlueprint('configdata', __name__, url_prefix='/config/data')\n\n\ndef _safe_remove(path, g=None):\n try:\n os.remove(path)\n except:\n LOG.exception('Exception removing {}'.format(path))\n\n\nclass _CDataYaml(object):\n def __init__(self, cpath, datafilename):\n self.cpath = cpath\n self.datafilename = datafilename\n\n def read(self):\n fdfname = self.datafilename+'.yml'\n\n lockfname = os.path.join(self.cpath, fdfname+'.lock')\n lock = filelock.FileLock(lockfname)\n\n os.listdir(self.cpath)\n if fdfname not in os.listdir(self.cpath):\n return jsonify(error={\n 'message': 'Unknown config data file'\n }), 400\n\n try:\n with lock.acquire(timeout=10):\n with open(os.path.join(self.cpath, fdfname), 'r') as f:\n result = yaml.safe_load(f)\n\n except Exception as e:\n return jsonify(error={\n 'message': 'Error loading config data file: %s' % str(e)\n }), 500\n\n return jsonify(result=result)\n\n def create(self):\n tdir = os.path.dirname(os.path.join(self.cpath, self.datafilename))\n\n if not os.path.samefile(self.cpath, tdir):\n return jsonify(error={'msg': 'Wrong config data filename'}), 400\n\n fdfname = os.path.join(self.cpath, self.datafilename+'.yml')\n\n lockfname = fdfname+'.lock'\n lock = filelock.FileLock(lockfname)\n\n try:\n body = request.get_json()\n except Exception as e:\n return jsonify(error={'message': str(e)}), 400\n\n try:\n with lock.acquire(timeout=10):\n with open(fdfname, 'w') as f:\n yaml.safe_dump(body, stream=f)\n except Exception as e:\n return jsonify(error={\n 'message': str(e)\n }), 500\n\n def append(self):\n tdir = os.path.dirname(os.path.join(self.cpath, self.datafilename))\n\n if not os.path.samefile(self.cpath, tdir):\n return jsonify(error={'msg': 'Wrong config data filename'}), 400\n\n cdfname = os.path.join(self.cpath, self.datafilename+'.yml')\n\n lockfname = cdfname+'.lock'\n lock = filelock.FileLock(lockfname)\n\n try:\n with lock.acquire(timeout=10):\n if not os.path.isfile(cdfname):\n config_data_file = []\n else:\n with open(cdfname, 'r') as f:\n config_data_file = yaml.safe_load(f)\n\n if type(config_data_file) != list:\n raise RuntimeError('Config data file is not a list')\n\n body = request.get_json()\n if body is None:\n return jsonify(error={\n 'message': 'No record in request'\n }), 400\n\n config_data_file.append(body)\n\n with open(cdfname, 'w') as f:\n yaml.safe_dump(config_data_file, stream=f)\n\n except Exception as e:\n return jsonify(error={\n 'message': 'Error appending to config data file: %s' % str(e)\n }), 500\n\n\nclass _CDataLocalDB(object):\n def __init__(self, cpath, datafilename):\n self.cpath = cpath\n self.datafilename = datafilename\n self.full_path = os.path.join(self.cpath, self.datafilename)\n\n def read(self):\n tdir = os.path.dirname(self.full_path)\n if not os.path.samefile(self.cpath, tdir):\n return jsonify(error={'msg': 'Wrong config data filename'}), 400\n\n result = []\n\n if not os.path.isfile(self.full_path+'.db'):\n return jsonify(result=[])\n\n try:\n conn = sqlite3.connect(self.full_path+'.db')\n\n for row in conn.execute('select * from indicators'):\n indicator = json.loads(row[2])\n indicator['indicator'] = row[0]\n indicator['type'] = row[1]\n indicator['_expiration_ts'] = row[3]\n indicator['_update_ts'] = row[4]\n result.append(indicator)\n\n sleep(0)\n\n finally:\n conn.close()\n\n return jsonify(result=result)\n\n def create(self):\n return jsonify(error=dict(message='Method not allowed on localdb files')), 400\n\n def _parse_text_data(self, data):\n result = []\n state = 'INIT'\n indicator = {}\n attribute = None\n\n for line in iter(data.splitlines()):\n if len(line) > 0 and line[0] == '#':\n continue\n\n if state == 'INIT':\n line = line.strip()\n if len(line) == 0:\n continue\n\n indicator['type'] = line\n state = 'TYPE'\n continue\n\n if state == 'TYPE':\n line = line.strip()\n if len(line) == 0:\n continue\n\n indicator['indicator'] = line\n state = 'INDICATOR'\n continue\n\n if state == 'INDICATOR':\n line = line.strip()\n if len(line) == 0:\n result.append(indicator)\n indicator = {}\n state = 'INIT'\n continue\n\n attribute = line\n state = 'ATTRIBUTE'\n continue\n\n if state == 'ATTRIBUTE':\n line = line.strip()\n\n indicator[attribute] = line\n if attribute == 'confidence':\n if not line.isdigit():\n LOG.error('Invalid confidence value: {!r}'.format(line))\n return None\n indicator[attribute] = int(line)\n\n elif attribute == 'ttl':\n if line.isdigit():\n indicator[attribute] = int(line)\n else:\n indicator[attribute] = 'disabled'\n\n state = 'INDICATOR'\n continue\n\n if state == 'INDICATOR':\n result.append(indicator)\n state = 'INIT'\n\n if state != 'INIT':\n LOG.error('Error parsing indicators, state: {}'.format(state))\n return None\n\n if len(result) == 0:\n return None\n\n return result\n\n def append(self):\n tdir = os.path.dirname(self.full_path)\n if not os.path.samefile(self.cpath, tdir):\n return jsonify(error={'msg': 'Wrong config data filename'}), 400\n\n record = request.get_json()\n if record is None:\n record = self._parse_text_data(request.data)\n\n if record is None:\n return jsonify(error={\n 'message': 'No valid record in request'\n }), 400\n\n indicators = [record]\n if isinstance(record, list):\n indicators = record\n\n now = int(time.time()*1000)\n updates = []\n for en, entry in enumerate(indicators):\n indicator = entry.pop('indicator', None)\n if indicator is None:\n return jsonify(error={\n 'message': 'entry {}: indicator field is missing'.format(en)\n }), 400\n type_ = entry.pop('type', None)\n if type_ is None:\n return jsonify(error={\n 'message': 'entry {}: type field is missing'.format(en)\n }), 400\n\n expiration_ts = entry.pop('ttl', None)\n if expiration_ts is not None:\n if isinstance(expiration_ts, int):\n expiration_ts = (expiration_ts*1000+now)\n else:\n expiration_ts = 'disabled'\n\n updates.append((\n indicator, type_, json.dumps(entry), expiration_ts, now, json.dumps([indicator, type_, entry])\n ))\n\n try:\n conn = sqlite3.connect(self.full_path+'.db')\n\n with conn:\n conn.execute('''create table if not exists indicators\n (indicator text, type text, attributes text, expiration_ts integer,\n update_ts integer, content text, primary key(indicator, type));''')\n conn.execute('''create index if not exists updateIndex on indicators(update_ts);''')\n\n conn.executemany('''insert or replace into indicators\n (indicator, type, attributes, expiration_ts, update_ts, content)\n values (?, ?, ?, ?, ?, ?);\n ''', updates)\n\n finally:\n conn.close()\n\n\nclass _CDataUploadOnly(object):\n def __init__(self, extension, cpath, datafilename):\n self.extension = extension\n self.cpath = cpath\n self.datafilename = datafilename\n\n def read(self):\n fdfname = '{}.{}'.format(self.datafilename, self.extension)\n\n os.listdir(self.cpath)\n if fdfname not in os.listdir(self.cpath):\n return jsonify(error={\n 'message': 'Unknown config data file'\n }), 400\n\n return jsonify(result='ok')\n\n def create(self):\n tdir = os.path.dirname(os.path.join(self.cpath, self.datafilename))\n\n if not os.path.samefile(self.cpath, tdir):\n return jsonify(error={'msg': 'Wrong config data filename'}), 400\n\n fdfname = os.path.join(self.cpath, '{}.{}'.format(self.datafilename, self.extension))\n\n if 'file' not in request.files:\n return jsonify(error={'messsage': 'No file'}), 400\n\n file = request.files['file']\n if file.filename == '':\n return jsonify(error={'message': 'No file'}), 400\n\n tf = NamedTemporaryFile(prefix='mm-extension-upload', delete=False)\n try:\n file.save(tf)\n tf.close()\n\n shutil.move(tf.name, fdfname)\n\n finally:\n _safe_remove(tf.name)\n\n\nclass _CDataCertificate(_CDataUploadOnly):\n def __init__(self, cpath, datafilename):\n super(_CDataCertificate, self).__init__(\n extension='crt',\n cpath=cpath,\n datafilename=datafilename\n )\n\n\nclass _CDataPrivateKey(_CDataUploadOnly):\n def __init__(self, cpath, datafilename):\n super(_CDataPrivateKey, self).__init__(\n extension='pem',\n cpath=cpath,\n datafilename=datafilename\n )\n\n\n# API for working with side configs and dynamic data files\n@BLUEPRINT.route('/', methods=['GET'], read_write=False)\ndef get_config_data(datafilename):\n cpath = os.path.dirname(os.environ.get('MM_CONFIG'))\n\n datafiletype = request.values.get('t', 'yaml')\n\n if datafiletype == 'yaml':\n return _CDataYaml(cpath, datafilename).read()\n elif datafiletype == 'cert':\n return _CDataCertificate(cpath, datafilename).read()\n elif datafiletype == 'pkey':\n return _CDataPrivateKey(cpath, datafilename).read()\n elif datafiletype == 'localdb':\n return _CDataLocalDB(cpath, datafilename).read()\n\n return jsonify(error=dict(message='Unknown data file type')), 400\n\n\n@BLUEPRINT.route('/', methods=['PUT'], read_write=True)\ndef save_config_data(datafilename):\n cpath = os.path.dirname(os.environ.get('MM_CONFIG'))\n\n datafiletype = request.values.get('t', 'yaml')\n\n if datafiletype == 'yaml':\n result = _CDataYaml(cpath, datafilename).create()\n elif datafiletype == 'cert':\n result = _CDataCertificate(cpath, datafilename).create()\n elif datafiletype == 'pkey':\n result = _CDataPrivateKey(cpath, datafilename).create()\n elif datafiletype == 'localdb':\n result = _CDataLocalDB(cpath, datafilename).create()\n else:\n return jsonify(error=dict(message='Unknown data file type')), 400\n\n if result is None:\n hup = request.args.get('h', None)\n if hup is not None:\n MMRpcClient.send_cmd(hup, 'hup', {'source': 'minemeld-web'})\n\n return jsonify(result='ok'), 200\n\n return result\n\n\n@BLUEPRINT.route('//append', methods=['POST'], read_write=True)\ndef append_config_data(datafilename):\n cpath = os.path.dirname(os.environ.get('MM_CONFIG'))\n\n datafiletype = request.values.get('t', 'yaml')\n\n if datafiletype == 'yaml':\n result = _CDataYaml(cpath, datafilename).append()\n elif datafiletype == 'localdb':\n result = _CDataLocalDB(cpath, datafilename).append()\n else:\n return jsonify(error=dict(message='Unknown data file type')), 400\n\n if result is None:\n hup = request.args.get('h', None)\n if hup is not None:\n MMRpcClient.send_cmd(hup, 'hup', {'source': 'minemeld-web'})\n\n return jsonify(result='ok'), 200\n\n return result\n" }, { "path": "minemeld/flask/events.py", "content": "import gevent\nimport gevent.queue\nimport redis\nimport werkzeug.local\nimport ujson as json\nfrom blinker import signal\n\nfrom flask import g\n\nfrom .logger import LOG\n\n\nSTATUS_EVENTS_SUBSCRIBER = None\n\n\nclass StatusEventsSubscriber(object):\n \"\"\"Subscribes to mm-status events from engine\n \"\"\"\n\n def __init__(self, connection_pool):\n self.SR = redis.StrictRedis(connection_pool=connection_pool)\n self._g = None\n self._signal = signal('mm-status')\n\n def _retry_wrap(self):\n while True:\n try:\n self._listen()\n\n except gevent.GreenletExit:\n break\n\n except:\n LOG.exception('Exception in event listener')\n\n def _listen(self):\n pubsub = self.SR.pubsub(ignore_subscribe_messages=True)\n pubsub.psubscribe('mm-engine-status.*')\n\n while pubsub.subscribed:\n response = pubsub.get_message(timeout=30.0)\n if response is None:\n continue\n\n if not bool(self._signal.receivers):\n LOG.info('no receivers')\n continue\n\n data = json.loads(response['data'])\n source = data.pop('source', '')\n\n self._signal.send(source, data=data)\n\n def start(self):\n if self._g is not None:\n return\n self._g = gevent.spawn(self._retry_wrap)\n\n\nclass EventsReceiver(object):\n def __init__(self):\n self._signal = signal('mm-status')\n self._signal.connect(self._signal_receiver)\n self._q = gevent.queue.Queue()\n self._iterator = self._generator()\n\n def _signal_receiver(self, sender, data):\n message = {\n 'source': sender\n }\n message.update(data)\n self._q.put(message)\n\n def _generator(self):\n yield 'data: ok\\n\\n'\n\n while True:\n try:\n message = self._q.get(timeout=5.0)\n yield 'data: '+json.dumps(message)+'\\n\\n'\n\n except gevent.queue.Empty:\n yield 'data: ping\\n\\n'\n continue\n\n yield 'data: { \"msg\": \"\" }\\n\\n'\n\n def __iter__(self):\n return self\n\n def next(self):\n result = next(self._iterator)\n\n return result\n\n def close(self):\n self._signal.disconnect(self._signal_receiver)\n\n\ndef get_EventsGenerator():\n result = getattr(g, '_events_generator', None)\n if result is None:\n result = EventsReceiver()\n g._events_generator = result\n\n return result\n\n\nEventsGenerator = werkzeug.local.LocalProxy(get_EventsGenerator)\n\n\ndef teardown(exception):\n eg = getattr(g, '_events_generator', None)\n if eg is not None:\n g._events_generator.close()\n g._events_generator = None\n\n\ndef init_app(app, redis_url):\n \"\"\"Initalize event generator in the app\n \n Args:\n app (object): Flask App\n redis_url (str): Redis URL for communicating with engine\n \"\"\"\n\n global STATUS_EVENTS_SUBSCRIBER\n\n redis_cp = redis.ConnectionPool.from_url(\n redis_url,\n max_connections=1\n )\n STATUS_EVENTS_SUBSCRIBER = StatusEventsSubscriber(connection_pool=redis_cp)\n STATUS_EVENTS_SUBSCRIBER.start()\n\n app.teardown_appcontext(teardown)\n" }, { "path": "minemeld/flask/extensionsapi.py", "content": "# Copyright 2015-2017 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport sys\nimport os\nimport os.path\nimport shutil\nimport functools\nimport subprocess\nimport uuid\nimport stat\nfrom tempfile import NamedTemporaryFile\n\nimport filelock\nfrom gevent import Timeout\nfrom gevent.subprocess import Popen\nfrom flask import jsonify, request\nfrom werkzeug.utils import secure_filename\n\nimport minemeld.extensions\nimport minemeld.loader\n\nfrom . import config\nfrom .jobs import JOBS_MANAGER\nfrom .prototypeapi import reset_prototype_paths\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\n\n\n__all__ = ['BLUEPRINT']\n\n\nBLUEPRINT = MMBlueprint('extensions', __name__, url_prefix='')\n\nDISABLE_NEW_EXTENSIONS = config.get('DISABLE_NEW_EXTENSIONS', False)\n\n\ndef _get_extensions():\n library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None)\n if library_directory is None:\n raise RuntimeError('MINEMELD_LOCAL_LIBRARY_PATH not set')\n\n return minemeld.extensions.extensions(library_directory)\n\n\ndef _build_activate_args(ext_path):\n pip_path = config.get('MINEMELD_PIP_PATH', None)\n if pip_path is None:\n raise RuntimeError('MINEMELD_PIP_PATH not set')\n\n library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None)\n if library_directory is None:\n raise RuntimeError('MINEMELD_LOCAL_LIBRARY_PATH not set')\n\n constraints_file = os.path.join(library_directory, 'constraints.txt')\n\n args = [\n pip_path,\n 'install',\n '-c', constraints_file\n ]\n if ext_path.endswith('.whl'):\n args.append(ext_path)\n else:\n args.append('-e')\n args.append(ext_path)\n\n return args\n\n\ndef _build_deactivate_args(extension_name):\n pip_path = config.get('MINEMELD_PIP_PATH', None)\n if pip_path is None:\n raise RuntimeError('MINEMELD_PIP_PATH not set')\n\n args = [\n pip_path,\n 'uninstall', '-y',\n extension_name\n ]\n\n return args\n\n\ndef _find_running_job(extension, jobs):\n for jobid, job in jobs.iteritems():\n if job['status'] != 'RUNNING':\n continue\n\n if job['name'] == extension.name:\n return jobid\n\n return None\n\n\ndef _load_frozen_paths():\n library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None)\n if library_directory is None:\n LOG.error('freeze not updated - MINEMELD_LOCAL_LIBRARY_PATH not set')\n return\n\n freeze_path = os.path.join(library_directory, 'freeze.txt')\n\n if not os.path.isfile(freeze_path):\n LOG.info('Extensions frigidaire not found, paths not loaded')\n return\n\n freeze_lock = filelock.FileLock('{}.lock'.format(freeze_path))\n with freeze_lock.acquire(timeout=30):\n with open(freeze_path, 'r') as ff:\n minemeld.extensions.load_frozen_paths(ff)\n\n\ndef _update_freeze_file():\n library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None)\n if library_directory is None:\n LOG.error('freeze not updated - MINEMELD_LOCAL_LIBRARY_PATH not set')\n return\n\n freeze_path = os.path.join(library_directory, 'freeze.txt')\n\n freeze_lock = filelock.FileLock('{}.lock'.format(freeze_path))\n with freeze_lock.acquire(timeout=30):\n with open(freeze_path, 'w+') as ff:\n frozen = minemeld.extensions.freeze(library_directory)\n for frozen_ext in frozen:\n ff.write('{}\\n'.format(frozen_ext))\n\n\ndef _extensions_changed(activated_path, deactivated_path, g):\n if g is not None and g.value == 0:\n # process was successful\n if deactivated_path is not None:\n try:\n sys.path.remove(deactivated_path)\n except ValueError:\n LOG.error('extensions_changed: Error removing {}'.format(deactivated_path))\n\n if activated_path is not None:\n if activated_path not in sys.path:\n sys.path.append(activated_path)\n\n minemeld.loader.bump_workingset()\n reset_prototype_paths()\n _update_freeze_file()\n\n\ndef _safe_remove(path, g=None):\n try:\n os.remove(path)\n except:\n LOG.exception('Exception removing {}'.format(path))\n\n\n@BLUEPRINT.route('/extensions', methods=['GET'], read_write=False)\ndef list_extensions():\n extensions = _get_extensions()\n\n jobs = JOBS_MANAGER.get_jobs('extensions')\n\n result = []\n for e in extensions:\n edict = e._asdict()\n rjobid = _find_running_job(e, jobs)\n if rjobid is not None:\n edict['running_job'] = rjobid\n\n result.append(edict)\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/extensions//activate', methods=['POST'], read_write=True)\ndef activate_extension(extension):\n if DISABLE_NEW_EXTENSIONS:\n return 'Disabled', 403\n\n params = request.get_json(silent=True)\n if params is None:\n return jsonify(error={'message': 'no params'}), 400\n\n ext_path = params.get('path', None)\n if ext_path is None:\n return jsonify(error={'message': 'path not specified'}), 400\n\n ext_version = params.get('version', None)\n if ext_version is None:\n return jsonify(error={'message': 'version not specified'}), 400\n\n extensions = _get_extensions()\n\n for e in extensions:\n if e.name != extension:\n continue\n\n if e.version != ext_version:\n continue\n\n if e.path == ext_path:\n break\n else:\n return jsonify(error={'message': 'extension not found'}), 400\n\n if not e.installed:\n return jsonify(error={'message': 'extension not installed'}), 400\n\n if e.activated:\n return jsonify(error={'messsage': 'extension already activated'}), 400\n\n jobs = JOBS_MANAGER.get_jobs('extensions')\n if _find_running_job(e, jobs) is not None:\n return jsonify(error={'message': 'pending job'}), 400\n\n jobid = JOBS_MANAGER.exec_job(\n job_group='extensions',\n description='activate {} v{}'.format(e.name, e.version),\n args=_build_activate_args(e.path),\n data={\n 'name': extension,\n 'version': ext_version,\n 'path': ext_path\n },\n callback=functools.partial(_extensions_changed, e.path, None)\n )\n\n return jsonify(result=jobid)\n\n\n@BLUEPRINT.route('/extensions//deactivate', methods=['GET', 'POST'], read_write=True)\ndef deactivate_extension(extension):\n if DISABLE_NEW_EXTENSIONS:\n return 'Disabled', 403\n\n params = request.get_json(silent=True)\n if params is None:\n return jsonify(error={'message': 'no params'}), 400\n\n ext_path = params.get('path', None)\n if ext_path is None:\n return jsonify(error={'message': 'path not specified'}), 400\n\n ext_version = params.get('version', None)\n if ext_version is None:\n return jsonify(error={'message': 'version not specified'}), 400\n\n extensions = _get_extensions()\n\n for e in extensions:\n if e.name != extension:\n continue\n\n if e.version != ext_version:\n continue\n\n if e.path == ext_path:\n break\n else:\n return jsonify(error={'message': 'extension not found'}), 400\n\n if not e.installed:\n return jsonify(error={'message': 'extension not installed'}), 400\n\n if not e.activated:\n return jsonify(error={'messsage': 'extension not activated'}), 400\n\n jobs = JOBS_MANAGER.get_jobs('extensions')\n if _find_running_job(e, jobs) is not None:\n return jsonify(error={'message': 'pending job'}), 400\n\n jobid = JOBS_MANAGER.exec_job(\n job_group='extensions',\n description='deactivate {} v{}'.format(e.name, e.version),\n args=_build_deactivate_args(extension),\n data={\n 'name': extension,\n 'version': ext_version,\n 'path': ext_path\n },\n callback=functools.partial(_extensions_changed, None, e.path)\n )\n\n return jsonify(result=jobid)\n\n\n@BLUEPRINT.route('/extensions//uninstall', methods=['GET', 'POST'], read_write=True)\ndef uninstall_extension(extension):\n if DISABLE_NEW_EXTENSIONS:\n return 'Disabled', 403\n\n params = request.get_json(silent=True)\n if params is None:\n return jsonify(error={'message': 'no params'}), 400\n\n ext_path = params.get('path', None)\n if ext_path is None:\n return jsonify(error={'message': 'path not specified'}), 400\n\n ext_version = params.get('version', None)\n if ext_version is None:\n return jsonify(error={'message': 'version not specified'}), 400\n\n extensions = _get_extensions()\n\n for e in extensions:\n if e.name != extension:\n continue\n\n if e.version != ext_version:\n continue\n\n if e.path == ext_path:\n break\n else:\n return jsonify(error={'message': 'extension not found'}), 400\n\n if not e.installed:\n return jsonify(error={'message': 'extension not installed'}), 400\n\n if e.activated:\n return jsonify(error={'messsage': 'extension activated'}), 400\n\n jobs = JOBS_MANAGER.get_jobs('extensions')\n if _find_running_job(e, jobs) is not None:\n return jsonify(error={'message': 'pending job'}), 400\n\n if e.path.endswith('.whl'):\n os.remove(e.path)\n\n else:\n shutil.rmtree(e.path)\n\n _extensions_changed(None, None, None)\n\n return jsonify(result='ok')\n\n\n@BLUEPRINT.route('/extensions', methods=['POST'], read_write=True)\ndef upload_extension():\n if DISABLE_NEW_EXTENSIONS:\n return 'Disabled', 403\n\n library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None)\n if library_directory is None:\n raise RuntimeError('MINEMELD_LOCAL_LIBRARY_PATH not set')\n\n if 'file' not in request.files:\n return jsonify(error={'messsage': 'No file'}), 400\n\n file = request.files['file']\n if file.filename == '':\n return jsonify(error={'message': 'No file'}), 400\n\n if not file or not file.filename.endswith('.whl'):\n return jsonify(error={'message': 'Incorrect filename'}), 400\n\n filename = secure_filename(file.filename)\n if filename != file.filename:\n return jsonify(error={'message': 'Incorrect filename'}), 400\n if os.path.basename(filename) != filename:\n return jsonify(error={'message': 'Incorrect filename'}), 400\n\n toks = filename.split('-', 5)\n if len(toks) != 4 and len(toks) != 5:\n return jsonify(error={'message': 'Invalid wheel filename'}), 400\n\n if toks[-1] != 'any.whl' and toks[-1] != 'linux_x86_64.whl':\n return jsonify(error={'message': 'Invalid wheel platform'}), 400\n\n if not toks[-3].startswith('py2') and not toks[-3].startswith('cp2'):\n return jsonify(error={'message': 'Invalid wheel python version'}), 400\n\n tf = NamedTemporaryFile(prefix='mm-extension-upload', delete=False)\n\n try:\n file.save(tf)\n tf.close()\n\n metadata = minemeld.extensions.get_metadata_from_wheel(tf.name, filename)\n if metadata is None:\n return jsonify(error={'message': 'Invalid MineMeld extension'}), 400\n\n full_filename = os.path.join(library_directory, filename)\n shutil.move(tf.name, full_filename)\n\n except (KeyError, ValueError) as e:\n LOG.error('Invalid extension: {}'.format(str(e)))\n return jsonify(error={'message': 'Invalid python wheel'}), 400\n\n finally:\n _safe_remove(tf.name)\n\n return jsonify(result='OK')\n\n\n@BLUEPRINT.route('/extensions/git-refs', methods=['GET'], read_write=False)\ndef get_git_refs():\n if DISABLE_NEW_EXTENSIONS:\n return 'Disabled', 403\n\n git_endpoint = request.values.get('ep', None)\n if git_endpoint is None:\n return jsonify(error={'message': 'Missing endpoint'}), 400\n\n if not git_endpoint.endswith('.git'):\n return jsonify(error={'message': 'Invalid git endpoint'}), 400\n\n git_path = config.get('MINEMELD_GIT_PATH', None)\n if git_path is None:\n return jsonify(error={'message': 'MINEMELD_GIT_PATH not set'}), 500\n\n git_args = [git_path, 'ls-remote', '-t', '-h', git_endpoint]\n\n git_process = Popen(\n args=git_args,\n close_fds=True,\n shell=False,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE\n )\n\n timeout = Timeout(20.0)\n timeout.start()\n try:\n git_stdout, git_stderr = git_process.communicate()\n\n except Timeout:\n git_process.kill()\n return jsonify(error={'message': 'Timeout accessing git repo'}), 400\n\n finally:\n timeout.cancel()\n\n if git_process.returncode != 0:\n LOG.error('Error running {}: {}'.format(git_args, git_stderr))\n return jsonify(error={'message': 'Error running git: {}'.format(git_stderr)}), 400\n\n return jsonify(result=[line.rsplit('/', 1)[-1] for line in git_stdout.splitlines()])\n\n\n@BLUEPRINT.route('/extensions/git-install', methods=['POST'], read_write=True)\ndef install_from_git():\n if DISABLE_NEW_EXTENSIONS:\n return 'Disabled', 403\n\n library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None)\n if library_directory is None:\n raise RuntimeError('MINEMELD_LOCAL_LIBRARY_PATH not set')\n\n params = request.get_json(silent=True)\n if params is None:\n return jsonify(error={'message': 'no params'}), 400\n\n git_endpoint = params.get('ep', None)\n if git_endpoint is None:\n return jsonify(error={'message': 'Missing endpoint'}), 400\n\n if not git_endpoint.endswith('.git'):\n return jsonify(error={'message': 'Invalid git endpoint'}), 400\n\n git_ref = params.get('ref', None)\n if git_ref is None:\n return jsonify(error={'message': 'Missing git ref'}), 400\n\n git_path = config.get('MINEMELD_GIT_PATH', None)\n if git_path is None:\n return jsonify(error={'message': 'MINEMELD_GIT_PATH not set'}), 500\n\n install_directory = os.path.join(\n library_directory,\n str(uuid.uuid4())\n )\n job_args = [\n \"mm-extension-from-git\",\n git_path,\n git_ref,\n git_endpoint,\n install_directory\n ]\n\n jobid = JOBS_MANAGER.exec_job(\n job_group='extensions-git',\n description='install from git {} branch {}'.format(git_endpoint, git_ref),\n args=job_args,\n data={\n 'endpoint': git_endpoint,\n 'ref': git_ref,\n 'path': install_directory\n }\n )\n\n return jsonify(result=jobid)\n\n\ndef init_app(app):\n _load_frozen_paths()\n" }, { "path": "minemeld/flask/feedredis.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport cStringIO\nimport json\nimport re\nfrom collections import defaultdict\nfrom contextlib import contextmanager\n\nimport unicodecsv\nfrom flask import request, jsonify, Response, stream_with_context\nfrom flask.ext.login import current_user\nfrom gevent import sleep\nfrom netaddr import IPRange, IPNetwork, IPSet, iprange_to_cidrs\n\nfrom .aaa import MMBlueprint\nfrom .cbfeed import CbFeedInfo, CbReport\nfrom .logger import LOG\nfrom .mmrpc import MMMaster\nfrom .redisclient import SR\n\n__all__ = ['BLUEPRINT']\n\nFEED_INTERVAL = 100\n_PROTOCOL_RE = re.compile('^(?:[a-z]+:)*//')\n_PORT_RE = re.compile('^([a-z0-9\\-\\.]+)(?:\\:[0-9]+)*')\n_INVALID_TOKEN_RE = re.compile('(?:[^\\./+=\\?&]+\\*[^\\./+=\\?&]*)|(?:[^\\./+=\\?&]*\\*[^\\./+=\\?&]+)')\n_BROAD_PATTERN = re.compile('^(?:\\*\\.)+[a-zA-Z]+(?::[0-9]+)?$')\n_IPV4_MASK_RE = re.compile('^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(\\\\/[0-9]+)?$')\n_IPV4_RANGE_RE = re.compile(\n '^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}-[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$')\n\nBLUEPRINT = MMBlueprint('feeds', __name__, url_prefix='/feeds')\n\n\ndef _translate_ip_ranges(indicator, value=None):\n if value is not None and value['type'] != 'IPv4':\n return [indicator]\n\n try:\n ip_range = IPRange(*indicator.split('-', 1))\n\n except (AddrFormatError, ValueError, TypeError):\n return [indicator]\n\n return [str(x) if x.size != 1 else str(x.network) for x in ip_range.cidrs()]\n\n\ndef _extract_cidrs(indicator):\n try:\n parsed = iprange_to_cidrs(*indicator.split('-', 1)) if '-' in indicator else [IPNetwork(indicator)]\n\n except (AddrFormatError, ValueError, TypeError):\n raise Exception('Invalid IP Address in summarization: {!r}'.format(indicator))\n\n return parsed\n\n\n@contextmanager\ndef _buffer():\n result = cStringIO.StringIO()\n\n try:\n yield result\n finally:\n result.close()\n\n\ndef generate_panosurl_feed(feed, start, num, desc, value, **kwargs):\n zrange = SR.zrange\n if desc:\n zrange = SR.zrevrange\n\n if num is None:\n num = (1 << 32) - 1\n\n cstart = start\n\n while cstart < (start + num):\n ilist = zrange(feed, cstart,\n cstart - 1 + min(start + num - cstart, FEED_INTERVAL))\n\n for i in ilist:\n i = i.lower()\n\n i = _PROTOCOL_RE.sub('', i)\n\n withport = i\n i = _PORT_RE.sub('\\g<1>', i)\n LOG.debug('{} => {}'.format(withport, i))\n if withport != i and 'sp' not in kwargs:\n # port removed, but strip port not enabled\n # ignore entry\n continue\n\n old = i\n i = _INVALID_TOKEN_RE.sub('*', i)\n if old != i:\n # url changed, invalid tokens detected\n if 'di' in kwargs:\n # drop invalid in params, drop entry\n continue\n \n # check if the pattern is now too broad\n hostname = i\n if '/' in hostname:\n hostname, _ = hostname.split('/', 1)\n if _BROAD_PATTERN.match(hostname) is not None:\n continue\n\n if '/' not in i and not 'nsl' in kwargs:\n i = i + '/' # add a slash at the end of the hostname\n\n # for PAN-OS *.domain.com does not match domain.com\n # we should provide both\n # this could generate more than num entries in the egress feed\n if i.startswith('*.'):\n yield i[2:] + '\\n'\n\n yield i + '\\n'\n\n if len(ilist) < 100:\n break\n\n cstart += 100\n\n\ndef generate_plain_feed(feed, start, num, desc, value, **kwargs):\n zrange = SR.zrange\n if desc:\n zrange = SR.zrevrange\n\n if num is None:\n num = (1 << 32) - 1\n\n translate_ip_ranges = kwargs.pop('translate_ip_ranges', False)\n\n should_aggregate = 'sum' in kwargs\n if should_aggregate:\n translate_ip_ranges = False\n temp_set = set()\n\n cstart = start\n\n while cstart < (start + num):\n ilist = zrange(feed, cstart,\n cstart - 1 + min(start + num - cstart, FEED_INTERVAL))\n\n if should_aggregate:\n for i in ilist:\n for n in _extract_cidrs(i):\n temp_set.add(n)\n\n else:\n if translate_ip_ranges:\n ilist = [xi for i in ilist for xi in _translate_ip_ranges(i)]\n\n yield '\\n'.join(ilist) + '\\n'\n\n if len(ilist) < 100:\n break\n\n cstart += 100\n\n if should_aggregate:\n ip_set = IPSet(temp_set)\n for cidr in ip_set.iter_cidrs():\n yield str(cidr) + '\\n'\n\n\ndef generate_json_feed(feed, start, num, desc, value, **kwargs):\n zrange = SR.zrange\n if desc:\n zrange = SR.zrevrange\n\n if num is None:\n num = (1 << 32) - 1\n\n translate_ip_ranges = kwargs.pop('translate_ip_ranges', False)\n\n if value == 'json':\n yield '[\\n'\n\n cstart = start\n firstelement = True\n\n while cstart < (start + num):\n ilist = zrange(feed, cstart,\n cstart - 1 + min(start + num - cstart, FEED_INTERVAL))\n\n result = cStringIO.StringIO()\n\n for indicator in ilist:\n v = SR.hget(feed + '.value', indicator)\n\n xindicators = [indicator]\n if translate_ip_ranges and '-' in indicator:\n xindicators = _translate_ip_ranges(indicator, None if v is None else json.loads(v))\n\n if v is None:\n v = 'null'\n\n for i in xindicators:\n if value == 'json' and not firstelement:\n result.write(',\\n')\n\n if value == 'json-seq':\n result.write('\\x1E')\n\n result.write('{\"indicator\":')\n result.write(json.dumps(i))\n result.write(',\"value\":')\n result.write(v)\n result.write('}')\n\n if value == 'json-seq':\n result.write('\\n')\n\n firstelement = False\n\n yield result.getvalue()\n\n result.close()\n\n if len(ilist) < 100:\n break\n\n cstart += 100\n\n if value == 'json':\n yield ']\\n'\n\n\ndef generate_csv_feed(feed, start, num, desc, value, **kwargs):\n def _is_atomic_type(fv):\n return (isinstance(fv, unicode) or isinstance(fv, str) or isinstance(fv, int) or isinstance(fv, bool))\n\n def _format_field_value(fv):\n if _is_atomic_type(fv):\n return fv\n\n if isinstance(fv, list):\n ok = True\n for fve in fv:\n ok &= _is_atomic_type(fve)\n\n if ok:\n return ','.join(fv)\n\n return json.dumps(fv)\n\n zrange = SR.zrange\n if desc:\n zrange = SR.zrevrange\n\n if num is None:\n num = (1 << 32) - 1\n\n translate_ip_ranges = kwargs.pop('translate_ip_ranges', False)\n\n # extract name of fields and column names\n columns = []\n fields = []\n for addf in kwargs.pop('f', []):\n if '|' in addf:\n fname, cname = addf.rsplit('|', 1)\n else:\n fname = addf\n cname = addf\n columns.append(cname)\n fields.append(fname)\n\n # if no fields are specified, only indicator is generated\n if len(fields) == 0:\n fields = ['indicator']\n columns = ['indicator']\n\n # check if header should be generated\n header = kwargs.pop('h', None)\n if header is None:\n header = True\n else:\n header = int(header[0])\n\n # check if bom should be generated\n ubom = kwargs.pop('ubom', None)\n if ubom is None:\n ubom = False\n else:\n ubom = int(ubom[0])\n\n cstart = start\n\n if ubom:\n LOG.debug('BOM')\n yield '\\xef\\xbb\\xbf'\n\n with _buffer() as current_line:\n w = unicodecsv.DictWriter(\n current_line,\n fieldnames=columns,\n encoding='utf-8'\n )\n\n if header:\n w.writeheader()\n yield current_line.getvalue()\n\n while cstart < (start + num):\n ilist = zrange(feed, cstart,\n cstart - 1 + min(start + num - cstart, FEED_INTERVAL))\n\n for indicator in ilist:\n v = SR.hget(feed + '.value', indicator)\n v = None if v is None else json.loads(v)\n\n xindicators = [indicator]\n if translate_ip_ranges and '-' in indicator:\n xindicators = _translate_ip_ranges(indicator, v)\n\n for i in xindicators:\n fieldvalues = {}\n\n for f, c in zip(fields, columns):\n if f == 'indicator':\n fieldvalues[c] = i\n continue\n\n if v is not None and f in v:\n fieldvalues[c] = _format_field_value(v[f])\n\n current_line.truncate(0)\n w.writerow(fieldvalues)\n yield current_line.getvalue()\n\n if len(ilist) < FEED_INTERVAL:\n break\n\n cstart += FEED_INTERVAL\n\n\ndef generate_mwg_feed(feed, start, num, desc, value, **kwargs):\n zrange = SR.zrange\n if desc:\n zrange = SR.zrevrange\n\n if num is None:\n num = (1 << 32) - 1\n\n translate_ip_ranges = kwargs.pop('translate_ip_ranges', False)\n type_ = kwargs.get('t', None)\n if type_ is None:\n type_ = 'string'\n else:\n type_ = type_[0]\n translate_ip_ranges |= type_ == 'ip'\n\n yield 'type={}\\n'.format(type_)\n\n cstart = start\n while cstart < (start + num):\n ilist = zrange(feed, cstart,\n cstart - 1 + min(start + num - cstart, FEED_INTERVAL))\n\n for indicator in ilist:\n v = SR.hget(feed + '.value', indicator)\n v = None if v is None else json.loads(v)\n\n xindicators = [indicator]\n if translate_ip_ranges and '-' in indicator:\n xindicators = _translate_ip_ranges(indicator, v)\n\n sources = 'from minemeld'\n if v is not None:\n sources = v.get('sources', 'from minemeld')\n if isinstance(sources, list):\n sources = ','.join(sources)\n\n for i in xindicators:\n yield '\"{}\" \"{}\"\\n'.format(\n i.replace('\"', '\\\\\"'),\n sources.replace('\"', '\\\\\"')\n )\n\n if len(ilist) < 100:\n break\n\n cstart += 100\n\n\n# This formatter implements BlueCoat custom URL format as described at\n# https://www.bluecoat.com/documents/download/a366dc73-d455-4859-b92a-c96bd034cb4c/f849f1e3-a906-4ee8-924e-a2061dfe3cdf\n# It expects the value 'bc_category' in the indicator. The value can be either a single string or a list of strings.\n# Optional feed arguments:\n# ca : Indicator's attribute that hosts the BlueCoat category. Defaults to 'bc_category'\n# cd : Default BlueCoat category for indicators that do not have 'catattr'. This argument can appear multiple\n# times and it will be handled as a list of categories the indicator belongs to. If not present then\n# indicators without 'catattr' will be discarded.\ndef generate_bluecoat_feed(feed, start, num, desc, value, **kwargs):\n zrange = SR.zrange\n ilist = zrange(feed, 0, (1 << 32) - 1)\n bc_dict = defaultdict(list)\n flag_category_default = kwargs.get('cd', None)\n flag_category_attr = kwargs.get('ca', ['bc_category'])[0]\n\n for i in ilist:\n sleep(0)\n v = SR.hget(feed + '.value', i)\n v = None if v is None else json.loads(v)\n i = i.lower()\n i = _PROTOCOL_RE.sub('', i)\n i = _INVALID_TOKEN_RE.sub('*', i)\n\n if v is None:\n if flag_category_default is None:\n continue\n else:\n bc_cat_list = flag_category_default\n else:\n bc_cat_attr = v.get(flag_category_attr, None)\n if isinstance(bc_cat_attr, list):\n bc_cat_list = bc_cat_attr\n elif isinstance(bc_cat_attr, basestring):\n bc_cat_list = [bc_cat_attr]\n elif flag_category_default is not None:\n bc_cat_list = flag_category_default\n else:\n continue\n\n for bc_cat in bc_cat_list:\n bc_dict[bc_cat].append(i)\n\n for key, value in bc_dict.iteritems():\n yield 'define category {}\\n'.format(key)\n for ind in value:\n yield ind + '\\n'\n yield 'end\\n'\n\n\ndef generate_carbon_black(feed, start, num, desc, value, **kwargs):\n zrange = SR.zrange\n ilist = zrange(feed, 0, (1 << 32) - 1)\n mm_to_cb = {\"IPv4\": \"ipv4\",\n \"domain\": \"dns\",\n \"md5\": \"md5\"}\n ind_by_type = {\"dns\": [],\n \"md5\": []}\n\n # Let's stream the information as soon as we have it\n yield \"{\\n\\\"feedinfo\\\": {\\n\"\n cb_feed_info = CbFeedInfo(name=feed)\n for cb_info_parts in cb_feed_info.iterate():\n yield \" \" + cb_info_parts\n yield \"\\n},\\n\\\"reports\\\": [{\"\n\n report_args = dict()\n report_args[\"id\"] = feed + \"_report\"\n\n report_title = kwargs.get('rt', [\"MieneMeld Generated Report\"])\n if report_title is not None:\n report_title = report_title[0]\n report_args[\"title\"] = report_title\n\n report_score = kwargs.get('rs', None)\n if report_score is not None:\n try:\n report_score = int(report_score[0])\n except ValueError:\n report_score = None\n report_args[\"score\"] = report_score\n\n cb_report = CbReport(**report_args)\n\n for cb_report_parts in cb_report.iterate():\n yield \" \" + cb_report_parts\n yield \", \\\"iocs\\\": {\"\n yield \" \\\"ipv4\\\": [\"\n\n # Loop though all indicators\n # Only indicators of type IPv4, domain and md5 can be exported to Carbon Black\n ipv4_line = None\n for i in ilist:\n sleep(0)\n v = SR.hget(feed + '.value', i)\n v = None if v is None else json.loads(v)\n if v is None:\n continue\n v_type = v.get(\"type\", None)\n if v_type not in mm_to_cb:\n continue\n if v_type in (\"domain\", \"md5\"):\n ind_by_type[mm_to_cb[v_type]].append(i.lower())\n continue\n\n # Carbon Black do not support IPv4 networks not ranges. We must expand them.\n ip_range = None\n if _IPV4_MASK_RE.match(i):\n ip_range = IPSet(IPNetwork(i))\n elif _IPV4_RANGE_RE.match(i):\n range_parts = i.split(\"-\")\n ip_range = IPRange(range_parts[0], range_parts[1])\n for ip_addr in ip_range:\n if ipv4_line is not None:\n yield ipv4_line + \",\"\n ipv4_line = \"\\\"{}\\\"\".format(str(ip_addr))\n yield (\"\" if ipv4_line is None else ipv4_line) + \"],\"\n yield \"\\\"dns\\\": {},\".format(json.dumps(ind_by_type[\"dns\"]))\n yield \"\\\"md5\\\": {}\".format(json.dumps(ind_by_type[\"md5\"]))\n yield \"}}]}\"\n\n\n_FEED_FORMATS = {\n 'json': {\n 'formatter': generate_json_feed,\n 'mimetype': 'application/json'\n },\n 'json-seq': {\n 'formatter': generate_json_feed,\n 'mimetype': 'application/json-seq'\n },\n 'panosurl': {\n 'formatter': generate_panosurl_feed,\n 'mimetype': 'text/plain'\n },\n 'mwg': {\n 'formatter': generate_mwg_feed,\n 'mimetype': 'text/plain'\n },\n 'bluecoat': {\n 'formatter': generate_bluecoat_feed,\n 'mimetype': 'text/plain'\n },\n 'carbonblack': {\n 'formatter': generate_carbon_black,\n 'mimetype': 'application/json'\n },\n 'csv': {\n 'formatter': generate_csv_feed,\n 'mimetype': 'text/csv'\n }\n}\n\n\n@BLUEPRINT.route('/', methods=['GET'], feeds=True, read_write=False)\ndef get_feed_content(feed):\n if not current_user.check_feed(feed):\n return 'Unauthorized', 401\n\n # check if feed exists\n status = MMMaster.status()\n tr = status.get('result', None)\n if tr is None:\n LOG.error(\"Error retrieving status from MMMaster: {!r}\".format(status.get('error', 'error')))\n return 'Internal error', 500\n\n nname = 'mbus:slave:' + feed\n if nname not in tr:\n return 'Unknown feed', 404\n nclass = tr[nname].get('class', None)\n if nclass != 'minemeld.ft.redis.RedisSet':\n return 'Unknown feed', 404\n\n start = request.values.get('s')\n if start is None:\n start = 0\n try:\n start = int(start)\n if start < 0:\n raise ValueError()\n except ValueError:\n LOG.error(\"Invalid request, s not a non-negative integer: %s\", start)\n return 's should be a positive integer', 400\n\n num = request.values.get('n')\n if num is not None:\n try:\n num = int(num)\n if num <= 0:\n raise ValueError()\n except ValueError:\n LOG.error(\"Invalid request, n not a positive integer: %s\", num)\n return 'n should be a positive integer', 400\n else:\n num = None\n\n desc = request.values.get('d')\n desc = (False if desc is None else True)\n\n value = request.values.get('v')\n if value is not None and value not in _FEED_FORMATS:\n return 'unknown format', 400\n\n kwargs = {}\n kwargs['translate_ip_ranges'] = int(request.values.get('tr', 0)) # generate IP ranges\n\n # move to kwargs all the additional parameters, pop the predefined\n kwargs.update(request.values.to_dict(flat=False))\n kwargs.pop('s', None)\n kwargs.pop('n', None)\n kwargs.pop('d', None)\n kwargs.pop('v', None)\n kwargs.pop('tr', None)\n\n mimetype = 'text/plain'\n formatter = generate_plain_feed\n if value in _FEED_FORMATS:\n formatter = _FEED_FORMATS[value]['formatter']\n mimetype = _FEED_FORMATS[value]['mimetype']\n\n return Response(\n stream_with_context(\n formatter(feed, start, num, desc, value, **kwargs)\n ),\n mimetype=mimetype\n )\n" }, { "path": "minemeld/flask/jobs.py", "content": "# Copyright 2017 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport uuid\nimport tempfile\nimport subprocess\nimport shutil\nimport json\nimport time\nimport signal\nfrom collections import namedtuple, defaultdict\n\nimport redis\nimport psutil\nimport werkzeug.local\nimport gevent\nfrom gevent.subprocess import Popen\nfrom flask import g\n\nfrom . import REDIS_URL\nfrom . import config\nfrom .logger import LOG\n\n\n__all__ = ['init_app', 'JOBS_MANAGER']\n\n\nREDIS_CP = redis.ConnectionPool.from_url(\n REDIS_URL,\n max_connections=int(os.environ.get('REDIS_MAX_CONNECTIONS', 5))\n)\n\nREDIS_JOBS_GROUP_PREFIX = 'mm-jobs-{}'\n\n\n_Job = namedtuple('_Job', field_names=['glet', 'timeout_glet'])\n\n\nclass JobsManager(object):\n def __init__(self, connection_pool):\n self.SR = redis.StrictRedis(connection_pool=connection_pool)\n self.running_jobs = defaultdict(dict)\n\n def _safe_rmtree(self, path):\n shutil.rmtree(path, ignore_errors=True)\n\n def _safe_remove(self, path):\n try:\n os.remove(path)\n except:\n pass\n\n def _get_job_status(self, jobpid, jobhash):\n try:\n jobprocess = psutil.Process(pid=jobpid)\n\n except psutil.NoSuchProcess:\n return {\n 'status': 'DONE',\n 'returncode': None\n }\n\n if hash(jobprocess) != jobhash:\n return {\n 'status': 'DONE',\n 'returncode': None\n }\n\n return {\n 'status': 'RUNNING'\n }\n\n def _collect_job(self, jobdata):\n if 'collected' in jobdata:\n return\n\n tempdir = jobdata.get('cwd', None)\n if tempdir is not None:\n self._safe_rmtree(tempdir)\n\n jobdata['collected'] = True\n\n def _job_monitor_glet(self, job_group, jobid, description, args, data):\n jobname = (REDIS_JOBS_GROUP_PREFIX+'-{}').format(job_group, jobid)\n joblogfile = os.path.join(\n config.get('MINEMELD_LOG_DIRECTORY_PATH', '/tmp'),\n '{}.log'.format(jobname)\n )\n jobtempdir = tempfile.mkdtemp(prefix=jobname)\n\n LOG.info('Executing job {} - {} cwd: {} logfile: {}'.format(jobname, args, jobtempdir, joblogfile))\n\n try:\n with open(joblogfile, 'w+') as logfile:\n jobprocess = Popen(\n args=args,\n close_fds=True,\n cwd=jobtempdir,\n shell=False,\n stdout=logfile,\n stderr=subprocess.STDOUT\n )\n\n except OSError:\n self._safe_remove(joblogfile)\n self._safe_rmtree(jobtempdir)\n LOG.exception('Error starting job {}'.format(jobname))\n return\n\n jobpsproc = psutil.Process(pid=jobprocess.pid)\n\n jobdata = data\n if jobdata is None:\n jobdata = {}\n\n jobdata['create_time'] = int(time.time()*1000)\n jobdata['description'] = description\n jobdata['job_id'] = jobid\n jobdata['pid'] = jobpsproc.pid\n jobdata['hash'] = hash(jobpsproc)\n jobdata['logfile'] = joblogfile\n jobdata['cwd'] = jobtempdir\n jobdata['status'] = 'RUNNING'\n\n self.SR.hset(\n REDIS_JOBS_GROUP_PREFIX.format(job_group),\n jobid,\n json.dumps(jobdata)\n )\n\n jobprocess.wait()\n\n if jobprocess.returncode != 0:\n jobdata['status'] = 'ERROR'\n else:\n jobdata['status'] = 'DONE'\n jobdata['returncode'] = jobprocess.returncode\n jobdata['end_time'] = int(time.time()*1000)\n\n self._collect_job(jobdata)\n\n self.SR.hset(\n REDIS_JOBS_GROUP_PREFIX.format(job_group),\n jobid,\n json.dumps(jobdata)\n )\n\n job = self.running_jobs[job_group].pop(jobid, None)\n if job is not None and job.timeout_glet is not None:\n job.timeout_glet.kill()\n\n return jobprocess.returncode\n\n def _job_timeout_glet(self, job_group, jobid, timeout):\n gevent.sleep(timeout)\n\n prefix = REDIS_JOBS_GROUP_PREFIX.format(job_group)\n\n jobdata = self.SR.hget(prefix, jobid)\n if jobdata is None:\n return\n\n jobdata = json.loads(jobdata)\n status = jobdata.get('status', None)\n if status != 'RUNNING':\n LOG.info('Timeout for job {}-{} triggered but status not running'.format(prefix, jobid))\n return\n\n pid = jobdata.get('pid', None)\n if pid is None:\n LOG.error('Timeout for job {}-{} triggered but no pid available'.format(prefix, jobid))\n return\n\n LOG.error('Timeout for job {}-{} triggered, sending TERM signal'.format(prefix, jobid))\n os.kill(pid, signal.SIGTERM)\n\n def delete_job(self, job_group, jobid):\n prefix = REDIS_JOBS_GROUP_PREFIX.format(job_group)\n\n jobdata = self.SR.hget(prefix, jobid)\n if jobdata is None:\n return\n\n jobdata = json.loads(jobdata)\n\n logfile = jobdata.get('logfile', None)\n if logfile is not None:\n self._safe_remove(logfile)\n\n self._collect_job(jobdata)\n\n self.SR.hdel(prefix, jobid)\n\n def get_jobs(self, job_group):\n prefix = REDIS_JOBS_GROUP_PREFIX.format(job_group)\n result = {}\n\n jobs_map = self.SR.hgetall(prefix)\n\n for jobid, jobdata in jobs_map.iteritems():\n try:\n jobdata = json.loads(jobdata)\n\n if jobdata['status'] == 'RUNNING':\n jobpid = jobdata['pid']\n job_status = self._get_job_status(jobpid, jobdata['hash'])\n jobdata.update(job_status)\n\n result[jobid] = jobdata\n\n except (ValueError, KeyError, psutil.ZombieProcess, psutil.AccessDenied):\n LOG.error('Invalid job value - deleting job {}::{}'.format(job_group, jobid))\n self.delete_job(job_group, jobid)\n continue\n\n if jobdata['status'] == 'DONE' and 'collected' not in jobdata:\n if jobid not in self.running_jobs[job_group]:\n self._collect_job(jobdata)\n self.SR.hset(job_group, jobid, json.dumps(jobdata))\n\n return result\n\n def exec_job(self, job_group, description, args, data=None, callback=None, timeout=None):\n jobid = str(uuid.uuid4())\n\n glet = gevent.spawn(\n self._job_monitor_glet,\n job_group, jobid, description, args, data\n )\n if callback is not None:\n glet.link(callback)\n\n timeout_glet = None\n if timeout is not None:\n timeout_glet = gevent.spawn(self._job_timeout_glet, job_group, jobid, timeout)\n\n self.running_jobs[job_group][jobid] = _Job(glet=glet, timeout_glet=timeout_glet)\n\n return jobid\n\n\ndef get_JobsManager():\n jobsmgr = getattr(g, '_jobs_manager', None)\n if jobsmgr is None:\n jobsmgr = JobsManager(connection_pool=REDIS_CP)\n g._jobs_manager = jobsmgr\n return jobsmgr\n\n\ndef teardown(exception):\n jobsmgr = getattr(g, '_jobs_manager', None)\n if jobsmgr is not None:\n g._jobs_manager = None\n LOG.info(\n 'redis connection pool: in use: {} available: {}'.format(\n len(REDIS_CP._in_use_connections),\n len(REDIS_CP._available_connections)\n )\n )\n\n\nJOBS_MANAGER = werkzeug.local.LocalProxy(get_JobsManager)\n\n\ndef init_app(app):\n app.teardown_appcontext(teardown)\n" }, { "path": "minemeld/flask/jobsapi.py", "content": "# Copyright 2015-2017 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport os.path\n\nfrom flask import send_from_directory, jsonify\n\nfrom .jobs import JOBS_MANAGER\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\n\n\n__all__ = ['BLUEPRINT']\n\n\nBLUEPRINT = MMBlueprint('jobs', __name__, url_prefix='/jobs')\n\n\n@BLUEPRINT.route('/', methods=['GET'], read_write=False)\ndef get_jobs(job_group):\n jobs = JOBS_MANAGER.get_jobs(job_group)\n\n return jsonify(result=jobs)\n\n\n@BLUEPRINT.route('//', methods=['GET'], read_write=False)\ndef get_job(job_group, jobid):\n jobs = JOBS_MANAGER.get_jobs(job_group)\n if jobid not in jobs:\n return jsonify(error={'message': 'job unknown'}), 400\n\n return jsonify(result=jobs[jobid])\n\n\n@BLUEPRINT.route('///log', methods=['GET'], read_write=False)\ndef get_job_log(job_group, jobid):\n jobs = JOBS_MANAGER.get_jobs(job_group)\n if jobid not in jobs:\n return jsonify(error={'message': 'job unknown'}), 400\n\n job = jobs[jobid]\n\n return send_from_directory(\n os.path.dirname(job['logfile']),\n os.path.basename(job['logfile']),\n as_attachment=True\n )\n" }, { "path": "minemeld/flask/logger.py", "content": "import logging\nimport json\n\nfrom flask import current_app\n\n# [2017-01-16 20:32:07 +0000] [15997] [INFO]\nLOG_FORMAT = '[%(asctime)s] [%(process)d] [%(levelname)s] %(message)s'\nLOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S %Z'\n\n\nclass MMLogger(object):\n def __init__(self):\n self.system_logger = logging.getLogger('minemeld')\n self._init_logger(self.system_logger)\n\n self.system_logger.info('MMLogger started')\n\n def init_app(self, app):\n del app.logger.handlers[:]\n\n self._init_logger(app.logger)\n\n def _init_logger(self, logger):\n logger.propagate = False\n logger.setLevel(logging.DEBUG)\n\n handler = logging.StreamHandler()\n handler.setFormatter(logging.Formatter(\n fmt=LOG_FORMAT,\n datefmt=LOG_DATE_FORMAT\n ))\n logger.addHandler(handler)\n\n def debug(self, *args, **kwargs):\n if current_app:\n current_app.logger.debug(*args, **kwargs)\n else:\n self.system_logger.debug(*args, **kwargs)\n\n def info(self, *args, **kwargs):\n if current_app:\n current_app.logger.info(*args, **kwargs)\n else:\n self.system_logger.info(*args, **kwargs)\n\n def warning(self, *args, **kwargs):\n if current_app:\n current_app.logger.warning(*args, **kwargs)\n else:\n self.system_logger.warning(*args, **kwargs)\n\n def error(self, *args, **kwargs):\n if current_app:\n current_app.logger.error(*args, **kwargs)\n else:\n self.system_logger.error(*args, **kwargs)\n\n def critical(self, *args, **kwargs):\n if current_app:\n current_app.logger.critical(*args, **kwargs)\n else:\n self.system_logger.critical(*args, **kwargs)\n\n def exception(self, *args, **kwargs):\n if current_app:\n current_app.logger.exception(*args, **kwargs)\n else:\n self.system_logger.exception(*args, **kwargs)\n\n def audit(self, user_id, action_name, params, msg=None):\n audit_params = dict(user=user_id, action=action_name, params=params, msg=msg)\n self.info('AUDIT - {}'.format(json.dumps(audit_params)))\n\n\nLOG = MMLogger()\n" }, { "path": "minemeld/flask/loginapi.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom flask import request, jsonify\nimport flask.ext.login\n\nfrom . import aaa\nfrom .logger import LOG\n\n\n__all__ = ['BLUEPRINT']\n\n\nBLUEPRINT = aaa.MMBlueprint('login', __name__, url_prefix='')\n\n\n@BLUEPRINT.route('/login', methods=['GET', 'POST'], login_required=False, read_write=False)\ndef login():\n username = request.values.get('u')\n if username is None:\n return jsonify(error='Missing username'), 400\n\n password = request.values.get('p')\n if password is None:\n return jsonify(error='Missing password'), 400\n\n user = aaa.check_admin_user(username, password)\n if user is None:\n return jsonify(error=\"Wrong credentials\"), 401\n\n flask.ext.login.login_user(user)\n return 'OK'\n\n\n@BLUEPRINT.route('/logout', methods=['GET'], login_required=False, read_write=False)\ndef logout():\n flask.ext.login.logout_user()\n return 'OK'\n" }, { "path": "minemeld/flask/logsapi.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom flask import send_from_directory, jsonify\n\nfrom . import config\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\n\n\n__all__ = ['BLUEPRINT']\n\n\nBLUEPRINT = MMBlueprint('logs', __name__, url_prefix='/logs')\n\n\n@BLUEPRINT.route('/minemeld-engine.log', methods=['GET'], read_write=True)\ndef get_minemeld_engine_log():\n log_directory = config.get('MINEMELD_LOG_DIRECTORY_PATH', None)\n if log_directory is None:\n return jsonify(error={'message': 'LOG_DIRECTORY not set'}), 500\n\n return send_from_directory(log_directory, 'minemeld-engine.log', as_attachment=True)\n\n\n@BLUEPRINT.route('/minemeld-web.log', methods=['GET'], read_write=True)\ndef get_minemeld_web_log():\n log_directory = config.get('MINEMELD_LOG_DIRECTORY_PATH', None)\n if log_directory is None:\n return jsonify(error={'message': 'LOG_DIRECTORY not set'}), 500\n\n return send_from_directory(log_directory, 'minemeld-web.log', as_attachment=True)\n" }, { "path": "minemeld/flask/main.py", "content": "from . import create_app\n\napp = create_app()\n" }, { "path": "minemeld/flask/metricsapi.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport os.path\nimport hashlib\n\nimport rrdtool\n\nfrom flask import request, jsonify\n\nimport minemeld.collectd\n\nfrom . import config\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\n\n\n__all__ = ['BLUEPRINT']\n\n\nRRD_PATH = config.get('RRD_PATH', '/var/lib/collectd/rrd/minemeld/')\nRRD_SOCKET_PATH = config.get('RRD_SOCKET_PATH', '/var/run/collectd.sock')\nALLOWED_CF = ['MAX', 'MIN', 'AVERAGE']\n\n\nBLUEPRINT = MMBlueprint('metrics', __name__, url_prefix='/metrics')\n\n\ndef _list_metrics(prefix=None):\n result = os.listdir(RRD_PATH)\n\n if prefix is not None:\n result = [m for m in result if m.startswith(prefix)]\n\n return result\n\n\ndef _fetch_metric(cc, metric, type_=None,\n cf='MAX', dt=86400, r=1800):\n dirname = os.path.join(RRD_PATH, metric)\n\n if type_ is None:\n rrdname = os.listdir(dirname)[0]\n type_ = rrdname.replace('.rrd', '')\n else:\n rrdname = type_+'.rrd'\n if rrdname not in os.listdir(dirname):\n raise RuntimeError('Unknown metric type')\n\n cc.flush(identifier='minemeld/%s/%s' % (metric, type_))\n\n (start, end, step), metrics, data = rrdtool.fetch(\n str(os.path.join(dirname, rrdname)),\n cf,\n '--start', '-%d' % dt,\n '--resolution', '%d' % r\n )\n\n result = []\n\n if type_ != 'minemeld_delta':\n curts = start\n for v in data:\n result.append([curts, v[0]])\n curts += step\n else:\n curts = start+step\n ov = data[0][0]\n for v in data[1:]:\n cv = v[0]\n if cv is not None and ov is not None:\n if cv >= ov:\n cv = cv - ov\n result.append([curts, cv])\n\n ov = v[0]\n curts += step\n\n return result\n\n\n@BLUEPRINT.route('/', read_write=False)\ndef get_metrics():\n return jsonify(result=_list_metrics())\n\n\n@BLUEPRINT.route('/minemeld/', read_write=False)\ndef get_node_type_metrics(nodetype):\n cf = str(request.args.get('cf', 'MAX')).upper()\n if cf not in ALLOWED_CF:\n return jsonify(error={'message': 'Unknown function'}), 400\n\n try:\n dt = int(request.args.get('dt', '86400'))\n except ValueError:\n return jsonify(error={'message': 'Invalid delta'}), 400\n if dt < 0:\n return jsonify(error={'message': 'Invalid delta'}), 400\n\n try:\n resolution = int(request.args.get('r', '1800'))\n except ValueError:\n return jsonify(error={'message': 'Invalid resolution'}), 400\n if resolution < 0:\n return jsonify(error={'message': 'Invalid resolution'}), 400\n\n type_ = request.args.get('t', None)\n\n metrics = _list_metrics(prefix='minemeld.'+nodetype+'.')\n\n cc = minemeld.collectd.CollectdClient(RRD_SOCKET_PATH)\n\n result = []\n for m in metrics:\n v = _fetch_metric(cc, m, cf=cf, dt=dt, r=resolution, type_=type_)\n\n _, _, mname = m.split('.', 2)\n\n result.append({\n 'metric': mname,\n 'values': v\n })\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/minemeld', read_write=False)\ndef get_global_metrics():\n cf = str(request.args.get('cf', 'MAX')).upper()\n if cf not in ALLOWED_CF:\n return jsonify(error={'message': 'Unknown function'}), 400\n\n try:\n dt = int(request.args.get('dt', '86400'))\n except ValueError:\n return jsonify(error={'message': 'Invalid delta'}), 400\n if dt < 0:\n return jsonify(error={'message': 'Invalid delta'}), 400\n\n try:\n resolution = int(request.args.get('r', '1800'))\n except ValueError:\n return jsonify(error={'message': 'Invalid resolution'}), 400\n if resolution < 0:\n return jsonify(error={'message': 'Invalid resolution'}), 400\n\n type_ = request.args.get('t', None)\n\n metrics = _list_metrics(prefix='minemeld.')\n metrics = [m for m in metrics if 'minemeld.sources' not in m]\n metrics = [m for m in metrics if 'minemeld.outputs' not in m]\n metrics = [m for m in metrics if 'minemeld.transits' not in m]\n\n cc = minemeld.collectd.CollectdClient(RRD_SOCKET_PATH)\n\n result = []\n for m in metrics:\n v = _fetch_metric(cc, m, cf=cf, dt=dt, r=resolution, type_=type_)\n\n _, mname = m.split('.', 1)\n\n result.append({\n 'metric': mname,\n 'values': v\n })\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/', read_write=False)\ndef get_node_metrics(node):\n cf = str(request.args.get('cf', 'MAX')).upper()\n if cf not in ALLOWED_CF:\n return jsonify(error={'message': 'Unknown function'}), 400\n\n try:\n dt = int(request.args.get('dt', '86400'))\n except ValueError:\n return jsonify(error={'message': 'Invalid delta'}), 400\n if dt < 0:\n return jsonify(error={'message': 'Invalid delta'}), 400\n\n try:\n resolution = int(request.args.get('r', '1800'))\n except ValueError:\n return jsonify(error={'message': 'Invalid resolution'}), 400\n if resolution < 0:\n return jsonify(error={'message': 'Invalid resolution'}), 400\n\n type_ = request.args.get('t', None)\n\n node = hashlib.md5(node).hexdigest()[:10]\n metrics = _list_metrics(prefix=node+'.')\n\n cc = minemeld.collectd.CollectdClient(RRD_SOCKET_PATH)\n\n result = []\n for m in metrics:\n v = _fetch_metric(cc, m, cf=cf, dt=dt, r=resolution, type_=type_)\n\n _, mname = m.split('.', 1)\n\n result.append({\n 'metric': mname,\n 'values': v\n })\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('//', methods=['GET'], read_write=False)\ndef get_metric(node, metric):\n cf = str(request.args.get('cf', 'MAX')).upper()\n if cf not in ALLOWED_CF:\n return jsonify(error={'message': 'Unknown function'}), 400\n\n try:\n dt = int(request.args.get('dt', '86400'))\n except ValueError:\n return jsonify(error={'message': 'Invalid delta'}), 400\n if dt < 0:\n return jsonify(error={'message': 'Invalid delta'}), 400\n\n try:\n resolution = int(request.args.get('r', '1800'))\n except ValueError:\n return jsonify(error={'message': 'Invalid resolution'}), 400\n if resolution < 0:\n return jsonify(error={'message': 'Invalid resolution'}), 400\n\n type_ = request.args.get('t', 'minemeld_counter')\n\n node = hashlib.md5(node).hexdigest()[:10]\n metric = node+'.'+metric\n\n if metric not in _list_metrics():\n return jsonify(error={'message': 'Unknown metric'}), 404\n\n cc = minemeld.collectd.CollectdClient(RRD_SOCKET_PATH)\n\n try:\n result = _fetch_metric(cc, metric, type_=type_, cf=cf,\n dt=dt, r=resolution)\n except RuntimeError as e:\n return jsonify(error={'message': str(e)}), 400\n\n return jsonify(result=result)\n" }, { "path": "minemeld/flask/mmrpc.py", "content": "import json\n\nimport gevent\nimport gevent.event\nimport gevent.queue\nimport werkzeug.local\n\nfrom flask import g\n\nimport minemeld.comm\nfrom minemeld.mgmtbus import MGMTBUS_PREFIX, MGMTBUS_MASTER\n\nfrom . import config\nfrom .logger import LOG\n\n\n__all__ = ['init_app', 'MMMaster', 'MMRpcClient']\n\n\nclass _MMMasterConnection(object):\n def __init__(self):\n self.comm = None\n\n tconfig = config.get('MGMTBUS', {})\n self.comm_class = tconfig.get('class', 'ZMQRedis')\n self.comm_config = tconfig.get('config', {})\n\n def _open_channel(self):\n if self.comm is not None:\n return\n\n self.comm = minemeld.comm.factory(\n self.comm_class,\n self.comm_config\n )\n self.comm.start()\n\n def _send_cmd(self, method, params={}):\n self._open_channel()\n\n return self.comm.send_rpc(\n MGMTBUS_MASTER,\n method,\n params,\n timeout=10.0\n )\n\n def status(self):\n return self._send_cmd('status')\n\n def stop(self):\n if self.comm is not None:\n self.comm.stop()\n self.comm = None\n\n\nclass _MMRpcClient(object):\n def __init__(self):\n self.comm = None\n\n tconfig = config.get('MGMTBUS', {})\n self.comm_class = tconfig.get('class', 'ZMQRedis')\n self.comm_config = tconfig.get('config', {})\n\n def _open_channel(self):\n if self.comm is not None:\n return\n\n self.comm = minemeld.comm.factory(\n self.comm_class,\n self.comm_config\n )\n self.comm.start()\n\n def send_raw_cmd(self, target, method, params={}, timeout=10):\n self._open_channel()\n\n return self.comm.send_rpc(target, method, params, timeout=timeout)\n\n def send_cmd(self, target, method, params={}, timeout=10):\n target = '{}directslave:{}'.format(MGMTBUS_PREFIX, target)\n\n return self.send_raw_cmd(target, method, params=params, timeout=timeout)\n\n def stop(self):\n if self.comm is not None:\n self.comm.stop()\n self.comm = None\n\n\ndef get_mmmaster():\n r = getattr(g, 'MMMaster', None)\n if r is None:\n r = _MMMasterConnection()\n g.MMMaster = r\n return r\n\n\nMMMaster = werkzeug.LocalProxy(get_mmmaster) # pylint:disable=E1101\n\n\ndef get_mmrpcclient():\n r = getattr(g, 'MMRpcClient', None)\n if r is None:\n r = _MMRpcClient()\n g.MMRpcClient = r\n return r\n\n\nMMRpcClient = werkzeug.LocalProxy(get_mmrpcclient) # pylint:disable=E1101\n\n\ndef teardown(exception):\n r = getattr(g, 'MMMaster', None)\n if r is not None:\n g.MMMaster.stop()\n g.MMMaster = None\n\n r = getattr(g, 'MMRpcClient', None)\n if r is not None:\n g.MMRpcClient.stop()\n g.MMRpcClient = None\n\n\ndef init_app(app):\n app.teardown_appcontext(teardown)\n" }, { "path": "minemeld/flask/prototypeapi.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport sys\nimport os\nimport os.path\nimport json\n\nimport yaml\nimport filelock\nfrom flask import jsonify, request\n\nimport minemeld.loader\n\nfrom . import config\nfrom .utils import running_config, committed_config\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\n\n\n__all__ = ['BLUEPRINT']\n\n\nPROTOTYPE_ENV = 'MINEMELD_PROTOTYPE_PATH'\nLOCAL_PROTOTYPE_PATH = 'MINEMELD_LOCAL_PROTOTYPE_PATH'\n\nBLUEPRINT = MMBlueprint('prototype', __name__, url_prefix='')\n\nPROTOTYPE_PATHS = None\n\n\ndef _prototype_paths():\n global PROTOTYPE_PATHS\n if PROTOTYPE_PATHS is not None:\n return PROTOTYPE_PATHS\n\n paths = config.get(PROTOTYPE_ENV, None)\n if paths is None:\n raise RuntimeError('{} environment variable not set'.format(PROTOTYPE_ENV))\n paths = paths.split(':')\n\n prototype_eps = minemeld.loader.map(minemeld.loader.MM_PROTOTYPES_ENTRYPOINT)\n for pname, mmep in prototype_eps.iteritems():\n if not mmep.loadable:\n LOG.info('Prototype entry point {} not loadable, ignored'.format(pname))\n continue\n try:\n # even if old dist is no longer available, old module could be cached\n cmodule = sys.modules.get(mmep.ep.module_name, None)\n cmodule_path = getattr(cmodule, '__path__', None)\n if cmodule is not None and cmodule_path is not None:\n if not cmodule_path[0].startswith(mmep.ep.dist.location):\n LOG.info('Invalidating cache for {}'.format(mmep.ep.module_name))\n sys.modules.pop(mmep.ep.module_name)\n\n ep = mmep.ep.load()\n # we add prototype paths in front, to let extensions override default protos\n paths.insert(0, ep())\n except:\n LOG.exception('Exception loading paths from {}'.format(pname))\n\n PROTOTYPE_PATHS = paths\n\n return paths\n\n\ndef _local_library_path(prototypename):\n toks = prototypename.split('.', 1)\n if len(toks) != 2:\n raise ValueError('bad prototype name')\n library, prototype = toks\n\n if os.path.basename(library) != library:\n raise ValueError('bad library name, nice try')\n if library != 'minemeldlocal':\n raise ValueError('invalid library')\n library_filename = library+'.yml'\n\n local_path = config.get(LOCAL_PROTOTYPE_PATH)\n if local_path is None:\n paths = os.getenv(PROTOTYPE_ENV, None)\n if paths is None:\n raise RuntimeError(\n '%s environment variable not set' %\n (PROTOTYPE_ENV)\n )\n\n paths = paths.split(':')\n for p in paths:\n if '/local/' in p:\n local_path = p\n break\n\n if local_path is None:\n raise RuntimeError(\n 'No local path in %s' % PROTOTYPE_ENV\n )\n\n library_path = os.path.join(local_path, library_filename)\n\n return library_path, prototype\n\n\n@BLUEPRINT.route('/prototype', methods=['GET'], read_write=False)\ndef list_prototypes():\n paths = _prototype_paths()\n\n prototypes = {}\n for p in paths:\n try:\n for plibrary in os.listdir(p):\n if not plibrary.endswith('.yml'):\n continue\n\n plibraryname, _ = plibrary.rsplit('.', 1)\n\n with open(os.path.join(p, plibrary), 'r') as f:\n pcontents = yaml.safe_load(f)\n\n if plibraryname not in prototypes:\n prototypes[plibraryname] = pcontents\n continue\n\n # oldest has precedence\n newprotos = pcontents.get('prototypes', {})\n currprotos = prototypes[plibraryname].get('prototypes', {})\n newprotos.update(currprotos)\n prototypes[plibraryname]['prototypes'] = newprotos\n\n except:\n LOG.exception('Error loading libraries from %s', p)\n\n return jsonify(result=prototypes)\n\n\n@BLUEPRINT.route('/prototype/', methods=['GET'], read_write=False)\ndef get_prototype(prototypename):\n toks = prototypename.split('.', 1)\n if len(toks) != 2:\n return jsonify(error={'message': 'bad prototype name'}), 400\n library, prototype = toks\n\n if os.path.basename(library) != library:\n return jsonify(error={'message': 'bad library name, nice try'}), 400\n library_filename = library+'.yml'\n\n paths = _prototype_paths()\n\n for path in paths:\n full_library_name = os.path.join(path, library_filename)\n if not os.path.isfile(full_library_name):\n continue\n\n with open(full_library_name, 'r') as f:\n library_contents = yaml.safe_load(f)\n\n prototypes = library_contents.get('prototypes', None)\n if prototypes is None:\n continue\n\n if prototype not in prototypes:\n continue\n\n curr_prototype = prototypes[prototype]\n\n result = {\n 'class': curr_prototype['class'],\n 'developmentStatus': None,\n 'config': None,\n 'nodeType': None,\n 'description': None,\n 'indicatorTypes': None,\n 'tags': None\n }\n\n if 'config' in curr_prototype:\n result['config'] = yaml.dump(\n curr_prototype['config'],\n indent=4,\n default_flow_style=False\n )\n\n if 'development_status' in curr_prototype:\n result['developmentStatus'] = curr_prototype['development_status']\n\n if 'node_type' in curr_prototype:\n result['nodeType'] = curr_prototype['node_type']\n\n if 'description' in curr_prototype:\n result['description'] = curr_prototype['description']\n\n if 'indicator_types' in curr_prototype:\n result['indicatorTypes'] = curr_prototype['indicator_types']\n\n if 'tags' in curr_prototype:\n result['tags'] = curr_prototype['tags']\n\n return jsonify(result=result), 200\n\n\n@BLUEPRINT.route('/prototype/', methods=['POST'], read_write=True)\ndef add_local_prototype(prototypename):\n AUTHOR_ = 'minemeld-web'\n DESCRIPTION_ = 'Local prototype library managed via MineMeld WebUI'\n\n try:\n library_path, prototype = _local_library_path(prototypename)\n\n except ValueError as e:\n return jsonify(error={'message': str(e)}), 400\n\n lock = filelock.FileLock('{}.lock'.format(library_path))\n with lock.acquire(timeout=10):\n if os.path.isfile(library_path):\n with open(library_path, 'r') as f:\n library_contents = yaml.safe_load(f)\n if not isinstance(library_contents, dict):\n library_contents = {}\n if 'description' not in library_contents:\n library_contents['description'] = DESCRIPTION_\n if 'prototypes' not in library_contents:\n library_contents['prototypes'] = {}\n if 'author' not in library_contents:\n library_contents['author'] = AUTHOR_\n else:\n library_contents = {\n 'author': AUTHOR_,\n 'description': DESCRIPTION_,\n 'prototypes': {}\n }\n\n try:\n incoming_prototype = request.get_json()\n except Exception as e:\n return jsonify(error={'message': str(e)}), 400\n\n new_prototype = {\n 'class': incoming_prototype['class'],\n }\n\n if 'config' in incoming_prototype:\n try:\n new_prototype['config'] = yaml.safe_load(\n incoming_prototype['config']\n )\n except Exception as e:\n return jsonify(error={'message': 'invalid YAML in config'}), 400\n\n if 'developmentStatus' in incoming_prototype:\n new_prototype['development_status'] = \\\n incoming_prototype['developmentStatus']\n\n if 'nodeType' in incoming_prototype:\n new_prototype['node_type'] = incoming_prototype['nodeType']\n\n if 'description' in incoming_prototype:\n new_prototype['description'] = incoming_prototype['description']\n\n if 'indicatorTypes' in incoming_prototype:\n new_prototype['indicator_types'] = incoming_prototype['indicatorTypes']\n\n if 'tags' in incoming_prototype:\n new_prototype['tags'] = incoming_prototype['tags']\n\n library_contents['prototypes'][prototype] = new_prototype\n\n with open(library_path, 'w') as f:\n yaml.safe_dump(library_contents, f, indent=4, default_flow_style=False)\n\n return jsonify(result='OK'), 200\n\n\n@BLUEPRINT.route('/prototype/', methods=['DELETE'], read_write=True)\ndef delete_local_prototype(prototypename):\n try:\n library_path, prototype = _local_library_path(prototypename)\n\n except ValueError as e:\n return jsonify(error={'message': str(e)}), 400\n\n if not os.path.isfile(library_path):\n return jsonify(error={'message': 'missing local prototype library'}), 400\n\n # check if the proto is in use in running or committed config\n rcconfig = running_config()\n for nodename, nodevalue in rcconfig.get('nodes', {}).iteritems():\n if 'prototype' not in nodevalue:\n continue\n if nodevalue['prototype'] == prototypename:\n return jsonify(error={'message': 'prototype in use in running config'}), 400\n\n ccconfig = committed_config()\n for nodename, nodevalue in ccconfig.get('nodes', {}).iteritems():\n if 'prototype' not in nodevalue:\n continue\n if nodevalue['prototype'] == prototypename:\n return jsonify(error={'message': 'prototype in use in committed config'}), 400\n\n lock = filelock.FileLock('{}.lock'.format(library_path))\n with lock.acquire(timeout=10):\n with open(library_path, 'r') as f:\n library_contents = yaml.safe_load(f)\n\n if not isinstance(library_contents, dict):\n return jsonify(error={'message': 'invalid local prototype library'}), 400\n\n library_contents['prototypes'].pop(prototype, None)\n\n with open(library_path, 'w') as f:\n yaml.safe_dump(library_contents, f, indent=4, default_flow_style=False)\n\n return jsonify(result='OK'), 200\n\n\ndef reset_prototype_paths():\n global PROTOTYPE_PATHS\n PROTOTYPE_PATHS = None\n" }, { "path": "minemeld/flask/redisclient.py", "content": "import os\n\nimport redis\nimport werkzeug.local\n\nfrom flask import g\n\nfrom . import REDIS_URL\nfrom .logger import LOG\n\n\n__all__ = ['init_app', 'SR']\n\n\nREDIS_CP = redis.ConnectionPool.from_url(\n REDIS_URL,\n max_connections=int(os.environ.get('REDIS_MAX_CONNECTIONS', 200))\n)\n\n\ndef get_SR():\n SR = getattr(g, '_redis_client', None)\n if SR is None:\n SR = redis.StrictRedis(connection_pool=REDIS_CP)\n g._redis_client = SR\n return SR\n\n\ndef teardown(exception):\n SR = getattr(g, '_redis_client', None)\n if SR is not None:\n g._redis_client = None\n LOG.debug(\n 'redis connection pool: in use: {} available: {}'.format(\n len(REDIS_CP._in_use_connections),\n len(REDIS_CP._available_connections)\n )\n )\n\n\nSR = werkzeug.local.LocalProxy(get_SR)\n\n\ndef init_app(app):\n app.teardown_appcontext(teardown)\n" }, { "path": "minemeld/flask/session.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nfrom datetime import timedelta\nfrom uuid import uuid4\n\nimport ujson\nimport redis\nimport werkzeug.datastructures\nimport flask.sessions\n\nfrom .logger import LOG\n\n\nSESSION_EXPIRATION_ENV = 'SESSION_EXPIRATION'\nDEFAULT_SESSION_EXPIRATION = 10\n\n\nclass RedisSession(werkzeug.datastructures.CallbackDict, flask.sessions.SessionMixin):\n def __init__(self, initial=None, sid=None, new=False):\n def on_update(self):\n self.modified = True\n werkzeug.datastructures.CallbackDict.__init__(self, initial, on_update)\n self.sid = sid\n self.new = new\n self.modified = False\n\n\nclass RedisSessionInterface(flask.sessions.SessionInterface):\n serializer = ujson\n session_class = RedisSession\n\n def __init__(self, redis_=None, prefix='mm-session:'):\n if redis_ is None:\n redis_ = redis.StrictRedis()\n self.redis = redis_\n self.prefix = prefix\n self.expirtaion_delta = timedelta(\n minutes=int(os.environ.get(\n SESSION_EXPIRATION_ENV,\n DEFAULT_SESSION_EXPIRATION\n ))\n )\n\n def generate_sid(self):\n return str(uuid4())\n\n def get_redis_expiration_time(self, app, session):\n return timedelta(minutes=10)\n\n def open_session(self, app, request):\n LOG.debug(\n 'redis session connection pool: in use: {} available: {}'.format(\n len(self.redis.connection_pool._in_use_connections),\n len(self.redis.connection_pool._available_connections)\n )\n )\n sid = request.cookies.get(app.session_cookie_name)\n if not sid:\n sid = self.generate_sid()\n return self.session_class(sid=sid, new=True)\n\n val = self.redis.get(self.prefix + sid)\n if val is not None:\n data = self.serializer.loads(val)\n return self.session_class(data, sid=sid)\n\n return self.session_class(sid=sid, new=True)\n\n def save_session(self, app, session, response):\n domain = self.get_cookie_domain(app)\n if 'user_id' not in session:\n self.redis.delete(self.prefix + session.sid)\n\n if session.modified:\n response.delete_cookie(\n app.session_cookie_name,\n domain=domain\n )\n return\n\n redis_exp = self.get_redis_expiration_time(app, session)\n cookie_exp = self.get_expiration_time(app, session)\n val = self.serializer.dumps(dict(session))\n self.redis.setex(\n self.prefix + session.sid,\n int(redis_exp.total_seconds()),\n val\n )\n\n response.set_cookie(\n app.session_cookie_name,\n session.sid,\n expires=cookie_exp,\n httponly=True,\n domain=domain\n )\n\n\ndef init_app(app, redis_url):\n redis_cp = redis.ConnectionPool.from_url(\n redis_url,\n max_connections=int(os.environ.get('REDIS_SESSIONS_MAX_CONNECTIONS', 20))\n )\n\n app.session_interface = RedisSessionInterface(\n redis_=redis.StrictRedis(connection_pool=redis_cp)\n )\n app.config.update(\n SESSION_COOKIE_NAME='mm-session',\n SESSION_COOKIE_SECURE=True\n )\n" }, { "path": "minemeld/flask/sns.py", "content": "import requests\nimport json\nimport os.path\nimport uuid\nimport minemeld\nfrom .logger import LOG\nfrom . import config\n\nTYPEHELLO = 'hello'\nTYPEMKWISH = 'mkwish'\nTYPESTATS = 'stats'\nSNS_API_URL = None\nSNS_ENABLED = None\nUUID_FILENAME = None\nSNS_OBJ = None\nSNS_AVAILABLE = False\n\n\nclass Sns(object):\n def __init__(self, path):\n self.api_url = SNS_API_URL\n self.filename = os.path.join(path, UUID_FILENAME)\n self.mm_version = minemeld.__version__\n self.init_ok = self._init_uuid()\n\n def get_status(self):\n return self.init_ok\n\n def _init_uuid(self):\n uuid_error = uuid.UUID(bytes='\\x01' * 16).hex\n # Test case 1: UUIDFILENAME file does not exist. We try to create a new uuid and store in the filesystem\n if not os.path.isfile(self.filename):\n self.uuid = uuid.uuid4().hex\n # If we've failed to send the one-in-a-lifetime hello we just don't store the uuid to try again next boot\n if self._hello_world():\n try:\n with open(self.filename, 'w') as f:\n f.write(self.uuid)\n LOG.debug('New uuid file created.')\n except Exception as e:\n # Let the caller know uuid was not saved meaning sns might not be ready\n LOG.exception('Something went wrong creating the uuid file: {}'.format(self.filename))\n return False\n LOG.debug('Instance uuid = {}'.format(self.uuid))\n LOG.debug('MineMeld cloud notification service is ready.')\n return True\n LOG.info('MineMeld cloud notification service is not available.')\n return False\n # Test case 2: UUIDFILENAME exists but we can't open it (permissions issues?)\n try:\n f = open(self.filename)\n except IOError:\n self.uuid = uuid_error\n LOG.exception('Failure opening the uuid file {}'.format(self.filename))\n return True\n r_uuid = f.readline().strip()\n f.close()\n # Test case 3: We can read UUIDFILENAME but the content is not a valid UUID4\n try:\n val = uuid.UUID(r_uuid, version=4)\n except ValueError:\n self.uuid = uuid_error\n LOG.info('Invalid uuid value in the file: {}'.format(r_uuid))\n return True\n self.uuid = val.hex if val.hex == r_uuid else uuid_error\n LOG.debug('Instance uuid = {}'.format(self.uuid))\n return True\n\n def _send_message(self, kvmessage):\n kvmessage['uuid'] = self.uuid\n kvmessage['version'] = self.mm_version\n try:\n r = requests.post(self.api_url,\n data=json.dumps(kvmessage),\n timeout=5,\n headers={'Content-Type': 'application/json'})\n except Exception as e:\n LOG.exception('Failure sending the message to the sns cloud provider')\n return False\n if r.status_code == requests.codes.ok:\n response = r.json()\n if response.get('response', '') == 'ok':\n return True\n return False\n\n def _hello_world(self):\n return self._send_message({'type': TYPEHELLO, 'message': 'Hello world!'})\n\n def make_wish(self, message):\n LOG.debug('Sending new wish message to SNS')\n return self._send_message({'type': TYPEMKWISH, 'message': message})\n\n def send_stats(self, stats):\n stats['type'] = TYPESTATS\n return self._send_message(stats)\n\n\ndef init_app():\n global SNS_OBJ\n global SNS_AVAILABLE\n global UUID_FILENAME\n global SNS_API_URL\n global SNS_ENABLED\n SNS_API_URL = config.get('SNS_URL', 'https://minemeld-notifications.panw.io/0.9/')\n SNS_ENABLED = config.get('SNS_ENABLED', False)\n UUID_FILENAME = config.get('UUID_FILE', 'uu.id4')\n if not SNS_ENABLED:\n return\n SNS_OBJ = Sns(config.API_CONFIG_PATH)\n SNS_AVAILABLE = SNS_OBJ.get_status()\n" }, { "path": "minemeld/flask/statusapi.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport os.path\nimport uuid\nimport functools\nimport time\nfrom zipfile import ZipFile\nfrom tempfile import NamedTemporaryFile, gettempdir\n\nimport gevent\nimport psutil\nimport yaml\nfrom flask import Response, stream_with_context, jsonify, request, send_file\n\nfrom . import config\nfrom .mmrpc import MMMaster\nfrom .mmrpc import MMRpcClient\nfrom .events import EventsGenerator\nfrom .redisclient import SR\nfrom .aaa import MMBlueprint, enable_prevent_write, disable_prevent_write\nfrom .logger import LOG\nfrom .jobs import JOBS_MANAGER\nfrom .utils import safe_remove, committed_config_path\nfrom .sns import SNS_OBJ, SNS_AVAILABLE\nfrom minemeld import __version__\n\n__all__ = ['BLUEPRINT']\n\nBLUEPRINT = MMBlueprint('status', __name__, url_prefix='/status')\n\n\nclass _PubSubWrapper(object):\n def __init__(self, subscription, pattern=False):\n self.subscription = subscription\n self.pattern = pattern\n\n self.pubsub = SR.pubsub(ignore_subscribe_messages=True)\n if pattern:\n self.pubsub.psubscribe(subscription)\n else:\n self.pubsub.subscribe(subscription)\n\n self.generator = self._msg_generator()\n\n def _listen(self):\n while self.pubsub.subscribed:\n response = self.pubsub.get_message(timeout=5.0)\n yield response\n\n def _msg_generator(self):\n yield 'data: ok\\n\\n'\n\n for message in self._listen():\n if message is None:\n yield 'data: ping\\n\\n'\n continue\n\n message = message['data']\n\n if message == '':\n break\n\n yield 'data: ' + message + '\\n\\n'\n\n yield 'data: { \"msg\": \"\" }\\n\\n'\n\n def __iter__(self):\n return self\n\n def next(self):\n return next(self.generator)\n\n def close(self):\n if self.pattern:\n self.pubsub.punsubscribe(self.subscription)\n else:\n self.pubsub.unsubscribe(self.subscription)\n\n self.pubsub.close()\n self.pubsub = None\n\n\n@BLUEPRINT.route('/events/query/', read_write=False)\ndef get_query_events(quuid):\n try:\n uuid.UUID(quuid)\n except ValueError:\n return jsonify(error={'message': 'Bad query uuid'}), 400\n\n swc_response = stream_with_context(\n _PubSubWrapper('mm-traced-q.' + quuid)\n )\n r = Response(swc_response, mimetype='text/event-stream')\n\n return r\n\n\n@BLUEPRINT.route('/events/status', read_write=False)\ndef get_status_events():\n swc_response = stream_with_context(EventsGenerator)\n r = Response(swc_response, mimetype='text/event-stream')\n\n return r\n\n\n@BLUEPRINT.route('/system', methods=['GET'], read_write=False)\ndef get_system_status():\n data_path = config.get('MINEMELD_LOCAL_PATH', None)\n if data_path is None:\n jsonify(error={'message': 'MINEMELD_LOCAL_PATH not set'}), 500\n\n res = {}\n res['cpu'] = psutil.cpu_percent(interval=1, percpu=True)\n res['memory'] = psutil.virtual_memory().percent\n res['swap'] = psutil.swap_memory().percent\n res['disk'] = psutil.disk_usage(data_path).percent\n res['sns'] = SNS_AVAILABLE\n\n return jsonify(result=res, timestamp=int(time.time() * 1000))\n\n\n@BLUEPRINT.route('/info', methods=['GET'], read_write=False)\ndef get_system_info():\n res = {}\n res['sns'] = SNS_AVAILABLE\n res['version'] = __version__\n\n return jsonify(result=res)\n\n\n@BLUEPRINT.route('/minemeld', methods=['GET'], read_write=False)\ndef get_minemeld_status():\n status = MMMaster.status()\n\n tr = status.get('result', None)\n if tr is None:\n return jsonify(error={'message': status.get('error', 'error')}), 400\n\n result = []\n for f, v in tr.iteritems():\n _, _, v['name'] = f.split(':', 2)\n result.append(v)\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/config', methods=['GET'], read_write=False)\ndef get_minemeld_running_config():\n rcpath = os.path.join(\n os.path.dirname(os.environ.get('MM_CONFIG')),\n 'running-config.yml'\n )\n with open(rcpath, 'r') as f:\n rcconfig = yaml.safe_load(f)\n\n return jsonify(result=rcconfig)\n\n\n# XXX this should be moved to a different endpoint\n@BLUEPRINT.route('//hup', methods=['GET', 'POST'], read_write=False)\ndef hup_node(nodename):\n status = MMMaster.status()\n tr = status.get('result', None)\n if tr is None:\n return jsonify(error={'message': status.get('error', 'error')}), 400\n\n nname = 'mbus:slave:' + nodename\n if nname not in tr:\n return jsonify(error={'message': 'Unknown node'}), 404\n\n MMRpcClient.send_cmd(nodename, 'hup', {'source': 'minemeld-web'})\n\n return jsonify(result='ok'), 200\n\n\n# XXX this should be moved to a different endpoint\n@BLUEPRINT.route('//signal/', methods=['GET', 'POST'], read_write=False)\ndef signal_node(nodename, signalname):\n status = MMMaster.status()\n tr = status.get('result', None)\n if tr is None:\n return jsonify(error={'message': status.get('error', 'error')}), 400\n\n nname = 'mbus:slave:' + nodename\n if nname not in tr:\n return jsonify(error={'message': 'Unknown node'}), 404\n\n params = request.get_json(silent=True)\n if params is None:\n params = {}\n\n params.update({\n 'source': 'minemeld-web',\n 'signal': signalname\n })\n\n MMRpcClient.send_cmd(\n target=nodename,\n method='signal',\n params=params\n )\n\n return jsonify(result='ok'), 200\n\n\ndef _clean_local_backup(local_backup_file, g):\n def _safe_remove(path):\n LOG.info('Removing backup {}'.format(local_backup_file))\n try:\n os.remove(path)\n except:\n pass\n\n if g.value != 0:\n _safe_remove(local_backup_file)\n return\n\n LOG.info('Removing backup {} in 300s'.format(local_backup_file))\n gevent.spawn_later(300, _safe_remove, local_backup_file)\n\n\n# XXX this should be moved to a different endpoint\n@BLUEPRINT.route('/backup', methods=['POST'], read_write=True)\ndef generate_local_backup():\n params = request.get_json(silent=True)\n if params is None:\n return jsonify(error={'message': 'missing request body'}), 400\n\n password = params.get('p', None)\n if password is None:\n return jsonify(error={'message': 'missing p paramater in request body'}), 400\n\n sevenz_path = config.get('MINEMELD_7Z_PATH', None)\n if sevenz_path is None:\n return jsonify(error={'message': 'MINEMELD_7Z_PATH not set'}), 500\n\n # create temp zip file\n tf = NamedTemporaryFile(prefix='mm-local-backup', suffix='.zip', delete=False)\n tf.close()\n # initialize the zip structure inside the file\n ZipFile(tf.name, 'w').close()\n\n # build args\n args = [sevenz_path, 'a', '-p{}'.format(password), '-y', tf.name]\n\n proto_path = config.get('MINEMELD_LOCAL_PROTOTYPE_PATH', None)\n if proto_path is not None:\n args.append(proto_path)\n certs_path = config.get('MINEMELD_LOCAL_CERTS_PATH', None)\n if certs_path is not None:\n args.append(certs_path)\n config_path = os.path.dirname(os.environ.get('MM_CONFIG'))\n args.append(config_path)\n\n jobs = JOBS_MANAGER.get_jobs(job_group='status-backup')\n for jobid, jobdata in jobs.iteritems():\n if jobdata == 'RUNNING':\n return jsonify(error={'message': 'a backup job is already running'}), 400\n\n jobid = JOBS_MANAGER.exec_job(\n job_group='status-backup',\n description='local backup',\n args=args,\n data={\n 'result-file': tf.name\n },\n callback=functools.partial(_clean_local_backup, tf.name)\n )\n\n return jsonify(result=jobid)\n\n\n# XXX this should be moved to a different endpoint\n@BLUEPRINT.route('/backup/', methods=['GET'], read_write=True)\ndef get_local_backup(jobid):\n jobs = JOBS_MANAGER.get_jobs(job_group='status-backup')\n\n if jobid not in jobs:\n return jsonify(error={'message': 'unknown job'}), 404\n\n return send_file(jobs[jobid]['result-file'])\n\n\n@BLUEPRINT.route('/backup/import', methods=['POST'], read_write=True)\ndef import_local_backup():\n if 'file' not in request.files:\n return jsonify(error={'messsage': 'No file in request'}), 400\n\n file = request.files['file']\n if file.filename == '':\n return jsonify(error={'message': 'No file'}), 400\n\n tf = NamedTemporaryFile(prefix='mm-import-backup', delete=False)\n try:\n file.save(tf)\n tf.close()\n\n with ZipFile(tf.name, 'r') as zf:\n contents = zf.namelist()\n\n except Exception, e:\n safe_remove(tf.name)\n raise e\n\n ibid = os.path.basename(tf.name)[16:]\n result = {\n 'id': ibid,\n 'configuration': 'config/committed-config.yml' in contents,\n 'localPrototypes': 'prototypes/minemeldlocal.yml' in contents,\n 'feedsAAA': True,\n 'localCertificates': False\n }\n\n # check for feeds AAA files\n testfile = os.path.join(\n 'config/api',\n config.APIConfigDict(attribute='FEEDS_USERS_ATTRS', level=50).filename\n )\n result['feedsAAA'] &= testfile in contents\n\n testfile = os.path.join(\n 'config/api',\n config.APIConfigDict(attribute='FEEDS_ATTRS', level=50).filename\n )\n result['feedsAAA'] &= testfile in contents\n\n testfile = os.path.join(\n 'config/api',\n 'feeds.htpasswd'\n )\n result['feedsAAA'] &= testfile in contents\n\n # check for local certificates, there should be at least one\n # to flag certs as available\n for fname in contents:\n if fname.startswith('certs/site/') and not fname.endswith('/'):\n result['localCertificates'] = True\n break\n\n if not (result['configuration'] or result['localPrototypes'] or result['feedsAAA']):\n safe_remove(tf.name)\n return jsonify(error={'message': 'Invalid MineMeld backup'}), 400\n\n return jsonify(result=result)\n\n\ndef _cleanup_after_restore(backup_file, locker, g):\n disable_prevent_write(locker)\n safe_remove(backup_file)\n\n\n@BLUEPRINT.route('/backup/import//restore', methods=['POST'], read_write=True)\ndef restore_local_backup(backup_id):\n restore_path = config.get('MINEMELD_RESTORE_PATH', None)\n if restore_path is None:\n return jsonify(error={'message': 'MINEMELD_RESTORE_PATH not set'}), 500\n\n params = request.get_json(silent=True)\n if params is None:\n return jsonify(error={'message': 'missing request body'}), 400\n\n password = params.get('p', None)\n restore_configuration = params.get('configuration', False)\n restore_prototypes = params.get('localPrototypes', False)\n restore_feeds_aaa = params.get('feedsAAA', False)\n restore_certificates = params.get('localCertificates', False)\n\n if not (restore_configuration or restore_prototypes or restore_feeds_aaa):\n return jsonify(error={'message': 'Nothing to do'}), 400\n\n backup_file = os.path.join(gettempdir(), 'mm-import-backup{}'.format(backup_id))\n if not os.path.samefile(os.path.dirname(backup_file), gettempdir()):\n return jsonify(error={'message': 'Invalid backup id'}), 400\n if not os.path.exists(backup_file):\n return jsonify(error={'message': 'Invalid backup id'}), 404\n\n locker = 'restore-backup-{}-{}'.format(backup_id, int(time.time()))\n enable_prevent_write(locker)\n try:\n jobs = JOBS_MANAGER.get_jobs(job_group='status-backup')\n for jobid, jobdata in jobs.iteritems():\n if jobdata == 'RUNNING':\n disable_prevent_write(locker)\n return jsonify(error={'message': 'a backup job is running'}), 400\n\n jobs = JOBS_MANAGER.get_jobs(job_group='restore-backup')\n for jobid, jobdata in jobs.iteritems():\n if jobdata == 'RUNNING':\n disable_prevent_write(locker)\n return jsonify(error={'message': 'a restore job is running'}), 400\n\n args = [restore_path]\n if restore_configuration:\n p = os.path.dirname(committed_config_path())\n args.extend(['--configuration-path', p])\n\n if restore_prototypes:\n p = config.get('MINEMELD_LOCAL_PROTOTYPE_PATH', None)\n if p is None:\n return jsonify(error={'message': 'MINEMELD_LOCAL_PROTOTYPE_PATH not set'}), 500\n\n args.extend(['--prototypes-path', p])\n\n if restore_feeds_aaa:\n args.extend([\n '--feeds-aaa-path',\n os.path.join(os.path.dirname(committed_config_path()), 'api')\n ])\n args.extend([\n '--feeds-aaa',\n 'feeds.htpasswd'\n ])\n args.extend([\n '--feeds-aaa',\n config.APIConfigDict(attribute='FEEDS_ATTRS', level=50).filename\n ])\n args.extend([\n '--feeds-aaa',\n config.APIConfigDict(attribute='FEEDS_USERS_ATTRS', level=50).filename\n ])\n\n if restore_certificates:\n p = config.get('MINEMELD_LOCAL_CERTS_PATH', None)\n if p is None:\n LOG.error('MINEMELD_LOCAL_CERTS_PATH not set, local certificates not restored')\n\n else:\n args.extend([\n '--certificates-path',\n p\n ])\n\n if password is not None:\n args.extend(['--password', password])\n\n args.append(backup_file)\n\n jobid = JOBS_MANAGER.exec_job(\n job_group='restore-backup',\n description='restore backup',\n args=args,\n callback=functools.partial(_cleanup_after_restore, backup_file, locker),\n timeout=200\n )\n\n except:\n disable_prevent_write(locker)\n raise\n\n return jsonify(result=jobid)\n\n\n@BLUEPRINT.route('/mkwish', methods=['POST'], read_write=False)\ndef sns_wish():\n request.get_data()\n message = request.data\n success = SNS_OBJ.make_wish(message)\n if success:\n return jsonify(result='ok')\n return jsonify(error={'messsage': 'Error sending the message'}), 400\n" }, { "path": "minemeld/flask/supervisorapi.py", "content": "# Copyright 2015-2017 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport time\nfrom signal import SIGHUP\n\nimport psutil\nimport gevent\nimport xmlrpclib\nimport supervisor.xmlrpc\n\nfrom flask import jsonify\n\nfrom . import config\nfrom .supervisorclient import MMSupervisor\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\n\n\n__all__ = ['BLUEPRINT']\n\n\nBLUEPRINT = MMBlueprint('supervisor', __name__, url_prefix='')\n\n\ndef _restart_engine():\n LOG.info('Restarting minemeld-engine')\n\n supervisorurl = config.get('SUPERVISOR_URL',\n 'unix:///var/run/supervisor.sock')\n sserver = xmlrpclib.ServerProxy(\n 'http://127.0.0.1',\n transport=supervisor.xmlrpc.SupervisorTransport(\n None,\n None,\n supervisorurl\n )\n )\n\n try:\n result = sserver.supervisor.stopProcess('minemeld-engine', False)\n if not result:\n LOG.error('Stop minemeld-engine returned False')\n return\n\n except xmlrpclib.Fault as e:\n LOG.error('Error stopping minemeld-engine: {!r}'.format(e))\n\n LOG.info('Stopped minemeld-engine for API request')\n\n now = time.time()\n info = None\n while (time.time()-now) < 60*10*1000:\n info = sserver.supervisor.getProcessInfo('minemeld-engine')\n if info['statename'] in ('FATAL', 'STOPPED', 'UNKNOWN', 'EXITED'):\n break\n gevent.sleep(5)\n else:\n LOG.error('Timeout during minemeld-engine restart')\n return\n\n sserver.supervisor.startProcess('minemeld-engine', False)\n LOG.info('Started minemeld-engine')\n\n\n@BLUEPRINT.route('/supervisor', methods=['GET'], read_write=False)\ndef service_status():\n try:\n supervisorstate = MMSupervisor.supervisor.getState()\n\n except:\n LOG.exception(\"Exception connecting to supervisor\")\n return jsonify(result={'statename': 'STOPPED'})\n\n supervisorstate['processes'] = {}\n pinfo = MMSupervisor.supervisor.getAllProcessInfo()\n for p in pinfo:\n process = {\n 'statename': p['statename'],\n 'start': p['start'],\n 'children': None\n }\n\n try:\n ps = psutil.Process(pid=p['pid'])\n process['children'] = len(ps.children())\n\n except:\n LOG.exception(\"Error retrieving childen of %d\" % p['pid'])\n\n supervisorstate['processes'][p['name']] = process\n\n return jsonify(result=supervisorstate)\n\n\n@BLUEPRINT.route('/supervisor/minemeld-engine/start', methods=['GET'], read_write=True)\ndef start_minemeld_engine():\n result = MMSupervisor.supervisor.startProcess('minemeld-engine', False)\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/supervisor/minemeld-engine/stop', methods=['GET'], read_write=True)\ndef stop_minemeld_engine():\n result = MMSupervisor.supervisor.stopProcess('minemeld-engine', False)\n\n return jsonify(result=result)\n\n\n@BLUEPRINT.route('/supervisor/minemeld-engine/restart', methods=['GET'], read_write=True)\ndef restart_minemeld_engine():\n info = MMSupervisor.supervisor.getProcessInfo('minemeld-engine')\n if info['statename'] == 'STARTING' or info['statename'] == 'STOPPING':\n return jsonify(error={\n 'message': ('minemeld-engine not in RUNNING state: %s' %\n info['statename'])\n }), 400\n\n gevent.spawn(_restart_engine)\n\n return jsonify(result='OK')\n\n\n@BLUEPRINT.route('/supervisor/minemeld-web/hup', methods=['GET'], read_write=True)\ndef hup_minemeld_web():\n info = MMSupervisor.supervisor.getProcessInfo('minemeld-web')\n\n apipid = info['pid']\n os.kill(apipid, SIGHUP)\n\n return jsonify(result='OK')\n" }, { "path": "minemeld/flask/supervisorclient.py", "content": "from flask import g\n\nimport psutil # noqa\nimport xmlrpclib\nimport supervisor.xmlrpc\nimport werkzeug.local\n\nfrom . import config\n\n\n__all__ = ['init_app', 'MMSupervisor']\n\n\ndef get_Supervisor():\n sserver = getattr(g, '_supervisor', None)\n if sserver is None:\n supervisorurl = config.get('SUPERVISOR_URL',\n 'unix:///var/run/supervisor.sock')\n sserver = xmlrpclib.ServerProxy(\n 'http://127.0.0.1',\n transport=supervisor.xmlrpc.SupervisorTransport(\n None,\n None,\n supervisorurl\n )\n )\n g._supervisor = sserver\n\n return sserver\n\n\nMMSupervisor = werkzeug.local.LocalProxy(get_Supervisor)\n\n\ndef teardown(exception):\n SR = getattr(g, '_supervisor', None)\n if SR is not None:\n g._supervisor = None\n\n\ndef init_app(app):\n app.teardown_appcontext(teardown)\n" }, { "path": "minemeld/flask/taxiicollmgmt.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport re\n\nimport libtaxii\nimport libtaxii.messages_11\nimport libtaxii.constants\n\nfrom flask import request\nfrom flask.ext.login import current_user\n\nfrom . import config\nfrom .taxiiutils import taxii_check, taxii_make_response, get_taxii_feeds\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\n\n\n__all__ = ['BLUEPRINT']\n\n\nHOST_RE = re.compile('^[a-zA-Z\\d-]{1,63}(?:\\.[a-zA-Z\\d-]{1,63})*(?::[0-9]{1,5})*$')\n\nBLUEPRINT = MMBlueprint('taxiicollmgmt', __name__, url_prefix='')\n\n\n@BLUEPRINT.route('/taxii-collection-management-service', methods=['POST'], feeds=True, read_write=False)\n@taxii_check\ndef taxii_collection_mgmt_service():\n taxii_feeds = get_taxii_feeds()\n authorized_feeds = filter(\n current_user.check_feed,\n taxii_feeds\n )\n if len(authorized_feeds) == 0:\n return 'Unauthorized', 401\n\n server_host = config.get('TAXII_HOST', None)\n if server_host is None:\n server_host = request.headers.get('Host', None)\n if server_host is None:\n return 'Missing Host header', 400\n\n if HOST_RE.match(server_host) is None:\n return 'Invalid Host header', 400\n\n tm = libtaxii.messages_11.get_message_from_xml(request.data)\n if tm.message_type != \\\n libtaxii.constants.MSG_COLLECTION_INFORMATION_REQUEST:\n return 'Invalid message, invalid Message Type', 400\n\n cir = libtaxii.messages_11.CollectionInformationResponse(\n libtaxii.messages_11.generate_message_id(),\n tm.message_id\n )\n\n for feed in authorized_feeds:\n cii = libtaxii.messages_11.CollectionInformation(\n feed,\n '{} Data Feed'.format(feed),\n ['urn:stix.mitre.org:xml:1.1.1'],\n True\n )\n si = libtaxii.messages_11.PollingServiceInstance(\n 'urn:taxii.mitre.org:protocol:http:1.0',\n 'https://{}/taxii-poll-service'.format(server_host),\n ['urn:taxii.mitre.org:message:xml:1.1']\n )\n cii.polling_service_instances.append(si)\n cir.collection_informations.append(cii)\n\n return taxii_make_response(cir)\n" }, { "path": "minemeld/flask/taxiidiscovery.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport re\n\nimport libtaxii\nimport libtaxii.messages_11\nimport libtaxii.constants\n\nfrom flask import request\nfrom flask.ext.login import current_user\n\nfrom . import config\nfrom .taxiiutils import get_taxii_feeds, taxii_check, taxii_make_response\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\n\n\n__all__ = ['BLUEPRINT']\n\n\nBLUEPRINT = MMBlueprint('taxiidiscovery', __name__, url_prefix='')\n\nHOST_RE = re.compile('^[a-zA-Z\\d-]{1,63}(?:\\.[a-zA-Z\\d-]{1,63})*(?::[0-9]{1,5})*$')\n\n_SERVICE_INSTANCES = [\n {\n 'type': libtaxii.constants.SVC_DISCOVERY,\n 'path': 'taxii-discovery-service'\n },\n {\n 'type': libtaxii.constants.SVC_COLLECTION_MANAGEMENT,\n 'path': 'taxii-collection-management-service'\n },\n {\n 'type': libtaxii.constants.SVC_POLL,\n 'path': 'taxii-poll-service'\n }\n]\n\n\n@BLUEPRINT.route('/taxii-discovery-service', methods=['POST'], feeds=True, read_write=False)\n@taxii_check\ndef taxii_discovery_service():\n taxii_feeds = get_taxii_feeds()\n authorized = next(\n (tf for tf in taxii_feeds if current_user.check_feed(tf)),\n None\n )\n if authorized is None:\n return 'Unauthorized', 401\n\n server_host = config.get('TAXII_HOST', None)\n if server_host is None:\n server_host = request.headers.get('Host', None)\n if server_host is None:\n return 'Missing Host header', 400\n\n if HOST_RE.match(server_host) is None:\n return 'Invalid Host header', 400\n\n tm = libtaxii.messages_11.get_message_from_xml(request.data)\n if tm.message_type != libtaxii.constants.MSG_DISCOVERY_REQUEST:\n return 'Invalid message, invalid Message Type', 400\n\n dresp = libtaxii.messages_11.DiscoveryResponse(\n libtaxii.messages_11.generate_message_id(),\n tm.message_id\n )\n\n for si in _SERVICE_INSTANCES:\n sii = libtaxii.messages_11.ServiceInstance(\n si['type'],\n 'urn:taxii.mitre.org:services:1.1',\n 'urn:taxii.mitre.org:protocol:http:1.0',\n \"https://{}/{}\".format(server_host, si['path']),\n ['urn:taxii.mitre.org:message:xml:1.1'],\n available=True\n )\n dresp.service_instances.append(sii)\n\n return taxii_make_response(dresp)\n" }, { "path": "minemeld/flask/taxiipoll.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport datetime\n\nimport pytz\nimport lz4.frame\n\nimport libtaxii\nimport libtaxii.constants\nimport stix.core\n\nfrom flask import request, Response, stream_with_context\nfrom flask.ext.login import current_user\n\nfrom .redisclient import SR\nfrom .taxiiutils import taxii_check, get_taxii_feeds\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\nfrom minemeld.ft.utils import dt_to_millisec\n\n\n__all__ = ['BLUEPRINT']\n\n\nBLUEPRINT = MMBlueprint('taxiipoll', __name__, url_prefix='')\n\n\n_TAXII_POLL_RESPONSE_HEADER = \"\"\"\n\n%(inclusive_end_timestamp_label)s\n\"\"\"\n\n\ndef _oldest_indicator_timestamp(feed):\n olist = SR.zrevrange(\n feed, 0, 0,\n withscores=True\n )\n if len(olist) == 0:\n return None\n\n ots = int(olist[0][1])/1000\n\n return datetime.datetime.fromtimestamp(ots, pytz.utc)\n\n\ndef _indicators_feed(feed, excbegtime, incendtime):\n if excbegtime is None:\n excbegtime = 0\n else:\n excbegtime = dt_to_millisec(excbegtime) + 1\n incendtime = dt_to_millisec(incendtime)\n\n cstart = 0\n while True:\n indicators = SR.zrangebyscore(\n feed, excbegtime, incendtime,\n start=cstart, num=100\n )\n if indicators is None:\n break\n\n for i in indicators:\n value = SR.hget(feed + '.value', i)\n\n if value.startswith('lz4'):\n try:\n value = lz4.frame.decompress(value[3:])\n value = stix.core.STIXPackage.from_json(value)\n value = value.to_xml(\n ns_dict={'https://go.paloaltonetworks.com/minemeld': 'minemeld'}\n )\n\n except ValueError:\n continue\n\n yield value\n\n if len(indicators) < 100:\n break\n\n cstart += 100\n\n\ndef data_feed_11(rmsgid, cname, excbegtime, incendtime):\n tfeeds = get_taxii_feeds()\n if cname not in tfeeds:\n return 'Invalid message, unknown feed', 400\n\n if not incendtime:\n incendtime = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)\n\n def _resp_generator():\n # yield the opening tag of the Poll Response\n resp_header = _TAXII_POLL_RESPONSE_HEADER % {\n 'collection_name': cname,\n 'message_id': libtaxii.messages_11.generate_message_id(),\n 'in_response_to': rmsgid,\n 'inclusive_end_timestamp_label': incendtime.isoformat()\n }\n if excbegtime is not None:\n resp_header += (\n '' +\n excbegtime.isoformat() +\n ''\n )\n\n yield resp_header\n\n # yield the content blocks\n for i in _indicators_feed(cname, excbegtime, incendtime):\n cb1 = libtaxii.messages_11.ContentBlock(\n content_binding=libtaxii.constants.CB_STIX_XML_11,\n content=i\n )\n yield cb1.to_xml()+'\\n'\n\n # yield the closing tag\n yield ''\n\n return Response(\n response=stream_with_context(_resp_generator()),\n status=200,\n headers={\n 'X-TAXII-Content-Type': 'urn:taxii.mitre.org:message:xml:1.1',\n 'X-TAXII-Protocol': 'urn:taxii.mitre.org:protocol:http:1.0'\n },\n mimetype='application/xml'\n )\n\n\n@BLUEPRINT.route('/taxii-poll-service', methods=['POST'], feeds=True, read_write=False)\n@taxii_check\ndef taxii_poll_service():\n taxiict = request.headers['X-TAXII-Content-Type']\n if taxiict == 'urn:taxii.mitre.org:message:xml:1.1':\n tm = libtaxii.messages_11.get_message_from_xml(request.data)\n if tm.message_type != libtaxii.constants.MSG_POLL_REQUEST:\n return 'Invalid message', 400\n\n cname = tm.collection_name\n excbegtime = tm.exclusive_begin_timestamp_label\n incendtime = tm.inclusive_end_timestamp_label\n\n if not current_user.check_feed(cname):\n return 'Unauthorized', 401\n\n return data_feed_11(tm.message_id, cname, excbegtime, incendtime)\n\n elif taxiict == 'urn:taxii.mitre.org:message:xml:1.0':\n # old TAXII 1.0 not supported yet\n return 'Invalid message', 400\n\n else:\n return 'Invalid message', 400\n" }, { "path": "minemeld/flask/taxiiutils.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport functools\n\nfrom flask import request\nfrom flask import make_response\n\nfrom .mmrpc import MMMaster\nfrom .logger import LOG\n\n\ndef taxii_make_response(m11):\n h = {\n 'Content-Type': \"application/xml\",\n 'X-TAXII-Content-Type': 'urn:taxii.mitre.org:message:xml:1.1',\n 'X-TAXII-Protocol': 'urn:taxii.mitre.org:protocol:http:1.0'\n }\n r = make_response((m11.to_xml(pretty_print=True), 200, h))\n return r\n\n\ndef taxii_make_response_10(m10):\n h = {\n 'Content-Type': \"application/xml\",\n 'X-TAXII-Content-Type': 'urn:taxii.mitre.org:message:xml:1.0',\n 'X-TAXII-Protocol': 'urn:taxii.mitre.org:protocol:http:1.0'\n }\n r = make_response((m10.to_xml(pretty_print=True), 200, h))\n return r\n\n\ndef taxii_check(f):\n @functools.wraps(f)\n def check(*args, **kwargs):\n tct = request.headers.get('X-TAXII-Content-Type', None)\n if tct not in [\n 'urn:taxii.mitre.org:message:xml:1.1',\n 'urn:taxii.mitre.org:message:xml:1.0'\n ]:\n return 'Invalid TAXII Headers', 400\n tct = request.headers.get('X-TAXII-Protocol', None)\n if tct not in [\n 'urn:taxii.mitre.org:protocol:http:1.0',\n 'urn:taxii.mitre.org:protocol:https:1.0'\n ]:\n return 'Invalid TAXII Headers', 400\n tct = request.headers.get('X-TAXII-Services', None)\n if tct not in [\n 'urn:taxii.mitre.org:services:1.1',\n 'urn:taxii.mitre.org:services:1.0'\n ]:\n return 'Invalid TAXII Headers', 400\n return f(*args, **kwargs)\n return check\n\n\ndef get_taxii_feeds():\n # check if feed exists\n status = MMMaster.status()\n status = status.get('result', None)\n if status is None:\n raise RuntimeError('Error retrieving engine status')\n\n result = []\n for node, node_status in status.iteritems():\n class_ = node_status.get('class', None)\n if class_ != 'minemeld.ft.taxii.DataFeed':\n continue\n\n _, _, feedname = node.split(':', 2)\n result.append(feedname)\n\n return result\n" }, { "path": "minemeld/flask/tracedapi.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport uuid\n\nfrom flask import request, jsonify\n\nfrom . import config\nfrom .mmrpc import MMRpcClient\nfrom .jobs import JOBS_MANAGER\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\n\nimport minemeld.traced\n\n\n__all__ = ['BLUEPRINT']\n\n\nBLUEPRINT = MMBlueprint('traced', __name__, url_prefix='/traced')\n\n\n@BLUEPRINT.route('/query', read_write=False)\ndef traced_query():\n query_uuid = request.args.get('uuid', None)\n if query_uuid is None:\n return jsonify(error={'message': 'query UUID missing'}), 400\n try:\n uuid.UUID(query_uuid)\n except ValueError:\n return jsonify(error={'message': 'invalid query UUID'}), 400\n\n timestamp = request.args.get('ts', None)\n if timestamp is not None:\n try:\n timestamp = int(timestamp)\n except ValueError:\n return jsonify(error={'message': 'invalid timestamp'}), 400\n\n counter = request.args.get('c', None)\n if counter is not None:\n try:\n counter = int(counter)\n except ValueError:\n return jsonify(error={'message': 'invalid counter'}), 400\n\n num_lines = request.args.get('nl', None)\n if num_lines is not None:\n try:\n num_lines = int(num_lines)\n except ValueError:\n return jsonify(error={'message': 'invalid num_lines'}), 400\n\n query = request.args.get('q', \"\")\n\n result = MMRpcClient.send_raw_cmd(minemeld.traced.QUERY_QUEUE, 'query', {\n 'uuid': query_uuid,\n 'timestamp': timestamp,\n 'counter': counter,\n 'num_lines': num_lines,\n 'query': query\n })\n\n return jsonify(result=result), 200\n\n\n@BLUEPRINT.route('/query//kill', read_write=False)\ndef traced_kill_query(query_uuid):\n try:\n uuid.UUID(query_uuid)\n except ValueError:\n return jsonify(error={'message': 'invalid query UUID'}), 400\n\n result = MMRpcClient.send_raw_cmd(minemeld.traced.QUERY_QUEUE, 'kill_query', {\n 'uuid': query_uuid\n })\n\n return jsonify(result=result), 200\n\n\n@BLUEPRINT.route('/purge-all', read_write=True)\ndef traced_purge_all():\n traced_purge_path = config.get('MINEMELD_TRACED_PURGE_PATH', None)\n if traced_purge_path is None:\n return jsonify(error={'message': 'MINEMELD_TRACED_PURGE_PATH not set'}), 500\n\n jobs = JOBS_MANAGER.get_jobs(job_group='traced-purge')\n for jobid, jobdata in jobs.iteritems():\n if jobdata == 'RUNNING':\n return jsonify(error={'message': 'a trace purge job is already running'}), 400\n\n jobid = JOBS_MANAGER.exec_job(\n job_group='traced-purge',\n description='purge all traces',\n args=[traced_purge_path, '--all'],\n data={}\n )\n\n return jsonify(result=jobid)\n" }, { "path": "minemeld/flask/utils.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport re\n\nimport yaml\n\nfrom .logger import LOG\n\n\nclass DirSnapshot(object):\n def __init__(self, path, regex=None):\n self._entries = self._init_snapshot(path, regex)\n\n def _init_snapshot(self, path, regex):\n result = set()\n\n files = os.listdir(path)\n pattern = re.compile(regex) if regex is not None else None\n for f in files:\n if pattern is not None:\n if pattern.match(f) is None:\n continue\n\n mtime = os.stat(os.path.join(path, f)).st_mtime\n result.add('%s_%d' % (f, int(mtime)))\n\n return result\n\n def __eq__(self, other):\n return self._entries == other._entries\n\n def __ne__(self, other):\n return self._entries != other._entries\n\n\ndef running_config_path():\n rcpath = os.path.join(\n os.path.dirname(os.environ.get('MM_CONFIG')),\n 'running-config.yml'\n )\n\n return rcpath\n\n\ndef committed_config_path():\n ccpath = os.path.join(\n os.path.dirname(os.environ.get('MM_CONFIG')),\n 'committed-config.yml'\n )\n\n return ccpath\n\n\ndef running_config():\n with open(running_config_path(), 'r') as f:\n rcconfig = yaml.safe_load(f)\n\n return rcconfig\n\n\ndef committed_config():\n with open(committed_config_path(), 'r') as f:\n ccconfig = yaml.safe_load(f)\n\n return ccconfig\n\n\ndef safe_remove(path):\n try:\n os.remove(path)\n except:\n LOG.exception('Exception removing {}'.format(path))\n" }, { "path": "minemeld/flask/validateapi.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport yaml\n\nfrom flask import jsonify, request\n\nimport minemeld.ft.condition\nfrom .aaa import MMBlueprint\nfrom .logger import LOG\n\n\n__all__ = ['BLUEPRINT']\n\n\nBLUEPRINT = MMBlueprint('validate', __name__, url_prefix='/validate')\n\n\ndef _return_validation_error(msg):\n return jsonify(error={\n 'message': msg\n }), 400\n\n\n@BLUEPRINT.route('/syslogminerrule', methods=['POST'], read_write=False)\ndef validate_syslogminerrule():\n try:\n crule = request.data\n\n except Exception as e:\n return _return_validation_error(\n 'Error accessing request body: %s' % str(e)\n )\n\n try:\n crule = yaml.safe_load(crule)\n\n except Exception as e:\n return _return_validation_error(\n 'YAML not valid: %s' % str(e)\n )\n\n if 'name' not in crule:\n return _return_validation_error('\"name\" is required')\n\n conditions = crule.get('conditions', None)\n if conditions is None or len(conditions) == 0:\n return _return_validation_error(\n 'no \"conditions\" in rule'\n )\n\n for c in conditions:\n try:\n minemeld.ft.condition.Condition(c)\n except Exception as e:\n return _return_validation_error(\n 'Condition %s is not valid' % c\n )\n\n indicators = crule.get('indicators', None)\n if type(indicators) != list:\n return _return_validation_error(\n 'no \"indicators\" in rule'\n )\n\n for i in indicators:\n if type(i) != str:\n return _return_validation_error(\n 'wrong indicator format: %s' % i\n )\n\n return jsonify(result='ok')\n" }, { "path": "minemeld/ft/__init__.py", "content": "from minemeld.loader import load, MM_NODES_ENTRYPOINT\n\n\ndef factory(classname, name, chassis, config):\n node_class = load(MM_NODES_ENTRYPOINT, classname)\n\n return node_class(\n name=name,\n chassis=chassis,\n config=config\n )\n\n\nclass ft_states(object):\n READY = 0\n CONNECTED = 1\n REBUILDING = 2\n RESET = 3\n INIT = 4\n STARTED = 5\n CHECKPOINT = 6\n IDLE = 7\n STOPPED = 8\n" }, { "path": "minemeld/ft/actorbase.py", "content": "import logging\nfrom collections import namedtuple\n\nimport gevent\nfrom gevent.queue import Queue\n\nfrom minemeld.ft.base import BaseFT, _counting\n\n\nLOG = logging.getLogger(__name__)\n\nActorCommand = namedtuple('ActorCommand', ['command', 'kwargs_'])\n\n\nclass ActorBaseFT(BaseFT):\n def __init__(self, *args, **kwargs):\n super(ActorBaseFT, self).__init__(*args, **kwargs)\n\n self._actor_queue = Queue(maxsize=1)\n self._actor_glet = None\n\n @_counting('rebuild.queued')\n def command_rebuild(self):\n pass\n\n @_counting('checkpoint.queued')\n def checkpoint(self, **kwargs):\n self._actor_queue.put(ActorCommand(command='checkpoint', kwargs_=kwargs))\n\n @_counting('update.queued')\n def update(self, **kwargs):\n self._actor_queue.put(ActorCommand(command='update', kwargs_=kwargs))\n\n @_counting('withdraw.queued')\n def withdraw(self, **kwargs):\n self._actor_queue.put(ActorCommand(command='withdraw', kwargs_=kwargs))\n\n def _actor_loop(self):\n while True:\n acommand = self._actor_queue.get()\n\n if acommand.command == 'checkpoint':\n method = super(ActorBaseFT, self).checkpoint\n elif acommand.command == 'update':\n method = super(ActorBaseFT, self).update\n elif acommand.command == 'withdraw':\n method = super(ActorBaseFT, self).withdraw\n elif acommand.command == 'rebuild':\n method = self._rebuild\n else:\n LOG.error('{} - unknown command {}'.format(self.name, acommand.command))\n\n try:\n method(**acommand.kwargs_)\n except gevent.GreenletExit:\n break\n except:\n LOG.exception('{} - error executing {!r}'.format(self.name, acommand))\n\n def start(self):\n super(ActorBaseFT, self).start()\n\n if self._actor_glet is not None:\n return\n\n self._actor_glet = gevent.spawn(self._actor_loop)\n\n def stop(self):\n super(ActorBaseFT, self).stop()\n\n if self._actor_glet is None:\n return\n\n self._actor_glet.kill()\n self._actor_glet = None\n" }, { "path": "minemeld/ft/anomali.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements minemeld.ft.anomali.Intelligence, the Miner node for\nAnomali Intelligence API.\n\"\"\"\n\nimport os\nimport yaml\nimport netaddr\nimport pytz\nimport datetime\nimport requests\nimport logging\n\nfrom . import basepoller\nfrom .utils import interval_in_sec, dt_to_millisec\n\nLOG = logging.getLogger(__name__)\n\n\n_API_BASE = 'https://api.threatstream.com'\n_API_ENDPOINT = '/api/v2/intelligence/'\n\n\nclass Intelligence(basepoller.BasePollerFT):\n def __init__(self, name, chassis, config):\n super(Intelligence, self).__init__(name, chassis, config)\n\n self.last_run = None\n\n def configure(self):\n super(Intelligence, self).configure()\n\n self.url = self.config.get('url', None)\n self.polling_timeout = self.config.get('polling_timeout', 20)\n self.verify_cert = self.config.get('verify_cert', True)\n\n self.prefix = self.config.get('prefix', 'anomali')\n self.fields = self.config.get('fields', None)\n self.query = self.config.get('query', None)\n initial_interval = self.config.get('initial_interval', '3600')\n self.initial_interval = interval_in_sec(initial_interval)\n if self.initial_interval is None:\n LOG.error(\n '%s - wrong initial_interval format: %s',\n self.name, initial_interval\n )\n self.initial_interval = 3600\n\n self.api_key = None\n self.username = None\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.api_key = sconfig.get('api_key', None)\n if self.api_key is not None:\n LOG.info('%s - API Key set', self.name)\n\n self.username = sconfig.get('username', None)\n if self.username is not None:\n LOG.info('%s - username set', self.name)\n\n def _calc_age_out(self, indicator, attributes):\n etsattribute = self.prefix+'_expiration_ts'\n if etsattribute in attributes:\n original_ets = attributes[etsattribute]\n LOG.debug('%s - original_ets: %s', self.name, original_ets)\n original_ets = original_ets[:19]\n\n ets = datetime.datetime.strptime(\n original_ets,\n '%Y-%m-%dT%H:%M:%S'\n ).replace(tzinfo=pytz.UTC)\n LOG.debug('%s - expiration_ts set for %s', self.name, indicator)\n return dt_to_millisec(ets)\n\n return super(Intelligence, self)._calc_age_out(indicator, attributes)\n\n def _process_item(self, item):\n if 'value' not in item:\n LOG.debug('%s - value not in %s', self.name, item)\n return [[None, None]]\n\n indicator = item['value']\n if not (isinstance(indicator, str) or\n isinstance(indicator, unicode)):\n LOG.error(\n '%s - Wrong indicator type: %s - %s',\n self.name, indicator, type(indicator)\n )\n return [[None, None]]\n\n fields = self.fields\n if fields is None:\n fields = item.keys()\n fields.remove('value')\n\n attributes = {}\n for field in fields:\n if field not in item:\n continue\n attributes['%s_%s' % (self.prefix, field)] = item[field]\n\n if 'confidence' in item:\n attributes['confidence'] = item['confidence']\n\n if item['type'] == 'domain':\n attributes['type'] = 'domain'\n\n elif item['type'] == 'url':\n attributes['type'] = 'URL'\n\n elif item['type'] == 'ip':\n try:\n n = netaddr.IPNetwork(indicator)\n except:\n LOG.error('%s - Invald IP address: %s', self.name, indicator)\n return [[None, None]]\n\n if n.version == 4:\n attributes['type'] = 'IPv4'\n elif n.version == 6:\n attributes['type'] = 'IPv6'\n else:\n LOG.error('%s - Unknown ip version: %d', self.name, n.version)\n return [[None, None]]\n\n else:\n LOG.info(\n '%s - indicator type %s not supported',\n self.name,\n item['type']\n )\n return [[None, None]]\n\n return [[indicator, attributes]]\n\n def _build_iterator(self, now):\n if self.api_key is None or self.username is None:\n raise RuntimeError('%s - credentials not set' % self.name)\n\n if self.last_run is None:\n now = datetime.datetime.fromtimestamp(now/1000.0, pytz.UTC)\n dtinterval = datetime.timedelta(seconds=self.initial_interval)\n origin = now - dtinterval\n else:\n origin = datetime.datetime.fromtimestamp(\n self.last_run/1000.0,\n pytz.UTC\n )\n\n q = '(modified_ts>=%s)' % origin.strftime('%Y-%m-%dT%H:%M:%S')\n if self.query:\n q = '(%s AND %s)' % (q, self.query)\n\n params = dict(\n username=self.username,\n api_key=self.api_key,\n limit=100,\n q=q\n )\n LOG.debug('%s - query params: %s', self.name, params)\n\n rkwargs = dict(\n stream=True,\n verify=self.verify_cert,\n timeout=self.polling_timeout,\n params=params\n )\n\n r = requests.get(\n _API_BASE+_API_ENDPOINT,\n **rkwargs\n )\n\n while True:\n try:\n r.raise_for_status()\n except:\n LOG.error(\n '%s - exception in request: %s %s',\n self.name, r.status_code, r.content\n )\n raise\n\n cjson = r.json()\n if 'objects' not in cjson:\n LOG.error('%s - no objects in response', self.name)\n return\n\n objects = cjson['objects']\n for o in objects:\n yield o\n\n if 'meta' not in cjson:\n return\n\n if 'next' not in cjson['meta']:\n return\n\n next_url = cjson['meta']['next']\n if next_url is None:\n return\n\n LOG.debug('%s - requesting next items', self.name)\n rkwargs.pop('params', None)\n r = requests.get(\n _API_BASE+cjson['meta']['next'],\n **rkwargs\n )\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(Intelligence, self).hup(source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n" }, { "path": "minemeld/ft/auscert.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport requests\nimport logging\nimport itertools\nimport os\nimport yaml\n\nfrom . import http\n\nLOG = logging.getLogger(__name__)\n\n\nclass MaliciousURLFeed(http.HttpFT):\n def configure(self):\n super(MaliciousURLFeed, self).configure()\n\n self.api_key = None\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.api_key = sconfig.get('api_key', None)\n if self.api_key is not None:\n LOG.info('%s - api_key set', self.name)\n\n def _build_iterator(self, now):\n if self.api_key is None:\n raise RuntimeError(\n '{} - API Key not set, '\n 'poll not performed'.format(self.name)\n )\n\n rkwargs = dict(\n stream=True,\n verify=self.verify_cert,\n timeout=self.polling_timeout\n )\n \n rkwargs[\"headers\"] = { \n 'API-Key': self.api_key \n }\n\n session = requests.Session()\n\n r = session.get(\n self.url,\n **rkwargs\n )\n\n # if api_key is wrong we'll get a 403 response code\n if r.status_code == 403: \n raise RuntimeError(\n '{} - not authorized (Invalid API Key?)'.format(self.name)\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.debug('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n result = r.iter_lines()\n if self.ignore_regex is not None:\n result = itertools.ifilter(\n lambda x: self.ignore_regex.match(x) is None,\n result\n )\n\n return result\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(MaliciousURLFeed, self).hup(source)\n" }, { "path": "minemeld/ft/autofocus.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport os\nimport yaml\nimport netaddr\nimport netaddr.core\nimport pan.afapi\nimport ujson\nimport re\n\nfrom . import basepoller\n\nLOG = logging.getLogger(__name__)\n\nDOMAIN_RE = re.compile('^[a-zA-Z\\d-]{,63}(\\.[a-zA-Z\\d-]{,63})*$')\n\n\nclass ExportList(basepoller.BasePollerFT):\n def configure(self):\n super(ExportList, self).configure()\n\n self.api_key = None\n self.label = None\n\n self.hostname = self.config.get('autofocus_hostname', None)\n self.verify_cert = self.config.get('verify_cert', None)\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.api_key = sconfig.get('api_key', None)\n if self.api_key is not None:\n LOG.info('%s - api key set', self.name)\n\n self.label = sconfig.get('label', None)\n\n def _process_item(self, row):\n indicator = row\n\n result = {}\n result['type'] = self._type_of_indicator(indicator)\n result['autofocus_label'] = self.label\n\n return [[indicator, result]]\n\n def _check_for_ip(self, indicator):\n if '-' in indicator:\n # check for address range\n a1, a2 = indicator.split('-', 1)\n\n try:\n a1 = netaddr.IPAddress(a1)\n a2 = netaddr.IPAddress(a2)\n\n if a1.version == a2.version:\n if a1.version == 6:\n return 'IPv6'\n if a1.version == 4:\n return 'IPv4'\n\n except:\n return None\n\n return None\n\n if '/' in indicator:\n # check for network\n try:\n ip = netaddr.IPNetwork(indicator)\n\n except:\n return None\n\n if ip.version == 4:\n return 'IPv4'\n if ip.version == 6:\n return 'IPv6'\n\n return None\n\n try:\n ip = netaddr.IPAddress(indicator)\n except:\n return None\n\n if ip.version == 4:\n return 'IPv4'\n if ip.version == 6:\n return 'IPv6'\n\n return None\n\n def _type_of_indicator(self, indicator):\n ipversion = self._check_for_ip(indicator)\n if ipversion is not None:\n return ipversion\n\n if DOMAIN_RE.match(indicator):\n return 'domain'\n\n return 'URL'\n\n def _build_iterator(self, now):\n if self.api_key is None or self.label is None:\n raise RuntimeError(\n '%s - api_key or label not set, poll not performed' % self.name\n )\n\n body = {\n 'label': self.label,\n 'panosFormatted': True\n }\n\n af = pan.afapi.PanAFapi(\n hostname=self.hostname,\n verify_cert=self.verify_cert,\n api_key=self.api_key\n )\n\n r = af.export(data=ujson.dumps(body))\n r.raise_for_status()\n\n return r.json.get('export_list', [])\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(ExportList, self).hup(source=source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n" }, { "path": "minemeld/ft/azure.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport itertools\nimport functools\nfrom collections import defaultdict\n\nimport requests\nimport netaddr\nimport lxml.etree\nimport bs4\n\nfrom . import basepoller\n\nLOG = logging.getLogger(__name__)\n\nAZUREXML_URL = \\\n 'https://www.microsoft.com/EN-US/DOWNLOAD/confirmation.aspx?id=41653'\n\nAZURE_CLOUD_TO_URL = {\n 'public': 'https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519',\n 'usgov': 'http://www.microsoft.com/en-us/download/confirmation.aspx?id=57063',\n 'china': 'https://www.microsoft.com/en-us/download/confirmation.aspx?id=57062',\n 'germany': 'https://www.microsoft.com/en-us/download/confirmation.aspx?id=57064'\n}\n\nREGIONS_XPATH = '/AzurePublicIpAddresses/Region'\n\n\ndef _build_IPv4(nodename, region, iprange):\n iprange = iprange.get('Subnet', None)\n if iprange is None:\n LOG.error('%s - No Subnet', nodename)\n return {}\n\n try:\n netaddr.IPNetwork(iprange)\n except:\n LOG.exception('%s - Invalid ip range: %s', nodename, iprange)\n return {}\n\n item = {\n 'indicator': iprange,\n 'type': 'IPv4',\n 'confidence': 100,\n 'azure_region': region,\n 'sources': ['azure.xml']\n }\n return item\n\n\ndef _build_IP(nodename, address_prefix, **keywords):\n try:\n ap = netaddr.IPNetwork(address_prefix)\n except Exception:\n LOG.exception('%s - Invalid ip range: %s', nodename, address_prefix)\n return {}\n\n if ap.version == 4:\n type_ = 'IPv4'\n elif ap.version == 6:\n type_ = 'IPv6'\n else:\n LOG.error('{} - Unknown IP version: {}'.format(nodename, ap.version))\n return {}\n\n item = {\n 'indicator': address_prefix,\n 'type': type_,\n 'confidence': 100,\n 'sources': [nodename]\n }\n item.update(keywords)\n\n return item\n\n\nclass AzureXML(basepoller.BasePollerFT):\n def configure(self):\n super(AzureXML, self).configure()\n\n self.polling_timeout = self.config.get('polling_timeout', 20)\n self.verify_cert = self.config.get('verify_cert', True)\n\n def _process_item(self, item):\n indicator = item.pop('indicator', None)\n return [[indicator, item]]\n\n def _build_request(self, now):\n r = requests.Request(\n 'GET',\n AZUREXML_URL\n )\n\n return r.prepare()\n\n def _build_iterator(self, now):\n _iterators = []\n\n rkwargs = dict(\n stream=False,\n verify=self.verify_cert,\n timeout=self.polling_timeout\n )\n\n r = requests.get(\n AZUREXML_URL,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.error('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n html_soup = bs4.BeautifulSoup(r.content, \"lxml\")\n a = html_soup.find('a', class_='failoverLink')\n if a is None:\n LOG.error('%s - failoverLink not found', self.name)\n raise RuntimeError('{} - failoverLink not found'.format(self.name))\n LOG.debug('%s - download link: %s', self.name, a['href'])\n\n rkwargs = dict(\n stream=True,\n verify=self.verify_cert,\n timeout=self.polling_timeout\n )\n\n r = requests.get(\n a['href'],\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.error('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n parser = lxml.etree.XMLParser()\n for chunk in r.iter_content(chunk_size=10 * 1024):\n parser.feed(chunk)\n rtree = parser.close()\n\n regions = rtree.xpath(REGIONS_XPATH)\n\n for r in regions:\n LOG.debug('%s - Extracting region: %s', self.name, r.get('Name'))\n\n ipranges = r.xpath('IpRange')\n _iterators.append(itertools.imap(\n functools.partial(_build_IPv4, self.name, r.get('Name')),\n ipranges\n ))\n\n return itertools.chain(*_iterators)\n\n\nclass AzureJSON(basepoller.BasePollerFT):\n def configure(self):\n super(AzureJSON, self).configure()\n\n self.polling_timeout = self.config.get('polling_timeout', 20)\n self.verify_cert = self.config.get('verify_cert', True)\n self.cloud = self.config.get('cloud', 'public')\n self.url = AZURE_CLOUD_TO_URL[self.cloud]\n\n def _process_item(self, item):\n indicator = item.pop('indicator', None)\n return [[indicator, item]]\n\n def _build_iterator(self, now):\n _iterators = []\n\n rkwargs = dict(\n stream=False,\n verify=self.verify_cert,\n timeout=self.polling_timeout\n )\n\n r = requests.get(\n self.url,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.error('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n html_soup = bs4.BeautifulSoup(r.content, \"lxml\")\n a = html_soup.find('a', class_='failoverLink')\n if a is None:\n LOG.error('%s - failoverLink not found', self.name)\n raise RuntimeError('{} - failoverLink not found'.format(self.name))\n LOG.debug('%s - download link: %s', self.name, a['href'])\n\n rkwargs = dict(\n stream=True,\n verify=self.verify_cert,\n timeout=self.polling_timeout\n )\n\n r = requests.get(\n a['href'],\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.error('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n rtree = r.json()\n\n values = rtree.get('values', None)\n if values is None:\n LOG.error('{} - no values in JSON response'.format(self.name))\n return []\n\n for v in values:\n LOG.debug('{} - Extracting value: {!r}'.format(self.name, v.get('id', None)))\n\n id_ = v.get('id', None)\n name = v.get('name', None)\n\n props = v.get('properties', None)\n if props is None:\n LOG.error('{} - no properties in value'.format(self.name))\n continue\n\n region = props.get('region', None)\n platform = props.get('platform', None)\n system_service = props.get('systemService', None)\n address_prefixes = props.get('addressPrefixes', [])\n _iterators.append(itertools.imap(\n functools.partial(\n _build_IP,\n self.name,\n azure_name=name,\n azure_id=id_,\n azure_region=region,\n azure_platform=platform,\n azure_system_service=system_service\n ),\n address_prefixes\n ))\n\n # aggregate indicators\n aggregated_indicators = defaultdict(lambda: dict(\n azure_name_list=set([]),\n azure_id_list=set(([])),\n azure_region_list=set([]),\n azure_platform_list=set([]),\n azure_system_service_list=set([])\n ))\n for i in itertools.chain(*_iterators):\n cv = aggregated_indicators[i['indicator']]\n cv.update(i)\n\n for k, v in i.iteritems():\n cv[k] = v\n if k.startswith('azure_'):\n cv['{}_list'.format(k)].add(str(v).lower())\n\n # convert sets into lists\n for iv in aggregated_indicators.values():\n for k in iv.keys():\n if isinstance(iv[k], set):\n iv[k] = list(iv[k])\n\n return iter(aggregated_indicators.values())\n" }, { "path": "minemeld/ft/bambenek.py", "content": "\"\"\"\nThis module implements a thin wrapper class around minemeld.ft.csv.CSVFT\nto mine Bambenek Consulting feeds\n\"\"\"\n\nfrom __future__ import absolute_import\n\nimport logging\n\nfrom . import csv\n\nLOG = logging.getLogger(__name__)\n\n\nclass Miner(csv.CSVFT):\n pass\n" }, { "path": "minemeld/ft/base.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements minemeld.ft.base.BaseFT, the base class for nodes.\n\"\"\"\n\nfrom __future__ import absolute_import\n\nimport logging\nimport copy\nimport os\nimport collections\nimport json\n\nimport gevent\n\nfrom . import condition\nfrom . import ft_states\nfrom . import utils\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass _Filters(object):\n \"\"\"Implements a set of filters to be applied to indicators.\n Used by mineneld.ft.base.BaseFT for ingress and egress filters.\n\n Args:\n filters (list): list of filters.\n \"\"\"\n def __init__(self, filters):\n self.filters = []\n\n for f in filters:\n cf = {\n 'name': f.get('name', 'filter_%d' % len(self.filters)),\n 'conditions': [],\n 'actions': []\n }\n\n fconditions = f.get('conditions', None)\n if fconditions is None:\n fconditions = []\n for c in fconditions:\n cf['conditions'].append(condition.Condition(c))\n\n for a in f.get('actions'):\n cf['actions'].append(a)\n\n self.filters.append(cf)\n\n def apply(self, origin=None, method=None, indicator=None, value=None):\n if value is None:\n d = {}\n else:\n d = copy.copy(value)\n\n if indicator is not None:\n d['__indicator'] = indicator\n\n if method is not None:\n d['__method'] = method\n\n if origin is not None:\n d['__origin'] = origin\n\n for f in self.filters:\n LOG.debug(\"evaluating filter %s\", f['name'])\n\n r = True\n for c in f['conditions']:\n r &= c.eval(d)\n\n if not r:\n continue\n\n for a in f['actions']:\n if a == 'accept':\n if value is None:\n return indicator, None\n\n d.pop('__indicator')\n d.pop('__origin', None)\n d.pop('__method', None)\n\n return indicator, d\n\n elif a == 'drop':\n return None, None\n\n LOG.debug(\"no matching filter, default accept\")\n\n if value is None:\n return indicator, None\n\n d.pop('__indicator')\n d.pop('__origin', None)\n d.pop('__method', None)\n\n return indicator, d\n\n\ndef _counting(statsname):\n \"\"\"Decorator for counting calls to decorated instance methods.\n Counters are stored in statistics attribute of the instance.\n\n Args:\n statsname (str): name of the counter to increment\n \"\"\"\n def _counter_out(f):\n def _counter(self, *args, **kwargs):\n self.statistics[statsname] += 1\n f(self, *args, **kwargs)\n self.publish_status()\n return _counter\n return _counter_out\n\n\nclass BaseFT(object):\n \"\"\"Implements base class of MineMeld engine nodes.\n\n **Config parameters**\n\n :infilters: inbound filter set. Filters to be applied to\n received indicators.\n :outfilters: outbound filter set. Filters to be applied to\n transmitted indicators.\n\n **Filter set**\n Each filter set is a list of filters. Filters are verified from top\n to bottom, and the first matching filter is applied. Default action\n is **accept**.\n Each filter is a dictionary with 3 keys:\n\n :name: name of the filter.\n :conditions: list of boolean expressions to match on the\n indicator.\n :actions: list of actions to be applied to the indicator.\n Currently the only supported actions are **accept** and **drop**\n\n In addition to the atttributes in the indicator value, filters can\n match on 3 special attributes:\n\n :__indicator: the indicator itself.\n :__method: the method of the message, **update** or **withdraw**.\n :__origin: the name of the node who sent the indicator.\n\n **Condition**\n A condition in the filter, is boolean expression composed by a JMESPath\n expression, an operator (<, <=, ==, >=, >, !=) and a value.\n\n Example:\n Example config in YAML::\n\n infilters:\n - name: accept withdraws\n conditions:\n - __method == 'withdraw'\n actions:\n - accept\n - name: accept URL\n conditions:\n - type == 'URL'\n actions:\n - accept\n - name: drop all\n actions:\n - drop\n outfilters:\n - name: accept all (default)\n actions:\n - accept\n\n Args:\n name (str): node name, should be unique inside the graph\n chassis (object): parent chassis instance\n config (dict): node config.\n \"\"\"\n def __init__(self, name, chassis, config):\n self.name = name\n\n self.chassis = chassis\n\n self._original_config = copy.deepcopy(config)\n self.config = config\n self.configure()\n\n self.inputs = []\n self.output = None\n\n self.statistics = collections.defaultdict(int)\n\n self.read_checkpoint()\n\n self.chassis.request_mgmtbus_channel(self)\n\n self._state = ft_states.READY\n\n self._last_status_publish = None\n self._throttled_publish_status = utils.GThrottled(self._internal_publish_status, 3000)\n self._clock = 0\n\n self._disable_full_trace = 'MM_DISABLE_FULL_TRACE' in os.environ\n self._disable_full_trace_glet = None\n\n @property\n def state(self):\n return self._state\n\n @state.setter\n def state(self, value):\n LOG.info(\"%s - transitioning to state %d\", self.name, value)\n self._state = value\n\n if value >= ft_states.INIT and value <= ft_states.STOPPED:\n self.publish_status(force=True)\n\n def read_checkpoint(self):\n \"\"\"Reads checkpoint file from disk.\n\n First line of the checkpoint file is a UUID, the *checkpoint* received\n before stopping. The second line is a dictionary in JSON with the class\n of the node and the config. The third line is a dictionary in JSON\n with the persistent state of the node.\n\n Checkpoint files are used to check if the saved state on disk is\n consistent with the current running config. If the state is not\n consistent `last_checkpoint` is set to None, to indicate that the state\n stored on disk is not valid or inexistent.\n\n Called by `__init__`.\n \"\"\"\n self.last_checkpoint = None\n\n config = {\n 'class': (self.__class__.__module__+'.'+self.__class__.__name__),\n 'config': self._original_config\n }\n config = json.dumps(config, sort_keys=True)\n\n try:\n with open(self.name+'.chkp', 'r') as f:\n contents = f.read()\n if contents[0] == '{':\n # new format\n contents = json.loads(contents)\n self.last_checkpoint = contents['checkpoint']\n saved_config = contents['config']\n saved_state = contents['state']\n\n else:\n # old format\n lines = contents.splitlines()\n self.last_checkpoint = lines[0]\n\n saved_config = ''\n if len(lines) > 1:\n # this to support a really old format\n # where only checkpoint value was saved\n saved_config = lines[1]\n\n saved_state = None\n\n LOG.debug('%s - restored checkpoint: %s', self.name, self.last_checkpoint)\n\n # old_status is missing in old releases\n # stick to the old behavior\n if saved_config and saved_config != config:\n LOG.info(\n '%s - saved config does not match new config',\n self.name\n )\n self.last_checkpoint = None\n return\n\n LOG.info(\n '%s - saved config matches new config',\n self.name\n )\n\n if saved_state is not None:\n self._saved_state_restore(saved_state)\n\n except (ValueError, IOError):\n LOG.exception('%s - Error reading last checkpoint', self.name)\n self.last_checkpoint = None\n\n def create_checkpoint(self, value):\n \"\"\"Saves checkpoint file to disk.\n\n Called by `checkpoint`.\n\n Args:\n value (str): received *checkpoint*\n \"\"\"\n config = {\n 'class': (self.__class__.__module__+'.'+self.__class__.__name__),\n 'config': self._original_config\n }\n\n contents = {\n 'checkpoint': value,\n 'config': json.dumps(config, sort_keys=True),\n 'state': self._saved_state_create()\n }\n\n with open(self.name+'.chkp', 'w') as f:\n f.write(json.dumps(contents))\n f.write('\\n')\n\n def remove_checkpoint(self):\n try:\n os.remove('{}.chkp'.format(self.name))\n\n except (IOError, OSError):\n pass\n\n def _saved_state_restore(self, saved_state):\n pass\n\n def _saved_state_create(self):\n return {}\n\n def configure(self):\n \"\"\"Applies the config settings stored in `self.config`.\n\n Called by `__init__`.\n\n When this method is changed to add/remove new parameters, the class\n docstring should be updated.\n \"\"\"\n self.infilters = _Filters(self.config.get('infilters', []))\n self.outfilters = _Filters(self.config.get('outfilters', []))\n\n def connect(self, inputs, output):\n if self.state != ft_states.READY:\n LOG.error('connect called in non ready FT')\n raise AssertionError('connect called in non ready FT')\n\n for i in inputs:\n LOG.info(\"%s - requesting fabric sub channel for %s\", self.name, i)\n self.chassis.request_sub_channel(\n self.name,\n self,\n i,\n allowed_methods=['update', 'withdraw', 'checkpoint']\n )\n self.inputs = inputs\n self.inputs_checkpoint = {}\n\n if output:\n self.output = self.chassis.request_pub_channel(self.name)\n\n self.chassis.request_rpc_channel(\n self.name,\n self,\n allowed_methods=[\n 'update',\n 'withdraw',\n 'checkpoint',\n 'get',\n 'get_all',\n 'get_range',\n 'length'\n ]\n )\n\n self.state = ft_states.CONNECTED\n\n def apply_infilters(self, origin, method, indicator, value):\n return self.infilters.apply(\n origin=origin,\n method=method,\n indicator=indicator,\n value=value\n )\n\n def apply_outfilters(self, origin, method, indicator, value):\n return self.outfilters.apply(\n origin=origin,\n method=method,\n indicator=indicator,\n value=value\n )\n\n def do_rpc(self, dftname, method, block=True, timeout=30, **kwargs):\n return self.chassis.send_rpc(self.name, dftname, method, kwargs,\n block=block, timeout=timeout)\n\n @_counting('update.tx')\n def emit_update(self, indicator, value):\n if self.output is None:\n return\n\n self.trace('EMIT_UPDATE', indicator, value=value)\n\n indicator, value = self.apply_outfilters(\n origin=self.name,\n method='update',\n indicator=indicator,\n value=value\n )\n\n if indicator is None:\n return\n\n if value is not None:\n for k in value.keys():\n if k[0] in ['_', '$']:\n value.pop(k)\n\n self.output.publish(\"update\", {\n 'source': self.name,\n 'indicator': indicator,\n 'value': value\n })\n\n @_counting('withdraw.tx')\n def emit_withdraw(self, indicator, value=None):\n if self.output is None:\n return\n\n self.trace('EMIT_WITHDRAW', indicator, value=value)\n\n indicator, value = self.apply_outfilters(\n origin=self.name,\n method='withdraw',\n indicator=indicator,\n value=value\n )\n\n if indicator is None:\n return\n\n if value is not None:\n for k in value.keys():\n if k[0] in ['_', '$']:\n value.pop(k)\n\n self.output.publish(\"withdraw\", {\n 'source': self.name,\n 'indicator': indicator,\n 'value': value\n })\n\n @_counting('checkpoint.tx')\n def emit_checkpoint(self, value):\n if self.output is None:\n return\n\n self.output.publish('checkpoint', {\n 'source': self.name,\n 'value': value\n })\n\n @_counting('update.rx')\n def update(self, source=None, indicator=None, value=None):\n LOG.debug('%s {%s} - update from %s value %s',\n self.name, self.state, source, value)\n\n if not self._disable_full_trace:\n self.trace('RECVD_UPDATE', indicator, source_node=source, value=value)\n\n if self.state not in [ft_states.STARTED, ft_states.CHECKPOINT]:\n self.statistics['error.wrong_state'] += 1\n return\n\n if source in self.inputs_checkpoint:\n LOG.error(\"update received from checkpointed source\")\n raise AssertionError(\"update received from checkpointed source\")\n\n if value is not None:\n for k in value.keys():\n if k.startswith(\"_\"):\n value.pop(k)\n\n fltindicator, fltvalue = self.apply_infilters(\n origin=source,\n method='update',\n indicator=indicator,\n value=value\n )\n\n if fltindicator is None:\n if not self._disable_full_trace:\n self.trace('DROP_UPDATE', indicator, source_node=source, value=value)\n\n self.filtered_withdraw(\n source=source,\n indicator=indicator,\n value=value\n )\n return\n\n self.trace('ACCEPT_UPDATE', indicator, source_node=source, value=value)\n self.filtered_update(\n source=source,\n indicator=fltindicator,\n value=fltvalue\n )\n\n @_counting('update.processed')\n def filtered_update(self, source=None, indicator=None, value=None):\n raise NotImplementedError('%s: update' % self.name)\n\n @_counting('withdraw.rx')\n def withdraw(self, source=None, indicator=None, value=None):\n LOG.debug('%s {%s} - withdraw from %s value %s',\n self.name, self.state, source, value)\n\n if not self._disable_full_trace:\n self.trace('RECVD_WITHDRAW', indicator, source_node=source, value=value)\n\n if self.state not in [ft_states.STARTED, ft_states.CHECKPOINT]:\n self.statistics['error.wrong_state'] += 1\n return\n\n if source in self.inputs_checkpoint:\n LOG.error(\"withdraw received from checkpointed source\")\n raise AssertionError(\"withdraw received from checkpointed source\")\n\n fltindicator, fltvalue = self.apply_infilters(\n origin=source,\n method='withdraw',\n indicator=indicator,\n value=value\n )\n\n if fltindicator is None:\n if not self._disable_full_trace:\n self.trace('DROP_WITHDRAW', indicator, source_node=source, value=value)\n return\n\n if fltvalue is not None:\n for k in fltvalue.keys():\n if k.startswith(\"_\"):\n fltvalue.pop(k)\n\n self.trace('ACCEPT_WITHDRAW', indicator, source_node=source, value=value)\n self.filtered_withdraw(\n source=source,\n indicator=indicator,\n value=value\n )\n\n @_counting('withdraw.processed')\n def filtered_withdraw(self, source=None, indicator=None, value=None):\n raise NotImplementedError('%s: withdraw' % self.name)\n\n @_counting('checkpoint.rx')\n def checkpoint(self, source=None, value=None):\n LOG.debug('%s {%s} - checkpoint from %s value %s',\n self.name, self.state, source, value)\n\n if self.state not in [ft_states.STARTED, ft_states.CHECKPOINT]:\n LOG.error(\"%s {%s} - checkpoint received with state not STARTED \"\n \"or CHECKPOINT\",\n self.name, self.state)\n raise AssertionError(\"checkpoint received with state not STARTED \"\n \"or CHECKPOINT\")\n\n for v in self.inputs_checkpoint.values():\n if v != value:\n LOG.error(\"different checkpoint value received\")\n raise AssertionError(\"different checkpoint value received\")\n\n self.inputs_checkpoint[source] = value\n\n if len(self.inputs_checkpoint) != len(self.inputs):\n self.state = ft_states.CHECKPOINT\n return\n\n self.state = ft_states.IDLE\n self.create_checkpoint(value)\n self.last_checkpoint = value\n self.emit_checkpoint(value)\n\n def _full_trace_timeout(self, timeout):\n \"\"\"To be used as greenlet for disabling full trace after a specific time\n \"\"\"\n gevent.sleep(timeout)\n self._disable_full_trace = True\n LOG.debug('{} - full trace disabled'.format(self.name))\n\n def enable_full_trace(self, timeout=600):\n \"\"\"Enables full trace\n \"\"\"\n\n # if full trace is already enabled, do nothing\n if self._disable_full_trace is False:\n return\n\n self._disable_full_trace_glet = gevent.spawn(\n self._full_trace_timeout,\n timeout\n )\n self._disable_full_trace = False\n LOG.debug('{} - full trace enabled'.format(self.name))\n\n def publish_status(self, force=False):\n if force:\n self._internal_publish_status()\n\n self._throttled_publish_status()\n\n def _internal_publish_status(self):\n self._last_status_publish = utils.utc_millisec()\n status = self.mgmtbus_status()\n self.chassis.publish_status(\n timestamp=utils.utc_millisec(),\n nodename=self.name,\n status=status\n )\n\n def mgmtbus_state_info(self):\n return {\n 'checkpoint': self.last_checkpoint,\n 'state': self.state,\n 'is_source': len(self.inputs) == 0\n }\n\n def mgmtbus_initialize(self):\n self.state = ft_states.INIT\n self.remove_checkpoint()\n self.initialize()\n return 'OK'\n\n def mgmtbus_rebuild(self):\n self.state = ft_states.REBUILDING\n self.remove_checkpoint()\n self.rebuild()\n self.state = ft_states.INIT\n return 'OK'\n\n def mgmtbus_reset(self):\n self.state = ft_states.RESET\n self.remove_checkpoint()\n self.reset()\n self.state = ft_states.INIT\n return 'OK'\n\n def mgmtbus_status(self):\n try:\n # if node is not ready yet to publish the length\n length = self.length()\n except:\n length = None\n\n result = {\n 'clock': self._clock,\n 'class': (self.__class__.__module__+'.'+self.__class__.__name__),\n 'state': self.state,\n 'statistics': self.statistics,\n 'length': length,\n 'inputs': self.inputs,\n 'output': (self.output is not None),\n 'trace': not self._disable_full_trace\n }\n self._clock += 1\n return result\n\n def mgmtbus_checkpoint(self, value=None):\n if len(self.inputs) != 0:\n return 'ignored'\n\n self.state = ft_states.IDLE\n self.create_checkpoint(value)\n self.last_checkpoint = value\n self.emit_checkpoint(value)\n\n return 'OK'\n\n def mgmtbus_hup(self, source=None):\n self.hup(source=source)\n\n def mgmtbus_signal(self, source=None, signal=None, **kwargs):\n if signal == 'trace':\n self.enable_full_trace()\n return self._disable_full_trace\n\n raise NotImplementedError('{}: signal - not implemented'.format(self.name))\n\n def initialize(self):\n pass\n\n def rebuild(self):\n pass\n\n def reset(self):\n pass\n\n def get_state(self):\n return self.state\n\n def get(self, source=None, indicator=None):\n raise NotImplementedError('%s: get - not implemented' % self.name)\n\n def get_all(self, source=None):\n raise NotImplementedError('%s: get_all - not implemented' % self.name)\n\n def get_range(self, source=None, index=None, from_key=None, to_key=None):\n raise NotImplementedError('%s: get_range - not implemented' %\n self.name)\n\n def length(self, source=None):\n raise NotImplementedError('%s: length - not implemented' % self.name)\n\n def hup(self, source=None):\n raise NotImplementedError('%s: hup - not implemented' % self.name)\n\n def trace(self, action, indicator, **kwargs):\n if self.state not in [ft_states.STARTED, ft_states.CHECKPOINT]:\n LOG.debug(\n \"%s - trace called in wrong state %s\",\n self.name,\n self.state\n )\n return\n\n trace = {\n 'indicator': indicator,\n 'op': action,\n }\n trace.update(kwargs)\n self.chassis.log(\n timestamp=utils.utc_millisec(),\n nodename=self.name,\n log_type='TRACE',\n value=trace\n )\n\n def start(self):\n LOG.debug(\"%s - start called\", self.name)\n\n if self.state != ft_states.INIT:\n LOG.error(\"start on not INIT FT\")\n raise AssertionError(\"start on not INIT FT\")\n\n self.state = ft_states.STARTED\n\n def stop(self):\n LOG.debug(\"%s - stop called\", self.name)\n\n if self._disable_full_trace_glet is not None:\n self._disable_full_trace_glet.kill()\n self._disable_full_trace_glet = None\n\n if self.state not in [ft_states.IDLE, ft_states.STARTED]:\n LOG.error(\"stop on not IDLE or STARTED FT\")\n raise AssertionError(\"stop on not IDLE or STARTED FT\")\n\n self._throttled_publish_status.cancel()\n\n self.state = ft_states.STOPPED\n\n @staticmethod\n def gc(name, config=None):\n try:\n os.remove('{}.chkp'.format(name))\n except:\n pass\n" }, { "path": "minemeld/ft/basepoller.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements minemeld.ft.basepoller.BasePollerFT, a base class for\nminers retrieving indicators by periodically polling an external source.\n\"\"\"\n\nimport logging\nimport copy\nimport random\nimport collections\nimport sys\nimport shutil\n\nimport gevent\nimport gevent.event\nimport gevent.queue\n\nfrom . import base\nfrom . import ft_states\nfrom .table import Table\nfrom .utils import utc_millisec\nfrom .utils import RWLock\nfrom .utils import parse_age_out\n\nLOG = logging.getLogger(__name__)\n\n_MAX_AGE_OUT = ((1 << 32)-1)*1000 # 2106-02-07 6:28:15\n\n\nclass _BaseBPTable(object):\n def __init__(self, table):\n self.table = table\n\n def get(self, indicator, itype=None):\n return self.table.get(indicator)\n\n def delete(self, indicator, itype=None):\n self.table.delete(indicator)\n\n def put(self, indicator, value):\n self.table.put(indicator, value)\n\n def query(self, *args, **kwargs):\n return self.table.query(*args, **kwargs)\n\n def length(self):\n return self.table.num_indicators\n\n def close(self):\n self.table.close()\n\n def __del__(self):\n self.close()\n\n\nclass _BPTable_v0(_BaseBPTable):\n def __init__(self, table):\n super(_BPTable_v0, self).__init__(table)\n\n self.table.create_index('_age_out')\n self.table.create_index('_withdrawn')\n self.table.create_index('_last_run')\n\n\nclass _BPTable_v1(_BaseBPTable):\n def __init__(self, table, type_in_key):\n super(_BPTable_v1, self).__init__(table)\n\n self.table.create_index('_age_out')\n self.table.create_index('_withdrawn')\n self.table.create_index('_last_run')\n\n self.type_in_key = type_in_key\n\n cmetadata = self.table.get_custom_metadata()\n if cmetadata is None:\n _custom_metadata = dict(version=1, type_in_key=type_in_key)\n self.table.set_custom_metadata(_custom_metadata)\n else:\n if cmetadata.get('type_in_key', None) != self.type_in_key:\n raise RuntimeError('Can\\'t change type in key of an existing table')\n\n def get(self, indicator, itype=None):\n if self.type_in_key:\n indicator = self._type_key(indicator, itype)\n\n return self.table.get(indicator)\n\n def delete(self, indicator, itype=None):\n if self.type_in_key:\n indicator = self._type_key(indicator, itype)\n\n return self.table.delete(indicator)\n\n def put(self, indicator, value):\n if self.type_in_key:\n itype = value.get('type', None)\n indicator = self._type_key(indicator, itype)\n\n return self.table.put(indicator, value)\n\n def query(self, *args, **kwargs):\n if not self.type_in_key:\n return self.table.query(*args, **kwargs)\n\n if kwargs.get('include_value', False):\n return self._type_key_query_with_value(*args, **kwargs)\n\n return self._type_key_query(*args, **kwargs)\n\n def _type_key_query(self, *args, **kwargs):\n for key in self.table.query(*args, **kwargs):\n yield self._type_key_indicator(key)\n\n def _type_key_query_with_value(self, *args, **kwargs):\n for key, value in self.table.query(*args, **kwargs):\n yield self._type_key_indicator(key), value\n\n def _type_key(self, indicator, itype):\n if itype is None:\n raise RuntimeError('Type None in table with type in key')\n\n return u'{}::{}'.format(itype, indicator)\n\n def _type_key_indicator(self, key):\n return key.split('::', 1)[1]\n\n\ndef _bptable_factory(name, truncate=False, type_in_key=False):\n table = Table(name, truncate=truncate)\n\n metadata = table.get_custom_metadata()\n if metadata is not None:\n version = metadata.get('version', None)\n if version is None:\n raise RuntimeError('{} - table with metadata but no version'.format(name))\n\n if version == 1:\n return _BPTable_v1(table, type_in_key=type_in_key)\n\n raise RuntimeError('{} - table with unknown version: {}'.format(name, version))\n\n # no metadata, could be a new table or an old one\n if table.num_indicators > 0:\n if type_in_key:\n raise RuntimeError('Old BPtable0 can\\'t be used with multiple indicator types')\n\n return _BPTable_v0(table)\n\n # new table\n return _BPTable_v1(table, type_in_key=type_in_key)\n\n\nclass IndicatorStatus(object):\n D_MASK = 1\n F_MASK = 2\n A_MASK = 4\n W_MASK = 8\n\n NX = 0\n NFNANW = D_MASK\n XFNANW = D_MASK | F_MASK\n NFXANW = D_MASK | A_MASK\n XFXANW = D_MASK | F_MASK | A_MASK\n NFNAXW = D_MASK | W_MASK\n XFNAXW = D_MASK | F_MASK | W_MASK\n NFXAXW = D_MASK | A_MASK | W_MASK\n XFXAXW = D_MASK | F_MASK | A_MASK | W_MASK\n\n def __init__(self, indicator, attributes, itable, now, in_feed_threshold):\n self.state = 0\n\n self.cv = itable.get(indicator, itype=attributes.get('type', None))\n if self.cv is None:\n return\n self.state = self.state | IndicatorStatus.D_MASK\n\n if self.cv['_age_out'] < now:\n self.state = self.state | IndicatorStatus.A_MASK\n\n if self.cv['_last_run'] >= in_feed_threshold:\n self.state = self.state | IndicatorStatus.F_MASK\n\n if self.cv.get('_withdrawn', None) is not None:\n self.state = self.state | IndicatorStatus.W_MASK\n\n\nclass BasePollerFT(base.BaseFT):\n \"\"\"Implements base class for polling miners.\n\n **Config parameters**\n :source_name: name of the source. This is placed in the\n *sources* attribute of the generated indicators. Default: name\n of the node.\n :attributes: dictionary of attributes for the generated indicators.\n This dictionary is used as template for the value of the generated\n indicators. Default: empty\n :interval: polling interval in seconds. Default: 3600.\n :num_retries: in case of failure, how many times the miner should\n try to reach the source. If this number is exceeded, the miner\n waits until the next polling time to try again. Default: 2\n :age_out: age out policies to apply to the indicators.\n Default: age out check interval 3600 seconds, sudden death enabled,\n default age out interval 30 days.\n\n **Age out policy**\n Age out policy is described by a dictionary with at least 3 keys:\n\n :interval: number of seconds between successive age out checks.\n :sudden_death: boolean, if *true* indicators are immediately aged out\n when they disappear from the feed.\n :default: age out interval. After this interval an indicator is aged\n out even if it is still present in the feed. If *null*, no age out\n interval is applied.\n\n Additional keys can be used to specify age out interval per indicator\n *type*.\n\n **Age out interval**\n Age out intervals have the following format::\n\n +\n\n *base attribute* can be *last_seen*, if the age out interval should be\n calculated based on the last time the indicator was found in the feed,\n or *first_seen*, if instead the age out interval should be based on the\n time the indicator was first seen in the feed. If not specified\n *first_seen* is used.\n\n *interval* is the length of the interval expressed in seconds. Suffixes\n *d*, *h* and *m* can be used to specify days, hours or minutes.\n\n Example:\n Example config in YAML for a feed where indicators should be aged out\n only when they are removed from the feed::\n\n source_name: example.persistent_feed\n interval: 600\n age_out:\n default: null\n sudden_death: true\n interval: 300\n attributes:\n type: IPv4\n confidence: 100\n share_level: green\n direction: inbound\n\n Example config in YAML for a feed where indicators are aged out when\n they disappear from the feed and 30 days after they have seen for the\n first time in the feed::\n\n source_name: example.long_running_feed\n interval: 3600\n age_out:\n default: first_seen+30d\n sudden_death: true\n interval: 1800\n attributes:\n type: URL\n confidence: 50\n share_level: green\n\n Example config in YAML for a feed where indicators are aged 30 days\n after they have seen for the last time in the feed::\n\n source_name: example.delta_feed\n interval: 3600\n age_out:\n default: last_seen+30d\n sudden_death: false\n interval: 1800\n attributes:\n type: URL\n confidence: 50\n share_level: green\n\n Args:\n name (str): node name, should be unique inside the graph\n chassis (object): parent chassis instance\n config (dict): node config.\n \"\"\"\n\n _AGE_OUT_BASES = None\n _DEFAULT_AGE_OUT_BASE = None\n\n def __init__(self, name, chassis, config):\n self.table = None\n self.agg_table = None\n\n self._actor_queue = gevent.queue.Queue(maxsize=128)\n self._actor_glet = None\n self._actor_commands_ts = collections.defaultdict(int)\n self._poll_glet = None\n self._age_out_glet = None\n self._emit_counter = 0\n\n self.last_run = None\n self.last_successful_run = None\n self.last_ageout_run = None\n self._sub_state = None\n self._sub_state_message = None\n\n self.poll_event = gevent.event.Event()\n\n self.state_lock = RWLock()\n\n super(BasePollerFT, self).__init__(name, chassis, config)\n\n def configure(self):\n super(BasePollerFT, self).configure()\n\n self.source_name = self.config.get('source_name', self.name)\n self.attributes = self.config.get('attributes', {})\n self.multiple_indicator_types = self.config.get('multiple_indicator_types', False)\n self.interval = self.config.get('interval', 3600)\n self.num_retries = self.config.get('num_retries', 2)\n\n self.aggregate_indicators = self.config.get('aggregate_indicators', False)\n self.aggregate_use_partial = self.config.get('aggregate_use_partial', False)\n\n _age_out = self.config.get('age_out', {})\n\n self.age_out = {\n 'interval': _age_out.get('interval', 3600),\n 'sudden_death': _age_out.get('sudden_death', True),\n 'default': parse_age_out(\n _age_out.get('default', '30d'),\n age_out_bases=self._AGE_OUT_BASES,\n default_base=self._DEFAULT_AGE_OUT_BASE\n )\n }\n for k, v in _age_out.iteritems():\n if k in self.age_out:\n continue\n self.age_out[k] = parse_age_out(v)\n\n def _saved_state_restore(self, saved_state):\n self.last_run = saved_state.get('last_run', None)\n self.last_successful_run = saved_state.get(\n 'last_successful_run',\n None\n )\n\n def _saved_state_create(self):\n return {\n 'last_run': self.last_run,\n 'last_successful_run': self.last_successful_run\n }\n\n def _saved_state_reset(self):\n self.last_successful_run = None\n self.last_run = None\n\n def _initialize_table(self, truncate=False):\n self.table = _bptable_factory(\n self.name,\n truncate=truncate,\n type_in_key=self.multiple_indicator_types\n )\n\n def initialize(self):\n self._initialize_table()\n\n def rebuild(self):\n self._actor_queue.put(\n (utc_millisec(), 'rebuild')\n )\n self._initialize_table(truncate=(self.last_checkpoint is None))\n\n def reset(self):\n self._saved_state_reset()\n self._initialize_table(truncate=True)\n\n @base.BaseFT.state.setter\n def state(self, value):\n LOG.debug(\"%s - acquiring state write lock\", self.name)\n self.state_lock.lock()\n # this is weird ! from stackoverflow 10810369\n super(BasePollerFT, self.__class__).state.fset(self, value)\n self.state_lock.unlock()\n LOG.debug(\"%s - releasing state write lock\", self.name)\n\n def _controlled_emit_update(self, indicator, value):\n self._emit_counter += 1\n if self._emit_counter == 15937:\n gevent.sleep(0.001)\n self._emit_counter = 0\n self.emit_update(indicator, value)\n\n def _controlled_emit_withdraw(self, indicator, value):\n self._emit_counter += 1\n if self._emit_counter == 15937:\n gevent.sleep(0.001)\n self._emit_counter = 0\n self.emit_withdraw(indicator=indicator, value=value)\n\n def _age_out(self):\n with self.state_lock:\n if self.state != ft_states.STARTED:\n return\n\n try:\n now = utc_millisec()\n\n for i, v in self.table.query(index='_age_out',\n to_key=now-1,\n include_value=True):\n LOG.debug('%s - %s %s aged out', self.name, i, v)\n\n if v.get('_withdrawn', None) is not None:\n continue\n\n self._controlled_emit_withdraw(\n indicator=i,\n value=v\n )\n v['_withdrawn'] = now\n self.table.put(i, v)\n\n self.statistics['aged_out'] += 1\n\n self.last_ageout_run = now\n\n except gevent.GreenletExit:\n raise\n\n except:\n LOG.exception('Exception in _age_out')\n\n def _flush(self):\n with self.state_lock:\n if self.state != ft_states.STARTED:\n return\n\n try:\n now = utc_millisec()\n\n for i, v in self.table.query(include_value=True):\n if v.get('_withdrawn', None) is not None:\n continue\n\n self._controlled_emit_withdraw(\n indicator=i,\n value=v\n )\n v['_withdrawn'] = now\n v['_last_run'] = 0\n self.table.put(i, v)\n\n self.statistics['flushed'] += 1\n\n except gevent.GreenletExit:\n raise\n\n except:\n LOG.exception('Exception in _flush')\n\n def _sudden_death(self):\n if self.last_successful_run is None:\n return\n\n with self.state_lock:\n if self.state != ft_states.STARTED:\n return\n\n LOG.debug('checking sudden death for %d', self.last_successful_run)\n\n for i, v in self.table.query(index='_last_run',\n to_key=self.last_successful_run-1,\n include_value=True):\n LOG.debug('%s - %s %s sudden death', self.name, i, v)\n\n v['_age_out'] = self.last_successful_run-1\n self.table.put(i, v)\n self.statistics['removed'] += 1\n\n def _collect_garbage(self):\n now = utc_millisec()\n\n with self.state_lock:\n if self.state != ft_states.STARTED:\n return\n\n for i, v in self.table.query(index='_withdrawn',\n to_key=now,\n include_value=True):\n if v.get('_last_run', 0) >= (self.last_successful_run-1):\n continue\n\n LOG.debug('%s - %s collected', self.name, i)\n self.table.delete(i, itype=v.get('type', None))\n self.statistics['garbage_collected'] += 1\n\n def _compare_attributes(self, oa, na):\n default_attrs = ['sources', 'last_seen', 'first_seen']\n default_attrs.extend(self.attributes.keys())\n\n for k in oa:\n if k in default_attrs or k[0] in ('_', '$'):\n continue\n if k not in na:\n return False\n for k in na:\n if oa.get(k, None) != na[k]:\n return False\n return True\n\n def _update_attributes(self, current, _new, current_run, new_run):\n x = {k:v for k,v in current.iteritems() \n if k in _new or\n k in self.attributes or\n k in ['sources', 'last_seen', 'first_seen'] or\n k[0] in ('_', '$')}\n x.update(_new)\n\n return x\n\n def _aggregate_iterator(self, iterator):\n self.agg_table = _bptable_factory(\n '{}.aggregate-temp'.format(self.name),\n truncate=True,\n type_in_key=True\n )\n\n for nitem, item in enumerate(iterator):\n if nitem != 0 and nitem % 1024 == 0:\n gevent.sleep(0.001)\n\n with self.state_lock:\n if self.state != ft_states.STARTED:\n LOG.info(\n '%s - state not STARTED, aggregation not performed',\n self.name\n )\n self.agg_table.close()\n return False\n\n try:\n ipairs = self._process_item(item)\n\n except gevent.GreenletExit:\n raise\n\n except:\n self.statistics['error.parsing'] += 1\n LOG.exception('%s - Exception parsing %s', self.name, item)\n continue\n\n for indicator, attributes in ipairs:\n self.agg_table.put(indicator, attributes)\n\n return True\n\n def _aggregate_process_item(self, item):\n return [item]\n\n def _polling_loop(self):\n LOG.info(\"Polling %s\", self.name)\n\n now = utc_millisec()\n\n with self.state_lock:\n if self.state != ft_states.STARTED:\n LOG.info(\n '%s - state not STARTED, polling not performed',\n self.name\n )\n return False\n\n iterator = self._build_iterator(now)\n\n if iterator is None:\n return False\n\n process_item = self._process_item\n aggregation_exc = None\n if self.aggregate_indicators:\n if self.agg_table is not None:\n self.agg_table.close()\n self.agg_table = None\n\n try:\n if not self._aggregate_iterator(iterator):\n return False\n except:\n # if aggregate_use_partial is True, we store exception\n # and handle partial results\n if not self.aggregate_use_partial:\n raise\n LOG.info('{} - Exception during aggregation, storing'.format(self.name))\n aggregation_exc = sys.exc_info()\n\n process_item = self._aggregate_process_item\n iterator = self.agg_table.query(include_value=True)\n\n for nitem, item in enumerate(iterator):\n if nitem != 0 and nitem % 1024 == 0:\n gevent.sleep(0.001)\n\n with self.state_lock:\n if self.state != ft_states.STARTED:\n break\n\n try:\n ipairs = process_item(item)\n\n except gevent.GreenletExit:\n raise\n\n except:\n self.statistics['error.parsing'] += 1\n LOG.exception('%s - Exception parsing %s', self.name, item)\n continue\n\n for indicator, attributes in ipairs:\n if indicator is None:\n LOG.debug('%s - indicator is None for item %s',\n self.name, item)\n continue\n\n in_feed_threshold = self.last_successful_run\n if in_feed_threshold is None:\n in_feed_threshold = now - self.interval*1000\n\n istatus = IndicatorStatus(\n indicator=indicator,\n attributes=attributes,\n itable=self.table,\n now=now,\n in_feed_threshold=in_feed_threshold\n )\n\n if istatus.state in [IndicatorStatus.NX,\n IndicatorStatus.NFNANW,\n IndicatorStatus.NFXANW,\n IndicatorStatus.NFXAXW,\n IndicatorStatus.NFNAXW]:\n v = copy.copy(self.attributes)\n v['sources'] = [self.source_name]\n v['last_seen'] = now\n v['first_seen'] = now\n v['_last_run'] = now\n v.update(attributes)\n v['_age_out'] = self._calc_age_out(indicator, v)\n\n self.statistics['added'] += 1\n self.table.put(indicator, v)\n\n if v['_age_out'] >= now:\n self._controlled_emit_update(indicator, v)\n\n LOG.debug('%s - added %s %s', self.name, indicator, v)\n\n elif istatus.state == IndicatorStatus.XFNANW:\n v = istatus.cv\n\n eq = self._compare_attributes(v, attributes)\n\n old_last_run = v['_last_run']\n v['_last_run'] = now\n\n v = self._update_attributes(\n v, attributes,\n old_last_run, now\n )\n\n v['_age_out'] = self._calc_age_out(indicator, v)\n\n self.table.put(indicator, v)\n\n # emit updates if different and not aged out\n if not eq and v['_age_out'] >= now:\n self._controlled_emit_update(indicator, v)\n\n elif istatus.state == IndicatorStatus.XFXANW:\n v = istatus.cv\n v['_last_run'] = now\n self.table.put(indicator, v)\n\n elif istatus.state in [IndicatorStatus.XFXAXW,\n IndicatorStatus.XFNAXW]:\n v = istatus.cv\n v['_last_run'] = now\n v['_withdrawn'] = now\n self.table.put(indicator, v)\n\n else:\n LOG.error('%s - indicator state unhandled: %s',\n self.name, istatus.state)\n continue\n\n if self.agg_table is not None:\n iterator.close()\n self.agg_table.close()\n self.agg_table = None\n shutil.rmtree('{}.aggregate-temp'.format(self.name))\n\n if aggregation_exc is not None:\n LOG.info('{} - Reraising exception happened during aggregation'.format(self.name))\n raise aggregation_exc[0], aggregation_exc[1], aggregation_exc[2]\n\n return True\n\n def _rebuild(self):\n with self.state_lock:\n if self.state != ft_states.STARTED:\n return\n\n self.sub_state = 'REBUILDING'\n\n for i, v in self.table.query(include_value=True):\n self._controlled_emit_update(i, v)\n\n def _poll(self):\n tryn = 0\n\n while tryn < self.num_retries:\n lastrun = utc_millisec()\n\n try:\n self.sub_state = 'POLLING'\n\n performed = self._polling_loop()\n if performed:\n self.last_successful_run = lastrun\n\n _result = 'SUCCESS'\n break\n\n except gevent.GreenletExit:\n raise\n\n except Exception as e:\n try:\n _error_msg = str(e)\n except UnicodeDecodeError:\n _error_msg = repr(e)\n\n _result = ('ERROR', _error_msg)\n\n self.statistics['error.polling'] += 1\n\n LOG.exception(\"Exception in polling loop for %s: %s\",\n self.name, str(e))\n\n tryn += 1\n gevent.sleep(random.randint(1, 5))\n\n LOG.debug(\"%s - End of polling - #indicators: %d\",\n self.name, self.table.length())\n\n self.last_run = lastrun\n self.sub_state = _result\n\n def _actor_loop(self):\n while True:\n timestamp, command = self._actor_queue.get()\n LOG.info('%s - command: %d %s', self.name, timestamp, command)\n\n try:\n last_ts = self._actor_commands_ts[command]\n if timestamp < last_ts:\n LOG.info(\n '%s - command %s, old timestamp - ignored',\n self.name,\n command\n )\n continue\n\n if command == 'poll':\n self._poll()\n\n elif command == 'age_out':\n self._age_out()\n\n elif command == 'sudden_death':\n self._sudden_death()\n\n elif command == 'gc':\n self._collect_garbage()\n\n elif command == 'rebuild':\n self._rebuild()\n\n elif command == 'flush':\n self._flush()\n\n else:\n LOG.error('%s - unknown command: %s', self.name, command)\n\n except gevent.GreenletExit:\n raise\n\n except:\n LOG.exception(\n '%s - exception executing command %s', self.name, command\n )\n\n self._actor_commands_ts[command] = utc_millisec()\n\n def _poll_loop(self):\n # wait to poll until after the first ageout run\n while self.last_ageout_run is None:\n gevent.sleep(1)\n\n # if last_run is not None it means we have restored\n # a previous state, wait until poll time\n if self.last_run is not None:\n self.sub_state = 'WAITING'\n\n LOG.info(\n '%s - restored last run, waiting until the next poll time',\n self.name\n )\n\n try:\n self._huppable_wait(\n (self.last_run+self.interval*1000)-utc_millisec()\n )\n except gevent.GreenletExit:\n return\n\n while True:\n with self.state_lock:\n if self.state != ft_states.STARTED:\n break\n\n self._actor_queue.put(\n (utc_millisec(), 'poll')\n )\n\n if self.age_out['sudden_death']:\n self._actor_queue.put(\n (utc_millisec(), 'sudden_death')\n )\n\n self._actor_queue.put(\n (utc_millisec(), 'age_out')\n )\n self._actor_queue.put(\n (utc_millisec(), 'gc')\n )\n\n try:\n self._huppable_wait(self.interval*1000)\n except gevent.GreenletExit:\n break\n\n def _age_out_loop(self):\n while True:\n with self.state_lock:\n if self.state != ft_states.STARTED:\n break\n\n self._actor_queue.put(\n (utc_millisec(), 'age_out')\n )\n\n if self.age_out['interval'] is None:\n break\n\n try:\n gevent.sleep(self.age_out['interval'])\n except gevent.GreenletExit:\n break\n\n def _calc_age_out(self, indicator, attributes):\n t = attributes.get('type', None)\n if t is None or t not in self.age_out:\n sel = self.age_out['default']\n else:\n sel = self.age_out[t]\n\n if sel is None:\n return _MAX_AGE_OUT\n\n b = attributes[sel['base']]\n\n return b + sel['offset']\n\n def _huppable_wait(self, deltat):\n while deltat < 0:\n LOG.warning(\n 'Time for processing exceeded interval for %s',\n self.name\n )\n deltat += self.interval*1000\n\n LOG.info('hup is clear: %r', self.poll_event.is_set())\n hup_called = self.poll_event.wait(timeout=deltat/1000.0)\n if hup_called:\n LOG.debug('%s - clearing poll event', self.name)\n self.poll_event.clear()\n\n def mgmtbus_status(self):\n result = super(BasePollerFT, self).mgmtbus_status()\n result['last_run'] = self.last_run\n result['last_successful_run'] = self.last_successful_run\n result['sub_state'] = self.sub_state[0]\n\n if self.sub_state[1] is not None:\n result['sub_state_message'] = self.sub_state[1]\n\n return result\n\n def mgmtbus_signal(self, source=None, signal=None, **kwargs):\n if signal != 'flush':\n super(BasePollerFT, self).mgmtbus_signal(\n source=source,\n signal=signal,\n **kwargs\n )\n\n self._actor_queue.put(\n (utc_millisec(), 'flush')\n )\n self._actor_queue.put(\n (utc_millisec(), 'gc')\n )\n\n @property\n def sub_state(self):\n return (self._sub_state, self._sub_state_message)\n\n @sub_state.setter\n def sub_state(self, value):\n if (type(value) == tuple):\n self._sub_state = value[0]\n self._sub_state_message = value[1]\n else:\n self._sub_state = value\n self._sub_state_message = None\n\n self.publish_status(force=True)\n\n def hup(self, source=None):\n LOG.info('%s - hup received, force polling', self.name)\n self.poll_event.set()\n\n def length(self, source=None):\n return self.table.length()\n\n def start(self):\n super(BasePollerFT, self).start()\n\n if self._actor_glet is not None:\n return\n\n self._actor_glet = gevent.spawn(\n self._actor_loop\n )\n self._poll_glet = gevent.spawn_later(\n random.randint(0, 2),\n self._poll_loop\n )\n\n self._age_out_glet = gevent.spawn(\n self._age_out_loop\n )\n\n def stop(self):\n super(BasePollerFT, self).stop()\n\n if self._actor_glet is None:\n return\n\n self._actor_glet.kill()\n self._poll_glet.kill()\n self._age_out_glet.kill()\n\n self.table.close()\n\n LOG.info(\"%s - # indicators: %d\", self.name, self.table.length())\n\n @staticmethod\n def gc(name, config=None):\n base.BaseFT.gc(name, config=config)\n shutil.rmtree(name, ignore_errors=True)\n shutil.rmtree('{}.aggregate-temp'.format(name), ignore_errors=True)\n" }, { "path": "minemeld/ft/cif.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport os\n\nimport arrow\nimport ujson\nimport yaml\n\nimport cifsdk.client\nimport cifsdk.constants\n\nfrom . import basepoller\n\nLOG = logging.getLogger(__name__)\n\n\nclass Feed(basepoller.BasePollerFT):\n def configure(self):\n super(Feed, self).configure()\n\n self.token = None\n\n self.remote = self.config.get('remote', None)\n self.verify_cert = self.config.get('verify_cert', True)\n self.filters = self.config.get('filters', None)\n self.initial_days = self.config.get('initial_days', 7)\n self.prefix = self.config.get('prefix', 'cif')\n\n self.fields = self.config.get('fields', cifsdk.constants.FIELDS)\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.token = sconfig.get('token', None)\n if self.token is not None:\n LOG.info('%s - token set', self.name)\n\n self.remote = sconfig.get('remote', self.remote)\n self.verify_cert = sconfig.get('verify_cert', self.verify_cert)\n filters = sconfig.get('filters', self.filters)\n if filters is not None:\n if self.filters is not None:\n self.filters.update(filters)\n else:\n self.filters = filters\n\n def _process_item(self, item):\n indicator = item.get('observable', None)\n if indicator is None:\n LOG.error('%s - no observable in item', self.name)\n return [[None, None]]\n\n otype = item.get('otype', None)\n if otype is None:\n LOG.error('%s - no otype in item', self.name)\n return [[None, None]]\n\n if otype == 'ipv4':\n type_ = 'IPv4'\n elif otype == 'ipv6':\n type_ = 'IPv6'\n elif otype == 'fqdn':\n type_ = 'domain'\n elif otype == 'url':\n type_ = 'URL'\n else:\n LOG.error('%s - unahndled otype %s', self.name, otype)\n return [[None, None]]\n\n attributes = {\n 'type': type_\n }\n for field in self.fields:\n if field in ['observable', 'otype', 'confidence']:\n continue\n\n if field not in item:\n continue\n attributes['%s_%s' % (self.prefix, field)] = item[field]\n\n if 'confidence' in item:\n attributes['confidence'] = item['confidence']\n\n LOG.debug('%s - %s: %s', self.name, indicator, attributes)\n\n return [[indicator, attributes]]\n\n def _build_iterator(self, now):\n if self.token is None or self.remote is None or self.filters is None:\n LOG.info(\n '%s - token, remote or filters not set, poll not performed',\n self.name\n )\n raise RuntimeError(\n '%s - token, remote or filters not set, poll not performed' % self.name\n )\n\n filters = {}\n filters.update(self.filters)\n\n days = filters.pop('days', self.initial_days)\n now = arrow.get(now/1000.0)\n\n filters['reporttimeend'] = '{0}Z'.format(\n now.format('YYYY-MM-DDTHH:mm:ss')\n )\n if self.last_successful_run is None:\n filters['reporttime'] = '{0}Z'.format(\n now.shift(days=-days).format('YYYY-MM-DDTHH:mm:ss')\n )\n else:\n filters['reporttime'] = '{0}Z'.format(\n arrow.get(self.last_successful_run/1000.0).format('YYYY-MM-DDTHH:mm:ss')\n )\n LOG.debug('%s - filters: %s', self.name, filters)\n\n cifclient = cifsdk.client.Client(\n token=self.token,\n remote=self.remote,\n verify_ssl=self.verify_cert,\n timeout=900\n )\n\n try:\n ret = cifclient.search(filters=filters, decode=False)\n\n except SystemExit as e:\n raise RuntimeError(str(e))\n\n return ujson.loads(ret)\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(Feed, self).hup(source=source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n" }, { "path": "minemeld/ft/ciscoise.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport os\nimport yaml\nimport minemeld.packages.ise.ers\nfrom . import basepoller\n\nLOG = logging.getLogger(__name__)\n\n\nclass ErsSgt(basepoller.BasePollerFT):\n def configure(self):\n super(ErsSgt, self).configure()\n\n self.kwargs = {}\n for x in ['hostname', 'username', 'password',\n 'verify_cert', 'timeout']:\n if x == 'verify_cert':\n self.kwargs['verify'] = self.config.get(x, None)\n else:\n self.kwargs[x] = self.config.get(x, None)\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n self.prefix = self.config.get('prefix', 'ise_sgt')\n\n d = self.kwargs.copy()\n if d['password']:\n d['password'] = '*' * 6\n LOG.debug('%s prefix: %s', d, self.prefix)\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except IOError as e:\n LOG.info('%s - No side config: %s', self.name, e)\n return\n\n if sconfig is None:\n LOG.info('%s - Empty side config: %s', self.name,\n self.side_config_path)\n return\n\n for x in ['hostname', 'username', 'password',\n 'verify_cert', 'timeout']:\n v = sconfig.get(x, None)\n if v is not None and x == 'verify_cert':\n self.kwargs['verify'] = v\n elif v is not None:\n self.kwargs[x] = v\n\n def _process_item(self, item):\n return [[item['ip'], {'type': 'IPv4', self.prefix: item['sgt']}]]\n\n def _build_iterator(self, now):\n def indicators(ips_sgts_map):\n LOG.debug('SGT indicators #%d %s', len(api.ips_sgts_map),\n api.ips_sgts_map)\n for item in ips_sgts_map:\n yield {'ip': item, 'sgt': ips_sgts_map[item]}\n\n try:\n api = minemeld.packages.ise.ers.IseErs(**self.kwargs)\n except minemeld.packages.ise.ers.IseErsError as e:\n # missing arguments\n x = '%s: poll not performed: %s' % (self.name, e)\n LOG.info('%s', x)\n raise RuntimeError(x)\n\n api.sgts_ips_map()\n\n return indicators(api.ips_sgts_map)\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(ErsSgt, self).hup(source=source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n" }, { "path": "minemeld/ft/cofense.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements minemeld.ft.cofense.Triage, the Miner node for\nCofense Triage API.\n\"\"\"\n\nimport os\nimport yaml\nimport requests\nimport itertools\nimport logging\nimport math\nimport pytz\nfrom datetime import timedelta\nfrom urlparse import urljoin\n\nfrom . import basepoller\nfrom .utils import interval_in_sec, EPOCH\n\nLOG = logging.getLogger(__name__)\n\n\n_TRIAGE_API_CALL_PATH = '/api/public/v1/triage_threat_indicators'\n_API_USER_AGENT = 'Cofense Intelligence (minemeld)'\n\n_RESULTS_PER_PAGE = 50\n\n\nclass Triage(basepoller.BasePollerFT):\n def configure(self):\n super(Triage, self).configure()\n\n self.polling_timeout = self.config.get('polling_timeout', 20)\n\n self.prefix = self.config.get('prefix', 'cofense')\n initial_interval = self.config.get('initial_interval', '30d')\n self.initial_interval = interval_in_sec(initial_interval)\n if self.initial_interval is None:\n LOG.error(\n '%s - wrong initial_interval format: %s',\n self.name, initial_interval\n )\n self.initial_interval = interval_in_sec('30d')\n\n self.source_name = self.config.get('source_name', 'cofense.triage')\n self.headers = {'user-agent': _API_USER_AGENT}\n self.confidence_map = self.config.get('confidence_map', {\n 'Malicious': 100,\n 'Suspicious': 50\n })\n\n self.verify_cert = self.config.get('verify_cert', True)\n self.api_domain = self.config.get('api_domain', None)\n self.api_account = self.config.get('api_account', None)\n self.api_token = self.config.get('api_token', None)\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n api_domain = sconfig.get('api_domain', None)\n if api_domain is not None:\n self.api_domain = api_domain\n LOG.info('%s - API Domain set', self.name)\n\n api_account = sconfig.get('api_account', None)\n if api_account is not None:\n self.api_account = api_account\n LOG.info('%s - API Account set', self.name)\n\n api_token = sconfig.get('api_token', None)\n if api_token is not None:\n self.api_token = api_token\n LOG.info('%s - API Token set', self.name)\n\n verify_cert = sconfig.get('verify_cert', None)\n if verify_cert is not None:\n self.verify_cert = verify_cert\n LOG.info('%s - Verify Cert set', self.name)\n\n def _process_item(self, item):\n LOG.debug('{} - item: {}'.format(self.name, item))\n\n report_id = item.get('report_id', None)\n type_ = item.get('threat_key', None)\n indicator = item.get('threat_value', None)\n level = item.get('threat_level', None)\n if type_ is None or indicator is None:\n LOG.error('{} - entry with no value or type: {!r}'.format(self.name, item))\n return []\n\n if level not in self.confidence_map:\n LOG.info('{} - threat_level {} not in cofidence map: indicator ignored'.format(self.name, level))\n return []\n\n if type_ == 'URL':\n type_ = 'URL'\n elif type_ == 'Domain':\n type_ = 'Domain'\n elif type_ == 'MD5':\n type_ = 'md5'\n elif type_ == 'SHA256':\n type_ = 'sha256'\n else:\n LOG.error('{} - unknown indicator type: {!r}'.format(self.name, item))\n return []\n\n value = dict(type=type_)\n if report_id is not None:\n value['{}_report_id'.format(self.prefix)] = report_id\n if level is not None:\n value['{}_threat_level'.format(self.prefix)] = level\n\n value['confidence'] = self.confidence_map[level]\n \n return [[indicator, value]]\n\n def _build_iterator(self, now):\n if self.api_domain is None or self.api_account is None or self.api_token is None:\n raise RuntimeError('%s - credentials not set' % self.name)\n\n poll_start = self.last_successful_run\n if self.last_successful_run is None:\n poll_start = now - (self.initial_interval * 1000)\n dt_poll_start = EPOCH + timedelta(milliseconds=poll_start)\n\n LOG.debug('{} - polling start: {}'.format(self.name, dt_poll_start))\n num_of_pages = self._check_number_of_pages(dt_poll_start)\n LOG.info(\"{} - polling: start date: {!r} number of pages: {!r}\".format(\n self.name, dt_poll_start, num_of_pages\n ))\n\n return self._iterate_over_pages(dt_poll_start, num_of_pages)\n\n def _check_number_of_pages(self, dt_poll_start):\n r = self._perform_api_call(dt_poll_start)\n\n total_entries = r.headers['Total']\n LOG.info('{} - polling total entries: {}'.format(self.name, total_entries))\n\n return int(math.ceil(int(total_entries)/float(_RESULTS_PER_PAGE)))\n\n def _iterate_over_pages(self, start_date, num_of_pages):\n for page_num in xrange(1, num_of_pages+1):\n r = self._perform_api_call(start_date, page_num)\n\n processed_data = r.json()\n for entry in processed_data:\n yield entry\n\n def _perform_api_call(self, start_date, page=None):\n headers = self.headers.copy()\n headers['Authorization'] = 'Token token={}:{}'.format(self.api_account, self.api_token)\n\n params = {\n \"per_page\": _RESULTS_PER_PAGE,\n \"start_date\": start_date.strftime('%Y-%m-%dT%H:%M')\n }\n if page is not None:\n params['page'] = page\n\n request_url = urljoin(self.api_domain, _TRIAGE_API_CALL_PATH)\n r = requests.get(request_url, \n params=params,\n verify=self.verify_cert,\n headers=headers\n )\n\n r.raise_for_status()\n\n return r\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(Triage, self).hup(source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n" }, { "path": "minemeld/ft/condition/BoolExpr.g4", "content": "grammar BoolExpr;\n\n@header {\n# flake8: noqa\n}\n\nbooleanExpression\n : expression comparator value\n ;\n\nexpression\n : JAVASCRIPTIDENTIFIER\n | functionExpression\n ;\n\nfunctionExpression\n : JAVASCRIPTIDENTIFIER ( noArgs | oneOrMoreArgs )\n ;\n\nnoArgs\n : '(' ')'\n ;\n\noneOrMoreArgs\n : '(' expression (',' (expression|value))* ')'\n ;\n\nJAVASCRIPTIDENTIFIER\n : [a-zA-Z_$][a-zA-Z_$0-9]*\n ;\n\ncomparator\n : '<'\n | '<='\n | '=='\n | '>='\n | '>'\n | '!='\n ;\n\nvalue\n : STRING\n | NUMBER\n | 'true'\n | 'false'\n | 'null'\n ;\n\nSTRING \n : '\"' (~[\"\\\\])* '\"' \n | '\\'' (~['\\\\])* '\\'' \n ;\n\nNUMBER : '-'? INT ;\n\nfragment INT : '0' | [1-9] [0-9]* ; // no leading zeros\n\nWS : [ \\t\\n\\r]+ -> skip ;\n" }, { "path": "minemeld/ft/condition/BoolExpr.tokens", "content": "T__0=1\nT__1=2\nT__2=3\nT__3=4\nT__4=5\nT__5=6\nT__6=7\nT__7=8\nT__8=9\nT__9=10\nT__10=11\nT__11=12\nJAVASCRIPTIDENTIFIER=13\nSTRING=14\nNUMBER=15\nWS=16\n'('=1\n')'=2\n','=3\n'<'=4\n'<='=5\n'=='=6\n'>='=7\n'>'=8\n'!='=9\n'true'=10\n'false'=11\n'null'=12\n" }, { "path": "minemeld/ft/condition/BoolExprLexer.py", "content": "# Generated from BoolExpr.g4 by ANTLR 4.7.1\n# encoding: utf-8\nfrom __future__ import print_function\nfrom antlr4 import *\nfrom io import StringIO\nimport sys\n\n\n# flake8: noqa\n\n\ndef serializedATN():\n with StringIO() as buf:\n buf.write(u\"\\3\\u608b\\ua72a\\u8133\\ub9ed\\u417c\\u3be7\\u7786\\u5964\\2\")\n buf.write(u\"\\22z\\b\\1\\4\\2\\t\\2\\4\\3\\t\\3\\4\\4\\t\\4\\4\\5\\t\\5\\4\\6\\t\\6\\4\\7\")\n buf.write(u\"\\t\\7\\4\\b\\t\\b\\4\\t\\t\\t\\4\\n\\t\\n\\4\\13\\t\\13\\4\\f\\t\\f\\4\\r\\t\")\n buf.write(u\"\\r\\4\\16\\t\\16\\4\\17\\t\\17\\4\\20\\t\\20\\4\\21\\t\\21\\4\\22\\t\\22\")\n buf.write(u\"\\3\\2\\3\\2\\3\\3\\3\\3\\3\\4\\3\\4\\3\\5\\3\\5\\3\\6\\3\\6\\3\\6\\3\\7\\3\\7\")\n buf.write(u\"\\3\\7\\3\\b\\3\\b\\3\\b\\3\\t\\3\\t\\3\\n\\3\\n\\3\\n\\3\\13\\3\\13\\3\\13\\3\")\n buf.write(u\"\\13\\3\\13\\3\\f\\3\\f\\3\\f\\3\\f\\3\\f\\3\\f\\3\\r\\3\\r\\3\\r\\3\\r\\3\\r\")\n buf.write(u\"\\3\\16\\3\\16\\7\\16N\\n\\16\\f\\16\\16\\16Q\\13\\16\\3\\17\\3\\17\\7\\17\")\n buf.write(u\"U\\n\\17\\f\\17\\16\\17X\\13\\17\\3\\17\\3\\17\\3\\17\\7\\17]\\n\\17\\f\")\n buf.write(u\"\\17\\16\\17`\\13\\17\\3\\17\\5\\17c\\n\\17\\3\\20\\5\\20f\\n\\20\\3\\20\")\n buf.write(u\"\\3\\20\\3\\21\\3\\21\\3\\21\\7\\21m\\n\\21\\f\\21\\16\\21p\\13\\21\\5\\21\")\n buf.write(u\"r\\n\\21\\3\\22\\6\\22u\\n\\22\\r\\22\\16\\22v\\3\\22\\3\\22\\2\\2\\23\\3\")\n buf.write(u\"\\3\\5\\4\\7\\5\\t\\6\\13\\7\\r\\b\\17\\t\\21\\n\\23\\13\\25\\f\\27\\r\\31\")\n buf.write(u\"\\16\\33\\17\\35\\20\\37\\21!\\2#\\22\\3\\2\\t\\6\\2&&C\\\\aac|\\7\\2&\")\n buf.write(u\"&\\62;C\\\\aac|\\4\\2$$^^\\4\\2))^^\\3\\2\\63;\\3\\2\\62;\\5\\2\\13\\f\")\n buf.write(u\"\\17\\17\\\"\\\"\\2\\u0080\\2\\3\\3\\2\\2\\2\\2\\5\\3\\2\\2\\2\\2\\7\\3\\2\\2\")\n buf.write(u\"\\2\\2\\t\\3\\2\\2\\2\\2\\13\\3\\2\\2\\2\\2\\r\\3\\2\\2\\2\\2\\17\\3\\2\\2\\2\")\n buf.write(u\"\\2\\21\\3\\2\\2\\2\\2\\23\\3\\2\\2\\2\\2\\25\\3\\2\\2\\2\\2\\27\\3\\2\\2\\2\")\n buf.write(u\"\\2\\31\\3\\2\\2\\2\\2\\33\\3\\2\\2\\2\\2\\35\\3\\2\\2\\2\\2\\37\\3\\2\\2\\2\")\n buf.write(u\"\\2#\\3\\2\\2\\2\\3%\\3\\2\\2\\2\\5\\'\\3\\2\\2\\2\\7)\\3\\2\\2\\2\\t+\\3\\2\")\n buf.write(u\"\\2\\2\\13-\\3\\2\\2\\2\\r\\60\\3\\2\\2\\2\\17\\63\\3\\2\\2\\2\\21\\66\\3\\2\")\n buf.write(u\"\\2\\2\\238\\3\\2\\2\\2\\25;\\3\\2\\2\\2\\27@\\3\\2\\2\\2\\31F\\3\\2\\2\\2\")\n buf.write(u\"\\33K\\3\\2\\2\\2\\35b\\3\\2\\2\\2\\37e\\3\\2\\2\\2!q\\3\\2\\2\\2#t\\3\\2\")\n buf.write(u\"\\2\\2%&\\7*\\2\\2&\\4\\3\\2\\2\\2\\'(\\7+\\2\\2(\\6\\3\\2\\2\\2)*\\7.\\2\")\n buf.write(u\"\\2*\\b\\3\\2\\2\\2+,\\7>\\2\\2,\\n\\3\\2\\2\\2-.\\7>\\2\\2./\\7?\\2\\2/\")\n buf.write(u\"\\f\\3\\2\\2\\2\\60\\61\\7?\\2\\2\\61\\62\\7?\\2\\2\\62\\16\\3\\2\\2\\2\\63\")\n buf.write(u\"\\64\\7@\\2\\2\\64\\65\\7?\\2\\2\\65\\20\\3\\2\\2\\2\\66\\67\\7@\\2\\2\\67\")\n buf.write(u\"\\22\\3\\2\\2\\289\\7#\\2\\29:\\7?\\2\\2:\\24\\3\\2\\2\\2;<\\7v\\2\\2<=\")\n buf.write(u\"\\7t\\2\\2=>\\7w\\2\\2>?\\7g\\2\\2?\\26\\3\\2\\2\\2@A\\7h\\2\\2AB\\7c\\2\")\n buf.write(u\"\\2BC\\7n\\2\\2CD\\7u\\2\\2DE\\7g\\2\\2E\\30\\3\\2\\2\\2FG\\7p\\2\\2GH\")\n buf.write(u\"\\7w\\2\\2HI\\7n\\2\\2IJ\\7n\\2\\2J\\32\\3\\2\\2\\2KO\\t\\2\\2\\2LN\\t\\3\")\n buf.write(u\"\\2\\2ML\\3\\2\\2\\2NQ\\3\\2\\2\\2OM\\3\\2\\2\\2OP\\3\\2\\2\\2P\\34\\3\\2\")\n buf.write(u\"\\2\\2QO\\3\\2\\2\\2RV\\7$\\2\\2SU\\n\\4\\2\\2TS\\3\\2\\2\\2UX\\3\\2\\2\\2\")\n buf.write(u\"VT\\3\\2\\2\\2VW\\3\\2\\2\\2WY\\3\\2\\2\\2XV\\3\\2\\2\\2Yc\\7$\\2\\2Z^\\7\")\n buf.write(u\")\\2\\2[]\\n\\5\\2\\2\\\\[\\3\\2\\2\\2]`\\3\\2\\2\\2^\\\\\\3\\2\\2\\2^_\\3\\2\")\n buf.write(u\"\\2\\2_a\\3\\2\\2\\2`^\\3\\2\\2\\2ac\\7)\\2\\2bR\\3\\2\\2\\2bZ\\3\\2\\2\\2\")\n buf.write(u\"c\\36\\3\\2\\2\\2df\\7/\\2\\2ed\\3\\2\\2\\2ef\\3\\2\\2\\2fg\\3\\2\\2\\2g\")\n buf.write(u\"h\\5!\\21\\2h \\3\\2\\2\\2ir\\7\\62\\2\\2jn\\t\\6\\2\\2km\\t\\7\\2\\2lk\")\n buf.write(u\"\\3\\2\\2\\2mp\\3\\2\\2\\2nl\\3\\2\\2\\2no\\3\\2\\2\\2or\\3\\2\\2\\2pn\\3\")\n buf.write(u\"\\2\\2\\2qi\\3\\2\\2\\2qj\\3\\2\\2\\2r\\\"\\3\\2\\2\\2su\\t\\b\\2\\2ts\\3\\2\")\n buf.write(u\"\\2\\2uv\\3\\2\\2\\2vt\\3\\2\\2\\2vw\\3\\2\\2\\2wx\\3\\2\\2\\2xy\\b\\22\\2\")\n buf.write(u\"\\2y$\\3\\2\\2\\2\\13\\2OV^benqv\\3\\b\\2\\2\")\n return buf.getvalue()\n\n\nclass BoolExprLexer(Lexer):\n\n atn = ATNDeserializer().deserialize(serializedATN())\n\n decisionsToDFA = [ DFA(ds, i) for i, ds in enumerate(atn.decisionToState) ]\n\n T__0 = 1\n T__1 = 2\n T__2 = 3\n T__3 = 4\n T__4 = 5\n T__5 = 6\n T__6 = 7\n T__7 = 8\n T__8 = 9\n T__9 = 10\n T__10 = 11\n T__11 = 12\n JAVASCRIPTIDENTIFIER = 13\n STRING = 14\n NUMBER = 15\n WS = 16\n\n channelNames = [ u\"DEFAULT_TOKEN_CHANNEL\", u\"HIDDEN\" ]\n\n modeNames = [ u\"DEFAULT_MODE\" ]\n\n literalNames = [ u\"\",\n u\"'('\", u\"')'\", u\"','\", u\"'<'\", u\"'<='\", u\"'=='\", u\"'>='\", u\"'>'\", \n u\"'!='\", u\"'true'\", u\"'false'\", u\"'null'\" ]\n\n symbolicNames = [ u\"\",\n u\"JAVASCRIPTIDENTIFIER\", u\"STRING\", u\"NUMBER\", u\"WS\" ]\n\n ruleNames = [ u\"T__0\", u\"T__1\", u\"T__2\", u\"T__3\", u\"T__4\", u\"T__5\", \n u\"T__6\", u\"T__7\", u\"T__8\", u\"T__9\", u\"T__10\", u\"T__11\", \n u\"JAVASCRIPTIDENTIFIER\", u\"STRING\", u\"NUMBER\", u\"INT\", \n u\"WS\" ]\n\n grammarFileName = u\"BoolExpr.g4\"\n\n def __init__(self, input=None, output=sys.stdout):\n super(BoolExprLexer, self).__init__(input, output=output)\n self.checkVersion(\"4.7.1\")\n self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache())\n self._actions = None\n self._predicates = None\n\n\n" }, { "path": "minemeld/ft/condition/BoolExprLexer.tokens", "content": "T__0=1\nT__1=2\nT__2=3\nT__3=4\nT__4=5\nT__5=6\nT__6=7\nT__7=8\nT__8=9\nT__9=10\nT__10=11\nT__11=12\nJAVASCRIPTIDENTIFIER=13\nSTRING=14\nNUMBER=15\nWS=16\n'('=1\n')'=2\n','=3\n'<'=4\n'<='=5\n'=='=6\n'>='=7\n'>'=8\n'!='=9\n'true'=10\n'false'=11\n'null'=12\n" }, { "path": "minemeld/ft/condition/BoolExprListener.py", "content": "# Generated from BoolExpr.g4 by ANTLR 4.7.1\nfrom antlr4 import *\n\n# flake8: noqa\n\n\n# This class defines a complete listener for a parse tree produced by BoolExprParser.\nclass BoolExprListener(ParseTreeListener):\n\n # Enter a parse tree produced by BoolExprParser#booleanExpression.\n def enterBooleanExpression(self, ctx):\n pass\n\n # Exit a parse tree produced by BoolExprParser#booleanExpression.\n def exitBooleanExpression(self, ctx):\n pass\n\n\n # Enter a parse tree produced by BoolExprParser#expression.\n def enterExpression(self, ctx):\n pass\n\n # Exit a parse tree produced by BoolExprParser#expression.\n def exitExpression(self, ctx):\n pass\n\n\n # Enter a parse tree produced by BoolExprParser#functionExpression.\n def enterFunctionExpression(self, ctx):\n pass\n\n # Exit a parse tree produced by BoolExprParser#functionExpression.\n def exitFunctionExpression(self, ctx):\n pass\n\n\n # Enter a parse tree produced by BoolExprParser#noArgs.\n def enterNoArgs(self, ctx):\n pass\n\n # Exit a parse tree produced by BoolExprParser#noArgs.\n def exitNoArgs(self, ctx):\n pass\n\n\n # Enter a parse tree produced by BoolExprParser#oneOrMoreArgs.\n def enterOneOrMoreArgs(self, ctx):\n pass\n\n # Exit a parse tree produced by BoolExprParser#oneOrMoreArgs.\n def exitOneOrMoreArgs(self, ctx):\n pass\n\n\n # Enter a parse tree produced by BoolExprParser#comparator.\n def enterComparator(self, ctx):\n pass\n\n # Exit a parse tree produced by BoolExprParser#comparator.\n def exitComparator(self, ctx):\n pass\n\n\n # Enter a parse tree produced by BoolExprParser#value.\n def enterValue(self, ctx):\n pass\n\n # Exit a parse tree produced by BoolExprParser#value.\n def exitValue(self, ctx):\n pass\n\n\n" }, { "path": "minemeld/ft/condition/BoolExprParser.py", "content": "# Generated from BoolExpr.g4 by ANTLR 4.7.1\n# encoding: utf-8\nfrom __future__ import print_function\nfrom antlr4 import *\nfrom io import StringIO\nimport sys\n\n\n# flake8: noqa\n\ndef serializedATN():\n with StringIO() as buf:\n buf.write(u\"\\3\\u608b\\ua72a\\u8133\\ub9ed\\u417c\\u3be7\\u7786\\u5964\\3\")\n buf.write(u\"\\22\\63\\4\\2\\t\\2\\4\\3\\t\\3\\4\\4\\t\\4\\4\\5\\t\\5\\4\\6\\t\\6\\4\\7\\t\")\n buf.write(u\"\\7\\4\\b\\t\\b\\3\\2\\3\\2\\3\\2\\3\\2\\3\\3\\3\\3\\5\\3\\27\\n\\3\\3\\4\\3\\4\")\n buf.write(u\"\\3\\4\\5\\4\\34\\n\\4\\3\\5\\3\\5\\3\\5\\3\\6\\3\\6\\3\\6\\3\\6\\3\\6\\5\\6&\")\n buf.write(u\"\\n\\6\\7\\6(\\n\\6\\f\\6\\16\\6+\\13\\6\\3\\6\\3\\6\\3\\7\\3\\7\\3\\b\\3\\b\")\n buf.write(u\"\\3\\b\\2\\2\\t\\2\\4\\6\\b\\n\\f\\16\\2\\4\\3\\2\\6\\13\\4\\2\\f\\16\\20\\21\")\n buf.write(u\"\\2/\\2\\20\\3\\2\\2\\2\\4\\26\\3\\2\\2\\2\\6\\30\\3\\2\\2\\2\\b\\35\\3\\2\\2\")\n buf.write(u\"\\2\\n \\3\\2\\2\\2\\f.\\3\\2\\2\\2\\16\\60\\3\\2\\2\\2\\20\\21\\5\\4\\3\\2\")\n buf.write(u\"\\21\\22\\5\\f\\7\\2\\22\\23\\5\\16\\b\\2\\23\\3\\3\\2\\2\\2\\24\\27\\7\\17\")\n buf.write(u\"\\2\\2\\25\\27\\5\\6\\4\\2\\26\\24\\3\\2\\2\\2\\26\\25\\3\\2\\2\\2\\27\\5\\3\")\n buf.write(u\"\\2\\2\\2\\30\\33\\7\\17\\2\\2\\31\\34\\5\\b\\5\\2\\32\\34\\5\\n\\6\\2\\33\")\n buf.write(u\"\\31\\3\\2\\2\\2\\33\\32\\3\\2\\2\\2\\34\\7\\3\\2\\2\\2\\35\\36\\7\\3\\2\\2\")\n buf.write(u\"\\36\\37\\7\\4\\2\\2\\37\\t\\3\\2\\2\\2 !\\7\\3\\2\\2!)\\5\\4\\3\\2\\\"%\\7\")\n buf.write(u\"\\5\\2\\2#&\\5\\4\\3\\2$&\\5\\16\\b\\2%#\\3\\2\\2\\2%$\\3\\2\\2\\2&(\\3\\2\")\n buf.write(u\"\\2\\2\\'\\\"\\3\\2\\2\\2(+\\3\\2\\2\\2)\\'\\3\\2\\2\\2)*\\3\\2\\2\\2*,\\3\\2\")\n buf.write(u\"\\2\\2+)\\3\\2\\2\\2,-\\7\\4\\2\\2-\\13\\3\\2\\2\\2./\\t\\2\\2\\2/\\r\\3\\2\")\n buf.write(u\"\\2\\2\\60\\61\\t\\3\\2\\2\\61\\17\\3\\2\\2\\2\\6\\26\\33%)\")\n return buf.getvalue()\n\n\nclass BoolExprParser ( Parser ):\n\n grammarFileName = \"BoolExpr.g4\"\n\n atn = ATNDeserializer().deserialize(serializedATN())\n\n decisionsToDFA = [ DFA(ds, i) for i, ds in enumerate(atn.decisionToState) ]\n\n sharedContextCache = PredictionContextCache()\n\n literalNames = [ u\"\", u\"'('\", u\"')'\", u\"','\", u\"'<'\", u\"'<='\", \n u\"'=='\", u\"'>='\", u\"'>'\", u\"'!='\", u\"'true'\", u\"'false'\", \n u\"'null'\" ]\n\n symbolicNames = [ u\"\", u\"\", u\"\", u\"\", \n u\"\", u\"\", u\"\", u\"\", \n u\"\", u\"\", u\"\", u\"\", \n u\"\", u\"JAVASCRIPTIDENTIFIER\", u\"STRING\", \n u\"NUMBER\", u\"WS\" ]\n\n RULE_booleanExpression = 0\n RULE_expression = 1\n RULE_functionExpression = 2\n RULE_noArgs = 3\n RULE_oneOrMoreArgs = 4\n RULE_comparator = 5\n RULE_value = 6\n\n ruleNames = [ u\"booleanExpression\", u\"expression\", u\"functionExpression\", \n u\"noArgs\", u\"oneOrMoreArgs\", u\"comparator\", u\"value\" ]\n\n EOF = Token.EOF\n T__0=1\n T__1=2\n T__2=3\n T__3=4\n T__4=5\n T__5=6\n T__6=7\n T__7=8\n T__8=9\n T__9=10\n T__10=11\n T__11=12\n JAVASCRIPTIDENTIFIER=13\n STRING=14\n NUMBER=15\n WS=16\n\n def __init__(self, input, output=sys.stdout):\n super(BoolExprParser, self).__init__(input, output=output)\n self.checkVersion(\"4.7.1\")\n self._interp = ParserATNSimulator(self, self.atn, self.decisionsToDFA, self.sharedContextCache)\n self._predicates = None\n\n\n\n class BooleanExpressionContext(ParserRuleContext):\n\n def __init__(self, parser, parent=None, invokingState=-1):\n super(BoolExprParser.BooleanExpressionContext, self).__init__(parent, invokingState)\n self.parser = parser\n\n def expression(self):\n return self.getTypedRuleContext(BoolExprParser.ExpressionContext,0)\n\n\n def comparator(self):\n return self.getTypedRuleContext(BoolExprParser.ComparatorContext,0)\n\n\n def value(self):\n return self.getTypedRuleContext(BoolExprParser.ValueContext,0)\n\n\n def getRuleIndex(self):\n return BoolExprParser.RULE_booleanExpression\n\n def enterRule(self, listener):\n if hasattr(listener, \"enterBooleanExpression\"):\n listener.enterBooleanExpression(self)\n\n def exitRule(self, listener):\n if hasattr(listener, \"exitBooleanExpression\"):\n listener.exitBooleanExpression(self)\n\n\n\n\n def booleanExpression(self):\n\n localctx = BoolExprParser.BooleanExpressionContext(self, self._ctx, self.state)\n self.enterRule(localctx, 0, self.RULE_booleanExpression)\n try:\n self.enterOuterAlt(localctx, 1)\n self.state = 14\n self.expression()\n self.state = 15\n self.comparator()\n self.state = 16\n self.value()\n except RecognitionException as re:\n localctx.exception = re\n self._errHandler.reportError(self, re)\n self._errHandler.recover(self, re)\n finally:\n self.exitRule()\n return localctx\n\n class ExpressionContext(ParserRuleContext):\n\n def __init__(self, parser, parent=None, invokingState=-1):\n super(BoolExprParser.ExpressionContext, self).__init__(parent, invokingState)\n self.parser = parser\n\n def JAVASCRIPTIDENTIFIER(self):\n return self.getToken(BoolExprParser.JAVASCRIPTIDENTIFIER, 0)\n\n def functionExpression(self):\n return self.getTypedRuleContext(BoolExprParser.FunctionExpressionContext,0)\n\n\n def getRuleIndex(self):\n return BoolExprParser.RULE_expression\n\n def enterRule(self, listener):\n if hasattr(listener, \"enterExpression\"):\n listener.enterExpression(self)\n\n def exitRule(self, listener):\n if hasattr(listener, \"exitExpression\"):\n listener.exitExpression(self)\n\n\n\n\n def expression(self):\n\n localctx = BoolExprParser.ExpressionContext(self, self._ctx, self.state)\n self.enterRule(localctx, 2, self.RULE_expression)\n try:\n self.state = 20\n self._errHandler.sync(self)\n la_ = self._interp.adaptivePredict(self._input,0,self._ctx)\n if la_ == 1:\n self.enterOuterAlt(localctx, 1)\n self.state = 18\n self.match(BoolExprParser.JAVASCRIPTIDENTIFIER)\n pass\n\n elif la_ == 2:\n self.enterOuterAlt(localctx, 2)\n self.state = 19\n self.functionExpression()\n pass\n\n\n except RecognitionException as re:\n localctx.exception = re\n self._errHandler.reportError(self, re)\n self._errHandler.recover(self, re)\n finally:\n self.exitRule()\n return localctx\n\n class FunctionExpressionContext(ParserRuleContext):\n\n def __init__(self, parser, parent=None, invokingState=-1):\n super(BoolExprParser.FunctionExpressionContext, self).__init__(parent, invokingState)\n self.parser = parser\n\n def JAVASCRIPTIDENTIFIER(self):\n return self.getToken(BoolExprParser.JAVASCRIPTIDENTIFIER, 0)\n\n def noArgs(self):\n return self.getTypedRuleContext(BoolExprParser.NoArgsContext,0)\n\n\n def oneOrMoreArgs(self):\n return self.getTypedRuleContext(BoolExprParser.OneOrMoreArgsContext,0)\n\n\n def getRuleIndex(self):\n return BoolExprParser.RULE_functionExpression\n\n def enterRule(self, listener):\n if hasattr(listener, \"enterFunctionExpression\"):\n listener.enterFunctionExpression(self)\n\n def exitRule(self, listener):\n if hasattr(listener, \"exitFunctionExpression\"):\n listener.exitFunctionExpression(self)\n\n\n\n\n def functionExpression(self):\n\n localctx = BoolExprParser.FunctionExpressionContext(self, self._ctx, self.state)\n self.enterRule(localctx, 4, self.RULE_functionExpression)\n try:\n self.enterOuterAlt(localctx, 1)\n self.state = 22\n self.match(BoolExprParser.JAVASCRIPTIDENTIFIER)\n self.state = 25\n self._errHandler.sync(self)\n la_ = self._interp.adaptivePredict(self._input,1,self._ctx)\n if la_ == 1:\n self.state = 23\n self.noArgs()\n pass\n\n elif la_ == 2:\n self.state = 24\n self.oneOrMoreArgs()\n pass\n\n\n except RecognitionException as re:\n localctx.exception = re\n self._errHandler.reportError(self, re)\n self._errHandler.recover(self, re)\n finally:\n self.exitRule()\n return localctx\n\n class NoArgsContext(ParserRuleContext):\n\n def __init__(self, parser, parent=None, invokingState=-1):\n super(BoolExprParser.NoArgsContext, self).__init__(parent, invokingState)\n self.parser = parser\n\n\n def getRuleIndex(self):\n return BoolExprParser.RULE_noArgs\n\n def enterRule(self, listener):\n if hasattr(listener, \"enterNoArgs\"):\n listener.enterNoArgs(self)\n\n def exitRule(self, listener):\n if hasattr(listener, \"exitNoArgs\"):\n listener.exitNoArgs(self)\n\n\n\n\n def noArgs(self):\n\n localctx = BoolExprParser.NoArgsContext(self, self._ctx, self.state)\n self.enterRule(localctx, 6, self.RULE_noArgs)\n try:\n self.enterOuterAlt(localctx, 1)\n self.state = 27\n self.match(BoolExprParser.T__0)\n self.state = 28\n self.match(BoolExprParser.T__1)\n except RecognitionException as re:\n localctx.exception = re\n self._errHandler.reportError(self, re)\n self._errHandler.recover(self, re)\n finally:\n self.exitRule()\n return localctx\n\n class OneOrMoreArgsContext(ParserRuleContext):\n\n def __init__(self, parser, parent=None, invokingState=-1):\n super(BoolExprParser.OneOrMoreArgsContext, self).__init__(parent, invokingState)\n self.parser = parser\n\n def expression(self, i=None):\n if i is None:\n return self.getTypedRuleContexts(BoolExprParser.ExpressionContext)\n else:\n return self.getTypedRuleContext(BoolExprParser.ExpressionContext,i)\n\n\n def value(self, i=None):\n if i is None:\n return self.getTypedRuleContexts(BoolExprParser.ValueContext)\n else:\n return self.getTypedRuleContext(BoolExprParser.ValueContext,i)\n\n\n def getRuleIndex(self):\n return BoolExprParser.RULE_oneOrMoreArgs\n\n def enterRule(self, listener):\n if hasattr(listener, \"enterOneOrMoreArgs\"):\n listener.enterOneOrMoreArgs(self)\n\n def exitRule(self, listener):\n if hasattr(listener, \"exitOneOrMoreArgs\"):\n listener.exitOneOrMoreArgs(self)\n\n\n\n\n def oneOrMoreArgs(self):\n\n localctx = BoolExprParser.OneOrMoreArgsContext(self, self._ctx, self.state)\n self.enterRule(localctx, 8, self.RULE_oneOrMoreArgs)\n self._la = 0 # Token type\n try:\n self.enterOuterAlt(localctx, 1)\n self.state = 30\n self.match(BoolExprParser.T__0)\n self.state = 31\n self.expression()\n self.state = 39\n self._errHandler.sync(self)\n _la = self._input.LA(1)\n while _la==BoolExprParser.T__2:\n self.state = 32\n self.match(BoolExprParser.T__2)\n self.state = 35\n self._errHandler.sync(self)\n token = self._input.LA(1)\n if token in [BoolExprParser.JAVASCRIPTIDENTIFIER]:\n self.state = 33\n self.expression()\n pass\n elif token in [BoolExprParser.T__9, BoolExprParser.T__10, BoolExprParser.T__11, BoolExprParser.STRING, BoolExprParser.NUMBER]:\n self.state = 34\n self.value()\n pass\n else:\n raise NoViableAltException(self)\n\n self.state = 41\n self._errHandler.sync(self)\n _la = self._input.LA(1)\n\n self.state = 42\n self.match(BoolExprParser.T__1)\n except RecognitionException as re:\n localctx.exception = re\n self._errHandler.reportError(self, re)\n self._errHandler.recover(self, re)\n finally:\n self.exitRule()\n return localctx\n\n class ComparatorContext(ParserRuleContext):\n\n def __init__(self, parser, parent=None, invokingState=-1):\n super(BoolExprParser.ComparatorContext, self).__init__(parent, invokingState)\n self.parser = parser\n\n\n def getRuleIndex(self):\n return BoolExprParser.RULE_comparator\n\n def enterRule(self, listener):\n if hasattr(listener, \"enterComparator\"):\n listener.enterComparator(self)\n\n def exitRule(self, listener):\n if hasattr(listener, \"exitComparator\"):\n listener.exitComparator(self)\n\n\n\n\n def comparator(self):\n\n localctx = BoolExprParser.ComparatorContext(self, self._ctx, self.state)\n self.enterRule(localctx, 10, self.RULE_comparator)\n self._la = 0 # Token type\n try:\n self.enterOuterAlt(localctx, 1)\n self.state = 44\n _la = self._input.LA(1)\n if not((((_la) & ~0x3f) == 0 and ((1 << _la) & ((1 << BoolExprParser.T__3) | (1 << BoolExprParser.T__4) | (1 << BoolExprParser.T__5) | (1 << BoolExprParser.T__6) | (1 << BoolExprParser.T__7) | (1 << BoolExprParser.T__8))) != 0)):\n self._errHandler.recoverInline(self)\n else:\n self._errHandler.reportMatch(self)\n self.consume()\n except RecognitionException as re:\n localctx.exception = re\n self._errHandler.reportError(self, re)\n self._errHandler.recover(self, re)\n finally:\n self.exitRule()\n return localctx\n\n class ValueContext(ParserRuleContext):\n\n def __init__(self, parser, parent=None, invokingState=-1):\n super(BoolExprParser.ValueContext, self).__init__(parent, invokingState)\n self.parser = parser\n\n def STRING(self):\n return self.getToken(BoolExprParser.STRING, 0)\n\n def NUMBER(self):\n return self.getToken(BoolExprParser.NUMBER, 0)\n\n def getRuleIndex(self):\n return BoolExprParser.RULE_value\n\n def enterRule(self, listener):\n if hasattr(listener, \"enterValue\"):\n listener.enterValue(self)\n\n def exitRule(self, listener):\n if hasattr(listener, \"exitValue\"):\n listener.exitValue(self)\n\n\n\n\n def value(self):\n\n localctx = BoolExprParser.ValueContext(self, self._ctx, self.state)\n self.enterRule(localctx, 12, self.RULE_value)\n self._la = 0 # Token type\n try:\n self.enterOuterAlt(localctx, 1)\n self.state = 46\n _la = self._input.LA(1)\n if not((((_la) & ~0x3f) == 0 and ((1 << _la) & ((1 << BoolExprParser.T__9) | (1 << BoolExprParser.T__10) | (1 << BoolExprParser.T__11) | (1 << BoolExprParser.STRING) | (1 << BoolExprParser.NUMBER))) != 0)):\n self._errHandler.recoverInline(self)\n else:\n self._errHandler.reportMatch(self)\n self.consume()\n except RecognitionException as re:\n localctx.exception = re\n self._errHandler.reportError(self, re)\n self._errHandler.recover(self, re)\n finally:\n self.exitRule()\n return localctx\n\n\n\n\n\n" }, { "path": "minemeld/ft/condition/__init__.py", "content": "from .BoolExprParser import BoolExprParser # noqa\nfrom .BoolExprLexer import BoolExprLexer # noqa\nfrom .BoolExprListener import BoolExprListener # noqa\nfrom .interface import Condition # noqa\n" }, { "path": "minemeld/ft/condition/interface.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport jmespath\nimport logging\nimport antlr4\nimport operator\n\nfrom .BoolExprParser import BoolExprParser # noqa\nfrom .BoolExprLexer import BoolExprLexer # noqa\nfrom .BoolExprListener import BoolExprListener # noqa\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass _BECompiler(BoolExprListener):\n def exitExpression(self, ctx):\n self.expression = jmespath.compile(ctx.getText())\n\n def exitComparator(self, ctx):\n comparator = ctx.getText()\n if comparator == '==':\n self.comparator = operator.eq\n elif comparator == '<':\n self.comparator = operator.lt\n elif comparator == '<=':\n self.comparator = operator.le\n elif comparator == '>':\n self.comparator = operator.gt\n elif comparator == '>=':\n self.comparator = operator.ge\n elif comparator == '!=':\n self.comparator = operator.ne\n\n def exitValue(self, ctx):\n if ctx.STRING() is not None:\n self.value = ctx.STRING().getText()[1:-1]\n elif ctx.NUMBER() is not None:\n self.value = int(ctx.NUMBER().getText())\n elif ctx.getText() == 'null':\n self.value = None\n elif ctx.getText() == 'false':\n self.value = False\n elif ctx.getText() == 'true':\n self.value = True\n\n\nclass Condition(object):\n def __init__(self, s):\n self.expression, self.comparator, self.value = self._parse_boolexpr(s)\n\n def _parse_boolexpr(self, s):\n lexer = BoolExprLexer(\n antlr4.InputStream(s)\n )\n stream = antlr4.CommonTokenStream(lexer)\n parser = BoolExprParser(stream)\n tree = parser.booleanExpression()\n\n eb = _BECompiler()\n walker = antlr4.ParseTreeWalker()\n walker.walk(eb, tree)\n\n return eb.expression, eb.comparator, eb.value\n\n def eval(self, i):\n try:\n r = self.expression.search(i)\n except jmespath.exceptions.JMESPathError:\n LOG.debug(\"Exception in eval: \", exc_info=True)\n r = None\n\n # XXX this is a workaround for a bug in JMESPath\n if r == 'null':\n r = None\n\n return self.comparator(r, self.value)\n" }, { "path": "minemeld/ft/csv.py", "content": "# Copyright 2015-2020 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements minemeld.ft.csv.CSVFT, the Miner node for csv\nfeeds over HTTP/HTTPS.\n\"\"\"\n\nfrom __future__ import absolute_import\n\nimport logging\nimport re\nimport os.path\nimport itertools\nimport csv\nimport requests\nimport yaml\nimport shutil\nfrom urllib3.response import GzipDecoder\n\nfrom . import basepoller\n\nLOG = logging.getLogger(__name__)\n\n\nclass CSVFT(basepoller.BasePollerFT):\n \"\"\"Implements class for miners of csv feeds over http/https.\n\n **Config parameters**\n :url: URL of the feed.\n :polling_timeout: timeout of the polling request in seconds.\n Default: 20\n :verify_cert: boolean, if *true* feed HTTPS server certificate is\n verified. Default: *true*\n :ignore_regex: Python regular expression for lines that should be\n ignored. Default: *null*\n :fieldnames: list of field names in the file. If *null* the values\n in the first row of the file are used as names. Default: *null*\n :delimiter: see `csv Python module `_.\n Default: ,\n :doublequote: see `csv Python module `_.\n Default: true\n :escapechar: see `csv Python module `_.\n Default: null\n :quotechar: see `csv Python module `_.\n Default: \"\n :skipinitialspace: see `csv Python module `_.\n Default: false\n\n Example:\n Example config in YAML::\n\n url: https://sslbl.abuse.ch/blacklist/sslipblacklist.csv\n ignore_regex: '^#'\n fieldnames:\n - indicator\n - port\n - sslblabusech_type\n\n Args:\n name (str): node name, should be unique inside the graph\n chassis (object): parent chassis instance\n config (dict): node config.\n \"\"\"\n def configure(self):\n super(CSVFT, self).configure()\n\n self.polling_timeout = self.config.get('polling_timeout', 20)\n self.url = self.config.get('url', None)\n self.verify_cert = self.config.get('verify_cert', True)\n\n self.username = self.config.get('username', None)\n self.password = self.config.get('password', None)\n\n self.ignore_regex = self.config.get('ignore_regex', None)\n if self.ignore_regex is not None:\n self.ignore_regex = re.compile(self.ignore_regex)\n\n self.fieldnames = self.config.get('fieldnames', None)\n\n self.dialect = {\n 'delimiter': self.config.get('delimiter', ','),\n 'doublequote': self.config.get('doublequote', True),\n 'escapechar': self.config.get('escapechar', None),\n 'quotechar': self.config.get('quotechar', '\"'),\n 'skipinitialspace': self.config.get('skipinitialspace', False)\n }\n\n self.decode_gzip = self.config.get('decode_gzip', False)\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n username = sconfig.get('username', None)\n if username is not None:\n self.username = username\n LOG.info('%s - username set', self.name)\n\n password = sconfig.get('password', None)\n if password is not None:\n self.password = password\n LOG.info('%s - password set', self.name)\n\n def _process_item(self, item):\n item.pop(None, None) # I love this\n\n indicator = item.pop('indicator', None)\n return [[indicator, item]]\n\n def _build_request(self, now):\n auth = None\n if self.username is not None and self.password is not None:\n auth = (self.username, self.password)\n\n r = requests.Request(\n 'GET',\n self.url,\n auth=auth\n )\n\n return r.prepare()\n\n def _build_iterator(self, now):\n def _debug(x):\n LOG.info('{!r}'.format(x))\n return x\n\n _session = requests.Session()\n\n prepreq = self._build_request(now)\n\n # this is to honour the proxy environment variables\n rkwargs = _session.merge_environment_settings(\n prepreq.url,\n {}, None, None, None # defaults\n )\n rkwargs['stream'] = True\n rkwargs['verify'] = self.verify_cert\n rkwargs['timeout'] = self.polling_timeout\n\n r = _session.send(prepreq, **rkwargs)\n\n try:\n r.raise_for_status()\n except:\n LOG.debug('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n response = r.raw\n if self.decode_gzip:\n response = self._gzipped_line_splitter(r)\n\n if self.ignore_regex is not None:\n response = itertools.ifilter(\n lambda x: self.ignore_regex.match(x) is None,\n response\n )\n\n csvreader = csv.DictReader(\n response,\n fieldnames=self.fieldnames,\n **self.dialect\n )\n\n return csvreader\n\n def _gzipped_line_splitter(self, response):\n # same logic used in urllib32.response.iter_lines\n pending = None\n\n decoder = GzipDecoder()\n chunks = itertools.imap(\n decoder.decompress,\n response.iter_content(chunk_size=1024*1024)\n )\n\n for chunk in chunks:\n if pending is not None:\n chunk = pending + chunk\n\n lines = chunk.splitlines()\n\n if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:\n pending = lines.pop()\n else:\n pending = None\n\n for line in lines:\n yield line\n\n if pending is not None:\n yield pending\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(CSVFT, self).hup(source=source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n shutil.rmtree('{}_temp'.format(name), ignore_errors=True)\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n" }, { "path": "minemeld/ft/dag.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport yaml\nimport netaddr\nimport os\nimport re\nimport collections\nimport itertools\nimport shutil\n\nimport gevent\nimport gevent.queue\nimport gevent.event\n\nimport pan.xapi\n\nfrom . import base\nfrom . import actorbase\nfrom . import table\nfrom .utils import utc_millisec\n\nLOG = logging.getLogger(__name__)\n\nSUBRE = re.compile(\"[^A-Za-z0-9_]\")\n\n\nclass DevicePusher(gevent.Greenlet):\n def __init__(self, device, prefix, watermark, attributes, persistent):\n super(DevicePusher, self).__init__()\n\n self.device = device\n self.xapi = pan.xapi.PanXapi(\n tag=self.device.get('tag', None),\n api_username=self.device.get('api_username', None),\n api_password=self.device.get('api_password', None),\n api_key=self.device.get('api_key', None),\n port=self.device.get('port', None),\n hostname=self.device.get('hostname', None),\n serial=self.device.get('serial', None)\n )\n\n self.prefix = prefix\n self.attributes = attributes\n self.watermark = watermark\n self.persistent = persistent\n\n self.q = gevent.queue.Queue()\n\n def put(self, op, address, value):\n LOG.debug('adding %s:%s to device queue', op, address)\n self.q.put([op, address, value])\n\n def _get_registered_ip_tags(self, ip):\n self.xapi.op(\n cmd='%s' % ip,\n vsys=self.device.get('vsys', None),\n cmd_xml=False\n )\n\n entries = self.xapi.element_root.findall('./result/entry')\n if entries is None or len(entries) == 0:\n LOG.warning('%s: ip %s has no tags', self.device.get('hostname', None), ip)\n return None\n\n tags = [member.text for member in entries[0].findall('./tag/member')\n if member.text and member.text.startswith(self.prefix)]\n\n return tags\n\n def _get_all_registered_ips(self):\n cmd = (\n '' %\n (self.prefix, self.watermark)\n )\n self.xapi.op(\n cmd=cmd,\n vsys=self.device.get('vsys', None),\n cmd_xml=False\n )\n\n entries = self.xapi.element_root.findall('./result/entry')\n if not entries:\n return\n\n for entry in entries:\n ip = entry.get(\"ip\")\n\n yield ip, self._get_registered_ip_tags(ip)\n\n def _dag_message(self, type_, addresses):\n message = [\n \"\",\n \"1.0\",\n \"update\",\n \"\"\n ]\n persistent = ''\n if type_ == 'register':\n persistent = ' persistent=\"%d\"' % (1 if self.persistent else 0)\n message.append('<%s>' % type_)\n\n if addresses is not None and len(addresses) != 0:\n akeys = sorted(addresses.keys())\n for a in akeys:\n message.append(\n '' % (a, persistent)\n )\n\n tags = sorted(addresses[a])\n if tags is not None:\n message.append('')\n for t in tags:\n message.append('%s' % t)\n message.append('')\n\n message.append('')\n\n message.append('' % type_)\n message.append('')\n\n return ''.join(message)\n\n def _user_id(self, cmd=None):\n try:\n self.xapi.user_id(cmd=cmd,\n vsys=self.device.get('vsys', None))\n\n except gevent.GreenletExit:\n raise\n\n except pan.xapi.PanXapiError as e:\n LOG.debug('%s', e)\n if 'already exists, ignore' in str(e):\n pass\n elif 'does not exist, ignore unreg' in str(e):\n pass\n elif 'Failed to register' in str(e):\n pass\n else:\n LOG.exception('XAPI exception in pusher for device %s: %s',\n self.device.get('hostname', None), str(e))\n raise\n\n def _tags_from_value(self, value):\n result = []\n\n def _tag(t, v):\n if type(v) == unicode:\n v = v.encode('ascii', 'replace')\n else:\n v = str(v)\n\n v = SUBRE.sub('_', v)\n tag = '%s%s_%s' % (self.prefix, t, v)\n\n return tag\n\n for t in self.attributes:\n if t in value:\n if t == 'confidence':\n confidence = value[t]\n if confidence < 50:\n tag = '%s%s_low' % (self.prefix, t)\n elif confidence < 75:\n tag = '%s%s_medium' % (self.prefix, t)\n else:\n tag = '%s%s_high' % (self.prefix, t)\n\n result.append(tag)\n\n else:\n LOG.debug('%s %s %s', t, value[t], type(value[t]))\n if isinstance(value[t], list):\n for v in value[t]:\n LOG.debug('%s', v)\n result.append(_tag(t, v))\n else:\n result.append(_tag(t, value[t]))\n\n else:\n # XXX noop for this case?\n result.append('%s%s_unknown' % (self.prefix, t))\n\n LOG.debug('%s', result)\n\n return set(result) # XXX eliminate duplicates\n\n def _push(self, op, address, value):\n tags = []\n\n tags.append('%s%s' % (self.prefix, self.watermark))\n\n tags += self._tags_from_value(value)\n\n if len(tags) == 0:\n tags = None\n\n msg = self._dag_message(op, {address: tags})\n\n self._user_id(cmd=msg)\n\n def _init_resync(self):\n ctags = collections.defaultdict(set)\n while True:\n op, address, value = self.q.get()\n if op == 'EOI':\n break\n\n if op != 'init':\n raise RuntimeError(\n 'DevicePusher %s - wrong op %s received in init phase' %\n (self.device.get('hostname', None), op)\n )\n\n ctags[address].add('%s%s' % (self.prefix, self.watermark))\n for t in self._tags_from_value(value):\n ctags[address].add(t)\n\n LOG.debug('%s', ctags)\n\n register = collections.defaultdict(list)\n unregister = collections.defaultdict(list)\n for a, atags in self._get_all_registered_ips():\n regtags = set()\n if atags is not None:\n for t in atags:\n regtags.add(t)\n\n added = ctags[a] - regtags\n removed = regtags - ctags[a]\n\n for t in added:\n register[a].append(t)\n\n for t in removed:\n unregister[a].append(t)\n\n ctags.pop(a)\n\n # ips not in firewall\n for a, atags in ctags.iteritems():\n register[a] = atags\n\n LOG.debug('register %s', register)\n LOG.debug('unregister %s', unregister)\n\n # XXX use constant for chunk size\n if len(register) != 0:\n addrs = iter(register)\n for i in xrange(0, len(register), 1000):\n rmsg = self._dag_message(\n 'register',\n {k: register[k] for k in itertools.islice(addrs, 1000)}\n )\n self._user_id(cmd=rmsg)\n\n if len(unregister) != 0:\n addrs = iter(unregister)\n for i in xrange(0, len(unregister), 1000):\n urmsg = self._dag_message(\n 'unregister',\n {k: unregister[k] for k in itertools.islice(addrs, 1000)}\n )\n self._user_id(cmd=urmsg)\n\n def _run(self):\n self._init_resync()\n\n while True:\n try:\n op, address, value = self.q.peek()\n self._push(op, address, value)\n self.q.get() # discard processed message\n\n except gevent.GreenletExit:\n break\n\n except pan.xapi.PanXapiError as e:\n LOG.exception('XAPI exception in pusher for device %s: %s',\n self.device.get('hostname', None), str(e))\n raise\n\n\nclass DagPusher(actorbase.ActorBaseFT):\n def __init__(self, name, chassis, config):\n self.devices = []\n self.device_pushers = []\n\n self.device_list_glet = None\n self.device_list_mtime = None\n\n self.ageout_glet = None\n self.last_ageout_run = None\n\n self.hup_event = gevent.event.Event()\n\n super(DagPusher, self).__init__(name, chassis, config)\n\n def configure(self):\n super(DagPusher, self).configure()\n\n self.device_list_path = self.config.get('device_list', None)\n if self.device_list_path is None:\n self.device_list_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_device_list.yml' % self.name\n )\n self.age_out = self.config.get('age_out', 3600)\n self.age_out_interval = self.config.get('age_out_interval', None)\n self.tag_prefix = self.config.get('tag_prefix', 'mmld_')\n self.tag_watermark = self.config.get('tag_watermark', 'pushed')\n self.tag_attributes = self.config.get(\n 'tag_attributes',\n ['confidence', 'direction']\n )\n self.persistent_registered_ips = self.config.get(\n 'persistent_registered_ips',\n True\n )\n\n def _initialize_table(self, truncate=False):\n self.table = table.Table(self.name, truncate=truncate)\n self.table.create_index('_age_out')\n\n def initialize(self):\n self._initialize_table()\n\n def rebuild(self):\n self.rebuild_flag = True\n self._initialize_table(truncate=True)\n\n def reset(self):\n self._initialize_table(truncate=True)\n\n def _validate_ip(self, indicator, value):\n type_ = value.get('type', None)\n if type_ not in ['IPv4', 'IPv6']:\n LOG.error('%s - invalid indicator type, ignored: %s',\n self.name, type_)\n self.statistics['ignored'] += 1\n return\n\n if '-' in indicator:\n i1, i2 = indicator.split('-', 1)\n if i1 != i2:\n LOG.error('%s - indicator range must be equal, ignored: %s',\n self.name, indicator)\n self.statistics['ignored'] += 1\n return\n indicator = i1\n\n try:\n address = netaddr.IPNetwork(indicator)\n except netaddr.core.AddrFormatError as e:\n LOG.error('%s - invalid IP address received, ignored: %s',\n self.name, e)\n self.statistics['ignored'] += 1\n return\n\n if address.size != 1:\n LOG.error('%s - IP network received, ignored: %s',\n self.name, address)\n self.statistics['ignored'] += 1\n return\n\n if type_ == 'IPv4' and address.version != 4 or \\\n type_ == 'IPv6' and address.version != 6:\n LOG.error('%s - IP version mismatch, ignored',\n self.name)\n self.statistics['ignored'] += 1\n return\n\n return address\n\n @base._counting('update.processed')\n def filtered_update(self, source=None, indicator=None, value=None):\n address = self._validate_ip(indicator, value)\n if address is None:\n return\n\n current_value = self.table.get(str(address))\n\n now = utc_millisec()\n age_out = now+self.age_out*1000\n\n value['_age_out'] = age_out\n\n self.statistics['added'] += 1\n self.table.put(str(address), value)\n LOG.debug('%s - #indicators: %d', self.name, self.length())\n\n value.pop('_age_out')\n\n uflag = False\n if current_value is not None:\n for t in self.tag_attributes:\n cv = current_value.get(t, None)\n nv = value.get(t, None)\n if isinstance(cv, list) or isinstance(nv, list):\n uflag |= set(cv) != set(nv)\n else:\n uflag |= cv != nv\n\n LOG.debug('uflag %s current %s new %s', uflag, current_value, value)\n\n for p in self.device_pushers:\n if uflag:\n p.put('unregister', str(address), current_value)\n p.put('register', str(address), value)\n\n @base._counting('withdraw.processed')\n def filtered_withdraw(self, source=None, indicator=None, value=None):\n address = self._validate_ip(indicator, value)\n if address is None:\n return\n\n current_value = self.table.get(str(address))\n if current_value is None:\n LOG.warning('%s - unknown indicator received, ignored: %s',\n self.name, address)\n self.statistics['ignored'] += 1\n return\n\n current_value.pop('_age_out', None)\n\n self.statistics['removed'] += 1\n self.table.delete(str(address))\n LOG.debug('%s - #indicators: %d', self.name, self.length())\n\n for p in self.device_pushers:\n p.put('unregister', str(address), current_value)\n\n def _age_out_run(self):\n while True:\n try:\n now = utc_millisec()\n\n LOG.debug('now: %s', now)\n\n for i, v in self.table.query(index='_age_out',\n to_key=now-1,\n include_value=True):\n LOG.debug('%s - %s %s aged out', self.name, i, v)\n\n for dp in self.device_pushers:\n dp.put(\n op='unregister',\n address=i,\n value=v\n )\n\n self.statistics['aged_out'] += 1\n self.table.delete(i)\n\n self.last_ageout_run = now\n LOG.debug('%s - #indicators: %d', self.name, self.length())\n\n except gevent.GreenletExit:\n break\n\n except Exception:\n LOG.exception('Exception in _age_out_loop')\n\n try:\n gevent.sleep(self.age_out_interval)\n except gevent.GreenletExit:\n break\n\n def _spawn_device_pusher(self, device):\n dp = DevicePusher(\n device,\n self.tag_prefix,\n self.tag_watermark,\n self.tag_attributes,\n self.persistent_registered_ips\n )\n dp.link_exception(self._device_pusher_died)\n\n for i, v in self.table.query(include_value=True):\n LOG.debug('%s - addding %s to init', self.name, i)\n dp.put('init', i, v)\n dp.put('EOI', None, None)\n\n return dp\n\n def _device_pusher_died(self, g):\n try:\n g.get()\n\n except gevent.GreenletExit:\n pass\n\n except Exception:\n LOG.exception('%s - exception in greenlet for %s, '\n 'respawning in 60 seconds',\n self.name, g.device['hostname'])\n\n for idx in range(len(self.device_pushers)):\n if self.device_pushers[idx].device == g.device:\n break\n else:\n LOG.info('%s - device pusher for %s removed,' +\n ' respawning aborted',\n self.name, g.device['hostname'])\n g = None\n return\n\n dp = self._spawn_device_pusher(g.device)\n self.device_pushers[idx] = dp\n dp.start_later(60)\n\n def _load_device_list(self):\n with open(self.device_list_path, 'r') as dlf:\n dlist = yaml.safe_load(dlf)\n\n added = [d for i, d in enumerate(dlist) if d not in self.devices]\n removed = [i for i, d in enumerate(self.devices) if d not in dlist]\n\n dpushers = []\n for d in dlist:\n if d in added:\n dp = self._spawn_device_pusher(d)\n dpushers.append(dp)\n else:\n idx = self.devices.index(d)\n dpushers.append(self.device_pushers[idx])\n\n for idx in removed:\n self.device_pushers[idx].kill()\n\n self.device_pushers = dpushers\n self.devices = dlist\n\n for g in self.device_pushers:\n if g.value is None and not g.started:\n g.start()\n\n def _huppable_wait(self, wait_time):\n hup_called = self.hup_event.wait(timeout=wait_time)\n if hup_called:\n LOG.debug('%s - clearing poll event', self.name)\n self.hup_event.clear()\n\n def _device_list_monitor(self):\n if self.device_list_path is None:\n LOG.warning('%s - no device_list path configured', self.name)\n return\n\n while True:\n try:\n mtime = os.stat(self.device_list_path).st_mtime\n except OSError:\n LOG.debug('%s - error checking mtime of %s',\n self.name, self.device_list_path)\n self._huppable_wait(5)\n continue\n\n if mtime != self.device_list_mtime:\n self.device_list_mtime = mtime\n\n try:\n self._load_device_list()\n LOG.info('%s - device list loaded', self.name)\n except Exception:\n LOG.exception('%s - exception loading device list',\n self.name)\n\n self._huppable_wait(5)\n\n def mgmtbus_status(self):\n result = super(DagPusher, self).mgmtbus_status()\n\n result['devices'] = len(self.devices)\n\n return result\n\n def length(self, source=None):\n return self.table.num_indicators\n\n def start(self):\n super(DagPusher, self).start()\n\n if self.device_list_glet is not None:\n return\n\n self.device_list_glet = gevent.spawn_later(\n 2,\n self._device_list_monitor\n )\n\n if self.age_out_interval is not None:\n self.ageout_glet = gevent.spawn(self._age_out_run)\n\n def stop(self):\n super(DagPusher, self).stop()\n\n if self.device_list_glet is None:\n return\n\n for g in self.device_pushers:\n g.kill()\n\n self.device_list_glet.kill()\n\n if self.ageout_glet is not None:\n self.ageout_glet.kill()\n\n self.table.close()\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload device list', self.name)\n self.hup_event.set()\n\n @staticmethod\n def gc(name, config=None):\n actorbase.ActorBaseFT.gc(name, config=config)\n\n shutil.rmtree(name, ignore_errors=True)\n device_list_path = None\n if config is not None:\n device_list_path = config.get('device_list', None)\n if device_list_path is None:\n device_list_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_device_list.yml'.format(name)\n )\n\n try:\n os.remove(device_list_path)\n except OSError:\n pass\n" }, { "path": "minemeld/ft/dag_ng.py", "content": "# Copyright 2015-2019 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport yaml\nimport netaddr\nimport os\nimport re\nimport collections\nimport itertools\nimport shutil\nimport time\n\nimport gevent\nimport gevent.queue\nimport gevent.event\n\nimport pan.xapi\n\nfrom . import base\nfrom . import actorbase\nfrom . import table\nfrom .utils import utc_millisec\n\nLOG = logging.getLogger(__name__)\n\n# The tag name cannot contain the following:\n# - single quote\n# - double quote\n# - greater than one consecutive space\n# And cannot be the case insensitive words:\n# - and, or, not\nSUBRE = re.compile(\"['\\\"]| +\")\nCANARY_INDICATOR = '::/128' # RFC 4291: The Unspecified Address\nCANARY_TAG = 'canary_for_resync'\nCANARY_CHECK_SECONDS = 60*5\nMAX_CHUNK_SIZE = 512 # max IPs in register/unregister message\n\n\ndef _api_wrapper(x):\n from functools import wraps\n\n @wraps(x)\n def wrapper(self, *args, **kwargs):\n if self.valid_device_version is None:\n valid, version = self._valid_device_version()\n if not valid:\n self.valid_device_version = False\n LOG.error('%s: PAN-OS %s: must be 8.0 or greater',\n self.device.get('hostname', None), version)\n else:\n self.valid_device_version = True\n LOG.info('%s: PAN-OS %s',\n self.device.get('hostname', None), version)\n\n if self.valid_device_version:\n r = x(self, *args, **kwargs)\n return r\n else:\n return '' # iterable\n\n return wrapper\n\n\nclass DevicePusher(gevent.Greenlet):\n def __init__(self, device, prefix, watermark, attributes, persistent):\n super(DevicePusher, self).__init__()\n\n self.device = device\n self.xapi = pan.xapi.PanXapi(\n tag=self.device.get('tag', None),\n api_username=self.device.get('api_username', None),\n api_password=self.device.get('api_password', None),\n api_key=self.device.get('api_key', None),\n port=self.device.get('port', None),\n hostname=self.device.get('hostname', None),\n serial=self.device.get('serial', None)\n )\n\n self.valid_device_version = None\n self.prefix = prefix\n self.attributes = attributes\n self.watermark = watermark\n self.persistent = persistent\n\n self.q = gevent.queue.Queue()\n\n def _valid_device_version(self):\n try:\n self.xapi.ad_hoc({'type': 'version'}, modify_qs=True)\n except pan.xapi.PanXapiError as e:\n return False, None\n\n x = self.xapi.element_root.find('./result/sw-version')\n if x is not None:\n version = x.text\n else:\n LOG.error('%s: type=version request: no sw-version',\n self.device.get('hostname', None))\n return False, None\n\n major = version.split('.', 1)\n valid = False\n try:\n if int(major[0]) >= 8:\n valid = True\n except ValueError:\n pass\n\n return valid, version\n\n @_api_wrapper\n def _set_canary(self):\n addresses = {\n CANARY_INDICATOR: ['%s%s' % (self.prefix, CANARY_TAG)]\n }\n\n cmd = self._dag_message('register',\n addresses,\n persistent=False,\n timeout=CANARY_CHECK_SECONDS+60*2)\n self._user_id(cmd)\n\n def _test_canary(self):\n cmd = '''\\\n\n \n \n \n \n \n \n \n'''\n\n self.xapi.op(cmd=cmd % (self.prefix, CANARY_TAG),\n vsys=self.device.get('vsys', None))\n\n if self.xapi.element_root is None:\n return False\n\n x = self.xapi.element_root.find('./result/count')\n if x is None:\n LOG.error('%s: no count element in registered-ip response',\n self.device.get('hostname', None))\n return False\n\n try:\n count = int(x.text)\n except ValueError as e:\n LOG.error('%s: count invalid: %s: %s',\n self.device.get('hostname', None), x.text, e)\n return False\n\n # XXX sufficient check?\n if count != 1:\n return False\n\n self._set_canary() # reset timeout\n return True\n\n def put(self, op, address, value):\n LOG.debug('adding %s:%s to device queue', op, address)\n self.q.put([op, address, value])\n\n @_api_wrapper\n def _get_all_registered_ips(self):\n cmd = '''\\\n\n \n \n %d\n %d\n \n \n \n'''\n\n start = 1\n LIMIT = 500\n\n while True:\n self.xapi.op(cmd=cmd % (start, LIMIT),\n vsys=self.device.get('vsys', None))\n\n if self.xapi.element_root is None:\n return\n\n x = self.xapi.element_root.find('./result/count')\n if x is None:\n LOG.error('%s: no count element in registered-ip response',\n self.device.get('hostname', None))\n return\n\n try:\n count = int(x.text)\n except ValueError as e:\n LOG.error('%s: count invalid: %s: %s',\n self.device.get('hostname', None), x.text, e)\n return\n LOG.info('%s: count %d',\n self.device.get('hostname', None), count)\n\n entries = self.xapi.element_root.findall('./result/entry')\n for entry in entries:\n ip = entry.get('ip')\n members = entry.findall('./tag/member')\n tags = []\n for member in members:\n tags.append(member.text)\n\n if '%s%s' % (self.prefix, self.watermark) in tags:\n _tags = [x for x in tags if x.startswith(self.prefix)]\n try:\n _ip = netaddr.IPNetwork(ip)\n except netaddr.core.AddrFormatError as e:\n LOG.error('%s: invalid IP address from firewall: %s',\n self.device.get('hostname', None), e)\n yield ip, _tags\n\n # canonize host length address with prefix\n yield str(_ip), _tags\n\n if count < LIMIT:\n break\n\n start += LIMIT\n\n def _dag_message(self, type_, addresses,\n persistent=None, timeout=None):\n message = [\n \"\",\n # version element ignored \"1.0\",\n \"update\",\n \"\"\n ]\n\n _persistent = ''\n if type_ == 'register':\n if persistent is not None:\n _persistent = \\\n ' persistent=\"%d\"' % (1 if persistent else 0)\n else:\n _persistent = \\\n ' persistent=\"%d\"' % (1 if self.persistent else 0)\n message.append('<%s>' % type_)\n\n if addresses is not None and len(addresses) != 0:\n akeys = sorted(addresses.keys())\n for a in akeys:\n message.append(\n '' % (a, _persistent)\n )\n\n tags = sorted(addresses[a])\n if tags is not None:\n message.append('')\n for t in tags:\n if timeout is not None:\n # PAN-OS 9.0 and greater\n message.append('%s' %\n (timeout, t))\n else:\n message.append('%s' % t)\n message.append('')\n\n message.append('')\n\n message.append('' % type_)\n message.append('')\n\n return ''.join(message)\n\n @_api_wrapper\n def _user_id(self, cmd=None):\n try:\n self.xapi.user_id(cmd=cmd,\n vsys=self.device.get('vsys', None))\n\n except gevent.GreenletExit:\n raise\n\n except pan.xapi.PanXapiError as e:\n LOG.debug('%s', e)\n if 'already exists, ignore' in str(e):\n pass\n elif 'does not exist, ignore unreg' in str(e):\n pass\n elif 'Failed to register' in str(e):\n pass\n else:\n LOG.exception('XAPI exception in pusher for device %s: %s',\n self.device.get('hostname', None), str(e))\n raise\n\n def _tags_from_value(self, value):\n result = []\n\n def _tag(t, v):\n if type(v) == unicode:\n v = v.encode('ascii', 'replace')\n else:\n v = str(v)\n\n m = re.match('^(and|or|not)$', v, flags=re.IGNORECASE)\n if m:\n v = '_%s_' % m.group(0)\n else:\n v = SUBRE.sub('_', v)\n tag = '%s%s_%s' % (self.prefix, t, v)\n\n return tag\n\n for t in self.attributes:\n if t in value:\n if t == 'confidence':\n confidence = value[t]\n if confidence < 50:\n tag = '%s%s_low' % (self.prefix, t)\n elif confidence < 75:\n tag = '%s%s_medium' % (self.prefix, t)\n else:\n tag = '%s%s_high' % (self.prefix, t)\n\n result.append(tag)\n\n else:\n LOG.debug('%s %s %s', t, value[t], type(value[t]))\n if isinstance(value[t], list):\n for v in value[t]:\n LOG.debug('%s', v)\n result.append(_tag(t, v))\n else:\n result.append(_tag(t, value[t]))\n\n else:\n # XXX noop for this case?\n result.append('%s%s_unknown' % (self.prefix, t))\n\n LOG.debug('%s', result)\n\n return set(result) # XXX eliminate duplicates\n\n def _push(self, op, addresses):\n x = {}\n for address in addresses:\n x[address] = []\n x[address].append('%s%s' % (self.prefix, self.watermark))\n x[address] += self._tags_from_value(addresses[address])\n if len(x[address]) == 0:\n x[address] = None\n\n LOG.debug('%s', x)\n msg = self._dag_message(op, x)\n\n self._user_id(cmd=msg)\n\n def _init_resync(self):\n ctags = collections.defaultdict(set)\n while True:\n op, address, value = self.q.get()\n if op == 'EOI':\n break\n\n if op != 'init':\n raise RuntimeError(\n 'DevicePusher %s - wrong op %s received in init phase' %\n (self.device.get('hostname', None), op)\n )\n\n ctags[address].add('%s%s' % (self.prefix, self.watermark))\n for t in self._tags_from_value(value):\n ctags[address].add(t)\n\n LOG.debug('%s', ctags)\n\n register = collections.defaultdict(list)\n unregister = collections.defaultdict(list)\n for a, atags in self._get_all_registered_ips():\n regtags = set()\n if atags is not None:\n for t in atags:\n regtags.add(t)\n\n added = ctags[a] - regtags\n removed = regtags - ctags[a]\n\n for t in added:\n register[a].append(t)\n\n for t in removed:\n unregister[a].append(t)\n\n ctags.pop(a)\n\n # ips not in firewall\n for a, atags in ctags.iteritems():\n register[a] = atags\n\n LOG.debug('register %s', register)\n LOG.debug('unregister %s', unregister)\n\n if len(register) != 0:\n addrs = iter(register)\n for i in xrange(0, len(register), MAX_CHUNK_SIZE):\n rmsg = self._dag_message(\n 'register',\n {k: register[k] for k in itertools.islice(\n addrs, MAX_CHUNK_SIZE)}\n )\n self._user_id(cmd=rmsg)\n\n if len(unregister) != 0:\n addrs = iter(unregister)\n for i in xrange(0, len(unregister), MAX_CHUNK_SIZE):\n urmsg = self._dag_message(\n 'unregister',\n {k: unregister[k] for k in itertools.islice(\n addrs, MAX_CHUNK_SIZE)}\n )\n self._user_id(cmd=urmsg)\n\n self._set_canary()\n\n def _run(self):\n def _chunk(op, addresses):\n MAX_TIME = 1\n\n self.q.get() # discard processed message\n end = time.time() + MAX_TIME\n\n while True:\n try:\n _op, address, value = self.q.peek_nowait()\n except gevent.queue.Empty:\n break\n\n if _op != op:\n break\n addresses.update({address: value})\n self.q.get()\n\n if time.time() > end or len(addresses) == MAX_CHUNK_SIZE:\n break\n\n LOG.debug('%s chunks %d', op, len(addresses))\n\n self._init_resync()\n\n last_check = int(time.time())\n while True:\n now = int(time.time())\n elapsed = now - last_check\n LOG.debug('%s: elapsed %d', self.device.get('hostname', None),\n elapsed)\n if elapsed >= CANARY_CHECK_SECONDS:\n if self.valid_device_version and not self._test_canary():\n raise RuntimeError('%s: out of sync detected' %\n self.device.get('hostname', None))\n last_check = int(time.time())\n\n try:\n try:\n LOG.debug('%s: peek %d', self.device.get('hostname', None),\n CANARY_CHECK_SECONDS-elapsed)\n op, address, value = self.q.peek(\n timeout=CANARY_CHECK_SECONDS-elapsed)\n except gevent.queue.Empty:\n if self.valid_device_version and not self._test_canary():\n raise RuntimeError('%s: out of sync detected' %\n self.device.get('hostname', None))\n last_check = int(time.time())\n continue\n\n addresses = {address: value}\n _chunk(op, addresses)\n self._push(op, addresses)\n\n except gevent.GreenletExit:\n break\n\n except pan.xapi.PanXapiError as e:\n LOG.exception('XAPI exception in pusher for device %s: %s',\n self.device.get('hostname', None), str(e))\n raise\n\n\nclass DagPusher(actorbase.ActorBaseFT):\n def __init__(self, name, chassis, config):\n self.devices = []\n self.device_pushers = []\n\n self.device_list_glet = None\n self.device_list_mtime = None\n\n self.ageout_glet = None\n self.last_ageout_run = None\n\n self.hup_event = gevent.event.Event()\n\n super(DagPusher, self).__init__(name, chassis, config)\n\n def configure(self):\n super(DagPusher, self).configure()\n\n self.device_list_path = self.config.get('device_list', None)\n if self.device_list_path is None:\n self.device_list_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_device_list.yml' % self.name\n )\n self.age_out = self.config.get('age_out', 3600)\n self.age_out_interval = self.config.get('age_out_interval', None)\n self.tag_prefix = self.config.get('tag_prefix', 'mmld_')\n self.tag_watermark = self.config.get('tag_watermark', 'pushed')\n self.tag_attributes = self.config.get(\n 'tag_attributes',\n ['confidence', 'direction']\n )\n self.persistent_registered_ips = self.config.get(\n 'persistent_registered_ips',\n True\n )\n\n def _initialize_table(self, truncate=False):\n self.table = table.Table(self.name, truncate=truncate)\n self.table.create_index('_age_out')\n\n def initialize(self):\n self._initialize_table()\n\n def rebuild(self):\n self.rebuild_flag = True\n self._initialize_table(truncate=True)\n\n def reset(self):\n self._initialize_table(truncate=True)\n\n def _validate_ip(self, indicator, value):\n type_ = value.get('type', None)\n if type_ not in ['IPv4', 'IPv6']:\n LOG.error('%s - invalid indicator type, ignored: %s',\n self.name, type_)\n self.statistics['ignored'] += 1\n return\n\n if '-' in indicator:\n i1, i2 = indicator.split('-', 1)\n if i1 != i2:\n LOG.error('%s - indicator range must be equal, ignored: %s',\n self.name, indicator)\n self.statistics['ignored'] += 1\n return\n indicator = i1\n\n try:\n address = netaddr.IPNetwork(indicator)\n except netaddr.core.AddrFormatError as e:\n LOG.error('%s - invalid IP address received, ignored: %s',\n self.name, e)\n self.statistics['ignored'] += 1\n return\n\n if address.size != 1:\n LOG.error('%s - IP network received, ignored: %s',\n self.name, address)\n self.statistics['ignored'] += 1\n return\n\n if type_ == 'IPv4' and address.version != 4 or \\\n type_ == 'IPv6' and address.version != 6:\n LOG.error('%s - IP version mismatch, ignored',\n self.name)\n self.statistics['ignored'] += 1\n return\n\n return address\n\n @base._counting('update.processed')\n def filtered_update(self, source=None, indicator=None, value=None):\n address = self._validate_ip(indicator, value)\n if address is None:\n return\n\n current_value = self.table.get(str(address))\n\n now = utc_millisec()\n age_out = now+self.age_out*1000\n\n value['_age_out'] = age_out\n\n self.statistics['added'] += 1\n self.table.put(str(address), value)\n LOG.debug('%s - #indicators: %d', self.name, self.length())\n\n value.pop('_age_out')\n\n uflag = False\n if current_value is not None:\n for t in self.tag_attributes:\n cv = current_value.get(t, None)\n nv = value.get(t, None)\n if isinstance(cv, list) or isinstance(nv, list):\n uflag |= set(cv) != set(nv)\n else:\n uflag |= cv != nv\n\n LOG.debug('uflag %s current %s new %s', uflag, current_value, value)\n\n for p in self.device_pushers:\n if uflag:\n p.put('unregister', str(address), current_value)\n p.put('register', str(address), value)\n\n @base._counting('withdraw.processed')\n def filtered_withdraw(self, source=None, indicator=None, value=None):\n address = self._validate_ip(indicator, value)\n if address is None:\n return\n\n current_value = self.table.get(str(address))\n if current_value is None:\n LOG.warning('%s - unknown indicator received, ignored: %s',\n self.name, address)\n self.statistics['ignored'] += 1\n return\n\n current_value.pop('_age_out', None)\n\n self.statistics['removed'] += 1\n self.table.delete(str(address))\n LOG.debug('%s - #indicators: %d', self.name, self.length())\n\n for p in self.device_pushers:\n p.put('unregister', str(address), current_value)\n\n def _age_out_run(self):\n while True:\n try:\n now = utc_millisec()\n\n LOG.debug('now: %s', now)\n\n for i, v in self.table.query(index='_age_out',\n to_key=now-1,\n include_value=True):\n LOG.debug('%s - %s %s aged out', self.name, i, v)\n\n for dp in self.device_pushers:\n dp.put(\n op='unregister',\n address=i,\n value=v\n )\n\n self.statistics['aged_out'] += 1\n self.table.delete(i)\n\n self.last_ageout_run = now\n LOG.debug('%s - #indicators: %d', self.name, self.length())\n\n except gevent.GreenletExit:\n break\n\n except Exception:\n LOG.exception('Exception in _age_out_loop')\n\n try:\n gevent.sleep(self.age_out_interval)\n except gevent.GreenletExit:\n break\n\n def _spawn_device_pusher(self, device):\n dp = DevicePusher(\n device,\n self.tag_prefix,\n self.tag_watermark,\n self.tag_attributes,\n self.persistent_registered_ips\n )\n dp.link_exception(self._device_pusher_died)\n\n for i, v in self.table.query(include_value=True):\n LOG.debug('%s - addding %s to init', self.name, i)\n dp.put('init', i, v)\n dp.put('EOI', None, None)\n\n return dp\n\n def _device_pusher_died(self, g):\n def _restart(g):\n for idx in range(len(self.device_pushers)):\n if self.device_pushers[idx].device == g.device:\n break\n else:\n LOG.info('%s - device pusher for %s removed,' +\n ' respawning aborted',\n self.name, g.device['hostname'])\n g = None\n return\n\n dp = self._spawn_device_pusher(g.device)\n self.device_pushers[idx] = dp\n dp.start_later(60)\n\n try:\n g.get()\n\n except gevent.GreenletExit:\n pass\n\n except RuntimeError as e:\n LOG.error('%s: %s, respawning in 60 seconds',\n self.name, e)\n _restart(g)\n\n except Exception:\n LOG.exception('%s - exception in greenlet for %s, '\n 'respawning in 60 seconds',\n self.name, g.device['hostname'])\n _restart(g)\n\n def _load_device_list(self):\n with open(self.device_list_path, 'r') as dlf:\n dlist = yaml.safe_load(dlf)\n\n added = [d for i, d in enumerate(dlist) if d not in self.devices]\n removed = [i for i, d in enumerate(self.devices) if d not in dlist]\n\n dpushers = []\n for d in dlist:\n if d in added:\n dp = self._spawn_device_pusher(d)\n dpushers.append(dp)\n else:\n idx = self.devices.index(d)\n dpushers.append(self.device_pushers[idx])\n\n for idx in removed:\n self.device_pushers[idx].kill()\n\n self.device_pushers = dpushers\n self.devices = dlist\n\n for g in self.device_pushers:\n if g.value is None and not g.started:\n g.start()\n\n def _huppable_wait(self, wait_time):\n hup_called = self.hup_event.wait(timeout=wait_time)\n if hup_called:\n LOG.debug('%s - clearing poll event', self.name)\n self.hup_event.clear()\n\n def _device_list_monitor(self):\n if self.device_list_path is None:\n LOG.warning('%s - no device_list path configured', self.name)\n return\n\n while True:\n try:\n mtime = os.stat(self.device_list_path).st_mtime\n except OSError:\n LOG.debug('%s - error checking mtime of %s',\n self.name, self.device_list_path)\n self._huppable_wait(5)\n continue\n\n if mtime != self.device_list_mtime:\n self.device_list_mtime = mtime\n\n try:\n self._load_device_list()\n LOG.info('%s - device list loaded', self.name)\n except Exception:\n LOG.exception('%s - exception loading device list',\n self.name)\n\n self._huppable_wait(5)\n\n def mgmtbus_status(self):\n result = super(DagPusher, self).mgmtbus_status()\n\n result['devices'] = len(self.devices)\n\n return result\n\n def length(self, source=None):\n return self.table.num_indicators\n\n def start(self):\n super(DagPusher, self).start()\n\n if self.device_list_glet is not None:\n return\n\n self.device_list_glet = gevent.spawn_later(\n 2,\n self._device_list_monitor\n )\n\n if self.age_out_interval is not None:\n self.ageout_glet = gevent.spawn(self._age_out_run)\n\n def stop(self):\n super(DagPusher, self).stop()\n\n if self.device_list_glet is None:\n return\n\n for g in self.device_pushers:\n g.kill()\n\n self.device_list_glet.kill()\n\n if self.ageout_glet is not None:\n self.ageout_glet.kill()\n\n self.table.close()\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload device list', self.name)\n self.hup_event.set()\n\n @staticmethod\n def gc(name, config=None):\n actorbase.ActorBaseFT.gc(name, config=config)\n\n shutil.rmtree(name, ignore_errors=True)\n device_list_path = None\n if config is not None:\n device_list_path = config.get('device_list', None)\n if device_list_path is None:\n device_list_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_device_list.yml'.format(name)\n )\n\n try:\n os.remove(device_list_path)\n except OSError:\n pass\n" }, { "path": "minemeld/ft/google.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport itertools\nimport functools\nimport collections\nimport netaddr\nimport minemeld.packages.gdns.dig\n\nfrom . import basepoller\n\nLOG = logging.getLogger(__name__)\n\n_GOOGLE_DNS_SERVER = '8.8.8.8'\n\n\nclass GoogleSPF(basepoller.BasePollerFT):\n def configure(self):\n super(GoogleSPF, self).configure()\n\n self.polling_timeout = self.config.get('polling_timeout', 20)\n self.tries = self.config.get('tries', 3)\n self.verify_cert = self.config.get('verify_cert', True)\n self.udp_port = self.config.get('udp_port', None)\n self.tcp_port = self.config.get('tcp_port', 53)\n self.source_name = self.config.get('source_name', self.SOURCE_NAME)\n\n def _process_item(self, item):\n indicator = item.pop('indicator', None)\n return [[indicator, item]]\n\n def _resolve_spf(self, dig, name):\n LOG.debug('%s - Resolving SPF for %s', self.name, name)\n reply = dig.query(name, dig.NS_C_IN, dig.NS_T_TXT)\n spf = dig.parse_txt_reply(reply)\n if len(spf) > 1:\n raise RuntimeError(\n '%s - TXT record for %s has more than 1 block' %\n (self.name, name)\n )\n\n spf = spf[0]\n result = collections.defaultdict(list)\n spftoks = spf.split()\n\n if spftoks[0] != 'v=spf1':\n raise RuntimeError(\n '%s - Wrong SPF signature in SPF for %s' %\n (self.name, name)\n )\n\n for t in spftoks[1:]:\n toks = t.split(':', 1)\n if toks[0] in ['include', 'ip4', 'ip6']:\n result[toks[0]].append(toks[1])\n\n return result\n\n def _build_IPv4(self, netblock, ipnetwork):\n try:\n n = netaddr.IPNetwork(ipnetwork)\n if n.version != 4:\n raise ValueError('invalid ip4 network: %d' % n.version)\n except:\n LOG.exception('%s - Invalid ip4 network: %s', self.name, ipnetwork)\n return {}\n\n item = {\n 'indicator': ipnetwork,\n 'type': 'IPv4',\n 'confidence': 100,\n self.BLOCK_ATTRIBUTE: netblock,\n 'sources': [self.SOURCE_NAME]\n }\n return item\n\n def _build_IPv6(self, netblock, ipnetwork):\n try:\n n = netaddr.IPNetwork(ipnetwork)\n if n.version != 6:\n raise ValueError('invalid ip6 network: %d' % n.version)\n except:\n LOG.exception('%s - Invalid ip6 network: %s', self.name, ipnetwork)\n return {}\n\n item = {\n 'indicator': ipnetwork,\n 'type': 'IPv6',\n 'confidence': 100,\n self.BLOCK_ATTRIBUTE: netblock,\n 'sources': [self.SOURCE_NAME]\n }\n return item\n\n def _build_iterator(self, now):\n _iterators = []\n\n dig = minemeld.packages.gdns.dig.Dig(\n servers=[_GOOGLE_DNS_SERVER],\n udp_port=self.udp_port,\n tcp_port=self.tcp_port,\n tries=self.tries,\n timeout=self.polling_timeout\n )\n\n mainspf = self._resolve_spf(dig, self.ROOT_SPF)\n if 'include' not in mainspf:\n LOG.error(\n '%s - No includes in SPF' % self.name\n )\n return []\n\n for idomain in mainspf['include']:\n ispf = self._resolve_spf(dig, idomain)\n\n _iterators.append(itertools.imap(\n functools.partial(self._build_IPv4, idomain),\n ispf.get('ip4', [])\n ))\n _iterators.append(itertools.imap(\n functools.partial(self._build_IPv6, idomain),\n ispf.get('ip6', [])\n ))\n\n return itertools.chain(*_iterators)\n\n\nclass GoogleNetBlocks(GoogleSPF):\n ROOT_SPF = '_spf.google.com'\n SOURCE_NAME = 'google.netblocks'\n BLOCK_ATTRIBUTE = 'google_netblock'\n\n\nclass GoogleCloudNetBlocks(GoogleSPF):\n ROOT_SPF = '_cloud-netblocks.googleusercontent.com'\n SOURCE_NAME = 'google.cloudnetblocks'\n BLOCK_ATTRIBUTE = 'google_cloudnetblock'\n" }, { "path": "minemeld/ft/http.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements minemeld.ft.http.HttpFT, the Miner node for plain\ntext feeds over HTTP/HTTPS.\n\"\"\"\n\nimport requests\nimport logging\nimport re\nimport itertools\n\nfrom minemeld import __version__ as MM_VERSION\n\nfrom . import basepoller\n\nLOG = logging.getLogger(__name__)\n\n\nclass HttpFT(basepoller.BasePollerFT):\n \"\"\"Implements class for miners of plain text feeds over http/https.\n\n **Config parameters**\n :url: URL of the feed.\n :polling_timeout: timeout of the polling request in seconds.\n Default: 20\n :verify_cert: boolean, if *true* feed HTTPS server certificate is\n verified. Default: *true*\n :user_agent: string, value for the User-Agent header in HTTP\n request. If ``MineMeld``, MineMeld/ is used.\n Default: python ``requests`` default.\n :ignore_regex: Python regular expression for lines that should be\n ignored. Default: *null*\n :indicator: an *extraction dictionary* to extract the indicator from\n the line. If *null*, the text until the first whitespace or newline\n character is used as indicator. Default: *null*\n :fields: a dicionary of *extraction dictionaries* to extract\n additional attributes from each line. Default: {}\n :encoding: encoding of the feed, if not UTF-8. See\n ``str.decode`` for options. Default: *null*, meaning do\n nothing, (Assumes UTF-8).\n\n **Extraction dictionary**\n Extraction dictionaries contain the following keys:\n\n :regex: Python regular expression for searching the text.\n :transform: template to generate the final value from the result\n of the regular expression. Default: the entire match of the regex\n is used as extracted value.\n\n See Python `re `_ module for\n details about Python regular expressions and templates.\n\n Example:\n Example config in YAML where extraction dictionaries are used to\n extract the indicator and additional fields::\n\n url: https://www.dshield.org/block.txt\n ignore_regex: \"[#S].*\"\n indicator:\n regex: '^([0-9]{1,3}\\\\.[0-9]{1,3}\\\\.[0-9]{1,3}\\\\.[0-9]{1,3})\\\\t([0-9]{1,3}\\\\.[0-9]{1,3}\\\\.[0-9]{1,3}\\\\.[0-9]{1,3})'\n transform: '\\\\1-\\\\2'\n fields:\n dshield_nattacks:\n regex: '^.*\\\\t.*\\\\t[0-9]+\\\\t([0-9]+)'\n transform: '\\\\1'\n dshield_name:\n regex: '^.*\\\\t.*\\\\t[0-9]+\\\\t[0-9]+\\\\t([^\\\\t]+)'\n transform: '\\\\1'\n dshield_country:\n regex: '^.*\\\\t.*\\\\t[0-9]+\\\\t[0-9]+\\\\t[^\\\\t]+\\\\t([A-Z]+)'\n transform: '\\\\1'\n dshield_email:\n regex: '^.*\\\\t.*\\\\t[0-9]+\\\\t[0-9]+\\\\t[^\\\\t]+\\\\t[A-Z]+\\\\t(\\\\S+)'\n transform: '\\\\1'\n\n Example config in YAML where the text in each line until the first\n whitespace is used as indicator::\n\n url: https://ransomwaretracker.abuse.ch/downloads/CW_C2_URLBL.txt\n ignore_regex: '^#'\n\n Args:\n name (str): node name, should be unique inside the graph\n chassis (object): parent chassis instance\n config (dict): node config.\n \"\"\"\n def configure(self):\n super(HttpFT, self).configure()\n\n self.url = self.config.get('url', None)\n self.polling_timeout = self.config.get('polling_timeout', 20)\n self.verify_cert = self.config.get('verify_cert', True)\n self.user_agent = self.config.get('user_agent', None)\n self.encoding = self.config.get('encoding', None)\n\n self.username = self.config.get('username', None)\n self.password = self.config.get('password', None)\n\n self.ignore_regex = self.config.get('ignore_regex', None)\n if self.ignore_regex is not None:\n self.ignore_regex = re.compile(self.ignore_regex)\n\n self.indicator = self.config.get('indicator', None)\n\n if self.indicator is not None:\n if 'regex' in self.indicator:\n self.indicator['regex'] = re.compile(self.indicator['regex'])\n else:\n raise ValueError('%s - indicator stanza should have a regex',\n self.name)\n if 'transform' not in self.indicator:\n if self.indicator['regex'].groups > 0:\n LOG.warning('%s - no transform string for indicator'\n ' but pattern contains groups',\n self.name)\n self.indicator['transform'] = '\\g<0>'\n\n self.fields = self.config.get('fields', {})\n for f, fattrs in self.fields.iteritems():\n if 'regex' in fattrs:\n fattrs['regex'] = re.compile(fattrs['regex'])\n else:\n raise ValueError('%s - %s field does not have a regex',\n self.name, f)\n if 'transform' not in fattrs:\n if fattrs['regex'].groups > 0:\n LOG.warning('%s - no transform string for field %s'\n ' but pattern contains groups',\n self.name, f)\n fattrs['transform'] = '\\g<0>'\n\n def _process_item(self, line):\n line = line.strip()\n if not line:\n return [[None, None]]\n\n if self.indicator is None:\n indicator = line.split()[0]\n\n else:\n indicator = self.indicator['regex'].search(line)\n if indicator is None:\n return [[None, None]]\n\n indicator = indicator.expand(self.indicator['transform'])\n\n attributes = {}\n for f, fattrs in self.fields.iteritems():\n m = fattrs['regex'].search(line)\n\n if m is None:\n continue\n\n attributes[f] = m.expand(fattrs['transform'])\n\n try:\n i = int(attributes[f])\n except:\n pass\n else:\n attributes[f] = i\n\n return [[indicator, attributes]]\n\n def _build_iterator(self, now):\n rkwargs = dict(\n stream=True,\n verify=self.verify_cert,\n timeout=self.polling_timeout\n )\n\n if self.user_agent is not None:\n if self.user_agent == 'MineMeld':\n rkwargs['headers'] = {\n 'User-Agent': 'MineMeld/%s' % MM_VERSION\n }\n\n else:\n rkwargs['headers'] = {\n 'User-Agent': self.user_agent\n }\n\n if self.username is not None and self.password is not None:\n rkwargs['auth'] = (self.username, self.password)\n\n r = requests.get(\n self.url,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.debug('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n result = r.iter_lines()\n if self.ignore_regex is not None:\n result = itertools.ifilter(\n lambda x: self.ignore_regex.match(x) is None,\n result\n )\n if self.encoding is not None:\n result = itertools.imap(\n lambda x: x.decode(self.encoding).encode('utf_8'),\n result\n )\n\n return result\n" }, { "path": "minemeld/ft/ipop.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport netaddr\nimport uuid\nimport shutil\n\nfrom . import base\nfrom . import actorbase\nfrom . import table\nfrom . import st\nfrom .utils import utc_millisec\nfrom .utils import RESERVED_ATTRIBUTES\n\nLOG = logging.getLogger(__name__)\n\nWL_LEVEL = st.MAX_LEVEL\n\n\nclass MWUpdate(object):\n def __init__(self, start, end, uuids):\n self.start = start\n self.end = end\n self.uuids = set(uuids)\n\n s = netaddr.IPAddress(start)\n e = netaddr.IPAddress(end)\n self._indicator = '%s-%s' % (s, e)\n\n def indicator(self):\n return self._indicator\n\n def __repr__(self):\n return 'MWUpdate('+self._indicator+', %r)' % self.uuids\n\n def __hash__(self):\n return hash(self._indicator)\n\n def __eq__(self, other):\n return self.start == other.start and \\\n self.end == other.end\n\n\nclass AggregateIPv4FT(actorbase.ActorBaseFT):\n def __init__(self, name, chassis, config):\n self.active_requests = []\n\n super(AggregateIPv4FT, self).__init__(name, chassis, config)\n\n def configure(self):\n super(AggregateIPv4FT, self).configure()\n\n self.whitelist_prefixes = self.config.get('whitelist_prefixes', [])\n self.enable_list_merge = self.config.get('enable_list_merge', False)\n\n def _initialize_tables(self, truncate=False):\n self.table = table.Table(\n self.name,\n bloom_filter_bits=10,\n truncate=truncate\n )\n self.table.create_index('_id')\n self.st = st.ST(self.name+'_st', 32, truncate=truncate)\n\n def initialize(self):\n self._initialize_tables()\n\n def rebuild(self):\n self._initialize_tables(truncate=True)\n\n def reset(self):\n self._initialize_tables(truncate=True)\n\n def _indicator_key(self, indicator, source):\n return indicator+'\\x00'+source\n\n def _calc_indicator_value(self, uuids, additional_uuid=None, additional_value=None):\n mv = {'sources': []}\n for uuid_ in uuids:\n if uuid_ == additional_uuid:\n v = additional_value\n else:\n # uuid_ = str(uuid.UUID(bytes=uuid_))\n k, v = next(\n self.table.query('_id', from_key=uuid_, to_key=uuid_,\n include_value=True),\n (None, None)\n )\n if k is None:\n LOG.error(\"Unable to find key associated with uuid: %s\", uuid_)\n\n for vk in v:\n if vk in mv and vk in RESERVED_ATTRIBUTES:\n mv[vk] = RESERVED_ATTRIBUTES[vk](mv[vk], v[vk])\n else:\n if self.enable_list_merge and vk in mv and isinstance(mv[vk], list):\n if not isinstance(v[vk], list):\n mv[vk] = v[vk]\n else:\n mv[vk].extend(v[vk])\n else:\n mv[vk] = v[vk]\n\n return mv\n\n def _merge_values(self, origin, ov, nv):\n result = {'sources': []}\n\n result['_added'] = ov['_added']\n result['_id'] = ov['_id']\n\n for k in nv.keys():\n result[k] = nv[k]\n\n return result\n\n def _add_indicator(self, origin, indicator, value):\n added = False\n\n now = utc_millisec()\n ik = self._indicator_key(indicator, origin)\n\n v = self.table.get(ik)\n if v is None:\n v = {\n '_id': str(uuid.uuid4()),\n '_added': now\n }\n added = True\n self.statistics['added'] += 1\n\n v = self._merge_values(origin, v, value)\n v['_updated'] = now\n\n self.table.put(ik, v)\n\n return v, added\n\n def _calc_ipranges(self, start, end):\n \"\"\"Calc IP Ranges overlapping the range between start and end\n \n Args:\n start (int): start of the range\n end (int): end of the range\n \n Returns:\n set: set of ranges\n \"\"\"\n\n result = set()\n\n # collect the endpoint between start and end\n eps = set()\n for epaddr, _, _, _ in self.st.query_endpoints(start=start, stop=end):\n eps.add(epaddr)\n eps = sorted(eps)\n\n if len(eps) == 0:\n return result\n\n # walk thru the endpoints, tracking last endpoint\n # current level, active segments and segments levels\n oep = None\n oeplevel = -1\n live_ids = set()\n slevels = {}\n\n for epaddr in eps:\n # for each endpoint we track which segments are starting\n # and which ones are ending with that specific endpoint\n end_ids = set()\n start_ids = set()\n eplevel = 0\n for cuuid, clevel, cstart, cend in self.st.cover(epaddr):\n slevels[cuuid] = clevel\n\n if clevel > eplevel:\n eplevel = clevel\n if cstart == epaddr:\n start_ids.add(cuuid)\n if cend == epaddr:\n end_ids.add(cuuid)\n\n if cend != epaddr and cstart != epaddr:\n if cuuid not in live_ids:\n assert epaddr == eps[0]\n live_ids.add(cuuid)\n\n assert len(end_ids) + len(start_ids) > 0\n\n if len(start_ids) != 0:\n if oep is not None and oep != epaddr and len(live_ids) != 0:\n if oeplevel != WL_LEVEL:\n result.add(MWUpdate(oep, epaddr-1,\n live_ids))\n\n oep = epaddr\n oeplevel = eplevel\n live_ids = live_ids | start_ids\n\n if len(end_ids) != 0:\n if oep is not None and len(live_ids) != 0:\n if eplevel < WL_LEVEL:\n result.add(MWUpdate(oep, epaddr, live_ids))\n\n oep = epaddr+1\n live_ids = live_ids - end_ids\n\n oeplevel = eplevel\n if len(live_ids) != 0:\n oeplevel = max([slevels[id_] for id_ in live_ids])\n\n return result\n\n def _range_from_indicator(self, indicator):\n if '-' in indicator:\n start, end = map(\n lambda x: int(netaddr.IPAddress(x)),\n indicator.split('-', 1)\n )\n elif '/' in indicator:\n ipnet = netaddr.IPNetwork(indicator)\n start = int(ipnet.ip)\n end = start+ipnet.size-1\n else:\n start = int(netaddr.IPAddress(indicator))\n end = start\n\n if (not (start >= 0 and start <= 0xFFFFFFFF)) or \\\n (not (end >= 0 and end <= 0xFFFFFFFF)):\n LOG.error('%s - {%s} invalid IPv4 indicator',\n self.name, indicator)\n return None, None\n\n return start, end\n\n def _endpoints_from_range(self, start, end):\n \"\"\"Return last endpoint before range and first endpoint after range\n \n Args:\n start (int): range start\n end (int): range stop\n \n Returns:\n tuple: (last endpoint before, first endpoint after)\n \"\"\"\n\n rangestart = next(\n self.st.query_endpoints(start=0, stop=max(start-1, 0),\n reverse=True),\n None\n )\n if rangestart is not None:\n rangestart = rangestart[0]\n LOG.debug('%s - range start: %s', self.name, rangestart)\n\n rangestop = next(\n self.st.query_endpoints(reverse=False,\n start=min(end+1, self.st.max_endpoint),\n stop=self.st.max_endpoint,\n include_start=False),\n None\n )\n if rangestop is not None:\n rangestop = rangestop[0]\n LOG.debug('%s - range stop: %s', self.name, rangestop)\n\n return rangestart, rangestop\n\n @base._counting('update.processed')\n def filtered_update(self, source=None, indicator=None, value=None):\n vtype = value.get('type', None)\n if vtype != 'IPv4':\n self.statistics['update.ignored'] += 1\n return\n\n v, newindicator = self._add_indicator(source, indicator, value)\n\n start, end = self._range_from_indicator(indicator)\n if start is None or end is None:\n return\n\n level = 1\n for p in self.whitelist_prefixes:\n if source.startswith(p):\n level = WL_LEVEL\n break\n\n LOG.debug(\"%s - update: indicator: (%s) %s %s level: %s\",\n self.name, indicator, start, end, level)\n\n rangestart, rangestop = self._endpoints_from_range(start, end)\n\n rangesb = set(self._calc_ipranges(rangestart, rangestop))\n LOG.debug('%s - ranges before update: %s', self.name, rangesb)\n\n if not newindicator and level != WL_LEVEL:\n for u in rangesb:\n self.emit_update(\n u.indicator(),\n self._calc_indicator_value(u.uuids)\n )\n return\n\n uuidbytes = v['_id']\n self.st.put(uuidbytes, start, end, level=level)\n\n rangesa = set(self._calc_ipranges(rangestart, rangestop))\n LOG.debug('%s - ranges after update: %s', self.name, rangesa)\n\n added = rangesa-rangesb\n LOG.debug(\"%s - IP ranges added: %s\", self.name, added)\n\n removed = rangesb-rangesa\n LOG.debug(\"%s - IP ranges removed: %s\", self.name, removed)\n\n for u in added:\n self.emit_update(\n u.indicator(),\n self._calc_indicator_value(u.uuids)\n )\n\n for u in rangesa - added:\n for ou in rangesb:\n if u == ou and len(u.uuids ^ ou.uuids) != 0:\n LOG.debug(\"IP range updated: %s\", repr(u))\n self.emit_update(\n u.indicator(),\n self._calc_indicator_value(u.uuids)\n )\n\n for u in removed:\n self.emit_withdraw(\n u.indicator(),\n value=self._calc_indicator_value(u.uuids)\n )\n\n @base._counting('withdraw.processed')\n def filtered_withdraw(self, source=None, indicator=None, value=None):\n LOG.debug(\"%s - withdraw from %s - %s\", self.name, source, indicator)\n\n if value is not None and value.get('type', None) != 'IPv4':\n self.statistics['withdraw.ignored'] += 1\n return\n\n ik = self._indicator_key(indicator, source)\n\n v = self.table.get(ik)\n LOG.debug(\"%s - v: %s\", self.name, v)\n if v is None:\n return\n\n self.table.delete(ik)\n self.statistics['removed'] += 1\n\n start, end = self._range_from_indicator(indicator)\n if start is None or end is None:\n return\n\n level = 1\n for p in self.whitelist_prefixes:\n if source.startswith(p):\n level = WL_LEVEL\n break\n\n rangestart, rangestop = self._endpoints_from_range(start, end)\n\n rangesb = set(self._calc_ipranges(rangestart, rangestop))\n LOG.debug(\"ranges before: %s\", rangesb)\n\n uuidbytes = v['_id']\n self.st.delete(uuidbytes, start, end, level=level)\n\n rangesa = set(self._calc_ipranges(rangestart, rangestop))\n LOG.debug(\"ranges after: %s\", rangesa)\n\n added = rangesa-rangesb\n LOG.debug(\"IP ranges added: %s\", added)\n\n removed = rangesb-rangesa\n LOG.debug(\"IP ranges removed: %s\", removed)\n\n for u in added:\n self.emit_update(\n u.indicator(),\n self._calc_indicator_value(u.uuids)\n )\n\n for u in rangesa - added:\n for ou in rangesb:\n if u == ou and len(u.uuids ^ ou.uuids) != 0:\n LOG.debug(\"IP range updated: %s\", repr(u))\n self.emit_update(\n u.indicator(),\n self._calc_indicator_value(u.uuids)\n )\n\n for u in removed:\n self.emit_withdraw(\n u.indicator(),\n value=self._calc_indicator_value(\n u.uuids,\n additional_uuid=v['_id'],\n additional_value=v\n )\n )\n\n def _send_indicators(self, source=None, from_key=None, to_key=None):\n if from_key is None:\n from_key = 0\n if to_key is None:\n to_key = 0xFFFFFFFF\n\n result = self._calc_ipranges(from_key, to_key)\n for u in result:\n self.do_rpc(\n source,\n \"update\",\n indicator=u.indicator(),\n value=self._calc_indicator_value(u.uuids)\n )\n\n def get(self, source=None, indicator=None):\n if not type(indicator) in [str, unicode]:\n raise ValueError(\"Invalid indicator type\")\n\n indicator = int(netaddr.IPAddress(indicator))\n\n result = self._calc_ipranges(indicator, indicator)\n if len(result) == 0:\n return None\n\n u = result.pop()\n return self._calc_indicator_value(u.uuids)\n\n def get_all(self, source=None):\n self._send_indicators(source=source)\n return 'OK'\n\n def get_range(self, source=None, index=None, from_key=None, to_key=None):\n if index is not None:\n raise ValueError('Index not found')\n if from_key is not None:\n from_key = int(netaddr.IPAddress(from_key))\n if to_key is not None:\n to_key = int(netaddr.IPAddress(to_key))\n\n self._send_indicators(\n source=source,\n from_key=from_key,\n to_key=to_key\n )\n\n return 'OK'\n\n def length(self, source=None):\n return self.table.num_indicators\n\n def stop(self):\n super(AggregateIPv4FT, self).stop()\n\n for g in self.active_requests:\n g.kill()\n self.active_requests = []\n\n self.table.close()\n\n LOG.info(\"%s - # indicators: %d\", self.name, self.table.num_indicators)\n\n @staticmethod\n def gc(name, config=None):\n actorbase.ActorBaseFT.gc(name, config=config)\n\n shutil.rmtree(name, ignore_errors=True)\n shutil.rmtree('{}_st'.format(name), ignore_errors=True)\n" }, { "path": "minemeld/ft/json.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements minemeld.ft.json.SimpleJSON, the Miner node for JSON\nfeeds over HTTP/HTTPS.\n\"\"\"\n\nimport requests\nimport logging\nimport jmespath\nimport os\nimport yaml\n\nfrom . import basepoller\n\nLOG = logging.getLogger(__name__)\n\n\nclass SimpleJSON(basepoller.BasePollerFT):\n \"\"\"Implements class for miners of JSON feeds over http/https.\n\n **Config parameters**\n :url: URL of the feed.\n :polling_timeout: timeout of the polling request in seconds.\n Default: 20\n :verify_cert: boolean, if *true* feed HTTPS server certificate is\n verified. Default: *true*\n :username: string, for BasicAuth authentication (*password* required)\n :password: string, for BasicAuth authentication (*username* required)\n :client_cert_required: boolean, triggers client certificate authentication\n (requires *cert_file* and *key_file*)\n :cert_file: string, path to the client certificate\n :key_file: string, path to the private key of the client certificate\n :extractor: JMESPath expression for extracting the indicators from\n the JSON document. Default: @\n :indicator: the JSON attribute to use as indicator. Default: indicator\n :fields: list of JSON attributes to include in the indicator value.\n If *null* no additional attributes are extracted. Default: *null*\n :prefix: prefix to add to field names. Default: json\n\n :headers: Header parameters are optional to sepcify a user-agent or an api-token\n Example: headers = {'user-agent': 'my-app/0.0.1'} or Authorization: Bearer \n (curl -H \"Authorization: Bearer \" \"https://api-url.com/api/v1/iocs?first_seen_since=2016-1-1\")\n\n Example:\n Example config in YAML::\n\n url: https://ip-ranges.amazonaws.com/ip-ranges.json\n extractor: \"prefixes[?service=='AMAZON']\"\n prefix: aws\n indicator: ip_prefix\n headers: {'Authorization': '12345668900', 'user-agent': 'my-app/0.0.1'}\n fields:\n - region\n - service\n\n Args:\n name (str): node name, should be unique inside the graph\n chassis (object): parent chassis instance\n config (dict): node config.\n \"\"\"\n\n def configure(self):\n super(SimpleJSON, self).configure()\n\n self.url = self.config.get('url', None)\n self.polling_timeout = self.config.get('polling_timeout', 20)\n self.verify_cert = self.config.get('verify_cert', True)\n\n self.compile_error = None\n try:\n self.extractor = jmespath.compile(self.config.get('extractor', '@'))\n except Exception as e:\n LOG.debug('%s - exception in jmespath: %s',\n self.name, e)\n self.compile_error = \"{}\".format(e)\n\n self.indicator = self.config.get('indicator', 'indicator')\n self.prefix = self.config.get('prefix', 'json')\n self.fields = self.config.get('fields', None)\n\n self.username = self.config.get('username', None)\n self.password = self.config.get('password', None)\n\n self.headers = self.config.get('headers', None)\n\n # option for enabling client cert, default disabled\n self.client_cert_required = self.config.get('client_cert_required', False)\n self.key_file = self.config.get('key_file', None)\n if self.key_file is None and self.client_cert_required:\n self.key_file = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s.pem' % self.name\n )\n self.cert_file = self.config.get('cert_file', None)\n if self.cert_file is None and self.client_cert_required:\n self.cert_file = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s.crt' % self.name\n )\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n username = sconfig.get('username', None)\n password = sconfig.get('password', None)\n if username is not None and password is not None:\n self.username = username\n self.password = password\n LOG.info('{} - Loaded credentials from side config'.format(self.name))\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(SimpleJSON, self).hup(source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n\n client_cert_required = False\n if config is not None:\n client_cert_required = config.get('client_cert_required', False)\n\n if config is not None:\n cert_path = config.get('cert_file', None)\n if cert_path is None and client_cert_required:\n cert_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}.crt'.format(name)\n )\n\n if cert_path is not None:\n try:\n os.remove(cert_path)\n except:\n pass\n\n if config is not None:\n key_path = config.get('key_file', None)\n if key_path is None and client_cert_required:\n key_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}.pem'.format(name)\n )\n\n if key_path is not None:\n try:\n os.remove(key_path)\n except:\n pass\n\n def _process_item(self, item):\n if self.indicator not in item:\n LOG.debug('%s not in %s', self.indicator, item)\n return [[None, None]]\n\n indicator = item[self.indicator]\n if not (isinstance(indicator, str) or\n isinstance(indicator, unicode)):\n LOG.error(\n 'Wrong indicator type: %s - %s',\n indicator, type(indicator)\n )\n return [[None, None]]\n\n fields = self.fields\n if fields is None:\n fields = item.keys()\n fields.remove(self.indicator)\n\n attributes = {}\n for field in fields:\n if field not in item:\n continue\n attributes['%s_%s' % (self.prefix, field)] = item[field]\n\n return [[indicator, attributes]]\n\n def _build_iterator(self, now):\n if self.compile_error is not None:\n raise RuntimeError(self.compile_error)\n\n rkwargs = dict(\n stream=True,\n verify=self.verify_cert,\n timeout=self.polling_timeout\n )\n\n if self.username is not None and self.password is not None:\n rkwargs['auth'] = (self.username, self.password)\n\n if self.headers is not None:\n rkwargs['headers'] = self.headers\n\n if self.client_cert_required and self.key_file is not None and self.cert_file is not None:\n rkwargs['cert'] = (self.cert_file, self.key_file)\n\n r = requests.get(\n self.url,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.debug('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n result = self.extractor.search(r.json())\n\n return result\n" }, { "path": "minemeld/ft/local.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport yaml\nimport filelock\nimport os\n\nfrom . import basepoller\n\nLOG = logging.getLogger(__name__)\n\n\nclass YamlFT(basepoller.BasePollerFT):\n def __init__(self, name, chassis, config):\n self.file_monitor_mtime = None\n\n super(YamlFT, self).__init__(name, chassis, config)\n\n def configure(self):\n super(YamlFT, self).configure()\n\n self.path = self.config.get('path', None)\n if self.path is None:\n self.path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_indicators.yml' % self.name\n )\n self.lock_path = self.path+'.lock'\n\n def _flush(self):\n self.file_monitor_mtime = None\n super(YamlFT, self)._flush()\n\n def _process_item(self, item):\n indicator = item.pop('indicator', None)\n if indicator is None:\n return [[None, None]]\n\n item['sources'] = [self.name]\n\n return [[indicator, item]]\n\n def _load_yaml(self):\n with filelock.FileLock(self.lock_path).acquire(timeout=10):\n with open(self.path, 'r') as f:\n result = yaml.safe_load(f)\n\n if type(result) != list:\n raise RuntimeError(\n '%s - %s should be a list of indicators' %\n (self.name, self.path)\n )\n\n return result\n\n def _build_iterator(self, now):\n if self.path is None:\n LOG.warning('%s - no path configured', self.name)\n raise RuntimeError('%s - no path configured' % self.name)\n\n try:\n mtime = os.stat(self.path).st_mtime\n except OSError as e:\n if e.errno == 2: # no such file\n return None\n\n LOG.exception('%s - error checking mtime of %s',\n self.name, self.path)\n raise RuntimeError(\n '%s - error checking indicators list' % self.name\n )\n\n if mtime == self.file_monitor_mtime:\n return None\n\n self.file_monitor_mtime = mtime\n\n try:\n return self._load_yaml()\n\n except:\n LOG.exception('%s - exception loading indicators list', self.name)\n raise\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n path = None\n if config is not None:\n path = config.get('path', None)\n if path is None:\n path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_indicators.yml'.format(name)\n )\n lock_path = '{}.lock'.format(path)\n\n try:\n os.remove(path)\n except:\n pass\n\n try:\n os.remove(lock_path)\n except:\n pass\n\n\nclass YamlIPv4FT(YamlFT):\n def _process_item(self, item):\n item['type'] = 'IPv4'\n\n return super(YamlIPv4FT, self)._process_item(item)\n\n\nclass YamlURLFT(YamlFT):\n def _process_item(self, item):\n item['type'] = 'URL'\n\n return super(YamlURLFT, self)._process_item(item)\n\n\nclass YamlDomainFT(YamlFT):\n def _process_item(self, item):\n item['type'] = 'domain'\n\n return super(YamlDomainFT, self)._process_item(item)\n\n\nclass YamlIPv6FT(YamlFT):\n def _process_item(self, item):\n item['type'] = 'IPv6'\n\n return super(YamlIPv6FT, self)._process_item(item)\n" }, { "path": "minemeld/ft/localdb.py", "content": "# Copyright 2017-present Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport os\nimport os.path\nimport logging\nimport sqlite3\nfrom contextlib import contextmanager\n\nimport ujson as json\n\nfrom . import basepoller\nfrom . import ft_states\nfrom .utils import interval_in_sec, dt_to_millisec, utc_millisec\n\nLOG = logging.getLogger(__name__)\n\n_MAX_AGE_OUT = ((1 << 32)-1)*1000 # 2106-02-07 6:28:15\n\n\n@contextmanager\ndef dbconnection(path):\n conn = sqlite3.connect(path)\n\n yield conn\n\n conn.close()\n\n\nclass Miner(basepoller.BasePollerFT):\n def __init__(self, name, chassis, config):\n super(Miner, self).__init__(name, chassis, config)\n\n self.last_run = None\n\n def configure(self):\n if not 'age_out' in self.config:\n self.config['age_out'] = {\n 'interval': 1800,\n 'sudden_death': False,\n 'default': None\n }\n\n super(Miner, self).configure()\n\n self.default_ttl = self.config.get('default_ttl', 86400)\n\n self.path = self.config.get('path', None)\n if self.path is None:\n self.path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_indicators.db' % self.name\n )\n\n def _collect_garbage(self):\n if not os.path.isfile(self.path):\n return\n\n now = utc_millisec()\n\n with self.state_lock, dbconnection(self.path) as conn:\n if self.state != ft_states.STARTED:\n return\n\n with conn:\n for i, v in self.table.query(index='_withdrawn',\n to_key=now,\n include_value=True):\n # if v.get('_last_run', 0) >= (self.last_successful_run-1):\n # continue\n\n itype = v.get('type', None)\n\n conn.execute('delete from indicators where indicator=? and type=?;', (i, itype))\n\n self.table.delete(i, itype=itype)\n self.statistics['garbage_collected'] += 1\n\n def _calc_age_out(self, indicator, attributes):\n if isinstance(attributes['_expiration_ts'], int):\n return attributes['_expiration_ts']\n\n return _MAX_AGE_OUT\n\n def _process_item(self, item):\n indicator = item[0]\n value = json.loads(item[2])\n value['type'] = item[1]\n value['_expiration_ts'] = item[3]\n\n if value['_expiration_ts'] is None:\n # if none, expiration is set to update_ts+default_ttl\n value['_expiration_ts'] = item[4]+self.default_ttl*1000\n\n return [[indicator, value]]\n\n def _updates_iterator(self, last_successful_run):\n with dbconnection(self.path) as conn:\n for row in conn.execute('select * from indicators where update_ts >= ?', (last_successful_run,)):\n yield row\n\n def _build_iterator(self, now):\n if not os.path.isfile(self.path):\n return []\n\n last_successful_run = 0\n if self.last_successful_run is not None:\n last_successful_run = self.last_successful_run\n\n return self._updates_iterator(last_successful_run)\n\n def hup(self, source=None):\n super(Miner, self).hup(source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n path = None\n if config is not None:\n path = config.get('path', None)\n if path is None:\n path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_indicators.db'.format(name)\n )\n\n try:\n os.remove(path)\n except:\n pass\n" }, { "path": "minemeld/ft/logstash.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport ujson\nimport datetime\nimport socket\n\nfrom . import base\nfrom . import actorbase\n\nLOG = logging.getLogger(__name__)\n\n\nclass LogstashOutput(actorbase.ActorBaseFT):\n def __init__(self, name, chassis, config):\n super(LogstashOutput, self).__init__(name, chassis, config)\n\n self._ls_socket = None\n\n def configure(self):\n super(LogstashOutput, self).configure()\n\n self.logstash_host = self.config.get('logstash_host', '127.0.0.1')\n self.logstash_port = int(self.config.get('logstash_port', '5514'))\n\n def connect(self, inputs, output):\n output = False\n super(LogstashOutput, self).connect(inputs, output)\n\n def _connect_logstash(self):\n if self._ls_socket is not None:\n return\n\n _ls_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n _ls_socket.connect((self.logstash_host, self.logstash_port))\n\n self._ls_socket = _ls_socket\n\n def initialize(self):\n pass\n\n def rebuild(self):\n pass\n\n def reset(self):\n pass\n\n def _send_logstash(self, message, source=None, indicator=None, value=None):\n now = datetime.datetime.now()\n\n fields = {\n '@timestamp': now.isoformat()+'Z',\n '@version': 1,\n 'logstash_output_node': self.name,\n 'message': message\n }\n\n if indicator is not None:\n fields['@indicator'] = indicator\n\n if source is not None:\n fields['@origin'] = source\n\n if value is not None:\n fields.update(value)\n\n if 'last_seen' in fields:\n last_seen = datetime.datetime.fromtimestamp(\n float(fields['last_seen'])/1000.0\n )\n fields['last_seen'] = last_seen.isoformat()+'Z'\n\n if 'first_seen' in fields:\n first_seen = datetime.datetime.fromtimestamp(\n float(fields['first_seen'])/1000.0\n )\n fields['first_seen'] = first_seen.isoformat()+'Z'\n\n try:\n self._connect_logstash()\n self._ls_socket.sendall(ujson.dumps(fields)+'\\n')\n except:\n self._ls_socket = None\n raise\n\n self.statistics['message.sent'] += 1\n\n @base._counting('update.processed')\n def filtered_update(self, source=None, indicator=None, value=None):\n self._send_logstash(\n 'update',\n source=source,\n indicator=indicator,\n value=value\n )\n\n @base._counting('withdraw.processed')\n def filtered_withdraw(self, source=None, indicator=None, value=None):\n self._send_logstash(\n 'withdraw',\n source=source,\n indicator=indicator,\n value=value\n )\n\n def length(self, source=None):\n return 0\n" }, { "path": "minemeld/ft/mm.py", "content": "# Copyright 2017-present Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements minemeld.ft.mm.JSONSEQMiner, the Miner node for\nMineMeld JSON-SEQ feeds over HTTP/HTTPS.\n\"\"\"\n\nimport os.path\nimport logging\n\nimport requests\nimport yaml\nimport ujson\n\nfrom . import basepoller\n\nLOG = logging.getLogger(__name__)\n\n\nclass JSONSEQMiner(basepoller.BasePollerFT):\n \"\"\"Implements class for miners of MineMeld JSON-SEQ feeds over http/https.\n\n **Config parameters**\n :url: URL of the feed.\n :polling_timeout: timeout of the polling request in seconds.\n Default: 20\n :verify_cert: boolean, if *true* feed HTTPS server certificate is\n verified. Default: *true*\n :side_config_path: path to the side config with credentials for the feed\n\n Args:\n name (str): node name, should be unique inside the graph\n chassis (object): parent chassis instance\n config (dict): node config.\n \"\"\"\n def configure(self):\n super(JSONSEQMiner, self).configure()\n\n self.polling_timeout = self.config.get('polling_timeout', 20)\n self.verify_cert = self.config.get('verify_cert', True)\n\n self.url = self.config.get('url', None)\n\n self.username = None\n self.password = None\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n username = sconfig.get('username', None)\n password = sconfig.get('password', None)\n if username is not None and password is not None:\n self.username = username\n self.password = password\n LOG.info('{} - Loaded credentials from side config'.format(self.name))\n\n def _process_item(self, item):\n return [[item['indicator'], item['value']]]\n\n def _json_seq_iterator(self, r):\n for line in r.iter_lines(decode_unicode=True, delimiter='\\x1E'):\n if line:\n try:\n yield ujson.loads(line)\n except ValueError:\n LOG.error('{} - Error parsing {!r}'.format(self.name, line))\n\n def _build_iterator(self, now):\n if self.url is None:\n raise RuntimeError(\n '{} - feed url not set'.format(self.name)\n )\n\n rkwargs = dict(\n stream=True,\n verify=self.verify_cert,\n timeout=self.polling_timeout,\n params={'v': 'json-seq'}\n )\n\n if self.username is not None and self.password is not None:\n rkwargs['auth'] = (self.username, self.password)\n\n r = requests.get(\n self.url,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.debug('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n return self._json_seq_iterator(r)\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(JSONSEQMiner, self).hup(source)\n" }, { "path": "minemeld/ft/o365.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport itertools\nimport functools\nimport uuid\nimport os\nimport json\nfrom collections import defaultdict\n\nimport yaml\nimport netaddr\nimport requests\nimport lxml.etree\n\nfrom . import basepoller\n\nLOG = logging.getLogger(__name__)\n\nO365_URL = \\\n 'https://support.content.office.net/en-us/static/O365IPAddresses.xml'\nXPATH_FUNS_NS = 'http://minemeld.panw.io/o365functions'\nXPATH_FUNS_PREFIX = 'o365f'\nXPATH_PRODUCTS = \"/products/product/@name\"\nBASE_XPATH = \"/products/product[\" + XPATH_FUNS_PREFIX + \":lower-case(@name)='%s']\"\n\nO365_API_BASE_URL = 'https://endpoints.office.com'\n\ndef _build_IPv4(source, address):\n item = {\n 'indicator': address.text,\n 'type': 'IPv4',\n 'confidence': 100,\n 'sources': [source]\n }\n return item\n\n\ndef _build_IPv6(source, address):\n item = {\n 'indicator': address.text,\n 'type': 'IPv6',\n 'confidence': 100,\n 'sources': [source]\n }\n return item\n\n\ndef _build_URL(source, url):\n item = {\n 'indicator': url.text,\n 'type': 'URL',\n 'confidence': 100,\n 'sources': [source]\n }\n return item\n\n\ndef _xpath_lower_case(context, a):\n return [e.lower() for e in a]\n\n\nclass O365XML(basepoller.BasePollerFT):\n def configure(self):\n super(O365XML, self).configure()\n\n # register lower-case\n ns = lxml.etree.FunctionNamespace(XPATH_FUNS_NS)\n ns['lower-case'] = _xpath_lower_case\n self.prefixmap = {XPATH_FUNS_PREFIX: XPATH_FUNS_NS}\n\n self.polling_timeout = self.config.get('polling_timeout', 20)\n self.verify_cert = self.config.get('verify_cert', True)\n self.products = self.config.get('products', [])\n\n self.url = self.config.get('url', O365_URL)\n\n def _process_item(self, item):\n indicator = item.pop('indicator', None)\n return [[indicator, item]]\n\n def _build_request(self, now):\n r = requests.Request(\n 'GET',\n self.url\n )\n\n return r.prepare()\n\n def _o365_iterator(self, now):\n _iterators = []\n\n _session = requests.Session()\n _adapter = requests.adapters.HTTPAdapter(\n pool_connections=10,\n pool_maxsize=10,\n max_retries=3\n )\n _session.mount('https://', _adapter)\n\n prepreq = self._build_request(now)\n\n # this is to honour the proxy environment variables\n rkwargs = _session.merge_environment_settings(\n prepreq.url,\n {}, None, None, None # defaults\n )\n rkwargs['stream'] = True\n rkwargs['verify'] = self.verify_cert\n rkwargs['timeout'] = self.polling_timeout\n r = _session.send(prepreq, **rkwargs)\n\n try:\n r.raise_for_status()\n except:\n LOG.debug('%s - exception in request: %s %s',\n self.name, r.status_code, r.text)\n raise\n\n parser = lxml.etree.XMLParser()\n for chunk in r.iter_content(chunk_size=10 * 1024):\n parser.feed(chunk)\n rtree = parser.close()\n\n products = self.products\n if len(products) == 0:\n products = self._extract_products(rtree)\n\n for p in products:\n xpath = BASE_XPATH % p.lower()\n pIPv4s = rtree.xpath(\n xpath + \"/addresslist[@type='IPv4']/address\",\n namespaces=self.prefixmap\n )\n _iterators.append(itertools.imap(\n functools.partial(_build_IPv4, 'office365.%s' % p.lower()),\n pIPv4s\n ))\n\n pIPv6s = rtree.xpath(\n xpath + \"/addresslist[@type='IPv6']/address\",\n namespaces=self.prefixmap\n )\n _iterators.append(itertools.imap(\n functools.partial(_build_IPv6, 'office365.%s' % p.lower()),\n pIPv6s\n ))\n\n pURLs = rtree.xpath(\n xpath + \"/addresslist[@type='URL']/address\",\n namespaces=self.prefixmap\n )\n _iterators.append(itertools.imap(\n functools.partial(_build_URL, 'office365.%s' % p.lower()),\n pURLs\n ))\n\n return itertools.chain(*_iterators)\n\n def _build_iterator(self, now):\n oiterator = self._o365_iterator(now)\n\n idict = {}\n for i in oiterator:\n indicator = i['indicator']\n cvalue = idict.get(indicator, None)\n if cvalue is not None:\n i['sources'] = list(set(i['sources']) | set(cvalue['sources']))\n idict[indicator] = i\n\n return itertools.imap(lambda i: i[1], idict.iteritems())\n\n def _extract_products(self, rtree):\n products = rtree.xpath(XPATH_PRODUCTS)\n LOG.info('%s - found products: %r', self.name, products)\n return products\n\n\nO365_API_FIELDS = [\n 'id',\n 'expressRoute',\n 'notes',\n 'serviceArea',\n 'tcpPorts',\n 'udpPorts',\n 'category',\n 'required'\n]\n\n\nclass O365API(basepoller.BasePollerFT):\n def __init__(self, name, chassis, config):\n self.client_request_id = str(uuid.uuid4())\n self.latest_version = '0000000000'\n\n super(O365API, self).__init__(name, chassis, config)\n\n def configure(self):\n super(O365API, self).configure()\n\n self.polling_timeout = self.config.get('polling_timeout', 20)\n self.verify_cert = self.config.get('verify_cert', True)\n\n self.instance = self.config.get('instance', 'O365Worldwide')\n self.service_areas = self.config.get('service_areas', None)\n self.tenant_name = self.config.get('tenant_name', None)\n self.disable_integrations = self.config.get('disable_integrations', False)\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n disable_integrations = sconfig.get('disable_integrations', None)\n if disable_integrations is not None:\n self.disable_integrations = disable_integrations\n LOG.info('{} - Loaded side config'.format(self.name))\n\n def _saved_state_restore(self, saved_state):\n super(O365API, self)._saved_state_restore(saved_state)\n\n self.client_request_id = saved_state.get('client_request_id', None)\n self.latest_version = saved_state.get('latest_version', None)\n\n LOG.info('saved state: client_request_id: {} latest_version: {}'.format(\n self.client_request_id,\n self.latest_version\n ))\n\n def _saved_state_create(self):\n sstate = super(O365API, self)._saved_state_create()\n\n sstate['latest_version'] = self.latest_version\n sstate['client_request_id'] = self.client_request_id\n\n return sstate\n\n def _saved_state_reset(self):\n super(O365API, self)._saved_state_reset()\n\n self.client_request_id = str(uuid.uuid4())\n self.latest_version = '0000000000'\n\n def _check_version(self):\n rkwargs = dict(\n stream=False,\n verify=self.verify_cert,\n timeout=self.polling_timeout,\n params={\n 'clientrequestid': self.client_request_id\n }\n )\n\n url = '{}/version/{}'.format(\n O365_API_BASE_URL,\n self.instance\n )\n\n r = requests.get(\n url,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.debug('{} - exception in request: {} {!r}'.format(\n self.name, r.status_code, r.content\n ))\n raise\n\n version = r.json()\n\n LOG.debug('{} - version: {}'.format(self.name, version))\n\n if version['latest'] > self.latest_version:\n return version['latest']\n\n return\n\n def _process_item(self, item):\n return [item]\n\n def _analyze_item(self, item):\n result = []\n\n base_value = {}\n for wka in O365_API_FIELDS:\n if wka in item:\n base_value['o365_{}'.format(wka)] = item[wka]\n\n if self.disable_integrations and 'o365_notes' in base_value:\n if 'integration' in base_value['o365_notes'].lower():\n return result\n\n for url in item.get('urls', []):\n value = base_value.copy()\n value['type'] = 'URL'\n\n result.append([url, value])\n\n for ip in item.get('ips', []):\n try:\n parsed = netaddr.IPNetwork(ip)\n except (netaddr.AddrFormatError, ValueError):\n LOG.error('{} - Unknown IP version: {}'.format(self.name, ip))\n continue\n\n value = base_value.copy()\n if parsed.version == 4:\n value['type'] = 'IPv4'\n elif parsed.version == 6:\n value['type'] = 'IPv6'\n\n result.append([ip, value])\n\n return result\n\n def _iterator(self, array, latest_version):\n indicators = defaultdict(lambda: {'o365_{}_list'.format(f): set() for f in O365_API_FIELDS})\n\n for i in array:\n for ci, cv in self._analyze_item(i):\n oldv = indicators[ci]\n oldv.update(cv)\n for fn in O365_API_FIELDS:\n label = 'o365_{}'.format(fn)\n if label in cv:\n val = str(cv[label]).lower()\n if label in ['o365_tcpPorts', 'o365_udpPorts']:\n ports = val.split(',')\n for p in ports:\n oldv['{}_list'.format(label)].add(str(p))\n else:\n oldv['{}_list'.format(label)].add(str(val))\n\n for i, v in indicators.iteritems():\n for fn in O365_API_FIELDS:\n label = 'o365_{}_list'.format(fn)\n v[label] = list(v[label])\n yield [i, v]\n\n self.latest_version = latest_version\n\n def _build_iterator(self, now):\n latest_version = self._check_version()\n if latest_version is None:\n LOG.info('{} - Already latest version, polling not performed'.format(\n self.name\n ))\n return None\n\n rkwargs = dict(\n stream=False,\n verify=self.verify_cert,\n timeout=self.polling_timeout,\n params={\n 'clientrequestid': self.client_request_id\n }\n )\n if self.tenant_name is not None:\n rkwargs['params']['tenantname'] = self.tenant_name\n if self.service_areas is not None:\n rkwargs['params']['serviceareas'] = ','.join(self.service_areas)\n\n url = '{}/endpoints/{}'.format(\n O365_API_BASE_URL,\n self.instance\n )\n\n r = requests.get(\n url,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.debug('{} - exception in request: {} {!r}'.format(\n self.name, r.status_code, r.content\n ))\n raise\n\n return self._iterator(r.json(), latest_version)\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n self.latest_version = None\n super(O365API, self).hup(source=source)\n" }, { "path": "minemeld/ft/op.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport shutil\n\nfrom . import base\nfrom . import actorbase\nfrom . import table\nfrom .utils import utc_millisec\nfrom .utils import RESERVED_ATTRIBUTES\n\nLOG = logging.getLogger(__name__)\n\n\nclass AggregateFT(actorbase.ActorBaseFT):\n _ftclass = 'AggregateFT'\n\n def __init__(self, name, chassis, config):\n self.active_requests = []\n self.table = None\n\n super(AggregateFT, self).__init__(name, chassis, config)\n\n def configure(self):\n super(AggregateFT, self).configure()\n\n self.whitelist_prefixes = self.config.get('whitelist_prefixes', [])\n self.ignore_cases = self.config.get('ignore_cases', False)\n\n def _initialize_table(self, truncate=False):\n self.table = table.Table(self.name, truncate=truncate)\n\n def initialize(self):\n self._initialize_table()\n\n def rebuild(self):\n self._initialize_table(truncate=True)\n\n def reset(self):\n self._initialize_table(truncate=True)\n\n def _indicator_key(self, indicator, source):\n return indicator+'\\x00'+source\n\n def _is_whitelist(self, s):\n for p in self.whitelist_prefixes:\n if s.startswith(p):\n return True\n return False\n\n def _emit_update_indicator(self, indicator):\n LOG.debug(\"%s - emitting update: %s\", self.name, indicator)\n\n mv = {'sources': []}\n for s in self.inputs:\n if self._is_whitelist(s):\n continue\n\n v = self.table.get(self._indicator_key(indicator, s))\n if v is None:\n continue\n\n for k in v.keys():\n if k in mv and k in RESERVED_ATTRIBUTES:\n mv[k] = RESERVED_ATTRIBUTES[k](mv[k], v[k])\n else:\n mv[k] = v[k]\n\n if len(mv) > 1:\n self.emit_update(indicator, mv)\n\n def _merge_values(self, source, ov, nv):\n result = {'sources': []}\n\n result['_added'] = ov['_added']\n\n for k in nv.keys():\n result[k] = nv[k]\n\n return result\n\n def _add_indicator(self, source, indicator, value):\n now = utc_millisec()\n\n v = self.table.get(self._indicator_key(indicator, source))\n if v is None:\n v = {\n '_added': now,\n }\n\n v = self._merge_values(source, v, value)\n v['_updated'] = now\n\n self.table.put(self._indicator_key(indicator, source), v)\n\n return v\n\n @base._counting('update.processed')\n def filtered_update(self, source=None, indicator=None, value=None):\n if self.ignore_cases:\n indicator = indicator.lower()\n\n ebl = False\n ewl = False\n for i in self.inputs:\n v = self.table.exists(self._indicator_key(indicator, i))\n if self._is_whitelist(i):\n ewl |= v\n else:\n ebl |= v\n\n v = self._add_indicator(source, indicator, value)\n\n if self._is_whitelist(source):\n # update from whitelist\n if ewl:\n # already whitelisted, no updates\n return\n\n if ebl:\n self.emit_withdraw(indicator)\n\n else:\n if ewl:\n return\n self._emit_update_indicator(indicator)\n\n @base._counting('withdraw.processed')\n def filtered_withdraw(self, source=None, indicator=None, value=None):\n if self.ignore_cases:\n indicator = indicator.lower()\n\n ikey = self._indicator_key(indicator, source)\n\n cvalue = self.table.get(ikey)\n e = (cvalue is not None)\n if value is not None and cvalue is not None:\n if value.get('type', None) != cvalue.get('type', None):\n self.statistics['withdraw.ignored'] += 1\n return\n\n ebl = 0\n ewl = 0\n for i in self.inputs:\n v = int(self.table.exists(self._indicator_key(indicator, i)))\n if self._is_whitelist(i):\n ewl += v\n else:\n ebl += v\n\n self.table.delete(ikey)\n\n if self._is_whitelist(source):\n # withdraw from whitelist\n if e and ewl > 1:\n return\n\n if ebl != 0:\n self._emit_update_indicator(indicator)\n\n else:\n if ewl > 0:\n return\n\n if e:\n if ebl > 1:\n self._emit_update_indicator(indicator)\n else:\n self.emit_withdraw(indicator, value=cvalue)\n\n def get(self, source=None, indicator=None):\n mv = {}\n for s in self.inputs:\n v = self.table.get(self._indicator_key(indicator, s))\n if v is None:\n continue\n\n for k in v.keys():\n if k in mv and k in RESERVED_ATTRIBUTES:\n mv[k] = RESERVED_ATTRIBUTES[k](mv[k], v[k])\n else:\n mv[k] = v[k]\n\n return mv\n\n def get_all(self, source=None):\n return self.get_range(source=source)\n\n def get_range(self, source=None, index=None, from_key=None, to_key=None):\n if index is not None:\n raise ValueError(\"Index not found\")\n\n if to_key is not None:\n to_key = self._indicator_key(to_key, '\\x7F')\n\n cindicator = None\n cvalue = {}\n for k, v in self.table.query(index=index, from_key=from_key,\n to_key=to_key, include_value=True):\n indicator, _ = k.split('\\x00')\n if indicator == cindicator:\n for vk in v.keys():\n if vk in cvalue and vk in RESERVED_ATTRIBUTES:\n cvalue[vk] = RESERVED_ATTRIBUTES[vk](cvalue[vk], v[vk])\n else:\n cvalue[vk] = v[vk]\n\n else:\n if cindicator is not None:\n self.do_rpc(source, \"update\", indicator=cindicator,\n value=cvalue)\n cindicator = indicator\n cvalue = v\n\n if cindicator is not None:\n self.do_rpc(source, \"update\", indicator=cindicator,\n value=cvalue)\n\n return 'OK'\n\n def length(self, source=None):\n return self.table.num_indicators\n\n def stop(self):\n super(AggregateFT, self).stop()\n\n for g in self.active_requests:\n g.kill()\n self.active_requests = []\n\n self.table.close()\n\n LOG.info(\"%s - # indicators: %d\", self.name, self.table.num_indicators)\n\n @staticmethod\n def gc(name, config=None):\n actorbase.ActorBaseFT.gc(name, config=config)\n shutil.rmtree(name, ignore_errors=True)\n" }, { "path": "minemeld/ft/panos.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport gevent\nimport gevent.event\nimport minemeld.packages.panforest\nimport random\nimport copy\nimport re\n\nimport pan.xapi\n\nfrom . import base\nfrom . import table\nfrom .utils import utc_millisec\n\nLOG = logging.getLogger(__name__)\n\nclass CheckpointSet(Exception):\n pass\n\nclass InterruptablePanForest(minemeld.packages.panforest.PanForest):\n def __init__(self, wobject, xapi=None, log_type=None, filter=None,\n nlogs=None, format=None):\n super(InterruptablePanForest, self).__init__(\n xapi=xapi,\n log_type=log_type,\n filter=filter,\n nlogs=nlogs,\n format=format\n )\n self.wobject = wobject\n\n def sleep(self, t):\n value = self.wobject.wait(timeout=t)\n LOG.debug('value %s', value)\n if value is not None:\n raise CheckpointSet()\n\ndef _age_out_in_usecs(val):\n multipliers = {\n '': 1000,\n 'm': 60000,\n 'h': 3600000,\n 'd': 86400000\n }\n\n mo = re.match(\"([0-9]+)([dmh]?)\", val)\n if mo is None:\n return None\n\n return int(mo.group(1))*multipliers[mo.group(2)]\n\ndef _sleeper(slot, maxretries):\n c = 0\n while c < maxretries:\n yield slot*random.uniform(0, (2**c-1))\n c = c+1\n\n yield maxretries*slot\n\nclass PanOSLogsAPIFT(base.BaseFT):\n def __init__(self, name, chassis, config):\n self.glet = None\n\n self.age_out_glets = []\n\n self.tables = []\n self.active_requests = []\n self.rebuild_flag = False\n self.last_log = None\n self.idle_waitobject = gevent.event.AsyncResult()\n\n super(PanOSLogsAPIFT, self).__init__(name, chassis, config)\n\n def configure(self):\n super(PanOSLogsAPIFT, self).configure()\n\n self.source_name = self.config.get('source_name', self.name)\n self.tag = self.config.get('tag', None)\n self.hostname = self.config.get('hostname', None)\n self.api_key = self.config.get('api_key', None)\n self.api_username = self.config.get('api_username', None)\n self.api_password = self.config.get('api_password', None)\n self.log_type = self.config.get('log_type', None)\n self.filter = self.config.get('filter', None)\n self.sleeper_slot = int(self.config.get('sleeper_slot', '10'))\n self.maxretries = int(self.config.get('maxretries', '16'))\n self.fields = self.config.get('fields', [])\n self.age_out_interval = int(self.config.get('age_out_interval', '3600'))\n\n def _initialize_tables(self, truncate=False):\n for idx, field in enumerate(self.fields):\n t = table.Table(\n self.name+'_%d' % idx,\n truncate=truncate\n )\n t.create_index('last_seen')\n self.tables.append(t)\n\n def initialize(self):\n self._initialize_tables()\n\n def rebuild(self):\n self._initialize_tables()\n self.rebuild_flag = True\n\n def reset(self):\n self._initialize_tables(truncate=True)\n\n def emit_checkpoint(self, value):\n LOG.debug(\"%s - checkpoint set to %s\", self.name, value)\n self.idle_waitobject.set(value)\n\n def _age_out_loop(self, fieldidx):\n interval = self.fields[fieldidx].get('age_out', '30d')\n interval = _age_out_in_usecs(interval)\n t = self.tables[fieldidx]\n\n while True:\n try:\n now = utc_millisec()\n for i, v in t.query(index='last_seen', to_key=now-interval,\n include_value=True):\n LOG.debug('%s - %s %s aged out', self.name, i, v)\n self.emit_withdraw(indicator=i)\n t.delete(i)\n\n except gevent.GreenletExit:\n break\n\n except:\n LOG.exception('Exception in _age_out_loop')\n\n gevent.sleep(self.age_out_interval)\n\n def _run(self):\n if self.rebuild_flag:\n LOG.debug(\"rebuild flag set, resending current indicators\")\n # reinit flag is set, emit update for all the known indicators\n for t in self.tables:\n for i, v in t.query('last_seen', include_value=True):\n self.emit_update(i, v)\n\n sleeper = _sleeper(self.sleeper_slot, self.maxretries)\n checkpoint = None\n while True:\n try:\n xapi = pan.xapi.PanXapi(\n api_username=self.api_username,\n api_password=self.api_password,\n api_key=self.api_key,\n hostname=self.hostname,\n tag=self.tag,\n timeout=60\n )\n pf = InterruptablePanForest(\n self.idle_waitobject,\n xapi=xapi,\n log_type=self.log_type,\n filter=self.filter,\n format='python'\n )\n\n for log in pf.follow():\n sleeper = _sleeper(self.sleeper_slot, self.maxretries)\n\n self.statistics['log.processed'] += 1\n\n now = utc_millisec()\n\n for idx, field in enumerate(self.fields):\n if field['name'] in log:\n v = copy.copy(field['attributes'])\n v['last_seen'] = now\n self.tables[idx].put(log[field['name']], v)\n self.emit_update(indicator=log[field['name']], value=v)\n else:\n LOG.debug('%s - field %s not found', self.name, field['name'])\n\n if self.idle_waitobject.ready():\n break\n\n except gevent.GreenletExit:\n pass\n\n except CheckpointSet:\n LOG.debug('%s - CheckpointSet catched')\n pass\n\n except:\n LOG.exception(\"%s - exception in log loop\", self.name)\n\n try:\n checkpoint = self.idle_waitobject.get(timeout=next(sleeper))\n except gevent.Timeout:\n pass\n\n LOG.debug('%s - checkpoint: %s', self.name, checkpoint)\n if checkpoint is not None:\n super(PanOSLogsAPIFT, self).emit_checkpoint(checkpoint)\n break\n\n def length(self, source=None):\n return sum([t.num_indicators for t in self.tables])\n\n def start(self):\n super(PanOSLogsAPIFT, self).start()\n\n if self.glet is not None:\n return\n\n self.glet = gevent.spawn_later(random.randint(0, 2), self._run)\n\n for idx in range(len(self.fields)):\n self.age_out_glets.append(\n gevent.spawn(self._age_out_loop, idx)\n )\n\n def stop(self):\n super(PanOSLogsAPIFT, self).stop()\n\n if self.glet is None:\n return\n\n for g in self.active_requests:\n g.kill()\n\n self.glet.kill()\n for g in self.age_out_glets:\n g.kill()\n self.age_out_glets = None\n\n for t in self.tables:\n LOG.info(\"%s - # indicators: %d\", self.name, t.num_indicators)\n" }, { "path": "minemeld/ft/phishme.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements minemeld.ft.phishme.Intelligence, the Miner node for\nPhishMe Intelligence API.\n\"\"\"\n\nimport os\nimport yaml\nimport requests\nimport itertools\nimport logging\n\nfrom . import basepoller\nfrom .utils import interval_in_sec\n\nLOG = logging.getLogger(__name__)\n\n\n_API_BASE = 'https://www.threathq.com/apiv1'\n_API_THREAT_SEARCH = '/threat/search'\n_API_THREAT_UPDATE = '/threat/updates'\n_API_USER_AGENT = 'PhishMe Intelligence (minemeld)'\n\n_RESULTS_PER_PAGE = 10\n\n\nclass Intelligence(basepoller.BasePollerFT):\n def __init__(self, name, chassis, config):\n self.position = None\n\n super(Intelligence, self).__init__(name, chassis, config)\n\n def configure(self):\n super(Intelligence, self).configure()\n\n self.polling_timeout = self.config.get('polling_timeout', 20)\n self.verify_cert = self.config.get('verify_cert', True)\n\n self.prefix = self.config.get('prefix', 'phishme')\n initial_interval = self.config.get('initial_interval', '30d')\n self.initial_interval = interval_in_sec(initial_interval)\n if self.initial_interval is None:\n LOG.error(\n '%s - wrong initial_interval format: %s',\n self.name, initial_interval\n )\n self.initial_interval = interval_in_sec('30d')\n\n self.fields = self.config.get('fields', [\n 'threatDetailURL',\n 'label',\n 'threatType'\n ])\n self.confidence_map = self.config.get('confidence_map', {\n 'Major': 100,\n 'Moderate': 70,\n 'Minor': 34,\n 'None': 0\n })\n self.product = self.config.get('product', 'malware')\n self.source_name = self.config.get('source_name', 'phishme.intelligence')\n self.headers = {'user-agent': _API_USER_AGENT}\n\n self.api_key = None\n self.username = None\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.api_key = sconfig.get('api_key', None)\n if self.api_key is not None:\n LOG.info('%s - API Key set', self.name)\n\n self.username = sconfig.get('username', None)\n if self.username is not None:\n LOG.info('%s - username set', self.name)\n\n def _saved_state_restore(self, saved_state):\n super(Intelligence, self)._saved_state_restore(saved_state)\n self.position = saved_state.get('position', None)\n LOG.info('position from sstate: %s', self.position)\n\n def _saved_state_create(self):\n sstate = super(Intelligence, self)._saved_state_create()\n sstate['position'] = self.position\n\n return sstate\n\n def _saved_state_reset(self):\n super(Intelligence, self)._saved_state_reset()\n self.position = None\n\n def _update_attributes(self, current, _new, current_run, new_run):\n LOG.debug('current: %r', current)\n LOG.debug('_new: %r', _new)\n\n # create temp store for phishme spec values\n phishme_values = {}\n\n # loop over the phishme fields\n for f in self.fields:\n field_name = self.prefix+'_'+f\n\n newv = _new.get(field_name, None)\n if newv is None:\n continue\n\n phishme_values[field_name] = current.get(field_name, [])\n if newv[0] not in phishme_values[field_name]:\n phishme_values[field_name].append(newv[0])\n\n # add role\n field_name = self.prefix+'_role'\n newrole = _new.get(field_name, None)\n if newrole is not None:\n phishme_values[field_name] = current.get(field_name, [])\n if newrole[0] not in phishme_values[field_name]:\n phishme_values[field_name].append(newrole[0])\n\n # impact and confidence\n if _new['confidence'] < current['confidence']:\n phishme_values['confidence'] = current['confidence']\n phishme_values[self.prefix+'_impact'] = current[self.prefix+'_impact']\n\n LOG.debug(phishme_values)\n\n current.update(_new)\n current.update(phishme_values)\n\n return current\n\n def _convert_block(self, block):\n v = {}\n\n impact = block.get('impact', None)\n if impact is not None:\n v[self.prefix+'_impact'] = impact\n\n if impact in self.confidence_map:\n v['confidence'] = self.confidence_map[impact]\n\n role = block.get('role', None)\n if role is not None:\n v[self.prefix+'_role'] = [role]\n\n type_ = block.get('blockType', None)\n if type_ is None:\n LOG.error(\n '%s - no \"blockType\" attribute in block',\n self.name\n )\n return None, None\n\n if type_ == 'IPv4 Address':\n v['type'] = 'IPv4'\n elif type_ == 'Domain Name':\n v['type'] = 'domain'\n elif type_ == 'URL':\n v['type'] = 'URL'\n else:\n LOG.error('%s - unknown blockType: %s', self.name, type_)\n return None, None\n\n indicator = block.get('data', None)\n if indicator is None:\n LOG.error('%s - no \"data\" attribute in block', self.name)\n return None, None\n\n return indicator, v\n\n def _process_item(self, item):\n result = []\n\n block_set = item.get('blockSet', None)\n\n value = {}\n for f in self.fields:\n fv = item.get(f, None)\n if fv is None:\n continue\n value[self.prefix+'_'+f] = [fv]\n\n if block_set is not None:\n for block in block_set:\n indicator, v = self._convert_block(block)\n\n if indicator is not None:\n v.update(value)\n result.append([indicator, v])\n\n else:\n LOG.error('%s - no \"blockSet\" in item', self.name)\n result = [[None, None]]\n\n return result\n\n def _build_iterator(self, now):\n LOG.info('position: %s', self.position)\n\n if self.api_key is None or self.username is None:\n raise RuntimeError('%s - credentials not set' % self.name)\n\n if self.position is None:\n # backfill\n return itertools.chain(\n self._threathq_backfill(now),\n self._threathq_update(now)\n )\n\n # update\n return self._threathq_update(now)\n\n def _threathq_backfill(self, now):\n payload = {\n 'beginTimestamp': int(now/1000.0 - self.initial_interval),\n 'endTimestamp': int(now/1000.0),\n 'threatType': self.product,\n 'resultsPerPage': _RESULTS_PER_PAGE\n }\n\n cur_page = 0\n total_pages = 1\n\n while cur_page < total_pages:\n LOG.debug('%s - polling backfill %d/%d', self.name, cur_page, total_pages)\n\n payload['page'] = cur_page\n\n rkwargs = dict(\n verify=self.verify_cert,\n timeout=self.polling_timeout,\n params=payload,\n auth=(self.username, self.api_key),\n headers=self.headers\n )\n\n r = requests.post(\n _API_BASE+_API_THREAT_SEARCH,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.error(\n '%s - exception in request: %s %s',\n self.name, r.status_code, r.content\n )\n raise\n\n cjson = r.json()\n\n data = cjson.get('data', None)\n if 'data' is None:\n LOG.error('%s - no \"data\" in response', self.name)\n return\n\n page = data.get('page', None)\n if page is None:\n LOG.error('%s - no \"page\" in response', self.name)\n return\n total_pages = page.get('totalPages', None)\n if total_pages is None:\n LOG.error('%s - no \"totalPages\" in response', self.name)\n return\n LOG.debug('%s - total_pages set to %d', self.name, total_pages)\n\n threats = data.get('threats', [])\n for t in threats:\n yield t\n\n cur_page += 1\n\n def _threathq_update(self, now):\n changelog_size = 1000\n while changelog_size == 1000:\n if self.position is not None:\n payload = dict(position=self.position)\n else:\n payload = dict(timestamp=int(now/1000.0))\n\n rkwargs = dict(\n stream=True,\n verify=self.verify_cert,\n timeout=self.polling_timeout,\n params=payload,\n auth=(self.username, self.api_key),\n headers=self.headers\n )\n\n r = requests.post(\n _API_BASE+_API_THREAT_UPDATE,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.error(\n '%s - exception in request: %s %s',\n self.name, r.status_code, r.content\n )\n raise\n\n cjson = r.json()\n\n data = cjson.get('data', None)\n if data is None:\n LOG.error('%s - no \"data\" in update request', self.name)\n return\n\n changelog = data.get('changelog', None)\n if changelog is not None:\n changelog_size = len(changelog)\n\n else:\n LOG.info('%s - no \"changelog\" in update request', self.name)\n changelog_size = 0\n changelog = []\n\n thgen = self._retrieve_threats(\n self._group_changes_in_pages(\n itertools.ifilter(self._filter_changes, changelog)\n )\n )\n for t in thgen:\n yield t\n\n next_position = data.get('nextPosition', None)\n if next_position is None:\n LOG.error('%s - no nextPosition in update request', self.name)\n else:\n self.position = next_position\n\n def _group_changes_in_pages(self, ichanges):\n # I know I could use izip with *n, but really ?\n threatids = []\n for c in ichanges:\n id_ = str(c.get('threatId', None))\n if id_ is None:\n LOG.error('%s - change with no threatId', self.name)\n continue\n\n type_ = c.get('threatType', None)\n if type_ is None:\n LOG.error('%s - change with no threatType', self.name)\n continue\n\n if type_ == 'malware':\n id_ = 'm_' + id_\n elif type_ == 'phish':\n id_ = 'p_' + id_\n else:\n LOG.error('%s - unknown threatType: %s', self.name, type_)\n continue\n\n threatids.append(id_)\n if len(threatids) == _RESULTS_PER_PAGE:\n yield threatids\n threatids = []\n\n if len(threatids) != 0:\n yield threatids\n\n def _retrieve_threats(self, pages):\n for p in pages:\n payload = {\n 'resultsPerPage': _RESULTS_PER_PAGE,\n 'threatId': p\n }\n\n rkwargs = dict(\n verify=self.verify_cert,\n timeout=self.polling_timeout,\n params=payload,\n auth=(self.username, self.api_key),\n headers=self.headers\n )\n\n r = requests.post(\n _API_BASE+_API_THREAT_SEARCH,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.error(\n '%s - exception in request: %s %s',\n self.name, r.status_code, r.content\n )\n raise\n\n cjson = r.json()\n\n data = cjson.get('data', None)\n if data is None:\n LOG.error('%s - no \"data\" in search request', self.name)\n continue\n\n threats = data.get('threats', None)\n if threats is None:\n LOG.error('%s - no \"threats\" in search request', self.name)\n continue\n\n for t in threats:\n yield t\n\n def _filter_changes(self, change):\n if change.get('deleted', None):\n LOG.debug('%s - deleted change', self.name)\n return False\n\n if self.product == 'all':\n return True\n\n threat_type = change.get('threatType', None)\n if threat_type is None:\n LOG.error('%s - change with no threatType', self.name)\n return False\n\n if threat_type == 'malware' and self.product == 'malware':\n return True\n\n if threat_type == 'phish' and self.product == 'phish':\n return True\n\n return False\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(Intelligence, self).hup(source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n" }, { "path": "minemeld/ft/proofpoint.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport requests\nimport os\nimport shutil\nimport yaml\nimport datetime\nimport pytz\nimport netaddr\nimport netaddr.core\n\nfrom minemeld import __version__ as MM_VERSION\n\nfrom . import basepoller\nfrom . import table\nfrom .utils import dt_to_millisec\n\nLOG = logging.getLogger(__name__)\n\n_CATNAME = [\n \"CnC\",\n \"Bot\",\n \"Spam\",\n \"Drop\",\n \"SpywareCnC\",\n \"OnlineGaming\",\n \"DriveBySrc\",\n \"ChatServer\",\n \"TorNode\",\n \"Compromised\",\n \"P2P\",\n \"Proxy\",\n \"IPCheck\",\n \"Utility\",\n \"DDoSTarget\",\n \"Scanner\",\n \"Brute_Forcer\",\n \"FakeAV\",\n \"DynDNS\",\n \"Undesirable\",\n \"AbusedTLD\",\n \"SelfSignedSSL\",\n \"Blackhole\",\n \"RemoteAccessService\",\n \"P2PCnC\",\n \"Parking\",\n \"VPN\",\n \"EXE_Source\",\n \"Mobile_CnC\",\n \"Mobile_Spyware_CnC\",\n \"Skype_SuperNode\",\n \"Bitcoin_Related\",\n \"DDoSAttacker\"\n]\n\n\nclass ETIntelligence(basepoller.BasePollerFT):\n _FILE = None\n\n def __init__(self, name, chassis, config):\n self.ttable = None\n\n super(ETIntelligence, self).__init__(name, chassis, config)\n\n def configure(self):\n super(ETIntelligence, self).configure()\n\n self.polling_timeout = self.config.get('polling_timeout', 20)\n self.verify_cert = self.config.get('verify_cert', True)\n self.score_threshold = self.config.get('score_threshold', 50)\n\n self.source_name = 'proofpoint.etintelligence'\n\n self.auth_code = None\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.auth_code = sconfig.get('auth_code', None)\n if self.auth_code is not None:\n LOG.info('%s - authorization code set', self.name)\n\n monitored_categories = sconfig.get('monitored_categories', [])\n if type(monitored_categories) != list:\n LOG.error('%s - wrong monitored_categories format, should '\n 'be a list of ints', self.name)\n self.monitored_categories = []\n else:\n self.monitored_categories = monitored_categories\n\n def _process_row(self, row):\n indicator, category, score, first_seen, last_seen, ports = \\\n row.split(',')\n\n if indicator == 'ip' or indicator == 'domain':\n return None\n\n try:\n category = int(category)\n except ValueError:\n LOG.error('%s - wrong category format, ignored', self.name)\n return None\n\n if category not in self.monitored_categories:\n return None\n\n if category > 0 and category <= len(_CATNAME):\n category_name = _CATNAME[category-1]\n else:\n category_name = '%d' % category\n\n try:\n score = int(score)\n if score < 0 or score > 127:\n raise ValueError('wrong score format')\n except ValueError:\n LOG.error('%s - wrong score format, ignored', self.name)\n return None\n\n if score <= self.score_threshold:\n LOG.debug('%s - score below threshold, ignored', self.name)\n return None\n\n try:\n fs = datetime.datetime.strptime(first_seen, '%Y-%m-%d')\n fs = fs.replace(tzinfo=pytz.UTC)\n fs = dt_to_millisec(fs)\n except:\n LOG.exception('%s - wrong first_seen format, ignored', self.name)\n return None\n\n try:\n ls = datetime.datetime.strptime(last_seen, '%Y-%m-%d')\n ls = ls.replace(tzinfo=pytz.UTC)\n ls = dt_to_millisec(ls)\n except:\n LOG.exception('%s - wrong last_seen format, ignored', self.name)\n return None\n\n ports = ports.split()\n\n value = {\n 'proofpoint_etintelligence_max_score': score,\n 'proofpoint_etintelligence_last_seen': ls,\n 'proofpoint_etintelligence_first_seen': fs,\n 'proofpoint_etintelligence_ports': ports,\n 'proofpoint_etintelligence_categories': [category_name]\n }\n\n return [indicator, value]\n\n def _process_item(self, item):\n return [item]\n\n def _build_iterator(self, now):\n if self.auth_code is None or len(self.monitored_categories) == 0:\n raise RuntimeError(\n '%s - authorization code or categories not set, poll not performed' % self.name\n )\n\n LOG.info('%s - categories: %s', self.name, self.monitored_categories)\n\n if self.ttable is not None:\n self.ttable.close()\n self.ttable = None\n\n self.ttable = table.Table(self.name+'_temp', truncate=True)\n\n url = ('https://rules.emergingthreats.net/' +\n self.auth_code +\n '/reputation/' +\n self._FILE)\n\n rkwargs = dict(\n stream=True,\n verify=self.verify_cert,\n timeout=self.polling_timeout,\n headers={\n 'User-Agent': 'MineMeld/%s' % MM_VERSION\n }\n )\n\n r = requests.get(\n url,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.debug('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n for line in r.iter_lines():\n p = self._process_row(line)\n if p is None:\n continue\n\n i, nv = p\n\n ov = self.ttable.get(i)\n if ov is None:\n self.ttable.put(i, nv)\n else:\n if (ov['proofpoint_etintelligence_max_score'] <\n nv['proofpoint_etintelligence_max_score']):\n ov['proofpoint_etintelligence_max_score'] = \\\n nv['proofpoint_etintelligence_max_score']\n if (ov['proofpoint_etintelligence_first_seen'] >\n nv['proofpoint_etintelligence_first_seen']):\n ov['proofpoint_etintelligence_first_seen'] = \\\n nv['proofpoint_etintelligence_first_seen']\n if (ov['proofpoint_etintelligence_last_seen'] >\n nv['proofpoint_etintelligence_last_seen']):\n ov['proofpoint_etintelligence_last_seen'] = \\\n nv['proofpoint_etintelligence_last_seen']\n ov['proofpoint_etintelligence_ports'] += \\\n nv['proofpoint_etintelligence_ports']\n ov['proofpoint_etintelligence_categories'] += \\\n nv['proofpoint_etintelligence_categories']\n self.ttable.put(i, ov)\n\n return self.ttable.query(include_value=True)\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(ETIntelligence, self).hup(source=source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n shutil.rmtree('{}_temp'.format(name), ignore_errors=True)\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n\n\nclass EmergingThreatsIP(ETIntelligence):\n _FILE = 'detailed-iprepdata.txt'\n\n def _process_item(self, row):\n ipairs = super(EmergingThreatsIP, self)._process_item(row)\n\n result = []\n\n for i, v in ipairs:\n try:\n parsed_ip = netaddr.IPAddress(i)\n except:\n LOG.error('%s - invalid IP %s, ignored', self.name, i)\n continue\n\n if parsed_ip.version == 4:\n v['type'] = 'IPv4'\n elif parsed_ip.version == 6:\n v['type'] = 'IPv6'\n else:\n LOG.error('%s - unknown IP version %s, ignored', self.name, i)\n continue\n\n result.append([i, v])\n\n return result\n\n\nclass EmergingThreatsDomain(ETIntelligence):\n _FILE = 'detailed-domainrepdata.txt'\n\n def _process_item(self, row):\n ipairs = super(EmergingThreatsDomain, self)._process_item(row)\n\n result = []\n\n for i, v in ipairs:\n v['type'] = 'domain'\n\n result.append([i, v])\n\n return result\n" }, { "path": "minemeld/ft/recordedfuture.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport requests\nimport os\nimport ujson\nimport yaml\nimport netaddr\nimport netaddr.core\n\nfrom minemeld.ft import csv # changed from . to minemeld.ft\n\nLOG = logging.getLogger(__name__)\n\nclass IPRiskList(csv.CSVFT):\n def configure(self):\n super(IPRiskList, self).configure()\n\n self.source_name = 'recordedfuture.iprisklist'\n self.confidence = self.config.get('confidence', 80)\n\n self.token = None\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.token = sconfig.get('token', None)\n if self.token is not None:\n LOG.info('%s - token set', self.name)\n\n def _process_item(self, row):\n row.pop(None, None) # I love this\n\n result = {}\n\n indicator = row.get('Name', '')\n if indicator == '':\n return []\n\n try:\n if '/' in indicator:\n ip = netaddr.IPNetwork(indicator)\n else:\n ip = netaddr.IPAddress(indicator)\n except netaddr.core.AddrFormatError:\n LOG.exception(\"%s - failed parsing indicator\", self.name)\n return []\n\n if ip.version == 4:\n result['type'] = 'IPv4'\n elif ip.version == 6:\n result['type'] = 'IPv6'\n else:\n LOG.debug(\"%s - unknown IP version %d\", self.name, ip.version)\n return []\n\n risk = row.get('Risk', '')\n if risk != '':\n try:\n result['recordedfuture_risk'] = int(risk)\n result['confidence'] = (int(risk) * self.confidence) / 100\n except:\n LOG.debug(\"%s - invalid risk string: %s\",\n self.name, risk)\n\n riskstring = row.get('RiskString', '')\n if riskstring != '':\n result['recordedfuture_riskstring'] = riskstring\n\n edetails = row.get('EvidenceDetails', '')\n if edetails != '':\n try:\n edetails = ujson.loads(edetails)\n except:\n LOG.debug(\"%s - invalid JSON string in EvidenceDetails: %s\",\n self.name, edetails)\n else:\n edetails = edetails.get('EvidenceDetails', [])\n result['recordedfuture_evidencedetails'] = \\\n [ed['Rule'] for ed in edetails]\n\n result['recordedfuture_entityurl'] = \\\n 'https://app.recordedfuture.com/live/sc/entity/ip:' + indicator\n\n return [[indicator, result]]\n\n def _build_iterator(self, now):\n if self.token is None:\n raise RuntimeError(\n '%s - token not set, poll not performed' % self.name\n )\n\n return super(IPRiskList, self)._build_iterator(now)\n\n def _build_request(self, now):\n params = {'output_format': 'csv/splunk'}\n headers = {'X-RFToken': self.token}\n r = requests.Request(\n 'GET',\n 'https://api.recordedfuture.com/v2/ip/risklist',\n headers=headers, params=params,\n\n )\n\n return r.prepare()\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(IPRiskList, self).hup(source)\n\n @staticmethod\n def gc(name, config=None):\n csv.CSVFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n\n\n\nclass DomainRiskList(csv.CSVFT):\n def configure(self):\n super(DomainRiskList, self).configure()\n\n self.source_name = 'recordedfuture.domainriskList'\n self.confidence = self.config.get('confidence', 80)\n\n self.token = None\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.token = sconfig.get('token', None)\n if self.token is not None:\n LOG.info('%s - token set', self.name)\n\n def _process_item(self, row):\n row.pop(None, None) # I love this\n\n result = {}\n\n indicator = row.get('Name', '')\n if indicator == '':\n return []\n\n risk = row.get('Risk', '')\n if risk != '':\n try:\n result['recordedfuture_risk'] = int(risk)\n result['confidence'] = (int(risk) * self.confidence) / 100\n except:\n LOG.debug(\"%s - invalid risk string: %s\",\n self.name, risk)\n\n riskstring = row.get('RiskString', '')\n if riskstring != '':\n result['recordedfuture_riskstring'] = riskstring\n\n edetails = row.get('EvidenceDetails', '')\n if edetails != '':\n try:\n edetails = ujson.loads(edetails)\n except:\n LOG.debug(\"%s - invalid JSON string in EvidenceDetails: %s\",\n self.name, edetails)\n else:\n edetails = edetails.get('EvidenceDetails', [])\n result['recordedfuture_evidencedetails'] = \\\n [ed['Rule'] for ed in edetails]\n\n result['recordedfuture_entityurl'] = \\\n 'https://app.recordedfuture.com/live/sc/entity/idn:' + indicator\n\n return [[indicator, result]]\n\n def _build_iterator(self, now):\n if self.token is None:\n raise RuntimeError(\n '%s - token not set, poll not performed' % self.name\n )\n\n return super(DomainRiskList, self)._build_iterator(now)\n\n def _build_request(self, now):\n params = {'output_format': 'csv/splunk'}\n headers = {'X-RFToken': self.token}\n r = requests.Request(\n 'GET',\n 'https://api.recordedfuture.com/v2/domain/risklist',\n headers=headers, params=params,\n\n )\n\n return r.prepare()\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(DomainRiskList, self).hup(source)\n\n @staticmethod\n def gc(name, config=None):\n csv.CSVFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n\n\n\nclass MasterRiskList(csv.CSVFT):\n def configure(self):\n super(MasterRiskList, self).configure()\n self.source_name = 'recordedfuture.masterrisklist'\n self.confidence = self.config.get('confidence', 80)\n\n self.entity = None ## entity added\n self.token = None\n self.path = None ## fusion/ risklist path added\n self.api = None ## api type added\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.token = sconfig.get('token', None)\n if self.token is not None:\n LOG.info('%s - token set', self.name)\n\n self.path = sconfig.get('path', None)\n if self.path is not None:\n LOG.info('%s - path set', self.name)\n\n self.entity = sconfig.get('entity', None)\n if self.entity is not None:\n LOG.info('%s - entity set', self.name)\n\n self.api = sconfig.get('api', None)\n if self.api is not None:\n LOG.info('%s - API set', self.name)\n\n def _process_item(self, row):\n\n row.pop(None, None)\n url_key = 'recordedfuture_entityurl'\n base_url = 'https://app.recordedfuture.com/live/sc/entity/'\n result = {}\n indicator = row.get('Name', '')\n if indicator == '':\n return []\n if self.entity == 'ip':\n try:\n if '/' in indicator:\n ip = netaddr.IPNetwork(indicator)\n else:\n ip = netaddr.IPAddress(indicator)\n except netaddr.core.AddrFormatError:\n LOG.exception(\"%s - failed parsing indicator\", self.name)\n return []\n\n if ip.version == 4:\n result['type'] = 'IPv4'\n elif ip.version == 6:\n result['type'] = 'IPv6'\n else:\n LOG.debug(\"%s - unknown IP version %d\", self.name, ip.version)\n return []\n result[url_key] = '{}ip:{}'.format(base_url, indicator)\n elif self.entity == 'domain':\n result['type'] = 'domain'\n result[url_key] = '{}idn:{}'.format(base_url, indicator)\n elif self.entity == 'url':\n result['type'] = 'URL'\n result[url_key] = '{}url:{}'.format(base_url, indicator)\n elif self.entity == 'hash':\n algo = row.get('Algorithm', '')\n if algo != '':\n result['recordedfuture_algorithm'] = algo\n result['type'] = self._check_hash_type(indicator)\n result[url_key] = '{}hash:{}'.format(base_url, indicator)\n risk = row.get('Risk', '')\n if risk != '':\n try:\n result['recordedfuture_risk'] = int(risk)\n result['confidence'] = (int(risk) * self.confidence) / 100\n except:\n LOG.debug(\"%s - invalid risk string: %s\",\n self.name, risk)\n\n riskstring = row.get('RiskString', '')\n if riskstring != '':\n result['recordedfuture_riskstring'] = riskstring\n\n edetails = row.get('EvidenceDetails', '')\n if edetails != '':\n try:\n edetails = ujson.loads(edetails)\n except:\n LOG.debug(\"%s - invalid JSON string in EvidenceDetails: %s\",\n self.name, edetails)\n else:\n edetails = edetails.get('EvidenceDetails', [])\n result['recordedfuture_evidencedetails'] = \\\n [ed['Rule'] for ed in edetails]\n\n return [[indicator, result]]\n\n @staticmethod\n def _check_hash_type(entity):\n if len(entity) == 64:\n return 'sha256'\n elif len(entity) == 40:\n return 'sha1'\n elif len(entity) == 32:\n return 'md5'\n else:\n return ''\n\n def _build_iterator(self, now):\n if self.token is None:\n raise RuntimeError(\n '%s - token not set, poll not performed' % self.name\n )\n\n if self.entity is None:\n raise RuntimeError(\n '%s - entity not set, poll not performed' % self.name\n )\n\n if self.api is None:\n raise RuntimeError(\n '%s - api not set, poll not performed' % self.name\n )\n\n if self.api == 'fusion':\n if self.entity == 'ip':\n if self.path != None:\n if self.path.find('ip') == -1:\n raise RuntimeError(\n '%s - wrong file path for the given miner' % self.name\n )\n\n if self.entity == 'url':\n if self.path != None:\n if self.path.find('url') == -1:\n raise RuntimeError(\n '%s - wrong file path for the given miner' % self.name\n )\n\n if self.entity == 'hash':\n if self.path != None:\n if self.path.find('hash') == -1:\n raise RuntimeError(\n '%s - wrong file path for the given miner' % self.name\n )\n\n if self.entity == 'domain':\n if self.path != None:\n if self.path.find('domain') == -1:\n raise RuntimeError(\n '%s - wrong file path for the given miner' % self.name\n )\n\n return super(MasterRiskList, self)._build_iterator(now)\n\n def _build_request(self, now):\n if self.api == 'connectApi':\n if self.path is None:\n url = 'https://api.recordedfuture.com/v2/' + str(self.entity) + '/risklist'\n else:\n url = 'https://api.recordedfuture.com/v2/' + str(self.entity) + '/risklist?list=' + self.path\n\n params = {'output_format': 'csv/splunk'}\n headers = {'X-RFToken': self.token, 'X-RF-User-Agent': 'Minemeld v1.2',\n 'content-type': 'application/json'}\n\n r = requests.Request('GET', url, headers=headers, params=params)\n\n return r.prepare()\n\n if self.api == 'fusion':\n if self.path is None:\n url = '/public/risklists/default_' + str(self.entity) + '_risklist.csv'\n else:\n url = self.path\n\n url = url.replace('/', '%2F')\n params = {'output_format': 'csv/splunk'}\n headers = {'X-RFToken': self.token, 'X-RF-User-Agent': 'Minemeld v1.2', 'content-type': 'application/json'}\n re = requests.Request('GET', 'https://api.recordedfuture.com/v2/fusion/files/?path=' + url, headers=headers, params=params)\n\n return re.prepare()\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(MasterRiskList, self).hup(source)\n\n @staticmethod\n def gc(name, config=None):\n csvhelper.CSVFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n" }, { "path": "minemeld/ft/redis.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport redis\nimport os\nimport ujson as json\n\nfrom . import base\nfrom . import actorbase\n\nLOG = logging.getLogger(__name__)\n\n\nclass RedisSet(actorbase.ActorBaseFT):\n def __init__(self, name, chassis, config):\n self.redis_skey = name\n self.redis_skey_value = name+'.value'\n self.redis_skey_chkp = name+'.chkp'\n\n self.SR = None\n\n super(RedisSet, self).__init__(name, chassis, config)\n\n def configure(self):\n super(RedisSet, self).configure()\n\n self.redis_url = self.config.get('redis_url',\n os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock')\n )\n self.scoring_attribute = self.config.get(\n 'scoring_attribute',\n 'last_seen'\n )\n self.store_value = self.config.get('store_value', False)\n self.max_entries = self.config.get('max_entries', 1000 * 1000)\n\n def connect(self, inputs, output):\n output = False\n super(RedisSet, self).connect(inputs, output)\n\n def read_checkpoint(self):\n self._connect_redis()\n\n self.last_checkpoint = None\n\n config = {\n 'class': (self.__class__.__module__+'.'+self.__class__.__name__),\n 'config': self._original_config\n }\n config = json.dumps(config, sort_keys=True)\n\n try:\n contents = self.SR.get(self.redis_skey_chkp)\n if contents is None:\n raise ValueError('{} - last checkpoint not found'.format(self.name))\n\n if contents[0] == '{':\n # new format\n contents = json.loads(contents)\n self.last_checkpoint = contents['checkpoint']\n saved_config = contents['config']\n saved_state = contents['state']\n\n else:\n self.last_checkpoint = contents\n saved_config = ''\n saved_state = None\n\n LOG.debug('%s - restored checkpoint: %s', self.name, self.last_checkpoint)\n\n # old_status is missing in old releases\n # stick to the old behavior\n if saved_config and saved_config != config:\n LOG.info(\n '%s - saved config does not match new config',\n self.name\n )\n self.last_checkpoint = None\n return\n\n LOG.info(\n '%s - saved config matches new config',\n self.name\n )\n\n if saved_state is not None:\n self._saved_state_restore(saved_state)\n\n except (ValueError, IOError):\n LOG.exception('{} - Error reading last checkpoint'.format(self.name))\n self.last_checkpoint = None\n\n def create_checkpoint(self, value):\n self._connect_redis()\n\n config = {\n 'class': (self.__class__.__module__+'.'+self.__class__.__name__),\n 'config': self._original_config\n }\n\n contents = {\n 'checkpoint': value,\n 'config': json.dumps(config, sort_keys=True),\n 'state': self._saved_state_create()\n }\n\n self.SR.set(self.redis_skey_chkp, json.dumps(contents))\n\n def remove_checkpoint(self):\n self._connect_redis()\n self.SR.delete(self.redis_skey_chkp)\n\n def _connect_redis(self):\n if self.SR is not None:\n return\n\n self.SR = redis.StrictRedis.from_url(\n self.redis_url\n )\n\n def initialize(self):\n self._connect_redis()\n\n def rebuild(self):\n self._connect_redis()\n self.SR.delete(self.redis_skey)\n self.SR.delete(self.redis_skey_value)\n\n def reset(self):\n self._connect_redis()\n self.SR.delete(self.redis_skey)\n self.SR.delete(self.redis_skey_value)\n\n def _add_indicator(self, score, indicator, value):\n if self.length() >= self.max_entries:\n self.statistics['drop.overflow'] += 1\n return\n\n with self.SR.pipeline() as p:\n p.multi()\n\n p.zadd(self.redis_skey, score, indicator)\n if self.store_value:\n p.hset(self.redis_skey_value, indicator, json.dumps(value))\n\n result = p.execute()[0]\n\n self.statistics['added'] += result\n\n def _delete_indicator(self, indicator):\n with self.SR.pipeline() as p:\n p.multi()\n\n p.zrem(self.redis_skey, indicator)\n p.hdel(self.redis_skey_value, indicator)\n\n result = p.execute()[0]\n\n self.statistics['removed'] += result\n\n @base._counting('update.processed')\n def filtered_update(self, source=None, indicator=None, value=None):\n score = 0\n if self.scoring_attribute is not None:\n av = value.get(self.scoring_attribute, None)\n if type(av) == int or type(av) == long:\n score = av\n else:\n LOG.error(\"scoring_attribute is not int: %s\", type(av))\n score = 0\n\n self._add_indicator(score, indicator, value)\n\n @base._counting('withdraw.processed')\n def filtered_withdraw(self, source=None, indicator=None, value=None):\n self._delete_indicator(indicator)\n\n def length(self, source=None):\n return self.SR.zcard(self.redis_skey)\n\n @staticmethod\n def gc(name, config=None):\n actorbase.ActorBaseFT.gc(name, config=config)\n\n if config is None:\n config = {}\n\n redis_skey = name\n redis_skey_value = '{}.value'.format(name)\n redis_skey_chkp = '{}.chkp'.format(name)\n redis_url = config.get('redis_url',\n os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock')\n )\n\n cp = None\n try:\n cp = redis.ConnectionPool.from_url(\n url=redis_url\n )\n\n SR = redis.StrictRedis(connection_pool=cp)\n\n SR.delete(redis_skey)\n SR.delete(redis_skey_value)\n SR.delete(redis_skey_chkp)\n\n except Exception as e:\n raise RuntimeError(str(e))\n\n finally:\n if cp is not None:\n cp.disconnect()\n" }, { "path": "minemeld/ft/st.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nSimple segment tree implementation based on LevelDB.\n\n**KEYS**\n\nNumbers are 8-bit unsigned.\n\n- Segment key: (1, , , , )\n- Endpoint key: (1, , , , )\n\n**ENDPOINT**\n\n- Type: 0: START, 1: END\n\"\"\"\n\nimport plyvel\nimport struct\nimport logging\nimport shutil\nimport array\n\nLOG = logging.getLogger(__name__)\n\nMAX_LEVEL = 0xFE\nTYPE_START = 0x00\nTYPE_END = 0x1\n\n\nclass ST(object):\n def __init__(self, name, epsize, truncate=False,\n bloom_filter_bits=10, write_buffer_size=(4 << 20)):\n if truncate:\n try:\n shutil.rmtree(name)\n except:\n pass\n\n self.db = plyvel.DB(\n name,\n create_if_missing=True,\n write_buffer_size=write_buffer_size,\n bloom_filter_bits=bloom_filter_bits\n )\n self.epsize = epsize\n self.max_endpoint = (1 << epsize)-1\n\n self.num_endpoints = 0\n self.num_segments = 0\n\n def _split_interval(self, start, end, lower, upper):\n if start <= lower and upper <= end:\n return [(lower, upper)]\n\n mid = (lower+upper)/2\n\n result = []\n if start <= mid:\n result += self._split_interval(start, end, lower, mid)\n if end > mid:\n result += self._split_interval(start, end, mid+1, upper)\n\n return result\n\n def _segment_key(self, start, end, uuid_=None, level=None):\n res = array.array('B', [\n 1,\n (start >> 56) & 0xFF, (start >> 48) & 0xFF,\n (start >> 40) & 0xFF, (start >> 32) & 0xFF,\n (start >> 24) & 0xFF, (start >> 16) & 0xFF,\n (start >> 8) & 0xFF, start & 0xFF,\n (end >> 56) & 0xFF, (end >> 48) & 0xFF,\n (end >> 40) & 0xFF, (end >> 32) & 0xFF,\n (end >> 24) & 0xFF, (end >> 16) & 0xFF,\n (end >> 8) & 0xFF, end & 0xFF,\n ])\n\n if level is not None:\n res.append(level)\n if uuid_ is not None:\n for c in uuid_:\n res.append(ord(c))\n\n return res.tostring()\n\n def _split_segment_key(self, key):\n _, start, end, level = struct.unpack(\">BQQB\", key[:18])\n return start, end, level, key[18:]\n\n def _endpoint_key(self, endpoint, level=None, type_=None, uuid_=None):\n res = array.array('B', [\n 2,\n (endpoint >> 56) & 0xFF, (endpoint >> 48) & 0xFF,\n (endpoint >> 40) & 0xFF, (endpoint >> 32) & 0xFF,\n (endpoint >> 24) & 0xFF, (endpoint >> 16) & 0xFF,\n (endpoint >> 8) & 0xFF, endpoint & 0xFF\n ])\n\n if level is not None:\n res.append(level)\n if type_ is not None:\n res.append(type_)\n if uuid_ is not None:\n for c in uuid_:\n res.append(ord(c))\n\n return res.tostring()\n\n def _split_endpoint_key(self, k):\n _, endpoint, level, type_ = struct.unpack(\">BQBB\", k[:11])\n type_ = (True if type_ == TYPE_START else False)\n return endpoint, level, type_, k[11:]\n\n def close(self):\n self.db.close()\n\n def put(self, uuid_, start, end, level=0):\n si = self._split_interval(start, end, 0, self.max_endpoint)\n\n value = struct.pack(\">QQ\", start, end)\n\n batch = self.db.write_batch()\n\n for i in si:\n k = self._segment_key(i[0], i[1], uuid_=uuid_, level=level)\n batch.put(k, value)\n\n ks = self._endpoint_key(\n start,\n level=level,\n type_=TYPE_START,\n uuid_=uuid_\n )\n batch.put(ks, \"\\x00\")\n ke = self._endpoint_key(\n end,\n level=level,\n type_=TYPE_END,\n uuid_=uuid_\n )\n batch.put(ke, \"\\x00\")\n\n batch.write()\n\n self.num_endpoints += 2\n self.num_segments += len(si)\n\n def delete(self, uuid_, start, end, level=0):\n batch = self.db.write_batch()\n\n si = self._split_interval(start, end, 0, self.max_endpoint)\n for i in si:\n k = self._segment_key(i[0], i[1], uuid_=uuid_, level=level)\n batch.delete(k)\n\n ks = self._endpoint_key(\n start,\n level=level,\n type_=TYPE_START,\n uuid_=uuid_\n )\n batch.delete(ks)\n ke = self._endpoint_key(\n end,\n level=level,\n type_=TYPE_END,\n uuid_=uuid_\n )\n batch.delete(ke)\n\n batch.write()\n\n self.num_endpoints -= 2\n self.num_segments -= len(si)\n\n def cover(self, value):\n \"\"\"Iterate over segments covering value. Segment format:\n (uuid, level, start, end).\n \n Args:\n value (int): Address\n \"\"\"\n\n lower = 0\n upper = self.max_endpoint*2\n\n while True:\n mid = (lower+upper)/2\n if value <= mid:\n upper = mid\n else:\n lower = mid+1\n\n ks = self._segment_key(lower, upper)\n ke = self._segment_key(lower, upper, level=MAX_LEVEL+1)\n\n for k, v in self.db.iterator(start=ks, stop=ke, include_value=True,\n reverse=True, include_start=False,\n include_stop=False):\n _, _, level, uuid_ = self._split_segment_key(k)\n start, end = struct.unpack(\">QQ\", v)\n\n yield uuid_, level, start, end\n\n if lower == upper:\n break\n\n def query_endpoints(self, start=None, stop=None, reverse=False,\n include_start=True, include_stop=True):\n \"\"\"Iterate over endpoints between start and end. endpoints have the\n format (endpoint, level, type, uuid). Type: 0 - start, 1 - end\n\n start (int, optional): Defaults to None.\n stop (int, optional): Defaults to None.\n reverse (bool, optional): Defaults to False.\n include_start (bool, optional): Defaults to True.\n include_stop (bool, optional): Defaults to True.\n \"\"\"\n\n if start is None:\n start = self._endpoint_key(0)\n else:\n start = self._endpoint_key(start)\n if stop is None:\n stop = self._endpoint_key(self.max_endpoint, level=MAX_LEVEL+1)\n else:\n stop = self._endpoint_key(stop, level=MAX_LEVEL+1)\n\n di = self.db.iterator(\n start=start,\n stop=stop,\n reverse=reverse,\n include_value=False,\n include_start=include_start,\n include_stop=include_stop\n )\n for k in di:\n yield self._split_endpoint_key(k)\n" }, { "path": "minemeld/ft/syslog.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport shutil\n\nimport gevent\nimport gevent.queue\nimport gevent.event\n\nimport amqp\nimport ujson\nimport netaddr\nimport datetime\nimport socket\nimport random\nimport os\nimport yaml\nimport copy\nimport re\n\nfrom . import base\nfrom . import actorbase\nfrom . import table\nfrom . import ft_states\nfrom . import condition\nfrom .utils import utc_millisec\nfrom .utils import RWLock\nfrom .utils import parse_age_out\n\nLOG = logging.getLogger(__name__)\n\n_MAX_AGE_OUT = ((1 << 32)-1)*1000\n\n\nclass SyslogMatcher(actorbase.ActorBaseFT):\n def __init__(self, name, chassis, config):\n self.amqp_glet = None\n\n super(SyslogMatcher, self).__init__(name, chassis, config)\n\n self._ls_socket = None\n\n def configure(self):\n super(SyslogMatcher, self).configure()\n\n self.exchange = self.config.get('exchange', 'mmeld-syslog')\n self.rabbitmq_username = self.config.get('rabbitmq_username', 'guest')\n self.rabbitmq_password = self.config.get('rabbitmq_password', 'guest')\n\n self.input_types = self.config.get('input_types', {})\n\n self.logstash_host = self.config.get('logstash_host', None)\n self.logstash_port = self.config.get('logstash_port', 5514)\n\n def _initialize_tables(self, truncate=False):\n self.table_ipv4 = table.Table(self.name+'_ipv4', truncate=truncate)\n self.table_ipv4.create_index('_start')\n\n self.table_indicators = table.Table(\n self.name+'_indicators',\n truncate=truncate\n )\n\n self.table = table.Table(self.name, truncate=truncate)\n self.table.create_index('syslog_original_indicator')\n\n def initialize(self):\n self._initialize_tables()\n\n def rebuild(self):\n self._initialize_tables(truncate=True)\n\n def reset(self):\n self._initialize_tables(truncate=True)\n\n @base._counting('update.processed')\n def filtered_update(self, source=None, indicator=None, value=None):\n type_ = value.get('type', None)\n if type_ is None:\n LOG.error(\"%s - received update with no type, ignored\", self.name)\n return\n\n itype = self.input_types.get(source, None)\n if itype is None:\n LOG.debug('%s - no type associated to %s, added %s',\n self.name, source, type_)\n self.input_types[source] = type_\n itype = type_\n\n if itype != type_:\n LOG.error(\"%s - indicator of type %s received from \"\n \"source %s with type %s, ignored\",\n self.name, type_, source, itype)\n return\n\n if type_ == 'IPv4':\n start, end = map(netaddr.IPAddress, indicator.split('-', 1))\n\n LOG.debug('start: %d', start.value)\n\n value['_start'] = start.value\n value['_end'] = end.value\n\n self.table_ipv4.put(indicator, value)\n\n else:\n self.table_indicators.put(type_+indicator, value)\n\n @base._counting('withdraw.processed')\n def filtered_withdraw(self, source=None, indicator=None, value=None):\n itype = self.input_types.get(source, None)\n if itype is None:\n LOG.error('%s - withdraw from unknown source', self.name)\n return\n\n if itype == 'IPv4':\n v = self.table_ipv4.get(indicator)\n if v is not None:\n self.table_ipv4.delete(indicator)\n\n else:\n v = self.table_indicators.get(itype+indicator)\n if v is not None:\n self.table_indicators.delete(itype+indicator)\n\n if v is not None:\n for i, v in self.table.query(index='syslog_original_indicator',\n from_key=itype+indicator,\n to_key=itype+indicator,\n include_value=True):\n self.emit_withdraw(i, value=v)\n self.table.delete(i)\n\n def _handle_ip(self, ip, source=True, message=None):\n try:\n ipv = netaddr.IPAddress(ip)\n except:\n return\n\n if ipv.version != 4:\n return\n\n ipv = ipv.value\n\n iv = next(\n (self.table_ipv4.query(index='_start',\n to_key=ipv,\n include_value=True,\n include_start=True,\n reverse=True)),\n None\n )\n if iv is None:\n return\n\n i, v = iv\n\n if v['_end'] < ipv:\n return\n\n for s in v.get('sources', []):\n self.statistics['source.'+s] += 1\n self.statistics['total_matches'] += 1\n\n v['syslog_original_indicator'] = 'IPv4'+i\n\n self.table.put(ip, v)\n self.emit_update(ip, v)\n\n if message is not None:\n self._send_logstash(\n message='matched IPv4',\n indicator=i,\n value=v,\n session=message\n )\n\n def _handle_url(self, url, message=None):\n domain = url.split('/', 1)[0]\n\n v = self.table_indicators.get('domain'+domain)\n if v is None:\n return\n\n v['syslog_original_indicator'] = 'domain'+domain\n\n for s in v.get('sources', []):\n self.statistics[s] += 1\n self.statistics['total_matches'] += 1\n\n self.table.put(domain, v)\n self.emit_update(domain, v)\n\n if message is not None:\n self._send_logstash(\n message='matched domain',\n indicator=domain,\n value=v,\n session=message\n )\n\n @base._counting('syslog.processed')\n def _handle_syslog_message(self, message):\n src_ip = message.get('src_ip', None)\n if src_ip is not None:\n self._handle_ip(src_ip, message=message)\n\n dst_ip = message.get('dest_ip', None)\n if dst_ip is not None:\n self._handle_ip(dst_ip, source=False, message=message)\n\n url = message.get('url', None)\n if url is not None:\n self._handle_url(url, message=message)\n\n def _amqp_callback(self, msg):\n try:\n message = ujson.loads(msg.body)\n self._handle_syslog_message(message)\n\n except gevent.GreenletExit:\n raise\n\n except:\n LOG.exception(\n \"%s - exception handling syslog message\",\n self.name\n )\n\n def _amqp_consumer(self):\n while True:\n try:\n conn = amqp.connection.Connection(\n userid=self.rabbitmq_username,\n password=self.rabbitmq_password\n )\n channel = conn.channel()\n channel.exchange_declare(\n self.exchange,\n 'fanout',\n durable=False,\n auto_delete=False\n )\n q = channel.queue_declare(\n exclusive=False\n )\n\n channel.queue_bind(\n queue=q.queue,\n exchange=self.exchange,\n )\n channel.basic_consume(\n callback=self._amqp_callback,\n no_ack=True,\n exclusive=True\n )\n\n while True:\n conn.drain_events()\n\n except gevent.GreenletExit:\n break\n\n except:\n LOG.exception('%s - Exception in consumer glet', self.name)\n\n gevent.sleep(30)\n\n def _connect_logstash(self):\n if self._ls_socket is not None:\n return\n\n if self.logstash_host is None:\n return\n\n _ls_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n _ls_socket.connect((self.logstash_host, self.logstash_port))\n\n self._ls_socket = _ls_socket\n\n def _send_logstash(self, message=None, indicator=None,\n value=None, session=None):\n now = datetime.datetime.now()\n\n fields = {\n '@timestamp': now.isoformat()+'Z',\n '@version': 1,\n 'syslog_node': self.name,\n 'message': message\n }\n\n if indicator is not None:\n fields['indicator'] = indicator\n\n if value is not None:\n if 'last_seen' in value:\n last_seen = datetime.datetime.fromtimestamp(\n float(value['last_seen'])/1000.0\n )\n value['last_seen'] = last_seen.isoformat()+'Z'\n\n if 'first_seen' in fields:\n first_seen = datetime.datetime.fromtimestamp(\n float(value['first_seen'])/1000.0\n )\n value['first_seen'] = first_seen.isoformat()+'Z'\n\n fields['indicator_value'] = value\n\n if session is not None:\n session.pop('event.tags', None)\n fields['session'] = session\n\n try:\n self._connect_logstash()\n\n if self._ls_socket is not None:\n self._ls_socket.sendall(ujson.dumps(fields)+'\\n')\n\n except:\n self._ls_socket = None\n raise\n\n self.statistics['logstash.sent'] += 1\n\n def mgmtbus_status(self):\n result = super(SyslogMatcher, self).mgmtbus_status()\n\n return result\n\n def length(self, source=None):\n return (self.table_ipv4.num_indicators +\n self.table_indicators.num_indicators)\n\n def start(self):\n super(SyslogMatcher, self).start()\n\n self.amqp_glet = gevent.spawn_later(\n 2,\n self._amqp_consumer\n )\n\n def stop(self):\n super(SyslogMatcher, self).stop()\n\n if self.amqp_glet is None:\n return\n\n self.table.close()\n self.table_indicators.close()\n self.table_ipv4.close()\n\n @staticmethod\n def gc(name, config=None):\n actorbase.ActorBaseFT.gc(name, config=config)\n shutil.rmtree(name, ignore_errors=True)\n shutil.rmtree('{}_indicators'.format(name), ignore_errors=True)\n shutil.rmtree('{}_ipv4'.format(name), ignore_errors=True)\n\n\nclass SyslogMiner(base.BaseFT):\n def __init__(self, name, chassis, config):\n self.amqp_glet = None\n self.ageout_glet = None\n self._actor_glet = None\n self._actor_queue = gevent.queue.Queue(maxsize=128)\n self._msg_queue = gevent.queue.Queue(maxsize=1)\n self._do_process = gevent.event.Event()\n\n self.active_requests = []\n self.rebuild_flag = False\n self.last_ageout_run = None\n\n self.state_lock = RWLock()\n\n super(SyslogMiner, self).__init__(name, chassis, config)\n\n def configure(self):\n super(SyslogMiner, self).configure()\n\n self.source_name = self.config.get('source_name', self.name)\n self.attributes = self.config.get('attributes', {})\n\n _age_out = self.config.get('age_out', {})\n\n self.age_out = {\n 'interval': _age_out.get('interval', 3600),\n 'default': parse_age_out(_age_out.get('default', 'last_seen+1h'))\n }\n for k, v in _age_out.iteritems():\n if k in self.age_out:\n continue\n self.age_out[k] = parse_age_out(v)\n\n self.exchange = self.config.get('exchange', 'mmeld-syslog')\n self.rabbitmq_username = self.config.get('rabbitmq_username', 'guest')\n self.rabbitmq_password = self.config.get('rabbitmq_password', 'guest')\n\n self.indicator_mapping = self.config.get('indicator_mapping', {\n 'src_ip': 'IP',\n 'dest_ip': 'IP',\n 'misc': 'URL'\n })\n\n self.prefix = self.config.get('prefix', 'panossyslog')\n\n self.rules = []\n self.side_config_path = self.config.get('rules', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_rules.yml' % self.name\n )\n\n self._load_side_config()\n\n def _initialize_table(self, truncate=False):\n self.table = table.Table(self.name, truncate=truncate)\n self.table.create_index('_age_out')\n self.table.create_index('_withdrawn')\n\n def initialize(self):\n self._initialize_table()\n\n def rebuild(self):\n self._actor_queue.put(\n (utc_millisec(), 'rebuild')\n )\n self._initialize_table(truncate=(self.last_checkpoint is None))\n\n def reset(self):\n self._initialize_table(truncate=True)\n\n def _compile_rule(self, name, f):\n LOG.debug('%s - compiling rule %s: %s', self.name, name, f)\n result = {\n 'name': name,\n 'metric': 'rule.%s' % re.sub('[^a-zA-Z0-9]', '_', name),\n 'conditions': [],\n 'indicators': [],\n 'fields': []\n }\n\n conditions = f.get('conditions', None)\n if conditions is None or len(conditions) == 0:\n LOG.error('%s - no conditions in rule %s, ignored',\n self.name, name)\n return None\n for c in conditions:\n result['conditions'].append(condition.Condition(c))\n\n indicators = f.get('indicators', None)\n if type(indicators) != list:\n LOG.error('%s - no indicators list in rule %s, ignored',\n self.name, name)\n return None\n for i in indicators:\n if i not in self.indicator_mapping:\n LOG.error('%s - rule %s unknown type indicator %s, ignored',\n self.name, name, i)\n continue\n result['indicators'].append(i)\n if len(result['indicators']) == 0:\n LOG.error('%s - no valid indicators in rule %s, ignored',\n self.name, name)\n return None\n\n fields = f.get('fields', None)\n if fields is not None:\n if type(fields) != list:\n LOG.error('%s - wrong fields format in rule %s, ignored',\n self.name, name)\n return None\n\n result['fields'] = [fld for fld in fields if type(fld) == str]\n\n return result\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n rules = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading rules: %s', self.name, str(e))\n return\n\n if type(rules) != list:\n LOG.error('%s - Error loading rules: not a list', self.name)\n return\n\n newrules = []\n for idx, f in enumerate(rules):\n fname = f.get('name', None)\n if fname is None:\n LOG.error('%s - rule %d does not have a name, ignored',\n self.name, idx)\n continue\n\n cf = self._compile_rule(fname, f)\n if cf is not None:\n newrules.append(cf)\n\n self.rules = newrules\n\n @base.BaseFT.state.setter\n def state(self, value):\n self.state_lock.lock()\n # this is weird ! from stackoverflow 10810369\n super(SyslogMiner, self.__class__).state.fset(self, value)\n self.state_lock.unlock()\n\n def _command_rebuild(self):\n with self.state_lock:\n if self.state != ft_states.STARTED:\n return\n\n for i, v in self.table.query(include_value=True):\n indicator, _ = i.split('\\0', 1)\n self.emit_update(indicator=indicator, value=v)\n\n def _command_age_out(self):\n with self.state_lock:\n if self.state != ft_states.STARTED:\n return\n\n try:\n now = utc_millisec()\n\n for i, v in self.table.query(index='_age_out',\n to_key=now-1,\n include_value=True):\n indicator, _ = i.split('\\0', 1)\n\n self.emit_withdraw(indicator=indicator, value=v)\n self.table.delete(i)\n\n self.statistics['aged_out'] += 1\n\n self.last_ageout_run = now\n\n except gevent.GreenletExit:\n raise\n\n except:\n LOG.exception('Exception in _age_out_loop')\n\n def _calc_age_out(self, indicator, attributes):\n t = attributes.get('type', None)\n if t is None or t not in self.age_out:\n sel = self.age_out['default']\n else:\n sel = self.age_out[t]\n\n if sel is None:\n return _MAX_AGE_OUT\n\n b = attributes[sel['base']]\n\n return b + sel['offset']\n\n def _apply_rule(self, f, message):\n r = True\n for c in f['conditions']:\n r &= c.eval(message)\n\n if not r:\n return\n\n for i in f['indicators']:\n indicator = message.get(i, None)\n if indicator is None:\n continue\n\n value = {}\n\n for fld in f['fields']:\n fv = message.get(fld, None)\n if fv is not None:\n value['%s_%s' % (self.prefix, fld)] = fv\n\n type_ = self.indicator_mapping[i]\n\n if type_ == 'IP':\n pi = netaddr.IPAddress(indicator)\n if pi.version == 6:\n type_ = 'IPv6'\n elif pi.version == 4:\n type_ = 'IPv4'\n else:\n continue\n\n value['type'] = type_\n\n device = message.get('serial_number', 'unknown')\n\n yield [indicator, value, device]\n\n @base._counting('syslog.processed')\n def _handle_syslog_message(self, message):\n devices_attribute = '%s_devices' % self.prefix\n\n now = utc_millisec()\n\n for f in self.rules:\n for indicator, value, device in self._apply_rule(f, message):\n if indicator is None:\n continue\n\n self.statistics[f['metric']] += 1\n\n type_ = value.get('type', None)\n if type_ is None:\n LOG.error('%s - no type for indicator %s, ignored',\n self.name, indicator)\n continue\n\n ikey = indicator+'\\0'+type_\n cv = self.table.get(ikey)\n\n if cv is None:\n cv = copy.copy(self.attributes)\n cv['sources'] = [self.source_name]\n cv['last_seen'] = now\n cv['first_seen'] = now\n cv[devices_attribute] = [device]\n cv.update(value)\n cv['_age_out'] = self._calc_age_out(indicator, cv)\n\n self.statistics['added'] += 1\n self.table.put(ikey, cv)\n self.emit_update(indicator, cv)\n\n else:\n cv['last_seen'] = now\n cv.update(value)\n cv['_age_out'] = self._calc_age_out(indicator, cv)\n if device not in cv[devices_attribute]:\n cv[devices_attribute].append(device)\n\n self.table.put(ikey, cv)\n self.emit_update(indicator, cv)\n\n def _actor_loop(self):\n while True:\n msg = None\n\n try:\n msg = self._actor_queue.get(block=False)\n except gevent.queue.Empty:\n msg = None\n\n if msg is not None:\n _, command = msg\n if command == 'age_out':\n self._command_age_out()\n elif command == 'rebuild':\n self._command_rebuild()\n else:\n LOG.error('{} - unknown command {} - ignored'.format(\n self.name,\n command\n ))\n\n msg = None\n try:\n while self._msg_queue.qsize() != 0:\n msg = self._msg_queue.get(block=False)\n\n try:\n self._handle_syslog_message(msg)\n except gevent.GreenletExit:\n raise\n except:\n LOG.exception('{} - exception handling message'.format(self.name))\n\n except gevent.queue.Empty:\n pass\n\n self._do_process.wait()\n self._do_process.clear()\n\n def _age_out_loop(self):\n while True:\n self._actor_queue.put(\n (utc_millisec(), 'age_out')\n )\n self._do_process.set()\n\n try:\n gevent.sleep(self.age_out['interval'])\n\n except gevent.GreenletExit:\n break\n\n def _amqp_callback(self, msg):\n try:\n LOG.info(u'{}'.format(msg.body))\n message = ujson.loads(msg.body)\n self._msg_queue.put(message)\n self._do_process.set()\n\n except gevent.GreenletExit:\n raise\n\n except:\n LOG.exception(\n \"%s - exception handling syslog message\",\n self.name\n )\n\n def _amqp_consumer(self):\n while self.last_ageout_run is None:\n gevent.sleep(1)\n\n with self.state_lock:\n if self.state != ft_states.STARTED:\n LOG.error('{} - wrong state in amqp_consumer'.format(self.name))\n return\n\n while True:\n try:\n conn = amqp.connection.Connection(\n userid=self.rabbitmq_username,\n password=self.rabbitmq_password\n )\n channel = conn.channel()\n channel.exchange_declare(\n self.exchange,\n 'fanout',\n durable=False,\n auto_delete=False\n )\n q = channel.queue_declare(\n exclusive=False\n )\n\n channel.queue_bind(\n queue=q.queue,\n exchange=self.exchange,\n )\n channel.basic_consume(\n callback=self._amqp_callback,\n no_ack=True,\n exclusive=True\n )\n\n while True:\n conn.drain_events()\n\n except gevent.GreenletExit:\n break\n\n except:\n LOG.exception('%s - Exception in consumer glet', self.name)\n\n gevent.sleep(30)\n\n def length(self, source=None):\n return self.table.num_indicators\n\n def start(self):\n super(SyslogMiner, self).start()\n\n if self.amqp_glet is not None:\n return\n\n self.amqp_glet = gevent.spawn_later(\n random.randint(0, 2),\n self._amqp_consumer\n )\n self.ageout_glet = gevent.spawn(self._age_out_loop)\n self._actor_glet = gevent.spawn(self._actor_loop)\n\n def stop(self):\n super(SyslogMiner, self).stop()\n\n if self.amqp_glet is None:\n return\n\n for g in self.active_requests:\n g.kill()\n\n self.amqp_glet.kill()\n self.ageout_glet.kill()\n self._actor_glet.kill()\n\n self.table.close()\n\n LOG.info(\"%s - # indicators: %d\", self.name, self.table.num_indicators)\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload filters', self.name)\n self._load_side_config()\n\n @staticmethod\n def gc(name, config=None):\n base.BaseFT.gc(name, config=config)\n\n shutil.rmtree(name, ignore_errors=True)\n side_config_path = None\n if config is not None:\n side_config_path = config.get('rules', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_rules.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n" }, { "path": "minemeld/ft/table.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nTable implementation based on LevelDB (https://github.com/google/leveldb).\nThis is a sort of poor, lazy man implementation of IndexedDB schema.\n\n**KEYS**\n\nNumbers are 8-bit unsigned.\n\n- Schema Version: (0)\n- Index Last Global Id: (0,1, )\n- Last Update Key: (0,2)\n- Number of Indicators: (0,3)\n- Table Last Global ID: (0,4)\n- Custom Metadata: (0,5)\n- Indicator Version: (1,0,)\n- Indicator: (1,1,)\n\n**INDICATORS**\n\nEach indicators has 2 entries associated in the DB: a version and a value.\n\nThe version number is used to track indicator existance and versioning.\nWhen an indicator value is updated, its version number is incremented.\nThe version number is a 64-bit LSB unsigned int.\n\nThe value of an indicator is a 64-bit unsigned int LSB followed by a dump of\na dictionary of attributes in JSON format.\n\nTo iterate over all the indicators versions iterate from key (1,0) to key\n(1,1) excluded.\n\nNULL indicators are not allowed.\n\n**INDEXES**\n\nIndicators are stored in alphabetical order. Indexes are secondary indexes\non indicators attributes.\n\nEach index has an associated id in the range 0 - 255. The attribute associated\nto the index is stored at (0,1,), if the key does not exist the\nindex does not exist.\n\nThere is also a Last Global Id per index, used to index indicators with the\nsame attribute value. Each time a new indicator is added to the index, the\nLast Global Id is incremented. The Last Global Id of an index is stored at\n(2,,0) as a 64-bit LSB unsigned int.\n\nEach entry in the index is stored with a key\n(2,,0xF0,,) and value\n(,). depends on the type of attribute.\n\nWhen iterating over an index, the value of an index entry is loaded and if\nthe version does not match with current indicator version the index entry is\ndeleted. This permits a sort of lazy garbage collection.\n\nTo retrieve all the indicators with a specific attribute value just iterate\nover the keys (2,,0xF0,) and\n(2,,0xF0,,0xFF..FF)\n\"\"\"\n\nimport os\nimport plyvel\nimport struct\nimport ujson\nimport time\nimport logging\nimport shutil\nimport gevent\n\n\nSCHEMAVERSION_KEY = struct.pack(\"B\", 0)\nSTART_INDEX_KEY = struct.pack(\"BBB\", 0, 1, 0)\nEND_INDEX_KEY = struct.pack(\"BBB\", 0, 1, 0xFF)\nLAST_UPDATE_KEY = struct.pack(\"BB\", 0, 2)\nNUM_INDICATORS_KEY = struct.pack(\"BB\", 0, 3)\nTABLE_LAST_GLOBAL_ID = struct.pack(\"BB\", 0, 4)\nCUSTOM_METADATA = struct.pack(\"BB\", 0, 5)\n\nLOG = logging.getLogger(__name__)\n\n\nclass InvalidTableException(Exception):\n pass\n\n\nclass Table(object):\n def __init__(self, name, truncate=False, bloom_filter_bits=0):\n if truncate:\n try:\n shutil.rmtree(name)\n except:\n pass\n\n self.db = None\n self._compact_glet = None\n\n self.db = plyvel.DB(\n name,\n create_if_missing=True,\n bloom_filter_bits=bloom_filter_bits\n )\n self._read_metadata()\n\n self.compact_interval = int(os.environ.get('MM_TABLE_COMPACT_INTERVAL', 3600 * 6))\n self.compact_delay = int(os.environ.get('MM_TABLE_COMPACT_DELAY', 3600))\n self._compact_glet = gevent.spawn(self._compact_loop)\n\n def _init_db(self):\n self.last_update = 0\n self.indexes = {}\n self.num_indicators = 0\n self.last_global_id = 0\n\n batch = self.db.write_batch()\n batch.put(SCHEMAVERSION_KEY, struct.pack(\"B\", 1))\n batch.put(LAST_UPDATE_KEY, struct.pack(\">Q\", self.last_update))\n batch.put(NUM_INDICATORS_KEY, struct.pack(\">Q\", self.num_indicators))\n batch.put(TABLE_LAST_GLOBAL_ID, struct.pack(\">Q\", self.last_global_id))\n batch.write()\n\n def _read_metadata(self):\n sv = self._get(SCHEMAVERSION_KEY)\n if sv is None:\n return self._init_db()\n sv = struct.unpack(\"B\", sv)[0]\n if sv == 0:\n # add table last global id\n self._upgrade_from_s0()\n elif sv == 1:\n pass\n else:\n raise InvalidTableException(\"Schema version not supported\")\n\n self.indexes = {}\n ri = self.db.iterator(\n start=START_INDEX_KEY,\n stop=END_INDEX_KEY\n )\n with ri:\n for k, v in ri:\n _, _, indexid = struct.unpack(\"BBB\", k)\n if v in self.indexes:\n raise InvalidTableException(\"2 indexes with the same name\")\n self.indexes[v] = {\n 'id': indexid,\n 'last_global_id': 0\n }\n for i in self.indexes:\n lgi = self._get(self._last_global_id_key(self.indexes[i]['id']))\n if lgi is not None:\n self.indexes[i]['last_global_id'] = struct.unpack(\">Q\", lgi)[0]\n else:\n self.indexes[i]['last_global_id'] = -1\n\n t = self._get(LAST_UPDATE_KEY)\n if t is None:\n raise InvalidTableException(\"LAST_UPDATE_KEY not found\")\n self.last_update = struct.unpack(\">Q\", t)[0]\n\n t = self._get(NUM_INDICATORS_KEY)\n if t is None:\n raise InvalidTableException(\"NUM_INDICATORS_KEY not found\")\n self.num_indicators = struct.unpack(\">Q\", t)[0]\n\n t = self._get(TABLE_LAST_GLOBAL_ID)\n if t is None:\n raise InvalidTableException(\"TABLE_LAST_GLOBAL_ID not found\")\n self.last_global_id = struct.unpack(\">Q\", t)[0]\n\n def _get(self, key):\n try:\n result = self.db.get(key)\n except KeyError:\n return None\n\n return result\n\n def __del__(self):\n self.close()\n\n def get_custom_metadata(self):\n cmetadata = self._get(CUSTOM_METADATA)\n if cmetadata is None:\n return None\n return ujson.loads(cmetadata)\n\n def set_custom_metadata(self, metadata=None):\n if metadata is None:\n self.db.delete(CUSTOM_METADATA)\n return\n\n cmetadata = ujson.dumps(metadata)\n self.db.put(CUSTOM_METADATA, cmetadata)\n\n def close(self):\n if self.db is not None:\n self.db.close()\n\n if self._compact_glet is not None:\n self._compact_glet.kill()\n\n self.db = None\n self._compact_glet = None\n\n def exists(self, key):\n if type(key) == unicode:\n key = key.encode('utf8')\n\n ikeyv = self._indicator_key_version(key)\n return (self._get(ikeyv) is not None)\n\n def get(self, key):\n if type(key) == unicode:\n key = key.encode('utf8')\n\n ikey = self._indicator_key(key)\n value = self._get(ikey)\n if value is None:\n return None\n\n # skip version\n return ujson.loads(value[8:])\n\n def delete(self, key):\n if type(key) == unicode:\n key = key.encode('utf8')\n\n ikey = self._indicator_key(key)\n ikeyv = self._indicator_key_version(key)\n\n if self._get(ikeyv) is None:\n return\n\n batch = self.db.write_batch()\n batch.delete(ikey)\n batch.delete(ikeyv)\n self.num_indicators -= 1\n batch.put(NUM_INDICATORS_KEY, struct.pack(\">Q\", self.num_indicators))\n batch.write()\n\n def _indicator_key(self, key):\n return struct.pack(\"BB\", 1, 1) + key\n\n def _indicator_key_version(self, key):\n return struct.pack(\"BB\", 1, 0) + key\n\n def _index_key(self, idxid, value, lastidxid=None):\n key = struct.pack(\"BBB\", 2, idxid, 0xF0)\n\n if type(value) == unicode:\n value = value.encode('utf8')\n\n if type(value) == str:\n key += struct.pack(\">BL\", 0x0, len(value))+value\n elif type(value) == int or type(value) == long:\n key += struct.pack(\">BQ\", 0x1, value)\n else:\n raise ValueError(\"Unhandled value type: %s\" % type(value))\n\n if lastidxid is not None:\n key += struct.pack(\">Q\", lastidxid)\n\n return key\n\n def _last_global_id_key(self, idxid):\n return struct.pack(\"BBB\", 2, idxid, 0)\n\n def create_index(self, attribute):\n if attribute in self.indexes:\n return\n\n if len(self.indexes) == 0:\n idxid = 0\n else:\n idxid = max([i['id'] for i in self.indexes.values()])+1\n\n self.indexes[attribute] = {\n 'id': idxid,\n 'last_global_id': -1\n }\n\n batch = self.db.write_batch()\n batch.put(struct.pack(\"BBB\", 0, 1, idxid), attribute)\n batch.write()\n\n def put(self, key, value):\n if type(key) == unicode:\n key = key.encode('utf8')\n\n if type(value) != dict:\n raise ValueError()\n\n ikey = self._indicator_key(key)\n ikeyv = self._indicator_key_version(key)\n\n exists = self._get(ikeyv)\n self.last_global_id += 1\n cversion = self.last_global_id\n\n now = time.time()\n self.last_update = now\n\n batch = self.db.write_batch()\n batch.put(ikey, struct.pack(\">Q\", cversion)+ujson.dumps(value))\n batch.put(ikeyv, struct.pack(\">Q\", cversion))\n batch.put(LAST_UPDATE_KEY, struct.pack(\">Q\", self.last_update))\n batch.put(TABLE_LAST_GLOBAL_ID, struct.pack(\">Q\", self.last_global_id))\n\n if exists is None:\n self.num_indicators += 1\n batch.put(\n NUM_INDICATORS_KEY,\n struct.pack(\">Q\", self.num_indicators)\n )\n\n for iattr, index in self.indexes.iteritems():\n v = value.get(iattr, None)\n if v is None:\n continue\n\n index['last_global_id'] += 1\n\n idxkey = self._index_key(index['id'], v, index['last_global_id'])\n batch.put(idxkey, struct.pack(\">Q\", cversion) + key)\n\n batch.put(\n self._last_global_id_key(index['id']),\n struct.pack(\">Q\", index['last_global_id'])\n )\n\n batch.write()\n\n def query(self, index=None, from_key=None, to_key=None,\n include_value=False, include_stop=True, include_start=True,\n reverse=False):\n if type(from_key) is unicode:\n from_key = from_key.encode('ascii', 'replace')\n if type(to_key) is unicode:\n to_key = to_key.encode('ascii', 'replace')\n\n if index is None:\n return self._query_by_indicator(\n from_key=from_key,\n to_key=to_key,\n include_value=include_value,\n include_stop=include_stop,\n include_start=include_start,\n reverse=reverse\n )\n return self._query_by_index(\n index,\n from_key=from_key,\n to_key=to_key,\n include_value=include_value,\n include_stop=include_stop,\n include_start=include_start,\n reverse=reverse\n )\n\n def _query_by_indicator(self, from_key=None, to_key=None,\n include_value=False, include_stop=True,\n include_start=True, reverse=False):\n if from_key is None:\n from_key = struct.pack(\"BB\", 1, 1)\n include_stop = False\n else:\n from_key = self._indicator_key(from_key)\n\n if to_key is None:\n to_key = struct.pack(\"BB\", 1, 2)\n include_start = False\n else:\n to_key = self._indicator_key(to_key)\n\n ri = self.db.iterator(\n start=from_key,\n stop=to_key,\n include_stop=include_stop,\n include_start=include_start,\n reverse=reverse,\n include_value=False\n )\n with ri:\n for ekey in ri:\n ekey = ekey[2:]\n if include_value:\n yield ekey.decode('utf8', 'ignore'), self.get(ekey)\n else:\n yield ekey.decode('utf8', 'ignore')\n\n def _query_by_index(self, index, from_key=None, to_key=None,\n include_value=False, include_stop=True,\n include_start=True, reverse=False):\n if index not in self.indexes:\n raise ValueError()\n\n idxid = self.indexes[index]['id']\n\n if from_key is None:\n from_key = struct.pack(\"BBB\", 2, idxid, 0xF0)\n include_start = False\n else:\n from_key = self._index_key(idxid, from_key)\n\n if to_key is None:\n to_key = struct.pack(\"BBB\", 2, idxid, 0xF1)\n include_stop = False\n else:\n to_key = self._index_key(\n idxid,\n to_key,\n lastidxid=0xFFFFFFFFFFFFFFFF\n )\n\n ldeleted = 0\n ri = self.db.iterator(\n start=from_key,\n stop=to_key,\n include_value=True,\n include_start=include_start,\n include_stop=include_stop,\n reverse=reverse\n )\n with ri:\n for ikey, ekey in ri:\n iversion = struct.unpack(\">Q\", ekey[:8])[0]\n ekey = ekey[8:]\n\n evalue = self._get(self._indicator_key_version(ekey))\n if evalue is None:\n # LOG.debug(\"Key does not exist\")\n # key does not exist\n self.db.delete(ikey)\n ldeleted += 1\n continue\n\n cversion = struct.unpack(\">Q\", evalue)[0]\n if iversion != cversion:\n # index value is old\n # LOG.debug(\"Version mismatch\")\n self.db.delete(ikey)\n ldeleted += 1\n continue\n\n if include_value:\n yield ekey.decode('utf8', 'ignore'), self.get(ekey)\n else:\n yield ekey.decode('utf8', 'ignore')\n\n LOG.info('Deleted in scan of {}: {}'.format(index, ldeleted))\n\n def _compact_loop(self):\n gevent.sleep(self.compact_delay)\n\n while True:\n try:\n gevent.idle()\n\n counter = 0\n for idx in self.indexes.keys():\n for i in self.query(index=idx, include_value=False):\n if counter % 512 == 0:\n gevent.sleep(0.001) # yield to other greenlets\n counter += 1\n\n except gevent.GreenletExit:\n break\n except:\n LOG.exception('Exception in _compact_loop')\n\n try:\n gevent.sleep(self.compact_interval)\n\n except gevent.GreenletExit:\n break\n\n def _upgrade_from_s0(self):\n LOG.info('Upgrading from schema version 0 to schema version 1')\n\n LOG.info('Loading indexes...')\n indexes = {}\n ri = self.db.iterator(\n start=START_INDEX_KEY,\n stop=END_INDEX_KEY\n )\n with ri:\n for k, v in ri:\n _, _, indexid = struct.unpack(\"BBB\", k)\n if v in indexes:\n raise InvalidTableException(\"2 indexes with the same name\")\n indexes[v] = {\n 'id': indexid,\n 'last_global_id': 0\n }\n for i in indexes:\n lgi = self._get(self._last_global_id_key(indexes[i]['id']))\n if lgi is not None:\n indexes[i]['last_global_id'] = struct.unpack(\">Q\", lgi)[0]\n else:\n indexes[i]['last_global_id'] = -1\n\n LOG.info('Scanning indexes...')\n last_global_id = 0\n for i, idata in indexes.iteritems():\n from_key = struct.pack(\"BBB\", 2, idata['id'], 0xF0)\n include_start = False\n to_key = struct.pack(\"BBB\", 2, idata['id'], 0xF1)\n include_stop = False\n\n ri = self.db.iterator(\n start=from_key,\n stop=to_key,\n include_value=True,\n include_start=include_start,\n include_stop=include_stop,\n reverse=False\n )\n with ri:\n for ikey, ekey in ri:\n iversion = struct.unpack(\">Q\", ekey[:8])[0]\n if iversion > last_global_id:\n last_global_id = iversion+1\n\n LOG.info('Last global id: {}'.format(last_global_id))\n batch = self.db.write_batch()\n batch.put(SCHEMAVERSION_KEY, struct.pack(\"B\", 1))\n batch.put(TABLE_LAST_GLOBAL_ID, struct.pack(\">Q\", last_global_id))\n batch.write()\n" }, { "path": "minemeld/ft/taxii.py", "content": "# Copyright 2015-present Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport copy\nimport urlparse\nimport uuid\nimport os.path\nfrom datetime import datetime, timedelta\n\nimport pytz\nimport lz4.frame\nimport lxml.etree\nimport yaml\nimport redis\nimport gevent\nimport gevent.event\nimport netaddr\nimport werkzeug.urls\nfrom six import string_types\n\nimport libtaxii\nimport libtaxii.clients\nimport libtaxii.messages_11\nfrom libtaxii.constants import MSG_STATUS_MESSAGE, ST_SUCCESS\n\nimport stix.core.stix_package\nimport stix.core.stix_header\nimport stix.indicator\nimport stix.common.vocabs\nimport stix.common.information_source\nimport stix.common.identity\nimport stix.extensions.marking.ais\nimport stix.data_marking\nimport stix.extensions.marking.tlp\n\nimport stix_edh\n\nimport cybox.core\nimport cybox.objects.address_object\nimport cybox.objects.domain_name_object\nimport cybox.objects.uri_object\nimport cybox.objects.file_object\n\nimport mixbox.idgen\nimport mixbox.namespaces\n\nfrom . import basepoller\nfrom . import base\nfrom . import actorbase\nfrom .utils import dt_to_millisec, interval_in_sec, utc_millisec\n\n\n# stix_edh is imported to register the EDH data marking extensions, but it is not directly used.\n# Delete the symbol to silence the warning about the import being unnecessary and prevent the\n# PyCharm 'Optimize Imports' operation from removing the import.\ndel stix_edh\n\nLOG = logging.getLogger(__name__)\n\n\n_STIX_MINEMELD_HASHES = [\n 'ssdeep',\n 'md5',\n 'sha1',\n 'sha256',\n 'sha512'\n]\n\n\ndef set_id_namespace(uri, name):\n # maec and cybox\n NS = mixbox.namespaces.Namespace(uri, name)\n mixbox.idgen.set_id_namespace(NS)\n\n\nclass TaxiiClient(basepoller.BasePollerFT):\n def __init__(self, name, chassis, config):\n self.poll_service = None\n self.collection_mgmt_service = None\n self.last_taxii_run = None\n self.last_stix_package_ts = None\n\n super(TaxiiClient, self).__init__(name, chassis, config)\n\n def configure(self):\n super(TaxiiClient, self).configure()\n\n self.initial_interval = self.config.get('initial_interval', '1d')\n self.initial_interval = interval_in_sec(self.initial_interval)\n if self.initial_interval is None:\n LOG.error(\n '%s - wrong initial_interval format: %s',\n self.name, self.initial_interval\n )\n self.initial_interval = 86400\n self.max_poll_dt = self.config.get(\n 'max_poll_dt',\n 86400\n )\n\n # options for processing\n self.ip_version_auto_detect = self.config.get('ip_version_auto_detect', True)\n self.ignore_composition_operator = self.config.get('ignore_composition_operator', False)\n self.create_fake_indicator = self.config.get('create_fake_indicator', False)\n self.hash_priority = self.config.get('hash_priority', _STIX_MINEMELD_HASHES)\n self.lower_timestamp_precision = self.config.get('lower_timestamp_precision', False)\n\n self.discovery_service = self.config.get('discovery_service', None)\n self.collection = self.config.get('collection', None)\n\n # option for enabling client authentication\n self.client_credentials_required = self.config.get(\n 'client_credentials_required',\n True\n )\n self.username = self.config.get('username', None)\n self.password = self.config.get('password', None)\n if self.username is not None or self.password is not None:\n self.client_credentials_required = False\n\n # option for enabling client cert, default disabled\n self.client_cert_required = self.config.get('client_cert_required', False)\n self.key_file = self.config.get('key_file', None)\n if self.key_file is None and self.client_cert_required:\n self.key_file = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s.pem' % self.name\n )\n self.cert_file = self.config.get('cert_file', None)\n if self.cert_file is None and self.client_cert_required:\n self.cert_file = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s.crt' % self.name\n )\n\n self.subscription_id = None\n self.subscription_id_required = self.config.get('subscription_id_required', False)\n\n self.ca_file = self.config.get('ca_file', None)\n if self.ca_file is None:\n self.ca_file = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s-ca.crt' % self.name\n )\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self.prefix = self.config.get('prefix', self.name)\n\n self.confidence_map = self.config.get('confidence_map', {\n 'low': 40,\n 'medium': 60,\n 'high': 80\n })\n\n self._load_side_config()\n\n def _load_side_config(self):\n if not self.client_credentials_required and not self.subscription_id_required:\n LOG.info('{} - side config not needed'.format(self.name))\n return\n\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n if self.client_credentials_required:\n username = sconfig.get('username', None)\n password = sconfig.get('password', None)\n if username is not None and password is not None:\n self.username = username\n self.password = password\n LOG.info('{} - Loaded credentials from side config'.format(self.name))\n\n if self.subscription_id_required:\n subscription_id = sconfig.get('subscription_id', None)\n if subscription_id is not None:\n self.subscription_id = subscription_id\n LOG.info('{} - Loaded subscription id from side config'.format(self.name))\n\n def _saved_state_restore(self, saved_state):\n super(TaxiiClient, self)._saved_state_restore(saved_state)\n self.last_taxii_run = saved_state.get('last_taxii_run', None)\n LOG.info('last_taxii_run from sstate: %s', self.last_taxii_run)\n\n def _saved_state_create(self):\n sstate = super(TaxiiClient, self)._saved_state_create()\n sstate['last_taxii_run'] = self.last_taxii_run\n\n return sstate\n\n def _saved_state_reset(self):\n super(TaxiiClient, self)._saved_state_reset()\n self.last_taxii_run = None\n\n def _build_taxii_client(self):\n result = libtaxii.clients.HttpClient()\n\n up = urlparse.urlparse(self.discovery_service)\n\n if up.scheme == 'https':\n result.set_use_https(True)\n\n if self.username and self.password:\n if self.key_file and self.cert_file:\n result.set_auth_type(\n libtaxii.clients.HttpClient.AUTH_CERT_BASIC\n )\n result.set_auth_credentials({\n 'username': self.username,\n 'password': self.password,\n 'key_file': self.key_file,\n 'cert_file': self.cert_file\n })\n\n else:\n result.set_auth_type(\n libtaxii.clients.HttpClient.AUTH_BASIC\n )\n result.set_auth_credentials({\n 'username': self.username,\n 'password': self.password\n })\n\n else:\n if self.key_file and self.cert_file:\n result.set_auth_type(\n libtaxii.clients.HttpClient.AUTH_CERT\n )\n result.set_auth_credentials({\n 'key_file': self.key_file,\n 'cert_file': self.cert_file\n })\n\n else:\n result.set_auth_type(\n libtaxii.clients.HttpClient.AUTH_NONE\n )\n\n if self.ca_file is not None and os.path.isfile(self.ca_file):\n result.set_verify_server(\n verify_server=True,\n ca_file=self.ca_file\n )\n\n return result\n\n def _call_taxii_service(self, service_url, tc, request):\n up = urlparse.urlparse(service_url)\n hostname = up.hostname\n path = up.path\n port = up.port\n\n resp = tc.call_taxii_service2(\n hostname,\n path,\n libtaxii.constants.VID_TAXII_XML_11,\n request,\n port=port\n )\n\n return resp\n\n def _discover_services(self, tc):\n msg_id = libtaxii.messages_11.generate_message_id()\n request = libtaxii.messages_11.DiscoveryRequest(msg_id)\n request = request.to_xml()\n\n resp = self._call_taxii_service(self.discovery_service, tc, request)\n\n tm = libtaxii.get_message_from_http_response(resp, msg_id)\n\n LOG.debug('Discovery_Response {%s} %s',\n type(tm), tm.to_xml(pretty_print=True))\n\n if tm.message_type == MSG_STATUS_MESSAGE:\n raise RuntimeError('{} - Error retrieving collections: {} - {}'.format(\n self.name, tm.status_type, tm.message\n ))\n\n self.collection_mgmt_service = None\n for si in tm.service_instances:\n if si.service_type != libtaxii.constants.SVC_COLLECTION_MANAGEMENT:\n continue\n\n self.collection_mgmt_service = si.service_address\n break\n\n if self.collection_mgmt_service is None:\n raise RuntimeError('%s - collection management service not found' %\n self.name)\n\n def _check_collections(self, tc):\n msg_id = libtaxii.messages_11.generate_message_id()\n request = libtaxii.messages_11.CollectionInformationRequest(msg_id)\n request = request.to_xml()\n\n resp = self._call_taxii_service(self.collection_mgmt_service, tc, request)\n\n tm = libtaxii.get_message_from_http_response(resp, msg_id)\n\n LOG.debug('Collection_Information_Response {%s} %s',\n type(tm), tm.to_xml(pretty_print=True))\n\n if tm.message_type == MSG_STATUS_MESSAGE:\n raise RuntimeError('{} - Error retrieving collections: {} - {}'.format(\n self.name, tm.status_type, tm.message\n ))\n\n tci = None\n for ci in tm.collection_informations:\n if ci.collection_name != self.collection:\n continue\n\n tci = ci\n break\n\n if tci is None:\n raise RuntimeError('%s - collection %s not found' %\n (self.name, self.collection))\n\n if tci.polling_service_instances is None or \\\n len(tci.polling_service_instances) == 0:\n raise RuntimeError('%s - collection %s doesn\\'t support polling' %\n (self.name, self.collection))\n\n if tci.collection_type != libtaxii.constants.CT_DATA_FEED:\n raise RuntimeError(\n '%s - collection %s is not a data feed (%s)' %\n (self.name, self.collection, tci.collection_type)\n )\n\n for pi in tci.polling_service_instances:\n LOG.info('{} - message binding: {}'.format(\n self.name, pi.poll_message_bindings\n ))\n if pi.poll_message_bindings[0] == libtaxii.constants.VID_TAXII_XML_11:\n self.poll_service = pi.poll_address\n LOG.info('{} - poll service found'.format(self.name))\n break\n else:\n raise RuntimeError(\n '%s - collection %s does not support TAXII 1.1 message binding (%s)' %\n (self.name, self.collection, tci.collection_type)\n )\n\n LOG.debug('%s - poll service: %s',\n self.name, self.poll_service)\n\n def _poll_fulfillment_request(self, tc, result_id, result_part_number):\n msg_id = libtaxii.messages_11.generate_message_id()\n request = libtaxii.messages_11.PollFulfillmentRequest(\n message_id=msg_id,\n result_id=result_id,\n result_part_number=result_part_number,\n collection_name=self.collection\n )\n request = request.to_xml()\n\n resp = self._call_taxii_service(self.poll_service, tc, request)\n\n return libtaxii.get_message_from_http_response(resp, msg_id)\n\n def _poll_collection(self, tc, begin=None, end=None):\n msg_id = libtaxii.messages_11.generate_message_id()\n\n prargs = dict(\n message_id=msg_id,\n collection_name=self.collection,\n exclusive_begin_timestamp_label=begin,\n inclusive_end_timestamp_label=end,\n )\n if self.subscription_id_required:\n prargs['subscription_id'] = self.subscription_id\n else:\n pps = libtaxii.messages_11.PollParameters(\n response_type='FULL',\n allow_asynch=False\n )\n prargs['poll_parameters'] = pps\n\n request = libtaxii.messages_11.PollRequest(**prargs)\n\n LOG.debug('%s - first poll request %s',\n self.name, request.to_xml(pretty_print=True))\n\n request = request.to_xml()\n\n resp = self._call_taxii_service(self.poll_service, tc, request)\n\n tm = libtaxii.get_message_from_http_response(resp, msg_id)\n\n LOG.debug('%s - Poll_Response {%s} %s',\n self.name, type(tm), tm.to_xml(pretty_print=True))\n\n if tm.message_type == MSG_STATUS_MESSAGE:\n if tm.status_type == ST_SUCCESS:\n LOG.info('{} - TAXII Server returned success with no STIX packages'.format(\n self.name\n ))\n return []\n\n raise RuntimeError('{} - Error polling: {} - {}'.format(\n self.name, tm.status_type, tm.message\n ))\n\n stix_objects = {\n 'observables': {},\n 'indicators': {},\n 'ttps': {}\n }\n\n self._handle_content_blocks(\n tm.content_blocks,\n stix_objects\n )\n\n while tm.more:\n tm = self._poll_fulfillment_request(\n tc,\n result_id=tm.result_id,\n result_part_number=tm.result_part_number+1\n )\n\n LOG.debug('{} - Poll_Response {!r}'.format(\n self.name, tm.to_xml(pretty_print=True)\n ))\n\n if tm.message_type == MSG_STATUS_MESSAGE:\n if tm.status_type == ST_SUCCESS:\n break\n\n raise RuntimeError('{} - Error polling: {} - {}'.format(\n self.name, tm.status_type, tm.message\n ))\n\n self._handle_content_blocks(\n tm.content_blocks,\n stix_objects\n )\n\n LOG.debug('%s - stix_objects: %s', self.name, stix_objects)\n\n params = {\n 'ttps': stix_objects['ttps'],\n 'observables': stix_objects['observables']\n }\n\n if len(stix_objects['indicators']) == 0 and len(stix_objects['observables']) != 0:\n LOG.info('{} - TAXII Content contains observables but no indicators'.format(self.name))\n if self.create_fake_indicator:\n stix_objects['indicators']['minemeld:00000000-0000-0000-0000-000000000000'] = {\n 'observables': stix_objects['observables'].values(),\n 'ttps': []\n }\n\n return [[iid, iv, params]\n for iid, iv in stix_objects['indicators'].iteritems()]\n\n def _incremental_poll_collection(self, taxii_client, begin, end):\n cbegin = begin\n dt = timedelta(seconds=self.max_poll_dt)\n\n self.last_stix_package_ts = None\n\n while cbegin < end:\n cend = min(end, cbegin+dt)\n\n LOG.info('{} - polling {!r} to {!r}'.format(self.name, cbegin, cend))\n result = self._poll_collection(\n taxii_client,\n begin=cbegin,\n end=cend\n )\n\n for i in result:\n yield i\n\n if self.last_stix_package_ts is not None:\n self.last_taxii_run = self.last_stix_package_ts\n\n cbegin = cend\n\n def _handle_content_blocks(self, content_blocks, objects):\n try:\n for cb in content_blocks:\n if cb.content_binding.binding_id != \\\n libtaxii.constants.CB_STIX_XML_111:\n LOG.error('%s - Unsupported content binding: %s',\n self.name, cb.content_binding.binding_id)\n continue\n\n try:\n stixpackage = stix.core.stix_package.STIXPackage.from_xml(\n lxml.etree.fromstring(cb.content)\n )\n except Exception:\n LOG.exception(\n '%s - Exception parsing content block',\n self.name\n )\n continue\n\n if stixpackage.indicators:\n for i in stixpackage.indicators:\n ci = {}\n\n if i.timestamp is not None:\n ci = {\n 'timestamp': dt_to_millisec(i.timestamp),\n }\n\n if i.description is not None and i.description.structuring_format is None:\n # copy description only if there is no markup to avoid side-effects\n ci['description'] = i.description.value\n\n if i.confidence is not None:\n confidence = str(i.confidence.value).lower()\n if confidence in self.confidence_map:\n ci['confidence'] = \\\n self.confidence_map[confidence]\n\n os = []\n ttps = []\n\n if i.observables:\n for o in i.observables:\n os.append(self._decode_observable(o))\n if i.observable and len(os) == 0:\n os.append(self._decode_observable(i.observable))\n\n if i.indicated_ttps:\n for t in i.indicated_ttps:\n ttps.append(self._decode_ttp(t))\n\n ci['observables'] = os\n ci['ttps'] = ttps\n\n objects['indicators'][i.id_] = ci\n\n if stixpackage.observables:\n for o in stixpackage.observables:\n co = self._decode_observable(o)\n objects['observables'][o.id_] = co\n\n if stixpackage.ttps:\n for t in stixpackage.ttps:\n ct = self._decode_ttp(t)\n objects['ttps'][t.id_] = ct\n\n timestamp = stixpackage.timestamp\n if isinstance(timestamp, datetime):\n timestamp = dt_to_millisec(timestamp)\n if self.last_stix_package_ts is None or timestamp > self.last_stix_package_ts:\n LOG.debug('{} - last STIX package timestamp set to {!r}'.format(self.name, timestamp))\n self.last_stix_package_ts = timestamp\n\n except:\n LOG.exception(\"%s - exception in _handle_content_blocks\" %\n self.name)\n raise\n\n def _decode_observable(self, o):\n LOG.debug('observable: %s', o.to_dict())\n\n if o.idref:\n return {'idref': o.idref}\n\n odict = o.to_dict()\n result = {}\n\n oc = odict.get('observable_composition', None)\n if oc:\n ocoperator = oc.get('operator', None)\n if ocoperator != 'OR' and not self.ignore_composition_operator:\n LOG.error(\n '%s - Observable composition with %s not supported yet: %s',\n self.name, ocoperator, odict\n )\n return None\n\n result['type'] = '_cyboxOR'\n\n result['observables'] = []\n for nestedo in oc.get('observables', []):\n if 'idref' not in nestedo:\n LOG.error(\n '%s - only Observable references are supported in Observable Composition: %s',\n self.name, odict\n )\n return None\n result['observables'].append(nestedo['idref'])\n\n return result\n\n oo = odict.get('object', None)\n if oo is None:\n LOG.error('%s - no object in observable', self.name)\n return None\n\n op = oo.get('properties', None)\n if op is None:\n LOG.error('%s - no properties in observable object', self.name)\n return None\n\n return self._decode_object_properties(op, odict=odict)\n\n def _decode_object_properties(self, op, odict=None):\n result = {}\n\n ot = op.get('xsi:type', None)\n if ot is None:\n LOG.error('%s - no type in observable props', self.name)\n return None\n\n if ot == 'DomainNameObjectType':\n result['type'] = 'domain'\n\n ov = op.get('value', None)\n if ov is None:\n LOG.error('%s - no value in observable props', self.name)\n return None\n if not isinstance(ov, string_types):\n ov = ov.get('value', None)\n if ov is None:\n LOG.error('%s - no value in observable value', self.name)\n return None\n\n elif ot == 'FileObjectType':\n ov = ''\n\n if 'file_name' in op.keys():\n file_name = op.get('file_name')\n if isinstance(file_name, dict):\n ov = op['file_name'].get('value', None)\n result['type'] = 'file.name'\n else:\n ov = op['file_name']\n result['type'] = 'file.name'\n\n hashes = op.get('hashes', [])\n if not isinstance(hashes, list) or len(hashes) == 0:\n LOG.error('{} - FileObjectType with unhandled structure: {!r}'.format(\n self.name, op\n ))\n return None\n\n indicator_type = None\n cprio = -1\n indicator_hashes = {}\n\n for h in hashes:\n hvalue = h.get('simple_hash_value', None)\n if hvalue is None:\n continue\n\n if not isinstance(hvalue, string_types):\n if not isinstance(hvalue, dict):\n continue\n\n hvalue = hvalue.get('value', None)\n if hvalue is None:\n continue\n\n htype = h.get('type', None)\n if htype is None:\n continue\n\n elif isinstance(htype, string_types):\n htype = htype.lower()\n\n elif isinstance(htype, dict):\n htype = htype.get('value', None)\n if htype is None or not isinstance(htype, string_types):\n continue\n\n htype = htype.lower()\n if htype not in self.hash_priority:\n continue\n\n prio = self.hash_priority.index(htype)\n\n if prio > cprio:\n indicator_type = htype\n cprio = prio\n\n indicator_hashes[htype] = hvalue\n\n if indicator_type is None:\n LOG.error('{} - No valid hash found in FileObjectType: {!r}'.format(\n self.name, op\n ))\n return None\n\n if ov == '':\n ov = indicator_hashes[indicator_type]\n result['type'] = indicator_type\n\n for h, v in indicator_hashes.iteritems():\n if h == indicator_type:\n continue\n result['{}_{}'.format(self.prefix, h)] = v\n\n elif ot == 'SocketAddressObjectType':\n ip_address = op.get('ip_address', None)\n if ip_address is None:\n return None\n\n return self._decode_object_properties(ip_address)\n\n elif ot == 'AddressObjectType':\n ov = op.get('address_value', None)\n if ov is None:\n LOG.error('%s - no value in observable props', self.name)\n return None\n if not isinstance(ov, string_types):\n ov = ov.get('value', None)\n if ov is None:\n LOG.error('%s - no value in observable value', self.name)\n return None\n\n # set the IP Address type\n if not self.ip_version_auto_detect:\n addrcat = op.get('category', None)\n if addrcat == 'ipv6-addr':\n result['type'] = 'IPv6'\n elif addrcat == 'ipv4-addr':\n result['type'] = 'IPv4'\n elif addrcat == 'e-mail':\n result['type'] = 'email-addr'\n else:\n LOG.error('{} - unknown address category: {}'.format(self.name, addrcat))\n return None\n\n else:\n # some feeds do not set the IP Address type and it\n # defaults to ipv4-addr even if the IP is IPv6\n # this is to auto detect the type\n if type(ov) == list:\n address = ov[0]\n else:\n address = ov\n\n try:\n parsed = netaddr.IPNetwork(address)\n except (netaddr.AddrFormatError, ValueError):\n LOG.error('{} - Unknown IP version: {}'.format(self.name, address))\n return None\n\n if parsed.version == 4:\n result['type'] = 'IPv4'\n elif parsed.version == 6:\n result['type'] = 'IPv6'\n\n if result['type'] in ['IPv4', 'IPv6']:\n source = op.get('is_source', None)\n if source is True:\n result['direction'] = 'inbound'\n elif source is False:\n result['direction'] = 'outbound'\n\n if 'type' not in result:\n LOG.error('%s - no IP category and unknown version')\n return None\n\n elif ot == 'URIObjectType':\n result['type'] = 'URL'\n\n ov = op.get('value', None)\n if ov is None:\n LOG.error('%s - no value in observable props', self.name)\n return None\n if not isinstance(ov, string_types):\n ov = ov.get('value', None)\n if ov is None:\n LOG.error('%s - no value in observable value', self.name)\n return None\n\n elif ot == 'LinkObjectType':\n if op.get('type', 'URL') != 'URL':\n LOG.error('{} - Unhandled LinkObjectType type: {!r}'.format(self.name, op))\n return None\n\n result['type'] = 'URL'\n\n ov = op.get('value', None)\n if ov is None:\n LOG.error('%s - no value in observable props', self.name)\n return None\n if not isinstance(ov, string_types):\n ov = ov.get('value', None)\n if ov is None:\n LOG.error('%s - no value in observable value', self.name)\n return None\n\n elif ot == 'EmailMessageObjectType':\n result['type'] = 'email-message'\n\n ov = ''\n LOG.debug('EmailMessageObjectType OP: {!r}'.format(op))\n\n body = op.get('raw_body', None)\n if body is not None:\n result['body'] = body\n LOG.debug('EmailMessage Body: {!r}'.format(body))\n\n header = op.get('header', None)\n if header is not None:\n result['header'] = header\n try:\n ov = header.get('from').get('address_value').get('value')\n except Exception:\n LOG.error('{} - no email address listed'.format(self.name))\n\n subject = op.get('subject', None)\n if subject is not None:\n result['subject'] = subject\n if ov == '':\n ov = subject\n\n elif ot == 'ArtifactObjectType':\n ov = ''\n result['type'] = 'artifact'\n\n LOG.debug('ArtifactObjectType OV: {!r}'.format(ov))\n\n title = odict.get('title', None)\n if title is not None:\n ov = title\n result['title'] = title\n\n description = odict.get('description', None)\n if description is not None:\n result['description'] = description\n if ov == '':\n ov = description\n\n artifact = op['raw_artifact']\n if artifact is not None:\n result['artifact'] = artifact\n\n elif ot == 'PDFFileObjectType':\n ov = ''\n result['type'] = 'pdf-file'\n\n if 'file_name' in op.keys():\n file_name = op.get('file_name')\n if type(file_name) == dict:\n if file_name.get('value', None) is not None:\n ov = op['file_name'].get('value', None)\n else:\n ov = op['file_name']\n else:\n ov = file_name\n\n LOG.debug('PDFObjectType OV: {!r}'.format(ov))\n\n if 'file_path' in op.keys():\n result['file_path'] = op['file_path'].get('value', None)\n\n if 'file_size' in op.keys():\n result['file_size'] = op['file_size'].get('value', None)\n\n if 'metadata' in op.keys():\n result['metadata'] = op['metadata']\n\n if 'file_format' in op.keys():\n result['file_format'] = op['file_format']\n\n hashes = op.get('hashes', None)\n if hashes is not None:\n for i in hashes:\n if 'type' in i.keys():\n if isinstance(i['type'], string_types):\n hash_type = i['type']\n else:\n hash_type = i['type'].get('value', None)\n if 'simple_hash_value' in i.keys():\n if isinstance(i['simple_hash_value'], string_types):\n result[hash_type] = i['simple_hash_value']\n else:\n result[hash_type] = i['simple_hash_value'].get('value', None)\n\n elif ot == 'WhoisObjectType':\n ov = ''\n result['type'] = 'whois'\n LOG.debug('WhoisObjectType OV: {!r}'.format(ov))\n\n remarks = op.get('remarks', None)\n if remarks is not None:\n result['remarks'] = op['remarks']\n ov = remarks.split('\\n')[0]\n\n elif ot == 'HTTPSessionObjectType':\n ov = ''\n result['type'] = 'http-session'\n\n if 'http_request_response' in op.keys():\n tmp = op['http_request_response']\n\n if len(tmp) == 1:\n item = tmp[0]\n LOG.debug('HTTPSessionObjectType item: {!r}'.format(item))\n http_client_request = item.get('http_client_request', None)\n if http_client_request is not None:\n http_request_header = http_client_request.get('http_request_header', None)\n if http_request_header is not None:\n raw_header = http_request_header.get('raw_header', None)\n if raw_header is not None:\n result['header'] = raw_header\n ov = raw_header.split('\\n')[0]\n else:\n LOG.error('{} - multiple HTTPSessionObjectTypes not supported'.format(self.name))\n\n elif ot == 'PortObjectType':\n result['type'] = 'port'\n LOG.debug('PortObjectType OP: {!r}'.format(op))\n protocol = op.get('layer4_protocol', None)\n port = op.get('port_value', None)\n ov = '{}:{}'.format(protocol, port)\n\n elif ot == 'WindowsExecutableFileObjectType':\n ov = ''\n result['type'] = 'windows-executable'\n LOG.debug('WindowsExecutableFileObjectType OP: {!r}'.format(op))\n\n if 'file_name' in op.keys():\n if isinstance(op['file_name'], string_types):\n ov = op['file_name']\n else:\n ov = op['file_name'].get('value', None)\n\n if 'size_in_bytes' in op.keys():\n result['file_size'] = op['size_in_bytes']\n\n if 'file_format' in op.keys():\n result['file_format'] = op['file_format']\n\n hashes = op.get('hashes', None)\n if hashes is not None:\n for i in hashes:\n if 'type' in i.keys():\n if isinstance(i['type'], string_types):\n hash_type = i['type']\n else:\n hash_type = i['type'].get('value', None)\n if 'simple_hash_value' in i.keys():\n if isinstance(i['simple_hash_value'], string_types):\n result[hash_type] = i['simple_hash_value']\n else:\n result[hash_type] = i['simple_hash_value'].get('value', None)\n\n elif ot == 'CISCP:IndicatorTypeVocab-0.0':\n result['type'] = op['xsi:type']\n LOG.debug('CISCP:IndicatorTypeVocab-0.0 OP: {!r}'.format(op))\n ov = None\n LOG.error('{} - CISCP:IndicatorTypeVocab-0.0 Type not currently supported'.format(self.name))\n return None\n\n elif ot == 'WindowsRegistryKeyObjectType':\n result['type'] = op['xsi:type']\n LOG.debug('WindowsRegistryKeyObjectType OP: {!r}'.format(op))\n ov = None\n LOG.error('{} - WindowsRegistryKeyObjectType Type not currently supported'.format(self.name))\n return None\n\n elif ot == 'stixVocabs:IndicatorTypeVocab-1.0':\n result['type'] = op['xsi:type']\n LOG.debug('stixVocabs:IndicatorTypeVocab-1.0 OP: {!r}'.format(op))\n ov = None\n LOG.error('{} - stixVocabs:IndicatorTypeVocab-1.0 Type not currently supported'.format(self.name))\n return None\n\n elif ot == 'NetworkConnectionObjectType':\n result['type'] = 'NetworkConnection'\n LOG.debug('NetworkConnectionObjectType OP: {!r}'.format(op))\n ov = None\n LOG.error('{} - NetworkConnectionObjectType Type not currently supported'.format(self.name))\n return None\n\n else:\n LOG.error('{} - unknown type {} {!r}'.format(self.name, ot, op))\n return None\n\n result['indicator'] = ov\n\n LOG.debug('{!r}'.format(result))\n\n return result\n\n def _decode_ttp(self, t):\n tdict = t.to_dict()\n\n if 'ttp' in tdict:\n tdict = tdict['ttp']\n\n if 'idref' in tdict:\n return {'idref': tdict['idref']}\n\n if 'description' in tdict:\n return {'description': tdict['description']}\n\n if 'title' in tdict:\n return {'description': tdict['title']}\n\n return {'description': ''}\n\n def _process_item(self, item):\n result = []\n value = {}\n\n iid, iv, stix_objects = item\n\n value['%s_indicator' % self.prefix] = iid\n\n if 'description' in iv:\n value['{}_indicator_description'.format(self.prefix)] = iv['description']\n\n if 'confidence' in iv:\n value['confidence'] = iv['confidence']\n\n if len(iv['ttps']) != 0:\n ttp = iv['ttps'][0]\n if 'idref' in ttp:\n ttp = stix_objects['ttps'].get(ttp['idref'])\n\n if ttp is not None and 'description' in ttp:\n value['%s_ttp' % self.prefix] = ttp['description']\n\n composed_observables = []\n for o in iv['observables']:\n if o is None:\n continue\n\n v = copy.copy(value)\n\n ob = o\n if 'idref' in o:\n ob = stix_objects['observables'].get(o['idref'], None)\n v['%s_observable' % self.prefix] = o['idref']\n\n if ob is None:\n continue\n\n if ob['type'] == '_cyboxOR':\n for o in ob['observables']:\n composed_observables.append(o)\n continue\n\n v['type'] = ob['type']\n\n if type(ob['indicator']) == list:\n indicator = ob['indicator']\n else:\n indicator = [ob['indicator']]\n\n for i in indicator:\n result.append([i, v])\n\n for o in composed_observables:\n v = copy.copy(value)\n\n ob = stix_objects['observables'].get(o, None)\n v['%s_observable' % self.prefix] = o\n\n if ob is None:\n continue\n\n if ob['type'] == '_cyboxOR':\n LOG.error(\n '%s - Nested Observable Composition not supported',\n self.name\n )\n continue\n\n v['type'] = ob['type']\n\n if type(ob['indicator']) == list:\n indicator = ob['indicator']\n else:\n indicator = [ob['indicator']]\n\n for i in indicator:\n result.append([i, v])\n\n return result\n\n def _build_iterator(self, now):\n if self.client_credentials_required:\n if self.username is None or self.password is None:\n raise RuntimeError(\n '%s - username or password required and not set, poll not performed' % self.name\n )\n if self.cert_file is not None and not os.path.isfile(self.cert_file):\n raise RuntimeError(\n '%s - client cert required and not set, poll not performed' % self.name\n )\n if self.key_file is not None and not os.path.isfile(self.key_file):\n raise RuntimeError(\n '%s - client cert key required and not set, poll not performed' % self.name\n )\n if self.subscription_id_required and self.subscription_id is None:\n raise RuntimeError(\n '%s - subscription id required and not set, poll not performed' % self.name\n )\n\n tc = self._build_taxii_client()\n self._discover_services(tc)\n self._check_collections(tc)\n\n last_run = self.last_taxii_run\n max_back = now-(self.initial_interval*1000)\n if last_run is None or last_run < max_back:\n last_run = max_back\n\n begin = datetime.utcfromtimestamp(last_run/1000)\n begin = begin.replace(tzinfo=pytz.UTC)\n\n end = datetime.utcfromtimestamp(now/1000)\n end = end.replace(tzinfo=pytz.UTC)\n\n if self.lower_timestamp_precision:\n end = end.replace(second=0, microsecond=0)\n begin = begin.replace(second=0, microsecond=0)\n\n return self._incremental_poll_collection(\n taxii_client=tc,\n begin=begin,\n end=end\n )\n\n def _flush(self):\n self.last_taxii_run = None\n super(TaxiiClient, self)._flush()\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(TaxiiClient, self).hup(source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n\n client_cert_required = False\n if config is not None:\n client_cert_required = config.get('client_cert_required', False)\n\n cert_path = None\n if config is not None:\n cert_path = config.get('cert_file', None)\n if cert_path is None and client_cert_required:\n cert_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}.crt'.format(name)\n )\n\n if cert_path is not None:\n try:\n os.remove(cert_path)\n except:\n pass\n\n key_path = None\n if config is not None:\n key_path = config.get('key_file', None)\n if key_path is None and client_cert_required:\n key_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}.pem'.format(name)\n )\n\n if key_path is not None:\n try:\n os.remove(key_path)\n except:\n pass\n\n\ndef _stix_ip_observable(namespace, indicator, value):\n category = cybox.objects.address_object.Address.CAT_IPV4\n if value['type'] == 'IPv6':\n category = cybox.objects.address_object.Address.CAT_IPV6\n\n indicators = [indicator]\n if '-' in indicator:\n # looks like an IP Range, let's try to make it a CIDR\n a1, a2 = indicator.split('-', 1)\n if a1 == a2:\n # same IP\n indicators = [a1]\n else:\n # use netaddr builtin algo to summarize range into CIDR\n iprange = netaddr.IPRange(a1, a2)\n cidrs = iprange.cidrs()\n indicators = map(str, cidrs)\n\n observables = []\n for i in indicators:\n id_ = '{}:observable-{}'.format(\n namespace,\n uuid.uuid4()\n )\n\n ao = cybox.objects.address_object.Address(\n address_value=i,\n category=category\n )\n\n o = cybox.core.Observable(\n title='{}: {}'.format(value['type'], i),\n id_=id_,\n item=ao\n )\n\n observables.append(o)\n\n return observables\n\n\ndef _stix_email_addr_observable(namespace, indicator, value):\n category = cybox.objects.address_object.Address.CAT_EMAIL\n\n id_ = '{}:observable-{}'.format(\n namespace,\n uuid.uuid4()\n )\n\n ao = cybox.objects.address_object.Address(\n address_value=indicator,\n category=category\n )\n\n o = cybox.core.Observable(\n title='{}: {}'.format(value['type'], indicator),\n id_=id_,\n item=ao\n )\n\n return [o]\n\n\ndef _stix_domain_observable(namespace, indicator, value):\n id_ = '{}:observable-{}'.format(\n namespace,\n uuid.uuid4()\n )\n\n do = cybox.objects.domain_name_object.DomainName()\n do.value = indicator\n do.type_ = 'FQDN'\n\n o = cybox.core.Observable(\n title='FQDN: ' + indicator,\n id_=id_,\n item=do\n )\n\n return [o]\n\n\ndef _stix_url_observable(namespace, indicator, value):\n id_ = '{}:observable-{}'.format(\n namespace,\n uuid.uuid4()\n )\n\n uo = cybox.objects.uri_object.URI(\n value=indicator,\n type_=cybox.objects.uri_object.URI.TYPE_URL\n )\n\n o = cybox.core.Observable(\n title='URL: ' + indicator,\n id_=id_,\n item=uo\n )\n\n return [o]\n\n\ndef _stix_hash_observable(namespace, indicator, value):\n id_ = '{}:observable-{}'.format(\n namespace,\n uuid.uuid4()\n )\n\n uo = cybox.objects.file_object.File()\n uo.add_hash(indicator)\n\n o = cybox.core.Observable(\n title='{}: {}'.format(value['type'], indicator),\n id_=id_,\n item=uo\n )\n\n return [o]\n\n\n_TYPE_MAPPING = {\n 'IPv4': {\n 'indicator_type': stix.common.vocabs.IndicatorType.TERM_IP_WATCHLIST,\n 'mapper': _stix_ip_observable\n },\n 'IPv6': {\n 'indicator_type': stix.common.vocabs.IndicatorType.TERM_IP_WATCHLIST,\n 'mapper': _stix_ip_observable\n },\n 'URL': {\n 'indicator_type': stix.common.vocabs.IndicatorType.TERM_URL_WATCHLIST,\n 'mapper': _stix_url_observable\n },\n 'domain': {\n 'indicator_type': stix.common.vocabs.IndicatorType.TERM_DOMAIN_WATCHLIST,\n 'mapper': _stix_domain_observable\n },\n 'sha256': {\n 'indicator_type': stix.common.vocabs.IndicatorType.TERM_FILE_HASH_WATCHLIST,\n 'mapper': _stix_hash_observable\n },\n 'sha1': {\n 'indicator_type': stix.common.vocabs.IndicatorType.TERM_FILE_HASH_WATCHLIST,\n 'mapper': _stix_hash_observable\n },\n 'md5': {\n 'indicator_type': stix.common.vocabs.IndicatorType.TERM_FILE_HASH_WATCHLIST,\n 'mapper': _stix_hash_observable\n },\n 'email-addr': {\n 'indicator_type': stix.common.vocabs.IndicatorType.TERM_MALICIOUS_EMAIL,\n 'mapper': _stix_email_addr_observable\n }\n}\n\n\nclass DataFeed(actorbase.ActorBaseFT):\n def __init__(self, name, chassis, config):\n self.redis_skey = name\n self.redis_skey_value = name+'.value'\n self.redis_skey_chkp = name+'.chkp'\n\n self.SR = None\n self.ageout_glet = None\n\n super(DataFeed, self).__init__(name, chassis, config)\n\n def configure(self):\n super(DataFeed, self).configure()\n\n self.redis_url = self.config.get('redis_url',\n os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock')\n )\n\n self.namespace = self.config.get('namespace', 'minemeld')\n self.namespaceuri = self.config.get(\n 'namespaceuri',\n 'https://go.paloaltonetworks.com/minemeld'\n )\n\n self.age_out_interval = self.config.get('age_out_interval', '24h')\n self.age_out_interval = interval_in_sec(self.age_out_interval)\n if self.age_out_interval < 60:\n LOG.info('%s - age out interval too small, forced to 60 seconds')\n self.age_out_interval = 60\n\n self.max_entries = self.config.get('max_entries', 1000 * 1000)\n\n self.attributes_package_title = self.config.get('attributes_package_title', [])\n if not isinstance(self.attributes_package_title, list):\n LOG.error('{} - attributes_package_title should be a list - ignored')\n self.attributes_package_title = []\n\n self.attributes_package_description = self.config.get('attributes_package_description', [])\n if not isinstance(self.attributes_package_description, list):\n LOG.error('{} - attributes_package_description should be a list - ignored')\n self.attributes_package_description = []\n\n self.attributes_package_sdescription = self.config.get('attributes_package_short_description', [])\n if not isinstance(self.attributes_package_sdescription, list):\n LOG.error('{} - attributes_package_sdescription should be a list - ignored')\n self.attributes_package_sdescription = []\n\n self.attributes_package_information_source = self.config.get('attributes_package_information_source', [])\n if not isinstance(self.attributes_package_information_source, list):\n LOG.error('{} - attributes_package_information_source should be a list - ignored')\n self.attributes_package_information_source = []\n\n def connect(self, inputs, output):\n output = False\n super(DataFeed, self).connect(inputs, output)\n\n def read_checkpoint(self):\n self._connect_redis()\n self.last_checkpoint = self.SR.get(self.redis_skey_chkp)\n\n def create_checkpoint(self, value):\n self._connect_redis()\n self.SR.set(self.redis_skey_chkp, value)\n\n def remove_checkpoint(self):\n self._connect_redis()\n self.SR.delete(self.redis_skey_chkp)\n\n def _connect_redis(self):\n if self.SR is not None:\n return\n\n self.SR = redis.StrictRedis.from_url(\n self.redis_url\n )\n\n def _read_oldest_indicator(self):\n olist = self.SR.zrange(\n self.redis_skey, 0, 0,\n withscores=True\n )\n LOG.debug('%s - oldest: %s', self.name, olist)\n if len(olist) == 0:\n return None, None\n\n return int(olist[0][1]), olist[0][0]\n\n def initialize(self):\n self._connect_redis()\n\n def rebuild(self):\n self._connect_redis()\n self.SR.delete(self.redis_skey)\n self.SR.delete(self.redis_skey_value)\n\n def reset(self):\n self._connect_redis()\n self.SR.delete(self.redis_skey)\n self.SR.delete(self.redis_skey_value)\n\n def _add_indicator(self, score, indicator, value):\n if self.length() >= self.max_entries:\n LOG.info('dropped overflow')\n self.statistics['drop.overflow'] += 1\n return\n\n type_ = value['type']\n type_mapper = _TYPE_MAPPING.get(type_, None)\n if type_mapper is None:\n self.statistics['drop.unknown_type'] += 1\n LOG.error('%s - Unsupported indicator type: %s', self.name, type_)\n return\n\n set_id_namespace(self.namespaceuri, self.namespace)\n\n title = None\n if len(self.attributes_package_title) != 0:\n for pt in self.attributes_package_title:\n if pt not in value:\n continue\n\n title = '{}'.format(value[pt])\n break\n\n description = None\n if len(self.attributes_package_description) != 0:\n for pd in self.attributes_package_description:\n if pd not in value:\n continue\n\n description = '{}'.format(value[pd])\n break\n\n sdescription = None\n if len(self.attributes_package_sdescription) != 0:\n for pd in self.attributes_package_sdescription:\n if pd not in value:\n continue\n\n sdescription = '{}'.format(value[pd])\n break\n\n information_source = None\n if len(self.attributes_package_information_source) != 0:\n for isource in self.attributes_package_information_source:\n if isource not in value:\n continue\n\n information_source = '{}'.format(value[isource])\n break\n\n if information_source is not None:\n identity = stix.common.identity.Identity(name=information_source)\n information_source = stix.common.information_source.InformationSource(identity=identity)\n\n handling = None\n share_level = value.get('share_level', None)\n if share_level in ['white', 'green', 'amber', 'red']:\n marking_specification = stix.data_marking.MarkingSpecification()\n marking_specification.controlled_structure = \"//node() | //@*\"\n\n tlp = stix.extensions.marking.tlp.TLPMarkingStructure()\n tlp.color = share_level.upper()\n marking_specification.marking_structures.append(tlp)\n\n handling = stix.data_marking.Marking()\n handling.add_marking(marking_specification)\n\n header = None\n if (title is not None or\n description is not None or\n handling is not None or\n sdescription is not None or\n information_source is not None):\n header = stix.core.STIXHeader(\n title=title,\n description=description,\n handling=handling,\n short_description=sdescription,\n information_source=information_source\n )\n\n spid = '{}:indicator-{}'.format(\n self.namespace,\n uuid.uuid4()\n )\n sp = stix.core.STIXPackage(id_=spid, stix_header=header)\n\n observables = type_mapper['mapper'](self.namespace, indicator, value)\n\n for o in observables:\n id_ = '{}:indicator-{}'.format(\n self.namespace,\n uuid.uuid4()\n )\n\n if value['type'] == 'URL':\n eindicator = werkzeug.urls.iri_to_uri(indicator, safe_conversion=True)\n else:\n eindicator = indicator\n\n sindicator = stix.indicator.indicator.Indicator(\n id_=id_,\n title='{}: {}'.format(\n value['type'],\n eindicator\n ),\n description='{} indicator from {}'.format(\n value['type'],\n ', '.join(value['sources'])\n ),\n timestamp=datetime.utcnow().replace(tzinfo=pytz.utc)\n )\n\n confidence = value.get('confidence', None)\n if confidence is None:\n LOG.error('%s - indicator without confidence', self.name)\n sindicator.confidence = \"Unknown\" # We shouldn't be here\n elif confidence < 50:\n sindicator.confidence = \"Low\"\n elif confidence < 75:\n sindicator.confidence = \"Medium\"\n else:\n sindicator.confidence = \"High\"\n\n sindicator.add_indicator_type(type_mapper['indicator_type'])\n\n sindicator.add_observable(o)\n\n sp.add_indicator(sindicator)\n\n spackage = 'lz4'+lz4.frame.compress(\n sp.to_json(),\n compression_level=lz4.frame.COMPRESSIONLEVEL_MINHC\n )\n with self.SR.pipeline() as p:\n p.multi()\n\n p.zadd(self.redis_skey, score, spid)\n p.hset(self.redis_skey_value, spid, spackage)\n\n result = p.execute()[0]\n\n self.statistics['added'] += result\n\n def _delete_indicator(self, indicator_id):\n with self.SR.pipeline() as p:\n p.multi()\n\n p.zrem(self.redis_skey, indicator_id)\n p.hdel(self.redis_skey_value, indicator_id)\n\n result = p.execute()[0]\n\n self.statistics['removed'] += result\n\n def _age_out_run(self):\n while True:\n now = utc_millisec()\n low_watermark = now - self.age_out_interval*1000\n\n otimestamp, oindicator = self._read_oldest_indicator()\n LOG.debug(\n '{} - low watermark: {} otimestamp: {}'.format(\n self.name,\n low_watermark,\n otimestamp\n )\n )\n while otimestamp is not None and otimestamp < low_watermark:\n self._delete_indicator(oindicator)\n otimestamp, oindicator = self._read_oldest_indicator()\n\n wait_time = 30\n if otimestamp is not None:\n next_expiration = (\n (otimestamp + self.age_out_interval*1000) - now\n )\n wait_time = max(wait_time, next_expiration/1000 + 1)\n LOG.debug('%s - sleeping for %d secs', self.name, wait_time)\n\n gevent.sleep(wait_time)\n\n @base._counting('update.processed')\n def filtered_update(self, source=None, indicator=None, value=None):\n now = utc_millisec()\n\n self._add_indicator(now, indicator, value)\n\n @base._counting('withdraw.ignored')\n def filtered_withdraw(self, source=None, indicator=None, value=None):\n # this is a TAXII data feed, old indicators never expire\n pass\n\n def length(self, source=None):\n return self.SR.zcard(self.redis_skey)\n\n def start(self):\n super(DataFeed, self).start()\n\n self.ageout_glet = gevent.spawn(self._age_out_run)\n\n def stop(self):\n super(DataFeed, self).stop()\n\n self.ageout_glet.kill()\n\n LOG.info(\n \"%s - # indicators: %d\",\n self.name,\n self.SR.zcard(self.redis_skey)\n )\n\n @staticmethod\n def gc(name, config=None):\n actorbase.ActorBaseFT.gc(name, config=config)\n\n if config is None:\n config = {}\n\n redis_skey = name\n redis_skey_value = '{}.value'.format(name)\n redis_skey_chkp = '{}.chkp'.format(name)\n redis_url = config.get('redis_url',\n os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock')\n )\n\n cp = None\n try:\n cp = redis.ConnectionPool.from_url(\n redis_url\n )\n\n SR = redis.StrictRedis(connection_pool=cp)\n\n SR.delete(redis_skey)\n SR.delete(redis_skey_value)\n SR.delete(redis_skey_chkp)\n\n except Exception as e:\n raise RuntimeError(str(e))\n\n finally:\n if cp is not None:\n cp.disconnect()\n" }, { "path": "minemeld/ft/taxii2.py", "content": "# Copyright 2015-present Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport os\nimport yaml\nfrom datetime import datetime, timedelta\nfrom uuid import UUID\n\nimport pytz\nimport requests\nimport requests.structures\nimport ujson\n\nfrom stix2patterns.pattern import Pattern, ParseException\n\nfrom . import basepoller\n\nfrom .utils import dt_to_millisec, interval_in_sec\n\nLOG = logging.getLogger(__name__)\n\n_STIX2_TYPES_TO_MM_TYPES = {\n 'ipv4-addr': 'IPv4',\n 'ipv6-addr': 'IPv6',\n 'domain': 'domain',\n 'domain-name': 'domain',\n 'url': 'URL',\n 'file': None,\n 'md5': 'md5',\n 'sha-1': 'sha1',\n 'sha-256': 'sha256'\n}\n\n\nclass Taxii2Client(basepoller.BasePollerFT):\n def __init__(self, name, chassis, config):\n self.poll_service = None\n self.last_taxii2_run = None\n self.last_stix2_package_ts = None\n\n super(Taxii2Client, self).__init__(name, chassis, config)\n\n def configure(self):\n super(Taxii2Client, self).configure()\n\n self.initial_interval = self.config.get('initial_interval', '1d')\n self.initial_interval = interval_in_sec(self.initial_interval)\n if self.initial_interval is None:\n LOG.error('%s - wrong initial_interval format: %s', self.name, self.initial_interval)\n self.initial_interval = 86400\n self.max_poll_dt = self.config.get('max_poll_dt', 86400)\n\n # options for processing\n self.lower_timestamp_precision = self.config.get('lower_timestamp_precision', False)\n\n self.auth_type = self.config.get('auth_type', 'none')\n\n self.username = self.config.get('username', None)\n self.password = self.config.get('password', None)\n\n self.api_key = self.config.get('api_key', None)\n\n self.discovery_service = self.config.get('discovery_service', None)\n self.api_root = self.config.get('api_root', None)\n self.collection = self.config.get('collection', None)\n self.verify_cert = self.config.get('verify_cert', True)\n\n self.enabled = self.config.get('enabled', 'no')\n\n self.client = None\n self.taxii_collection = None\n self.taxii_version = None\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n # self.prefix = self.config.get('prefix', self.name)\n\n # self.confidence_map = self.config.get('confidence_map', {'low': 40, 'medium': 60, 'high': 80})\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n auth_type = sconfig.get('auth_type', None)\n username = sconfig.get('username', None)\n password = sconfig.get('password', None)\n api_key = sconfig.get('api_key', None)\n\n if api_key is not None:\n self.api_key = api_key\n LOG.info('{} - Loaded credentials from side config'.format(self.name))\n elif username is not None and password is not None:\n self.username = username\n self.password = password\n LOG.info('{} - Loaded credentials from side config'.format(self.name))\n\n discovery_service = sconfig.get('discovery_service', None)\n api_root = sconfig.get('api_root', None)\n collection = sconfig.get('collection', None)\n verify_cert = sconfig.get('verify_cert', None)\n\n enabled = sconfig.get('enabled', None)\n\n if discovery_service is not None:\n self.discovery_service = discovery_service\n LOG.info('{} - Loaded discovery service from side config'.format(self.name))\n\n if api_root is not None:\n self.api_root = api_root\n LOG.info('{} - Loaded api root from side config'.format(self.name))\n\n if collection is not None:\n self.collection = collection\n LOG.info('{} - Loaded collection from side config'.format(self.name))\n\n if verify_cert is not None:\n self.verify_cert = verify_cert\n LOG.info('{} - Loaded collection from side config'.format(self.name))\n\n if auth_type is not None:\n self.auth_type = auth_type\n LOG.info('{} - Loaded collection from side config'.format(self.name))\n\n if enabled is not None:\n self.enabled = enabled\n LOG.info('{} - Loaded collection from side config'.format(self.name))\n\n def _saved_state_restore(self, saved_state):\n super(Taxii2Client, self)._saved_state_restore(saved_state)\n self.last_taxii2_run = saved_state.get('last_taxii2_run', None)\n LOG.info('last_taxii2_run from sstate: %s', self.last_taxii2_run)\n\n def _saved_state_create(self):\n sstate = super(Taxii2Client, self)._saved_state_create()\n sstate['last_taxii2_run'] = self.last_taxii2_run\n\n return sstate\n\n def _saved_state_reset(self):\n super(Taxii2Client, self)._saved_state_reset()\n self.last_taxii2_run = None\n\n def _set_accept_header(self, session):\n content_types = {\n 'stix20': 'application/vnd.oasis.stix+json; version=2.0',\n 'taxii20': 'application/vnd.oasis.taxii+json; version=2.0',\n 'stix21': 'application/taxii+json; version=2.1',\n 'taxii21': 'application/taxii+json; version=2.1'\n }\n\n try:\n # Assume the server is TAXII 2.0\n self.taxii_version = '2.0'\n session.headers.update({'Accept': content_types['taxii20']})\n\n r1 = session.get(self.discovery_service)\n if r1.status_code == 406:\n # If the server is not TAXII 2.0, assume it is TAXII 2.1\n self.taxii_version = '2.1'\n session.headers.update({'Accept': content_types['taxii21']})\n\n r2 = session.get(self.discovery_service)\n if r2.status_code == 406:\n # The server supports neither\n raise RuntimeError('server does not support TAXII 2.0 nor TAXII 2.1')\n except Exception as e:\n raise RuntimeError('error contacting server. {}'.format(e))\n\n def _build_taxii2_client(self):\n session = requests.Session()\n session.verify = True if self.verify_cert == 'yes' else False\n\n if self.api_key:\n session.headers.update({'Authorization': 'Token {}'.format(self.api_key)})\n elif self.username and self.password:\n session.auth = (self.username, self.password)\n # session.auth = requests.auth.HTTPBasicAuth(self.username, self.password)\n else:\n pass\n\n # Check the TAXII server to ensure the correct Accept header is set\n self._set_accept_header(session)\n\n self.client = session\n\n def _get_api_root(self):\n if self.client:\n r = self.client.get(self.discovery_service)\n if r.status_code == requests.codes.ok:\n try:\n discovery = r.json()\n if 'api_roots' in discovery:\n api_roots = discovery['api_roots']\n for url in api_roots:\n # strip the trailing slash\n if url[:-1].endswith(self.api_root):\n self.api_root = url\n break\n else:\n raise RuntimeError('error getting api_root.'.format(r.status_code))\n except Exception as e:\n raise RuntimeError('error getting api_root. {}'.format(e))\n else:\n raise RuntimeError('error getting api_root. received code {}'.format(r.status_code))\n else:\n raise RuntimeError('client does not exist {}'.format(self.collection))\n\n def _is_uuid(self, val, ver):\n n = len(val)\n if n == 32 or n == 36:\n try:\n uuid_val = UUID(val, version=ver)\n except Exception:\n return False\n\n return str(uuid_val) == val\n else:\n return False\n\n def _get_collection(self):\n try:\n if self.client:\n collection_url = '{}collections/'.format(self.api_root)\n r = self.client.get(collection_url)\n if r.status_code == requests.codes.ok:\n collections = r.json()['collections']\n if self._is_uuid(self.collection, 3) or self._is_uuid(self.collection, 4):\n for c in collections:\n if c['id'] == self.collection:\n self.taxii_collection = c\n break\n else:\n self.taxii_collection = collections[0]\n else:\n msg = 'error getting collection {}. received code {}'.format(self.collection, r.status_code)\n raise RuntimeError(msg)\n else:\n raise RuntimeError('client does not exist {}'.format(self.collection))\n except RuntimeError as e:\n LOG.exception(e)\n except Exception as e:\n LOG.exception('collection {} was not found - {}'.format(self.collection, e))\n\n\n # noinspection PyMethodMayBeStatic\n def _clean_indicator(self, sub_pattern_value):\n indicator = str(sub_pattern_value)\n\n if indicator[0] == \"'\" and indicator[-1] == \"'\":\n return indicator[1:-1]\n else:\n return indicator\n\n # noinspection PyMethodMayBeStatic\n def _detect_and_map_type(self, i_type, sub_pattern_type):\n if i_type == 'file':\n sub_pattern_type = sub_pattern_type[-1].lower()\n\n return _STIX2_TYPES_TO_MM_TYPES.get(sub_pattern_type, None)\n else:\n return _STIX2_TYPES_TO_MM_TYPES.get(i_type, None)\n\n def _convert_stix2_obj_to_mm_obj(self, obj, rels, ttps):\n # Inspect the STIX2 Pattern\n # result\n # comparisons\n # type_dict\n # foo\n # 0\n # 0 (type)\n # 1 (op)\n # 2 (value)\n # bar\n # 0\n # 0 (type)\n # 1 (op)\n # 2 (value)\n # ...\n\n # noinspection PyBroadException\n try:\n pattern = obj['pattern']\n inspected_pattern = Pattern(pattern).inspect()\n comparisons = inspected_pattern.comparisons\n\n indicators = []\n\n # noinspection PyCompatibility\n for i_type, i_patterns in comparisons.iteritems():\n # The Pattern Inspector buckets each comparison expression in the observable expression based on type\n if i_type in _STIX2_TYPES_TO_MM_TYPES:\n # The Pattern Inspector reduces the observable expression into a flat list of comparison expressions\n for sub_pattern in i_patterns:\n (sub_pattern_type, sub_pattern_op, sub_pattern_value) = sub_pattern\n\n mm_type = self._detect_and_map_type(i_type, sub_pattern_type)\n\n if mm_type:\n indicator = self._clean_indicator(sub_pattern_value)\n value = {\n \"type\": mm_type\n }\n if 'confidence' in obj:\n value['confidence'] = obj['confidence']\n\n descriptions = [r[\"description\"].strip() for r in rels if \"description\" in r]\n if len(descriptions):\n value[\"description\"] = \", \".join(descriptions)\n\n techniques = [t[\"name\"].strip() for t in ttps]\n if len(techniques):\n value[\"techniques\"] = \", \".join(techniques)\n\n i = [indicator, value]\n indicators.append(i)\n\n return indicators\n except ParseException as e:\n LOG.warning('error parsing indicator pattern {}'.format(e))\n except Exception as e:\n LOG.error('exception parsing indicator pattern {}'.format(e))\n\n def _explore(self, root, types):\n objs = [root]\n while objs:\n obj = objs.pop()\n if isinstance(obj, dict):\n if 'type' in obj and obj['type'] in types:\n yield obj\n else:\n objs.extend(obj.values())\n elif isinstance(obj, list):\n objs.extend(obj)\n\n def _poll_taxii21_server(self):\n \"\"\"\n TAXII 2.1 uses a limit url query parameter and a 'more' true/false key in the returned data\n https://docs.oasis-open.org/cti/taxii/v2.1/csprd01/taxii-v2.1-csprd01.html#_Toc532988055\n :return: list of objects\n \"\"\"\n\n data = []\n params = {'limit': '100'}\n if self.last_stix2_package_ts:\n params['added_after'] = self.last_stix2_package_ts\n fetch_more = True\n\n while fetch_more:\n # Poll the server\n # Check the 'more' field in the response json to see if there is more data\n # Poll until there is no data\n url = '{}collections/{}/objects/'.format(self.api_root, self.taxii_collection['id'])\n\n r = self.client.get(url, params=params)\n\n if r.status_code in [200, 201, 206]:\n try:\n r_json = ujson.loads(r.text)\n # Filter objects by type in the data returned by the TAXII 2.x server\n types = ['indicator', 'attack-pattern', 'relationship']\n objs = self._explore(r_json, types)\n data.extend(objs)\n\n # Sort the objs in data by timestamp to find the most recent timestamp\n data.sort(key=lambda x: datetime.strptime(x['modified'], '%Y-%m-%dT%H:%M:%S.%fZ'))\n if len(data):\n ts = data[-1]['modified']\n params['added_after'] = ts\n self.last_stix2_package_ts = ts\n\n if 'more' in r_json and r_json['more'] is True:\n pass\n else:\n break\n except Exception as e:\n LOG.exception(e)\n break\n else:\n break\n\n return data\n\n def _poll_taxii20_server(self):\n \"\"\"\n TAXII 2.0 uses Range and Content-Range headers for pagination\n http://docs.oasis-open.org/cti/taxii/v2.0/cs01/taxii-v2.0-cs01.html#_Toc496542715\n :return: list of objects\n \"\"\"\n\n data = []\n size = 100\n params = {}\n if self.last_stix2_package_ts:\n params['added_after'] = self.last_stix2_package_ts\n fetch_more = True\n\n while fetch_more:\n # Poll the server\n # Check the response headers to see if there is paginated data\n # Poll until there is no data\n\n url = '{}collections/{}/objects/'.format(self.api_root, self.taxii_collection['id'])\n\n r = self.client.get(url, params=params)\n\n if r.status_code in [200, 201, 206]:\n try:\n r_json = ujson.loads(r.text)\n # Filter objects by type in the data returned by the TAXII 2.x server\n types = ['indicator', 'attack-pattern', 'relationship']\n objs = self._explore(r_json, types)\n data.extend(objs)\n\n # Sort the objs in data by timestamp to find the most recent timestamp\n data.sort(key=lambda x: datetime.strptime(x['modified'], '%Y-%m-%dT%H:%M:%S.%fZ'))\n\n if len(data):\n self.last_stix2_package_ts = data[-1]['modified']\n\n content_range = r.headers.get('Content-Range', None)\n if content_range and content_range.startswith('items '):\n content_range_start_end_size = content_range[6:]\n content_range_start_end, content_range_size = content_range_start_end_size.split('/')\n content_range_start, content_range_end = content_range_start_end.split('-')\n\n next_start = int(content_range_end) + 1\n # next_end = next_start + size\n next_end = next_start + (int(content_range_end) - int(content_range_start)) + 1\n\n if next_start < int(content_range_size):\n updated_content_range = 'items {}-{}'.format(next_start, next_end)\n self.client.headers.update({'Range': updated_content_range})\n else:\n break\n else:\n break\n except Exception as e:\n LOG.exception(e)\n break\n else:\n break\n\n return data\n\n def _poll_and_filter_collection(self, begin=None, end=None):\n if self.client:\n if self.taxii_collection:\n if self.taxii_version == '2.0':\n data = self._poll_taxii20_server()\n elif self.taxii_version == '2.1':\n data = self._poll_taxii21_server()\n else:\n # Unsupported\n data = []\n\n raw_objs = data\n\n # Sort objects by type in the data returned by the TAXII 2.x server\n objs = {}\n types = ['indicator', 'attack-pattern', 'relationship']\n for k in types:\n objs[k] = []\n\n for obj in raw_objs:\n if obj['type'] == 'indicator' and 'pattern' in obj:\n objs[obj['type']].append(obj)\n else:\n objs[obj['type']].append(obj)\n\n ids_to_ttps = {}\n for t in objs['attack-pattern']:\n ids_to_ttps[t['id']] = t\n\n indicators = []\n for i in objs['indicator']:\n i_rels = [x for x in objs['relationship'] if i['id'] == x['source_ref']]\n i_ttp_rels = [x for x in i_rels if x['target_ref'].startswith('attack-pattern')]\n i_ttps = [ids_to_ttps[x['target_ref']] for x in i_ttp_rels]\n\n mm_is = self._convert_stix2_obj_to_mm_obj(i, i_rels, i_ttps)\n if mm_is:\n # The indicator pattern is valid and was parsed\n indicators.extend(mm_is)\n\n return indicators\n else:\n raise RuntimeError('no collection {}'.format(self.collection))\n else:\n raise RuntimeError('client does not exist {}'.format(self.collection))\n\n def _incremental_poll_collection(self, begin, end):\n cbegin = begin\n dt = timedelta(seconds=self.max_poll_dt)\n\n # self.last_stix2_package_ts = None\n\n while cbegin < end:\n cend = min(end, cbegin + dt)\n\n LOG.info('{} - polling {!r} to {!r}'.format(self.name, cbegin, cend))\n result = self._poll_and_filter_collection(begin=cbegin, end=cend)\n\n for i in result:\n yield i\n\n if self.last_stix2_package_ts is not None:\n self.last_taxii2_run = self.last_stix2_package_ts\n\n cbegin = cend\n\n def _process_item(self, item):\n return [item]\n\n def _manage_time(self, now):\n last_run = self.last_taxii2_run\n if last_run:\n last_run = dt_to_millisec(datetime.strptime(self.last_taxii2_run, '%Y-%m-%dT%H:%M:%S.%fZ'))\n max_back = now - (self.initial_interval * 1000)\n if last_run is None or last_run < max_back:\n last_run = max_back\n\n begin = datetime.utcfromtimestamp(last_run / 1000)\n begin = begin.replace(tzinfo=pytz.UTC)\n\n end = datetime.utcfromtimestamp(now / 1000)\n end = end.replace(tzinfo=pytz.UTC)\n\n if self.lower_timestamp_precision:\n end = end.replace(second=0, microsecond=0)\n begin = begin.replace(second=0, microsecond=0)\n\n return begin, end\n\n def _check_args(self):\n if (self.username or self.password) and self.api_key:\n raise RuntimeError(\n '%s - username, password, and api_key cannot all be set, poll not performed' % self.name\n )\n\n if not self.discovery_service:\n raise RuntimeError(\n '%s - discovery_service required and not set, poll not performed' % self.name\n )\n\n if not self.api_root:\n raise RuntimeError(\n '%s - api_root required and not set, poll not performed' % self.name\n )\n\n if not self.collection:\n raise RuntimeError(\n '%s - collection required and not set, poll not performed' % self.name\n )\n\n if not self.enabled:\n raise RuntimeError(\n '%s - node is disabled, poll not performed' % self.name\n )\n\n def _build_iterator(self, now):\n self._check_args()\n\n self._build_taxii2_client()\n self._get_api_root()\n self._get_collection()\n\n self._check_args()\n\n (begin, end) = self._manage_time(now)\n\n return self._incremental_poll_collection(begin=begin, end=end)\n\n def _flush(self):\n self.last_taxii2_run = None\n super(Taxii2Client, self)._flush()\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(Taxii2Client, self).hup(source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except Exception:\n pass\n" }, { "path": "minemeld/ft/test.py", "content": "from __future__ import absolute_import\n\nimport logging\nimport gevent\n\nfrom . import base\nfrom .utils import utc_millisec\n\nimport netaddr\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass TestMiner(base.BaseFT):\n def __init__(self, name, chassis, config):\n super(TestMiner, self).__init__(name, chassis, config)\n\n self._glet = None\n\n def configure(self):\n super(TestMiner, self).configure()\n\n self.num_messages = self.config.get('num_messages', 100000)\n self.mps = self.config.get('mps', 1000)\n\n def initialize(self):\n pass\n\n def rebuild(self):\n pass\n\n def reset(self):\n pass\n\n def _run(self):\n cip = 0x0A000000\n\n v = {\n 'type': 'IPv4',\n 'confidence': 0,\n 'share_level': 'red'\n }\n\n LOG.info('%s - start sending messages: %d', self.name, utc_millisec())\n\n t1 = utc_millisec()\n for i in xrange(self.num_messages):\n ip = str(netaddr.IPAddress(i+cip))\n self.emit_update(ip, v)\n\n if ((i+1) % self.mps) == 0:\n now = utc_millisec()\n LOG.info('%d: %d', i+1, now - t1)\n\n if now - t1 < 1000:\n gevent.sleep((1000 - now + t1)/1000.0)\n\n t1 = now\n\n LOG.info('%s - all messages sent: %d', self.name, utc_millisec())\n\n def length(self, source=None):\n return 0\n\n def start(self):\n super(TestMiner, self).start()\n\n self._glet = gevent.spawn_later(\n 2,\n self._run\n )\n\n def stop(self):\n super(TestMiner, self).stop()\n\n if self._glet is None:\n return\n\n self._glet.kill()\n\n\nclass TestFeed(base.BaseFT):\n def __init__(self, name, chassis, config):\n super(TestFeed, self).__init__(name, chassis, config)\n\n def configure(self):\n super(TestFeed, self).configure()\n\n self.num_messages = self.config.get('num_messages', 100000)\n\n def read_checkpoint(self):\n self.last_checkpoint = None\n\n def create_checkpoint(self, value):\n pass\n\n def initialize(self):\n pass\n\n def rebuild(self):\n pass\n\n def reset(self):\n pass\n\n @base._counting('update.processed')\n def filtered_update(self, source=None, indicator=None, value=None):\n if self.statistics['update.processed'] == 1:\n LOG.info('%s - first message: %d', self.name, utc_millisec())\n elif self.statistics['update.processed'] == self.num_messages:\n LOG.info('%s - last message: %d', self.name, utc_millisec())\n\n @base._counting('withdraw.processed')\n def filtered_withdraw(self, source=None, indicator=None, value=None):\n pass\n\n def length(self, source=None):\n pass\n\n\nclass FaultyConfig(base.BaseFT):\n def configure(self):\n super(FaultyConfig, self).configure()\n\n raise RuntimeError('fault !')\n\n def initialize(self):\n pass\n\n def rebuild(self):\n pass\n\n def reset(self):\n pass\n\n def length(self, source=None):\n return 0\n\n\nclass FaultyInit(base.BaseFT):\n def __init__(self, name, chassis, config):\n raise RuntimeError('fault !')\n\n def configure(self):\n pass\n\n def initialize(self):\n pass\n\n def rebuild(self):\n pass\n\n def reset(self):\n pass\n\n def length(self, source=None):\n return 0\n" }, { "path": "minemeld/ft/threatconnect.py", "content": "import logging\nimport hmac\nimport hashlib\nimport base64\nimport time\nimport requests\nimport pytz\nimport os\nimport yaml\nimport re\n\nfrom netaddr import IPNetwork, AddrFormatError\nfrom urllib import quote\nfrom basepoller import BasePollerFT\nfrom utils import utc_millisec, dt_to_millisec\nfrom datetime import datetime\n\nLOG = logging.getLogger(__name__)\nGENERIC_INDICATOR_MAP = [\n {\"apiBranch\": \"emailAddresses\", \"apiEntity\": \"emailAddress\", \"indicator\": {\"address\": \"email-addr\"}},\n {\"apiBranch\": \"hosts\", \"apiEntity\": \"host\", \"indicator\": {\"hostName\": \"domain\"}},\n {\"apiBranch\": \"urls\", \"apiEntity\": \"url\", \"indicator\": {\"text\": \"URL\"}},\n {\"apiBranch\": \"files\", \"apiEntity\": \"file\", \"indicator\": {\"md5\": \"md5\", \"sha1\": \"sha1\", \"sha256\": \"sha256\"}},\n {\"apiBranch\": \"registryKeys\", \"apiEntity\": \"registryKey\", \"indicator\": None},\n {\"apiBranch\": \"userAgents\", \"apiEntity\": \"userAgent\", \"indicator\": None}\n]\n\nIP_INDICATOR_MAP = [\n {\"apiBranch\": \"addresses\", \"apiEntity\": \"address\", \"indicator\": [\"ip\"]},\n {\"apiBranch\": \"ipPorts\", \"apiEntity\": \"ipPort\", \"indicator\": None}\n]\n\nGROUP_TYPES = [\"adversaries\", \"campaigns\", \"documents\", \"emails\", \"incidents\", \"signatures\", \"threats\"]\n\nSHA256_PATTERN = \"[A-Fa-f0-9]{64}\"\nSHA1_PATTERN = \"[A-Fa-f0-9]{40}\"\nMD5_PATTERN = \"[A-Fa-f0-9]{32}\"\n\n\nclass ThreatConnect(object):\n api_secret = None\n api_key = None\n api_url = None\n api_base_uri = None\n signature = None\n api_timestamp = None\n owner = None\n hash_patterns = {\"sha256\": re.compile(SHA256_PATTERN), \"sha1\": re.compile(SHA1_PATTERN),\n \"md5\": re.compile(MD5_PATTERN)}\n\n def __init__(self, api_secret, api_key, api_url, api_base_uri, owner):\n self.api_secret = api_secret\n self.api_key = api_key\n self.api_url = api_url\n self.api_base_uri = api_base_uri\n self.owner = None if owner is None else quote(owner)\n\n def prepare_get(self, uri):\n self.api_timestamp = str(int(time.time()))\n message = '{}:GET:{}'.format(uri, self.api_timestamp)\n digest = hmac.new(self.api_secret, msg=message, digestmod=hashlib.sha256).digest()\n self.signature = 'TC {}:{}'.format(self.api_key, base64.b64encode(digest).decode())\n\n def __call__(self, r):\n r.headers['Authorization'] = self.signature\n r.headers['Timestamp'] = self.api_timestamp\n return r\n\n def _detect_ip_version(self, ip_addr):\n try:\n parsed = IPNetwork(ip_addr)\n except (AddrFormatError, ValueError):\n LOG.error('{} - Unknown IP version: {}'.format(self.name, ip_addr))\n return None\n\n if parsed.version == 4:\n return 'IPv4'\n\n if parsed.version == 6:\n return 'IPv6'\n\n return None\n\n def _detect_sha_version(self, hash):\n for hash_type, re_obj in self.hash_patterns.iteritems():\n if re_obj.match(hash) is not None:\n return hash_type\n return None\n\n def group_indicator_processing(self, item, group_type, group_id, f_seen, l_seen):\n attributes = {'tc_group_type': group_type, 'tc_group_id': group_id, 'first_seen': f_seen, 'last_seen': l_seen}\n indicator = item.get(\"summary\", None)\n confidence = item.get('threatAssessConfidence', None)\n if confidence is not None:\n attributes['confidence'] = int(confidence)\n tc_indicator_type = item.get(\"type\", None)\n if tc_indicator_type == \"Address\":\n attributes['type'] = self._detect_ip_version(indicator)\n elif tc_indicator_type == \"File\":\n attributes['type'] = self._detect_sha_version(indicator)\n elif tc_indicator_type == \"EmailAddress\":\n attributes['type'] = \"email-addr\"\n elif tc_indicator_type == \"URL\":\n attributes['type'] = \"URL\"\n elif tc_indicator_type == \"Host\":\n attributes['type'] = \"domain\"\n if tc_indicator_type is None or indicator is None:\n return []\n return [indicator, attributes]\n\n def general_processing(self, item, indicator_map, f_seen, l_seen):\n result = []\n for tc_indicator, mm_indicator in indicator_map.iteritems():\n indicator = item.get(tc_indicator, None)\n if indicator is None:\n continue\n attributes = {'type': mm_indicator, 'first_seen': f_seen, 'last_seen': l_seen}\n confidence = item.get('threatAssessConfidence', None)\n if confidence is not None:\n attributes['confidence'] = int(confidence)\n add_attributes = dict(indicator_map)\n add_attributes.pop(tc_indicator)\n for tc_attribute, mm_attribute in add_attributes.iteritems():\n value = item.get(tc_attribute, None)\n if value is None:\n continue\n attributes[mm_attribute] = value\n result.append([indicator, attributes])\n return result\n\n def ip_processing(self, item, indicator_list, f_seen, l_seen):\n result = []\n for tc_indicator in indicator_list:\n indicator = item.get(tc_indicator, None)\n if indicator is None:\n continue\n ip_type = self._detect_ip_version(indicator)\n if ip_type is None:\n continue\n attributes = {'type': ip_type, 'first_seen': f_seen, 'last_seen': l_seen}\n confidence = item.get('threatAssessConfidence', None)\n if confidence is not None:\n attributes['confidence'] = int(confidence)\n result.append([indicator, attributes])\n return result\n\n def _paginate_request(self, entry_point, entity, from_timestamp=None):\n if from_timestamp is not None:\n isotime = datetime.fromtimestamp(from_timestamp / 1000).replace(tzinfo=pytz.utc).isoformat()\n\n def do_call(start):\n api_request = entry_point + '?resultStart={}&resultLimit=100'.format(start)\n if from_timestamp is not None:\n api_request += \"&modifiedSince={}\".format(isotime)\n if self.owner is not None:\n api_request += '&owner={}'.format(self.owner)\n self.prepare_get(api_request)\n final_url = self.api_url + api_request\n response = requests.get(final_url, auth=self)\n doc = response.json()\n if doc[\"status\"] != \"Success\":\n raise RuntimeError(\"ThreatConnectAPI - {}\".format(doc.get(\"message\", \"unknown error\")))\n return doc\n\n r_data = do_call(0)\n pointer = 0\n if \"data\" not in r_data:\n return\n if \"resultCount\" not in r_data[\"data\"]:\n return\n result_count = r_data[\"data\"][\"resultCount\"]\n while True:\n items = r_data[\"data\"][entity]\n for item in items:\n yield item\n pointer += len(items)\n if result_count <= pointer:\n break\n r_data = do_call(pointer)\n\n def indicator_iterator(self, last_tc_run):\n from_timestamp = last_tc_run\n for a in IP_INDICATOR_MAP:\n indicator_list = a.get(\"indicator\", None)\n if indicator_list is None:\n continue\n for item in self._paginate_request(self.api_base_uri + \"/v2/indicators/\" + a[\"apiBranch\"], a[\"apiEntity\"],\n from_timestamp):\n yield (\"IP\", item, indicator_list)\n\n for a in GENERIC_INDICATOR_MAP:\n indicator_map = a.get(\"indicator\", None)\n if indicator_map is None:\n continue\n for item in self._paginate_request(self.api_base_uri + \"/v2/indicators/\" + a[\"apiBranch\"], a[\"apiEntity\"],\n from_timestamp):\n yield (\"GENERAL\", item, indicator_map)\n\n def groups_iterator(self, groups):\n for group_type, group_ids in groups.iteritems():\n for group_id in group_ids:\n for item in self._paginate_request(\n self.api_base_uri + \"/v2/groups/{}/{}/indicators\".format(group_type, group_id), \"indicator\"):\n yield (item, group_type, group_id)\n\n\nclass TCMiner(BasePollerFT):\n tc = None\n api_secret = None\n api_key = None\n api_url = None\n api_base_uri = None\n owner = None\n side_config_path = None\n last_tc_run = None\n\n def configure(self):\n super(TCMiner, self).configure()\n self.api_key = self.config.get('apikey', None)\n self.api_secret = self.config.get('apisecret', None)\n self.sndbox_fqdn = self.config.get('sndbox_fqdn', 'sandbox.threatconnect.com')\n sandbox = self.config.get('sandbox', False)\n if sandbox:\n self.api_url = 'https://{}'.format(self.sndbox_fqdn)\n self.api_base_uri = '/api'\n else:\n self.api_url = 'https://api.threatconnect.com'\n self.api_base_uri = ''\n self.owner = self.config.get('owner', None)\n if not (None in [self.api_key, self.api_secret]):\n self.tc = ThreatConnect(self.api_secret, self.api_key, self.api_url, self.api_base_uri, self.owner)\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.tc = None\n data_owner = sconfig.get('owner', self.owner)\n side_api_key = sconfig.get('apikey', self.api_key)\n side_api_secret = sconfig.get('apisecret', self.api_secret)\n if not (None in [side_api_key, side_api_secret]):\n self.tc = ThreatConnect(side_api_secret, side_api_key, self.api_url, self.api_base_uri, data_owner)\n\n def _saved_state_restore(self, saved_state):\n super(TCMiner, self)._saved_state_restore(saved_state)\n self.last_tc_run = saved_state.get('last_tc_run', None)\n LOG.info('last_tc_run from sstate: %s', self.last_tc_run)\n\n def _saved_state_create(self):\n sstate = super(TCMiner, self)._saved_state_create()\n sstate['last_tc_run'] = self.last_tc_run\n return sstate\n\n def _saved_state_reset(self):\n super(TCMiner, self)._saved_state_reset()\n self.last_tc_run = None\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(TCMiner, self).hup(source=source)\n\n @staticmethod\n def gc(name, config=None):\n BasePollerFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n finally:\n pass\n\n\nclass IndicatorsMiner(TCMiner):\n initial_interval = None\n\n def configure(self):\n super(IndicatorsMiner, self).configure()\n self.initial_interval = self.config.get('initial_interval', 30)\n\n def _build_iterator(self, now):\n if self.tc is None:\n raise RuntimeError(\n '{} - API Key or API Secret not set, '\n 'poll not performed'.format(self.name)\n )\n if self.last_successful_run is None:\n self.last_successful_run = utc_millisec() - self.initial_interval * 86400000.0\n if self.last_tc_run is None:\n self.last_tc_run = self.last_successful_run\n\n return self.tc.indicator_iterator(self.last_tc_run)\n\n def _process_item(self, item):\n tc_date_added = item[1].get('dateAdded', None)\n tc_last_modified = item[1].get('lastModified', None)\n f_seen = utc_millisec() if tc_date_added is None else dt_to_millisec(\n datetime.strptime(tc_date_added, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.utc))\n l_seen = utc_millisec() if tc_last_modified is None else dt_to_millisec(\n datetime.strptime(tc_last_modified, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.utc))\n if l_seen > self.last_tc_run:\n self.last_tc_run = l_seen\n if item[0] == \"IP\":\n return self.tc.ip_processing(item[1], item[2], f_seen, l_seen)\n if item[0] == \"GENERAL\":\n return self.tc.general_processing(item[1], item[2], f_seen, l_seen)\n return []\n\n\nclass GroupsMiner(TCMiner):\n groups = {}\n\n def configure(self):\n super(GroupsMiner, self).configure()\n groups = self.config.get('groups', None)\n if groups is not None and isinstance(groups, dict):\n for group_type in GROUP_TYPES:\n group_ids = groups.get(group_type, None)\n if group_ids is not None and isinstance(group_ids, list):\n self.groups[group_type] = group_ids\n\n def _build_iterator(self, now):\n if self.tc is None:\n raise RuntimeError(\n '{} - API Key or API Secret not set, '\n 'poll not performed'.format(self.name)\n )\n\n return self.tc.groups_iterator(self.groups)\n\n def _process_item(self, item):\n tc_date_added = item[0].get('dateAdded', None)\n tc_last_modified = item[0].get('lastModified', None)\n f_seen = utc_millisec() if tc_date_added is None else dt_to_millisec(\n datetime.strptime(tc_date_added, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.utc))\n l_seen = utc_millisec() if tc_last_modified is None else dt_to_millisec(\n datetime.strptime(tc_last_modified, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.utc))\n\n return [self.tc.group_indicator_processing(item[0], item[1], item[2], f_seen, l_seen)]\n" }, { "path": "minemeld/ft/threatq.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements minemeld.ft.threatq.Export, the Miner node for ThreatQ\nexport API.\n\"\"\"\n\nimport requests\nimport logging\nimport os\nimport yaml\nimport netaddr\n\nfrom . import basepoller\n\nLOG = logging.getLogger(__name__)\n\n\nclass Export(basepoller.BasePollerFT):\n \"\"\"Implements class for Miners of ThreatQ Export API.\n\n **Config parameters**\n :side_config (str): path to the side config file, defaults\n to CONFIGDIR/_side_config.yml\n :polling_timeout: timeout of the polling request in seconds.\n Default: 20\n\n **Side Config parameters**\n :url: URL of the feed.\n :polling_timeout: timeout of the polling request in seconds.\n Default: 20\n :verify_cert: boolean, if *true* feed HTTPS server certificate is\n verified. Default: *true*\n\n Example:\n Example side config in YAML::\n\n url: https://10.5.172.225/api/export/6e472a434efe34ceb5a99ff6c9a8124e/?token=xoZjB4ypoNQdnbQhVi0B\n verify_cert: false\n\n Args:\n name (str): node name, should be unique inside the graph\n chassis (object): parent chassis instance\n config (dict): node config.\n \"\"\"\n def configure(self):\n super(Export, self).configure()\n\n self.polling_timeout = self.config.get('polling_timeout', 20)\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.url = sconfig.get('url', None)\n if self.url is not None:\n LOG.info('%s - url set', self.name)\n\n self.verify_cert = sconfig.get('verify_cert', True)\n\n def _process_item(self, line):\n line = line.strip()\n if not line:\n return [[None, None]]\n\n itype, indicator = line.split(',', 1)\n\n attributes = {}\n if itype == 'IP Address':\n ipaddr = netaddr.IPAddress(indicator)\n\n if ipaddr.version == 4:\n attributes['type'] = 'IPv4'\n\n elif ipaddr.version == 6:\n attributes['type'] = 'IPv6'\n\n else:\n LOG.error(\n '%s - %s: unknown IP version %s',\n line,\n self.name,\n ipaddr.version\n )\n return [[None, None]]\n\n elif itype == 'CIDR Block':\n ipaddr = netaddr.IPNetwork(indicator)\n\n if ipaddr.version == 4:\n attributes['type'] = 'IPv4'\n\n elif ipaddr.version == 6:\n attributes['type'] = 'IPv6'\n\n else:\n LOG.error(\n '%s - %s: unknown IP version %s',\n line,\n self.name,\n ipaddr.version\n )\n return [[None, None]]\n\n elif itype == 'FQDN':\n attributes['type'] = 'domain'\n\n elif itype == 'URL':\n attributes['type'] = 'URL'\n\n else:\n LOG.error(\n '%s - unknown indicator type %s - ignored',\n self.name,\n itype\n )\n return [[None, None]]\n\n return [[indicator, attributes]]\n\n def _build_iterator(self, now):\n if self.url is None:\n raise RuntimeError(\n '%s - url not set, poll not performed' % self.name\n )\n\n rkwargs = dict(\n stream=True,\n verify=self.verify_cert,\n timeout=self.polling_timeout\n )\n\n r = requests.get(\n self.url,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.debug('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n result = r.iter_lines()\n\n return result\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(Export, self).hup(source=source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n" }, { "path": "minemeld/ft/tmt.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport requests\nimport os\nimport yaml\nimport itertools\nimport csv\nimport gevent\nimport shutil\n\nfrom . import basepoller\nfrom . import table\nfrom .utils import interval_in_sec\n\nLOG = logging.getLogger(__name__)\n\n\nclass DTIAPI(basepoller.BasePollerFT):\n _AGE_OUT_BASES = ['first_seen', 'last_seen', 'tmt_last_sample_timestamp']\n _DEFAULT_AGE_OUT_BASE = 'tmt_last_sample_timestamp'\n\n def __init__(self, name, chassis, config):\n self.ttable = None\n\n super(DTIAPI, self).__init__(name, chassis, config)\n\n def configure(self):\n super(DTIAPI, self).configure()\n\n self.polling_timeout = self.config.get('polling_timeout', 120)\n self.verify_cert = self.config.get('verify_cert', True)\n\n self.dialect = {\n 'delimiter': self.config.get('delimiter', ','),\n 'doublequote': self.config.get('doublequote', True),\n 'escapechar': self.config.get('escapechar', None),\n 'quotechar': self.config.get('quotechar', '\"'),\n 'skipinitialspace': self.config.get('skipinitialspace', False)\n }\n\n self.include_suspicious = self.config.get('include_suspicious', True)\n initial_interval = self.config.get('initial_interval', '2d')\n self.initial_interval = interval_in_sec(initial_interval)\n if self.initial_interval is None:\n LOG.error(\n '%s - wrong initial_interval format: %s',\n self.name, initial_interval\n )\n self.initial_interval = interval_in_sec('2d')\n\n self.source_name = 'themediatrust.dti'\n\n self.api_key = None\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.api_key = sconfig.get('api_key', None)\n if self.api_key is not None:\n LOG.info('%s - authorization code set', self.name)\n\n def _process_row(self, row):\n ip = row.pop('ip_addres', None)\n if ip == '0.0.0.0':\n ip = None\n\n domain = row.pop('host_name', None)\n\n value = {}\n for k, v in row.iteritems():\n if k == 'last_sample_timestamp':\n value['tmt_last_sample_timestamp'] = int(v)*1000\n continue\n\n key = k\n if not k.startswith('tmt'):\n key = 'tmt_%s' % k\n\n value[key] = [v]\n\n return ip, domain, value\n\n def _process_item(self, item):\n type_, indicator = item[0].split(':', 1)\n\n value = {}\n for k, v in item[1].iteritems():\n value[k] = v\n value['type'] = type_\n\n return [[indicator, value]]\n\n def _tmerge(self, indicator, value):\n ov = self.ttable.get(indicator)\n\n if ov is None:\n self.ttable.put(indicator, value)\n return\n\n for k, v in value.iteritems():\n if k == 'tmt_last_sample_timestamp':\n if v > ov[k]: # confusing, this is just for PEP8 sake\n ov[k] = v\n continue\n\n if v[0] not in ov[k]:\n ov[k].append(v)\n\n self.ttable.put(indicator, ov)\n\n def _build_iterator(self, now):\n if self.api_key is None:\n raise RuntimeError('%s - api_key not set' % self.name)\n\n if self.ttable is not None:\n self.ttable.close()\n self.ttable = None\n\n self.ttable = table.Table(self.name+'_temp', truncate=True)\n\n last_fetch = self.last_run\n if last_fetch is None:\n last_fetch = int(now/1000) - self.initial_interval\n\n params = dict(\n key=self.api_key,\n action='fjord_base',\n include_suspicious=(1 if self.include_suspicious else 0),\n last_fetch=last_fetch\n )\n\n rkwargs = dict(\n stream=True,\n verify=self.verify_cert,\n timeout=self.polling_timeout,\n params=params\n )\n\n r = requests.get(\n 'https://www.themediatrust.com/api',\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.debug('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n response = itertools.ifilter(\n lambda x: not x.startswith('got commandoptions'),\n r.raw\n )\n\n csvreader = csv.DictReader(\n response,\n **self.dialect\n )\n\n for row in csvreader:\n gevent.sleep(0)\n ip, domain, value = self._process_row(row)\n if ip is None and domain is None:\n continue\n\n if ip is not None:\n self._tmerge('IPv4:%s' % ip, value)\n\n if domain is not None:\n self._tmerge('domain:%s' % domain, value)\n\n return self.ttable.query(include_value=True)\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(DTIAPI, self).hup(source=source)\n\n @staticmethod\n def gc(name, config=None):\n basepoller.BasePollerFT.gc(name, config=config)\n\n shutil.rmtree('{}_temp'.format(name), ignore_errors=True)\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n" }, { "path": "minemeld/ft/utils.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport time\nimport operator\nimport functools\nimport datetime\nimport pytz\nimport re\n\nimport gevent\nimport gevent.lock\nimport gevent.event\n\n\nEPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.UTC)\n\n\ndef utc_millisec():\n return int(time.time()*1000)\n\n\ndef dt_to_millisec(dt):\n if dt.tzinfo == None:\n dt = dt.replace(tzinfo=pytz.UTC)\n delta = dt - EPOCH\n return int(delta.total_seconds()*1000)\n\n\ndef interval_in_sec(val):\n if isinstance(val, int):\n return val\n\n multipliers = {\n '': 1,\n 'm': 60,\n 'h': 3600,\n 'd': 86400\n }\n\n mo = re.match(\"([0-9]+)([dmh]?)\", val)\n if mo is None:\n return None\n\n return int(mo.group(1))*multipliers[mo.group(2)]\n\n\ndef age_out_in_millisec(val):\n multipliers = {\n '': 1000,\n 'm': 60000,\n 'h': 3600000,\n 'd': 86400000\n }\n\n mo = re.match(\"([0-9]+)([dmh]?)\", val)\n if mo is None:\n return None\n\n return int(mo.group(1))*multipliers[mo.group(2)]\n\n\ndef _merge_atomic_values(op, v1, v2):\n if op(v1, v2):\n return v2\n return v1\n\n\ndef _merge_array(v1, v2):\n for e in v2:\n if e not in v1:\n v1.append(e)\n return v1\n\n\nRESERVED_ATTRIBUTES = {\n 'sources': _merge_array,\n 'first_seen': functools.partial(_merge_atomic_values, operator.gt),\n 'last_seen': functools.partial(_merge_atomic_values, operator.lt),\n 'type': functools.partial(_merge_atomic_values, operator.eq),\n 'direction': functools.partial(_merge_atomic_values, operator.eq),\n 'confidence': functools.partial(_merge_atomic_values, operator.lt),\n 'country': functools.partial(_merge_atomic_values, operator.eq),\n 'AS': functools.partial(_merge_atomic_values, operator.eq)\n}\n\n\nclass RWLock(object):\n def __init__(self):\n self.num_readers = 0\n self.num_writers = 0\n\n self.m1 = gevent.lock.Semaphore(1)\n self.m2 = gevent.lock.Semaphore(1)\n self.m3 = gevent.lock.Semaphore(1)\n self.w = gevent.lock.Semaphore(1)\n self.r = gevent.lock.Semaphore(1)\n\n def lock(self):\n self.m2.acquire()\n\n self.num_writers += 1\n if self.num_writers == 1:\n self.r.acquire()\n\n self.m2.release()\n self.w.acquire()\n\n def unlock(self):\n self.w.release()\n self.m2.acquire()\n\n self.num_writers -= 1\n if self.num_writers == 0:\n self.r.release()\n\n self.m2.release()\n\n def rlock(self):\n self.m3.acquire()\n self.r.acquire()\n self.m1.acquire()\n\n self.num_readers += 1\n if self.num_readers == 1:\n self.w.acquire()\n\n self.m1.release()\n self.r.release()\n self.m3.release()\n\n def runlock(self):\n self.m1.acquire()\n\n self.num_readers -= 1\n if self.num_readers == 0:\n self.w.release()\n\n self.m1.release()\n\n def __enter__(self):\n self.rlock()\n\n def __exit__(self, type, value, traceback):\n self.runlock()\n\n\n_AGE_OUT_BASES = ['last_seen', 'first_seen']\n\n\ndef parse_age_out(s, age_out_bases=None, default_base=None):\n if s is None:\n return None\n\n if age_out_bases is None:\n age_out_bases = _AGE_OUT_BASES\n\n if default_base is None:\n default_base = 'first_seen'\n\n if default_base not in age_out_bases:\n raise ValueError('%s not in %s' % (default_base, age_out_bases))\n\n result = {}\n\n toks = s.split('+', 1)\n if len(toks) == 1:\n t = toks[0].strip()\n if t in age_out_bases:\n result['base'] = t\n result['offset'] = 0\n else:\n result['base'] = default_base\n result['offset'] = age_out_in_millisec(t)\n if result['offset'] is None:\n raise ValueError('Invalid age out offset %s' % t)\n else:\n base = toks[0].strip()\n if base not in age_out_bases:\n raise ValueError('Invalid age out base %s' % base)\n result['base'] = base\n result['offset'] = age_out_in_millisec(toks[1].strip())\n if result['offset'] is None:\n raise ValueError('Invalid age out offset %s' % t)\n\n return result\n\n\nclass GThrottled(object):\n def __init__(self, f, wait):\n self._timeout = None\n self._previous = 0\n self._cancelled = False\n\n self._args = []\n self._kwargs = {}\n\n self.f = f\n self.wait = wait\n\n def later(self):\n self._previous = utc_millisec()\n self._timeout = None\n\n self.f(*self._args, **self._kwargs)\n\n def __call__(self, *args, **kwargs):\n now = utc_millisec()\n remaining = self.wait - (now - self._previous)\n\n if self._cancelled:\n return\n\n if remaining <= 0 or remaining > self.wait:\n if self._timeout is not None:\n self._timeout.join(timeout=5)\n self._timeout = None\n self._previous = now\n self.f(*args, **kwargs)\n\n elif self._timeout is None:\n self._args = args\n self._kwargs = kwargs\n self._timeout = gevent.spawn_later(remaining/1000.0, self.later)\n\n else:\n self._args = args\n self._kwargs = kwargs\n\n def cancel(self):\n self._cancelled = True\n if self._timeout:\n self._timeout.join(timeout=5)\n if self._timeout is not None:\n self._timeout.kill()\n self._previous = 0\n self._timeout = None\n self._args = []\n self._kwargs = {}\n" }, { "path": "minemeld/ft/visa.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements minemeld.ft.visa.VTI, the Miner node for\nVisa Threat Intelligence API.\n\"\"\"\nimport logging\nimport requests\nimport re\n\nfrom . import json\nfrom utils import utc_millisec, dt_to_millisec\nfrom datetime import datetime\nfrom netaddr import IPNetwork, AddrFormatError\n\nLOG = logging.getLogger(__name__)\nVTI_INDICATOR_TYPES = {'Hash': 'HASH',\n 'IP': 'IP',\n 'Email': 'email-addr',\n 'URL': 'URL',\n 'FQDN': 'domain'}\nVTI_VICTIM_TYPES = ('Restaurant', 'Retail', 'Hospitality and Lodging', 'QSR',\n 'B2B', 'Supermarket', 'POS Integrator', 'Financial Institution',\n 'Cinema', 'Parking', 'Pharmacy', 'Telecommunications', 'Other Retail')\nSHA256_PATTERN = \"[A-Fa-f0-9]{64}\"\nSHA1_PATTERN = \"[A-Fa-f0-9]{40}\"\nMD5_PATTERN = \"[A-Fa-f0-9]{32}\"\n\n\nclass VTI(json.SimpleJSON):\n initial_interval = None\n indicator_type = None\n victim_type = None\n hash_patterns = {\"sha256\": re.compile(SHA256_PATTERN), \"sha1\": re.compile(SHA1_PATTERN),\n \"md5\": re.compile(MD5_PATTERN)}\n\n def configure(self):\n super(VTI, self).configure()\n self.initial_interval = self.config.get('initial_interval', '30')\n self.indicator_type = self.config.get('indicator_type', None)\n if self.indicator_type is not None and self.indicator_type not in VTI_INDICATOR_TYPES:\n self.indicator_type = None\n self.victim_type = self.config.get('victim_type', None)\n if self.victim_type is not None and self.victim_type not in VTI_VICTIM_TYPES:\n self.victim_type = None\n\n def _process_item(self, item):\n if self.indicator not in item:\n LOG.debug('%s not in %s', self.indicator, item)\n return [[None, None]]\n\n indicator = item[self.indicator]\n if not (isinstance(indicator, str) or\n isinstance(indicator, unicode)):\n LOG.error(\n 'Wrong indicator type: %s - %s',\n indicator, type(indicator)\n )\n return [[None, None]]\n\n indicator_type = item.get('indicatorType', None)\n if indicator_type is not None:\n indicator_type = VTI_INDICATOR_TYPES.get(indicator_type, None)\n if indicator_type == 'HASH':\n indicator_type = self._detect_sha_version(indicator)\n if indicator_type == 'IP':\n indicator_type = self._detect_ip_version(indicator)\n\n upload_date = item.get('uploadDate', None)\n if upload_date is None:\n upload_date = utc_millisec()\n else:\n try:\n dt = datetime.strptime(upload_date, '%Y-%m-%d')\n upload_date = dt_to_millisec(dt)\n except ValueError:\n upload_date = utc_millisec()\n if upload_date > self.last_vti_run:\n self.last_vti_run = upload_date\n\n fields = self.fields\n if fields is None:\n fields = item.keys()\n fields.remove(self.indicator)\n\n if 'indicatorType' in fields:\n fields.remove('indicatorType')\n if 'uploadDate' in fields:\n fields.remove('uploadDate')\n\n attributes = {'type': indicator_type, 'first_seen': upload_date, 'last_seen': upload_date}\n for field in fields:\n if field not in item:\n continue\n attributes['%s_%s' % (self.prefix, field)] = item[field]\n\n return [[indicator, attributes]]\n\n def _build_iterator(self, now):\n rkwargs = dict(\n stream=True,\n verify=self.verify_cert,\n timeout=self.polling_timeout\n )\n\n if self.headers is not None:\n rkwargs['headers'] = self.headers\n\n if self.username is not None and self.password is not None:\n rkwargs['auth'] = (self.username, self.password)\n else:\n raise RuntimeError('%s - credentials not set' % self.name)\n\n if self.client_cert_required and self.key_file is not None and self.cert_file is not None:\n rkwargs['cert'] = (self.cert_file, self.key_file)\n else:\n raise RuntimeError('%s - client certificate/key not set' % self.name)\n\n if self.last_successful_run is None:\n self.last_successful_run = utc_millisec() - self.initial_interval * 86400000.0\n if self.last_vti_run is None:\n self.last_vti_run = self.last_successful_run\n\n start_date = datetime.fromtimestamp(self.last_vti_run / 1000)\n end_date = datetime.fromtimestamp(utc_millisec() / 1000)\n\n payload = {'startDate': start_date.strftime('%Y-%m-%d'),\n 'endDate': end_date.strftime('%Y-%m-%d')}\n\n if self.indicator_type is not None:\n payload['indicatorType'] = self.indicator_type\n\n if self.victim_type is not None:\n payload['victimType'] = self.victim_type\n\n r = requests.get(\n self.url,\n params=payload,\n **rkwargs\n )\n\n try:\n r.raise_for_status()\n except:\n LOG.debug('%s - exception in request: %s %s',\n self.name, r.status_code, r.content)\n raise\n\n result = self.extractor.search(r.json())\n\n if result is None:\n result = []\n\n return result\n\n def _detect_ip_version(self, ip_addr):\n try:\n parsed = IPNetwork(ip_addr)\n except (AddrFormatError, ValueError):\n LOG.error('{} - Unknown IP version: {}'.format(self.name, ip_addr))\n return None\n\n if parsed.version == 4:\n return 'IPv4'\n\n if parsed.version == 6:\n return 'IPv6'\n\n return None\n\n def _detect_sha_version(self, hash_value):\n for hash_type, re_obj in self.hash_patterns.iteritems():\n if re_obj.match(hash_value) is not None:\n return hash_type\n return None\n\n def _saved_state_restore(self, saved_state):\n super(VTI, self)._saved_state_restore(saved_state)\n self.last_vti_run = saved_state.get('last_vti_run', None)\n LOG.info('last_vti_run from sstate: %s', self.last_vti_run)\n\n def _saved_state_create(self):\n sstate = super(VTI, self)._saved_state_create()\n sstate['last_vti_run'] = self.last_vti_run\n return sstate\n\n def _saved_state_reset(self):\n super(VTI, self)._saved_state_reset()\n self.last_vti_run = None\n" }, { "path": "minemeld/ft/vt.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements:\n- minemeld.ft.vt.Notifications, the Miner node for VirusTotal Notifications\n feed\n\"\"\"\n\nimport logging\nimport os\nimport yaml\n\nfrom . import json\n\nLOG = logging.getLogger(__name__)\n\n_VT_NOTIFICATIONS = 'https://www.virustotal.com/intelligence/hunting/notifications-feed/?key='\n\n\nclass Notifications(json.SimpleJSON):\n def __init__(self, name, chassis, config):\n super(Notifications, self).__init__(name, chassis, config)\n\n self.api_key = None\n\n def configure(self):\n self.config['url'] = None\n self.config['extractor'] = 'notifications'\n self.config['prefix'] = 'vt'\n\n super(Notifications, self).configure()\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n\n self._load_side_config()\n\n def _load_side_config(self):\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.api_key = sconfig.get('api_key', None)\n if self.api_key is not None:\n LOG.info('%s - api key set', self.name)\n self.url = _VT_NOTIFICATIONS + self.api_key\n\n def _process_item(self, item):\n result = []\n\n for htype in ['md5', 'sha256', 'sha1']:\n value = {self.prefix+'_'+k: v for k, v in item.iteritems()}\n indicator = value.pop(self.prefix+'_'+htype, None)\n value['type'] = htype\n\n if indicator is not None:\n result.append([indicator, value])\n\n return result\n\n def _build_iterator(self, now):\n if self.api_key is None:\n LOG.info('%s - API key not set', self.name)\n raise RuntimeError(\n '%s - API Key not set' % self.name\n )\n\n return super(Notifications, self)._build_iterator(now)\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n super(Notifications, self).hup(source=source)\n\n @staticmethod\n def gc(name, config=None):\n json.SimpleJSON.gc(name, config=config)\n\n side_config_path = None\n if config is not None:\n side_config_path = config.get('side_config', None)\n if side_config_path is None:\n side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '{}_side_config.yml'.format(name)\n )\n\n try:\n os.remove(side_config_path)\n except:\n pass\n" }, { "path": "minemeld/ft/xmpp.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import absolute_import\n\nimport logging\nimport ujson\nimport random\nimport gevent\nimport gevent.event\nimport gevent.queue\nimport yaml\nimport os\nimport sleekxmpp\nimport sleekxmpp.xmlstream\n\nfrom . import base\nfrom . import op\n\nLOG = logging.getLogger(__name__)\n\n\nclass XMPPOutput(base.BaseFT):\n def __init__(self, name, chassis, config):\n super(XMPPOutput, self).__init__(name, chassis, config)\n\n self._xmpp_client = None\n self._xmpp_glet = None\n self._publisher_glet = None\n\n self.q = gevent.queue.Queue()\n\n self._read_sequence_number()\n\n self._load_event = gevent.event.Event()\n self._xmpp_client_ready = gevent.event.Event()\n\n def configure(self):\n super(XMPPOutput, self).configure()\n\n self.server = self.config.get('server', None)\n self.port = self.config.get('port', 5222)\n self.pubsub_service = self.config.get('pubsub_service', None)\n self.node = self.config.get('node', None)\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n self._load_side_config()\n\n def _load_side_config(self):\n self.jid = None\n self.password = None\n\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.jid = sconfig.get('jid', None)\n if self.jid is not None:\n LOG.info('%s - jid set', self.name)\n\n self.password = sconfig.get('password', None)\n if self.password is not None:\n LOG.info('%s - password set', self.name)\n\n def connect(self, inputs, output):\n output = False\n super(XMPPOutput, self).connect(inputs, output)\n\n def initialize(self):\n pass\n\n def rebuild(self):\n self.sequence_number = None\n\n def reset(self):\n self.sequence_number = None\n\n def _read_sequence_number(self):\n self.sequence_number = None\n\n try:\n with open(self.name+'.seqn', 'r') as f:\n self.sequence_number = int(f.read().strip())\n os.remove(self.name+'.seqn')\n except IOError:\n pass\n\n def _write_sequence_number(self):\n if self.sequence_number is None:\n return\n\n with open(self.name+'.seqn', 'w') as f:\n f.write('%s' % self.sequence_number)\n\n @base._counting('update.processed')\n def filtered_update(self, source=None, indicator=None, value=None):\n self.q.put(['UPDATE', indicator, value])\n\n @base._counting('withdraw.processed')\n def filtered_withdraw(self, source=None, indicator=None, value=None):\n self.q.put(['WITHDRAW', indicator, value])\n\n def _xmpp_publish(self, cmd, data=None):\n if data is None:\n data = ''\n\n payload_xml = sleekxmpp.xmlstream.ET.Element('mm-command')\n command_xml = sleekxmpp.xmlstream.ET.SubElement(payload_xml, 'command')\n command_xml.text = cmd\n seqno_xml = sleekxmpp.xmlstream.ET.SubElement(payload_xml, 'seqno')\n seqno_xml.text = '%s' % self.sequence_number\n data_xml = sleekxmpp.xmlstream.ET.SubElement(payload_xml, 'data')\n data_xml.text = ujson.dumps(data)\n\n result = self._xmpp_client['xep_0060'].publish(\n self.pubsub_service,\n self.node,\n payload=payload_xml\n )\n LOG.debug('%s - xmpp publish: %s', self.name, result)\n\n self.sequence_number += 1\n\n self.statistics['xmpp.published'] += 1\n\n def _xmpp_session_start(self, event):\n LOG.debug('%s - _xmpp_session_start', self.name)\n self._xmpp_client.get_roster()\n self._xmpp_client.send_presence()\n\n if self.sequence_number is None:\n self.sequence_number = random.getrandbits(64)\n self._xmpp_publish('INIT')\n\n self._xmpp_client_ready.set()\n\n def _xmpp_disconnected(self, event):\n LOG.debug('%s - _xmpp_disconnected', self.name)\n self._xmpp_client_ready.clear()\n\n def _start_xmpp_client(self):\n if self._xmpp_client is not None:\n return\n\n if self.jid is None or self.password is None:\n raise RuntimeError('%s - jid or password not set', self.name)\n\n if self.server is None or self.port is None:\n raise RuntimeError('%s - server or port not set', self.name)\n\n if self.node is None or self.pubsub_service is None:\n raise RuntimeError(\n '%s - node or pubsub_service not set',\n self.name\n )\n\n self._xmpp_client = sleekxmpp.ClientXMPP(\n jid=self.jid,\n password=self.password\n )\n self._xmpp_client.register_plugin('xep_0030')\n self._xmpp_client.register_plugin('xep_0059')\n self._xmpp_client.register_plugin('xep_0060')\n self._xmpp_client.add_event_handler(\n 'session_start',\n self._xmpp_session_start\n )\n self._xmpp_client.add_event_handler(\n 'disconnected',\n self._xmpp_disconnected\n )\n\n if not self._xmpp_client.connect((self.server, self.port)):\n raise RuntimeError(\n '%s - error connecting to XMPP server',\n self.name\n )\n\n self._xmpp_client.process(block=True)\n\n def _publisher(self):\n while True:\n self._xmpp_client_ready.wait()\n\n try:\n while True:\n cmd, indicator, value = self.q.peek()\n if value is None:\n value = {}\n value['origins'] = [self.jid]\n self._xmpp_publish(cmd, {\n 'indicator': indicator,\n 'value': value\n })\n _ = self.q.get()\n\n except gevent.GreenletExit:\n break\n\n except Exception as e:\n LOG.exception('%s - Exception in publishing message', self.name)\n gevent.sleep(30)\n self.statistics['xmpp.publish_error'] += 1\n\n def _run(self):\n while True:\n try:\n self._start_xmpp_client()\n\n except RuntimeError() as e:\n LOG.error('%s - %s', self.name, str(e))\n self.statistics['xmpp.error'] += 1\n\n except gevent.GreenletExit:\n if self._xmpp_client is not None:\n self._xmpp_client.disconnect()\n break\n\n except Exception as e:\n LOG.exception('%s - error in starting XMPP client', self.name)\n\n try:\n if self._xmpp_client is not None:\n self._xmpp_client.disconnect()\n self._xmpp_client = None\n\n hup_called = self._load_event.wait(timeout=60)\n if hup_called:\n LOG.debug('%s - clearing load event', self.name)\n self._load_event.clear()\n\n except gevent.GreenletExit:\n break\n\n def length(self, source=None):\n return 0\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n self._load_event.set()\n\n def mgmtbus_checkpoint(self, value=None):\n self._write_sequence_number()\n return super(XMPPOutput, self).mgmtbus_checkpoint(value=value)\n\n def start(self):\n super(XMPPOutput, self).start()\n\n if self._xmpp_glet is not None:\n return\n self._xmpp_glet = gevent.spawn_later(random.randint(0, 2), self._run)\n self._publisher_glet = gevent.spawn_later(random.randint(0, 3), self._publisher)\n\n def stop(self):\n super(XMPPOutput, self).stop()\n\n if self._xmpp_client is None:\n return\n\n self._xmpp_glet.kill()\n self._publisher_glet.kill()\n\n\nclass XMPPMiner(op.AggregateFT):\n def __init__(self, name, chassis, config):\n super(XMPPMiner, self).__init__(name, chassis, config)\n\n self._xmpp_client = None\n self._xmpp_glet = None\n\n self._load_event = gevent.event.Event()\n\n def configure(self):\n super(XMPPMiner, self).configure()\n\n self.server = self.config.get('server', None)\n self.port = self.config.get('port', 5222)\n self.pubsub_service = self.config.get('pubsub_service', None)\n self.node = self.config.get('node', None)\n\n self.side_config_path = self.config.get('side_config', None)\n if self.side_config_path is None:\n self.side_config_path = os.path.join(\n os.environ['MM_CONFIG_DIR'],\n '%s_side_config.yml' % self.name\n )\n self._load_side_config()\n\n def _load_side_config(self):\n self.jid = None\n self.password = None\n\n try:\n with open(self.side_config_path, 'r') as f:\n sconfig = yaml.safe_load(f)\n\n except Exception as e:\n LOG.error('%s - Error loading side config: %s', self.name, str(e))\n return\n\n self.jid = sconfig.get('jid', None)\n if self.jid is not None:\n LOG.info('%s - jid set', self.name)\n\n self.password = sconfig.get('password', None)\n if self.password is not None:\n LOG.info('%s - password set', self.name)\n\n def _xmpp_session_start(self, event):\n LOG.debug('%s - _xmpp_session_start', self.name)\n self._xmpp_client.get_roster()\n self._xmpp_client.send_presence()\n\n result = self._xmpp_client['xep_0060'].subscribe(\n self.pubsub_service,\n self.node\n )\n LOG.debug('%s - subscribe result: %s', self.name, result)\n\n def _xmpp_publish(self, msg):\n LOG.debug('%s - _publish %s', self.name, msg)\n\n node_ = msg['pubsub_event']['items']['node']\n if node_ != self.node:\n return\n\n payload = msg['pubsub_event']['items']['item']['payload']\n if payload is None:\n return\n\n command = payload.find('{http://jabber.org/protocol/pubsub#event}command')\n if command is None:\n LOG.error(\n '%s - pubsub event received with no commands',\n self.name\n )\n return\n\n command = command.text\n if command == 'INIT':\n return\n\n data = payload.find('{http://jabber.org/protocol/pubsub#event}data')\n if data is None:\n LOG.error(\n '%s - pubsub event received with no data',\n self.name\n )\n return\n\n data = data.text\n data = ujson.loads(data)\n\n indicator = data.get('indicator', None)\n if indicator is None:\n LOG.error('%s - received command with no indicator', self.name)\n return\n\n value = data.get('value', None)\n if value is None:\n LOG.error('%s - received command with no value', self.name)\n return\n\n origins = value.get('origins', None)\n if origins is None:\n LOG.error('%s - received indicator with no origin', self.name)\n return\n\n if self.jid in origins:\n LOG.debug('%s - indicator already known, ignored', self.name)\n return\n\n if command == 'UPDATE':\n for o in origins:\n if o not in self.inputs:\n self.inputs.append(o)\n\n self.update(\n source=o,\n indicator=indicator,\n value=value\n )\n\n elif command == 'WITHDRAW':\n for o in origins:\n if o not in self.inputs:\n self.inputs.append(o)\n\n self.withdraw(\n source=o,\n indicator=indicator,\n value=value\n )\n\n else:\n LOG.error('%s - unknown command %s', self.name, command)\n\n def _start_xmpp_client(self):\n if self._xmpp_client is not None:\n return\n\n if self.jid is None or self.password is None:\n raise RuntimeError('%s - jid or password not set', self.name)\n\n if self.server is None or self.port is None:\n raise RuntimeError('%s - server or port not set', self.name)\n\n if self.node is None or self.pubsub_service is None:\n raise RuntimeError(\n '%s - node or pubsub_service not set',\n self.name\n )\n\n self._xmpp_client = sleekxmpp.ClientXMPP(\n jid=self.jid,\n password=self.password\n )\n self._xmpp_client.register_plugin('xep_0030')\n self._xmpp_client.register_plugin('xep_0059')\n self._xmpp_client.register_plugin('xep_0060')\n self._xmpp_client.add_event_handler(\n 'session_start',\n self._xmpp_session_start\n )\n self._xmpp_client.add_event_handler(\n 'pubsub_publish',\n self._xmpp_publish\n )\n\n if not self._xmpp_client.connect((self.server, self.port)):\n raise RuntimeError(\n '%s - error connecting to XMPP server',\n self.name\n )\n\n self._xmpp_client.process(block=True)\n\n def _run(self):\n while True:\n try:\n self._start_xmpp_client()\n\n except RuntimeError() as e:\n LOG.error('%s - %s', self.name, str(e))\n self.statistics['xmpp.error'] += 1\n\n except gevent.GreenletExit:\n if self._xmpp_client is not None:\n self._xmpp_client.disconnect()\n break\n\n except Exception as e:\n LOG.exception('%s - error in starting XMPP client', self.name)\n\n try:\n if self._xmpp_client is not None:\n self._xmpp_client.disconnect()\n self._xmpp_client = None\n\n hup_called = self._load_event.wait(timeout=60)\n if hup_called:\n LOG.debug('%s - clearing load event', self.name)\n self._load_event.clear()\n\n except gevent.GreenletExit:\n break \n\n def start(self):\n super(XMPPMiner, self).start()\n\n if self._xmpp_glet is not None:\n return\n self._xmpp_glet = gevent.spawn_later(random.randint(0, 2), self._run)\n\n def stop(self):\n super(XMPPMiner, self).stop()\n\n if self._xmpp_client is None:\n return\n\n self._xmpp_glet.kill()\n\n def hup(self, source=None):\n LOG.info('%s - hup received, reload side config', self.name)\n self._load_side_config()\n self._load_event.set()\n" }, { "path": "minemeld/loader.py", "content": "import logging\n\nimport pip\n\nfrom pkg_resources import working_set, WorkingSet\nfrom collections import namedtuple\n\nLOG = logging.getLogger(__name__)\n\nMM_NODES_ENTRYPOINT = 'minemeld_nodes'\nMM_NODES_GCS_ENTRYPOINT = 'minemeld_nodes_gcs'\nMM_NODES_VALIDATORS_ENTRYPOINT = 'minemeld_nodes_validators'\nMM_PROTOTYPES_ENTRYPOINT = 'minemeld_prototypes'\nMM_API_ENTRYPOINT = 'minemeld_api'\nMM_WEBUI_ENTRYPOINT = 'minemeld_webui'\n\nMMEntryPoint = namedtuple(\n 'MMEntryPoint',\n ['ep', 'name', 'loadable', 'conflicts']\n)\n\n_ENTRYPOINT_GROUPS = {}\n\n_WS = None\n\n\ndef _conflicts(requirements, installed):\n result = []\n\n for r in requirements:\n installed_dist = installed.get(r.project_name, None)\n if installed_dist is None:\n result.append('{} not installed'.format(r.project_name))\n continue\n\n if installed_dist.version not in r:\n result.append('{}=={} not compatible with {}'.format(\n installed_dist.project_name,\n installed_dist.version,\n str(r)\n ))\n\n return result\n\n\ndef _initialize_entry_point_group(entrypoint_group):\n global _WS\n\n installed = {d.project_name: d for d in working_set}\n\n if _WS is None:\n _WS = WorkingSet()\n\n cache = {}\n result = {}\n for ep in _WS.iter_entry_points(entrypoint_group):\n egg_name = ep.dist.egg_name()\n conflicts = cache.get(egg_name, None)\n if conflicts is None:\n conflicts = _conflicts(\n ep.dist.requires(),\n installed\n )\n cache[egg_name] = conflicts\n\n if len(conflicts) != 0:\n LOG.error('{} not loadable: {}'.format(\n ep.name,\n ', '.join(conflicts)\n ))\n result[ep.name] = MMEntryPoint(\n ep=ep,\n name=ep.name,\n conflicts=conflicts,\n loadable=(len(conflicts) == 0)\n )\n\n _ENTRYPOINT_GROUPS[entrypoint_group] = result\n\n\ndef bump_workingset():\n global _WS, _ENTRYPOINT_GROUPS\n\n _WS = None\n _ENTRYPOINT_GROUPS = {}\n\n\ndef list(entrypoint_group):\n if entrypoint_group not in _ENTRYPOINT_GROUPS:\n _initialize_entry_point_group(entrypoint_group)\n eg = _ENTRYPOINT_GROUPS[entrypoint_group]\n\n return eg.keys()\n\n\ndef map(entrypoint_group):\n if entrypoint_group not in _ENTRYPOINT_GROUPS:\n _initialize_entry_point_group(entrypoint_group)\n eg = _ENTRYPOINT_GROUPS[entrypoint_group]\n\n return eg\n\n\ndef load(entrypoint_group, entrypoint_name):\n LOG.info('Loading %s:%s', entrypoint_group, entrypoint_name)\n if entrypoint_group not in _ENTRYPOINT_GROUPS:\n _initialize_entry_point_group(entrypoint_group)\n eg = _ENTRYPOINT_GROUPS[entrypoint_group]\n\n mmep = eg.get(entrypoint_name, None)\n if mmep is None:\n raise RuntimeError('Unknown entry point: {}:{}'.format(entrypoint_group, entrypoint_name))\n\n if not mmep.loadable:\n raise RuntimeError('Entry point {}:{} not loadable: {}'.format(\n entrypoint_group,\n entrypoint_name,\n ', '.join(mmep.conflicts)\n ))\n\n return mmep.ep.load()\n" }, { "path": "minemeld/mgmtbus.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements master and slave hub classes for MineMeld engine\nmanagement bus.\n\nManagement bus master sends commands to all managemnt bus slaves by\nposting a message to a specific topic (MGMTBUS_PREFIX+'bus').\nSlaves subscribe to the topic, and when a command is received they\nreply back to the master by sending the answer to the queue\nMGMTBUS_PREFIX+'master'. Slaves connections are multiplexed via\nslave hub class.\n\nManagement bus is used to control the MineMeld engine graph and to\nperiodically retrieve metrics from all the nodes.\n\"\"\"\n\nfrom __future__ import absolute_import\n\nimport logging\nimport uuid\nimport collections\nimport time\nimport hashlib\nimport os\n\nimport gevent\nimport gevent.event\nimport gevent.lock\nimport gevent.timeout\n\nimport redis\nimport ujson\n\nimport minemeld.comm\nimport minemeld.ft\n\nfrom .collectd import CollectdClient\nfrom .startupplanner import plan\n\nLOG = logging.getLogger(__name__)\n\nMGMTBUS_PREFIX = \"mbus:\"\nMGMTBUS_TOPIC = MGMTBUS_PREFIX+'bus'\nMGMTBUS_CHASSIS_TOPIC = MGMTBUS_PREFIX+'chassisbus'\nMGMTBUS_MASTER = '@'+MGMTBUS_PREFIX+'master'\nMGMTBUS_LOG_TOPIC = MGMTBUS_PREFIX+'log'\nMGMTBUS_STATUS_TOPIC = MGMTBUS_PREFIX+'status'\n\n\nclass MgmtbusMaster(object):\n \"\"\"MineMeld engine management bus master\n\n Args:\n ftlist (list): list of nodes\n config (dict): config\n comm_class (string): communication backend to be used\n comm_config (dict): config for the communication backend\n \"\"\"\n def __init__(self, ftlist, config, comm_class, comm_config, num_chassis):\n super(MgmtbusMaster, self).__init__()\n\n self.ftlist = ftlist\n self.config = config\n self.comm_config = comm_config\n self.comm_class = comm_class\n self.num_chassis = num_chassis\n\n self._chassis = []\n self._all_chassis_ready = gevent.event.Event()\n\n self.graph_status = None\n\n self._start_timestamp = int(time.time())*1000\n self._status_lock = gevent.lock.Semaphore()\n self.status_glet = None\n self._status = {}\n\n self.SR = redis.StrictRedis.from_url(\n os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock')\n )\n\n self.comm = minemeld.comm.factory(self.comm_class, self.comm_config)\n self._out_channel = self.comm.request_pub_channel(MGMTBUS_TOPIC)\n self.comm.request_rpc_server_channel(\n name=MGMTBUS_MASTER,\n obj=self,\n allowed_methods=['rpc_status', 'rpc_chassis_ready'],\n method_prefix='rpc_'\n )\n self._slaves_rpc_client = self.comm.request_rpc_fanout_client_channel(\n MGMTBUS_TOPIC\n )\n self._chassis_rpc_client = self.comm.request_rpc_fanout_client_channel(\n MGMTBUS_CHASSIS_TOPIC\n )\n self.comm.request_rpc_server_channel(\n name=MGMTBUS_STATUS_TOPIC,\n obj=self,\n allowed_methods=['status']\n )\n\n def rpc_status(self):\n \"\"\"Returns collected status via RPC\n \"\"\"\n return self._status\n\n def rpc_chassis_ready(self, chassis_id=None):\n \"\"\"Chassis signal ready state via this RPC\n \"\"\"\n if chassis_id in self._chassis:\n LOG.error('duplicate chassis_id received in rpc_chassis_ready')\n return 'ok'\n\n self._chassis.append(chassis_id)\n if len(self._chassis) == self.num_chassis:\n self._all_chassis_ready.set()\n\n return 'ok'\n\n def wait_for_chassis(self, timeout=60):\n \"\"\"Wait for all the chassis signal ready state\n \"\"\"\n if self.num_chassis == 0: # empty config\n return\n\n if not self._all_chassis_ready.wait(timeout=timeout):\n raise RuntimeError('Timeout waiting for chassis')\n\n def start_chassis(self):\n self._send_cmd_and_wait(\n 'start',\n to_slaves=False, # chassis\n timeout=60\n )\n\n def _send_cmd(self, command, to_slaves=True, params=None, and_discard=False):\n \"\"\"Sends command to slaves or chassis over mgmt bus.\n\n Args:\n command (str): command\n params (dict): params of the command\n and_discard (bool): discard answer, don't wait\n to_slaves (bool): send command to nodes, otherwise to chassis\n\n Returns:\n returns a gevent.event.AsyncResult that is signaled\n when all the answers are collected\n \"\"\"\n if params is None:\n params = {}\n\n rpc_client = self._slaves_rpc_client\n num_results = len(self.ftlist)\n if not to_slaves:\n rpc_client = self._chassis_rpc_client\n num_results = self.num_chassis\n\n return rpc_client.send_rpc(\n command,\n params=params,\n and_discard=and_discard,\n num_results=num_results\n )\n\n def _send_cmd_and_wait(self, command, to_slaves=True, timeout=60):\n \"\"\"Simple wrapper around _send_cmd for raising exceptions\n \"\"\"\n revt = self._send_cmd(command, to_slaves=to_slaves)\n success = revt.wait(timeout=timeout)\n if success is None:\n LOG.critical('Timeout in {}'.format(command))\n raise RuntimeError('Timeout in {}'.format(command))\n result = revt.get(block=False)\n if result['errors'] > 0:\n LOG.critical('Errors reported in {}'.format(command))\n raise RuntimeError('Errors reported in {}'.format(command))\n\n return result\n\n def _send_node_cmd(self, nodename, command, params=None):\n \"\"\"Send command to a single node\n \"\"\"\n if params is None:\n params = {}\n\n try:\n result = self.comm.send_rpc(\n dest='{}directslave:{}'.format(MGMTBUS_PREFIX, nodename),\n method=command,\n params=params,\n timeout=60\n )\n except gevent.timeout.Timeout:\n msg = 'Timeout in {} to node {}'.format(command, nodename)\n LOG.error(msg)\n raise RuntimeError(msg)\n\n if result.get('result', None) is None:\n raise RuntimeError('Error in {} to node {}: {}'.format(\n command, nodename, result.get('error', '')\n ))\n\n return result['result']\n\n def init_graph(self, config):\n \"\"\"Initalizes graph by sending startup messages.\n\n Args:\n config (MineMeldConfig): config\n \"\"\"\n result = self._send_cmd_and_wait('state_info', timeout=60)\n LOG.info('state: {}'.format(result['answers']))\n LOG.info('changes: {!r}'.format(config.changes))\n\n state_info = {k.split(':', 2)[-1]: v for k, v in result['answers'].iteritems()}\n\n startup_plan = plan(config, state_info)\n for node, command in startup_plan.iteritems():\n LOG.info('{} <= {}'.format(node, command))\n self._send_node_cmd(node, command)\n\n self.graph_status = 'INIT'\n\n def checkpoint_graph(self, max_tries=60):\n \"\"\"Checkpoints the graph.\n\n Args:\n max_tries (int): number of minutes before giving up\n \"\"\"\n LOG.info('checkpoint_graph called, checking current state')\n\n if self.graph_status != 'INIT':\n LOG.info('graph status {}, checkpoint_graph ignored'.format(self.graph_status))\n return\n\n while True:\n revt = self._send_cmd('state_info')\n success = revt.wait(timeout=30)\n if success is None:\n LOG.error('timeout in state_info')\n gevent.sleep(60)\n continue\n\n result = revt.get(block=False)\n if result['errors'] > 0:\n LOG.critical('errors reported from nodes in ' +\n 'checkpoint_graph: %s',\n result['errors'])\n gevent.sleep(60)\n continue\n\n all_started = True\n for answer in result['answers'].values():\n if answer.get('state', None) != minemeld.ft.ft_states.STARTED:\n all_started = False\n break\n if not all_started:\n LOG.error('some nodes not started yet, waiting')\n gevent.sleep(60)\n continue\n\n break\n\n chkp = str(uuid.uuid4())\n LOG.info('Sending checkpoint {} to nodes'.format(chkp))\n for nodename in self.ftlist:\n self._send_node_cmd(nodename, 'checkpoint', params={'value': chkp})\n\n ntries = 0\n while ntries < max_tries:\n revt = self._send_cmd('state_info')\n success = revt.wait(timeout=60)\n if success is None:\n LOG.error(\"Error retrieving nodes states after checkpoint\")\n gevent.sleep(30)\n continue\n\n result = revt.get(block=False)\n\n cgraphok = True\n for answer in result['answers'].values():\n cgraphok &= (answer['checkpoint'] == chkp)\n if cgraphok:\n LOG.info('checkpoint graph - all good')\n break\n\n gevent.sleep(2)\n ntries += 1\n\n if ntries == max_tries:\n LOG.error('checkpoint_graph: nodes still not in '\n 'checkpoint state after max_tries')\n\n self.graph_status = 'CHECKPOINT'\n\n def _send_collectd_metrics(self, answers, interval):\n \"\"\"Send collected metrics from nodes to collectd.\n\n Args:\n answers (list): list of metrics\n interval (int): collection interval\n \"\"\"\n collectd_socket = self.config.get(\n 'COLLECTD_SOCKET',\n '/var/run/collectd.sock'\n )\n\n cc = CollectdClient(collectd_socket)\n\n gstats = collections.defaultdict(lambda: 0)\n\n for source, a in answers.iteritems():\n ntype = 'processors'\n if len(a.get('inputs', [])) == 0:\n ntype = 'miners'\n elif not a.get('output', False):\n ntype = 'outputs'\n\n stats = a.get('statistics', {})\n length = a.get('length', None)\n\n _, _, source = source.split(':', 2)\n source = hashlib.md5(source).hexdigest()[:10]\n\n for m, v in stats.iteritems():\n gstats[ntype+'.'+m] += v\n cc.putval(source+'.'+m, v,\n interval=interval,\n type_='minemeld_delta')\n\n if length is not None:\n gstats['length'] += length\n gstats[ntype+'.length'] += length\n cc.putval(\n source+'.length',\n length,\n type_='minemeld_counter',\n interval=interval\n )\n\n for gs, v in gstats.iteritems():\n type_ = 'minemeld_delta'\n if gs.endswith('length'):\n type_ = 'minemeld_counter'\n\n cc.putval('minemeld.'+gs, v, type_=type_, interval=interval)\n\n def _merge_status(self, nodename, status):\n currstatus = self._status.get(nodename, None)\n if currstatus is not None:\n if currstatus.get('clock', -1) > status.get('clock', -2):\n LOG.error('old clock: {} > {} - dropped'.format(\n currstatus.get('clock', -1),\n status.get('clock', -2)\n ))\n return\n\n self._status[nodename] = status\n\n try:\n source = nodename.split(':', 2)[2]\n self.SR.publish(\n 'mm-engine-status.'+source,\n ujson.dumps({\n 'source': source,\n 'timestamp': int(time.time())*1000,\n 'status': status\n })\n )\n\n except:\n LOG.exception('Error publishing status')\n\n def _status_loop(self):\n \"\"\"Greenlet that periodically retrieves metrics from nodes and sends\n them to collected.\n \"\"\"\n loop_interval = self.config.get('STATUS_INTERVAL', '60')\n try:\n loop_interval = int(loop_interval)\n except ValueError:\n LOG.error('invalid STATUS_INTERVAL settings, '\n 'reverting to default')\n loop_interval = 60\n\n while True:\n revt = self._send_cmd('status')\n success = revt.wait(timeout=30)\n if success is None:\n LOG.error('timeout in waiting for status updates from nodes')\n else:\n result = revt.get(block=False)\n\n with self._status_lock:\n for nodename, nodestatus in result['answers'].iteritems():\n self._merge_status(nodename, nodestatus)\n\n try:\n self._send_collectd_metrics(\n result['answers'],\n loop_interval\n )\n\n except:\n LOG.exception('Exception in _status_loop')\n\n gevent.sleep(loop_interval)\n\n def status(self, timestamp, **kwargs):\n source = kwargs.get('source', None)\n if source is None:\n LOG.error('no source in status report - dropped')\n return\n\n status = kwargs.get('status', None)\n if status is None:\n LOG.error('no status in status report - dropped')\n return\n\n if self._status_lock.locked():\n return\n\n with self._status_lock:\n if timestamp < self._start_timestamp:\n return\n\n self._merge_status('mbus:slave:'+source, status)\n\n def start_status_monitor(self):\n \"\"\"Starts status monitor greenlet.\n \"\"\"\n if self.status_glet is not None:\n LOG.error('double call to start_status')\n return\n\n self.status_glet = gevent.spawn(self._status_loop)\n\n def stop_status_monitor(self):\n \"\"\"Stops status monitor greenlet.\n \"\"\"\n if self.status_glet is None:\n return\n self.status_glet.kill()\n self.status_glet = None\n\n def start(self):\n self.comm.start()\n\n def stop(self):\n self.comm.stop()\n\n\nclass MgmtbusSlaveHub(object):\n \"\"\"Hub MineMeld engine management bus slaves. Each chassis\n has an instance of this class, and each node in the chassis\n request a channel to the management bus via this instance.\n\n Args:\n config (dict): config\n comm_class (string): communication backend to be used\n comm_config (dict): config for the communication backend\n \"\"\"\n\n def __init__(self, config, comm_class, comm_config):\n self.config = config\n self.comm_config = comm_config\n self.comm_class = comm_class\n\n self.comm = minemeld.comm.factory(self.comm_class, self.comm_config)\n\n def request_log_channel(self):\n LOG.debug(\"Adding log channel\")\n return self.comm.request_pub_channel(\n topic=MGMTBUS_LOG_TOPIC,\n multi_write=True\n )\n\n def send_status(self, params):\n self.comm.send_rpc(\n dest=MGMTBUS_STATUS_TOPIC,\n method='status',\n params=params,\n block=True\n )\n\n def request_chassis_rpc_channel(self, chassis):\n self.comm.request_rpc_server_channel(\n '{}chassis:{}'.format(MGMTBUS_PREFIX, chassis.chassis_id),\n chassis,\n allowed_methods=[\n 'mgmtbus_start'\n ],\n method_prefix='mgmtbus_',\n fanout=MGMTBUS_CHASSIS_TOPIC\n )\n\n def request_channel(self, node):\n self.comm.request_rpc_server_channel(\n '{}directslave:{}'.format(MGMTBUS_PREFIX, node.name),\n node,\n allowed_methods=[\n 'mgmtbus_state_info',\n 'mgmtbus_initialize',\n 'mgmtbus_rebuild',\n 'mgmtbus_reset',\n 'mgmtbus_status',\n 'mgmtbus_checkpoint',\n 'mgmtbus_hup',\n 'mgmtbus_signal'\n ],\n method_prefix='mgmtbus_'\n )\n self.comm.request_rpc_server_channel(\n '{}slave:{}'.format(MGMTBUS_PREFIX, node.name),\n node,\n allowed_methods=[\n 'mgmtbus_state_info',\n 'mgmtbus_initialize',\n 'mgmtbus_rebuild',\n 'mgmtbus_reset',\n 'mgmtbus_status',\n 'mgmtbus_checkpoint'\n ],\n method_prefix='mgmtbus_',\n fanout=MGMTBUS_TOPIC\n )\n\n def add_failure_listener(self, f):\n self.comm.add_failure_listener(f)\n\n def send_master_rpc(self, command, params=None, timeout=None):\n return self.comm.send_rpc(\n MGMTBUS_MASTER,\n command,\n params,\n timeout=timeout\n )\n\n def start(self):\n LOG.debug('mgmtbus start called')\n self.comm.start()\n\n def stop(self):\n self.comm.stop()\n\n\ndef master_factory(config, comm_class, comm_config, nodes, num_chassis):\n \"\"\"Factory of management bus master instances\n\n Args:\n config (dict): management bus master config\n comm_class (string): communication backend.\n Unused, ZMQRedis is always used\n comm_config (dict): config of the communication backend\n fts (list): list of nodes\n\n Returns:\n Instance of minemeld.mgmtbus.MgmtbusMaster class\n \"\"\"\n _ = comm_class # noqa\n\n return MgmtbusMaster(\n ftlist=nodes,\n config=config,\n comm_class='ZMQRedis',\n comm_config=comm_config,\n num_chassis=num_chassis\n )\n\n\ndef slave_hub_factory(config, comm_class, comm_config):\n \"\"\"Factory of management bus slave hub instances\n\n Args:\n config (dict): management bus master config\n comm_class (string): communication backend.\n Unused, ZMQRedis is always used\n comm_config (dict): config of the communication backend.\n\n Returns:\n Instance of minemeld.mgmtbus.MgmtbusSlaveHub class\n \"\"\"\n _ = comm_class # noqa\n\n return MgmtbusSlaveHub(\n config,\n 'ZMQRedis',\n comm_config\n )\n" }, { "path": "minemeld/packages/__init__.py", "content": "" }, { "path": "minemeld/packages/gdns/LICENSE", "content": "Except when otherwise stated (look at the beginning of each file) the software\nand the documentation in this project are copyrighted by:\n\n Denis Bilenko and the contributors, http://www.gevent.org\n\nand licensed under the MIT license:\n\n Permission is hereby granted, free of charge, to any person obtaining a copy\n of this software and associated documentation files (the \"Software\"), to deal\n in the Software without restriction, including without limitation the rights\n to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n copies of the Software, and to permit persons to whom the Software is\n furnished to do so, subject to the following conditions:\n\n The above copyright notice and this permission notice shall be included in\n all copies or substantial portions of the Software.\n\n THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n THE SOFTWARE.\n" }, { "path": "minemeld/packages/gdns/__init__.py", "content": "" }, { "path": "minemeld/packages/gdns/_ares.pyx", "content": "# Copyright (c) 2011-2012 Denis Bilenko. See LICENSE for details.\n# 2016 - Modified by Palo Alto Networks\n\ncimport cares\nimport sys\nfrom cpython cimport *\nfrom libc.string cimport memset\nfrom libc.stdlib cimport malloc, free\n\nfrom _socket import gaierror\n\n__all__ = ['channel']\n\n\nTIMEOUT = 1\n\nDEF EV_READ = 1\nDEF EV_WRITE = 2\n\n\ncdef extern from \"dnshelper.c\":\n int AF_INET\n int AF_INET6\n\n struct hostent:\n char* h_name\n int h_addrtype\n\n struct sockaddr_t \"sockaddr\":\n pass\n\n struct ares_channeldata:\n pass\n\n object parse_h_aliases(hostent*)\n object parse_h_addr_list(hostent*)\n void* create_object_from_hostent(void*)\n\n # this imports _socket lazily\n struct sockaddr_in6:\n pass\n int gevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, sockaddr_in6* sa6)\n\n\nARES_SUCCESS = cares.ARES_SUCCESS\nARES_ENODATA = cares.ARES_ENODATA\nARES_EFORMERR = cares.ARES_EFORMERR\nARES_ESERVFAIL = cares.ARES_ESERVFAIL\nARES_ENOTFOUND = cares.ARES_ENOTFOUND\nARES_ENOTIMP = cares.ARES_ENOTIMP\nARES_EREFUSED = cares.ARES_EREFUSED\nARES_EBADQUERY = cares.ARES_EBADQUERY\nARES_EBADNAME = cares.ARES_EBADNAME\nARES_EBADFAMILY = cares.ARES_EBADFAMILY\nARES_EBADRESP = cares.ARES_EBADRESP\nARES_ECONNREFUSED = cares.ARES_ECONNREFUSED\nARES_ETIMEOUT = cares.ARES_ETIMEOUT\nARES_EOF = cares.ARES_EOF\nARES_EFILE = cares.ARES_EFILE\nARES_ENOMEM = cares.ARES_ENOMEM\nARES_EDESTRUCTION = cares.ARES_EDESTRUCTION\nARES_EBADSTR = cares.ARES_EBADSTR\nARES_EBADFLAGS = cares.ARES_EBADFLAGS\nARES_ENONAME = cares.ARES_ENONAME\nARES_EBADHINTS = cares.ARES_EBADHINTS\nARES_ENOTINITIALIZED = cares.ARES_ENOTINITIALIZED\nARES_ELOADIPHLPAPI = cares.ARES_ELOADIPHLPAPI\nARES_EADDRGETNETWORKPARAMS = cares.ARES_EADDRGETNETWORKPARAMS\nARES_ECANCELLED = cares.ARES_ECANCELLED\n\nARES_FLAG_USEVC = cares.ARES_FLAG_USEVC\nARES_FLAG_PRIMARY = cares.ARES_FLAG_PRIMARY\nARES_FLAG_IGNTC = cares.ARES_FLAG_IGNTC\nARES_FLAG_NORECURSE = cares.ARES_FLAG_NORECURSE\nARES_FLAG_STAYOPEN = cares.ARES_FLAG_STAYOPEN\nARES_FLAG_NOSEARCH = cares.ARES_FLAG_NOSEARCH\nARES_FLAG_NOALIASES = cares.ARES_FLAG_NOALIASES\nARES_FLAG_NOCHECKRESP = cares.ARES_FLAG_NOCHECKRESP\n\n\n_ares_errors = dict([\n (cares.ARES_SUCCESS, 'ARES_SUCCESS'),\n (cares.ARES_ENODATA, 'ARES_ENODATA'),\n (cares.ARES_EFORMERR, 'ARES_EFORMERR'),\n (cares.ARES_ESERVFAIL, 'ARES_ESERVFAIL'),\n (cares.ARES_ENOTFOUND, 'ARES_ENOTFOUND'),\n (cares.ARES_ENOTIMP, 'ARES_ENOTIMP'),\n (cares.ARES_EREFUSED, 'ARES_EREFUSED'),\n (cares.ARES_EBADQUERY, 'ARES_EBADQUERY'),\n (cares.ARES_EBADNAME, 'ARES_EBADNAME'),\n (cares.ARES_EBADFAMILY, 'ARES_EBADFAMILY'),\n (cares.ARES_EBADRESP, 'ARES_EBADRESP'),\n (cares.ARES_ECONNREFUSED, 'ARES_ECONNREFUSED'),\n (cares.ARES_ETIMEOUT, 'ARES_ETIMEOUT'),\n (cares.ARES_EOF, 'ARES_EOF'),\n (cares.ARES_EFILE, 'ARES_EFILE'),\n (cares.ARES_ENOMEM, 'ARES_ENOMEM'),\n (cares.ARES_EDESTRUCTION, 'ARES_EDESTRUCTION'),\n (cares.ARES_EBADSTR, 'ARES_EBADSTR'),\n (cares.ARES_EBADFLAGS, 'ARES_EBADFLAGS'),\n (cares.ARES_ENONAME, 'ARES_ENONAME'),\n (cares.ARES_EBADHINTS, 'ARES_EBADHINTS'),\n (cares.ARES_ENOTINITIALIZED, 'ARES_ENOTINITIALIZED'),\n (cares.ARES_ELOADIPHLPAPI, 'ARES_ELOADIPHLPAPI'),\n (cares.ARES_EADDRGETNETWORKPARAMS, 'ARES_EADDRGETNETWORKPARAMS'),\n (cares.ARES_ECANCELLED, 'ARES_ECANCELLED')])\n\n\n# maps c-ares flag to _socket module flag\n_cares_flag_map = None\n\n\ncdef _prepare_cares_flag_map():\n global _cares_flag_map\n import _socket\n _cares_flag_map = [\n (getattr(_socket, 'NI_NUMERICHOST', 1), cares.ARES_NI_NUMERICHOST),\n (getattr(_socket, 'NI_NUMERICSERV', 2), cares.ARES_NI_NUMERICSERV),\n (getattr(_socket, 'NI_NOFQDN', 4), cares.ARES_NI_NOFQDN),\n (getattr(_socket, 'NI_NAMEREQD', 8), cares.ARES_NI_NAMEREQD),\n (getattr(_socket, 'NI_DGRAM', 16), cares.ARES_NI_DGRAM)]\n\n\ncpdef _convert_cares_flags(int flags, int default=cares.ARES_NI_LOOKUPHOST|cares.ARES_NI_LOOKUPSERVICE):\n if _cares_flag_map is None:\n _prepare_cares_flag_map()\n for socket_flag, cares_flag in _cares_flag_map:\n if socket_flag & flags:\n default |= cares_flag\n flags &= ~socket_flag\n if not flags:\n return default\n raise gaierror(-1, \"Bad value for ai_flags: 0x%x\" % flags)\n\n\ncpdef strerror(code):\n return '%s: %s' % (_ares_errors.get(code) or code, cares.ares_strerror(code))\n\n\nclass InvalidIP(ValueError):\n pass\n\n\ncdef void gevent_sock_state_callback(void *data, int s, int read, int write):\n if not data:\n return\n cdef channel ch = data\n ch._sock_state_callback(s, read, write)\n\n\ncdef class result:\n cdef public object value\n cdef public object exception\n\n def __init__(self, object value=None, object exception=None):\n self.value = value\n self.exception = exception\n\n def __repr__(self):\n if self.exception is None:\n return '%s(%r)' % (self.__class__.__name__, self.value)\n elif self.value is None:\n return '%s(exception=%r)' % (self.__class__.__name__, self.exception)\n else:\n return '%s(value=%r, exception=%r)' % (self.__class__.__name__, self.value, self.exception)\n # add repr_recursive precaution\n\n def successful(self):\n return self.exception is None\n\n def get(self):\n if self.exception is not None:\n raise self.exception\n return self.value\n\n\nclass ares_host_result(tuple):\n\n def __new__(cls, family, iterable):\n cdef object self = tuple.__new__(cls, iterable)\n self.family = family\n return self\n\n def __getnewargs__(self):\n return (self.family, tuple(self))\n\n\ncdef void gevent_ares_host_callback(void *arg, int status, int timeouts, hostent* host):\n cdef channel channel\n cdef object callback\n channel, callback = arg\n Py_DECREF(arg)\n cdef object host_result\n try:\n if status or not host:\n callback(result(None, gaierror(status, strerror(status))))\n else:\n try:\n host_result = ares_host_result(host.h_addrtype, (host.h_name, parse_h_aliases(host), parse_h_addr_list(host)))\n except:\n callback(result(None, sys.exc_info()[1]))\n else:\n callback(result(host_result))\n except:\n channel.loop.handle_error(callback, *sys.exc_info())\n\n\ncdef void gevent_ares_generic_callback(void *arg, int status, int timeouts, unsigned char *abuf, int alen):\n cdef channel channel\n cdef object callback\n channel, callback = arg\n Py_DECREF(arg)\n\n cdef bytes generic_result\n\n try:\n if status or not abuf:\n callback(result(None, gaierror(status, strerror(status))))\n else:\n try:\n generic_result = abuf[:alen]\n except:\n callback(result(None, sys.exc_info()[1]))\n else:\n callback(result(generic_result))\n except:\n channel.loop.handle_error(callback, *sys.exc_info())\n\n\ncdef void gevent_ares_nameinfo_callback(void *arg, int status, int timeouts, char *c_node, char *c_service):\n cdef channel channel\n cdef object callback\n channel, callback = arg\n Py_DECREF(arg)\n cdef object node\n cdef object service\n try:\n if status:\n callback(result(None, gaierror(status, strerror(status))))\n else:\n if c_node:\n node = PyBytes_FromString(c_node)\n else:\n node = None\n if c_service:\n service = PyBytes_FromString(c_service)\n else:\n service = None\n callback(result((node, service)))\n except:\n channel.loop.handle_error(callback, *sys.exc_info())\n\n\ncdef public class channel [object PyGeventAresChannelObject, type PyGeventAresChannel_Type]:\n\n cdef public object loop\n cdef ares_channeldata* channel\n cdef public dict _watchers\n cdef public object _timer\n\n def __init__(self, object loop, flags=None, timeout=None, tries=None, ndots=None,\n udp_port=None, tcp_port=None, servers=None):\n cdef ares_channeldata* channel = NULL\n cdef cares.ares_options options\n memset(&options, 0, sizeof(cares.ares_options))\n cdef int optmask = cares.ARES_OPT_SOCK_STATE_CB\n options.sock_state_cb = gevent_sock_state_callback\n options.sock_state_cb_data = self\n if flags is not None:\n options.flags = int(flags)\n optmask |= cares.ARES_OPT_FLAGS\n if timeout is not None:\n options.timeout = int(float(timeout) * 1000)\n optmask |= cares.ARES_OPT_TIMEOUTMS\n if tries is not None:\n options.tries = int(tries)\n optmask |= cares.ARES_OPT_TRIES\n if ndots is not None:\n options.ndots = int(ndots)\n optmask |= cares.ARES_OPT_NDOTS\n if udp_port is not None:\n options.udp_port = int(udp_port)\n optmask |= cares.ARES_OPT_UDP_PORT\n if tcp_port is not None:\n options.tcp_port = int(tcp_port)\n optmask |= cares.ARES_OPT_TCP_PORT\n cdef int result = cares.ares_library_init(cares.ARES_LIB_INIT_ALL) # ARES_LIB_INIT_WIN32 -DUSE_WINSOCK?\n if result:\n raise gaierror(result, strerror(result))\n result = cares.ares_init_options(&channel, &options, optmask)\n if result:\n raise gaierror(result, strerror(result))\n self._timer = loop.timer(TIMEOUT, TIMEOUT)\n self._watchers = {}\n self.channel = channel\n try:\n if servers is not None:\n self.set_servers(servers)\n self.loop = loop\n except:\n self.destroy()\n raise\n\n def __repr__(self):\n args = (self.__class__.__name__, id(self), self._timer, len(self._watchers))\n return '<%s at 0x%x _timer=%r _watchers[%s]>' % args\n\n def destroy(self):\n if self.channel:\n # XXX ares_library_cleanup?\n cares.ares_destroy(self.channel)\n self.channel = NULL\n self._watchers.clear()\n self._timer.stop()\n self.loop = None\n\n def __dealloc__(self):\n if self.channel:\n # XXX ares_library_cleanup?\n cares.ares_destroy(self.channel)\n self.channel = NULL\n\n def set_servers(self, servers=None):\n if not self.channel:\n raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')\n if not servers:\n servers = []\n if isinstance(servers, basestring):\n servers = servers.split(',')\n cdef int length = len(servers)\n cdef int result, index\n cdef char* string\n cdef cares.ares_addr_node* c_servers\n if length <= 0:\n result = cares.ares_set_servers(self.channel, NULL)\n else:\n c_servers = malloc(sizeof(cares.ares_addr_node) * length)\n if not c_servers:\n raise MemoryError\n try:\n index = 0\n for server in servers:\n string = server\n if cares.ares_inet_pton(AF_INET, string, &c_servers[index].addr) > 0:\n c_servers[index].family = AF_INET\n elif cares.ares_inet_pton(AF_INET6, string, &c_servers[index].addr) > 0:\n c_servers[index].family = AF_INET6\n else:\n raise InvalidIP(repr(string))\n c_servers[index].next = &c_servers[index] + 1\n index += 1\n if index >= length:\n break\n c_servers[length - 1].next = NULL\n index = cares.ares_set_servers(self.channel, c_servers)\n if index:\n raise ValueError(strerror(index))\n finally:\n free(c_servers)\n\n def num_servers(self):\n if not self.channel:\n return None\n\n cdef cares.ares_addr_node *servers\n cdef cares.ares_addr_node *curserver\n cdef int result\n cdef int num = 0\n\n result = cares.ares_get_servers(self.channel, &servers)\n\n if result != cares.ARES_SUCCESS:\n raise RuntimeError('Error retrieving channel servers: %d' % result)\n\n curserver = servers\n while curserver != NULL:\n num += 1\n curserver = curserver.next\n\n cares.ares_free_data(servers)\n\n return num\n\n # this crashes c-ares\n #def cancel(self):\n # cares.ares_cancel(self.channel)\n\n cdef _sock_state_callback(self, int socket, int read, int write):\n if not self.channel:\n return\n cdef object watcher = self._watchers.get(socket)\n cdef int events = 0\n if read:\n events |= EV_READ\n if write:\n events |= EV_WRITE\n if watcher is None:\n if not events:\n return\n watcher = self.loop.io(socket, events)\n self._watchers[socket] = watcher\n elif events:\n if watcher.events == events:\n return\n watcher.stop()\n watcher.events = events\n else:\n watcher.stop()\n self._watchers.pop(socket, None)\n if not self._watchers:\n self._timer.stop()\n return\n watcher.start(self._process_fd, watcher, pass_events=True)\n self._timer.again(self._on_timer)\n\n def _on_timer(self):\n cares.ares_process_fd(self.channel, cares.ARES_SOCKET_BAD, cares.ARES_SOCKET_BAD)\n\n def _process_fd(self, int events, object watcher):\n if not self.channel:\n return\n cdef int read_fd = watcher.fd\n cdef int write_fd = read_fd\n if not (events & EV_READ):\n read_fd = cares.ARES_SOCKET_BAD\n if not (events & EV_WRITE):\n write_fd = cares.ARES_SOCKET_BAD\n cares.ares_process_fd(self.channel, read_fd, write_fd)\n\n def gethostbyname(self, object callback, char* name, int family=AF_INET):\n if not self.channel:\n raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')\n # note that for file lookups still AF_INET can be returned for AF_INET6 request\n cdef object arg = (self, callback)\n Py_INCREF(arg)\n cares.ares_gethostbyname(self.channel, name, family, gevent_ares_host_callback, arg)\n\n def query(self, object callback, char *name, int dnsclass, int type_):\n if not self.channel:\n raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')\n\n cdef object arg = (self, callback)\n Py_INCREF(arg)\n\n cares.ares_query(\n self.channel,\n name,\n dnsclass,\n type_,\n gevent_ares_generic_callback,\n arg\n )\n\n def gethostbyaddr(self, object callback, char* addr):\n if not self.channel:\n raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')\n # will guess the family\n cdef char addr_packed[16]\n cdef int family\n cdef int length\n if cares.ares_inet_pton(AF_INET, addr, addr_packed) > 0:\n family = AF_INET\n length = 4\n elif cares.ares_inet_pton(AF_INET6, addr, addr_packed) > 0:\n family = AF_INET6\n length = 16\n else:\n raise InvalidIP(repr(addr))\n cdef object arg = (self, callback)\n Py_INCREF(arg)\n cares.ares_gethostbyaddr(self.channel, addr_packed, length, family, gevent_ares_host_callback, arg)\n\n def parse_txt_reply(self, bytes reply):\n cdef cares.ares_txt_reply *txtout\n cdef cares.ares_txt_reply *curtxt\n cdef int ok\n cdef list result = []\n\n ok = cares.ares_parse_txt_reply(reply, len(reply), &txtout)\n if ok != cares.ARES_SUCCESS:\n raise RuntimeError('Error parsing txt reply: %d' % ok)\n\n curtxt = txtout\n while curtxt:\n result.append(curtxt.txt[:curtxt.length])\n curtxt = txtout.next\n\n return result\n\n cpdef _getnameinfo(self, object callback, tuple sockaddr, int flags):\n if not self.channel:\n raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed')\n cdef char* hostp = NULL\n cdef int port = 0\n cdef int flowinfo = 0\n cdef int scope_id = 0\n cdef sockaddr_in6 sa6\n if not PyTuple_Check(sockaddr):\n raise TypeError('expected a tuple, got %r' % (sockaddr, ))\n PyArg_ParseTuple(sockaddr, \"si|ii\", &hostp, &port, &flowinfo, &scope_id)\n if port < 0 or port > 65535:\n raise gaierror(-8, 'Invalid value for port: %r' % port)\n cdef int length = gevent_make_sockaddr(hostp, port, flowinfo, scope_id, &sa6)\n if length <= 0:\n raise InvalidIP(repr(hostp))\n cdef object arg = (self, callback)\n Py_INCREF(arg)\n cdef sockaddr_t* x = &sa6\n cares.ares_getnameinfo(self.channel, x, length, flags, gevent_ares_nameinfo_callback, arg)\n\n def getnameinfo(self, object callback, tuple sockaddr, int flags):\n return self._getnameinfo(callback, sockaddr, _convert_cares_flags(flags))\n" }, { "path": "minemeld/packages/gdns/cares.pxd", "content": "cdef extern from \"ares.h\":\n struct ares_options:\n int flags\n void* sock_state_cb\n void* sock_state_cb_data\n int timeout\n int tries\n int ndots\n unsigned short udp_port\n unsigned short tcp_port\n char **domains\n int ndomains\n char* lookups\n\n int ARES_OPT_FLAGS\n int ARES_OPT_SOCK_STATE_CB\n int ARES_OPT_TIMEOUTMS\n int ARES_OPT_TRIES\n int ARES_OPT_NDOTS\n int ARES_OPT_TCP_PORT\n int ARES_OPT_UDP_PORT\n int ARES_OPT_SERVERS\n int ARES_OPT_DOMAINS\n int ARES_OPT_LOOKUPS\n\n int ARES_FLAG_USEVC\n int ARES_FLAG_PRIMARY\n int ARES_FLAG_IGNTC\n int ARES_FLAG_NORECURSE\n int ARES_FLAG_STAYOPEN\n int ARES_FLAG_NOSEARCH\n int ARES_FLAG_NOALIASES\n int ARES_FLAG_NOCHECKRESP\n\n int ARES_LIB_INIT_ALL\n int ARES_SOCKET_BAD\n\n int ARES_SUCCESS\n int ARES_ENODATA\n int ARES_EFORMERR\n int ARES_ESERVFAIL\n int ARES_ENOTFOUND\n int ARES_ENOTIMP\n int ARES_EREFUSED\n int ARES_EBADQUERY\n int ARES_EBADNAME\n int ARES_EBADFAMILY\n int ARES_EBADRESP\n int ARES_ECONNREFUSED\n int ARES_ETIMEOUT\n int ARES_EOF\n int ARES_EFILE\n int ARES_ENOMEM\n int ARES_EDESTRUCTION\n int ARES_EBADSTR\n int ARES_EBADFLAGS\n int ARES_ENONAME\n int ARES_EBADHINTS\n int ARES_ENOTINITIALIZED\n int ARES_ELOADIPHLPAPI\n int ARES_EADDRGETNETWORKPARAMS\n int ARES_ECANCELLED\n\n int ARES_NI_NOFQDN\n int ARES_NI_NUMERICHOST\n int ARES_NI_NAMEREQD\n int ARES_NI_NUMERICSERV\n int ARES_NI_DGRAM\n int ARES_NI_TCP\n int ARES_NI_UDP\n int ARES_NI_SCTP\n int ARES_NI_DCCP\n int ARES_NI_NUMERICSCOPE\n int ARES_NI_LOOKUPHOST\n int ARES_NI_LOOKUPSERVICE\n\n\n int ares_library_init(int flags)\n void ares_library_cleanup()\n int ares_init_options(void *channelptr, ares_options *options, int)\n int ares_init(void *channelptr)\n void ares_destroy(void *channelptr)\n void ares_gethostbyname(void* channel, char *name, int family, void* callback, void *arg)\n void ares_gethostbyaddr(void* channel, void *addr, int addrlen, int family, void* callback, void *arg)\n void ares_process_fd(void* channel, int read_fd, int write_fd)\n char* ares_strerror(int code)\n void ares_cancel(void* channel)\n void ares_getnameinfo(void* channel, void* sa, int salen, int flags, void* callback, void *arg)\n void ares_query(void* channel, const char *name, int dnsclass, int type, void* callback, void *arg)\n\n struct in_addr:\n pass\n\n struct ares_in6_addr:\n pass\n\n struct addr_union:\n in_addr addr4\n ares_in6_addr addr6\n\n struct ares_addr_node:\n ares_addr_node *next\n int family\n addr_union addr\n\n int ares_set_servers(void* channel, ares_addr_node *servers)\n int ares_get_servers(void* channel, ares_addr_node **servers)\n\n struct ares_txt_reply:\n ares_txt_reply *next\n unsigned char *txt\n int length\n\n int ares_parse_txt_reply(const unsigned char* abuf, int alen, ares_txt_reply **txt_out)\n\n void ares_free_data(void *dataptr)\n\n\ncdef extern from \"cares_pton.h\":\n int ares_inet_pton(int af, char *src, void *dst)\n" }, { "path": "minemeld/packages/gdns/cares_ntop.h", "content": "#include \n#define ares_inet_ntop(w,x,y,z) inet_ntop(w,x,y,z)\n" }, { "path": "minemeld/packages/gdns/cares_pton.h", "content": "#include \n#define ares_inet_pton(x,y,z) inet_pton(x,y,z)\n#define ares_inet_net_pton(w,x,y,z) inet_net_pton(w,x,y,z)\n" }, { "path": "minemeld/packages/gdns/dig.py", "content": "# Based on resolver_ares.py from gevent\n# Copyright (c) 2011 Denis Bilenko. See LICENSE for details.\n\nfrom __future__ import absolute_import\nimport os\nfrom _socket import gaierror\nfrom gevent.hub import Waiter, get_hub\nfrom minemeld.packages.gdns._ares import channel\n\n\nclass Dig(object):\n\n ares_class = channel\n\n # from arpa/nameser.h\n NS_C_INVALID = 0 # Cookie\n NS_C_IN = 1 # Internet\n NS_C_2 = 2 # unallocated/unsupported\n NS_C_CHAOS = 3 # MIT Chaos-net\n NS_C_HS = 4 # MIT Hesiod\n NS_C_NONE = 254 # prereq. sections in update requests\n NS_C_ANY = 255 # Wildcard match\n NS_C_MAX = 65536\n\n NS_T_INVALID = 0 # Cookie\n NS_T_A = 1 # Host address\n NS_T_NS = 2 # Authoritative server\n NS_T_MD = 3 # Mail destination\n NS_T_MF = 4 # Mail forwarder\n NS_T_CNAME = 5 # Canonical name\n NS_T_SOA = 6 # Start of authority zone\n NS_T_MB = 7 # Mailbox domain name\n NS_T_MG = 8 # Mail group member\n NS_T_MR = 9 # Mail rename name\n NS_T_NULL = 10 # Null resource record\n NS_T_WKS = 11 # Well known service\n NS_T_PTR = 12 # Domain name pointer\n NS_T_HINFO = 13 # Host information\n NS_T_MINFO = 14 # Mailbox information\n NS_T_MX = 15 # Mail routing information\n NS_T_TXT = 16 # Text strings\n NS_T_RP = 17 # Responsible person\n NS_T_AFSDB = 18 # AFS cell database\n NS_T_X25 = 19 # X_25 calling address\n NS_T_ISDN = 20 # ISDN calling address\n NS_T_RT = 21 # Router\n NS_T_NSAP = 22 # NSAP address\n NS_T_NSAP_PTR = 23 # Reverse NSAP lookup (deprecated)\n NS_T_SIG = 24 # Security signature\n NS_T_KEY = 25 # Security key\n NS_T_PX = 26 # X.400 mail mapping\n NS_T_GPOS = 27 # Geographical position (withdrawn)\n NS_T_AAAA = 28 # Ip6 Address\n NS_T_LOC = 29 # Location Information\n NS_T_NXT = 30 # Next domain (security)\n NS_T_EID = 31 # Endpoint identifier\n NS_T_NIMLOC = 32 # Nimrod Locator\n NS_T_SRV = 33 # Server Selection\n NS_T_ATMA = 34 # ATM Address\n NS_T_NAPTR = 35 # Naming Authority PoinTeR\n NS_T_KX = 36 # Key Exchange\n NS_T_CERT = 37 # Certification record\n NS_T_A6 = 38 # IPv6 address (deprecated, use NS_T_AAAA)\n NS_T_DNAME = 39 # Non-terminal DNAME (for IPv6)\n NS_T_SINK = 40 # Kitchen sink (experimentatl)\n NS_T_OPT = 41 # EDNS0 option (meta-RR)\n NS_T_APL = 42 # Address prefix list (RFC3123)\n NS_T_TKEY = 249 # Transaction key\n NS_T_TSIG = 250 # Transaction signature\n NS_T_IXFR = 251 # Incremental zone transfer\n NS_T_AXFR = 252 # Transfer zone of authority\n NS_T_MAILB = 253 # Transfer mailbox records\n NS_T_MAILA = 254 # Transfer mail agent records\n NS_T_ANY = 255 # Wildcard match\n NS_T_ZXFR = 256 # BIND-specific, nonstandard\n NS_T_MAX = 65536\n\n def __init__(self, hub=None, **kwargs):\n if hub is None:\n hub = get_hub()\n self.hub = hub\n\n self.ares = self.ares_class(hub.loop, **kwargs)\n self.pid = os.getpid()\n self.params = kwargs\n self.fork_watcher = hub.loop.fork(ref=False)\n self.fork_watcher.start(self._on_fork)\n\n def __repr__(self):\n return (\n '' %\n (id(self), self.ares)\n )\n\n def _on_fork(self):\n pid = os.getpid()\n if pid != self.pid:\n self.hub.loop.run_callback(self.ares.destroy)\n self.ares = self.ares_class(self.hub.loop, **self.params)\n self.pid = pid\n\n def close(self):\n if self.ares is not None:\n self.hub.loop.run_callback(self.ares.destroy)\n self.ares = None\n self.fork_watcher.stop()\n\n def query(self, name, dnsclass, type_):\n if isinstance(name, unicode):\n name = name.encode('ascii')\n elif not isinstance(name, str):\n raise TypeError('Expected string, not %s' % type(name).__name__)\n\n while True:\n ares = self.ares\n try:\n waiter = Waiter(self.hub)\n ares.query(waiter, name, dnsclass, type_)\n result = waiter.get()\n\n return result\n\n except gaierror:\n if ares is self.ares:\n raise\n\n def parse_txt_reply(self, reply):\n return self.ares.parse_txt_reply(reply)\n" }, { "path": "minemeld/packages/gdns/dnshelper.c", "content": "/* Copyright (c) 2011 Denis Bilenko. See LICENSE for details. */\n#include \"Python.h\"\n\n#ifdef HAVE_NETDB_H\n#include \n#endif\n\n#include \"ares.h\"\n\n#include \"cares_ntop.h\"\n#include \"cares_pton.h\"\n\n#if PY_VERSION_HEX < 0x02060000\n #define PyBytes_FromString PyString_FromString\n#endif\n\n\nstatic PyObject* _socket_error = 0;\n\nstatic PyObject*\nget_socket_object(PyObject** pobject, const char* name)\n{\n if (!*pobject) {\n PyObject* _socket;\n _socket = PyImport_ImportModule(\"_socket\");\n if (_socket) {\n *pobject = PyObject_GetAttrString(_socket, name);\n if (!*pobject) {\n PyErr_WriteUnraisable(Py_None);\n }\n Py_DECREF(_socket);\n }\n else {\n PyErr_WriteUnraisable(Py_None);\n }\n if (!*pobject) {\n *pobject = PyExc_IOError;\n }\n }\n return *pobject;\n}\n\n\nstatic int\ngevent_append_addr(PyObject* list, int family, void* src, char* tmpbuf, size_t tmpsize) {\n int status = -1;\n PyObject* tmp;\n if (ares_inet_ntop(family, src, tmpbuf, tmpsize)) {\n tmp = PyBytes_FromString(tmpbuf);\n if (tmp) {\n status = PyList_Append(list, tmp);\n Py_DECREF(tmp);\n }\n }\n return status;\n}\n\n\nstatic PyObject*\nparse_h_aliases(struct hostent *h)\n{\n char **pch;\n PyObject *result = NULL;\n PyObject *tmp;\n\n result = PyList_New(0);\n\n if (result && h->h_aliases) {\n for (pch = h->h_aliases; *pch != NULL; pch++) {\n if (*pch != h->h_name && strcmp(*pch, h->h_name)) {\n int status;\n tmp = PyBytes_FromString(*pch);\n if (tmp == NULL) {\n break;\n }\n\n status = PyList_Append(result, tmp);\n Py_DECREF(tmp);\n\n if (status) {\n break;\n }\n }\n }\n }\n\n return result;\n}\n\n\nstatic PyObject *\nparse_h_addr_list(struct hostent *h)\n{\n char **pch;\n PyObject *result = NULL;\n\n result = PyList_New(0);\n\n if (result) {\n switch (h->h_addrtype) {\n case AF_INET:\n {\n char tmpbuf[sizeof \"255.255.255.255\"];\n for (pch = h->h_addr_list; *pch != NULL; pch++) {\n if (gevent_append_addr(result, AF_INET, *pch, tmpbuf, sizeof(tmpbuf))) {\n break;\n }\n }\n break;\n }\n case AF_INET6:\n {\n char tmpbuf[sizeof(\"ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255\")];\n for (pch = h->h_addr_list; *pch != NULL; pch++) {\n if (gevent_append_addr(result, AF_INET6, *pch, tmpbuf, sizeof(tmpbuf))) {\n break;\n }\n }\n break;\n }\n default:\n PyErr_SetString(get_socket_object(&_socket_error, \"error\"), \"unsupported address family\");\n Py_DECREF(result);\n result = NULL;\n }\n }\n\n return result;\n}\n\n\nstatic int\ngevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, struct sockaddr_in6* sa6) {\n if ( ares_inet_pton(AF_INET, hostp, &((struct sockaddr_in*)sa6)->sin_addr.s_addr) > 0 ) {\n ((struct sockaddr_in*)sa6)->sin_family = AF_INET;\n ((struct sockaddr_in*)sa6)->sin_port = htons(port);\n return sizeof(struct sockaddr_in);\n }\n else if ( ares_inet_pton(AF_INET6, hostp, &sa6->sin6_addr.s6_addr) > 0 ) {\n sa6->sin6_family = AF_INET6;\n sa6->sin6_port = htons(port);\n sa6->sin6_flowinfo = flowinfo;\n sa6->sin6_scope_id = scope_id;\n return sizeof(struct sockaddr_in6);\n }\n return -1;\n}\n" }, { "path": "minemeld/packages/gevent_openssl/COPYING", "content": "Copyright (c) 2015, Menno Finlay-Smits\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n * Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in the\n documentation and/or other materials provided with the distribution.\n * Neither the name of Menno Finlay-Smits nor the names of its\n contributors may be used to endorse or promote products derived\n from this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL MENNO SMITS BE LIABLE FOR ANY\nDIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "path": "minemeld/packages/gevent_openssl/SSL.py", "content": "\"\"\"gevent_openssl.SSL - gevent compatibility with OpenSSL.SSL (pyOpenSSL)\n\"\"\"\n\nimport logging\n\nimport OpenSSL.SSL\nfrom gevent.socket import wait_read, wait_write\n\n_real_connection = OpenSSL.SSL.Connection\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass Connection(object):\n \"\"\"OpenSSL Connection wrapper\n \"\"\"\n\n _reverse_mapping = _real_connection._reverse_mapping\n\n def __init__(self, context, sock):\n self._context = context\n self._sock = sock\n self._connection = _real_connection(context, sock)\n\n def __getattr__(self, attr):\n return getattr(self._connection, attr)\n\n def __iowait(self, io_func, *args, **kwargs):\n fd = self._sock.fileno()\n timeout = self._sock.gettimeout()\n while True:\n try:\n return io_func(*args, **kwargs)\n except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError):\n wait_read(fd, timeout=timeout)\n except OpenSSL.SSL.WantWriteError:\n wait_write(fd, timeout=timeout)\n\n def accept(self):\n sock, addr = self._sock.accept()\n return Connection(self._context, sock), addr\n\n def do_handshake(self):\n # even if some sites are super sensible\n # to handshake timeouts (to avoid DDoS),\n # we have to make handshake not blocking to avoid issues\n # with firewalls or other middle boxes dropping the connection\n return self.__iowait(self._connection.do_handshake)\n\n def connect(self, *args, **kwargs):\n return self.__iowait(self._connection.connect, *args, **kwargs)\n\n def send(self, data, flags=0):\n return self.__send(self._connection.send, data, flags)\n\n def sendall(self, data, flags=0):\n # see https://github.com/mjs/gevent_openssl/issues/12\n # Note: all of the types supported by OpenSSL's Connection.sendall,\n # basestring, memoryview, and buffer, support len(...) and slicing,\n # so they are safe to use here.\n while len(data) > 0:\n res = self.send(data, flags)\n data = data[res:]\n\n def __send(self, send_method, data, flags=0):\n try:\n return self.__iowait(send_method, data, flags)\n except OpenSSL.SSL.SysCallError as e:\n if e[0] == -1 and not data:\n # errors when writing empty strings are expected and can be\n # ignored\n return 0\n raise\n\n def recv(self, bufsiz, flags=0):\n pending = self._connection.pending()\n if pending:\n return self._connection.recv(min(pending, bufsiz))\n try:\n return self.__iowait(self._connection.recv, bufsiz, flags)\n except OpenSSL.SSL.ZeroReturnError:\n return ''\n except OpenSSL.SSL.SysCallError as e:\n if e[0] == -1 and 'Unexpected EOF' in e[1]:\n # errors when reading empty strings are expected and can be\n # ignored\n return ''\n raise\n\n def shutdown(self):\n return self.__iowait(self._connection.shutdown)\n" }, { "path": "minemeld/packages/gevent_openssl/__init__.py", "content": "\"\"\"gevent_openssl - gevent compatibility with pyOpenSSL.\n\nUsage\n-----\n\nInstead of importing OpenSSL directly, do so in the following manner:\n\n..\n\n import gevent_openssl as OpenSSL\n\nor\n\n..\n\n import gevent_openssl; gevent_openssl.monkey_patch()\n\nAny calls that would have blocked the current thread will now only block the\ncurrent green thread.\n\nThis compatibility is accomplished by ensuring the nonblocking flag is\nset before any blocking operation and the OpenSSL file descriptor is\npolled internally to trigger needed events.\n\"\"\"\n\nfrom . import SSL as MySSL\nfrom OpenSSL import *\n\n\ndef monkey_patch():\n \"\"\"\n Monkey patches `OpenSSL.SSL.Connection`\n \"\"\"\n mod = __import__('OpenSSL').SSL\n mod.Connection = MySSL.Connection\n" }, { "path": "minemeld/packages/ise/__init__.py", "content": "import logging\n\nDEBUG1 = logging.DEBUG\nDEBUG2 = DEBUG1 - 1\nDEBUG3 = DEBUG2 - 1\n\nlogging.addLevelName(DEBUG2, 'DEBUG2')\nlogging.addLevelName(DEBUG3, 'DEBUG3')\n" }, { "path": "minemeld/packages/ise/ers.py", "content": "#\n# Copyright (c) 2016 Palo Alto Networks, Inc. \n#\n# Permission to use, copy, modify, and distribute this software for any\n# purpose with or without fee is hereby granted, provided that the above\n# copyright notice and this permission notice appear in all copies.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES\n# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF\n# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR\n# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES\n# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN\n# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF\n# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\n#\n\n'''Interface to the Cisco ISE ERS (External RESTful Services) API\n\nThe interface is specific to requirements for creating SGT mappings\non PAN-OS.\n\nSee ERS SDK page at: https://ise:9060/ers/sdk\n'''\n\nfrom collections import defaultdict\nimport inspect\nimport logging\nimport pprint\nimport xml.etree.ElementTree as etree\n\nfrom . import DEBUG1, DEBUG2, DEBUG3\n\ntry:\n import requests\nexcept ImportError:\n raise ValueError('Install requests library: '\n 'http://docs.python-requests.org/')\n\n# https://github.com/shazow/urllib3/issues/655\n# Requests treats None as forever\n_None = object()\n\n\nclass IseErsRequest:\n def __init__(self, name=None):\n self.name = name\n # python-requests\n self.response = None\n self.status_code = None\n self.reason = None\n self.headers = None\n self.encoding = None\n self.content = None\n self.text = None\n #\n self.xml_root = None\n self.obj = None\n\n def raise_for_status(self):\n if self.response is not None:\n try:\n self.response.raise_for_status()\n except requests.exceptions.HTTPError as e:\n raise IseErsError(e)\n\n\nclass IseErsError(Exception):\n pass\n\n\nclass IseErs:\n def __init__(self,\n hostname=None,\n username=None,\n password=None,\n verify=None,\n timeout=_None):\n if hostname is None:\n raise IseErsError('no hostname')\n if username is None:\n raise IseErsError('no username')\n if password is None:\n raise IseErsError('no password')\n\n self._log = logging.getLogger(__name__).log\n self.verify = verify\n if self.verify is False:\n requests.packages.urllib3.disable_warnings()\n self.timeout = timeout\n self._log(DEBUG2, 'timeout: %s', repr(timeout))\n self.uri = 'https://' + hostname + ':9060'\n self.auth = requests.auth.HTTPBasicAuth(username, password)\n\n def _request(self,\n uri,\n headers):\n\n kwargs = {}\n if self.verify is not None:\n kwargs['verify'] = self.verify\n if self.timeout is not _None:\n kwargs['timeout'] = self.timeout\n\n try:\n r = requests.get(url=uri,\n headers=headers,\n auth=self.auth,\n **kwargs)\n except (requests.exceptions.RequestException, ValueError) as e:\n raise IseErsError(e)\n\n return r\n\n def _set_attributes(self, r):\n x = IseErsRequest(inspect.stack()[1][3])\n # http://docs.python-requests.org/en/master/api/#requests.Response\n x.response = r\n x.status_code = r.status_code\n x.reason = r.reason\n x.headers = r.headers\n x.encoding = r.encoding\n self._log(DEBUG2, r.encoding)\n self._log(DEBUG2, r.request.headers) # XXX authorization header\n self._log(DEBUG2, r.headers)\n x.content = r.content # bytes\n x.text = r.text # Unicode\n self._log(DEBUG3, r.text)\n try:\n x.xml_root = etree.fromstring(r.content)\n except etree.ParseError as e:\n self._log(DEBUG1, 'ElementTree.fromstring ParseError: %s', e)\n\n if x.xml_root is not None:\n self._log(DEBUG1, 'root tag: %s', x.xml_root.tag)\n if x.xml_root.tag == '{ers.ise.cisco.com}ersResponse':\n message = x.xml_root.find('messages/message')\n if message is not None:\n x.obj = {}\n x.obj['error'] = {}\n for k in ['type', 'code']:\n if k in message.attrib:\n x.obj['error'][k] = message.attrib[k]\n title = message.findall('title')\n if title is not None:\n x.obj['error']['title'] = []\n for elem in title:\n x.obj['error']['title'].append(elem.text)\n\n self._log(DEBUG2, x.obj)\n\n return x\n\n def sgts_ips_map(self):\n r = self.sgt()\n r.raise_for_status()\n if 'sgt' not in r.obj:\n raise IseErsError('no response data for sgt()')\n _sgt = r.obj['sgt']\n\n r = self.sgmapping()\n r.raise_for_status()\n if 'sgmapping' not in r.obj:\n raise IseErsError('no response data for sgmapping()')\n _sgmapping = r.obj['sgmapping']\n\n _sgt_name_desc_map = {}\n _sgt_id_name_map = {}\n _sgts_ips_map = {}\n for x in _sgt:\n _sgt_name_desc_map[x['name']] = x['description']\n _sgt_id_name_map[x['id']] = x['name']\n _sgts_ips_map[x['name']] = []\n\n self._log(DEBUG2, pprint.pformat(_sgt_name_desc_map))\n self._log(DEBUG2, pprint.pformat(_sgt_id_name_map))\n\n for x in _sgmapping:\n r = self.sgmapping(id=x['id'])\n r.raise_for_status()\n if 'sgmapping_id' not in r.obj:\n raise IseErsError('no response data for sgmapping(%s)' %\n x['id'])\n _sgmapping_id = r.obj['sgmapping_id']\n if 'hostIp' in _sgmapping_id:\n if _sgmapping_id['sgt'] not in _sgt_id_name_map:\n pass # XXX refresh _sgt_id_name_map\n else:\n _sgts_ips_map[_sgt_id_name_map[_sgmapping_id['sgt']]].\\\n append(_sgmapping_id['hostIp'])\n\n self._log(DEBUG2, pprint.pformat(_sgts_ips_map))\n\n d = defaultdict(list)\n [d[v].append(k) for k in _sgts_ips_map for v in _sgts_ips_map[k]]\n\n self.sgt_name_desc_map = _sgt_name_desc_map\n self.sgts_ips_map = _sgts_ips_map\n self.ips_sgts_map = dict(d)\n\n return self.sgt_name_desc_map, self.sgts_ips_map, self.ips_sgts_map\n\n def sgt(self,\n id=None):\n '''Security Groups:\n Get-By-Id\n Get-All\n '''\n\n headers = {\n 'accept':\n 'application/vnd.com.cisco.ise.trustsec.sgt.1.0+xml'\n }\n path = '/ers/config/sgt'\n if id is not None:\n path += '/' + str(id)\n uri = self.uri + path\n\n r = self._request(uri=uri, headers=headers)\n x = self._set_attributes(r)\n\n if x.xml_root is not None:\n rk = 'sgt' # root key\n if x.xml_root.tag == '{ers.ise.cisco.com}searchResult':\n x.obj = {}\n x.obj[rk] = []\n for elem in x.xml_root.findall('resources/resource'):\n for k in ['name', 'id', 'description']:\n if k not in elem.attrib:\n raise IseErsError('missing attribute \\\"%s\\\": %s' %\n (k, elem.attrib))\n x.obj[rk].append(elem.attrib)\n\n elif x.xml_root.tag == '{trustsec.ers.ise.cisco.com}sgt':\n rk = 'sgt_id'\n x.obj = {}\n x.obj[rk] = {}\n for k in ['id', 'name', 'description']:\n if k not in x.xml_root.attrib:\n raise IseErsError('missing attribute \\\"%s\\\": %s' %\n (k, elem.attrib))\n else:\n x.obj[rk][k] = x.xml_root.attrib[k]\n for elem in x.xml_root.findall('*'):\n x.obj[rk][elem.tag] = elem.text\n\n self._log(DEBUG2, pprint.pformat(x.obj))\n\n return x\n\n def sgmapping(self,\n id=None):\n '''IP To SGT Mapping:\n Get-All\n Get-By-Id\n '''\n\n headers = {\n 'accept':\n 'application/vnd.com.cisco.ise.trustsec.sgmapping.1.0+xml'\n }\n path = '/ers/config/sgmapping'\n if id is not None:\n path += '/' + str(id)\n uri = self.uri + path\n\n r = self._request(uri=uri, headers=headers)\n x = self._set_attributes(r)\n\n if x.xml_root is not None:\n if x.xml_root.tag == '{ers.ise.cisco.com}searchResult':\n rk = 'sgmapping'\n x.obj = {}\n x.obj[rk] = []\n for elem in x.xml_root.findall('resources/resource'):\n for k in ['name', 'id']:\n if k not in elem.attrib:\n raise IseErsError('missing attribute \\\"%s\\\": %s' %\n (k, elem.attrib))\n x.obj[rk].append(elem.attrib)\n\n elif x.xml_root.tag == '{trustsec.ers.ise.cisco.com}sgMapping':\n rk = 'sgmapping_id'\n x.obj = {}\n x.obj[rk] = {}\n for elem in x.xml_root.findall('*'):\n x.obj[rk][elem.tag] = elem.text\n\n self._log(DEBUG2, pprint.pformat(x.obj))\n\n return x\n" }, { "path": "minemeld/packages/panforest/__init__.py", "content": "from . import forest\n\nPanForest = forest.PanForest\nPanForestError = forest.PanForestError\n" }, { "path": "minemeld/packages/panforest/forest.py", "content": "#\n# Copyright (c) 2015 Palo Alto Networks, Inc. \n#\n# Permission to use, copy, modify, and distribute this software for any\n# purpose with or without fee is hereby granted, provided that the above\n# copyright notice and this permission notice appear in all copies.\n#\n# THE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES\n# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF\n# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR\n# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES\n# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN\n# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF\n# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\n#\n\nimport pprint\nimport random\nimport sys\nimport time\nimport logging\n\nimport pan.config\n\n_NLOGS = 100\n\nLOG = logging.getLogger(__name__)\n\n\nclass PanForestError(Exception):\n def __init__(self, msg):\n self.msg = msg\n\n def __str__(self):\n if self.msg is None:\n return ''\n return self.msg\n\n\nclass PanForest(object):\n def __init__(self,\n xapi=None,\n log_type=None,\n filter=None,\n nlogs=None,\n format=None):\n self.xapi = xapi\n self.log_type = log_type\n self.filter = filter\n self.nlogs = _NLOGS if nlogs is None else nlogs\n if format not in ['xml', 'python', None]:\n raise PanForestError('Invalid format: %s' % format)\n self.format = 'python' if format is None else format\n\n def __iter__(self):\n return self.follow()\n\n def sleep(self, t):\n time.sleep(t)\n\n def follow(self):\n \"\"\"\\\n generator function to return log entries matching optional\n filter\n \"\"\"\n\n # filter time < now so we start at logs in current second\n now = int(time.time()) - 1\n filter = '(receive_time leq %d)' % now\n LOG.debug('filter: %s', filter)\n _, obj = self._log_get(nlogs=1,\n filter=filter)\n if not obj:\n raise PanForestError(\"Can't get last log\")\n\n count = self._log_count(obj)\n\n if count != 1:\n seqno = 0\n else:\n seqno = self._log_seqno(obj)\n\n LOG.debug('starting seqno: %s', seqno)\n\n filter = self._filter(seqno)\n nlogs = self.nlogs\n\n skip = 0\n sleeper = self._sleeper()\n\n while True:\n LOG.debug('skip: %d nlogs: %d filter: \"%s\"' %\n (skip, nlogs, filter))\n elem, obj = self._log_get(nlogs=nlogs,\n filter=filter,\n skip=skip)\n\n if not obj:\n raise PanForestError(\"Can't get log chunk\")\n\n count = self._log_count(obj)\n\n if count == 0:\n skip = 0\n filter = self._filter(seqno)\n\n try:\n wait = next(sleeper)\n except StopIteration:\n pass\n x = random.uniform(0, 0.5)\n LOG.debug('sleep: %d %d', wait, x)\n\n self.sleep(wait+x)\n continue\n\n elif count == nlogs:\n sleeper = self._sleeper()\n skip += nlogs\n t = self._log_seqno(obj)\n if t > seqno:\n seqno = t\n # don't update filter\n self.sleep(0)\n\n elif count < nlogs:\n sleeper = self._sleeper()\n t = self._log_seqno(obj)\n if t > seqno:\n seqno = t\n filter = self._filter(seqno)\n self.sleep(0)\n\n else:\n assert False, 'NOTREACHED'\n\n if self.format == 'python':\n for entry in self._log_entry(obj['logs'], 'entry'):\n yield entry\n elif self.format == 'xml':\n nodes = elem.findall('.')\n for node in nodes:\n yield node\n\n def tail(self, lines):\n elem, obj = self._log_get(nlogs=lines,\n filter=self.filter)\n if not obj:\n raise PanForestError(\"Can't get log\")\n\n count = self._log_count(obj)\n if count == 0:\n return None\n\n if self.format == 'python':\n return obj\n elif self.format == 'xml':\n return elem\n\n def _log_get(self, nlogs=None, skip=None, filter=None):\n try:\n self.xapi.log(log_type=self.log_type,\n skip=skip,\n nlogs=nlogs,\n filter=filter)\n except pan.xapi.PanXapiError as e:\n raise PanForestError('pan.xapi.PanXapi: %s' % e)\n\n path = './result/log/logs'\n elem = self.xapi.element_root.find(path)\n if elem is None:\n raise PanForestError('No %s in element_root' % path)\n\n obj = self._xml_python(elem)\n\n LOG.debug(pprint.pformat(obj))\n\n return elem, obj\n\n def _sleeper(self):\n \"\"\"return iterator of seconds to sleep until log match\"\"\"\n\n try:\n xrange(1)\n except NameError:\n _range = range\n else:\n _range = xrange\n\n x = _range(1, 60, 3)\n\n return iter(x)\n\n @staticmethod\n def _xml_python(elem, path=None):\n try:\n conf = pan.config.PanConfig(config=elem)\n except pan.config.PanConfigError as e:\n raise PanForestError('pan.config.PanConfigError: %s' % e)\n\n obj = conf.python(path)\n return obj\n\n def _log_count(self, obj):\n if not ('logs' in obj and 'count' in obj['logs']):\n raise PanForestError('logs count not found')\n\n try:\n count = int(obj['logs']['count'])\n except ValueError:\n raise PanForestError('count not numeric: %s' %\n obj['logs']['count'])\n\n LOG.debug('count: %d' % count)\n\n return count\n\n def _log_seqno(self, obj):\n x = self._log_entry(obj['logs']['entry'][0], 'seqno')\n try:\n n = int(x)\n except ValueError:\n raise PanForestError('seqno not numeric: %s' % x)\n\n return n\n\n @staticmethod\n def _log_entry(obj, key):\n if key not in obj:\n raise PanForestError('key not in entry: %s' % key)\n\n return obj[key]\n\n def _filter(self, seqno):\n s = 'not (seqno leq %d)' % seqno\n\n if self.filter:\n s = self.filter + ' and ' + s\n\n return s\n" }, { "path": "minemeld/run/__init__.py", "content": "" }, { "path": "minemeld/run/cacert_merge.py", "content": "# Copyright 2015-2017 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport logging\nimport argparse\nimport sys\nimport os\nimport os.path\n\nimport yaml\n\ntry:\n from ssl import create_default_context\nexcept ImportError:\n def create_default_context(cafile, cadata):\n print('WARNING: old python version (< 2.7.9) - certificate verification not performed')\n\ntry:\n import certifi\n CERTIFI_WHERE = certifi.where()\nexcept ImportError:\n # XXX Error?\n CERTIFI_WHERE = None\n\n\nLOG = logging.getLogger(__name__)\n\n\ndef main():\n logging.basicConfig(\n level=logging.DEBUG,\n format=\"%(asctime)s (%(process)d)%(module)s.%(funcName)s\"\n \" %(levelname)s: %(message)s\",\n datefmt=\"%Y-%m-%dT%H:%M:%S\"\n )\n\n parser = argparse.ArgumentParser(usage='%(prog)s [options] [cafile ...]')\n parser.add_argument('--no-merge-certifi',\n action='store_const',\n const=True,\n help='do not merge certifi CA bundle '\n '(default: merge \"%s\")' % CERTIFI_WHERE)\n parser.add_argument('--config',\n help='configuration file path (default: no config)')\n parser.add_argument('--dst',\n required=True,\n help='destination CA bundle path')\n parser.add_argument('cafile',\n nargs='*',\n help='local CA bundle file(s) '\n '(default: stdin)')\n args = parser.parse_args()\n\n try:\n certs = open(args.dst, 'w')\n except IOError as e:\n LOG.error('open: %s: %s' % (args.dst, e))\n return 1\n\n config = {\n 'no_merge_certifi': False\n }\n\n if args.config:\n with open(args.config, 'r') as f:\n loaded_config = yaml.safe_load(f)\n if loaded_config is not None:\n config.update(loaded_config)\n\n config.update({k: v for k, v in vars(args).iteritems() if v is not None})\n LOG.info('config: {}'.format(config))\n\n if not config['no_merge_certifi'] and CERTIFI_WHERE:\n try:\n with open(CERTIFI_WHERE) as f:\n buf = f.read()\n except IOError as e:\n LOG.error('%s: %s' % (CERTIFI_WHERE, e))\n return 1\n try:\n certs.write(buf)\n except IOError as e:\n LOG.error('%s: %s' % (args.dst, e))\n return 1\n\n if args.cafile:\n for x in args.cafile:\n files = [x]\n if os.path.isdir(x):\n files = [os.path.join(x, e) for e in os.listdir(x)]\n\n for fname in files:\n verify_cafile(cafile=fname)\n try:\n with open(fname) as f:\n buf = f.read()\n except IOError as e:\n LOG.error('%s: %s' % (fname, e))\n return 1\n try:\n certs.write(buf)\n except IOError as e:\n LOG.error('%s: %s' % (args.dst, e))\n return 1\n else:\n x = sys.stdin.read()\n try:\n x = unicode(x)\n except NameError:\n # 3.x\n pass\n verify_cafile(cadata=x)\n try:\n certs.write(x)\n except IOError as e:\n LOG.error('%s: %s' % (args.dst, e))\n return 1\n\n certs.close()\n verify_cafile(cafile=args.dst)\n\n return 0\n\n\ndef verify_cafile(cafile=None, cadata=None):\n try:\n create_default_context(cafile=cafile, cadata=cadata)\n except IOError as e:\n if cafile:\n LOG.error('Invalid cafile %s: %s' % (cafile, e))\n else:\n LOG.error('Invalid cadata: %s' % e)\n sys.exit(1)\n" }, { "path": "minemeld/run/config.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import print_function\n\nimport sys\nimport time\nimport os\nimport os.path\nimport logging\nimport shutil\nimport re\nimport json\nimport multiprocessing\nimport functools\nfrom collections import namedtuple\n\nimport yaml\nimport gevent.core\n\nimport minemeld.loader\n\n\n__all__ = ['load_config', 'validate_config', 'resolve_prototypes']\n\n\n# disables construction of timestamp objects\nyaml.SafeLoader.add_constructor(\n u'tag:yaml.org,2002:timestamp',\n yaml.SafeLoader.construct_yaml_str\n)\n\n\nLOG = logging.getLogger(__name__)\n\nCOMMITTED_CONFIG = 'committed-config.yml'\nRUNNING_CONFIG = 'running-config.yml'\nPROTOTYPE_ENV = 'MINEMELD_PROTOTYPE_PATH'\nMGMTBUS_NUM_CONNS_ENV = 'MGMTBUS_NUM_CONNS'\nFABRIC_NUM_CONNS_ENV = 'FABRIC_NUM_CONNS'\n\nCHANGE_ADDED = 0\nCHANGE_DELETED = 1\nCHANGE_INPUT_ADDED = 2\nCHANGE_INPUT_DELETED = 3\nCHANGE_OUTPUT_ENABLED = 4\nCHANGE_OUTPUT_DISABLED = 5\n\n_ConfigChange = namedtuple(\n '_ConfigChange',\n ['nodename', 'nodeclass', 'change', 'detail']\n)\n\n_Config = namedtuple(\n '_Config',\n ['nodes', 'fabric', 'mgmtbus', 'changes']\n)\n\n\nclass MineMeldConfigChange(_ConfigChange):\n def __new__(_cls, nodename, nodeclass, change, detail=None):\n return _ConfigChange.__new__(\n _cls,\n nodename=nodename,\n nodeclass=nodeclass,\n change=change,\n detail=detail\n )\n\n\nclass MineMeldConfig(_Config):\n def as_nset(self):\n result = set()\n for nname, nvalue in self.nodes.iteritems():\n result.add(\n json.dumps(\n [nname, nvalue.get('class', None)],\n sort_keys=True\n )\n )\n return result\n\n def compute_changes(self, oconfig):\n if oconfig is None:\n # oconfig is None, mark everything as added\n for nodename, nodeattrs in self.nodes.iteritems():\n self.changes.append(\n MineMeldConfigChange(nodename=nodename, nodeclass=nodeattrs['class'], change=CHANGE_ADDED)\n )\n return\n\n my_nset = self.as_nset()\n other_nset = oconfig.as_nset()\n\n deleted = other_nset - my_nset\n added = my_nset - other_nset\n untouched = my_nset & other_nset\n\n # mark delted as deleted\n for snode in deleted:\n nodename, nodeclass = json.loads(snode)\n change = MineMeldConfigChange(\n nodename=nodename,\n nodeclass=nodeclass,\n change=CHANGE_DELETED,\n detail=oconfig.nodes[nodename]\n )\n self.changes.append(change)\n\n # mark added as added\n for snode in added:\n nodename, nodeclass = json.loads(snode)\n change = MineMeldConfigChange(\n nodename=nodename,\n nodeclass=nodeclass,\n change=CHANGE_ADDED\n )\n self.changes.append(change)\n\n # check inputs/output for untouched\n for snode in untouched:\n nodename, nodeclass = json.loads(snode)\n\n my_inputs = set(self.nodes[nodename].get('inputs', []))\n other_inputs = set(oconfig.nodes[nodename].get('inputs', []))\n\n iadded = my_inputs - other_inputs\n ideleted = other_inputs - my_inputs\n\n for i in iadded:\n change = MineMeldConfigChange(\n nodename=nodename,\n nodeclass=nodeclass,\n change=CHANGE_INPUT_ADDED,\n detail=i\n )\n self.changes.append(change)\n\n for i in ideleted:\n change = MineMeldConfigChange(\n nodename=nodename,\n nodeclass=nodeclass,\n change=CHANGE_INPUT_DELETED,\n detail=i\n )\n self.changes.append(change)\n\n my_output = self.nodes[nodename].get('output', False)\n other_output = oconfig.nodes[nodename].get('output', False)\n\n if my_output == other_output:\n continue\n\n change_type = CHANGE_OUTPUT_DISABLED\n if my_output:\n change_type = CHANGE_OUTPUT_ENABLED\n\n change = MineMeldConfigChange(\n nodename=nodename,\n nodeclass=nodeclass,\n change=change_type\n )\n self.changes.append(change)\n\n @classmethod\n def from_dict(cls, dconfig=None):\n if dconfig is None:\n dconfig = {}\n\n fabric = dconfig.get('fabric', None)\n if fabric is None:\n fabric_num_conns = int(\n os.getenv(FABRIC_NUM_CONNS_ENV, 50)\n )\n\n fabric = {\n 'class': 'ZMQRedis',\n 'config': {\n 'num_connections': fabric_num_conns,\n 'priority': gevent.core.MINPRI # pylint:disable=E1101\n }\n }\n\n mgmtbus = dconfig.get('mgmtbus', None)\n if mgmtbus is None:\n mgmtbus_num_conns = int(\n os.getenv(MGMTBUS_NUM_CONNS_ENV, 10)\n )\n\n mgmtbus = {\n 'transport': {\n 'class': 'ZMQRedis',\n 'config': {\n 'num_connections': mgmtbus_num_conns,\n 'priority': gevent.core.MAXPRI # pylint:disable=E1101\n }\n },\n 'master': {},\n 'slave': {}\n }\n\n nodes = dconfig.get('nodes', None)\n if nodes is None:\n nodes = {}\n\n return cls(nodes=nodes, fabric=fabric, mgmtbus=mgmtbus, changes=[])\n\n\ndef _load_node_prototype(protoname, paths):\n proto_module, proto_name = protoname.rsplit('.', 1)\n\n pmodule = None\n pmprotos = {}\n for p in paths:\n pmpath = os.path.join(p, proto_module+'.yml')\n\n try:\n with open(pmpath, 'r') as pf:\n pmodule = yaml.safe_load(pf)\n\n if pmodule is None:\n pmodule = {}\n except IOError:\n pmodule = None\n continue\n\n pmprotos = pmodule.get('prototypes', {})\n\n if proto_name not in pmprotos:\n pmodule = None\n continue\n\n if 'class' not in pmprotos[proto_name]:\n pmodule = None\n continue\n\n return pmprotos[proto_name]\n\n raise RuntimeError('Unable to load prototype %s: '\n ' not found' % (protoname))\n\n\ndef _load_config_from_file(f):\n valid = True\n config = yaml.safe_load(f)\n\n if not isinstance(config, dict) and config is not None:\n raise ValueError('Invalid config YAML type')\n\n return valid, MineMeldConfig.from_dict(config)\n\n\ndef _load_and_validate_config_from_file(path):\n valid = False\n config = None\n\n if os.path.isfile(path):\n try:\n with open(path, 'r') as cf:\n valid, config = _load_config_from_file(cf)\n if not valid:\n LOG.error('Invalid config file {}'.format(path))\n except (RuntimeError, IOError):\n LOG.exception(\n 'Error loading config {}, config ignored'.format(path)\n )\n valid, config = False, None\n\n if valid and config is not None:\n valid = resolve_prototypes(config)\n\n if valid and config is not None:\n vresults = validate_config(config)\n if len(vresults) != 0:\n LOG.error('Invalid config {}: {}'.format(\n path,\n ', '.join(vresults)\n ))\n valid = False\n\n return valid, config\n\n\ndef _destroy_node(change, installed_nodes=None, installed_nodes_gcs=None):\n LOG.info('Destroying {!r}'.format(change))\n\n destroyed_name = change.nodename\n destroyed_class = change.nodeclass\n if destroyed_class is None:\n LOG.error('Node {} with no class destroyed'.format(destroyed_name))\n return 1\n\n # load node class GC from entry_point or from \"gc\" staticmethod of class\n node_gc = None\n mmep = installed_nodes_gcs.get(destroyed_class, None)\n if mmep is None:\n mmep = installed_nodes.get(destroyed_class, None)\n\n try:\n nodep = mmep.ep.load()\n\n if hasattr(nodep, 'gc'):\n node_gc = nodep.gc\n except ImportError:\n LOG.exception(\"Error checking node class {} for gc method\".format(destroyed_class))\n else:\n try:\n node_gc = mmep.ep.load()\n except ImportError:\n LOG.exception(\"Error resolving gc for class {}\".format(destroyed_class))\n if node_gc is None:\n LOG.error('Node {} with class {} with no garbage collector destroyed'.format(\n destroyed_name, destroyed_class\n ))\n return 1\n\n try:\n node_gc(\n destroyed_name,\n config=change.detail.get('config', None)\n )\n\n except:\n LOG.exception('Exception destroying old node {} of class {}'.format(\n destroyed_name, destroyed_class\n ))\n return 1\n\n return 0\n\n\ndef _destroy_old_nodes(config):\n # this destroys resources used by destroyed nodes\n # a nodes has been destroyed if a node with same\n # name & config does not exist in the new config\n # the case of different node config but same and name\n # and class is handled by node itself\n destroyed_nodes = [c for c in config.changes if c.change == CHANGE_DELETED]\n LOG.info('Destroyed nodes: {!r}'.format(destroyed_nodes))\n if len(destroyed_nodes) == 0:\n return\n\n installed_nodes = minemeld.loader.map(minemeld.loader.MM_NODES_ENTRYPOINT)\n installed_nodes_gcs = minemeld.loader.map(minemeld.loader.MM_NODES_GCS_ENTRYPOINT)\n\n dpool = multiprocessing.Pool()\n _bound_destroy_node = functools.partial(\n _destroy_node,\n installed_nodes=installed_nodes,\n installed_nodes_gcs=installed_nodes_gcs\n )\n dpool.imap_unordered(\n _bound_destroy_node,\n destroyed_nodes\n )\n dpool.close()\n dpool.join()\n dpool = None\n\n\ndef _load_config_from_dir(path):\n ccpath = os.path.join(\n path,\n COMMITTED_CONFIG\n )\n rcpath = os.path.join(\n path,\n RUNNING_CONFIG\n )\n\n ccvalid, cconfig = _load_and_validate_config_from_file(ccpath)\n rcvalid, rcconfig = _load_and_validate_config_from_file(rcpath)\n\n if not rcvalid and not ccvalid:\n # both running and canidate are not valid\n print(\n \"At least one of\", RUNNING_CONFIG,\n \"or\", COMMITTED_CONFIG,\n \"should exist in\", path,\n file=sys.stderr\n )\n sys.exit(1)\n\n elif rcvalid and not ccvalid:\n # running is valid but candidate is not\n return rcconfig\n\n elif not rcvalid and ccvalid:\n # candidate is valid while running is not\n LOG.info('Switching to candidate config')\n cconfig.compute_changes(rcconfig)\n LOG.info('Changes in config: {!r}'.format(cconfig.changes))\n _destroy_old_nodes(cconfig)\n if rcconfig is not None:\n shutil.copyfile(\n rcpath,\n '{}.{}'.format(rcpath, int(time.time()))\n )\n shutil.copyfile(ccpath, rcpath)\n return cconfig\n\n elif rcvalid and ccvalid:\n LOG.info('Switching to candidate config')\n cconfig.compute_changes(rcconfig)\n LOG.info('Changes in config: {!r}'.format(cconfig.changes))\n _destroy_old_nodes(cconfig)\n shutil.copyfile(\n rcpath,\n '{}.{}'.format(rcpath, int(time.time()))\n )\n shutil.copyfile(ccpath, rcpath)\n return cconfig\n\n\ndef _detect_cycles(nodes):\n # using Topoligical Sorting to detect cycles in graph, see Wikipedia\n graph = {}\n S = set()\n L = []\n\n for n in nodes:\n graph[n] = {\n 'inputs': [],\n 'outputs': []\n }\n\n for n, v in nodes.iteritems():\n for i in v.get('inputs', []):\n if i in graph:\n graph[i]['outputs'].append(n)\n graph[n]['inputs'].append(i)\n\n for n, v in graph.iteritems():\n if len(v['inputs']) == 0:\n S.add(n)\n\n while len(S) != 0:\n n = S.pop()\n L.append(n)\n\n for m in graph[n]['outputs']:\n graph[m]['inputs'].remove(n)\n if len(graph[m]['inputs']) == 0:\n S.add(m)\n graph[n]['outputs'] = []\n\n nedges = 0\n for n, v in graph.iteritems():\n nedges += len(v['inputs'])\n nedges += len(v['outputs'])\n\n return nedges == 0\n\n\ndef resolve_prototypes(config):\n # retrieve prototype dir from environment\n # used for main library and local library\n paths = os.getenv(PROTOTYPE_ENV, None)\n if paths is None:\n raise RuntimeError('Unable to load prototypes: %s '\n 'environment variable not set' %\n (PROTOTYPE_ENV))\n paths = paths.split(':')\n\n # add prototype dirs from extension to paths\n prototypes_entrypoints = minemeld.loader.map(minemeld.loader.MM_PROTOTYPES_ENTRYPOINT)\n for epname, mmep in prototypes_entrypoints.iteritems():\n if not mmep.loadable:\n LOG.info('Prototypes entrypoint {} not loadable'.format(epname))\n continue\n\n try:\n ep = mmep.ep.load()\n # we add prototype paths in front, to let extensions override default protos\n paths.insert(0, ep())\n\n except:\n LOG.exception(\n 'Exception retrieving path from prototype entrypoint {}'.format(epname)\n )\n\n # resolve all prototypes\n valid = True\n\n nodes_config = config.nodes\n for _, nconfig in nodes_config.iteritems():\n if 'prototype' in nconfig:\n try:\n nproto = _load_node_prototype(nconfig['prototype'], paths)\n except RuntimeError as e:\n LOG.error('Error loading prototype {}: {}'.format(\n nconfig['prototype'],\n str(e)\n ))\n valid = False\n continue\n\n nconfig.pop('prototype')\n\n nconfig['class'] = nproto['class']\n nproto_config = nproto.get('config', {})\n nproto_config.update(\n nconfig.get('config', {})\n )\n nconfig['config'] = nproto_config\n\n return valid\n\n\ndef validate_config(config):\n result = []\n\n nodes = config.nodes\n\n for n in nodes.keys():\n if re.match('^[a-zA-Z0-9_\\-]+$', n) is None: # pylint:disable=W1401\n result.append('%s node name is invalid' % n)\n\n for n, v in nodes.iteritems():\n for i in v.get('inputs', []):\n if i not in nodes:\n result.append('%s -> %s is unknown' % (n, i))\n continue\n\n if not nodes[i].get('output', False):\n result.append('%s -> %s output disabled' %\n (n, i))\n\n installed_nodes = minemeld.loader.map(minemeld.loader.MM_NODES_ENTRYPOINT)\n for n, v in nodes.iteritems():\n nclass = v.get('class', None)\n if nclass is None:\n result.append('No class in {}'.format(n))\n continue\n\n mmep = installed_nodes.get(nclass, None)\n if mmep is None:\n result.append(\n 'Unknown node class {} in {}'.format(nclass, n)\n )\n continue\n\n if not mmep.loadable:\n result.append(\n 'Class {} in {} not safe to load'.format(nclass, n)\n )\n\n if not _detect_cycles(nodes):\n result.append('loop detected')\n\n return result\n\n\ndef load_config(config_path):\n if os.path.isdir(config_path):\n return _load_config_from_dir(config_path)\n\n # this is just a file, as we can't do a delta\n # we just load it and mark all the nodes as added\n valid, config = _load_and_validate_config_from_file(config_path)\n if not valid:\n raise RuntimeError('Invalid config')\n config.compute_changes(None)\n\n return config\n" }, { "path": "minemeld/run/console.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport gevent\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport logging\nimport signal\nimport time\nimport uuid\nimport json\n\nimport click\n\nimport minemeld.comm\nimport minemeld.mgmtbus\nimport minemeld.traced\n\nLOG = logging.getLogger(__name__)\n\n\ndef _send_cmd(ctx, target, command, params=None, source=True):\n if params is None:\n params = {}\n\n if source:\n params['source'] = ctx.obj['SOURCE']\n return ctx.obj['COMM'].send_rpc(\n target,\n command,\n params\n )\n\n\ndef _print_json(obj):\n print json.dumps(\n obj,\n indent=4,\n sort_keys=True\n )\n\n\n@click.group()\n@click.option('--comm-class', default='ZMQRedis',\n metavar='CLASSNAME')\n@click.option('--verbose', count=True)\n@click.pass_context\ndef cli(ctx, verbose, comm_class):\n comm_class = str(comm_class)\n source = 'console-%d' % int(time.time())\n\n loglevel = logging.WARNING\n if verbose > 0:\n loglevel = logging.INFO\n if verbose > 1:\n loglevel = logging.DEBUG\n logging.basicConfig(\n level=loglevel,\n format=\"%(asctime)s (%(process)d)%(module)s.%(funcName)s\"\n \" %(levelname)s: %(message)s\",\n datefmt=\"%Y-%m-%dT%H:%M:%S\"\n )\n\n comm = minemeld.comm.factory(comm_class, {}) # XXX should support config\n\n gevent.signal(signal.SIGTERM, comm.stop)\n gevent.signal(signal.SIGQUIT, comm.stop)\n gevent.signal(signal.SIGINT, comm.stop)\n\n comm.start()\n\n ctx.obj['COMM'] = comm\n ctx.obj['SOURCE'] = source\n\n\n@cli.command()\n@click.argument('target')\n@click.pass_context\ndef length(ctx, target):\n if target is None:\n raise click.UsageError(message='target required')\n\n _print_json(_send_cmd(ctx, target, 'length'))\n\n ctx.obj['COMM'].stop()\n\n\n@cli.command()\n@click.argument('target')\n@click.pass_context\ndef hup(ctx, target):\n if target is None:\n raise click.UsageError(message='target required')\n\n target = 'mbus:directslave:'+target\n _print_json(_send_cmd(ctx, target, 'hup'))\n\n ctx.obj['COMM'].stop()\n\n\n@cli.command()\n@click.argument('target', required=False, default=None)\n@click.pass_context\ndef status(ctx, target):\n if target is None:\n target = minemeld.mgmtbus.MGMTBUS_MASTER\n else:\n target = 'mbus:directslave:'+target\n\n _print_json(_send_cmd(ctx, target, 'status', source=False))\n\n ctx.obj['COMM'].stop()\n\n\n@cli.command(name='signal')\n@click.argument('signal')\n@click.argument('target')\n@click.pass_context\ndef mm_signal(ctx, signal, target):\n target = 'mbus:directslave:'+target\n _print_json(_send_cmd(ctx, target, 'signal', source=False, params={'signal': signal}))\n\n ctx.obj['COMM'].stop()\n\n\n# XXX query should subscribe to the Redis topic to dump the\n# query results\n@cli.command()\n@click.argument('query')\n@click.option('--from-counter', default=None, type=int)\n@click.option('--from-timestamp', default=None, type=int)\n@click.option('--num-lines', default=100, type=int)\n@click.option('--query-uuid', default=None)\n@click.pass_context\ndef query(ctx, query, from_counter, from_timestamp, num_lines, query_uuid):\n if query_uuid is None:\n query_uuid = str(uuid.uuid4())\n\n _print_json(\n _send_cmd(\n ctx,\n minemeld.traced.QUERY_QUEUE,\n 'query',\n source=False,\n params={\n 'uuid': query_uuid,\n 'timestamp': from_timestamp,\n 'counter': from_counter,\n 'num_lines': num_lines,\n 'query': query\n }\n )\n )\n\n ctx.obj['COMM'].stop()\n\n\ndef main():\n cli(obj={}) # pylint:disable=E1123,E1120\n" }, { "path": "minemeld/run/extgit.py", "content": "# Copyright 2019 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport sys\nimport logging\nimport argparse\nimport subprocess\nimport os.path\nimport shutil\n\nfrom minemeld import __version__\n\n\ndef _parse_args():\n parser = argparse.ArgumentParser(\n description=\"Install MineMeld extension from git repo\"\n )\n parser.add_argument(\n '--version',\n action='version',\n version=__version__\n )\n parser.add_argument(\n 'git_path',\n action='store',\n metavar='GIT_PATH',\n help='path of git executable'\n )\n parser.add_argument(\n 'git_ref',\n action='store',\n metavar='GIT_REF',\n help='git reference'\n )\n parser.add_argument(\n 'git_endpoint',\n action='store',\n metavar='GIT_ENDPOINT',\n help='git endpoint'\n )\n parser.add_argument(\n 'install_directory',\n action='store',\n metavar='INSTALL_DIRECTORY',\n help='directory to install the extension into'\n )\n return parser.parse_args()\n\n\ndef main():\n logging.basicConfig(level=logging.DEBUG)\n\n args = _parse_args()\n\n git_args = [\n args.git_path,\n 'clone',\n '-b', args.git_ref,\n '--depth', '1',\n args.git_endpoint,\n args.install_directory\n ]\n logging.info('Calling git: {!r}'.format(git_args))\n subprocess.check_call(git_args)\n\n if not os.path.exists(os.path.join(args.install_directory, 'minemeld.json')):\n logging.error('minemeld.json does not exists in install directory - invalid extension')\n try:\n shutil.rmtree(args.install_directory)\n except Exception as _:\n logging.exception('Error removing install directory')\n\n sys.exit(1)\n\n sys.exit(0)\n" }, { "path": "minemeld/run/freeze.py", "content": "# Copyright 2017 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport sys\nimport logging\nimport argparse\n\nimport minemeld.extensions\nfrom minemeld import __version__\n\n\ndef _parse_args():\n parser = argparse.ArgumentParser(\n description=\"Freeze MineMeld extensions\"\n )\n parser.add_argument(\n '--version',\n action='version',\n version=__version__\n )\n parser.add_argument(\n 'library',\n action='store',\n metavar='LIBRARY',\n help='path of the MineMeld library directory'\n )\n parser.add_argument(\n 'outfile',\n action='store',\n metavar='OUTFILE',\n default='-',\n nargs='?',\n help='path of the file to write the output to. (default: stdout)'\n )\n return parser.parse_args()\n\n\ndef main():\n logging.basicConfig(level=logging.DEBUG)\n\n args = _parse_args()\n\n if args.outfile == '-':\n of = sys.stdout\n else:\n of = open(args.outfile, 'w+')\n\n frozenext = minemeld.extensions.freeze(args.library)\n for e in frozenext:\n of.write('{}\\n'.format(e))\n\n of.close()\n" }, { "path": "minemeld/run/launcher.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom __future__ import print_function\n\nimport gevent\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport os.path\nimport logging\nimport signal\nimport multiprocessing\nimport argparse\nimport os\nimport math\n\nimport psutil\n\nimport minemeld.chassis\nimport minemeld.mgmtbus\nimport minemeld.comm\nimport minemeld.run.config\n\nfrom minemeld import __version__\n\nLOG = logging.getLogger(__name__)\n\n\ndef _run_chassis(fabricconfig, mgmtbusconfig, fts):\n try:\n # lower priority to make master and web\n # more \"responsive\"\n os.nice(5)\n\n c = minemeld.chassis.Chassis(\n fabricconfig['class'],\n fabricconfig['config'],\n mgmtbusconfig\n )\n c.configure(fts)\n\n gevent.signal(signal.SIGUSR1, c.stop)\n\n while not c.fts_init():\n if c.poweroff.wait(timeout=0.1) is not None:\n break\n\n gevent.sleep(1)\n\n LOG.info('Nodes initialized')\n\n try:\n c.poweroff.wait()\n LOG.info('power off')\n\n except KeyboardInterrupt:\n LOG.error(\"We should not be here !\")\n c.stop()\n\n except:\n LOG.exception('Exception in chassis main procedure')\n raise\n\n\ndef _check_disk_space(num_nodes):\n free_disk_per_node = int(os.environ.get(\n 'MM_DISK_SPACE_PER_NODE',\n 10*1024 # default: 10MB per node\n ))\n needed_disk = free_disk_per_node*num_nodes*1024\n free_disk = psutil.disk_usage('.').free\n\n LOG.debug('Disk space - needed: {} available: {}'.format(needed_disk, free_disk))\n\n if free_disk <= needed_disk:\n LOG.critical(\n ('Not enough space left on the device, available: {} needed: {}'\n ' - please delete traces, logs and old engine versions and restart').format(\n free_disk, needed_disk\n )\n )\n return None\n\n return free_disk\n\n\ndef _parse_args():\n parser = argparse.ArgumentParser(\n description=\"Low-latency threat indicators processor\"\n )\n parser.add_argument(\n '--version',\n action='version',\n version=__version__\n )\n parser.add_argument(\n '--multiprocessing',\n default=0,\n type=int,\n action='store',\n metavar='NP',\n help='enable multiprocessing. NP is the number of chassis, '\n '0 to use two chassis per machine core (default)'\n )\n parser.add_argument(\n '--nodes-per-chassis',\n default=15.0,\n type=float,\n action='store',\n metavar='NPC',\n help='number of nodes per chassis (default 15)'\n )\n parser.add_argument(\n '--verbose',\n action='store_true',\n help='verbose'\n )\n parser.add_argument(\n 'config',\n action='store',\n metavar='CONFIG',\n help='path of the config file or of the config directory'\n )\n return parser.parse_args()\n\n\ndef _setup_environment(config):\n # make config dir available to nodes\n cdir = config\n if not os.path.isdir(cdir):\n cdir = os.path.dirname(config)\n os.environ['MM_CONFIG_DIR'] = cdir\n\n if not 'REQUESTS_CA_BUNDLE' in os.environ and 'MM_CA_BUNDLE' in os.environ:\n os.environ['REQUESTS_CA_BUNDLE'] = os.environ['MM_CA_BUNDLE']\n\n\ndef main():\n mbusmaster = None\n processes_lock = None\n processes = None\n disk_space_monitor_glet = None\n\n def _cleanup():\n if mbusmaster is not None:\n mbusmaster.checkpoint_graph()\n\n if processes_lock is None:\n return\n\n with processes_lock:\n if processes is None:\n return\n\n for p in processes:\n if not p.is_alive():\n continue\n\n try:\n os.kill(p.pid, signal.SIGUSR1)\n except OSError:\n continue\n\n while sum([int(t.is_alive()) for t in processes]) != 0:\n gevent.sleep(1)\n\n def _sigint_handler():\n LOG.info('SIGINT received')\n _cleanup()\n signal_received.set()\n\n def _sigterm_handler():\n LOG.info('SIGTERM received')\n _cleanup()\n signal_received.set()\n\n def _disk_space_monitor(num_nodes):\n while True:\n if _check_disk_space(num_nodes=num_nodes) is None:\n _cleanup()\n signal_received.set()\n break\n\n gevent.sleep(60)\n\n args = _parse_args()\n\n # logging\n loglevel = logging.INFO\n if args.verbose:\n loglevel = logging.DEBUG\n\n logging.basicConfig(\n level=loglevel,\n format=\"%(asctime)s (%(process)d)%(module)s.%(funcName)s\"\n \" %(levelname)s: %(message)s\",\n datefmt=\"%Y-%m-%dT%H:%M:%S\"\n )\n LOG.info(\"Starting mm-run.py version %s\", __version__)\n LOG.info(\"mm-run.py arguments: %s\", args)\n\n _setup_environment(args.config)\n\n # load and validate config\n config = minemeld.run.config.load_config(args.config)\n\n LOG.info(\"mm-run.py config: %s\", config)\n\n if _check_disk_space(num_nodes=len(config.nodes)) is None:\n LOG.critical('Not enough disk space available, exit')\n return 2\n\n np = args.multiprocessing\n if np == 0:\n np = multiprocessing.cpu_count()\n LOG.info('multiprocessing: #cores: %d', multiprocessing.cpu_count())\n LOG.info(\"multiprocessing: max #chassis: %d\", np)\n\n npc = args.nodes_per_chassis\n if npc <= 0:\n LOG.critical('nodes-per-chassis should be a positive integer')\n return 2\n\n np = min(\n int(math.ceil(len(config.nodes)/npc)),\n np\n )\n LOG.info(\"Number of chassis: %d\", np)\n\n ftlists = [{} for j in range(np)]\n j = 0\n for ft in config.nodes:\n pn = j % len(ftlists)\n ftlists[pn][ft] = config.nodes[ft]\n j += 1\n\n # cleanup\n if config.mgmtbus['transport']['class'] != config.fabric['class']:\n raise ValueError('mgmtbus class and fabric class should match')\n minemeld.comm.cleanup(config.fabric['class'], config.fabric['config'])\n\n signal.signal(signal.SIGINT, signal.SIG_IGN)\n signal.signal(signal.SIGTERM, signal.SIG_IGN)\n\n processes = []\n for g in ftlists:\n if len(g) == 0:\n continue\n\n p = multiprocessing.Process(\n target=_run_chassis,\n args=(\n config.fabric,\n config.mgmtbus,\n g\n )\n )\n processes.append(p)\n p.start()\n\n processes_lock = gevent.lock.BoundedSemaphore()\n signal_received = gevent.event.Event()\n\n gevent.signal(signal.SIGINT, _sigint_handler)\n gevent.signal(signal.SIGTERM, _sigterm_handler)\n\n try:\n mbusmaster = minemeld.mgmtbus.master_factory(\n config=config.mgmtbus['master'],\n comm_class=config.mgmtbus['transport']['class'],\n comm_config=config.mgmtbus['transport']['config'],\n nodes=config.nodes.keys(),\n num_chassis=len(processes)\n )\n mbusmaster.start()\n mbusmaster.wait_for_chassis(timeout=10)\n # here nodes are all CONNECTED, fabric and mgmtbus up, with mgmtbus\n # dispatching and fabric not dispatching\n mbusmaster.start_status_monitor()\n mbusmaster.init_graph(config)\n # here nodes are all INIT\n mbusmaster.start_chassis()\n # here nodes should all be starting\n\n except Exception:\n LOG.exception('Exception initializing graph')\n _cleanup()\n raise\n\n disk_space_monitor_glet = gevent.spawn(_disk_space_monitor, len(config.nodes))\n\n try:\n while not signal_received.wait(timeout=1.0):\n with processes_lock:\n r = [int(t.is_alive()) for t in processes]\n if sum(r) != len(processes):\n LOG.info(\"One of the chassis has stopped, exit\")\n break\n\n except KeyboardInterrupt:\n LOG.info(\"Ctrl-C received, exiting\")\n\n except:\n LOG.exception(\"Exception in main loop\")\n\n if disk_space_monitor_glet is not None:\n disk_space_monitor_glet.kill()\n" }, { "path": "minemeld/run/restore.py", "content": "#!/usr/bin/env python\n\nimport os\nimport signal\nimport time\nimport logging\nimport argparse\nimport xmlrpclib\nfrom zipfile import ZipFile\nfrom collections import deque\nfrom contextlib import contextmanager\n\nimport yaml\nimport redis\nimport supervisor.xmlrpc\n\n\nREDIS_KEY_PREFIX = 'mm:config:'\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass BFile(object):\n def __init__(self, zip_path, type_, extracted_path=None, target_path=None):\n self.zip_path = zip_path\n self.type_ = type_\n self.extracted_path = extracted_path\n self.target_path = target_path\n\n def __repr__(self):\n return '{}({}) => {}({})'.format(\n self.zip_path,\n self.type_,\n self.target_path,\n self.extracted_path\n )\n\n\ndef _parse_args():\n parser = argparse.ArgumentParser(\n description=\"Restore full MineMeld backup\"\n )\n parser.add_argument(\n '--dry-run',\n action='store_true',\n help='Dry run'\n )\n parser.add_argument(\n '--configuration-path',\n action='store',\n help='Path to restore configuration files to'\n )\n parser.add_argument(\n '--prototypes-path',\n action='store',\n help='Path to restore prototypes to'\n )\n parser.add_argument(\n '--feeds-aaa-path',\n action='store',\n help='Path to restore feeds AAA configuration to'\n )\n parser.add_argument(\n '--feeds-aaa',\n action='append',\n help='Restore feeds AAA configuration'\n )\n parser.add_argument(\n '--certificates-path',\n action='store',\n help='Path to restore certificates to'\n )\n parser.add_argument(\n '--password',\n action='store',\n help='Password for the backup file'\n )\n parser.add_argument(\n 'backup',\n action='store',\n help='path of the backup file'\n )\n return parser.parse_args()\n\n\nclass ContextManagerStack(object):\n def __init__(self):\n self._stack = deque()\n\n def enter(self, cm):\n result = cm.__enter__()\n self._stack.append(cm.__exit__)\n\n return result\n\n def __enter__(self):\n return self\n\n def __exit__(self, *exc_info):\n while self._stack:\n cb = self._stack.pop()\n cb(*exc_info)\n\n\n@contextmanager\ndef handle_minemeld_engine(supervisor_url):\n sserver = xmlrpclib.ServerProxy(\n 'http://127.0.0.1',\n transport=supervisor.xmlrpc.SupervisorTransport(\n None,\n None,\n supervisor_url\n )\n )\n\n # check supervisor state\n sstate = sserver.supervisor.getState()\n if sstate['statecode'] == 2: # FATAL\n raise RuntimeError('Supervisor state: 2')\n\n if sstate['statecode'] != 1:\n raise RuntimeError(\n \"Supervisor transitioning to a new state, restore not performed\"\n )\n\n # check minemeld-engine state\n pstate = sserver.supervisor.getProcessInfo('minemeld-engine')['statename']\n if pstate not in ['STOPPED', 'EXITED', 'FATAL', 'RUNNING']:\n raise RuntimeError(\n (\"minemeld-engine transitioning to a new state, \" +\n \"restore not performed\")\n )\n\n # if minemeld-engine state is running, stop it\n if pstate == 'RUNNING':\n result = sserver.supervisor.stopProcess('minemeld-engine', False)\n if not result:\n raise RuntimeError('Stop minemeld-engine returned False')\n\n LOG.info('Stopping minemeld-engine')\n\n now = time.time()\n info = None\n while (time.time()-now) < 60*15*1000:\n info = sserver.supervisor.getProcessInfo('minemeld-engine')\n if info['statename'] == 'STOPPED':\n break\n time.sleep(5)\n\n if info is not None and info['statename'] != 'STOPPED':\n raise RuntimeError('Timeout during minemeld-engine stop')\n else:\n LOG.info('minemeld-engine not running: {}'.format(pstate))\n\n yield\n\n # we restart only if no Exception have been raised by other tasks\n sserver.supervisor.startProcess('minemeld-engine', False)\n started_at = time.time()\n\n # check minemeld-engine state\n pstate = sserver.supervisor.getProcessInfo('minemeld-engine')['statename']\n while pstate != 'RUNNING':\n LOG.info('minemeld-engine state: {}'.format(pstate))\n\n if pstate == 'FATAL':\n raise RuntimeError('minemeld-engine failed to start')\n\n if (time.time() - started_at) > 40:\n raise RuntimeError('minemeld-engine didn\\'t start in 40 seconds')\n\n time.sleep(1)\n\n pstate = sserver.supervisor.getProcessInfo('minemeld-engine')['statename']\n\n LOG.info('Started minemeld-engine')\n\n\n@contextmanager\ndef extract_file(backup_id, bfile, efile, configuration_path, prototypes_path, feeds_aaa_path, certificates_path):\n if efile.type_ == 'configuration':\n new_path = configuration_path\n elif efile.type_ == 'prototypes':\n new_path = prototypes_path\n elif efile.type_ == 'feeds_aaa':\n new_path = feeds_aaa_path\n elif efile.type_ == 'certificates':\n new_path = certificates_path\n if efile.zip_path.startswith('certs/site'):\n new_path = os.path.join(certificates_path, '/site')\n else:\n raise RuntimeError('Unknown file type: {!r}'.format(efile))\n new_path = os.path.join(\n new_path,\n '{}'.format(os.path.basename(efile.zip_path))\n )\n extracted_path = '{}.{}'.format(new_path, backup_id)\n LOG.info('Extracting {} to {}'.format(efile.zip_path, extracted_path))\n efile.extracted_path = extracted_path\n efile.target_path = new_path\n\n fin = bfile.open(efile.zip_path, 'r')\n with open(extracted_path, 'w') as fout:\n while True:\n b = fin.read(1024 * 1024)\n if not b:\n break\n fout.write(b)\n\n try:\n yield\n\n except:\n try:\n os.remove(extracted_path)\n LOG.info('Removed temporary file {}'.format(extracted_path))\n except:\n pass\n\n\n@contextmanager\ndef backup_file(old_file_path):\n new_path = '{}.bak'.format(old_file_path)\n os.rename(old_file_path, new_path)\n\n try:\n yield\n\n except:\n try:\n os.rename(new_path, old_file_path)\n\n except:\n LOG.error('Error restoring {} to {}'.format(new_path, old_file_path))\n\n else:\n try:\n os.remove(new_path)\n except:\n LOG.error('Error removing backup {}'.format(new_path))\n\n\n@contextmanager\ndef restore_file(f):\n LOG.info('Moving {} to {}'.format(f.extracted_path, f.target_path))\n os.rename(f.extracted_path, f.target_path)\n\n try:\n yield\n\n except:\n try:\n os.remove(f.target_path)\n except:\n LOG.error('Error removing extracted file during recovery: {}'.format(f.target_path))\n\n\ndef _list_of_configuration_files(bfile, flist):\n dir_prefix = 'config/'\n\n committed_config_path = os.path.join(dir_prefix, 'committed-config.yml')\n if committed_config_path not in flist:\n raise RuntimeError('No committed-config in backup')\n\n committed_config = yaml.safe_load(bfile.open(committed_config_path, 'r'))\n result = [BFile(zip_path=committed_config_path, type_='configuration')]\n prefixes = [os.path.join(dir_prefix, nname) for nname in committed_config['nodes']]\n for c in flist:\n if not c.startswith(dir_prefix):\n continue\n for p in prefixes:\n if c.startswith(p):\n result.append(BFile(zip_path=c, type_='configuration'))\n break\n\n return result\n\n\ndef _list_of_prototypes_files(bfile, flist):\n dir_prefix = 'prototypes/'\n\n result = []\n for c in flist:\n if c == dir_prefix:\n continue\n if not c.startswith(dir_prefix):\n continue\n\n result.append(BFile(zip_path=c, type_='prototypes'))\n\n return result\n\n\ndef _list_of_certificates_files(bfile, flist):\n # from the backup file we restore certs/config.yml\n # and all the files in certs/site/\n\n result = []\n for c in flist:\n if c == 'certs/site/':\n continue\n if not c == 'certs/config.yml' and not c.startswith('certs/site/'):\n continue\n\n result.append(BFile(zip_path=c, type_='certificates'))\n\n return result\n\n\ndef _list_of_feeds_aaa_files(faaf, bfile, flist):\n faaf_path = os.path.join('config/api/', faaf)\n if faaf_path in flist:\n return [BFile(zip_path=faaf_path, type_='feeds_aaa')]\n\n return []\n\n\ndef _reload_candidate_config(supervisor_url):\n SR = redis.StrictRedis()\n ckeys = SR.keys('{}*'.format(REDIS_KEY_PREFIX))\n if ckeys:\n for ck in ckeys:\n LOG.info('Deleting {}'.format(ck))\n SR.delete(ck)\n\n LOG.info('Candidate config keys deleted')\n\n sserver = xmlrpclib.ServerProxy(\n 'http://127.0.0.1',\n transport=supervisor.xmlrpc.SupervisorTransport(\n None,\n None,\n supervisor_url\n )\n )\n\n # check supervisor state\n sstate = sserver.supervisor.getState()\n if sstate['statecode'] == 2: # FATAL\n raise RuntimeError('Supervisor state: 2')\n\n if sstate['statecode'] != 1:\n raise RuntimeError(\n \"Supervisor transitioning to a new state, restore not performed\"\n )\n\n # check minemeld-engine state\n pinfo = sserver.supervisor.getProcessInfo('minemeld-web')\n if pinfo['statename'] != 'RUNNING':\n raise RuntimeError('minemeld-web not running, reload not sent')\n\n os.kill(pinfo['pid'], signal.SIGHUP)\n\n LOG.info('API process reloaded')\n\n\ndef main():\n supervisor_url = 'unix:///var/run/minemeld/minemeld.sock'\n\n logging.basicConfig(\n level=logging.DEBUG,\n format='%(asctime)s %(levelname)s: %(message)s'\n )\n\n args = _parse_args()\n\n supervisor_url = os.environ.get(\n 'SUPERVISOR_URL',\n supervisor_url\n )\n\n LOG.info('restore started: {!r}'.format(args))\n\n with ContextManagerStack() as cmstack:\n backup_id = os.path.basename(args.backup)\n bfile = cmstack.enter(ZipFile(args.backup, 'r'))\n\n if args.password:\n bfile.setpassword(args.password)\n\n contents = bfile.namelist()\n\n files = []\n if args.configuration_path:\n files.extend(_list_of_configuration_files(bfile, contents))\n if args.prototypes_path:\n files.extend(_list_of_prototypes_files(bfile, contents))\n if args.certificates_path:\n files.extend(_list_of_certificates_files(bfile, contents))\n if args.feeds_aaa_path:\n if not args.feeds_aaa:\n LOG.warning('No feeds AAA config file specified')\n else:\n for faaf in args.feeds_aaa:\n files.extend(_list_of_feeds_aaa_files(faaf, bfile, contents))\n LOG.info('List of files to be restored: {}'.format(files))\n\n # stop/start minemeld-engine\n cmstack.enter(handle_minemeld_engine(supervisor_url))\n\n # extract files\n for f in files:\n cmstack.enter(extract_file(\n backup_id=backup_id,\n bfile=bfile,\n efile=f,\n configuration_path=args.configuration_path,\n prototypes_path=args.prototypes_path,\n feeds_aaa_path=args.feeds_aaa_path,\n certificates_path=args.certificates_path\n ))\n LOG.info('Extracted files: {}'.format(files))\n\n # check if I can move old files\n for f in files:\n LOG.info('Checking {} for write permissions'.format(f.target_path))\n if not os.path.exists(f.target_path):\n continue\n if not os.path.isfile(f.target_path):\n raise RuntimeError('{} is not a file !'.format(f.target_path))\n if not os.access(f.target_path, os.W_OK):\n raise RuntimeError('No permission to write to {}'.format(f.target_path))\n if not os.access(os.path.dirname(f.target_path), os.W_OK):\n raise RuntimeError('No permission to write to {}'.format(os.path.dirname(f.target_path)))\n\n # backup old files\n for f in files:\n if os.path.exists(f.target_path):\n cmstack.enter(backup_file(\n f.target_path\n ))\n\n # move new files\n for f in files:\n cmstack.enter(restore_file(f))\n\n try:\n _reload_candidate_config(supervisor_url)\n\n except:\n LOG.exception('Error reverting candidate config')\n" }, { "path": "minemeld/startupplanner.py", "content": "import logging\nfrom operator import itemgetter\nfrom collections import defaultdict\n\nimport networkx as nx\n\nfrom minemeld.run.config import CHANGE_INPUT_DELETED, CHANGE_ADDED, CHANGE_INPUT_ADDED\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass CheckpointNodes(object):\n def __init__(self):\n self.nodes = set()\n self.num_sources = 0\n\n\ndef _build_graph(config):\n graph = nx.DiGraph()\n\n # nodes\n for nodename, _ in config.nodes.iteritems():\n graph.add_node(nodename)\n\n # edges\n for nodename, nodevalue in config.nodes.iteritems():\n inputs = nodevalue.get('inputs', [])\n graph.add_edges_from([(i, nodename) for i in inputs])\n\n return graph\n\n\ndef _plan_subgraph(sg, config, state_info):\n LOG.info('state_info: {!r}'.format(state_info))\n LOG.info('planning for subgraph {!r}'.format(sg.nodes()))\n plan = {}\n\n checkpoints = defaultdict(CheckpointNodes)\n for nodename in sg:\n chkp = state_info[nodename].get('checkpoint', None)\n checkpoints[chkp].nodes.add(nodename)\n if state_info[nodename].get('is_source', False):\n checkpoints[chkp].num_sources += 1\n\n changes = defaultdict(list)\n for c in config.changes:\n if c.nodename in sg:\n changes[c.nodename].append(c)\n\n # if there are no checkpoints => reset\n if len(checkpoints) == 1 and None in checkpoints:\n LOG.info('No checkpoints, new graph: reset')\n for nodename in sg:\n plan[nodename] = 'reset'\n return plan\n\n # if there are no changes and all the nodes are at the same\n # checkpoint => initialize\n if len(checkpoints) == 1 and len(changes) == 0:\n LOG.info('No changes and all nodes have the same checkpoint: initialize')\n for nodename in sg:\n plan[nodename] = 'initialize'\n return plan\n\n # pick the most common checkpoint among sources as reference point\n scheckpoints = sorted(\n [(c, cn.num_sources) for c, cn in checkpoints.iteritems() if c is not None],\n key=itemgetter(1),\n reverse=True\n )\n quorum_checkpoint = None\n if len(scheckpoints) > 0:\n quorum_checkpoint = scheckpoints[0][0]\n LOG.info('Quorum checkpoint: {}'.format(quorum_checkpoint))\n\n # invalid nodes are nodes whose current state is not up to\n # date\n # - nodes with an old checkpoint\n # - nodes with no checkpoint but not added\n # - nodes that had an input deleted\n invalid_nodes = []\n for nodename in sg:\n if nodename not in checkpoints[quorum_checkpoint].nodes and nodename not in checkpoints[None].nodes:\n invalid_nodes.append(nodename)\n continue\n\n added = next((c for c in changes[nodename] if c.change == CHANGE_ADDED), None)\n if added is None and nodename in checkpoints[None].nodes:\n invalid_nodes.append(nodename)\n continue\n\n ideleted = next((c for c in changes[nodename] if c.change == CHANGE_INPUT_DELETED), None)\n if ideleted is not None:\n invalid_nodes.append(nodename)\n continue\n\n # there is at least one invalid node, we reset all the nodes except for the\n # sources with checkpoint == quorum_checkpoint\n # XXX this can be improved\n if len(invalid_nodes) > 0:\n for nodename in sg:\n if nodename in invalid_nodes:\n plan[nodename] = 'reset'\n continue\n\n if not state_info[nodename].get('is_source', False):\n plan[nodename] = 'reset'\n continue\n\n if nodename not in checkpoints[quorum_checkpoint].nodes:\n plan[nodename] = 'reset'\n continue\n\n plan[nodename] = 'rebuild'\n LOG.info('Invalid nodes detected ({}): {}'.format(invalid_nodes, plan))\n return plan\n\n # let's check added nodes and nodes added as inputs, if they have no ancestors\n # we can just initialize\n init_flag = True\n added_nodes = []\n for nodename, clist in changes.iteritems():\n added = next((c for c in clist if c.change == CHANGE_ADDED), None)\n if added is not None:\n if not state_info[nodename].get('is_source', False):\n init_flag = False\n break\n added_nodes.append(nodename)\n\n added_input_nodes = set()\n for nodename, clist in changes.iteritems():\n input_added = [c.detail for c in clist if c.change == CHANGE_INPUT_ADDED]\n for ainode in input_added:\n if not state_info[ainode].get('is_source', False):\n init_flag = False\n added_input_nodes.add(ainode)\n\n if init_flag:\n LOG.info('Only source nodes have been added: initialize')\n for nodename in sg:\n if nodename in added_nodes:\n plan[nodename] = 'reset'\n elif nodename in added_input_nodes:\n plan[nodename] = 'rebuild'\n else:\n plan[nodename] = 'initialize'\n return plan\n\n for nodename in sg:\n if not state_info[nodename].get('is_source', False):\n plan[nodename] = 'reset'\n continue\n\n if nodename not in checkpoints[quorum_checkpoint].nodes:\n plan[nodename] = 'reset'\n continue\n\n plan[nodename] = 'rebuild'\n LOG.info('Non-source nodes added ({}): {}'.format(added_nodes, plan))\n return plan\n\n\ndef plan(config, state_info):\n \"\"\"Defines a startup plan for the MineMeld graph.\n\n Args:\n config (MineMeldConfig): config\n state_info (dict): state_info for each node\n\n Returns a dictionary where keys are node names and\n values the satrtup command for the node.\n \"\"\"\n plan = {}\n\n graph = _build_graph(config)\n\n for subgraph in nx.weakly_connected_component_subgraphs(graph, copy=True):\n plan.update(_plan_subgraph(subgraph, config, state_info))\n\n return plan\n" }, { "path": "minemeld/supervisord/__init__.py", "content": "" }, { "path": "minemeld/supervisord/listener.py", "content": "import sys\nimport os\nimport logging\nimport time\n\nimport redis\nimport ujson\nfrom supervisor import childutils\n\nLOG = logging.getLogger(__name__)\n\n\ndef _handle_event(SR, engine_process_name, hdrs, payload):\n event = hdrs.get('eventname', None)\n if not event.startswith('PROCESS_STATE'):\n return\n event = event.split('_', 2)[-1]\n\n processname = None\n pkvs = payload.split()\n for pkv in pkvs:\n pkey, pvalue = pkv.split(':', 1)\n if pkey == 'processname':\n processname = pvalue\n break\n else:\n LOG.error('processname key not found in payload')\n return\n\n if processname != engine_process_name:\n return\n\n SR.publish(\n 'mm-engine-status.',\n ujson.dumps({\n 'source': '',\n 'timestamp': int(time.time())*1000,\n 'status': event\n })\n )\n\n\ndef main():\n logging.basicConfig(level=logging.DEBUG)\n\n engine_process_name = os.environ.get('MM_ENGINE_PROCESSNAME', 'minemeld-engine')\n\n SR = redis.StrictRedis.from_url(\n os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock')\n )\n\n while True:\n hdrs, payload = childutils.listener.wait(sys.stdin, sys.stdout)\n LOG.info('hdr: {!r} payload: {!r}'.format(hdrs, payload))\n\n try:\n _handle_event(SR, engine_process_name, hdrs, payload)\n\n except:\n LOG.exception('Exception in handling event')\n\n finally:\n childutils.listener.ok(sys.stdout)\n" }, { "path": "minemeld/traced/__init__.py", "content": "from minemeld.traced.queryprocessor import QUERY_QUEUE # noqa\n" }, { "path": "minemeld/traced/main.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements the main entry point to the mm-traced daemon\n\"\"\"\n\nfrom __future__ import print_function\n\nimport gevent\nimport gevent.event\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport argparse\nimport logging\nimport yaml\nimport functools\nimport signal\n\nfrom minemeld import __version__\n\nimport minemeld.comm\nimport minemeld.traced.storage\nimport minemeld.traced.writer\nimport minemeld.traced.queryprocessor\n\nLOG = logging.getLogger(__name__)\n\n\ndef _parse_args():\n parser = argparse.ArgumentParser(\n description=\"Tracing daemon for MineMeld engine\"\n )\n parser.add_argument(\n '--version',\n action='version',\n version=__version__\n )\n parser.add_argument(\n '--verbose',\n action='store_true',\n help='verbose'\n )\n parser.add_argument(\n 'config',\n action='store',\n metavar='CONFIG',\n help='path of the config file or of the config directory'\n )\n return parser.parse_args()\n\n\ndef _ioloop_failure(event):\n LOG.debug(\"loop failure\")\n event.set()\n\n\ndef main():\n def _sigint_handler():\n raise KeyboardInterrupt('Ctrl-C from _sigint_handler')\n\n def _sigterm_handler():\n raise KeyboardInterrupt('Ctrl-C from _sigterm_handler')\n\n def _cleanup():\n trace_writer.stop()\n query_processor.stop()\n store.stop()\n comm.stop()\n\n args = _parse_args()\n\n loglevel = logging.INFO\n if args.verbose:\n loglevel = logging.DEBUG\n\n logging.basicConfig(\n level=loglevel,\n format=\"%(asctime)s (%(process)d)%(module)s.%(funcName)s\"\n \" %(levelname)s: %(message)s\",\n datefmt=\"%Y-%m-%dT%H:%M:%S\"\n )\n LOG.info('Starting mm-traced version %s', __version__)\n LOG.info('mm-traced arguments: %s', args)\n\n with open(args.config, 'r') as f:\n config = yaml.safe_load(f)\n if config is None:\n config = {}\n\n LOG.info('mm-traced config: %s', config)\n\n store = minemeld.traced.storage.Store(config.get('store', None))\n\n transport_config = config.get('transport', {\n 'class': 'ZMQRedis',\n 'config': {\n 'num_connections': 1\n }\n })\n comm = minemeld.comm.factory(\n transport_config['class'],\n transport_config['config']\n )\n\n trace_writer = minemeld.traced.writer.Writer(\n comm,\n store,\n topic=config.get('topic', 'mbus:log'),\n config=config.get('writer', {})\n )\n\n query_processor = minemeld.traced.queryprocessor.QueryProcessor(\n comm,\n store,\n config=config.get('queryprocessor', {})\n )\n\n shutdown_event = gevent.event.Event()\n comm.add_failure_listener(\n functools.partial(_ioloop_failure, shutdown_event)\n )\n\n comm.start()\n\n gevent.signal(signal.SIGINT, _sigint_handler)\n gevent.signal(signal.SIGTERM, _sigterm_handler)\n\n try:\n shutdown_event.wait()\n\n except KeyboardInterrupt:\n pass\n\n except:\n LOG.exception('Exception')\n\n finally:\n _cleanup()\n" }, { "path": "minemeld/traced/purge.py", "content": "#!/usr/bin/env python\n\nimport logging\nimport os\nimport time\nimport sys\nimport argparse\nimport shutil\nimport json\nimport xmlrpclib\nimport supervisor.xmlrpc\n\nLOG = logging.getLogger(__name__)\n\n\ndef _parse_args():\n parser = argparse.ArgumentParser(\n description=\"Purge utility for old MineMeld traces\"\n )\n parser.add_argument(\n '--dry-run',\n action='store_true',\n help='Dry run'\n )\n parser.add_argument(\n '--all',\n action='store_true',\n help='Delete all traces'\n )\n parser.add_argument(\n 'config',\n action='store',\n metavar='CONFIG',\n nargs='?',\n help='path of the config file or of the config directory'\n )\n return parser.parse_args()\n\n\ndef stop_minemeld_traced(supervisor_url):\n sserver = xmlrpclib.ServerProxy(\n 'http://127.0.0.1',\n transport=supervisor.xmlrpc.SupervisorTransport(\n None,\n None,\n supervisor_url\n )\n )\n\n sstate = sserver.supervisor.getState()\n if sstate['statecode'] == 2: # FATAL\n return False\n if sstate['statecode'] != 1:\n LOG.critical(\n \"Supervisor transitioning to a new state, we will purge next time\"\n )\n sys.exit(1)\n\n pstate = sserver.supervisor.getProcessInfo('minemeld-traced')['statename']\n if pstate in ['STOPPED', 'EXITED', 'FATAL']:\n return False\n if pstate != 'RUNNING':\n LOG.critical(\n (\"minemeld-traced transitioning to a new state, \" +\n \"we will purge next time\")\n )\n sys.exit(1)\n\n result = sserver.supervisor.stopProcess('minemeld-traced', False)\n if not result:\n LOG.critical('Stop minemeld-traced returned False')\n sys.exit(1)\n\n LOG.info('Stopping minemeld-traced')\n\n now = time.time()\n info = None\n while (time.time()-now) < 60*15*1000:\n info = sserver.supervisor.getProcessInfo('minemeld-traced')\n if info['statename'] == 'STOPPED':\n break\n time.sleep(5)\n\n if info is not None and info['statename'] != 'STOPPED':\n LOG.critical('Timeout during minemeld-traced stop')\n sys.exit(1)\n\n return True\n\n\ndef start_minemeld_traced(supervisor_url):\n sserver = xmlrpclib.ServerProxy(\n 'http://127.0.0.1',\n transport=supervisor.xmlrpc.SupervisorTransport(\n None,\n None,\n supervisor_url\n )\n )\n\n sserver.supervisor.startProcess('minemeld-traced', False)\n LOG.info('Started minemeld-traced')\n\n\ndef main():\n trace_directory = '/opt/minemeld/local/trace'\n supervisor_url = 'unix:///var/run/minemeld/minemeld.sock'\n num_days = 30\n\n logging.basicConfig(\n level=logging.DEBUG,\n format='%(asctime)s %(levelname)s: %(message)s'\n )\n\n args = _parse_args()\n\n if args.config is not None:\n try:\n with open(args.config, 'r') as f:\n config = json.load(f)\n\n except (IOError, ValueError) as e:\n LOG.critical(\n 'Error loading config file %s: %s' % (args.config, str(e))\n )\n sys.exit(1)\n\n trace_directory = config.get('trace_directory', trace_directory)\n supervisor_url = config.get('supervisor_url', supervisor_url)\n num_days = config.get('num_days', num_days)\n\n trace_directory = os.environ.get(\n 'MINEMELD_TRACE_DIRECTORY',\n trace_directory\n )\n if not os.path.isdir(trace_directory):\n LOG.critical(\"%s is not a directory\", trace_directory)\n sys.exit(1)\n\n num_days = int(os.environ.get('MINEMELD_TRACE_NUM_DAYS', num_days))\n if num_days < 1:\n LOG.critical(\n 'MINEMELD_TRACE_NUM_DAYS should be greater than 1: %d',\n num_days\n )\n sys.exit(1)\n\n supervisor_url = os.environ.get(\n 'SUPERVISOR_URL',\n supervisor_url\n )\n\n LOG.info(\n \"mm-traced-purge started, #days: %d directory: %s\",\n num_days,\n trace_directory\n )\n\n now = time.time()\n today = now - (now % 86400)\n oldest = today - (num_days-1)*86400\n\n tables = os.listdir(trace_directory)\n tobe_removed = []\n for t in tables:\n try:\n d = int(t, 16)\n except ValueError:\n LOG.debug(\"Invalid table name: %s\", t)\n continue\n\n if d < oldest or args.all:\n LOG.info('Marking table %s for removal', t)\n tobe_removed.append(t)\n\n if len(tobe_removed) > 0 and not args.dry_run:\n trunning = stop_minemeld_traced(supervisor_url)\n\n for tr in tobe_removed:\n LOG.info(\"Removing %s\", tr)\n try:\n shutil.rmtree(os.path.join(trace_directory, tr))\n except:\n LOG.exception(\"Error removing table %s\", tr)\n\n if trunning:\n start_minemeld_traced(supervisor_url)\n" }, { "path": "minemeld/traced/queryprocessor.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements the query processor for mm-traced daemon\n\"\"\"\n\nimport logging\nimport calendar\nimport time\nimport ujson\nimport re\nimport os\n\nimport gevent\nimport greenlet\nimport gevent.lock\nimport gevent.event\nimport redis\n\nLOG = logging.getLogger(__name__)\n\nQUERY_QUEUE = 'mmtraced:query'\n\n_REGEX_SPECIAL_CHARS = [\n '[', '\\\\', '^', '$', '.', '|', '?', '*', '+', '(', ')'\n]\n\n\nclass Query(gevent.Greenlet):\n def __init__(self, store, query, timestamp, counter,\n num_lines, uuid, redis_config):\n self.uuid = uuid\n self.store = store\n\n self.query = query\n\n self.starting_timestamp = timestamp\n self.starting_counter = counter\n self.num_lines = num_lines\n\n self.redis_url = redis_config.get('redis_url',\n os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock')\n )\n\n super(Query, self).__init__()\n\n LOG.info(\"Query %s - %s\", uuid, query)\n self._parse_query(query)\n\n def _parse_query(self, query):\n query = query.strip()\n components = query.lower().split()\n\n field_specific = re.compile('^[\\w$]+:.*$')\n\n self.parsed_query = []\n for c in components:\n negate = False\n if c[0] == '-':\n negate = True\n c = c[1:]\n\n matching_re = c\n if field_specific.match(c) is not None:\n field, value = c.split(':', 1)\n\n efield = []\n for c in field:\n if c in _REGEX_SPECIAL_CHARS:\n efield.append('\\\\')\n efield.append(c)\n efield = ''.join(efield)\n\n evalue = []\n for c in value:\n if c in _REGEX_SPECIAL_CHARS:\n evalue.append('\\\\')\n evalue.append(c)\n evalue = ''.join(evalue)\n\n matching_re = (\n '\"%(field)s\":(?:\\[(?:\".*\",)*)?\"*[^\"]*%(value)s' %\n {\n 'field': efield,\n 'value': evalue\n }\n )\n LOG.debug(matching_re)\n\n self.parsed_query.append({\n 're': re.compile(matching_re, re.IGNORECASE),\n 'negate': negate\n })\n\n def _check_query(self, log):\n for q in self.parsed_query:\n occ = q['re'].search(log)\n if not ((occ is not None) ^ q['negate']):\n return False\n return True\n\n def _core_run(self):\n LOG.debug(\"Query %s started\", self.uuid)\n\n SR = redis.StrictRedis.from_url(\n self.redis_url\n )\n\n line_generator = self.store.iterate_backwards(\n self.uuid,\n self.starting_timestamp,\n self.starting_counter\n )\n\n try:\n num_generated_lines = 0\n while num_generated_lines < self.num_lines:\n line = next(line_generator, None)\n if not line:\n break\n\n gevent.sleep(0)\n\n if 'log' not in line:\n SR.publish('mm-traced-q.'+self.uuid, ujson.dumps(line))\n continue\n\n if self._check_query(line['log']):\n SR.publish('mm-traced-q.'+self.uuid, ujson.dumps(line))\n num_generated_lines += 1\n\n SR.publish(\n 'mm-traced-q.'+self.uuid,\n '{\"msg\": \"Loaded %d lines\"}' % num_generated_lines\n )\n\n finally:\n SR.publish('mm-traced-q.'+self.uuid, '')\n LOG.info(\"Query %s finished - %d\", self.uuid, num_generated_lines)\n\n def _run(self):\n try:\n self._core_run()\n\n finally:\n # make sure we release the tables if we stop in the middle\n # of an iteration\n self.store.release_all(self.uuid)\n\n\nclass QueryProcessor(object):\n def __init__(self, comm, store, config=None):\n if config is None:\n config = {}\n\n self._stop = gevent.event.Event()\n\n self.max_concurrency = config.get('max_concurrency', 10)\n self.redis_config = config.get('redis', {})\n self.store = store\n\n self.queries_lock = gevent.lock.BoundedSemaphore()\n self.queries = {}\n\n comm.request_rpc_server_channel(\n QUERY_QUEUE,\n self,\n allowed_methods=['query', 'kill_query']\n )\n\n def _query_finished(self, gquery):\n self.queries_lock.acquire()\n self.queries.pop(gquery.uuid, None)\n self.queries_lock.release()\n\n try:\n result = gquery.get()\n\n except:\n self.store.release_all(gquery.uuid)\n LOG.exception('Query finished with exception')\n return\n\n if isinstance(result, greenlet.GreenletExit):\n self.store.release_all(gquery.uuid)\n\n def query(self, uuid, query, timestamp=None, counter=None, num_lines=None):\n LOG.debug('Query called: {!r}'.format(query))\n\n if self._stop.is_set():\n raise RuntimeError('stopping')\n\n if timestamp is None:\n timestamp = int(calendar.timegm(time.gmtime())*1000)\n\n if counter is None:\n counter = 0xFFFFFFFFFFFFFFFF\n\n if num_lines is None:\n num_lines = 100\n\n self.queries_lock.acquire()\n LOG.debug('Locked')\n\n if len(self.queries) >= self.max_concurrency:\n self.queries_lock.release()\n raise RuntimeError('max number of concurrent queries reached')\n\n if uuid in self.queries:\n self.queries_lock.release()\n raise RuntimeError('UUID not unique')\n\n try:\n gquery = Query(\n self.store,\n query,\n timestamp,\n counter,\n num_lines,\n uuid,\n self.redis_config\n )\n gquery.link(self._query_finished)\n self.queries[uuid] = gquery\n gquery.start()\n\n finally:\n self.queries_lock.release()\n\n return 'OK'\n\n def kill_query(self, uuid):\n if self._stop.is_set():\n raise RuntimeError('stopping')\n\n self.queries_lock.acquire()\n if uuid in self.queries:\n self.queries[uuid].kill()\n self.queries_lock.release()\n\n return 'OK'\n\n def stop(self):\n LOG.info('QueryProcessor - stop called')\n\n if self._stop.is_set():\n return\n\n self._stop.set()\n self.queries_lock.acquire()\n for _, gquery in self.queries.iteritems():\n gquery.kill()\n self.queries_lock.release()\n" }, { "path": "minemeld/traced/storage.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements the storage mechansim for the mm-traced daemon\n\"\"\"\n\nimport logging\nimport datetime\nimport time\nimport Queue\nimport os\nimport os.path\n\nimport gevent.queue\nimport gevent.event\nimport gevent.lock\n\nimport plyvel\nimport pytz\n\nLOG = logging.getLogger(__name__)\n\nSTART_KEY = '%016x%015x' % (0, 0)\n\nTABLE_MAX_COUNTER_KEY = 'MAX_COUNTER'\n\n\nclass TableNotFound(Exception):\n pass\n\n\nclass Table(object):\n def __init__(self, name, create_if_missing=True):\n LOG.debug('New table: %s %s', name, create_if_missing)\n\n self.name = name\n self.last_used = None\n self.refs = []\n\n if not create_if_missing and not os.path.exists(name):\n raise TableNotFound('Table does not exists')\n\n try:\n self.db = plyvel.DB(\n name,\n create_if_missing=create_if_missing\n )\n\n except plyvel.Error as e:\n if not create_if_missing:\n raise TableNotFound(str(e))\n raise\n\n self.max_counter = None\n try:\n self.max_counter = self.db.get(TABLE_MAX_COUNTER_KEY)\n\n except KeyError:\n pass\n\n if self.max_counter is None:\n LOG.warning(\n 'MAX_ID key not found in %s',\n self.name\n )\n self.max_counter = -1\n\n else:\n self.max_counter = int(self.max_counter, 16)\n\n LOG.debug('Table %s - max id: %d', self.name, self.max_counter)\n\n def add_reference(self, refid):\n self.refs.append(refid)\n\n def remove_reference(self, refid):\n try:\n self.refs.remove(refid)\n\n except ValueError:\n LOG.warning(\n 'Attempt to remove non existing reference: %s - %s',\n refid,\n self.name\n )\n\n LOG.debug('{} #refs: {}'.format(self.name, self.ref_count()))\n\n def ref_count(self):\n return len(self.refs)\n\n def put(self, key, value):\n self.last_used = time.time()\n\n self.max_counter += 1\n new_max_counter = '%016x' % self.max_counter\n\n batch = self.db.write_batch()\n batch.put(key+new_max_counter, value)\n batch.put(TABLE_MAX_COUNTER_KEY, new_max_counter)\n batch.write()\n\n def backwards_iterator(self, timestamp, counter):\n return self.db.iterator(\n start=START_KEY,\n stop=('%016x%016x' % (timestamp, counter)),\n include_start=False,\n include_stop=True,\n reverse=True\n )\n\n def close(self):\n LOG.debug('{} - close'.format(self.name))\n self.db.close()\n\n @staticmethod\n def oldest_table():\n # XXX we should switch to something iterative\n entries = os.listdir('.')\n if len(entries) == 0:\n return None\n\n tables = []\n for e in entries:\n try:\n int(e, 16)\n except:\n continue\n\n tables.append(e)\n\n if len(tables) == 0:\n return None\n\n tables = sorted(tables)\n return tables[0]\n\n\ndef _lock_current_tables():\n \"\"\"Decorator for locking current_tables\n \"\"\"\n def _lock_out(f):\n def _lock_in(self, *args, **kwargs):\n self.current_tables_lock.acquire()\n\n try:\n result = f(self, *args, **kwargs)\n\n finally:\n self.current_tables_lock.release()\n\n return result\n return _lock_in\n return _lock_out\n\n\nclass Store(object):\n def __init__(self, config=None):\n if config is None:\n config = {}\n\n self._stop = gevent.event.Event()\n\n self.max_tables = config.get('max_tables', 5)\n\n self.current_tables = {}\n self.current_tables_lock = gevent.lock.BoundedSemaphore()\n\n self.max_written_timestamp = None\n self.max_written_counter = 0\n\n self.add_queue = gevent.queue.PriorityQueue()\n\n def _open_table(self, name, create_if_missing):\n table = Table(name, create_if_missing=create_if_missing)\n self.current_tables[name] = table\n\n return table\n\n def _close_table(self, table):\n table.close()\n self.current_tables.pop(table.name)\n\n def _add_table(self, name, priority, create_if_missing=True):\n self.current_tables_lock.acquire()\n if len(self.current_tables) < self.max_tables:\n try:\n result = self._open_table(\n name,\n create_if_missing=create_if_missing\n )\n finally:\n self.current_tables_lock.release()\n return result\n self.current_tables_lock.release()\n\n future_table = gevent.event.AsyncResult()\n self.add_queue.put((\n priority,\n (future_table, name, create_if_missing)\n ))\n self._process_queue()\n\n return future_table.get()\n\n def _get_table(self, day, ref, create_if_missing=True):\n table = self.current_tables.get(day, None)\n\n if table is None:\n prio = 99 if ref != 'write' else 1\n table = self._add_table(\n day,\n prio,\n create_if_missing=create_if_missing\n )\n\n table.add_reference(ref)\n\n return table\n\n @_lock_current_tables()\n def _process_queue_element(self, name, ftable, create_if_missing):\n if name in self.current_tables:\n ftable.set(self.current_tables[name])\n return True\n\n if len(self.current_tables) < self.max_tables:\n new_table = self._open_table(\n name,\n create_if_missing=create_if_missing\n )\n ftable.set(new_table)\n return True\n\n # garbage collect\n candidate = None\n for _, table in self.current_tables.iteritems():\n if table.ref_count() != 0:\n continue\n\n if candidate is None or candidate.last_used > table.last_used:\n candidate = table\n\n if candidate is None:\n return False\n\n self._close_table(candidate)\n\n new_table = self._open_table(\n name,\n create_if_missing=create_if_missing\n )\n ftable.set(new_table)\n\n return True\n\n def _process_queue(self):\n # this is for perf improvement\n if self.add_queue.empty():\n return\n\n try:\n while True:\n prio, (ftable, name, create_if_missing) = \\\n self.add_queue.get_nowait()\n\n result = self._process_queue_element(\n name,\n ftable,\n create_if_missing\n )\n if not result:\n prio = (prio - 1) if prio > 2 else prio\n self.add_queue.put((\n prio,\n (ftable, name, create_if_missing)\n ))\n return\n\n except IndexError:\n return\n\n except Queue.Empty:\n return\n\n def _release(self, table, ref):\n table.remove_reference(ref)\n\n self._process_queue()\n\n def write(self, timestamp, log):\n if self._stop.is_set():\n raise RuntimeError('stopping')\n\n tssec = timestamp/1000\n day = '%016x' % (tssec - (tssec % 86400))\n\n table = self._get_table(day, 'write')\n\n try:\n table.put(\n '%016x' % timestamp,\n log\n )\n\n finally:\n self._release(table, 'write')\n\n def iterate_backwards(self, ref, timestamp, counter):\n if self._stop.is_set():\n raise RuntimeError('stopping')\n\n tssec = timestamp/1000\n current_day = (tssec - (tssec % 86400))\n\n oldest_table = Table.oldest_table()\n if oldest_table is None:\n yield {'msg': 'No more logs to check'}\n return\n\n while True:\n table_name = '%016x' % current_day\n if table_name < oldest_table:\n yield {'msg': 'No more logs to check'}\n return\n\n day = datetime.datetime.fromtimestamp(\n current_day,\n pytz.UTC\n )\n day = '%04d-%02d-%02d' % (\n day.year,\n day.month,\n day.day\n )\n yield {'msg': 'Checking %s' % day}\n\n try:\n table = self._get_table(\n table_name,\n ref,\n create_if_missing=False\n )\n except TableNotFound:\n if current_day == 0:\n # XXX this is unreachable\n yield {'msg': 'This should be unreachable'}\n return\n\n current_day -= 86400\n continue\n\n table_iterator = table.backwards_iterator(\n timestamp=timestamp,\n counter=counter\n )\n\n for linets, line in table_iterator:\n yield {\n 'timestamp': int(linets[:16], 16),\n 'counter': int(linets[16:], 16),\n 'log': line\n }\n\n self._release(table, ref)\n\n if current_day == 0:\n yield {'msg': 'We haved reached the origins of time'}\n return\n\n current_day -= 86400\n\n def release_all(self, ref):\n if self._stop.is_set():\n raise RuntimeError('stopping')\n\n self.current_tables_lock.acquire()\n for t in self.current_tables.values():\n t.remove_reference(ref)\n self.current_tables_lock.release()\n\n self._process_queue()\n\n def stop(self):\n LOG.info('Store - stop called')\n\n if self._stop.is_set():\n return\n\n self._stop.set()\n self.current_tables_lock.acquire()\n for t in self.current_tables.keys():\n self.current_tables[t].close()\n self.current_tables.pop(t, None)\n self.current_tables_lock.release()\n" }, { "path": "minemeld/traced/writer.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements the writer class for logs\n\"\"\"\n\nimport logging\n\nimport psutil\nimport ujson\nimport gevent\nimport gevent.event\n\nLOG = logging.getLogger(__name__)\n\n\nclass DiskSpaceMonitor(gevent.Greenlet):\n def __init__(self, threshold, low_disk):\n self._threshold = threshold\n self._low_disk = low_disk\n\n super(DiskSpaceMonitor, self).__init__()\n\n def _run(self):\n while True:\n perc_used = psutil.disk_usage('.').percent\n\n if perc_used >= self._threshold:\n if not self._low_disk.is_set():\n self._low_disk.set()\n LOG.critical(\n 'Disk space used above threshold ({}%), writing disabled'.format(self._threshold)\n )\n\n else:\n if self._low_disk.is_set():\n self._low_disk.clear()\n LOG.info('Disk space used below threshold, writing restored')\n\n gevent.sleep(60)\n\n\nclass Writer(object):\n def __init__(self, comm, store, topic, config):\n self._stop = gevent.event.Event()\n self._low_disk = gevent.event.Event()\n\n self.store = store\n self.comm = comm\n self.comm.request_sub_channel(\n topic,\n self,\n allowed_methods=['log'],\n name='mbus:log:writer',\n multi_write=True\n )\n\n self._disk_monitor_glet = DiskSpaceMonitor(\n threshold=config.get('threshold', 70),\n low_disk=self._low_disk\n )\n self._disk_monitor_glet.start()\n\n def log(self, timestamp, **kwargs):\n if self._stop.is_set():\n return\n\n if self._low_disk.is_set():\n return\n\n self.store.write(timestamp, ujson.dumps(kwargs))\n\n def stop(self):\n LOG.info('Writer - stop called')\n\n if self._stop.is_set():\n return\n\n self._stop.set()\n self._disk_monitor_glet.kill()\n" }, { "path": "nodes.json", "content": "{\n \"minemeld.ft.anomali.Intelligence\": {\n \"class\": \"minemeld.ft.anomali:Intelligence\"\n },\n \"minemeld.ft.auscert.MaliciousURLFeed\": {\n \"class\": \"minemeld.ft.auscert:MaliciousURLFeed\"\n },\n \"minemeld.ft.autofocus.ExportList\": {\n \"class\": \"minemeld.ft.autofocus:ExportList\"\n },\n \"minemeld.ft.azure.AzureXML\": {\n \"class\": \"minemeld.ft.azure:AzureXML\"\n },\n \"minemeld.ft.azure.AzureJSON\": {\n \"class\": \"minemeld.ft.azure:AzureJSON\"\n },\n \"minemeld.ft.cif.Feed\": {\n \"class\": \"minemeld.ft.cif:Feed\"\n },\n \"minemeld.ft.ciscoise.ErsSgt\": {\n \"class\": \"minemeld.ft.ciscoise:ErsSgt\"\n },\n \"minemeld.ft.csv.CSVFT\": {\n \"class\": \"minemeld.ft.csv:CSVFT\"\n },\n \"minemeld.ft.dag.DagPusher\": {\n \"class\": \"minemeld.ft.dag:DagPusher\"\n },\n \"minemeld.ft.dag_ng.DagPusher\": {\n \"class\": \"minemeld.ft.dag_ng:DagPusher\"\n },\n \"minemeld.ft.google.GoogleNetBlocks\": {\n \"class\": \"minemeld.ft.google:GoogleNetBlocks\"\n },\n \"minemeld.ft.google.GoogleCloudNetBlocks\": {\n \"class\": \"minemeld.ft.google:GoogleCloudNetBlocks\"\n },\n \"minemeld.ft.google.GoogleSPF\": {\n \"class\": \"minemeld.ft.google:GoogleSPF\"\n },\n \"minemeld.ft.http.HttpFT\": {\n \"class\": \"minemeld.ft.http:HttpFT\"\n },\n \"minemeld.ft.ipop.AggregateIPv4FT\": {\n \"class\": \"minemeld.ft.ipop:AggregateIPv4FT\"\n },\n \"minemeld.ft.json.SimpleJSON\": {\n \"class\": \"minemeld.ft.json:SimpleJSON\"\n },\n \"minemeld.ft.local.YamlFT\": {\n \"class\": \"minemeld.ft.local:YamlFT\"\n },\n \"minemeld.ft.local.YamlIPv4FT\": {\n \"class\": \"minemeld.ft.local:YamlIPv4FT\"\n },\n \"minemeld.ft.local.YamlURLFT\": {\n \"class\": \"minemeld.ft.local:YamlURLFT\"\n },\n \"minemeld.ft.local.YamlDomainFT\": {\n \"class\": \"minemeld.ft.local:YamlDomainFT\"\n },\n \"minemeld.ft.local.YamlIPv6FT\": {\n \"class\": \"minemeld.ft.local:YamlIPv6FT\"\n },\n \"minemeld.ft.logstash.LogstashOutput\": {\n \"class\": \"minemeld.ft.logstash:LogstashOutput\"\n },\n \"minemeld.ft.o365.O365XML\": {\n \"class\": \"minemeld.ft.o365:O365XML\"\n },\n \"minemeld.ft.o365.O365API\": {\n \"class\": \"minemeld.ft.o365:O365API\"\n },\n \"minemeld.ft.op.AggregateFT\": {\n \"class\": \"minemeld.ft.op:AggregateFT\"\n },\n \"minemeld.ft.phishme.Intelligence\": {\n \"class\": \"minemeld.ft.phishme:Intelligence\"\n },\n \"minemeld.ft.proofpoint.ETIntelligence\": {\n \"class\": \"minemeld.ft.proofpoint:ETIntelligence\"\n },\n \"minemeld.ft.proofpoint.EmergingThreatsIP\": {\n \"class\": \"minemeld.ft.proofpoint:EmergingThreatsIP\"\n },\n \"minemeld.ft.proofpoint.EmergingThreatsDomain\": {\n \"class\": \"minemeld.ft.proofpoint:EmergingThreatsDomain\"\n },\n \"minemeld.ft.recordedfuture.IPRiskList\": {\n \"class\": \"minemeld.ft.recordedfuture:IPRiskList\"\n },\n\t\"minemeld.ft.recordedfuture.DomainRiskList\": {\n \"class\": \"minemeld.ft.recordedfuture:DomainRiskList\"\n },\n \"minemeld.ft.recordedfuture.MasterRiskList\": {\n \"class\": \"minemeld.ft.recordedfuture:MasterRiskList\"\n },\n \"minemeld.ft.redis.RedisSet\": {\n \"class\": \"minemeld.ft.redis:RedisSet\"\n },\n \"minemeld.ft.syslog.SyslogMatcher\": {\n \"class\": \"minemeld.ft.syslog:SyslogMatcher\"\n },\n \"minemeld.ft.syslog.SyslogMiner\": {\n \"class\": \"minemeld.ft.syslog:SyslogMiner\"\n },\n \"minemeld.ft.taxii.TaxiiClient\": {\n \"class\": \"minemeld.ft.taxii:TaxiiClient\"\n },\n \"minemeld.ft.taxii.DataFeed\": {\n \"class\": \"minemeld.ft.taxii:DataFeed\"\n },\n \"minemeld.ft.threatq.Export\": {\n \"class\": \"minemeld.ft.threatq:Export\"\n },\n \"minemeld.ft.tmt.DTIAPI\": {\n \"class\": \"minemeld.ft.tmt:DTIAPI\"\n },\n \"minemeld.ft.vt.Notifications\": {\n \"class\": \"minemeld.ft.vt:Notifications\"\n },\n \"minemeld.ft.mm.JSONSEQMiner\": {\n \"class\": \"minemeld.ft.mm:JSONSEQMiner\"\n },\n \"minemeld.ft.localdb.Miner\": {\n \"class\": \"minemeld.ft.localdb:Miner\"\n },\n \"minemeld.ft.threatconnect.IndicatorsMiner\": {\n \"class\": \"minemeld.ft.threatconnect:IndicatorsMiner\"\n },\n \"minemeld.ft.threatconnect.GroupsMiner\": {\n \"class\": \"minemeld.ft.threatconnect:GroupsMiner\"\n },\n \"minemeld.ft.visa.VTI\": {\n \"class\": \"minemeld.ft.visa:VTI\"\n },\n \"minemeld.ft.taxii2.Taxii2Client\": {\n \"class\": \"minemeld.ft.taxii2:Taxii2Client\"\n },\n \"minemeld.ft.cofense.Triage\": {\n \"class\": \"minemeld.ft.cofense:Triage\"\n },\n \"minemeld.ft.bambenek.Miner\": {\n \"class\": \"minemeld.ft.bambenek:Miner\"\n }\n}\n" }, { "path": "requirements-dev.txt", "content": "Sphinx==1.3.1\nsphinx-rtd-theme==0.1.8\ntox==2.1.1\nmock==1.3.0\ngit+https://github.com/PaloAltoNetworks/platter#egg=platter\ncython==0.24\nvirtualenv==16.7.9\n" }, { "path": "requirements-web.txt", "content": "Flask==0.12.4\ngunicorn==19.5.0\npsutil==5.6.6\nFlask-Login==0.2.11\npasslib==1.6.5\nrrdtool==0.1.2\nsupervisor==3.1.3\nblinker==1.4\n" }, { "path": "requirements.txt", "content": "pip>=9.0.1\namqp==1.4.6\ngevent==1.0.2\ngreenlet==0.4.7\nhiredis==0.2.0\nPyYAML==5.4\nredis==2.10.5\nrequests==2.20.0\nplyvel==0.9\nnetaddr==0.7.18\njmespath==0.7.1\nclick==4.1\npan-python==0.10.0\nstix==1.1.1.8\ncybox==2.1.0.17\nsix==1.11.0\nlxml==4.6.3\nstix-edh==1.0.0\nlibtaxii==1.1.107\npytz==2015.4\ncertifi\nujson==1.34\nfilelock==2.0.4\nsleekxmpp==1.3.1\nbeautifulsoup4==4.4.1\ncifsdk==2.0.14\nlz4==2.2.1\nnetworkx==1.11\nunicodecsv==0.14.1\nWerkzeug==0.12.2\npyzmq==16.0.3\nstix2-patterns==1.1.0\n" }, { "path": "scripts/prebuild-script.sh", "content": "#!/bin/bash\n\npip install cython==0.24\n\n" }, { "path": "setup.py", "content": "# Copyright 2015-2017 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport json\nfrom setuptools import Extension, setup, find_packages\n\ntry:\n from Cython.Build import cythonize\nexcept ImportError:\n # this is for platter\n cythonize = lambda x: x\n\nimport sys\nimport os.path\nsys.path.insert(0, os.path.abspath('.'))\nfrom minemeld import __version__\n\nwith open('nodes.json') as f:\n _entry_points = {\n 'minemeld_nodes': [],\n 'minemeld_nodes_gcs': [],\n 'minemeld_nodes_validators': []\n }\n _nodes = json.load(f)\n for node, v in _nodes.iteritems():\n _entry_points['minemeld_nodes'].append(\n '{} = {}'.format(node, v['class'])\n )\n if 'gc' in v:\n _entry_points['minemeld_nodes_gcs'].append(\n '{} = {}'.format(node, v['gc'])\n )\n if 'validator' in v:\n _entry_points['minemeld_nodes_validators'].append(\n '{} = {}'.format(node, v['validator'])\n )\n\n\nwith open('requirements.txt') as f:\n _requirements = f.read().splitlines()\n\nwith open('README.md') as f:\n _long_description = f.read()\n\n_packages = find_packages(exclude=[\"*.tests\", \"*.tests.*\", \"tests.*\", \"tests\"])\n\nGDNS = Extension(\n name='minemeld.packages.gdns._ares',\n sources=['minemeld/packages/gdns/_ares.pyx'],\n include_dirs=[],\n libraries=['cares'],\n define_macros=[('HAVE_NETDB_H', '')],\n depends=['minemeld/packages/gdns/dnshelper.c']\n)\n\nsetup(\n name='minemeld-core',\n version=__version__,\n url='https://github.com/PaloAltoNetworks-BD/minemeld-core',\n author='Palo Alto Networks',\n author_email='techbizdev@paloaltonetworks.com',\n description='An extensible indicator processing framework',\n classifiers=[\n 'Development Status :: 3 - Alpha',\n 'License :: OSI Approved :: Apache Software License',\n 'Programming Language :: Python :: 2.7',\n 'Topic :: Security',\n 'Topic :: Internet'\n ],\n long_description=_long_description,\n packages=_packages,\n provides=['minemeld'],\n install_requires=_requirements,\n ext_modules=cythonize([GDNS]),\n entry_points={\n 'console_scripts': [\n 'mm-run = minemeld.run.launcher:main',\n 'mm-console = minemeld.run.console:main',\n 'mm-traced = minemeld.traced.main:main',\n 'mm-traced-purge = minemeld.traced.purge:main',\n 'mm-supervisord-listener = minemeld.supervisord.listener:main',\n 'mm-extensions-freeze = minemeld.run.freeze:main',\n 'mm-cacert-merge = minemeld.run.cacert_merge:main',\n 'mm-restore = minemeld.run.restore:main',\n 'mm-extension-from-git = minemeld.run.extgit:main'\n ],\n 'minemeld_nodes': _entry_points['minemeld_nodes'],\n 'minemeld_nodes_gcs': _entry_points['minemeld_nodes_gcs'],\n 'minemeld_nodes_validators': _entry_points['minemeld_nodes_validators']\n }\n)\n" }, { "path": "tests/comm_mock.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements mock classes for minemed.comm tests\n\"\"\"\n\nclass MockComm(object):\n def __init__(self, config):\n self.config = config\n\n self.sub_channels = []\n self.rpc_server_channels = []\n\n def request_sub_channel(self, topic, obj=None, allowed_methods=None, name=None):\n self.sub_channels.append({\n 'topic': topic,\n 'obj': obj,\n 'allowed_methods': allowed_methods,\n 'name': name\n })\n\n def request_rpc_server_channel(self, name, obj=None, allowed_methods=[],\n method_prefix='', fanout=None):\n self.rpc_server_channels.append({\n 'name': name,\n 'obj': obj,\n 'allowed_methods': allowed_methods,\n 'method_prefix': method_prefix,\n 'fanout': fanout\n })\n\ndef comm_factory(config):\n return MockComm(config)\n" }, { "path": "tests/empty.yml", "content": "" }, { "path": "tests/feeds.htpasswd", "content": "user1:$apr1$SdhtTFdb$br1vVDVDr3r/ZYTo0aj.L0\nguest:$apr1$fUbzwJlK$tXcBNLcN8zhXNGjgB7M6./\n" }, { "path": "tests/integration/basic/DomainHC%3Fv%3Dcarbonblack.result", "content": "{\n\"feedinfo\": {\n \"category\": \"MineMeld\", \"icon\": \"iVBORw0KGgoAAAANSUhEUgAAASwAAAFLCAYAAABsjLGXAAAMFmlDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnltSCAktEAEpoTdBehUIHQQB6WAjJAFCCZAQVOzIooJrQcWCFV0Bsa0FkLUiioVFwF4XRFRW1sWCDZU3KaDP1753vm/u/Dlzzpn/zD13MgOAsi07NzcLVQEgW5AvjAryZSYkJjFJPUABUAEFGACczRHl+kRGhgEoo/0/y7tbAJH0160lsf51/L+KKpcn4gCAREKcwhVxsiE+BgCuyckV5gNAaIN6o9n5uRI8CLG6EBIEgIhLcJoMa0pwigxPkNrERPlBzAKATGWzhWkAKEl4Mws4aTCOkoSjrYDLF0C8FWIvTjqbC/EDiCdkZ+dArEyG2Dzluzhp/xQzZSwmm502hmW5SIXszxflZrHn/p/L8b8lO0s8OochbNR0YXCUJGe4bjWZOaESTIX4pCAlPAJiNYgv8blSewm+ly4OjpXbD3BEfnDNAAMAFHDZ/qEQ60DMEGfG+sixPVso9YX2aDg/PyRGjlOEOVHy+GiBICs8TB5neTovZBRv54kCokdtUvmBIRDDSkOPFabHxMt4oi0F/LhwiJUg7hBlRofKfR8VpvuFj9oIxVESzsYQv00VBkbJbDDNbNFoXpgNhy2dC9YCxspPjwmW+WIJPFFC2CgHLs8/QMYB4/IEsXJuGKwu3yi5b0luVqTcHtvOywqKkq0zdlhUED3q25UPC0y2DtjjDPbkSPlc73LzI2Nk3HAUhAE/4A+YQAxbCsgBGYDfPtAwAH/JRgIBGwhBGuABa7lm1CNeOiKAz2hQCP6CiAdEY36+0lEeKID6L2Na2dMapEpHC6QemeApxNm4Nu6Fe+Bh8MmCzR53xd1G/ZjKo7MSA4j+xGBiINFijAcHss6CTQj4/0YXCnsezE7CRTCaw7d4hKeETsJjwk1CN+EuiANPpFHkVrP4RcIfmDPBFNANowXKs0v5PjvcFLJ2wn1xT8gfcscZuDawxh1hJj64N8zNCWq/Zyge4/ZtLX+cT8L6+3zkeiVLJSc5i5SxN+M3ZvVjFL/v1ogL+9AfLbHl2FGsFTuHXcZOYg2AiZ3BGrE27JQEj1XCE2kljM4WJeWWCePwR21s62z7bT//MDdbPr9kvUT5vDn5ko/BLyd3rpCflp7P9IG7MY8ZIuDYTGDa29q5ACDZ22VbxxuGdM9GGFe+6fLOAuBWCpVp33RsIwBOPAWA/u6bzug1LPc1AJzq4IiFBTKdZDsGBPiPoQy/Ci2gB4yAOczHHjgDD8ACAWAyiAAxIBHMhCueDrIh59lgPlgCSkAZWAM2gC1gB9gNasABcAQ0gJPgHLgIroIOcBPch3XRB16AQfAODCMIQkJoCB3RQvQRE8QKsUdcES8kAAlDopBEJBlJQwSIGJmPLEXKkHJkC7ILqUV+RU4g55DLSCdyF+lB+pHXyCcUQ6moOqqLmqITUVfUBw1FY9AZaBqahxaixegqdBNahe5H69Fz6FX0JtqNvkCHMIApYgzMALPGXDE/LAJLwlIxIbYQK8UqsCrsINYE3/N1rBsbwD7iRJyOM3FrWJvBeCzOwfPwhfhKfAteg9fjLfh1vAcfxL8SaAQdghXBnRBCSCCkEWYTSggVhL2E44QL8LvpI7wjEokMohnRBX6XicQM4jziSuI24iHiWWInsZc4RCKRtEhWJE9SBIlNyieVkDaT9pPOkLpIfaQPZEWyPtmeHEhOIgvIReQK8j7yaXIX+Rl5WEFFwUTBXSFCgaswV2G1wh6FJoVrCn0KwxRVihnFkxJDyaAsoWyiHKRcoDygvFFUVDRUdFOcqshXXKy4SfGw4iXFHsWPVDWqJdWPOp0qpq6iVlPPUu9S39BoNFMai5ZEy6etotXSztMe0T4o0ZVslEKUuEqLlCqV6pW6lF4qKyibKPsoz1QuVK5QPqp8TXlARUHFVMVPha2yUKVS5YTKbZUhVbqqnWqEarbqStV9qpdVn6uR1EzVAtS4asVqu9XOq/XSMboR3Y/OoS+l76FfoPepE9XN1EPUM9TL1A+ot6sPaqhpOGrEaczRqNQ4pdHNwBimjBBGFmM14wjjFuPTON1xPuN441aMOziua9x7zfGaLE2eZqnmIc2bmp+0mFoBWplaa7UatB5q49qW2lO1Z2tv176gPTBefbzHeM740vFHxt/TQXUsdaJ05uns1mnTGdLV0w3SzdXdrHted0CPocfSy9Bbr3dar1+fru+lz9dfr39G/0+mBtOHmcXcxGxhDhroGAQbiA12GbQbDBuaGcYaFhkeMnxoRDFyNUo1Wm/UbDRorG88xXi+cZ3xPRMFE1eTdJONJq0m703NTONNl5k2mD430zQLMSs0qzN7YE4z9zbPM68yv2FBtHC1yLTYZtFhiVo6WaZbVlpes0KtnK34VtusOicQJrhNEEyomnDbmmrtY11gXWfdY8OwCbMpsmmweTnReGLSxLUTWyd+tXWyzbLdY3vfTs1usl2RXZPda3tLe459pf0NB5pDoMMih0aHV45WjjzH7Y53nOhOU5yWOTU7fXF2cRY6H3TudzF2SXbZ6nLbVd010nWl6yU3gpuv2yK3k24f3Z3d892PuP/tYe2R6bHP4/kks0m8SXsm9XoaerI9d3l2ezG9kr12enV7G3izvau8H7OMWFzWXtYzHwufDJ/9Pi99bX2Fvsd93/u5+y3wO+uP+Qf5l/q3B6gFxAZsCXgUaBiYFlgXOBjkFDQv6GwwITg0eG3w7RDdEE5IbcjgZJfJCya3hFJDo0O3hD4OswwThjVNQadMnrJuyoNwk3BBeEMEiAiJWBfxMNIsMi/yt6nEqZFTK6c+jbKLmh/VGk2PnhW9L/pdjG/M6pj7seax4tjmOOW46XG1ce/j/ePL47sTJiYsSLiaqJ3IT2xMIiXFJe1NGpoWMG3DtL7pTtNLpt+aYTZjzozLM7VnZs08NUt5FnvW0WRCcnzyvuTP7Ah2FXsoJSRla8ogx4+zkfOCy+Ku5/bzPHnlvGepnqnlqc/TPNPWpfWne6dXpA/w/fhb+K8ygjN2ZLzPjMiszhzJis86lE3OTs4+IVATZApacvRy5uR05lrlluR257nnbcgbFIYK94oQ0QxRY746POa0ic3FP4l7CrwKKgs+zI6bfXSO6hzBnLa5lnNXzH1WGFj4yzx8Hmde83yD+Uvm9yzwWbBrIbIwZWHzIqNFxYv6FgctrllCWZK55Pci26LyordL45c2FesWLy7u/Snop7oSpRJhye1lHst2LMeX85e3r3BYsXnF11Ju6ZUy27KKss8rOSuv/Gz386afR1alrmpf7bx6+xriGsGaW2u919aUq5YXlveum7Kufj1zfen6txtmbbhc4VixYyNlo3hj96awTY2bjTev2fx5S/qWm5W+lYe26mxdsfX9Nu62ru2s7Qd36O4o2/FpJ3/nnV1Bu+qrTKsqdhN3F+x+uiduT+svrr/U7tXeW7b3S7Wgursmqqal1qW2dp/OvtV1aJ24rn//9P0dB/wPNB60PrjrEONQ2WFwWHz4z1+Tf711JPRI81HXowePmRzbepx+vLQeqZ9bP9iQ3tDdmNjYeWLyieYmj6bjv9n8Vn3S4GTlKY1Tq09TThefHjlTeGbobO7ZgXNp53qbZzXfP59w/kbL1Jb2C6EXLl0MvHi+1af1zCXPSycvu18+ccX1SsNV56v1bU5tx393+v14u3N7/TWXa40dbh1NnZM6T3d5d5277n/94o2QG1dvht/svBV7687t6be773DvPL+bdffVvYJ7w/cXPyA8KH2o8rDikc6jqj8s/jjU7dx9qse/p+1x9OP7vZzeF09ETz73FT+lPa14pv+s9rn985P9gf0df077s+9F7ovhgZK/VP/a+tL85bG/WX+3DSYM9r0Svhp5vfKN1pvqt45vm4cihx69y343/L70g9aHmo+uH1s/xX96Njz7M+nzpi8WX5q+hn59MJI9MpLLFrKlRwEMNjQ1FYDX1QDQEuHZoQMAipLs7iUVRHZflCLwn7DsfiYVZwCqWQDELgYgDJ5RtsNmAjEV9pKjdwwLoA4OY00uolQHe1ksKrzBED6MjLzRBYDUBMAX4cjI8LaRkS97INm7AJzNk935JEKE5/udEyWoo++PQfCD/AMf7G3o0obnYAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAAgRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iCiAgICAgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjk5NjwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj45MDI8L2V4aWY6UGl4ZWxYRGltZW5zaW9uPgogICAgICAgICA8dGlmZjpPcmllbnRhdGlvbj4xPC90aWZmOk9yaWVudGF0aW9uPgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4KTlGaRAAAQABJREFUeAHsvQmYXNlVJvi2WHPRmirtS2qXajNZA8ZuqJDBXxsGA20mhdtuFg9QYMyOh6U/BoWmjcHdA3xA21DFzLDaMMpv+gNc2GCXrZRtDIbKqrJdUqm0lfaUlJJyz4h46/z/vfFSKeUWEflexI1U3CplbO/dd+6555577lk1rdVaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgYeCgzoD8UoW4NsYUARDARBgDV3VO/rO6l3dR2cXn9DQyeD3t6DgablA13X8NpqLQy0MNDCQJ0xEASafizoNY8fz1n5QDMqeTyv5z/eW8n1D9M1LYQ8TLPdGmvdMJAP8saGgU+aP/7UgDPzoZCwrK9c+4P1CTfTYSTc1UEwldb1dKFgF+52ZFcMP/bIj92ccb1+/HjePHQo78747qF+22JYD/X0twYfNQbIqJ7u7zcOHeqfZjJfPpff5enFb/W94rd6vn3QDSZ3ekExbeiJFI5/hh8E+N8pWXp2zDCSr+pG6gupoOPv3rrnw68QvnxeM44cyWu6nvejhrfZ+msxrGabsRa8SmKAx7d+LWce0iWjOnvnLzuv3vnKuxxv/D1+UHxrpk3P6oanua6tObamBWA9vu8LZRWYlmbopmaYgZZMmZppWtrkuOaZWuazycTa//b0rv/z8xz08SBnhf0riYQ6ANViWHVAcusRyxsD1Dcd1vs8jvLkrWPtg8P9P+75Yz+ZyjrdWuBpxaKn+R7eaLqvQ06CRt0Q2imhgJ+JG/wEaQvsTNMNzcq2W5pdMPEp/acr/L0f/Kb9//kOdWEzpbeZdz8M71sM62GY5dYYY8PAsy/2JEI91Qun3/+fvGD0aLbD7y4Vbc0u+WRS4E9gP1pQkcJ9GlAdzC3QPPxvda5M6ZOj+uWEseo9b9v3R/+UP65Z+UPa9JFz+p6H4E2LYT0Ek9waYjwYOA7GcQiM4+zg73ddHHv5jxKpqXe5rqPZRd8Fj8LaCswInkwXByeVMZJ2UfdSxqp3vW3fH//dwypptRhWBBTV6uLhwgD1VX041h3WNe9L537y0KQz/PFMu7thYtzxtIASlWZFjpFAd6yklvCh2Uqaa7/rbXv+8O8fRqZVnZga+Sy0OmxhoLkwQGZ19Kimk1mdOPPMeybsW5+zkqUNE2OurQc8+sXArIgiPUh4juYYpqeV3Nt9J87/8mPUZZFpNRcGlwZti2EtDX+tux8yDBzt10y4GfjHz/zAT3jm8Mc1w9PtokZfqySVVXGiAxr5hOsEdqZNyxRKV/7sxeDFBJlWPp9/aNZxrAiOc/JafbcwUG8MhDqrL5372e8vaYN/bTsO3ROo/K63lON0rEgm7ImV//u37Xv2Q8cCDVZKTVgp642Tej/voeHM9UZs63nLCwO0BlLB/qULv/otE/aNv3KgXNcaw6wgx+nG1KStuf74L754/fe2klkdO9YbhYJf+UlrMSzlp6gFYKMxQGZA14WvXfr4qqnStT9Pph098HQHR8B6S1YSFUFgeq7mZDv9lePjZ9/HL7t6hx6K01JjEN5oCmw9v4WBSjEAJfthrU+ExAwVjv/3dHth+8RYAF91LVlpFzFdZ9i2rcFBtRfxiR/WdTBQOKKi0Uq5bFtLwlq2U9saWBQYeHbgGQuq9OALZ3/+e4zkxHvGxxwyr0QUfS+xDwP+XhrCfg7864Vfe4J99fcfWvbHwpaEtUSqad2+fDFQllgouVifOf1D/0fCcsC7dDKshq8bxh9C+POybZo5XrzxjYDpxeU7E/dG1pKw7uGi9a6FgfswEEos/Wd/5j+lsqXHi1OeBz6hhBQjcmUhdlp4fvnuXgJ+Itcvjq73DWKZfWj4TrHM8NkazjLBQFm6cvFqfvb0D33AQrQNvTfj9rWqFn0IqoaxsiCOhHkdJ0SIXcs5Y2lLwqqWQlrXPxQY6O8/KiSpL1745ZxhlZ4qTrnMbayEdDU9AdS0u65m6enNQXBshfwebGsZtxbDWsaT2xpa7RgYyuWFtc12br8rmYbYopmuatIVbAFGqRRoCb24bXj4lS1ytCdbDKv2aW/d2cJA82GAx6rQc9z1x3o8HLvwn3KMgAzUx4kwm9EtvNklMX2vsEXzYX5xiFsS1uI4al3xkGGgr69XrIvPvd67KQi8XY4tMoMqt1bIQeF65XW24dUfOsBpGhj4pHKMNUryaSnda8SmsNJg3w1LNrEbWabpCLZi0EyrVFONmG38bV1d0mvccYNupLVa47m+nNHGgzYXBHoS4Yy6Ye3jjz09A8s6prDFsOYigTm+Y0T8009rxpmOT+qDIApYYmBCFpuZ0HXIW5AlScvzrX480KychrBU7bC/nK02HOxybZCrnmhrY3515LlSTeEOgEh4TBPIJMqePy4YFumybOGcQZfLZ4ZaDGuRuQyQr7uvr087fDhPor3Pz4Um76/cPdo2OvzVtGms0dfoB0tPbv+5cRhvvEM6o/gPi94fxkRri6BV6Z9P5HKY537CeAAMgJyBWnflWplhGboL0tT1LbdufXT9unUfuNHXd5jH12UpaSk4DWrQBerBWYdO5H0ITIJJXbny5cxVv/8bbGf0m31/6smSO7QDkKZBzSuCwM4iw4dhGtYUnKDHTCM7lDRWfDGZ7Pj7N2/79QGOKJ/XjAMHevXDh2WxAjVG2YLiQQzM9GP61Ml3fTHbZv67qQn4DuiN925/EFYo3DUU2fEPrvQMCwdD3dz/VFfXfxlYzhtkS8J6gAoYmX/qVF8QFq986epvPzE2dek9r008+92BVtqX7QCP9x0tDU2VTxMNLUiivgA99myUanI0wyiCeMa+fWRcP/LC6R/5x2RixW9/687f+Zym9WnLmZgeQGVTfjyq5THB+eBTZ9/Xpduj2x2GDrLGjWIHLEoaBCkF2CBO+ZmsaRSKE4/iK7FBNiXyKwC6xbBCJIHfPDvQYx1+qk9U6v3nSx9+61TxygeHxgfemWn3TNN2tBLKNY2P6DjqgVxIMfwrX8NeQEWCssHOPLzzE4kO5ztKxYnv+MxrP/DX7cbWD75l729cI9PK5fqpB1NsGdwbxsP67oAm/ZgMR9uCydnkkmFVWGK+3jgjqaWNQDdBbFbCM7Spyd2EAbS1bOlKOVNtvSedz2O1Xs4wcx69dPkjGz//+o/+6UTx1S9ZmaHv9bVJc2K05JaKyICk4TJdZ/Q+GT3/Ud95/z+pnLXQHyL6jWBq3Hdsu6RlOkvvHvPODrxw5offzrS2zAtO5Sj6aDWFMDA8sEqsCRz7H01nTcbiUBek5DyRZtNlkYOSPtI1P05UYiPkbqkkzIRvKe2hl7DEEU3Pu3lg8bOv/+gP3pp4+XezHd7qwhgyOo6bDvYvE6xF4IkEUmVDcqIgwRxF4yOOg8Lkj/jexGc+//qPfOBte//vj4G0DDAtENjyzmFUJc4aevmenjNimk3D2GMluOrpmxlJua5Ix0UgaSFMcasFh3I9FOzxi9tBT0nQE2tL44rlx7MeaglLKNYh7ZCSPnPqB/4wkRr7MzNhr0YFFIfSFJlNWUHFS5bSSDlJpxi4PmprGqmJj/af+emfyUOhD0Fr+VHVUjDV4HtzWr+wrgWa8wQqyWPZqykFkx1ZZFiIboSgbqBoK77xd9y69aEtRGHZUsi3y6o9tAzreAAr4KG8++L531rxmdPvfSG7wv6JqSnbc2yWFBcJ2qJnJDhKYhF4U5NFzTdv/96Jsx88nIfu/niQe+glXRVWFY/o1CuePHkM2USN7XQYRYueDpY4WAKE3VRLYPVa+Eem6iLS0TTddsNwtrL7rq7lGaLzUDKs8Bj4tUu/ueqO/crnM+3Ot42NQKsuswvFHZGP/g3X84ta0b3+xy8P/u72Q3q/ewz+Xkuk49btS8QAvO3EehhM/AOyH3g7yiE5yjEsDpMSFo+DlLKkVkF329oMvL/1GH/P5a4rCTdhW0p76BgW3Rao9L5y5VjmxtSrn850+N8wPuqUgESmva3XJFuOrdlglJ13Rk/9N07gYR15w5epopTja4bW1S9DcowgscO09DbPUzckhxJWmltfmWJRSAfiIbQbvr5L4vo5IR42A96rgfGhYlgU+UPHzdMTn/rLTIfzTROjYB3YrKpBWkTXJibGbLhwjf8vXzj7q0+jz+B4f74lZUWE3KV0Y3uTsBByaejKWgi5tZJhkV9R2sJfnVklNG1yWVsKHyqG1d+vC4bwudefOZLuKLwLkhWU6w0rKAB1iekm055mezc/QJI7kYNnfas1DAMnTvQL/MMXcx8lGDT5t2EQzf1gAsWFmzahvSpfQvWbg5BHfLP5zp1Pdcqv82X5q3zRMnh5aBiW0FuhEOYXzv1vb3X9kfzEGE+BOqa8bsfAWeSCZ5vFgouCmIV3/PMbv7kd5OU/LAUxZyGjwV9QD0SrLcHwA/sgpHFF2ZUEi7qr5P2rFwyL7Ku0NQi+vInj6O/vv/8KftnkbdkNaK754FGQeqsgOG4VnGu/l4BUg2SN8GgXMTVz3VKX72CQgnVH89NZv6PkXD7Eh4apTeoCQOsh0xiQITkoPfN6fi1MhdscGzRiqOlyQl5KZkUrId9LMUoeCZNJLYHvhMf79OCW0ZuHgmE9N/CUcBv4/Nm/fn+23espTnowAsPHqsGNtAZXat+AnOdpzrcQnPBY0mDQHrrHhyE5U8HwRi9wN7uUVmSQqFK4IHOiGJgCzSAkZ/pIiK/AZ3U3kyHYQ/sl0P3yZRn9XfYMK0DYDUNuzp79y07XG//5UgmCFb1tlGmIo4CXsueVRMZIHksoESoD3kMCSJcm/ZZsvXQgnTENTAL5gprzAF4qFO73MywwKgbki9ODoKVcbvmF6Cx7hhWe4y+5X/qP6ay3Ax7BmFHE/6nSYI/m8cPQjS0n3/joegnWUTUXiio4iwGOM2Fq4cDenUgy2tkgnajZQB2hhfBBAH14Jvv+WJjMj2Lig5c09WdxVGrqESwAPCUVism8xA1Gf8gUvEq1uD2hx4IDYHG9b42sA6g3+soZAxYYWuuniDEw+LxMLYxAhMeZNgj/KblpkP1wt6XT6GxWpKOKjocIVWPr5OSzG9rafnyQKbxx+exLI8Zfvbpb1gyr7Lnsff619/cUg6FvKhbEpqmOdCVmGelJfc/vbE8abVaGO+PXesXxhOmWW60eGChvbNJC6LvdmA5xFlRxlVPJznAcKt35/v4mNz9NK6yfmrpMSyEY1vIq+7Wsj4Td5VQhdnDnze0dhgGtBKUtpXZOpsSCP5aXTTsgwuui5PjAQF4pGO9fFMvvUxiS8y9X85uh0hYhOWBZyq0NEgW5apIhOWRYeP8AoYh0ONksEozoUwc5UwPlNcD3y6EpNylRIvVCz7DcNQNXTB6Oh7P2pCifV2tf2CmhLXXhweoKL+Xnn1+e+bhrxU/c94UhOa7jgGG5q+gxrpJZZub4ScBprFpaCOduhp9MetBjTe3i7z09zylJ83PDvvi3y/ZISCag6zJ/OqSYx0QqY0X1EuCqyApIxfvozvB4Un5dVsS2ODk29oqSfeexTJupTU46SNYYMNOUeg1AhRZC5nSfzbfgj8Ua1b77RBl4dY0HNWB3GUtY8lj1wmv/YQ2Y13am38Dszp7fGpAW+S1grcjfgDyBXvfw8HNbZP8yc0Dkz2p1OAsDQ0P9IA6hE9qlG9g+AiaDVY9WCCQlP/pgLUDIiClkiE5pGwZh4nq8qGlAmDURFXyxbCWssjuDb/vGViSNlLm559qQKkBSnJeQCJEURMQIGUZhhavd2YavLveXMwfE+exW34JJYbuQR3A/KD4RQGzBvgZ+JXiYUigStAJORQlrfuh0vYRK1dTFjYz8Fje/i8gFQsFkWUhay1jCwhShwUx9EI6AOvxTSIMLbEzy+nr/FUQIRorqJx5KSkGFckvoseoNx8P6vKNH84ImguDFLISRLS5ipTAnSq4LcFEtAWjDkJy554whOhiAYXdoTmkrr1lOm5+SEzP3RFT37dBQl9iEzMDaa4koUTVThQgixCwkDRoE4IXvu8LpD5VPuE22WswYOHBAmv2/cuFvNnlBqRsZZ/HExsaYzjVkclVCJpL2gV74fp7dV1gK27D5edqQSOY3V3/N+t2yPBJS/xAq3APdf8JHBaSFZrhRkzdNhBCsTIDoUQjUx8N8RogO4TgE5I0Ccdk/V6YS7tN8w+g2LS2NjAdQ+KiJdG5uzOFOCyFTX83DsDBnukjmh0uEpXA5bX5YJsuxcf/RtE+d/ekUXnbQEXCB2RXXNuqPIELYo6Adhe4BbmKBv31s7BNrJTz5+WmyUQAvs+d2dHxS4Lho3z6QxqEQxUekeUbBcZKqp0NyJInPC6Uo8htMJ/MTm9+8FzfRD8uSYYUVQwz7xhZQ4w6bSkhQoorzco8IA8NmcaagsKVYPAN/ILbl5aUsx6TW3wsXBuTRWzf2+fCFo1SrFoQSGtIJ0yEzad/iTSbz0wJvaxAsr2R+y/JIOJ1TSte3myZyc7s+7brKeY0+QIQ8h3jQPZjF0jgdXV8peykvC+vO4ous/leQOYUWQs+f2M/CbmqyK6nR4FEQxhmpI8D7+Rs3P1KXve3u3S9y8zu1XDY/JaWO+Seisl86OsbFdGL9P4bqvbhJV1LMJ0ndR4Sa9FIOggmheF9uXsqVzV49r8oLOrk08vwq0MhWh0dysjAFG3kprYMLWwinAdcRhgaP9yDh+6bQY2nl9DnTVzTpm2XJsC70dAsxH+rTPSrPy2wiLBcS8O0ny3C3pKtYJ1AeuW8OvbTR86c2eeBXEF+UY1gEiJtbBRbCe9hCTH02i/vKyfwGwvQ5965oynfLjmFRzEfJrPJC95UNyZmLCEFBSJmMODDN7g6CsylaCPP5/LKbI1VWSuif5CfcfcmUiSK33ELU9AonZFS4V0oMXAczk/n19Mj0OargvlY4Kh1/rf034D4p5v/DyR9ZDel+s8vqvYqG5MwmQl2n7kHXnO2jo38tFO9HjrQU73ER0VCun4KLViiN7k6kuMep6asXjj+MIRRAh1/O88qDLYO4A21cnDLwGexYTWY8zxDm/HrZMawww2jgO5sxQ1tEbm4oJuYcvQJf3k+EsuR4IhFkPW9MFBIIpQAFQF1uIOiHGQiBBl89JO3D20o4QQOwQLBoIZw7ad98AHHzc3m+3TY+/udMDKmF1vP57miG75cdwwqRrhvFfekUcnOLkBz1dpZ5iJCM1WMhAd8dFilxwvG0XqPFAKQNSt6CRbneRLdgWFzeCjYCKYwztB9V2Dg6RBlBrCqsLxTOMpnfsqjItOwY1lBOhuQg/9reZJoEqKaYfx8RimUzTYlYS6xa7wuPd3gpi7PK9K+tNxFh4KjgTq9c+cNNkL+77ZKwECq5HshbZ5f1WgQNOP4xvXtb1kLG5MKy2fyUnKBFpmL+n6lw1/rKFkIXCndcej8zmP/eOv8iiTCQZur7ns3KJ6C0oIjqw8xJL9KD3HdF68PSMRDmzbftSVoI1woLoYIKd3JVEjSzjFLKqoKcxa2JBFLNeDKZHza/Km5fOo7j6GFZMSyREkQucFQb13cKMV/so3GgrvY+CRIph7smK/jy/T0wWUiA+hR/5927R4UoXy4kUPsDW3fOwkCYPtvR7jyaaUvQqVhdSRbACQthmVZmDWaBL6gRQVC9SOaHzY920HuktsB9qv60rBhWuLA/9dp7t+BYJUNyKrcE13WOKGHdr3APHx+IkuOG4axx3clt8tuWpTDETlSv4+MydbDrFbsN04a7gCkk86j6j7ofQSvslPyn4oZEX3STCQrbIa2zcDDurqqDip9UrwuXGcOSCzthZDYbprZKpIqdKbzUC6uVPAf73DxEyB3QbW+nhnW4nBtr1TKbp0oQFO81h3IyoR183p5UOX022UsYQyhYTVXyEUxPkNZ13e8eHf2QcJNpdkvhsloIz5UrhPje5GOZjDCpQJOqnggsiBCAMdXtXESIhMmBAVsBCE2E6Gjac0rv/vGyluh7p24QVIGXwPL84hZPKrCqYgXRQzV3j6QPqg1ESjdBLHNfN/e3MnICDg6dtu1t4TUync7cVzfDt8uKYe0ZPyOm1DDMXZCwQJPKxTsLmuBxkETIQNa5GwgNugffL4S6B6/ZdQ9zj7Mx3/ZpfQLzLw9+dFOgueWkfdDwKNYIEGklCVoWMYT4XCWQuFyHtM4cBzKZXy53vcou1ELKvEtGLTArgkY/JF0AcGa3H5devur5XwkixHBIhKK2HAjyQQoic3IcUcxz+8jIxxCYy5Z/8DL5detv1Rjo6v+YwGXg2lv9oNTBzWHWJFTdazw3ULSmw2iVFsJpYCitI9kMTI2+CILu75e6u+kLmuzNsmFYiLkTYv6L15/NYr1vRUoZToWSizwkwtBCOJtmZIgO0oNs9v1LQvcQevDPvrb1Ta0YmCzdfDQLXSE2CNj+1UuLLMYFXjq3cabSUctjYaDLZH6HDiFZaRNbCpcNwzpQjrkbGX0RC9zbJkNy1GRYVFwtTIR0+guCTMYwPXvk0UpJs3VdZRg4Uc6Xj7q6OwPNBpEoqjrAcHhQncc4U9FgyZwYUA+V3eYg+GobbgL15ZXcyCsZ0LJJ4BdW7zV1o9s3jTSq+JanuxI01O8aUIuQ+5g5cgGqwU+6i3xGVmHK2c9bloPTH8fR6MYFTH8kwuF4Y0+wDCEZlpiXRgM3x/MpUaRxJAzpZo5LFvmK0jockf3S9tu3+yitv97MyfyWjYQVzlrJm0BZL+wniobkEE7aL6czR86zUrCqAhEGqRdFbiwssqYW5cP5afRrWNZraOh0B3z1NruOTVpRch2Qi1LPmQDB1M5R5ZEwmYTa1NR2Ev/NHFCv5ETVQtQnTvSLXdMwdYS0YNcsb0q19BXnPSERUukuiHBeMYsOpCK2aOf1659EKja4xZZr6MUJ33LvOyzrda30wgYvKO4QgriiZb1CXSdrEXJfm5dUFp407H26y2R+jjN0YOFL1f91WTAsLHw9n5divufZ+1m9V8VGgiMRMi5sMSLEmMxybqwdicRxoXg/Uq6hp+LYmgWm0A/JcUu7DMtLgVZgRVMTelIxJXE6ji6lcWNkMj8wrlC9wJ2wKduyYFihEvGlwV/tQmj6NiGZIJpQxRkJiZBm6kUaKvj6AUT5jOY5e8W1vQcXv2uRTh/2n6fLejmD+9KwJ4MduNzwVMQLGQ0V7mRYhLT2xoB6HDnKtQLAoOk0q+SYFxvjsmBYYeT95NjExsC3NwkLYVBxNtnFcBTt76A8OuFzV1+MCOFD42aojwvGhSi/XPJyR4vQ6nr7SLmsV6Cb+zwPhhll5SvJqFg4VbTFiGVBNOioosNkfvq2mzf/+BF56dGmZFjLwkrYJSqC9Gm2XjiQyiT0QqHkY3KUY8akOTKq+4hwAbLhzs+ME6xeTSJbLnm55YKp/19KFWjiOGS7w/utBGZkAfzXH8J7TySt1B5DeK8f+Y76UL4rbNSNyxvx5mazWgqVW9QSwdX9PVOuCOIFzu5EkvK9oewZnUQ4XwzhHKPmsRBSgL2s8nLPMc46fSWlipHga4weQFkvrGJwsDo9vKrHUA1LPWeFZb0W65tjRCZb1hefKkvrzRlQvywY1uDzsiJI4HvIzc29Sc3zOXUSJEIGsvJ9BU340MBDY+edO78kFO+adnhZzFkFY4/hEpnN48LlL2/w/MmNTCFMn4YYHrSkLgkQyaOqsl6LPtHwUyy04U+KWgEIqF/0DhUvaPojYVnMFy4NQeDuYKoQNCWJkGKfIEJARygrABJeylDG6f5K6Ft24JarzexDA/gb2kLcBcbEvkRKt4pFbhuKbm7AFFUH3J0ERS8Rc1wnMr7WFSmLnn9eptdZYrd1v73pGVaflDi8E+d+YUvBvrbDgVcvJlhJOZ+EJ4iwzLAqmG24YOteW4dhToyNktC+WME9rUvmwUBY1muieGt3IuNqxaJQHai5BkAsNM5QhRBR2XIwLIZMFneAeRk4G8KhQ+j0ouCH82A8+q+b/ngxHZITWEgV4q3iLqKoWkJslZVaCGdMNagKNGX4QveAEB0hTc74vfW2MgxMl/WC3fUxWWS0shvrfRU5iDTOROn9LEN0wKe2F+7+ziY5puZTLzQ9wwqJqejefjzbZiK1jOZTLAm/V+V1JhFWC5OwFPrFx3gfCLnp83JXO/4oroc0wTM4p0Gz/THk++cBXTkyEUMlkPTTmz9fmrisqj9kgEJnp9krC1phC28Oj8hVddTgi5ueYQ0N9QsixEruRuofkiDV7spRItdLLUQIaVEvlbC49KD7xo28KIgZOso2mHaa7PFHBU1cHPr/NuBcVC7rZShJ/6SVmWW9okA0j4Ho12vDpu6618XmF0W/9e5DzfN7hVjABPD0x60SXKr4pLAQkn0px66k4rQWIiShOQ69lJ1NpjW6FaO71aw+NJynRrU+7YCgijtTN1HWa6KLCVdANcpRCgEiCTN8SyR4xPvogNQDw3A1sOld6JYZQJpOvaDkDkNkVtLCYGAGB0Os2sIKIRR9VWshEdZopmbuE49FKQJ3XOyMA+Xc9aqNU2V4ugdeELTuasMH020mcBooeyakhBVNSM79M4LdHRlAwBB9mcwPa6Xp1AtNzbDCyPvzBQYHOyI3N+Y6DGa4f7Ya/GkpRIgQHd+yIE4avgheDUtUNXhITfX4EGcle2ynYZbgLtAkZb0ixDJo0HCpyAqCrUFwvF12nVdwi59/0E3NsMLIe983t5uWlsaRUOHIMCZiK4v3lPmraJAGhA8Njr0iRCcnS1Q1FaFVMdxYLj10QmbzCFBUVAQCR3nSihBikoaIhgiT9kXYNzgVivTCKgXXhtu3P1e2FDZXzcumZlhh5L3j3nk0nWV6Bl3JyHsSIY+qDMkRrXpWo7siN5bfHQTH2tEXeFhz7YyRrrsqOyPD1/I8/kAf6E5uVbqsF0hEGGdiOCeQBrmpIwNICu+EHksTcbhVIrSBlzc1w7pQjrzX9cRuqiQg8lbPCuqAfLIpUdYLRFhmWVU+FT40UvG+9c6dr5RDdJprZ6xywJFeHpb1Onf34xsR5gTVAZ2LuXzVa6SPmcaZKIHk+mAGECbz0/y75ZqXn4zyEbEjtGmthBL50kKIuLBHTYbkKIh6gkQ/i+nacjVxLHEkDDJpI1m0Jw+iy9PN6EMTOzXP84CwrNd4YWKLF0ytYDy5YFc1zcU8D4nga9IKQVpKWa/FwOC6EZu7rglH5L6+gaayFDaxhCWPRG8M/8lKKK42O7AQgmEpyLJmE2ENQGJkupdKk9pKYmdsFaVYbGnO/r1k3zhI52IsWmXLegG2WCyEITbIqEVMYTAhkkIePtxctQKalmH19/cL2K/dvrAJSsQtzMWmajK2SIgQhCyUxYH9DWXigzZCTQYdLg5VXj9Wdi52PXtXoJe4rykmW0lMCaDAUBapqLREtKKsKvWhgb59bOzZtbKzfA176BLBqPH2pj0STo/XKO1Noh5IqWRjvmlfUavdI8KlnVi5M7osDotUM0FwMqnrB22hTG7xrAUnnEwduBPOxZ42+Tid2siwxLwseGdjfuQuzAD5uODj8OmIjGR+61335ga8uV3e/JviaNi0ElYYeW+7o3uTOCpxJfMP/inXJBEuNZBVh0kayuLA3XXr1p9u4SD7+poveLXekxM6F4O5w7nY3uKoXtYLFFxFvrQa0BkW6UXRL2+M+tCmas3KsKYj78GiDjI4OLYtaYnTya2cFsKlEiGlKegegkRCQ/VeW5ike1tFKRadndC5+NUrz23wvKJwLqaIteiNdb6AAFHEScL1RWQZxfuYgES3updMekhbM7WTw2wmfWhTMiwsXs4mxSrN8ad2C4YV1/TyIUtokgjLqW7RzxKIEEcb3W1rQwJobVyUr28VpVh8YkLnYjdwujXDzjJJAw6ES5iGxZ9Z6xUkaGZoqKCiUq2PmL6P+lCk3hZFevFl0+hDm5JhoaSoILiXLv/BRrzZbotsBuoVnSCQkghRZDCCJUL9A6oVo09XxBS2ilJMr79533R0XBeYn3Ku78uA2YNdRZQPb95H1vwD9+E4YghnAwSlHpL5aUFpOzZ/JHonmZJS1W9NybDCsl4FZ3iT7xfXeczNrWiqW9IBiZBK86WSBPugSRq6GBFTiM/03hYLUn1SawyEFy48J5TJwNJez6eFUF2S5/ySVuJvZX2o5u0aHv5IOUTnaFPQkbqzt8CsdZezFTje8KOZtgQZgbIKd54BBcPieJbIsbADo74c9XV+99DQf6aFB62leJd4mP2XEin8jATDctyRA2VVwuwLFfiGpEFeRafRJZJJBaMJUCuAl9krHafYVJETTcmwwsh7Tysh8h75fZA/uIJZqvslIRGmQyJc4h5G+7zj+Mja4K01DHs7B9TyeF9oWvPEOPjUUIev+VsdFucjEhVsPA4y/xUjIvg+5kYcuB0dJnJj3Son82uOsl9NybByMlsBJtZ+glVy8J/SRJiIiAix8pA1Uvc62g3Nd+4KxXvMhN3k3ct4y1euPr/eCwqbheqAbliKNQLEHZdJ+1gGjvwqbiARUyiS+SGH4U6io7//ufjZJB+0xNZ0DIs6G+yReAksz3cQec/TYOzzWzWaQyKssqxXJc8RRSnApAXDasaskZUMMoprQulT1yf3mpaf8qhuV3VzA2Shwj2KsS/WB9cRl44fTAhLYVkIiJtPLgbWor83HcMql/XS/uXib+Hs7e3gEUmqtBcda90v4PKg13IUFsIQeHBrZI2kpbAgGBaYd9OYpMMx1Ov1hNYvHjVRGtyVSjMbrU7NjZqLEsQi8qUBujqJOtBjuUCGv/nKlS9npBCQVxM3Mwim6RjWvbJe3lYEGXQ0jYUwOiqUinct2DU29uE1ci7VJ7QZNFe/t+WkfXjgo5DGVRWu5BEQrCLeGMIH0c6YQkpY9o5U6jOb5K/qpyxq2ljCKe/2wUzW1MbHNQ9zDTlGrUb+RPUuFe5RblsQ5cGwOFZnk+eN0sJzp1WUgvi4v/HIgyaMMSV3eDeOhJgPFo6Jbue4/4m1fyJEYdK++kFHS6GvJRJBWvPcboBwLjxC1z6S+O9sOgnrxHSlj2A3JCwyg/rNcRXzMYsIo+NacB4NvGzWNG37tig73ipKMdfEHBUYHxz/QhfkCJT1EhZCJemdVkEq20VITv2oGU9EVXEEevnBLZEbay4sqvZdU0lYmFhsmtKvxvUnH8OmCYal6K5JIsTyiIMIWZQCO6OpT/kip1FPT3NYeOpJ/KFz8a3hM4+43vh6yFew1PjKMSxyVfIoYSEEdHwvOG19kMW9D4ZCK8yxJiTS+jy6tqcoN4ELDeOolhdzefbOpzqhed7q4hCOCVZuDCER0kIYRyArjzsyN1YhzI0lTKUL4e5h+y10Lna10YOpjAmLMmKaxCFdPUxQwgothGRY9WvY/RlT6I8JCYvCAIWC+j2/+icpt9gXGsKBPqkUHBn9+gYvKG6n0hB7kpJjIBHSQsgMXTEQIRSmtBRqO65cORZaeJTEw0LzGedvA9pzovvJ0lC3mbChIzLB1GOYiQgGQaiiioaoBhwq+WwbhlNd39YsyfyaisjDyHsnsHdbiSAZYD9QdTsgEWbAsAR8ka8TEhqld2d7KvVvVLyjqW/hkXDW5+/g8zJpn6b7KOuFjMh1PWlVPkaSBjc1Wgjr3SCplw04hU2FwuBG+Xy16aipdFhhWa+SN7Qvjcoftm3QFTBR74le7HkhEU6X9Vrshqp/p4Un0NJpI+16U9RjnW0GC0/Vw6zxBh6Z0YQ+xvbGtplgWNw46s8SFh8AYaKFUORL4+Vih1v8voiuYOpVP5NBxl57jAH1XysbcJRVMTQVw/pIWNZLM/Z7PkRZnrfrO8EV0UlIhMxtJBZJ9DDqhqG7mUxgjY2IKjrPR5mELZ/PG0x819uLkA3UrTszIFO07Bk/EwwNdQWneg+iKmKeQxPDqwgpdbyor6+PJwfvjVt/v/7C3b/dxWNPoCONIs/pijWpOijrOhsAHgw4XjrlGaViYbdEjTxKK4amaXCahmGVd03B+R1/bG+CysLoGcE0YpbyhkS4tLJeiz8d+EDSBnAM3RXVoHGHkCgWv3P2FcRtGEHQq/X5up6f0Vff7Bs0fIfN4nh/ziIDO3y4T6kdubdXwjw6NbjBCSZXB/TUU5BaCBV5FKUrBj7zfb1JmnMPgwQe7AoXmZ6e8lEasKjYmoZhlZP2BVfHXlhz6vontjkOlYU4/Su2a4ZEKGIIYyVCXXNFEjZ3TxAct3T9EKpei6MQ6b6ihurRRr/Wb+D4BGTeI9STt4613x15aZ1hJtbY7s1NU/btjkQiXUyYnUP4N2gkskNv0fN3D2n9vE87hvyEWl+vpgrjCo/HRf3Wo/BXM8Yn6M8QKFkqh+RLXSf1WOQb9W50bvZYqDEo7OCzpaWwOjqqJ8xNw7DCyh6DIy9v8IOpDQGXiqImWNJdaKaOjwhlEjbsjDuHh/+BCtPLIVNfjIDyYFRPo0waJCliUUhTJ87//GOOM/W2QHO+5dqdvz/o+VNb4JebSqQMK4s0JAHqqBVLt7WifbuoF81bnz39Qy9ZZvsLmczev32z/jNXURJDO348Z+Fo6oHoG7D07o16qFzWy7Ynd2hpkbSP06AcrQskgWHQmoyXBklYugj18gOve3LyIxvb2n75ejnHmlJSczi7yk1iCNh8r65bOJBKmWahIORYzrOSLbQQkijjAVIq3k0zWOl5hZ14zOX+fuGTNuM49wBqeIzT8uahMqP68pXfWV2cev3drjf5Hwulq2/OthsWrGowZkDnwzgzoLhUwjqydeGfAz5kQHeWRpjL1mTK36rrpe8dHfuXD33+9fd/fHX28d98csv7r3EToeR2/7HyATji/IjnHy4zYdQgfJLnZtXLetFC2CgOTwmLBhxYnFcXCs5mvLkeSqhxTlOtfePQ0hwtLOuFtMi7rCQFA4M7QDy8YAko4dQTqXXIHMmxu+3IjRV4Y+VUM5+cFx/HjvViH9c0MqsbN/687XOvP/Mr4+MDL5uZkY8mssV/B0nNmhi3vckxz7GLyDzik0npcBtBlzIlPYbFTBHgZY7mTY17zvhY0dPMqZWJtrsfuDnxxZePv/5zP4rLcZ7I+8cC+Tw+s55NLH0qCpB+yPFQmh6SIXjYvHipJ2wPPovHwdBC+OBv9fpMNQLg8JnMz/cHlc+x1hwMa8auaWge/Gp45q7XlFb3nGkiFOyhunurv5rrkEvUL2eNHJhTujoe5C2hX8JC/vxrz7znlbuf+mqybew3jcTU1snxkluY9FwfDAqdca+Hm0jAeu6kDep9Zi52vic7ENeBmZmua/jjI7ajm1NdZvbmH79w+sf+MgjeSB/W+1CUrBFM66iA99TgX+CY7O62UcsRECtJ5yTh6ZCcxtEz8KX7BvZ/IGkXQNLCIzXfq9aUnMgHkTRj10TJk8ntPpXNam6ago/S6hPGEM5c7Q+Oa6mfiQLJvKeLUoBJ3MdghF6JUtXXLn181WdP/8jHjczox62Us3N8pIjIJhaf1akWwL/aIgbAvQw8MeE6hjc+VvAynWPvfeH1D30aOzeqU/d5oWS31LFWfv8BgXI7KG72vKkVLOsFBhvnNFQO2owrCRB3CLq+sG4l+VWjgAQdUSIFLU0KizPy4M+ioxmgN/RtUzCssKzX1679X5uA1p22LU6DysE+TYSQP0IijHl2dVaDZvn6wcFfRVYCVoPuncYLleCHDvW7Xzn/oaduTH765XT7+HsmJ0u+XYQTmw5GBUkpOvgQjA2z7cjdkp3pHM999vQPC98CSnY8dkT3nIV7Ghh4QYy/YF9/lMYCcALqD+r2/IWhu/9XSuNpbBXceBonYAl+btDqjrW1LQhehEs2VtzRvJI4mybu+1Gp2qfyrmmPb/T8whrlk/YBq/UgQvAB4fGO8vXrLauwhbNGZ0826pDIrE6c/YXvHrFPfdFKT20bH3VsiEMkREpVcTQ4muiJsWHHyXQWv/szr73vKB8S+njF8cAH+3x+XGaugGNxd6AV8XOEPPnBhy3xMydCZBllP43kWJCSbZsAFHeMjv6NqMYUVswmaCq1pmBY4a5Z0u4+lm2HYCB9hpTcATi59QtkZdYGzW9vt6A4HRV6LFp4jgc5izqkz51+f68dXPtb3SikiwWNoeJJ4C5WvMn+dRN6LWjfx3/9M6d/5K2EhdJeHQhfz+ekP5nrj0HXiSfGOtraR0T2EMYQ8n2j4US9y8A0tSy0LTsJTm/vQSUx1xQMKyzr5XqFbniNQDlokhSVayERMoawTkRIovItPA9ehyKn0ebNGbgt9Lv9Z37x7Y5/5xhS4Gquw2MRlen1agHWou6kMtSxTf5XPpXSXtxHQ/iX0aiJxwQp37NRJYc8WkiU9Rp4xc/hcZBqA+o6GytdCZABie61t0MScIZEkV5Nm9/iXPEgY7iwKRjWofKuiSrPT7LyMf5TkvsLIgRGqXSvFxHCS51JK7BKp0T1k927P136yvkP7yl51/vMhKP5rkjLWg/p5j7yBESJqQnPS2f9t3zutR/7Yf7Yrx2K9Xx2QJOqg1du/OkGXyuhQEmdto37Rr74B7HL4LIkNhoyLDUEQXJ6ZLXQ9TLDmtvivPjo4r1CeYYldmW5ayJ8sATLD3WojRagZ0/KNBHCEUBYCOsEJPAD/YMgtF03bnywjZCN2Wf+LN3mrHBsDZqJekpWs/ACZurhjGb/FH+h5Cfmc9Zl0XzR1f8xuZG5xe2BbrexZiV2DjU3N0BG/RX9sFRpMpnfuGBYkEuVtBQqz7D6NBF5r71y449gIXS6HZEHSlExn0QIGYJEKPb2+lCiVJgGpe2PPPJI22dP/+yvZVcU3zwJx048PlkfEOZ9ilmYRGVuy+75l3NHD/Gq/v54pSw+Y7x4eX+mDbscjjn4qBBLIHTlBgIRus760kr49NmvENWF9R3J/EZGPr5KXoAjtmJNeYYV7pqBU9qKGJF24Vej6K5JLtUIIuRR1IDf56XBwm/Y3t1fmhwvgMz0WI9fFdIxloEBXRbCfYKJd/KeMGKhwvuruiwsUIL4IQSEI4ZQzXhnsZlxy6WuUxmOAFBQphDHwuIm172wXiJevWR+dddtVEWBMy6edG48mmkztLExtct6hYGsM0CP+y1iaEQ4kDXinftRX5tE1SY6qNfmCBo1sNCCI50zmYfzFh5fwcGEXxZ1b1E+C0wbXcpAbscbPsjSJCqX9aKkwM0tUiQsCaGsBO0HaUSKejLH+2sqxhQqL2GFuyaUIbsgYWFHipbQlzTHD9wsiRBaI35fJ0qkdMUj6ISra9fGRnxEhIml+gBoDfsI8Awa6zxvbPuZwedWS0CORi5YhAVK7gRnO6GL2WbbiCFUNSQHSKGeU+g660QnFRAApWEvnfY1zx7bxeujTApZwfMrukRpCWvmrun6o48bdOoGw1Jnju/hmFLOfUQY+ZK896yZ70Jt3p0itKS+aSRQqUcl/BAaHadT27+rnb/xLwIrR2cOIKL3YYGSKxf+cb0XTG4VGjwF9VdEAOdHxBDiA98LpESEh6V0AwlYhHpB1pqZFDIEeSldR3av0hJWuGsGwekOVGna4sDsBbOrcjCHMyrKetWRCENiL0G1POzokLTUYlb3USlOg15bMTZe2lV2dHRNZ7dh+WlIWUxDoAovuB8VwEKMFZXue1Z1H1j2i9WYit28D5sheFhsU1YdaOWrlVv8M0cR7ppfu/4PG32tuE34AcosAjMvU+I95zUkwroBhGfSW3oczIohhXyvWiP/Rg4tLILgjtYxMUX4jmj5yFfBmQHp6Fj0bu2mkl8PDDo1KIgRKVVRfyXmK3JMLIUCmBSSBTv8nbdvf2iT7Cn64/tSIFSaYYW7JhwAd5mml6KHZHgEWsqg47iXdBcSYb1okLjgUXTUjmNEEfUJTmUhYTkW56V3bvykYFhxcJEXygVKQCIHfR9KM0UJhbRBRpXG0V20OJBR89QxKSRvLq1JaD6z2KKpZSlUmmF1lHfNgju4L5UVUw3HQ/V2zVlEKGc61r98JmmdySKnUGSBE8nvlGsQrgzpHTlG2JiNFIBHCiqOLXofUqKwf8cd2a3aMYZwzWxEB6XxSJEw8wG1v6eCGCE6ll7yb4rY1HLZr9p7jPhOpRlWWNYLiqv9zBxJBVbE44+sO+6a9SZCYoP6K5FhRlHMyOA+Cjz6dSK7L5Yd+6gY/XDw8koIdDscWAj5wMgmN8KOhHEGkClmIZwxQh1B0BSz/J38sqdHZr+YcUFD3yprJeSuiSZ2TThD7jMYeq8kCcpjWb2S9s2kFqKDDIvKGinEzPxVjfdgIHCJglhspF8lRF3IJhE1ZCETPH/xX2EhnFgfwMUD4kvkz1kq3ASIUhVppVFlvSoZg8hy4U+K2FRcL9ZgJffV4xqFJSy5a44GX14d6MZWWdZLvV0zJMKZZb3qMXHhM4o4DnIRKLc6JUwCNBHT57t3Qpijfp1mgubIvmTKSMLSBfOguhbCtOUw26GqDXosxKZq/pYgOImssVrAwrqqAKsMILMRIpV9Z658FUn7JjaImGcFd03CzVUpFO6zBxHbN6R3Hi+KDPVXtGG6oL/SjFJBKyYSm84SzKFcF9EVaQvDfSbtOzutJI6DKFCChyjHEiBtCqg603txeEhB8KTwohqYjCnkFDndIyN9QvF+5Ig6indlGVYYFuAH4/uxa1rCr0bRXZN0WG8LIVe8YFigedVInrCJBhWuCeUe0DPRlknfEN/JoszlC6J5OXVU7BkwPASPq1qghA7POuI9TT0RJMwdx7HNFUwhZtEhRqXGEJ1AMwy/zXX9HYQsXIsqQKkswwp3zZJ9G2W9HCxKVXdNaaaeTtpXh1klhZNJMd2T7StsIQR8JrLUmUb2wjdu+vAwUdPbeyxSmZC6znxexhCWvLEdvoiOV4+FU7pCUDZcGkw30BOfwBY3nEgQTuVCzcqWQmw07o0DnDOVmqoMa7oYJs7QjwkiVGwfCieR+6OwEBKTdYSR2jwH0pVDW4R663MaPSa0y0gxeAVnIB+MhcHPEWPpqBj9ufH/sQ4+E7sYQwiEKEjXGDckrITRMbGq7al/Qm2MG8kkbV6qSVhi6jBHvmaY5l5+QkxhpJuMeEKNfxScWE4h8AUGz93T9ka6Vd01iXOuvmkLYY2TUMttXKV0ZyAlqcqv5CyiBpi15gzHeOBI9KD2lbOMjo3e2uD6413IsIonKalwDxIJFNbWjRtrO3peC7QV1xgBQGD5R6UGGANWzPX88YOECxuiiKhXAUYlGVZY1uvMVSj9Am2nXYJfiA55WrEmlgZgYiBrI8zUwkKoHLnfmyShvYJmCQvgNL8dHuiJfA67y2W9iqVbB9NZy4JeT0VNNodPAwTSIq+4yA9AxGkTVR/QlJtByAtGSZTS07aPjHxMqWR+kRMQZ2DpTebmngqGNrrBBMp6kTVAWaNgozBYb4W7wAaei0Lx0Yss0eEYEPpmcdLUOpIbL7LbPePvjHxxhgVKnGBih2Ei/ZCiSfs0xJVZVprs+xRx4RtrztsousZ3/KxWC1CnEFo3FqMt3BJlv1QJ0VEQWZo2XdbLu/toJosSVhrtv+pqasiwBDeNfDnOT8a029OlQV2swGOaYqemD2czm85xJLmcVI7PP6rqfzlxQvYZoEAJzjHKIgR8FHnBdC1prXuNo9SDxKmiTF5BMauOlFMRjkX5OCTzSwbGuKjGpGlqlP1SkmFN75reVLduFrErWaBE1eZUQkQ1BANZBXR1kAH5HE4ale2OyhZCoMeETtkyO6/sXfvuu3KZHIl0EmdaCF1/aquHonp1mIKKVvzMi0TSycA3XdtCBVtTMCzL6r7sealbCRRfo0Fi5vVqvNd9JvPz9YndhGdgIK8EapVkWIfKZb08v/QkvaTxnxLIepCQuProSpOs9x6JZ7IWh+IWQj+ZSlACPIcFWYzDQtjXd1jQ78lbf4Ic5P5OBwhBYhnlaJoOWBZdGILkzbXtj14iHa1a9QPXQNjXaCmkdelB2lLhs8iNFXiPE5aeHjVCdNSbXFgGsU1yDi3Xn9ziChd39Q4+ID8Aeb+FkN/Vo/E51F8pSeUhAgKIf4GJgPAucRw8cCAG4adXPsx1Jze6fmEVNzf1KAUwQn9FCyGcRq9vXvO2QUINJu4ZRscp6Twqx6HSX+JROuEWuiW8tBQ2XnBQjmGFZb1ODf4FwgLcbu6aaPXiBVXRDBkGHUYpZdWTefBZJcYQ4o2SiAF80NmA5qncM2TQc1cuclDDGMKJycGD2TaT8fHkkpE/pyqimOdigw60esdJHv+Ow6jMy+A4cNYw8DZq17R5YKjua4bocO0F3Tdv/sYjvDeUaKvrJ9qrlWNYYVkvxyls9bRCh6KOy2IWyDBYDJN6rHoxLK5GhuQUlDZDgFvRQjhlae1paSHM5X4ychQNDfWLPl3EvWmGKFCijL/Q/csUITlGGv9MIW1uPvcdVCJoprn264UChoAIJnyMHD/3w1DdJ9A2gqAJkr02mfRETGFvWaKtrqdor1aOYYXDm/KvcdekFIHQcTVKVoWw8VVQF7jHtIVw5o8xveczybBoIRQ5sGJ6zpK7JbeysAYDYzSRWC0WaV9f5EGE+uFeaSH0g8knOCFCub1k4CPvgNNm2piwhNn1dfZ+1XqE08gj14WSbfhwIGXjdQq1QFgK2yC52vbNRwmYCjGFyjGssKyX73ko61VUlQgFYXFbrGcMIR9K3UJoISTVK0blBFEAZVo49+grLu9f+y6RVuZU78FIQRV6aqnrTHi+A10nszQoqMEiI4LA6TspL6mZbxA966a2Cz1HMvmWS7pmXU4mBf+KFD98zhIbgIKxwPJglQ52si8Vyn4pxbAohuaxOxM5CLN4nEcuRXdNoT8ygb0U/hHOurTyc6haoLSu4PKUaAA3SSSSDE44D8kBpY44r/mIsXRUrPJXr39sva+Vdtsl4VqiFD2XaSJIMHYrMK9uW/HON/jdyZMHcKDXtJUrv2tYhyLeojRaPyrioytqoC9E6CAzljYVJvNr+JFbqQk+ehTsCi0IbrV7gbPFsQWtKwVjONPkqknD9S2DGqX6NSIoDMkRyKrfoyt+EhXuOA5qKWvN67zpmHQdixRP/f3CHU0ruvZWlKXvFEkCFVS40+vZws5mmdnrq1btGCE+ent7ufDJpcDL01+3oJDH9hMpftj30psukvkhbcO2IDibIgMjxI1sSjGDAwdkorBXrv6PDX5Q3MHwACBIKRg5WUzo4geFYHX7k0Y6uQ2l4qewQusAJuiaGFHepQF7sqGnNCMwRRhK98AzkSOno+O64NcF+/r+TJtILOWKqeEEKdS4wq0Eshhqqa8RrGM0KuOYOFDGiaGZ5+Fei01avdAzMFWU/SLF2TtHRj4uQnQabSmMnJCWQitdXdL9H0LoLt1w0oi0oG1cwcZE/ZOoZ7wLZunNI3wPIYIzG2sjKijPkWHJQP9YH1dj50K8MqcmDb+zbZtQuI+Pb4wcN8+Py+IIEGCg67RV3NcE/pBVFCSc0pKJzAV+ETLvECeGtuJrhQLldfUshYAJyfzgiqsHHVBCbCf84Rrl+0Y0pRhWuGtO2df2prNAF/AFLq8cy/KhYetoWw9v86ljhua92ta2BnOHRNgxNq54IsIFbTNpH99HzgWigR8hOVAl66nhtN4pFmkuJ/WS0XQve8mXoyFcf1SUo1KPSiSc4FbCQgjG9FV+EzKqXO6kmD6kZL1cKpkl5CJR0FJIrOpuR4euwVIokvnlcrJgrRxd/f8qxbA+cuE5bjVUP+5HWA6wpRR407NDBY3vwXPZ2vBFzci8ZllJQj39e1xvKKirA7YAAEAASURBVG1KCyHIiBxLwQawEIbCqjDt57d3fedtCeKRSJEjiiJgEtCS+LfFdXgaVJBl6QbEE1/3nHSpPb36CoEcKjMqBBMLnKxZ872XYJy4lGKNOOi4eY1ajYSGGE1dJvNDVGFDYVSGI4Dw7hXD9Ib34bOKJIjJo5nah5k6g2wJyZPQ05zW9TZ8z3mMn4vQ/0pYCPE0FRuOaEEymYGF0HwDIoObhx6EOpsoYT1QLoowcOH310OXuIMOjpiU+JFf5SCQaxjxlPBkD7QrB9a/4yJvP1VmVJoGzKDp+lNTWpC+YkIxT5JXrnEHAs0FiiTzU4ZhhUn7hoM3VgI9WxUuhulLM7V1+c2bv/OqZ7S/ViySzKiDiJfkuCJVLuslF5uPdapraWutCMnBOSJyRhI6XPtGaQc8nNo9MCwIXJE/R46n9r9ko2REycSKS7q+cYo9HdGktAn2Ghw7Ji2Fmm5+TVoKa39WbHcCtSyxh01n29DQ30KXhUFp+YbhWiGGJS2EVwb71yPoeZNLKZ90qFqD6Gchb4pltl8laAhgvVQspqGD4IKJVpKYOXQiIlS4z/xepfeUpGCvhz0gQ1OYSIvc1Z+LfA5Dj+sJ5+reLIRbkAn1h5E/Z8m4xf5lWSJsUDBvMqiZ0mZXOb7SNLPnXFccCaHJIkNQqelgWJSyiqhT+G/IisHWuLJfyjCskAg9d2KflQwSwmNNwV2Ty4OJ6Qwj8TKnrlDIXcb2czlF0V/sPnyJp/HQqXKWUTAO6JADozChuR2ZXYJhDcUQQxhGQxi6tZeRW9gmFFvkPOpB3EbSPt+1IG2uEdbSkEGF1JHLyXe+n361iGyMuAfnwvBXVV4D+GL5WiplpCy9JIpShGu1ERAqw7DCsl6T9o1dyRTFK1PJXVM6RaYQyNomiHDdunUTutZx3hLZNeMhN9IwxQf1LYSI4gUeLDNzN5veKJTMCPGPlq7BFfNlKcTxRg5Q18miCdE+ZOm9UTmAOROphv3AFBLW7F5lUdlkcu0Vz0uOSGEsPil99vMr+gY2Ad3NZLg1jItkfogqrOjGOC5ShmEdllprrHj/cc8Hw1KOBAX6ee4zSxBzDL1NmKnFt7r/KkNRAHRsUFOlHFoIVfXBgvIKTpJwudBTZzd3fpOIIYy6DmFeRkMEt4Lj7WAJ2xycV4B0Zej43iJl1WuIfl5mdG12h2Des6VNWaNxxYoP4vfgagIOpnHS0D3YqntHqubGEPiOSOYXpqWurpdorlZiooEMOhaJxY5dU+GyXpIIdT87viaz6XI4BYax6lTcBQUoYZUgc8bq7BUOqMZXnswsK4Xae22X2EU+DgthORri7Ll/e8TzJ7e5IuZZCKA1Qh3bbYghpJpAv75r3fcLfPRqvTzVTzfqs8AH8KK7ht5+rqx4j23Tm35w1W8Q4Is8T4Ff2MVb83moUwF31d1EcIMSDCu0EL5x6+/XAxEo68VimOqV9eI+QyIElV0BEV4GZZUnTYelULwVW2QE8zJnF8rHEAIf3HsSescrHMDT/dFLPqGntWUVdiFbZ1ZYCPlQ9RqiITQw7xVMEU0dBxnTLGY0MKAJ5acXeKdlELTcuNUajo4QHfBaXdsZJvNrlKVQCYYVFsMcnRrc4Abj6zxk08QJUVEiROUTcwV9jJw+SBAkrFTqTRehg7iZxKkQ39+3i0ZBeAIbIHVRJSeKDmPog4sRWg5d87JaKtF+no/o6Hgm8jk80yE9rafs27vTWW7zuppuadjVEokU0SBqMj77omRM/GJm6+mBHIpmWStfK5WILvViZwFTOZlfaV0y6TS07JcSDCsshunpwwfTGdiZqN/D0p85sUq8x5nHohu3rkkl6qmDlKi0trbvvAlwrzFvN+h01i4aBeyhS4OCWBHDw6BhIcROXAym2qxNYpFe6Pn2yJn3qgvS0xoy1QFf6bJevuHYupZNyLJee8qMaTYtyBAdqOFep6UQLVYpffbzK/qGyfy8bNYyvXIyv4GBVQ3hHQ156IMoGu+RgawFd6TbsFjWy8TMxbLuH3x0VZ/hpGh4DgorGGuFyb678FayJwgXDPjNfD2OggLEAjk3E1cwhpATph5mCBSC5pC0L2l13t61/onr+AZe3aciBZW4PnxYxiV6/tgeUYeQD1KsYb6wtQUmaQWHZJFiJ6eFjOlBYGWIThBsu+L7yZsKl/2CxAg9VjmZX095zT44mrg/K8GwDh0tE2Fgo6wXUjTINRr32KvqXxChSHUL6AxLpE3pERJEr8ChoZtnobFAnxHrU4gMPJyVw+nWoKqEBZuJl0xy/AxVknmfQq/uqhC94MV58u5Aelzr212Hgrjg5wveVe8fwVdhLYVSw0veXt2x74p8/nwZV48Ipr527U8N4rbrapf9gg0/mBTJ/I6W12y9cdtwhgURRdeE1YHn5ImtwqVBQYYFToF0sdw400NrEruuyomiBDHERQQ21fHVEsx4+CJakR698wGskhP5+UoOIpK/zEtpmkk4Sa56gx0+6NUdxUP6yh7Wlwqvr/P8QjkaQkUWHvhUD2ATu75z9fcIaRNBOYIxzcYDPUyllG5o7Sj7NfsKRb5BqhlsEBAFsWYNaSmMeHOuYKANZ1hhWa8zd/o2wmjf7UCU4PxVAHtdL8EkwUKY4IZ+bdemd00TYX9/TsBhmqmLTBMist1iq4wSOHZG9QYIWz1xojxQsHJ4aTOyJCkshKdiLOtVdIb3Wgkj7VHkjFqijWjiDOQUtIyO16AuwCqff9pA6ZxesQ6hJjpnGCaoP1LyiWhEGsp+kQDt7rt3j2KtsslCtvJ9ff42nGH1atITuliY2Oz6EyiGCZagHLuSkwGPX5ipBRFypVB3pYVOdLb9zbQcXkkyf7dgLdFMIFFBRiVCchTFC2YMEHqGa6e1tpSUsJ7WctEgYEYvJ8rvS/bIzlSGoSzSXWDGJYq8ZdAzMq7qsqwXsowumLECm56YWUtb83UG0mO+o5XSI8AKaV2W/fJXQKO6NYIua+qi4QwrjEsquTcOZNstzYPpB5uOgkuTRJggZELhTiIExoMjR/JiO1y//t9PoujXZYbolHfNmiZk5k3smIhgWS+1YwhhIYTFAU6cE6a+RSiZ7+V9mjmiJb7P9XOjQHID7VEIvMD+EvuL4XbMFwEzHaQWRlkvYU2uNAAcdsU3SiUNZb/ml8hiALmiLnkMBMq9jg6kB3eGRDK/cO1W1EFEFzWcYYXFMG2vsCvQYSFE0jMgRjWGJYiQlVkyyXX3ESGZE+AVzn+6oX89kWBeo+hkeiJC+bJeXKHQ7yXMzsF9699xi7TZO533KRpKJU3ky2o8x5vYxTLqSsYQcrgwGrt2Av56UsLSFpE2c7mcYMSJxP/0BjyyIKXzlBG9P18EM4GU5QBV10UQNMp+Cbgj6LfiLhrKsEiEoZka1geU9eKWGd1irxgLi11ItwXQkGubqHqkn5eX52bclS+/zyBNCCWsCJ1ewbGUL+sFFp1kWa9AP4uFxrxPgPoIJzOydrScg+nkld9ZDQrZKXSdKqakxeQnEE+p6dbV9Su/9SIRkMvBIrFgk1L6ypXvHUaEx6C6Zb8gZGGN+v6EKKxa3qy5p9atNZRhlWV6IiHt+MWtrmNj4HqDYZoT94HFXU9LXN2y6ulLvGImEfb394ubEPQLHQTcMiLUQZAaGJJDx9G6UoYYUaV/CBz80xJr4dohLIQL6mwq7XXmdQf6ZA6mscLkI55WXO8x2EXBiuA8ELLqdUJPD27sfKqiFNHlhS/sg4aWKSfzU2/jBl3DGdalWmR7EBxD8LkWhKX5Zs5VnO8byhzCkkGv3/yLDUFgw0IoVqVy61JkIQARggwHt6x4y105IXJX5PuhIZkmJJFYccVxrFGkCcEYoiE4YETor/iqaoOFEGsOFlRdF6XYh7t7Iqerrl5ZUck17u5LJs2U0HUqRyliU4H7C6RsIy1w0ftA0r755jAs+wWm8EbUnjHzPbPa7zHJus1g86C0bWzoJZHMLyzNV21ftV4fOWFVA0gYyFpymeq22E43DxE9W00ndbmWZb0wW0ZW1pYTRMi9VLZTp6RT4IoV33UFs3kNCwqLd+kFBbgew5AcVVPKSMbsmcUpS8umtojj8p7xd07jJsTRUl/PDMgYwpJX6E6koL9C2AEWt3Isi/5orMlomSmBi2/vrqwm4/j4GYkzY8VXpZQeUC8aOR6XMg84CbHsFytZZ2y9KPRYveWNZCn9VnNvQxlWWNZr0r6CYpikPTUDWSURJpG0z7pA5HY/QIShpRAFBbD/ZM6SuWExLbkRI7QQql7WizoXU7OG015WJDXMLaqzqR41z/QMYDtjs6HrxItyrAogyWk3bSgdE0GH2Nz2VFiTMZeTUrrl61dKJcNR0VIosW54WSbz8yZ2czYGyhsJ39ejNZRhXQjLemnBHs8vggYbCs6c+CYRYqOzStBNJfXV99WWC28QhCr1VrjUe52FB5baxJrEoqTCnf/wDFUbwlAweKPjIhxqhYUwaoU7d3aMXyiuPd/eTguhio1BG9C5G14p5ZjJjkuEsXL3Dimlu8b/fBF+pJelP99iyvrGYAHzgZUaiHqQF8rB6PWCZOkrq0ZIKc6HFkLblaluVdw1uZtjsaC2XMLOJMzLHO5cRBg6/+lm5ylbpgmJxPmPMYSUshTmV6gilAaOzAvgKjZCNiJXuIf50mghRBa5XS79PDAnNZJenLf5VAdguq5uS7/9Ih90r6zXYo89wj1KW7fuEFJuJ67EmXJ7MUgW+p0bJ4POA10m88Marmux44YxrHICMDDrW+2+74pUt6B6BYkQZuoU0WRc2ph55yVO5lxEmMvxF6hLTaQJQbIzDGXJY2EHwkKI1yV3RuDiaBQr8H/SWiX80w4ciR7UMIZwzLv7iK85az1ycOr5lWtI0QDpOmG2X+rq2jdO8CoNAAe5SMEF9+BA+XVly36BFHnkxcaxc3QUG4ho+brNRcMYVpi079VbnwIRFrZIM3X0xC4RuoS/ICN4cSMkJ3t13bqDE+xpbiLMiXOK72++jMzmN5gmBHMrdk2+q7aRAngzPdzVUr3eG4lYZPA5QxVCHBEM4eFeqVf3vV4Wf9fVLwPMC4XbB1JpM+lzTyeXVK0BLLo0YLcSzDu/SEjOg+BDShfrERWZznueMNxQjqyZhh7sP4rPPJ3LEB1nneuOCUthPct+NYxhdfV/TBBcYE/tMUxP6UBWKtF9FLvkhM9PhPgFbe3aXxrEYgLDEifCJRGbsBDy9KPe0uRQuZJQhDAwC5OB15naKkKWhsrKY3FBRH/CPn3D324luC8sHFAc0WOr6gZzRFnT8MFokJFW4OLpMgOqtKNQSkcl8a8VsVOR/6nFrjgSmGcRPpfJIE7Nu3mQ39QzRKdhDIsDZRstDe5mICt8BihLqLc0USAVyUK0pJG9QHjnI0JBsNM7avYURXpIIFxdVTdyOSJClPViMDjeL4nzVQ1BxTeIsl6mnhm29FWXxV0RV/Vin6eO9onh+15R5GICPpRDB2U+wAXHSnil+YaQsCrG4vSF0lIYBNmrrmuNq1r2CyOFrg65JTS/m6AjRKdu89EwhnXiRL9YzDDfHoTlh341dRv0NH0s/gbUp5k28lyZZvu9sl7z3Dcw0EOxCvMZiDQhVErU2ihVUVXATKOq+mAJh1qEoRh6+uzuDd85xLFGXdYLKNSZe4l9wwtoKy2E/I6f1WoQNZkvzU2OdCa2XyJss8t6LQaxLPvV1fVfLmKMV6nAx6hrJ6LFHlf772XSnnpTuYuaNuZaHt8QhoXRThNhyb4rUt2qGMhKziNqy/mpsZS15hoRvBAR9vR0COLS9ZWvlhAoDWKTDKyGmeGKDC2ENdxel1sEN0fSPstIiwWan5Ywo3x8XjCnfzr3wXVB4Oy2YdCAGqUhdLvwqFiTESEOun79sa3vu8xrHyzrtfD9oDapeMeL7hh65g2qItCUY1jcMJjMD9XZt2EtwyoMrsoDcR1agyb+qBjc2NhX1sAps1uW9VKTCMUu5wdXv2HzT17kfCxMhFKkB91dKBQwq9KxrHqCK99RRAy12jGExAglrPbYynqFFkIYZVACzhcWQmGW5KNVauDeVAOgIjhSRCNPF6cfhFADiGKTA+2cZOYPtFr6qOGx1dwiy35hA9lz586vbZB3yjVdTS+1XNsQhhUS4YXhr673/IkNnksW7TcElkWQBgmLpdc73hBEKDbBhYhQOv+lUgcvwcEBITrky9XrsUCs0yE5grMvAmQjfuZi9JG6SfMycGnInCcMcZT1Gi5XZyn4sBBmjAROhDx+qIgWWJNRNUnzhcL92YEekXKo+rnJy1uMjrOOzSWBNK7KNVn2CyralaDUzRI8GZweN6gNQUZ3mQhdbepAIq0nEBQGIuQyVayJXdMk8zhJyI7TarNgOyJ2w87OH78NVfn1paQJ4apUOcsojgA4Cmoo6+UXssmtoihHHGW99vTIGDtsHLAQgkoYJ6VggwrWZLKRpLWyHJJTazzlSUFDvm+eoj8fmlBkKTZkqC81r70dyevcmyLVjKbVp+xXQxjWgPacwH/JGdphJR1qijAz6km+LOvlolRTOrla7JpwaliQbiB1sIy3wKmvZV6txfmPWCDnFhZCHAn5Xj3MEA2YNcgQkD6HNmU2wJUj+rJe7POEJo0zvl94AotEIoc/qNWowDHsIkNWNBFPOVc0RCUg95WtrJ2J9cz8cVdRSyGGgvk3OCHGLo6rv/+5upBpQxjWYA8LpdKj13sSXu4Yu3rCFcATOmWHXuueLiSI+WvLcTSiBU8/LZ3/8OmCTNdUvTKS6FDeQoiIoWQKHEszTq9c+S3DHP0ReqlF2KjIzZdjCMGshIUwwu6j64pJ+8Tx37q2NvP4G+y41oyroZU1veqXr+HULTJ/gBTVlCoxKUEw8QTHm8vJNc33cba6M6yZRGh7o9t8X/CuOMdYY9+ytpzmJ293ZndclZ1IHdVCHWLiRDPNlSj7JeiMOK5qIZN9i7JeVd0ln1uvv+Tmhs40wB1igcqyXtWNczFYw3xpXzj7011Q8O5y4eMEkVO93Q1ELTzcA+Pa3o3vqShp3/xjl8ODtO7BgRSB9OoNtww7cvhj7er6liD4VAqbLLCQj52fxP6AByemr69PPPPG+D+ugzpil40MhrSnPHhd4z/L2nKQj649uvF9wqWhkiwE5eSj8NvyLyFNiM00IWhVsR5eTP0Vj0AKIkZMDRgW3MM4lbKsV1cMZb0gpshnGRZCQILVyMWEFwVpBfMkHIUNS0ZDQC1Q7ZzLkXIpcOGXdaWBf4ZhYVXud2FXcb8imR8p1dkxMvJPdQvRqTvD0nrlIX1o5AYthGtY1gtNyXUpasvpHa+D+CgqAcbFwcyVCwqsWvW2N3A9RHoWpSDrqazxCbxaxBAu/rjKOo38KjDgwENlmKTWnll9QXafi/wpYQyh7QztT2dNixZCJV0aIPXRyI0XoeucoRaoCSfTmT/0ladQRYdNRa4lYgqxIbd53uROAlmPEJ26M6yQCEve4KMgQlZ18EH+Ci5NbHMwU4MURVCvrC1XyZEnL5iTrn/POHPAV1P2izcSEaKsV1nhTkJQrQFOOEmaGtK8TCS09acJX61K5oXGFsYQYia2cyrQVHRLIzpk0j4jI/zRFhpTJb/lcvIq3QzOMPsopC4F1wdh0mkpRDK/UZF9tJKxLfWaujOssKxXwZ3YYRgljBplvSDnL3UgEd8viRC7W8pYJxTulWYhoEh//Lgs+6UZ1qusoALdMXlRRY2IoIWQKZ84ORXfWFHv0VwEGOFzhG3faL92YEOvDMmJuKwXIQ1jCFGWXtkYQu5oJhxGUZNxXDfNi4Q7ZLR8X1vLUaLX0ulHL+PIfb3sz6cgKfAsAGkj0ESdwnqU/ao3w9IP90o/miAovolJuTnhtU1qvHdR8eSAYZmaJyrBaFqu4geGDpSGkTwHfxrcB3Gp0oYrhYUQJKugtqY8igB5vekUpbGsV4E0K/JYVDrGCq4DaUyHb6H/LT70V/yuglvrfAlwAWkTsbBXv7X7By/y4b2ajAmsHZC8WBPt7T92EyNWvuxXoBX3cayg19jz7NeVYQlVjlAqwsnOn9zieSzBoeCyhM6KTopgOFfXtD2GXQ7sKicZLd8v1sKCAoaR/XrZUlixDoIrUvmkfQIBSNpnrJLHZaCKkuVieKnu97xgTi98/UcfCTR3t80sCFLLX103cV8NsChh6YaJkJynHDJVtCXhgrhEP2Jt6lrbSYQoslMhdcU9nGr6J4wlJAaAlNU9PPy78Hpnk/Mm30f/t64MK0x1e+7uxzcgrB2BrMKloc4wLI5EkYVA1OoyB3dvOCyOPNVIEGFBAYTDXikWjQKOT1h8lRExKT20EC4OaWOuIOtgWS9MnEij0j3wTORzGIZv4TGPYLGuooRFJDZmxAs8FcyF4VsI1hCqg+dqDsm5/xlh5g84lJ/X4aFbhd3m/o5i/RQYDiyFKNG3ySsMbpCPijdEJ3JCWxg/BwTBTRUKm11/cqV0wVJOf4UhUC9BHU2bWJCytlw1EoT011qz5vsuYWVXXPaLyMGupUHPqqLcGU4tJsxH0j7NzyY2iOPyeIWVYcIOKnkdHrggaNPxR5Bl1EiCX1HCUI5hYboQkoPSXkZSpB/aMy4zdlQyxoWuCTN/mNoaFOflNlZ75o+FnrOk3zAbNJqlUmZKM4fFsVDTZP3IJfW7wM11ZVgDAy+I5xXd6wez7ZCjoV/GPwWJUBAgATtP3FVaW47XyiZ1ELr+lgLU9iJNCBnRYk0gBNdR8CSiKrhlsS5j+J3MnCZ8c9gLkhf4gFwVx+VKAdoTpurRElstGC6AG+WORBiLcEYrTfmuoaXE8XjpCvcQQzLzh6/7FwvQEmLjU48khE5R9zNpyIGat4uQx132SzCQEEVxvz4/LuONSo6zE4VTldw0iAMwCrOEc1lCXyuq91ZaWy7EH3UQ9Pwu9/VauQJK+POcr2ROuE9YBx3IMGReKjYyDoahmEbmXM+2Z2Ip68Vx57ScYFB+UJIxhCoio4wLXU9e2rD6TYJ5z1WgpDbQpZSeTPZcBDleqTXzR23Pru4u+ppogS1CdHqm60dW10elV9eVYeXL8UZ+MPZEoKJHjcQadrPAcG3LQz2Bi/yqFh+j0PNb17OvO45gP8T1okITk/bRrYHMS8VGjUUCFTbAUlnWy4MsWbNX93zjgzSKrvOCYZl6ehMSxc13aWO/BzISyAqaMNv+9eC6D0xgdvXo4imPCFpZufK9w1DoX1tK5o84kUQ6lXUi3R18Dj7DUhifmqduDCufz/McgbEECc93NruqWghFvmpx5LnctfKtlzgJteyauRzvpKUxcUqURaogrxF5lEjah1e+V7PBqQz0mLA6hc4GDjgxgJoXfQ4HL69MWCt22A6syWBhiuGDByLDw2HQ1Ns/SdiO9+dMQLnoplTJODBc6tnl+gxSNWX+qOQ5EVwjyn4FgbtrcPBXu9hfGAMaQd+zuqgbwzpwRFoPTg3+xQZfK3YzkBUzqxoRQgaSZb0MI3Vt5+q3jxJjc5f1moXLB76QOohE4pFLrmMN04EUw12QmPmjCMlZ8KoHHlPHjxgBVXGoDINUwJomFO6VOtRWB+YRcbk5nkbsbxYmMgUlLGZoSGmGW0pdXpF4+tMEOFcOy6purPNfjRAdsT4hYZ0HVeLC+CSX+aFY+BdsXiJEB+hYk077G3l1bzkGdOE7a/u1bgwrHIOrlbYFmt3OYpjYjNRjWMCj8KvRLaG/qv3II50HV6z4jqtgzSj7RVRzvc/diAieksmwlJMlyiBjxgJY8I3ClGdnrc3lkBzJmOceVc3fCjx1du6/4/nFK8yVvgDqan7Ikm6EH0M6k0S2ilX/75t2/IcRRjeEx9gl9Tvj5lwu/NApyn7hU8X+fOGd8b+y7Jfmt7WZZqk0eJDPizOmsG4MKxzEZOny/kwb5Wb1asuJyWX5nsBiYQVhIZyvrNdihFAW6fHylGMYmdPlZH5zMix+SYY1M2nfYv035HdghszcNFK3VnSsFxksas37tBD8xF0+L49Dum5eFzGdYFkL3VPP30C7vmH6iamx1PiKxL6P8dlDQ70xwCc3A9OzLtu2VRSJGxaR0uuJh/KzxF5rmYj41K2d/C7Osl91Y1gncv1Crg+8YA8kLKB94eNRAxDPR5LoTBtijml0iFQhS4MjJy2FvndOKk3nX3SUqhg/yLAcvo+B+pc2FN5NJTMkRfx3Zufqw+K4XEnKnVoe/PQReRwyjew/g0WKh9fSTxz3YLv12trTkK46f/9NO95/EYHx5uHDfZCNo21Hj0pL4er1vW+AJlD2a2EpPdqnV96b2Jxxua9NCkuheFv57VVdWReGhYPQdObIkj/8KLWJHGRVkNblYpgHsVP4bmoqrXde4SNzuZ9cApw5CbXR+apd4tuFCwow3x8zNXDLUrVBnwJDQlqkAZZJ++KZx5wm8Z4I0p+bmoC/HvIpAidLmItoMArrqJPJGpCurJPbE9/3IfZ6KuJMqyGkR47kxXghpU/Bn+9yNZk/wj7q88qiFODXvrE9CF5MgLliicej7qkLwzp6NC/WYBDcasfRf6tj26A8Nct6iUBW7GY927/vopzsXiEZ1jbxJ8MFdm6xmEIiiFlGqeVSmWEROrB1IX0Od/fESD+HBd6/eXf+JUhYX85mEdxJ/8RGNmRSweATpSLiKFNrf2LHjkPF40HeypddMKIGTS58mfkDJT8QU8gQoHg2iKXADpgQU0jCLe28fv0T5WR+R2Mh4xgJ7h4KDhyQFsKXrv7dej+Y3I5UHILu712hzDvhxY2QnEu6vgOerQATVFN7kyJ9e/vuNxzXvEXnP/Q3JwMkZxMWwtofFved2DR9q1SALd/ICgvhnp5aK8MsDioX6/HjOUpVwFnyj2Cc5P+cjXATWLyTSK/QRSaCbFsCB9TsTx/a+XtfInyH9DyjNWJsedE3rNZnXY/LtYrMHzFCdX/XlLBENo2ViYSznb/19/fHwlti6fT+wWhaV28YXzS5C4WBMog/ks48D17Y6M/Q0Qiv9EA/SVDKea1qXiBHj8oBZbM/dxOM6sZ8zn9kiTwKqmwhBKtA0j5D871gzDJXiDCUHNUWMbZc7riQqN6+7//568Kk/nIGCR/BvmJmEHMOCLMT+CtWJWEhtX7n7fv//L/jsGZAuRy7xIeFLwDS9QyqiQt0q6XQk+gCeetuZyd2FH9ExBTmcshfGUOrC8PqGPikAH6qdGNPKgukB4aaqhrdRwlui4UVhASh5fJLQjl1EFJdR6lKOv/xyDez8SORM9NC+MAlMy9v3Hsyc8b06amLG3e89aYE5EisoGIRCCmLrwk988vMTwYNI+vBx8oo70OyfJbf3pmwJkeNZ9+x/xO/KH/PU9yLdfx8zr3MH51XbDsxXk3mj/vGEfOHkK59XS8HQQ/EMkd1YVgXLkwDf8D3cR5c0jErHsyDaZD4kKccoq2WEhIW5NolPUwSdK/AMdQfCGWRZ5oHOyU6aCFkTn9UdlC1+Qn4QyHy4vwO/VAxn48+JGeugR861O9Suf/tB/70s76f+O0Vq1IM/ohdshGwIPQIQmTQuSppTo2bf/zv9//VT/B7VoeJS281GwdSrdDV9Y1XIOQh+yhyFbHotpKNS6ggso/ijThCRw1m7AyL1oLDh6WytOTe3UuRQ8UGIwAshJBo3dToys4NwkI4NBSFU+SQYEGBvuoVivQY/yyRnhcwQwPFTnUbdSf0wWoXCvcDR3rrxlpPncoLzLxj/199cGJE+0wHNCX4wo4XVzoiQD2zvT1lFkaTv/uOA3/1DJ937Fgvwm9knGO8zw97l2PX9e8s6Wb2LP35FF1CusNd19d3XL/+bFZu1vnIaSR2hhUm7bt798UVgH67YwsJK/KBhNNb+yvKenH30vxrj617+ir7OXVK7m6193nvTsPwLhaLMPwyN+ccjVlGSYhz/jjH9fX8isRHqy5jkD3fFfqrsJhIPeBAHKp/LOglo9ey7o53T47pL3WuSCTxkUwrajaPrSNw4dycsMx2zSus++Db9//FL/DZlKzi8Ldi3/M1gXv4efF3GD1Qp7AOS3Y+YBb+HgwLegPN2ZLJvLZOXhp9Mr/YRx9mjrw89dIjKLKy0WNZL1D/wmNvzK8kBsvoRJ7yg2L3PnLkSASLISfE97Vrv/E8Dnxw/qMe6J6ymogIQ3LUxAoXSjkkZ8J3E+aq1zg70eV9qmyuD+t9Hq1y3/L4bw2vTb3pbYWx1InOlckklghXyVIV8Zxn9BG4yZRutnWkLLfY8W8rUo+99dDeP/ht0msezKq+ktU9vIRlvyy9E4H0Yulw3UZAm/eeEcE7YSmEY3HWtmUVnTC6JYK+p7uInWGFO7HjD+9LpowkUt0C0fE4lU2PqqY32Mag0Qx0T0gQ0TlF5gVh6fr7RhBmgjQhAuX3EZtgWGBrSnJx4rIckgP47mQ9SxyXNVlesiZM13oT9VlkWk/t/JXR1ft/9u2lydV/1JZtM9Jt8FXShV4LDqaomLzIYpa/w8IimJRgVDqcQa32zrTl2ZlzfmHtT3373j/55m/c/itfziNGkByxfjqr2djJ5eR3umW+TrUCdFixr9vZUCz6DdCpe9ksXjRv56JX13hB7AMfyvWLxVko3d6ZSOE4iIwc+MNRKdMADE9jyIGlo6xXl5AgwnxWSwVypkiP+sBfZ2gLJRb2yz9EBJXtNpYPJ0P8gFelGlYsXRp0PfH6N+3/zTuErbd3qZVhahuhUMLjePgUYjS/bc8fvj9p7H5n4Kx7KZPJmNl2FIw3A8Qq8D+hN3XwhkQ3/YpfRP0dHNGNZEYHk0pYlpXUnGLmK/5U1zNrVr77iUN7P/pRWCaFRJc/pLm0UtYGbTR39fVJXWoQbLjk+8lblWT+iObJ1fbCZYQWOI/zJY6yX8Ixj53H1U4dDddg8BjO4PdWaVwPrKFfYhmMw3BwEEymNSFh1dDNArfk0H0/HmKg7BdZlFRg8wYwNM0BCxd1CPmTos0wRKGFCwSvLH1y42lI4/GQ+iREUGhv2ZZ/HoaMT//zhV///oJ9472BN/HN6UywyrSQWAKpqsSOwHVERKOxjgCrNJcKZtErmecC2/x80lr7N0/v+Ui/ZEx/oB0PclZOO+7h81KPmpHgR24OurZ69S9dv3nz3Tfh8b4OOdYaykTnGhis4JgKbAnIjcXfqfrAR6Ax5AFz3VXdd7EyLAAPYKXPjO2O7bRE1Qn1ViVwiqBeHFO91M215q7LRGE0FkI5Gf398hWlMb9aKk3ygwksTC8jFp3A/4qF+IYw33s1jeRL/CRDcgYaxrAIQ6hP4hGxzFg+ga8/8cqVP9w0MnXycd/xdxa8oQNYPrBYmYHvuTrSOk+lrDWn4TV+ecOKnacf2/AM6yqWx/Ff4Sics04gp5X0XleHTgEj+QAzu/q3bv2vJ62E8xjf40uhjCc+1GgBQnR4ZNV3jY7+/OoVK373LqpNAZH5yJhrrAyrbCEMzg4e67o0+tmddgkbFshGillqoJhQ0AgmSq/7+o09m3/wuqb9UPnIEw3Rhs5/UJFdLpW0KWalg/GB4qY4a4QWQiENqIOWEBIQW2CWCijMoftn+WVYICK8oJGvPCJyY+zXDpmH9H7vyS3vZ9obkfpmcbh+XGOmha7+vJ7LHSlLVP2L39aAK2TZL/gz+t55ptsJAnjRKtbIVOnagNf1sBg+AvDAsKK1FMbKsEILYcEdXu/5k+tkam71FO6w2/lgIkhD0HEKu4PIxYRXCj0RNbpH9Glr1nzzxVs3n0fZL2N3sQhsYIIpZikdkgPpEzmwDCRcHEvo7SJpX04UiOiPCDdL74Zzhl7AuDS9r6/X7OqSvm/Tlsw+WAjKaTBDIxB/O4X0LYf1PCSsPG7nP3Vbj4jbHIDiYjWS+V0FoJSuotlQIxy1SOaXThupQvHWo+j3tYGBVVTNRiaNx8qwusvATro3DqQyZmJySihvOAClGuVt00zgWOafJ2AbvqvH1PLT3vkRwCpFYlgKi7duHIbzn/b/t3ctwHFd5fme+9inVi9bluTYsWVLduxAEqLwKBC8TlKYSR/DlK5oM9BCmCYlbaADocx0OmjVTttpIRAIBUISkiYpBYlJQ2GghQySeTUhUbETbMd2/I5tET+l1Ur7uI9+/zl7beNIiXb37u6Rfc6Mdle7d+/9z7f3fvc//7OP7Hl0ulGwKAWN0mu66uQbohU7ZDygRbomhHzBqfhBzlfYSuarSzWXW3Ou94KUKMh9icofuu7uF30Kz/poZGMt9ClE9Yac0UOz7+/fHehpXVPyyPSLtl5FO7tGN3nRvgC1luBOBgozKqI8RsTsKLX1CrYKAV1IxIkkMQhqJ4JHufD0SMb2ApTOmv4Q/GiVPjCXPIRYde3Z3JMupeRIyq2VTnFRfE8EMYdCG/fDgHEEpACpg1wFBAcCrRo8bfYNYo9jgV7zNb1OtghbMoSn3nKYBa5cCQfoAi23C6hE4Dn7Sb5K2nq99rySfPK6bj5HEcEYFPPuUYVRmdt64dQTcjMmuuTUMSXntTG9lLYY5CdNc/PtJ2DCODZf5Y9GI0KXOLX9wjL9cpIF/5NzILALv2aERUKmSxHdBTu7Cl4avuxpNKCvOD5sVSaizxFpdGRZ7LcO0Oe1qFPuHxe/5QszMzbcp6RUebBfnRf27m8kzzNdJLpt44mhsCaGbwOSR8RLQxIQgJZOC0Xc1SLPl3oEyDh5tP2i0Aanb2Liz3mKTpBtv2pGWL6QL048sgwBsL3ce0CoyzbArHS30pl1rK/75uNCPHE3C1bUJFeNbTv6IjhA1OcGWZL9ihZY8gHDZ4+EcKbn0CXHsMOcsM4asoMFR+3ttRHwNm1K8usVD/s8HuwenOby2odf2BbQU3iKDk7qdgTnLqdvBdn2q2aEBTWFj9O5M5c53mybIy5MKa9LeAhRRTPK7VcgD9Bq8GIiboir9CtXPngKAaS7qaEAQv88hK1IulLmPx/IHAsQzXppWcdVh+idWmqf/IjqYV4EkknxETOaeeUP/EfXLz+vxCcyPIq2X01NuKrcDC81E2ROYc0Iy1865ItHN0ZjpoEqo6jtJeEdATYapJzgl9f308+dRvdePNXiJIAulyx5ZUPPUE4hzHpoIiTDSTaPDJDPArGG0OX56q5PZGlJ4gdszvMN9XYNEfADkGEjOpzP6wVE8tPNVbYziO72CBPCA9PXEhx+HGIQ0NSMsI4fH+NAup69hunwEMKjgQs0eNWlOhSIQfUC1BxEQG+lXW2qbn+v+m3/hGOsaWx2hsKwPCTW1oQcX1WOBX+IG4yLOuKm1vxt+o7femvB31cbBopAElH4tMNly67fCy39pXCYbnq4qiQbPom6bv4aIdoIlzsIMWtGWFi3CpvN2V5l0t0JaC2Gfh+4KItWETerQwLQZBC4zrkP/4Rj7LKf5QtsXzRCGToCpzm/0NA3mWOh1Eph1jp0+bLXccLawgNGGyrUJX7wNCcnxgamdc06TOWQYL2QjrBwD+Z2LCgpl3vP3ldq+xWMslITwgLpg2Q5+ZsoiXy5LdrkyKZd0cnPlzz41Q+v6RnYR29sSdaOQGg5hfuhsWxZelpjoccjYUNDmXT5bpHAAWK5kUgIAbWJB3vaPngmjTy7RpZYod/mUh90TY2i3A3hgDTCX51f+UMmbHBCl3IK7d5TPc9Qig5GOpDrvyaE5VcZffrIvWjrVegtws2JUaNj0a4rHGBWXrSPRY92sCsytJfBABM155ZKeCOKWuSxzJRto90eloWSBQAiIRiB/1Y2Y57oaLkKLbaAS2k5Mvec1Lv1QiCZTPNDwav9oiNt2y/EBXDjLGtxnMJqEjiotl81IpGNnE1DmrMShNXqUBxPKQCRhJdnoHUVL6hn8jrlVKyt1io246VRNH1F59e2wTr0eGebSUxOPlR5BjwkpF1FjPZ/ubrrwy+LigjpwOwQ8kx08Unit/1ytPDzeR4TI2WRD7r+7UQC5e/dfKBtv2pCWOPjT/L9ZmYPb4g18TwUqisUiEoY6CnmUSF1qBJGE18Odiduq4uMY2NJjo9pRj/rzDpaSPcsCnEIdG4V7gx2h2IUhfDy2dj/ber7wudoN1u2JBVZVYhn0F/zPW6GEUbbL2MawQM4Z+WzD/t2DthA1gsM1gRyDtWEsL6bETmE0Kr6RLntmhymqnMBvzL5LI0C7lLMjXANa11md11IQ5RE0fSWpY8+DSvRIyuWmCg1wOy6sOWroUa1oZhr5WdMLx5e9WEYInlJYmoC8WpfU5/VEwGRU7h06U0HYchC2y8yvMvc9su5UqAzEkiUQE2YZDApljhFN4MSE3XhgLLPGLgF4CHEWrsYmm1uaj5MO6hvFLdok9URafmUmbPPRMKMWlc1cmkIi56roa65Fja6PvHWNelfkIGXyLVscNUXaohAml9QvO0Xi7xIQc++NlPDg1ay61IXHXcVqsNGhKmlesN74ISFuzFP6sXZH8IfPIT8fG+48nAh4qjh6FJTSsj4Ulv3ew/S5zu04Np6XXi8C/8nW9azz/ZbrO3Bg60R85PdCZ2SoMGjDWF4XASe3dIaNvLZpq9uXv/5uyEvitrxxg4Xiq7+byACdH6AoBCWSZU/nBdKTU0aKNG8hxZ9CjVv1bFjh0uewuqL+QVOWBsHhVDPHbl3mevleiiigbJd5p1Wgz6g2xTy5DTLTOxfyVbOkhiD2iC/e9VLpP7+cc7mnZ0jX7Xy7uNL2y0Ttiw08KzroDnbzW1hKz/d/sRN6x+4nY6OuyJWhPLZRuqKjKQHgw2UnyIIQN5Z4A3puAe+rufuAqBBLJanIbUrruvZXto+iBSdwAmrlEIIbWG6B/eAhAO1ATeFOl+DC4ATtylKesavzJN6g2vrtYBjlzahuyV1EqZ/r0gs/6Ax42yPxk302qt1V2NfRrJZeS51Ui5klnzzhnVfeg99Qk1LVQqOj5F8z8mkkElH5Q9RQ13CkCHhZHPicfQBYR4nrCCQDJywfBadmp1YH2sinuJF/qUiLK5WI9/ZsQ247lt3E5BBtfUq90ehTsKUY8iW3Du1pjX+B2beOYXqrCHsR9w7y93hArbHjwFOZEXDdNGKPWbkMy1fumH9l/4IGhXvsExdaRawG7VJgxDw234Z4eUHXNdA2y8SREptmGt9nlfkhnc/06Ma2AInLD+HUGf6eo9syJK4688HiYyUuGj1Ii+VoG8//7NGvGZszCZ7VkfHI7tXxiM3GgX3eDRucE1LkEtgUtEJBEXdc+MJ3dKcWNbNLb/1pivu/ws6AmlWiqwCw7pmO0qlRG5ee/vgUfyWE5ZFSjqd1XINMimQwQ1Uuo4kK2V6VKW8BE1YbKCUQ5h3z2wkYZGkKR2QdDdC4xFctuFT7dHeQwTm8eN3NFTO664bLxJp9az4+ta+ROv1Rt7Y1YJW7OQLBulTyEMV8iGSnuJL4AWMxnUzFo8axdm27y9pvfa65LrPPITPeCt2RVZ0Jsg/SoZ33vaLsch2XgII2rFskuP616mYH0Jl1p46lWoR8lXnKQyUsOAhpOJJ3oS3LY6ggdXFIrcpB3qMYH4U0VgBUB67ousDPKQhlUo1/Acn0roPpHXZZV/bdf3St71Jn2GPNsUtFolj8QbCoqBO4EvLNSIv+ptr8M/AQSWS8mz0vNFjCcOMRCOsOBt9RiuuGLhp/f03X7f8rhfoePQFlSc4F5TyvifafuEk8Ny9osntfKdD4+ZAXptS26/lth0vVR+tzlMIPSO4sXGjEObgwf9ahqJ9q3iTcE5hwR0jkD1Bj0brKiyOmnYBU4c0DAwpfvHbQVo8FWbJR6Yw1z/55f73PX4mn/+bWNR8o2GhUA8CXYsoBA9hSfmiFtLnyU0vefsnFgppzAoZOvWwy07pBScXe9LSWx66YcPd3/IxLC0Bi9wt6L+pnhcFAv39Cf67W3oCzXlPkszceSOT8NCwKKfQQ/hQzClmKEVnTyolWrBVKmeghNXRcSWunxGsPPJ9uqHFiV3RvhrvSTeQ9EzR5S43uN833m/ero0jAEOOQcGa5D0c2DHivaHnsScg1RNP7b3l5uxMMYUGrJuga/VEooZuotQDBYzwlTehDK4izIsw17u2ebTohJ93WfQHrfGe/37Lqru4N5RmOOqlzaTGG4cq4zoBsihHBycslzn7Z1BbDecB9bik92S63iALs6NRw5wssjUE8/h4pir5AiWs3YnvcGFmiqd6IzEYTaYYrQkDPQZNuuqho0sO6ClidXCD+7oMtfVCk0qJBnkPSRzStojA3rL269/Dv9/bs+fO5sPe9AZWYL2zucwGx9ORrSnshNFQZzZiLN0esjKHY7H1u67tvrVUo15MjPaVTI6WOhynxZvqcZEiIJrzUtuvnP1z9AhwV+TzPEVHQk0LEHv21QT0vn2UU1j5tRYombTtO9t89EpucJeK7M+elxCN+hAiaFTPv0jvJpOiSeXZLSR6Uco7ZAgWNI4f7/D6+u6lpeLTpb9XlxRL3VEN39M6vJQ2ghLVlGZT1Q3u1Y+nPq0jAoO4yaa1lpaPnZo49odHEPEOwpKv4LZYAXBlsIfAEWE8ZII535SxcNgCIyyooyQEN1wXncw63oN94XLUb0t4UyyTGZ5jTVzWdt0BcWBxt6qfEOUdSfy4IqePUp82bRrjjgye+zhC+8LDb7RiT4KEYd9CwcDNmsoFLA/txbE12VxxzVEanKuzENp+5d8so+TECwVuc/V6T5/+QGtb28NnRDG/NGexcmUOjLBKQrgve8NNW1/47mq7iBWNlAZ30dYLtqBjqztSvxaAibtVueA1YvtS5YQ5PJqcuUoijTVCNHXMOiMArRs3rjG0uvT2uuR+kVN95ik6kK7DtrOUUwjCqtxTyO/UQeA8UhLi4KG98BDmVlDRPqqUHMS+A90HeJ08hDqLbeeeQdwB5PydA5212tlFiAC0aD50L7qNtBgYiuh6rkhzEXuqySP3FEajehjl0jfQEfxsmEqOFhhhnWvrdWI9ak1HbElzCD0EiDEGxZKxvQTYsJbi1SUqAU99RyHQWASEpxBxOQfzeW1G0mJ+ZCpywiF0+HFYD+HlFyGsBLvACMs/eM7JrAlFQPKIVsR7smlYvK0XpeSEjLZtJLNPtL786lkhsHgQGOamga6u1+/HDZgX84OCJZuGBThhb6NH5nFPoVZFGafACGtLckzYVTzv9aSVcpOgbL88EKOifTYarxuefkCIl5RNSiWPQmBBCJDqQoOxj80yT99bqo0lHWHBO8DQJoDWqquEvLx7VEXKTCCERZ6AdMlD6Lgza6lyJQJGpQMOYHkW9dXS9COdS96xj8DbUsO2XrR/NRQCNUQAXJDkjjNcbzvp3MaQ8brjbb+gxaybmvrjpQIPMEYFIxDCGir1HHvu4D+1gfXXUidlrL0C2XcFc5r/KxSAhRpYhh5+aW37b0/ShrVv6zW/OOoThUBQCHjM2EnB0BjyXXdQAnnbL09bMjOT6xJzrsxTGMjkNo6Ig085WXgI8928rRcytYVgMj1SWy9iUuN5kqoebb1kmr2S5eJDYGxMzIk0LCrmh0HXnWxaFhGWixSdMHSGjSTk+Pi+ivihoi/RAc8fHSnKIdS0ontmAxIdI1ivImjx/C0keQ2xEIWl6XqYewjr1dZLktkrMS5CBCj7gaZlGM0HUC75JJaFdOXJRlgkohuydLApW0X/+Mnb9LqcEQhh7R4XOYS2U+ixwsTyLJCWPuVMZEHboq1XseAgBita17ZeC5JNbaQQqACBHTtE45T29uRR8NQxal8PyhIOsAr2V8uvEIvC/H6NOEayIhkDIazb+scphAFVGgpXy+hUFQAxV3gIzdmQFeY1sPy7k/hcPSoEFh8Cg4NprLLAUWwA16DJi/nJeA2S3keeQhRxWkEoV1p9tGrCEmAJRve8wmryEMo44BtEWy8Er3nakRWrbj5AMu5I1a+tl4yYKJkWPwIgAiguKXEdM28PhToQOcg2iCd4HTem9U1OptpJvpGRktxlCFs1YYkcQk0jDyE4tJeEQriodJCROqobumbqkb0r2Vsb0tarjN9FbaoQKAMBURQPaf1byfBOznB8mU55mQazqZiEp7VnszlefbSUr1+WjFUTlp9DKDyEdodwX8rI8ZRDCA1L03YSQo1o61XWL6M2VggsEIGznkLT3gvCKqJkMhQtOSronjeFUk6hEYbqUHFOYdWE5ae25Iu/3ogqmCEXJVHJtHaeoA1/CWEglKc7aFEa0hMvkECNauvVcDCUABcdAn77LMPo2oegaBTzwzqRLFtyDeIEbpYxGbusUtGqJixek4kk0dzVJm9nIF8fQvrlgJZOy1VUsOMaVqWAqe8pBGRDwDdgL0FvS5zme8hWiyEbYZ2VCcFFV9E/PtHS64WOqglrhzbCgXGdWe6uJG1moQev33YIGEXagutaJ1qaug/QcZWHsH7oqyPVA4EkL42s66EfURcdYYyvx3EXfgxfJgRDck8h7N/cw7nwPYCOy9n4wm3J8o+MIO4WRNmWy10qJYb3Ltyu8f+Lon3oyj7R3/3JIyRPKiUy3Rsvm5JAIRAEAiKAVNPi/zM5idpOmkb9oKVSHsAN6FNIaXv6yomJd8Z9Aitn9lURlp9D+OM9d3ag2M26ktuyqn2WI/xCt4XT0qWifYYe2wljpJNOU2lZ6YySC52O2k4hMAcCohv0smX3b3U9/WfxOFe4eHzkHBs36i1efRRaTZeumwkhBFSeMkZV5LKxVGXUYeFOZOMspRxCHL0sAcqQteJNqbCFgZAG17P30E66f7ef/5oV71B9USEgGQKkrZyr3BB5mMcWyRdehPZ6JBQ7OTnZNkMQDg2VB2RVhHV6vI1/37Yn4CHUQw4s7zi8jISFlBxNC1ttpZQc0YSyPKjU1goBuREYGkpy84zr9n0jO63tikTQHVTjncJlEVyUd2LW0b6+f4eDANVSEKlfjnBVEda6/t38YNBf4CHkTMUBK0eAOmzLq4zmc1g7O8V9dDzfs1mHY6tDKATqhgA1KCEta/ny9AzyZT+DQgR07LIIoZbCkhnGQF9lqIKHuGCi609Z8lVFWFvQsYMO7Lqz10gX9XEOedeCmxcWrCNNiTX76e1UFSVaz+1WvVIIyIjAKLdbLe189MHJSe3ppiZYbxkTlbIaLC6ISscqTDNY9Cckyhjv+lOeUBUT1vkeQjgbV5KHUNKBon2YpseOXLf8rhNCxsGyWF3SeSmxFAKvQIC0GOrwTc+mlfhoNku9qzzyGDb6AnVRi87IZNwZj7WOkeCVNKOomLD8xMXRXbctRVP6XrsIDpDPyMcVYjL06Zr1KwJJeQgJBTUuZgREt/CkuXTpQ097WvhvW1pCFJdFmlcjb9RuLIYlKjOe7Oy8dy8pPEh/LptEKyYsrKv48GyrE4vSJQ6FfsjYh5BIFMVPUVHwRRJ406Zk5XMWU1aPCoFFgIBYGnZ1feMfz5z2Hm9ttUjLQqJHQwaI0jNyOfAVi39RSJA06hqH5ecQFo2TG8Ix3cKKkDNWQ+CY/6DUzRUeQlez9Cbe1mv+TdUnCoGLBwFaEg4Pp7jVvWC/9X2nT3tPEWlBs4G/vO6jCC0P5WXMxzs7H/6h0K7GKooRq1jb8D1tsPuvojrpGKTeSRbSgOZHkA3F+WdN3T5AQvpy02s1FAIXMwIDAyMoRZA0V6782Gw01vo7MMJvbWuzQpgzGeHrtTy0kdsYmpx0Mq7W9nGBd+XNiysmLD+HEE0nSjmEMkaOe4j7IA+hdqSn98b9BFZKUyk54qRRj5cCAoyN2URaLS0PngpHOm6YmmSj7e0h3whfkZZTBm60f5OCtnUj/Gfd3V85QLIwNlLxcXlPszIE4JuSSof1Z8lgxuAhpJUXwp0k068gLPc5wq7KAAAJP0lEQVQQQhVF0b4BXrSPBFdDIXApIeCTFmNfPo3r9MZfT7z3gaYm69ZCwdYKBY+0LeKBoC8MG5ea2dxsaVOT+l2d3d/4Ji1RQVZV2dEq0rD8HMLv7flghydxDiGUXrT1gobliZIy5O7FD1MvVfhSuibUXCVHgEjLD3fo6h7+UGY6cmuxaJxpaaGQb3jLeKwWKykhVU2Grq8CanKZ4bAJsmJ/3dk9fDcpOalU+V7BCyWpiLD8PoRG0e7EDttlzSHEPUO3wec6M3fwiScvnL76XyFw6SAgwh3SOpFHd/djD5nWqqumJqOPGEZIa242LMPgvURJA6K/cm/scMTzAFXW1mqFigXjdC4ff3dn98inhZGdPIRl7/MVP05FhHV6jWiC6OqzG8IRPewg8xl7DlqlfIWwZb7BU3IKeSyXdUsU7Rsrcw9qc4XARYYAFfujKT37bL+1ZMlnD3d2P/anurf8bVOZ2IjjmPmWFstEpQeTl9RiHpZ1rAiiIQIju5OD1/yZvwaxlUjKgeNNb242LROd1U9PGt8MRTqu7ep65NueJzyVQZAVjsnXrvRc1ljXL5KHmaNfbsU0LZ+jgHvEZso0YGNDUoLh2OxEyAhxg7uocDgmk5RKFoVA3REQ5DFeJDIZGhrxlnR+4ecQ4ucnTvzVhqnMSURY5n4fq8SrEgkdbVk93p6L7NTUrwHaEjdgGwiYQOl4VEEBmyFoPJt1Tk9Oed/X9cR9XV3/9mOaVMnATmQX2KjI6J7UkiCoMeh39tVeZZwX2ATm3VHJfuUU3SMnt90woWlfKbuUxbz7Vh8oBC4CBHxvHRELrmcs6e6hlcjfed7wP7z88k9el53OvtnVnGs0d/pKpptxGOxb0Xs0ojE9j1XlJJStSbx+HtmBT7le/KfLux85RLCA1KC8pKB8VWdgnwvispdxtB711bvvb3/Pj6JN+uaZabIUScdcdjxhmrlp87F3bfyP93O5CYEA1tFzAaneUwgsdgTOLd9eGXZAicunTg015fPjoUiku9jWdtO0aN56btZ0jSGh2SBb2bl3g31VgYaVBsmlvf/85btbocSsPZtDWK6JLth5vGJvpLnSKtXQ47vpwyEAqdUQyFcIoN5QCCwyBM5pXHTtpPTx8X16f/84NC+yXXEPIq9hJab1Vf5EJCe2+z1sk8ZlVzuyogOWTVh+H8L26Iq2rHOsg3IIz2ovYiYyPMLg7hqFnK41Wav+lwTaJINUSgaFwCJAQKyguJbFAzzp+hZOwyE8+2OQ6wQlksN24/4HNX0um7D8HEJPN3ot3YwWi3nMBlOUaeBugHQAw7PDe9f2vf0pEm1LUtjdZBJTyaIQWAwICALjlzhIyh9p/0Vdn8smLF+6kBHpMJCWVCjmXBCWXB5Cz3PDkZBhzyaeWMY2T4+OauZmlq7ZutrHRD0rBBQCtUWgbKLxi25Z+tKjszOOMBQJfbG2ki5077TWZq41kzGdFmP1o/S148nUeXeGhe5IbacQUAjIhkDZhDVSmkHRMV6CIjOrIw4DS1xpCAFNGp14IqRZRvvwtWs+vo2EG2DVpwTI9sMpeRQClyICZROWXw99zdobjxl69CiVlgFlSUJY0K5018plQsXWWN8/0w/qacM0R0nkuxRPMTVnhUBwCJRNWBTan0ZgWBe7Osu08C+sEMxgiG8ITqQq9gTtKtEc1cJm5+euXfGRbWnYri6MFali7+qrCgGFQIMRKJuwSN5NWpp/L2QmvuM6VKeZ0vYaO6DnFcMxZs1mYjs39d0yRNIMJkXeVGMlU0dXCCgEgkKgIsLaMqTxBMr2xBu/U8yF9oTCjW7YyBxmuFYxH3JaQ2s+wNh1M6OjaWhXirCCOlHUfhQCMiBQsWZEtXUoBH909x0fNaMn78lM5m2EY1UcJlEpGLCpI6zCYbF4DHnlnR/a1HfP10aRG7UZ9X8q3af6nkJAISAnAhVpWDSVLVsoEFPTkn3/+sVsRn8uGufm9/o2bGTQrEBWTYkoc3MtnyKyIvvaZq2yAvc0HzUUAgoBeRGoWMOiKfla1tiuO6/PORM/9pitoTIWhfPzbh01nTZKRWi6ZzYlIloh2/T3N13xwKdg+kd8BR6oJoYaCgGFwEWHQMUaFiFBS8I0lobJ9ff+xNBid8Tj1JADVQuh9tQKKTAskVEhFNFRK8zSijPNd3KywpvpoTS4SpFVrbBX+1UINBqBqjWh0YcPUPtU/UOpbc/ccls/iyW0ZCFvgwihbgVY1E/kM2koIaaxRLNl5mbZcUtvTd24/v6vp9OankymWTqtjOyNPqHU8RUCtUSgqiWhLxh1zPE1mx/sfP/HzXDuM47jaMWCR00bqaVQxcchjQokBY3NY+EoGgYxQ4NncqQ10v2RN/V8emLY0wyUSKQSGGoZ6P8g6lkhcJEiUNWS0MeEyAraDS9u/84Nj97tOi0pp2ieTLRQ00ZSwBiKqJI3b8GkQuTjYGubbFKRGModJiKGnY/+ys23Drxzw2MDRFb3oS71AK/Vs+D9+iKrZ4WAQmARIlCx5jPXXKluzsiIpg8MaM5PD6WXZ7N7P61ps7dE4kzLzdqabXsuLEzQhnQiJAo3ReofbOTEarxzNNETvfTMUNjQQmgTVMjhAzv8TNhc8pV39L7vUcRYkSeSDXspHTmCNbOVzTU/9Z5CQCHQWAQCJSx/KufHQY3uuuPtBW/6LzXXeVck7rUyHZ5EuBJtFP5zqag9vmSAvwx029ARfwpzmDabBUm57KCuh34YMVq/dX3vPU9Ci+PkxPdNYQtqCejDrZ4VApcMAjUhLEKPurxS40TftrT18JcvyxReekfePvF218326iy8xPPsDjQNhL4VOm17uRNho3lCN5qeDZmRZxLRjVuvXDYw7f8SFEJBsV/KsO4jop4VAgqBwBEgoknzLhq/uWvP2xPePvlA+86XvgjiOhz9zU/Ff8PDmjHqpU0y6s/1uXpPIaAQUAjUBAHSuIi8YLOa19BP4Qnk9RPbpefdriYCqp0qBBQC0iPQEM2FDO3kMBzShvjxB7VBmLJggVd2KelPGCWgQkAhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQkAhoBCoOQL/D8pAub21P/M1AAAAAElFTkSuQmCC\", \"icon_small\": \"iVBORw0KGgoAAAANSUhEUgAAADIAAAAyCAYAAAAeP4ixAAAMFmlDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnltSCAktEAEpoTdBehUIHQQB6WAjJAFCCZAQVOzIooJrQcWCFV0Bsa0FkLUiioVFwF4XRFRW1sWCDZU3KaDP1753vm/u/Dlzzpn/zD13MgOAsi07NzcLVQEgW5AvjAryZSYkJjFJPUABUAEFGACczRHl+kRGhgEoo/0/y7tbAJH0160lsf51/L+KKpcn4gCAREKcwhVxsiE+BgCuyckV5gNAaIN6o9n5uRI8CLG6EBIEgIhLcJoMa0pwigxPkNrERPlBzAKATGWzhWkAKEl4Mws4aTCOkoSjrYDLF0C8FWIvTjqbC/EDiCdkZ+dArEyG2Dzluzhp/xQzZSwmm502hmW5SIXszxflZrHn/p/L8b8lO0s8OochbNR0YXCUJGe4bjWZOaESTIX4pCAlPAJiNYgv8blSewm+ly4OjpXbD3BEfnDNAAMAFHDZ/qEQ60DMEGfG+sixPVso9YX2aDg/PyRGjlOEOVHy+GiBICs8TB5neTovZBRv54kCokdtUvmBIRDDSkOPFabHxMt4oi0F/LhwiJUg7hBlRofKfR8VpvuFj9oIxVESzsYQv00VBkbJbDDNbNFoXpgNhy2dC9YCxspPjwmW+WIJPFFC2CgHLs8/QMYB4/IEsXJuGKwu3yi5b0luVqTcHtvOywqKkq0zdlhUED3q25UPC0y2DtjjDPbkSPlc73LzI2Nk3HAUhAE/4A+YQAxbCsgBGYDfPtAwAH/JRgIBGwhBGuABa7lm1CNeOiKAz2hQCP6CiAdEY36+0lEeKID6L2Na2dMapEpHC6QemeApxNm4Nu6Fe+Bh8MmCzR53xd1G/ZjKo7MSA4j+xGBiINFijAcHss6CTQj4/0YXCnsezE7CRTCaw7d4hKeETsJjwk1CN+EuiANPpFHkVrP4RcIfmDPBFNANowXKs0v5PjvcFLJ2wn1xT8gfcscZuDawxh1hJj64N8zNCWq/Zyge4/ZtLX+cT8L6+3zkeiVLJSc5i5SxN+M3ZvVjFL/v1ogL+9AfLbHl2FGsFTuHXcZOYg2AiZ3BGrE27JQEj1XCE2kljM4WJeWWCePwR21s62z7bT//MDdbPr9kvUT5vDn5ko/BLyd3rpCflp7P9IG7MY8ZIuDYTGDa29q5ACDZ22VbxxuGdM9GGFe+6fLOAuBWCpVp33RsIwBOPAWA/u6bzug1LPc1AJzq4IiFBTKdZDsGBPiPoQy/Ci2gB4yAOczHHjgDD8ACAWAyiAAxIBHMhCueDrIh59lgPlgCSkAZWAM2gC1gB9gNasABcAQ0gJPgHLgIroIOcBPch3XRB16AQfAODCMIQkJoCB3RQvQRE8QKsUdcES8kAAlDopBEJBlJQwSIGJmPLEXKkHJkC7ILqUV+RU4g55DLSCdyF+lB+pHXyCcUQ6moOqqLmqITUVfUBw1FY9AZaBqahxaixegqdBNahe5H69Fz6FX0JtqNvkCHMIApYgzMALPGXDE/LAJLwlIxIbYQK8UqsCrsINYE3/N1rBsbwD7iRJyOM3FrWJvBeCzOwfPwhfhKfAteg9fjLfh1vAcfxL8SaAQdghXBnRBCSCCkEWYTSggVhL2E44QL8LvpI7wjEokMohnRBX6XicQM4jziSuI24iHiWWInsZc4RCKRtEhWJE9SBIlNyieVkDaT9pPOkLpIfaQPZEWyPtmeHEhOIgvIReQK8j7yaXIX+Rl5WEFFwUTBXSFCgaswV2G1wh6FJoVrCn0KwxRVihnFkxJDyaAsoWyiHKRcoDygvFFUVDRUdFOcqshXXKy4SfGw4iXFHsWPVDWqJdWPOp0qpq6iVlPPUu9S39BoNFMai5ZEy6etotXSztMe0T4o0ZVslEKUuEqLlCqV6pW6lF4qKyibKPsoz1QuVK5QPqp8TXlARUHFVMVPha2yUKVS5YTKbZUhVbqqnWqEarbqStV9qpdVn6uR1EzVAtS4asVqu9XOq/XSMboR3Y/OoS+l76FfoPepE9XN1EPUM9TL1A+ot6sPaqhpOGrEaczRqNQ4pdHNwBimjBBGFmM14wjjFuPTON1xPuN441aMOziua9x7zfGaLE2eZqnmIc2bmp+0mFoBWplaa7UatB5q49qW2lO1Z2tv176gPTBefbzHeM740vFHxt/TQXUsdaJ05uns1mnTGdLV0w3SzdXdrHted0CPocfSy9Bbr3dar1+fru+lz9dfr39G/0+mBtOHmcXcxGxhDhroGAQbiA12GbQbDBuaGcYaFhkeMnxoRDFyNUo1Wm/UbDRorG88xXi+cZ3xPRMFE1eTdJONJq0m703NTONNl5k2mD430zQLMSs0qzN7YE4z9zbPM68yv2FBtHC1yLTYZtFhiVo6WaZbVlpes0KtnK34VtusOicQJrhNEEyomnDbmmrtY11gXWfdY8OwCbMpsmmweTnReGLSxLUTWyd+tXWyzbLdY3vfTs1usl2RXZPda3tLe459pf0NB5pDoMMih0aHV45WjjzH7Y53nOhOU5yWOTU7fXF2cRY6H3TudzF2SXbZ6nLbVd010nWl6yU3gpuv2yK3k24f3Z3d892PuP/tYe2R6bHP4/kks0m8SXsm9XoaerI9d3l2ezG9kr12enV7G3izvau8H7OMWFzWXtYzHwufDJ/9Pi99bX2Fvsd93/u5+y3wO+uP+Qf5l/q3B6gFxAZsCXgUaBiYFlgXOBjkFDQv6GwwITg0eG3w7RDdEE5IbcjgZJfJCya3hFJDo0O3hD4OswwThjVNQadMnrJuyoNwk3BBeEMEiAiJWBfxMNIsMi/yt6nEqZFTK6c+jbKLmh/VGk2PnhW9L/pdjG/M6pj7seax4tjmOOW46XG1ce/j/ePL47sTJiYsSLiaqJ3IT2xMIiXFJe1NGpoWMG3DtL7pTtNLpt+aYTZjzozLM7VnZs08NUt5FnvW0WRCcnzyvuTP7Ah2FXsoJSRla8ogx4+zkfOCy+Ku5/bzPHnlvGepnqnlqc/TPNPWpfWne6dXpA/w/fhb+K8ygjN2ZLzPjMiszhzJis86lE3OTs4+IVATZApacvRy5uR05lrlluR257nnbcgbFIYK94oQ0QxRY746POa0ic3FP4l7CrwKKgs+zI6bfXSO6hzBnLa5lnNXzH1WGFj4yzx8Hmde83yD+Uvm9yzwWbBrIbIwZWHzIqNFxYv6FgctrllCWZK55Pci26LyordL45c2FesWLy7u/Snop7oSpRJhye1lHst2LMeX85e3r3BYsXnF11Ju6ZUy27KKss8rOSuv/Gz386afR1alrmpf7bx6+xriGsGaW2u919aUq5YXlveum7Kufj1zfen6txtmbbhc4VixYyNlo3hj96awTY2bjTev2fx5S/qWm5W+lYe26mxdsfX9Nu62ru2s7Qd36O4o2/FpJ3/nnV1Bu+qrTKsqdhN3F+x+uiduT+svrr/U7tXeW7b3S7Wgursmqqal1qW2dp/OvtV1aJ24rn//9P0dB/wPNB60PrjrEONQ2WFwWHz4z1+Tf711JPRI81HXowePmRzbepx+vLQeqZ9bP9iQ3tDdmNjYeWLyieYmj6bjv9n8Vn3S4GTlKY1Tq09TThefHjlTeGbobO7ZgXNp53qbZzXfP59w/kbL1Jb2C6EXLl0MvHi+1af1zCXPSycvu18+ccX1SsNV56v1bU5tx393+v14u3N7/TWXa40dbh1NnZM6T3d5d5277n/94o2QG1dvht/svBV7687t6be773DvPL+bdffVvYJ7w/cXPyA8KH2o8rDikc6jqj8s/jjU7dx9qse/p+1x9OP7vZzeF09ETz73FT+lPa14pv+s9rn985P9gf0df077s+9F7ovhgZK/VP/a+tL85bG/WX+3DSYM9r0Svhp5vfKN1pvqt45vm4cihx69y343/L70g9aHmo+uH1s/xX96Njz7M+nzpi8WX5q+hn59MJI9MpLLFrKlRwEMNjQ1FYDX1QDQEuHZoQMAipLs7iUVRHZflCLwn7DsfiYVZwCqWQDELgYgDJ5RtsNmAjEV9pKjdwwLoA4OY00uolQHe1ksKrzBED6MjLzRBYDUBMAX4cjI8LaRkS97INm7AJzNk935JEKE5/udEyWoo++PQfCD/AMf7G3o0obnYAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAAgVpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iCiAgICAgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjEwMjI8L2V4aWY6UGl4ZWxZRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpQaXhlbFhEaW1lbnNpb24+OTE2PC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPHRpZmY6T3JpZW50YXRpb24+MTwvdGlmZjpPcmllbnRhdGlvbj4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC94OnhtcG1ldGE+Cj6cgV0AAA+rSURBVGgFzVprcFXHfd/dc859IfGSBAaHGlkgg4TAiZKm7bSxnDpN/MhrYhHbsfOa1E0m0/RbP3Xq637px2YmM52J+6GeTp2HNAmNXRtjZxKRh5PJoEkISIDAwjgEMEQCrMe9uuec3f5+/z1HlpBkE7dOvXDv2bOP//u1e6XV26g5p/Sg6jeXh9eZzqkx19c3lGqt3NuIxDcmxbmqWW5Vtbr8+HJr/9/HBgb6g5yIp0fue++zo/d/4cCxT96Tj71lzNAEMuA6R/ZmnzkTT4/037B/9BPPPz92rxs6s9f9+Own3f6RTzx74MJDqwj7jZhZVp0rEUVgA64/oN2ib8GQWskkVoKxcJwC2bt3MD3kHo6MtvvWtkR3xHGa1qbjeOpKI17bWvhgOln7a+7ZdM9T81pbCCPvXzcjlByJ36sH04GRauH5F/vXkCGtq7a6gn3nSFZ6PjbcG3Lud6OTd5Ur5k8u/26uobTWkE+Eh6rXElUx7k+55nzvcMrnSu26GCETlNyzI/3rnxnt/2qTOjIS19Kx/aP3/veB4/3tVTDzZjTT2dssEcmq9F3awEqdMlCx9BxkZK1TrUW3g7CrWlmYwYr0rjiRc05p50xYlQytXR/8XVRQ20ygNqxrje62afotrqVmaCr5vut46j41lElZ714YZAkHHGobW7UqtFtV7eRmgffIyIrw35CRLuU3W23/cX1bqefyxFwtbtjUpi6ZuFS3kOQf/3DsgVuJiDngOhiQJRAQrceNjPQXYEw7k8SCdEyRA+oFojFQSXPFrJ54Nb2Fm4aGLr05RsQZ4ROAoZ2zfz47HVNUEd7p8AFwurVNgWovq24iansdRJxf2HIBjWt7E6zophjSBzwRBKkFD6ZgVApGVGLj3Qv3Ltd/AwnCMtGeO37fJnRuSmJarvLWDOZSq2xTUaly2Hgn1/X1tYGW62s505EynYWSKUHDFjgEXw6hHCjvMNoJI5curQxfoka+8drn0NAQGbU2adwSRLoV6gcfYr8ZRq0iZ8FZKozoTHvYc90MwbH2FIqBatTTFJsQrWhWaEBUCZ2mppSzu5xDgNQ6FZvDGi5Z2F5XI2PNU15CRu8ulYXnlAP8EBLgGQO1wMZ3nD376RYCRoTxe/jyOu1g3xAoBAzt9oDIRZRZgGTSKAXazM0Rvr55cvLBGz245eG/LiPrxm8WZIiCu4W6BXLAuwtgZiZNKUJElbntRJRp0eNc4ZtSBTn20KGHI/R3JpS6l4//hoQKMKuCgUYS54JQrW80YoEPDMvSvOwgoRIZw67H4HaJ4vmSNaKOIDZYRbIKDm9MQyJXc67FfOEyz8FBH92uFKa3ANNWREFqJPc9JAzANYgqoA7dtAJnCXTaQ1DDw5mVXAN3RUZyZN8/cv9G7OkQW9WMXrAoqIPISkAWos9kBkC9hN3b67V4DZ5Fr239PozGodpRKJhmCEkcHSDnG2jPONPi8HAO8cPxzErmF2adFRlR/X4FDKfDBLolVz+R0cz4JDKYl0oSvBknEqPDC7N++7LfY5lUtUp2RUUhgYx4uBQUGl0Sjk/T0An9ULkujtNKloO/IiN5eHSh7S5V6HoqAVwKH3B8KwWCVM/BNMDZ9osX+2/gzOhof8SyhghZZF5biz2c1U0IgT4/ZACFQMoEiMrIUtC6ZPhGA8HAuY4LFx7aQPi5tbCftxXDbx6zAfxW460XshEhSYgRZKHwpKEtBxNZ34gTSu1Cd/dgI0cAtPCzQRBSNVJgol7y5YzTB0b3dqfUpuiAUlKOwkcihI8IBAn3NOsoMi2plYBysa1taYZfnhFIglUuQUESu4hM6gmInWjxcRGQExmkZjAZr1oVRsmVmBn+B8+d2HtXmqT3gcAtyKAnQFsVxF/IC8tqVannT/ztJuTTrUyyaJrSF23jlY4eArbMeDaTVauC8NWplPB/2tfHLYub53vxmKqqqsjoqRMPtyLOb/NRxclaSVgZsiyqyG7mAlRH73nq6N5/KJXV05Xm4KFCUfeta4v+BpH2uWdOfmo1NTE8fE7sVLmr28NIr/U1FnMdiwYQj2fue6JvzwizIaW2x5O6NMMvq5HbsoxeSGodqbKbYy81b2BEhk+ODOkYPLhw6mpDTTSC+4tFF85MJxhUDSw09Xpqm5qinpmZ+P3Y9l/jjcvEGVub9lQQtqcbCReH0J7QTXnQP/g+rxIwIQwrX6rkAYXMA5a0ZTUy1twpGnFB0lVeJbx6M8s2cXeJw1jFviyGxV1tuLBWZxmr+ClggkGtMDsTI6jZCYypprZpzmGf7YG20cMXFgkMvGKDKwUBEgvnME4GcUZpSK6x286f72+TicxqfB8BIu8sfPIqRt6t3RPQECAbIuKHEuOzDDtmH10ZoBvVIVsQwlwj2gMQFzLRODMZhZVxrv3FE/tjPsFrN8O2mAzfuBaVllPBK0UT/SjE4Ze88IMmjGDtDTDfDg5cW0EsYYTE3X77UMLFTrseVKXsSkNPHB3lgy6Fws88gzSxOWQDjpJ0rsViGzIEKfPS+ztvPk8g1aqyPz31xQ2Ygu8hSyHfcZxrWTyiBH0xNG5fAG8H4fPIAZcOj6iQSMi+toJYwghQiZR/cvzvm+Fc27MDDyGSSmsA36hwIjLBBPtCAr6gDU1XwkLZT+LQg0aAwphjdPQBHKI4/Go82wEaN0rEYtTjUgxkxI9XIv0TmhLkIXMULiUT0FC1FUaurSCWMDKYnQhr7uJWbIKjSx0kxIHItAKfiV34I8B8gX0oQrJyHXUxlcdcQML4zU00HZB4RMauKolYxsH3fJIV38M6uIGAwWlNj61p3TCSxOoKcodAEt2jx3oPYHdxMHd49tmWMJJndOtiImN2ZukujBBMhEoxToOfQf0/j2hj3oxUTUjyCylB/MPDhQ2U4aEpHCay3NFTl96KMz8XCfXCOdIk1wZBCO19bQ42+yKSLLdJcMBTZ1rqPHPmgXWcyK2HvSWMHOQoGqNKII4qgMgI8Rn6TKyjX4c6PSJZmWOguA7FySKhCiuRPExg9NycqylTHsOImv5lk/geproIB2rwRSh0hr1BvWYbFV06ybUAcDRi1vV4ZUAKV+U2RVG9nQMqsx72ljDyCC6OOeFc2sPrmLyhh6SkgpmpVCX16BSEeXJqKmF4ChLQU08QT4gXH6yj8hwJwb+XN8UbzxIOC77vHf887oDstsz3ZD2xRZA+aujfNBebXuZaqPMwPmyEyh6ZTpubeWVgpYBU6rVSZREjlBCJOHSuWgEyHHiEJ1kDaBYhEedede7iml1n67Z0FuHzYhEENFCP0jeJkSjxn6woOjqC0rHu7mrj64d6SYEq2fgmDN6Y+V6O39JMgf/YO9u/eoXrEEgO17y9wn4lehG8ZTgHjXL2wXUHGZSWA5KXQbVX3iemzv0RKNnC0gQ0CVFCGBgBp2Nf2f6VuQ0bBqcB8iTtGPicODpQEXL2kOIJFP2awDtbmsXREzt7S7mMjIdCC+tInOccPWOc+BKH0rR4Ej4xBcKB3+sGPXF4ECkOj2I095/FpsXfJQjEuPotvNngLQmwiJY4zvAHA/IRSBaqoxyrscBf0MgMjN740iYQRpTaKisCHe6m72EH/YWi5mFDGhL4L3xH6c2bd/wWyF7KHF4kD3aQGCX3dJ6U2k3xDtrT7EH4785en9FTF++JCkAG+4KIRa9Api0tzYa/yveAw18mUAUtINMbpUuyXGCAtGbrQaJHub6WvCLEwKnO0ozQWORABE6vay1FM6+mP7yze/BJTjhc9DHvoDsqeUjGOKMQPChbtWX16tl2DjyS3T4KNxxgO/jokKgKPtWTOZqf8N9BbQYUR2YkH4xUcvTqdKqQ0RnffEMHFMN5Jb6eqc+Vz3Dizm3vldLkQzu/9W+4rfwPzBtkcjBjLC6vnwhLLR/nOn8Iy5zY6iM0JzYPFkJlLquEOAn5E+Pw8PhijVBFLB8GBgYCZ92ORVEF8RJ1EMFdKOu14wIZXwjmp+qJe0VB/MDkJe6RSmAAnLEPv/uxWSCH8qrzd8N3dX33MzbR727U3Ud0EnTf1f3dBz/Q8dhVf1nNUn/+GuqIaADc5jjxlGiIiJpleH8RTvVK6+ryd7wtuw68o+GyAw9NGeQhyttCIQhQhpx83/avXcq2aL3xyVeOnP74CdRTG01sYxCMe0ex+QQRjj8NvMC1Q0N9UM8QArS4rcgYmXkYU/x4LTwKPN6csguMYTi/PlavJ7PwwwovKLBUGGJaQKj3x2RchFNQ84zkx8dY1bbDP5rgVGRBNjJosA4CGWJWPI9zPYtLBLbHipXgfQiVRayDJ6pg7fpi+epkfBqR5+sk9GBfH4gYYnfel3iWZxXRh7mcAVkgXz4azcw0/6ZcunIGDr+T5xqQAd/BYQZFHZ6dzn22pPXjdWpyocoyOHF3AbcK8MH8VpEmI0UbJCwRKGOCrq97O/Y9MXVl7p9gRq4QSdGfgolnorB4xz27v3FZfiDKJJ0hkAeP0hTEUiY8s5Rye/vjddjECUYuvOemK9eoCN9bLl2a2uJhjniJ8yW/bOBdbFY9Yy8jFv+rsD6bKhMUXwu91E8G+q927nsENVJH2tB/meryzju7vnP3HZ3fGM9/IPLIfr9vb45Eog5DgPOb8Y7LTWdR0hdtHO/kBH9uENPCJNb6ywZs7ZJ7KjDhd+McCkdPYzdZUuYUx3KmX7P5KvZXT2OKHyLXvLLJbyo59mabMeYwcwcaYzZFR76SIqJePbC8jJCQnflIFURX3ffH798Y1+baJaOz9PESByP4oSJRp2/vevwCNqr+/gHYvOeTzHBv7jdkEkLBfC4Y7vj9W/4TBaR/fLaGDIaaDpjo8IKYDoMkJw5PPxNGsmOjbcym22BsG7JbxfxkgYxOV7JHSQ4JhkhERHzP2/+F9HNY/ukdfvPmfcfOn/voC62thb+YmGjMQXAR/CNB5MIvGvodXEs/E2fPLxsQDxYdeEQn4B/Mgw9fB12+2Scgj+yt+/Zm6/+QIAjd5ycnG6daWopl3J+Fa9ZEZSlIjf53UuAQAUUj63ovU2VkrUdqcUQIAJIGHrTcazktoTf/JdbPvrXf9FtaAArUU6dPf+w9brLxJWD8M1COFO3+84Ybn8x+iB1MhZHRRwcpc5rhVj7RkeM3ng0cZ8u1aXu6bDZKcjv46Gs5Qda+xV9yaU1zbh9kef/PC9H5IOU9WUzrttv65Inj9zc5XGkKy0VcLq1eUyjPTqf1wJjP3d79r9NyIY0/GlgI7A/R13IDXzXO9fG2GTmNff8XGDn+zIDklX23/+i996EQ+DL4KYGpQ84E/3L3zm+P/W9yQo7sD/Ykt8shoyaWG39bj2VEC0PsUxNva4Iz4v4H8vqTm++oi2AAAAAASUVORK5C\", \"version\": \"0.1\", \"provider_url\": \"https://live.paloaltonetworks.com/t5/MineMeld/ct-p/MineMeld\", \"display_name\": \"MineMeld Feed\", \"summary\": \"Indicators routed through MineMeld\", \"tech_data\": \"Indicators routed through MineMeld\", \"name\": \"DomainHC\"\n},\n\"reports\": [{ \"timestamp\": 1550566791, \"id\": \"DomainHC_report\", \"link\": \"https://live.paloaltonetworks.com/t5/MineMeld/ct-p/MineMeld\", \"score\": 100, \"description\": \"MineMeld Generated Report\", \"title\": \"MieneMeld Generated Report\", \"iocs\": { \"ipv4\": [],\"dns\": [\"25z5g623wpqpdwis.onion.to\", \"27c73bq66y4xqoh7.dorfact.at\", \"27lelchgcvs2wpm7.3lhjyx.top\", \"27lelchgcvs2wpm7.7jiff7.top\", \"27lelchgcvs2wpm7.7zv8o2.top\", \"27lelchgcvs2wpm7.9ildst.top\", \"27lelchgcvs2wpm7.adevf4.top\", \"27lelchgcvs2wpm7.ag082d.top\", \"27lelchgcvs2wpm7.apperloads.win\", \"27lelchgcvs2wpm7.asd3r3.top\", \"27lelchgcvs2wpm7.b7mciu.top\", \"27lelchgcvs2wpm7.bedrastic.bid\", \"27lelchgcvs2wpm7.bestfordownload.click\", \"27lelchgcvs2wpm7.bonbestal.asia\", \"27lelchgcvs2wpm7.fm0cga.top\", \"27lelchgcvs2wpm7.h9ihx3.top\", \"27lelchgcvs2wpm7.laverhants.link\", \"27lelchgcvs2wpm7.liopakerb.black\", \"27lelchgcvs2wpm7.marksgain.kim\", \"27lelchgcvs2wpm7.nfgpeb.top\", \"27lelchgcvs2wpm7.redefined.click\", \"27lelchgcvs2wpm7.rt4e34.win\", \"27lelchgcvs2wpm7.tankbe.pro\", \"27lelchgcvs2wpm7.thyx30.top\", \"27lelchgcvs2wpm7.uboys5.top\", \"27lelchgcvs2wpm7.vrid8l.top\", \"27lelchgcvs2wpm7.wins4n.win\", \"27lelchgcvs2wpm7.wishsends.mobi\", \"27lelchgcvs2wpm7.xkfi59.top\", \"27lelchgcvs2wpm7.xmvr54.top\", \"2bdfb.spinakrosa.at\", \"2gdb4.leoraorage.at\", \"2ymh2gnnbg6pgq2r.gremsot.pl\", \"2ymh2gnnbg6pgq2r.winregion.tw\", \"32kl2rwsjvqjeui7.onion.cab\", \"32kl2rwsjvqjeui7.onion.to\", \"32kl2rwsjvqjeui7.tor2web.org\", \"37kddsserrt.xyz\", \"3qbyaoohkcqkzrz6.bestxprice.ch\", \"3qbyaoohkcqkzrz6.livecamshow.ch\", \"3qbyaoohkcqkzrz6.torclassik.li\", \"3qbyaoohkcqkzrz6.torcommunity.ch\", \"3qbyaoohkcqkzrz6.tordonator.li\", \"3qbyaoohkcqkzrz6.tordoor.li\", \"3qbyaoohkcqkzrz6.torgate.es\", \"3qbyaoohkcqkzrz6.torgateway.li\", \"3qbyaoohkcqkzrz6.tormain.li\", \"3qbyaoohkcqkzrz6.tormaster.ch\", \"3qbyaoohkcqkzrz6.tormaster.fr\", \"3qbyaoohkcqkzrz6.torplanet.eu\", \"3qbyaoohkcqkzrz6.torprovider.li\", \"3qbyaoohkcqkzrz6.torreactor.li\", \"3qbyaoohkcqkzrz6.torstation.li\", \"4kqd3hmqgptupi3p.0vgu64.top\", \"4kqd3hmqgptupi3p.143h2a.top\", \"4kqd3hmqgptupi3p.1tvjk1.top\", \"4kqd3hmqgptupi3p.1zp109.bid\", \"4kqd3hmqgptupi3p.249isv.bid\", \"4kqd3hmqgptupi3p.2y4t6f.bid\", \"4kqd3hmqgptupi3p.3arvfd.top\", \"4kqd3hmqgptupi3p.3lhjyx.top\", \"4kqd3hmqgptupi3p.43wjor.top\", \"4kqd3hmqgptupi3p.4j11jt.bid\", \"4kqd3hmqgptupi3p.4k9xlx.top\", \"4kqd3hmqgptupi3p.5b4ej6.bid\", \"4kqd3hmqgptupi3p.5ctoeb.bid\", \"4kqd3hmqgptupi3p.62er3d.top\", \"4kqd3hmqgptupi3p.6h03gw.top\", \"4kqd3hmqgptupi3p.6j7jcn.bid\", \"4kqd3hmqgptupi3p.6ntrb6.top\", \"4kqd3hmqgptupi3p.6ogy3i.top\", \"4kqd3hmqgptupi3p.7w9p1n.bid\", \"4kqd3hmqgptupi3p.859rkn.top\", \"4kqd3hmqgptupi3p.8kcfnk.bid\", \"4kqd3hmqgptupi3p.91006j.bid\", \"4kqd3hmqgptupi3p.9ildst.top\", \"4kqd3hmqgptupi3p.a0g0o7.bid\", \"4kqd3hmqgptupi3p.adevf4.top\", \"4kqd3hmqgptupi3p.anypicked.red\", \"4kqd3hmqgptupi3p.as5su5.top\", \"4kqd3hmqgptupi3p.asfall.in\", \"4kqd3hmqgptupi3p.athere.in\", \"4kqd3hmqgptupi3p.b7mciu.top\", \"4kqd3hmqgptupi3p.barberryshin.casa\", \"4kqd3hmqgptupi3p.bestergo.pw\", \"4kqd3hmqgptupi3p.bigfooters.loan\", \"4kqd3hmqgptupi3p.bnctf6.top\", \"4kqd3hmqgptupi3p.bookjumps.us\", \"4kqd3hmqgptupi3p.boxsame.kim\", \"4kqd3hmqgptupi3p.boxtimed.gdn\", \"4kqd3hmqgptupi3p.breakown.loan\", \"4kqd3hmqgptupi3p.byeraser.lol\", \"4kqd3hmqgptupi3p.carrygain.kim\", \"4kqd3hmqgptupi3p.cfu46r.bid\", \"4kqd3hmqgptupi3p.chargecar.vip\", \"4kqd3hmqgptupi3p.choiceher.win\", \"4kqd3hmqgptupi3p.clockhate.loan\", \"4kqd3hmqgptupi3p.cm5ohx.bid\", \"4kqd3hmqgptupi3p.csv7o6.bid\", \"4kqd3hmqgptupi3p.cutslifes.bid\", \"4kqd3hmqgptupi3p.dd4xo3.top\", \"4kqd3hmqgptupi3p.dkrie7.top\", \"4kqd3hmqgptupi3p.dmvute.top\", \"4kqd3hmqgptupi3p.dozensby.loan\", \"4kqd3hmqgptupi3p.easyits.black\", \"4kqd3hmqgptupi3p.effortany.win\", \"4kqd3hmqgptupi3p.endsdoubt.loan\", \"4kqd3hmqgptupi3p.eventeach.gdn\", \"4kqd3hmqgptupi3p.ezm0r5.top\", \"4kqd3hmqgptupi3p.f0jlbj.bid\", \"4kqd3hmqgptupi3p.fairlies.link\", \"4kqd3hmqgptupi3p.foodtopic.mobi\", \"4kqd3hmqgptupi3p.g7kcux.bid\", \"4kqd3hmqgptupi3p.gameswarm.loan\", \"4kqd3hmqgptupi3p.gapplayed.link\", \"4kqd3hmqgptupi3p.getsbug.kim\", \"4kqd3hmqgptupi3p.gg4dgp.bid\", \"4kqd3hmqgptupi3p.gio6f6.bid\", \"4kqd3hmqgptupi3p.gletterstan.trade\", \"4kqd3hmqgptupi3p.goodslet.win\", \"4kqd3hmqgptupi3p.goshare.red\", \"4kqd3hmqgptupi3p.gs2ka7.top\", \"4kqd3hmqgptupi3p.he81tz.bid\", \"4kqd3hmqgptupi3p.heardbids.date\", \"4kqd3hmqgptupi3p.heldbegun.kim\", \"4kqd3hmqgptupi3p.hessale.pw\", \"4kqd3hmqgptupi3p.holescase.pw\", \"4kqd3hmqgptupi3p.homehuge.top\", \"4kqd3hmqgptupi3p.hotcopies.bid\", \"4kqd3hmqgptupi3p.inforcing.pw\", \"4kqd3hmqgptupi3p.insystem.men\", \"4kqd3hmqgptupi3p.itdrink.club\", \"4kqd3hmqgptupi3p.ix1upt.bid\", \"4kqd3hmqgptupi3p.jal9lk.bid\", \"4kqd3hmqgptupi3p.k7oud1.top\", \"4kqd3hmqgptupi3p.kml2o2.top\", \"4kqd3hmqgptupi3p.l6k4x7.bid\", \"4kqd3hmqgptupi3p.laterugly.win\", \"4kqd3hmqgptupi3p.liescale.in\", \"4kqd3hmqgptupi3p.liesshall.bid\", \"4kqd3hmqgptupi3p.lobulz.bid\", \"4kqd3hmqgptupi3p.lorrydo.lol\", \"4kqd3hmqgptupi3p.masterany.red\", \"4kqd3hmqgptupi3p.meetbinds.pw\", \"4kqd3hmqgptupi3p.metmet.win\", \"4kqd3hmqgptupi3p.metpast.site\", \"4kqd3hmqgptupi3p.mi3596.bid\", \"4kqd3hmqgptupi3p.mtxtul.top\", \"4kqd3hmqgptupi3p.mustspace.us\", \"4kqd3hmqgptupi3p.myaddress.link\", \"4kqd3hmqgptupi3p.namefalls.pro\", \"4kqd3hmqgptupi3p.nameuser.site\", \"4kqd3hmqgptupi3p.nearlybut.us\", \"4kqd3hmqgptupi3p.needmight.win\", \"4kqd3hmqgptupi3p.newrange.link\", \"4kqd3hmqgptupi3p.nextask.loan\", \"4kqd3hmqgptupi3p.nh47ri.bid\", \"4kqd3hmqgptupi3p.nxmu0x.bid\", \"4kqd3hmqgptupi3p.o8hpwj.top\", \"4kqd3hmqgptupi3p.outputon.asia\", \"4kqd3hmqgptupi3p.ownamount.pro\", \"4kqd3hmqgptupi3p.p79b8l.bid\", \"4kqd3hmqgptupi3p.pairsraw.loan\", \"4kqd3hmqgptupi3p.pap44w.top\", \"4kqd3hmqgptupi3p.powersno.link\", \"4kqd3hmqgptupi3p.pushstory.bid\", \"4kqd3hmqgptupi3p.r21wmw.top\", \"4kqd3hmqgptupi3p.rsi6gn.top\", \"4kqd3hmqgptupi3p.salethe.gdn\", \"4kqd3hmqgptupi3p.sayssales.bid\", \"4kqd3hmqgptupi3p.scoreable.bid\", \"4kqd3hmqgptupi3p.seemby.loan\", \"4kqd3hmqgptupi3p.sel7rg.bid\", \"4kqd3hmqgptupi3p.selfcrash.site\", \"4kqd3hmqgptupi3p.sentowing.trade\", \"4kqd3hmqgptupi3p.sitcalls.us\", \"4kqd3hmqgptupi3p.sk8r54.top\", \"4kqd3hmqgptupi3p.somegave.info\", \"4kqd3hmqgptupi3p.stageend.link\", \"4kqd3hmqgptupi3p.stopsage.gdn\", \"4kqd3hmqgptupi3p.storingus.gdn\", \"4kqd3hmqgptupi3p.tankplain.date\", \"4kqd3hmqgptupi3p.termprior.men\", \"4kqd3hmqgptupi3p.themevery.win\", \"4kqd3hmqgptupi3p.thyx30.top\", \"4kqd3hmqgptupi3p.tieslaws.link\", \"4kqd3hmqgptupi3p.todaynine.loan\", \"4kqd3hmqgptupi3p.twz1ga.top\", \"4kqd3hmqgptupi3p.uwckha.top\", \"4kqd3hmqgptupi3p.v11z5e.top\", \"4kqd3hmqgptupi3p.valueshes.bid\", \"4kqd3hmqgptupi3p.variedtax.kim\", \"4kqd3hmqgptupi3p.vkm4l6.top\", \"4kqd3hmqgptupi3p.wallluck.date\", \"4kqd3hmqgptupi3p.whmykv.bid\", \"4kqd3hmqgptupi3p.wins4n.top\", \"4kqd3hmqgptupi3p.wz139z.top\", \"4kqd3hmqgptupi3p.xmfru5.top\", \"4kqd3hmqgptupi3p.y12acl.bid\", \"4kqd3hmqgptupi3p.y5j7e6.top\", \"4kqd3hmqgptupi3p.yg767p.bid\", \"4kqd3hmqgptupi3p.yoursdoor.lol\", \"4kqd3hmqgptupi3p.z8ijgn.bid\", \"4kqd3hmqgptupi3p.z97f9v.bid\", \"4rebaopfgrewe.top\", \"4w5wihkwyhsav2ha.dreamtest.at\", \"4w5wihkwyhsav2ha.fastdances.at\", \"4w5wihkwyhsav2ha.grandhaus.at\", \"4w5wihkwyhsav2ha.payfactor.at\", \"52uo5k3t73ypjije.01fake.bid\", \"52uo5k3t73ypjije.086ux2.top\", \"52uo5k3t73ypjije.0n5joc.top\", \"52uo5k3t73ypjije.0nyi6l.bid\", \"52uo5k3t73ypjije.0vgu64.top\", \"52uo5k3t73ypjije.11pmnz.top\", \"52uo5k3t73ypjije.1bipa9.top\", \"52uo5k3t73ypjije.1de02r.top\", \"52uo5k3t73ypjije.1f1dw3.bid\", \"52uo5k3t73ypjije.1g0vo2.bid\", \"52uo5k3t73ypjije.1pma4t.bid\", \"52uo5k3t73ypjije.1ufr2v.bid\", \"52uo5k3t73ypjije.209kai.bid\", \"52uo5k3t73ypjije.249isv.bid\", \"52uo5k3t73ypjije.26lpul.bid\", \"52uo5k3t73ypjije.2gbbja.top\", \"52uo5k3t73ypjije.2llgoy.bid\", \"52uo5k3t73ypjije.2y4t6f.bid\", \"52uo5k3t73ypjije.2ym6om.bid\", \"52uo5k3t73ypjije.31wkhu.top\", \"52uo5k3t73ypjije.33dofy.top\", \"52uo5k3t73ypjije.35u068.bid\", \"52uo5k3t73ypjije.3di24a.top\", \"52uo5k3t73ypjije.3gpdgx.bid\", \"52uo5k3t73ypjije.3lhjyx.top\", \"52uo5k3t73ypjije.3rr6ao.top\", \"52uo5k3t73ypjije.3zotov.bid\", \"52uo5k3t73ypjije.40wiai.top\", \"52uo5k3t73ypjije.43l7lm.bid\", \"52uo5k3t73ypjije.43wjor.top\", \"52uo5k3t73ypjije.495iru.top\", \"52uo5k3t73ypjije.4jub4e.bid\", \"52uo5k3t73ypjije.4k9xlx.top\", \"52uo5k3t73ypjije.4n592s.top\", \"52uo5k3t73ypjije.4nf7ij.top\", \"52uo5k3t73ypjije.4oyhvh.top\", \"52uo5k3t73ypjije.4pjetv.bid\", \"52uo5k3t73ypjije.4xiiup.bid\", \"52uo5k3t73ypjije.4yl1hr.bid\", \"52uo5k3t73ypjije.4ynpjd.top\", \"52uo5k3t73ypjije.50cs7p.bid\", \"52uo5k3t73ypjije.56185u.bid\", \"52uo5k3t73ypjije.5ctoeb.bid\", \"52uo5k3t73ypjije.5ittco.bid\", \"52uo5k3t73ypjije.5kb3dl.top\", \"52uo5k3t73ypjije.5o4bjf.bid\", \"52uo5k3t73ypjije.5tb8hy.bid\", \"52uo5k3t73ypjije.5vhk5r.bid\", \"52uo5k3t73ypjije.5zxii2.bid\", \"52uo5k3t73ypjije.62er3d.top\", \"52uo5k3t73ypjije.68xmf9.bid\", \"52uo5k3t73ypjije.6ec2xb.bid\", \"52uo5k3t73ypjije.6j7jcn.bid\", \"52uo5k3t73ypjije.6w3rkc.bid\", \"52uo5k3t73ypjije.7156et.bid\", \"52uo5k3t73ypjije.7asel7.top\", \"52uo5k3t73ypjije.7j6htz.bid\", \"52uo5k3t73ypjije.7jiff7.top\", \"52uo5k3t73ypjije.7ud98m.bid\", \"52uo5k3t73ypjije.7wrwp4.top\", \"52uo5k3t73ypjije.80yabh.bid\", \"52uo5k3t73ypjije.86rhzr.bid\", \"52uo5k3t73ypjije.8a0sf6.top\", \"52uo5k3t73ypjije.8cjlyt.bid\", \"52uo5k3t73ypjije.8hphyr.top\", \"52uo5k3t73ypjije.8i8dt4.top\", \"52uo5k3t73ypjije.8kcfnk.bid\", \"52uo5k3t73ypjije.8rrxd9.bid\", \"52uo5k3t73ypjije.8rxv74.bid\", \"52uo5k3t73ypjije.91006j.bid\", \"52uo5k3t73ypjije.94ycl8.bid\", \"52uo5k3t73ypjije.95ovzy.top\", \"52uo5k3t73ypjije.9bjnlk.bid\", \"52uo5k3t73ypjije.9cd81s.bid\", \"52uo5k3t73ypjije.9ildst.top\", \"52uo5k3t73ypjije.9kxz23.bid\", \"52uo5k3t73ypjije.9nj8ex.top\", \"52uo5k3t73ypjije.9sfrr0.bid\", \"52uo5k3t73ypjije.9tftgh.bid\", \"52uo5k3t73ypjije.a0g0o7.bid\", \"52uo5k3t73ypjije.a2uzpe.top\", \"52uo5k3t73ypjije.aclox4.bid\", \"52uo5k3t73ypjije.ahvshc.top\", \"52uo5k3t73ypjije.ai7hur.bid\", \"52uo5k3t73ypjije.ajolkg.bid\", \"52uo5k3t73ypjije.aryh7f.bid\", \"52uo5k3t73ypjije.asxjdp.top\", \"52uo5k3t73ypjije.b2s4ch.bid\", \"52uo5k3t73ypjije.b7mciu.top\", \"52uo5k3t73ypjije.b8ll6n.top\", \"52uo5k3t73ypjije.bar8sc.bid\", \"52uo5k3t73ypjije.bcjl1h.top\", \"52uo5k3t73ypjije.bipa9k.bid\", \"52uo5k3t73ypjije.bipnnp.bid\", \"52uo5k3t73ypjije.bj9eea.bid\", \"52uo5k3t73ypjije.bnctf6.top\", \"52uo5k3t73ypjije.bp9mn8.bid\", \"52uo5k3t73ypjije.bt7r70.top\", \"52uo5k3t73ypjije.c3fz3z.bid\", \"52uo5k3t73ypjije.c7ex9n.top\", \"52uo5k3t73ypjije.catfills.mobi\", \"52uo5k3t73ypjije.cc0r87.bid\", \"52uo5k3t73ypjije.cfu46r.bid\", \"52uo5k3t73ypjije.cjc2jn.top\", \"52uo5k3t73ypjije.cm5ohx.bid\", \"52uo5k3t73ypjije.cm898n.bid\", \"52uo5k3t73ypjije.cmfkru.top\", \"52uo5k3t73ypjije.cpvwgx.bid\", \"52uo5k3t73ypjije.csdbnk.bid\", \"52uo5k3t73ypjije.csj0k5.top\", \"52uo5k3t73ypjije.csv7o6.bid\", \"52uo5k3t73ypjije.cto5ee.bid\", \"52uo5k3t73ypjije.czzg7f.bid\", \"52uo5k3t73ypjije.daigy0.top\", \"52uo5k3t73ypjije.das34.com\", \"52uo5k3t73ypjije.dd4xo3.top\", \"52uo5k3t73ypjije.ddwub3.top\", \"52uo5k3t73ypjije.deg5xr.top\", \"52uo5k3t73ypjije.dkrie7.top\", \"52uo5k3t73ypjije.dkriur.top\", \"52uo5k3t73ypjije.dkro3u.top\", \"52uo5k3t73ypjije.dmrueo.top\", \"52uo5k3t73ypjije.dmvute.top\", \"52uo5k3t73ypjije.dsv023.bid\", \"52uo5k3t73ypjije.dvuybv.bid\", \"52uo5k3t73ypjije.e32d1o.bid\", \"52uo5k3t73ypjije.e6in0v.top\", \"52uo5k3t73ypjije.e78hjo.bid\", \"52uo5k3t73ypjije.e8hua8.top\", \"52uo5k3t73ypjije.ei9evn.top\", \"52uo5k3t73ypjije.en3oyw.bid\", \"52uo5k3t73ypjije.eoivrm.bid\", \"52uo5k3t73ypjije.ep493u.top\", \"52uo5k3t73ypjije.er05vm.bid\", \"52uo5k3t73ypjije.ezm0r5.top\", \"52uo5k3t73ypjije.f0jlbj.bid\", \"52uo5k3t73ypjije.f242v5.bid\", \"52uo5k3t73ypjije.f3z72p.bid\", \"52uo5k3t73ypjije.fe98iy.top\", \"52uo5k3t73ypjije.fi50le.bid\", \"52uo5k3t73ypjije.fkgrie.top\", \"52uo5k3t73ypjije.g0ots2.top\", \"52uo5k3t73ypjije.g0spln.bid\", \"52uo5k3t73ypjije.g5196b.bid\", \"52uo5k3t73ypjije.gg4dgp.bid\", \"52uo5k3t73ypjije.gio6f6.bid\", \"52uo5k3t73ypjije.givxuf.bid\", \"52uo5k3t73ypjije.gmnjz7.bid\", \"52uo5k3t73ypjije.gnee6i.top\", \"52uo5k3t73ypjije.gnuvaw.bid\", \"52uo5k3t73ypjije.goztus.bid\", \"52uo5k3t73ypjije.gpy3tc.top\", \"52uo5k3t73ypjije.gtnfgj.top\", \"52uo5k3t73ypjije.gu7eao.bid\", \"52uo5k3t73ypjije.gvoafg.bid\", \"52uo5k3t73ypjije.h3ss4t.bid\", \"52uo5k3t73ypjije.hawtzr.bid\", \"52uo5k3t73ypjije.hbd7m4.bid\", \"52uo5k3t73ypjije.hhc366.bid\", \"52uo5k3t73ypjije.hlu8yz.top\", \"52uo5k3t73ypjije.hossy3.bid\", \"52uo5k3t73ypjije.hv42mo.bid\", \"52uo5k3t73ypjije.i5cgcw.top\", \"52uo5k3t73ypjije.i6gn9s.bid\", \"52uo5k3t73ypjije.i8zh1k.bid\", \"52uo5k3t73ypjije.iait3w.bid\", \"52uo5k3t73ypjije.ibngww.top\", \"52uo5k3t73ypjije.ie7t8k.top\", \"52uo5k3t73ypjije.ih9te2.bid\", \"52uo5k3t73ypjije.ij0cia.bid\", \"52uo5k3t73ypjije.imhhwm.top\", \"52uo5k3t73ypjije.insystem.men\", \"52uo5k3t73ypjije.izyclz.bid\", \"52uo5k3t73ypjije.j8873f.bid\", \"52uo5k3t73ypjije.j92msu.top\", \"52uo5k3t73ypjije.jal9lk.bid\", \"52uo5k3t73ypjije.jg6jtw.top\", \"52uo5k3t73ypjije.js43vy.bid\", \"52uo5k3t73ypjije.k0dcd2.bid\", \"52uo5k3t73ypjije.k21zey.bid\", \"52uo5k3t73ypjije.k56185.top\", \"52uo5k3t73ypjije.k7oud1.top\", \"52uo5k3t73ypjije.k8ytej.bid\", \"52uo5k3t73ypjije.k9z7pm.top\", \"52uo5k3t73ypjije.ka0te8.top\", \"52uo5k3t73ypjije.kas17.com\", \"52uo5k3t73ypjije.kcufx4.top\", \"52uo5k3t73ypjije.kml2o2.top\", \"52uo5k3t73ypjije.kswcuk.top\", \"52uo5k3t73ypjije.kt70uk.bid\", \"52uo5k3t73ypjije.ku824r.bid\", \"52uo5k3t73ypjije.kwnw1b.bid\", \"52uo5k3t73ypjije.kyjw0g.bid\", \"52uo5k3t73ypjije.kzhzuc.top\", \"52uo5k3t73ypjije.kzo8mc.top\", \"52uo5k3t73ypjije.kzwor6.top\", \"52uo5k3t73ypjije.l6ry3h.bid\", \"52uo5k3t73ypjije.laugk2.top\", \"52uo5k3t73ypjije.lba61x.top\", \"52uo5k3t73ypjije.ldsl8m.bid\", \"52uo5k3t73ypjije.lethints.date\", \"52uo5k3t73ypjije.lh9ax3.bid\", \"52uo5k3t73ypjije.li8wfu.bid\", \"52uo5k3t73ypjije.lib2vi.top\", \"52uo5k3t73ypjije.lio2wr.bid\", \"52uo5k3t73ypjije.loanshown.info\", \"52uo5k3t73ypjije.lrraca.bid\", \"52uo5k3t73ypjije.lwbi59.top\", \"52uo5k3t73ypjije.m33d4b.bid\", \"52uo5k3t73ypjije.m5fgoi.top\", \"52uo5k3t73ypjije.m6j75a.bid\", \"52uo5k3t73ypjije.mbwxyg.bid\", \"52uo5k3t73ypjije.mfgb1h.top\", \"52uo5k3t73ypjije.mn1kms.bid\", \"52uo5k3t73ypjije.msu96b.top\", \"52uo5k3t73ypjije.mtxtul.top\", \"52uo5k3t73ypjije.myurv5.bid\", \"52uo5k3t73ypjije.n41n1a.top\", \"52uo5k3t73ypjije.n6kswi.top\", \"52uo5k3t73ypjije.n8niwa.bid\", \"52uo5k3t73ypjije.nb83bp.bid\", \"52uo5k3t73ypjije.neekll.bid\", \"52uo5k3t73ypjije.nh47ri.bid\", \"52uo5k3t73ypjije.nmapwy.bid\", \"52uo5k3t73ypjije.nxmu0x.bid\", \"52uo5k3t73ypjije.o08a6d.top\", \"52uo5k3t73ypjije.o0hwme.bid\", \"52uo5k3t73ypjije.o5xcnd.bid\", \"52uo5k3t73ypjije.o6fa2g.bid\", \"52uo5k3t73ypjije.o8hpwj.bid\", \"52uo5k3t73ypjije.o8hpwj.top\", \"52uo5k3t73ypjije.o9w43w.bid\", \"52uo5k3t73ypjije.oef1sh.bid\", \"52uo5k3t73ypjije.ojesoa.bid\", \"52uo5k3t73ypjije.ojx58b.bid\", \"52uo5k3t73ypjije.omrexj.top\", \"52uo5k3t73ypjije.ooulp2.bid\", \"52uo5k3t73ypjije.ovpgod.top\", \"52uo5k3t73ypjije.p0lxvm.bid\", \"52uo5k3t73ypjije.p2lsgr.top\", \"52uo5k3t73ypjije.p5dxeh.bid\", \"52uo5k3t73ypjije.pap44w.top\", \"52uo5k3t73ypjije.pfija1.bid\", \"52uo5k3t73ypjije.pop81.com\", \"52uo5k3t73ypjije.poplenjohs.review\", \"52uo5k3t73ypjije.pr2zwz.bid\", \"52uo5k3t73ypjije.r21wmw.top\", \"52uo5k3t73ypjije.r2ok0b.bid\", \"52uo5k3t73ypjije.r4z3o5.bid\", \"52uo5k3t73ypjije.rdmwha.bid\", \"52uo5k3t73ypjije.red4is.top\", \"52uo5k3t73ypjije.rexjyp.bid\", \"52uo5k3t73ypjije.rgdk0u.top\", \"52uo5k3t73ypjije.rl0bdw.top\", \"52uo5k3t73ypjije.rnkj09.top\", \"52uo5k3t73ypjije.rv50gt.bid\", \"52uo5k3t73ypjije.s2xb1s.bid\", \"52uo5k3t73ypjije.sdfztr.bid\", \"52uo5k3t73ypjije.self56.top\", \"52uo5k3t73ypjije.sg62es.top\", \"52uo5k3t73ypjije.skri59.top\", \"52uo5k3t73ypjije.snwy26.top\", \"52uo5k3t73ypjije.sotn58.bid\", \"52uo5k3t73ypjije.srmlzh.bid\", \"52uo5k3t73ypjije.ssh3ln.bid\", \"52uo5k3t73ypjije.sx90yk.bid\", \"52uo5k3t73ypjije.sxjdpg.bid\", \"52uo5k3t73ypjije.thyx30.top\", \"52uo5k3t73ypjije.ti4wic.top\", \"52uo5k3t73ypjije.to6maq.top\", \"52uo5k3t73ypjije.twz1ga.top\", \"52uo5k3t73ypjije.txszfs.top\", \"52uo5k3t73ypjije.tzgwdf.top\", \"52uo5k3t73ypjije.u2r7tm.bid\", \"52uo5k3t73ypjije.u36ik0.bid\", \"52uo5k3t73ypjije.u50s89.bid\", \"52uo5k3t73ypjije.ujtwhg.top\", \"52uo5k3t73ypjije.ul8ib9.bid\", \"52uo5k3t73ypjije.un8niw.top\", \"52uo5k3t73ypjije.uv39h5.bid\", \"52uo5k3t73ypjije.uw3r6a.top\", \"52uo5k3t73ypjije.uw7w05.bid\", \"52uo5k3t73ypjije.uwazu7.bid\", \"52uo5k3t73ypjije.uwckha.bid\", \"52uo5k3t73ypjije.uwckha.top\", \"52uo5k3t73ypjije.ux93ip.top\", \"52uo5k3t73ypjije.v11z5e.top\", \"52uo5k3t73ypjije.v9y6z8.bid\", \"52uo5k3t73ypjije.veupl2.top\", \"52uo5k3t73ypjije.vkm4l6.top\", \"52uo5k3t73ypjije.vkslju.bid\", \"52uo5k3t73ypjije.vlo18w.bid\", \"52uo5k3t73ypjije.vmotsf.bid\", \"52uo5k3t73ypjije.vor28o.bid\", \"52uo5k3t73ypjije.vt3dg6.bid\", \"52uo5k3t73ypjije.w6sj06.bid\", \"52uo5k3t73ypjije.w8yolm.bid\", \"52uo5k3t73ypjije.wg00sp.bid\", \"52uo5k3t73ypjije.whmykv.bid\", \"52uo5k3t73ypjije.whosewine.lol\", \"52uo5k3t73ypjije.wht5py.top\", \"52uo5k3t73ypjije.wins4n.win\", \"52uo5k3t73ypjije.wl52rt.bid\", \"52uo5k3t73ypjije.wrd4fo.top\", \"52uo5k3t73ypjije.ws1uet.top\", \"52uo5k3t73ypjije.wz139z.top\", \"52uo5k3t73ypjije.x2kl7t.top\", \"52uo5k3t73ypjije.x3nnbd.top\", \"52uo5k3t73ypjije.x7fylp.bid\", \"52uo5k3t73ypjije.x9a6yb.bid\", \"52uo5k3t73ypjije.x9kjcn.bid\", \"52uo5k3t73ypjije.x9le66.top\", \"52uo5k3t73ypjije.xab7m0.top\", \"52uo5k3t73ypjije.xglk6h.bid\", \"52uo5k3t73ypjije.xjb384.bid\", \"52uo5k3t73ypjije.xmfru5.top\", \"52uo5k3t73ypjije.xtppp8.bid\", \"52uo5k3t73ypjije.y12acl.bid\", \"52uo5k3t73ypjije.y5j7e6.top\", \"52uo5k3t73ypjije.ye42cp.bid\", \"52uo5k3t73ypjije.yg767p.bid\", \"52uo5k3t73ypjije.yn8krm.bid\", \"52uo5k3t73ypjije.yrd7v5.bid\", \"52uo5k3t73ypjije.yty0gm.bid\", \"52uo5k3t73ypjije.yv7l4b.top\", \"52uo5k3t73ypjije.yw4629.top\", \"52uo5k3t73ypjije.ywszbe.bid\", \"52uo5k3t73ypjije.z6a7f1.bid\", \"52uo5k3t73ypjije.z8ijgn.bid\", \"52uo5k3t73ypjije.z97f9v.bid\", \"52uo5k3t73ypjije.zclw5i.top\", \"52uo5k3t73ypjije.zcwrhe.bid\", \"52uo5k3t73ypjije.zd3p2g.top\", \"52uo5k3t73ypjije.zda7bk.top\", \"52uo5k3t73ypjije.zed84j.bid\", \"52uo5k3t73ypjije.zhvlh1.bid\", \"52uo5k3t73ypjije.zxtezv.bid\", \"52uo5k3t73ypjije.zzis8p.bid\", \"5rport45vcdef345adfkksawe.bematvocal.at\", \"6dtxgqam4crv6rr6.onion.cab\", \"6g4ds.froekuge.com\", \"74nfnjhlq45nkgws4hbdbk45wekfjhqw4talefgnv.curryfort.at\", \"88fga.ketteaero.com\", \"8b4bb47tiaolhy4uhhlfaqerg.sofarany.at\", \"94dbhbj3l4blaeyfgl7q45glbaer.giponfeste.at\", \"974gfbjhb23hbfkyfaby3byqlyuebvly5q254y.mendilobo.com\", \"9hrds.wolfcrap.at\", \"a64gfdsjhb4htbiwaysbdvukyft5q.zobodine.at\", \"aa12111.top\", \"aarnknthc.xyz\", \"abvtqhwodwjmi.work\", \"acbstypdrijslr.ru\", \"accemfsqovkd.pw\", \"acjhwpdjhlhbncf.click\", \"aechjic.pw\", \"ahsqbeospcdrngfv.info\", \"ahuqfrqk54v3vnzj.1vcxfn.bid\", \"ahuqfrqk54v3vnzj.45yu0p.bid\", \"ahuqfrqk54v3vnzj.4h16v3.top\", \"ahuqfrqk54v3vnzj.6avw2a.bid\", \"ahuqfrqk54v3vnzj.7y1266.top\", \"ahuqfrqk54v3vnzj.8kiec2.top\", \"ahuqfrqk54v3vnzj.9sfk22.bid\", \"ahuqfrqk54v3vnzj.bds4sn.top\", \"ahuqfrqk54v3vnzj.bz7k7l.top\", \"ahuqfrqk54v3vnzj.c8jxpp.top\", \"ahuqfrqk54v3vnzj.cb3pul.top\", \"ahuqfrqk54v3vnzj.dxzr2l.top\", \"ahuqfrqk54v3vnzj.ewg6uf.bid\", \"ahuqfrqk54v3vnzj.g4dc5s.bid\", \"ahuqfrqk54v3vnzj.h4lu4i.bid\", \"ahuqfrqk54v3vnzj.i81wik.bid\", \"ahuqfrqk54v3vnzj.kj3f52.bid\", \"ahuqfrqk54v3vnzj.l7g2sv.bid\", \"ahuqfrqk54v3vnzj.n3oyw7.bid\", \"ahuqfrqk54v3vnzj.roep3o.top\", \"ahuqfrqk54v3vnzj.sg9lxh.bid\", \"ahuqfrqk54v3vnzj.tjubo1.top\", \"ahuqfrqk54v3vnzj.u9fcji.bid\", \"ahuqfrqk54v3vnzj.uzeb6r.bid\", \"ahuqfrqk54v3vnzj.v5neyw.bid\", \"ahuqfrqk54v3vnzj.vgxcci.top\", \"ahuqfrqk54v3vnzj.x90yk1.bid\", \"ahuqfrqk54v3vnzj.xs2xeh.bid\", \"ahuqfrqk54v3vnzj.zn90h4.bid\", \"ampjsppmftmfdblpt.info\", \"anbqjdoyw6wkmpeu.oldtrees.at\", \"applesnoutsthings.bid\", \"aqmip.fr\", \"arddxjkwrp.xyz\", \"as3ws.fopyirr.com\", \"avsxrcoq2q5fgrw2.13inb1.top\", \"avsxrcoq2q5fgrw2.17vj7b.top\", \"avsxrcoq2q5fgrw2.199ovv.top\", \"avsxrcoq2q5fgrw2.1gtx3p.top\", \"avsxrcoq2q5fgrw2.1mwipu.top\", \"avsxrcoq2q5fgrw2.1nsnuh.top\", \"avsxrcoq2q5fgrw2.2wfe60.top\", \"avsxrcoq2q5fgrw2.5m2n7x.top\", \"avsxrcoq2q5fgrw2.5s96fr.top\", \"avsxrcoq2q5fgrw2.79j8fm.top\", \"avsxrcoq2q5fgrw2.8l4jpw.top\", \"avsxrcoq2q5fgrw2.9c431m.bid\", \"avsxrcoq2q5fgrw2.arpbxw.top\", \"avsxrcoq2q5fgrw2.ayjy5d.top\", \"avsxrcoq2q5fgrw2.dgjpgy.top\", \"avsxrcoq2q5fgrw2.et7izd.top\", \"avsxrcoq2q5fgrw2.ewg6uf.bid\", \"avsxrcoq2q5fgrw2.h44l3d.bid\", \"avsxrcoq2q5fgrw2.ihuk7s.top\", \"avsxrcoq2q5fgrw2.j4cser.bid\", \"avsxrcoq2q5fgrw2.lbxvhk.top\", \"avsxrcoq2q5fgrw2.lxvmhm.top\", \"avsxrcoq2q5fgrw2.nbz4dn.top\", \"avsxrcoq2q5fgrw2.p93w1x.bid\", \"avsxrcoq2q5fgrw2.r1sjrp.top\", \"avsxrcoq2q5fgrw2.rys9pj.top\", \"avsxrcoq2q5fgrw2.tjdup0.top\", \"avsxrcoq2q5fgrw2.uunmkj.top\", \"avsxrcoq2q5fgrw2.vestjb.top\", \"avsxrcoq2q5fgrw2.vofy7f.top\", \"avsxrcoq2q5fgrw2.w22p3v.top\", \"avsxrcoq2q5fgrw2.w5hilw.top\", \"avsxrcoq2q5fgrw2.wgx4go.top\", \"avsxrcoq2q5fgrw2.y1fx4w.top\", \"avsxrcoq2q5fgrw2.y9kxz2.bid\", \"avsxrcoq2q5fgrw2.yr1h37.top\", \"avsxrcoq2q5fgrw2.z0mkoc.top\", \"avsxrcoq2q5fgrw2.zi842m.bid\", \"avxdypmdbo.pw\", \"axnemuevqnstqyflb.work\", \"b4youfred5485jgsa3453f.italazudda.com\", \"barjhxoye.info\", \"bciuemfaapyf.biz\", \"bddadevlpkwrrmud.xyz\", \"bfd45u8ehdklrfqwlhbhjbgqw.niptana.at\", \"bkdjvmmkwgkvgw.su\", \"blxbymhjva.info\", \"bnjhx.eu\", \"bqbbsfdw.be\", \"bqukfjfv.org\", \"bwcfinnt.work\", \"bwpegsfa.info\", \"bxlrywuuobje.pw\", \"cdxbbpngq.pw\", \"cerberhhyed5frqa.305iot.top\", \"cerberhhyed5frqa.305iot.win\", \"cerberhhyed5frqa.45gf4t.win\", \"cerberhhyed5frqa.45kgok.win\", \"cerberhhyed5frqa.5kti58.win\", \"cerberhhyed5frqa.ad34ft.win\", \"cerberhhyed5frqa.adevf4.win\", \"cerberhhyed5frqa.alri58.win\", \"cerberhhyed5frqa.as13fd.win\", \"cerberhhyed5frqa.asxce4.win\", \"cerberhhyed5frqa.azlto5.win\", \"cerberhhyed5frqa.cmr95i.top\", \"cerberhhyed5frqa.cmr95i.win\", \"cerberhhyed5frqa.cmti5o.win\", \"cerberhhyed5frqa.cneo59.top\", \"cerberhhyed5frqa.cneo59.win\", \"cerberhhyed5frqa.dk59jg.win\", \"cerberhhyed5frqa.dkrti5.top\", \"cerberhhyed5frqa.er48rt.win\", \"cerberhhyed5frqa.fgfid6.win\", \"cerberhhyed5frqa.fkr84i.win\", \"cerberhhyed5frqa.fkri48.win\", \"cerberhhyed5frqa.gkfit9.top\", \"cerberhhyed5frqa.gkfit9.win\", \"cerberhhyed5frqa.kipfgs65s.com\", \"cerberhhyed5frqa.lfotp5.top\", \"cerberhhyed5frqa.li4loi.win\", \"cerberhhyed5frqa.lib2vi.win\", \"cerberhhyed5frqa.m5fgoi.win\", \"cerberhhyed5frqa.m5gid4.top\", \"cerberhhyed5frqa.m5gid4.win\", \"cerberhhyed5frqa.m5gips.win\", \"cerberhhyed5frqa.mix3hi.win\", \"cerberhhyed5frqa.moneu5.win\", \"cerberhhyed5frqa.oneswi.win\", \"cerberhhyed5frqa.qor499.top\", \"cerberhhyed5frqa.raress.win\", \"cerberhhyed5frqa.sdfiso.win\", \"cerberhhyed5frqa.sims6n.win\", \"cerberhhyed5frqa.ti4wic.win\", \"cerberhhyed5frqa.to6maq.win\", \"cerberhhyed5frqa.vmfu48.win\", \"cerberhhyed5frqa.we34re.top\", \"cerberhhyed5frqa.we34re.win\", \"cerberhhyed5frqa.werti4.win\", \"cerberhhyed5frqa.wet4io.win\", \"cerberhhyed5frqa.wewiso.win\", \"cerberhhyed5frqa.workju.win\", \"cerberhhyed5frqa.xltnet.win\", \"cerberhhyed5frqa.xmfhr6.win\", \"cerberhhyed5frqa.xmfir0.top\", \"cerberhhyed5frqa.xmfir0.win\", \"cerberhhyed5frqa.xmfjr7.top\", \"cerberhhyed5frqa.xmfkr8.top\", \"cerberhhyed5frqa.xmfu59.win\", \"cerberhhyed5frqa.xo59ok.win\", \"cerberhhyed5frqa.xtrvb4.win\", \"cerberhhyed5frqa.zgf48j.win\", \"chromebewfk.top\", \"citointechnologiesalefor.top\", \"clhyelmwnuqhigecp.pw\", \"corefitness.info\", \"cpawdrtxfjkwrkkl.pw\", \"cpyrltela.pw\", \"crosseunity.top\", \"cudcfybkk.pw\", \"cwprfpjtmjb.biz\", \"cxlgwofgrjfoaa.info\", \"d34fa.lasmeio.com\", \"dd7bsndhr45nfksdnkferfer.javakale.at\", \"de2nuvwegoo32oqv.torbook.li\", \"de2nuvwegoo32oqv.tordrims.li\", \"de2nuvwegoo32oqv.torfigth.li\", \"de2nuvwegoo32oqv.tormilki.li\", \"de2nuvwegoo32oqv.torminimals.li\", \"de2nuvwegoo32oqv.torspaces.li\", \"de2nuvwegoo32oqv.tortelevision.li\", \"de2nuvwegoo32oqv.tortodorf.li\", \"de2nuvwegoo32oqv.torworks.li\", \"dkoipg.pw\", \"dltvwp.it\", \"dmwajvm.fr\", \"dolfexalto.com\", \"domainstop.top\", \"dqtfhkgskushlum.org\", \"dtojlhpasjk.pw\", \"dvmbtgoobxcc.pw\", \"dwytqrgblrynsgtew.org\", \"dyoravdkiavfkbkx.pw\", \"earthspiruitr.top\", \"eaxpifdtwsv.biz\", \"ecjfdaqmmyusxntwl.work\", \"egerdpkvutvodmtsy.pw\", \"egovrxvuspxck.be\", \"eoalsoub.pw\", \"eppilxqwyqdhmpdsn.pw\", \"eqtrtdavtnr.pw\", \"euduudaehipk.pw\", \"exnqhgk.xyz\", \"eypdxikxsufj.pw\", \"eywlmqugxx.info\", \"f4dsbjhb45wfiuqeib4fkqeg.meccaledgy.at\", \"f5xraa2y2ybtrefz.onion.to\", \"fdehgchykmiqwdg.info\", \"ffoqr3ug7m726zou.04hyxg.top\", \"ffoqr3ug7m726zou.0v7hry.bid\", \"ffoqr3ug7m726zou.1321z6.top\", \"ffoqr3ug7m726zou.13inb1.top\", \"ffoqr3ug7m726zou.14gmtu.top\", \"ffoqr3ug7m726zou.17vj7b.top\", \"ffoqr3ug7m726zou.1967qy.top\", \"ffoqr3ug7m726zou.1feasu.top\", \"ffoqr3ug7m726zou.1gtx3p.top\", \"ffoqr3ug7m726zou.1mwipu.top\", \"ffoqr3ug7m726zou.1nsnuh.top\", \"ffoqr3ug7m726zou.2fu7bc.top\", \"ffoqr3ug7m726zou.2msuuj.top\", \"ffoqr3ug7m726zou.2rl0pv.top\", \"ffoqr3ug7m726zou.4tkb0d.top\", \"ffoqr3ug7m726zou.5e4u7d.bid\", \"ffoqr3ug7m726zou.5hmjh7.bid\", \"ffoqr3ug7m726zou.5m2n7x.top\", \"ffoqr3ug7m726zou.735giv.top\", \"ffoqr3ug7m726zou.8uvtsg.top\", \"ffoqr3ug7m726zou.9yim37.top\", \"ffoqr3ug7m726zou.ac7zvz.top\", \"ffoqr3ug7m726zou.b31wkh.bid\", \"ffoqr3ug7m726zou.b4abvx.top\", \"ffoqr3ug7m726zou.bd7tlu.top\", \"ffoqr3ug7m726zou.bdlvdy.top\", \"ffoqr3ug7m726zou.bpuhab.top\", \"ffoqr3ug7m726zou.bwei9h.top\", \"ffoqr3ug7m726zou.ca15sj.top\", \"ffoqr3ug7m726zou.do9wwg.top\", \"ffoqr3ug7m726zou.e1e7w2.top\", \"ffoqr3ug7m726zou.efebgv.top\", \"ffoqr3ug7m726zou.f5x6ws.top\", \"ffoqr3ug7m726zou.ffsm1a.bid\", \"ffoqr3ug7m726zou.gwz8gh.top\", \"ffoqr3ug7m726zou.hajw7w.bid\", \"ffoqr3ug7m726zou.hpwom3.top\", \"ffoqr3ug7m726zou.hy6dxo.bid\", \"ffoqr3ug7m726zou.hzrekn.top\", \"ffoqr3ug7m726zou.i4ucg2.bid\", \"ffoqr3ug7m726zou.iocvou.top\", \"ffoqr3ug7m726zou.jye7lt.top\", \"ffoqr3ug7m726zou.kfymbh.top\", \"ffoqr3ug7m726zou.l4dlll.bid\", \"ffoqr3ug7m726zou.l6r7i9.top\", \"ffoqr3ug7m726zou.lc1xfc.top\", \"ffoqr3ug7m726zou.le6611.bid\", \"ffoqr3ug7m726zou.lruwth.top\", \"ffoqr3ug7m726zou.m3cvi8.top\", \"ffoqr3ug7m726zou.momg04.top\", \"ffoqr3ug7m726zou.ndnmuk.top\", \"ffoqr3ug7m726zou.ptnbfm.top\", \"ffoqr3ug7m726zou.rssh3l.bid\", \"ffoqr3ug7m726zou.rxmbsm.top\", \"ffoqr3ug7m726zou.rzt69n.top\", \"ffoqr3ug7m726zou.rzvhne.top\", \"ffoqr3ug7m726zou.s611js.top\", \"ffoqr3ug7m726zou.sg9lxh.bid\", \"ffoqr3ug7m726zou.smd95z.top\", \"ffoqr3ug7m726zou.tsrwj3.top\", \"ffoqr3ug7m726zou.tx0igu.bid\", \"ffoqr3ug7m726zou.u9fcji.bid\", \"ffoqr3ug7m726zou.ud9z0v.top\", \"ffoqr3ug7m726zou.ukswcu.bid\", \"ffoqr3ug7m726zou.umvv28.top\", \"ffoqr3ug7m726zou.utebcd.top\", \"ffoqr3ug7m726zou.v0xn1i.bid\", \"ffoqr3ug7m726zou.vjso7r.top\", \"ffoqr3ug7m726zou.w22p3v.top\", \"ffoqr3ug7m726zou.w67y8u.bid\", \"ffoqr3ug7m726zou.wf912u.bid\", \"ffoqr3ug7m726zou.wmvsh0.top\", \"ffoqr3ug7m726zou.wwa4tu.top\", \"ffoqr3ug7m726zou.wx2n44.top\", \"ffoqr3ug7m726zou.x8p2m7.bid\", \"ffoqr3ug7m726zou.x9ap4h.top\", \"ffoqr3ug7m726zou.xe1ws1.top\", \"ffoqr3ug7m726zou.y9kxz2.bid\", \"ffoqr3ug7m726zou.yjo0z9.top\", \"ffoqr3ug7m726zou.yur4j5.top\", \"ffoqr3ug7m726zou.yv3uwa.bid\", \"ffoqr3ug7m726zou.zee0xr.top\", \"ffoqr3ug7m726zou.zio9yg.bid\", \"ffoqr3ug7m726zou.zjfbxy.top\", \"ffoqr3ug7m726zou.zkxb17.top\", \"ffoqr3ug7m726zou.zn90h4.bid\", \"ffoqr3ug7m726zou.zpjpsf.top\", \"ffoqr3ug7m726zou.zu3fzc.bid\", \"fhvjsmtkirihxh.xyz\", \"fitga.ru\", \"fmirgordkhig.xyz\", \"fnarsipfqe.pw\", \"fnjyygovdjyemga.xyz\", \"fnmi62725zfti2vy.13inb1.top\", \"fnmi62725zfti2vy.17vj7b.top\", \"fnmi62725zfti2vy.1gtx3p.top\", \"fnmi62725zfti2vy.o08ra6.top\", \"fnmi62725zfti2vy.p9wol3.top\", \"fnmi62725zfti2vy.vwgxhm.bid\", \"fooplodanx.top\", \"fpashgkepwtoqdjg.pw\", \"fqoapcjolfwwenqx.pw\", \"fqtdrnqmeofknd.biz\", \"ftoxmpdipwobp4qy.10nzk9.top\", \"ftoxmpdipwobp4qy.17vj7b.top\", \"ftoxmpdipwobp4qy.199ovv.top\", \"ftoxmpdipwobp4qy.1gtx3p.top\", \"ftoxmpdipwobp4qy.1nsnuh.top\", \"ftoxmpdipwobp4qy.7pnxn9.top\", \"ftoxmpdipwobp4qy.lxvmhm.top\", \"fuuasvhpsvuihlnje.pw\", \"fuuwnsv.pw\", \"fyqtguo.biz\", \"g4dhhg53jsdjnnkjwjrfyiouh3o4u4th.vinerteen.com\", \"gccxqpuuylioxoip.pw\", \"gfcuxnaek.ru\", \"gfkuwflbhsjdabnu4nfukerfqwlfwr4rw.ringbalor.com\", \"gfwncoyhbdvggns.pw\", \"gguaxufrt.pw\", \"gitybdjgbxd.nl\", \"glhxgchhfemcjgr.pw\", \"gnsquwmgukkpgpt.pw\", \"govementruystd.top\", \"gsebqsi.ru\", \"gsmdqrmqddqtuv.xyz\", \"gvludcvhcrjwmgq.in\", \"gwbak.nickymaru.com\", \"gwe32fdr74bhfsyujb34gfszfv.zatcurr.com\", \"h3ds4.maconslab.com\", \"h54dc.leverdaze.at\", \"h5nuwefkuh134ljngkasdbasfg.corolbugan.com\", \"hjhqmbxyinislkkt.11bwgu.top\", \"hjhqmbxyinislkkt.127axt.top\", \"hjhqmbxyinislkkt.12bsy8.top\", \"hjhqmbxyinislkkt.12bxp9.top\", \"hjhqmbxyinislkkt.12ct4c.top\", \"hjhqmbxyinislkkt.12gsjz.top\", \"hjhqmbxyinislkkt.12m58x.top\", \"hjhqmbxyinislkkt.12zucf.top\", \"hjhqmbxyinislkkt.13bcem.top\", \"hjhqmbxyinislkkt.13eymq.top\", \"hjhqmbxyinislkkt.13fmby.top\", \"hjhqmbxyinislkkt.13khiv.top\", \"hjhqmbxyinislkkt.13kn4l.top\", \"hjhqmbxyinislkkt.13qgdd.top\", \"hjhqmbxyinislkkt.13ydzv.top\", \"hjhqmbxyinislkkt.142djp.top\", \"hjhqmbxyinislkkt.14dr1s.top\", \"hjhqmbxyinislkkt.14klmz.top\", \"hjhqmbxyinislkkt.14o2wp.top\", \"hjhqmbxyinislkkt.14stvt.top\", \"hjhqmbxyinislkkt.14yppf.top\", \"hjhqmbxyinislkkt.15e8hv.top\", \"hjhqmbxyinislkkt.15mwt4.top\", \"hjhqmbxyinislkkt.15u3kg.top\", \"hjhqmbxyinislkkt.16ke1t.top\", \"hjhqmbxyinislkkt.16l1zt.top\", \"hjhqmbxyinislkkt.17kc8y.top\", \"hjhqmbxyinislkkt.17rm9b.top\", \"hjhqmbxyinislkkt.18f5bw.top\", \"hjhqmbxyinislkkt.18lmhb.top\", \"hjhqmbxyinislkkt.18nepv.top\", \"hjhqmbxyinislkkt.18yzmj.top\", \"hjhqmbxyinislkkt.18zrup.top\", \"hjhqmbxyinislkkt.19b6nk.top\", \"hjhqmbxyinislkkt.19hj4f.top\", \"hjhqmbxyinislkkt.19s7gy.top\", \"hjhqmbxyinislkkt.19xdpm.top\", \"hjhqmbxyinislkkt.19xvyd.top\", \"hjhqmbxyinislkkt.1a2xx3.top\", \"hjhqmbxyinislkkt.1a8u1r.top\", \"hjhqmbxyinislkkt.1aajb7.top\", \"hjhqmbxyinislkkt.1aamtz.top\", \"hjhqmbxyinislkkt.1accfa.top\", \"hjhqmbxyinislkkt.1acfka.top\", \"hjhqmbxyinislkkt.1adh2r.top\", \"hjhqmbxyinislkkt.1aq4sz.top\", \"hjhqmbxyinislkkt.1aqq5k.top\", \"hjhqmbxyinislkkt.1b8tmn.top\", \"hjhqmbxyinislkkt.1bas8q.top\", \"hjhqmbxyinislkkt.1bcnad.top\", \"hjhqmbxyinislkkt.1bcxcs.top\", \"hjhqmbxyinislkkt.1bu9xu.top\", \"hjhqmbxyinislkkt.1c1ajf.top\", \"hjhqmbxyinislkkt.1cdqfv.top\", \"hjhqmbxyinislkkt.1cnkik.top\", \"hjhqmbxyinislkkt.1csesc.top\", \"hjhqmbxyinislkkt.1dq6nd.top\", \"hjhqmbxyinislkkt.1dvqvh.top\", \"hjhqmbxyinislkkt.1e47tj.top\", \"hjhqmbxyinislkkt.1eagrj.top\", \"hjhqmbxyinislkkt.1eeyaj.top\", \"hjhqmbxyinislkkt.1efxa8.top\", \"hjhqmbxyinislkkt.1fgsmc.top\", \"hjhqmbxyinislkkt.1fnjrj.top\", \"hjhqmbxyinislkkt.1fttxm.top\", \"hjhqmbxyinislkkt.1fy93v.top\", \"hjhqmbxyinislkkt.1fygsg.top\", \"hjhqmbxyinislkkt.1fzjn3.top\", \"hjhqmbxyinislkkt.1fzz7a.top\", \"hjhqmbxyinislkkt.1gjpzp.top\", \"hjhqmbxyinislkkt.1gqrpq.top\", \"hjhqmbxyinislkkt.1gredn.top\", \"hjhqmbxyinislkkt.1grvue.top\", \"hjhqmbxyinislkkt.1gswwp.top\", \"hjhqmbxyinislkkt.1gu5um.top\", \"hjhqmbxyinislkkt.1gunao.top\", \"hjhqmbxyinislkkt.1gvyo8.top\", \"hjhqmbxyinislkkt.1gxfxt.top\", \"hjhqmbxyinislkkt.1gzjuc.top\", \"hjhqmbxyinislkkt.1hapca.top\", \"hjhqmbxyinislkkt.1j43kf.top\", \"hjhqmbxyinislkkt.1jmip6.top\", \"hjhqmbxyinislkkt.1jnhdc.top\", \"hjhqmbxyinislkkt.1jwuaa.top\", \"hjhqmbxyinislkkt.1k6bas.top\", \"hjhqmbxyinislkkt.1kge5a.top\", \"hjhqmbxyinislkkt.1khwro.top\", \"hjhqmbxyinislkkt.1kjhhf.top\", \"hjhqmbxyinislkkt.1kraqn.top\", \"hjhqmbxyinislkkt.1kw51p.top\", \"hjhqmbxyinislkkt.1lqrja.top\", \"hjhqmbxyinislkkt.1ltyev.top\", \"hjhqmbxyinislkkt.1mat7v.top\", \"hjhqmbxyinislkkt.1mee2x.top\", \"hjhqmbxyinislkkt.1mqvsc.top\", \"hjhqmbxyinislkkt.1mswjm.top\", \"hjhqmbxyinislkkt.1mvku2.top\", \"hjhqmbxyinislkkt.1mwvgh.top\", \"hjhqmbxyinislkkt.1nm62r.top\", \"hjhqmbxyinislkkt.1npg9s.top\", \"hjhqmbxyinislkkt.1ntyds.top\", \"hjhqmbxyinislkkt.1pcvko.top\", \"hjhqmbxyinislkkt.1ppto6.top\", \"hjhqmbxyinislkkt.1pxbfh.top\", \"hjhqmbxyinislkkt.1q7pwb.top\", \"hjhqmbxyinislkkt.1qjl23.top\", \"hjhqmbxyinislkkt.1qk2un.top\", \"hjhqmbxyinislkkt.1w5iy8.top\", \"hjhqmbxyinislkkt.1xynaz.top\", \"hmndhdbscgru.pw\", \"honourableud.top\", \"hppfsslyeyseudg.biz\", \"hrfgd74nfksjdcnnklnwefvdsf.materdunst.com\", \"htankds.info\", \"hycninyxuaa.xyz\", \"i01001.dgn.vn\", \"i3ezlvkoi7fwyood.onion.to\", \"i3ezlvkoi7fwyood.tor2web.org\", \"i5ndw.titlecorta.at\", \"ibjgnqsthdyp.pw\", \"ibtfqftkgi.pw\", \"ifohvkxmyp.biz\", \"igoodsnd.wang\", \"ik4dm.mazerunci.at\", \"iqfyujpvubwawc.pw\", \"irhng84nfaslbv243ljtblwqjrb.pinnafaon.at\", \"irudhkunrlfu25fhkaqw34blr5qlby4tgq43t.orrisbirth.com\", \"iuieylpvfurcvmpk.pw\", \"jfmiondv.xyz\", \"jghbktqepe.pw\", \"jhdgh.club\", \"jhomitevd2abj3fk.onion.to\", \"juhacjacjckclqf.pw\", \"jxqdry.ru\", \"jymhmkdaxfbl.click\", \"k234s.ascotsprue.com\", \"k34ew.keyedgell.com\", \"k3cxd.pileanoted.com\", \"k47d3.proporr.com\", \"k4restportgonst34d23r.oftpony.at\", \"kbv5s.kylepasse.at\", \"kcdfajaxngiff.info\", \"kciylimohteftc.pw\", \"kh5jfnvkk5twerfnku5twuilrnglnuw45yhlw.vealsithe.com\", \"kjkwjqvqrjocpi.xyz\", \"kkd47eh4hdjshb5t.angortra.at\", \"kkr4hbwdklf234bfl84uoqleflqwrfqwuelfh.brazabaya.com\", \"kpybuhnosdrm.in\", \"kqlxtqptsmys.in\", \"ks-davis.com\", \"ktlgpiilbj.biz\", \"kwontdmplpnbl.pw\", \"kypsuw.pw\", \"l123d.feustude.at\", \"lcrdceiajmiar.org\", \"lfdachijzuwx4bc4.0ndl3j.bid\", \"lfdachijzuwx4bc4.6szfn7.top\", \"lfdachijzuwx4bc4.83zw1f.bid\", \"lfdachijzuwx4bc4.8dlgyg.bid\", \"lfdachijzuwx4bc4.af38vz.top\", \"lfdachijzuwx4bc4.ci221p.top\", \"lfdachijzuwx4bc4.djintc.bid\", \"lfdachijzuwx4bc4.e6cf2t.bid\", \"lfdachijzuwx4bc4.eujvrw.bid\", \"lfdachijzuwx4bc4.ev99l6.bid\", \"lfdachijzuwx4bc4.ex9n9v.top\", \"lfdachijzuwx4bc4.fe6cf2.top\", \"lfdachijzuwx4bc4.fwzxnb.bid\", \"lfdachijzuwx4bc4.iuzppd.top\", \"lfdachijzuwx4bc4.le2brr.bid\", \"lfdachijzuwx4bc4.m7f27y.bid\", \"lfdachijzuwx4bc4.twyjdx.bid\", \"lfdachijzuwx4bc4.tx0igu.bid\", \"lfdachijzuwx4bc4.u9fcji.bid\", \"lfdachijzuwx4bc4.vrgdrs.top\", \"lfdachijzuwx4bc4.w4629d.top\", \"lfdachijzuwx4bc4.x4tk5c.bid\", \"lfdachijzuwx4bc4.zreknv.bid\", \"lollyoff.info\", \"lookingpersonals.top\", \"lpholfnvwbukqwye.onion.cab\", \"lpholfnvwbukqwye.onion.to\", \"lrmficvqs.pw\", \"ltpwqva.xyz\", \"luvenxj.uk\", \"lvanwwbyabcfevyi.pw\", \"lyrnvane.pw\", \"macooptwafkwchtpo.pw\", \"mmhmtea.pw\", \"mphtadhci5mrdlju.onion.to\", \"mphtadhci5mrdlju.tor2web.org\", \"muuojcu.xyz\", \"mwqwverayognn.pw\", \"mxyfasm.pw\", \"mz7oyb3v32vshcvk.bidobject.li\", \"mz7oyb3v32vshcvk.getstar.li\", \"mz7oyb3v32vshcvk.torapples.li\", \"mz7oyb3v32vshcvk.torlongor.li\", \"mz7oyb3v32vshcvk.tormidle.at\", \"mz7oyb3v32vshcvk.toysworlds.at\", \"newgiftnd.wang\", \"newgiftst.top\", \"nhhyxorxbxarxe.org\", \"nikessysleys.top\", \"nlpqflkbvkdde.eu\", \"nn54djhfnrnm4dnjnerfsd.replylaten.at\", \"nnrtsdf34dsjhb23rsdf.spannflow.com\", \"nwcpgymgh.work\", \"o4dm3.leaama.at\", \"odgtnkmq.pw\", \"oehknf74ohqlfnpq9rhfgcq93g.hateflux.com\", \"ohpbdikmrrhr.pw\", \"ohplsuljopekq.biz\", \"ojmekzw4mujvqeju.bioserv.at\", \"ojmekzw4mujvqeju.dreamtest.at\", \"ojmekzw4mujvqeju.fineboy.at\", \"ojmekzw4mujvqeju.minitili.at\", \"omeaswslhgdw.xyz\", \"oqwygprskqv65j72.12kb9j.top\", \"oqwygprskqv65j72.13gpqd.top\", \"oqwygprskqv65j72.13rdvu.top\", \"oqwygprskqv65j72.14jqyo.top\", \"oqwygprskqv65j72.17q8f6.top\", \"oqwygprskqv65j72.1aj1bb.top\", \"oqwygprskqv65j72.1d88b8.top\", \"oqwygprskqv65j72.1dofqx.top\", \"oqwygprskqv65j72.1fdlhn.top\", \"oqwygprskqv65j72.1fs9pz.top\", \"oqwygprskqv65j72.1gam57.top\", \"oqwygprskqv65j72.1gqj8x.top\", \"oqwygprskqv65j72.1hbdbx.top\", \"oqwygprskqv65j72.1j1x2b.top\", \"oqwygprskqv65j72.1jquw7.top\", \"oqwygprskqv65j72.1kh9ct.top\", \"oqwygprskqv65j72.1mudaw.top\", \"oqwygprskqv65j72.1nzpby.top\", \"ozfin.ru\", \"p27dokhpz2n7nvgr.12a63k.top\", \"p27dokhpz2n7nvgr.12c8ff.top\", \"p27dokhpz2n7nvgr.12gzrv.top\", \"p27dokhpz2n7nvgr.12hxjv.top\", \"p27dokhpz2n7nvgr.12nwsv.top\", \"p27dokhpz2n7nvgr.12smak.top\", \"p27dokhpz2n7nvgr.12t3rn.top\", \"p27dokhpz2n7nvgr.12ulcz.top\", \"p27dokhpz2n7nvgr.12umzf.top\", \"p27dokhpz2n7nvgr.12uzfa.top\", \"p27dokhpz2n7nvgr.12vpkc.top\", \"p27dokhpz2n7nvgr.1321z6.top\", \"p27dokhpz2n7nvgr.133chr.top\", \"p27dokhpz2n7nvgr.135nt3.top\", \"p27dokhpz2n7nvgr.13g2v9.top\", \"p27dokhpz2n7nvgr.13gmvm.top\", \"p27dokhpz2n7nvgr.13ixv2.top\", \"p27dokhpz2n7nvgr.13upky.top\", \"p27dokhpz2n7nvgr.13upnc.top\", \"p27dokhpz2n7nvgr.13wm9b.top\", \"p27dokhpz2n7nvgr.13xwn9.top\", \"p27dokhpz2n7nvgr.14ewqv.top\", \"p27dokhpz2n7nvgr.14gmtu.top\", \"p27dokhpz2n7nvgr.14kfoz.top\", \"p27dokhpz2n7nvgr.14udep.top\", \"p27dokhpz2n7nvgr.15jznv.top\", \"p27dokhpz2n7nvgr.15l2ub.top\", \"p27dokhpz2n7nvgr.15nhsf.top\", \"p27dokhpz2n7nvgr.15oqwp.top\", \"p27dokhpz2n7nvgr.15rnwa.top\", \"p27dokhpz2n7nvgr.15wmdx.top\", \"p27dokhpz2n7nvgr.168w5y.top\", \"p27dokhpz2n7nvgr.16ay2s.top\", \"p27dokhpz2n7nvgr.16bwhs.top\", \"p27dokhpz2n7nvgr.16fohp.top\", \"p27dokhpz2n7nvgr.16nxpn.top\", \"p27dokhpz2n7nvgr.16qpet.top\", \"p27dokhpz2n7nvgr.173w9w.top\", \"p27dokhpz2n7nvgr.17g6gc.top\", \"p27dokhpz2n7nvgr.17gvad.top\", \"p27dokhpz2n7nvgr.17m14u.top\", \"p27dokhpz2n7nvgr.17ryrs.top\", \"p27dokhpz2n7nvgr.17u2yg.top\", \"p27dokhpz2n7nvgr.18dawg.top\", \"p27dokhpz2n7nvgr.18kkhl.top\", \"p27dokhpz2n7nvgr.18kmtt.top\", \"p27dokhpz2n7nvgr.195heb.top\", \"p27dokhpz2n7nvgr.1967qy.top\", \"p27dokhpz2n7nvgr.1a7ivn.top\", \"p27dokhpz2n7nvgr.1a7wnt.top\", \"p27dokhpz2n7nvgr.1aghep.top\", \"p27dokhpz2n7nvgr.1ajohk.top\", \"p27dokhpz2n7nvgr.1apgrn.top\", \"p27dokhpz2n7nvgr.1apkjn.top\", \"p27dokhpz2n7nvgr.1aweql.top\", \"p27dokhpz2n7nvgr.1axzcw.top\", \"p27dokhpz2n7nvgr.1azkux.top\", \"p27dokhpz2n7nvgr.1b3qjy.top\", \"p27dokhpz2n7nvgr.1bj4k9.top\", \"p27dokhpz2n7nvgr.1bniyw.top\", \"p27dokhpz2n7nvgr.1bvadx.top\", \"p27dokhpz2n7nvgr.1bywu2.top\", \"p27dokhpz2n7nvgr.1bzolk.top\", \"p27dokhpz2n7nvgr.1cauz3.top\", \"p27dokhpz2n7nvgr.1cb19l.top\", \"p27dokhpz2n7nvgr.1cbcpy.top\", \"p27dokhpz2n7nvgr.1cewld.top\", \"p27dokhpz2n7nvgr.1cggqc.top\", \"p27dokhpz2n7nvgr.1cglxz.top\", \"p27dokhpz2n7nvgr.1chy1m.top\", \"p27dokhpz2n7nvgr.1cknbd.top\", \"p27dokhpz2n7nvgr.1cpb4z.top\", \"p27dokhpz2n7nvgr.1cpy1q.top\", \"p27dokhpz2n7nvgr.1cq7gd.top\", \"p27dokhpz2n7nvgr.1cvmb4.top\", \"p27dokhpz2n7nvgr.1cw65b.top\", \"p27dokhpz2n7nvgr.1czh7o.top\", \"p27dokhpz2n7nvgr.1d8d9w.top\", \"p27dokhpz2n7nvgr.1d8m97.top\", \"p27dokhpz2n7nvgr.1daq6h.top\", \"p27dokhpz2n7nvgr.1dlcbk.top\", \"p27dokhpz2n7nvgr.1dp6un.top\", \"p27dokhpz2n7nvgr.1dsdm4.top\", \"p27dokhpz2n7nvgr.1dyzdh.top\", \"p27dokhpz2n7nvgr.1dz7gk.top\", \"p27dokhpz2n7nvgr.1ebvqb.top\", \"p27dokhpz2n7nvgr.1eeb86.top\", \"p27dokhpz2n7nvgr.1em2j4.top\", \"p27dokhpz2n7nvgr.1enbyr.top\", \"p27dokhpz2n7nvgr.1evjph.top\", \"p27dokhpz2n7nvgr.1fel3k.top\", \"p27dokhpz2n7nvgr.1fgywm.top\", \"p27dokhpz2n7nvgr.1fqwek.top\", \"p27dokhpz2n7nvgr.1fu8p3.top\", \"p27dokhpz2n7nvgr.1gnlsi.top\", \"p27dokhpz2n7nvgr.1gqqsc.top\", \"p27dokhpz2n7nvgr.1gvql3.top\", \"p27dokhpz2n7nvgr.1gy9bo.top\", \"p27dokhpz2n7nvgr.1h23cc.top\", \"p27dokhpz2n7nvgr.1hkjl3.top\", \"p27dokhpz2n7nvgr.1hpvzl.top\", \"p27dokhpz2n7nvgr.1hw36d.top\", \"p27dokhpz2n7nvgr.1jemdr.top\", \"p27dokhpz2n7nvgr.1jh5kv.top\", \"p27dokhpz2n7nvgr.1jhnvt.top\", \"p27dokhpz2n7nvgr.1jpb8w.top\", \"p27dokhpz2n7nvgr.1js3tl.top\", \"p27dokhpz2n7nvgr.1jw2lx.top\", \"p27dokhpz2n7nvgr.1jyhqc.top\", \"p27dokhpz2n7nvgr.1jzmjr.top\", \"p27dokhpz2n7nvgr.1kja1j.top\", \"p27dokhpz2n7nvgr.1kq4l8.top\", \"p27dokhpz2n7nvgr.1ktjse.top\", \"p27dokhpz2n7nvgr.1kyjw7.top\", \"p27dokhpz2n7nvgr.1l4zyd.top\", \"p27dokhpz2n7nvgr.1lcteo.top\", \"p27dokhpz2n7nvgr.1lfyy4.top\", \"p27dokhpz2n7nvgr.1lt2pn.top\", \"p27dokhpz2n7nvgr.1m3xsy.top\", \"p27dokhpz2n7nvgr.1mfakx.top\", \"p27dokhpz2n7nvgr.1mfdt8.top\", \"p27dokhpz2n7nvgr.1mir1h.top\", \"p27dokhpz2n7nvgr.1ms2rx.top\", \"p27dokhpz2n7nvgr.1mwipu.top\", \"p27dokhpz2n7nvgr.1nhkou.top\", \"p27dokhpz2n7nvgr.1nmrtq.top\", \"p27dokhpz2n7nvgr.1nprob.top\", \"p27dokhpz2n7nvgr.1p5fwl.top\", \"p27dokhpz2n7nvgr.1pbfky.top\", \"p27dokhpz2n7nvgr.1pbu64.top\", \"p27dokhpz2n7nvgr.1pglcs.top\", \"p27dokhpz2n7nvgr.1plugt.top\", \"p27dokhpz2n7nvgr.1psts4.top\", \"p27dokhpz2n7nvgr.1pymg3.top\", \"p27dokhpz2n7nvgr.1vjnyh.top\", \"p27dokhpz2n7nvgr.1wmvk2.top\", \"p54dhkus4tlkfashdb6vjetgsdfg.greetingshere.at\", \"pagaldaily.com\", \"pdlbtnfhtoxghb.org\", \"pe2cku7pebkpgeko.13inb1.top\", \"pe2cku7pebkpgeko.199ovv.top\", \"pe2cku7pebkpgeko.1cb19l.top\", \"pe2cku7pebkpgeko.1gtx3p.top\", \"pe2cku7pebkpgeko.1mwipu.top\", \"pe2cku7pebkpgeko.1plugt.top\", \"pe2cku7pebkpgeko.1pr21c.top\", \"pe2cku7pebkpgeko.582h0n.top\", \"pe2cku7pebkpgeko.5hmjh7.bid\", \"pe2cku7pebkpgeko.ahovbr.top\", \"pe2cku7pebkpgeko.bw9e2z.top\", \"pe2cku7pebkpgeko.dj68hn.top\", \"pe2cku7pebkpgeko.hclz73.top\", \"pe2cku7pebkpgeko.kwrd4f.bid\", \"pe2cku7pebkpgeko.p93w1x.bid\", \"pe2cku7pebkpgeko.pkx86a.top\", \"pe2cku7pebkpgeko.prbuoi.top\", \"pe2cku7pebkpgeko.r1sjrp.top\", \"pe2cku7pebkpgeko.reu88i.top\", \"pe2cku7pebkpgeko.rjf9yn.top\", \"pe2cku7pebkpgeko.tsrwj3.top\", \"pe2cku7pebkpgeko.ttx0ig.top\", \"pe2cku7pebkpgeko.utebcd.top\", \"pe2cku7pebkpgeko.va3ibn.top\", \"pe2cku7pebkpgeko.vfe2f1.top\", \"pe2cku7pebkpgeko.yjo0z9.top\", \"pe2cku7pebkpgeko.z5xfkc.top\", \"pennysgoods.top\", \"plfbvdrpvsm.pw\", \"pmenboeqhyrpvomq.0nyi6l.bid\", \"pmenboeqhyrpvomq.0vgu64.top\", \"pmenboeqhyrpvomq.2agglf.top\", \"pmenboeqhyrpvomq.4pzclh.top\", \"pmenboeqhyrpvomq.58na23.top\", \"pmenboeqhyrpvomq.5b1s82.top\", \"pmenboeqhyrpvomq.7s0g3v.top\", \"pmenboeqhyrpvomq.89m6y8.bid\", \"pmenboeqhyrpvomq.8kcfnk.bid\", \"pmenboeqhyrpvomq.9ildst.top\", \"pmenboeqhyrpvomq.9nkxd3.top\", \"pmenboeqhyrpvomq.a4coac.top\", \"pmenboeqhyrpvomq.afteghonte.lol\", \"pmenboeqhyrpvomq.as5su5.top\", \"pmenboeqhyrpvomq.asxjdp.top\", \"pmenboeqhyrpvomq.azwsxe.top\", \"pmenboeqhyrpvomq.b7mciu.top\", \"pmenboeqhyrpvomq.bnctf6.top\", \"pmenboeqhyrpvomq.cmri58.top\", \"pmenboeqhyrpvomq.e6in0v.top\", \"pmenboeqhyrpvomq.enanhb.bid\", \"pmenboeqhyrpvomq.factordo.site\", \"pmenboeqhyrpvomq.fm0cga.top\", \"pmenboeqhyrpvomq.g0ots2.top\", \"pmenboeqhyrpvomq.gletterstan.trade\", \"pmenboeqhyrpvomq.gnuvaw.bid\", \"pmenboeqhyrpvomq.hasterlyston.cloud\", \"pmenboeqhyrpvomq.hwh75t.top\", \"pmenboeqhyrpvomq.ibngww.top\", \"pmenboeqhyrpvomq.k7oud1.top\", \"pmenboeqhyrpvomq.ka0te8.top\", \"pmenboeqhyrpvomq.kswcuk.top\", \"pmenboeqhyrpvomq.li4loi.top\", \"pmenboeqhyrpvomq.loopsay.link\", \"pmenboeqhyrpvomq.m54tkp.bid\", \"pmenboeqhyrpvomq.mtxtul.top\", \"pmenboeqhyrpvomq.n41n1a.top\", \"pmenboeqhyrpvomq.n80yab.top\", \"pmenboeqhyrpvomq.nh47ri.bid\", \"pmenboeqhyrpvomq.o08a6d.top\", \"pmenboeqhyrpvomq.o8hpwj.top\", \"pmenboeqhyrpvomq.p8rruv.top\", \"pmenboeqhyrpvomq.pap44w.top\", \"pmenboeqhyrpvomq.paypoints.red\", \"pmenboeqhyrpvomq.r21wmw.top\", \"pmenboeqhyrpvomq.rnkj09.top\", \"pmenboeqhyrpvomq.s71vsc.top\", \"pmenboeqhyrpvomq.self56.top\", \"pmenboeqhyrpvomq.shutlazy.casa\", \"pmenboeqhyrpvomq.swissprogramms.bid\", \"pmenboeqhyrpvomq.t4hvl4.bid\", \"pmenboeqhyrpvomq.thyx30.top\", \"pmenboeqhyrpvomq.txszfs.top\", \"pmenboeqhyrpvomq.v11z5e.top\", \"pmenboeqhyrpvomq.viceled.pw\", \"pmenboeqhyrpvomq.vkm4l6.top\", \"pmenboeqhyrpvomq.wn4h1k.top\", \"pmenboeqhyrpvomq.wrd4fo.top\", \"pmenboeqhyrpvomq.x1kofw.top\", \"pmenboeqhyrpvomq.xneyvm.top\", \"pmenboeqhyrpvomq.xx6jck.top\", \"pmenboeqhyrpvomq.y5j7e6.top\", \"pmenboeqhyrpvomq.y7fjr4.bid\", \"pmenboeqhyrpvomq.yw4629.top\", \"pnyviolg.eu\", \"po4dbsjbneljhrlbvaueqrgveatv.bonmawp.at\", \"poimoiyreque5.pw\", \"polaerunity.top\", \"ponmaredimare.top\", \"pornohd24.com\", \"preeqlultgfifg.pw\", \"prest54538hnksjn4kjfwdbhwere.hotchunman.com\", \"pts764gt354fder34fsqw45gdfsavadfgsfg.kraskula.com\", \"pvwinlrmwvccuo.eu\", \"qbqrfyeqqvcvv.pw\", \"qcwbrevxrotoepsp.pw\", \"qdesslfdcmd.pw\", \"qdvkdyvrtpjc.pw\", \"qfjhpgbefuhenjp7.1225wj.top\", \"qfjhpgbefuhenjp7.12efwa.top\", \"qfjhpgbefuhenjp7.12f53x.top\", \"qfjhpgbefuhenjp7.12u5fl.top\", \"qfjhpgbefuhenjp7.13iuvw.top\", \"qfjhpgbefuhenjp7.143kzi.top\", \"qfjhpgbefuhenjp7.158ugp.top\", \"qfjhpgbefuhenjp7.16g9ub.top\", \"qfjhpgbefuhenjp7.17cwdi.top\", \"qfjhpgbefuhenjp7.17ipn9.top\", \"qfjhpgbefuhenjp7.17xukb.top\", \"qfjhpgbefuhenjp7.18dwag.top\", \"qfjhpgbefuhenjp7.18ggbf.top\", \"qfjhpgbefuhenjp7.18rkju.top\", \"qfjhpgbefuhenjp7.19ckzf.top\", \"qfjhpgbefuhenjp7.1a2jzy.top\", \"qfjhpgbefuhenjp7.1cosak.top\", \"qfjhpgbefuhenjp7.1e1jbc.top\", \"qfjhpgbefuhenjp7.1e1y8p.top\", \"qfjhpgbefuhenjp7.1fcfjn.top\", \"qfjhpgbefuhenjp7.1jfjhb.top\", \"qfjhpgbefuhenjp7.1jrkyn.top\", \"qfjhpgbefuhenjp7.1mkwry.top\", \"qfjhpgbefuhenjp7.1mnsg6.top\", \"qfuxosx.eu\", \"qlwnvdjwro.pw\", \"qqonof.info\", \"qqtphtlhny.pw\", \"qsbfwgtedexirbyoq.pw\", \"qvdgqayo.pw\", \"rastypasty34.top\", \"rbg4hfbilrf7to452p89hrfq.boonmower.com\", \"rbwubtpsyokqn.info\", \"real346real.top\", \"remoteunityrety.top\", \"renaulrtcenturytrick.top\", \"rkiywansamtu.top\", \"rolerxunitywsto.top\", \"rootaleyz.top\", \"rowerpovertort.top\", \"rqfsctpgpuani.pw\", \"rrcspgfghsjnklts.pw\", \"rzss2zfue73dfvmj.onlinerpgame.ch\", \"rzss2zfue73dfvmj.truewargame.ch\", \"sdwempsovemtr.yt\", \"seelkqtkkqxvq.click\", \"semiconductry.top\", \"sgowntfjwkybawi.pw\", \"sgrnhwyqxdk.pw\", \"sondr5344ygfweyjbfkw4fhsefv.heliofetch.at\", \"sonicfopase.top\", \"sqrgvbgfyya.org\", \"sqsigig.pw\", \"ssvylrn.pw\", \"stevnxwq.pw\", \"stgg5jv6mqiibmax.toradmin.li\", \"stgg5jv6mqiibmax.toranimals.li\", \"stgg5jv6mqiibmax.torbrouke.li\", \"stgg5jv6mqiibmax.torclasses.li\", \"stgg5jv6mqiibmax.torclever.li\", \"stgg5jv6mqiibmax.torcreator.li\", \"stgg5jv6mqiibmax.torking.li\", \"stgg5jv6mqiibmax.torpice.li\", \"stgg5jv6mqiibmax.torpoint.ch\", \"stgg5jv6mqiibmax.torshop.li\", \"sumnitdomains.top\", \"svkjhguk.ru\", \"svvgyjweurxn.click\", \"swfqg.in\", \"sxflmtgxerkpgwlnp.pw\", \"t54ndnku456ngkwsudqer.wallymac.com\", \"tdhyjfxltpj.pw\", \"tes543berda73i48fsdfsd.keratadze.at\", \"topgearspoilytyrdc.top\", \"toxnwbkoulii.pw\", \"toytyaclucomunit.top\", \"tqlcjh.fr\", \"tregretryfaltervipo.top\", \"trxswbwxhr.xyz\", \"tswsgajtwhqkosd.su\", \"tt54rfdjhb34rfbnknaerg.milerteddy.com\", \"ttoyqvq.pw\", \"tuouyunittyewr.top\", \"twbers4hmi6dc65f.onion.cab\", \"twbers4hmi6dc65f.onion.to\", \"twbers4hmi6dc65f.tor2web.org\", \"u24er.ovaarmor.com\", \"u54bbnhf354fbkh254tbkhjbgy8258gnkwerg.tahaplap.com\", \"ubisortdasert.top\", \"uetwvrlnee.fr\", \"uhgmnigjpf.biz\", \"uhhvhjqowpgopq.xyz\", \"uhjxayhpisr.pw\", \"uhufnlsad7bhf4ykqfbevmxergwrth.himfinn.com\", \"uiredn4njfsa4234bafb32ygjdawfvs.frascuft.com\", \"uj5nj.onanwhit.com\", \"umjjvccteg.biz\", \"unintyregullyar.top\", \"unittogreas.top\", \"unityharerteraz.top\", \"unityrulesyur.top\", \"unixbroungs.top\", \"unocl45trpuoefft.054t69.bid\", \"unocl45trpuoefft.06j7o0.top\", \"unocl45trpuoefft.086ux2.top\", \"unocl45trpuoefft.0evktl.top\", \"unocl45trpuoefft.0kousz.bid\", \"unocl45trpuoefft.0kv6tw.bid\", \"unocl45trpuoefft.0vgu64.top\", \"unocl45trpuoefft.18xhww.bid\", \"unocl45trpuoefft.1cn41a.bid\", \"unocl45trpuoefft.1de02r.top\", \"unocl45trpuoefft.1v3bnu.top\", \"unocl45trpuoefft.249isv.bid\", \"unocl45trpuoefft.2y4t6f.bid\", \"unocl45trpuoefft.308an1.top\", \"unocl45trpuoefft.31wkhu.top\", \"unocl45trpuoefft.36u6mp.bid\", \"unocl45trpuoefft.3n9lut.bid\", \"unocl45trpuoefft.42wunw.bid\", \"unocl45trpuoefft.4bb9vz.bid\", \"unocl45trpuoefft.4k98id.top\", \"unocl45trpuoefft.54drms.bid\", \"unocl45trpuoefft.54m2k3.bid\", \"unocl45trpuoefft.5o3euy.bid\", \"unocl45trpuoefft.5v3uvc.bid\", \"unocl45trpuoefft.60c61d.bid\", \"unocl45trpuoefft.6w3rkc.bid\", \"unocl45trpuoefft.75tdcj.bid\", \"unocl45trpuoefft.78of7m.bid\", \"unocl45trpuoefft.791sd5.bid\", \"unocl45trpuoefft.7cevps.bid\", \"unocl45trpuoefft.7eup7k.bid\", \"unocl45trpuoefft.7tooul.bid\", \"unocl45trpuoefft.88wz5p.bid\", \"unocl45trpuoefft.8kcfnk.bid\", \"unocl45trpuoefft.8uwckh.top\", \"unocl45trpuoefft.9bjnlk.bid\", \"unocl45trpuoefft.9lnito.top\", \"unocl45trpuoefft.9lx4s6.bid\", \"unocl45trpuoefft.9u3iy1.top\", \"unocl45trpuoefft.a3migu.bid\", \"unocl45trpuoefft.a4v4c3.bid\", \"unocl45trpuoefft.ageshere.club\", \"unocl45trpuoefft.ahhc36.top\", \"unocl45trpuoefft.at593l.bid\", \"unocl45trpuoefft.at9gwv.bid\", \"unocl45trpuoefft.awspm2.top\", \"unocl45trpuoefft.barzc4.bid\", \"unocl45trpuoefft.bjahwh.bid\", \"unocl45trpuoefft.c3fz3z.bid\", \"unocl45trpuoefft.c4issd.bid\", \"unocl45trpuoefft.c9kp0o.bid\", \"unocl45trpuoefft.ceikto.bid\", \"unocl45trpuoefft.cgf59i.top\", \"unocl45trpuoefft.cifbp9.bid\", \"unocl45trpuoefft.ckw9fm.top\", \"unocl45trpuoefft.cm5ohx.bid\", \"unocl45trpuoefft.csdbnk.bid\", \"unocl45trpuoefft.csv7o6.bid\", \"unocl45trpuoefft.cypz3w.top\", \"unocl45trpuoefft.czzg7f.bid\", \"unocl45trpuoefft.dwkofh.top\", \"unocl45trpuoefft.dyo7c9.top\", \"unocl45trpuoefft.efebgv.bid\", \"unocl45trpuoefft.eloppu.bid\", \"unocl45trpuoefft.emogew.bid\", \"unocl45trpuoefft.eo6rzt.bid\", \"unocl45trpuoefft.ev6i0x.bid\", \"unocl45trpuoefft.eyohd2.top\", \"unocl45trpuoefft.f17bam.bid\", \"unocl45trpuoefft.freshsdog.loan\", \"unocl45trpuoefft.frn62e.top\", \"unocl45trpuoefft.gg4dgp.bid\", \"unocl45trpuoefft.gio6f6.bid\", \"unocl45trpuoefft.givxuf.bid\", \"unocl45trpuoefft.hawtzr.bid\", \"unocl45trpuoefft.he81tz.bid\", \"unocl45trpuoefft.hur45z.bid\", \"unocl45trpuoefft.hvh2gb.bid\", \"unocl45trpuoefft.hxrd02.bid\", \"unocl45trpuoefft.hynwbs.top\", \"unocl45trpuoefft.hyr1h3.bid\", \"unocl45trpuoefft.i1wcrl.bid\", \"unocl45trpuoefft.i561zy.bid\", \"unocl45trpuoefft.ibngww.top\", \"unocl45trpuoefft.idw6s5.bid\", \"unocl45trpuoefft.igpfcu.bid\", \"unocl45trpuoefft.igrj6t.bid\", \"unocl45trpuoefft.ih301a.bid\", \"unocl45trpuoefft.ii2yoh.bid\", \"unocl45trpuoefft.ilm071.bid\", \"unocl45trpuoefft.j0cia7.bid\", \"unocl45trpuoefft.j404oy.bid\", \"unocl45trpuoefft.j8exy2.bid\", \"unocl45trpuoefft.jcife9.bid\", \"unocl45trpuoefft.jdf4je.bid\", \"unocl45trpuoefft.jjogbj.top\", \"unocl45trpuoefft.jnd0bj.bid\", \"unocl45trpuoefft.jsotn5.top\", \"unocl45trpuoefft.jvrh8g.bid\", \"unocl45trpuoefft.k56185.top\", \"unocl45trpuoefft.kf1gxm.bid\", \"unocl45trpuoefft.kg5bof.bid\", \"unocl45trpuoefft.kml2o2.top\", \"unocl45trpuoefft.knowhands.us\", \"unocl45trpuoefft.ks3ghp.bid\", \"unocl45trpuoefft.kswcuk.top\", \"unocl45trpuoefft.l05l27.top\", \"unocl45trpuoefft.l69xgc.bid\", \"unocl45trpuoefft.l97i5a.bid\", \"unocl45trpuoefft.lak8wd.bid\", \"unocl45trpuoefft.larebg.bid\", \"unocl45trpuoefft.lcyznu.bid\", \"unocl45trpuoefft.lio2wr.bid\", \"unocl45trpuoefft.lk0bzc.top\", \"unocl45trpuoefft.ll3zot.bid\", \"unocl45trpuoefft.lzskva.bid\", \"unocl45trpuoefft.m03t72.bid\", \"unocl45trpuoefft.m33d4b.bid\", \"unocl45trpuoefft.m9a225.top\", \"unocl45trpuoefft.mbwxyg.bid\", \"unocl45trpuoefft.md9eyv.bid\", \"unocl45trpuoefft.meetsface.win\", \"unocl45trpuoefft.metpast.date\", \"unocl45trpuoefft.mezy7j.bid\", \"unocl45trpuoefft.moonsides.faith\", \"unocl45trpuoefft.n20b1c.top\", \"unocl45trpuoefft.n41n1a.top\", \"unocl45trpuoefft.n94lrn.bid\", \"unocl45trpuoefft.na2iuz.bid\", \"unocl45trpuoefft.nmit4p.bid\", \"unocl45trpuoefft.noyl9o.bid\", \"unocl45trpuoefft.nz6emv.bid\", \"unocl45trpuoefft.o2dval.top\", \"unocl45trpuoefft.o8hpwj.top\", \"unocl45trpuoefft.og5ezh.top\", \"unocl45trpuoefft.on2420.bid\", \"unocl45trpuoefft.ozlrnx.bid\", \"unocl45trpuoefft.p1gneb.bid\", \"unocl45trpuoefft.p2ix1u.bid\", \"unocl45trpuoefft.p4sr76.top\", \"unocl45trpuoefft.pap44w.top\", \"unocl45trpuoefft.pbprju.bid\", \"unocl45trpuoefft.piy4l3.bid\", \"unocl45trpuoefft.ptneek.bid\", \"unocl45trpuoefft.r21wmw.top\", \"unocl45trpuoefft.r2vai7.bid\", \"unocl45trpuoefft.rgbb50.bid\", \"unocl45trpuoefft.rie9py.bid\", \"unocl45trpuoefft.rslh9a.top\", \"unocl45trpuoefft.s7b63k.bid\", \"unocl45trpuoefft.sirchi.bid\", \"unocl45trpuoefft.sp4o1t.bid\", \"unocl45trpuoefft.tcly4s.bid\", \"unocl45trpuoefft.tfmmby.bid\", \"unocl45trpuoefft.thanreal.link\", \"unocl45trpuoefft.ttabop.bid\", \"unocl45trpuoefft.u64rj2.top\", \"unocl45trpuoefft.uaol08.bid\", \"unocl45trpuoefft.ukwnvw.bid\", \"unocl45trpuoefft.um1x6z.bid\", \"unocl45trpuoefft.uog1ky.bid\", \"unocl45trpuoefft.uso3z0.bid\", \"unocl45trpuoefft.uw3r6a.top\", \"unocl45trpuoefft.uwckha.top\", \"unocl45trpuoefft.v4kx51.bid\", \"unocl45trpuoefft.v50gtu.bid\", \"unocl45trpuoefft.vfuvsv.bid\", \"unocl45trpuoefft.vi5iko.bid\", \"unocl45trpuoefft.vkm4l6.top\", \"unocl45trpuoefft.vkslju.bid\", \"unocl45trpuoefft.vlwbcz.bid\", \"unocl45trpuoefft.vmomcc.bid\", \"unocl45trpuoefft.whmykv.bid\", \"unocl45trpuoefft.wl8t6k.bid\", \"unocl45trpuoefft.wlvxd6.bid\", \"unocl45trpuoefft.wz139z.top\", \"unocl45trpuoefft.x9kjcn.bid\", \"unocl45trpuoefft.x9le66.top\", \"unocl45trpuoefft.xf38wp.bid\", \"unocl45trpuoefft.xlxd92.bid\", \"unocl45trpuoefft.y721yz.top\", \"unocl45trpuoefft.ye4f7k.bid\", \"unocl45trpuoefft.yky1uf.bid\", \"unocl45trpuoefft.ytbyhs.bid\", \"unocl45trpuoefft.yty0gm.bid\", \"unocl45trpuoefft.zbj2kc.bid\", \"unocl45trpuoefft.zdamew.bid\", \"unocl45trpuoefft.zgheyh.bid\", \"unocl45trpuoefft.zjems2.bid\", \"unocl45trpuoefft.zn9cme.bid\", \"urulvtffwoq.xyz\", \"uuwflbmjmi.eu\", \"uvcmlfca.biz\", \"uxvvm.us\", \"uxwavkmttywsuynt.pw\", \"vcabbvhrqhot.pw\", \"vewrb.italisumo.at\", \"vpuroeit.pw\", \"vrvis6ndra5jeggj.livegaming.ch\", \"vrvis6ndra5jeggj.livewargaming.ch\", \"vrvis6ndra5jeggj.onlinebattlefield.ch\", \"vrympoqs5ra34nfo.bigbird.at\", \"vrympoqs5ra34nfo.bigclear.at\", \"vrympoqs5ra34nfo.smartbus.at\", \"vrympoqs5ra34nfo.torhelper.pl\", \"vujqbcditgsqxe.fr\", \"vyohacxzoue32vvk.0ayn1s.top\", \"vyohacxzoue32vvk.0ot7em.bid\", \"vyohacxzoue32vvk.0vtwzy.top\", \"vyohacxzoue32vvk.1m47ka.bid\", \"vyohacxzoue32vvk.23fvxw.bid\", \"vyohacxzoue32vvk.2hr4fs.top\", \"vyohacxzoue32vvk.34o9h1.bid\", \"vyohacxzoue32vvk.3buvlc.bid\", \"vyohacxzoue32vvk.3m370u.top\", \"vyohacxzoue32vvk.3peyo3.bid\", \"vyohacxzoue32vvk.3t3hyf.top\", \"vyohacxzoue32vvk.5a5vmh.top\", \"vyohacxzoue32vvk.5i0ukv.bid\", \"vyohacxzoue32vvk.5m2n7x.top\", \"vyohacxzoue32vvk.5s96fr.top\", \"vyohacxzoue32vvk.6wkz70.bid\", \"vyohacxzoue32vvk.79j8fm.top\", \"vyohacxzoue32vvk.7a07br.bid\", \"vyohacxzoue32vvk.7jrv53.bid\", \"vyohacxzoue32vvk.7m7ujm.bid\", \"vyohacxzoue32vvk.8g1k17.bid\", \"vyohacxzoue32vvk.ac7zvz.top\", \"vyohacxzoue32vvk.axu3u8.bid\", \"vyohacxzoue32vvk.b14kkk.bid\", \"vyohacxzoue32vvk.c4cwr4.bid\", \"vyohacxzoue32vvk.c8jxpp.top\", \"vyohacxzoue32vvk.chnbyl.bid\", \"vyohacxzoue32vvk.cp3yme.top\", \"vyohacxzoue32vvk.d7h6yx.top\", \"vyohacxzoue32vvk.dgjpgy.top\", \"vyohacxzoue32vvk.dks71o.bid\", \"vyohacxzoue32vvk.ean5e7.top\", \"vyohacxzoue32vvk.ewfp5y.bid\", \"vyohacxzoue32vvk.ezb568.top\", \"vyohacxzoue32vvk.fp6fj6.top\", \"vyohacxzoue32vvk.fsly47.top\", \"vyohacxzoue32vvk.g7rst5.bid\", \"vyohacxzoue32vvk.gjbmis.top\", \"vyohacxzoue32vvk.h2xun1.top\", \"vyohacxzoue32vvk.ibar8s.top\", \"vyohacxzoue32vvk.jb4uh0.top\", \"vyohacxzoue32vvk.jnv1df.top\", \"vyohacxzoue32vvk.joco7r.top\", \"vyohacxzoue32vvk.jwi2ek.bid\", \"vyohacxzoue32vvk.k9p80d.top\", \"vyohacxzoue32vvk.kfymbh.top\", \"vyohacxzoue32vvk.kwrd4f.bid\", \"vyohacxzoue32vvk.l4dlll.bid\", \"vyohacxzoue32vvk.mayrwf.top\", \"vyohacxzoue32vvk.mpduf5.bid\", \"vyohacxzoue32vvk.ncw0rp.top\", \"vyohacxzoue32vvk.nta934.top\", \"vyohacxzoue32vvk.o08ra6.top\", \"vyohacxzoue32vvk.o5b17o.top\", \"vyohacxzoue32vvk.p9su2u.top\", \"vyohacxzoue32vvk.pr52ni.top\", \"vyohacxzoue32vvk.r31sot.top\", \"vyohacxzoue32vvk.r3b2sh.top\", \"vyohacxzoue32vvk.roep3o.top\", \"vyohacxzoue32vvk.ss8doe.top\", \"vyohacxzoue32vvk.t6ueop.bid\", \"vyohacxzoue32vvk.u8e2dz.top\", \"vyohacxzoue32vvk.ug6ewx.top\", \"vyohacxzoue32vvk.vjso7r.top\", \"vyohacxzoue32vvk.w22p3v.top\", \"vyohacxzoue32vvk.w67y8u.bid\", \"vyohacxzoue32vvk.x83zw1.top\", \"vyohacxzoue32vvk.xsf5a8.top\", \"vyohacxzoue32vvk.xy2rlg.bid\", \"vyohacxzoue32vvk.zmn16h.top\", \"vyohacxzoue32vvk.zn90h4.bid\", \"vyohacxzoue32vvk.zp9i1l.bid\", \"vyohacxzoue32vvk.zu3fzc.bid\", \"vyohacxzoue32vvk.zz3w5l.bid\", \"w6bfg4hahn5bfnlsafgchkvg5fwsfvrt.hareuna.at\", \"waduavfijwkanvf.xyz\", \"wbaskcsxiffiax.info\", \"wdvxeval.ru\", \"wersalitrestyws.top\", \"wjfkoqueatxdmqw.biz\", \"wjtqjleommc4z46i.249isv.bid\", \"wjtqjleommc4z46i.2y4t6f.bid\", \"wjtqjleommc4z46i.35rof4.bid\", \"wjtqjleommc4z46i.35u068.bid\", \"wjtqjleommc4z46i.44vva6.bid\", \"wjtqjleommc4z46i.4bb9vz.bid\", \"wjtqjleommc4z46i.54vw9b.bid\", \"wjtqjleommc4z46i.5n5y6v.bid\", \"wjtqjleommc4z46i.5r1sol.bid\", \"wjtqjleommc4z46i.7hu6og.bid\", \"wjtqjleommc4z46i.8a9r2h.bid\", \"wjtqjleommc4z46i.993hev.bid\", \"wjtqjleommc4z46i.9sellg.bid\", \"wjtqjleommc4z46i.9ule2e.bid\", \"wjtqjleommc4z46i.au6d1d.bid\", \"wjtqjleommc4z46i.bipa9k.bid\", \"wjtqjleommc4z46i.c3fz3z.bid\", \"wjtqjleommc4z46i.cc0r87.bid\", \"wjtqjleommc4z46i.cdyd2z.bid\", \"wjtqjleommc4z46i.cgab48.bid\", \"wjtqjleommc4z46i.cm5ohx.bid\", \"wjtqjleommc4z46i.csv7o6.bid\", \"wjtqjleommc4z46i.cto5ee.bid\", \"wjtqjleommc4z46i.d11zjd.bid\", \"wjtqjleommc4z46i.e53rg4.bid\", \"wjtqjleommc4z46i.eag72x.top\", \"wjtqjleommc4z46i.efyh72.bid\", \"wjtqjleommc4z46i.f0jlbj.bid\", \"wjtqjleommc4z46i.fw1bwy.bid\", \"wjtqjleommc4z46i.fwfu4t.bid\", \"wjtqjleommc4z46i.gg4dgp.bid\", \"wjtqjleommc4z46i.h8prbu.top\", \"wjtqjleommc4z46i.hom07d.bid\", \"wjtqjleommc4z46i.i8zh1k.bid\", \"wjtqjleommc4z46i.idw6s5.bid\", \"wjtqjleommc4z46i.ilmgcl.bid\", \"wjtqjleommc4z46i.izyclz.bid\", \"wjtqjleommc4z46i.j0n83w.bid\", \"wjtqjleommc4z46i.jal9lk.bid\", \"wjtqjleommc4z46i.jujthy.bid\", \"wjtqjleommc4z46i.kt70uk.bid\", \"wjtqjleommc4z46i.kyjw0g.bid\", \"wjtqjleommc4z46i.kzhzuc.top\", \"wjtqjleommc4z46i.ldsl8m.bid\", \"wjtqjleommc4z46i.m33d4b.bid\", \"wjtqjleommc4z46i.n8ln0w.bid\", \"wjtqjleommc4z46i.nh47ri.bid\", \"wjtqjleommc4z46i.nnbdlh.bid\", \"wjtqjleommc4z46i.nxmu0x.bid\", \"wjtqjleommc4z46i.o8hpwj.top\", \"wjtqjleommc4z46i.obx4vo.bid\", \"wjtqjleommc4z46i.oodvxp.bid\", \"wjtqjleommc4z46i.p41khf.bid\", \"wjtqjleommc4z46i.pmnz7a.bid\", \"wjtqjleommc4z46i.salethe.gdn\", \"wjtqjleommc4z46i.srmlzh.bid\", \"wjtqjleommc4z46i.srtos7.bid\", \"wjtqjleommc4z46i.t4jp3w.bid\", \"wjtqjleommc4z46i.u36ik0.bid\", \"wjtqjleommc4z46i.uv39h5.bid\", \"wjtqjleommc4z46i.uwckha.top\", \"wjtqjleommc4z46i.vh6vss.bid\", \"wjtqjleommc4z46i.w3r6a4.bid\", \"wjtqjleommc4z46i.whmykv.bid\", \"wjtqjleommc4z46i.xjwlms.bid\", \"wjtqjleommc4z46i.y12acl.bid\", \"wjtqjleommc4z46i.y2ijlz.bid\", \"wjtqjleommc4z46i.y7603i.bid\", \"wjtqjleommc4z46i.yfr0o1.bid\", \"wjtqjleommc4z46i.z7uxzg.bid\", \"wjtqjleommc4z46i.z97f9v.bid\", \"wjtqjleommc4z46i.zclhx9.bid\", \"wor4d.slewirk.at\", \"wpvvusso.xyz\", \"wqxvsxppjivs.pw\", \"wrubyjtvqhxaqkh.pw\", \"wtxvmsikbmtbq.pw\", \"wvltrlrnf.xyz\", \"www.1axb.com\", \"www.chromebewfk.top\", \"www.chromefastl.top\", \"www.chromehakc.top\", \"www.cleverdotl.top\", \"www.ddiopoola.top\", \"www.dealkolld.top\", \"www.dokjasura.top\", \"www.fkauueeepla.top\", \"www.flowerxpo.top\", \"www.foolalexas.top\", \"www.googlefoad.top\", \"www.newsectorbs.top\", \"www.newtonpaiva.br\", \"www.watherfka.top\", \"www.weekendlk.top\", \"x5sbb5gesp6kzwsh.frontmain.pl\", \"x5sbb5gesp6kzwsh.frontymen.pl\", \"x5sbb5gesp6kzwsh.homewind.pl\", \"x5sbb5gesp6kzwsh.mailteam.pl\", \"x5sbb5gesp6kzwsh.questpul.pl\", \"xfyubqmldwvuyar.yt\", \"xhrnfffaixawpuob.pw\", \"xmniabhrfafptwx.pw\", \"xofguhypjgvxrm.pw\", \"xpcx6erilkjced3j.16hwwh.top\", \"xpcx6erilkjced3j.16umxg.top\", \"xpcx6erilkjced3j.17gcun.top\", \"xpcx6erilkjced3j.18ey8e.top\", \"xpcx6erilkjced3j.19kdeh.top\", \"xpcx6erilkjced3j.1blery.top\", \"xpcx6erilkjced3j.1cgbcv.top\", \"xpcx6erilkjced3j.1ebjjq.top\", \"xpcx6erilkjced3j.1j9jad.top\", \"xpcx6erilkjced3j.1jyrty.top\", \"xpcx6erilkjced3j.1mfmkz.top\", \"xpcx6erilkjced3j.1mpsnr.top\", \"xpcx6erilkjced3j.1n5mod.top\", \"xrhwryizf5mui7a5.50mb1c.bid\", \"xrhwryizf5mui7a5.djintc.bid\", \"xrhwryizf5mui7a5.g72xh8.top\", \"xrhwryizf5mui7a5.h44l3d.bid\", \"xrhwryizf5mui7a5.j4cser.bid\", \"xrhwryizf5mui7a5.jhrb5a.top\", \"xrhwryizf5mui7a5.r8c85p.top\", \"xrhwryizf5mui7a5.rt01jw.top\", \"xrhwryizf5mui7a5.uw9x7z.bid\", \"xrhwryizf5mui7a5.vgxcci.top\", \"xvchcbeqxkd.pw\", \"xyhhuxa.be\", \"y4bxj.adozeuds.com\", \"yavmxpiqfwmubk.pw\", \"yaynawvtuqcarjwc.pw\", \"ycvcjbhgkmsiyhdd.info\", \"yofkhfskdyiqo.biz\", \"ytcijiooxdtlbevrh.info\", \"ytrest84y5i456hghadefdsd.pontogrot.com\", \"yuertao.pw\", \"yuysikankhqvdwdv.xyz\", \"ywjgjvpuyitnbiw.info\", \"yyre45dbvn2nhbefbmh.begumvelic.at\", \"zjfq4lnfbs7pncr5.onion.to\", \"zjfq4lnfbs7pncr5.tor2web.org\", \"ztuw5bvuuapzdfya.klimbim.pl\", \"zutzt67dcxr6mxcn.onion.to\"],\"md5\": []}}]}" }, { "path": "tests/integration/basic/IPv4.lst", "content": "103.249.88.244-103.249.88.244\n103.89.88.88-103.89.88.88\n104.18.34.162-104.18.34.162\n108.170.60.189-108.170.60.189\n109.230.199.159-109.230.199.159\n109.230.199.169-109.230.199.169\n109.230.199.30-109.230.199.30\n125.209.82.158-125.209.82.158\n136.25.2.43-136.25.2.43\n137.74.131.18-137.74.131.18\n138.197.148.53-138.197.148.53\n140.82.48.224-140.82.48.224\n144.76.215.117-144.76.215.117\n162.244.32.180-162.244.32.180\n173.254.223.115-173.254.223.115\n173.46.85.161-173.46.85.161\n173.46.85.168-173.46.85.168\n173.46.85.19-173.46.85.19\n173.46.85.205-173.46.85.205\n173.46.85.22-173.46.85.22\n173.46.85.234-173.46.85.234\n173.46.85.60-173.46.85.60\n173.46.85.68-173.46.85.68\n173.46.85.71-173.46.85.71\n173.46.85.86-173.46.85.86\n173.46.85.98-173.46.85.98\n178.162.132.90-178.162.132.90\n178.239.21.106-178.239.21.106\n179.43.176.148-179.43.176.148\n18.221.114.76-18.221.114.76\n181.129.146.34-181.129.146.34\n181.129.171.34-181.129.171.34\n181.129.93.226-181.129.93.226\n181.215.247.164-181.215.247.164\n181.215.47.171-181.215.47.171\n185.125.205.69-185.125.205.69\n185.125.205.73-185.125.205.73\n185.125.205.75-185.125.205.75\n185.125.205.78-185.125.205.78\n185.125.205.79-185.125.205.79\n185.125.205.91-185.125.205.91\n185.127.27.238-185.127.27.238\n185.141.62.213-185.141.62.213\n185.156.174.115-185.156.174.115\n185.158.248.92-185.158.248.92\n185.158.249.233-185.158.249.233\n185.158.251.60-185.158.251.60\n185.174.173.128-185.174.173.128\n185.189.149.187-185.189.149.187\n185.202.174.91-185.202.174.91\n185.203.118.6-185.203.118.6\n185.205.210.139-185.205.210.139\n185.212.47.103-185.212.47.103\n185.22.65.5-185.22.65.5\n185.223.163.26-185.223.163.26\n185.231.153.46-185.231.153.46\n185.236.203.53-185.236.203.53\n185.236.203.60-185.236.203.60\n185.244.30.101-185.244.30.101\n185.244.30.105-185.244.30.105\n185.244.30.106-185.244.30.106\n185.244.30.109-185.244.30.109\n185.244.30.111-185.244.30.111\n185.244.30.113-185.244.30.113\n185.244.30.114-185.244.30.114\n185.244.30.120-185.244.30.120\n185.244.30.121-185.244.30.121\n185.244.30.124-185.244.30.124\n185.244.30.93-185.244.30.93\n185.77.129.11-185.77.129.11\n186.147.161.204-186.147.161.204\n186.167.66.51-186.167.66.51\n187.19.17.132-187.19.17.132\n192.227.248.175-192.227.248.175\n192.99.212.140-192.99.212.140\n193.29.56.44-193.29.56.44\n194.5.98.104-194.5.98.104\n194.5.98.139-194.5.98.139\n194.5.98.148-194.5.98.148\n194.5.98.193-194.5.98.193\n194.5.98.194-194.5.98.194\n194.5.98.226-194.5.98.226\n194.5.98.38-194.5.98.38\n194.5.98.56-194.5.98.56\n194.5.99.119-194.5.99.119\n194.5.99.136-194.5.99.136\n194.5.99.158-194.5.99.158\n194.5.99.159-194.5.99.159\n194.5.99.175-194.5.99.175\n194.5.99.2-194.5.99.2\n194.5.99.207-194.5.99.207\n194.5.99.226-194.5.99.226\n194.5.99.250-194.5.99.250\n194.5.99.59-194.5.99.59\n194.5.99.63-194.5.99.63\n194.5.99.67-194.5.99.67\n194.5.99.7-194.5.99.7\n194.5.99.97-194.5.99.97\n194.68.225.63-194.68.225.63\n194.76.225.59-194.76.225.59\n194.99.20.254-194.99.20.254\n195.123.212.149-195.123.212.149\n195.123.213.169-195.123.213.169\n195.123.227.20-195.123.227.20\n195.123.245.214-195.123.245.214\n195.123.245.90-195.123.245.90\n199.21.106.189-199.21.106.189\n202.63.242.48-202.63.242.48\n204.95.99.204-204.95.99.204\n209.58.186.245-209.58.186.245\n212.47.194.15-212.47.194.15\n212.73.150.215-212.73.150.215\n213.152.161.138-213.152.161.138\n24.217.192.131-24.217.192.131\n24.247.181.155-24.247.181.155\n24.247.182.169-24.247.182.169\n24.247.182.240-24.247.182.240\n24.247.182.253-24.247.182.253\n31.171.152.103-31.171.152.103\n31.171.152.105-31.171.152.105\n31.171.152.106-31.171.152.106\n31.171.152.107-31.171.152.107\n31.7.188.40-31.7.188.40\n35.198.61.54-35.198.61.54\n37.59.134.55-37.59.134.55\n45.249.90.124-45.249.90.124\n45.55.36.231-45.55.36.231\n46.166.173.109-46.166.173.109\n46.17.45.29-46.17.45.29\n46.17.47.216-46.17.47.216\n46.183.223.10-46.183.223.10\n47.44.54.70-47.44.54.70\n5.2.64.188-5.2.64.188\n5.2.67.66-5.2.67.66\n5.206.225.115-5.206.225.115\n5.8.88.125-5.8.88.125\n54.37.86.44-54.37.86.44\n54.38.146.43-54.38.146.43\n68.111.123.100-68.111.123.100\n68.183.249.84-68.183.249.84\n72.189.124.41-72.189.124.41\n76.107.90.235-76.107.90.235\n78.155.220.198-78.155.220.198\n81.177.141.211-81.177.141.211\n81.177.180.174-81.177.180.174\n82.199.134.139-82.199.134.139\n82.199.134.156-82.199.134.156\n83.166.245.213-83.166.245.213\n85.217.170.62-85.217.170.62\n87.236.22.142-87.236.22.142\n91.192.100.16-91.192.100.16\n91.192.100.27-91.192.100.27\n91.192.100.3-91.192.100.3\n91.192.100.40-91.192.100.40\n91.192.100.44-91.192.100.44\n91.192.100.48-91.192.100.48\n91.192.100.52-91.192.100.52\n92.222.10.99-92.222.10.99\n93.115.26.171-93.115.26.171\n94.103.83.137-94.103.83.137\n94.185.86.56-94.185.86.56\n94.237.44.31-94.237.44.31\n95.213.251.165-95.213.251.165\n95.47.161.68-95.47.161.68\n97.87.175.152-97.87.175.152\n1.1.1.0-1.1.1.33\n1.1.2.0-1.1.2.255\n1.1.3.10-1.1.3.44\n" }, { "path": "tests/integration/basic/IPv4HC%3Fs%3D5%26n%3D10.result", "content": "104.18.34.162-104.18.34.162\n108.170.60.189-108.170.60.189\n109.230.199.159-109.230.199.159\n109.230.199.169-109.230.199.169\n109.230.199.30-109.230.199.30\n125.209.82.158-125.209.82.158\n136.25.2.43-136.25.2.43\n137.74.131.18-137.74.131.18\n138.197.148.53-138.197.148.53\n140.82.48.224-140.82.48.224\n" }, { "path": "tests/integration/basic/IPv4HC%3Fv%3Dcsv%26f%3Dconfidence%26f%3Dsources%7Cfeeds%26f%3Dindicator%7Cclientip%26tr%3D1.result", "content": "confidence,feeds,clientip\r\n100,localdb,1.1.1.0/27\r\n100,localdb,1.1.1.32/31\r\n100,localdb,1.1.2.0/24\r\n100,localdb,1.1.3.10/31\r\n100,localdb,1.1.3.12/30\r\n100,localdb,1.1.3.16/28\r\n100,localdb,1.1.3.32/29\r\n100,localdb,1.1.3.40/30\r\n100,localdb,1.1.3.44\r\n100,localdb,103.249.88.244\r\n100,localdb,103.89.88.88\r\n100,localdb,104.18.34.162\r\n100,localdb,108.170.60.189\r\n100,localdb,109.230.199.159\r\n100,localdb,109.230.199.169\r\n100,localdb,109.230.199.30\r\n100,localdb,125.209.82.158\r\n100,localdb,136.25.2.43\r\n100,localdb,137.74.131.18\r\n100,localdb,138.197.148.53\r\n100,localdb,140.82.48.224\r\n100,localdb,144.76.215.117\r\n100,localdb,162.244.32.180\r\n100,localdb,173.254.223.115\r\n100,localdb,173.46.85.161\r\n100,localdb,173.46.85.168\r\n100,localdb,173.46.85.19\r\n100,localdb,173.46.85.205\r\n100,localdb,173.46.85.22\r\n100,localdb,173.46.85.234\r\n100,localdb,173.46.85.60\r\n100,localdb,173.46.85.68\r\n100,localdb,173.46.85.71\r\n100,localdb,173.46.85.86\r\n100,localdb,173.46.85.98\r\n100,localdb,178.162.132.90\r\n100,localdb,178.239.21.106\r\n100,localdb,179.43.176.148\r\n100,localdb,18.221.114.76\r\n100,localdb,181.129.146.34\r\n100,localdb,181.129.171.34\r\n100,localdb,181.129.93.226\r\n100,localdb,181.215.247.164\r\n100,localdb,181.215.47.171\r\n100,localdb,185.125.205.69\r\n100,localdb,185.125.205.73\r\n100,localdb,185.125.205.75\r\n100,localdb,185.125.205.78\r\n100,localdb,185.125.205.79\r\n100,localdb,185.125.205.91\r\n100,localdb,185.127.27.238\r\n100,localdb,185.141.62.213\r\n100,localdb,185.156.174.115\r\n100,localdb,185.158.248.92\r\n100,localdb,185.158.249.233\r\n100,localdb,185.158.251.60\r\n100,localdb,185.174.173.128\r\n100,localdb,185.189.149.187\r\n100,localdb,185.202.174.91\r\n100,localdb,185.203.118.6\r\n100,localdb,185.205.210.139\r\n100,localdb,185.212.47.103\r\n100,localdb,185.22.65.5\r\n100,localdb,185.223.163.26\r\n100,localdb,185.231.153.46\r\n100,localdb,185.236.203.53\r\n100,localdb,185.236.203.60\r\n100,localdb,185.244.30.101\r\n100,localdb,185.244.30.105\r\n100,localdb,185.244.30.106\r\n100,localdb,185.244.30.109\r\n100,localdb,185.244.30.111\r\n100,localdb,185.244.30.113\r\n100,localdb,185.244.30.114\r\n100,localdb,185.244.30.120\r\n100,localdb,185.244.30.121\r\n100,localdb,185.244.30.124\r\n100,localdb,185.244.30.93\r\n100,localdb,185.77.129.11\r\n100,localdb,186.147.161.204\r\n100,localdb,186.167.66.51\r\n100,localdb,187.19.17.132\r\n100,localdb,192.227.248.175\r\n100,localdb,192.99.212.140\r\n100,localdb,193.29.56.44\r\n100,localdb,194.5.98.104\r\n100,localdb,194.5.98.139\r\n100,localdb,194.5.98.148\r\n100,localdb,194.5.98.193\r\n100,localdb,194.5.98.194\r\n100,localdb,194.5.98.226\r\n100,localdb,194.5.98.38\r\n100,localdb,194.5.98.56\r\n100,localdb,194.5.99.119\r\n100,localdb,194.5.99.136\r\n100,localdb,194.5.99.158\r\n100,localdb,194.5.99.159\r\n100,localdb,194.5.99.175\r\n100,localdb,194.5.99.2\r\n100,localdb,194.5.99.207\r\n100,localdb,194.5.99.226\r\n100,localdb,194.5.99.250\r\n100,localdb,194.5.99.59\r\n100,localdb,194.5.99.63\r\n100,localdb,194.5.99.67\r\n100,localdb,194.5.99.7\r\n100,localdb,194.5.99.97\r\n100,localdb,194.68.225.63\r\n100,localdb,194.76.225.59\r\n100,localdb,194.99.20.254\r\n100,localdb,195.123.212.149\r\n100,localdb,195.123.213.169\r\n100,localdb,195.123.227.20\r\n100,localdb,195.123.245.214\r\n100,localdb,195.123.245.90\r\n100,localdb,199.21.106.189\r\n100,localdb,202.63.242.48\r\n100,localdb,204.95.99.204\r\n100,localdb,209.58.186.245\r\n100,localdb,212.47.194.15\r\n100,localdb,212.73.150.215\r\n100,localdb,213.152.161.138\r\n100,localdb,24.217.192.131\r\n100,localdb,24.247.181.155\r\n100,localdb,24.247.182.169\r\n100,localdb,24.247.182.240\r\n100,localdb,24.247.182.253\r\n100,localdb,31.171.152.103\r\n100,localdb,31.171.152.105\r\n100,localdb,31.171.152.106\r\n100,localdb,31.171.152.107\r\n100,localdb,31.7.188.40\r\n100,localdb,35.198.61.54\r\n100,localdb,37.59.134.55\r\n100,localdb,45.249.90.124\r\n100,localdb,45.55.36.231\r\n100,localdb,46.166.173.109\r\n100,localdb,46.17.45.29\r\n100,localdb,46.17.47.216\r\n100,localdb,46.183.223.10\r\n100,localdb,47.44.54.70\r\n100,localdb,5.2.64.188\r\n100,localdb,5.2.67.66\r\n100,localdb,5.206.225.115\r\n100,localdb,5.8.88.125\r\n100,localdb,54.37.86.44\r\n100,localdb,54.38.146.43\r\n100,localdb,68.111.123.100\r\n100,localdb,68.183.249.84\r\n100,localdb,72.189.124.41\r\n100,localdb,76.107.90.235\r\n100,localdb,78.155.220.198\r\n100,localdb,81.177.141.211\r\n100,localdb,81.177.180.174\r\n100,localdb,82.199.134.139\r\n100,localdb,82.199.134.156\r\n100,localdb,83.166.245.213\r\n100,localdb,85.217.170.62\r\n100,localdb,87.236.22.142\r\n100,localdb,91.192.100.16\r\n100,localdb,91.192.100.27\r\n100,localdb,91.192.100.3\r\n100,localdb,91.192.100.40\r\n100,localdb,91.192.100.44\r\n100,localdb,91.192.100.48\r\n100,localdb,91.192.100.52\r\n100,localdb,92.222.10.99\r\n100,localdb,93.115.26.171\r\n100,localdb,94.103.83.137\r\n100,localdb,94.185.86.56\r\n100,localdb,94.237.44.31\r\n100,localdb,95.213.251.165\r\n100,localdb,95.47.161.68\r\n100,localdb,97.87.175.152\r\n" }, { "path": "tests/integration/basic/IPv4HC%3Fv%3Djson%26tr%3D1.result", "content": "[\n{\"indicator\":\"1.1.1.0/27\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"1.1.1.32/31\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"1.1.2.0/24\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"1.1.3.10/31\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"1.1.3.12/30\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"1.1.3.16/28\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"1.1.3.32/29\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"1.1.3.40/30\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"1.1.3.44\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"103.249.88.244\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"103.89.88.88\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"104.18.34.162\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"108.170.60.189\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"109.230.199.159\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"109.230.199.169\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"109.230.199.30\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"125.209.82.158\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"136.25.2.43\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"137.74.131.18\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"138.197.148.53\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"140.82.48.224\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"144.76.215.117\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"162.244.32.180\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"173.254.223.115\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"173.46.85.161\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"173.46.85.168\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"173.46.85.19\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"173.46.85.205\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"173.46.85.22\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"173.46.85.234\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"173.46.85.60\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"173.46.85.68\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"173.46.85.71\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"173.46.85.86\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"173.46.85.98\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"178.162.132.90\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"178.239.21.106\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"179.43.176.148\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"18.221.114.76\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"181.129.146.34\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"181.129.171.34\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"181.129.93.226\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"181.215.247.164\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"181.215.47.171\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.125.205.69\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.125.205.73\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.125.205.75\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.125.205.78\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.125.205.79\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.125.205.91\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.127.27.238\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.141.62.213\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.156.174.115\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.158.248.92\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.158.249.233\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.158.251.60\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.174.173.128\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.189.149.187\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.202.174.91\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.203.118.6\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.205.210.139\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.212.47.103\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.22.65.5\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.223.163.26\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.231.153.46\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.236.203.53\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.236.203.60\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.244.30.101\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.244.30.105\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.244.30.106\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.244.30.109\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.244.30.111\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.244.30.113\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.244.30.114\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.244.30.120\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.244.30.121\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.244.30.124\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.244.30.93\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"185.77.129.11\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"186.147.161.204\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"186.167.66.51\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"187.19.17.132\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"192.227.248.175\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"192.99.212.140\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"193.29.56.44\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.98.104\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.98.139\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.98.148\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.98.193\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.98.194\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.98.226\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.98.38\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.98.56\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.119\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.136\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.158\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.159\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.175\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.2\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.207\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.226\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.250\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.59\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.63\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.67\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.7\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.5.99.97\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.68.225.63\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.76.225.59\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"194.99.20.254\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"195.123.212.149\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"195.123.213.169\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"195.123.227.20\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"195.123.245.214\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"195.123.245.90\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"199.21.106.189\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"202.63.242.48\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"204.95.99.204\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"209.58.186.245\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"212.47.194.15\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"212.73.150.215\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"213.152.161.138\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"24.217.192.131\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"24.247.181.155\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"24.247.182.169\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"24.247.182.240\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"24.247.182.253\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"31.171.152.103\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"31.171.152.105\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"31.171.152.106\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"31.171.152.107\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"31.7.188.40\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"35.198.61.54\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"37.59.134.55\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"45.249.90.124\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"45.55.36.231\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"46.166.173.109\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"46.17.45.29\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"46.17.47.216\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"46.183.223.10\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"47.44.54.70\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"5.2.64.188\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"5.2.67.66\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"5.206.225.115\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"5.8.88.125\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"54.37.86.44\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"54.38.146.43\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"68.111.123.100\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"68.183.249.84\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"72.189.124.41\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"76.107.90.235\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"78.155.220.198\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"81.177.141.211\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"81.177.180.174\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"82.199.134.139\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"82.199.134.156\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"83.166.245.213\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"85.217.170.62\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"87.236.22.142\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"91.192.100.16\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"91.192.100.27\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"91.192.100.3\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"91.192.100.40\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"91.192.100.44\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"91.192.100.48\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"91.192.100.52\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"92.222.10.99\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"93.115.26.171\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"94.103.83.137\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"94.185.86.56\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"94.237.44.31\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"95.213.251.165\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"95.47.161.68\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}},\n{\"indicator\":\"97.87.175.152\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}]\n" }, { "path": "tests/integration/basic/IPv4HC%3Fv%3Djson-seq.result", "content": "\u001e{\"indicator\":\"1.1.1.0-1.1.1.33\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"1.1.2.0-1.1.2.255\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"1.1.3.10-1.1.3.44\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"103.249.88.244-103.249.88.244\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"103.89.88.88-103.89.88.88\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"104.18.34.162-104.18.34.162\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"108.170.60.189-108.170.60.189\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"109.230.199.159-109.230.199.159\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"109.230.199.169-109.230.199.169\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"109.230.199.30-109.230.199.30\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"125.209.82.158-125.209.82.158\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"136.25.2.43-136.25.2.43\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"137.74.131.18-137.74.131.18\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"138.197.148.53-138.197.148.53\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"140.82.48.224-140.82.48.224\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"144.76.215.117-144.76.215.117\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"162.244.32.180-162.244.32.180\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"173.254.223.115-173.254.223.115\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"173.46.85.161-173.46.85.161\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"173.46.85.168-173.46.85.168\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"173.46.85.19-173.46.85.19\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"173.46.85.205-173.46.85.205\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"173.46.85.22-173.46.85.22\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"173.46.85.234-173.46.85.234\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"173.46.85.60-173.46.85.60\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"173.46.85.68-173.46.85.68\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"173.46.85.71-173.46.85.71\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"173.46.85.86-173.46.85.86\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"173.46.85.98-173.46.85.98\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"178.162.132.90-178.162.132.90\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"178.239.21.106-178.239.21.106\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"179.43.176.148-179.43.176.148\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"18.221.114.76-18.221.114.76\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"181.129.146.34-181.129.146.34\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"181.129.171.34-181.129.171.34\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"181.129.93.226-181.129.93.226\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"181.215.247.164-181.215.247.164\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"181.215.47.171-181.215.47.171\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.125.205.69-185.125.205.69\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.125.205.73-185.125.205.73\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.125.205.75-185.125.205.75\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.125.205.78-185.125.205.78\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.125.205.79-185.125.205.79\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.125.205.91-185.125.205.91\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.127.27.238-185.127.27.238\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.141.62.213-185.141.62.213\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.156.174.115-185.156.174.115\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.158.248.92-185.158.248.92\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.158.249.233-185.158.249.233\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.158.251.60-185.158.251.60\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.174.173.128-185.174.173.128\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.189.149.187-185.189.149.187\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.202.174.91-185.202.174.91\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.203.118.6-185.203.118.6\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.205.210.139-185.205.210.139\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.212.47.103-185.212.47.103\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.22.65.5-185.22.65.5\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.223.163.26-185.223.163.26\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.231.153.46-185.231.153.46\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.236.203.53-185.236.203.53\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.236.203.60-185.236.203.60\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.244.30.101-185.244.30.101\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.244.30.105-185.244.30.105\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.244.30.106-185.244.30.106\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.244.30.109-185.244.30.109\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.244.30.111-185.244.30.111\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.244.30.113-185.244.30.113\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.244.30.114-185.244.30.114\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.244.30.120-185.244.30.120\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.244.30.121-185.244.30.121\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.244.30.124-185.244.30.124\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.244.30.93-185.244.30.93\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"185.77.129.11-185.77.129.11\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"186.147.161.204-186.147.161.204\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"186.167.66.51-186.167.66.51\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"187.19.17.132-187.19.17.132\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"192.227.248.175-192.227.248.175\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"192.99.212.140-192.99.212.140\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"193.29.56.44-193.29.56.44\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.98.104-194.5.98.104\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.98.139-194.5.98.139\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.98.148-194.5.98.148\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.98.193-194.5.98.193\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.98.194-194.5.98.194\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.98.226-194.5.98.226\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.98.38-194.5.98.38\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.98.56-194.5.98.56\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.119-194.5.99.119\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.136-194.5.99.136\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.158-194.5.99.158\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.159-194.5.99.159\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.175-194.5.99.175\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.2-194.5.99.2\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.207-194.5.99.207\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.226-194.5.99.226\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.250-194.5.99.250\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.59-194.5.99.59\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.63-194.5.99.63\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.67-194.5.99.67\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.7-194.5.99.7\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.5.99.97-194.5.99.97\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.68.225.63-194.68.225.63\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.76.225.59-194.76.225.59\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"194.99.20.254-194.99.20.254\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"195.123.212.149-195.123.212.149\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"195.123.213.169-195.123.213.169\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"195.123.227.20-195.123.227.20\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"195.123.245.214-195.123.245.214\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"195.123.245.90-195.123.245.90\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"199.21.106.189-199.21.106.189\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"202.63.242.48-202.63.242.48\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"204.95.99.204-204.95.99.204\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"209.58.186.245-209.58.186.245\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"212.47.194.15-212.47.194.15\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"212.73.150.215-212.73.150.215\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"213.152.161.138-213.152.161.138\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"24.217.192.131-24.217.192.131\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"24.247.181.155-24.247.181.155\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"24.247.182.169-24.247.182.169\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"24.247.182.240-24.247.182.240\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"24.247.182.253-24.247.182.253\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"31.171.152.103-31.171.152.103\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"31.171.152.105-31.171.152.105\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"31.171.152.106-31.171.152.106\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"31.171.152.107-31.171.152.107\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"31.7.188.40-31.7.188.40\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"35.198.61.54-35.198.61.54\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"37.59.134.55-37.59.134.55\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"45.249.90.124-45.249.90.124\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"45.55.36.231-45.55.36.231\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"46.166.173.109-46.166.173.109\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"46.17.45.29-46.17.45.29\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"46.17.47.216-46.17.47.216\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"46.183.223.10-46.183.223.10\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"47.44.54.70-47.44.54.70\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"5.2.64.188-5.2.64.188\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"5.2.67.66-5.2.67.66\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"5.206.225.115-5.206.225.115\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"5.8.88.125-5.8.88.125\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"54.37.86.44-54.37.86.44\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"54.38.146.43-54.38.146.43\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"68.111.123.100-68.111.123.100\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"68.183.249.84-68.183.249.84\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"72.189.124.41-72.189.124.41\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"76.107.90.235-76.107.90.235\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"78.155.220.198-78.155.220.198\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"81.177.141.211-81.177.141.211\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"81.177.180.174-81.177.180.174\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"82.199.134.139-82.199.134.139\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"82.199.134.156-82.199.134.156\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"83.166.245.213-83.166.245.213\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"85.217.170.62-85.217.170.62\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"87.236.22.142-87.236.22.142\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"91.192.100.16-91.192.100.16\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"91.192.100.27-91.192.100.27\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"91.192.100.3-91.192.100.3\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"91.192.100.40-91.192.100.40\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"91.192.100.44-91.192.100.44\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"91.192.100.48-91.192.100.48\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"91.192.100.52-91.192.100.52\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"92.222.10.99-92.222.10.99\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"93.115.26.171-93.115.26.171\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"94.103.83.137-94.103.83.137\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"94.185.86.56-94.185.86.56\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"94.237.44.31-94.237.44.31\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"95.213.251.165-95.213.251.165\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"95.47.161.68-95.47.161.68\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n\u001e{\"indicator\":\"97.87.175.152-97.87.175.152\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1550570114477,\"type\":\"IPv4\",\"share_level\":\"red\",\"last_seen\":1550570114477}}\n" }, { "path": "tests/integration/basic/IPv4HC%3Fv%3Dmwg.result", "content": "type=string\n\"1.1.1.0-1.1.1.33\" \"localdb\"\n\"1.1.2.0-1.1.2.255\" \"localdb\"\n\"1.1.3.10-1.1.3.44\" \"localdb\"\n\"103.249.88.244-103.249.88.244\" \"localdb\"\n\"103.89.88.88-103.89.88.88\" \"localdb\"\n\"104.18.34.162-104.18.34.162\" \"localdb\"\n\"108.170.60.189-108.170.60.189\" \"localdb\"\n\"109.230.199.159-109.230.199.159\" \"localdb\"\n\"109.230.199.169-109.230.199.169\" \"localdb\"\n\"109.230.199.30-109.230.199.30\" \"localdb\"\n\"125.209.82.158-125.209.82.158\" \"localdb\"\n\"136.25.2.43-136.25.2.43\" \"localdb\"\n\"137.74.131.18-137.74.131.18\" \"localdb\"\n\"138.197.148.53-138.197.148.53\" \"localdb\"\n\"140.82.48.224-140.82.48.224\" \"localdb\"\n\"144.76.215.117-144.76.215.117\" \"localdb\"\n\"162.244.32.180-162.244.32.180\" \"localdb\"\n\"173.254.223.115-173.254.223.115\" \"localdb\"\n\"173.46.85.161-173.46.85.161\" \"localdb\"\n\"173.46.85.168-173.46.85.168\" \"localdb\"\n\"173.46.85.19-173.46.85.19\" \"localdb\"\n\"173.46.85.205-173.46.85.205\" \"localdb\"\n\"173.46.85.22-173.46.85.22\" \"localdb\"\n\"173.46.85.234-173.46.85.234\" \"localdb\"\n\"173.46.85.60-173.46.85.60\" \"localdb\"\n\"173.46.85.68-173.46.85.68\" \"localdb\"\n\"173.46.85.71-173.46.85.71\" \"localdb\"\n\"173.46.85.86-173.46.85.86\" \"localdb\"\n\"173.46.85.98-173.46.85.98\" \"localdb\"\n\"178.162.132.90-178.162.132.90\" \"localdb\"\n\"178.239.21.106-178.239.21.106\" \"localdb\"\n\"179.43.176.148-179.43.176.148\" \"localdb\"\n\"18.221.114.76-18.221.114.76\" \"localdb\"\n\"181.129.146.34-181.129.146.34\" \"localdb\"\n\"181.129.171.34-181.129.171.34\" \"localdb\"\n\"181.129.93.226-181.129.93.226\" \"localdb\"\n\"181.215.247.164-181.215.247.164\" \"localdb\"\n\"181.215.47.171-181.215.47.171\" \"localdb\"\n\"185.125.205.69-185.125.205.69\" \"localdb\"\n\"185.125.205.73-185.125.205.73\" \"localdb\"\n\"185.125.205.75-185.125.205.75\" \"localdb\"\n\"185.125.205.78-185.125.205.78\" \"localdb\"\n\"185.125.205.79-185.125.205.79\" \"localdb\"\n\"185.125.205.91-185.125.205.91\" \"localdb\"\n\"185.127.27.238-185.127.27.238\" \"localdb\"\n\"185.141.62.213-185.141.62.213\" \"localdb\"\n\"185.156.174.115-185.156.174.115\" \"localdb\"\n\"185.158.248.92-185.158.248.92\" \"localdb\"\n\"185.158.249.233-185.158.249.233\" \"localdb\"\n\"185.158.251.60-185.158.251.60\" \"localdb\"\n\"185.174.173.128-185.174.173.128\" \"localdb\"\n\"185.189.149.187-185.189.149.187\" \"localdb\"\n\"185.202.174.91-185.202.174.91\" \"localdb\"\n\"185.203.118.6-185.203.118.6\" \"localdb\"\n\"185.205.210.139-185.205.210.139\" \"localdb\"\n\"185.212.47.103-185.212.47.103\" \"localdb\"\n\"185.22.65.5-185.22.65.5\" \"localdb\"\n\"185.223.163.26-185.223.163.26\" \"localdb\"\n\"185.231.153.46-185.231.153.46\" \"localdb\"\n\"185.236.203.53-185.236.203.53\" \"localdb\"\n\"185.236.203.60-185.236.203.60\" \"localdb\"\n\"185.244.30.101-185.244.30.101\" \"localdb\"\n\"185.244.30.105-185.244.30.105\" \"localdb\"\n\"185.244.30.106-185.244.30.106\" \"localdb\"\n\"185.244.30.109-185.244.30.109\" \"localdb\"\n\"185.244.30.111-185.244.30.111\" \"localdb\"\n\"185.244.30.113-185.244.30.113\" \"localdb\"\n\"185.244.30.114-185.244.30.114\" \"localdb\"\n\"185.244.30.120-185.244.30.120\" \"localdb\"\n\"185.244.30.121-185.244.30.121\" \"localdb\"\n\"185.244.30.124-185.244.30.124\" \"localdb\"\n\"185.244.30.93-185.244.30.93\" \"localdb\"\n\"185.77.129.11-185.77.129.11\" \"localdb\"\n\"186.147.161.204-186.147.161.204\" \"localdb\"\n\"186.167.66.51-186.167.66.51\" \"localdb\"\n\"187.19.17.132-187.19.17.132\" \"localdb\"\n\"192.227.248.175-192.227.248.175\" \"localdb\"\n\"192.99.212.140-192.99.212.140\" \"localdb\"\n\"193.29.56.44-193.29.56.44\" \"localdb\"\n\"194.5.98.104-194.5.98.104\" \"localdb\"\n\"194.5.98.139-194.5.98.139\" \"localdb\"\n\"194.5.98.148-194.5.98.148\" \"localdb\"\n\"194.5.98.193-194.5.98.193\" \"localdb\"\n\"194.5.98.194-194.5.98.194\" \"localdb\"\n\"194.5.98.226-194.5.98.226\" \"localdb\"\n\"194.5.98.38-194.5.98.38\" \"localdb\"\n\"194.5.98.56-194.5.98.56\" \"localdb\"\n\"194.5.99.119-194.5.99.119\" \"localdb\"\n\"194.5.99.136-194.5.99.136\" \"localdb\"\n\"194.5.99.158-194.5.99.158\" \"localdb\"\n\"194.5.99.159-194.5.99.159\" \"localdb\"\n\"194.5.99.175-194.5.99.175\" \"localdb\"\n\"194.5.99.2-194.5.99.2\" \"localdb\"\n\"194.5.99.207-194.5.99.207\" \"localdb\"\n\"194.5.99.226-194.5.99.226\" \"localdb\"\n\"194.5.99.250-194.5.99.250\" \"localdb\"\n\"194.5.99.59-194.5.99.59\" \"localdb\"\n\"194.5.99.63-194.5.99.63\" \"localdb\"\n\"194.5.99.67-194.5.99.67\" \"localdb\"\n\"194.5.99.7-194.5.99.7\" \"localdb\"\n\"194.5.99.97-194.5.99.97\" \"localdb\"\n\"194.68.225.63-194.68.225.63\" \"localdb\"\n\"194.76.225.59-194.76.225.59\" \"localdb\"\n\"194.99.20.254-194.99.20.254\" \"localdb\"\n\"195.123.212.149-195.123.212.149\" \"localdb\"\n\"195.123.213.169-195.123.213.169\" \"localdb\"\n\"195.123.227.20-195.123.227.20\" \"localdb\"\n\"195.123.245.214-195.123.245.214\" \"localdb\"\n\"195.123.245.90-195.123.245.90\" \"localdb\"\n\"199.21.106.189-199.21.106.189\" \"localdb\"\n\"202.63.242.48-202.63.242.48\" \"localdb\"\n\"204.95.99.204-204.95.99.204\" \"localdb\"\n\"209.58.186.245-209.58.186.245\" \"localdb\"\n\"212.47.194.15-212.47.194.15\" \"localdb\"\n\"212.73.150.215-212.73.150.215\" \"localdb\"\n\"213.152.161.138-213.152.161.138\" \"localdb\"\n\"24.217.192.131-24.217.192.131\" \"localdb\"\n\"24.247.181.155-24.247.181.155\" \"localdb\"\n\"24.247.182.169-24.247.182.169\" \"localdb\"\n\"24.247.182.240-24.247.182.240\" \"localdb\"\n\"24.247.182.253-24.247.182.253\" \"localdb\"\n\"31.171.152.103-31.171.152.103\" \"localdb\"\n\"31.171.152.105-31.171.152.105\" \"localdb\"\n\"31.171.152.106-31.171.152.106\" \"localdb\"\n\"31.171.152.107-31.171.152.107\" \"localdb\"\n\"31.7.188.40-31.7.188.40\" \"localdb\"\n\"35.198.61.54-35.198.61.54\" \"localdb\"\n\"37.59.134.55-37.59.134.55\" \"localdb\"\n\"45.249.90.124-45.249.90.124\" \"localdb\"\n\"45.55.36.231-45.55.36.231\" \"localdb\"\n\"46.166.173.109-46.166.173.109\" \"localdb\"\n\"46.17.45.29-46.17.45.29\" \"localdb\"\n\"46.17.47.216-46.17.47.216\" \"localdb\"\n\"46.183.223.10-46.183.223.10\" \"localdb\"\n\"47.44.54.70-47.44.54.70\" \"localdb\"\n\"5.2.64.188-5.2.64.188\" \"localdb\"\n\"5.2.67.66-5.2.67.66\" \"localdb\"\n\"5.206.225.115-5.206.225.115\" \"localdb\"\n\"5.8.88.125-5.8.88.125\" \"localdb\"\n\"54.37.86.44-54.37.86.44\" \"localdb\"\n\"54.38.146.43-54.38.146.43\" \"localdb\"\n\"68.111.123.100-68.111.123.100\" \"localdb\"\n\"68.183.249.84-68.183.249.84\" \"localdb\"\n\"72.189.124.41-72.189.124.41\" \"localdb\"\n\"76.107.90.235-76.107.90.235\" \"localdb\"\n\"78.155.220.198-78.155.220.198\" \"localdb\"\n\"81.177.141.211-81.177.141.211\" \"localdb\"\n\"81.177.180.174-81.177.180.174\" \"localdb\"\n\"82.199.134.139-82.199.134.139\" \"localdb\"\n\"82.199.134.156-82.199.134.156\" \"localdb\"\n\"83.166.245.213-83.166.245.213\" \"localdb\"\n\"85.217.170.62-85.217.170.62\" \"localdb\"\n\"87.236.22.142-87.236.22.142\" \"localdb\"\n\"91.192.100.16-91.192.100.16\" \"localdb\"\n\"91.192.100.27-91.192.100.27\" \"localdb\"\n\"91.192.100.3-91.192.100.3\" \"localdb\"\n\"91.192.100.40-91.192.100.40\" \"localdb\"\n\"91.192.100.44-91.192.100.44\" \"localdb\"\n\"91.192.100.48-91.192.100.48\" \"localdb\"\n\"91.192.100.52-91.192.100.52\" \"localdb\"\n\"92.222.10.99-92.222.10.99\" \"localdb\"\n\"93.115.26.171-93.115.26.171\" \"localdb\"\n\"94.103.83.137-94.103.83.137\" \"localdb\"\n\"94.185.86.56-94.185.86.56\" \"localdb\"\n\"94.237.44.31-94.237.44.31\" \"localdb\"\n\"95.213.251.165-95.213.251.165\" \"localdb\"\n\"95.47.161.68-95.47.161.68\" \"localdb\"\n\"97.87.175.152-97.87.175.152\" \"localdb\"\n" }, { "path": "tests/integration/basic/IPv4HC.result", "content": "1.1.1.0-1.1.1.33\n1.1.2.0-1.1.2.255\n1.1.3.10-1.1.3.44\n103.249.88.244-103.249.88.244\n103.89.88.88-103.89.88.88\n104.18.34.162-104.18.34.162\n108.170.60.189-108.170.60.189\n109.230.199.159-109.230.199.159\n109.230.199.169-109.230.199.169\n109.230.199.30-109.230.199.30\n125.209.82.158-125.209.82.158\n136.25.2.43-136.25.2.43\n137.74.131.18-137.74.131.18\n138.197.148.53-138.197.148.53\n140.82.48.224-140.82.48.224\n144.76.215.117-144.76.215.117\n162.244.32.180-162.244.32.180\n173.254.223.115-173.254.223.115\n173.46.85.161-173.46.85.161\n173.46.85.168-173.46.85.168\n173.46.85.19-173.46.85.19\n173.46.85.205-173.46.85.205\n173.46.85.22-173.46.85.22\n173.46.85.234-173.46.85.234\n173.46.85.60-173.46.85.60\n173.46.85.68-173.46.85.68\n173.46.85.71-173.46.85.71\n173.46.85.86-173.46.85.86\n173.46.85.98-173.46.85.98\n178.162.132.90-178.162.132.90\n178.239.21.106-178.239.21.106\n179.43.176.148-179.43.176.148\n18.221.114.76-18.221.114.76\n181.129.146.34-181.129.146.34\n181.129.171.34-181.129.171.34\n181.129.93.226-181.129.93.226\n181.215.247.164-181.215.247.164\n181.215.47.171-181.215.47.171\n185.125.205.69-185.125.205.69\n185.125.205.73-185.125.205.73\n185.125.205.75-185.125.205.75\n185.125.205.78-185.125.205.78\n185.125.205.79-185.125.205.79\n185.125.205.91-185.125.205.91\n185.127.27.238-185.127.27.238\n185.141.62.213-185.141.62.213\n185.156.174.115-185.156.174.115\n185.158.248.92-185.158.248.92\n185.158.249.233-185.158.249.233\n185.158.251.60-185.158.251.60\n185.174.173.128-185.174.173.128\n185.189.149.187-185.189.149.187\n185.202.174.91-185.202.174.91\n185.203.118.6-185.203.118.6\n185.205.210.139-185.205.210.139\n185.212.47.103-185.212.47.103\n185.22.65.5-185.22.65.5\n185.223.163.26-185.223.163.26\n185.231.153.46-185.231.153.46\n185.236.203.53-185.236.203.53\n185.236.203.60-185.236.203.60\n185.244.30.101-185.244.30.101\n185.244.30.105-185.244.30.105\n185.244.30.106-185.244.30.106\n185.244.30.109-185.244.30.109\n185.244.30.111-185.244.30.111\n185.244.30.113-185.244.30.113\n185.244.30.114-185.244.30.114\n185.244.30.120-185.244.30.120\n185.244.30.121-185.244.30.121\n185.244.30.124-185.244.30.124\n185.244.30.93-185.244.30.93\n185.77.129.11-185.77.129.11\n186.147.161.204-186.147.161.204\n186.167.66.51-186.167.66.51\n187.19.17.132-187.19.17.132\n192.227.248.175-192.227.248.175\n192.99.212.140-192.99.212.140\n193.29.56.44-193.29.56.44\n194.5.98.104-194.5.98.104\n194.5.98.139-194.5.98.139\n194.5.98.148-194.5.98.148\n194.5.98.193-194.5.98.193\n194.5.98.194-194.5.98.194\n194.5.98.226-194.5.98.226\n194.5.98.38-194.5.98.38\n194.5.98.56-194.5.98.56\n194.5.99.119-194.5.99.119\n194.5.99.136-194.5.99.136\n194.5.99.158-194.5.99.158\n194.5.99.159-194.5.99.159\n194.5.99.175-194.5.99.175\n194.5.99.2-194.5.99.2\n194.5.99.207-194.5.99.207\n194.5.99.226-194.5.99.226\n194.5.99.250-194.5.99.250\n194.5.99.59-194.5.99.59\n194.5.99.63-194.5.99.63\n194.5.99.67-194.5.99.67\n194.5.99.7-194.5.99.7\n194.5.99.97-194.5.99.97\n194.68.225.63-194.68.225.63\n194.76.225.59-194.76.225.59\n194.99.20.254-194.99.20.254\n195.123.212.149-195.123.212.149\n195.123.213.169-195.123.213.169\n195.123.227.20-195.123.227.20\n195.123.245.214-195.123.245.214\n195.123.245.90-195.123.245.90\n199.21.106.189-199.21.106.189\n202.63.242.48-202.63.242.48\n204.95.99.204-204.95.99.204\n209.58.186.245-209.58.186.245\n212.47.194.15-212.47.194.15\n212.73.150.215-212.73.150.215\n213.152.161.138-213.152.161.138\n24.217.192.131-24.217.192.131\n24.247.181.155-24.247.181.155\n24.247.182.169-24.247.182.169\n24.247.182.240-24.247.182.240\n24.247.182.253-24.247.182.253\n31.171.152.103-31.171.152.103\n31.171.152.105-31.171.152.105\n31.171.152.106-31.171.152.106\n31.171.152.107-31.171.152.107\n31.7.188.40-31.7.188.40\n35.198.61.54-35.198.61.54\n37.59.134.55-37.59.134.55\n45.249.90.124-45.249.90.124\n45.55.36.231-45.55.36.231\n46.166.173.109-46.166.173.109\n46.17.45.29-46.17.45.29\n46.17.47.216-46.17.47.216\n46.183.223.10-46.183.223.10\n47.44.54.70-47.44.54.70\n5.2.64.188-5.2.64.188\n5.2.67.66-5.2.67.66\n5.206.225.115-5.206.225.115\n5.8.88.125-5.8.88.125\n54.37.86.44-54.37.86.44\n54.38.146.43-54.38.146.43\n68.111.123.100-68.111.123.100\n68.183.249.84-68.183.249.84\n72.189.124.41-72.189.124.41\n76.107.90.235-76.107.90.235\n78.155.220.198-78.155.220.198\n81.177.141.211-81.177.141.211\n81.177.180.174-81.177.180.174\n82.199.134.139-82.199.134.139\n82.199.134.156-82.199.134.156\n83.166.245.213-83.166.245.213\n85.217.170.62-85.217.170.62\n87.236.22.142-87.236.22.142\n91.192.100.16-91.192.100.16\n91.192.100.27-91.192.100.27\n91.192.100.3-91.192.100.3\n91.192.100.40-91.192.100.40\n91.192.100.44-91.192.100.44\n91.192.100.48-91.192.100.48\n91.192.100.52-91.192.100.52\n92.222.10.99-92.222.10.99\n93.115.26.171-93.115.26.171\n94.103.83.137-94.103.83.137\n94.185.86.56-94.185.86.56\n94.237.44.31-94.237.44.31\n95.213.251.165-95.213.251.165\n95.47.161.68-95.47.161.68\n97.87.175.152-97.87.175.152\n" }, { "path": "tests/integration/basic/README.md", "content": "# Simple script to exercise and test feed output formats.\n\n## Usage\n\n```\nexport MM_URL=https://192.168.55.169\nexport MM_USERNAME=admin\nexport MM_PASSWORD=minemeld\n\n./test.py\n```\n\n## Warning\n\nURL.lst contains a list of malicious URLs, do not click\n" }, { "path": "tests/integration/basic/URL.lst", "content": "http://7-eleven-handbags.com/X1rZYp.php\nhttp://8vs.com/6jezbr.php\nhttp://abdal.com.ua/7_jzay.php\nhttp://aditaborai.com.br/WgNGXe.php\nhttp://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php\nhttp://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php\nhttp://allstarpaintbody.com/lrQ2bG.php\nhttp://americancorner.udp.cl/etloxW.php\nhttp://ample-sun.eu/4BKEt7.php\nhttp://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php\nhttp://anoukdelecluse.nl/lGZLB1.php\nhttp://appeum.com/wp-content/themes/cc.php\nhttp://arot.altervista.org/KHTUdq.php\nhttp://arttoday.sk/mE8MKJ.php\nhttp://ascortimisoara.ro/kWIH5V.php\nhttp://aspectdesigns.com.au/0rTVlG.php\nhttp://audetlaw.com/LnVAdF.php\nhttp://autohaus-seevetal.com/9x6UwK.php\nhttp://avancarvisual.com.br/wp-content/themes/twentytwelve/VzkgnX.php\nhttp://babylicious.ie/s1GHUZ.php\nhttp://balkanium.altervista.org/p3er4s.php\nhttp://beachhouseplans.com/wp-admin/js/5d8gMe.php\nhttp://best-service.jp/olxu2Y.php\nhttp://beyondthedog.net/edHDvf.php\nhttp://bigboattravel.com/uploads/3/5/4/5/3545341/header_images/NthjHz.php\nhttp://bisofit.com/QXwm4I.php\nhttp://bktrade.kiev.ua/76b3ZQ.php\nhttp://boilersandfurnaces.com/uploads/3/5/1/6/3516773/RPyH2q.php\nhttp://bolizarsospos.com/0l0vp1va6b2\nhttp://bolizarsospos.com/1cslstk2qv121\nhttp://bolizarsospos.com/1xb81c28qs2db\nhttp://bolizarsospos.com/22o1210hbpw\nhttp://bolizarsospos.com/2h6t511wpuvnych\nhttp://bolizarsospos.com/379gz635s3j946\nhttp://bolizarsospos.com/4kpy8ju42x137\nhttp://bolizarsospos.com/503qu7boexyk\nhttp://bolizarsospos.com/574xl5yme0gdz\nhttp://bolizarsospos.com/5gpf7ecxhf\nhttp://bolizarsospos.com/5hmwl5qvpz2f3gc\nhttp://bolizarsospos.com/6m50uk8ty1031\nhttp://bolizarsospos.com/6tvpgu93q4wx5t\nhttp://bolizarsospos.com/703hjdr3ez72\nhttp://bolizarsospos.com/73075bdj8meb\nhttp://bolizarsospos.com/7gr904pzv6\nhttp://bolizarsospos.com/7ms68qsdfj0jt\nhttp://bolizarsospos.com/89e8f40k8zcn38\nhttp://bolizarsospos.com/8eo5zwhh4zndwwa\nhttp://bolizarsospos.com/8tsdhjccoxz6c\nhttp://bolizarsospos.com/94g2mr36b4\nhttp://bolizarsospos.com/9bqdnk2h58ty2l\nhttp://bolizarsospos.com/9hul78mtg1n63\nhttp://bolizarsospos.com/b0slgvfxvyf\nhttp://bolizarsospos.com/b3amhlkiar2c\nhttp://bolizarsospos.com/b8g7g560612\nhttp://bolizarsospos.com/bo5ha9ild1zjukv\nhttp://bolizarsospos.com/cannzqzrum14o4c\nhttp://bolizarsospos.com/ch3eq62ad8k\nhttp://bolizarsospos.com/ci72o4ruf2y87\nhttp://bolizarsospos.com/d5i52z8cgv5\nhttp://bolizarsospos.com/d65v4fx21f\nhttp://bolizarsospos.com/d7jly5f09tqj\nhttp://bolizarsospos.com/di53su4z7uqvj\nhttp://bolizarsospos.com/dypi31624z\nhttp://bolizarsospos.com/e5tkclwq9w0\nhttp://bolizarsospos.com/e887nn5k9pb6\nhttp://bolizarsospos.com/f1s0y87wrwo\nhttp://bolizarsospos.com/fgivit1drjuh\nhttp://bolizarsospos.com/fpkirizbrzxc5\nhttp://bolizarsospos.com/fxoztyxp320q\nhttp://bolizarsospos.com/g5k4uvxghygg7r\nhttp://bolizarsospos.com/gvi00me81aabu\nhttp://bolizarsospos.com/hq5drme48h\nhttp://bolizarsospos.com/hzpz767vze9\nhttp://bolizarsospos.com/ifkhfc5369az88\nhttp://bolizarsospos.com/iijoama0ynrowtp\nhttp://bolizarsospos.com/j4hzoz8cgdeza\nhttp://bolizarsospos.com/kka7641ov7\nhttp://bolizarsospos.com/ko679ybid6ys58\nhttp://bolizarsospos.com/kxdmlkhmuyf9\nhttp://bolizarsospos.com/kzqnheutxkjwhr\nhttp://bolizarsospos.com/mi5b67bilrfu\nhttp://bolizarsospos.com/n2csus3eo1tyg\nhttp://bolizarsospos.com/nve4m67l83\nhttp://bolizarsospos.com/o0nyjlre41o3\nhttp://bolizarsospos.com/pgjcokoi2kisu\nhttp://bolizarsospos.com/pl36lz43r6r7\nhttp://bolizarsospos.com/q3xryv3mh1\nhttp://bolizarsospos.com/qely217wcjdl7b\nhttp://bolizarsospos.com/qo9ux20lo1\nhttp://bolizarsospos.com/qu9ajlxsiw\nhttp://bolizarsospos.com/r45byxsjhz\nhttp://bolizarsospos.com/raph9xccgxt\nhttp://bolizarsospos.com/rb05hez1r044\nhttp://bolizarsospos.com/rdjg0eb5r0qs\nhttp://bolizarsospos.com/ri86nx23dhqbmch\nhttp://bolizarsospos.com/rjotoddb4n7hl\nhttp://bolizarsospos.com/rof06587c1x2y3t\nhttp://bolizarsospos.com/s40o542jt7v\nhttp://bolizarsospos.com/sb2zarf5vy\nhttp://bolizarsospos.com/uamuxps7y98\nhttp://bolizarsospos.com/uiyi9dkf5bs\nhttp://bolizarsospos.com/v13rw8n8w2\nhttp://bolizarsospos.com/vzum6ywdedxjtd\nhttp://bolizarsospos.com/walqb5xzunmr\nhttp://bolizarsospos.com/wilqkaz24rnqli\nhttp://bolizarsospos.com/x1tg111bara5\nhttp://bolizarsospos.com/x753k2s01gnd5b\nhttp://bolizarsospos.com/x7lfazpjuuiel\nhttp://bolizarsospos.com/xgw1o6gt9h8k9g\nhttp://bolizarsospos.com/xjp3zmw6glginuq\nhttp://bolizarsospos.com/yias364ajr\nhttp://bolizarsospos.com/zyayxp2kpay\nhttp://bucksmedia.go2cloud.org/aff_c\nhttp://building.msu.ac.th/q3Bslr.php\nhttp://businessaviators.com/r1doyF.php\nhttp://challengestrata.com.au/fP_BXS.php\nhttp://cheapshirts.us/zVnMrG.php\nhttp://chong.joelle.free.fr/_L43PH.php\nhttp://connectao.com/wp-content/themes/twentyeleven/cc.php\nhttp://*.feyda.net/hOeDr4.php\nhttp://d3mpd.fe.uns.ac.id/XPgmur.php\nhttp://daffamedia.com/wp-content/plugins/wp_module/img5.php\nhttp://dechehang.com/GZ2QRn.php\nhttp://definitionen.de/v7GVES.php\nhttp://dichiro.com/WaIrd6.php\nhttp://dillardvideo.com/wp-admin/network/2.php\nhttp://dining-bar.com/BQ_Ln4.php\nhttp://domaine-cassillac.com/4q3esU.php\nhttp://double-wing.de/DZkCLR.php\nhttp://drdigitalmd.com/img1.php\nhttp://eatside.es/xZQGXV.php\nhttp://ecocalsots.com/N79GTA.php\nhttp://ecolux-comfort.com/nPAbsy.php\nhttp://elcoachingempresarial.com/wp-admin/user/2.php\nhttp://emprende21.es/oTIq7A.php\nhttp://estudiobarco.com.ar/5TFv7E.php\nhttp://event-travel.co.uk/3K6Psd.php\nhttp://feuerwehr-stadt-riesa.de/UFiPOq.php\nhttp://foundersomaha.net/wp-includes/Text/Diff/Renderer/ap3.php\nhttp://frame3d.de/ItGJKd.php\nhttp://fun-pop.com/Ks1rCc.php\nhttp://genedillardart.com/wp-admin/network/3.php\nhttp://gibdd.ws/J7D65p.php\nhttp://glitchygaming.com/r07QZu.php\nhttp://grochowina.net/UnvPso.php\nhttp://haarsaloncindy.nl/XzF03r.php\nhttp://hamilton150.co.nz/LmfuMZ.php\nhttp://icsot.na.its.ac.id/8vwRUX.php\nhttp://igatha.com/h4MeKJ.php\nhttp://ilovesport.kiev.ua/z8X9T7.php\nhttp://imagescameraclub.com/j7b5kK.php\nhttp://inspirenetworks.in/vAqu3L.php\nhttp://italyprego.com/Lf2dcA.php\nhttp://jadwalpialadunia.in/rG4Rdi.php\nhttp://jambola.com/LuylWV.php\nhttp://jauregia.net/img5.php\nhttp://konyavakfi.nl/Zmje1r.php\nhttp://kuruyaprak.com/OTLuKo.php\nhttp://kvnysoho.com/eHafFT.php\nhttp://lazymoosestamping.com/YRfbgB.php\nhttp://london-escorts-agency.org.uk/fdnmyD.php\nhttp://lzclient.com/img4.php\nhttp://madisonbootcamps.com/gWQ3wp.php\nhttp://mangohills.net/RxIoCE.php\nhttp://marciogerhardtsouza.com.br/mPCsDz.php\nhttp://marcortes.com/img5.php\nhttp://markossolomon.com/F1q7QX.php\nhttp://maternalserenity.co.uk/I_NwPg.php\nhttp://millsmanagement.nl/AnOgVK.php\nhttp://mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php\nhttp://naimselmonaj.com/QoYx31.php\nhttp://nonnuoccaobang.com/BRdoDL.php\nhttp://nupleta.com.br/KoHV09.php\nhttp://openroadsolutions.com/FJ2dOw.php\nhttp://oregonreversemortgage.com/Rwafp_.php\nhttp://p237996.mybestmv.com/adServe/domainClick\nhttp://paintituppottery.com/6cmeb2.php\nhttp://patrianossa.com.br/u8LkzD.php\nhttp://portalmaismidia.com.br/tnSmIb.php\nhttp://portret-tekening.nl/mNQVts.php\nhttp://procrediti.com.ua/d6yGOX.php\nhttp://rajsima87.com/img2.php\nhttp://recaswine.ro/dXlq0Y.php\nhttp://silstop.pl/Si0cCJ.php\nhttp://smartnote.co/2NxVzA.php\nhttp://studiolegalecsb.it/iQcNfC.php\nhttp://takaram.ir/gjOREZ.php\nhttp://takatei.com/rfYI4L.php\nhttp://tcblog.de/mXdVTh.php\nhttp://theassemblyguy.co.nz/vpFAbQ.php\nhttp://trion.com.ph/jdKAap.php\nhttp://tusrecetas.net/JbElN7.php\nhttp://tutorialswalk.info/wp-content/themes/Defne/img2.php\nhttp://viralcrazies.com/iFHt4C.php\nhttp://vsedveri-33.ru/\nhttp://weberteam.hu/WCTdO5.php\nhttp://www.001edizioni.com/NZwt_a.php\nhttp://www.bishopbell.co.uk/enRmcC.php\nhttp://www.chemes.eu/wp-content/themes/decoy2/redux-framework/ReduxCore/inc/fields/info/2.php\nhttp://www.decorandoimoveis.com/QEO5yh.php\nhttp://www.feddoctor.com/Oe1LMr.php\nhttp://www.gjscomputerservices.com.au/S1_rvm.php\nhttp://www.granmarquise.com.br/6f_8ei.php\nhttp://www.hanecaklaw.com/\nhttp://www.hanoiguidedtours.com/iQ2q1f.php\nhttp://www.healthstafftravel.com.au/oyBbUs.php\nhttp://www.plexipr.com/vAHzWX.php\nhttp://www.rippedknees.co.uk/TXmJcq.php\nhttp://www.taoblu.com/wp-content/plugins/wp_module/sbML0j.php\nhttp://www.vishvagujarat.com/5of9dt.php\nhttp://www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkG.php\n*abc.example.com/foobar?a=:80\nhttps://*abc.*test.com\nhttp://www.example.com:1000\n" }, { "path": "tests/integration/basic/URLHC%3Fs%3D5%26n%3D10.result", "content": "http://aditaborai.com.br/WgNGXe.php\nhttp://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php\nhttp://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php\nhttp://allstarpaintbody.com/lrQ2bG.php\nhttp://americancorner.udp.cl/etloxW.php\nhttp://ample-sun.eu/4BKEt7.php\nhttp://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php\nhttp://anoukdelecluse.nl/lGZLB1.php\nhttp://appeum.com/wp-content/themes/cc.php\nhttp://arot.altervista.org/KHTUdq.php\n" }, { "path": "tests/integration/basic/URLHC%3Fv%3Dbluecoat%26cd%3Dtest.result", "content": "define category test\n*.example.com/foobar?a=:80\n*.feyda.net/hoedr4.php\n7-eleven-handbags.com/x1rzyp.php\n8vs.com/6jezbr.php\nabdal.com.ua/7_jzay.php\naditaborai.com.br/wgngxe.php\nairconditioning12601.com/uploads/3/5/7/6/3576233/v5k3za.php\nallreadytravel.com/uploads/3/5/4/9/3549731/header_images/tomae1.php\nallstarpaintbody.com/lrq2bg.php\namericancorner.udp.cl/etloxw.php\nample-sun.eu/4bket7.php\nanime-tuner.square7.ch/wp-content/themes/twentyeleven/mstgk_.php\nanoukdelecluse.nl/lgzlb1.php\nappeum.com/wp-content/themes/cc.php\narot.altervista.org/khtudq.php\narttoday.sk/me8mkj.php\nascortimisoara.ro/kwih5v.php\naspectdesigns.com.au/0rtvlg.php\naudetlaw.com/lnvadf.php\nautohaus-seevetal.com/9x6uwk.php\navancarvisual.com.br/wp-content/themes/twentytwelve/vzkgnx.php\nbabylicious.ie/s1ghuz.php\nbalkanium.altervista.org/p3er4s.php\nbeachhouseplans.com/wp-admin/js/5d8gme.php\nbest-service.jp/olxu2y.php\nbeyondthedog.net/edhdvf.php\nbigboattravel.com/uploads/3/5/4/5/3545341/header_images/nthjhz.php\nbisofit.com/qxwm4i.php\nbktrade.kiev.ua/76b3zq.php\nboilersandfurnaces.com/uploads/3/5/1/6/3516773/rpyh2q.php\nbolizarsospos.com/0l0vp1va6b2\nbolizarsospos.com/1cslstk2qv121\nbolizarsospos.com/1xb81c28qs2db\nbolizarsospos.com/22o1210hbpw\nbolizarsospos.com/2h6t511wpuvnych\nbolizarsospos.com/379gz635s3j946\nbolizarsospos.com/4kpy8ju42x137\nbolizarsospos.com/503qu7boexyk\nbolizarsospos.com/574xl5yme0gdz\nbolizarsospos.com/5gpf7ecxhf\nbolizarsospos.com/5hmwl5qvpz2f3gc\nbolizarsospos.com/6m50uk8ty1031\nbolizarsospos.com/6tvpgu93q4wx5t\nbolizarsospos.com/703hjdr3ez72\nbolizarsospos.com/73075bdj8meb\nbolizarsospos.com/7gr904pzv6\nbolizarsospos.com/7ms68qsdfj0jt\nbolizarsospos.com/89e8f40k8zcn38\nbolizarsospos.com/8eo5zwhh4zndwwa\nbolizarsospos.com/8tsdhjccoxz6c\nbolizarsospos.com/94g2mr36b4\nbolizarsospos.com/9bqdnk2h58ty2l\nbolizarsospos.com/9hul78mtg1n63\nbolizarsospos.com/b0slgvfxvyf\nbolizarsospos.com/b3amhlkiar2c\nbolizarsospos.com/b8g7g560612\nbolizarsospos.com/bo5ha9ild1zjukv\nbolizarsospos.com/cannzqzrum14o4c\nbolizarsospos.com/ch3eq62ad8k\nbolizarsospos.com/ci72o4ruf2y87\nbolizarsospos.com/d5i52z8cgv5\nbolizarsospos.com/d65v4fx21f\nbolizarsospos.com/d7jly5f09tqj\nbolizarsospos.com/di53su4z7uqvj\nbolizarsospos.com/dypi31624z\nbolizarsospos.com/e5tkclwq9w0\nbolizarsospos.com/e887nn5k9pb6\nbolizarsospos.com/f1s0y87wrwo\nbolizarsospos.com/fgivit1drjuh\nbolizarsospos.com/fpkirizbrzxc5\nbolizarsospos.com/fxoztyxp320q\nbolizarsospos.com/g5k4uvxghygg7r\nbolizarsospos.com/gvi00me81aabu\nbolizarsospos.com/hq5drme48h\nbolizarsospos.com/hzpz767vze9\nbolizarsospos.com/ifkhfc5369az88\nbolizarsospos.com/iijoama0ynrowtp\nbolizarsospos.com/j4hzoz8cgdeza\nbolizarsospos.com/kka7641ov7\nbolizarsospos.com/ko679ybid6ys58\nbolizarsospos.com/kxdmlkhmuyf9\nbolizarsospos.com/kzqnheutxkjwhr\nbolizarsospos.com/mi5b67bilrfu\nbolizarsospos.com/n2csus3eo1tyg\nbolizarsospos.com/nve4m67l83\nbolizarsospos.com/o0nyjlre41o3\nbolizarsospos.com/pgjcokoi2kisu\nbolizarsospos.com/pl36lz43r6r7\nbolizarsospos.com/q3xryv3mh1\nbolizarsospos.com/qely217wcjdl7b\nbolizarsospos.com/qo9ux20lo1\nbolizarsospos.com/qu9ajlxsiw\nbolizarsospos.com/r45byxsjhz\nbolizarsospos.com/raph9xccgxt\nbolizarsospos.com/rb05hez1r044\nbolizarsospos.com/rdjg0eb5r0qs\nbolizarsospos.com/ri86nx23dhqbmch\nbolizarsospos.com/rjotoddb4n7hl\nbolizarsospos.com/rof06587c1x2y3t\nbolizarsospos.com/s40o542jt7v\nbolizarsospos.com/sb2zarf5vy\nbolizarsospos.com/uamuxps7y98\nbolizarsospos.com/uiyi9dkf5bs\nbolizarsospos.com/v13rw8n8w2\nbolizarsospos.com/vzum6ywdedxjtd\nbolizarsospos.com/walqb5xzunmr\nbolizarsospos.com/wilqkaz24rnqli\nbolizarsospos.com/x1tg111bara5\nbolizarsospos.com/x753k2s01gnd5b\nbolizarsospos.com/x7lfazpjuuiel\nbolizarsospos.com/xgw1o6gt9h8k9g\nbolizarsospos.com/xjp3zmw6glginuq\nbolizarsospos.com/yias364ajr\nbolizarsospos.com/zyayxp2kpay\nbucksmedia.go2cloud.org/aff_c\nbuilding.msu.ac.th/q3bslr.php\nbusinessaviators.com/r1doyf.php\nchallengestrata.com.au/fp_bxs.php\ncheapshirts.us/zvnmrg.php\nchong.joelle.free.fr/_l43ph.php\nconnectao.com/wp-content/themes/twentyeleven/cc.php\nd3mpd.fe.uns.ac.id/xpgmur.php\ndaffamedia.com/wp-content/plugins/wp_module/img5.php\ndechehang.com/gz2qrn.php\ndefinitionen.de/v7gves.php\ndichiro.com/waird6.php\ndillardvideo.com/wp-admin/network/2.php\ndining-bar.com/bq_ln4.php\ndomaine-cassillac.com/4q3esu.php\ndouble-wing.de/dzkclr.php\ndrdigitalmd.com/img1.php\neatside.es/xzqgxv.php\necocalsots.com/n79gta.php\necolux-comfort.com/npabsy.php\nelcoachingempresarial.com/wp-admin/user/2.php\nemprende21.es/otiq7a.php\nestudiobarco.com.ar/5tfv7e.php\nevent-travel.co.uk/3k6psd.php\nfeuerwehr-stadt-riesa.de/ufipoq.php\nfoundersomaha.net/wp-includes/text/diff/renderer/ap3.php\nframe3d.de/itgjkd.php\nfun-pop.com/ks1rcc.php\ngenedillardart.com/wp-admin/network/3.php\ngibdd.ws/j7d65p.php\nglitchygaming.com/r07qzu.php\ngrochowina.net/unvpso.php\nhaarsaloncindy.nl/xzf03r.php\nhamilton150.co.nz/lmfumz.php\nicsot.na.its.ac.id/8vwrux.php\nigatha.com/h4mekj.php\nilovesport.kiev.ua/z8x9t7.php\nimagescameraclub.com/j7b5kk.php\ninspirenetworks.in/vaqu3l.php\nitalyprego.com/lf2dca.php\njadwalpialadunia.in/rg4rdi.php\njambola.com/luylwv.php\njauregia.net/img5.php\nkonyavakfi.nl/zmje1r.php\nkuruyaprak.com/otluko.php\nkvnysoho.com/ehafft.php\nlazymoosestamping.com/yrfbgb.php\nlondon-escorts-agency.org.uk/fdnmyd.php\nlzclient.com/img4.php\nmadisonbootcamps.com/gwq3wp.php\nmangohills.net/rxioce.php\nmarciogerhardtsouza.com.br/mpcsdz.php\nmarcortes.com/img5.php\nmarkossolomon.com/f1q7qx.php\nmaternalserenity.co.uk/i_nwpg.php\nmillsmanagement.nl/anogvk.php\nmmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php\nnaimselmonaj.com/qoyx31.php\nnonnuoccaobang.com/brdodl.php\nnupleta.com.br/kohv09.php\nopenroadsolutions.com/fj2dow.php\noregonreversemortgage.com/rwafp_.php\np237996.mybestmv.com/adserve/domainclick\npaintituppottery.com/6cmeb2.php\npatrianossa.com.br/u8lkzd.php\nportalmaismidia.com.br/tnsmib.php\nportret-tekening.nl/mnqvts.php\nprocrediti.com.ua/d6ygox.php\nrajsima87.com/img2.php\nrecaswine.ro/dxlq0y.php\nsilstop.pl/si0ccj.php\nsmartnote.co/2nxvza.php\nstudiolegalecsb.it/iqcnfc.php\ntakaram.ir/gjorez.php\ntakatei.com/rfyi4l.php\ntcblog.de/mxdvth.php\ntheassemblyguy.co.nz/vpfabq.php\ntrion.com.ph/jdkaap.php\ntusrecetas.net/jbeln7.php\ntutorialswalk.info/wp-content/themes/defne/img2.php\nviralcrazies.com/ifht4c.php\nvsedveri-33.ru/\nweberteam.hu/wctdo5.php\nwww.001edizioni.com/nzwt_a.php\nwww.bishopbell.co.uk/enrmcc.php\nwww.chemes.eu/wp-content/themes/decoy2/redux-framework/reduxcore/inc/fields/info/2.php\nwww.decorandoimoveis.com/qeo5yh.php\nwww.example.com:1000\nwww.feddoctor.com/oe1lmr.php\nwww.gjscomputerservices.com.au/s1_rvm.php\nwww.granmarquise.com.br/6f_8ei.php\nwww.hanecaklaw.com/\nwww.hanoiguidedtours.com/iq2q1f.php\nwww.healthstafftravel.com.au/oybbus.php\nwww.plexipr.com/vahzwx.php\nwww.rippedknees.co.uk/txmjcq.php\nwww.taoblu.com/wp-content/plugins/wp_module/sbml0j.php\nwww.vishvagujarat.com/5of9dt.php\nwww.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkg.php\n*.*.com\nend\n" }, { "path": "tests/integration/basic/URLHC%3Fv%3Dbluecoat.result", "content": "" }, { "path": "tests/integration/basic/URLHC%3Fv%3Dcsv%26f%3Dconfidence%26f%3Dsources%7Cfeeds%26f%3Dindicator%7Curl.result", "content": "confidence,feeds,url\r\n100,localdb,*abc.example.com/foobar?a=:80\r\n100,localdb,http://*.feyda.net/hOeDr4.php\r\n100,localdb,http://7-eleven-handbags.com/X1rZYp.php\r\n100,localdb,http://8vs.com/6jezbr.php\r\n100,localdb,http://abdal.com.ua/7_jzay.php\r\n100,localdb,http://aditaborai.com.br/WgNGXe.php\r\n100,localdb,http://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php\r\n100,localdb,http://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php\r\n100,localdb,http://allstarpaintbody.com/lrQ2bG.php\r\n100,localdb,http://americancorner.udp.cl/etloxW.php\r\n100,localdb,http://ample-sun.eu/4BKEt7.php\r\n100,localdb,http://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php\r\n100,localdb,http://anoukdelecluse.nl/lGZLB1.php\r\n100,localdb,http://appeum.com/wp-content/themes/cc.php\r\n100,localdb,http://arot.altervista.org/KHTUdq.php\r\n100,localdb,http://arttoday.sk/mE8MKJ.php\r\n100,localdb,http://ascortimisoara.ro/kWIH5V.php\r\n100,localdb,http://aspectdesigns.com.au/0rTVlG.php\r\n100,localdb,http://audetlaw.com/LnVAdF.php\r\n100,localdb,http://autohaus-seevetal.com/9x6UwK.php\r\n100,localdb,http://avancarvisual.com.br/wp-content/themes/twentytwelve/VzkgnX.php\r\n100,localdb,http://babylicious.ie/s1GHUZ.php\r\n100,localdb,http://balkanium.altervista.org/p3er4s.php\r\n100,localdb,http://beachhouseplans.com/wp-admin/js/5d8gMe.php\r\n100,localdb,http://best-service.jp/olxu2Y.php\r\n100,localdb,http://beyondthedog.net/edHDvf.php\r\n100,localdb,http://bigboattravel.com/uploads/3/5/4/5/3545341/header_images/NthjHz.php\r\n100,localdb,http://bisofit.com/QXwm4I.php\r\n100,localdb,http://bktrade.kiev.ua/76b3ZQ.php\r\n100,localdb,http://boilersandfurnaces.com/uploads/3/5/1/6/3516773/RPyH2q.php\r\n100,localdb,http://bolizarsospos.com/0l0vp1va6b2\r\n100,localdb,http://bolizarsospos.com/1cslstk2qv121\r\n100,localdb,http://bolizarsospos.com/1xb81c28qs2db\r\n100,localdb,http://bolizarsospos.com/22o1210hbpw\r\n100,localdb,http://bolizarsospos.com/2h6t511wpuvnych\r\n100,localdb,http://bolizarsospos.com/379gz635s3j946\r\n100,localdb,http://bolizarsospos.com/4kpy8ju42x137\r\n100,localdb,http://bolizarsospos.com/503qu7boexyk\r\n100,localdb,http://bolizarsospos.com/574xl5yme0gdz\r\n100,localdb,http://bolizarsospos.com/5gpf7ecxhf\r\n100,localdb,http://bolizarsospos.com/5hmwl5qvpz2f3gc\r\n100,localdb,http://bolizarsospos.com/6m50uk8ty1031\r\n100,localdb,http://bolizarsospos.com/6tvpgu93q4wx5t\r\n100,localdb,http://bolizarsospos.com/703hjdr3ez72\r\n100,localdb,http://bolizarsospos.com/73075bdj8meb\r\n100,localdb,http://bolizarsospos.com/7gr904pzv6\r\n100,localdb,http://bolizarsospos.com/7ms68qsdfj0jt\r\n100,localdb,http://bolizarsospos.com/89e8f40k8zcn38\r\n100,localdb,http://bolizarsospos.com/8eo5zwhh4zndwwa\r\n100,localdb,http://bolizarsospos.com/8tsdhjccoxz6c\r\n100,localdb,http://bolizarsospos.com/94g2mr36b4\r\n100,localdb,http://bolizarsospos.com/9bqdnk2h58ty2l\r\n100,localdb,http://bolizarsospos.com/9hul78mtg1n63\r\n100,localdb,http://bolizarsospos.com/b0slgvfxvyf\r\n100,localdb,http://bolizarsospos.com/b3amhlkiar2c\r\n100,localdb,http://bolizarsospos.com/b8g7g560612\r\n100,localdb,http://bolizarsospos.com/bo5ha9ild1zjukv\r\n100,localdb,http://bolizarsospos.com/cannzqzrum14o4c\r\n100,localdb,http://bolizarsospos.com/ch3eq62ad8k\r\n100,localdb,http://bolizarsospos.com/ci72o4ruf2y87\r\n100,localdb,http://bolizarsospos.com/d5i52z8cgv5\r\n100,localdb,http://bolizarsospos.com/d65v4fx21f\r\n100,localdb,http://bolizarsospos.com/d7jly5f09tqj\r\n100,localdb,http://bolizarsospos.com/di53su4z7uqvj\r\n100,localdb,http://bolizarsospos.com/dypi31624z\r\n100,localdb,http://bolizarsospos.com/e5tkclwq9w0\r\n100,localdb,http://bolizarsospos.com/e887nn5k9pb6\r\n100,localdb,http://bolizarsospos.com/f1s0y87wrwo\r\n100,localdb,http://bolizarsospos.com/fgivit1drjuh\r\n100,localdb,http://bolizarsospos.com/fpkirizbrzxc5\r\n100,localdb,http://bolizarsospos.com/fxoztyxp320q\r\n100,localdb,http://bolizarsospos.com/g5k4uvxghygg7r\r\n100,localdb,http://bolizarsospos.com/gvi00me81aabu\r\n100,localdb,http://bolizarsospos.com/hq5drme48h\r\n100,localdb,http://bolizarsospos.com/hzpz767vze9\r\n100,localdb,http://bolizarsospos.com/ifkhfc5369az88\r\n100,localdb,http://bolizarsospos.com/iijoama0ynrowtp\r\n100,localdb,http://bolizarsospos.com/j4hzoz8cgdeza\r\n100,localdb,http://bolizarsospos.com/kka7641ov7\r\n100,localdb,http://bolizarsospos.com/ko679ybid6ys58\r\n100,localdb,http://bolizarsospos.com/kxdmlkhmuyf9\r\n100,localdb,http://bolizarsospos.com/kzqnheutxkjwhr\r\n100,localdb,http://bolizarsospos.com/mi5b67bilrfu\r\n100,localdb,http://bolizarsospos.com/n2csus3eo1tyg\r\n100,localdb,http://bolizarsospos.com/nve4m67l83\r\n100,localdb,http://bolizarsospos.com/o0nyjlre41o3\r\n100,localdb,http://bolizarsospos.com/pgjcokoi2kisu\r\n100,localdb,http://bolizarsospos.com/pl36lz43r6r7\r\n100,localdb,http://bolizarsospos.com/q3xryv3mh1\r\n100,localdb,http://bolizarsospos.com/qely217wcjdl7b\r\n100,localdb,http://bolizarsospos.com/qo9ux20lo1\r\n100,localdb,http://bolizarsospos.com/qu9ajlxsiw\r\n100,localdb,http://bolizarsospos.com/r45byxsjhz\r\n100,localdb,http://bolizarsospos.com/raph9xccgxt\r\n100,localdb,http://bolizarsospos.com/rb05hez1r044\r\n100,localdb,http://bolizarsospos.com/rdjg0eb5r0qs\r\n100,localdb,http://bolizarsospos.com/ri86nx23dhqbmch\r\n100,localdb,http://bolizarsospos.com/rjotoddb4n7hl\r\n100,localdb,http://bolizarsospos.com/rof06587c1x2y3t\r\n100,localdb,http://bolizarsospos.com/s40o542jt7v\r\n100,localdb,http://bolizarsospos.com/sb2zarf5vy\r\n100,localdb,http://bolizarsospos.com/uamuxps7y98\r\n100,localdb,http://bolizarsospos.com/uiyi9dkf5bs\r\n100,localdb,http://bolizarsospos.com/v13rw8n8w2\r\n100,localdb,http://bolizarsospos.com/vzum6ywdedxjtd\r\n100,localdb,http://bolizarsospos.com/walqb5xzunmr\r\n100,localdb,http://bolizarsospos.com/wilqkaz24rnqli\r\n100,localdb,http://bolizarsospos.com/x1tg111bara5\r\n100,localdb,http://bolizarsospos.com/x753k2s01gnd5b\r\n100,localdb,http://bolizarsospos.com/x7lfazpjuuiel\r\n100,localdb,http://bolizarsospos.com/xgw1o6gt9h8k9g\r\n100,localdb,http://bolizarsospos.com/xjp3zmw6glginuq\r\n100,localdb,http://bolizarsospos.com/yias364ajr\r\n100,localdb,http://bolizarsospos.com/zyayxp2kpay\r\n100,localdb,http://bucksmedia.go2cloud.org/aff_c\r\n100,localdb,http://building.msu.ac.th/q3Bslr.php\r\n100,localdb,http://businessaviators.com/r1doyF.php\r\n100,localdb,http://challengestrata.com.au/fP_BXS.php\r\n100,localdb,http://cheapshirts.us/zVnMrG.php\r\n100,localdb,http://chong.joelle.free.fr/_L43PH.php\r\n100,localdb,http://connectao.com/wp-content/themes/twentyeleven/cc.php\r\n100,localdb,http://d3mpd.fe.uns.ac.id/XPgmur.php\r\n100,localdb,http://daffamedia.com/wp-content/plugins/wp_module/img5.php\r\n100,localdb,http://dechehang.com/GZ2QRn.php\r\n100,localdb,http://definitionen.de/v7GVES.php\r\n100,localdb,http://dichiro.com/WaIrd6.php\r\n100,localdb,http://dillardvideo.com/wp-admin/network/2.php\r\n100,localdb,http://dining-bar.com/BQ_Ln4.php\r\n100,localdb,http://domaine-cassillac.com/4q3esU.php\r\n100,localdb,http://double-wing.de/DZkCLR.php\r\n100,localdb,http://drdigitalmd.com/img1.php\r\n100,localdb,http://eatside.es/xZQGXV.php\r\n100,localdb,http://ecocalsots.com/N79GTA.php\r\n100,localdb,http://ecolux-comfort.com/nPAbsy.php\r\n100,localdb,http://elcoachingempresarial.com/wp-admin/user/2.php\r\n100,localdb,http://emprende21.es/oTIq7A.php\r\n100,localdb,http://estudiobarco.com.ar/5TFv7E.php\r\n100,localdb,http://event-travel.co.uk/3K6Psd.php\r\n100,localdb,http://feuerwehr-stadt-riesa.de/UFiPOq.php\r\n100,localdb,http://foundersomaha.net/wp-includes/Text/Diff/Renderer/ap3.php\r\n100,localdb,http://frame3d.de/ItGJKd.php\r\n100,localdb,http://fun-pop.com/Ks1rCc.php\r\n100,localdb,http://genedillardart.com/wp-admin/network/3.php\r\n100,localdb,http://gibdd.ws/J7D65p.php\r\n100,localdb,http://glitchygaming.com/r07QZu.php\r\n100,localdb,http://grochowina.net/UnvPso.php\r\n100,localdb,http://haarsaloncindy.nl/XzF03r.php\r\n100,localdb,http://hamilton150.co.nz/LmfuMZ.php\r\n100,localdb,http://icsot.na.its.ac.id/8vwRUX.php\r\n100,localdb,http://igatha.com/h4MeKJ.php\r\n100,localdb,http://ilovesport.kiev.ua/z8X9T7.php\r\n100,localdb,http://imagescameraclub.com/j7b5kK.php\r\n100,localdb,http://inspirenetworks.in/vAqu3L.php\r\n100,localdb,http://italyprego.com/Lf2dcA.php\r\n100,localdb,http://jadwalpialadunia.in/rG4Rdi.php\r\n100,localdb,http://jambola.com/LuylWV.php\r\n100,localdb,http://jauregia.net/img5.php\r\n100,localdb,http://konyavakfi.nl/Zmje1r.php\r\n100,localdb,http://kuruyaprak.com/OTLuKo.php\r\n100,localdb,http://kvnysoho.com/eHafFT.php\r\n100,localdb,http://lazymoosestamping.com/YRfbgB.php\r\n100,localdb,http://london-escorts-agency.org.uk/fdnmyD.php\r\n100,localdb,http://lzclient.com/img4.php\r\n100,localdb,http://madisonbootcamps.com/gWQ3wp.php\r\n100,localdb,http://mangohills.net/RxIoCE.php\r\n100,localdb,http://marciogerhardtsouza.com.br/mPCsDz.php\r\n100,localdb,http://marcortes.com/img5.php\r\n100,localdb,http://markossolomon.com/F1q7QX.php\r\n100,localdb,http://maternalserenity.co.uk/I_NwPg.php\r\n100,localdb,http://millsmanagement.nl/AnOgVK.php\r\n100,localdb,http://mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php\r\n100,localdb,http://naimselmonaj.com/QoYx31.php\r\n100,localdb,http://nonnuoccaobang.com/BRdoDL.php\r\n100,localdb,http://nupleta.com.br/KoHV09.php\r\n100,localdb,http://openroadsolutions.com/FJ2dOw.php\r\n100,localdb,http://oregonreversemortgage.com/Rwafp_.php\r\n100,localdb,http://p237996.mybestmv.com/adServe/domainClick\r\n100,localdb,http://paintituppottery.com/6cmeb2.php\r\n100,localdb,http://patrianossa.com.br/u8LkzD.php\r\n100,localdb,http://portalmaismidia.com.br/tnSmIb.php\r\n100,localdb,http://portret-tekening.nl/mNQVts.php\r\n100,localdb,http://procrediti.com.ua/d6yGOX.php\r\n100,localdb,http://rajsima87.com/img2.php\r\n100,localdb,http://recaswine.ro/dXlq0Y.php\r\n100,localdb,http://silstop.pl/Si0cCJ.php\r\n100,localdb,http://smartnote.co/2NxVzA.php\r\n100,localdb,http://studiolegalecsb.it/iQcNfC.php\r\n100,localdb,http://takaram.ir/gjOREZ.php\r\n100,localdb,http://takatei.com/rfYI4L.php\r\n100,localdb,http://tcblog.de/mXdVTh.php\r\n100,localdb,http://theassemblyguy.co.nz/vpFAbQ.php\r\n100,localdb,http://trion.com.ph/jdKAap.php\r\n100,localdb,http://tusrecetas.net/JbElN7.php\r\n100,localdb,http://tutorialswalk.info/wp-content/themes/Defne/img2.php\r\n100,localdb,http://viralcrazies.com/iFHt4C.php\r\n100,localdb,http://vsedveri-33.ru/\r\n100,localdb,http://weberteam.hu/WCTdO5.php\r\n100,localdb,http://www.001edizioni.com/NZwt_a.php\r\n100,localdb,http://www.bishopbell.co.uk/enRmcC.php\r\n100,localdb,http://www.chemes.eu/wp-content/themes/decoy2/redux-framework/ReduxCore/inc/fields/info/2.php\r\n100,localdb,http://www.decorandoimoveis.com/QEO5yh.php\r\n100,localdb,http://www.example.com:1000\r\n100,localdb,http://www.feddoctor.com/Oe1LMr.php\r\n100,localdb,http://www.gjscomputerservices.com.au/S1_rvm.php\r\n100,localdb,http://www.granmarquise.com.br/6f_8ei.php\r\n100,localdb,http://www.hanecaklaw.com/\r\n100,localdb,http://www.hanoiguidedtours.com/iQ2q1f.php\r\n100,localdb,http://www.healthstafftravel.com.au/oyBbUs.php\r\n100,localdb,http://www.plexipr.com/vAHzWX.php\r\n100,localdb,http://www.rippedknees.co.uk/TXmJcq.php\r\n100,localdb,http://www.taoblu.com/wp-content/plugins/wp_module/sbML0j.php\r\n100,localdb,http://www.vishvagujarat.com/5of9dt.php\r\n100,localdb,http://www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkG.php\r\n100,localdb,https://*abc.*test.com\r\n" }, { "path": "tests/integration/basic/URLHC%3Fv%3Djson%26tr%3D1.result", "content": "[\n{\"indicator\":\"*abc.example.com/foobar?a=:80\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://*.feyda.net/hOeDr4.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://7-eleven-handbags.com/X1rZYp.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://8vs.com/6jezbr.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://abdal.com.ua/7_jzay.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://aditaborai.com.br/WgNGXe.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://allstarpaintbody.com/lrQ2bG.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://americancorner.udp.cl/etloxW.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://ample-sun.eu/4BKEt7.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://anoukdelecluse.nl/lGZLB1.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://appeum.com/wp-content/themes/cc.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://arot.altervista.org/KHTUdq.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://arttoday.sk/mE8MKJ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://ascortimisoara.ro/kWIH5V.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://aspectdesigns.com.au/0rTVlG.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://audetlaw.com/LnVAdF.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://autohaus-seevetal.com/9x6UwK.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://avancarvisual.com.br/wp-content/themes/twentytwelve/VzkgnX.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://babylicious.ie/s1GHUZ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://balkanium.altervista.org/p3er4s.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://beachhouseplans.com/wp-admin/js/5d8gMe.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://best-service.jp/olxu2Y.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://beyondthedog.net/edHDvf.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bigboattravel.com/uploads/3/5/4/5/3545341/header_images/NthjHz.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bisofit.com/QXwm4I.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bktrade.kiev.ua/76b3ZQ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://boilersandfurnaces.com/uploads/3/5/1/6/3516773/RPyH2q.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/0l0vp1va6b2\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/1cslstk2qv121\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/1xb81c28qs2db\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/22o1210hbpw\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/2h6t511wpuvnych\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/379gz635s3j946\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/4kpy8ju42x137\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/503qu7boexyk\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/574xl5yme0gdz\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/5gpf7ecxhf\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/5hmwl5qvpz2f3gc\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/6m50uk8ty1031\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/6tvpgu93q4wx5t\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/703hjdr3ez72\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/73075bdj8meb\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/7gr904pzv6\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/7ms68qsdfj0jt\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/89e8f40k8zcn38\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/8eo5zwhh4zndwwa\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/8tsdhjccoxz6c\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/94g2mr36b4\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/9bqdnk2h58ty2l\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/9hul78mtg1n63\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/b0slgvfxvyf\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/b3amhlkiar2c\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/b8g7g560612\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/bo5ha9ild1zjukv\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/cannzqzrum14o4c\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/ch3eq62ad8k\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/ci72o4ruf2y87\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/d5i52z8cgv5\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/d65v4fx21f\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/d7jly5f09tqj\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/di53su4z7uqvj\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/dypi31624z\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/e5tkclwq9w0\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/e887nn5k9pb6\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/f1s0y87wrwo\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/fgivit1drjuh\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/fpkirizbrzxc5\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/fxoztyxp320q\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/g5k4uvxghygg7r\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/gvi00me81aabu\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/hq5drme48h\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/hzpz767vze9\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/ifkhfc5369az88\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/iijoama0ynrowtp\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/j4hzoz8cgdeza\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/kka7641ov7\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/ko679ybid6ys58\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/kxdmlkhmuyf9\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/kzqnheutxkjwhr\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/mi5b67bilrfu\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/n2csus3eo1tyg\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/nve4m67l83\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/o0nyjlre41o3\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/pgjcokoi2kisu\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/pl36lz43r6r7\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/q3xryv3mh1\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/qely217wcjdl7b\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/qo9ux20lo1\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/qu9ajlxsiw\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/r45byxsjhz\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/raph9xccgxt\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/rb05hez1r044\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/rdjg0eb5r0qs\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/ri86nx23dhqbmch\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/rjotoddb4n7hl\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/rof06587c1x2y3t\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/s40o542jt7v\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/sb2zarf5vy\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/uamuxps7y98\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/uiyi9dkf5bs\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/v13rw8n8w2\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/vzum6ywdedxjtd\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/walqb5xzunmr\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/wilqkaz24rnqli\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/x1tg111bara5\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/x753k2s01gnd5b\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/x7lfazpjuuiel\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/xgw1o6gt9h8k9g\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/xjp3zmw6glginuq\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/yias364ajr\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bolizarsospos.com/zyayxp2kpay\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://bucksmedia.go2cloud.org/aff_c\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://building.msu.ac.th/q3Bslr.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://businessaviators.com/r1doyF.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://challengestrata.com.au/fP_BXS.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://cheapshirts.us/zVnMrG.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://chong.joelle.free.fr/_L43PH.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://connectao.com/wp-content/themes/twentyeleven/cc.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://d3mpd.fe.uns.ac.id/XPgmur.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://daffamedia.com/wp-content/plugins/wp_module/img5.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://dechehang.com/GZ2QRn.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://definitionen.de/v7GVES.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://dichiro.com/WaIrd6.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://dillardvideo.com/wp-admin/network/2.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://dining-bar.com/BQ_Ln4.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://domaine-cassillac.com/4q3esU.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://double-wing.de/DZkCLR.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://drdigitalmd.com/img1.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://eatside.es/xZQGXV.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://ecocalsots.com/N79GTA.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://ecolux-comfort.com/nPAbsy.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://elcoachingempresarial.com/wp-admin/user/2.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://emprende21.es/oTIq7A.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://estudiobarco.com.ar/5TFv7E.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://event-travel.co.uk/3K6Psd.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://feuerwehr-stadt-riesa.de/UFiPOq.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://foundersomaha.net/wp-includes/Text/Diff/Renderer/ap3.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://frame3d.de/ItGJKd.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://fun-pop.com/Ks1rCc.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://genedillardart.com/wp-admin/network/3.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://gibdd.ws/J7D65p.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://glitchygaming.com/r07QZu.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://grochowina.net/UnvPso.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://haarsaloncindy.nl/XzF03r.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://hamilton150.co.nz/LmfuMZ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://icsot.na.its.ac.id/8vwRUX.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://igatha.com/h4MeKJ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://ilovesport.kiev.ua/z8X9T7.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://imagescameraclub.com/j7b5kK.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://inspirenetworks.in/vAqu3L.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://italyprego.com/Lf2dcA.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://jadwalpialadunia.in/rG4Rdi.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://jambola.com/LuylWV.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://jauregia.net/img5.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://konyavakfi.nl/Zmje1r.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://kuruyaprak.com/OTLuKo.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://kvnysoho.com/eHafFT.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://lazymoosestamping.com/YRfbgB.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://london-escorts-agency.org.uk/fdnmyD.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://lzclient.com/img4.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://madisonbootcamps.com/gWQ3wp.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://mangohills.net/RxIoCE.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://marciogerhardtsouza.com.br/mPCsDz.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://marcortes.com/img5.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://markossolomon.com/F1q7QX.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://maternalserenity.co.uk/I_NwPg.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://millsmanagement.nl/AnOgVK.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://naimselmonaj.com/QoYx31.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://nonnuoccaobang.com/BRdoDL.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://nupleta.com.br/KoHV09.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://openroadsolutions.com/FJ2dOw.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://oregonreversemortgage.com/Rwafp_.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://p237996.mybestmv.com/adServe/domainClick\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://paintituppottery.com/6cmeb2.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://patrianossa.com.br/u8LkzD.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://portalmaismidia.com.br/tnSmIb.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://portret-tekening.nl/mNQVts.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://procrediti.com.ua/d6yGOX.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://rajsima87.com/img2.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://recaswine.ro/dXlq0Y.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://silstop.pl/Si0cCJ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://smartnote.co/2NxVzA.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://studiolegalecsb.it/iQcNfC.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://takaram.ir/gjOREZ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://takatei.com/rfYI4L.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://tcblog.de/mXdVTh.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://theassemblyguy.co.nz/vpFAbQ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://trion.com.ph/jdKAap.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://tusrecetas.net/JbElN7.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://tutorialswalk.info/wp-content/themes/Defne/img2.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://viralcrazies.com/iFHt4C.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://vsedveri-33.ru/\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://weberteam.hu/WCTdO5.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.001edizioni.com/NZwt_a.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.bishopbell.co.uk/enRmcC.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.chemes.eu/wp-content/themes/decoy2/redux-framework/ReduxCore/inc/fields/info/2.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.decorandoimoveis.com/QEO5yh.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.example.com:1000\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.feddoctor.com/Oe1LMr.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.gjscomputerservices.com.au/S1_rvm.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.granmarquise.com.br/6f_8ei.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.hanecaklaw.com/\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.hanoiguidedtours.com/iQ2q1f.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.healthstafftravel.com.au/oyBbUs.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.plexipr.com/vAHzWX.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.rippedknees.co.uk/TXmJcq.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.taoblu.com/wp-content/plugins/wp_module/sbML0j.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.vishvagujarat.com/5of9dt.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"http://www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkG.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}},\n{\"indicator\":\"https://*abc.*test.com\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}]\n" }, { "path": "tests/integration/basic/URLHC%3Fv%3Djson-seq.result", "content": "\u001e{\"indicator\":\"*abc.example.com/foobar?a=:80\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://*.feyda.net/hOeDr4.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://7-eleven-handbags.com/X1rZYp.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://8vs.com/6jezbr.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://abdal.com.ua/7_jzay.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://aditaborai.com.br/WgNGXe.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://allstarpaintbody.com/lrQ2bG.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://americancorner.udp.cl/etloxW.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://ample-sun.eu/4BKEt7.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://anoukdelecluse.nl/lGZLB1.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://appeum.com/wp-content/themes/cc.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://arot.altervista.org/KHTUdq.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://arttoday.sk/mE8MKJ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://ascortimisoara.ro/kWIH5V.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://aspectdesigns.com.au/0rTVlG.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://audetlaw.com/LnVAdF.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://autohaus-seevetal.com/9x6UwK.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://avancarvisual.com.br/wp-content/themes/twentytwelve/VzkgnX.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://babylicious.ie/s1GHUZ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://balkanium.altervista.org/p3er4s.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://beachhouseplans.com/wp-admin/js/5d8gMe.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://best-service.jp/olxu2Y.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://beyondthedog.net/edHDvf.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bigboattravel.com/uploads/3/5/4/5/3545341/header_images/NthjHz.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bisofit.com/QXwm4I.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bktrade.kiev.ua/76b3ZQ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://boilersandfurnaces.com/uploads/3/5/1/6/3516773/RPyH2q.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/0l0vp1va6b2\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/1cslstk2qv121\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/1xb81c28qs2db\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/22o1210hbpw\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/2h6t511wpuvnych\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/379gz635s3j946\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/4kpy8ju42x137\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/503qu7boexyk\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/574xl5yme0gdz\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/5gpf7ecxhf\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/5hmwl5qvpz2f3gc\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/6m50uk8ty1031\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/6tvpgu93q4wx5t\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/703hjdr3ez72\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/73075bdj8meb\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/7gr904pzv6\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/7ms68qsdfj0jt\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/89e8f40k8zcn38\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/8eo5zwhh4zndwwa\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/8tsdhjccoxz6c\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/94g2mr36b4\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/9bqdnk2h58ty2l\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/9hul78mtg1n63\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/b0slgvfxvyf\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/b3amhlkiar2c\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/b8g7g560612\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/bo5ha9ild1zjukv\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/cannzqzrum14o4c\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/ch3eq62ad8k\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/ci72o4ruf2y87\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/d5i52z8cgv5\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/d65v4fx21f\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/d7jly5f09tqj\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/di53su4z7uqvj\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/dypi31624z\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/e5tkclwq9w0\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/e887nn5k9pb6\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/f1s0y87wrwo\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/fgivit1drjuh\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/fpkirizbrzxc5\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/fxoztyxp320q\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/g5k4uvxghygg7r\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/gvi00me81aabu\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/hq5drme48h\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/hzpz767vze9\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/ifkhfc5369az88\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/iijoama0ynrowtp\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/j4hzoz8cgdeza\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/kka7641ov7\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/ko679ybid6ys58\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/kxdmlkhmuyf9\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/kzqnheutxkjwhr\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/mi5b67bilrfu\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/n2csus3eo1tyg\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/nve4m67l83\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/o0nyjlre41o3\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/pgjcokoi2kisu\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/pl36lz43r6r7\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/q3xryv3mh1\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/qely217wcjdl7b\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/qo9ux20lo1\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/qu9ajlxsiw\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/r45byxsjhz\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/raph9xccgxt\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/rb05hez1r044\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/rdjg0eb5r0qs\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/ri86nx23dhqbmch\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/rjotoddb4n7hl\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/rof06587c1x2y3t\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/s40o542jt7v\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/sb2zarf5vy\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/uamuxps7y98\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/uiyi9dkf5bs\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/v13rw8n8w2\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/vzum6ywdedxjtd\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/walqb5xzunmr\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/wilqkaz24rnqli\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/x1tg111bara5\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/x753k2s01gnd5b\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/x7lfazpjuuiel\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/xgw1o6gt9h8k9g\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/xjp3zmw6glginuq\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/yias364ajr\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bolizarsospos.com/zyayxp2kpay\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://bucksmedia.go2cloud.org/aff_c\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://building.msu.ac.th/q3Bslr.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://businessaviators.com/r1doyF.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://challengestrata.com.au/fP_BXS.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://cheapshirts.us/zVnMrG.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://chong.joelle.free.fr/_L43PH.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://connectao.com/wp-content/themes/twentyeleven/cc.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://d3mpd.fe.uns.ac.id/XPgmur.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://daffamedia.com/wp-content/plugins/wp_module/img5.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://dechehang.com/GZ2QRn.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://definitionen.de/v7GVES.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://dichiro.com/WaIrd6.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://dillardvideo.com/wp-admin/network/2.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://dining-bar.com/BQ_Ln4.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://domaine-cassillac.com/4q3esU.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://double-wing.de/DZkCLR.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://drdigitalmd.com/img1.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://eatside.es/xZQGXV.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://ecocalsots.com/N79GTA.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://ecolux-comfort.com/nPAbsy.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://elcoachingempresarial.com/wp-admin/user/2.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://emprende21.es/oTIq7A.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://estudiobarco.com.ar/5TFv7E.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://event-travel.co.uk/3K6Psd.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://feuerwehr-stadt-riesa.de/UFiPOq.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://foundersomaha.net/wp-includes/Text/Diff/Renderer/ap3.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://frame3d.de/ItGJKd.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://fun-pop.com/Ks1rCc.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://genedillardart.com/wp-admin/network/3.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://gibdd.ws/J7D65p.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://glitchygaming.com/r07QZu.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://grochowina.net/UnvPso.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://haarsaloncindy.nl/XzF03r.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://hamilton150.co.nz/LmfuMZ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://icsot.na.its.ac.id/8vwRUX.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://igatha.com/h4MeKJ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://ilovesport.kiev.ua/z8X9T7.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://imagescameraclub.com/j7b5kK.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://inspirenetworks.in/vAqu3L.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://italyprego.com/Lf2dcA.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://jadwalpialadunia.in/rG4Rdi.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://jambola.com/LuylWV.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://jauregia.net/img5.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://konyavakfi.nl/Zmje1r.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://kuruyaprak.com/OTLuKo.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://kvnysoho.com/eHafFT.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://lazymoosestamping.com/YRfbgB.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://london-escorts-agency.org.uk/fdnmyD.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://lzclient.com/img4.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://madisonbootcamps.com/gWQ3wp.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://mangohills.net/RxIoCE.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://marciogerhardtsouza.com.br/mPCsDz.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://marcortes.com/img5.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://markossolomon.com/F1q7QX.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://maternalserenity.co.uk/I_NwPg.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://millsmanagement.nl/AnOgVK.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://naimselmonaj.com/QoYx31.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://nonnuoccaobang.com/BRdoDL.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://nupleta.com.br/KoHV09.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://openroadsolutions.com/FJ2dOw.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://oregonreversemortgage.com/Rwafp_.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://p237996.mybestmv.com/adServe/domainClick\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://paintituppottery.com/6cmeb2.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://patrianossa.com.br/u8LkzD.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://portalmaismidia.com.br/tnSmIb.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://portret-tekening.nl/mNQVts.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://procrediti.com.ua/d6yGOX.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://rajsima87.com/img2.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://recaswine.ro/dXlq0Y.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://silstop.pl/Si0cCJ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://smartnote.co/2NxVzA.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://studiolegalecsb.it/iQcNfC.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://takaram.ir/gjOREZ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://takatei.com/rfYI4L.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://tcblog.de/mXdVTh.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://theassemblyguy.co.nz/vpFAbQ.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://trion.com.ph/jdKAap.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://tusrecetas.net/JbElN7.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://tutorialswalk.info/wp-content/themes/Defne/img2.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://viralcrazies.com/iFHt4C.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://vsedveri-33.ru/\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://weberteam.hu/WCTdO5.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.001edizioni.com/NZwt_a.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.bishopbell.co.uk/enRmcC.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.chemes.eu/wp-content/themes/decoy2/redux-framework/ReduxCore/inc/fields/info/2.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.decorandoimoveis.com/QEO5yh.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.example.com:1000\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.feddoctor.com/Oe1LMr.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.gjscomputerservices.com.au/S1_rvm.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.granmarquise.com.br/6f_8ei.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.hanecaklaw.com/\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.hanoiguidedtours.com/iQ2q1f.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.healthstafftravel.com.au/oyBbUs.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.plexipr.com/vAHzWX.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.rippedknees.co.uk/TXmJcq.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.taoblu.com/wp-content/plugins/wp_module/sbML0j.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.vishvagujarat.com/5of9dt.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"http://www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkG.php\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n\u001e{\"indicator\":\"https://*abc.*test.com\",\"value\":{\"sources\":[\"localdb\"],\"confidence\":100,\"first_seen\":1561029985300,\"type\":\"URL\",\"share_level\":\"red\",\"last_seen\":1561029985300}}\n" }, { "path": "tests/integration/basic/URLHC%3Fv%3Dmwg.result", "content": "type=string\n\"*abc.example.com/foobar?a=:80\" \"localdb\"\n\"http://*.feyda.net/hOeDr4.php\" \"localdb\"\n\"http://7-eleven-handbags.com/X1rZYp.php\" \"localdb\"\n\"http://8vs.com/6jezbr.php\" \"localdb\"\n\"http://abdal.com.ua/7_jzay.php\" \"localdb\"\n\"http://aditaborai.com.br/WgNGXe.php\" \"localdb\"\n\"http://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php\" \"localdb\"\n\"http://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php\" \"localdb\"\n\"http://allstarpaintbody.com/lrQ2bG.php\" \"localdb\"\n\"http://americancorner.udp.cl/etloxW.php\" \"localdb\"\n\"http://ample-sun.eu/4BKEt7.php\" \"localdb\"\n\"http://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php\" \"localdb\"\n\"http://anoukdelecluse.nl/lGZLB1.php\" \"localdb\"\n\"http://appeum.com/wp-content/themes/cc.php\" \"localdb\"\n\"http://arot.altervista.org/KHTUdq.php\" \"localdb\"\n\"http://arttoday.sk/mE8MKJ.php\" \"localdb\"\n\"http://ascortimisoara.ro/kWIH5V.php\" \"localdb\"\n\"http://aspectdesigns.com.au/0rTVlG.php\" \"localdb\"\n\"http://audetlaw.com/LnVAdF.php\" \"localdb\"\n\"http://autohaus-seevetal.com/9x6UwK.php\" \"localdb\"\n\"http://avancarvisual.com.br/wp-content/themes/twentytwelve/VzkgnX.php\" \"localdb\"\n\"http://babylicious.ie/s1GHUZ.php\" \"localdb\"\n\"http://balkanium.altervista.org/p3er4s.php\" \"localdb\"\n\"http://beachhouseplans.com/wp-admin/js/5d8gMe.php\" \"localdb\"\n\"http://best-service.jp/olxu2Y.php\" \"localdb\"\n\"http://beyondthedog.net/edHDvf.php\" \"localdb\"\n\"http://bigboattravel.com/uploads/3/5/4/5/3545341/header_images/NthjHz.php\" \"localdb\"\n\"http://bisofit.com/QXwm4I.php\" \"localdb\"\n\"http://bktrade.kiev.ua/76b3ZQ.php\" \"localdb\"\n\"http://boilersandfurnaces.com/uploads/3/5/1/6/3516773/RPyH2q.php\" \"localdb\"\n\"http://bolizarsospos.com/0l0vp1va6b2\" \"localdb\"\n\"http://bolizarsospos.com/1cslstk2qv121\" \"localdb\"\n\"http://bolizarsospos.com/1xb81c28qs2db\" \"localdb\"\n\"http://bolizarsospos.com/22o1210hbpw\" \"localdb\"\n\"http://bolizarsospos.com/2h6t511wpuvnych\" \"localdb\"\n\"http://bolizarsospos.com/379gz635s3j946\" \"localdb\"\n\"http://bolizarsospos.com/4kpy8ju42x137\" \"localdb\"\n\"http://bolizarsospos.com/503qu7boexyk\" \"localdb\"\n\"http://bolizarsospos.com/574xl5yme0gdz\" \"localdb\"\n\"http://bolizarsospos.com/5gpf7ecxhf\" \"localdb\"\n\"http://bolizarsospos.com/5hmwl5qvpz2f3gc\" \"localdb\"\n\"http://bolizarsospos.com/6m50uk8ty1031\" \"localdb\"\n\"http://bolizarsospos.com/6tvpgu93q4wx5t\" \"localdb\"\n\"http://bolizarsospos.com/703hjdr3ez72\" \"localdb\"\n\"http://bolizarsospos.com/73075bdj8meb\" \"localdb\"\n\"http://bolizarsospos.com/7gr904pzv6\" \"localdb\"\n\"http://bolizarsospos.com/7ms68qsdfj0jt\" \"localdb\"\n\"http://bolizarsospos.com/89e8f40k8zcn38\" \"localdb\"\n\"http://bolizarsospos.com/8eo5zwhh4zndwwa\" \"localdb\"\n\"http://bolizarsospos.com/8tsdhjccoxz6c\" \"localdb\"\n\"http://bolizarsospos.com/94g2mr36b4\" \"localdb\"\n\"http://bolizarsospos.com/9bqdnk2h58ty2l\" \"localdb\"\n\"http://bolizarsospos.com/9hul78mtg1n63\" \"localdb\"\n\"http://bolizarsospos.com/b0slgvfxvyf\" \"localdb\"\n\"http://bolizarsospos.com/b3amhlkiar2c\" \"localdb\"\n\"http://bolizarsospos.com/b8g7g560612\" \"localdb\"\n\"http://bolizarsospos.com/bo5ha9ild1zjukv\" \"localdb\"\n\"http://bolizarsospos.com/cannzqzrum14o4c\" \"localdb\"\n\"http://bolizarsospos.com/ch3eq62ad8k\" \"localdb\"\n\"http://bolizarsospos.com/ci72o4ruf2y87\" \"localdb\"\n\"http://bolizarsospos.com/d5i52z8cgv5\" \"localdb\"\n\"http://bolizarsospos.com/d65v4fx21f\" \"localdb\"\n\"http://bolizarsospos.com/d7jly5f09tqj\" \"localdb\"\n\"http://bolizarsospos.com/di53su4z7uqvj\" \"localdb\"\n\"http://bolizarsospos.com/dypi31624z\" \"localdb\"\n\"http://bolizarsospos.com/e5tkclwq9w0\" \"localdb\"\n\"http://bolizarsospos.com/e887nn5k9pb6\" \"localdb\"\n\"http://bolizarsospos.com/f1s0y87wrwo\" \"localdb\"\n\"http://bolizarsospos.com/fgivit1drjuh\" \"localdb\"\n\"http://bolizarsospos.com/fpkirizbrzxc5\" \"localdb\"\n\"http://bolizarsospos.com/fxoztyxp320q\" \"localdb\"\n\"http://bolizarsospos.com/g5k4uvxghygg7r\" \"localdb\"\n\"http://bolizarsospos.com/gvi00me81aabu\" \"localdb\"\n\"http://bolizarsospos.com/hq5drme48h\" \"localdb\"\n\"http://bolizarsospos.com/hzpz767vze9\" \"localdb\"\n\"http://bolizarsospos.com/ifkhfc5369az88\" \"localdb\"\n\"http://bolizarsospos.com/iijoama0ynrowtp\" \"localdb\"\n\"http://bolizarsospos.com/j4hzoz8cgdeza\" \"localdb\"\n\"http://bolizarsospos.com/kka7641ov7\" \"localdb\"\n\"http://bolizarsospos.com/ko679ybid6ys58\" \"localdb\"\n\"http://bolizarsospos.com/kxdmlkhmuyf9\" \"localdb\"\n\"http://bolizarsospos.com/kzqnheutxkjwhr\" \"localdb\"\n\"http://bolizarsospos.com/mi5b67bilrfu\" \"localdb\"\n\"http://bolizarsospos.com/n2csus3eo1tyg\" \"localdb\"\n\"http://bolizarsospos.com/nve4m67l83\" \"localdb\"\n\"http://bolizarsospos.com/o0nyjlre41o3\" \"localdb\"\n\"http://bolizarsospos.com/pgjcokoi2kisu\" \"localdb\"\n\"http://bolizarsospos.com/pl36lz43r6r7\" \"localdb\"\n\"http://bolizarsospos.com/q3xryv3mh1\" \"localdb\"\n\"http://bolizarsospos.com/qely217wcjdl7b\" \"localdb\"\n\"http://bolizarsospos.com/qo9ux20lo1\" \"localdb\"\n\"http://bolizarsospos.com/qu9ajlxsiw\" \"localdb\"\n\"http://bolizarsospos.com/r45byxsjhz\" \"localdb\"\n\"http://bolizarsospos.com/raph9xccgxt\" \"localdb\"\n\"http://bolizarsospos.com/rb05hez1r044\" \"localdb\"\n\"http://bolizarsospos.com/rdjg0eb5r0qs\" \"localdb\"\n\"http://bolizarsospos.com/ri86nx23dhqbmch\" \"localdb\"\n\"http://bolizarsospos.com/rjotoddb4n7hl\" \"localdb\"\n\"http://bolizarsospos.com/rof06587c1x2y3t\" \"localdb\"\n\"http://bolizarsospos.com/s40o542jt7v\" \"localdb\"\n\"http://bolizarsospos.com/sb2zarf5vy\" \"localdb\"\n\"http://bolizarsospos.com/uamuxps7y98\" \"localdb\"\n\"http://bolizarsospos.com/uiyi9dkf5bs\" \"localdb\"\n\"http://bolizarsospos.com/v13rw8n8w2\" \"localdb\"\n\"http://bolizarsospos.com/vzum6ywdedxjtd\" \"localdb\"\n\"http://bolizarsospos.com/walqb5xzunmr\" \"localdb\"\n\"http://bolizarsospos.com/wilqkaz24rnqli\" \"localdb\"\n\"http://bolizarsospos.com/x1tg111bara5\" \"localdb\"\n\"http://bolizarsospos.com/x753k2s01gnd5b\" \"localdb\"\n\"http://bolizarsospos.com/x7lfazpjuuiel\" \"localdb\"\n\"http://bolizarsospos.com/xgw1o6gt9h8k9g\" \"localdb\"\n\"http://bolizarsospos.com/xjp3zmw6glginuq\" \"localdb\"\n\"http://bolizarsospos.com/yias364ajr\" \"localdb\"\n\"http://bolizarsospos.com/zyayxp2kpay\" \"localdb\"\n\"http://bucksmedia.go2cloud.org/aff_c\" \"localdb\"\n\"http://building.msu.ac.th/q3Bslr.php\" \"localdb\"\n\"http://businessaviators.com/r1doyF.php\" \"localdb\"\n\"http://challengestrata.com.au/fP_BXS.php\" \"localdb\"\n\"http://cheapshirts.us/zVnMrG.php\" \"localdb\"\n\"http://chong.joelle.free.fr/_L43PH.php\" \"localdb\"\n\"http://connectao.com/wp-content/themes/twentyeleven/cc.php\" \"localdb\"\n\"http://d3mpd.fe.uns.ac.id/XPgmur.php\" \"localdb\"\n\"http://daffamedia.com/wp-content/plugins/wp_module/img5.php\" \"localdb\"\n\"http://dechehang.com/GZ2QRn.php\" \"localdb\"\n\"http://definitionen.de/v7GVES.php\" \"localdb\"\n\"http://dichiro.com/WaIrd6.php\" \"localdb\"\n\"http://dillardvideo.com/wp-admin/network/2.php\" \"localdb\"\n\"http://dining-bar.com/BQ_Ln4.php\" \"localdb\"\n\"http://domaine-cassillac.com/4q3esU.php\" \"localdb\"\n\"http://double-wing.de/DZkCLR.php\" \"localdb\"\n\"http://drdigitalmd.com/img1.php\" \"localdb\"\n\"http://eatside.es/xZQGXV.php\" \"localdb\"\n\"http://ecocalsots.com/N79GTA.php\" \"localdb\"\n\"http://ecolux-comfort.com/nPAbsy.php\" \"localdb\"\n\"http://elcoachingempresarial.com/wp-admin/user/2.php\" \"localdb\"\n\"http://emprende21.es/oTIq7A.php\" \"localdb\"\n\"http://estudiobarco.com.ar/5TFv7E.php\" \"localdb\"\n\"http://event-travel.co.uk/3K6Psd.php\" \"localdb\"\n\"http://feuerwehr-stadt-riesa.de/UFiPOq.php\" \"localdb\"\n\"http://foundersomaha.net/wp-includes/Text/Diff/Renderer/ap3.php\" \"localdb\"\n\"http://frame3d.de/ItGJKd.php\" \"localdb\"\n\"http://fun-pop.com/Ks1rCc.php\" \"localdb\"\n\"http://genedillardart.com/wp-admin/network/3.php\" \"localdb\"\n\"http://gibdd.ws/J7D65p.php\" \"localdb\"\n\"http://glitchygaming.com/r07QZu.php\" \"localdb\"\n\"http://grochowina.net/UnvPso.php\" \"localdb\"\n\"http://haarsaloncindy.nl/XzF03r.php\" \"localdb\"\n\"http://hamilton150.co.nz/LmfuMZ.php\" \"localdb\"\n\"http://icsot.na.its.ac.id/8vwRUX.php\" \"localdb\"\n\"http://igatha.com/h4MeKJ.php\" \"localdb\"\n\"http://ilovesport.kiev.ua/z8X9T7.php\" \"localdb\"\n\"http://imagescameraclub.com/j7b5kK.php\" \"localdb\"\n\"http://inspirenetworks.in/vAqu3L.php\" \"localdb\"\n\"http://italyprego.com/Lf2dcA.php\" \"localdb\"\n\"http://jadwalpialadunia.in/rG4Rdi.php\" \"localdb\"\n\"http://jambola.com/LuylWV.php\" \"localdb\"\n\"http://jauregia.net/img5.php\" \"localdb\"\n\"http://konyavakfi.nl/Zmje1r.php\" \"localdb\"\n\"http://kuruyaprak.com/OTLuKo.php\" \"localdb\"\n\"http://kvnysoho.com/eHafFT.php\" \"localdb\"\n\"http://lazymoosestamping.com/YRfbgB.php\" \"localdb\"\n\"http://london-escorts-agency.org.uk/fdnmyD.php\" \"localdb\"\n\"http://lzclient.com/img4.php\" \"localdb\"\n\"http://madisonbootcamps.com/gWQ3wp.php\" \"localdb\"\n\"http://mangohills.net/RxIoCE.php\" \"localdb\"\n\"http://marciogerhardtsouza.com.br/mPCsDz.php\" \"localdb\"\n\"http://marcortes.com/img5.php\" \"localdb\"\n\"http://markossolomon.com/F1q7QX.php\" \"localdb\"\n\"http://maternalserenity.co.uk/I_NwPg.php\" \"localdb\"\n\"http://millsmanagement.nl/AnOgVK.php\" \"localdb\"\n\"http://mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php\" \"localdb\"\n\"http://naimselmonaj.com/QoYx31.php\" \"localdb\"\n\"http://nonnuoccaobang.com/BRdoDL.php\" \"localdb\"\n\"http://nupleta.com.br/KoHV09.php\" \"localdb\"\n\"http://openroadsolutions.com/FJ2dOw.php\" \"localdb\"\n\"http://oregonreversemortgage.com/Rwafp_.php\" \"localdb\"\n\"http://p237996.mybestmv.com/adServe/domainClick\" \"localdb\"\n\"http://paintituppottery.com/6cmeb2.php\" \"localdb\"\n\"http://patrianossa.com.br/u8LkzD.php\" \"localdb\"\n\"http://portalmaismidia.com.br/tnSmIb.php\" \"localdb\"\n\"http://portret-tekening.nl/mNQVts.php\" \"localdb\"\n\"http://procrediti.com.ua/d6yGOX.php\" \"localdb\"\n\"http://rajsima87.com/img2.php\" \"localdb\"\n\"http://recaswine.ro/dXlq0Y.php\" \"localdb\"\n\"http://silstop.pl/Si0cCJ.php\" \"localdb\"\n\"http://smartnote.co/2NxVzA.php\" \"localdb\"\n\"http://studiolegalecsb.it/iQcNfC.php\" \"localdb\"\n\"http://takaram.ir/gjOREZ.php\" \"localdb\"\n\"http://takatei.com/rfYI4L.php\" \"localdb\"\n\"http://tcblog.de/mXdVTh.php\" \"localdb\"\n\"http://theassemblyguy.co.nz/vpFAbQ.php\" \"localdb\"\n\"http://trion.com.ph/jdKAap.php\" \"localdb\"\n\"http://tusrecetas.net/JbElN7.php\" \"localdb\"\n\"http://tutorialswalk.info/wp-content/themes/Defne/img2.php\" \"localdb\"\n\"http://viralcrazies.com/iFHt4C.php\" \"localdb\"\n\"http://vsedveri-33.ru/\" \"localdb\"\n\"http://weberteam.hu/WCTdO5.php\" \"localdb\"\n\"http://www.001edizioni.com/NZwt_a.php\" \"localdb\"\n\"http://www.bishopbell.co.uk/enRmcC.php\" \"localdb\"\n\"http://www.chemes.eu/wp-content/themes/decoy2/redux-framework/ReduxCore/inc/fields/info/2.php\" \"localdb\"\n\"http://www.decorandoimoveis.com/QEO5yh.php\" \"localdb\"\n\"http://www.example.com:1000\" \"localdb\"\n\"http://www.feddoctor.com/Oe1LMr.php\" \"localdb\"\n\"http://www.gjscomputerservices.com.au/S1_rvm.php\" \"localdb\"\n\"http://www.granmarquise.com.br/6f_8ei.php\" \"localdb\"\n\"http://www.hanecaklaw.com/\" \"localdb\"\n\"http://www.hanoiguidedtours.com/iQ2q1f.php\" \"localdb\"\n\"http://www.healthstafftravel.com.au/oyBbUs.php\" \"localdb\"\n\"http://www.plexipr.com/vAHzWX.php\" \"localdb\"\n\"http://www.rippedknees.co.uk/TXmJcq.php\" \"localdb\"\n\"http://www.taoblu.com/wp-content/plugins/wp_module/sbML0j.php\" \"localdb\"\n\"http://www.vishvagujarat.com/5of9dt.php\" \"localdb\"\n\"http://www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkG.php\" \"localdb\"\n\"https://*abc.*test.com\" \"localdb\"\n" }, { "path": "tests/integration/basic/URLHC%3Fv%3Dpanosurl%26di%3D1.result", "content": "feyda.net/hoedr4.php\n*.feyda.net/hoedr4.php\n7-eleven-handbags.com/x1rzyp.php\n8vs.com/6jezbr.php\nabdal.com.ua/7_jzay.php\naditaborai.com.br/wgngxe.php\nairconditioning12601.com/uploads/3/5/7/6/3576233/v5k3za.php\nallreadytravel.com/uploads/3/5/4/9/3549731/header_images/tomae1.php\nallstarpaintbody.com/lrq2bg.php\namericancorner.udp.cl/etloxw.php\nample-sun.eu/4bket7.php\nanime-tuner.square7.ch/wp-content/themes/twentyeleven/mstgk_.php\nanoukdelecluse.nl/lgzlb1.php\nappeum.com/wp-content/themes/cc.php\narot.altervista.org/khtudq.php\narttoday.sk/me8mkj.php\nascortimisoara.ro/kwih5v.php\naspectdesigns.com.au/0rtvlg.php\naudetlaw.com/lnvadf.php\nautohaus-seevetal.com/9x6uwk.php\navancarvisual.com.br/wp-content/themes/twentytwelve/vzkgnx.php\nbabylicious.ie/s1ghuz.php\nbalkanium.altervista.org/p3er4s.php\nbeachhouseplans.com/wp-admin/js/5d8gme.php\nbest-service.jp/olxu2y.php\nbeyondthedog.net/edhdvf.php\nbigboattravel.com/uploads/3/5/4/5/3545341/header_images/nthjhz.php\nbisofit.com/qxwm4i.php\nbktrade.kiev.ua/76b3zq.php\nboilersandfurnaces.com/uploads/3/5/1/6/3516773/rpyh2q.php\nbolizarsospos.com/0l0vp1va6b2\nbolizarsospos.com/1cslstk2qv121\nbolizarsospos.com/1xb81c28qs2db\nbolizarsospos.com/22o1210hbpw\nbolizarsospos.com/2h6t511wpuvnych\nbolizarsospos.com/379gz635s3j946\nbolizarsospos.com/4kpy8ju42x137\nbolizarsospos.com/503qu7boexyk\nbolizarsospos.com/574xl5yme0gdz\nbolizarsospos.com/5gpf7ecxhf\nbolizarsospos.com/5hmwl5qvpz2f3gc\nbolizarsospos.com/6m50uk8ty1031\nbolizarsospos.com/6tvpgu93q4wx5t\nbolizarsospos.com/703hjdr3ez72\nbolizarsospos.com/73075bdj8meb\nbolizarsospos.com/7gr904pzv6\nbolizarsospos.com/7ms68qsdfj0jt\nbolizarsospos.com/89e8f40k8zcn38\nbolizarsospos.com/8eo5zwhh4zndwwa\nbolizarsospos.com/8tsdhjccoxz6c\nbolizarsospos.com/94g2mr36b4\nbolizarsospos.com/9bqdnk2h58ty2l\nbolizarsospos.com/9hul78mtg1n63\nbolizarsospos.com/b0slgvfxvyf\nbolizarsospos.com/b3amhlkiar2c\nbolizarsospos.com/b8g7g560612\nbolizarsospos.com/bo5ha9ild1zjukv\nbolizarsospos.com/cannzqzrum14o4c\nbolizarsospos.com/ch3eq62ad8k\nbolizarsospos.com/ci72o4ruf2y87\nbolizarsospos.com/d5i52z8cgv5\nbolizarsospos.com/d65v4fx21f\nbolizarsospos.com/d7jly5f09tqj\nbolizarsospos.com/di53su4z7uqvj\nbolizarsospos.com/dypi31624z\nbolizarsospos.com/e5tkclwq9w0\nbolizarsospos.com/e887nn5k9pb6\nbolizarsospos.com/f1s0y87wrwo\nbolizarsospos.com/fgivit1drjuh\nbolizarsospos.com/fpkirizbrzxc5\nbolizarsospos.com/fxoztyxp320q\nbolizarsospos.com/g5k4uvxghygg7r\nbolizarsospos.com/gvi00me81aabu\nbolizarsospos.com/hq5drme48h\nbolizarsospos.com/hzpz767vze9\nbolizarsospos.com/ifkhfc5369az88\nbolizarsospos.com/iijoama0ynrowtp\nbolizarsospos.com/j4hzoz8cgdeza\nbolizarsospos.com/kka7641ov7\nbolizarsospos.com/ko679ybid6ys58\nbolizarsospos.com/kxdmlkhmuyf9\nbolizarsospos.com/kzqnheutxkjwhr\nbolizarsospos.com/mi5b67bilrfu\nbolizarsospos.com/n2csus3eo1tyg\nbolizarsospos.com/nve4m67l83\nbolizarsospos.com/o0nyjlre41o3\nbolizarsospos.com/pgjcokoi2kisu\nbolizarsospos.com/pl36lz43r6r7\nbolizarsospos.com/q3xryv3mh1\nbolizarsospos.com/qely217wcjdl7b\nbolizarsospos.com/qo9ux20lo1\nbolizarsospos.com/qu9ajlxsiw\nbolizarsospos.com/r45byxsjhz\nbolizarsospos.com/raph9xccgxt\nbolizarsospos.com/rb05hez1r044\nbolizarsospos.com/rdjg0eb5r0qs\nbolizarsospos.com/ri86nx23dhqbmch\nbolizarsospos.com/rjotoddb4n7hl\nbolizarsospos.com/rof06587c1x2y3t\nbolizarsospos.com/s40o542jt7v\nbolizarsospos.com/sb2zarf5vy\nbolizarsospos.com/uamuxps7y98\nbolizarsospos.com/uiyi9dkf5bs\nbolizarsospos.com/v13rw8n8w2\nbolizarsospos.com/vzum6ywdedxjtd\nbolizarsospos.com/walqb5xzunmr\nbolizarsospos.com/wilqkaz24rnqli\nbolizarsospos.com/x1tg111bara5\nbolizarsospos.com/x753k2s01gnd5b\nbolizarsospos.com/x7lfazpjuuiel\nbolizarsospos.com/xgw1o6gt9h8k9g\nbolizarsospos.com/xjp3zmw6glginuq\nbolizarsospos.com/yias364ajr\nbolizarsospos.com/zyayxp2kpay\nbucksmedia.go2cloud.org/aff_c\nbuilding.msu.ac.th/q3bslr.php\nbusinessaviators.com/r1doyf.php\nchallengestrata.com.au/fp_bxs.php\ncheapshirts.us/zvnmrg.php\nchong.joelle.free.fr/_l43ph.php\nconnectao.com/wp-content/themes/twentyeleven/cc.php\nd3mpd.fe.uns.ac.id/xpgmur.php\ndaffamedia.com/wp-content/plugins/wp_module/img5.php\ndechehang.com/gz2qrn.php\ndefinitionen.de/v7gves.php\ndichiro.com/waird6.php\ndillardvideo.com/wp-admin/network/2.php\ndining-bar.com/bq_ln4.php\ndomaine-cassillac.com/4q3esu.php\ndouble-wing.de/dzkclr.php\ndrdigitalmd.com/img1.php\neatside.es/xzqgxv.php\necocalsots.com/n79gta.php\necolux-comfort.com/npabsy.php\nelcoachingempresarial.com/wp-admin/user/2.php\nemprende21.es/otiq7a.php\nestudiobarco.com.ar/5tfv7e.php\nevent-travel.co.uk/3k6psd.php\nfeuerwehr-stadt-riesa.de/ufipoq.php\nfoundersomaha.net/wp-includes/text/diff/renderer/ap3.php\nframe3d.de/itgjkd.php\nfun-pop.com/ks1rcc.php\ngenedillardart.com/wp-admin/network/3.php\ngibdd.ws/j7d65p.php\nglitchygaming.com/r07qzu.php\ngrochowina.net/unvpso.php\nhaarsaloncindy.nl/xzf03r.php\nhamilton150.co.nz/lmfumz.php\nicsot.na.its.ac.id/8vwrux.php\nigatha.com/h4mekj.php\nilovesport.kiev.ua/z8x9t7.php\nimagescameraclub.com/j7b5kk.php\ninspirenetworks.in/vaqu3l.php\nitalyprego.com/lf2dca.php\njadwalpialadunia.in/rg4rdi.php\njambola.com/luylwv.php\njauregia.net/img5.php\nkonyavakfi.nl/zmje1r.php\nkuruyaprak.com/otluko.php\nkvnysoho.com/ehafft.php\nlazymoosestamping.com/yrfbgb.php\nlondon-escorts-agency.org.uk/fdnmyd.php\nlzclient.com/img4.php\nmadisonbootcamps.com/gwq3wp.php\nmangohills.net/rxioce.php\nmarciogerhardtsouza.com.br/mpcsdz.php\nmarcortes.com/img5.php\nmarkossolomon.com/f1q7qx.php\nmaternalserenity.co.uk/i_nwpg.php\nmillsmanagement.nl/anogvk.php\nmmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php\nnaimselmonaj.com/qoyx31.php\nnonnuoccaobang.com/brdodl.php\nnupleta.com.br/kohv09.php\nopenroadsolutions.com/fj2dow.php\noregonreversemortgage.com/rwafp_.php\np237996.mybestmv.com/adserve/domainclick\npaintituppottery.com/6cmeb2.php\npatrianossa.com.br/u8lkzd.php\nportalmaismidia.com.br/tnsmib.php\nportret-tekening.nl/mnqvts.php\nprocrediti.com.ua/d6ygox.php\nrajsima87.com/img2.php\nrecaswine.ro/dxlq0y.php\nsilstop.pl/si0ccj.php\nsmartnote.co/2nxvza.php\nstudiolegalecsb.it/iqcnfc.php\ntakaram.ir/gjorez.php\ntakatei.com/rfyi4l.php\ntcblog.de/mxdvth.php\ntheassemblyguy.co.nz/vpfabq.php\ntrion.com.ph/jdkaap.php\ntusrecetas.net/jbeln7.php\ntutorialswalk.info/wp-content/themes/defne/img2.php\nviralcrazies.com/ifht4c.php\nvsedveri-33.ru/\nweberteam.hu/wctdo5.php\nwww.001edizioni.com/nzwt_a.php\nwww.bishopbell.co.uk/enrmcc.php\nwww.chemes.eu/wp-content/themes/decoy2/redux-framework/reduxcore/inc/fields/info/2.php\nwww.decorandoimoveis.com/qeo5yh.php\nwww.feddoctor.com/oe1lmr.php\nwww.gjscomputerservices.com.au/s1_rvm.php\nwww.granmarquise.com.br/6f_8ei.php\nwww.hanecaklaw.com/\nwww.hanoiguidedtours.com/iq2q1f.php\nwww.healthstafftravel.com.au/oybbus.php\nwww.plexipr.com/vahzwx.php\nwww.rippedknees.co.uk/txmjcq.php\nwww.taoblu.com/wp-content/plugins/wp_module/sbml0j.php\nwww.vishvagujarat.com/5of9dt.php\nwww.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkg.php\n" }, { "path": "tests/integration/basic/URLHC%3Fv%3Dpanosurl%26sp%3D1%26nsl%3D1.result", "content": "example.com/foobar?a=:80\n*.example.com/foobar?a=:80\nfeyda.net/hoedr4.php\n*.feyda.net/hoedr4.php\n7-eleven-handbags.com/x1rzyp.php\n8vs.com/6jezbr.php\nabdal.com.ua/7_jzay.php\naditaborai.com.br/wgngxe.php\nairconditioning12601.com/uploads/3/5/7/6/3576233/v5k3za.php\nallreadytravel.com/uploads/3/5/4/9/3549731/header_images/tomae1.php\nallstarpaintbody.com/lrq2bg.php\namericancorner.udp.cl/etloxw.php\nample-sun.eu/4bket7.php\nanime-tuner.square7.ch/wp-content/themes/twentyeleven/mstgk_.php\nanoukdelecluse.nl/lgzlb1.php\nappeum.com/wp-content/themes/cc.php\narot.altervista.org/khtudq.php\narttoday.sk/me8mkj.php\nascortimisoara.ro/kwih5v.php\naspectdesigns.com.au/0rtvlg.php\naudetlaw.com/lnvadf.php\nautohaus-seevetal.com/9x6uwk.php\navancarvisual.com.br/wp-content/themes/twentytwelve/vzkgnx.php\nbabylicious.ie/s1ghuz.php\nbalkanium.altervista.org/p3er4s.php\nbeachhouseplans.com/wp-admin/js/5d8gme.php\nbest-service.jp/olxu2y.php\nbeyondthedog.net/edhdvf.php\nbigboattravel.com/uploads/3/5/4/5/3545341/header_images/nthjhz.php\nbisofit.com/qxwm4i.php\nbktrade.kiev.ua/76b3zq.php\nboilersandfurnaces.com/uploads/3/5/1/6/3516773/rpyh2q.php\nbolizarsospos.com/0l0vp1va6b2\nbolizarsospos.com/1cslstk2qv121\nbolizarsospos.com/1xb81c28qs2db\nbolizarsospos.com/22o1210hbpw\nbolizarsospos.com/2h6t511wpuvnych\nbolizarsospos.com/379gz635s3j946\nbolizarsospos.com/4kpy8ju42x137\nbolizarsospos.com/503qu7boexyk\nbolizarsospos.com/574xl5yme0gdz\nbolizarsospos.com/5gpf7ecxhf\nbolizarsospos.com/5hmwl5qvpz2f3gc\nbolizarsospos.com/6m50uk8ty1031\nbolizarsospos.com/6tvpgu93q4wx5t\nbolizarsospos.com/703hjdr3ez72\nbolizarsospos.com/73075bdj8meb\nbolizarsospos.com/7gr904pzv6\nbolizarsospos.com/7ms68qsdfj0jt\nbolizarsospos.com/89e8f40k8zcn38\nbolizarsospos.com/8eo5zwhh4zndwwa\nbolizarsospos.com/8tsdhjccoxz6c\nbolizarsospos.com/94g2mr36b4\nbolizarsospos.com/9bqdnk2h58ty2l\nbolizarsospos.com/9hul78mtg1n63\nbolizarsospos.com/b0slgvfxvyf\nbolizarsospos.com/b3amhlkiar2c\nbolizarsospos.com/b8g7g560612\nbolizarsospos.com/bo5ha9ild1zjukv\nbolizarsospos.com/cannzqzrum14o4c\nbolizarsospos.com/ch3eq62ad8k\nbolizarsospos.com/ci72o4ruf2y87\nbolizarsospos.com/d5i52z8cgv5\nbolizarsospos.com/d65v4fx21f\nbolizarsospos.com/d7jly5f09tqj\nbolizarsospos.com/di53su4z7uqvj\nbolizarsospos.com/dypi31624z\nbolizarsospos.com/e5tkclwq9w0\nbolizarsospos.com/e887nn5k9pb6\nbolizarsospos.com/f1s0y87wrwo\nbolizarsospos.com/fgivit1drjuh\nbolizarsospos.com/fpkirizbrzxc5\nbolizarsospos.com/fxoztyxp320q\nbolizarsospos.com/g5k4uvxghygg7r\nbolizarsospos.com/gvi00me81aabu\nbolizarsospos.com/hq5drme48h\nbolizarsospos.com/hzpz767vze9\nbolizarsospos.com/ifkhfc5369az88\nbolizarsospos.com/iijoama0ynrowtp\nbolizarsospos.com/j4hzoz8cgdeza\nbolizarsospos.com/kka7641ov7\nbolizarsospos.com/ko679ybid6ys58\nbolizarsospos.com/kxdmlkhmuyf9\nbolizarsospos.com/kzqnheutxkjwhr\nbolizarsospos.com/mi5b67bilrfu\nbolizarsospos.com/n2csus3eo1tyg\nbolizarsospos.com/nve4m67l83\nbolizarsospos.com/o0nyjlre41o3\nbolizarsospos.com/pgjcokoi2kisu\nbolizarsospos.com/pl36lz43r6r7\nbolizarsospos.com/q3xryv3mh1\nbolizarsospos.com/qely217wcjdl7b\nbolizarsospos.com/qo9ux20lo1\nbolizarsospos.com/qu9ajlxsiw\nbolizarsospos.com/r45byxsjhz\nbolizarsospos.com/raph9xccgxt\nbolizarsospos.com/rb05hez1r044\nbolizarsospos.com/rdjg0eb5r0qs\nbolizarsospos.com/ri86nx23dhqbmch\nbolizarsospos.com/rjotoddb4n7hl\nbolizarsospos.com/rof06587c1x2y3t\nbolizarsospos.com/s40o542jt7v\nbolizarsospos.com/sb2zarf5vy\nbolizarsospos.com/uamuxps7y98\nbolizarsospos.com/uiyi9dkf5bs\nbolizarsospos.com/v13rw8n8w2\nbolizarsospos.com/vzum6ywdedxjtd\nbolizarsospos.com/walqb5xzunmr\nbolizarsospos.com/wilqkaz24rnqli\nbolizarsospos.com/x1tg111bara5\nbolizarsospos.com/x753k2s01gnd5b\nbolizarsospos.com/x7lfazpjuuiel\nbolizarsospos.com/xgw1o6gt9h8k9g\nbolizarsospos.com/xjp3zmw6glginuq\nbolizarsospos.com/yias364ajr\nbolizarsospos.com/zyayxp2kpay\nbucksmedia.go2cloud.org/aff_c\nbuilding.msu.ac.th/q3bslr.php\nbusinessaviators.com/r1doyf.php\nchallengestrata.com.au/fp_bxs.php\ncheapshirts.us/zvnmrg.php\nchong.joelle.free.fr/_l43ph.php\nconnectao.com/wp-content/themes/twentyeleven/cc.php\nd3mpd.fe.uns.ac.id/xpgmur.php\ndaffamedia.com/wp-content/plugins/wp_module/img5.php\ndechehang.com/gz2qrn.php\ndefinitionen.de/v7gves.php\ndichiro.com/waird6.php\ndillardvideo.com/wp-admin/network/2.php\ndining-bar.com/bq_ln4.php\ndomaine-cassillac.com/4q3esu.php\ndouble-wing.de/dzkclr.php\ndrdigitalmd.com/img1.php\neatside.es/xzqgxv.php\necocalsots.com/n79gta.php\necolux-comfort.com/npabsy.php\nelcoachingempresarial.com/wp-admin/user/2.php\nemprende21.es/otiq7a.php\nestudiobarco.com.ar/5tfv7e.php\nevent-travel.co.uk/3k6psd.php\nfeuerwehr-stadt-riesa.de/ufipoq.php\nfoundersomaha.net/wp-includes/text/diff/renderer/ap3.php\nframe3d.de/itgjkd.php\nfun-pop.com/ks1rcc.php\ngenedillardart.com/wp-admin/network/3.php\ngibdd.ws/j7d65p.php\nglitchygaming.com/r07qzu.php\ngrochowina.net/unvpso.php\nhaarsaloncindy.nl/xzf03r.php\nhamilton150.co.nz/lmfumz.php\nicsot.na.its.ac.id/8vwrux.php\nigatha.com/h4mekj.php\nilovesport.kiev.ua/z8x9t7.php\nimagescameraclub.com/j7b5kk.php\ninspirenetworks.in/vaqu3l.php\nitalyprego.com/lf2dca.php\njadwalpialadunia.in/rg4rdi.php\njambola.com/luylwv.php\njauregia.net/img5.php\nkonyavakfi.nl/zmje1r.php\nkuruyaprak.com/otluko.php\nkvnysoho.com/ehafft.php\nlazymoosestamping.com/yrfbgb.php\nlondon-escorts-agency.org.uk/fdnmyd.php\nlzclient.com/img4.php\nmadisonbootcamps.com/gwq3wp.php\nmangohills.net/rxioce.php\nmarciogerhardtsouza.com.br/mpcsdz.php\nmarcortes.com/img5.php\nmarkossolomon.com/f1q7qx.php\nmaternalserenity.co.uk/i_nwpg.php\nmillsmanagement.nl/anogvk.php\nmmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php\nnaimselmonaj.com/qoyx31.php\nnonnuoccaobang.com/brdodl.php\nnupleta.com.br/kohv09.php\nopenroadsolutions.com/fj2dow.php\noregonreversemortgage.com/rwafp_.php\np237996.mybestmv.com/adserve/domainclick\npaintituppottery.com/6cmeb2.php\npatrianossa.com.br/u8lkzd.php\nportalmaismidia.com.br/tnsmib.php\nportret-tekening.nl/mnqvts.php\nprocrediti.com.ua/d6ygox.php\nrajsima87.com/img2.php\nrecaswine.ro/dxlq0y.php\nsilstop.pl/si0ccj.php\nsmartnote.co/2nxvza.php\nstudiolegalecsb.it/iqcnfc.php\ntakaram.ir/gjorez.php\ntakatei.com/rfyi4l.php\ntcblog.de/mxdvth.php\ntheassemblyguy.co.nz/vpfabq.php\ntrion.com.ph/jdkaap.php\ntusrecetas.net/jbeln7.php\ntutorialswalk.info/wp-content/themes/defne/img2.php\nviralcrazies.com/ifht4c.php\nvsedveri-33.ru/\nweberteam.hu/wctdo5.php\nwww.001edizioni.com/nzwt_a.php\nwww.bishopbell.co.uk/enrmcc.php\nwww.chemes.eu/wp-content/themes/decoy2/redux-framework/reduxcore/inc/fields/info/2.php\nwww.decorandoimoveis.com/qeo5yh.php\nwww.example.com\nwww.feddoctor.com/oe1lmr.php\nwww.gjscomputerservices.com.au/s1_rvm.php\nwww.granmarquise.com.br/6f_8ei.php\nwww.hanecaklaw.com/\nwww.hanoiguidedtours.com/iq2q1f.php\nwww.healthstafftravel.com.au/oybbus.php\nwww.plexipr.com/vahzwx.php\nwww.rippedknees.co.uk/txmjcq.php\nwww.taoblu.com/wp-content/plugins/wp_module/sbml0j.php\nwww.vishvagujarat.com/5of9dt.php\nwww.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkg.php\n" }, { "path": "tests/integration/basic/URLHC%3Fv%3Dpanosurl%26sp%3D1.result", "content": "example.com/foobar?a=:80\n*.example.com/foobar?a=:80\nfeyda.net/hoedr4.php\n*.feyda.net/hoedr4.php\n7-eleven-handbags.com/x1rzyp.php\n8vs.com/6jezbr.php\nabdal.com.ua/7_jzay.php\naditaborai.com.br/wgngxe.php\nairconditioning12601.com/uploads/3/5/7/6/3576233/v5k3za.php\nallreadytravel.com/uploads/3/5/4/9/3549731/header_images/tomae1.php\nallstarpaintbody.com/lrq2bg.php\namericancorner.udp.cl/etloxw.php\nample-sun.eu/4bket7.php\nanime-tuner.square7.ch/wp-content/themes/twentyeleven/mstgk_.php\nanoukdelecluse.nl/lgzlb1.php\nappeum.com/wp-content/themes/cc.php\narot.altervista.org/khtudq.php\narttoday.sk/me8mkj.php\nascortimisoara.ro/kwih5v.php\naspectdesigns.com.au/0rtvlg.php\naudetlaw.com/lnvadf.php\nautohaus-seevetal.com/9x6uwk.php\navancarvisual.com.br/wp-content/themes/twentytwelve/vzkgnx.php\nbabylicious.ie/s1ghuz.php\nbalkanium.altervista.org/p3er4s.php\nbeachhouseplans.com/wp-admin/js/5d8gme.php\nbest-service.jp/olxu2y.php\nbeyondthedog.net/edhdvf.php\nbigboattravel.com/uploads/3/5/4/5/3545341/header_images/nthjhz.php\nbisofit.com/qxwm4i.php\nbktrade.kiev.ua/76b3zq.php\nboilersandfurnaces.com/uploads/3/5/1/6/3516773/rpyh2q.php\nbolizarsospos.com/0l0vp1va6b2\nbolizarsospos.com/1cslstk2qv121\nbolizarsospos.com/1xb81c28qs2db\nbolizarsospos.com/22o1210hbpw\nbolizarsospos.com/2h6t511wpuvnych\nbolizarsospos.com/379gz635s3j946\nbolizarsospos.com/4kpy8ju42x137\nbolizarsospos.com/503qu7boexyk\nbolizarsospos.com/574xl5yme0gdz\nbolizarsospos.com/5gpf7ecxhf\nbolizarsospos.com/5hmwl5qvpz2f3gc\nbolizarsospos.com/6m50uk8ty1031\nbolizarsospos.com/6tvpgu93q4wx5t\nbolizarsospos.com/703hjdr3ez72\nbolizarsospos.com/73075bdj8meb\nbolizarsospos.com/7gr904pzv6\nbolizarsospos.com/7ms68qsdfj0jt\nbolizarsospos.com/89e8f40k8zcn38\nbolizarsospos.com/8eo5zwhh4zndwwa\nbolizarsospos.com/8tsdhjccoxz6c\nbolizarsospos.com/94g2mr36b4\nbolizarsospos.com/9bqdnk2h58ty2l\nbolizarsospos.com/9hul78mtg1n63\nbolizarsospos.com/b0slgvfxvyf\nbolizarsospos.com/b3amhlkiar2c\nbolizarsospos.com/b8g7g560612\nbolizarsospos.com/bo5ha9ild1zjukv\nbolizarsospos.com/cannzqzrum14o4c\nbolizarsospos.com/ch3eq62ad8k\nbolizarsospos.com/ci72o4ruf2y87\nbolizarsospos.com/d5i52z8cgv5\nbolizarsospos.com/d65v4fx21f\nbolizarsospos.com/d7jly5f09tqj\nbolizarsospos.com/di53su4z7uqvj\nbolizarsospos.com/dypi31624z\nbolizarsospos.com/e5tkclwq9w0\nbolizarsospos.com/e887nn5k9pb6\nbolizarsospos.com/f1s0y87wrwo\nbolizarsospos.com/fgivit1drjuh\nbolizarsospos.com/fpkirizbrzxc5\nbolizarsospos.com/fxoztyxp320q\nbolizarsospos.com/g5k4uvxghygg7r\nbolizarsospos.com/gvi00me81aabu\nbolizarsospos.com/hq5drme48h\nbolizarsospos.com/hzpz767vze9\nbolizarsospos.com/ifkhfc5369az88\nbolizarsospos.com/iijoama0ynrowtp\nbolizarsospos.com/j4hzoz8cgdeza\nbolizarsospos.com/kka7641ov7\nbolizarsospos.com/ko679ybid6ys58\nbolizarsospos.com/kxdmlkhmuyf9\nbolizarsospos.com/kzqnheutxkjwhr\nbolizarsospos.com/mi5b67bilrfu\nbolizarsospos.com/n2csus3eo1tyg\nbolizarsospos.com/nve4m67l83\nbolizarsospos.com/o0nyjlre41o3\nbolizarsospos.com/pgjcokoi2kisu\nbolizarsospos.com/pl36lz43r6r7\nbolizarsospos.com/q3xryv3mh1\nbolizarsospos.com/qely217wcjdl7b\nbolizarsospos.com/qo9ux20lo1\nbolizarsospos.com/qu9ajlxsiw\nbolizarsospos.com/r45byxsjhz\nbolizarsospos.com/raph9xccgxt\nbolizarsospos.com/rb05hez1r044\nbolizarsospos.com/rdjg0eb5r0qs\nbolizarsospos.com/ri86nx23dhqbmch\nbolizarsospos.com/rjotoddb4n7hl\nbolizarsospos.com/rof06587c1x2y3t\nbolizarsospos.com/s40o542jt7v\nbolizarsospos.com/sb2zarf5vy\nbolizarsospos.com/uamuxps7y98\nbolizarsospos.com/uiyi9dkf5bs\nbolizarsospos.com/v13rw8n8w2\nbolizarsospos.com/vzum6ywdedxjtd\nbolizarsospos.com/walqb5xzunmr\nbolizarsospos.com/wilqkaz24rnqli\nbolizarsospos.com/x1tg111bara5\nbolizarsospos.com/x753k2s01gnd5b\nbolizarsospos.com/x7lfazpjuuiel\nbolizarsospos.com/xgw1o6gt9h8k9g\nbolizarsospos.com/xjp3zmw6glginuq\nbolizarsospos.com/yias364ajr\nbolizarsospos.com/zyayxp2kpay\nbucksmedia.go2cloud.org/aff_c\nbuilding.msu.ac.th/q3bslr.php\nbusinessaviators.com/r1doyf.php\nchallengestrata.com.au/fp_bxs.php\ncheapshirts.us/zvnmrg.php\nchong.joelle.free.fr/_l43ph.php\nconnectao.com/wp-content/themes/twentyeleven/cc.php\nd3mpd.fe.uns.ac.id/xpgmur.php\ndaffamedia.com/wp-content/plugins/wp_module/img5.php\ndechehang.com/gz2qrn.php\ndefinitionen.de/v7gves.php\ndichiro.com/waird6.php\ndillardvideo.com/wp-admin/network/2.php\ndining-bar.com/bq_ln4.php\ndomaine-cassillac.com/4q3esu.php\ndouble-wing.de/dzkclr.php\ndrdigitalmd.com/img1.php\neatside.es/xzqgxv.php\necocalsots.com/n79gta.php\necolux-comfort.com/npabsy.php\nelcoachingempresarial.com/wp-admin/user/2.php\nemprende21.es/otiq7a.php\nestudiobarco.com.ar/5tfv7e.php\nevent-travel.co.uk/3k6psd.php\nfeuerwehr-stadt-riesa.de/ufipoq.php\nfoundersomaha.net/wp-includes/text/diff/renderer/ap3.php\nframe3d.de/itgjkd.php\nfun-pop.com/ks1rcc.php\ngenedillardart.com/wp-admin/network/3.php\ngibdd.ws/j7d65p.php\nglitchygaming.com/r07qzu.php\ngrochowina.net/unvpso.php\nhaarsaloncindy.nl/xzf03r.php\nhamilton150.co.nz/lmfumz.php\nicsot.na.its.ac.id/8vwrux.php\nigatha.com/h4mekj.php\nilovesport.kiev.ua/z8x9t7.php\nimagescameraclub.com/j7b5kk.php\ninspirenetworks.in/vaqu3l.php\nitalyprego.com/lf2dca.php\njadwalpialadunia.in/rg4rdi.php\njambola.com/luylwv.php\njauregia.net/img5.php\nkonyavakfi.nl/zmje1r.php\nkuruyaprak.com/otluko.php\nkvnysoho.com/ehafft.php\nlazymoosestamping.com/yrfbgb.php\nlondon-escorts-agency.org.uk/fdnmyd.php\nlzclient.com/img4.php\nmadisonbootcamps.com/gwq3wp.php\nmangohills.net/rxioce.php\nmarciogerhardtsouza.com.br/mpcsdz.php\nmarcortes.com/img5.php\nmarkossolomon.com/f1q7qx.php\nmaternalserenity.co.uk/i_nwpg.php\nmillsmanagement.nl/anogvk.php\nmmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php\nnaimselmonaj.com/qoyx31.php\nnonnuoccaobang.com/brdodl.php\nnupleta.com.br/kohv09.php\nopenroadsolutions.com/fj2dow.php\noregonreversemortgage.com/rwafp_.php\np237996.mybestmv.com/adserve/domainclick\npaintituppottery.com/6cmeb2.php\npatrianossa.com.br/u8lkzd.php\nportalmaismidia.com.br/tnsmib.php\nportret-tekening.nl/mnqvts.php\nprocrediti.com.ua/d6ygox.php\nrajsima87.com/img2.php\nrecaswine.ro/dxlq0y.php\nsilstop.pl/si0ccj.php\nsmartnote.co/2nxvza.php\nstudiolegalecsb.it/iqcnfc.php\ntakaram.ir/gjorez.php\ntakatei.com/rfyi4l.php\ntcblog.de/mxdvth.php\ntheassemblyguy.co.nz/vpfabq.php\ntrion.com.ph/jdkaap.php\ntusrecetas.net/jbeln7.php\ntutorialswalk.info/wp-content/themes/defne/img2.php\nviralcrazies.com/ifht4c.php\nvsedveri-33.ru/\nweberteam.hu/wctdo5.php\nwww.001edizioni.com/nzwt_a.php\nwww.bishopbell.co.uk/enrmcc.php\nwww.chemes.eu/wp-content/themes/decoy2/redux-framework/reduxcore/inc/fields/info/2.php\nwww.decorandoimoveis.com/qeo5yh.php\nwww.example.com/\nwww.feddoctor.com/oe1lmr.php\nwww.gjscomputerservices.com.au/s1_rvm.php\nwww.granmarquise.com.br/6f_8ei.php\nwww.hanecaklaw.com/\nwww.hanoiguidedtours.com/iq2q1f.php\nwww.healthstafftravel.com.au/oybbus.php\nwww.plexipr.com/vahzwx.php\nwww.rippedknees.co.uk/txmjcq.php\nwww.taoblu.com/wp-content/plugins/wp_module/sbml0j.php\nwww.vishvagujarat.com/5of9dt.php\nwww.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkg.php\n" }, { "path": "tests/integration/basic/URLHC%3Fv%3Dpanosurl.result", "content": "example.com/foobar?a=:80\n*.example.com/foobar?a=:80\nfeyda.net/hoedr4.php\n*.feyda.net/hoedr4.php\n7-eleven-handbags.com/x1rzyp.php\n8vs.com/6jezbr.php\nabdal.com.ua/7_jzay.php\naditaborai.com.br/wgngxe.php\nairconditioning12601.com/uploads/3/5/7/6/3576233/v5k3za.php\nallreadytravel.com/uploads/3/5/4/9/3549731/header_images/tomae1.php\nallstarpaintbody.com/lrq2bg.php\namericancorner.udp.cl/etloxw.php\nample-sun.eu/4bket7.php\nanime-tuner.square7.ch/wp-content/themes/twentyeleven/mstgk_.php\nanoukdelecluse.nl/lgzlb1.php\nappeum.com/wp-content/themes/cc.php\narot.altervista.org/khtudq.php\narttoday.sk/me8mkj.php\nascortimisoara.ro/kwih5v.php\naspectdesigns.com.au/0rtvlg.php\naudetlaw.com/lnvadf.php\nautohaus-seevetal.com/9x6uwk.php\navancarvisual.com.br/wp-content/themes/twentytwelve/vzkgnx.php\nbabylicious.ie/s1ghuz.php\nbalkanium.altervista.org/p3er4s.php\nbeachhouseplans.com/wp-admin/js/5d8gme.php\nbest-service.jp/olxu2y.php\nbeyondthedog.net/edhdvf.php\nbigboattravel.com/uploads/3/5/4/5/3545341/header_images/nthjhz.php\nbisofit.com/qxwm4i.php\nbktrade.kiev.ua/76b3zq.php\nboilersandfurnaces.com/uploads/3/5/1/6/3516773/rpyh2q.php\nbolizarsospos.com/0l0vp1va6b2\nbolizarsospos.com/1cslstk2qv121\nbolizarsospos.com/1xb81c28qs2db\nbolizarsospos.com/22o1210hbpw\nbolizarsospos.com/2h6t511wpuvnych\nbolizarsospos.com/379gz635s3j946\nbolizarsospos.com/4kpy8ju42x137\nbolizarsospos.com/503qu7boexyk\nbolizarsospos.com/574xl5yme0gdz\nbolizarsospos.com/5gpf7ecxhf\nbolizarsospos.com/5hmwl5qvpz2f3gc\nbolizarsospos.com/6m50uk8ty1031\nbolizarsospos.com/6tvpgu93q4wx5t\nbolizarsospos.com/703hjdr3ez72\nbolizarsospos.com/73075bdj8meb\nbolizarsospos.com/7gr904pzv6\nbolizarsospos.com/7ms68qsdfj0jt\nbolizarsospos.com/89e8f40k8zcn38\nbolizarsospos.com/8eo5zwhh4zndwwa\nbolizarsospos.com/8tsdhjccoxz6c\nbolizarsospos.com/94g2mr36b4\nbolizarsospos.com/9bqdnk2h58ty2l\nbolizarsospos.com/9hul78mtg1n63\nbolizarsospos.com/b0slgvfxvyf\nbolizarsospos.com/b3amhlkiar2c\nbolizarsospos.com/b8g7g560612\nbolizarsospos.com/bo5ha9ild1zjukv\nbolizarsospos.com/cannzqzrum14o4c\nbolizarsospos.com/ch3eq62ad8k\nbolizarsospos.com/ci72o4ruf2y87\nbolizarsospos.com/d5i52z8cgv5\nbolizarsospos.com/d65v4fx21f\nbolizarsospos.com/d7jly5f09tqj\nbolizarsospos.com/di53su4z7uqvj\nbolizarsospos.com/dypi31624z\nbolizarsospos.com/e5tkclwq9w0\nbolizarsospos.com/e887nn5k9pb6\nbolizarsospos.com/f1s0y87wrwo\nbolizarsospos.com/fgivit1drjuh\nbolizarsospos.com/fpkirizbrzxc5\nbolizarsospos.com/fxoztyxp320q\nbolizarsospos.com/g5k4uvxghygg7r\nbolizarsospos.com/gvi00me81aabu\nbolizarsospos.com/hq5drme48h\nbolizarsospos.com/hzpz767vze9\nbolizarsospos.com/ifkhfc5369az88\nbolizarsospos.com/iijoama0ynrowtp\nbolizarsospos.com/j4hzoz8cgdeza\nbolizarsospos.com/kka7641ov7\nbolizarsospos.com/ko679ybid6ys58\nbolizarsospos.com/kxdmlkhmuyf9\nbolizarsospos.com/kzqnheutxkjwhr\nbolizarsospos.com/mi5b67bilrfu\nbolizarsospos.com/n2csus3eo1tyg\nbolizarsospos.com/nve4m67l83\nbolizarsospos.com/o0nyjlre41o3\nbolizarsospos.com/pgjcokoi2kisu\nbolizarsospos.com/pl36lz43r6r7\nbolizarsospos.com/q3xryv3mh1\nbolizarsospos.com/qely217wcjdl7b\nbolizarsospos.com/qo9ux20lo1\nbolizarsospos.com/qu9ajlxsiw\nbolizarsospos.com/r45byxsjhz\nbolizarsospos.com/raph9xccgxt\nbolizarsospos.com/rb05hez1r044\nbolizarsospos.com/rdjg0eb5r0qs\nbolizarsospos.com/ri86nx23dhqbmch\nbolizarsospos.com/rjotoddb4n7hl\nbolizarsospos.com/rof06587c1x2y3t\nbolizarsospos.com/s40o542jt7v\nbolizarsospos.com/sb2zarf5vy\nbolizarsospos.com/uamuxps7y98\nbolizarsospos.com/uiyi9dkf5bs\nbolizarsospos.com/v13rw8n8w2\nbolizarsospos.com/vzum6ywdedxjtd\nbolizarsospos.com/walqb5xzunmr\nbolizarsospos.com/wilqkaz24rnqli\nbolizarsospos.com/x1tg111bara5\nbolizarsospos.com/x753k2s01gnd5b\nbolizarsospos.com/x7lfazpjuuiel\nbolizarsospos.com/xgw1o6gt9h8k9g\nbolizarsospos.com/xjp3zmw6glginuq\nbolizarsospos.com/yias364ajr\nbolizarsospos.com/zyayxp2kpay\nbucksmedia.go2cloud.org/aff_c\nbuilding.msu.ac.th/q3bslr.php\nbusinessaviators.com/r1doyf.php\nchallengestrata.com.au/fp_bxs.php\ncheapshirts.us/zvnmrg.php\nchong.joelle.free.fr/_l43ph.php\nconnectao.com/wp-content/themes/twentyeleven/cc.php\nd3mpd.fe.uns.ac.id/xpgmur.php\ndaffamedia.com/wp-content/plugins/wp_module/img5.php\ndechehang.com/gz2qrn.php\ndefinitionen.de/v7gves.php\ndichiro.com/waird6.php\ndillardvideo.com/wp-admin/network/2.php\ndining-bar.com/bq_ln4.php\ndomaine-cassillac.com/4q3esu.php\ndouble-wing.de/dzkclr.php\ndrdigitalmd.com/img1.php\neatside.es/xzqgxv.php\necocalsots.com/n79gta.php\necolux-comfort.com/npabsy.php\nelcoachingempresarial.com/wp-admin/user/2.php\nemprende21.es/otiq7a.php\nestudiobarco.com.ar/5tfv7e.php\nevent-travel.co.uk/3k6psd.php\nfeuerwehr-stadt-riesa.de/ufipoq.php\nfoundersomaha.net/wp-includes/text/diff/renderer/ap3.php\nframe3d.de/itgjkd.php\nfun-pop.com/ks1rcc.php\ngenedillardart.com/wp-admin/network/3.php\ngibdd.ws/j7d65p.php\nglitchygaming.com/r07qzu.php\ngrochowina.net/unvpso.php\nhaarsaloncindy.nl/xzf03r.php\nhamilton150.co.nz/lmfumz.php\nicsot.na.its.ac.id/8vwrux.php\nigatha.com/h4mekj.php\nilovesport.kiev.ua/z8x9t7.php\nimagescameraclub.com/j7b5kk.php\ninspirenetworks.in/vaqu3l.php\nitalyprego.com/lf2dca.php\njadwalpialadunia.in/rg4rdi.php\njambola.com/luylwv.php\njauregia.net/img5.php\nkonyavakfi.nl/zmje1r.php\nkuruyaprak.com/otluko.php\nkvnysoho.com/ehafft.php\nlazymoosestamping.com/yrfbgb.php\nlondon-escorts-agency.org.uk/fdnmyd.php\nlzclient.com/img4.php\nmadisonbootcamps.com/gwq3wp.php\nmangohills.net/rxioce.php\nmarciogerhardtsouza.com.br/mpcsdz.php\nmarcortes.com/img5.php\nmarkossolomon.com/f1q7qx.php\nmaternalserenity.co.uk/i_nwpg.php\nmillsmanagement.nl/anogvk.php\nmmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php\nnaimselmonaj.com/qoyx31.php\nnonnuoccaobang.com/brdodl.php\nnupleta.com.br/kohv09.php\nopenroadsolutions.com/fj2dow.php\noregonreversemortgage.com/rwafp_.php\np237996.mybestmv.com/adserve/domainclick\npaintituppottery.com/6cmeb2.php\npatrianossa.com.br/u8lkzd.php\nportalmaismidia.com.br/tnsmib.php\nportret-tekening.nl/mnqvts.php\nprocrediti.com.ua/d6ygox.php\nrajsima87.com/img2.php\nrecaswine.ro/dxlq0y.php\nsilstop.pl/si0ccj.php\nsmartnote.co/2nxvza.php\nstudiolegalecsb.it/iqcnfc.php\ntakaram.ir/gjorez.php\ntakatei.com/rfyi4l.php\ntcblog.de/mxdvth.php\ntheassemblyguy.co.nz/vpfabq.php\ntrion.com.ph/jdkaap.php\ntusrecetas.net/jbeln7.php\ntutorialswalk.info/wp-content/themes/defne/img2.php\nviralcrazies.com/ifht4c.php\nvsedveri-33.ru/\nweberteam.hu/wctdo5.php\nwww.001edizioni.com/nzwt_a.php\nwww.bishopbell.co.uk/enrmcc.php\nwww.chemes.eu/wp-content/themes/decoy2/redux-framework/reduxcore/inc/fields/info/2.php\nwww.decorandoimoveis.com/qeo5yh.php\nwww.feddoctor.com/oe1lmr.php\nwww.gjscomputerservices.com.au/s1_rvm.php\nwww.granmarquise.com.br/6f_8ei.php\nwww.hanecaklaw.com/\nwww.hanoiguidedtours.com/iq2q1f.php\nwww.healthstafftravel.com.au/oybbus.php\nwww.plexipr.com/vahzwx.php\nwww.rippedknees.co.uk/txmjcq.php\nwww.taoblu.com/wp-content/plugins/wp_module/sbml0j.php\nwww.vishvagujarat.com/5of9dt.php\nwww.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkg.php\n" }, { "path": "tests/integration/basic/URLHC.result", "content": "*abc.example.com/foobar?a=:80\nhttp://*.feyda.net/hOeDr4.php\nhttp://7-eleven-handbags.com/X1rZYp.php\nhttp://8vs.com/6jezbr.php\nhttp://abdal.com.ua/7_jzay.php\nhttp://aditaborai.com.br/WgNGXe.php\nhttp://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php\nhttp://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php\nhttp://allstarpaintbody.com/lrQ2bG.php\nhttp://americancorner.udp.cl/etloxW.php\nhttp://ample-sun.eu/4BKEt7.php\nhttp://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php\nhttp://anoukdelecluse.nl/lGZLB1.php\nhttp://appeum.com/wp-content/themes/cc.php\nhttp://arot.altervista.org/KHTUdq.php\nhttp://arttoday.sk/mE8MKJ.php\nhttp://ascortimisoara.ro/kWIH5V.php\nhttp://aspectdesigns.com.au/0rTVlG.php\nhttp://audetlaw.com/LnVAdF.php\nhttp://autohaus-seevetal.com/9x6UwK.php\nhttp://avancarvisual.com.br/wp-content/themes/twentytwelve/VzkgnX.php\nhttp://babylicious.ie/s1GHUZ.php\nhttp://balkanium.altervista.org/p3er4s.php\nhttp://beachhouseplans.com/wp-admin/js/5d8gMe.php\nhttp://best-service.jp/olxu2Y.php\nhttp://beyondthedog.net/edHDvf.php\nhttp://bigboattravel.com/uploads/3/5/4/5/3545341/header_images/NthjHz.php\nhttp://bisofit.com/QXwm4I.php\nhttp://bktrade.kiev.ua/76b3ZQ.php\nhttp://boilersandfurnaces.com/uploads/3/5/1/6/3516773/RPyH2q.php\nhttp://bolizarsospos.com/0l0vp1va6b2\nhttp://bolizarsospos.com/1cslstk2qv121\nhttp://bolizarsospos.com/1xb81c28qs2db\nhttp://bolizarsospos.com/22o1210hbpw\nhttp://bolizarsospos.com/2h6t511wpuvnych\nhttp://bolizarsospos.com/379gz635s3j946\nhttp://bolizarsospos.com/4kpy8ju42x137\nhttp://bolizarsospos.com/503qu7boexyk\nhttp://bolizarsospos.com/574xl5yme0gdz\nhttp://bolizarsospos.com/5gpf7ecxhf\nhttp://bolizarsospos.com/5hmwl5qvpz2f3gc\nhttp://bolizarsospos.com/6m50uk8ty1031\nhttp://bolizarsospos.com/6tvpgu93q4wx5t\nhttp://bolizarsospos.com/703hjdr3ez72\nhttp://bolizarsospos.com/73075bdj8meb\nhttp://bolizarsospos.com/7gr904pzv6\nhttp://bolizarsospos.com/7ms68qsdfj0jt\nhttp://bolizarsospos.com/89e8f40k8zcn38\nhttp://bolizarsospos.com/8eo5zwhh4zndwwa\nhttp://bolizarsospos.com/8tsdhjccoxz6c\nhttp://bolizarsospos.com/94g2mr36b4\nhttp://bolizarsospos.com/9bqdnk2h58ty2l\nhttp://bolizarsospos.com/9hul78mtg1n63\nhttp://bolizarsospos.com/b0slgvfxvyf\nhttp://bolizarsospos.com/b3amhlkiar2c\nhttp://bolizarsospos.com/b8g7g560612\nhttp://bolizarsospos.com/bo5ha9ild1zjukv\nhttp://bolizarsospos.com/cannzqzrum14o4c\nhttp://bolizarsospos.com/ch3eq62ad8k\nhttp://bolizarsospos.com/ci72o4ruf2y87\nhttp://bolizarsospos.com/d5i52z8cgv5\nhttp://bolizarsospos.com/d65v4fx21f\nhttp://bolizarsospos.com/d7jly5f09tqj\nhttp://bolizarsospos.com/di53su4z7uqvj\nhttp://bolizarsospos.com/dypi31624z\nhttp://bolizarsospos.com/e5tkclwq9w0\nhttp://bolizarsospos.com/e887nn5k9pb6\nhttp://bolizarsospos.com/f1s0y87wrwo\nhttp://bolizarsospos.com/fgivit1drjuh\nhttp://bolizarsospos.com/fpkirizbrzxc5\nhttp://bolizarsospos.com/fxoztyxp320q\nhttp://bolizarsospos.com/g5k4uvxghygg7r\nhttp://bolizarsospos.com/gvi00me81aabu\nhttp://bolizarsospos.com/hq5drme48h\nhttp://bolizarsospos.com/hzpz767vze9\nhttp://bolizarsospos.com/ifkhfc5369az88\nhttp://bolizarsospos.com/iijoama0ynrowtp\nhttp://bolizarsospos.com/j4hzoz8cgdeza\nhttp://bolizarsospos.com/kka7641ov7\nhttp://bolizarsospos.com/ko679ybid6ys58\nhttp://bolizarsospos.com/kxdmlkhmuyf9\nhttp://bolizarsospos.com/kzqnheutxkjwhr\nhttp://bolizarsospos.com/mi5b67bilrfu\nhttp://bolizarsospos.com/n2csus3eo1tyg\nhttp://bolizarsospos.com/nve4m67l83\nhttp://bolizarsospos.com/o0nyjlre41o3\nhttp://bolizarsospos.com/pgjcokoi2kisu\nhttp://bolizarsospos.com/pl36lz43r6r7\nhttp://bolizarsospos.com/q3xryv3mh1\nhttp://bolizarsospos.com/qely217wcjdl7b\nhttp://bolizarsospos.com/qo9ux20lo1\nhttp://bolizarsospos.com/qu9ajlxsiw\nhttp://bolizarsospos.com/r45byxsjhz\nhttp://bolizarsospos.com/raph9xccgxt\nhttp://bolizarsospos.com/rb05hez1r044\nhttp://bolizarsospos.com/rdjg0eb5r0qs\nhttp://bolizarsospos.com/ri86nx23dhqbmch\nhttp://bolizarsospos.com/rjotoddb4n7hl\nhttp://bolizarsospos.com/rof06587c1x2y3t\nhttp://bolizarsospos.com/s40o542jt7v\nhttp://bolizarsospos.com/sb2zarf5vy\nhttp://bolizarsospos.com/uamuxps7y98\nhttp://bolizarsospos.com/uiyi9dkf5bs\nhttp://bolizarsospos.com/v13rw8n8w2\nhttp://bolizarsospos.com/vzum6ywdedxjtd\nhttp://bolizarsospos.com/walqb5xzunmr\nhttp://bolizarsospos.com/wilqkaz24rnqli\nhttp://bolizarsospos.com/x1tg111bara5\nhttp://bolizarsospos.com/x753k2s01gnd5b\nhttp://bolizarsospos.com/x7lfazpjuuiel\nhttp://bolizarsospos.com/xgw1o6gt9h8k9g\nhttp://bolizarsospos.com/xjp3zmw6glginuq\nhttp://bolizarsospos.com/yias364ajr\nhttp://bolizarsospos.com/zyayxp2kpay\nhttp://bucksmedia.go2cloud.org/aff_c\nhttp://building.msu.ac.th/q3Bslr.php\nhttp://businessaviators.com/r1doyF.php\nhttp://challengestrata.com.au/fP_BXS.php\nhttp://cheapshirts.us/zVnMrG.php\nhttp://chong.joelle.free.fr/_L43PH.php\nhttp://connectao.com/wp-content/themes/twentyeleven/cc.php\nhttp://d3mpd.fe.uns.ac.id/XPgmur.php\nhttp://daffamedia.com/wp-content/plugins/wp_module/img5.php\nhttp://dechehang.com/GZ2QRn.php\nhttp://definitionen.de/v7GVES.php\nhttp://dichiro.com/WaIrd6.php\nhttp://dillardvideo.com/wp-admin/network/2.php\nhttp://dining-bar.com/BQ_Ln4.php\nhttp://domaine-cassillac.com/4q3esU.php\nhttp://double-wing.de/DZkCLR.php\nhttp://drdigitalmd.com/img1.php\nhttp://eatside.es/xZQGXV.php\nhttp://ecocalsots.com/N79GTA.php\nhttp://ecolux-comfort.com/nPAbsy.php\nhttp://elcoachingempresarial.com/wp-admin/user/2.php\nhttp://emprende21.es/oTIq7A.php\nhttp://estudiobarco.com.ar/5TFv7E.php\nhttp://event-travel.co.uk/3K6Psd.php\nhttp://feuerwehr-stadt-riesa.de/UFiPOq.php\nhttp://foundersomaha.net/wp-includes/Text/Diff/Renderer/ap3.php\nhttp://frame3d.de/ItGJKd.php\nhttp://fun-pop.com/Ks1rCc.php\nhttp://genedillardart.com/wp-admin/network/3.php\nhttp://gibdd.ws/J7D65p.php\nhttp://glitchygaming.com/r07QZu.php\nhttp://grochowina.net/UnvPso.php\nhttp://haarsaloncindy.nl/XzF03r.php\nhttp://hamilton150.co.nz/LmfuMZ.php\nhttp://icsot.na.its.ac.id/8vwRUX.php\nhttp://igatha.com/h4MeKJ.php\nhttp://ilovesport.kiev.ua/z8X9T7.php\nhttp://imagescameraclub.com/j7b5kK.php\nhttp://inspirenetworks.in/vAqu3L.php\nhttp://italyprego.com/Lf2dcA.php\nhttp://jadwalpialadunia.in/rG4Rdi.php\nhttp://jambola.com/LuylWV.php\nhttp://jauregia.net/img5.php\nhttp://konyavakfi.nl/Zmje1r.php\nhttp://kuruyaprak.com/OTLuKo.php\nhttp://kvnysoho.com/eHafFT.php\nhttp://lazymoosestamping.com/YRfbgB.php\nhttp://london-escorts-agency.org.uk/fdnmyD.php\nhttp://lzclient.com/img4.php\nhttp://madisonbootcamps.com/gWQ3wp.php\nhttp://mangohills.net/RxIoCE.php\nhttp://marciogerhardtsouza.com.br/mPCsDz.php\nhttp://marcortes.com/img5.php\nhttp://markossolomon.com/F1q7QX.php\nhttp://maternalserenity.co.uk/I_NwPg.php\nhttp://millsmanagement.nl/AnOgVK.php\nhttp://mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php\nhttp://naimselmonaj.com/QoYx31.php\nhttp://nonnuoccaobang.com/BRdoDL.php\nhttp://nupleta.com.br/KoHV09.php\nhttp://openroadsolutions.com/FJ2dOw.php\nhttp://oregonreversemortgage.com/Rwafp_.php\nhttp://p237996.mybestmv.com/adServe/domainClick\nhttp://paintituppottery.com/6cmeb2.php\nhttp://patrianossa.com.br/u8LkzD.php\nhttp://portalmaismidia.com.br/tnSmIb.php\nhttp://portret-tekening.nl/mNQVts.php\nhttp://procrediti.com.ua/d6yGOX.php\nhttp://rajsima87.com/img2.php\nhttp://recaswine.ro/dXlq0Y.php\nhttp://silstop.pl/Si0cCJ.php\nhttp://smartnote.co/2NxVzA.php\nhttp://studiolegalecsb.it/iQcNfC.php\nhttp://takaram.ir/gjOREZ.php\nhttp://takatei.com/rfYI4L.php\nhttp://tcblog.de/mXdVTh.php\nhttp://theassemblyguy.co.nz/vpFAbQ.php\nhttp://trion.com.ph/jdKAap.php\nhttp://tusrecetas.net/JbElN7.php\nhttp://tutorialswalk.info/wp-content/themes/Defne/img2.php\nhttp://viralcrazies.com/iFHt4C.php\nhttp://vsedveri-33.ru/\nhttp://weberteam.hu/WCTdO5.php\nhttp://www.001edizioni.com/NZwt_a.php\nhttp://www.bishopbell.co.uk/enRmcC.php\nhttp://www.chemes.eu/wp-content/themes/decoy2/redux-framework/ReduxCore/inc/fields/info/2.php\nhttp://www.decorandoimoveis.com/QEO5yh.php\nhttp://www.example.com:1000\nhttp://www.feddoctor.com/Oe1LMr.php\nhttp://www.gjscomputerservices.com.au/S1_rvm.php\nhttp://www.granmarquise.com.br/6f_8ei.php\nhttp://www.hanecaklaw.com/\nhttp://www.hanoiguidedtours.com/iQ2q1f.php\nhttp://www.healthstafftravel.com.au/oyBbUs.php\nhttp://www.plexipr.com/vAHzWX.php\nhttp://www.rippedknees.co.uk/TXmJcq.php\nhttp://www.taoblu.com/wp-content/plugins/wp_module/sbML0j.php\nhttp://www.vishvagujarat.com/5of9dt.php\nhttp://www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkG.php\nhttps://*abc.*test.com\n" }, { "path": "tests/integration/basic/domain.lst", "content": "25z5g623wpqpdwis.onion.to\n27c73bq66y4xqoh7.dorfact.at\n27lelchgcvs2wpm7.3lhjyx.top\n27lelchgcvs2wpm7.7jiff7.top\n27lelchgcvs2wpm7.7zv8o2.top\n27lelchgcvs2wpm7.9ildst.top\n27lelchgcvs2wpm7.adevf4.top\n27lelchgcvs2wpm7.ag082d.top\n27lelchgcvs2wpm7.apperloads.win\n27lelchgcvs2wpm7.asd3r3.top\n27lelchgcvs2wpm7.b7mciu.top\n27lelchgcvs2wpm7.bedrastic.bid\n27lelchgcvs2wpm7.bestfordownload.click\n27lelchgcvs2wpm7.bonbestal.asia\n27lelchgcvs2wpm7.fm0cga.top\n27lelchgcvs2wpm7.h9ihx3.top\n27lelchgcvs2wpm7.laverhants.link\n27lelchgcvs2wpm7.liopakerb.black\n27lelchgcvs2wpm7.marksgain.kim\n27lelchgcvs2wpm7.nfgpeb.top\n27lelchgcvs2wpm7.redefined.click\n27lelchgcvs2wpm7.rt4e34.win\n27lelchgcvs2wpm7.tankbe.pro\n27lelchgcvs2wpm7.thyx30.top\n27lelchgcvs2wpm7.uboys5.top\n27lelchgcvs2wpm7.vrid8l.top\n27lelchgcvs2wpm7.wins4n.win\n27lelchgcvs2wpm7.wishsends.mobi\n27lelchgcvs2wpm7.xkfi59.top\n27lelchgcvs2wpm7.xmvr54.top\n2bdfb.spinakrosa.at\n2gdb4.leoraorage.at\n2ymh2gnnbg6pgq2r.gremsot.pl\n2ymh2gnnbg6pgq2r.winregion.tw\n32kl2rwsjvqjeui7.onion.cab\n32kl2rwsjvqjeui7.onion.to\n32kl2rwsjvqjeui7.tor2web.org\n37kddsserrt.xyz\n3qbyaoohkcqkzrz6.bestxprice.ch\n3qbyaoohkcqkzrz6.livecamshow.ch\n3qbyaoohkcqkzrz6.torclassik.li\n3qbyaoohkcqkzrz6.torcommunity.ch\n3qbyaoohkcqkzrz6.tordonator.li\n3qbyaoohkcqkzrz6.tordoor.li\n3qbyaoohkcqkzrz6.torgate.es\n3qbyaoohkcqkzrz6.torgateway.li\n3qbyaoohkcqkzrz6.tormain.li\n3qbyaoohkcqkzrz6.tormaster.ch\n3qbyaoohkcqkzrz6.tormaster.fr\n3qbyaoohkcqkzrz6.torplanet.eu\n3qbyaoohkcqkzrz6.torprovider.li\n3qbyaoohkcqkzrz6.torreactor.li\n3qbyaoohkcqkzrz6.torstation.li\n4kqd3hmqgptupi3p.0vgu64.top\n4kqd3hmqgptupi3p.143h2a.top\n4kqd3hmqgptupi3p.1tvjk1.top\n4kqd3hmqgptupi3p.1zp109.bid\n4kqd3hmqgptupi3p.249isv.bid\n4kqd3hmqgptupi3p.2y4t6f.bid\n4kqd3hmqgptupi3p.3arvfd.top\n4kqd3hmqgptupi3p.3lhjyx.top\n4kqd3hmqgptupi3p.43wjor.top\n4kqd3hmqgptupi3p.4j11jt.bid\n4kqd3hmqgptupi3p.4k9xlx.top\n4kqd3hmqgptupi3p.5b4ej6.bid\n4kqd3hmqgptupi3p.5ctoeb.bid\n4kqd3hmqgptupi3p.62er3d.top\n4kqd3hmqgptupi3p.6h03gw.top\n4kqd3hmqgptupi3p.6j7jcn.bid\n4kqd3hmqgptupi3p.6ntrb6.top\n4kqd3hmqgptupi3p.6ogy3i.top\n4kqd3hmqgptupi3p.7w9p1n.bid\n4kqd3hmqgptupi3p.859rkn.top\n4kqd3hmqgptupi3p.8kcfnk.bid\n4kqd3hmqgptupi3p.91006j.bid\n4kqd3hmqgptupi3p.9ildst.top\n4kqd3hmqgptupi3p.a0g0o7.bid\n4kqd3hmqgptupi3p.adevf4.top\n4kqd3hmqgptupi3p.anypicked.red\n4kqd3hmqgptupi3p.as5su5.top\n4kqd3hmqgptupi3p.asfall.in\n4kqd3hmqgptupi3p.athere.in\n4kqd3hmqgptupi3p.b7mciu.top\n4kqd3hmqgptupi3p.barberryshin.casa\n4kqd3hmqgptupi3p.bestergo.pw\n4kqd3hmqgptupi3p.bigfooters.loan\n4kqd3hmqgptupi3p.bnctf6.top\n4kqd3hmqgptupi3p.bookjumps.us\n4kqd3hmqgptupi3p.boxsame.kim\n4kqd3hmqgptupi3p.boxtimed.gdn\n4kqd3hmqgptupi3p.breakown.loan\n4kqd3hmqgptupi3p.byeraser.lol\n4kqd3hmqgptupi3p.carrygain.kim\n4kqd3hmqgptupi3p.cfu46r.bid\n4kqd3hmqgptupi3p.chargecar.vip\n4kqd3hmqgptupi3p.choiceher.win\n4kqd3hmqgptupi3p.clockhate.loan\n4kqd3hmqgptupi3p.cm5ohx.bid\n4kqd3hmqgptupi3p.csv7o6.bid\n4kqd3hmqgptupi3p.cutslifes.bid\n4kqd3hmqgptupi3p.dd4xo3.top\n4kqd3hmqgptupi3p.dkrie7.top\n4kqd3hmqgptupi3p.dmvute.top\n4kqd3hmqgptupi3p.dozensby.loan\n4kqd3hmqgptupi3p.easyits.black\n4kqd3hmqgptupi3p.effortany.win\n4kqd3hmqgptupi3p.endsdoubt.loan\n4kqd3hmqgptupi3p.eventeach.gdn\n4kqd3hmqgptupi3p.ezm0r5.top\n4kqd3hmqgptupi3p.f0jlbj.bid\n4kqd3hmqgptupi3p.fairlies.link\n4kqd3hmqgptupi3p.foodtopic.mobi\n4kqd3hmqgptupi3p.g7kcux.bid\n4kqd3hmqgptupi3p.gameswarm.loan\n4kqd3hmqgptupi3p.gapplayed.link\n4kqd3hmqgptupi3p.getsbug.kim\n4kqd3hmqgptupi3p.gg4dgp.bid\n4kqd3hmqgptupi3p.gio6f6.bid\n4kqd3hmqgptupi3p.gletterstan.trade\n4kqd3hmqgptupi3p.goodslet.win\n4kqd3hmqgptupi3p.goshare.red\n4kqd3hmqgptupi3p.gs2ka7.top\n4kqd3hmqgptupi3p.he81tz.bid\n4kqd3hmqgptupi3p.heardbids.date\n4kqd3hmqgptupi3p.heldbegun.kim\n4kqd3hmqgptupi3p.hessale.pw\n4kqd3hmqgptupi3p.holescase.pw\n4kqd3hmqgptupi3p.homehuge.top\n4kqd3hmqgptupi3p.hotcopies.bid\n4kqd3hmqgptupi3p.inforcing.pw\n4kqd3hmqgptupi3p.insystem.men\n4kqd3hmqgptupi3p.itdrink.club\n4kqd3hmqgptupi3p.ix1upt.bid\n4kqd3hmqgptupi3p.jal9lk.bid\n4kqd3hmqgptupi3p.k7oud1.top\n4kqd3hmqgptupi3p.kml2o2.top\n4kqd3hmqgptupi3p.l6k4x7.bid\n4kqd3hmqgptupi3p.laterugly.win\n4kqd3hmqgptupi3p.liescale.in\n4kqd3hmqgptupi3p.liesshall.bid\n4kqd3hmqgptupi3p.lobulz.bid\n4kqd3hmqgptupi3p.lorrydo.lol\n4kqd3hmqgptupi3p.masterany.red\n4kqd3hmqgptupi3p.meetbinds.pw\n4kqd3hmqgptupi3p.metmet.win\n4kqd3hmqgptupi3p.metpast.site\n4kqd3hmqgptupi3p.mi3596.bid\n4kqd3hmqgptupi3p.mtxtul.top\n4kqd3hmqgptupi3p.mustspace.us\n4kqd3hmqgptupi3p.myaddress.link\n4kqd3hmqgptupi3p.namefalls.pro\n4kqd3hmqgptupi3p.nameuser.site\n4kqd3hmqgptupi3p.nearlybut.us\n4kqd3hmqgptupi3p.needmight.win\n4kqd3hmqgptupi3p.newrange.link\n4kqd3hmqgptupi3p.nextask.loan\n4kqd3hmqgptupi3p.nh47ri.bid\n4kqd3hmqgptupi3p.nxmu0x.bid\n4kqd3hmqgptupi3p.o8hpwj.top\n4kqd3hmqgptupi3p.outputon.asia\n4kqd3hmqgptupi3p.ownamount.pro\n4kqd3hmqgptupi3p.p79b8l.bid\n4kqd3hmqgptupi3p.pairsraw.loan\n4kqd3hmqgptupi3p.pap44w.top\n4kqd3hmqgptupi3p.powersno.link\n4kqd3hmqgptupi3p.pushstory.bid\n4kqd3hmqgptupi3p.r21wmw.top\n4kqd3hmqgptupi3p.rsi6gn.top\n4kqd3hmqgptupi3p.salethe.gdn\n4kqd3hmqgptupi3p.sayssales.bid\n4kqd3hmqgptupi3p.scoreable.bid\n4kqd3hmqgptupi3p.seemby.loan\n4kqd3hmqgptupi3p.sel7rg.bid\n4kqd3hmqgptupi3p.selfcrash.site\n4kqd3hmqgptupi3p.sentowing.trade\n4kqd3hmqgptupi3p.sitcalls.us\n4kqd3hmqgptupi3p.sk8r54.top\n4kqd3hmqgptupi3p.somegave.info\n4kqd3hmqgptupi3p.stageend.link\n4kqd3hmqgptupi3p.stopsage.gdn\n4kqd3hmqgptupi3p.storingus.gdn\n4kqd3hmqgptupi3p.tankplain.date\n4kqd3hmqgptupi3p.termprior.men\n4kqd3hmqgptupi3p.themevery.win\n4kqd3hmqgptupi3p.thyx30.top\n4kqd3hmqgptupi3p.tieslaws.link\n4kqd3hmqgptupi3p.todaynine.loan\n4kqd3hmqgptupi3p.twz1ga.top\n4kqd3hmqgptupi3p.uwckha.top\n4kqd3hmqgptupi3p.v11z5e.top\n4kqd3hmqgptupi3p.valueshes.bid\n4kqd3hmqgptupi3p.variedtax.kim\n4kqd3hmqgptupi3p.vkm4l6.top\n4kqd3hmqgptupi3p.wallluck.date\n4kqd3hmqgptupi3p.whmykv.bid\n4kqd3hmqgptupi3p.wins4n.top\n4kqd3hmqgptupi3p.wz139z.top\n4kqd3hmqgptupi3p.xmfru5.top\n4kqd3hmqgptupi3p.y12acl.bid\n4kqd3hmqgptupi3p.y5j7e6.top\n4kqd3hmqgptupi3p.yg767p.bid\n4kqd3hmqgptupi3p.yoursdoor.lol\n4kqd3hmqgptupi3p.z8ijgn.bid\n4kqd3hmqgptupi3p.z97f9v.bid\n4rebaopfgrewe.top\n4w5wihkwyhsav2ha.dreamtest.at\n4w5wihkwyhsav2ha.fastdances.at\n4w5wihkwyhsav2ha.grandhaus.at\n4w5wihkwyhsav2ha.payfactor.at\n52uo5k3t73ypjije.01fake.bid\n52uo5k3t73ypjije.086ux2.top\n52uo5k3t73ypjije.0n5joc.top\n52uo5k3t73ypjije.0nyi6l.bid\n52uo5k3t73ypjije.0vgu64.top\n52uo5k3t73ypjije.11pmnz.top\n52uo5k3t73ypjije.1bipa9.top\n52uo5k3t73ypjije.1de02r.top\n52uo5k3t73ypjije.1f1dw3.bid\n52uo5k3t73ypjije.1g0vo2.bid\n52uo5k3t73ypjije.1pma4t.bid\n52uo5k3t73ypjije.1ufr2v.bid\n52uo5k3t73ypjije.209kai.bid\n52uo5k3t73ypjije.249isv.bid\n52uo5k3t73ypjije.26lpul.bid\n52uo5k3t73ypjije.2gbbja.top\n52uo5k3t73ypjije.2llgoy.bid\n52uo5k3t73ypjije.2y4t6f.bid\n52uo5k3t73ypjije.2ym6om.bid\n52uo5k3t73ypjije.31wkhu.top\n52uo5k3t73ypjije.33dofy.top\n52uo5k3t73ypjije.35u068.bid\n52uo5k3t73ypjije.3di24a.top\n52uo5k3t73ypjije.3gpdgx.bid\n52uo5k3t73ypjije.3lhjyx.top\n52uo5k3t73ypjije.3rr6ao.top\n52uo5k3t73ypjije.3zotov.bid\n52uo5k3t73ypjije.40wiai.top\n52uo5k3t73ypjije.43l7lm.bid\n52uo5k3t73ypjije.43wjor.top\n52uo5k3t73ypjije.495iru.top\n52uo5k3t73ypjije.4jub4e.bid\n52uo5k3t73ypjije.4k9xlx.top\n52uo5k3t73ypjije.4n592s.top\n52uo5k3t73ypjije.4nf7ij.top\n52uo5k3t73ypjije.4oyhvh.top\n52uo5k3t73ypjije.4pjetv.bid\n52uo5k3t73ypjije.4xiiup.bid\n52uo5k3t73ypjije.4yl1hr.bid\n52uo5k3t73ypjije.4ynpjd.top\n52uo5k3t73ypjije.50cs7p.bid\n52uo5k3t73ypjije.56185u.bid\n52uo5k3t73ypjije.5ctoeb.bid\n52uo5k3t73ypjije.5ittco.bid\n52uo5k3t73ypjije.5kb3dl.top\n52uo5k3t73ypjije.5o4bjf.bid\n52uo5k3t73ypjije.5tb8hy.bid\n52uo5k3t73ypjije.5vhk5r.bid\n52uo5k3t73ypjije.5zxii2.bid\n52uo5k3t73ypjije.62er3d.top\n52uo5k3t73ypjije.68xmf9.bid\n52uo5k3t73ypjije.6ec2xb.bid\n52uo5k3t73ypjije.6j7jcn.bid\n52uo5k3t73ypjije.6w3rkc.bid\n52uo5k3t73ypjije.7156et.bid\n52uo5k3t73ypjije.7asel7.top\n52uo5k3t73ypjije.7j6htz.bid\n52uo5k3t73ypjije.7jiff7.top\n52uo5k3t73ypjije.7ud98m.bid\n52uo5k3t73ypjije.7wrwp4.top\n52uo5k3t73ypjije.80yabh.bid\n52uo5k3t73ypjije.86rhzr.bid\n52uo5k3t73ypjije.8a0sf6.top\n52uo5k3t73ypjije.8cjlyt.bid\n52uo5k3t73ypjije.8hphyr.top\n52uo5k3t73ypjije.8i8dt4.top\n52uo5k3t73ypjije.8kcfnk.bid\n52uo5k3t73ypjije.8rrxd9.bid\n52uo5k3t73ypjije.8rxv74.bid\n52uo5k3t73ypjije.91006j.bid\n52uo5k3t73ypjije.94ycl8.bid\n52uo5k3t73ypjije.95ovzy.top\n52uo5k3t73ypjije.9bjnlk.bid\n52uo5k3t73ypjije.9cd81s.bid\n52uo5k3t73ypjije.9ildst.top\n52uo5k3t73ypjije.9kxz23.bid\n52uo5k3t73ypjije.9nj8ex.top\n52uo5k3t73ypjije.9sfrr0.bid\n52uo5k3t73ypjije.9tftgh.bid\n52uo5k3t73ypjije.a0g0o7.bid\n52uo5k3t73ypjije.a2uzpe.top\n52uo5k3t73ypjije.aclox4.bid\n52uo5k3t73ypjije.ahvshc.top\n52uo5k3t73ypjije.ai7hur.bid\n52uo5k3t73ypjije.ajolkg.bid\n52uo5k3t73ypjije.aryh7f.bid\n52uo5k3t73ypjije.asxjdp.top\n52uo5k3t73ypjije.b2s4ch.bid\n52uo5k3t73ypjije.b7mciu.top\n52uo5k3t73ypjije.b8ll6n.top\n52uo5k3t73ypjije.bar8sc.bid\n52uo5k3t73ypjije.bcjl1h.top\n52uo5k3t73ypjije.bipa9k.bid\n52uo5k3t73ypjije.bipnnp.bid\n52uo5k3t73ypjije.bj9eea.bid\n52uo5k3t73ypjije.bnctf6.top\n52uo5k3t73ypjije.bp9mn8.bid\n52uo5k3t73ypjije.bt7r70.top\n52uo5k3t73ypjije.c3fz3z.bid\n52uo5k3t73ypjije.c7ex9n.top\n52uo5k3t73ypjije.catfills.mobi\n52uo5k3t73ypjije.cc0r87.bid\n52uo5k3t73ypjije.cfu46r.bid\n52uo5k3t73ypjije.cjc2jn.top\n52uo5k3t73ypjije.cm5ohx.bid\n52uo5k3t73ypjije.cm898n.bid\n52uo5k3t73ypjije.cmfkru.top\n52uo5k3t73ypjije.cpvwgx.bid\n52uo5k3t73ypjije.csdbnk.bid\n52uo5k3t73ypjije.csj0k5.top\n52uo5k3t73ypjije.csv7o6.bid\n52uo5k3t73ypjije.cto5ee.bid\n52uo5k3t73ypjije.czzg7f.bid\n52uo5k3t73ypjije.daigy0.top\n52uo5k3t73ypjije.das34.com\n52uo5k3t73ypjije.dd4xo3.top\n52uo5k3t73ypjije.ddwub3.top\n52uo5k3t73ypjije.deg5xr.top\n52uo5k3t73ypjije.dkrie7.top\n52uo5k3t73ypjije.dkriur.top\n52uo5k3t73ypjije.dkro3u.top\n52uo5k3t73ypjije.dmrueo.top\n52uo5k3t73ypjije.dmvute.top\n52uo5k3t73ypjije.dsv023.bid\n52uo5k3t73ypjije.dvuybv.bid\n52uo5k3t73ypjije.e32d1o.bid\n52uo5k3t73ypjije.e6in0v.top\n52uo5k3t73ypjije.e78hjo.bid\n52uo5k3t73ypjije.e8hua8.top\n52uo5k3t73ypjije.ei9evn.top\n52uo5k3t73ypjije.en3oyw.bid\n52uo5k3t73ypjije.eoivrm.bid\n52uo5k3t73ypjije.ep493u.top\n52uo5k3t73ypjije.er05vm.bid\n52uo5k3t73ypjije.ezm0r5.top\n52uo5k3t73ypjije.f0jlbj.bid\n52uo5k3t73ypjije.f242v5.bid\n52uo5k3t73ypjije.f3z72p.bid\n52uo5k3t73ypjije.fe98iy.top\n52uo5k3t73ypjije.fi50le.bid\n52uo5k3t73ypjije.fkgrie.top\n52uo5k3t73ypjije.g0ots2.top\n52uo5k3t73ypjije.g0spln.bid\n52uo5k3t73ypjije.g5196b.bid\n52uo5k3t73ypjije.gg4dgp.bid\n52uo5k3t73ypjije.gio6f6.bid\n52uo5k3t73ypjije.givxuf.bid\n52uo5k3t73ypjije.gmnjz7.bid\n52uo5k3t73ypjije.gnee6i.top\n52uo5k3t73ypjije.gnuvaw.bid\n52uo5k3t73ypjije.goztus.bid\n52uo5k3t73ypjije.gpy3tc.top\n52uo5k3t73ypjije.gtnfgj.top\n52uo5k3t73ypjije.gu7eao.bid\n52uo5k3t73ypjije.gvoafg.bid\n52uo5k3t73ypjije.h3ss4t.bid\n52uo5k3t73ypjije.hawtzr.bid\n52uo5k3t73ypjije.hbd7m4.bid\n52uo5k3t73ypjije.hhc366.bid\n52uo5k3t73ypjije.hlu8yz.top\n52uo5k3t73ypjije.hossy3.bid\n52uo5k3t73ypjije.hv42mo.bid\n52uo5k3t73ypjije.i5cgcw.top\n52uo5k3t73ypjije.i6gn9s.bid\n52uo5k3t73ypjije.i8zh1k.bid\n52uo5k3t73ypjije.iait3w.bid\n52uo5k3t73ypjije.ibngww.top\n52uo5k3t73ypjije.ie7t8k.top\n52uo5k3t73ypjije.ih9te2.bid\n52uo5k3t73ypjije.ij0cia.bid\n52uo5k3t73ypjije.imhhwm.top\n52uo5k3t73ypjije.insystem.men\n52uo5k3t73ypjije.izyclz.bid\n52uo5k3t73ypjije.j8873f.bid\n52uo5k3t73ypjije.j92msu.top\n52uo5k3t73ypjije.jal9lk.bid\n52uo5k3t73ypjije.jg6jtw.top\n52uo5k3t73ypjije.js43vy.bid\n52uo5k3t73ypjije.k0dcd2.bid\n52uo5k3t73ypjije.k21zey.bid\n52uo5k3t73ypjije.k56185.top\n52uo5k3t73ypjije.k7oud1.top\n52uo5k3t73ypjije.k8ytej.bid\n52uo5k3t73ypjije.k9z7pm.top\n52uo5k3t73ypjije.ka0te8.top\n52uo5k3t73ypjije.kas17.com\n52uo5k3t73ypjije.kcufx4.top\n52uo5k3t73ypjije.kml2o2.top\n52uo5k3t73ypjije.kswcuk.top\n52uo5k3t73ypjije.kt70uk.bid\n52uo5k3t73ypjije.ku824r.bid\n52uo5k3t73ypjije.kwnw1b.bid\n52uo5k3t73ypjije.kyjw0g.bid\n52uo5k3t73ypjije.kzhzuc.top\n52uo5k3t73ypjije.kzo8mc.top\n52uo5k3t73ypjije.kzwor6.top\n52uo5k3t73ypjije.l6ry3h.bid\n52uo5k3t73ypjije.laugk2.top\n52uo5k3t73ypjije.lba61x.top\n52uo5k3t73ypjije.ldsl8m.bid\n52uo5k3t73ypjije.lethints.date\n52uo5k3t73ypjije.lh9ax3.bid\n52uo5k3t73ypjije.li8wfu.bid\n52uo5k3t73ypjije.lib2vi.top\n52uo5k3t73ypjije.lio2wr.bid\n52uo5k3t73ypjije.loanshown.info\n52uo5k3t73ypjije.lrraca.bid\n52uo5k3t73ypjije.lwbi59.top\n52uo5k3t73ypjije.m33d4b.bid\n52uo5k3t73ypjije.m5fgoi.top\n52uo5k3t73ypjije.m6j75a.bid\n52uo5k3t73ypjije.mbwxyg.bid\n52uo5k3t73ypjije.mfgb1h.top\n52uo5k3t73ypjije.mn1kms.bid\n52uo5k3t73ypjije.msu96b.top\n52uo5k3t73ypjije.mtxtul.top\n52uo5k3t73ypjije.myurv5.bid\n52uo5k3t73ypjije.n41n1a.top\n52uo5k3t73ypjije.n6kswi.top\n52uo5k3t73ypjije.n8niwa.bid\n52uo5k3t73ypjije.nb83bp.bid\n52uo5k3t73ypjije.neekll.bid\n52uo5k3t73ypjije.nh47ri.bid\n52uo5k3t73ypjije.nmapwy.bid\n52uo5k3t73ypjije.nxmu0x.bid\n52uo5k3t73ypjije.o08a6d.top\n52uo5k3t73ypjije.o0hwme.bid\n52uo5k3t73ypjije.o5xcnd.bid\n52uo5k3t73ypjije.o6fa2g.bid\n52uo5k3t73ypjije.o8hpwj.bid\n52uo5k3t73ypjije.o8hpwj.top\n52uo5k3t73ypjije.o9w43w.bid\n52uo5k3t73ypjije.oef1sh.bid\n52uo5k3t73ypjije.ojesoa.bid\n52uo5k3t73ypjije.ojx58b.bid\n52uo5k3t73ypjije.omrexj.top\n52uo5k3t73ypjije.ooulp2.bid\n52uo5k3t73ypjije.ovpgod.top\n52uo5k3t73ypjije.p0lxvm.bid\n52uo5k3t73ypjije.p2lsgr.top\n52uo5k3t73ypjije.p5dxeh.bid\n52uo5k3t73ypjije.pap44w.top\n52uo5k3t73ypjije.pfija1.bid\n52uo5k3t73ypjije.pop81.com\n52uo5k3t73ypjije.poplenjohs.review\n52uo5k3t73ypjije.pr2zwz.bid\n52uo5k3t73ypjije.r21wmw.top\n52uo5k3t73ypjije.r2ok0b.bid\n52uo5k3t73ypjije.r4z3o5.bid\n52uo5k3t73ypjije.rdmwha.bid\n52uo5k3t73ypjije.red4is.top\n52uo5k3t73ypjije.rexjyp.bid\n52uo5k3t73ypjije.rgdk0u.top\n52uo5k3t73ypjije.rl0bdw.top\n52uo5k3t73ypjije.rnkj09.top\n52uo5k3t73ypjije.rv50gt.bid\n52uo5k3t73ypjije.s2xb1s.bid\n52uo5k3t73ypjije.sdfztr.bid\n52uo5k3t73ypjije.self56.top\n52uo5k3t73ypjije.sg62es.top\n52uo5k3t73ypjije.skri59.top\n52uo5k3t73ypjije.snwy26.top\n52uo5k3t73ypjije.sotn58.bid\n52uo5k3t73ypjije.srmlzh.bid\n52uo5k3t73ypjije.ssh3ln.bid\n52uo5k3t73ypjije.sx90yk.bid\n52uo5k3t73ypjije.sxjdpg.bid\n52uo5k3t73ypjije.thyx30.top\n52uo5k3t73ypjije.ti4wic.top\n52uo5k3t73ypjije.to6maq.top\n52uo5k3t73ypjije.twz1ga.top\n52uo5k3t73ypjije.txszfs.top\n52uo5k3t73ypjije.tzgwdf.top\n52uo5k3t73ypjije.u2r7tm.bid\n52uo5k3t73ypjije.u36ik0.bid\n52uo5k3t73ypjije.u50s89.bid\n52uo5k3t73ypjije.ujtwhg.top\n52uo5k3t73ypjije.ul8ib9.bid\n52uo5k3t73ypjije.un8niw.top\n52uo5k3t73ypjije.uv39h5.bid\n52uo5k3t73ypjije.uw3r6a.top\n52uo5k3t73ypjije.uw7w05.bid\n52uo5k3t73ypjije.uwazu7.bid\n52uo5k3t73ypjije.uwckha.bid\n52uo5k3t73ypjije.uwckha.top\n52uo5k3t73ypjije.ux93ip.top\n52uo5k3t73ypjije.v11z5e.top\n52uo5k3t73ypjije.v9y6z8.bid\n52uo5k3t73ypjije.veupl2.top\n52uo5k3t73ypjije.vkm4l6.top\n52uo5k3t73ypjije.vkslju.bid\n52uo5k3t73ypjije.vlo18w.bid\n52uo5k3t73ypjije.vmotsf.bid\n52uo5k3t73ypjije.vor28o.bid\n52uo5k3t73ypjije.vt3dg6.bid\n52uo5k3t73ypjije.w6sj06.bid\n52uo5k3t73ypjije.w8yolm.bid\n52uo5k3t73ypjije.wg00sp.bid\n52uo5k3t73ypjije.whmykv.bid\n52uo5k3t73ypjije.whosewine.lol\n52uo5k3t73ypjije.wht5py.top\n52uo5k3t73ypjije.wins4n.win\n52uo5k3t73ypjije.wl52rt.bid\n52uo5k3t73ypjije.wrd4fo.top\n52uo5k3t73ypjije.ws1uet.top\n52uo5k3t73ypjije.wz139z.top\n52uo5k3t73ypjije.x2kl7t.top\n52uo5k3t73ypjije.x3nnbd.top\n52uo5k3t73ypjije.x7fylp.bid\n52uo5k3t73ypjije.x9a6yb.bid\n52uo5k3t73ypjije.x9kjcn.bid\n52uo5k3t73ypjije.x9le66.top\n52uo5k3t73ypjije.xab7m0.top\n52uo5k3t73ypjije.xglk6h.bid\n52uo5k3t73ypjije.xjb384.bid\n52uo5k3t73ypjije.xmfru5.top\n52uo5k3t73ypjije.xtppp8.bid\n52uo5k3t73ypjije.y12acl.bid\n52uo5k3t73ypjije.y5j7e6.top\n52uo5k3t73ypjije.ye42cp.bid\n52uo5k3t73ypjije.yg767p.bid\n52uo5k3t73ypjije.yn8krm.bid\n52uo5k3t73ypjije.yrd7v5.bid\n52uo5k3t73ypjije.yty0gm.bid\n52uo5k3t73ypjije.yv7l4b.top\n52uo5k3t73ypjije.yw4629.top\n52uo5k3t73ypjije.ywszbe.bid\n52uo5k3t73ypjije.z6a7f1.bid\n52uo5k3t73ypjije.z8ijgn.bid\n52uo5k3t73ypjije.z97f9v.bid\n52uo5k3t73ypjije.zclw5i.top\n52uo5k3t73ypjije.zcwrhe.bid\n52uo5k3t73ypjije.zd3p2g.top\n52uo5k3t73ypjije.zda7bk.top\n52uo5k3t73ypjije.zed84j.bid\n52uo5k3t73ypjije.zhvlh1.bid\n52uo5k3t73ypjije.zxtezv.bid\n52uo5k3t73ypjije.zzis8p.bid\n5rport45vcdef345adfkksawe.bematvocal.at\n6dtxgqam4crv6rr6.onion.cab\n6g4ds.froekuge.com\n74nfnjhlq45nkgws4hbdbk45wekfjhqw4talefgnv.curryfort.at\n88fga.ketteaero.com\n8b4bb47tiaolhy4uhhlfaqerg.sofarany.at\n94dbhbj3l4blaeyfgl7q45glbaer.giponfeste.at\n974gfbjhb23hbfkyfaby3byqlyuebvly5q254y.mendilobo.com\n9hrds.wolfcrap.at\na64gfdsjhb4htbiwaysbdvukyft5q.zobodine.at\naa12111.top\naarnknthc.xyz\nabvtqhwodwjmi.work\nacbstypdrijslr.ru\naccemfsqovkd.pw\nacjhwpdjhlhbncf.click\naechjic.pw\nahsqbeospcdrngfv.info\nahuqfrqk54v3vnzj.1vcxfn.bid\nahuqfrqk54v3vnzj.45yu0p.bid\nahuqfrqk54v3vnzj.4h16v3.top\nahuqfrqk54v3vnzj.6avw2a.bid\nahuqfrqk54v3vnzj.7y1266.top\nahuqfrqk54v3vnzj.8kiec2.top\nahuqfrqk54v3vnzj.9sfk22.bid\nahuqfrqk54v3vnzj.bds4sn.top\nahuqfrqk54v3vnzj.bz7k7l.top\nahuqfrqk54v3vnzj.c8jxpp.top\nahuqfrqk54v3vnzj.cb3pul.top\nahuqfrqk54v3vnzj.dxzr2l.top\nahuqfrqk54v3vnzj.ewg6uf.bid\nahuqfrqk54v3vnzj.g4dc5s.bid\nahuqfrqk54v3vnzj.h4lu4i.bid\nahuqfrqk54v3vnzj.i81wik.bid\nahuqfrqk54v3vnzj.kj3f52.bid\nahuqfrqk54v3vnzj.l7g2sv.bid\nahuqfrqk54v3vnzj.n3oyw7.bid\nahuqfrqk54v3vnzj.roep3o.top\nahuqfrqk54v3vnzj.sg9lxh.bid\nahuqfrqk54v3vnzj.tjubo1.top\nahuqfrqk54v3vnzj.u9fcji.bid\nahuqfrqk54v3vnzj.uzeb6r.bid\nahuqfrqk54v3vnzj.v5neyw.bid\nahuqfrqk54v3vnzj.vgxcci.top\nahuqfrqk54v3vnzj.x90yk1.bid\nahuqfrqk54v3vnzj.xs2xeh.bid\nahuqfrqk54v3vnzj.zn90h4.bid\nampjsppmftmfdblpt.info\nanbqjdoyw6wkmpeu.oldtrees.at\napplesnoutsthings.bid\naqmip.fr\narddxjkwrp.xyz\nas3ws.fopyirr.com\navsxrcoq2q5fgrw2.13inb1.top\navsxrcoq2q5fgrw2.17vj7b.top\navsxrcoq2q5fgrw2.199ovv.top\navsxrcoq2q5fgrw2.1gtx3p.top\navsxrcoq2q5fgrw2.1mwipu.top\navsxrcoq2q5fgrw2.1nsnuh.top\navsxrcoq2q5fgrw2.2wfe60.top\navsxrcoq2q5fgrw2.5m2n7x.top\navsxrcoq2q5fgrw2.5s96fr.top\navsxrcoq2q5fgrw2.79j8fm.top\navsxrcoq2q5fgrw2.8l4jpw.top\navsxrcoq2q5fgrw2.9c431m.bid\navsxrcoq2q5fgrw2.arpbxw.top\navsxrcoq2q5fgrw2.ayjy5d.top\navsxrcoq2q5fgrw2.dgjpgy.top\navsxrcoq2q5fgrw2.et7izd.top\navsxrcoq2q5fgrw2.ewg6uf.bid\navsxrcoq2q5fgrw2.h44l3d.bid\navsxrcoq2q5fgrw2.ihuk7s.top\navsxrcoq2q5fgrw2.j4cser.bid\navsxrcoq2q5fgrw2.lbxvhk.top\navsxrcoq2q5fgrw2.lxvmhm.top\navsxrcoq2q5fgrw2.nbz4dn.top\navsxrcoq2q5fgrw2.p93w1x.bid\navsxrcoq2q5fgrw2.r1sjrp.top\navsxrcoq2q5fgrw2.rys9pj.top\navsxrcoq2q5fgrw2.tjdup0.top\navsxrcoq2q5fgrw2.uunmkj.top\navsxrcoq2q5fgrw2.vestjb.top\navsxrcoq2q5fgrw2.vofy7f.top\navsxrcoq2q5fgrw2.w22p3v.top\navsxrcoq2q5fgrw2.w5hilw.top\navsxrcoq2q5fgrw2.wgx4go.top\navsxrcoq2q5fgrw2.y1fx4w.top\navsxrcoq2q5fgrw2.y9kxz2.bid\navsxrcoq2q5fgrw2.yr1h37.top\navsxrcoq2q5fgrw2.z0mkoc.top\navsxrcoq2q5fgrw2.zi842m.bid\navxdypmdbo.pw\naxnemuevqnstqyflb.work\nb4youfred5485jgsa3453f.italazudda.com\nbarjhxoye.info\nbciuemfaapyf.biz\nbddadevlpkwrrmud.xyz\nbfd45u8ehdklrfqwlhbhjbgqw.niptana.at\nbkdjvmmkwgkvgw.su\nblxbymhjva.info\nbnjhx.eu\nbqbbsfdw.be\nbqukfjfv.org\nbwcfinnt.work\nbwpegsfa.info\nbxlrywuuobje.pw\ncdxbbpngq.pw\ncerberhhyed5frqa.305iot.top\ncerberhhyed5frqa.305iot.win\ncerberhhyed5frqa.45gf4t.win\ncerberhhyed5frqa.45kgok.win\ncerberhhyed5frqa.5kti58.win\ncerberhhyed5frqa.ad34ft.win\ncerberhhyed5frqa.adevf4.win\ncerberhhyed5frqa.alri58.win\ncerberhhyed5frqa.as13fd.win\ncerberhhyed5frqa.asxce4.win\ncerberhhyed5frqa.azlto5.win\ncerberhhyed5frqa.cmr95i.top\ncerberhhyed5frqa.cmr95i.win\ncerberhhyed5frqa.cmti5o.win\ncerberhhyed5frqa.cneo59.top\ncerberhhyed5frqa.cneo59.win\ncerberhhyed5frqa.dk59jg.win\ncerberhhyed5frqa.dkrti5.top\ncerberhhyed5frqa.er48rt.win\ncerberhhyed5frqa.fgfid6.win\ncerberhhyed5frqa.fkr84i.win\ncerberhhyed5frqa.fkri48.win\ncerberhhyed5frqa.gkfit9.top\ncerberhhyed5frqa.gkfit9.win\ncerberhhyed5frqa.kipfgs65s.com\ncerberhhyed5frqa.lfotp5.top\ncerberhhyed5frqa.li4loi.win\ncerberhhyed5frqa.lib2vi.win\ncerberhhyed5frqa.m5fgoi.win\ncerberhhyed5frqa.m5gid4.top\ncerberhhyed5frqa.m5gid4.win\ncerberhhyed5frqa.m5gips.win\ncerberhhyed5frqa.mix3hi.win\ncerberhhyed5frqa.moneu5.win\ncerberhhyed5frqa.oneswi.win\ncerberhhyed5frqa.qor499.top\ncerberhhyed5frqa.raress.win\ncerberhhyed5frqa.sdfiso.win\ncerberhhyed5frqa.sims6n.win\ncerberhhyed5frqa.ti4wic.win\ncerberhhyed5frqa.to6maq.win\ncerberhhyed5frqa.vmfu48.win\ncerberhhyed5frqa.we34re.top\ncerberhhyed5frqa.we34re.win\ncerberhhyed5frqa.werti4.win\ncerberhhyed5frqa.wet4io.win\ncerberhhyed5frqa.wewiso.win\ncerberhhyed5frqa.workju.win\ncerberhhyed5frqa.xltnet.win\ncerberhhyed5frqa.xmfhr6.win\ncerberhhyed5frqa.xmfir0.top\ncerberhhyed5frqa.xmfir0.win\ncerberhhyed5frqa.xmfjr7.top\ncerberhhyed5frqa.xmfkr8.top\ncerberhhyed5frqa.xmfu59.win\ncerberhhyed5frqa.xo59ok.win\ncerberhhyed5frqa.xtrvb4.win\ncerberhhyed5frqa.zgf48j.win\nchromebewfk.top\ncitointechnologiesalefor.top\nclhyelmwnuqhigecp.pw\ncorefitness.info\ncpawdrtxfjkwrkkl.pw\ncpyrltela.pw\ncrosseunity.top\ncudcfybkk.pw\ncwprfpjtmjb.biz\ncxlgwofgrjfoaa.info\nd34fa.lasmeio.com\ndd7bsndhr45nfksdnkferfer.javakale.at\nde2nuvwegoo32oqv.torbook.li\nde2nuvwegoo32oqv.tordrims.li\nde2nuvwegoo32oqv.torfigth.li\nde2nuvwegoo32oqv.tormilki.li\nde2nuvwegoo32oqv.torminimals.li\nde2nuvwegoo32oqv.torspaces.li\nde2nuvwegoo32oqv.tortelevision.li\nde2nuvwegoo32oqv.tortodorf.li\nde2nuvwegoo32oqv.torworks.li\ndkoipg.pw\ndltvwp.it\ndmwajvm.fr\ndolfexalto.com\ndomainstop.top\ndqtfhkgskushlum.org\ndtojlhpasjk.pw\ndvmbtgoobxcc.pw\ndwytqrgblrynsgtew.org\ndyoravdkiavfkbkx.pw\nearthspiruitr.top\neaxpifdtwsv.biz\necjfdaqmmyusxntwl.work\negerdpkvutvodmtsy.pw\negovrxvuspxck.be\neoalsoub.pw\neppilxqwyqdhmpdsn.pw\neqtrtdavtnr.pw\neuduudaehipk.pw\nexnqhgk.xyz\neypdxikxsufj.pw\neywlmqugxx.info\nf4dsbjhb45wfiuqeib4fkqeg.meccaledgy.at\nf5xraa2y2ybtrefz.onion.to\nfdehgchykmiqwdg.info\nffoqr3ug7m726zou.04hyxg.top\nffoqr3ug7m726zou.0v7hry.bid\nffoqr3ug7m726zou.1321z6.top\nffoqr3ug7m726zou.13inb1.top\nffoqr3ug7m726zou.14gmtu.top\nffoqr3ug7m726zou.17vj7b.top\nffoqr3ug7m726zou.1967qy.top\nffoqr3ug7m726zou.1feasu.top\nffoqr3ug7m726zou.1gtx3p.top\nffoqr3ug7m726zou.1mwipu.top\nffoqr3ug7m726zou.1nsnuh.top\nffoqr3ug7m726zou.2fu7bc.top\nffoqr3ug7m726zou.2msuuj.top\nffoqr3ug7m726zou.2rl0pv.top\nffoqr3ug7m726zou.4tkb0d.top\nffoqr3ug7m726zou.5e4u7d.bid\nffoqr3ug7m726zou.5hmjh7.bid\nffoqr3ug7m726zou.5m2n7x.top\nffoqr3ug7m726zou.735giv.top\nffoqr3ug7m726zou.8uvtsg.top\nffoqr3ug7m726zou.9yim37.top\nffoqr3ug7m726zou.ac7zvz.top\nffoqr3ug7m726zou.b31wkh.bid\nffoqr3ug7m726zou.b4abvx.top\nffoqr3ug7m726zou.bd7tlu.top\nffoqr3ug7m726zou.bdlvdy.top\nffoqr3ug7m726zou.bpuhab.top\nffoqr3ug7m726zou.bwei9h.top\nffoqr3ug7m726zou.ca15sj.top\nffoqr3ug7m726zou.do9wwg.top\nffoqr3ug7m726zou.e1e7w2.top\nffoqr3ug7m726zou.efebgv.top\nffoqr3ug7m726zou.f5x6ws.top\nffoqr3ug7m726zou.ffsm1a.bid\nffoqr3ug7m726zou.gwz8gh.top\nffoqr3ug7m726zou.hajw7w.bid\nffoqr3ug7m726zou.hpwom3.top\nffoqr3ug7m726zou.hy6dxo.bid\nffoqr3ug7m726zou.hzrekn.top\nffoqr3ug7m726zou.i4ucg2.bid\nffoqr3ug7m726zou.iocvou.top\nffoqr3ug7m726zou.jye7lt.top\nffoqr3ug7m726zou.kfymbh.top\nffoqr3ug7m726zou.l4dlll.bid\nffoqr3ug7m726zou.l6r7i9.top\nffoqr3ug7m726zou.lc1xfc.top\nffoqr3ug7m726zou.le6611.bid\nffoqr3ug7m726zou.lruwth.top\nffoqr3ug7m726zou.m3cvi8.top\nffoqr3ug7m726zou.momg04.top\nffoqr3ug7m726zou.ndnmuk.top\nffoqr3ug7m726zou.ptnbfm.top\nffoqr3ug7m726zou.rssh3l.bid\nffoqr3ug7m726zou.rxmbsm.top\nffoqr3ug7m726zou.rzt69n.top\nffoqr3ug7m726zou.rzvhne.top\nffoqr3ug7m726zou.s611js.top\nffoqr3ug7m726zou.sg9lxh.bid\nffoqr3ug7m726zou.smd95z.top\nffoqr3ug7m726zou.tsrwj3.top\nffoqr3ug7m726zou.tx0igu.bid\nffoqr3ug7m726zou.u9fcji.bid\nffoqr3ug7m726zou.ud9z0v.top\nffoqr3ug7m726zou.ukswcu.bid\nffoqr3ug7m726zou.umvv28.top\nffoqr3ug7m726zou.utebcd.top\nffoqr3ug7m726zou.v0xn1i.bid\nffoqr3ug7m726zou.vjso7r.top\nffoqr3ug7m726zou.w22p3v.top\nffoqr3ug7m726zou.w67y8u.bid\nffoqr3ug7m726zou.wf912u.bid\nffoqr3ug7m726zou.wmvsh0.top\nffoqr3ug7m726zou.wwa4tu.top\nffoqr3ug7m726zou.wx2n44.top\nffoqr3ug7m726zou.x8p2m7.bid\nffoqr3ug7m726zou.x9ap4h.top\nffoqr3ug7m726zou.xe1ws1.top\nffoqr3ug7m726zou.y9kxz2.bid\nffoqr3ug7m726zou.yjo0z9.top\nffoqr3ug7m726zou.yur4j5.top\nffoqr3ug7m726zou.yv3uwa.bid\nffoqr3ug7m726zou.zee0xr.top\nffoqr3ug7m726zou.zio9yg.bid\nffoqr3ug7m726zou.zjfbxy.top\nffoqr3ug7m726zou.zkxb17.top\nffoqr3ug7m726zou.zn90h4.bid\nffoqr3ug7m726zou.zpjpsf.top\nffoqr3ug7m726zou.zu3fzc.bid\nfhvjsmtkirihxh.xyz\nfitga.ru\nfmirgordkhig.xyz\nfnarsipfqe.pw\nfnjyygovdjyemga.xyz\nfnmi62725zfti2vy.13inb1.top\nfnmi62725zfti2vy.17vj7b.top\nfnmi62725zfti2vy.1gtx3p.top\nfnmi62725zfti2vy.o08ra6.top\nfnmi62725zfti2vy.p9wol3.top\nfnmi62725zfti2vy.vwgxhm.bid\nfooplodanx.top\nfpashgkepwtoqdjg.pw\nfqoapcjolfwwenqx.pw\nfqtdrnqmeofknd.biz\nftoxmpdipwobp4qy.10nzk9.top\nftoxmpdipwobp4qy.17vj7b.top\nftoxmpdipwobp4qy.199ovv.top\nftoxmpdipwobp4qy.1gtx3p.top\nftoxmpdipwobp4qy.1nsnuh.top\nftoxmpdipwobp4qy.7pnxn9.top\nftoxmpdipwobp4qy.lxvmhm.top\nfuuasvhpsvuihlnje.pw\nfuuwnsv.pw\nfyqtguo.biz\ng4dhhg53jsdjnnkjwjrfyiouh3o4u4th.vinerteen.com\ngccxqpuuylioxoip.pw\ngfcuxnaek.ru\ngfkuwflbhsjdabnu4nfukerfqwlfwr4rw.ringbalor.com\ngfwncoyhbdvggns.pw\ngguaxufrt.pw\ngitybdjgbxd.nl\nglhxgchhfemcjgr.pw\ngnsquwmgukkpgpt.pw\ngovementruystd.top\ngsebqsi.ru\ngsmdqrmqddqtuv.xyz\ngvludcvhcrjwmgq.in\ngwbak.nickymaru.com\ngwe32fdr74bhfsyujb34gfszfv.zatcurr.com\nh3ds4.maconslab.com\nh54dc.leverdaze.at\nh5nuwefkuh134ljngkasdbasfg.corolbugan.com\nhjhqmbxyinislkkt.11bwgu.top\nhjhqmbxyinislkkt.127axt.top\nhjhqmbxyinislkkt.12bsy8.top\nhjhqmbxyinislkkt.12bxp9.top\nhjhqmbxyinislkkt.12ct4c.top\nhjhqmbxyinislkkt.12gsjz.top\nhjhqmbxyinislkkt.12m58x.top\nhjhqmbxyinislkkt.12zucf.top\nhjhqmbxyinislkkt.13bcem.top\nhjhqmbxyinislkkt.13eymq.top\nhjhqmbxyinislkkt.13fmby.top\nhjhqmbxyinislkkt.13khiv.top\nhjhqmbxyinislkkt.13kn4l.top\nhjhqmbxyinislkkt.13qgdd.top\nhjhqmbxyinislkkt.13ydzv.top\nhjhqmbxyinislkkt.142djp.top\nhjhqmbxyinislkkt.14dr1s.top\nhjhqmbxyinislkkt.14klmz.top\nhjhqmbxyinislkkt.14o2wp.top\nhjhqmbxyinislkkt.14stvt.top\nhjhqmbxyinislkkt.14yppf.top\nhjhqmbxyinislkkt.15e8hv.top\nhjhqmbxyinislkkt.15mwt4.top\nhjhqmbxyinislkkt.15u3kg.top\nhjhqmbxyinislkkt.16ke1t.top\nhjhqmbxyinislkkt.16l1zt.top\nhjhqmbxyinislkkt.17kc8y.top\nhjhqmbxyinislkkt.17rm9b.top\nhjhqmbxyinislkkt.18f5bw.top\nhjhqmbxyinislkkt.18lmhb.top\nhjhqmbxyinislkkt.18nepv.top\nhjhqmbxyinislkkt.18yzmj.top\nhjhqmbxyinislkkt.18zrup.top\nhjhqmbxyinislkkt.19b6nk.top\nhjhqmbxyinislkkt.19hj4f.top\nhjhqmbxyinislkkt.19s7gy.top\nhjhqmbxyinislkkt.19xdpm.top\nhjhqmbxyinislkkt.19xvyd.top\nhjhqmbxyinislkkt.1a2xx3.top\nhjhqmbxyinislkkt.1a8u1r.top\nhjhqmbxyinislkkt.1aajb7.top\nhjhqmbxyinislkkt.1aamtz.top\nhjhqmbxyinislkkt.1accfa.top\nhjhqmbxyinislkkt.1acfka.top\nhjhqmbxyinislkkt.1adh2r.top\nhjhqmbxyinislkkt.1aq4sz.top\nhjhqmbxyinislkkt.1aqq5k.top\nhjhqmbxyinislkkt.1b8tmn.top\nhjhqmbxyinislkkt.1bas8q.top\nhjhqmbxyinislkkt.1bcnad.top\nhjhqmbxyinislkkt.1bcxcs.top\nhjhqmbxyinislkkt.1bu9xu.top\nhjhqmbxyinislkkt.1c1ajf.top\nhjhqmbxyinislkkt.1cdqfv.top\nhjhqmbxyinislkkt.1cnkik.top\nhjhqmbxyinislkkt.1csesc.top\nhjhqmbxyinislkkt.1dq6nd.top\nhjhqmbxyinislkkt.1dvqvh.top\nhjhqmbxyinislkkt.1e47tj.top\nhjhqmbxyinislkkt.1eagrj.top\nhjhqmbxyinislkkt.1eeyaj.top\nhjhqmbxyinislkkt.1efxa8.top\nhjhqmbxyinislkkt.1fgsmc.top\nhjhqmbxyinislkkt.1fnjrj.top\nhjhqmbxyinislkkt.1fttxm.top\nhjhqmbxyinislkkt.1fy93v.top\nhjhqmbxyinislkkt.1fygsg.top\nhjhqmbxyinislkkt.1fzjn3.top\nhjhqmbxyinislkkt.1fzz7a.top\nhjhqmbxyinislkkt.1gjpzp.top\nhjhqmbxyinislkkt.1gqrpq.top\nhjhqmbxyinislkkt.1gredn.top\nhjhqmbxyinislkkt.1grvue.top\nhjhqmbxyinislkkt.1gswwp.top\nhjhqmbxyinislkkt.1gu5um.top\nhjhqmbxyinislkkt.1gunao.top\nhjhqmbxyinislkkt.1gvyo8.top\nhjhqmbxyinislkkt.1gxfxt.top\nhjhqmbxyinislkkt.1gzjuc.top\nhjhqmbxyinislkkt.1hapca.top\nhjhqmbxyinislkkt.1j43kf.top\nhjhqmbxyinislkkt.1jmip6.top\nhjhqmbxyinislkkt.1jnhdc.top\nhjhqmbxyinislkkt.1jwuaa.top\nhjhqmbxyinislkkt.1k6bas.top\nhjhqmbxyinislkkt.1kge5a.top\nhjhqmbxyinislkkt.1khwro.top\nhjhqmbxyinislkkt.1kjhhf.top\nhjhqmbxyinislkkt.1kraqn.top\nhjhqmbxyinislkkt.1kw51p.top\nhjhqmbxyinislkkt.1lqrja.top\nhjhqmbxyinislkkt.1ltyev.top\nhjhqmbxyinislkkt.1mat7v.top\nhjhqmbxyinislkkt.1mee2x.top\nhjhqmbxyinislkkt.1mqvsc.top\nhjhqmbxyinislkkt.1mswjm.top\nhjhqmbxyinislkkt.1mvku2.top\nhjhqmbxyinislkkt.1mwvgh.top\nhjhqmbxyinislkkt.1nm62r.top\nhjhqmbxyinislkkt.1npg9s.top\nhjhqmbxyinislkkt.1ntyds.top\nhjhqmbxyinislkkt.1pcvko.top\nhjhqmbxyinislkkt.1ppto6.top\nhjhqmbxyinislkkt.1pxbfh.top\nhjhqmbxyinislkkt.1q7pwb.top\nhjhqmbxyinislkkt.1qjl23.top\nhjhqmbxyinislkkt.1qk2un.top\nhjhqmbxyinislkkt.1w5iy8.top\nhjhqmbxyinislkkt.1xynaz.top\nhmndhdbscgru.pw\nhonourableud.top\nhppfsslyeyseudg.biz\nhrfgd74nfksjdcnnklnwefvdsf.materdunst.com\nhtankds.info\nhycninyxuaa.xyz\ni01001.dgn.vn\ni3ezlvkoi7fwyood.onion.to\ni3ezlvkoi7fwyood.tor2web.org\ni5ndw.titlecorta.at\nibjgnqsthdyp.pw\nibtfqftkgi.pw\nifohvkxmyp.biz\nigoodsnd.wang\nik4dm.mazerunci.at\niqfyujpvubwawc.pw\nirhng84nfaslbv243ljtblwqjrb.pinnafaon.at\nirudhkunrlfu25fhkaqw34blr5qlby4tgq43t.orrisbirth.com\niuieylpvfurcvmpk.pw\njfmiondv.xyz\njghbktqepe.pw\njhdgh.club\njhomitevd2abj3fk.onion.to\njuhacjacjckclqf.pw\njxqdry.ru\njymhmkdaxfbl.click\nk234s.ascotsprue.com\nk34ew.keyedgell.com\nk3cxd.pileanoted.com\nk47d3.proporr.com\nk4restportgonst34d23r.oftpony.at\nkbv5s.kylepasse.at\nkcdfajaxngiff.info\nkciylimohteftc.pw\nkh5jfnvkk5twerfnku5twuilrnglnuw45yhlw.vealsithe.com\nkjkwjqvqrjocpi.xyz\nkkd47eh4hdjshb5t.angortra.at\nkkr4hbwdklf234bfl84uoqleflqwrfqwuelfh.brazabaya.com\nkpybuhnosdrm.in\nkqlxtqptsmys.in\nks-davis.com\nktlgpiilbj.biz\nkwontdmplpnbl.pw\nkypsuw.pw\nl123d.feustude.at\nlcrdceiajmiar.org\nlfdachijzuwx4bc4.0ndl3j.bid\nlfdachijzuwx4bc4.6szfn7.top\nlfdachijzuwx4bc4.83zw1f.bid\nlfdachijzuwx4bc4.8dlgyg.bid\nlfdachijzuwx4bc4.af38vz.top\nlfdachijzuwx4bc4.ci221p.top\nlfdachijzuwx4bc4.djintc.bid\nlfdachijzuwx4bc4.e6cf2t.bid\nlfdachijzuwx4bc4.eujvrw.bid\nlfdachijzuwx4bc4.ev99l6.bid\nlfdachijzuwx4bc4.ex9n9v.top\nlfdachijzuwx4bc4.fe6cf2.top\nlfdachijzuwx4bc4.fwzxnb.bid\nlfdachijzuwx4bc4.iuzppd.top\nlfdachijzuwx4bc4.le2brr.bid\nlfdachijzuwx4bc4.m7f27y.bid\nlfdachijzuwx4bc4.twyjdx.bid\nlfdachijzuwx4bc4.tx0igu.bid\nlfdachijzuwx4bc4.u9fcji.bid\nlfdachijzuwx4bc4.vrgdrs.top\nlfdachijzuwx4bc4.w4629d.top\nlfdachijzuwx4bc4.x4tk5c.bid\nlfdachijzuwx4bc4.zreknv.bid\nlollyoff.info\nlookingpersonals.top\nlpholfnvwbukqwye.onion.cab\nlpholfnvwbukqwye.onion.to\nlrmficvqs.pw\nltpwqva.xyz\nluvenxj.uk\nlvanwwbyabcfevyi.pw\nlyrnvane.pw\nmacooptwafkwchtpo.pw\nmmhmtea.pw\nmphtadhci5mrdlju.onion.to\nmphtadhci5mrdlju.tor2web.org\nmuuojcu.xyz\nmwqwverayognn.pw\nmxyfasm.pw\nmz7oyb3v32vshcvk.bidobject.li\nmz7oyb3v32vshcvk.getstar.li\nmz7oyb3v32vshcvk.torapples.li\nmz7oyb3v32vshcvk.torlongor.li\nmz7oyb3v32vshcvk.tormidle.at\nmz7oyb3v32vshcvk.toysworlds.at\nnewgiftnd.wang\nnewgiftst.top\nnhhyxorxbxarxe.org\nnikessysleys.top\nnlpqflkbvkdde.eu\nnn54djhfnrnm4dnjnerfsd.replylaten.at\nnnrtsdf34dsjhb23rsdf.spannflow.com\nnwcpgymgh.work\no4dm3.leaama.at\nodgtnkmq.pw\noehknf74ohqlfnpq9rhfgcq93g.hateflux.com\nohpbdikmrrhr.pw\nohplsuljopekq.biz\nojmekzw4mujvqeju.bioserv.at\nojmekzw4mujvqeju.dreamtest.at\nojmekzw4mujvqeju.fineboy.at\nojmekzw4mujvqeju.minitili.at\nomeaswslhgdw.xyz\noqwygprskqv65j72.12kb9j.top\noqwygprskqv65j72.13gpqd.top\noqwygprskqv65j72.13rdvu.top\noqwygprskqv65j72.14jqyo.top\noqwygprskqv65j72.17q8f6.top\noqwygprskqv65j72.1aj1bb.top\noqwygprskqv65j72.1d88b8.top\noqwygprskqv65j72.1dofqx.top\noqwygprskqv65j72.1fdlhn.top\noqwygprskqv65j72.1fs9pz.top\noqwygprskqv65j72.1gam57.top\noqwygprskqv65j72.1gqj8x.top\noqwygprskqv65j72.1hbdbx.top\noqwygprskqv65j72.1j1x2b.top\noqwygprskqv65j72.1jquw7.top\noqwygprskqv65j72.1kh9ct.top\noqwygprskqv65j72.1mudaw.top\noqwygprskqv65j72.1nzpby.top\nozfin.ru\np27dokhpz2n7nvgr.12a63k.top\np27dokhpz2n7nvgr.12c8ff.top\np27dokhpz2n7nvgr.12gzrv.top\np27dokhpz2n7nvgr.12hxjv.top\np27dokhpz2n7nvgr.12nwsv.top\np27dokhpz2n7nvgr.12smak.top\np27dokhpz2n7nvgr.12t3rn.top\np27dokhpz2n7nvgr.12ulcz.top\np27dokhpz2n7nvgr.12umzf.top\np27dokhpz2n7nvgr.12uzfa.top\np27dokhpz2n7nvgr.12vpkc.top\np27dokhpz2n7nvgr.1321z6.top\np27dokhpz2n7nvgr.133chr.top\np27dokhpz2n7nvgr.135nt3.top\np27dokhpz2n7nvgr.13g2v9.top\np27dokhpz2n7nvgr.13gmvm.top\np27dokhpz2n7nvgr.13ixv2.top\np27dokhpz2n7nvgr.13upky.top\np27dokhpz2n7nvgr.13upnc.top\np27dokhpz2n7nvgr.13wm9b.top\np27dokhpz2n7nvgr.13xwn9.top\np27dokhpz2n7nvgr.14ewqv.top\np27dokhpz2n7nvgr.14gmtu.top\np27dokhpz2n7nvgr.14kfoz.top\np27dokhpz2n7nvgr.14udep.top\np27dokhpz2n7nvgr.15jznv.top\np27dokhpz2n7nvgr.15l2ub.top\np27dokhpz2n7nvgr.15nhsf.top\np27dokhpz2n7nvgr.15oqwp.top\np27dokhpz2n7nvgr.15rnwa.top\np27dokhpz2n7nvgr.15wmdx.top\np27dokhpz2n7nvgr.168w5y.top\np27dokhpz2n7nvgr.16ay2s.top\np27dokhpz2n7nvgr.16bwhs.top\np27dokhpz2n7nvgr.16fohp.top\np27dokhpz2n7nvgr.16nxpn.top\np27dokhpz2n7nvgr.16qpet.top\np27dokhpz2n7nvgr.173w9w.top\np27dokhpz2n7nvgr.17g6gc.top\np27dokhpz2n7nvgr.17gvad.top\np27dokhpz2n7nvgr.17m14u.top\np27dokhpz2n7nvgr.17ryrs.top\np27dokhpz2n7nvgr.17u2yg.top\np27dokhpz2n7nvgr.18dawg.top\np27dokhpz2n7nvgr.18kkhl.top\np27dokhpz2n7nvgr.18kmtt.top\np27dokhpz2n7nvgr.195heb.top\np27dokhpz2n7nvgr.1967qy.top\np27dokhpz2n7nvgr.1a7ivn.top\np27dokhpz2n7nvgr.1a7wnt.top\np27dokhpz2n7nvgr.1aghep.top\np27dokhpz2n7nvgr.1ajohk.top\np27dokhpz2n7nvgr.1apgrn.top\np27dokhpz2n7nvgr.1apkjn.top\np27dokhpz2n7nvgr.1aweql.top\np27dokhpz2n7nvgr.1axzcw.top\np27dokhpz2n7nvgr.1azkux.top\np27dokhpz2n7nvgr.1b3qjy.top\np27dokhpz2n7nvgr.1bj4k9.top\np27dokhpz2n7nvgr.1bniyw.top\np27dokhpz2n7nvgr.1bvadx.top\np27dokhpz2n7nvgr.1bywu2.top\np27dokhpz2n7nvgr.1bzolk.top\np27dokhpz2n7nvgr.1cauz3.top\np27dokhpz2n7nvgr.1cb19l.top\np27dokhpz2n7nvgr.1cbcpy.top\np27dokhpz2n7nvgr.1cewld.top\np27dokhpz2n7nvgr.1cggqc.top\np27dokhpz2n7nvgr.1cglxz.top\np27dokhpz2n7nvgr.1chy1m.top\np27dokhpz2n7nvgr.1cknbd.top\np27dokhpz2n7nvgr.1cpb4z.top\np27dokhpz2n7nvgr.1cpy1q.top\np27dokhpz2n7nvgr.1cq7gd.top\np27dokhpz2n7nvgr.1cvmb4.top\np27dokhpz2n7nvgr.1cw65b.top\np27dokhpz2n7nvgr.1czh7o.top\np27dokhpz2n7nvgr.1d8d9w.top\np27dokhpz2n7nvgr.1d8m97.top\np27dokhpz2n7nvgr.1daq6h.top\np27dokhpz2n7nvgr.1dlcbk.top\np27dokhpz2n7nvgr.1dp6un.top\np27dokhpz2n7nvgr.1dsdm4.top\np27dokhpz2n7nvgr.1dyzdh.top\np27dokhpz2n7nvgr.1dz7gk.top\np27dokhpz2n7nvgr.1ebvqb.top\np27dokhpz2n7nvgr.1eeb86.top\np27dokhpz2n7nvgr.1em2j4.top\np27dokhpz2n7nvgr.1enbyr.top\np27dokhpz2n7nvgr.1evjph.top\np27dokhpz2n7nvgr.1fel3k.top\np27dokhpz2n7nvgr.1fgywm.top\np27dokhpz2n7nvgr.1fqwek.top\np27dokhpz2n7nvgr.1fu8p3.top\np27dokhpz2n7nvgr.1gnlsi.top\np27dokhpz2n7nvgr.1gqqsc.top\np27dokhpz2n7nvgr.1gvql3.top\np27dokhpz2n7nvgr.1gy9bo.top\np27dokhpz2n7nvgr.1h23cc.top\np27dokhpz2n7nvgr.1hkjl3.top\np27dokhpz2n7nvgr.1hpvzl.top\np27dokhpz2n7nvgr.1hw36d.top\np27dokhpz2n7nvgr.1jemdr.top\np27dokhpz2n7nvgr.1jh5kv.top\np27dokhpz2n7nvgr.1jhnvt.top\np27dokhpz2n7nvgr.1jpb8w.top\np27dokhpz2n7nvgr.1js3tl.top\np27dokhpz2n7nvgr.1jw2lx.top\np27dokhpz2n7nvgr.1jyhqc.top\np27dokhpz2n7nvgr.1jzmjr.top\np27dokhpz2n7nvgr.1kja1j.top\np27dokhpz2n7nvgr.1kq4l8.top\np27dokhpz2n7nvgr.1ktjse.top\np27dokhpz2n7nvgr.1kyjw7.top\np27dokhpz2n7nvgr.1l4zyd.top\np27dokhpz2n7nvgr.1lcteo.top\np27dokhpz2n7nvgr.1lfyy4.top\np27dokhpz2n7nvgr.1lt2pn.top\np27dokhpz2n7nvgr.1m3xsy.top\np27dokhpz2n7nvgr.1mfakx.top\np27dokhpz2n7nvgr.1mfdt8.top\np27dokhpz2n7nvgr.1mir1h.top\np27dokhpz2n7nvgr.1ms2rx.top\np27dokhpz2n7nvgr.1mwipu.top\np27dokhpz2n7nvgr.1nhkou.top\np27dokhpz2n7nvgr.1nmrtq.top\np27dokhpz2n7nvgr.1nprob.top\np27dokhpz2n7nvgr.1p5fwl.top\np27dokhpz2n7nvgr.1pbfky.top\np27dokhpz2n7nvgr.1pbu64.top\np27dokhpz2n7nvgr.1pglcs.top\np27dokhpz2n7nvgr.1plugt.top\np27dokhpz2n7nvgr.1psts4.top\np27dokhpz2n7nvgr.1pymg3.top\np27dokhpz2n7nvgr.1vjnyh.top\np27dokhpz2n7nvgr.1wmvk2.top\np54dhkus4tlkfashdb6vjetgsdfg.greetingshere.at\npagaldaily.com\npdlbtnfhtoxghb.org\npe2cku7pebkpgeko.13inb1.top\npe2cku7pebkpgeko.199ovv.top\npe2cku7pebkpgeko.1cb19l.top\npe2cku7pebkpgeko.1gtx3p.top\npe2cku7pebkpgeko.1mwipu.top\npe2cku7pebkpgeko.1plugt.top\npe2cku7pebkpgeko.1pr21c.top\npe2cku7pebkpgeko.582h0n.top\npe2cku7pebkpgeko.5hmjh7.bid\npe2cku7pebkpgeko.ahovbr.top\npe2cku7pebkpgeko.bw9e2z.top\npe2cku7pebkpgeko.dj68hn.top\npe2cku7pebkpgeko.hclz73.top\npe2cku7pebkpgeko.kwrd4f.bid\npe2cku7pebkpgeko.p93w1x.bid\npe2cku7pebkpgeko.pkx86a.top\npe2cku7pebkpgeko.prbuoi.top\npe2cku7pebkpgeko.r1sjrp.top\npe2cku7pebkpgeko.reu88i.top\npe2cku7pebkpgeko.rjf9yn.top\npe2cku7pebkpgeko.tsrwj3.top\npe2cku7pebkpgeko.ttx0ig.top\npe2cku7pebkpgeko.utebcd.top\npe2cku7pebkpgeko.va3ibn.top\npe2cku7pebkpgeko.vfe2f1.top\npe2cku7pebkpgeko.yjo0z9.top\npe2cku7pebkpgeko.z5xfkc.top\npennysgoods.top\nplfbvdrpvsm.pw\npmenboeqhyrpvomq.0nyi6l.bid\npmenboeqhyrpvomq.0vgu64.top\npmenboeqhyrpvomq.2agglf.top\npmenboeqhyrpvomq.4pzclh.top\npmenboeqhyrpvomq.58na23.top\npmenboeqhyrpvomq.5b1s82.top\npmenboeqhyrpvomq.7s0g3v.top\npmenboeqhyrpvomq.89m6y8.bid\npmenboeqhyrpvomq.8kcfnk.bid\npmenboeqhyrpvomq.9ildst.top\npmenboeqhyrpvomq.9nkxd3.top\npmenboeqhyrpvomq.a4coac.top\npmenboeqhyrpvomq.afteghonte.lol\npmenboeqhyrpvomq.as5su5.top\npmenboeqhyrpvomq.asxjdp.top\npmenboeqhyrpvomq.azwsxe.top\npmenboeqhyrpvomq.b7mciu.top\npmenboeqhyrpvomq.bnctf6.top\npmenboeqhyrpvomq.cmri58.top\npmenboeqhyrpvomq.e6in0v.top\npmenboeqhyrpvomq.enanhb.bid\npmenboeqhyrpvomq.factordo.site\npmenboeqhyrpvomq.fm0cga.top\npmenboeqhyrpvomq.g0ots2.top\npmenboeqhyrpvomq.gletterstan.trade\npmenboeqhyrpvomq.gnuvaw.bid\npmenboeqhyrpvomq.hasterlyston.cloud\npmenboeqhyrpvomq.hwh75t.top\npmenboeqhyrpvomq.ibngww.top\npmenboeqhyrpvomq.k7oud1.top\npmenboeqhyrpvomq.ka0te8.top\npmenboeqhyrpvomq.kswcuk.top\npmenboeqhyrpvomq.li4loi.top\npmenboeqhyrpvomq.loopsay.link\npmenboeqhyrpvomq.m54tkp.bid\npmenboeqhyrpvomq.mtxtul.top\npmenboeqhyrpvomq.n41n1a.top\npmenboeqhyrpvomq.n80yab.top\npmenboeqhyrpvomq.nh47ri.bid\npmenboeqhyrpvomq.o08a6d.top\npmenboeqhyrpvomq.o8hpwj.top\npmenboeqhyrpvomq.p8rruv.top\npmenboeqhyrpvomq.pap44w.top\npmenboeqhyrpvomq.paypoints.red\npmenboeqhyrpvomq.r21wmw.top\npmenboeqhyrpvomq.rnkj09.top\npmenboeqhyrpvomq.s71vsc.top\npmenboeqhyrpvomq.self56.top\npmenboeqhyrpvomq.shutlazy.casa\npmenboeqhyrpvomq.swissprogramms.bid\npmenboeqhyrpvomq.t4hvl4.bid\npmenboeqhyrpvomq.thyx30.top\npmenboeqhyrpvomq.txszfs.top\npmenboeqhyrpvomq.v11z5e.top\npmenboeqhyrpvomq.viceled.pw\npmenboeqhyrpvomq.vkm4l6.top\npmenboeqhyrpvomq.wn4h1k.top\npmenboeqhyrpvomq.wrd4fo.top\npmenboeqhyrpvomq.x1kofw.top\npmenboeqhyrpvomq.xneyvm.top\npmenboeqhyrpvomq.xx6jck.top\npmenboeqhyrpvomq.y5j7e6.top\npmenboeqhyrpvomq.y7fjr4.bid\npmenboeqhyrpvomq.yw4629.top\npnyviolg.eu\npo4dbsjbneljhrlbvaueqrgveatv.bonmawp.at\npoimoiyreque5.pw\npolaerunity.top\nponmaredimare.top\npornohd24.com\npreeqlultgfifg.pw\nprest54538hnksjn4kjfwdbhwere.hotchunman.com\npts764gt354fder34fsqw45gdfsavadfgsfg.kraskula.com\npvwinlrmwvccuo.eu\nqbqrfyeqqvcvv.pw\nqcwbrevxrotoepsp.pw\nqdesslfdcmd.pw\nqdvkdyvrtpjc.pw\nqfjhpgbefuhenjp7.1225wj.top\nqfjhpgbefuhenjp7.12efwa.top\nqfjhpgbefuhenjp7.12f53x.top\nqfjhpgbefuhenjp7.12u5fl.top\nqfjhpgbefuhenjp7.13iuvw.top\nqfjhpgbefuhenjp7.143kzi.top\nqfjhpgbefuhenjp7.158ugp.top\nqfjhpgbefuhenjp7.16g9ub.top\nqfjhpgbefuhenjp7.17cwdi.top\nqfjhpgbefuhenjp7.17ipn9.top\nqfjhpgbefuhenjp7.17xukb.top\nqfjhpgbefuhenjp7.18dwag.top\nqfjhpgbefuhenjp7.18ggbf.top\nqfjhpgbefuhenjp7.18rkju.top\nqfjhpgbefuhenjp7.19ckzf.top\nqfjhpgbefuhenjp7.1a2jzy.top\nqfjhpgbefuhenjp7.1cosak.top\nqfjhpgbefuhenjp7.1e1jbc.top\nqfjhpgbefuhenjp7.1e1y8p.top\nqfjhpgbefuhenjp7.1fcfjn.top\nqfjhpgbefuhenjp7.1jfjhb.top\nqfjhpgbefuhenjp7.1jrkyn.top\nqfjhpgbefuhenjp7.1mkwry.top\nqfjhpgbefuhenjp7.1mnsg6.top\nqfuxosx.eu\nqlwnvdjwro.pw\nqqonof.info\nqqtphtlhny.pw\nqsbfwgtedexirbyoq.pw\nqvdgqayo.pw\nrastypasty34.top\nrbg4hfbilrf7to452p89hrfq.boonmower.com\nrbwubtpsyokqn.info\nreal346real.top\nremoteunityrety.top\nrenaulrtcenturytrick.top\nrkiywansamtu.top\nrolerxunitywsto.top\nrootaleyz.top\nrowerpovertort.top\nrqfsctpgpuani.pw\nrrcspgfghsjnklts.pw\nrzss2zfue73dfvmj.onlinerpgame.ch\nrzss2zfue73dfvmj.truewargame.ch\nsdwempsovemtr.yt\nseelkqtkkqxvq.click\nsemiconductry.top\nsgowntfjwkybawi.pw\nsgrnhwyqxdk.pw\nsondr5344ygfweyjbfkw4fhsefv.heliofetch.at\nsonicfopase.top\nsqrgvbgfyya.org\nsqsigig.pw\nssvylrn.pw\nstevnxwq.pw\nstgg5jv6mqiibmax.toradmin.li\nstgg5jv6mqiibmax.toranimals.li\nstgg5jv6mqiibmax.torbrouke.li\nstgg5jv6mqiibmax.torclasses.li\nstgg5jv6mqiibmax.torclever.li\nstgg5jv6mqiibmax.torcreator.li\nstgg5jv6mqiibmax.torking.li\nstgg5jv6mqiibmax.torpice.li\nstgg5jv6mqiibmax.torpoint.ch\nstgg5jv6mqiibmax.torshop.li\nsumnitdomains.top\nsvkjhguk.ru\nsvvgyjweurxn.click\nswfqg.in\nsxflmtgxerkpgwlnp.pw\nt54ndnku456ngkwsudqer.wallymac.com\ntdhyjfxltpj.pw\ntes543berda73i48fsdfsd.keratadze.at\ntopgearspoilytyrdc.top\ntoxnwbkoulii.pw\ntoytyaclucomunit.top\ntqlcjh.fr\ntregretryfaltervipo.top\ntrxswbwxhr.xyz\ntswsgajtwhqkosd.su\ntt54rfdjhb34rfbnknaerg.milerteddy.com\nttoyqvq.pw\ntuouyunittyewr.top\ntwbers4hmi6dc65f.onion.cab\ntwbers4hmi6dc65f.onion.to\ntwbers4hmi6dc65f.tor2web.org\nu24er.ovaarmor.com\nu54bbnhf354fbkh254tbkhjbgy8258gnkwerg.tahaplap.com\nubisortdasert.top\nuetwvrlnee.fr\nuhgmnigjpf.biz\nuhhvhjqowpgopq.xyz\nuhjxayhpisr.pw\nuhufnlsad7bhf4ykqfbevmxergwrth.himfinn.com\nuiredn4njfsa4234bafb32ygjdawfvs.frascuft.com\nuj5nj.onanwhit.com\numjjvccteg.biz\nunintyregullyar.top\nunittogreas.top\nunityharerteraz.top\nunityrulesyur.top\nunixbroungs.top\nunocl45trpuoefft.054t69.bid\nunocl45trpuoefft.06j7o0.top\nunocl45trpuoefft.086ux2.top\nunocl45trpuoefft.0evktl.top\nunocl45trpuoefft.0kousz.bid\nunocl45trpuoefft.0kv6tw.bid\nunocl45trpuoefft.0vgu64.top\nunocl45trpuoefft.18xhww.bid\nunocl45trpuoefft.1cn41a.bid\nunocl45trpuoefft.1de02r.top\nunocl45trpuoefft.1v3bnu.top\nunocl45trpuoefft.249isv.bid\nunocl45trpuoefft.2y4t6f.bid\nunocl45trpuoefft.308an1.top\nunocl45trpuoefft.31wkhu.top\nunocl45trpuoefft.36u6mp.bid\nunocl45trpuoefft.3n9lut.bid\nunocl45trpuoefft.42wunw.bid\nunocl45trpuoefft.4bb9vz.bid\nunocl45trpuoefft.4k98id.top\nunocl45trpuoefft.54drms.bid\nunocl45trpuoefft.54m2k3.bid\nunocl45trpuoefft.5o3euy.bid\nunocl45trpuoefft.5v3uvc.bid\nunocl45trpuoefft.60c61d.bid\nunocl45trpuoefft.6w3rkc.bid\nunocl45trpuoefft.75tdcj.bid\nunocl45trpuoefft.78of7m.bid\nunocl45trpuoefft.791sd5.bid\nunocl45trpuoefft.7cevps.bid\nunocl45trpuoefft.7eup7k.bid\nunocl45trpuoefft.7tooul.bid\nunocl45trpuoefft.88wz5p.bid\nunocl45trpuoefft.8kcfnk.bid\nunocl45trpuoefft.8uwckh.top\nunocl45trpuoefft.9bjnlk.bid\nunocl45trpuoefft.9lnito.top\nunocl45trpuoefft.9lx4s6.bid\nunocl45trpuoefft.9u3iy1.top\nunocl45trpuoefft.a3migu.bid\nunocl45trpuoefft.a4v4c3.bid\nunocl45trpuoefft.ageshere.club\nunocl45trpuoefft.ahhc36.top\nunocl45trpuoefft.at593l.bid\nunocl45trpuoefft.at9gwv.bid\nunocl45trpuoefft.awspm2.top\nunocl45trpuoefft.barzc4.bid\nunocl45trpuoefft.bjahwh.bid\nunocl45trpuoefft.c3fz3z.bid\nunocl45trpuoefft.c4issd.bid\nunocl45trpuoefft.c9kp0o.bid\nunocl45trpuoefft.ceikto.bid\nunocl45trpuoefft.cgf59i.top\nunocl45trpuoefft.cifbp9.bid\nunocl45trpuoefft.ckw9fm.top\nunocl45trpuoefft.cm5ohx.bid\nunocl45trpuoefft.csdbnk.bid\nunocl45trpuoefft.csv7o6.bid\nunocl45trpuoefft.cypz3w.top\nunocl45trpuoefft.czzg7f.bid\nunocl45trpuoefft.dwkofh.top\nunocl45trpuoefft.dyo7c9.top\nunocl45trpuoefft.efebgv.bid\nunocl45trpuoefft.eloppu.bid\nunocl45trpuoefft.emogew.bid\nunocl45trpuoefft.eo6rzt.bid\nunocl45trpuoefft.ev6i0x.bid\nunocl45trpuoefft.eyohd2.top\nunocl45trpuoefft.f17bam.bid\nunocl45trpuoefft.freshsdog.loan\nunocl45trpuoefft.frn62e.top\nunocl45trpuoefft.gg4dgp.bid\nunocl45trpuoefft.gio6f6.bid\nunocl45trpuoefft.givxuf.bid\nunocl45trpuoefft.hawtzr.bid\nunocl45trpuoefft.he81tz.bid\nunocl45trpuoefft.hur45z.bid\nunocl45trpuoefft.hvh2gb.bid\nunocl45trpuoefft.hxrd02.bid\nunocl45trpuoefft.hynwbs.top\nunocl45trpuoefft.hyr1h3.bid\nunocl45trpuoefft.i1wcrl.bid\nunocl45trpuoefft.i561zy.bid\nunocl45trpuoefft.ibngww.top\nunocl45trpuoefft.idw6s5.bid\nunocl45trpuoefft.igpfcu.bid\nunocl45trpuoefft.igrj6t.bid\nunocl45trpuoefft.ih301a.bid\nunocl45trpuoefft.ii2yoh.bid\nunocl45trpuoefft.ilm071.bid\nunocl45trpuoefft.j0cia7.bid\nunocl45trpuoefft.j404oy.bid\nunocl45trpuoefft.j8exy2.bid\nunocl45trpuoefft.jcife9.bid\nunocl45trpuoefft.jdf4je.bid\nunocl45trpuoefft.jjogbj.top\nunocl45trpuoefft.jnd0bj.bid\nunocl45trpuoefft.jsotn5.top\nunocl45trpuoefft.jvrh8g.bid\nunocl45trpuoefft.k56185.top\nunocl45trpuoefft.kf1gxm.bid\nunocl45trpuoefft.kg5bof.bid\nunocl45trpuoefft.kml2o2.top\nunocl45trpuoefft.knowhands.us\nunocl45trpuoefft.ks3ghp.bid\nunocl45trpuoefft.kswcuk.top\nunocl45trpuoefft.l05l27.top\nunocl45trpuoefft.l69xgc.bid\nunocl45trpuoefft.l97i5a.bid\nunocl45trpuoefft.lak8wd.bid\nunocl45trpuoefft.larebg.bid\nunocl45trpuoefft.lcyznu.bid\nunocl45trpuoefft.lio2wr.bid\nunocl45trpuoefft.lk0bzc.top\nunocl45trpuoefft.ll3zot.bid\nunocl45trpuoefft.lzskva.bid\nunocl45trpuoefft.m03t72.bid\nunocl45trpuoefft.m33d4b.bid\nunocl45trpuoefft.m9a225.top\nunocl45trpuoefft.mbwxyg.bid\nunocl45trpuoefft.md9eyv.bid\nunocl45trpuoefft.meetsface.win\nunocl45trpuoefft.metpast.date\nunocl45trpuoefft.mezy7j.bid\nunocl45trpuoefft.moonsides.faith\nunocl45trpuoefft.n20b1c.top\nunocl45trpuoefft.n41n1a.top\nunocl45trpuoefft.n94lrn.bid\nunocl45trpuoefft.na2iuz.bid\nunocl45trpuoefft.nmit4p.bid\nunocl45trpuoefft.noyl9o.bid\nunocl45trpuoefft.nz6emv.bid\nunocl45trpuoefft.o2dval.top\nunocl45trpuoefft.o8hpwj.top\nunocl45trpuoefft.og5ezh.top\nunocl45trpuoefft.on2420.bid\nunocl45trpuoefft.ozlrnx.bid\nunocl45trpuoefft.p1gneb.bid\nunocl45trpuoefft.p2ix1u.bid\nunocl45trpuoefft.p4sr76.top\nunocl45trpuoefft.pap44w.top\nunocl45trpuoefft.pbprju.bid\nunocl45trpuoefft.piy4l3.bid\nunocl45trpuoefft.ptneek.bid\nunocl45trpuoefft.r21wmw.top\nunocl45trpuoefft.r2vai7.bid\nunocl45trpuoefft.rgbb50.bid\nunocl45trpuoefft.rie9py.bid\nunocl45trpuoefft.rslh9a.top\nunocl45trpuoefft.s7b63k.bid\nunocl45trpuoefft.sirchi.bid\nunocl45trpuoefft.sp4o1t.bid\nunocl45trpuoefft.tcly4s.bid\nunocl45trpuoefft.tfmmby.bid\nunocl45trpuoefft.thanreal.link\nunocl45trpuoefft.ttabop.bid\nunocl45trpuoefft.u64rj2.top\nunocl45trpuoefft.uaol08.bid\nunocl45trpuoefft.ukwnvw.bid\nunocl45trpuoefft.um1x6z.bid\nunocl45trpuoefft.uog1ky.bid\nunocl45trpuoefft.uso3z0.bid\nunocl45trpuoefft.uw3r6a.top\nunocl45trpuoefft.uwckha.top\nunocl45trpuoefft.v4kx51.bid\nunocl45trpuoefft.v50gtu.bid\nunocl45trpuoefft.vfuvsv.bid\nunocl45trpuoefft.vi5iko.bid\nunocl45trpuoefft.vkm4l6.top\nunocl45trpuoefft.vkslju.bid\nunocl45trpuoefft.vlwbcz.bid\nunocl45trpuoefft.vmomcc.bid\nunocl45trpuoefft.whmykv.bid\nunocl45trpuoefft.wl8t6k.bid\nunocl45trpuoefft.wlvxd6.bid\nunocl45trpuoefft.wz139z.top\nunocl45trpuoefft.x9kjcn.bid\nunocl45trpuoefft.x9le66.top\nunocl45trpuoefft.xf38wp.bid\nunocl45trpuoefft.xlxd92.bid\nunocl45trpuoefft.y721yz.top\nunocl45trpuoefft.ye4f7k.bid\nunocl45trpuoefft.yky1uf.bid\nunocl45trpuoefft.ytbyhs.bid\nunocl45trpuoefft.yty0gm.bid\nunocl45trpuoefft.zbj2kc.bid\nunocl45trpuoefft.zdamew.bid\nunocl45trpuoefft.zgheyh.bid\nunocl45trpuoefft.zjems2.bid\nunocl45trpuoefft.zn9cme.bid\nurulvtffwoq.xyz\nuuwflbmjmi.eu\nuvcmlfca.biz\nuxvvm.us\nuxwavkmttywsuynt.pw\nvcabbvhrqhot.pw\nvewrb.italisumo.at\nvpuroeit.pw\nvrvis6ndra5jeggj.livegaming.ch\nvrvis6ndra5jeggj.livewargaming.ch\nvrvis6ndra5jeggj.onlinebattlefield.ch\nvrympoqs5ra34nfo.bigbird.at\nvrympoqs5ra34nfo.bigclear.at\nvrympoqs5ra34nfo.smartbus.at\nvrympoqs5ra34nfo.torhelper.pl\nvujqbcditgsqxe.fr\nvyohacxzoue32vvk.0ayn1s.top\nvyohacxzoue32vvk.0ot7em.bid\nvyohacxzoue32vvk.0vtwzy.top\nvyohacxzoue32vvk.1m47ka.bid\nvyohacxzoue32vvk.23fvxw.bid\nvyohacxzoue32vvk.2hr4fs.top\nvyohacxzoue32vvk.34o9h1.bid\nvyohacxzoue32vvk.3buvlc.bid\nvyohacxzoue32vvk.3m370u.top\nvyohacxzoue32vvk.3peyo3.bid\nvyohacxzoue32vvk.3t3hyf.top\nvyohacxzoue32vvk.5a5vmh.top\nvyohacxzoue32vvk.5i0ukv.bid\nvyohacxzoue32vvk.5m2n7x.top\nvyohacxzoue32vvk.5s96fr.top\nvyohacxzoue32vvk.6wkz70.bid\nvyohacxzoue32vvk.79j8fm.top\nvyohacxzoue32vvk.7a07br.bid\nvyohacxzoue32vvk.7jrv53.bid\nvyohacxzoue32vvk.7m7ujm.bid\nvyohacxzoue32vvk.8g1k17.bid\nvyohacxzoue32vvk.ac7zvz.top\nvyohacxzoue32vvk.axu3u8.bid\nvyohacxzoue32vvk.b14kkk.bid\nvyohacxzoue32vvk.c4cwr4.bid\nvyohacxzoue32vvk.c8jxpp.top\nvyohacxzoue32vvk.chnbyl.bid\nvyohacxzoue32vvk.cp3yme.top\nvyohacxzoue32vvk.d7h6yx.top\nvyohacxzoue32vvk.dgjpgy.top\nvyohacxzoue32vvk.dks71o.bid\nvyohacxzoue32vvk.ean5e7.top\nvyohacxzoue32vvk.ewfp5y.bid\nvyohacxzoue32vvk.ezb568.top\nvyohacxzoue32vvk.fp6fj6.top\nvyohacxzoue32vvk.fsly47.top\nvyohacxzoue32vvk.g7rst5.bid\nvyohacxzoue32vvk.gjbmis.top\nvyohacxzoue32vvk.h2xun1.top\nvyohacxzoue32vvk.ibar8s.top\nvyohacxzoue32vvk.jb4uh0.top\nvyohacxzoue32vvk.jnv1df.top\nvyohacxzoue32vvk.joco7r.top\nvyohacxzoue32vvk.jwi2ek.bid\nvyohacxzoue32vvk.k9p80d.top\nvyohacxzoue32vvk.kfymbh.top\nvyohacxzoue32vvk.kwrd4f.bid\nvyohacxzoue32vvk.l4dlll.bid\nvyohacxzoue32vvk.mayrwf.top\nvyohacxzoue32vvk.mpduf5.bid\nvyohacxzoue32vvk.ncw0rp.top\nvyohacxzoue32vvk.nta934.top\nvyohacxzoue32vvk.o08ra6.top\nvyohacxzoue32vvk.o5b17o.top\nvyohacxzoue32vvk.p9su2u.top\nvyohacxzoue32vvk.pr52ni.top\nvyohacxzoue32vvk.r31sot.top\nvyohacxzoue32vvk.r3b2sh.top\nvyohacxzoue32vvk.roep3o.top\nvyohacxzoue32vvk.ss8doe.top\nvyohacxzoue32vvk.t6ueop.bid\nvyohacxzoue32vvk.u8e2dz.top\nvyohacxzoue32vvk.ug6ewx.top\nvyohacxzoue32vvk.vjso7r.top\nvyohacxzoue32vvk.w22p3v.top\nvyohacxzoue32vvk.w67y8u.bid\nvyohacxzoue32vvk.x83zw1.top\nvyohacxzoue32vvk.xsf5a8.top\nvyohacxzoue32vvk.xy2rlg.bid\nvyohacxzoue32vvk.zmn16h.top\nvyohacxzoue32vvk.zn90h4.bid\nvyohacxzoue32vvk.zp9i1l.bid\nvyohacxzoue32vvk.zu3fzc.bid\nvyohacxzoue32vvk.zz3w5l.bid\nw6bfg4hahn5bfnlsafgchkvg5fwsfvrt.hareuna.at\nwaduavfijwkanvf.xyz\nwbaskcsxiffiax.info\nwdvxeval.ru\nwersalitrestyws.top\nwjfkoqueatxdmqw.biz\nwjtqjleommc4z46i.249isv.bid\nwjtqjleommc4z46i.2y4t6f.bid\nwjtqjleommc4z46i.35rof4.bid\nwjtqjleommc4z46i.35u068.bid\nwjtqjleommc4z46i.44vva6.bid\nwjtqjleommc4z46i.4bb9vz.bid\nwjtqjleommc4z46i.54vw9b.bid\nwjtqjleommc4z46i.5n5y6v.bid\nwjtqjleommc4z46i.5r1sol.bid\nwjtqjleommc4z46i.7hu6og.bid\nwjtqjleommc4z46i.8a9r2h.bid\nwjtqjleommc4z46i.993hev.bid\nwjtqjleommc4z46i.9sellg.bid\nwjtqjleommc4z46i.9ule2e.bid\nwjtqjleommc4z46i.au6d1d.bid\nwjtqjleommc4z46i.bipa9k.bid\nwjtqjleommc4z46i.c3fz3z.bid\nwjtqjleommc4z46i.cc0r87.bid\nwjtqjleommc4z46i.cdyd2z.bid\nwjtqjleommc4z46i.cgab48.bid\nwjtqjleommc4z46i.cm5ohx.bid\nwjtqjleommc4z46i.csv7o6.bid\nwjtqjleommc4z46i.cto5ee.bid\nwjtqjleommc4z46i.d11zjd.bid\nwjtqjleommc4z46i.e53rg4.bid\nwjtqjleommc4z46i.eag72x.top\nwjtqjleommc4z46i.efyh72.bid\nwjtqjleommc4z46i.f0jlbj.bid\nwjtqjleommc4z46i.fw1bwy.bid\nwjtqjleommc4z46i.fwfu4t.bid\nwjtqjleommc4z46i.gg4dgp.bid\nwjtqjleommc4z46i.h8prbu.top\nwjtqjleommc4z46i.hom07d.bid\nwjtqjleommc4z46i.i8zh1k.bid\nwjtqjleommc4z46i.idw6s5.bid\nwjtqjleommc4z46i.ilmgcl.bid\nwjtqjleommc4z46i.izyclz.bid\nwjtqjleommc4z46i.j0n83w.bid\nwjtqjleommc4z46i.jal9lk.bid\nwjtqjleommc4z46i.jujthy.bid\nwjtqjleommc4z46i.kt70uk.bid\nwjtqjleommc4z46i.kyjw0g.bid\nwjtqjleommc4z46i.kzhzuc.top\nwjtqjleommc4z46i.ldsl8m.bid\nwjtqjleommc4z46i.m33d4b.bid\nwjtqjleommc4z46i.n8ln0w.bid\nwjtqjleommc4z46i.nh47ri.bid\nwjtqjleommc4z46i.nnbdlh.bid\nwjtqjleommc4z46i.nxmu0x.bid\nwjtqjleommc4z46i.o8hpwj.top\nwjtqjleommc4z46i.obx4vo.bid\nwjtqjleommc4z46i.oodvxp.bid\nwjtqjleommc4z46i.p41khf.bid\nwjtqjleommc4z46i.pmnz7a.bid\nwjtqjleommc4z46i.salethe.gdn\nwjtqjleommc4z46i.srmlzh.bid\nwjtqjleommc4z46i.srtos7.bid\nwjtqjleommc4z46i.t4jp3w.bid\nwjtqjleommc4z46i.u36ik0.bid\nwjtqjleommc4z46i.uv39h5.bid\nwjtqjleommc4z46i.uwckha.top\nwjtqjleommc4z46i.vh6vss.bid\nwjtqjleommc4z46i.w3r6a4.bid\nwjtqjleommc4z46i.whmykv.bid\nwjtqjleommc4z46i.xjwlms.bid\nwjtqjleommc4z46i.y12acl.bid\nwjtqjleommc4z46i.y2ijlz.bid\nwjtqjleommc4z46i.y7603i.bid\nwjtqjleommc4z46i.yfr0o1.bid\nwjtqjleommc4z46i.z7uxzg.bid\nwjtqjleommc4z46i.z97f9v.bid\nwjtqjleommc4z46i.zclhx9.bid\nwor4d.slewirk.at\nwpvvusso.xyz\nwqxvsxppjivs.pw\nwrubyjtvqhxaqkh.pw\nwtxvmsikbmtbq.pw\nwvltrlrnf.xyz\nwww.1axb.com\nwww.chromebewfk.top\nwww.chromefastl.top\nwww.chromehakc.top\nwww.cleverdotl.top\nwww.ddiopoola.top\nwww.dealkolld.top\nwww.dokjasura.top\nwww.fkauueeepla.top\nwww.flowerxpo.top\nwww.foolalexas.top\nwww.googlefoad.top\nwww.newsectorbs.top\nwww.newtonpaiva.br\nwww.watherfka.top\nwww.weekendlk.top\nx5sbb5gesp6kzwsh.frontmain.pl\nx5sbb5gesp6kzwsh.frontymen.pl\nx5sbb5gesp6kzwsh.homewind.pl\nx5sbb5gesp6kzwsh.mailteam.pl\nx5sbb5gesp6kzwsh.questpul.pl\nxfyubqmldwvuyar.yt\nxhrnfffaixawpuob.pw\nxmniabhrfafptwx.pw\nxofguhypjgvxrm.pw\nxpcx6erilkjced3j.16hwwh.top\nxpcx6erilkjced3j.16umxg.top\nxpcx6erilkjced3j.17gcun.top\nxpcx6erilkjced3j.18ey8e.top\nxpcx6erilkjced3j.19kdeh.top\nxpcx6erilkjced3j.1blery.top\nxpcx6erilkjced3j.1cgbcv.top\nxpcx6erilkjced3j.1ebjjq.top\nxpcx6erilkjced3j.1j9jad.top\nxpcx6erilkjced3j.1jyrty.top\nxpcx6erilkjced3j.1mfmkz.top\nxpcx6erilkjced3j.1mpsnr.top\nxpcx6erilkjced3j.1n5mod.top\nxrhwryizf5mui7a5.50mb1c.bid\nxrhwryizf5mui7a5.djintc.bid\nxrhwryizf5mui7a5.g72xh8.top\nxrhwryizf5mui7a5.h44l3d.bid\nxrhwryizf5mui7a5.j4cser.bid\nxrhwryizf5mui7a5.jhrb5a.top\nxrhwryizf5mui7a5.r8c85p.top\nxrhwryizf5mui7a5.rt01jw.top\nxrhwryizf5mui7a5.uw9x7z.bid\nxrhwryizf5mui7a5.vgxcci.top\nxvchcbeqxkd.pw\nxyhhuxa.be\ny4bxj.adozeuds.com\nyavmxpiqfwmubk.pw\nyaynawvtuqcarjwc.pw\nycvcjbhgkmsiyhdd.info\nyofkhfskdyiqo.biz\nytcijiooxdtlbevrh.info\nytrest84y5i456hghadefdsd.pontogrot.com\nyuertao.pw\nyuysikankhqvdwdv.xyz\nywjgjvpuyitnbiw.info\nyyre45dbvn2nhbefbmh.begumvelic.at\nzjfq4lnfbs7pncr5.onion.to\nzjfq4lnfbs7pncr5.tor2web.org\nztuw5bvuuapzdfya.klimbim.pl\nzutzt67dcxr6mxcn.onion.to\n" }, { "path": "tests/integration/basic/gen-results.sh", "content": "#!/bin/bash\n\ncurl -# -k -o IPv4HC.result \"https://127.0.0.1/feeds/IPv4HC\" -o IPv4HC.result\ncurl -# -k -o IPv4HC%3Fs%3D5%26n%3D10.result \"https://127.0.0.1/feeds/IPv4HC?s=5&n=10\"\ncurl -# -k -o IPv4HC%3Fv%3Djson%26tr%3D1.result \"https://127.0.0.1/feeds/IPv4HC?v=json&tr=1\"\ncurl -# -k -o IPv4HC%3Fv%3Djson-seq.result \"https://127.0.0.1/feeds/IPv4HC?v=json-seq\"\ncurl -# -k -o IPv4HC%3Fv%3Dmwg.result \"https://127.0.0.1/feeds/IPv4HC?v=mwg\"\ncurl -# -k -o IPv4HC%3Fv%3Dcsv%26f%3Dconfidence%26f%3Dsources%7Cfeeds%26f%3Dindicator%7Cclientip%26tr%3D1.result \"https://127.0.0.1/feeds/IPv4HC?v=csv&f=confidence&f=sources|feeds&f=indicator|clientip&tr=1\"\ncurl -# -k -o URLHC.result \"https://127.0.0.1/feeds/URLHC\"\ncurl -# -k -o URLHC%3Fs%3D5%26n%3D10.result \"https://127.0.0.1/feeds/URLHC?s=5&n=10\"\ncurl -# -k -o URLHC%3Fv%3Djson%26tr%3D1.result \"https://127.0.0.1/feeds/URLHC?v=json&tr=1\"\ncurl -# -k -o URLHC%3Fv%3Djson-seq.result \"https://127.0.0.1/feeds/URLHC?v=json-seq\"\ncurl -# -k -o URLHC%3Fv%3Dmwg.result \"https://127.0.0.1/feeds/URLHC?v=mwg\"\ncurl -# -k -o URLHC%3Fv%3Dcsv%26f%3Dconfidence%26f%3Dsources%7Cfeeds%26f%3Dindicator%7Curl.result \"https://127.0.0.1/feeds/URLHC?v=csv&f=confidence&f=sources|feeds&f=indicator|url\"\ncurl -# -k -o URLHC%3Fv%3Dbluecoat.result \"https://127.0.0.1/feeds/URLHC?v=bluecoat\"\ncurl -# -k -o URLHC%3Fv%3Dbluecoat%26cd%3Dtest.result \"https://127.0.0.1/feeds/URLHC?v=bluecoat&cd=test\"\ncurl -# -k -o URLHC%3Fv%3Dpanosurl.result \"https://127.0.0.1/feeds/URLHC?v=panosurl\"\ncurl -# -k -o URLHC%3Fv%3Dpanosurl%26sp%3D1.result \"https://127.0.0.1/feeds/URLHC?v=panosurl&sp=1\"\ncurl -# -k -o URLHC%3Fv%3Dpanosurl%26di%3D1.result \"https://127.0.0.1/feeds/URLHC?v=panosurl&di=1\"\ncurl -# -k -o DomainHC%3Fv%3Dcarbonblack.result \"https://127.0.0.1/feeds/DomainHC?v=carbonblack\"\n" }, { "path": "tests/integration/basic/test.py", "content": "#!/usr/bin/env python2\n\nimport logging\nimport time\nimport urllib\nimport os\nimport sys\nimport re\n\nfrom itertools import izip_longest\n\nimport requests\nimport urllib3\n\n\nurllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n\nLOG = logging.getLogger(__name__)\n\n\ndef get_full_config(mmurl, username, password):\n response = requests.get(\n '{}/config/full'.format(mmurl),\n auth=(username, password),\n verify=False\n )\n response.raise_for_status()\n\n return response.json()['result']\n\n\ndef delete_node(nodeid, version, mmurl, username, password):\n response = requests.delete(\n '{}/config/node/{}'.format(mmurl, nodeid),\n params={'version': version},\n auth=(username, password),\n verify=False\n )\n response.raise_for_status()\n\n\ndef create_node(version, nodename, prototype, inputs, output, mmurl, username, password):\n response = requests.post(\n '{}/config/node'.format(mmurl),\n auth=(username, password),\n verify=False,\n json={\n 'name': nodename,\n 'version': version,\n 'properties': {\n 'prototype': prototype,\n 'inputs': inputs,\n 'output': output\n }\n },\n headers={\n 'Content-Type': 'application/json'\n }\n )\n response.raise_for_status()\n\n\ndef commit_and_restart(mmurl, username, password):\n full_config = get_full_config(mmurl, username, password)\n\n response = requests.post(\n '{}/config/commit'.format(mmurl),\n auth=(username, password),\n verify=False,\n json={\n 'version': full_config['version'],\n },\n headers={\n 'Content-Type': 'application/json'\n }\n )\n response.raise_for_status()\n\n response = requests.get(\n '{}/supervisor/minemeld-engine/restart'.format(mmurl),\n auth=(username, password),\n verify=False\n )\n response.raise_for_status()\n\n\ndef delete_config(mmurl, username, password):\n LOG.info('Deleting config...')\n\n full_config = get_full_config(mmurl, username, password)\n\n for idx, n in enumerate(full_config['nodes']):\n if not n:\n continue\n \n delete_node(\n idx,\n n['version'],\n mmurl,\n username,\n password\n )\n\n full_config = get_full_config(mmurl, username, password)\n nnodes = len([n for n in full_config['nodes'] if n])\n if nnodes != 0:\n raise RuntimeError('Config not deleted: {!r}'.format(full_config))\n\n\ndef create_config(mmurl, username, password):\n LOG.info('Creating new config...')\n\n full_config = get_full_config(mmurl, username, password)\n\n # miner\n create_node(\n version=full_config['version'],\n nodename='localdb',\n prototype='stdlib.localDB',\n output=True,\n inputs=[],\n mmurl=mmurl,\n username=username,\n password=password\n )\n\n # URL flow\n create_node(\n version=full_config['version'],\n nodename='URLAggregator',\n prototype='stdlib.aggregatorURL',\n output=True,\n inputs=['localdb'],\n mmurl=mmurl,\n username=username,\n password=password\n )\n create_node(\n version=full_config['version'],\n nodename='URLHC',\n prototype='stdlib.feedHCWithValue',\n output=True,\n inputs=['URLAggregator'],\n mmurl=mmurl,\n username=username,\n password=password\n )\n\n # domain flow\n create_node(\n version=full_config['version'],\n nodename='DomainAggregator',\n prototype='stdlib.aggregatorDomain',\n output=True,\n inputs=['localdb'],\n mmurl=mmurl,\n username=username,\n password=password\n )\n create_node(\n version=full_config['version'],\n nodename='DomainHC',\n prototype='stdlib.feedHCWithValue',\n output=True,\n inputs=['DomainAggregator'],\n mmurl=mmurl,\n username=username,\n password=password\n )\n\n # IPv4\n create_node(\n version=full_config['version'],\n nodename='IPv4Aggregator',\n prototype='stdlib.aggregatorIPv4Generic',\n output=True,\n inputs=['localdb'],\n mmurl=mmurl,\n username=username,\n password=password\n )\n create_node(\n version=full_config['version'],\n nodename='IPv4HC',\n prototype='stdlib.feedHCWithValue',\n output=True,\n inputs=['IPv4Aggregator'],\n mmurl=mmurl,\n username=username,\n password=password\n )\n\n\ndef wait_for_restart(mmurl, username, password):\n LOG.info('Waiting for restart...')\n\n now = time.time()\n\n while time.time() < (now + 300):\n response = requests.get(\n '{}/supervisor'.format(mmurl),\n auth=(username, password),\n verify=False\n )\n response.raise_for_status()\n\n supervisor_status = response.json()['result']\n engine_status = supervisor_status['processes']['minemeld-engine']['statename']\n\n if engine_status == 'RUNNING':\n break\n\n time.sleep(10)\n\n else:\n raise RuntimeError('engine did not restart in 5 minutes')\n\ndef push_indicators(mmurl, username, password):\n LOG.info('Pushing indicators...')\n\n with open('IPv4.lst', 'r') as f:\n ipv4_iocs = f.readlines()\n\n with open('URL.lst', 'r') as f:\n url_iocs = f.readlines()\n\n with open('domain.lst', 'r') as f:\n domain_iocs = f.readlines()\n\n num_ipv4_iocs = len(ipv4_iocs)\n ipv4_iocs = ['IPv4\\n{}\\n\\n'.format(ioc) for ioc in ipv4_iocs]\n\n num_url_iocs = len(url_iocs)\n url_iocs = ['URL\\n{}\\n\\n'.format(ioc) for ioc in url_iocs]\n\n num_domain_iocs = len(domain_iocs)\n domain_iocs = ['domain\\n{}\\n\\n'.format(ioc) for ioc in domain_iocs]\n\n response = requests.post(\n '{}/config/data/localdb_indicators/append'.format(mmurl),\n params={\n 'h': 'localdb',\n 't': 'localdb'\n },\n auth=(username, password),\n headers={'Content-Type': 'application/text'},\n data=''.join(ipv4_iocs)+''.join(url_iocs)+''.join(domain_iocs),\n verify=False\n )\n response.raise_for_status()\n\n # wait for URLs to propagate\n LOG.info('Waiting for URLs to propagate...')\n now = time.time()\n while time.time() < (now + 300):\n response = requests.get(\n '{}/feeds/URLHC'.format(mmurl),\n verify=False\n )\n response.raise_for_status()\n\n if len(response.content.splitlines()) == num_url_iocs:\n break\n \n time.sleep(10)\n\n else:\n raise RuntimeError('URL IOCs did not propagate in 5 minutes')\n\n # wait for IPv4s to propagate\n LOG.info('Waiting for IPv4s to propagate...')\n now = time.time()\n while time.time() < (now + 300):\n response = requests.get(\n '{}/feeds/IPv4HC'.format(mmurl),\n verify=False\n )\n response.raise_for_status()\n\n if len(response.content.splitlines()) == num_ipv4_iocs:\n break\n \n time.sleep(10)\n\n else:\n raise RuntimeError('IPv4 IOCs did not propagate in 5 minutes')\n\n # wait for domains to propagate\n LOG.info('Waiting for domains to propagate...')\n now = time.time()\n while time.time() < (now + 300):\n response = requests.get(\n '{}/feeds/DomainHC'.format(mmurl),\n verify=False\n )\n response.raise_for_status()\n\n if len(response.content.splitlines()) == num_domain_iocs:\n break\n\n time.sleep(10)\n \n else:\n raise RuntimeError('domain IOCs did not propagate in 5 minutes')\n\n\ndef remove_timestamps(s):\n s = re.sub(r'\\\"first_seen\\\":[0-9]+', '\\\"first_seen\\\":0', s)\n s = re.sub(r'\\\"last_seen\\\":[0-9]+', '\\\"last_seen\\\":0', s)\n s = re.sub(r'\\\"timestamp\\\": [0-9]+', '\\\"timestamp\\\": 0', s)\n\n return s\n\n\ndef check_feeds(mmurl):\n check_result = True\n\n local_files = os.listdir('.')\n\n for fname in local_files:\n if not fname.endswith('.result'):\n continue\n\n req, _ = fname.split('.', 1)\n\n with open(fname, 'r') as f:\n result = f.readlines()\n\n LOG.info('Checking {}...'.format(urllib.unquote(req)))\n response = requests.get(\n '{}/feeds/{}'.format(mmurl, urllib.unquote(req)),\n verify=False\n )\n response.raise_for_status()\n\n clines = response.content.splitlines()\n\n for idx, (cl, rl) in enumerate(izip_longest(result, clines)):\n cl = remove_timestamps(cl.strip())\n rl = remove_timestamps(rl.strip())\n\n if cl != rl:\n LOG.error('{} does not match'.format(urllib.unquote(req)))\n LOG.error('L{} expected: {!r}'.format(idx, rl))\n LOG.error('L{} result: {!r}'.format(idx, cl))\n \n check_result = False\n break\n\n return check_result\n\ndef main():\n logging.basicConfig(level=logging.INFO)\n\n mmurl = os.environ.get('MM_URL', 'https://127.0.0.1')\n username = os.environ.get('MM_USERNAME', 'admin')\n password = os.environ.get('MM_PASSWORD', 'minemeld')\n\n delete_config(mmurl, username, password)\n commit_and_restart(mmurl, username, password)\n wait_for_restart(mmurl, username, password)\n\n create_config(mmurl, username, password)\n commit_and_restart(mmurl, username, password)\n wait_for_restart(mmurl, username, password)\n\n push_indicators(mmurl, username, password)\n\n if not check_feeds(mmurl):\n sys.exit(1)\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "tests/panos_mock.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport xml.etree.ElementTree\nimport re\nimport os.path\nimport collections\nimport logging\n\nMYDIR = os.path.dirname(__file__)\nSUBRE = re.compile(\"[^A-Za-z0-9_]\")\n\nLOG = logging.getLogger(__name__)\n\n\nclass PanXapiMock(object):\n def __init__(self, **kwargs):\n self.hostname = kwargs.get('hostname', 'xapi')\n self.counters = collections.defaultdict(lambda: 0)\n\n self.user_id_calls = []\n\n def _parse_answer(self, cmd, arg, counter):\n file_name = '%s_%s_%s_%d' % (self.hostname, cmd, arg, counter)\n file_name = SUBRE.sub('_', file_name)\n file_name = os.path.join(MYDIR, file_name+'.xml')\n\n LOG.debug(file_name)\n\n try:\n os.stat(file_name)\n\n except OSError:\n file_name = '%s_%s_%s' % (self.hostname, cmd, arg)\n file_name = SUBRE.sub('_', file_name)\n file_name = os.path.join(MYDIR, file_name+'.xml')\n\n os.stat(file_name)\n\n self.element_root = xml.etree.ElementTree.parse(file_name).getroot()\n\n def get(self, **kwargs):\n xpath = kwargs.get('xpath', None)\n if xpath is None:\n raise RuntimeError('no xpath in get')\n\n c = self.counters['get:::'+xpath]\n self.counters['get:::'+xpath] += 1\n\n self._parse_answer('get', xpath, c)\n\n def show(self, **kwargs):\n xpath = kwargs.get('xpath', None)\n if xpath is None:\n raise RuntimeError('no xpath in show')\n\n c = self.counters['show:::'+xpath]\n self.counters['show:::'+xpath] += 1\n\n self._parse_answer('show', xpath, c)\n\n def op(self, **kwargs):\n cmd = kwargs.get('cmd', None)\n if cmd is None:\n raise RuntimeError('no cmd in op')\n\n c = self.counters['op:::'+cmd]\n self.counters['op:::'+cmd] += 1\n\n self._parse_answer('op', cmd, c)\n\n def user_id(self, **kwargs):\n cmd = kwargs.get('cmd', None)\n if cmd is None:\n raise RuntimeError('no cmd in user_id')\n\n self.user_id_calls.append(cmd)\n\n\ndef factory(**kwargs):\n return PanXapiMock(**kwargs)\n" }, { "path": "tests/st_profile.py", "content": "#!/usr/bin/env python\n\n# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport uuid\nimport random\nimport tempfile\nimport time\n\nimport minemeld.ft.st\n\nTABLENAME = tempfile.mktemp(prefix='minemeld.ftsttest')\n\n\ndef queries(st):\n # t1 = time.time()\n # j = 0\n # for j in xrange(num_queries):\n # q = random.randint(0, 0xFFFFFFFF)\n # t2 = time.time()\n # dt = t2-t1\n\n # t1 = time.time()\n\n j = 0\n for j in xrange(num_queries):\n q = random.randint(0, 0xFFFFFFFF)\n next(st.cover(q), None)\n # t2 = time.time()\n\n # print \"TIME: Queried %d times in %d\" % (num_queries, (t2-t1-dt))\n\nif __name__ == '__main__':\n num_intervals = 100000\n num_queries = num_intervals\n\n st = minemeld.ft.st.ST(TABLENAME, 32, truncate=True)\n\n t1 = time.time()\n for j in xrange(num_intervals):\n end = random.randint(0, 0xFFFFFFFF)\n if random.randint(0, 1) == 0:\n end = end & 0xFFFFFF00\n start = end + 0xFF\n else:\n start = end\n sid = uuid.uuid4().bytes\n t2 = time.time()\n dt = t2-t1\n\n t1 = time.time()\n for j in xrange(num_intervals):\n end = random.randint(0, 0xFFFFFFFF)\n if random.randint(0, 1) == 0:\n start = end & 0xFFFFFF00\n end = start + 0xFF\n else:\n start = end\n sid = uuid.uuid4().bytes\n st.put(sid, start, end)\n t2 = time.time()\n print \"TIME: Inserted %d intervals in %d\" % (num_intervals, (t2-t1-dt))\n\n queries(st)\n" }, { "path": "tests/test-prototype-1.yml", "content": "nodes:\n testprototype:\n prototype: testproto.test\n\n" }, { "path": "tests/test_comm_amqp.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport unittest\n\nimport minemeld.comm.amqp\n\n\nclass MineMeldCommAMQP(unittest.TestCase):\n def test_01_rpc(self):\n class A(object):\n def f(self):\n return 'ok'\n\n a = A()\n\n ac = minemeld.comm.amqp.AMQP({})\n ac.request_rpc_server_channel('a', a, allowed_methods=['f'])\n ac.start()\n\n result = ac.send_rpc('a', 'f', {}, timeout=1)\n self.assertEqual(result['result'], 'ok')\n\n ac.stop()\n\n def test_02_pubsub(self):\n class A(object):\n counter = 0\n\n def f(self):\n self.counter += 1\n\n a = A()\n\n ac = minemeld.comm.amqp.AMQP({})\n ac.request_sub_channel('a', a, allowed_methods=['f'])\n pc = ac.request_pub_channel('a')\n ac.start()\n\n pc.publish('f')\n gevent.sleep(0.1)\n\n self.assertEqual(a.counter, 1)\n\n ac.stop()\n\n def test_03_rpc_fanout(self):\n class A(object):\n def __init__(self, n):\n self.n = n\n\n def f(self):\n return self.n\n\n a1 = A(1)\n a2 = A(2)\n\n ac = minemeld.comm.amqp.AMQP({})\n ac.request_rpc_server_channel('a1', a1, allowed_methods=['f'],\n fanout='test')\n ac.request_rpc_server_channel('a2', a2, allowed_methods=['f'],\n fanout='test')\n client = ac.request_rpc_fanout_client_channel('test')\n ac.start()\n\n evt = client.send_rpc('f', params={}, num_results=2)\n success = evt.wait(timeout=5)\n\n self.assertNotEqual(success, None)\n\n result = evt.get(block=False)\n\n self.assertEqual(result['answers'], {'a1': 1, 'a2': 2})\n\n ac.stop()\n" }, { "path": "tests/test_device_list.yml", "content": "- name: test\n hostname: 192.168.55.152\n api_username: admin\n api_password: admin\n- name: test2\n tag: ngfw1\n" }, { "path": "tests/test_device_list2.yml", "content": "- name: test2\n tag: ngfw2\n- name: test\n hostname: 192.168.55.152\n api_username: admin\n api_password: admin\n" }, { "path": "tests/test_flask_aaa.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT autofocus tests\n\nUnit tests for minemeld.ft.autofocus\n\"\"\"\n\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport unittest\nimport mock\nimport shutil\nimport logging\nimport os\nimport os.path\nimport base64\nimport passlib.apache\nimport xmltodict\n\nos.environ['MM_CONFIG'] = '.'\nos.environ['API_CONFIG_LOCK'] = os.path.join('.', 'api-config.lock')\n\nimport minemeld.flask.main\nimport minemeld.flask.feedredis\n\nLOG = logging.getLogger(__name__)\nMYDIR = os.path.dirname(__file__)\nTAXII_POLL_REQUEST = \"\"\"\n 2014-12-19T00:00:00Z\n 2014-12-19T12:00:00Z\n \n FULL\n \n\"\"\"\n\n\ndef _authorization_header(username, password):\n return 'Basic '+base64.b64encode(username+':'+password)\n\n\nclass MineMeldFlaskAAATests(unittest.TestCase):\n def setUp(self):\n try:\n shutil.rmtree('./api')\n except OSError:\n pass\n\n os.mkdir('api')\n\n minemeld.flask.main.app.config.update(\n DEBUG=True\n )\n self.app = minemeld.flask.main.app.test_client()\n\n def tearDown(self):\n try:\n shutil.rmtree('./api')\n except OSError:\n pass\n\n def _taxii_discovery_request(self, username=None, password=None):\n headers = {\n 'X-TAXII-Content-Type': 'urn:taxii.mitre.org:message:xml:1.1',\n 'X-TAXII-Protocol': 'urn:taxii.mitre.org:protocol:http:1.0',\n 'X-TAXII-Services': 'urn:taxii.mitre.org:services:1.1'\n }\n if username is not None:\n headers['Authorization'] = _authorization_header(username, password)\n resp = self.app.post(\n '/taxii-discovery-service',\n headers=headers,\n data=''\n )\n\n return resp\n\n def _taxii_collection_request(self, username=None, password=None):\n headers = {\n 'X-TAXII-Content-Type': 'urn:taxii.mitre.org:message:xml:1.1',\n 'X-TAXII-Protocol': 'urn:taxii.mitre.org:protocol:http:1.0',\n 'X-TAXII-Services': 'urn:taxii.mitre.org:services:1.1'\n }\n if username is not None:\n headers['Authorization'] = _authorization_header(username, password)\n resp = self.app.post(\n '/taxii-collection-management-service',\n headers=headers,\n data=''\n )\n\n return resp\n\n def _taxii_poll_request(self, collection, username=None, password=None):\n headers = {\n 'X-TAXII-Content-Type': 'urn:taxii.mitre.org:message:xml:1.1',\n 'X-TAXII-Protocol': 'urn:taxii.mitre.org:protocol:http:1.0',\n 'X-TAXII-Services': 'urn:taxii.mitre.org:services:1.1'\n }\n if username is not None:\n headers['Authorization'] = _authorization_header(username, password)\n resp = self.app.post(\n '/taxii-poll-service',\n headers=headers,\n data=TAXII_POLL_REQUEST % collection\n )\n\n return resp \n\n def _num_collections(self, resp):\n ans = xmltodict.parse(resp.data)\n\n collections = ans['taxii_11:Collection_Information_Response']['taxii_11:Collection']\n if isinstance(collections, list):\n return len(collections)\n return 1\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.feedredis.MMMaster')\n def test_feeds_auth_disabled(self, mmmastermock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': False\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n mmmastermock.status.return_value = {\n 'mbus:slave:feed1': {\n 'class': 'minemeld.ft.redis.RedisSet'\n }\n }\n\n resp = self.app.get('/feeds/feed1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('guest', 'guest')\n })\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('user1', 'password1')\n })\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('admin', 'password')\n })\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.feedredis.MMMaster')\n def test_feeds_single_tag(self, mmmastermock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n mmmastermock.status.return_value = {\n 'mbus:slave:feed1': {\n 'class': 'minemeld.ft.redis.RedisSet'\n }\n }\n\n resp = self.app.get('/feeds/feed1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('guest', 'guest')\n })\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('user1', 'password1')\n })\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('admin', 'password')\n })\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.feedredis.MMMaster')\n def test_feeds_two_tags(self, mmmastermock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential', 'open']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n mmmastermock.status.return_value = {\n 'mbus:slave:feed1': {\n 'class': 'minemeld.ft.redis.RedisSet'\n }\n }\n\n resp = self.app.get('/feeds/feed1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('guest', 'guest')\n })\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('user1', 'password1')\n })\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('admin', 'password')\n })\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.feedredis.MMMaster')\n def test_feeds_two_and_two(self, mmmastermock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential', 'open']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n mmmastermock.status.return_value = {\n 'mbus:slave:feed1': {\n 'class': 'minemeld.ft.redis.RedisSet'\n }\n }\n\n resp = self.app.get('/feeds/feed1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('guest', 'guest')\n })\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('user1', 'password1')\n })\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('admin', 'password')\n })\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.feedredis.MMMaster')\n def test_feeds_any(self, mmmastermock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['any']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n mmmastermock.status.return_value = {\n 'mbus:slave:feed1': {\n 'class': 'minemeld.ft.redis.RedisSet'\n }\n }\n\n resp = self.app.get('/feeds/feed1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('guest', 'guest')\n })\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('user1', 'password1')\n })\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('admin', 'password')\n })\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.feedredis.MMMaster')\n def test_feeds_anonymous(self, mmmastermock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['anonymous', 'confidential']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n mmmastermock.status.return_value = {\n 'mbus:slave:feed1': {\n 'class': 'minemeld.ft.redis.RedisSet'\n }\n }\n\n resp = self.app.get('/feeds/feed1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('guest', 'guest')\n })\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('user1', 'password1')\n })\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('admin', 'password')\n })\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.feedredis.MMMaster')\n def test_feeds_anonymous_2(self, mmmastermock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['anonymous']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n mmmastermock.status.return_value = {\n 'mbus:slave:feed1': {\n 'class': 'minemeld.ft.redis.RedisSet'\n }\n }\n\n resp = self.app.get('/feeds/feed1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('guest', 'guest')\n })\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('user1', 'password1')\n })\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('admin', 'password')\n })\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.feedredis.MMMaster')\n def test_feeds_no_tags(self, mmmastermock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {}\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n mmmastermock.status.return_value = {\n 'mbus:slave:feed1': {\n 'class': 'minemeld.ft.redis.RedisSet'\n }\n }\n\n resp = self.app.get('/feeds/feed1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('guest', 'guest')\n })\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('user1', 'password1')\n })\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('admin', 'password')\n })\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.feedredis.MMMaster')\n def test_feeds_no_user_tags(self, mmmastermock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n mmmastermock.status.return_value = {\n 'mbus:slave:feed1': {\n 'class': 'minemeld.ft.redis.RedisSet'\n }\n }\n\n resp = self.app.get('/feeds/feed1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('guest', 'guest')\n })\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('user1', 'password1')\n })\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': _authorization_header('admin', 'password')\n })\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.feedredis.MMMaster')\n def test_feeds_malformed(self, mmmastermock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n mmmastermock.status.return_value = {\n 'mbus:slave:feed1': {\n 'class': 'minemeld.ft.redis.RedisSet'\n }\n }\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': 'invalid authorization'\n })\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': 'Basic YWJjZGVmCg'\n })\n self.assertEqual(resp.status_code, 401)\n\n resp = self.app.get('/feeds/feed1', headers={\n 'Authorization': 'Basic '+base64.b64encode('invalidauth')\n })\n self.assertEqual(resp.status_code, 401)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.taxiidiscovery.get_taxii_feeds', return_value=['feed1'])\n @mock.patch('minemeld.flask.taxiicollmgmt.get_taxii_feeds', return_value=['feed1'])\n def test_taxii_auth_disabled(self, gtfmock1, gtfmock2, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': False,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n resp = self._taxii_discovery_request()\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='guest', password='guest')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='user1', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='admin', password='password')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='user1', password='password2')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='admin', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request()\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='guest', password='guest')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='user1', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='admin', password='password')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='user1', password='password2')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='admin', password='password2')\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.taxiidiscovery.get_taxii_feeds', return_value=['feed1', 'feed2'])\n @mock.patch('minemeld.flask.taxiicollmgmt.get_taxii_feeds', return_value=['feed1', 'feed2'])\n def test_taxii_services_tag(self, gtfmock1, gtfmock2, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential']\n },\n 'feed2': {\n 'tags': ['disabled']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n resp = self._taxii_discovery_request()\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_discovery_request(username='guest', password='guest')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_discovery_request(username='user1', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='admin', password='password')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='user1', password='password2')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_discovery_request(username='admin', password='password1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_collection_request()\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_collection_request(username='guest', password='guest')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_collection_request(username='user1', password='password1')\n self.assertEqual(self._num_collections(resp), 1)\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='admin', password='password')\n self.assertEqual(self._num_collections(resp), 2)\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='user1', password='password2')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_collection_request(username='admin', password='password2')\n self.assertEqual(resp.status_code, 401)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.taxiidiscovery.get_taxii_feeds', return_value=['feed1', 'feed2'])\n @mock.patch('minemeld.flask.taxiicollmgmt.get_taxii_feeds', return_value=['feed1', 'feed2'])\n def test_taxii_anonymous(self, gtfmock1, gtfmock2, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential']\n },\n 'feed2': {\n 'tags': ['anonymous']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n resp = self._taxii_discovery_request()\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='guest', password='guest')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_discovery_request(username='user1', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='admin', password='password')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='user1', password='password2')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='admin', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request()\n self.assertEqual(self._num_collections(resp), 1)\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='guest', password='guest')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_collection_request(username='user1', password='password1')\n self.assertEqual(self._num_collections(resp), 1)\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='admin', password='password')\n self.assertEqual(self._num_collections(resp), 2)\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='user1', password='password2')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='admin', password='password2')\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.taxiidiscovery.get_taxii_feeds', return_value=['feed1', 'feed2'])\n @mock.patch('minemeld.flask.taxiicollmgmt.get_taxii_feeds', return_value=['feed1', 'feed2'])\n def test_taxii_any(self, gtfmock1, gtfmock2, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential']\n },\n 'feed2': {\n 'tags': ['any']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n resp = self._taxii_discovery_request()\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_discovery_request(username='guest', password='guest')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='user1', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='admin', password='password')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='user1', password='password2')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_discovery_request(username='admin', password='password1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_collection_request()\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_collection_request(username='guest', password='guest')\n self.assertEqual(self._num_collections(resp), 1)\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='user1', password='password1')\n self.assertEqual(self._num_collections(resp), 2)\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='admin', password='password')\n self.assertEqual(self._num_collections(resp), 2)\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='user1', password='password2')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_collection_request(username='admin', password='password2')\n self.assertEqual(resp.status_code, 401)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.taxiidiscovery.get_taxii_feeds', return_value=['feed1', 'feed2'])\n @mock.patch('minemeld.flask.taxiicollmgmt.get_taxii_feeds', return_value=['feed1', 'feed2'])\n def test_taxii_any_anonymous(self, gtfmock1, gtfmock2, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential']\n },\n 'feed2': {\n 'tags': ['any', 'anonymous']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n resp = self._taxii_discovery_request()\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='guest', password='guest')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='user1', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='admin', password='password')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='user1', password='password2')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_discovery_request(username='admin', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request()\n self.assertEqual(self._num_collections(resp), 1)\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='guest', password='guest')\n self.assertEqual(self._num_collections(resp), 1)\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='user1', password='password1')\n self.assertEqual(self._num_collections(resp), 2)\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='admin', password='password')\n self.assertEqual(self._num_collections(resp), 2)\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='user1', password='password2')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_collection_request(username='admin', password='password2')\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.taxiipoll.get_taxii_feeds', return_value=['feed1', 'feed2'])\n def test_taxiipoll_single_tag(self, gtfmock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n resp = self._taxii_poll_request('feed1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_poll_request('feed1', username='guest', password='guest')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_poll_request('feed1', username='user1', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='admin', password='password')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='user1', password='password2')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_poll_request('feed1', username='admin', password='password1')\n self.assertEqual(resp.status_code, 401)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.taxiipoll.get_taxii_feeds', return_value=['feed1', 'feed2'])\n def test_taxiipolll_two_tags(self, gtfmock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential', 'open']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n resp = self._taxii_poll_request('feed1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_poll_request('feed1', username='guest', password='guest')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='user1', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='admin', password='password')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='user1', password='password2')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_poll_request('feed1', username='admin', password='password1')\n self.assertEqual(resp.status_code, 401)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.taxiipoll.get_taxii_feeds', return_value=['feed1', 'feed2'])\n def test_taxiipoll_two_and_two(self, gtfmock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['confidential', 'open']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n resp = self._taxii_poll_request('feed1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_poll_request('feed1', username='guest', password='guest')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='user1', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='admin', password='password')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='user1', password='password2')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_poll_request('feed1', username='admin', password='password1')\n self.assertEqual(resp.status_code, 401)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.taxiipoll.get_taxii_feeds', return_value=['feed1', 'feed2'])\n def test_taxiipoll_any(self, gtfmock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['any']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n resp = self._taxii_poll_request('feed1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_poll_request('feed1', username='guest', password='guest')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='user1', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='admin', password='password')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='user1', password='password2')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_poll_request('feed1', username='admin', password='password1')\n self.assertEqual(resp.status_code, 401)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.taxiipoll.get_taxii_feeds', return_value=['feed1', 'feed2'])\n def test_taxiipoll_anonymous(self, gtfmock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['anonymous', 'confidential']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n resp = self._taxii_poll_request('feed1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='guest', password='guest')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_poll_request('feed1', username='user1', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='admin', password='password')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='user1', password='password2')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='admin', password='password1')\n self.assertEqual(resp.status_code, 200)\n\n @mock.patch.dict('minemeld.flask.config.os.environ', {\n 'MM_CONFIG': '.',\n 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'),\n })\n @mock.patch('minemeld.flask.config.init')\n @mock.patch('minemeld.flask.config.get')\n @mock.patch('minemeld.flask.taxiipoll.get_taxii_feeds', return_value=['feed1', 'feed2'])\n def test_taxiipoll_anonymous_2(self, gtfmock, configmock, configinitmock):\n _config_attrs = {\n 'API_AUTH_ENABLED': True,\n 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')),\n 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')),\n 'FEEDS_AUTH_ENABLED': True,\n 'FEEDS_USERS_ATTRS': {\n 'guest': {\n 'tags': ['open', 'test']\n },\n 'user1': {\n 'tags': ['confidential']\n }\n },\n 'FEEDS_ATTRS': {\n 'feed1': {\n 'tags': ['anonymous']\n }\n }\n }\n\n def _config_get(attribute, default=None):\n if attribute in _config_attrs:\n return _config_attrs[attribute]\n return default\n\n configmock.configure_mock(side_effect=_config_get)\n\n resp = self._taxii_poll_request('feed1')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='guest', password='guest')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_poll_request('feed1', username='user1', password='password1')\n self.assertEqual(resp.status_code, 401)\n\n resp = self._taxii_poll_request('feed1', username='admin', password='password')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='user1', password='password2')\n self.assertEqual(resp.status_code, 200)\n\n resp = self._taxii_poll_request('feed1', username='admin', password='password1')\n self.assertEqual(resp.status_code, 200)\n" }, { "path": "tests/test_ft_autofocus.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT autofocus tests\n\nUnit tests for minemeld.ft.autofocus\n\"\"\"\n\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport unittest\nimport mock\nimport time\nimport shutil\nimport logging\nimport gc\nimport calendar\nimport os.path\n\nimport minemeld.ft.autofocus\n\nFTNAME = 'testft-%d' % int(time.time())\n\nLOG = logging.getLogger(__name__)\n\nCUR_LOGICAL_TIME = 0\n\nMYDIR = os.path.dirname(__file__)\n\n\ndef logical_millisec(*args):\n return CUR_LOGICAL_TIME\n\n\ndef gevent_event_mock_factory():\n result = mock.Mock()\n result.wait.side_effect = gevent.GreenletExit()\n\n return result\n\n\nclass MineMeldAutofocusFTTests(unittest.TestCase):\n def setUp(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n def tearDown(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch.object(calendar, 'timegm', side_effect=logical_millisec)\n def test_type_of_indicators(self, um_mock, sleep_mock, event_mock,\n spawnl_mock, spawn_mock):\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n config = {\n 'side_config': os.path.join(MYDIR, 'dummy.yml')\n }\n\n a = minemeld.ft.autofocus.ExportList(FTNAME, chassis, config)\n\n inputs = []\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 1)\n self.assertEqual(spawn_mock.call_count, 3)\n\n self.assertEqual(a._type_of_indicator('1.1.1.1'), 'IPv4')\n self.assertEqual(a._type_of_indicator('1.1.1.2-1.1.1.5'), 'IPv4')\n self.assertEqual(a._type_of_indicator('1.1.1.0/24'), 'IPv4')\n self.assertEqual(a._type_of_indicator('www.google.com'), 'domain')\n self.assertEqual(a._type_of_indicator('www.google.com/test'), 'URL')\n self.assertEqual(a._type_of_indicator('https://www.google.com'), 'URL')\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n" }, { "path": "tests/test_ft_base.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT Table tests\n\nUnit tests for minemeld.ft.base\n\"\"\"\n\nimport unittest\nimport mock\nimport gevent\n\nimport minemeld.ft.base\nimport minemeld.ft\n\n\nclass MineMeldFTBaseTests(unittest.TestCase):\n @mock.patch.object(minemeld.ft.base.BaseFT, 'configure',\n return_value=None)\n def test_init(self, configure_mock):\n config = {}\n chassis = mock.Mock()\n\n bcls = minemeld.ft.base.BaseFT\n\n b = bcls('test', chassis, config)\n\n self.assertEqual(b.name, 'test')\n self.assertEqual(b.chassis, chassis)\n self.assertEqual(b.config, config)\n self.assertItemsEqual(b.inputs, [])\n self.assertEqual(b.output, None)\n self.assertEqual(b.configure.call_count, 1)\n self.assertEqual(b.state, minemeld.ft.ft_states.READY)\n\n def test_connect_io(self):\n ftname = 'test'\n\n config = {}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n\n b = minemeld.ft.base.BaseFT(ftname, chassis, config)\n\n inputs = ['a', 'b', 'c']\n output = True\n\n b.connect(inputs, output)\n\n self.assertItemsEqual(b.inputs, inputs)\n self.assertEqual(b.output, ochannel)\n\n icalls = []\n for i in inputs:\n icalls.append(\n mock.call(ftname, b, i,\n allowed_methods=['update', 'withdraw', 'checkpoint'])\n )\n\n chassis.request_sub_channel.assert_has_calls(\n icalls,\n any_order=True\n )\n\n chassis.request_rpc_channel.assert_called_once_with(\n ftname,\n b,\n allowed_methods=[\n 'update',\n 'withdraw',\n 'checkpoint',\n 'get',\n 'get_all',\n 'get_range',\n 'length'\n ]\n )\n\n chassis.request_pub_channel.assert_called_once_with(ftname)\n\n def test_rpc(self):\n ftname = 'test'\n\n config = {}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n\n b = minemeld.ft.base.BaseFT(ftname, chassis, config)\n\n inputs = []\n output = False\n\n b.connect(inputs, output)\n\n chassis.request_sub_channel.assert_not_called()\n chassis.request_pub_channel.assert_not_called()\n chassis.request_rpc_channel.assert_called_once_with(\n ftname,\n b,\n allowed_methods=[\n 'update',\n 'withdraw',\n 'checkpoint',\n 'get',\n 'get_all',\n 'get_range',\n 'length'\n ]\n )\n\n b.do_rpc('destft', 'rpcmethod', a=1, b=1)\n\n chassis.send_rpc.assert_called_once_with(\n ftname,\n 'destft',\n 'rpcmethod',\n {'a': 1, 'b': 1},\n timeout=30,\n block=True\n )\n\n def test_emit(self):\n ftname = 'test'\n\n config = {}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n\n b = minemeld.ft.base.BaseFT(ftname, chassis, config)\n\n inputs = []\n output = True\n\n b.connect(inputs, output)\n\n chassis.request_sub_channel.assert_not_called()\n chassis.request_pub_channel.assert_called_once_with(ftname)\n chassis.request_rpc_channel.assert_called_once_with(\n ftname,\n b,\n allowed_methods=[\n 'update',\n 'withdraw',\n 'checkpoint',\n 'get',\n 'get_all',\n 'get_range',\n 'length'\n ]\n )\n\n b.emit_update('testi', {'test': 'v'})\n\n self.assertEqual(ochannel.publish.call_count, 1)\n self.assertEqual(ochannel.publish.call_args[0][0],\n 'update')\n self.assertEqual(ochannel.publish.call_args[0][1]['indicator'],\n 'testi')\n self.assertEqual(ochannel.publish.call_args[0][1]['value']['test'],\n 'v')\n\n b.emit_withdraw('testi', {'test': 'v'})\n\n self.assertEqual(ochannel.publish.call_count, 2)\n self.assertEqual(ochannel.publish.call_args[0][0],\n 'withdraw')\n self.assertEqual(ochannel.publish.call_args[0][1]['indicator'],\n 'testi')\n self.assertEqual(ochannel.publish.call_args[0][1]['value']['test'],\n 'v')\n\n def test_emit_filtered(self):\n ftname = 'test'\n\n config = {\n 'outfilters': [\n {\n 'name': 'rule1',\n 'conditions': [\n \"direction == 'inbound'\",\n \"type == 'IPv4'\"\n ],\n 'actions': ['accept']\n },\n {\n 'name': 'rule2',\n 'actions': ['drop']\n }\n ]\n }\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n\n b = minemeld.ft.base.BaseFT(ftname, chassis, config)\n\n inputs = []\n output = True\n\n b.connect(inputs, output)\n\n chassis.request_sub_channel.assert_not_called()\n chassis.request_pub_channel.assert_called_once_with(ftname)\n chassis.request_rpc_channel.assert_called_once_with(\n ftname,\n b,\n allowed_methods=[\n 'update',\n 'withdraw',\n 'checkpoint',\n 'get',\n 'get_all',\n 'get_range',\n 'length'\n ]\n )\n\n b.emit_update('testi', {'type': 'IPv6', 'direction': 'inbound'})\n self.assertEqual(ochannel.publish.call_count, 0)\n\n ochannel.publish.reset_mock()\n b.emit_withdraw('testi', {'type': 'IPv6', 'direction': 'inbound'})\n self.assertEqual(ochannel.publish.call_count, 0)\n\n ochannel.publish.reset_mock()\n b.emit_update('testi', {'type': 'IPv4', 'direction': 'inbound'})\n self.assertEqual(ochannel.publish.call_count, 1)\n\n ochannel.publish.reset_mock()\n b.emit_update('testi', {'type': 'IPv4', 'direction': 'outbound'})\n self.assertEqual(ochannel.publish.call_count, 0)\n\n ochannel.publish.reset_mock()\n b.emit_update('testi', {'type': 'IPv6', 'direction': 'inbound'})\n self.assertEqual(ochannel.publish.call_count, 0)\n\n ochannel.publish.reset_mock()\n b.emit_update('testi', {'type': 'IPv6', 'direction': 'outbound'})\n self.assertEqual(ochannel.publish.call_count, 0)\n\n def test_full_trace(self):\n config = {}\n chassis = mock.Mock()\n\n bcls = minemeld.ft.base.BaseFT\n\n b = bcls('test', chassis, config)\n\n b._disable_full_trace = True\n b.enable_full_trace(timeout=1)\n self.assertEqual(b._disable_full_trace, False)\n gevent.sleep(1.5)\n self.assertEqual(b._disable_full_trace, True)\n" }, { "path": "tests/test_ft_basepoller.py", "content": "# -*- coding: utf-8 -*-\n# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT basepoller tests\n\nUnit tests for minemeld.ft.basepoller\n\"\"\"\n\nimport gevent.hub\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport unittest\nimport mock\nimport time\nimport shutil\nimport logging\nimport gc\n\nimport minemeld.ft.basepoller\nimport minemeld.ft.table\n\nFTNAME = 'testft-%d' % int(time.time())\n\nLOG = logging.getLogger(__name__)\n\nCUR_LOGICAL_TIME = 0\n\n\ndef logical_millisec(*args):\n return CUR_LOGICAL_TIME*1000\n\n\ndef gevent_event_mock_factory():\n result = mock.Mock()\n result.wait.side_effect = gevent.GreenletExit()\n\n return result\n\n\nclass DeltaFeed(minemeld.ft.basepoller.BasePollerFT):\n def __init__(self, name, chassis):\n config = {\n 'age_out': {\n 'default': 'last_seen+4',\n 'sudden_death': False\n }\n }\n super(DeltaFeed, self).__init__(name, chassis, config)\n\n self.cur_iterator = 0\n\n self.iterators = [\n ['A', 'B', 'C'],\n ['D', 'E', 'F']\n ]\n\n def _build_iterator(self, now):\n r = []\n if self.cur_iterator < len(self.iterators):\n r = self.iterators[self.cur_iterator]\n\n self.cur_iterator += 1\n\n return r\n\n def _process_item(self, item):\n return [[item, {'type': 'IPv4'}]]\n\n\nclass RollingFeed(minemeld.ft.basepoller.BasePollerFT):\n def __init__(self, name, chassis):\n config = {\n 'age_out': {\n 'default': 'last_seen+4',\n 'sudden_death': True\n }\n }\n super(RollingFeed, self).__init__(name, chassis, config)\n\n self.cur_iterator = 0\n\n self.iterators = [\n ['A', 'B', 'C'],\n ['B', 'C', 'D']\n ]\n\n def _build_iterator(self, now):\n r = []\n if self.cur_iterator < len(self.iterators):\n r = self.iterators[self.cur_iterator]\n\n self.cur_iterator += 1\n\n return r\n\n def _process_item(self, item):\n return [[item, {'type': 'IPv4'}]]\n\n\nclass RollingFeedFirst(minemeld.ft.basepoller.BasePollerFT):\n def __init__(self, name, chassis):\n config = {\n 'age_out': {\n 'default': 'first_seen+1',\n 'sudden_death': True\n }\n }\n super(RollingFeedFirst, self).__init__(name, chassis, config)\n\n self.cur_iterator = 0\n\n self.iterators = [\n ['A', 'B', 'C'],\n ['B', 'C', 'D'],\n ['B', 'E', 'F'],\n ['E', 'F', 'G'],\n ['B', 'F', 'G']\n ]\n\n def _build_iterator(self, now):\n r = []\n if self.cur_iterator < len(self.iterators):\n r = self.iterators[self.cur_iterator]\n\n self.cur_iterator += 1\n\n return r\n\n def _process_item(self, item):\n return [[item, {'type': 'IPv4'}]]\n\n\nclass RollingFeedFirst2(minemeld.ft.basepoller.BasePollerFT):\n def __init__(self, name, chassis):\n config = {\n 'age_out': {\n 'default': 'first_seen+1',\n 'sudden_death': True\n }\n }\n super(RollingFeedFirst2, self).__init__(name, chassis, config)\n\n self.cur_iterator = 0\n\n self.iterators = [\n ['A'],\n ['A'],\n ['A'],\n [],\n ['A']\n ]\n\n def _build_iterator(self, now):\n r = []\n if self.cur_iterator < len(self.iterators):\n r = self.iterators[self.cur_iterator]\n\n self.cur_iterator += 1\n\n return r\n\n def _process_item(self, item):\n return [[item, {'type': 'IPv4'}]]\n\n\nclass PermanentFeed(minemeld.ft.basepoller.BasePollerFT):\n def __init__(self, name, chassis):\n config = {\n 'age_out': {\n 'default': None,\n 'sudden_death': True\n }\n }\n super(PermanentFeed, self).__init__(name, chassis, config)\n\n self.cur_iterator = 0\n\n self.iterators = [\n ['A', 'B', 'C'],\n ['B', 'C', 'D']\n ]\n\n def _build_iterator(self, now):\n r = []\n if self.cur_iterator < len(self.iterators):\n r = self.iterators[self.cur_iterator]\n\n self.cur_iterator += 1\n\n return r\n\n def _process_item(self, item):\n return [[item, {'type': 'IPv4'}]]\n\n\nclass SuperPermanentFeed(minemeld.ft.basepoller.BasePollerFT):\n def __init__(self, name, chassis):\n config = {\n 'age_out': {\n 'interval': None,\n 'default': None,\n 'sudden_death': True\n }\n }\n super(SuperPermanentFeed, self).__init__(name, chassis, config)\n\n self.cur_iterator = 0\n\n self.iterators = [\n ['A', 'B', 'C'],\n ['B', 'C', 'D']\n ]\n\n def _build_iterator(self, now):\n r = []\n if self.cur_iterator < len(self.iterators):\n r = self.iterators[self.cur_iterator]\n\n self.cur_iterator += 1\n\n return r\n\n def _process_item(self, item):\n return [[item, {'type': 'IPv4'}]]\n\n\nclass PermanentFeedWithType(minemeld.ft.basepoller.BasePollerFT):\n def __init__(self, name, chassis):\n config = {\n 'multiple_indicator_types': True,\n 'age_out': {\n 'default': None,\n 'sudden_death': True\n }\n }\n super(PermanentFeedWithType, self).__init__(name, chassis, config)\n\n self.cur_iterator = 0\n\n self.iterators = [\n ['IPv4@A', 'domain@B', 'domain@C'],\n ['IPv4@B', 'domain@C', 'IPv4@D']\n ]\n\n def _build_iterator(self, now):\n r = []\n if self.cur_iterator < len(self.iterators):\n r = self.iterators[self.cur_iterator]\n\n self.cur_iterator += 1\n\n return r\n\n def _process_item(self, item):\n it, i = item.split('@', 1)\n return [[i, {'type': it}]]\n\n\nclass PermanentFeedWithTypeAggregated(minemeld.ft.basepoller.BasePollerFT):\n def __init__(self, name, chassis):\n config = {\n 'multiple_indicator_types': True,\n 'aggregate_indicators': True,\n 'age_out': {\n 'default': None,\n 'sudden_death': True\n }\n }\n super(PermanentFeedWithTypeAggregated, self).__init__(name, chassis, config)\n\n self.cur_iterator = 0\n\n self.iterators = [\n ['IPv4@A@1', 'domain@B@1', 'domain@C@1', 'IPv4@A@2'],\n ['IPv4@B@1', 'domain@C@1', 'IPv4@D@1', 'IPv4@D@2']\n ]\n\n def _build_iterator(self, now):\n r = []\n if self.cur_iterator < len(self.iterators):\n r = self.iterators[self.cur_iterator]\n\n self.cur_iterator += 1\n\n return r\n\n def _process_item(self, item):\n it, i, v = item.split('@', 2)\n return [[i, {'type': it, 'attribute': v}]]\n\n\nclass DeltaFeedWithTypeAggregatedFaulty(minemeld.ft.basepoller.BasePollerFT):\n def __init__(self, name, chassis):\n config = {\n 'multiple_indicator_types': True,\n 'aggregate_indicators': True,\n 'aggregate_use_partial': True,\n 'age_out': {\n 'default': 'last_seen+4',\n 'sudden_death': False\n }\n }\n super(DeltaFeedWithTypeAggregatedFaulty, self).__init__(name, chassis, config)\n\n self.cur_iterator = 0\n\n self.cur_iterator = 1\n self.last_time = None\n\n self.iterators = [\n ['IPv4@A@1', 'domain@B@1', 'domain@C@1', 'IPv4@A@2'],\n ['IPv4@B@1', 'domain@C@1', 'IPv4@D@1', 'IPv4@D@2']\n ]\n\n def _iterator(self, iterator):\n for i in iterator:\n yield i\n raise RuntimeError('BAM !')\n\n def _build_iterator(self, now):\n if self.last_time is None:\n self.last_time = CUR_LOGICAL_TIME\n\n if self.last_time == CUR_LOGICAL_TIME:\n self.cur_iterator -= 1\n\n self.last_time = CUR_LOGICAL_TIME\n\n LOG.info('cur_iterator: {} time: {}'.format(self.cur_iterator, CUR_LOGICAL_TIME))\n r = []\n if self.cur_iterator < len(self.iterators):\n r = self._iterator(self.iterators[self.cur_iterator])\n\n self.cur_iterator += 1\n\n return r\n\n def _process_item(self, item):\n it, i, v = item.split('@', 2)\n return [[i, {'type': it, 'attribute': v}]]\n\n\nclass MineMeldFTBasePollerTests(unittest.TestCase):\n def setUp(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n def tearDown(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_delta_feed(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock):\n global CUR_LOGICAL_TIME\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = DeltaFeed(FTNAME, chassis)\n\n inputs = []\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 2)\n self.assertEqual(spawn_mock.call_count, 3)\n\n CUR_LOGICAL_TIME = 1\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n self.assertEqual(um_mock.call_count, 1)\n\n CUR_LOGICAL_TIME = 2\n a._poll()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 3)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n\n CUR_LOGICAL_TIME = 3\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n\n CUR_LOGICAL_TIME = 4\n a._poll()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 6)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 0)\n\n CUR_LOGICAL_TIME = 5\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n\n CUR_LOGICAL_TIME = 6\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n\n CUR_LOGICAL_TIME = 7\n a._age_out()\n self.assertEqual(a.statistics['aged_out'], 3)\n\n CUR_LOGICAL_TIME = 8\n a._poll()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 6)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 3)\n self.assertEqual(a.length(), 3)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_rolling_feed(self, um_mock, event_mock, sleep_mock,\n spawnl_mock, spawn_mock):\n global CUR_LOGICAL_TIME\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = RollingFeed(FTNAME, chassis)\n\n inputs = []\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 2)\n self.assertEqual(spawn_mock.call_count, 3)\n\n CUR_LOGICAL_TIME = 1\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n self.assertEqual(um_mock.call_count, 1)\n\n CUR_LOGICAL_TIME = 2\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 3)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n\n CUR_LOGICAL_TIME = 3\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n\n CUR_LOGICAL_TIME = 4\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 4)\n self.assertEqual(a.statistics.get('removed', 0), 1)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 1)\n\n CUR_LOGICAL_TIME = 5\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 1)\n\n CUR_LOGICAL_TIME = 6\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 1)\n\n CUR_LOGICAL_TIME = 7\n a._age_out()\n self.assertEqual(a.statistics['aged_out'], 3)\n\n CUR_LOGICAL_TIME = 8\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 4)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 4)\n self.assertEqual(a.length(), 0)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_rolling_feed_first(self, um_mock, event_mock, sleep_mock,\n spawnl_mock, spawn_mock):\n global CUR_LOGICAL_TIME\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = RollingFeedFirst(FTNAME, chassis)\n\n inputs = []\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 2)\n self.assertEqual(spawn_mock.call_count, 3)\n\n CUR_LOGICAL_TIME = 1\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n self.assertEqual(um_mock.call_count, 1)\n\n CUR_LOGICAL_TIME = 2\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 3)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n\n CUR_LOGICAL_TIME = 3\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 4)\n self.assertEqual(a.statistics.get('removed', 0), 1)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 1)\n self.assertEqual(a.statistics['aged_out'], 1)\n\n CUR_LOGICAL_TIME = 4\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 6)\n self.assertEqual(a.statistics.get('removed', 0), 3)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 3)\n self.assertEqual(a.statistics['aged_out'], 4)\n\n CUR_LOGICAL_TIME = 5\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 7)\n self.assertEqual(a.statistics.get('removed', 0), 4)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 4)\n self.assertEqual(a.statistics['aged_out'], 4)\n\n CUR_LOGICAL_TIME = 6\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 8)\n self.assertEqual(a.statistics.get('removed', 0), 5)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 5)\n self.assertEqual(a.statistics['aged_out'], 6)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_rolling_feed_first2(self, um_mock, event_mock, sleep_mock,\n spawnl_mock, spawn_mock):\n global CUR_LOGICAL_TIME\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = RollingFeedFirst2(FTNAME, chassis)\n\n inputs = []\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 2)\n self.assertEqual(spawn_mock.call_count, 3)\n\n CUR_LOGICAL_TIME = 1\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n self.assertEqual(um_mock.call_count, 1)\n\n CUR_LOGICAL_TIME = 2\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 1)\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 0)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n\n CUR_LOGICAL_TIME = 3\n a._age_out()\n self.assertEqual(a.statistics['added'], 1)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 0)\n self.assertEqual(a.statistics['aged_out'], 0)\n\n CUR_LOGICAL_TIME = 4\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 1)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 0)\n self.assertEqual(a.statistics['aged_out'], 1)\n\n CUR_LOGICAL_TIME = 5\n a._age_out()\n self.assertEqual(a.statistics['added'], 1)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 0)\n self.assertEqual(a.statistics['aged_out'], 1)\n\n CUR_LOGICAL_TIME = 6\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 1)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 0)\n self.assertEqual(a.statistics['aged_out'], 1)\n\n CUR_LOGICAL_TIME = 7\n a._age_out()\n self.assertEqual(a.statistics['added'], 1)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 0)\n self.assertEqual(a.statistics['aged_out'], 1)\n\n CUR_LOGICAL_TIME = 8\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 1)\n self.assertEqual(a.statistics['removed'], 1)\n self.assertEqual(a.statistics['garbage_collected'], 1)\n self.assertEqual(a.statistics['aged_out'], 1)\n\n CUR_LOGICAL_TIME = 9\n a._age_out()\n self.assertEqual(a.statistics['added'], 1)\n self.assertEqual(a.statistics['removed'], 1)\n self.assertEqual(a.statistics['garbage_collected'], 1)\n self.assertEqual(a.statistics['aged_out'], 1)\n\n CUR_LOGICAL_TIME = 10\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 2)\n self.assertEqual(a.statistics['removed'], 1)\n self.assertEqual(a.statistics['garbage_collected'], 1)\n self.assertEqual(a.statistics['aged_out'], 1)\n\n CUR_LOGICAL_TIME = 11\n a._age_out()\n self.assertEqual(a.statistics['added'], 2)\n self.assertEqual(a.statistics['removed'], 1)\n self.assertEqual(a.statistics['garbage_collected'], 1)\n self.assertEqual(a.statistics['aged_out'], 1)\n\n CUR_LOGICAL_TIME = 12\n a._age_out()\n self.assertEqual(a.statistics['added'], 2)\n self.assertEqual(a.statistics['removed'], 1)\n self.assertEqual(a.statistics['garbage_collected'], 1)\n self.assertEqual(a.statistics['aged_out'], 2)\n self.assertEqual(a.statistics['withdraw.tx'], 2)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_permanent_feed(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n global CUR_LOGICAL_TIME\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = PermanentFeed(FTNAME, chassis)\n\n inputs = []\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 2)\n self.assertEqual(spawn_mock.call_count, 3)\n\n CUR_LOGICAL_TIME = 1\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n self.assertEqual(um_mock.call_count, 1)\n\n CUR_LOGICAL_TIME = 2\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 3)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n\n CUR_LOGICAL_TIME = 3\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n\n CUR_LOGICAL_TIME = 4\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 4)\n self.assertEqual(a.statistics.get('removed', 0), 1)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 1)\n\n CUR_LOGICAL_TIME = 5\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 1)\n\n CUR_LOGICAL_TIME = 6\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 1)\n\n CUR_LOGICAL_TIME = 7\n a._age_out()\n self.assertEqual(a.statistics['aged_out'], 1)\n\n CUR_LOGICAL_TIME = 8\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 4)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 4)\n self.assertEqual(a.length(), 0)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_superpermanent_feed(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n global CUR_LOGICAL_TIME\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = PermanentFeed(FTNAME, chassis)\n\n inputs = []\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 2)\n self.assertEqual(spawn_mock.call_count, 3)\n\n CUR_LOGICAL_TIME = 1\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n self.assertEqual(um_mock.call_count, 1)\n\n CUR_LOGICAL_TIME = 2\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 3)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n\n CUR_LOGICAL_TIME = 4\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 4)\n self.assertEqual(a.statistics.get('removed', 0), 1)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 1)\n self.assertEqual(a.statistics.get('aged_out', 0), 1)\n\n CUR_LOGICAL_TIME = 8\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 4)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 4)\n self.assertEqual(a.length(), 0)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_drop_old_ops(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock):\n global CUR_LOGICAL_TIME\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = DeltaFeed(FTNAME, chassis)\n\n inputs = []\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 2)\n self.assertEqual(spawn_mock.call_count, 3)\n\n a._actor_queue.put((0, 'age_out'))\n a._actor_queue.put((999, 'age_out'))\n\n CUR_LOGICAL_TIME = 1\n try:\n a._actor_loop()\n except gevent.hub.LoopExit:\n pass\n self.assertEqual(a.last_ageout_run, 1000)\n self.assertEqual(um_mock.call_count, 2)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_permanentwithtype_feed(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n global CUR_LOGICAL_TIME\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = PermanentFeedWithType(FTNAME, chassis)\n\n inputs = []\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 2)\n self.assertEqual(spawn_mock.call_count, 3)\n\n CUR_LOGICAL_TIME = 1\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n self.assertEqual(um_mock.call_count, 1)\n\n CUR_LOGICAL_TIME = 2\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 3)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n\n CUR_LOGICAL_TIME = 3\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n\n CUR_LOGICAL_TIME = 4\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 5)\n self.assertEqual(a.statistics.get('removed', 0), 2)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 2)\n\n CUR_LOGICAL_TIME = 5\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 2)\n\n CUR_LOGICAL_TIME = 6\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 2)\n\n CUR_LOGICAL_TIME = 7\n a._age_out()\n self.assertEqual(a.statistics['aged_out'], 2)\n\n CUR_LOGICAL_TIME = 8\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 5)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 5)\n self.assertEqual(a.length(), 0)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_bptable_1(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n t = minemeld.ft.table.Table(FTNAME, truncate=True)\n bpt0 = minemeld.ft.basepoller._BPTable_v0(t)\n\n bpt0.put('A', {'v': 1})\n A = bpt0.get('A')\n self.assertEqual(A['v'], 1)\n\n A, V = next(bpt0.query(include_value=True))\n self.assertEqual(V['v'], 1)\n\n bpt0.delete('A')\n A = bpt0.get('A')\n self.assertEqual(A, None)\n\n bpt0.close()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_bptable_2(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n t = minemeld.ft.table.Table(FTNAME, truncate=True)\n bpt1 = minemeld.ft.basepoller._BPTable_v1(t, type_in_key=True)\n\n bpt1.put('A', {'type': 1})\n\n A = next(bpt1.query(include_value=False))\n self.assertEqual(A, 'A')\n\n bpt1.close()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_bptable_3(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n t = minemeld.ft.table.Table(FTNAME, truncate=True)\n bpt1 = minemeld.ft.basepoller._BPTable_v1(t, type_in_key=True)\n bpt1.close()\n\n with self.assertRaises(RuntimeError):\n t = minemeld.ft.table.Table(FTNAME, truncate=False)\n minemeld.ft.basepoller._BPTable_v1(t, type_in_key=False)\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_bptable_4(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n t = minemeld.ft.table.Table(FTNAME, truncate=True)\n bpt1 = minemeld.ft.basepoller._BPTable_v1(t, type_in_key=True)\n\n with self.assertRaises(RuntimeError):\n bpt1.put('A', {'a': 1})\n\n bpt1.close()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_bptable_5(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n t = minemeld.ft.table.Table(FTNAME, truncate=True)\n bpt1 = minemeld.ft.basepoller._BPTable_v1(t, type_in_key=True)\n bpt1.close()\n\n bpt1 = minemeld.ft.basepoller._bptable_factory(FTNAME, truncate=False, type_in_key=True)\n bpt1.close()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_bptable_6(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n t = minemeld.ft.table.Table(FTNAME, truncate=True)\n bpt0 = minemeld.ft.basepoller._BPTable_v0(t)\n bpt0.put('A', {'v': 1})\n bpt0.close()\n\n bpt1 = minemeld.ft.basepoller._bptable_factory(FTNAME, truncate=False, type_in_key=False)\n bpt1.close()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_bptable_7(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n t = minemeld.ft.table.Table(FTNAME, truncate=True)\n bpt0 = minemeld.ft.basepoller._BPTable_v0(t)\n bpt0.put('A', {'v': 1})\n bpt0.delete('A')\n bpt0.close()\n\n bpt1 = minemeld.ft.basepoller._bptable_factory(FTNAME, truncate=False, type_in_key=True)\n bpt1.close()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_bptable_8(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n t = minemeld.ft.table.Table(FTNAME, truncate=True)\n bpt1 = minemeld.ft.basepoller._BPTable_v1(t, type_in_key=True)\n bpt1.put(indicator=u'☃.net/påth', value={u'☃.net/påth': 1, 'type': u'☃.net/påth'})\n t = bpt1.get(u'☃.net/påth', itype=u'☃.net/påth')\n self.assertNotEqual(t, None)\n\n k, v = next(bpt1.query(include_value=True))\n self.assertEqual(k, u'☃.net/påth')\n self.assertEqual(v, {u'☃.net/påth': 1, 'type': u'☃.net/påth'})\n\n bpt1.close()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_permanentwithtype_feed_agg(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n global CUR_LOGICAL_TIME\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = PermanentFeedWithTypeAggregated(FTNAME, chassis)\n\n inputs = []\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 2)\n self.assertEqual(spawn_mock.call_count, 3)\n\n CUR_LOGICAL_TIME = 1\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n self.assertEqual(um_mock.call_count, 1)\n\n CUR_LOGICAL_TIME = 2\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 3)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n\n CUR_LOGICAL_TIME = 3\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n\n CUR_LOGICAL_TIME = 4\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 5)\n self.assertEqual(a.statistics.get('removed', 0), 2)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 2)\n\n CUR_LOGICAL_TIME = 5\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 2)\n\n CUR_LOGICAL_TIME = 6\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 2)\n\n CUR_LOGICAL_TIME = 7\n a._age_out()\n self.assertEqual(a.statistics['aged_out'], 2)\n\n CUR_LOGICAL_TIME = 8\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 5)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 5)\n self.assertEqual(a.length(), 0)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep')\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_permanentwithtype_feed_agg2(self, um_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n global CUR_LOGICAL_TIME\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = DeltaFeedWithTypeAggregatedFaulty(FTNAME, chassis)\n\n inputs = []\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 2)\n self.assertEqual(spawn_mock.call_count, 3)\n\n CUR_LOGICAL_TIME = 1\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n self.assertEqual(um_mock.call_count, 1)\n\n CUR_LOGICAL_TIME = 2\n a._poll()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 3)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n\n CUR_LOGICAL_TIME = 3\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n\n CUR_LOGICAL_TIME = 4\n a._poll()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 5)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 0)\n\n CUR_LOGICAL_TIME = 5\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n\n CUR_LOGICAL_TIME = 6\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n\n CUR_LOGICAL_TIME = 7\n a._age_out()\n self.assertEqual(a.statistics['aged_out'], 3)\n\n CUR_LOGICAL_TIME = 8\n a._poll()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 5)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 3)\n self.assertEqual(a.length(), 2)\n\n CUR_LOGICAL_TIME = 9\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['aged_out'], 5)\n self.assertEqual(a.length(), 0)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n" }, { "path": "tests/test_ft_boolexpr.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT Table tests\n\nUnit tests for minemeld.ft.boolexpr\n\"\"\"\n\nimport unittest\nimport jmespath\nimport operator\nimport logging\n\nimport antlr4\n\nimport minemeld.ft.condition\n\n\nLOG = logging.getLogger(__name__)\n\n\nclass ExprBuilder(minemeld.ft.condition.BoolExprListener):\n def __init__(self):\n self.expression = None\n self.comparator = None\n self.value = None\n\n def exitExpression(self, ctx):\n self.expression = ctx.getText()\n\n def exitComparator(self, ctx):\n comparator = ctx.getText()\n if comparator == '==':\n self.comparator = operator.eq\n elif comparator == '<':\n self.comparator = operator.lt\n elif comparator == '<=':\n self.comparator = operator.le\n elif comparator == '>':\n self.comparator = operator.gt\n elif comparator == '>=':\n self.comparator = operator.ge\n elif comparator == '!=':\n self.comparator = operator.ne\n\n def exitValue(self, ctx):\n if ctx.STRING() is not None:\n self.value = ctx.STRING().getText()[1:-1]\n elif ctx.NUMBER() is not None:\n self.value = int(ctx.NUMBER().getText())\n elif ctx.getText() == 'null':\n self.value = None\n elif ctx.getText() == 'false':\n self.value = False\n elif ctx.getText() == 'true':\n self.value = True\n\n\ndef _parse_string(s):\n lexer = minemeld.ft.condition.BoolExprLexer(\n antlr4.InputStream(s)\n )\n stream = antlr4.CommonTokenStream(lexer)\n parser = minemeld.ft.condition.BoolExprParser(stream)\n tree = parser.booleanExpression()\n\n eb = ExprBuilder()\n walker = antlr4.ParseTreeWalker()\n walker.walk(eb, tree)\n\n return eb\n\n\ndef _eval_expression(eb, i):\n ce = jmespath.compile(eb.expression)\n\n try:\n r = ce.search(i)\n except jmespath.exceptions.JMESPathError:\n LOG.exception(\"Exception in searching\")\n r = None\n\n LOG.info(\"r: %s value: %s\", r, eb.value)\n\n return eb.comparator(r, eb.value)\n\n\nclass MineMeldFTBaseTests(unittest.TestCase):\n def test_simple(self):\n eb = _parse_string('sources == \"http://dshield.org/blocklist\"')\n self.assertEqual(eb.expression, u'sources')\n self.assertEqual(eb.comparator, operator.eq)\n self.assertEqual(eb.value, 'http://dshield.org/blocklist')\n\n def test_func(self):\n eb = _parse_string('length(max(dshield_nattacks)) == '\n '\"http://dshield.org/blocklist\"')\n self.assertEqual(eb.expression, u'length(max(dshield_nattacks))')\n self.assertEqual(eb.comparator, operator.eq)\n self.assertEqual(eb.value, 'http://dshield.org/blocklist')\n\n def test_eval(self):\n i = {\n 'sources': [1, 2],\n 'type': 'IPv4'\n }\n\n c = minemeld.ft.condition.Condition('length(sources) > 1')\n self.assertTrue(c.eval(i))\n\n c = minemeld.ft.condition.Condition('length(b) > 1')\n self.assertFalse(c.eval(i))\n\n c = minemeld.ft.condition.Condition('type(b) == null')\n self.assertTrue(c.eval(i))\n\n c = minemeld.ft.condition.Condition('length(b) == null')\n self.assertTrue(c.eval(i))\n\n c = minemeld.ft.condition.Condition(\"starts_with(type, 'IP') \"\n \"== true\")\n self.assertTrue(c.eval(i))\n\n c = minemeld.ft.condition.Condition(\"type == 'IPv4'\")\n self.assertTrue(c.eval(i))\n" }, { "path": "tests/test_ft_dag.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT dag tests\n\nUnit tests for minemeld.ft.dag\n\"\"\"\n\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport unittest\nimport mock\nimport time\nimport shutil\nimport logging\nimport gc\nimport calendar\nimport os\nimport yaml\nimport functools\nimport pan.xapi\nimport panos_mock\n\nimport minemeld.ft.dag\n\nFTNAME = 'testft-%d' % int(time.time())\nDLIST_NAME = 'dag-dlist-%d.yml' % int(time.time())\n\nLOG = logging.getLogger(__name__)\n\nCUR_LOGICAL_TIME = 0\n\nMYDIR = os.path.dirname(__file__)\n\nGEVENT_SLEEP = gevent.sleep\n\n\ndef logical_millisec(*args):\n return CUR_LOGICAL_TIME\n\n\ndef gevent_event_mock_factory():\n result = mock.Mock()\n result.wait.side_effect = gevent.GreenletExit()\n\n return result\n\n\ndef device_pusher_mock_factory(device, prefix, watermark, attributes, persistence):\n def _start_se(x):\n x.started = True\n\n result = mock.MagicMock(started=False, device=device, value=None)\n result.start = mock.Mock(side_effect=functools.partial(_start_se, result))\n\n return result\n\n\nclass MineMeldFTDagPusherTests(unittest.TestCase):\n def setUp(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n try:\n os.remove(DLIST_NAME)\n except:\n pass\n\n def tearDown(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n try:\n os.remove(DLIST_NAME)\n except:\n pass\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch.object(minemeld.ft.dag.DagPusher, '_huppable_wait', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch.object(calendar, 'timegm', side_effect=logical_millisec)\n @mock.patch('minemeld.ft.dag.DevicePusher',\n side_effect=device_pusher_mock_factory)\n def test_device_list_load(self, dp_mock, timegm_mock, event_mock, hw_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n device_list_path = os.path.join(MYDIR, 'test_device_list.yml')\n device_list_path2 = os.path.join(MYDIR, 'test_device_list2.yml')\n\n with open(device_list_path, 'r') as f:\n dlist = yaml.safe_load(f)\n\n with open(device_list_path2, 'r') as f:\n dlist2 = yaml.safe_load(f)\n\n shutil.copyfile(device_list_path, DLIST_NAME)\n\n config = {\n 'device_list': DLIST_NAME\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.dag.DagPusher(FTNAME, chassis, config)\n\n inputs = []\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 1)\n self.assertEqual(spawn_mock.call_count, 1)\n\n # 1st round\n try:\n a._device_list_monitor()\n except gevent.GreenletExit:\n pass\n hw_mock.assert_called_with(5)\n self.assertEqual(len(a.devices), len(dlist))\n self.assertEqual(len(a.device_pushers), len(dlist))\n self.assertEqual(dp_mock.call_count, len(dlist))\n\n for i, d in enumerate(dlist):\n self.assertEqual(a.devices[i], d)\n self.assertEqual(a.device_pushers[i].start.call_count, 1)\n self.assertEqual(a.device_pushers[i].device, d)\n\n # 2nd round\n GEVENT_SLEEP(1)\n shutil.copyfile(device_list_path, DLIST_NAME)\n\n hw_mock.reset_mock()\n dp_mock.reset_mock()\n\n try:\n a._device_list_monitor()\n except gevent.GreenletExit:\n pass\n hw_mock.assert_called_with(5)\n self.assertEqual(len(a.devices), len(dlist))\n self.assertEqual(len(a.device_pushers), len(dlist))\n self.assertEqual(dp_mock.call_count, 0)\n\n for i, d in enumerate(dlist):\n self.assertEqual(a.devices[i], d)\n self.assertEqual(a.device_pushers[i].start.call_count, 1)\n self.assertEqual(a.device_pushers[i].device, d)\n\n # 3rd round\n GEVENT_SLEEP(1)\n shutil.copyfile(device_list_path2, DLIST_NAME)\n\n hw_mock.reset_mock()\n dp_mock.reset_mock()\n\n try:\n a._device_list_monitor()\n except gevent.GreenletExit:\n pass\n hw_mock.assert_called_with(5)\n self.assertEqual(len(a.devices), len(dlist2))\n self.assertEqual(len(a.device_pushers), len(dlist2))\n self.assertEqual(dp_mock.call_count, 1)\n\n for i, d in enumerate(dlist2):\n self.assertEqual(a.devices[i], d)\n self.assertEqual(a.device_pushers[i].start.call_count, 1)\n self.assertEqual(a.device_pushers[i].device, d)\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch.object(calendar, 'timegm', side_effect=logical_millisec)\n @mock.patch('minemeld.ft.dag.DevicePusher',\n side_effect=device_pusher_mock_factory)\n def test_uw(self, dp_mock, timegm_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n device_list_path = os.path.join(MYDIR, 'test_device_list.yml')\n\n shutil.copyfile(device_list_path, DLIST_NAME)\n\n config = {\n 'device_list': DLIST_NAME\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.dag.DagPusher(FTNAME, chassis, config)\n\n inputs = ['a']\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 1)\n self.assertEqual(spawn_mock.call_count, 1)\n\n try:\n a._device_list_monitor()\n except gevent.GreenletExit:\n pass\n\n a.update('a', indicator='127.0.0.1', value={\n 'type': 'IPv4',\n 'confidence': 100\n })\n\n for d in a.device_pushers:\n d.put.assert_called_with(\n 'register',\n '127.0.0.1',\n {\n 'type': 'IPv4',\n 'confidence': 100\n }\n )\n\n for d in a.device_pushers:\n d.put.reset_mock()\n\n a.withdraw('a', indicator='127.0.0.1')\n for d in a.device_pushers:\n d.put.assert_called_with(\n 'unregister',\n '127.0.0.1',\n {\n 'type': 'IPv4',\n 'confidence': 100\n }\n )\n\n for d in a.device_pushers:\n d.put.reset_mock()\n\n a.withdraw('a', indicator='127.0.0.1')\n for d in a.device_pushers:\n self.assertEqual(d.put.call_count, 0)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch.object(calendar, 'timegm', side_effect=logical_millisec)\n @mock.patch('minemeld.ft.dag.DevicePusher',\n side_effect=device_pusher_mock_factory)\n def test_uinvalid(self, dp_mock, timegm_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n device_list_path = os.path.join(MYDIR, 'test_device_list.yml')\n\n shutil.copyfile(device_list_path, DLIST_NAME)\n\n config = {\n 'device_list': DLIST_NAME\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.dag.DagPusher(FTNAME, chassis, config)\n\n inputs = ['a']\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 1)\n self.assertEqual(spawn_mock.call_count, 1)\n\n try:\n a._device_list_monitor()\n except gevent.GreenletExit:\n pass\n\n a.update('a', indicator='1.1.1.1-1.1.1.3', value={\n 'type': 'IPv4',\n 'confidence': 100\n })\n\n self.assertEqual(a.length(), 0)\n\n a.update('a', indicator='1.1.1.0/24', value={\n 'type': 'IPv4',\n 'confidence': 100\n })\n\n self.assertEqual(a.length(), 0)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch.object(calendar, 'timegm', side_effect=logical_millisec)\n @mock.patch('minemeld.ft.dag.DevicePusher',\n side_effect=device_pusher_mock_factory)\n def test_unicast1(self, dp_mock, timegm_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n device_list_path = os.path.join(MYDIR, 'test_device_list.yml')\n\n shutil.copyfile(device_list_path, DLIST_NAME)\n\n config = {\n 'device_list': DLIST_NAME\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.dag.DagPusher(FTNAME, chassis, config)\n\n inputs = ['a']\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 1)\n self.assertEqual(spawn_mock.call_count, 1)\n\n try:\n a._device_list_monitor()\n except gevent.GreenletExit:\n pass\n\n a.update('a', indicator='1.1.1.1-1.1.1.1', value={\n 'type': 'IPv4',\n 'confidence': 100\n })\n\n for d in a.device_pushers:\n d.put.assert_called_with(\n 'register',\n '1.1.1.1',\n {\n 'type': 'IPv4',\n 'confidence': 100\n }\n )\n\n for d in a.device_pushers:\n d.put.reset_mock()\n\n a.withdraw('a', indicator='1.1.1.1-1.1.1.1')\n for d in a.device_pushers:\n d.put.assert_called_with(\n 'unregister',\n '1.1.1.1',\n {\n 'type': 'IPv4',\n 'confidence': 100\n }\n )\n\n for d in a.device_pushers:\n d.put.reset_mock()\n\n a.withdraw('a', indicator='1.1.1.1-1.1.1.1')\n for d in a.device_pushers:\n self.assertEqual(d.put.call_count, 0)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch.object(calendar, 'timegm', side_effect=logical_millisec)\n @mock.patch('minemeld.ft.dag.DevicePusher',\n side_effect=device_pusher_mock_factory)\n def test_unicast2(self, dp_mock, timegm_mock, event_mock,\n sleep_mock, spawnl_mock, spawn_mock):\n device_list_path = os.path.join(MYDIR, 'test_device_list.yml')\n\n shutil.copyfile(device_list_path, DLIST_NAME)\n\n config = {\n 'device_list': DLIST_NAME\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.dag.DagPusher(FTNAME, chassis, config)\n\n inputs = ['a']\n output = False\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 1)\n self.assertEqual(spawn_mock.call_count, 1)\n\n try:\n a._device_list_monitor()\n except gevent.GreenletExit:\n pass\n\n a.update('a', indicator='1.1.1.1/32', value={\n 'type': 'IPv4',\n 'confidence': 100\n })\n\n for d in a.device_pushers:\n d.put.assert_called_with(\n 'register',\n '1.1.1.1',\n {\n 'type': 'IPv4',\n 'confidence': 100\n }\n )\n\n for d in a.device_pushers:\n d.put.reset_mock()\n\n a.withdraw('a', indicator='1.1.1.1/32')\n for d in a.device_pushers:\n d.put.assert_called_with(\n 'unregister',\n '1.1.1.1',\n {\n 'type': 'IPv4',\n 'confidence': 100\n }\n )\n\n for d in a.device_pushers:\n d.put.reset_mock()\n\n a.withdraw('a', indicator='1.1.1.1/32')\n for d in a.device_pushers:\n self.assertEqual(d.put.call_count, 0)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(pan.xapi, 'PanXapi', side_effect=panos_mock.factory)\n def test_devicepusher_dag_message(self, panxapi_mock):\n RESULT_REG = '1.0updateab'\n RESULT_UNREG = '1.0updateab'\n\n dp = minemeld.ft.dag.DevicePusher(\n {'tag': 'test'},\n 'mmeld_',\n 'test',\n [],\n False\n )\n\n reg = dp._dag_message('register', {'192.168.1.1': ['a', 'b']})\n self.assertEqual(reg, RESULT_REG)\n\n unreg = dp._dag_message('unregister', {'192.168.1.1': ['a', 'b']})\n self.assertEqual(unreg, RESULT_UNREG)\n\n @mock.patch.object(pan.xapi, 'PanXapi', side_effect=panos_mock.factory)\n def test_devicepusher_tags_from_value(self, panxapi_mock):\n dp = minemeld.ft.dag.DevicePusher(\n {'tag': 'test'},\n 'mmeld_',\n 'test',\n ['confidence', 'direction'],\n False\n )\n\n tags = dp._tags_from_value({'confidence': 49, 'direction': 'inbound'})\n self.assertEqual(tags, set(['mmeld_confidence_low', 'mmeld_direction_inbound']))\n\n tags = dp._tags_from_value({'confidence': 50})\n self.assertEqual(tags, set(['mmeld_confidence_medium', 'mmeld_direction_unknown']))\n\n tags = dp._tags_from_value({'confidence': 75, 'direction': 'outbound'})\n self.assertEqual(tags, set(['mmeld_confidence_high', 'mmeld_direction_outbound']))\n\n @mock.patch.object(pan.xapi, 'PanXapi', side_effect=panos_mock.factory)\n def test_devicepusher_get_all_registered_ips(self, panxapi_mock):\n dp = minemeld.ft.dag.DevicePusher(\n {'hostname': 'test_ft_dag_devicepusher'},\n 'mmeld_',\n 'test',\n ['confidence', 'direction'],\n False\n )\n\n result = dp._get_all_registered_ips()\n self.assertEqual(next(result), ('192.168.1.1', ['mmeld_test', 'mmeld_confidence_100', 'mmeld_pushed']))\n self.assertEqual(next(result), ('192.168.1.2', ['mmeld_test', 'mmeld_confidence_100']))\n\n @mock.patch.object(pan.xapi, 'PanXapi', side_effect=panos_mock.factory)\n def test_devicepusher_push(self, panxapi_mock):\n dp = minemeld.ft.dag.DevicePusher(\n {'hostname': 'test_ft_dag_devicepusher'},\n 'mmeld_',\n 'test',\n ['confidence', 'direction'],\n False\n )\n\n dp._push('register', '192.168.1.10', {'confidence': 40, 'direction': 'inbound'})\n self.assertEqual(\n dp.xapi.user_id_calls[0],\n '1.0updatemmeld_confidence_lowmmeld_direction_inboundmmeld_test'\n )\n\n @mock.patch.object(pan.xapi, 'PanXapi', side_effect=panos_mock.factory)\n def test_devicepusher_init_resync(self, panxapi_mock):\n dp = minemeld.ft.dag.DevicePusher(\n {'hostname': 'test_ft_dag_devicepusher'},\n 'mmeld_',\n 'test',\n ['confidence', 'direction'],\n False\n )\n\n dp.put('init', '192.168.1.1', {'confidence': 75, 'direction': 'inbound'})\n dp.put('init', '192.168.1.10', {'confidence': 80})\n dp.put('EOI', None, None)\n dp._init_resync()\n\n self.assertEqual(\n dp.xapi.user_id_calls[0],\n '1.0updatemmeld_confidence_highmmeld_direction_inboundmmeld_confidence_highmmeld_direction_unknownmmeld_test'\n )\n self.assertEqual(\n dp.xapi.user_id_calls[1],\n '1.0updatemmeld_confidence_100mmeld_pushedmmeld_confidence_100mmeld_test'\n )\n" }, { "path": "tests/test_ft_dag_devicepusher_op__show__object__registered_ip__ip_192_168_1_1__ip___registered_ip___object___show__0.xml", "content": "\n \n \n \n mmeld_test\n mmeld_confidence_100\n mmeld_pushed\n \n \n 1\n \n\n" }, { "path": "tests/test_ft_dag_devicepusher_op__show__object__registered_ip__ip_192_168_1_2__ip___registered_ip___object___show__0.xml", "content": "\n \n \n \n mmeld_test\n mmeld_confidence_100\n \n \n 1\n \n\n" }, { "path": "tests/test_ft_dag_devicepusher_op__show__object__registered_ip__tag__entry_name__mmeld_test_____tag___registered_ip___object___show__0.xml", "content": "\n \n \n \n mmeld_test\n \n \n \n \n mmeld_test\n \n \n 2\n \n\n" }, { "path": "tests/test_ft_ipop.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT ipop tests\n\nUnit tests for minemeld.ft.ipop\n\"\"\"\n\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport unittest\nimport mock\nimport time\nimport shutil\nimport logging\nimport netaddr\nimport random\n\nimport guppy # noqa\nimport pdb # noqa\nimport gc # noqa\n\nfrom nose.plugins.attrib import attr\n\nimport minemeld.ft.ipop\n\nLOG = logging.getLogger(__name__)\nFTNAME = 'testft-%d' % int(time.time())\n\n\ndef check_for_rpc(call_args_list, check_list, all_here=False):\n LOG.debug(\"call_args_list: %s\", call_args_list)\n\n found = []\n for chk in check_list:\n LOG.debug(\"checking: %s\", chk)\n\n for j in xrange(len(call_args_list)):\n if j in found:\n continue\n\n args = call_args_list[j][0]\n\n if args[0] != chk['method']:\n continue\n if args[1]['indicator'] != chk['indicator']:\n continue\n\n chkvalue = chk.get('value', None)\n if chkvalue is None:\n found.append(j)\n LOG.debug(\"found @%d\", j)\n break\n\n argsvalue = args[1].get('value', None)\n if chkvalue is not None and argsvalue is None:\n continue\n\n failed = False\n for k in chkvalue.keys():\n if k not in argsvalue:\n failed = True\n break\n if chkvalue[k] != argsvalue[k]:\n failed = True\n break\n if failed:\n continue\n\n found.append(j)\n LOG.debug(\"found @%d\", j)\n break\n\n c1 = len(found) == len(check_list)\n\n if not all_here:\n return c1\n\n c2 = len(found) == len(call_args_list)\n\n return c1+c2 == 2\n\n\nclass MineMeldFTIPOpTests(unittest.TestCase):\n def setUp(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n try:\n shutil.rmtree(FTNAME+\"_st\")\n except:\n pass\n\n def tearDown(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n try:\n shutil.rmtree(FTNAME+\"_st\")\n except:\n pass\n\n def test_calc_ipranges(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='192.168.0.1', value={\n 'type': 'IPv4',\n 's1$a': 1,\n 'sources': ['s1s']\n })\n self.assertEqual(ochannel.publish.call_count, 1)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'update')\n self.assertEqual(pargs[1]['indicator'], '192.168.0.1-192.168.0.1')\n\n ochannel.publish.reset_mock()\n a.filtered_update('s1', indicator='192.168.0.1-192.168.0.3', value={\n 'type': 'IPv4',\n 's1$b': 1,\n 'sources': ['s1s']\n })\n self.assertTrue(check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.1-192.168.0.1',\n 'value': {\n 's1$a': 1,\n 's1$b': 1,\n 'sources': ['s1s']\n }\n },\n {\n 'method': 'update',\n 'indicator': '192.168.0.2-192.168.0.3',\n 'value': {\n 's1$b': 1,\n 'sources': ['s1s']\n }\n }\n ],\n all_here=True\n ))\n\n ochannel.publish.reset_mock()\n a.filtered_update('s1', indicator='192.168.0.2-192.168.0.2', value={\n 'type': 'IPv4',\n 's1$c': 1,\n 'sources': ['s1s']\n })\n self.assertTrue(check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.2-192.168.0.2',\n 'value': {\n 's1$b': 1,\n 's1$c': 1,\n 'sources': ['s1s']\n }\n },\n {\n 'method': 'withdraw',\n 'indicator': '192.168.0.2-192.168.0.3',\n 'value': {\n 's1$b': 1,\n 'sources': ['s1s']\n }\n },\n {\n 'method': 'update',\n 'indicator': '192.168.0.3-192.168.0.3',\n 'value': {\n 's1$b': 1,\n 'sources': ['s1s']\n }\n }\n ],\n all_here=True\n ))\n\n ochannel.publish.reset_mock()\n a.filtered_update('s1', indicator='255.255.255.255', value={\n 'type': 'IPv4',\n 's1$e': 1,\n 'sources': ['s1s']\n })\n self.assertTrue(check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '255.255.255.255-255.255.255.255',\n 'value': {\n 'sources': ['s1s']\n }\n }\n ],\n all_here=True\n ))\n\n ochannel.publish.reset_mock()\n a.filtered_update('s1', indicator='0.0.0.0', value={\n 'type': 'IPv4',\n 's1$e': 1,\n 'sources': ['s1s']\n })\n self.assertTrue(check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '0.0.0.0-0.0.0.0',\n 'value': {\n 'sources': ['s1s']\n }\n }\n ],\n all_here=True\n ))\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n def test_uwl(self):\n config = {\n 'whitelist_prefixes': ['s2']\n }\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='192.168.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.0-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='192.168.0.0/24', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 's1$b': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.1.0-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n },\n {\n 'method': 'withdraw',\n 'indicator': '192.168.0.0-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n def test_uwl2(self):\n config = {\n 'whitelist_prefixes': ['s2']\n }\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='192.168.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.0-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='192.168.0.1', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 's1$b': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.0-192.168.0.0',\n 'value': {\n 's1$a': 1\n }\n },\n {\n 'method': 'update',\n 'indicator': '192.168.0.2-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n },\n {\n 'method': 'withdraw',\n 'indicator': '192.168.0.0-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n def test_uwl3(self):\n config = {\n 'whitelist_prefixes': ['s2']\n }\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='192.168.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.0-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='192.168.0.1', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 's1$b': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.0-192.168.0.0',\n 'value': {\n 's1$a': 1\n }\n },\n {\n 'method': 'update',\n 'indicator': '192.168.0.2-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n },\n {\n 'method': 'withdraw',\n 'indicator': '192.168.0.0-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='192.168.0.2', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 's1$b': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.3-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n },\n {\n 'method': 'withdraw',\n 'indicator': '192.168.0.2-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n def test_overlap_by_one(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='192.168.0.1-192.168.0.3', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.1-192.168.0.3'\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='192.168.0.3-192.168.0.4', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 's1$b': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.1-192.168.0.2'\n },\n {\n 'method': 'update',\n 'indicator': '192.168.0.3-192.168.0.3'\n },\n {\n 'method': 'update',\n 'indicator': '192.168.0.4-192.168.0.4'\n },\n {\n 'method': 'withdraw',\n 'indicator': '192.168.0.1-192.168.0.3'\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n def test_overlap_by_lastrange(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='8.8.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '8.8.0.0-8.8.255.255'\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='8.8.255.0/24', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 's1$b': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '8.8.0.0-8.8.254.255'\n },\n {\n 'method': 'update',\n 'indicator': '8.8.255.0-8.8.255.255'\n },\n {\n 'method': 'withdraw',\n 'indicator': '8.8.0.0-8.8.255.255'\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n def test_3overlaps(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='10.1.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '10.1.0.0-10.1.255.255'\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='10.1.1.0/24', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 's1$b': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '10.1.0.0-10.1.0.255'\n },\n {\n 'method': 'update',\n 'indicator': '10.1.1.0-10.1.1.255'\n },\n {\n 'method': 'update',\n 'indicator': '10.1.2.0-10.1.255.255'\n },\n {\n 'method': 'withdraw',\n 'indicator': '10.1.0.0-10.1.255.255'\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='10.1.1.128/25', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 's1$c': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '10.1.1.128-10.1.1.255',\n 's1$a': 1,\n 's1$b': 1,\n 's1$c': 1\n },\n {\n 'method': 'update',\n 'indicator': '10.1.1.0-10.1.1.127'\n },\n {\n 'method': 'withdraw',\n 'indicator': '10.1.1.0-10.1.1.255'\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n def test_2overlaps(self):\n config = {\n 'whitelist_prefixes': ['s2']\n }\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='10.1.0.0/24', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '10.1.0.0-10.1.0.255'\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='10.1.0.10', value={\n 'type': 'IPv4',\n 'sources': ['s2s']\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '10.1.0.11-10.1.0.255'\n },\n {\n 'method': 'update',\n 'indicator': '10.1.0.0-10.1.0.9'\n },\n {\n 'method': 'withdraw',\n 'indicator': '10.1.0.0-10.1.0.255'\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='10.1.0.25', value={\n 'type': 'IPv4',\n 'sources': ['s2s']\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '10.1.0.11-10.1.0.24',\n 's1$a': 1\n },\n {\n 'method': 'update',\n 'indicator': '10.1.0.26-10.1.0.255'\n },\n {\n 'method': 'withdraw',\n 'indicator': '10.1.0.11-10.1.0.255'\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n def test_attr_override(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2', 's3']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='10.1.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 'direction': 'inbound',\n 'first_seen': 10,\n 'last_seen': 25,\n 'confidence': 20\n })\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='10.1.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 'direction': 'inbound',\n 'first_seen': 5,\n 'last_seen': 20,\n 'confidence': 30\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '10.1.0.0-10.1.255.255',\n 'value': {\n 'direction': 'inbound',\n 'first_seen': 5,\n 'last_seen': 25,\n 'confidence': 30\n }\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n def test_uw(self):\n config = {\n 'whitelist_prefixes': ['s2']\n }\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='192.168.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.0-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_withdraw('s1', indicator='192.168.0.0/16')\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'withdraw',\n 'indicator': '192.168.0.0-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n def test_2uw(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='192.168.0.0', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n })\n a.filtered_update('s1', indicator='192.168.1.0', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 's1$a': 1\n })\n\n ochannel.publish.reset_mock()\n a.filtered_withdraw('s1', indicator='192.168.0.0')\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'withdraw',\n 'indicator': '192.168.0.0-192.168.0.0',\n 'value': {\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n }\n }\n ],\n all_here=True\n )\n )\n\n a.filtered_update('s1', indicator='192.168.0.0', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n })\n\n ochannel.publish.reset_mock()\n a.filtered_withdraw('s1', indicator='192.168.1.0')\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'withdraw',\n 'indicator': '192.168.1.0-192.168.1.0'\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n def test_uw_wrongtype(self):\n config = {\n 'whitelist_prefixes': ['s2']\n }\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='192.168.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.0-192.168.255.255',\n 'value': {\n 's1$a': 1\n }\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_withdraw('s1', indicator='192.168.0.0/16', value={'type': 'domain'})\n self.assertEqual(ochannel.publish.call_count, 0)\n\n ochannel.publish.reset_mock()\n a.filtered_withdraw('s1', indicator='192.168.0.0/16', value={'type': 'IPv4'})\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'withdraw',\n 'indicator': '192.168.0.0-192.168.255.255'\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n def test_updated_indicator(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n ochannel.publish.reset_mock()\n a.filtered_update('s1', indicator='192.168.0.0', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 1\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.0-192.168.0.0',\n 'value': {\n 's1$a': 1\n }\n }\n ],\n all_here=True\n )\n )\n\n ochannel.publish.reset_mock()\n a.filtered_update('s1', indicator='192.168.0.0', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 's1$a': 2\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '192.168.0.0-192.168.0.0',\n 'value': {\n 's1$a': 2\n }\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n @attr('slow')\n def test_stress_1(self):\n num_intervals = 100000\n\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n t1 = time.time()\n for j in xrange(num_intervals):\n start = random.randint(0, 0xFFFFFFFF)\n if random.randint(0, 4) == 0:\n start = start & 0xFFFFFF00\n end = start + 255\n else:\n end = start\n end = netaddr.IPAddress(end)\n start = netaddr.IPAddress(start)\n ochannel.publish.reset_mock()\n t2 = time.time()\n dt = t2-t1\n\n t1 = time.time()\n for j in xrange(num_intervals):\n start = random.randint(0, 0xFFFFFFFF)\n if random.randint(0, 4) == 0:\n start = start & 0xFFFFFF00\n end = start + 255\n else:\n end = start\n end = netaddr.IPAddress(end)\n start = netaddr.IPAddress(start)\n ochannel.publish.reset_mock()\n a.filtered_update('s1', indicator='%s-%s' % (start, end), value={\n 'type': 'IPv4',\n 'sources': ['s1s']\n })\n t2 = time.time()\n print \"TIME: Inserted %d intervals in %d\" % (num_intervals, (t2-t1-dt))\n\n t1 = time.time()\n for j in xrange(num_intervals):\n ochannel.publish.reset_mock()\n a.filtered_update('s1', indicator='%s' % (start), value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 'count': j\n })\n t2 = time.time()\n print \"TIME: Updated %d intervals in %d\" % (num_intervals, (t2-t1-dt))\n\n a.stop()\n\n a.st.db.close()\n a = None\n\n @attr('slow')\n def test_stress_2(self):\n num_intervals = 200\n\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n t1 = time.time()\n for _ in xrange(num_intervals):\n end = random.randint(0, 0xFFFFFFFF)\n start = random.randint(0, end)\n end = netaddr.IPAddress(end)\n start = netaddr.IPAddress(start)\n ochannel.publish.reset_mock()\n t2 = time.time()\n dt = t2-t1\n\n t1 = time.time()\n for j in xrange(num_intervals):\n end = random.randint(0, 0xFFFFFFFF)\n start = random.randint(0, end)\n end = netaddr.IPAddress(end)\n start = netaddr.IPAddress(start)\n ochannel.publish.reset_mock()\n a.filtered_update('s1', indicator='%s-%s' % (start, end), value={\n 'type': 'IPv4',\n 'sources': ['s1s']\n })\n t2 = time.time()\n print \"TIME: Inserted %d intervals in %d\" % (num_intervals, (t2-t1-dt))\n\n a.stop()\n\n a.st.db.close()\n a = None\n" }, { "path": "tests/test_ft_local.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT local tests\n\nUnit tests for minemeld.ft.local\n\"\"\"\n\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport unittest\nimport mock\nimport time\nimport shutil\nimport logging\nimport gc\nimport calendar\nimport os.path\nimport yaml\n\nimport minemeld.ft.local\n\nFTNAME = 'testft-%d' % int(time.time())\nLOCALDB_NAME = 'local-%d.yml' % int(time.time())\n\nLOG = logging.getLogger(__name__)\n\nCUR_LOGICAL_TIME = 0\n\nMYDIR = os.path.dirname(__file__)\n\n\ndef logical_millisec(*args):\n return CUR_LOGICAL_TIME*1000\n\n\ndef gevent_event_mock_factory():\n result = mock.Mock()\n result.wait.side_effect = gevent.GreenletExit()\n\n return result\n\n\nclass MineMeldYamlFTTests(unittest.TestCase):\n def setUp(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n try:\n os.remove(LOCALDB_NAME)\n except:\n pass\n\n def tearDown(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n try:\n os.remove(LOCALDB_NAME)\n except:\n pass\n\n @mock.patch.object(gevent, 'spawn')\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit())\n @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory)\n @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec)\n def test_yaml(self, um_mock, sleep_mock, event_mock,\n spawnl_mock, spawn_mock):\n global CUR_LOGICAL_TIME\n\n localdb_path = os.path.join(MYDIR, 'test_localdb.yml')\n localdb_path2 = os.path.join(MYDIR, 'test_localdb2.yml')\n\n with open(localdb_path, 'r') as f:\n localdb = yaml.safe_load(f)\n localdb = [k['indicator'] for k in localdb]\n\n with open(localdb_path2, 'r') as f:\n localdb2 = yaml.safe_load(f)\n localdb2 = [k['indicator'] for k in localdb2]\n\n shutil.copyfile(localdb_path, LOCALDB_NAME)\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n config = {\n 'path': LOCALDB_NAME,\n 'age_out': {\n 'default': None,\n 'sudden_death': True\n }\n }\n\n a = minemeld.ft.local.YamlFT(FTNAME, chassis, config)\n\n inputs = []\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 1)\n self.assertEqual(spawn_mock.call_count, 3)\n\n CUR_LOGICAL_TIME = 1\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n self.assertEqual(um_mock.call_count, 1)\n\n CUR_LOGICAL_TIME = 2\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], len(localdb))\n self.assertEqual(a.statistics.get('removed', 0), 0)\n\n lsp = a.last_successful_run\n\n CUR_LOGICAL_TIME = 3\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics.get('aged_out', 0), 0)\n self.assertEqual(a.statistics.get('removed', 0), 0)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 0)\n self.assertEqual(a.last_successful_run, lsp)\n\n LOG.debug('%d', a.statistics['added'])\n\n shutil.copyfile(localdb_path2, LOCALDB_NAME)\n\n CUR_LOGICAL_TIME = 4\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(\n a.statistics['added'],\n len(set(localdb) | set(localdb2))\n )\n self.assertEqual(\n a.statistics.get('removed', 0),\n len(set(localdb2)-set(localdb))\n )\n self.assertEqual(a.statistics.get('garbage_collected', 0), 1)\n\n CUR_LOGICAL_TIME = 5\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 1)\n\n CUR_LOGICAL_TIME = 6\n a._age_out()\n self.assertEqual(a.statistics.get('aged_out', 0), 1)\n\n CUR_LOGICAL_TIME = 7\n a._age_out()\n self.assertEqual(a.statistics['aged_out'], 1)\n\n CUR_LOGICAL_TIME = 8\n a._poll()\n a._sudden_death()\n a._age_out()\n a._collect_garbage()\n self.assertEqual(a.statistics['added'], 3)\n self.assertEqual(a.statistics.get('garbage_collected', 0), 1)\n self.assertEqual(a.length(), 2)\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n" }, { "path": "tests/test_ft_logstash.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT Logstash tests\n\nUnit tests for minemeld.ft.logstash\n\"\"\"\n\nimport unittest\nimport mock\nimport time\n\nimport minemeld.ft.logstash\n\nFTNAME = 'testft-%d' % int(time.time())\n\n\nclass MineMeldFTLogstashOutputTests(unittest.TestCase):\n def test_uw(self):\n config = {\n 'logstash_host': '127.0.0.1',\n 'logstash_port': 5514\n }\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n b = minemeld.ft.logstash.LogstashOutput(FTNAME, chassis, config)\n\n inputs = ['a', 'b', 'c']\n output = False\n\n b.connect(inputs, output)\n b.mgmtbus_initialize()\n\n b.start()\n time.sleep(1)\n\n b.update('a', indicator='testi', value={'test': 'v'})\n self.assertEqual(b.statistics['message.sent'], 1)\n\n b.withdraw('a', indicator='testi')\n self.assertEqual(b.statistics['message.sent'], 2)\n\n b.stop()\n" }, { "path": "tests/test_ft_op.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT op tests\n\nUnit tests for minemeld.ft.op\n\"\"\"\n\nimport gevent\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport unittest\nimport mock\nimport time\nimport shutil\nimport logging\n\nimport guppy # noqa\nimport pdb # noqa\nimport gc # noqa\n\nimport minemeld.ft.op\n\nFTNAME = 'testft-%d' % int(time.time())\n\nLOG = logging.getLogger(__name__)\n\n\ndef check_for_rpc(call_args_list, check_list, all_here=False, offset=0):\n LOG.debug(\"call_args_list: %s\", call_args_list)\n\n found = []\n for chk in check_list:\n LOG.debug(\"checking: %s\", chk)\n\n for j in xrange(len(call_args_list)):\n if j in found:\n continue\n\n args = call_args_list[j][0]\n LOG.debug(\"args: %s\", args[offset+0])\n\n if args[offset+0] != chk['method']:\n continue\n if args[offset+1]['indicator'] != chk['indicator']:\n continue\n\n chkvalue = chk.get('value', None)\n if chkvalue is None:\n found.append(j)\n LOG.debug(\"found @%d\", j)\n break\n\n argsvalue = args[offset+1].get('value', None)\n if chkvalue is not None and argsvalue is None:\n continue\n\n failed = False\n for k in chkvalue.keys():\n if k not in argsvalue:\n failed = True\n break\n if chkvalue[k] != argsvalue[k]:\n failed = True\n break\n if failed:\n continue\n\n found.append(j)\n LOG.debug(\"found @%d\", j)\n break\n\n c1 = len(found) == len(check_list)\n\n if not all_here:\n return c1\n\n c2 = len(found) == len(call_args_list)\n\n return c1+c2 == 2\n\n\nclass MineMeldFTOpTests(unittest.TestCase):\n def setUp(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n def tearDown(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n def test_aggregate_2u(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']})\n self.assertEqual(ochannel.publish.call_count, 1)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'update')\n self.assertEqual(pargs[1]['indicator'], 'i')\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s'])\n self.assertEqual(pargs[1]['value']['s1$a'], 1)\n\n a.filtered_update('s2', indicator='i', value={'s2$a': 1, 'sources': ['s2s']})\n self.assertEqual(ochannel.publish.call_count, 2)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'update')\n self.assertEqual(pargs[1]['indicator'], 'i')\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s', 's2s'])\n self.assertEqual(pargs[1]['value']['s2$a'], 1)\n self.assertEqual(pargs[1]['value']['s1$a'], 1)\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n def test_aggregate_uwl(self):\n config = {\n 'whitelist_prefixes': ['s2']\n }\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']})\n self.assertEqual(ochannel.publish.call_count, 1)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'update')\n self.assertEqual(pargs[1]['indicator'], 'i')\n self.assertEqual(pargs[1]['value']['s1$a'], 1)\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s'])\n\n a.filtered_update('s2', indicator='i', value={'s2$a': 1, 'sources': ['s2s']})\n self.assertEqual(ochannel.publish.call_count, 2)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'withdraw')\n self.assertEqual(pargs[1]['indicator'], 'i')\n self.assertIn('value', pargs[1])\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n def test_aggregate_2uw(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']})\n pargs = ochannel.publish.call_args[0]\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s'])\n\n a.filtered_update('s2', indicator='i', value={'s2$a': 1, 'sources': ['s2s']})\n pargs = ochannel.publish.call_args[0]\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s', 's2s'])\n\n a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']})\n pargs = ochannel.publish.call_args[0]\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s', 's2s'])\n\n a.filtered_withdraw('s2', indicator='i')\n pargs = ochannel.publish.call_args[0]\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s'])\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n def test_aggregate_2u2w(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']})\n self.assertEqual(ochannel.publish.call_count, 1)\n pargs = ochannel.publish.call_args[0]\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s'])\n\n a.filtered_update('s2', indicator='i', value={'s2$a': 1, 'sources': ['s2s']})\n self.assertEqual(ochannel.publish.call_count, 2)\n pargs = ochannel.publish.call_args[0]\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s', 's2s'])\n\n a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']})\n self.assertEqual(ochannel.publish.call_count, 3)\n pargs = ochannel.publish.call_args[0]\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s', 's2s'])\n\n a.filtered_withdraw('s2', indicator='i')\n self.assertEqual(ochannel.publish.call_count, 4)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'update')\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s'])\n\n a.filtered_withdraw('s1', indicator='i')\n self.assertEqual(ochannel.publish.call_count, 5)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'withdraw')\n self.assertIn('value', pargs[1])\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n def test_aggregate_u2w_difftypes(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'type': 'a', 'sources': ['s1s']})\n self.assertEqual(ochannel.publish.call_count, 1)\n pargs = ochannel.publish.call_args[0]\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s'])\n\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='i2', value={'s1$a': 1, 'type': 'a', 'sources': ['s1s']})\n self.assertEqual(ochannel.publish.call_count, 1)\n pargs = ochannel.publish.call_args[0]\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s'])\n\n ochannel.publish.reset_mock()\n a.filtered_withdraw('s1', indicator='i', value={'type': 'b'})\n self.assertEqual(ochannel.publish.call_count, 0)\n\n ochannel.publish.reset_mock()\n a.filtered_withdraw('s1', indicator='i', value={'type': 'a'})\n self.assertEqual(ochannel.publish.call_count, 1)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'withdraw')\n\n ochannel.publish.reset_mock()\n a.filtered_withdraw('s2', indicator='i2')\n self.assertEqual(ochannel.publish.call_count, 1)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'withdraw')\n self.assertIn('value', pargs[1])\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n def test_aggregate_uwlwl(self):\n config = {\n 'whitelist_prefixes': ['s2', 's3']\n }\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2', 's3']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']})\n self.assertEqual(ochannel.publish.call_count, 1)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'update')\n self.assertEqual(pargs[1]['indicator'], 'i')\n self.assertEqual(pargs[1]['value']['s1$a'], 1)\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s'])\n\n a.filtered_update('s2', indicator='i', value={'s2$a': 1, 'sources': ['s2s']})\n self.assertEqual(ochannel.publish.call_count, 2)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'withdraw')\n self.assertEqual(pargs[1]['indicator'], 'i')\n self.assertIn('value', pargs[1])\n\n a.filtered_update('s3', indicator='i', value={'s3$a': 1, 'sources': ['s3s']})\n self.assertEqual(ochannel.publish.call_count, 2)\n\n a.filtered_withdraw('s3', indicator='i')\n self.assertEqual(ochannel.publish.call_count, 2)\n\n a.filtered_withdraw('s2', indicator='i')\n self.assertEqual(ochannel.publish.call_count, 3)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'update')\n self.assertEqual(pargs[1]['indicator'], 'i')\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s'])\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n def test_infilters(self):\n config = {\n 'infilters': [\n {\n 'name': 'rule1',\n 'conditions': [\n 'type(sources) == null'\n ],\n 'actions': [\n 'drop'\n ]\n }\n ]\n }\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2', 's3']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.update(source='s1', indicator='i', value={'s1a': 1, 'sources': ['s1s']})\n gevent.sleep(0.1)\n self.assertEqual(ochannel.publish.call_count, 1)\n\n ochannel.publish.reset_mock()\n a.update(source='s2', indicator='i', value={'s2a': 1})\n gevent.sleep(0.1)\n self.assertEqual(ochannel.publish.call_count, 0)\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n def test_infilters_2u(self):\n config = {\n 'infilters': [\n {\n 'name': 'rule1',\n 'conditions': [\n 'type(sources) == null'\n ],\n 'actions': [\n 'drop'\n ]\n }\n ]\n }\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2', 's3']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.update(source='s1', indicator='i', value={'s1a': 1, 'sources': ['s1s']})\n gevent.sleep(0.1)\n self.assertEqual(ochannel.publish.call_count, 1)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'update')\n self.assertEqual(pargs[1]['indicator'], 'i')\n self.assertEqual(pargs[1]['value']['s1a'], 1)\n self.assertListEqual(pargs[1]['value']['sources'], ['s1s'])\n\n ochannel.publish.reset_mock()\n a.update(source='s1', indicator='i', value={'s1a': 1})\n gevent.sleep(0.1)\n self.assertEqual(ochannel.publish.call_count, 1)\n pargs = ochannel.publish.call_args[0]\n self.assertEqual(pargs[0], 'withdraw')\n self.assertEqual(pargs[1]['indicator'], 'i')\n self.assertIn('value', pargs[1])\n\n a.stop()\n\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n def test_attr_override(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2', 's3']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='10.1.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 'direction': 'inbound',\n 'first_seen': 10,\n 'last_seen': 25,\n 'confidence': 20\n })\n ochannel.publish.reset_mock()\n a.filtered_update('s2', indicator='10.1.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 'direction': 'inbound',\n 'first_seen': 5,\n 'last_seen': 20,\n 'confidence': 30\n })\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '10.1.0.0/16',\n 'value': {\n 'sources': ['s1s', 's2s'],\n 'direction': 'inbound',\n 'first_seen': 5,\n 'last_seen': 25,\n 'confidence': 30\n }\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a = None\n gc.collect()\n\n def test_get_all(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n chassis.send_rpc.return_value = {'error': None, 'result': 'OK'}\n\n a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2', 's3']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='10.1.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 'direction': 'inbound',\n 'first_seen': 10,\n 'last_seen': 25,\n 'confidence': 20\n })\n a.filtered_update('s2', indicator='10.1.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 'direction': 'inbound',\n 'first_seen': 5,\n 'last_seen': 20,\n 'confidence': 30\n })\n a.filtered_update('s3', indicator='10.1.1.0/24', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 'direction': 'inbound',\n 'first_seen': 5,\n 'last_seen': 20,\n 'confidence': 30\n })\n a.get_all(source='test')\n self.assertTrue(\n check_for_rpc(\n chassis.send_rpc.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '10.1.0.0/16',\n 'value': {\n 'sources': ['s2s'],\n 'direction': 'inbound',\n 'first_seen': 5,\n 'last_seen': 25,\n 'confidence': 30\n }\n },\n {\n 'method': 'update',\n 'indicator': '10.1.1.0/24',\n 'value': {\n 'sources': ['s1s'],\n 'direction': 'inbound',\n 'first_seen': 5,\n 'last_seen': 20,\n 'confidence': 30\n }\n }\n ],\n all_here=True,\n offset=2\n )\n )\n\n a.stop()\n\n a = None\n gc.collect()\n\n def test_get_range(self):\n config = {}\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n chassis.send_rpc.return_value = {'error': None, 'result': 'OK'}\n\n a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config)\n\n inputs = ['s1', 's2', 's3']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('s1', indicator='10.1.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 'direction': 'inbound',\n 'first_seen': 10,\n 'last_seen': 25,\n 'confidence': 20\n })\n a.filtered_update('s2', indicator='10.1.0.0/16', value={\n 'type': 'IPv4',\n 'sources': ['s2s'],\n 'direction': 'inbound',\n 'first_seen': 5,\n 'last_seen': 20,\n 'confidence': 30\n })\n a.filtered_update('s3', indicator='10.1.1.0/24', value={\n 'type': 'IPv4',\n 'sources': ['s1s'],\n 'direction': 'inbound',\n 'first_seen': 5,\n 'last_seen': 20,\n 'confidence': 30\n })\n a.get_range(source='test', from_key='10.1.0.0/16',\n to_key='10.1.1.0/24')\n self.assertTrue(\n check_for_rpc(\n chassis.send_rpc.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '10.1.0.0/16',\n 'value': {\n 'sources': ['s2s'],\n 'direction': 'inbound',\n 'first_seen': 5,\n 'last_seen': 25,\n 'confidence': 30\n }\n },\n {\n 'method': 'update',\n 'indicator': '10.1.1.0/24',\n 'value': {\n 'sources': ['s1s'],\n 'direction': 'inbound',\n 'first_seen': 5,\n 'last_seen': 20,\n 'confidence': 30\n }\n }\n ],\n all_here=True,\n offset=2\n )\n )\n\n a.stop()\n\n a = None\n gc.collect()\n" }, { "path": "tests/test_ft_redis.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT Redis tests\n\nUnit tests for minemeld.ft.redis\n\"\"\"\n\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport unittest\nimport mock\nimport redis\nimport time\n\nimport minemeld.ft.redis\n\nFTNAME = 'testft-%d' % int(time.time())\n\n\nclass MineMeldFTRedisTests(unittest.TestCase):\n def setUp(self):\n SR = redis.StrictRedis()\n SR.delete(FTNAME)\n\n def tearDown(self):\n SR = redis.StrictRedis()\n SR.delete(FTNAME)\n\n def test_init(self):\n config = {}\n chassis = mock.Mock()\n\n b = minemeld.ft.redis.RedisSet(FTNAME, chassis, config)\n\n self.assertEqual(b.name, FTNAME)\n self.assertEqual(b.chassis, chassis)\n self.assertEqual(b.config, config)\n self.assertItemsEqual(b.inputs, [])\n self.assertEqual(b.output, None)\n self.assertEqual(b.redis_skey, FTNAME)\n self.assertNotEqual(b.SR, None)\n self.assertEqual(b.redis_url, 'unix:///var/run/redis/redis.sock')\n\n def test_connect_io(self):\n config = {}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n\n b = minemeld.ft.redis.RedisSet(FTNAME, chassis, config)\n\n inputs = ['a', 'b', 'c']\n output = True\n\n b.connect(inputs, output)\n b.mgmtbus_initialize()\n\n self.assertItemsEqual(b.inputs, inputs)\n self.assertEqual(b.output, None)\n\n icalls = []\n for i in inputs:\n icalls.append(\n mock.call(\n FTNAME, b, i,\n allowed_methods=[\n 'update', 'withdraw', 'checkpoint'\n ]\n )\n )\n chassis.request_sub_channel.assert_has_calls(\n icalls,\n any_order=True\n )\n\n chassis.request_rpc_channel.assert_called_once_with(\n FTNAME,\n b,\n allowed_methods=[\n 'update',\n 'withdraw',\n 'checkpoint',\n 'get',\n 'get_all',\n 'get_range',\n 'length'\n ]\n )\n\n chassis.request_pub_channel.assert_not_called()\n\n def test_uw(self):\n config = {}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n b = minemeld.ft.redis.RedisSet(FTNAME, chassis, config)\n\n inputs = ['a', 'b', 'c']\n output = False\n\n b.connect(inputs, output)\n b.mgmtbus_initialize()\n\n b.start()\n time.sleep(1)\n\n SR = redis.StrictRedis()\n\n b.filtered_update('a', indicator='testi', value={'test': 'v'})\n sm = SR.zrange(FTNAME, 0, -1)\n self.assertEqual(len(sm), 1)\n self.assertIn('testi', sm)\n\n b.filtered_withdraw('a', indicator='testi')\n sm = SR.zrange(FTNAME, 0, -1)\n self.assertEqual(len(sm), 0)\n\n b.stop()\n self.assertNotEqual(b.SR, None)\n\n def test_stats(self):\n config = {}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n b = minemeld.ft.redis.RedisSet(FTNAME, chassis, config)\n\n inputs = ['a', 'b', 'c']\n output = False\n\n b.connect(inputs, output)\n b.mgmtbus_reset()\n\n b.start()\n time.sleep(1)\n\n b.filtered_update('a', indicator='testi', value={'test': 'v'})\n self.assertEqual(b.length(), 1)\n status = b.mgmtbus_status()\n self.assertEqual(status['statistics']['added'], 1)\n\n b.filtered_update('a', indicator='testi', value={'test': 'v2'})\n self.assertEqual(b.length(), 1)\n status = b.mgmtbus_status()\n self.assertEqual(status['statistics']['added'], 1)\n self.assertEqual(status['statistics']['removed'], 0)\n\n b.filtered_withdraw('a', indicator='testi')\n self.assertEqual(b.length(), 0)\n status = b.mgmtbus_status()\n self.assertEqual(status['statistics']['removed'], 1)\n\n b.stop()\n\n def test_store_value(self):\n config = {'store_value': True}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n b = minemeld.ft.redis.RedisSet(FTNAME, chassis, config)\n\n inputs = ['a', 'b', 'c']\n output = False\n\n b.connect(inputs, output)\n b.mgmtbus_reset()\n\n b.start()\n time.sleep(1)\n\n SR = redis.StrictRedis()\n\n b.filtered_update('a', indicator='testi', value={'test': 'v'})\n sm = SR.zrange(FTNAME, 0, -1)\n self.assertEqual(len(sm), 1)\n self.assertIn('testi', sm)\n sm = SR.hlen(FTNAME+'.value')\n self.assertEqual(sm, 1)\n\n b.filtered_withdraw('a', indicator='testi')\n sm = SR.zrange(FTNAME, 0, -1)\n self.assertEqual(len(sm), 0)\n sm = SR.hlen(FTNAME+'.value')\n self.assertEqual(sm, 0)\n\n b.stop()\n self.assertNotEqual(b.SR, None)\n\n def test_store_value_overflow(self):\n config = {'store_value': True}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n b = minemeld.ft.redis.RedisSet(FTNAME, chassis, config)\n b.max_entries = 1\n\n inputs = ['a', 'b', 'c']\n output = False\n\n b.connect(inputs, output)\n b.mgmtbus_reset()\n\n b.start()\n time.sleep(1)\n\n SR = redis.StrictRedis()\n\n b.filtered_update('a', indicator='testi', value={'test': 'v'})\n sm = SR.zrange(FTNAME, 0, -1)\n self.assertEqual(len(sm), 1)\n self.assertIn('testi', sm)\n sm = SR.hlen(FTNAME+'.value')\n self.assertEqual(sm, 1)\n\n b.filtered_update('a', indicator='testio', value={'test': 'v'})\n self.assertEqual(b.statistics['drop.overflow'], 1)\n sm = SR.zrange(FTNAME, 0, -1)\n self.assertEqual(len(sm), 1)\n self.assertIn('testi', sm)\n sm = SR.hlen(FTNAME+'.value')\n self.assertEqual(sm, 1)\n\n b.filtered_withdraw('a', indicator='testi')\n sm = SR.zrange(FTNAME, 0, -1)\n self.assertEqual(len(sm), 0)\n sm = SR.hlen(FTNAME+'.value')\n self.assertEqual(sm, 0)\n\n b.filtered_update('a', indicator='testio', value={'test': 'v'})\n self.assertEqual(b.statistics['drop.overflow'], 1)\n sm = SR.zrange(FTNAME, 0, -1)\n self.assertEqual(len(sm), 1)\n self.assertIn('testio', sm)\n sm = SR.hlen(FTNAME+'.value')\n self.assertEqual(sm, 1)\n\n b.stop()\n self.assertNotEqual(b.SR, None)\n" }, { "path": "tests/test_ft_st.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT ST tests\n\nUnit tests for minemeld.ft.st\n\"\"\"\n\nimport unittest\nimport tempfile\nimport shutil\nimport random\nimport uuid\nimport time\n\nfrom nose.plugins.attrib import attr\n\nimport minemeld.ft.st\n\nTABLENAME = tempfile.mktemp(prefix='minemeld.ftsttest')\nNUM_ELEMENTS = 10000\n\n\nclass MineMeldFTSTTests(unittest.TestCase):\n def setUp(self):\n try:\n shutil.rmtree(TABLENAME)\n except:\n pass\n\n def tearDown(self):\n try:\n shutil.rmtree(TABLENAME)\n except:\n pass\n\n def test_add_delete(self):\n st = minemeld.ft.st.ST(TABLENAME, 8, truncate=True)\n\n sid = uuid.uuid4().bytes\n\n st.put(sid, 1, 5, 1)\n st.delete(sid, 1, 5, 1)\n\n st.close()\n\n def test_query_endpoints_forward(self):\n st = minemeld.ft.st.ST(TABLENAME, 8, truncate=True)\n\n sid1 = uuid.uuid4().bytes\n sid2 = uuid.uuid4().bytes\n\n st.put(sid1, 1, 70, 1)\n st.put(sid2, 50, 100, 1)\n\n eps = [ep[0] for ep in st.query_endpoints(\n start=0,\n stop=st.max_endpoint,\n reverse=False,\n include_start=False,\n include_stop=False\n )]\n\n self.assertEqual(eps, [1, 50, 70, 100])\n\n st.close()\n\n def test_query_endpoints_reverse(self):\n st = minemeld.ft.st.ST(TABLENAME, 8, truncate=True)\n\n sid1 = uuid.uuid4().bytes\n sid2 = uuid.uuid4().bytes\n\n st.put(sid1, 1, 70, 1)\n st.put(sid2, 50, 100, 1)\n\n eps = [ep[0] for ep in st.query_endpoints(\n start=0,\n stop=st.max_endpoint,\n reverse=True,\n include_start=False,\n include_stop=False\n )]\n\n self.assertEqual(eps, [100, 70, 50, 1])\n\n st.close()\n\n def test_basic_cover(self):\n st = minemeld.ft.st.ST(TABLENAME, 8, truncate=True)\n\n sid = uuid.uuid4().bytes\n\n st.put(sid, 1, 5, 1)\n for i in range(1, 6):\n ci = st.cover(i)\n\n interval = next(ci, None)\n self.assertEqual(interval[0], sid)\n self.assertEqual(interval[1], 1)\n self.assertEqual(interval[2], 1)\n self.assertEqual(interval[3], 5)\n\n interval2 = next(ci, None)\n self.assertEqual(interval2, None)\n\n st.close()\n\n def test_cover_overlap(self):\n st = minemeld.ft.st.ST(TABLENAME, 8, truncate=True)\n\n sid1 = uuid.uuid4().bytes\n sid2 = uuid.uuid4().bytes\n\n st.put(sid1, 1, 5, 1)\n st.put(sid2, 3, 7, 2)\n\n ci = st.cover(1)\n\n interval = next(ci, None)\n self.assertEqual(interval[0], sid1)\n self.assertEqual(interval[1], 1)\n self.assertEqual(interval[2], 1)\n self.assertEqual(interval[3], 5)\n\n interval = next(ci, None)\n self.assertEqual(interval, None)\n\n ci = st.cover(3)\n\n intervals = [i for i in st.cover(3)]\n self.assertEqual(len(intervals), 2)\n self.assertEqual(intervals[0][0], sid1)\n self.assertEqual(intervals[0][1], 1)\n self.assertEqual(intervals[0][2], 1)\n self.assertEqual(intervals[0][3], 5)\n self.assertEqual(intervals[1][0], sid2)\n self.assertEqual(intervals[1][1], 2)\n self.assertEqual(intervals[1][2], 3)\n self.assertEqual(intervals[1][3], 7)\n\n ci = st.cover(7)\n\n interval = next(ci, None)\n self.assertEqual(interval[0], sid2)\n self.assertEqual(interval[1], 2)\n self.assertEqual(interval[2], 3)\n self.assertEqual(interval[3], 7)\n\n interval = next(ci, None)\n self.assertEqual(interval, None)\n\n st.close()\n\n def test_cover_overlap2(self):\n st = minemeld.ft.st.ST(TABLENAME, 8, truncate=True)\n\n sid1 = uuid.uuid4().bytes\n sid2 = uuid.uuid4().bytes\n\n st.put(sid1, 3, 7, 1)\n st.put(sid2, 3, 7, 2)\n\n intervals = [i for i in st.cover(3)]\n self.assertEqual(len(intervals), 2)\n self.assertEqual(intervals[0][0], sid2)\n self.assertEqual(intervals[0][1], 2)\n self.assertEqual(intervals[0][2], 3)\n self.assertEqual(intervals[0][3], 7)\n self.assertEqual(intervals[1][0], sid1)\n self.assertEqual(intervals[1][1], 1)\n self.assertEqual(intervals[1][2], 3)\n self.assertEqual(intervals[1][3], 7)\n\n st.close()\n\n def _random_map(self, nbits=10, nintervals=1000):\n epmax = (1 << nbits)-1\n\n rmap = [set() for i in xrange(epmax+1)]\n\n st = minemeld.ft.st.ST(TABLENAME, nbits, truncate=True)\n\n for j in xrange(nintervals):\n sid = uuid.uuid4().bytes\n end = random.randint(0, epmax)\n start = random.randint(0, epmax)\n if end < start:\n start, end = end, start\n st.put(sid, start, end, level=1)\n\n for k in xrange(start, end+1):\n rmap[k].add(sid)\n\n eps = []\n for ep, lvl, t, id_ in st.query_endpoints():\n if ep == 0 or ep == epmax:\n self.assertTrue(len(rmap[ep]) > 0)\n else:\n c = len(rmap[ep] ^ rmap[ep-1]) + len(rmap[ep] ^ rmap[ep+1])\n self.assertTrue(\n c > 0,\n msg=\"no change detected @ep %d: \"\n \"%r %r %r\" % (ep, rmap[ep-1], rmap[ep], rmap[ep+1])\n )\n eps.append(ep)\n\n for e in eps:\n intervals = [x[0] for x in st.cover(e)]\n intervals.sort()\n self.assertListEqual(intervals, sorted(rmap[e]))\n\n st.close()\n\n def test_random_map_fast(self):\n self._random_map()\n\n @attr('slow')\n def test_random_map_fast2(self):\n self._random_map(nintervals=2000)\n\n def test_255(self):\n st = minemeld.ft.st.ST(TABLENAME, 32, truncate=True)\n sid = uuid.uuid4().bytes\n st.put(sid, 0, 0xFF)\n self.assertEqual(st.num_segments, 1)\n self.assertEqual(st.num_endpoints, 2)\n st.close()\n\n @attr('slow')\n def test_stress_0(self):\n num_intervals = 100000\n st = minemeld.ft.st.ST(TABLENAME, 32, truncate=True)\n\n t1 = time.time()\n for j in xrange(num_intervals):\n end = random.randint(0, 0xFFFFFFFF)\n if random.randint(0, 1) == 0:\n end = end & 0xFFFFFF00\n start = end + 0xFF\n else:\n start = end\n sid = uuid.uuid4().bytes\n t2 = time.time()\n dt = t2-t1\n\n t1 = time.time()\n for j in xrange(num_intervals):\n end = random.randint(0, 0xFFFFFFFF)\n if random.randint(0, 1) == 0:\n start = end & 0xFFFFFF00\n end = start + 0xFF\n else:\n start = end\n sid = uuid.uuid4().bytes\n st.put(sid, start, end)\n t2 = time.time()\n print \"TIME: Inserted %d intervals in %d\" % (num_intervals, (t2-t1-dt))\n\n self.assertEqual(st.num_segments, num_intervals)\n self.assertEqual(st.num_endpoints, num_intervals*2)\n\n st.close()\n\n @attr('slow')\n def test_stress_1(self):\n num_intervals = 100000\n st = minemeld.ft.st.ST(TABLENAME, 32, truncate=True)\n\n t1 = time.time()\n for j in xrange(num_intervals):\n end = random.randint(0, 0xFFFFFFFF)\n start = random.randint(0, end)\n sid = uuid.uuid4().bytes\n t2 = time.time()\n dt = t2-t1\n\n t1 = time.time()\n for j in xrange(num_intervals):\n end = random.randint(0, 0xFFFFFFFF)\n start = random.randint(0, end)\n sid = uuid.uuid4().bytes\n st.put(sid, start, end)\n t2 = time.time()\n print \"TIME: Inserted %d intervals in %d\" % (num_intervals, (t2-t1-dt))\n\n num_queries = 100000\n\n t1 = time.time()\n j = 0\n for j in xrange(num_queries):\n q = random.randint(0, 0xFFFFFFFF)\n next(st.cover(q), None)\n t2 = time.time()\n print \"TIME: Queried %d times in %d\" % (num_queries, (t2-t1))\n\n st.close()\n" }, { "path": "tests/test_ft_syslog.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT syslog tests\n\nUnit tests for minemeld.ft.syslog\n\"\"\"\n\nimport unittest\nimport shutil\nimport time\nimport logging\nimport mock\nimport gevent\nimport socket\nimport gc\n\nimport minemeld.ft.syslog\n\nFTNAME = 'testft-%d' % int(time.time())\n\nLOG = logging.getLogger(__name__)\n\n\ndef check_for_rpc(call_args_list, check_list, all_here=False):\n LOG.debug(\"call_args_list: %s\", call_args_list)\n\n found = []\n for chk in check_list:\n LOG.debug(\"checking: %s\", chk)\n\n for j in xrange(len(call_args_list)):\n if j in found:\n continue\n\n args = call_args_list[j][0]\n\n if args[0] != chk['method']:\n continue\n if args[1]['indicator'] != chk['indicator']:\n continue\n\n chkvalue = chk.get('value', None)\n if chkvalue is None:\n found.append(j)\n LOG.debug(\"found @%d\", j)\n break\n\n argsvalue = args[1].get('value', None)\n if chkvalue is not None and argsvalue is None:\n continue\n\n failed = False\n for k in chkvalue.keys():\n if k not in argsvalue:\n failed = True\n break\n if chkvalue[k] != argsvalue[k]:\n failed = True\n break\n if failed:\n continue\n\n found.append(j)\n LOG.debug(\"found @%d\", j)\n break\n\n c1 = len(found) == len(check_list)\n\n if not all_here:\n return c1\n\n c2 = len(found) == len(call_args_list)\n\n return c1+c2 == 2\n\n\nclass MineMeldFTSyslogMatcherests(unittest.TestCase):\n def setUp(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n try:\n shutil.rmtree(FTNAME+\"_ipv4\")\n except:\n pass\n\n try:\n shutil.rmtree(FTNAME+\"_indicators\")\n except:\n pass\n\n def tearDown(self):\n try:\n shutil.rmtree(FTNAME)\n except:\n pass\n\n try:\n shutil.rmtree(FTNAME+\"_ipv4\")\n except:\n pass\n\n try:\n shutil.rmtree(FTNAME+\"_indicators\")\n except:\n pass\n\n @mock.patch.object(gevent, 'spawn_later')\n def test_handle_ip_01(self, spawnl_mock):\n config = {\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config)\n\n inputs = ['a']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 2)\n\n a.filtered_update('a', indicator='1.1.1.1-1.1.1.2', value={\n 'type': 'IPv4',\n 'confidence': 100\n })\n self.assertEqual(a.length(), 1)\n\n a._handle_ip('1.1.1.1')\n self.assertEqual(a.table.num_indicators, 1)\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': '1.1.1.1',\n 'value': {\n 'syslog_original_indicator':\n 'IPv4'+'1.1.1.1-1.1.1.2',\n 'type': 'IPv4',\n 'confidence': 100\n }\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn_later')\n def test_handle_ip_02(self, spawnl_mock):\n config = {\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config)\n\n inputs = ['a']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('a', indicator='1.1.1.1-1.1.1.2', value={\n 'type': 'IPv4',\n 'confidence': 100\n })\n a._handle_ip('1.1.1.1')\n\n ochannel.publish.reset_mock()\n a.filtered_withdraw('a', indicator='1.1.1.1-1.1.1.2')\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'withdraw',\n 'indicator': '1.1.1.1',\n 'value': {\n 'type': 'IPv4',\n 'confidence': 100\n }\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn_later')\n def test_handle_ip_03(self, spawnl_mock):\n config = {\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config)\n\n inputs = ['a']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('a', indicator='1.1.1.1-1.1.1.2', value={\n 'type': 'IPv4',\n 'confidence': 100\n })\n a.filtered_update('a', indicator='1.1.1.4-1.1.1.5', value={\n 'type': 'IPv4',\n 'confidence': 100\n })\n a._handle_ip('1.1.1.3')\n self.assertEqual(ochannel.publish.call_count, 0)\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn_later')\n def test_handle_url_01(self, spawnl_mock):\n config = {\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config)\n\n inputs = ['a']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n self.assertEqual(spawnl_mock.call_count, 2)\n\n a.filtered_update('a', indicator='www.example.com', value={\n 'type': 'domain',\n 'confidence': 100\n })\n self.assertEqual(a.length(), 1)\n\n a._handle_url('www.example.com/cgi/addressbook.php')\n self.assertEqual(a.table.num_indicators, 1)\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'update',\n 'indicator': 'www.example.com',\n 'value': {\n 'syslog_original_indicator':\n 'domain'+'www.example.com',\n 'type': 'domain',\n 'confidence': 100\n }\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn_later')\n def test_handle_url_02(self, spawnl_mock):\n config = {\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config)\n\n inputs = ['a']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('a', indicator='www.example.com', value={\n 'type': 'domain',\n 'confidence': 100\n })\n a._handle_url('www.example.com/cgi/addressbook.php')\n\n ochannel.publish.reset_mock()\n a.filtered_withdraw('a', indicator='www.example.com')\n self.assertEqual(a.table_indicators.num_indicators, 0)\n self.assertTrue(\n check_for_rpc(\n ochannel.publish.call_args_list,\n [\n {\n 'method': 'withdraw',\n 'indicator': 'www.example.com',\n 'value': {\n 'type': 'domain',\n 'confidence': 100\n }\n }\n ],\n all_here=True\n )\n )\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(socket, 'socket')\n def test_logstash_url(self, socket_socket, spawnl_mock):\n config = {\n 'logstash_host': '127.0.0.1'\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n mock_socket = mock.Mock()\n socket_socket.return_value = mock_socket\n\n a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config)\n\n inputs = ['a']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('a', indicator='www.example.com', value={\n 'type': 'domain',\n 'confidence': 100\n })\n a._handle_url(\n 'www.example.com/cgi/addressbook.php',\n message={\n 'session_id': 666\n }\n )\n\n self.assertEqual(mock_socket.connect.call_count, 1)\n self.assertEqual(mock_socket.sendall.call_count, 1)\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(socket, 'socket')\n def test_logstash_ip(self, socket_socket, spawnl_mock):\n config = {\n 'logstash_host': '127.0.0.1'\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n mock_socket = mock.Mock()\n socket_socket.return_value = mock_socket\n\n a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config)\n\n inputs = ['a']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('a', indicator='1.1.1.0-1.1.1.20', value={\n 'type': 'IPv4',\n 'confidence': 100\n })\n a._handle_ip('1.1.1.1', message={\n 'session_id': 666\n })\n\n self.assertEqual(mock_socket.connect.call_count, 1)\n self.assertEqual(mock_socket.sendall.call_count, 1)\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n\n @mock.patch.object(gevent, 'spawn_later')\n @mock.patch.object(socket, 'socket')\n def test_logstash_event_tags(self, socket_socket, spawnl_mock):\n config = {\n 'logstash_host': '127.0.0.1'\n }\n\n chassis = mock.Mock()\n\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n mock_socket = mock.Mock()\n socket_socket.return_value = mock_socket\n\n a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config)\n\n inputs = ['a']\n output = True\n\n a.connect(inputs, output)\n a.mgmtbus_initialize()\n a.start()\n\n a.filtered_update('a', indicator='1.1.1.0-1.1.1.20', value={\n 'type': 'IPv4',\n 'confidence': 100\n })\n a._handle_ip('1.1.1.1', message={\n 'session_id': 666,\n 'event.tags': [1, 2]\n })\n\n self.assertEqual(mock_socket.connect.call_count, 1)\n self.assertEqual(mock_socket.sendall.call_count, 1)\n self.assertEqual(\n 'session_id' in mock_socket.sendall.call_args[0][0],\n True\n )\n self.assertEqual(\n 'event.tags' in mock_socket.sendall.call_args[0][0],\n False\n )\n\n a.stop()\n\n a = None\n chassis = None\n rpcmock = None\n ochannel = None\n\n gc.collect()\n" }, { "path": "tests/test_ft_table.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT Table tests\n\nUnit tests for minemeld.ft.table\n\"\"\"\n\nimport unittest\nimport tempfile\nimport shutil\nimport random\nimport time\n\nimport minemeld.ft.table\n\nfrom nose.plugins.attrib import attr\n\nTABLENAME = tempfile.mktemp(prefix='minemeld.fttabletest')\nNUM_ELEMENTS = 10000\n\n\nclass MineMeldFTTableTests(unittest.TestCase):\n def setUp(self):\n try:\n shutil.rmtree(TABLENAME)\n except:\n pass\n\n def tearDown(self):\n try:\n shutil.rmtree(TABLENAME)\n except:\n pass\n\n def test_truncate(self):\n table = minemeld.ft.table.Table(TABLENAME)\n table.put('key', {'a': 1})\n table.close()\n table = None\n\n table = minemeld.ft.table.Table(TABLENAME)\n self.assertEqual(table.num_indicators, 1)\n table.close()\n table = None\n\n table = minemeld.ft.table.Table(TABLENAME, truncate=True)\n self.assertEqual(table.num_indicators, 0)\n table.close()\n\n def test_insert(self):\n table = minemeld.ft.table.Table(TABLENAME)\n table.create_index('a')\n\n for i in range(NUM_ELEMENTS):\n value = {'a': random.randint(0, 500)}\n key = 'i%d' % i\n table.put(key, value)\n\n self.assertEqual(table.num_indicators, NUM_ELEMENTS)\n table.close()\n\n def test_index_query(self):\n table = minemeld.ft.table.Table(TABLENAME)\n table.create_index('a')\n\n num_below_500 = 0\n for i in xrange(NUM_ELEMENTS):\n value = {'a': random.randint(0, 1000)}\n key = 'i%d' % i\n table.put(key, value)\n if value['a'] <= 500:\n num_below_500 += 1\n\n j = 0\n for k, v in table.query('a', from_key=0, to_key=500,\n include_value=True):\n j += 1\n\n self.assertEqual(j, num_below_500)\n table.close()\n\n def test_index_query_2(self):\n table = minemeld.ft.table.Table(TABLENAME)\n table.create_index('a')\n\n for i in xrange(NUM_ELEMENTS):\n value = {'a': 1483184218151+random.randint(600, 1000)}\n key = 'i%d' % i\n table.put(key, value)\n\n num_below_500 = 0\n for i in xrange(NUM_ELEMENTS):\n value = {'a': 1483184218151+random.randint(0, 1000)}\n key = 'i%d' % i\n table.put(key, value)\n if value['a'] <= 1483184218151+500:\n num_below_500 += 1\n\n j = 0\n for k, v in table.query('a', to_key=1483184218151+500,\n include_value=True):\n j += 1\n\n self.assertEqual(j, num_below_500)\n table.close()\n\n def test_index_query_3(self):\n table = minemeld.ft.table.Table(TABLENAME)\n table.create_index('a')\n\n for i in xrange(NUM_ELEMENTS):\n value = {'a': 1483184218151+random.randint(0, 500)}\n key = 'i%d' % i\n table.put(key, value)\n\n for i in xrange(NUM_ELEMENTS):\n key = 'i%d' % i\n table.delete(key)\n\n num_below_500 = 0\n for i in xrange(NUM_ELEMENTS):\n value = {'a': 1483184218151+random.randint(0, 1000)}\n key = 'i%d' % i\n table.put(key, value)\n if value['a'] <= 1483184218151+500:\n num_below_500 += 1\n\n j = 0\n for k, v in table.query('a', to_key=1483184218151+500,\n include_value=True):\n j += 1\n\n self.assertEqual(j, num_below_500)\n table.close()\n\n def test_query(self):\n table = minemeld.ft.table.Table(TABLENAME)\n table.create_index('a')\n\n for i in range(NUM_ELEMENTS):\n value = {'a': random.randint(0, 500)}\n key = 'i%d' % i\n table.put(key, value)\n\n j = 0\n for k, v in table.query(include_value=True):\n j += 1\n\n self.assertEqual(j, NUM_ELEMENTS)\n table.close()\n\n def test_exists(self):\n table = minemeld.ft.table.Table(TABLENAME)\n table.create_index('a')\n\n for i in range(NUM_ELEMENTS):\n value = {'a': random.randint(0, 500)}\n key = 'i%d' % i\n table.put(key, value)\n\n for i in range(NUM_ELEMENTS):\n j = random.randint(0, NUM_ELEMENTS-1)\n self.assertTrue(\n table.exists('i%d' % j),\n msg=\"i%d does not exists\" % j\n )\n table.close()\n\n def test_not_exists(self):\n table = minemeld.ft.table.Table(TABLENAME)\n table.create_index('a')\n\n for i in range(NUM_ELEMENTS):\n value = {'a': random.randint(0, 500)}\n key = 'i%d' % i\n table.put(key, value)\n\n for i in range(NUM_ELEMENTS):\n j = random.randint(NUM_ELEMENTS, 2*NUM_ELEMENTS)\n self.assertFalse(table.exists('i%d' % j))\n table.close()\n\n def test_update(self):\n table = minemeld.ft.table.Table(TABLENAME)\n table.create_index('a')\n\n table.put('k1', {'a': 1})\n table.put('k2', {'a': 1})\n table.put('k1', {'a': 2})\n\n ok = 0\n rk = None\n for k in table.query('a', from_key=0, to_key=1):\n rk = k\n ok += 1\n\n self.assertEqual(rk, 'k2')\n self.assertEqual(ok, 1)\n table.close()\n\n @attr('slow')\n def test_random(self):\n # create table\n table = minemeld.ft.table.Table(TABLENAME)\n table.create_index('a')\n\n # local dict\n d = {}\n\n # add 10000 elements to the table\n # with an 'a' attribute in range 0,500\n for i in range(NUM_ELEMENTS):\n value = {'a': random.randint(0, 500)}\n key = 'i%d' % i\n d[key] = value\n table.put(key, value)\n\n # check number of indicators added\n self.assertEqual(table.num_indicators, len(d.keys()))\n\n # check sorted query retrieval\n flatdict = sorted(d.items(), key=lambda x: x[1]['a'])\n j = 0\n for k, v in table.query('a', from_key=0, to_key=500,\n include_value=True):\n de = flatdict[j]\n self.assertEqual(de[1]['a'], v['a'])\n j = j+1\n\n # 1000 random add or delete\n for j in range(1000):\n op = random.randint(0, 1)\n\n if op == 0:\n # delete\n i = 'i%d' % random.randint(0, 2000)\n if i in d:\n del d[i]\n table.delete(i)\n elif op == 1:\n # add\n i = 'i%d' % random.randint(0, 2000)\n v = {'a': random.randint(0, 500)}\n table.put(i, v)\n d[i] = v\n\n # check num of indicators\n self.assertEqual(table.num_indicators, len(d.keys()))\n flatdict = sorted(d.items(), key=lambda x: x[1]['a'])\n j = 0\n for k, v in table.query('a', from_key=0, to_key=500,\n include_value=True):\n de = flatdict[j]\n # check sorting\n self.assertEqual(de[1]['a'], v['a'])\n j = j+1\n\n # close table\n table.close()\n table = None\n\n # reopen\n table = minemeld.ft.table.Table(TABLENAME)\n table.create_index('a')\n\n self.assertEqual(table.num_indicators, len(d.keys()))\n\n # check sort again\n flatdict = sorted(d.items(), key=lambda x: x[1]['a'])\n j = 0\n for k, v in table.query('a', from_key=0, to_key=500,\n include_value=True):\n de = flatdict[j]\n self.assertEqual(de[1]['a'], v['a'])\n j = j+1\n\n table.close()\n\n @attr('slow')\n def test_write(self):\n # create table\n table = minemeld.ft.table.Table(TABLENAME)\n\n # local dict\n d = {}\n\n t1 = time.time()\n for i in xrange(100000):\n value = {'a': random.randint(0, 500)}\n key = 'i%d' % i\n d[key] = value\n table.put(key, value)\n t2 = time.time()\n print 'TIME: Written %d elements in %s secs' % (100000, t2-t1)\n\n # check number of indicators added\n self.assertEqual(table.num_indicators, len(d.keys()))\n\n table.close()\n table = None\n" }, { "path": "tests/test_ft_taxii.py", "content": "# -*- coding: utf-8 -*-\n\n# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT TAXII tests\n\nUnit tests for minemeld.ft.taxii\n\"\"\"\n\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport unittest\nimport mock\n\nimport redis\nimport gevent\nimport greenlet\nimport time\nimport xmltodict\nimport os\nimport libtaxii.constants\nimport re\nimport lz4\nimport json\n\nimport minemeld.ft.taxii\nimport minemeld.ft\n\nFTNAME = 'testft-%d' % int(time.time())\n\nMYDIR = os.path.dirname(__file__)\n\n\nclass MockTaxiiContentBlock(object):\n def __init__(self, stix_xml):\n class _Binding(object):\n def __init__(self, id_):\n self.binding_id = id_\n\n self.content = stix_xml\n self.content_binding = _Binding(libtaxii.constants.CB_STIX_XML_111)\n\n\nclass MineMeldFTTaxiiTests(unittest.TestCase):\n @mock.patch.object(gevent, 'Greenlet')\n def test_taxiiclient_parse(self, glet_mock):\n config = {\n 'side_config': 'dummy.yml',\n 'ca_file': 'dummy.crt'\n }\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n b = minemeld.ft.taxii.TaxiiClient(FTNAME, chassis, config)\n\n inputs = []\n output = False\n\n b.connect(inputs, output)\n b.mgmtbus_initialize()\n\n b.start()\n\n testfiles = os.listdir(MYDIR)\n testfiles = filter(\n lambda x: x.startswith('test_ft_taxii_stix_package_'),\n testfiles\n )\n\n for t in testfiles:\n with open(os.path.join(MYDIR, t), 'r') as f:\n sxml = f.read()\n\n mo = re.match('test_ft_taxii_stix_package_([A-Za-z0-9]+)_([0-9]+)_.*', t)\n self.assertNotEqual(mo, None)\n type_ = mo.group(1)\n num_indicators = int(mo.group(2))\n\n stix_objects = {\n 'observables': {},\n 'indicators': {},\n 'ttps': {}\n }\n\n content_blocks = [\n MockTaxiiContentBlock(sxml)\n ]\n\n b._handle_content_blocks(\n content_blocks,\n stix_objects\n )\n\n params = {\n 'ttps': stix_objects['ttps'],\n 'observables': stix_objects['observables']\n }\n indicators = [[iid, iv, params] for iid, iv in stix_objects['indicators'].iteritems()]\n\n for i in indicators:\n result = b._process_item(i)\n\n self.assertEqual(len(result), num_indicators)\n\n if type_ != 'any':\n for r in result:\n self.assertEqual(r[1]['type'], type_)\n\n b.stop()\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_datafeed_init(self, glet_mock, SR_mock):\n config = {}\n chassis = mock.Mock()\n\n b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config)\n\n self.assertEqual(b.name, FTNAME)\n self.assertEqual(b.chassis, chassis)\n self.assertEqual(b.config, config)\n self.assertItemsEqual(b.inputs, [])\n self.assertEqual(b.output, None)\n self.assertEqual(b.redis_skey, FTNAME)\n self.assertEqual(b.redis_skey_chkp, FTNAME+'.chkp')\n self.assertEqual(b.redis_skey_value, FTNAME+'.value')\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_datafeed_update_ip(self, glet_mock, SR_mock):\n config = {}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config)\n\n inputs = ['a']\n output = False\n\n b.connect(inputs, output)\n b.mgmtbus_initialize()\n\n b.start()\n # __init__ + get chkp + delete chkp\n self.assertEqual(len(SR_mock.mock_calls), 6)\n SR_mock.reset_mock()\n SR_mock.return_value.zcard.return_value = 1\n\n # unicast\n b.filtered_update(\n 'a',\n indicator='1.1.1.1',\n value={\n 'type': 'IPv4',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n break\n else:\n self.fail(msg='hset not found')\n\n self.assertEqual(args[2].startswith('lz4'), True)\n stixdict = json.loads(lz4.uncompress(args[2][3:]))\n\n indicator = stixdict['indicators'][0]\n cyboxprops = indicator['observable']['object']['properties']\n self.assertEqual(cyboxprops['address_value'], '1.1.1.1')\n self.assertEqual(cyboxprops['xsi:type'], 'AddressObjectType')\n SR_mock.reset_mock()\n\n # CIDR\n b.filtered_update(\n 'a',\n indicator='1.1.1.0/24',\n value={\n 'type': 'IPv4',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n break\n else:\n self.fail(msg='hset not found')\n\n self.assertEqual(args[2].startswith('lz4'), True)\n stixdict = json.loads(lz4.uncompress(args[2][3:]))\n\n indicator = stixdict['indicators'][0]\n cyboxprops = indicator['observable']['object']['properties']\n self.assertEqual(cyboxprops['address_value'], '1.1.1.0/24')\n self.assertEqual(cyboxprops['xsi:type'], 'AddressObjectType')\n SR_mock.reset_mock()\n\n # fake range\n b.filtered_update(\n 'a',\n indicator='1.1.1.1-1.1.1.1',\n value={\n 'type': 'IPv4',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n break\n else:\n self.fail(msg='hset not found')\n\n self.assertEqual(args[2].startswith('lz4'), True)\n stixdict = json.loads(lz4.uncompress(args[2][3:]))\n\n indicator = stixdict['indicators'][0]\n cyboxprops = indicator['observable']['object']['properties']\n self.assertEqual(cyboxprops['address_value'], '1.1.1.1')\n self.assertEqual(cyboxprops['xsi:type'], 'AddressObjectType')\n SR_mock.reset_mock()\n\n # fake range 2\n b.filtered_update(\n 'a',\n indicator='1.1.1.0-1.1.1.31',\n value={\n 'type': 'IPv4',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n break\n else:\n self.fail(msg='hset not found')\n\n self.assertEqual(args[2].startswith('lz4'), True)\n stixdict = json.loads(lz4.uncompress(args[2][3:]))\n\n indicator = stixdict['indicators'][0]\n cyboxprops = indicator['observable']['object']['properties']\n self.assertEqual(cyboxprops['address_value'], '1.1.1.0/27')\n self.assertEqual(cyboxprops['xsi:type'], 'AddressObjectType')\n SR_mock.reset_mock()\n SR_mock.return_value.zcard.return_value = 1\n\n # real range\n b.filtered_update(\n 'a',\n indicator='1.1.1.0-1.1.1.33',\n value={\n 'type': 'IPv4',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n break\n else:\n self.fail(msg='hset not found')\n\n self.assertEqual(args[2].startswith('lz4'), True)\n stixdict = json.loads(lz4.uncompress(args[2][3:]))\n\n indicator = stixdict['indicators']\n cyboxprops = indicator[0]['observable']['object']['properties']\n self.assertEqual(cyboxprops['address_value'], '1.1.1.0/27')\n self.assertEqual(cyboxprops['xsi:type'], 'AddressObjectType')\n cyboxprops = indicator[1]['observable']['object']['properties']\n self.assertEqual(cyboxprops['address_value'], '1.1.1.32/31')\n self.assertEqual(cyboxprops['xsi:type'], 'AddressObjectType')\n SR_mock.reset_mock()\n\n b.stop()\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_datafeed_update_domain(self, glet_mock, SR_mock):\n config = {}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config)\n\n inputs = ['a']\n output = False\n\n b.connect(inputs, output)\n b.mgmtbus_initialize()\n\n b.start()\n # __init__ + get chkp + delete chkp\n self.assertEqual(len(SR_mock.mock_calls), 6)\n SR_mock.reset_mock()\n SR_mock.return_value.zcard.return_value = 1\n\n # unicast\n b.filtered_update(\n 'a',\n indicator='example.com',\n value={\n 'type': 'domain',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n break\n else:\n self.fail(msg='hset not found')\n\n self.assertEqual(args[2].startswith('lz4'), True)\n stixdict = json.loads(lz4.decompress(args[2][3:]))\n\n indicator = stixdict['indicators'][0]\n cyboxprops = indicator['observable']['object']['properties']\n self.assertEqual(cyboxprops['value'], 'example.com')\n self.assertEqual(cyboxprops['type'], 'FQDN')\n SR_mock.reset_mock()\n\n b.stop()\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_datafeed_update_url(self, glet_mock, SR_mock):\n config = {}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config)\n\n inputs = ['a']\n output = False\n\n b.connect(inputs, output)\n b.mgmtbus_initialize()\n\n b.start()\n # __init__ + get chkp + delete chkp\n self.assertEqual(len(SR_mock.mock_calls), 6)\n SR_mock.reset_mock()\n SR_mock.return_value.zcard.return_value = 1\n\n # unicast\n b.filtered_update(\n 'a',\n indicator='www.example.com/admin.php',\n value={\n 'type': 'URL',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n break\n else:\n self.fail(msg='hset not found')\n\n self.assertEqual(args[2].startswith('lz4'), True)\n stixdict = json.loads(lz4.decompress(args[2][3:]))\n\n indicator = stixdict['indicators'][0]\n cyboxprops = indicator['observable']['object']['properties']\n self.assertEqual(cyboxprops['type'], 'URL')\n self.assertEqual(cyboxprops['value'], 'www.example.com/admin.php')\n SR_mock.reset_mock()\n\n b.stop()\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_datafeed_unicode_url(self, glet_mock, SR_mock):\n config = {}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config)\n\n inputs = ['a']\n output = False\n\n b.connect(inputs, output)\n b.mgmtbus_initialize()\n\n b.start()\n # __init__ + get chkp + delete chkp\n self.assertEqual(len(SR_mock.mock_calls), 6)\n SR_mock.reset_mock()\n SR_mock.return_value.zcard.return_value = 1\n\n # unicast\n b.filtered_update(\n 'a',\n indicator=u'☃.net/påth',\n value={\n 'type': 'URL',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n break\n else:\n self.fail(msg='hset not found')\n\n self.assertEqual(args[2].startswith('lz4'), True)\n stixdict = json.loads(lz4.decompress(args[2][3:]))\n\n indicator = stixdict['indicators'][0]\n cyboxprops = indicator['observable']['object']['properties']\n self.assertEqual(cyboxprops['type'], 'URL')\n self.assertEqual(cyboxprops['value'], u'\\u2603.net/p\\xe5th')\n SR_mock.reset_mock()\n\n b.stop()\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_datafeed_overflow(self, glet_mock, SR_mock):\n config = {}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config)\n\n inputs = ['a']\n output = False\n\n b.connect(inputs, output)\n b.mgmtbus_initialize()\n\n b.start()\n # __init__ + get chkp + delete chkp\n self.assertEqual(len(SR_mock.mock_calls), 6)\n SR_mock.reset_mock()\n SR_mock.return_value.zcard.return_value = b.max_entries\n\n # unicast\n b.filtered_update(\n 'a',\n indicator=u'☃.net/påth',\n value={\n 'type': 'URL',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n self.fail(msg='hset found')\n\n self.assertEqual(b.statistics['drop.overflow'], 1)\n\n SR_mock.reset_mock()\n SR_mock.return_value.zcard.return_value = b.max_entries - 1\n\n # unicast\n b.filtered_update(\n 'a',\n indicator=u'☃.net/påth',\n value={\n 'type': 'URL',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n break\n else:\n self.fail(msg='hset not found')\n\n self.assertEqual(args[2].startswith('lz4'), True)\n stixdict = json.loads(lz4.decompress(args[2][3:]))\n\n indicator = stixdict['indicators'][0]\n cyboxprops = indicator['observable']['object']['properties']\n self.assertEqual(cyboxprops['type'], 'URL')\n self.assertEqual(cyboxprops['value'], u'\\u2603.net/p\\xe5th')\n\n b.stop()\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_datafeed_update_hash(self, glet_mock, SR_mock):\n config = {}\n chassis = mock.Mock()\n\n chassis.request_sub_channel.return_value = None\n ochannel = mock.Mock()\n chassis.request_pub_channel.return_value = ochannel\n chassis.request_rpc_channel.return_value = None\n rpcmock = mock.Mock()\n rpcmock.get.return_value = {'error': None, 'result': 'OK'}\n chassis.send_rpc.return_value = rpcmock\n\n b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config)\n\n inputs = ['a']\n output = False\n\n b.connect(inputs, output)\n b.mgmtbus_initialize()\n\n b.start()\n # __init__ + get chkp + delete chkp\n self.assertEqual(len(SR_mock.mock_calls), 6)\n SR_mock.reset_mock()\n SR_mock.return_value.zcard.return_value = 1\n\n # sha1\n b.filtered_update(\n 'a',\n indicator='a6a5418b4d67d9f3a33cbf184b25ac7f9fa87d33',\n value={\n 'type': 'sha1',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n break\n else:\n self.fail(msg='hset not found')\n\n self.assertEqual(args[2].startswith('lz4'), True)\n stixdict = json.loads(lz4.decompress(args[2][3:]))\n\n indicator = stixdict['indicators'][0]\n cyboxprops = indicator['observable']['object']['properties']\n self.assertEqual(cyboxprops['hashes'][0]['simple_hash_value'], 'a6a5418b4d67d9f3a33cbf184b25ac7f9fa87d33')\n self.assertEqual(cyboxprops['hashes'][0]['type']['value'], 'SHA1')\n SR_mock.reset_mock()\n\n # md5\n b.filtered_update(\n 'a',\n indicator='e23fadd6ceef8c618fc1c65191d846fa',\n value={\n 'type': 'md5',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n break\n else:\n self.fail(msg='hset not found')\n\n self.assertEqual(args[2].startswith('lz4'), True)\n stixdict = json.loads(lz4.decompress(args[2][3:]))\n\n indicator = stixdict['indicators'][0]\n cyboxprops = indicator['observable']['object']['properties']\n self.assertEqual(cyboxprops['hashes'][0]['simple_hash_value'], 'e23fadd6ceef8c618fc1c65191d846fa')\n self.assertEqual(cyboxprops['hashes'][0]['type']['value'], 'MD5')\n SR_mock.reset_mock()\n\n # sha256\n b.filtered_update(\n 'a',\n indicator='a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9',\n value={\n 'type': 'sha256',\n 'confidence': 100,\n 'share_level': 'green',\n 'sources': ['test.1']\n }\n )\n for call in SR_mock.mock_calls:\n name, args, kwargs = call\n if name == '().pipeline().__enter__().hset':\n break\n else:\n self.fail(msg='hset not found')\n\n self.assertEqual(args[2].startswith('lz4'), True)\n stixdict = json.loads(lz4.decompress(args[2][3:]))\n\n indicator = stixdict['indicators'][0]\n cyboxprops = indicator['observable']['object']['properties']\n self.assertEqual(cyboxprops['hashes'][0]['simple_hash_value'], 'a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9')\n self.assertEqual(cyboxprops['hashes'][0]['type']['value'], 'SHA256')\n SR_mock.reset_mock()\n\n b.stop()\n" }, { "path": "tests/test_ft_taxii_stix_package_IPv4_1_3.xml", "content": "\n\n \n mmtest1 watchlist that contains IP information.\n Indicators - Watchlist\n \n \n \n IP Watchlist\n Sample IP Address Indicator for this watchlist. This contains one indicator with a set of three IP addresses in the watchlist.\n \n \n \n 10.0.0.0\n \n \n \n \n \n\n" }, { "path": "tests/test_ft_taxii_stix_package_IPv4_3_1.xml", "content": "\n\n \n mmtest1 watchlist that contains IP information.\n Indicators - Watchlist\n \n \n \n IP Watchlist\n Sample IP Address Indicator for this watchlist. This contains one indicator with a set of three IP addresses in the watchlist.\n \n \n \n 10.0.0.0##comma##10.0.0.1##comma##10.0.0.2\n \n \n \n \n \n\n" }, { "path": "tests/test_ft_taxii_stix_package_IPv4_3_2.xml", "content": "\n\n \n mmtest1 watchlist that contains IP information.\n Indicators - Watchlist\n \n \n \n IP Watchlist\n Sample IP Address Indicator for this watchlist. This contains one indicator with a set of three IP addresses in the watchlist.\n \n \n \n 10.0.0.0##comma##10.0.0.1##comma##10.0.0.2\n \n \n \n \n \n\n" }, { "path": "tests/test_ft_taxii_stix_package_IPv6_3_1.xml", "content": "\n\n \n mmtest1 watchlist that contains IP information.\n Indicators - Watchlist\n \n \n \n IP Watchlist\n Sample IP Address Indicator for this watchlist. This contains one indicator with a set of three IP addresses in the watchlist.\n \n \n \n 2607:f8b0:4004:803::1015##comma##2607:f8b0:4004:803::1016##comma##2607:f8b0:4004:803::1017\n \n \n \n \n \n\n" }, { "path": "tests/test_ft_taxii_stix_package_IPv6_3_2.xml", "content": "\n\n \n mmtest1 watchlist that contains IP information.\n Indicators - Watchlist\n \n \n \n IP Watchlist\n Sample IP Address Indicator for this watchlist. This contains one indicator with a set of three IP addresses in the watchlist.\n \n \n \n 2607:f8b0:4004:803::1015##comma##2607:f8b0:4004:803::1016##comma##2607:f8b0:4004:803::1017\n \n \n \n \n \n\n" }, { "path": "tests/test_localdb.yml", "content": "- indicator: 1.1.1.1\n type: IPv4\n- indicator: 1.1.1.2\n type: IPv4\n" }, { "path": "tests/test_localdb2.yml", "content": "- indicator: 1.1.1.2\n type: IPv4\n- indicator: 1.1.1.3\n type: IPv4\n" }, { "path": "tests/test_run_config.py", "content": "# Copyright 2015 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT run config tests\n\nUnit tests for minemeld.run.config\n\"\"\"\n\nimport unittest\nimport mock\n\nimport os\nimport os.path\n\nimport minemeld.run.config\n\nMYDIR = os.path.dirname(__file__)\n\nos.environ['MINEMELD_PROTOTYPE_PATH'] = MYDIR\n\n\nclass MineMeldRunConfigTests(unittest.TestCase):\n def test_defaults_from_file(self):\n emptypath = os.path.join(MYDIR, 'empty.yml')\n\n config = minemeld.run.config.load_config(emptypath)\n\n self.assertEqual(config.fabric['class'], 'AMQP')\n self.assertEqual(config.fabric['config'], {'num_connections': 5})\n self.assertEqual(config.mgmtbus['transport']['class'], 'AMQP')\n self.assertEqual(config.mgmtbus['transport']['config'], {'num_connections': 1})\n self.assertEqual(config.mgmtbus['master'], {})\n self.assertEqual(config.mgmtbus['slave'], {})\n\n def test_prototype_1(self):\n protopath = os.path.join(MYDIR, 'test-prototype-1.yml')\n\n config = minemeld.run.config.load_config(protopath)\n\n self.assertEqual(config.nodes['testprototype']['class'], 'minemeld.ft.http.HttpFT')\n self.assertEqual(\n config.nodes['testprototype']['config'],\n {'useless1': 1}\n )\n\n def test_validate_config_1(self):\n config = {\n 'nodes': {\n 'n1': {\n 'inputs': ['n2', 'n3']\n },\n 'n2': {\n 'output': True\n }\n }\n }\n\n msgs = minemeld.run.config.validate_config(\n minemeld.run.config.MineMeldConfig.from_dict(config)\n )\n\n self.assertEqual(len(msgs), 3)\n\n def test_validate_config_2(self):\n config = {\n 'nodes': {\n 'n1': {\n 'inputs': ['n2']\n },\n 'n2': {\n 'output': False\n }\n }\n }\n\n msgs = minemeld.run.config.validate_config(\n minemeld.run.config.MineMeldConfig.from_dict(config)\n )\n\n self.assertEqual(len(msgs), 3)\n\n def test_validate_config_3(self):\n config = {\n 'nodes': {\n 'n1': {\n 'output': True,\n 'inputs': ['n2', 'n4']\n },\n 'n2': {\n 'output': True\n },\n 'n3': {\n 'output': True,\n 'inputs': ['n1']\n },\n 'n4': {\n 'output': True,\n 'inputs': ['n3']\n }\n }\n }\n\n msgs = minemeld.run.config.validate_config(\n minemeld.run.config.MineMeldConfig.from_dict(config)\n )\n\n self.assertEqual(len(msgs), 5)\n\n def test_validate_config_4(self):\n config = {\n 'nodes': {\n '../config/running-config.yml': {\n 'output': True\n },\n '': {\n 'output': True\n }\n }\n }\n\n msgs = minemeld.run.config.validate_config(\n minemeld.run.config.MineMeldConfig.from_dict(config)\n )\n\n self.assertEqual(len(msgs), 4)\n" }, { "path": "tests/test_startupplanner.py", "content": "# Copyright 2015-2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"FT run config tests\n\nUnit tests for minemeld.run.config\n\"\"\"\n\nimport unittest\nimport mock\n\nimport os\nimport os.path\n\nfrom minemeld.run.config import CHANGE_ADDED, CHANGE_DELETED, CHANGE_INPUT_ADDED, CHANGE_INPUT_DELETED\nfrom minemeld.run.config import MineMeldConfig, MineMeldConfigChange\nimport minemeld.startupplanner\n\n\nclass MineMeldStartupPlanner(unittest.TestCase):\n def test_subgraphs_1(self):\n nodes = {\n 'm1': {\n 'inputs': []\n },\n 'm2': {},\n 'm3': {},\n 'p1': {\n 'inputs': ['m1', 'm2']\n },\n 'p2': {\n 'inputs': ['m3']\n },\n 'o1': {\n 'inputs': ['m1']\n },\n 'o2': {\n 'inputs': ['p1']\n },\n 'o3': {\n 'inputs': ['p2']\n }\n }\n state_info = {\n 'm1': {\n 'checkpoint': None,\n 'is_source': True\n },\n 'm2': {\n 'checkpoint': None,\n 'is_source': True\n },\n 'm3': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'p1': {\n 'checkpoint': None,\n 'is_source': False\n },\n 'p2': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'o1': {\n 'checkpoint': None,\n 'is_source': False\n },\n 'o2': {\n 'checkpoint': None,\n 'is_source': False\n },\n 'o3': {\n 'checkpoint': 'a',\n 'is_source': False\n }\n }\n\n config = MineMeldConfig.from_dict(dict(nodes=nodes))\n config.compute_changes(config)\n\n self.assertEqual(len(config.changes), 0)\n\n plan = minemeld.startupplanner.plan(config, state_info)\n\n self.assertEqual(plan['m1'], 'reset')\n self.assertEqual(plan['m2'], 'reset')\n self.assertEqual(plan['p1'], 'reset')\n self.assertEqual(plan['o1'], 'reset')\n self.assertEqual(plan['o2'], 'reset')\n self.assertEqual(plan['m3'], 'initialize')\n self.assertEqual(plan['p2'], 'initialize')\n self.assertEqual(plan['o3'], 'initialize')\n\n def test_new_miner(self):\n nodes_before = {\n 'm1': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n nodes_after = {\n 'm1': {\n 'inputs': []\n },\n 'm2': {},\n 'p1': {\n 'inputs': ['m1', 'm2']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n state_info = {\n 'm1': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'm2': {\n 'checkpoint': None,\n 'is_source': True\n },\n 'p1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'o1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n }\n\n config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after))\n config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before))\n config_after.compute_changes(config_before)\n\n self.assertEqual(len(config_after.changes), 2)\n\n plan = minemeld.startupplanner.plan(config_after, state_info)\n\n self.assertEqual(plan['m1'], 'initialize')\n self.assertEqual(plan['m2'], 'reset')\n self.assertEqual(plan['p1'], 'initialize')\n self.assertEqual(plan['o1'], 'initialize')\n\n def test_removed_output(self):\n nodes_before = {\n 'm1': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n 'o2': {\n 'inputs': ['p1']\n },\n }\n nodes_after = {\n 'm1': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n state_info = {\n 'm1': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'm2': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'p1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'o1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n }\n\n config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after))\n config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before))\n config_after.compute_changes(config_before)\n\n self.assertEqual(len(config_after.changes), 1)\n\n plan = minemeld.startupplanner.plan(config_after, state_info)\n\n self.assertEqual(plan['m1'], 'initialize')\n self.assertEqual(plan['p1'], 'initialize')\n self.assertEqual(plan['o1'], 'initialize')\n\n def test_added_chain(self):\n nodes_after = {\n 'm1': {\n 'inputs': []\n },\n 'm2': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1', 'm2']\n },\n 'p2': {\n 'inputs': ['m1']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n 'o2': {\n 'inputs': ['p2']\n }\n }\n nodes_before = {\n 'm1': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1']\n },\n 'o1': {\n 'inputs': ['p1']\n }\n }\n state_info = {\n 'm1': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'm2': {\n 'checkpoint': None,\n 'is_source': True\n },\n 'p1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'p2': {\n 'checkpoint': None,\n 'is_source': False\n },\n 'o1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'o2': {\n 'checkpoint': None,\n 'is_source': False\n },\n }\n\n config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after))\n config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before))\n config_after.compute_changes(config_before)\n\n self.assertEqual(len(config_after.changes), 4)\n\n plan = minemeld.startupplanner.plan(config_after, state_info)\n\n self.assertEqual(plan['m1'], 'rebuild')\n self.assertEqual(plan['m2'], 'reset')\n self.assertEqual(plan['p1'], 'reset')\n self.assertEqual(plan['p2'], 'reset')\n self.assertEqual(plan['o1'], 'reset')\n self.assertEqual(plan['o2'], 'reset')\n\n def test_invalid_chain_1(self):\n nodes_after = {\n 'm1': {\n 'inputs': []\n },\n 'm2': {\n 'inputs': []\n },\n 'm3': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1', 'm2', 'm3']\n },\n 'p2': {\n 'inputs': ['m1']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n 'o2': {\n 'inputs': ['p2']\n }\n }\n nodes_before = {\n 'm1': {\n 'inputs': []\n },\n 'm2': {\n 'inputs': []\n },\n 'm3': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1', 'm2', 'm3']\n },\n 'o1': {\n 'inputs': ['p1']\n }\n }\n state_info = {\n 'm1': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'm2': {\n 'checkpoint': 'b',\n 'is_source': True\n },\n 'm3': {\n 'checkpoint': 'b',\n 'is_source': True\n },\n 'p1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'p2': {\n 'checkpoint': None,\n 'is_source': False\n },\n 'o1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'o2': {\n 'checkpoint': None,\n 'is_source': False\n },\n }\n\n config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after))\n config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before))\n config_after.compute_changes(config_before)\n\n self.assertEqual(len(config_after.changes), 2)\n\n plan = minemeld.startupplanner.plan(config_after, state_info)\n\n self.assertEqual(plan['m1'], 'reset')\n self.assertEqual(plan['m2'], 'rebuild')\n self.assertEqual(plan['m3'], 'rebuild')\n self.assertEqual(plan['p1'], 'reset')\n self.assertEqual(plan['p2'], 'reset')\n self.assertEqual(plan['o1'], 'reset')\n self.assertEqual(plan['o2'], 'reset')\n\n def test_invalid_node_1(self):\n nodes_before = {\n 'm1': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n nodes_after = {\n 'm1': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n state_info = {\n 'm1': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'p1': {\n 'checkpoint': 'b',\n 'is_source': False\n },\n 'o1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n }\n\n config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after))\n config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before))\n config_after.compute_changes(config_before)\n\n self.assertEqual(len(config_after.changes), 0)\n\n plan = minemeld.startupplanner.plan(config_after, state_info)\n\n self.assertEqual(plan['m1'], 'rebuild')\n self.assertEqual(plan['p1'], 'reset')\n self.assertEqual(plan['o1'], 'reset')\n\n def test_invalid_node_2(self):\n nodes_before = {\n 'm1': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n nodes_after = {\n 'm1': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n state_info = {\n 'm1': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'p1': {\n 'checkpoint': None,\n 'is_source': False\n },\n 'o1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n }\n\n config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after))\n config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before))\n config_after.compute_changes(config_before)\n\n self.assertEqual(len(config_after.changes), 0)\n\n plan = minemeld.startupplanner.plan(config_after, state_info)\n\n self.assertEqual(plan['m1'], 'rebuild')\n self.assertEqual(plan['p1'], 'reset')\n self.assertEqual(plan['o1'], 'reset')\n\n def test_invalid_node_3(self):\n nodes_before = {\n 'm1': {\n 'inputs': []\n },\n 'm2': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1', 'm2']\n },\n 'p2': {\n 'inputs': ['m1', 'm2']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n nodes_after = {\n 'm1': {\n 'inputs': []\n },\n 'm2': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1', 'm2']\n },\n 'p2': {\n 'inputs': ['m2']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n state_info = {\n 'm1': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'm2': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'p1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'p2': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'o1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n }\n\n config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after))\n config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before))\n config_after.compute_changes(config_before)\n\n self.assertEqual(len(config_after.changes), 1)\n\n plan = minemeld.startupplanner.plan(config_after, state_info)\n\n self.assertEqual(plan['m1'], 'rebuild')\n self.assertEqual(plan['m2'], 'rebuild')\n self.assertEqual(plan['p1'], 'reset')\n self.assertEqual(plan['p2'], 'reset')\n self.assertEqual(plan['o1'], 'reset')\n\n def test_existing_source_added(self):\n nodes_before = {\n 'm1': {\n 'inputs': []\n },\n 'm2': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n nodes_after = {\n 'm1': {\n 'inputs': []\n },\n 'm2': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1', 'm2']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n state_info = {\n 'm1': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'm2': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'p1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'o1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n }\n\n config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after))\n config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before))\n config_after.compute_changes(config_before)\n\n self.assertEqual(len(config_after.changes), 1)\n\n plan = minemeld.startupplanner.plan(config_after, state_info)\n\n self.assertEqual(plan['m1'], 'initialize')\n self.assertEqual(plan['m2'], 'rebuild')\n self.assertEqual(plan['p1'], 'initialize')\n self.assertEqual(plan['o1'], 'initialize')\n\n def test_non_existing_source_added(self):\n nodes_before = {\n 'm1': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n nodes_after = {\n 'm1': {\n 'inputs': []\n },\n 'm2': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1', 'm2']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n state_info = {\n 'm1': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'm2': {\n 'checkpoint': None,\n 'is_source': True\n },\n 'p1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'o1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n }\n\n config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after))\n config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before))\n config_after.compute_changes(config_before)\n\n self.assertEqual(len(config_after.changes), 2)\n\n plan = minemeld.startupplanner.plan(config_after, state_info)\n\n self.assertEqual(plan['m1'], 'initialize')\n self.assertEqual(plan['m2'], 'reset')\n self.assertEqual(plan['p1'], 'initialize')\n self.assertEqual(plan['o1'], 'initialize')\n\n def test_non_source_existing_input_added(self):\n nodes_before = {\n 'm1': {\n 'inputs': []\n },\n 'm2': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1']\n },\n 'p2': {\n 'inputs': ['m2']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n nodes_after = {\n 'm1': {\n 'inputs': []\n },\n 'm2': {\n 'inputs': []\n },\n 'p1': {\n 'inputs': ['m1', 'p2']\n },\n 'p2': {\n 'inputs': ['m2']\n },\n 'o1': {\n 'inputs': ['p1']\n },\n }\n state_info = {\n 'm1': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'm2': {\n 'checkpoint': 'a',\n 'is_source': True\n },\n 'p1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'p2': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n 'o1': {\n 'checkpoint': 'a',\n 'is_source': False\n },\n }\n\n config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after))\n config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before))\n config_after.compute_changes(config_before)\n\n self.assertEqual(len(config_after.changes), 1)\n\n plan = minemeld.startupplanner.plan(config_after, state_info)\n\n self.assertEqual(plan['m1'], 'rebuild')\n self.assertEqual(plan['m2'], 'rebuild')\n self.assertEqual(plan['p1'], 'reset')\n self.assertEqual(plan['p2'], 'reset')\n self.assertEqual(plan['o1'], 'reset')\n" }, { "path": "tests/test_traced_queryprocessor.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nUnit tests for minemeld.traced.queryprocessor\n\"\"\"\n\nimport gevent.monkey\ngevent.monkey.patch_all(thread=False, select=False)\n\nimport redis\nimport gevent\nimport greenlet\nimport unittest\nimport tempfile\nimport shutil\nimport random\nimport time\nimport mock\nimport json\nimport ujson\nimport logging\nimport guppy\nimport gc\n\nimport minemeld.traced.queryprocessor\n\nimport traced_mock\nimport comm_mock\n\nTABLENAME = tempfile.mktemp(prefix='minemeld.traced.storagetest')\n\nLOG = logging.getLogger(__name__)\n\nclass MineMeldTracedStorage(unittest.TestCase):\n def setUp(self):\n traced_mock.table_cleanup()\n traced_mock.query_cleanup()\n\n def tearDown(self):\n traced_mock.table_cleanup()\n traced_mock.query_cleanup()\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_query_1(self, glet_mock, SR_mock):\n store = traced_mock.store_factory()\n\n q = minemeld.traced.queryprocessor.Query(\n store,\n \"log\",\n 0, 0,\n 100,\n 'uuid-test',\n {}\n )\n q._run()\n\n self.assertGreater(len(SR_mock.mock_calls), 1)\n\n num_logs = 0\n eoq = False\n for call in SR_mock.mock_calls[1:]:\n name, args, kwargs = call\n self.assertEqual(name, '().publish')\n self.assertEqual(args[0], 'mm-traced-q.uuid-test')\n\n if args[1] == '':\n eoq = True\n else:\n line = json.loads(args[1])\n if 'log' in line:\n num_logs += 1\n\n self.assertEqual(num_logs, 0)\n self.assertEqual(eoq, True)\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_query_pos(self, glet_mock, SR_mock):\n store = traced_mock.store_factory()\n\n store.write(1*86400*1000, 'log0')\n\n q = minemeld.traced.queryprocessor.Query(\n store,\n \"log\",\n 3*86400*1000, 0,\n 100,\n 'uuid-test',\n {}\n )\n q._run()\n self.assertGreater(len(SR_mock.mock_calls), 1)\n\n num_logs = 0\n eoq = False\n for call in SR_mock.mock_calls[1:]:\n name, args, kwargs = call\n self.assertEqual(name, '().publish')\n self.assertEqual(args[0], 'mm-traced-q.uuid-test')\n\n if args[1] == '':\n eoq = True\n else:\n line = json.loads(args[1])\n if 'log' in line:\n num_logs += 1\n\n self.assertEqual(num_logs, 1)\n self.assertEqual(eoq, True)\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_query_and(self, glet_mock, SR_mock):\n store = traced_mock.store_factory()\n\n store.write(1*86400*1000, 'log0')\n store.write(1*86400*1000, 'log1')\n store.write(2*86400*1000, 'pog1')\n\n q = minemeld.traced.queryprocessor.Query(\n store,\n \"log -0\",\n 3*86400*1000, 0,\n 100,\n 'uuid-test',\n {}\n )\n q._run()\n LOG.debug(SR_mock.mock_calls)\n self.assertGreater(len(SR_mock.mock_calls), 1)\n\n num_logs = 0\n eoq = False\n for call in SR_mock.mock_calls[1:]:\n name, args, kwargs = call\n self.assertEqual(name, '().publish')\n self.assertEqual(args[0], 'mm-traced-q.uuid-test')\n\n if args[1] == '':\n eoq = True\n else:\n line = json.loads(args[1])\n if 'log' in line:\n num_logs += 1\n\n self.assertEqual(num_logs, 1)\n self.assertEqual(eoq, True)\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_query_fs1(self, glet_mock, SR_mock):\n store = traced_mock.store_factory()\n\n store.write(1*86400*1000, ujson.dumps({\n \"field1\": \"foo\",\n \"field2\": \"bar\",\n \"field3\": [\"foo\", \"bar\"],\n \"field4\": 12345678\n }))\n store.write(1*86400*1000, ujson.dumps({\n \"field1\": \"bar\",\n \"field2\": \"foo\",\n \"field3\": [\"foo\", \"bar\"],\n \"field4\": 12345679\n }))\n\n q = minemeld.traced.queryprocessor.Query(\n store,\n \"field1:foo\",\n 3*86400*1000, 0,\n 100,\n 'uuid-test',\n {}\n )\n q._run()\n LOG.debug(SR_mock.mock_calls)\n self.assertGreater(len(SR_mock.mock_calls), 1)\n\n num_logs = 0\n eoq = False\n for call in SR_mock.mock_calls[1:]:\n name, args, kwargs = call\n self.assertEqual(name, '().publish')\n self.assertEqual(args[0], 'mm-traced-q.uuid-test')\n\n if args[1] == '':\n eoq = True\n else:\n line = json.loads(args[1])\n if 'log' in line:\n num_logs += 1\n\n self.assertEqual(num_logs, 1)\n self.assertEqual(eoq, True)\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_query_fs2(self, glet_mock, SR_mock):\n store = traced_mock.store_factory()\n\n store.write(1*86400*1000, ujson.dumps({\n \"field1\": \"foo\",\n \"field2\": \"bar\",\n \"field3\": [\"foo\", \"bar\"],\n \"field4\": 12345678\n }))\n store.write(1*86400*1000, ujson.dumps({\n \"field1\": \"bar\",\n \"field2\": \"foo\",\n \"field3\": [\"foo\", \"bar\"],\n \"field4\": 12345679\n }))\n\n q = minemeld.traced.queryprocessor.Query(\n store,\n \"field1:foo field2:foo\",\n 3*86400*1000, 0,\n 100,\n 'uuid-test',\n {}\n )\n q._run()\n LOG.debug(SR_mock.mock_calls)\n self.assertGreater(len(SR_mock.mock_calls), 1)\n\n num_logs = 0\n eoq = False\n for call in SR_mock.mock_calls[1:]:\n name, args, kwargs = call\n self.assertEqual(name, '().publish')\n self.assertEqual(args[0], 'mm-traced-q.uuid-test')\n\n if args[1] == '':\n eoq = True\n else:\n line = json.loads(args[1])\n if 'log' in line:\n num_logs += 1\n\n self.assertEqual(num_logs, 0)\n self.assertEqual(eoq, True)\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_query_fs3(self, glet_mock, SR_mock):\n store = traced_mock.store_factory()\n\n store.write(1*86400*1000, ujson.dumps({\n \"field1\": \"foo\",\n \"field2\": \"bar\",\n \"field3\": [\"foo\", \"bar\"],\n \"field4\": 12345678\n }))\n store.write(1*86400*1000, ujson.dumps({\n \"field1\": \"bar\",\n \"field2\": \"foo\",\n \"field3\": [\"foo\", \"bar\"],\n \"field4\": 12345679\n }))\n\n q = minemeld.traced.queryprocessor.Query(\n store,\n \"field3:bar FIELD2:foo\",\n 3*86400*1000, 0,\n 100,\n 'uuid-test',\n {}\n )\n q._run()\n LOG.debug(SR_mock.mock_calls)\n self.assertGreater(len(SR_mock.mock_calls), 1)\n\n num_logs = 0\n eoq = False\n for call in SR_mock.mock_calls[1:]:\n name, args, kwargs = call\n self.assertEqual(name, '().publish')\n self.assertEqual(args[0], 'mm-traced-q.uuid-test')\n\n if args[1] == '':\n eoq = True\n else:\n line = json.loads(args[1])\n if 'log' in line:\n num_logs += 1\n\n self.assertEqual(num_logs, 1)\n self.assertEqual(eoq, True)\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_query_fs4(self, glet_mock, SR_mock):\n store = traced_mock.store_factory()\n\n store.write(1*86400*1000, ujson.dumps({\n \"field1\": \"foo\",\n \"field2\": \"bar\",\n \"field3\": [\"foo\", \"bar\"],\n \"field4\": 12345678\n }))\n store.write(1*86400*1000, ujson.dumps({\n \"field1\": \"bar\",\n \"field2\": \"foo\",\n \"field3\": [\"foo\", \"bar\"],\n \"field4\": 12345679\n }))\n\n q = minemeld.traced.queryprocessor.Query(\n store,\n \"field3:foo FIELD3:Bar\",\n 3*86400*1000, 0,\n 100,\n 'uuid-test',\n {}\n )\n q._run()\n LOG.debug(SR_mock.mock_calls)\n self.assertGreater(len(SR_mock.mock_calls), 1)\n\n num_logs = 0\n eoq = False\n for call in SR_mock.mock_calls[1:]:\n name, args, kwargs = call\n self.assertEqual(name, '().publish')\n self.assertEqual(args[0], 'mm-traced-q.uuid-test')\n\n if args[1] == '':\n eoq = True\n else:\n line = json.loads(args[1])\n if 'log' in line:\n num_logs += 1\n\n self.assertEqual(num_logs, 2)\n self.assertEqual(eoq, True)\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_query_fs5(self, glet_mock, SR_mock):\n store = traced_mock.store_factory()\n\n store.write(1*86400*1000, ujson.dumps({\n \"field1\": \"1foo1\",\n \"field2\": \"2bar2\",\n \"field3\": [\"5foo5\", \"6bar6\"],\n \"field4\": 12345678\n }))\n store.write(1*86400*1000, ujson.dumps({\n \"field1\": \"3bar3\",\n \"field2\": \"4foo4\",\n \"field3\": [\"8foo8\", \"7bar7\"],\n \"field4\": 12345679\n }))\n\n q = minemeld.traced.queryprocessor.Query(\n store,\n \"field3:foo -field4:679\",\n 3*86400*1000, 0,\n 100,\n 'uuid-test',\n {}\n )\n q._run()\n LOG.debug(SR_mock.mock_calls)\n self.assertGreater(len(SR_mock.mock_calls), 1)\n\n num_logs = 0\n eoq = False\n for call in SR_mock.mock_calls[1:]:\n name, args, kwargs = call\n self.assertEqual(name, '().publish')\n self.assertEqual(args[0], 'mm-traced-q.uuid-test')\n\n if args[1] == '':\n eoq = True\n else:\n line = json.loads(args[1])\n if 'log' in line:\n num_logs += 1\n\n self.assertEqual(num_logs, 1)\n self.assertEqual(eoq, True)\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_query_wcard(self, glet_mock, SR_mock):\n store = traced_mock.store_factory()\n\n store.write(1*86400*1000, ujson.dumps({\n \"field1\": \"1*oo1\",\n \"field2\": \"2*ar2\",\n \"field3\": [\"5foo5\", \"6bar6\"],\n \"field4\": 12345678\n }))\n store.write(1*86400*1000, ujson.dumps({\n \"field1\": \"3bar3\",\n \"field2\": \"4foo4\",\n \"field3\": [\"8*oo8\", \"7bar7\"],\n \"field4\": 12345678\n }))\n\n q = minemeld.traced.queryprocessor.Query(\n store,\n \"field3:*oo -field4:67?\",\n 3*86400*1000, 0,\n 100,\n 'uuid-test',\n {}\n )\n q._run()\n LOG.debug(SR_mock.mock_calls)\n self.assertGreater(len(SR_mock.mock_calls), 1)\n\n num_logs = 0\n eoq = False\n for call in SR_mock.mock_calls[1:]:\n name, args, kwargs = call\n self.assertEqual(name, '().publish')\n self.assertEqual(args[0], 'mm-traced-q.uuid-test')\n\n if args[1] == '':\n eoq = True\n else:\n line = json.loads(args[1])\n if 'log' in line:\n num_logs += 1\n\n self.assertEqual(num_logs, 1)\n self.assertEqual(eoq, True)\n\n @mock.patch.object(redis, 'StrictRedis')\n @mock.patch.object(gevent, 'Greenlet')\n def test_query_empty(self, glet_mock, SR_mock):\n store = traced_mock.store_factory()\n\n q = minemeld.traced.queryprocessor.Query(\n store,\n \"field3:foo -field4:679\",\n 3*86400*1000, 0,\n 100,\n 'uuid-test',\n {}\n )\n q._run()\n\n num_logs = 0\n eoq = False\n for call in SR_mock.mock_calls[1:]:\n name, args, kwargs = call\n self.assertEqual(name, '().publish')\n self.assertEqual(args[0], 'mm-traced-q.uuid-test')\n\n if args[1] == '':\n eoq = True\n else:\n line = json.loads(args[1])\n if 'log' in line:\n num_logs += 1\n\n self.assertEqual(num_logs, 0)\n self.assertEqual(eoq, True)\n\n @mock.patch.object(minemeld.traced.queryprocessor, 'Query', side_effect=traced_mock.query_factory)\n def test_queryprocessor_1(self, query_mock):\n comm = comm_mock.comm_factory({})\n store = traced_mock.store_factory()\n\n qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)\n\n self.assertEqual(\n comm.rpc_server_channels[0]['name'],\n minemeld.traced.queryprocessor.QUERY_QUEUE\n )\n self.assertEqual(\n comm.rpc_server_channels[0]['allowed_methods'],\n ['query', 'kill_query']\n )\n\n qp.query('uuid-test-1', \"test query\")\n self.assertEqual(len(traced_mock.MOCK_QUERIES), 1)\n\n qp.query('uuid-test-2', \"test query\")\n self.assertEqual(len(traced_mock.MOCK_QUERIES), 2)\n\n gevent.sleep(0)\n\n traced_mock.MOCK_QUERIES[0].finish_event.set()\n\n gevent.sleep(0)\n\n qp.stop()\n gevent.sleep(0)\n gevent.sleep(0)\n\n self.assertEqual(traced_mock.MOCK_QUERIES[0].get(), None)\n self.assertIsInstance(\n traced_mock.MOCK_QUERIES[1].get(),\n greenlet.GreenletExit\n )\n self.assertNotIn('uuid-test-1', store.release_alls)\n self.assertIn('uuid-test-2', store.release_alls)\n\n @mock.patch.object(minemeld.traced.queryprocessor, 'Query', side_effect=traced_mock.query_factory)\n def test_queryprocessor_2(self, query_mock):\n comm = comm_mock.comm_factory({})\n store = traced_mock.store_factory()\n\n qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)\n\n self.assertEqual(\n comm.rpc_server_channels[0]['name'],\n minemeld.traced.queryprocessor.QUERY_QUEUE\n )\n self.assertEqual(\n comm.rpc_server_channels[0]['allowed_methods'],\n ['query', 'kill_query']\n )\n\n qp.query('uuid-test-1', \"test query\")\n self.assertEqual(len(traced_mock.MOCK_QUERIES), 1)\n\n qp.query('uuid-test-2', \"bad\")\n self.assertEqual(len(traced_mock.MOCK_QUERIES), 2)\n\n gevent.sleep(0)\n\n traced_mock.MOCK_QUERIES[0].finish_event.set()\n traced_mock.MOCK_QUERIES[1].finish_event.set()\n\n gevent.sleep(0)\n\n qp.stop()\n gevent.sleep(0)\n gevent.sleep(0)\n\n self.assertEqual(traced_mock.MOCK_QUERIES[0].get(), None)\n self.assertRaises(\n RuntimeError,\n traced_mock.MOCK_QUERIES[1].get\n )\n self.assertNotIn('uuid-test-1', store.release_alls)\n self.assertIn('uuid-test-2', store.release_alls)\n\n @mock.patch.object(minemeld.traced.queryprocessor, 'Query', side_effect=traced_mock.query_factory)\n def test_queryprocessor_3(self, query_mock):\n comm = comm_mock.comm_factory({})\n store = traced_mock.store_factory()\n\n qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)\n\n self.assertEqual(\n comm.rpc_server_channels[0]['name'],\n minemeld.traced.queryprocessor.QUERY_QUEUE\n )\n self.assertEqual(\n comm.rpc_server_channels[0]['allowed_methods'],\n ['query', 'kill_query']\n )\n\n qp.query('uuid-test-1', \"test query\")\n gevent.sleep(0)\n\n qp.kill_query('uuid-test-1')\n gevent.sleep(0)\n\n self.assertIsInstance(\n traced_mock.MOCK_QUERIES[0].get(),\n greenlet.GreenletExit\n )\n self.assertEqual(\n len(qp.queries),\n 0\n )\n self.assertIn('uuid-test-1', store.release_alls)\n\n qp.stop()\n gevent.sleep(0)\n\n @mock.patch.object(minemeld.traced.queryprocessor, 'Query', side_effect=traced_mock.query_factory)\n def test_queryprocessor_4(self, query_mock):\n comm = comm_mock.comm_factory({})\n store = traced_mock.store_factory()\n\n qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store)\n\n self.assertEqual(\n comm.rpc_server_channels[0]['name'],\n minemeld.traced.queryprocessor.QUERY_QUEUE\n )\n self.assertEqual(\n comm.rpc_server_channels[0]['allowed_methods'],\n ['query', 'kill_query']\n )\n\n qp.stop()\n gevent.sleep(0)\n\n self.assertRaises(\n RuntimeError,\n qp.query,\n 'test-uuid-1', \"test\"\n )\n self.assertRaises(\n RuntimeError,\n qp.kill_query,\n 'test-uuid-1'\n )\n\n @mock.patch.object(minemeld.traced.queryprocessor, 'Query', side_effect=traced_mock.query_factory)\n def test_queryprocessor_5(self, query_mock):\n comm = comm_mock.comm_factory({})\n store = traced_mock.store_factory()\n\n qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store, {'max_concurrency': 2})\n\n qp.query('uuid-test-1', \"test query\")\n gevent.sleep(0)\n\n qp.query('uuid-test-2', \"test query\")\n gevent.sleep(0)\n\n self.assertRaises(\n RuntimeError,\n qp.query,\n 'uuid-test-3', \"test query\"\n )\n gevent.sleep(0)\n\n traced_mock.MOCK_QUERIES[0].finish_event.set()\n gevent.sleep(0.2)\n self.assertEqual(len(qp.queries), 1)\n\n self.assertEqual(\n qp.query('uuid-test-4', \"test query\"),\n 'OK'\n )\n gevent.sleep(0)\n\n qp.stop()\n gevent.sleep(0)\n" }, { "path": "tests/test_traced_storage.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nUnit tests for minemeld.traced.storage\n\"\"\"\n\nimport unittest\nimport tempfile\nimport shutil\nimport random\nimport time\nimport mock\nimport logging\n\nfrom nose.plugins.attrib import attr\n\nimport minemeld.traced.storage\n\nimport traced_mock\n\nTABLENAME = tempfile.mktemp(prefix='minemeld.traced.storagetest')\n\nLOG = logging.getLogger(__name__)\n\n\nclass MineMeldTracedStorage(unittest.TestCase):\n def setUp(self):\n traced_mock.table_cleanup()\n\n try:\n shutil.rmtree(TABLENAME)\n except:\n pass\n\n def tearDown(self):\n traced_mock.table_cleanup()\n\n try:\n shutil.rmtree(TABLENAME)\n except:\n pass\n\n def test_table_constructor(self):\n self.assertRaises(\n minemeld.traced.storage.TableNotFound,\n minemeld.traced.storage.Table, TABLENAME, create_if_missing=False\n )\n\n table = minemeld.traced.storage.Table(TABLENAME, create_if_missing=True)\n self.assertEqual(table.max_counter, -1)\n\n table.close()\n table = None\n\n table = minemeld.traced.storage.Table(TABLENAME, create_if_missing=False)\n self.assertEqual(table.max_counter, -1)\n\n table.close()\n table = None\n\n def test_table_write(self):\n table = minemeld.traced.storage.Table(TABLENAME, create_if_missing=True)\n table.put('%016x' % 0, 'value0')\n self.assertEqual(table.max_counter, 0)\n table.close()\n table = None\n\n table = minemeld.traced.storage.Table(TABLENAME, create_if_missing=False)\n iterator = table.backwards_iterator(1, 0xFFFFFFFFFFFFFFFF)\n ts, line = next(iterator)\n self.assertEqual(line, 'value0')\n self.assertEqual(int(ts[:16], 16), 0)\n self.assertEqual(int(ts[16:], 16), 0)\n self.assertRaises(StopIteration, next, iterator)\n table.close()\n table = None\n\n def test_table_references(self):\n table = minemeld.traced.storage.Table(TABLENAME, create_if_missing=True)\n self.assertEqual(table.ref_count(), 0)\n\n table.add_reference('ref1')\n self.assertEqual(table.ref_count(), 1)\n\n table.add_reference('ref2')\n self.assertEqual(table.ref_count(), 2)\n\n table.remove_reference('ref1')\n self.assertEqual(table.ref_count(), 1)\n\n table.remove_reference('ref1')\n self.assertEqual(table.ref_count(), 1)\n\n table.remove_reference('ref2')\n self.assertEqual(table.ref_count(), 0)\n\n def test_table_oldest(self):\n old_ = '%016x' % (3*86400)\n new_ = '%016x' % (4*86400)\n\n oldest = minemeld.traced.storage.Table.oldest_table()\n self.assertEqual(oldest, None)\n\n table = minemeld.traced.storage.Table(old_, create_if_missing=True)\n table.close()\n\n table = minemeld.traced.storage.Table(new_, create_if_missing=True)\n table.close()\n\n oldest = minemeld.traced.storage.Table.oldest_table()\n self.assertEqual(oldest, old_)\n\n shutil.rmtree(old_)\n shutil.rmtree(new_)\n\n def test_store_simple(self):\n store = minemeld.traced.storage.Store()\n store.stop()\n self.assertEqual(len(store.current_tables), 0)\n\n @mock.patch.object(minemeld.traced.storage, 'Table', side_effect=traced_mock.table_factory)\n def test_store_write(self, table_mock):\n store = minemeld.traced.storage.Store()\n store.write(0*86400*1000, 'log0')\n self.assertEqual(traced_mock.MOCK_TABLES[0].name, '%016x' % 0)\n\n store.write(1*86400*1000, 'log1')\n self.assertEqual(traced_mock.MOCK_TABLES[1].name, '%016x' % (86400*1))\n\n store.write(2*86400*1000, 'log2')\n self.assertEqual(traced_mock.MOCK_TABLES[2].name, '%016x' % (86400*2))\n\n store.write(3*86400*1000, 'log3')\n self.assertEqual(traced_mock.MOCK_TABLES[3].name, '%016x' % (86400*3))\n\n store.write(4*86400*1000, 'log4')\n self.assertEqual(traced_mock.MOCK_TABLES[4].name, '%016x' % (86400*4))\n\n store.write(5*86400*1000, 'log5')\n self.assertEqual(traced_mock.MOCK_TABLES[5].name, '%016x' % (86400*5))\n self.assertNotIn('%016x' % 0, store.current_tables)\n\n store.write(6*86400*1000, 'log6')\n self.assertEqual(traced_mock.MOCK_TABLES[6].name, '%016x' % (86400*6))\n self.assertNotIn('%016x' % 86400, store.current_tables)\n\n store.stop()\n self.assertEqual(len(store.current_tables), 0)\n\n @mock.patch.object(minemeld.traced.storage, 'Table', side_effect=traced_mock.table_factory)\n def test_store_iterate_backwards(self, table_mock):\n _oldest_table_mock = mock.MagicMock(side_effect=traced_mock.MockTable.oldest_table)\n table_mock.attach_mock(_oldest_table_mock, 'oldest_table')\n\n store = minemeld.traced.storage.Store()\n store.write(1*86400*1000, 'log0')\n store.write(2*86400*1000, 'log1')\n store.write(3*86400*1000, 'log2')\n store.write(4*86400*1000, 'log3')\n store.write(5*86400*1000, 'log4')\n self.assertEqual(minemeld.traced.storage.Table.oldest_table(), '%016x' % 86400)\n\n iterator = store.iterate_backwards(\n ref='test-iter1',\n timestamp=6*86400*1000,\n counter=0xFFFFFFFFFFFFFFFF\n )\n self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-07')\n self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-06')\n self.assertEqual(next(iterator)['log'], 'log4')\n self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-05')\n self.assertEqual(next(iterator)['log'], 'log3')\n self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-04')\n self.assertEqual(next(iterator)['log'], 'log2')\n self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-03')\n self.assertEqual(next(iterator)['log'], 'log1')\n self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-02')\n self.assertEqual(next(iterator)['log'], 'log0')\n self.assertEqual(next(iterator)['msg'], 'No more logs to check')\n self.assertRaises(StopIteration, next, iterator)\n\n store.stop()\n store.stop() # just for coverage\n\n @mock.patch.object(minemeld.traced.storage, 'Table', side_effect=traced_mock.table_factory)\n def test_store_iterate_backwards_2(self, table_mock):\n _oldest_table_mock = mock.MagicMock(side_effect=traced_mock.MockTable.oldest_table)\n table_mock.attach_mock(_oldest_table_mock, 'oldest_table')\n\n store = minemeld.traced.storage.Store()\n store.write(0*86400*1000, 'log0')\n store.write(2*86400*1000, 'log1')\n self.assertEqual(minemeld.traced.storage.Table.oldest_table(), '%016x' % 0)\n\n iterator = store.iterate_backwards(\n ref='test-iter1',\n timestamp=3*86400*1000,\n counter=0xFFFFFFFFFFFFFFFF\n )\n self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-04')\n self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-03')\n self.assertEqual(next(iterator)['log'], 'log1')\n self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-02')\n self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-01')\n self.assertEqual(next(iterator)['log'], 'log0')\n self.assertEqual(next(iterator)['msg'], 'We haved reached the origins of time')\n self.assertRaises(StopIteration, next, iterator)\n\n store.stop()\n\n @mock.patch.object(minemeld.traced.storage, 'Table', side_effect=traced_mock.table_factory)\n def test_store_iterate_backwards_empty(self, table_mock):\n _oldest_table_mock = mock.MagicMock(side_effect=traced_mock.MockTable.oldest_table)\n table_mock.attach_mock(_oldest_table_mock, 'oldest_table')\n\n store = minemeld.traced.storage.Store()\n self.assertEqual(minemeld.traced.storage.Table.oldest_table(), None)\n\n iterator = store.iterate_backwards(\n ref='test-iter1',\n timestamp=3*86400*1000,\n counter=0xFFFFFFFFFFFFFFFF\n )\n self.assertEqual(next(iterator)['msg'], 'No more logs to check')\n self.assertRaises(StopIteration, next, iterator)\n table_mock.assert_not_called()\n\n store.stop()\n\n @attr('slow')\n def test_stress_1(self):\n num_lines = 200000\n store = minemeld.traced.storage.Store()\n\n t1 = time.time()\n for j in xrange(num_lines):\n value = '{ \"log\": %d }' % random.randint(0, 0xFFFFFFFF)\n t2 = time.time()\n dt = t2-t1\n\n t1 = time.time()\n for j in xrange(num_lines):\n value = '{ \"log\": %d }' % random.randint(0, 0xFFFFFFFF)\n store.write(j, value)\n t2 = time.time()\n print \"TIME: Inserted %d lines in %d sec\" % (num_lines, (t2-t1-dt))\n\n store.stop()\n shutil.rmtree('1970-01-01')\n" }, { "path": "tests/test_traced_writer.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nUnit tests for minemeld.traced.writer\n\"\"\"\n\nimport unittest\nimport tempfile\nimport shutil\nimport random\nimport time\nimport mock\nimport logging\nimport ujson\n\nimport minemeld.traced.writer\n\nimport comm_mock\nimport traced_mock\n\nLOG = logging.getLogger(__name__)\n\n\nclass MineMeldTracedStorage(unittest.TestCase):\n def test_writer(self):\n config = {}\n comm = comm_mock.comm_factory(config)\n store = traced_mock.store_factory()\n\n writer = minemeld.traced.writer.Writer(comm, store, 'TESTTOPIC')\n self.assertEqual(comm.sub_channels[0]['topic'], 'TESTTOPIC')\n self.assertEqual(comm.sub_channels[0]['allowed_methods'], ['log'])\n\n writer.log(0, log='testlog')\n self.assertEqual(store.writes[0]['timestamp'], 0)\n self.assertEqual(store.writes[0]['log'], ujson.dumps({'log': 'testlog'}))\n\n writer.stop()\n writer.log(0, log='testlog')\n self.assertEqual(len(store.writes), 1)\n\n writer.stop() # just for coverage\n" }, { "path": "tests/testproto.yml", "content": "prototypes:\n test:\n config:\n useless1: 1\n class: minemeld.ft.http.HttpFT\n" }, { "path": "tests/traced_mock.py", "content": "# Copyright 2016 Palo Alto Networks, Inc\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\"\"\"\nThis module implements mock classes for minemed.traced tests\n\"\"\"\n\nimport gevent\nimport gevent.event\n\nimport logging\n\nfrom minemeld.traced.storage import TableNotFound\n\nLOG = logging.getLogger(__name__)\n\n\nCLOCK = -1\ndef _get_clock():\n global CLOCK\n\n CLOCK += 1\n return CLOCK\n\nMOCK_TABLES = []\n\nclass MockTable(object):\n def __init__(self, name, create_if_missing=True):\n self.name = name\n self.create_if_missing = create_if_missing\n\n self.last_used = None\n self.refs = []\n\n self.db_open = True\n self.db = {}\n\n self.max_counter = -1\n\n def add_reference(self, refid):\n self.refs.append(refid)\n\n def remove_reference(self, refid):\n try:\n self.refs.remove(refid)\n\n except ValueError:\n pass\n\n def ref_count(self):\n return len(self.refs)\n\n def put(self, key, value):\n self.last_used = _get_clock()\n\n self.max_counter += 1\n new_max_counter = '%016x' % self.max_counter\n\n self.db[key+new_max_counter] = value\n\n def backwards_iterator(self, timestamp, counter):\n starting_key = '%016x%016x' % (timestamp, counter)\n items = [[k, v] for k, v in self.db.iteritems() if k <= starting_key]\n items = sorted(items, cmp=lambda x, y: cmp(x[0], y[0]), reverse=True)\n return items\n\n def close(self):\n self.db_open = False\n\n @staticmethod\n def oldest_table():\n tables = [t.name for t in MOCK_TABLES]\n LOG.debug(tables)\n if len(tables) == 0:\n return None\n\n return sorted(tables)[0]\n\ndef table_factory(name, create_if_missing=True):\n table = next((t for t in MOCK_TABLES if t.name == name), None)\n if table is not None:\n return table\n\n if not create_if_missing:\n raise TableNotFound()\n\n mt = MockTable(name, create_if_missing=create_if_missing)\n MOCK_TABLES.append(mt)\n return mt\n\ndef table_cleanup():\n global MOCK_TABLES\n MOCK_TABLES = []\n\nclass MockStore(object):\n def __init__(self, config=None):\n if config is None:\n config = {}\n\n self.config = config\n self.writes = []\n self.db = {}\n self.counter = 0\n self.release_alls = []\n\n def write(self, timestamp, log):\n self.writes.append({\n 'timestamp': timestamp,\n 'log': log\n })\n self.db['%016x%016x' % (timestamp, self.counter)] = log\n self.counter += 1\n\n def iterate_backwards(self, ref, timestamp, counter):\n starting_key = '%016x%016x' % (timestamp, counter)\n items = [[k, v] for k, v in self.db.iteritems() if k <= starting_key]\n items = sorted(items, cmp=lambda x, y: cmp(x[0], y[0]), reverse=True)\n\n for c, i in enumerate(items):\n if c % 1 == 0:\n yield {'msg': 'test message'}\n yield {'timestamp': i[0], 'log': i[1]}\n\n def release_all(self, ref):\n self.release_alls.append(ref)\n\ndef store_factory(config=None):\n return MockStore(config=config)\n\nMOCK_QUERIES = []\n\nclass MockQuery(gevent.Greenlet):\n def __init__(self, store, query, timestamp, counter, \n num_lines, uuid, redis_config):\n self.store = store\n self.query = query\n self.timestamp = timestamp\n self.counter = counter\n self.num_lines = num_lines\n self.uuid = uuid\n self.redis_config = redis_config\n\n self.finish_event = gevent.event.Event()\n\n super(MockQuery, self).__init__()\n\n def kill(self):\n LOG.debug(\"%s killed\", self.uuid)\n super(MockQuery, self).kill()\n\n def _run(self):\n LOG.debug(\"%s started\", self.uuid)\n self.finish_event.wait()\n LOG.debug(\"%s finished\", self.uuid)\n\nclass MockEQuery(gevent.Greenlet):\n def __init__(self, store, query, timestamp, counter, \n num_lines, uuid, redis_config):\n self.store = store\n self.query = query\n self.timestamp = timestamp\n self.counter = counter\n self.num_lines = num_lines\n self.uuid = uuid\n self.redis_config = redis_config\n\n self.finish_event = gevent.event.Event()\n\n super(MockEQuery, self).__init__()\n\n def kill(self):\n LOG.debug(\"%s killed\", self.uuid)\n super(MockEQuery, self).kill()\n\n def _run(self):\n LOG.debug(\"%s started\", self.uuid)\n self.finish_event.wait()\n raise RuntimeError(\"BAD BAD QUERY!\")\n\ndef query_factory(store, query, timestamp, counter, \n num_lines, uuid, redis_config):\n\n if query == \"bad\":\n mqf = MockEQuery\n else:\n mqf = MockQuery\n\n mq = mqf(store, query, timestamp, counter, \n num_lines, uuid, redis_config)\n MOCK_QUERIES.append(mq)\n\n return mq\n\ndef query_cleanup():\n global MOCK_QUERIES\n MOCK_QUERIES = []\n" }, { "path": "tests/traced_storage_profile.py", "content": "#!/usr/bin/env python\n\nimport shutil\nimport minemeld.traced.storage\nimport random\nimport time\n\nif __name__ == \"__main__\":\n num_lines = 200000\n store = minemeld.traced.storage.Store()\n\n t1 = time.time()\n for j in xrange(num_lines):\n value = '{ \"log\": %d }' % random.randint(0, 0xFFFFFFFF)\n t2 = time.time()\n dt = t2-t1\n\n t1 = time.time()\n for j in xrange(num_lines):\n value = '{ \"log\": %d }' % random.randint(0, 0xFFFFFFFF)\n store.write(j, value)\n t2 = time.time()\n print \"TIME: Inserted %d lines in %d sec\" % (num_lines, (t2-t1-dt))\n\n store.stop()\n # shutil.rmtree('1970-01-01')\n\n" }, { "path": "tests/wsgi.htpasswd", "content": "admin:$apr1$pNXvvCp5$c4VXxDzt9waMeAEFRH7p8.\n" }, { "path": "tox.ini", "content": "[tox]\nenvlist = py27, flake8\nskipsdist = True\n\n[testenv:py27]\nbasedeps = mock\n nose\n coverage\n guppy\n xmltodict\nchangedir = {envtmpdir}\nsetenv = PYTHONPATH = {toxinidir}\ndeps = {[testenv:py27]basedeps}\n -r{toxinidir}/requirements.txt\n -r{toxinidir}/requirements-web.txt\ncommands = nosetests -a '!slow' -s {posargs}\n\n[testenv:flake8]\ndeps = flake8\ncommands = flake8 --ignore E402,E226 --max-line-length=100\n\n[testenv:stress]\nbasepython = python2.7\nbasedeps = mock\n nose\n guppy\nchangedir = {envtmpdir}\nsetenv = PYTHONPATH = {toxinidir}\ndeps = {[testenv:py27]basedeps}\n -r{toxinidir}/requirements.txt\n -r{toxinidir}/requirements-web.txt\ncommands = nosetests -s --logging-level=INFO -a 'slow' {posargs}\n\n[testenv:profile]\nbasepython = python2.7\nbasedeps = mock\n nose\n guppy\nchangedir = {envtmpdir}\nsetenv = PYTHONPATH = {toxinidir}\ndeps = {[testenv:py27]basedeps}\n -r{toxinidir}/requirements.txt\n -r{toxinidir}/requirements-web.txt\ncommands = nosetests -s --logging-level=INFO --with-profile --profile-stats-file profile.log -a 'slow' {posargs}\n\n" } ]