Repository: PaloAltoNetworks/minemeld-core Branch: master Commit: d9c08b2a5a94 Files: 221 Total size: 1.7 MB Directory structure: gitextract_8s2k2lhp/ ├── .circleci/ │ └── config.yml ├── .gitignore ├── AUTHORS ├── LICENSE ├── NOTICE ├── README.md ├── docs/ │ ├── Makefile │ ├── architecture.rst │ ├── conf.py │ ├── configapi.rst │ ├── index.rst │ ├── internals.rst │ ├── license.rst │ ├── messages.rst │ ├── metricsapi.rst │ ├── nodeconfig.rst │ ├── schema-indicator-0-1.json │ ├── schema.rst │ ├── scripts.rst │ └── statusapi.rst ├── minemeld/ │ ├── __init__.py │ ├── chassis.py │ ├── collectd.py │ ├── comm/ │ │ ├── __init__.py │ │ └── zmqredis.py │ ├── extensions/ │ │ ├── __init__.py │ │ └── manager.py │ ├── fabric.py │ ├── flask/ │ │ ├── __init__.py │ │ ├── aaa.py │ │ ├── aaaapi.py │ │ ├── cbfeed.py │ │ ├── config.py │ │ ├── configapi.py │ │ ├── configdataapi.py │ │ ├── events.py │ │ ├── extensionsapi.py │ │ ├── feedredis.py │ │ ├── jobs.py │ │ ├── jobsapi.py │ │ ├── logger.py │ │ ├── loginapi.py │ │ ├── logsapi.py │ │ ├── main.py │ │ ├── metricsapi.py │ │ ├── mmrpc.py │ │ ├── prototypeapi.py │ │ ├── redisclient.py │ │ ├── session.py │ │ ├── sns.py │ │ ├── statusapi.py │ │ ├── supervisorapi.py │ │ ├── supervisorclient.py │ │ ├── taxiicollmgmt.py │ │ ├── taxiidiscovery.py │ │ ├── taxiipoll.py │ │ ├── taxiiutils.py │ │ ├── tracedapi.py │ │ ├── utils.py │ │ └── validateapi.py │ ├── ft/ │ │ ├── __init__.py │ │ ├── actorbase.py │ │ ├── anomali.py │ │ ├── auscert.py │ │ ├── autofocus.py │ │ ├── azure.py │ │ ├── bambenek.py │ │ ├── base.py │ │ ├── basepoller.py │ │ ├── cif.py │ │ ├── ciscoise.py │ │ ├── cofense.py │ │ ├── condition/ │ │ │ ├── BoolExpr.g4 │ │ │ ├── BoolExpr.tokens │ │ │ ├── BoolExprLexer.py │ │ │ ├── BoolExprLexer.tokens │ │ │ ├── BoolExprListener.py │ │ │ ├── BoolExprParser.py │ │ │ ├── __init__.py │ │ │ └── interface.py │ │ ├── csv.py │ │ ├── dag.py │ │ ├── dag_ng.py │ │ ├── google.py │ │ ├── http.py │ │ ├── ipop.py │ │ ├── json.py │ │ ├── local.py │ │ ├── localdb.py │ │ ├── logstash.py │ │ ├── mm.py │ │ ├── o365.py │ │ ├── op.py │ │ ├── panos.py │ │ ├── phishme.py │ │ ├── proofpoint.py │ │ ├── recordedfuture.py │ │ ├── redis.py │ │ ├── st.py │ │ ├── syslog.py │ │ ├── table.py │ │ ├── taxii.py │ │ ├── taxii2.py │ │ ├── test.py │ │ ├── threatconnect.py │ │ ├── threatq.py │ │ ├── tmt.py │ │ ├── utils.py │ │ ├── visa.py │ │ ├── vt.py │ │ └── xmpp.py │ ├── loader.py │ ├── mgmtbus.py │ ├── packages/ │ │ ├── __init__.py │ │ ├── gdns/ │ │ │ ├── LICENSE │ │ │ ├── __init__.py │ │ │ ├── _ares.pyx │ │ │ ├── cares.pxd │ │ │ ├── cares_ntop.h │ │ │ ├── cares_pton.h │ │ │ ├── dig.py │ │ │ └── dnshelper.c │ │ ├── gevent_openssl/ │ │ │ ├── COPYING │ │ │ ├── SSL.py │ │ │ └── __init__.py │ │ ├── ise/ │ │ │ ├── __init__.py │ │ │ └── ers.py │ │ └── panforest/ │ │ ├── __init__.py │ │ └── forest.py │ ├── run/ │ │ ├── __init__.py │ │ ├── cacert_merge.py │ │ ├── config.py │ │ ├── console.py │ │ ├── extgit.py │ │ ├── freeze.py │ │ ├── launcher.py │ │ └── restore.py │ ├── startupplanner.py │ ├── supervisord/ │ │ ├── __init__.py │ │ └── listener.py │ └── traced/ │ ├── __init__.py │ ├── main.py │ ├── purge.py │ ├── queryprocessor.py │ ├── storage.py │ └── writer.py ├── nodes.json ├── requirements-dev.txt ├── requirements-web.txt ├── requirements.txt ├── scripts/ │ └── prebuild-script.sh ├── setup.py ├── tests/ │ ├── comm_mock.py │ ├── empty.yml │ ├── feeds.htpasswd │ ├── integration/ │ │ └── basic/ │ │ ├── DomainHC%3Fv%3Dcarbonblack.result │ │ ├── IPv4.lst │ │ ├── IPv4HC%3Fs%3D5%26n%3D10.result │ │ ├── IPv4HC%3Fv%3Dcsv%26f%3Dconfidence%26f%3Dsources%7Cfeeds%26f%3Dindicator%7Cclientip%26tr%3D1.result │ │ ├── IPv4HC%3Fv%3Djson%26tr%3D1.result │ │ ├── IPv4HC%3Fv%3Djson-seq.result │ │ ├── IPv4HC%3Fv%3Dmwg.result │ │ ├── IPv4HC.result │ │ ├── README.md │ │ ├── URL.lst │ │ ├── URLHC%3Fs%3D5%26n%3D10.result │ │ ├── URLHC%3Fv%3Dbluecoat%26cd%3Dtest.result │ │ ├── URLHC%3Fv%3Dbluecoat.result │ │ ├── URLHC%3Fv%3Dcsv%26f%3Dconfidence%26f%3Dsources%7Cfeeds%26f%3Dindicator%7Curl.result │ │ ├── URLHC%3Fv%3Djson%26tr%3D1.result │ │ ├── URLHC%3Fv%3Djson-seq.result │ │ ├── URLHC%3Fv%3Dmwg.result │ │ ├── URLHC%3Fv%3Dpanosurl%26di%3D1.result │ │ ├── URLHC%3Fv%3Dpanosurl%26sp%3D1%26nsl%3D1.result │ │ ├── URLHC%3Fv%3Dpanosurl%26sp%3D1.result │ │ ├── URLHC%3Fv%3Dpanosurl.result │ │ ├── URLHC.result │ │ ├── domain.lst │ │ ├── gen-results.sh │ │ └── test.py │ ├── panos_mock.py │ ├── st_profile.py │ ├── test-prototype-1.yml │ ├── test_comm_amqp.py │ ├── test_device_list.yml │ ├── test_device_list2.yml │ ├── test_flask_aaa.py │ ├── test_ft_autofocus.py │ ├── test_ft_base.py │ ├── test_ft_basepoller.py │ ├── test_ft_boolexpr.py │ ├── test_ft_dag.py │ ├── test_ft_dag_devicepusher_op__show__object__registered_ip__ip_192_168_1_1__ip___registered_ip___object___show__0.xml │ ├── test_ft_dag_devicepusher_op__show__object__registered_ip__ip_192_168_1_2__ip___registered_ip___object___show__0.xml │ ├── test_ft_dag_devicepusher_op__show__object__registered_ip__tag__entry_name__mmeld_test_____tag___registered_ip___object___show__0.xml │ ├── test_ft_ipop.py │ ├── test_ft_local.py │ ├── test_ft_logstash.py │ ├── test_ft_op.py │ ├── test_ft_redis.py │ ├── test_ft_st.py │ ├── test_ft_syslog.py │ ├── test_ft_table.py │ ├── test_ft_taxii.py │ ├── test_ft_taxii_stix_package_IPv4_1_3.xml │ ├── test_ft_taxii_stix_package_IPv4_3_1.xml │ ├── test_ft_taxii_stix_package_IPv4_3_2.xml │ ├── test_ft_taxii_stix_package_IPv6_3_1.xml │ ├── test_ft_taxii_stix_package_IPv6_3_2.xml │ ├── test_localdb.yml │ ├── test_localdb2.yml │ ├── test_run_config.py │ ├── test_startupplanner.py │ ├── test_traced_queryprocessor.py │ ├── test_traced_storage.py │ ├── test_traced_writer.py │ ├── testproto.yml │ ├── traced_mock.py │ ├── traced_storage_profile.py │ └── wsgi.htpasswd └── tox.ini ================================================ FILE CONTENTS ================================================ ================================================ FILE: .circleci/config.yml ================================================ version: 2.1 orbs: aws-s3: circleci/aws-s3@1.0.4 jobs: build-bionic: docker: - image: circleci/buildpack-deps:bionic steps: - run: name: Update & Upgrade command: sudo apt update && sudo apt upgrade -y - checkout - run: name: Install deps command: > sudo apt install -y gcc git python-minimal python2.7-dev libffi-dev libssl-dev make g++ libleveldb-dev librrd-dev libxslt1-dev libc-ares-dev libsnappy-dev python-pip - run: name: Install Python dev requirements command: sudo -H pip install -r requirements-dev.txt - run: name: Show Python Version command: /usr/bin/env python -V - run: name: Build package command: > platter build --virtualenv-version 16.7.9 -r requirements-web.txt --prebuild-script scripts/prebuild-script.sh - run: name: Rename files command: for file in dist/*; do mv "$file" "${file%.tar.gz}.$CIRCLE_SHA1.bionic"; done - persist_to_workspace: root: dist paths: - . build-xenial: docker: - image: circleci/buildpack-deps:xenial steps: - run: name: Update & Upgrade command: sudo apt update && sudo apt upgrade -y - checkout - run: name: Install deps command: > sudo apt install -y gcc git python-minimal python2.7-dev libffi-dev libssl-dev make g++ libleveldb-dev librrd-dev libxslt1-dev libc-ares-dev libsnappy-dev python-pip - run: name: Install Python dev requirements command: sudo -H pip install -r requirements-dev.txt - run: name: Show Python Version command: /usr/bin/env python -V - run: name: Build package command: > platter build --virtualenv-version 16.7.9 -r requirements-web.txt --prebuild-script scripts/prebuild-script.sh - run: name: Rename files command: for file in dist/*; do mv "$file" "${file%.tar.gz}.$CIRCLE_SHA1.xenial"; done - persist_to_workspace: root: dist paths: - . deploy: docker: - image: circleci/python:2.7 steps: - run: name: Create workspace dir command: mkdir ~/workspace - attach_workspace: at: ~/workspace - run: name: Workspace contents command: find ~/workspace - aws-s3/sync: from: ~/workspace to: 's3://minemeld/' arguments: | --acl public-read workflows: version: 2 xenial: jobs: - build-xenial: filters: tags: only: '/.*/' - deploy: requires: - build-xenial filters: branches: ignore: '/.*/' tags: only: '/.*/' bionic: jobs: - build-bionic: filters: tags: only: '/.*/' - deploy: requires: - build-bionic filters: branches: ignore: '/.*/' tags: only: '/.*/' ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover # Translations *.mo *.pot # Django stuff: *.log # Sphinx documentation docs/_build/ # PyBuilder target/ # PyCharm .idea/ .vscode/ _ares.c _ares.h ================================================ FILE: AUTHORS ================================================ Luigi Mori Kevin Steves Phil Da Silva Kevin Stilwell iDev Jonas Eichinger Egon Kidmose Dan James Ben Carroll John Marion ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS ================================================ FILE: NOTICE ================================================ MineMeld Core Copyright 2016 Palo Alto Networks This product includes software developed at Palo Alto Networks Inc. (http://www.paloaltonetworks.com/). ================================================ FILE: README.md ================================================ # minemeld-core This repo contains the code for the engine and the API of MineMeld, an extensible Threat Intelligence processing framework. For details check the MineMeld [Wiki](https://github.com/PaloAltoNetworks/minemeld/wiki) ================================================ FILE: docs/Makefile ================================================ # Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " applehelp to make an Apple Help Book" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" @echo " coverage to run coverage check of the documentation (if enabled)" clean: rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/minemeld.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/minemeld.qhc" applehelp: $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp @echo @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." @echo "N.B. You won't be able to view it unless you put it in" \ "~/Library/Documentation/Help or install it in your application" \ "bundle." devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/minemeld" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/minemeld" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." coverage: $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage @echo "Testing of coverage in the sources finished, look at the " \ "results in $(BUILDDIR)/coverage/python.txt." xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." ================================================ FILE: docs/architecture.rst ================================================ Architecture ============ Processing graph ---------------- The core processing engine of MineMeld is based on a Direct Acyclic Graph of nodes. Indicators are retrieved or received by Miner nodes and then pushed to downstream nodes via *update* messages. Nodes can also signal to downstream nodes the removal of an indicator (in case of expiration, ...) by sending a *withdraw* message. Each node in the graph is independent from other nodes, there is no global state or global clock inside the engine. All the nodes are independent and asynchrnous. Each node is also responsabile for maintaining its own state. This architecture trades memory and disk for flexibility. .. image:: images/updates.png .. image:: images/withdraws.png Node ---- Each node may have 0 or more inputs and 0 or more output. Rather obviuosly if a node has 0 inputs it is considered a Miner node, if a node has 0 ouput it is considered an Output node. Each node also offers a RPC interface, for direct out of band requests, and a connection to a *management bus* for status checks and management commands coming from the *management bus master*. .. image:: images/nodes.png The connections between nodes are implemented with a pubsub mechanism over the *fabric*. Each node sends its downstream message to a *topic* named as the node, and all the downstream nodes are subscribers of this topic. .. image:: images/topics.png Runtime architecture -------------------- To take advantage of all the cores available on the system, the engine by default automatically splits the nodes of the graph into multiple processes called *chassis*. Each *chassis* has a dedicated connection to the *fabric* and to the *management bus*. The *master* process monitors the health and the metrics of each *chassis* using the *management bus*. The *master* process is also responsible for synchronzing the nodes when the engine starts and stops, to ensure that the state of the graph is consistent. At shutdown this is achieved using a super simplified version of the Chandy-Lamport checkpoint algorithm. Both the *management bus* and the *fabric* are implemented using an external message broker, RabbitMQ. .. image:: images/chassis-architecture.png ================================================ FILE: docs/conf.py ================================================ # -*- coding: utf-8 -*- # # minemeld-core documentation build configuration file, created by # sphinx-quickstart on Thu Aug 6 14:14:33 2015. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # flake8: noqa import sys import os import shlex # from http://blog.rtwilson.com/how-to-make-your-sphinx-documentation-compile-with-readthedocs-when-youre-using-numpy-and-scipy/ import mock MOCK_MODULES = ['plyvel'] for mod_name in MOCK_MODULES: sys.modules[mod_name] = mock.Mock() # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('..')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.napoleon' ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'minemeld-core' copyright = u'2015, Palo Alto Networks' author = u'Palo Alto Networks' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = '0.9' # The full version, including alpha/beta/rc tags. release = '0.9.70.post5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all # documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'alabaster' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. #html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' #html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # Now only 'ja' uses this config value #html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. #html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. htmlhelp_basename = 'minemeld-coredoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', # Latex figure (float) alignment #'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'minemeld-core.tex', u'modindex_common_prefixer-wagon Documentation', u'Palo Alto Networks', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ (master_doc, 'minemeld-core', u'minemeld-core Documentation', [author], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'minemeld-core', u'minemeld-core Documentation', author, 'minemeld-core', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False ================================================ FILE: docs/configapi.rst ================================================ Configuration API ================= Configuration API can be used to change a temporary configuration. The temp configuration is applied when the /config/commit API is called. The temp configuration has a *version* attribute. The version changes every time the temp configuration is reinitialized or reloaded from the current MineMeld running configuration. Some API calls (commit, create node, ...) require a version parameter, if the supplied version is different from the current temp config version a 409 error is returned. Each node has a *version* attribute. The node version changes every time the node config is changed. Some API calls (set node, ...) require a version parameter. If the supplied version is different from the current node version a 409 error is returned. Authentication -------------- Authentication is performed via basic authentication. A 401 error is returned in case of invalid credentials: :: $ curl -u 'admin:goodpassword' -i http://127.0.0.1/config/info HTTP/1.1 200 OK [...] $ curl -u 'admin:baspassword' -i http://127.0.0.1/config/info HTTP/1.1 401 UNAUTHORIZED [...] .. warning:: Use HTTPS with a trusted certificate in production ! Configuration information ------------------------- :: $ curl -u 'admin:admin' -i http://127.0.0.1/config/info HTTP/1.1 200 OK Server: nginx/1.4.6 (Ubuntu) Date: Sun, 06 Sep 2015 14:51:41 GMT Content-Type: application/json Content-Length: 141 Connection: keep-alive { "result": { "fabric": false, "mgmtbus": false, "num_nodes": 8, "version": "58102565-1b93-4095-9130-84556496b84b" } } Reload configuration -------------------- Reload current running configuration as temporary configuration. Config version is changed. Returns new config version. :: $ curl -u 'admin:admin' -i http://127.0.0.1/config/reload HTTP/1.1 200 OK Server: nginx/1.4.6 (Ubuntu) Date: Sun, 06 Sep 2015 14:52:46 GMT Content-Type: application/json Content-Length: 54 Connection: keep-alive { "result": "b2482473-1e9f-4a24-a8b7-7296f1dfb856" } Create node ----------- Create a new node. *version* attribute is required, and should be the config version. Returns the new node id and version. :: $ curl -XPOST -H 'Content-Type: application/json' -u 'admin:admin' -i http://127.0.0.1/config/node -d '{ "name": "spamhaus_EDROP2", "properties": { "class": "HTTP", "config": { "attributes": { "direction": "inbound", "type": "IPv4" }, "cchar": ";", "source_name": "http://www.spamhaus.org/drop/edrop.txt", "split_char": ";", "url": "http://www.spamhaus.org/drop/edrop.txt" }, "output": true }, "version": "b2482473-1e9f-4a24-a8b7-7296f1dfb856" }' HTTP/1.1 200 OK Server: nginx/1.4.6 (Ubuntu) Date: Sun, 06 Sep 2015 14:59:28 GMT Content-Type: application/json Content-Length: 91 Connection: keep-alive { "result": { "id": 9, "version": "b2482473-1e9f-4a24-a8b7-7296f1dfb856+0" } } Get node configuration ---------------------- :: $ curl -u 'admin:admin' -i http://127.0.0.1/config/node/9 HTTP/1.1 200 OK Server: nginx/1.4.6 (Ubuntu) Date: Sun, 06 Sep 2015 15:01:00 GMT Content-Type: application/json Content-Length: 479 Connection: keep-alive { "result": { "name": "spamhaus_EDROP2", "properties": { "class": "HTTP", "config": { "attributes": { "direction": "inbound", "type": "IPv4" }, "cchar": ";", "source_name": "http://www.spamhaus.org/drop/edrop.txt", "split_char": ";", "url": "http://www.spamhaus.org/drop/edrop.txt" }, "output": true }, "version": "b2482473-1e9f-4a24-a8b7-7296f1dfb856+0" } } Change node configuration ------------------------- *version* is the current node version. :: $ curl -XPUT -u 'admin:admin' -H 'Content-Type: application/json' -i http://127.0.0.1/config/node/8 -d '{ "name": "spamhaus_EDROP2", "properties": { "class": "HTTP", "config": { "attributes": { "direction": "inbound", "type": "IPv4" }, "cchar": ";", "source_name": "http://www.spamhaus.org/drop/edrop2.txt", "split_char": ";", "url": "http://www.spamhaus.org/drop/edrop.txt" }, "output": true }, "version": "b2482473-1e9f-4a24-a8b7-7296f1dfb856+0" }' HTTP/1.1 200 OK Server: nginx/1.4.6 (Ubuntu) Date: Sun, 06 Sep 2015 15:24:25 GMT Content-Type: application/json Content-Length: 56 Connection: keep-alive { "result": "b2482473-1e9f-4a24-a8b7-7296f1dfb856+1" } Delete node ----------- Delete a node. *version* is the current node version. :: $ curl -XDELETE -H 'Content-type: application/json' -u 'admin:admin' -i http://127.0.0.1/config/node/9 -d '{"version": "b2482473-1e9f-4a24-a8b7-7296f1dfb856+0"}' HTTP/1.1 200 OK Server: nginx/1.4.6 (Ubuntu) Date: Sun, 06 Sep 2015 15:17:42 GMT Content-Type: application/json Content-Length: 20 Connection: keep-alive { "result": "OK" } Commit configuration -------------------- *version* is the current configuration version. :: $ curl -XPOST -H 'Content-Type: application/json' -u 'admin:admin' -i http://127.0.0.1/config/commit -d '{"version": "b2482473-1e9f-4a24-a8b7- 7296f1dfb856"}' HTTP/1.1 200 OK Server: nginx/1.4.6 (Ubuntu) Date: Sun, 06 Sep 2015 15:31:26 GMT Content-Type: application/json Content-Length: 20 Connection: keep-alive { "result": "OK" } ================================================ FILE: docs/index.rst ================================================ .. MineMeld documentation master file, created by sphinx-quickstart on Thu Aug 6 14:14:33 2015. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Welcome to MineMeld's documentation! ======================================= Contents: .. toctree:: :maxdepth: 1 architecture messages scripts schema statusapi configapi metricsapi internals license Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` ================================================ FILE: docs/internals.rst ================================================ Internals ========= minemeld.ft.table ------------------- .. automodule:: minemeld.ft.table minemeld.ft.st ---------------- .. automodule:: minemeld.ft.st ================================================ FILE: docs/license.rst ================================================ LICENSE ======= Copyright (c) 2015, Palo Alto Networks Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ================================================ FILE: docs/messages.rst ================================================ Messages ======== Intra-node protocol ------------------- The protocol used between nodes is super simple. There are 3 messages: update ****** update(indicator, value) :indicator: string :value: a dictionary of attributes for the indicator Notifies a new indicator or an update of the attributes associated to an indicator. withdraw ******** withdraw(indicator[, value]) :indicator: string :value: (optional) a dictionary of attributes for the indicator Notifies a withdraw of the indicator checkpoint ********** checkpoint(id) Used as a processing barrier for the graph when the graph is being stopped. ================================================ FILE: docs/metricsapi.rst ================================================ Metrics API =========== Metrics API can be used to retrieve historical statistics of the system and MineMeld nodes. Metrics list ------------ :: $ curl -i -u 'admin:admin' http://127.0.0.1/metrics HTTP/1.1 200 OK Server: nginx/1.4.6 (Ubuntu) Date: Tue, 08 Sep 2015 21:41:07 GMT Content-Type: application/json Content-Length: 1040 Connection: keep-alive { "result": [ "spamhaus_EDROP.update.tx", "df-run", "inboundaggregator.update.rx", "dshield_blocklist.length", "df-run-user", "zeustracker_badips.length", "spamhaus_DROP.length", "df-root", "df-sys-fs-cgroup", "spamhaus_EDROP.length", [...] ] } Metric history -------------- Metric history can be retrieved using /metrics/ endpoint. :: $ curl -i -u 'admin:admin' http://127.0.0.1/metrics/outboundaggregator.length HTTP/1.1 200 OK Server: nginx/1.4.6 (Ubuntu) Date: Tue, 08 Sep 2015 21:42:52 GMT Content-Type: application/json Content-Length: 1779 Connection: keep-alive { "result": [ [ 1441661340, null ], [...] ] } ================================================ FILE: docs/nodeconfig.rst ================================================ Node config =========== The set of config parameters supported by a node depends on the node class. .. note:: This document has been extracted from the docstrings of the python code. For the most updated documentation check the original source code. Base class ---------- All nodes support these parameters. Parameters +++++++++++++++++ :infilters: inbound *filter set*. Filters to apply to received indicators. :outfilters: outbound *filter set*. Filters to apply to transmitted indicators. Filter set ++++++++++ Each filter set is a list of filters. Filters are checked from top to bottom, the first matching filter is applied and following filters are not checked. Default action is **accept**. Each filter is a dictionary with 3 keys: :name: name of the filter. :conditions: list of boolean expressions to match on the indicator and indicator value. :actions: list of actions to apply to the indicator. Currently the only supported actions are **accept** and **drop** In addition to the atttributes in the indicator value, filters can match on 3 special attributes: :__indicator: the indicator itself. :__method: the method of the message, **update** or **withdraw**. :__origin: the name of the node who sent the indicator. Condition +++++++++ A condition in the filter is a boolean expression composed by: a JMESPath expression, an operator (<, <=, ==, >=, >, !=) and a value. Example +++++++ Example config in YAML:: infilters: - name: accept withdraws conditions: - __method == 'withdraw' actions: - accept - name: accept URL conditions: - type == 'URL' actions: - accept - name: drop all actions: - drop outfilters: - name: accept all (default) actions: - accept Base poller class ----------------- In addition to `Base class` config parameters, the base poller class support the following parameters. Config parameters +++++++++++++++++ :source_name: name of the source. This is added to the *sources* attribute of the generated indicators. Default: name of the node. :attributes: dictionary of attributes for the generated indicators. This dictionary is used as template for the value of the generated indicators. Default: empty :interval: polling interval in seconds. Default: 3600. :num_retries: how many times the miner should try to reach the source in case of failure. If this number is exceeded, the miner waits until the next polling time to try again. Default: 2 :age_out: age out policies to apply to the indicators. Default: age out check interval 3600 seconds, sudden death enabled, default age out interval 30 days. Age out policy ++++++++++++++ Age out policy is described by a dictionary with at least 3 keys: :interval: number of seconds between successive age out checks. :sudden_death: boolean, if *true* indicators are immediately aged out when they disappear from the feed. :default: age out interval. After this interval an indicator is aged out even if it is still present in the feed. If *null*, no age out interval is applied. Additional keys can be used to specify age out interval per indicator *type*. Age out interval ++++++++++++++++ Age out intervals have the following format:: + *base attribute* can be *last_seen*, if the age out interval should be calculated based on the last time the indicator was found in the feed, or *first_seen*, if instead the age out interval should be based on the time the indicator was first seen in the feed. If not specified *first_seen* is used. *interval* is the length of the interval expressed in seconds. Suffixes *d*, *h* and *m* can be used to specify days, hours or minutes. Example +++++++ Example config in YAML for a feed where indicators should be aged out only when they are removed from the feed:: source_name: example.persistent_feed interval: 600 age_out: default: null sudden_death: true interval: 300 attributes: type: IPv4 confidence: 100 share_level: green direction: inbound Example config in YAML for a feed where indicators are aged out when they disappear from the feed and 30 days after they have seen for the first time in the feed:: source_name: example.long_running_feed interval: 3600 age_out: default: first_seen+30d sudden_death: true interval: 1800 attributes: type: URL confidence: 50 share_level: green Example config in YAML for a feed where indicators are aged 30 days after they have seen for the last time in the feed:: source_name: example.delta_feed interval: 3600 age_out: default: last_seen+30d sudden_death: false interval: 1800 attributes: type: URL confidence: 50 share_level: green minemeld.ft.http.HttpFT ----------------------- In addition to `Base poller class` config parameters, the base poller class support the following parameters. Parameters +++++++++++++++++ :url: URL of the feed. :polling_timeout: timeout of the polling request in seconds. Default: 20 :verify_cert: boolean, if *true* feed HTTPS server certificate is verified. Default: *true* :ignore_regex: Python regular expression for lines that should be ignored. Default: *null* :indicator: an *extraction dictionary* to extract the indicator from the line. If *null*, the text until the first whitespace or newline character is used as indicator. Default: *null* :fields: a dicionary of *extraction dictionaries* to extract additional attributes from each line. Default: {} Extraction dictionary +++++++++++++++++++++ Extraction dictionaries contain the following keys: :regex: Python regular expression for searching the text. :transform: template to generate the final value from the result of the regular expression. Default: the entire match of the regex is used as extracted value. See Python `re `_ module for details about Python regular expressions and templates. Example +++++++ Example config in YAML where extraction dictionaries are used to extract the indicator and additional fields:: url: https://www.dshield.org/block.txt ignore_regex: "[#S].*" indicator: regex: '^([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})\t([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})' transform: '\1-\2' fields: dshield_nattacks: regex: '^.*\t.*\t[0-9]+\t([0-9]+)' transform: '\1' dshield_name: regex: '^.*\t.*\t[0-9]+\t[0-9]+\t([^\t]+)' transform: '\1' dshield_country: regex: '^.*\t.*\t[0-9]+\t[0-9]+\t[^\t]+\t([A-Z]+)' transform: '\1' dshield_email: regex: '^.*\t.*\t[0-9]+\t[0-9]+\t[^\t]+\t[A-Z]+\t(\S+)' transform: '\1' Example config in YAML where the text in each line until the first whitespace is used as indicator:: url: https://ransomwaretracker.abuse.ch/downloads/CW_C2_URLBL.txt ignore_regex: '^#' For a complete config example check **dshield.block** prototype. minemeld.ft.csv.CSVFT --------------------- In addition to `Base poller class` config parameters, the base poller class support the following parameters. Parameters ++++++++++ :url: URL of the feed. :polling_timeout: timeout of the polling request in seconds. Default: 20 :verify_cert: boolean, if *true* feed HTTPS server certificate is verified. Default: *true* :ignore_regex: Python regular expression for lines that should be ignored. Default: *null* :fieldnames: list of field names in the file. If *null* the values in the first row of the file are used as names. Default: *null* :delimiter: see `csv Python module `_. Default: , :doublequote: see `csv Python module `_. Default: true :escapechar: see `csv Python module `_. Default: null :quotechar: see `csv Python module `_. Default: " :skipinitialspace: see `csv Python module `_. Default: false Example +++++++ Example config in YAML:: url: https://sslbl.abuse.ch/blacklist/sslipblacklist.csv ignore_regex: '^#' fieldnames: - indicator - port - sslblabusech_type For a complete config example check **sslabusech.ipblacklist** prototype. minemeld.ft.json.SimpleJSON --------------------------- In addition to `Base poller class` config parameters, the base poller class support the following parameters. Parameters ++++++++++ :url: URL of the feed. :polling_timeout: timeout of the polling request in seconds. Default: 20 :verify_cert: boolean, if *true* feed HTTPS server certificate is verified. Default: *true* :extractor: JMESPath expression for extracting the indicators from the JSON document. Default: @ :indicator: the JSON attribute to use as indicator. Default: indicator :fields: list of JSON attributes to include in the indicator value. If *null* no additional attributes are extracted. Default: *null* :prefix: prefix to add to field names. Default: json Example +++++++ Example config in YAML:: url: https://ip-ranges.amazonaws.com/ip-ranges.json extractor: "prefixes[?service=='AMAZON']" prefix: aws indicator: ip_prefix fields: - region - service For a complete config example check **aws.AMAZON** prototype. ================================================ FILE: docs/schema-indicator-0-1.json ================================================ { "id": "https://www.paloaltonetworks.com/minemeld-indicator-schema-0-1#", "$schema": "http://json-schema.org/draft-04/schema#", "description": "schema for minemeld attributes", "type": "object", "required": [ "type" ], "properties": { "type": { "description": "type of the indicator", "type": "string", "enum": [ "IPv4", "IPv6", "domain", "URL", "sha512", "sha256", "sha1", "md5", "ssdeep", "mutex", "windows-registry-value", "user-agent.fragment", "file.name", "process.command_line", "email-addr", "autonomous-system" ] }, "direction": { "description": "direction of the session, applies to IPv4", "type": "string", "enum": ["inbound", "outbound"] }, "first_seen": { "type": "integer", "format": "utc-millisec", "description": "time the indicator has been seen for the first time. <" }, "last_seen": { "type": "integer", "format": "utc-millisec", "description": "time the indicator has been seen for the last time. >" }, "sources": { "description": "list of sources for this indicator", "type": "array", "items": { "type": "string", "format": "uri" } }, "confidence": { "type": "integer", "description": "confidence in the indicator 0-100", "minimum": 0, "maximum": 100 }, "share_level": { "description": "share level of indicator", "type": "string", "enum": ["white", "green", "amber", "red"] }, "country": { "type": "string", "description": "ISO country code (IPv4 and IPv6 only)", "minLength": 2, "maxLength": 2 }, "AS": { "type": "string", "description": "Autonmous system (IPv4 and IPv6 only)" } }, "patternProperties": { "^_[a-zA-Z0-9$_]*$": { "description": "private properties" }, "^$[a-zA-Z0-9$_]*$": { "description": "reserved, temporary properties" } } } ================================================ FILE: docs/schema.rst ================================================ Indicator value schema ====================== .. literalinclude:: schema-indicator-0-1.json :language: json :linenos: ================================================ FILE: docs/scripts.rst ================================================ mm-run ========= mm-run is a simple script that can be used as frontend to the minemeld library. Usage ----- :: $ mm-run --help usage: mm-run.py [-h] [--version] [--multiprocessing NP] [--verbose] CONFIG Low-latency threat indicators processor positional arguments: CONFIG path of the config file or of the config directory optional arguments: -h, --help show this help message and exit --version show program's version number and exit --multiprocessing NP enable multiprocessing. NP is the number of processes, 0 to use a process per machine core --verbose verbose Configuration ------------- CONFIG parameter on the command line can point to a configuration file or to a configuration directory. If CONIG is a directory, mm-run will check for commited-config.yml and running-config.yml files. If only running-config.yml exists, mm-run assumes that the config has not been changed and the processing continues where it was stopped. If only canidate-config.yml exists, mm-run copies the file to running-config.yml and reinitializes the processing. If both files exist, candidate-config.yml is copied over running-config.yml and the processing is reinitialized if the 2 files are different. If CONFIG instead is a path to a configuration file, the configuration will be considered as new and the processing is reinitialized. nodes ~~~~~ The **nodes** section contains the desription of the processing DAG. It is composed by a list of descriptions of nodes. Each node config has the following general format: :: nodename: # name of the node config: # list of parameters for the node, depends on the node class class: nodeclass # class of the node inputs: # list of upstream nodes - node1 - node2 output: true|false # if the node should generate updates & withdraws Node can also be based on prototypes, in that case the *config* and *class* sections are omitted as they are specified inside the prototype. :: nodename: # name of the node prototype: prototype1 # name of the prototype to be used inputs: # list of upstream nodes - node1 - node2 output: true|false # if the node should generate updates & withdraws Example 1 ^^^^^^^^^ :: spamhaus_DROP: config: source_name: http://www.spamhaus.org/drop/drop.txt attributes: type: IPv4 direction: inbound cchar: ; split_char: ; url: http://www.spamhaus.org/drop/drop.txt class: HTTP output: true This describes a node with the following properties: :name: spamhaus_DROP :class: HTTP :inputs: *none*, this is a Miner :output: enabled, this node will emit indicators :config: specific configuration for this node class Example 2 ^^^^^^^^^ :: inboundaggregator: config: infilters: - name: accept inbound IPv4 conditions: - type == 'IPv4' - direction == 'inbound' actions: - accept - name: drop all actions: - drop class: AggregatorIPv4 output: true inputs: - spamhaus_DROP - spamhaus_EDROP - dshield_blocklist :name: inboundaggregator :class: AggregatorIPv4 :inputs: this node will receive indicators from spamhaus_DROP, spamhaus_EDROP, ... :output: enabled, this node will emit indicators :config: specific configuration for this node class Example 3 ^^^^^^^^^ :: spamhaus_DROP: output: true prototype: spamhaus.DROP :name: spamhaus_DROP :inputs: *none*, this is a Miner :output: enabled, this node will emit indicators :prototype: *config* and *class* of this node will be loaded from the spamhaus.DROP prototype ================================================ FILE: docs/statusapi.rst ================================================ Status API ========== Status API can be used to retrieve status information about minemeld core engine and the system. MineMeld status --------------- :: $ curl -i -u 'admin:admin' http://127.0.0.1/status/minemeld HTTP/1.1 200 OK Server: nginx/1.4.6 (Ubuntu) Date: Tue, 08 Sep 2015 21:34:47 GMT Content-Type: application/json Content-Length: 1878 Connection: keep-alive { "result": { "dshield_blocklist": { "inputs": [], "length": 20, "output": true, "state": 5, "statistics": { "update.tx": 20 } }, [...] } System status ------------- Reports percent usage of CPUs, disk, memory and swap. :: $curl -i -u 'admin:admin' http://127.0.0.1/status/system HTTP/1.1 200 OK Server: nginx/1.4.6 (Ubuntu) Date: Tue, 08 Sep 2015 21:37:31 GMT Content-Type: application/json Content-Length: 108 Connection: keep-alive { "result": { "cpu": [ 0.0 ], "disk": 17.7, "memory": 25.1, "swap": 0.0 } } ================================================ FILE: minemeld/__init__.py ================================================ """ minemeld ======== MineMeld core engine """ __version__ = '0.9.70.post5' ================================================ FILE: minemeld/chassis.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ minemeld.chassis A chassis instance contains a list of nodes and a fabric. Nodes communicate using the fabric. """ import os import logging import gevent import gevent.queue import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import minemeld.mgmtbus import minemeld.ft import minemeld.fabric LOG = logging.getLogger(__name__) STATE_REPORT_INTERVAL = 10 class Chassis(object): """Chassis class Args: fabricclass (str): class for the fabric fabricconfig (dict): config dictionary for fabric, class specific mgmtbusconfig (dict): config dictionary for mgmt bus """ def __init__(self, fabricclass, fabricconfig, mgmtbusconfig): self.chassis_id = os.getpid() self.fts = {} self.poweroff = gevent.event.AsyncResult() self.fabric_class = fabricclass self.fabric_config = fabricconfig self.fabric = minemeld.fabric.factory( self.fabric_class, self, self.fabric_config ) self.mgmtbus = minemeld.mgmtbus.slave_hub_factory( mgmtbusconfig['slave'], mgmtbusconfig['transport']['class'], mgmtbusconfig['transport']['config'] ) self.mgmtbus.add_failure_listener(self.mgmtbus_failed) self.mgmtbus.request_chassis_rpc_channel(self) self.log_channel_queue = gevent.queue.Queue(maxsize=128) self.log_channel = self.mgmtbus.request_log_channel() self.log_glet = None self.status_channel_queue = gevent.queue.Queue(maxsize=128) self.status_glet = None def _dynamic_load(self, classname): modname, classname = classname.rsplit('.', 1) imodule = __import__(modname, globals(), locals(), [classname]) cls = getattr(imodule, classname) return cls def get_ft(self, ftname): return self.fts.get(ftname, None) def configure(self, config): """configures the chassis instance Args: config (list): list of FTs """ newfts = {} for ft in config: ftconfig = config[ft] LOG.debug(ftconfig) # new FT newfts[ft] = minemeld.ft.factory( ftconfig['class'], name=ft, chassis=self, config=ftconfig.get('config', {}) ) newfts[ft].connect( ftconfig.get('inputs', []), ftconfig.get('output', False) ) self.fts = newfts # XXX should be moved to constructor self.mgmtbus.start() self.fabric.start() self.mgmtbus.send_master_rpc( 'chassis_ready', params={'chassis_id': self.chassis_id}, timeout=10 ) def request_mgmtbus_channel(self, ft): self.mgmtbus.request_channel(ft) def request_rpc_channel(self, ftname, ft, allowed_methods=None): if allowed_methods is None: allowed_methods = [] self.fabric.request_rpc_channel(ftname, ft, allowed_methods) def request_pub_channel(self, ftname): return self.fabric.request_pub_channel(ftname) def request_sub_channel(self, ftname, ft, subname, allowed_methods=None): if allowed_methods is None: allowed_methods = [] self.fabric.request_sub_channel(ftname, ft, subname, allowed_methods) def send_rpc(self, sftname, dftname, method, params, block, timeout): return self.fabric.send_rpc(sftname, dftname, method, params, block=block, timeout=timeout) def _log_actor(self): while True: try: params = self.log_channel_queue.get() self.log_channel.publish( method='log', params=params ) except Exception: LOG.exception('Error sending log') def log(self, timestamp, nodename, log_type, value): self.log_channel_queue.put({ 'timestamp': timestamp, 'source': nodename, 'log_type': log_type, 'log': value }) def _status_actor(self): while True: try: params = self.status_channel_queue.get() self.mgmtbus.send_status( params=params ) except Exception: LOG.exception('Error publishing status') def publish_status(self, timestamp, nodename, status): self.status_channel_queue.put({ 'timestamp': timestamp, 'source': nodename, 'status': status }) def fabric_failed(self): self.stop() def mgmtbus_failed(self): LOG.critical('chassis - mgmtbus failed') self.stop() def mgmtbus_start(self): LOG.info('chassis - start received from mgmtbus') self.start() return 'ok' def fts_init(self): for ft in self.fts.values(): if ft.get_state() < minemeld.ft.ft_states.INIT: return False return True def stop(self): LOG.info("chassis stop called") if self.log_glet is not None: self.log_glet.kill() if self.status_glet is not None: self.status_glet.kill() if self.fabric is None: return for ftname, ft in self.fts.iteritems(): try: ft.stop() except: LOG.exception('Error stopping {}'.format(ftname)) LOG.info('Stopping fabric') self.fabric.stop() LOG.info('Stopping mgmtbus') self.mgmtbus.stop() LOG.info('chassis - stopped') self.poweroff.set(value='stop') def start(self): LOG.info("chassis start called") self.log_glet = gevent.spawn(self._log_actor) self.status_glet = gevent.spawn(self._status_actor) for ftname, ft in self.fts.iteritems(): LOG.debug("starting %s", ftname) ft.start() self.fabric.start_dispatching() ================================================ FILE: minemeld/collectd.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ minemeld.collectd Provides a client to collectd for storing metrics. """ import socket import logging LOG = logging.getLogger(__name__) class CollectdClient(object): """Collectd client. Args: path (str): path to the collectd unix socket """ def __init__(self, path): self.path = path self.socket = None def _open_socket(self): if self.socket is not None: return _socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) _socket.connect(self.path) self.socket = _socket def _readline(self): result = '' data = None while data != '\n': data = self.socket.recv(1) if data == '\n' or data is None: return result result += data def _send_cmd(self, command): self._open_socket() self.socket.send(command+'\n') ans = self._readline() status, message = ans.split(None, 1) status = int(status) if status < 0: raise RuntimeError('Error communicating with collectd %s' % message) message = [message] for _ in range(status): message.append(self._readline()) return status, '\n'.join(message) def flush(self, identifier=None, timeout=None): cmd = 'FLUSH' if timeout is not None: cmd += ' timeout=%d' % timeout if identifier is not None: cmd += ' identifier=%s' % identifier self._send_cmd( cmd ) def putval(self, identifier, value, timestamp='N', type_='minemeld_counter', hostname='minemeld', interval=None): if isinstance(timestamp, int): timestamp = '%d' % timestamp identifier = '/'.join([hostname, identifier, type_]) command = 'PUTVAL %s' % identifier if interval is not None: command += ' interval=%d' % interval command += ' %s:%d' % (timestamp, value) self._send_cmd(command) ================================================ FILE: minemeld/comm/__init__.py ================================================ from __future__ import absolute_import from .zmqredis import ZMQRedis def factory(commclass, config): if commclass == 'ZMQRedis': return ZMQRedis(config) return ZMQRedis(config) def cleanup(commclass, config): return ZMQRedis.cleanup(config) ================================================ FILE: minemeld/comm/zmqredis.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # disable import error # pylint:disable=E1101 """ This module implements ZMQ and Redis communication class for mgmtbus and fabric. """ from __future__ import absolute_import import logging import uuid import os import time import gevent import gevent.event import ujson as json from errno import EAGAIN import redis import zmq.green as zmq LOG = logging.getLogger(__name__) class RedisPubChannel(object): def __init__(self, topic, connection_pool): self.topic = topic self.prefix = 'mm:topic:{}'.format(self.topic) self.connection_pool = connection_pool self.SR = None self.num_publish = 0 def connect(self): if self.SR is not None: return self.SR = redis.StrictRedis( connection_pool=self.connection_pool ) def disconnect(self): if self.SR is None: return self.SR = None def lagger(self): # get status of subscribers subscribersc = self.SR.lrange( '{}:subscribers'.format(self.prefix), 0, -1 ) subscribersc = [int(sc) for sc in subscribersc] # check the lagger minsubc = self.num_publish if len(subscribersc) != 0: minsubc = min(subscribersc) return minsubc def gc(self, lagger): minhighbits = lagger >> 12 minqname = '{}:queue:{:013X}'.format( self.prefix, minhighbits ) # delete all the lists before the lagger queues = self.SR.keys('{}:queue:*'.format(self.prefix)) LOG.debug('topic {} - queues: {!r}'.format(self.topic, queues)) queues = [q for q in queues if q < minqname] LOG.debug('topic {} - queues to be deleted: {!r}'.format(self.topic, queues)) if len(queues) != 0: LOG.debug('topic {} - deleting {!r}'.format( self.topic, queues )) self.SR.delete(*queues) def publish(self, method, params=None): high_bits = self.num_publish >> 12 low_bits = self.num_publish & 0xfff if (low_bits % 128) == 127: lagger = self.lagger() LOG.debug('topic {} - sent {} lagger {}'.format( self.topic, self.num_publish, lagger )) while (self.num_publish - lagger) > 1024: LOG.debug('topic {} - waiting lagger delta: {}'.format( self.topic, self.num_publish - lagger )) gevent.sleep(0.1) lagger = self.lagger() if low_bits == 0xfff: # we are switching to a new list, gc self.gc(lagger) msg = { 'method': method, 'params': params } qname = '{}:queue:{:013X}'.format( self.prefix, high_bits ) self.SR.rpush(qname, json.dumps(msg)) self.num_publish += 1 class ZMQRpcFanoutClientChannel(object): def __init__(self, fanout): self.socket = None self.reply_socket = None self.context = None self.fanout = fanout self.active_rpcs = {} def run(self): while True: LOG.debug('RPC Fanout reply recving from {}:reply'.format(self.fanout)) body = self.reply_socket.recv_json() LOG.debug('RPC Fanout reply from {}:reply recvd: {!r}'.format(self.fanout, body)) self.reply_socket.send('OK') LOG.debug('RPC Fanout reply from {}:reply recvd: {!r} - ok'.format(self.fanout, body)) source = body.get('source', None) if source is None: LOG.error('No source in reply in ZMQRpcFanoutClientChannel {}'.format(self.fanout)) continue id_ = body.get('id', None) if id_ is None: LOG.error('No id in reply in ZMQRpcFanoutClientChannel {} from {}'.format(self.fanout, source)) continue actreq = self.active_rpcs.get(id_, None) if actreq is None: LOG.error('Unknown id {} in reply in ZMQRpcFanoutClientChannel {} from {}'.format(id_, self.fanout, source)) continue result = body.get('result', None) if result is None: actreq['errors'] += 1 errmsg = body.get('error', 'no error in reply') LOG.error('Error in RPC reply from {}: {}'.format(source, errmsg)) else: actreq['answers'][source] = result LOG.debug('RPC Fanout state: {!r}'.format(actreq)) if len(actreq['answers'])+actreq['errors'] >= actreq['num_results']: actreq['event'].set({ 'answers': actreq['answers'], 'errors': actreq['errors'] }) self.active_rpcs.pop(id_) gevent.sleep(0) def send_rpc(self, method, params=None, num_results=0, and_discard=False): if self.socket is None: raise RuntimeError('Not connected') if params is None: params = {} id_ = str(uuid.uuid1()) body = { 'reply_to': '{}:reply'.format(self.fanout), 'method': method, 'id': id_, 'params': params } event = gevent.event.AsyncResult() if num_results == 0: event.set({ 'answers': {}, 'errors': 0 }) return event self.active_rpcs[id_] = { 'cmd': method, 'answers': {}, 'num_results': num_results, 'event': event, 'errors': 0, 'discard': and_discard } LOG.debug('RPC Fanout Client: send multipart to {}: {!r}'.format(self.fanout, json.dumps(body))) self.socket.send_multipart([ '{}'.format(self.fanout), json.dumps(body) ]) LOG.debug('RPC Fanout Client: send multipart to {}: {!r} - done'.format(self.fanout, json.dumps(body))) gevent.sleep(0) return event def connect(self, context): if self.socket is not None: return self.context = context self.socket = context.socket(zmq.PUB) self.socket.bind('ipc:///var/run/minemeld/{}'.format(self.fanout)) self.reply_socket = context.socket(zmq.REP) self.reply_socket.bind('ipc:///var/run/minemeld/{}:reply'.format(self.fanout)) def disconnect(self): if self.socket is None: return self.socket.close(linger=0) self.reply_socket.close(linger=0) self.socket = None self.reply_socket = None class ZMQRpcServerChannel(object): def __init__(self, name, obj, allowed_methods=None, method_prefix='', fanout=None): if allowed_methods is None: allowed_methods = [] self.name = name self.obj = obj self.allowed_methods = allowed_methods self.method_prefix = method_prefix self.fanout = fanout self.context = None self.socket = None def _send_result(self, reply_to, id_, result=None, error=None): ans = { 'source': self.name, 'id': id_, 'result': result, 'error': error } if self.fanout is not None: reply_socket = self.context.socket(zmq.REQ) reply_socket.connect('ipc:///var/run/minemeld/{}'.format(reply_to)) LOG.debug('RPC Server {} result to {}'.format(self.name, reply_to)) reply_socket.send_json(ans) reply_socket.recv() LOG.debug('RPC Server {} result to {} - done'.format(self.name, reply_to)) reply_socket.close(linger=0) LOG.debug('RPC Server {} result to {} - closed'.format(self.name, reply_to)) reply_socket = None else: self.socket.send_multipart([reply_to, '', json.dumps(ans)]) def run(self): if self.socket is None: LOG.error('Run called with invalid socket in RPC server channel: {}'.format(self.name)) while True: LOG.debug('RPC Server receiving from {} - {}'.format(self.name, self.fanout)) toks = self.socket.recv_multipart() LOG.debug('RPC Server recvd from {} - {}: {!r}'.format(self.name, self.fanout, toks)) if self.fanout is not None: reply_to, body = toks reply_to = reply_to+':reply' else: reply_to, _, body = toks body = json.loads(body) LOG.debug('RPC command to {}: {!r}'.format(self.name, body)) method = body.get('method', None) id_ = body.get('id', None) params = body.get('params', {}) if method is None: LOG.error('No method in msg body') return if id_ is None: LOG.error('No id in msg body') return method = self.method_prefix+method if method not in self.allowed_methods: LOG.error('Method not allowed in RPC server channel {}: {}'.format(self.name, method)) self._send_result(reply_to, id_, error='Method not allowed') m = getattr(self.obj, method, None) if m is None: LOG.error('Method {} not defined in RPC server channel {}'.format(method, self.name)) self._send_result(reply_to, id_, error='Method not defined') try: result = m(**params) except gevent.GreenletExit: raise except Exception as e: self._send_result(reply_to, id_, error=str(e)) else: self._send_result(reply_to, id_, result=result) def connect(self, context): if self.socket is not None: return self.context = context if self.fanout is not None: # we are subscribers self.socket = self.context.socket(zmq.SUB) self.socket.connect('ipc:///var/run/minemeld/{}'.format(self.fanout)) self.socket.setsockopt(zmq.SUBSCRIBE, b'') # set the filter to empty to recv all messages else: # we are a router self.socket = self.context.socket(zmq.ROUTER) if self.name[0] == '@': address = 'ipc://@/var/run/minemeld/{}:rpc'.format( self.name[1:] ) else: address = 'ipc:///var/run/minemeld/{}:rpc'.format( self.name ) self.socket.bind(address) def disconnect(self): if self.socket is not None: self.socket.close(linger=0) self.socket = None class ZMQPubChannel(object): def __init__(self, topic): self.socket = None self.reply_socket = None self.context = None self.topic = topic def publish(self, method, params=None): if self.socket is None: raise RuntimeError('Not connected') if params is None: params = {} id_ = str(uuid.uuid1()) body = { 'method': method, 'id': id_, 'params': params } try: self.socket.send_json( obj=body, flags=zmq.NOBLOCK ) except zmq.ZMQError: LOG.error('Topic {} queue full - dropping message'.format(self.topic)) gevent.sleep(0) def connect(self, context): if self.socket is not None: return self.context = context self.socket = context.socket(zmq.PUB) self.socket.bind('ipc:///var/run/minemeld/{}'.format(self.topic)) def disconnect(self): if self.socket is None: return self.socket.close(linger=0) self.socket = None class ZMQSubChannel(object): def __init__(self, name, obj, allowed_methods=None, method_prefix='', topic=None): if allowed_methods is None: allowed_methods = [] self.name = name self.obj = obj self.allowed_methods = allowed_methods self.method_prefix = method_prefix self.topic = topic self.context = None self.socket = None def run(self): if self.socket is None: LOG.error('Run called with invalid socket in ZMQ Pub channel: {}'.format(self.name)) while True: LOG.debug('ZMQPub {} receiving'.format(self.name)) body = self.socket.recv_json() LOG.debug('ZMQPub {} recvd: {!r}'.format(self.name, body)) method = body.get('method', None) id_ = body.get('id', None) params = body.get('params', {}) if method is None: LOG.error('No method in msg body') return if id_ is None: LOG.error('No id in msg body') return method = self.method_prefix+method if method not in self.allowed_methods: LOG.error('Method not allowed in RPC server channel {}: {}'.format(self.name, method)) continue m = getattr(self.obj, method, None) if m is None: LOG.error('Method {} not defined in RPC server channel {}'.format(method, self.name)) continue try: m(**params) except gevent.GreenletExit: raise except Exception: LOG.exception('Exception in ZMQPub {}'.format(self.name)) def connect(self, context): if self.socket is not None: return self.context = context self.socket = self.context.socket(zmq.SUB) self.socket.connect('ipc:///var/run/minemeld/{}'.format(self.topic)) self.socket.setsockopt(zmq.SUBSCRIBE, b'') # set the filter to empty to recv all messages def disconnect(self): if self.socket is not None: self.socket.close(linger=0) self.socket = None class RedisSubChannel(object): def __init__(self, topic, connection_pool, object_, allowed_methods, name=None): self.topic = topic self.prefix = 'mm:topic:{}'.format(self.topic) self.channel = None self.name = name self.object = object_ self.allowed_methods = allowed_methods self.connection_pool = connection_pool self.num_callbacks = 0 self.sub_number = None def _callback(self, msg): try: msg = json.loads(msg) except ValueError: LOG.error("invalid message received") return method = msg.get('method', None) params = msg.get('params', {}) if method is None: LOG.error("Message without method field") return if method not in self.allowed_methods: LOG.error("Method not allowed: %s", method) return m = getattr(self.object, method, None) if m is None: LOG.error('Method %s not defined', method) return try: m(**params) except gevent.GreenletExit: raise except: LOG.exception('Exception in handling %s on topic %s ' 'with params %s', method, self.topic, params) self.num_callbacks += 1 def connect(self): subscribers_key = '{}:subscribers'.format(self.prefix) SR = redis.StrictRedis( connection_pool=self.connection_pool ) self.sub_number = SR.rpush( subscribers_key, 0 ) self.sub_number -= 1 LOG.debug('Sub Number {} on {}'.format(self.sub_number, subscribers_key)) def disconnect(self): pass class ZMQRedis(object): def __init__(self, config): self.context = None self.rpc_server_channels = {} self.pub_channels = [] self.mw_pub_channels = [] self.sub_channels = [] self.mw_sub_channels = [] self.rpc_fanout_clients_channels = [] self.active_rpcs = {} self.ioloops = [] self.failure_listeners = [] self.redis_config = { 'url': os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock') } self.redis_cp = redis.ConnectionPool.from_url( self.redis_config['url'] ) def add_failure_listener(self, listener): self.failure_listeners.append(listener) def request_rpc_server_channel(self, name, obj=None, allowed_methods=None, method_prefix='', fanout=None): if allowed_methods is None: allowed_methods = [] if name in self.rpc_server_channels: return self.rpc_server_channels[name] = ZMQRpcServerChannel( name, obj, method_prefix=method_prefix, allowed_methods=allowed_methods, fanout=fanout ) def request_rpc_fanout_client_channel(self, topic): c = ZMQRpcFanoutClientChannel(topic) self.rpc_fanout_clients_channels.append(c) return c def request_pub_channel(self, topic, multi_write=False): if not multi_write: redis_pub_channel = RedisPubChannel( topic=topic, connection_pool=self.redis_cp ) self.pub_channels.append(redis_pub_channel) return redis_pub_channel zmq_pub_channel = ZMQPubChannel(topic=topic) self.mw_pub_channels.append(zmq_pub_channel) return zmq_pub_channel def request_sub_channel(self, topic, obj=None, allowed_methods=None, name=None, max_length=None, multi_write=False): if allowed_methods is None: allowed_methods = [] if not multi_write: subchannel = RedisSubChannel( topic=topic, connection_pool=self.redis_cp, object_=obj, allowed_methods=allowed_methods, name=name ) self.sub_channels.append(subchannel) return subchannel = ZMQSubChannel( name=name, obj=obj, allowed_methods=allowed_methods, topic=topic ) self.mw_sub_channels.append(subchannel) def send_rpc(self, dest, method, params, block=True, timeout=None): if self.context is None: LOG.error('send_rpc to {} when not connected'.format(dest)) return id_ = str(uuid.uuid1()) body = { 'method': method, 'id': id_, 'params': params } socket = self.context.socket(zmq.REQ) if dest[0] == '@': address = 'ipc://@/var/run/minemeld/{}:rpc'.format( dest[1:] ) else: address = 'ipc:///var/run/minemeld/{}:rpc'.format( dest ) socket.connect(address) socket.setsockopt(zmq.LINGER, 0) socket.send_json(body) LOG.debug('RPC sent to {}:rpc for method {}'.format(dest, method)) if not block: socket.close(linger=0) return if timeout is not None: # zmq green does not support RCVTIMEO if socket.poll(flags=zmq.POLLIN, timeout=int(timeout*1000)) != 0: result = socket.recv_json(flags=zmq.NOBLOCK) else: socket.close(linger=0) raise RuntimeError('Timeout in RPC') else: result = socket.recv_json() socket.close(linger=0) return result def _ioloop(self, executor): executor.run() def _sub_ioloop(self, schannel): LOG.debug('start draining messages on topic {}'.format(schannel.topic)) counter = 0 SR = redis.StrictRedis(connection_pool=self.redis_cp) subscribers_key = '{}:subscribers'.format(schannel.prefix) while True: base = counter & 0xfff top = min(base + 127, 0xfff) msgs = SR.lrange( '{}:queue:{:013X}'.format(schannel.prefix, counter >> 12), base, top ) for m in msgs: LOG.debug('topic {} - {!r}'.format( schannel.topic, m )) schannel._callback(m) counter += len(msgs) if len(msgs) > 0: SR.lset( subscribers_key, schannel.sub_number, counter ) if len(msgs) < (top - base + 1): gevent.sleep(1.0) else: gevent.sleep(0) def _ioloop_failure(self, g): LOG.error('_ioloop_failure') try: g.get() except gevent.GreenletExit: return except: LOG.exception("_ioloop_failure: exception in ioloop") for l in self.failure_listeners: l() def start(self, start_dispatching=True): self.context = zmq.Context() for rfcc in self.rpc_fanout_clients_channels: rfcc.connect(self.context) for rpcc in self.rpc_server_channels.values(): rpcc.connect(self.context) for sc in self.sub_channels: sc.connect() for mwsc in self.mw_sub_channels: mwsc.connect(self.context) for pc in self.pub_channels: pc.connect() for mwpc in self.mw_pub_channels: mwpc.connect(self.context) if start_dispatching: self.start_dispatching() def start_dispatching(self): for rfcc in self.rpc_fanout_clients_channels: g = gevent.spawn(self._ioloop, rfcc) self.ioloops.append(g) g.link_exception(self._ioloop_failure) for rpcc in self.rpc_server_channels.values(): g = gevent.spawn(self._ioloop, rpcc) self.ioloops.append(g) g.link_exception(self._ioloop_failure) for schannel in self.sub_channels: g = gevent.spawn(self._sub_ioloop, schannel) self.ioloops.append(g) g.link_exception(self._ioloop_failure) for mwschannel in self.mw_sub_channels: g = gevent.spawn(self._ioloop, mwschannel) self.ioloops.append(g) g.link_exception(self._ioloop_failure) def stop(self): # kill ioloops for j in xrange(len(self.ioloops)): self.ioloops[j].unlink(self._ioloop_failure) self.ioloops[j].kill() self.ioloops[j] = None self.ioloops = None # close channels for rpcc in self.rpc_server_channels.values(): try: rpcc.disconnect() except Exception: LOG.debug("exception in disconnect: ", exc_info=True) for pc in self.pub_channels: try: pc.disconnect() except Exception: LOG.debug("exception in disconnect: ", exc_info=True) for mwpc in self.mw_pub_channels: try: mwpc.disconnect() except Exception: LOG.debug("exception in disconnect: ", exc_info=True) for sc in self.sub_channels: try: sc.disconnect() except Exception: LOG.debug("exception in disconnect: ", exc_info=True) for mwsc in self.mw_sub_channels: try: mwsc.disconnect() except Exception: LOG.debug("exception in disconnect: ", exc_info=True) for rfc in self.rpc_fanout_clients_channels: try: rfc.disconnect() except Exception: LOG.debug("exception in disconnect: ", exc_info=True) self.context.destroy() @staticmethod def cleanup(config): redis_cp = redis.ConnectionPool.from_url( os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock') ) SR = redis.StrictRedis(connection_pool=redis_cp) tkeys = SR.keys(pattern='mm:topic:*') if len(tkeys) > 0: LOG.info('Deleting old keys: {}'.format(len(tkeys))) SR.delete(*tkeys) SR = None redis_cp = None ================================================ FILE: minemeld/extensions/__init__.py ================================================ from .manager import * # noqa ================================================ FILE: minemeld/extensions/manager.py ================================================ import sys import os import os.path import json import logging from email.parser import Parser from collections import namedtuple from zipfile import ZipFile from pkg_resources import EntryPoint, parse_version import minemeld.loader LOG = logging.getLogger(__name__) __all__ = [ 'get_metadata_from_wheel', 'activated_extensions', 'installed_extensions', 'extensions', 'freeze', 'load_frozen_paths' ] METADATA_MAP = { 'name': 'Name', 'version': 'Version', 'author': 'Author', 'author_email': 'Author-email', 'description': 'Summary', 'url': 'Home-page' } InstalledExtension = namedtuple( 'InstalledExtension', [ 'name', 'version', 'author', 'author_email', 'description', 'url', 'path', 'entry_points' ] ) ActivatedExtension = namedtuple( 'ActivatedExtension', [ 'name', 'version', 'author', 'author_email', 'description', 'url', 'location', 'entry_points' ] ) ExternalExtension = namedtuple( 'ExternalExtension', [ 'name', 'version', 'author', 'author_email', 'description', 'url', 'path', 'activated', 'installed', 'entry_points' ] ) def _egg_link_path(dist): for path_item in sys.path: egg_link = os.path.join(path_item, dist.project_name + '.egg-link') if os.path.isfile(egg_link): return egg_link return None def _read_metadata(metadata_str): return Parser().parsestr(metadata_str) def _read_entry_points(ep_contents): ep_map = EntryPoint.parse_map(ep_contents) for _, epgroup in ep_map.iteritems(): for epname, ep in epgroup.iteritems(): epgroup[epname] = str(ep) return ep_map def _activated_extensions(): epgroups = ( minemeld.loader.MM_NODES_ENTRYPOINT, minemeld.loader.MM_NODES_GCS_ENTRYPOINT, minemeld.loader.MM_NODES_VALIDATORS_ENTRYPOINT, minemeld.loader.MM_PROTOTYPES_ENTRYPOINT, minemeld.loader.MM_API_ENTRYPOINT, minemeld.loader.MM_WEBUI_ENTRYPOINT ) activated_extensions = {} for epgroup in epgroups: for _, epvalue in minemeld.loader.map(epgroup).iteritems(): if epvalue.ep.dist.project_name == 'minemeld-core': continue location = 'site-packages' egg_link = _egg_link_path(epvalue.ep.dist) if egg_link is not None: with open(egg_link, 'r') as f: location = f.readline().strip() metadata = { 'name': epvalue.ep.dist.project_name, 'version': epvalue.ep.dist.version, 'author': None, 'author_email': None, 'description': None, 'url': None, 'entry_points': None } if egg_link: try: with open(os.path.join(location, 'minemeld.json'), 'r') as f: dist_metadata = json.load(f) for k in metadata.keys(): metadata[k] = dist_metadata.get(k, None) except (IOError, OSError) as excpt: LOG.error('Error loading metatdata from {}: {}'.format(location, str(excpt))) elif epvalue.ep.dist.has_metadata('METADATA'): dist_metadata = _read_metadata( epvalue.ep.dist.get_metadata('METADATA') ) for k in metadata.keys(): if k in METADATA_MAP and METADATA_MAP[k] in dist_metadata: metadata[k] = dist_metadata[METADATA_MAP[k]] if epvalue.ep.dist.has_metadata('entry_points.txt'): metadata['entry_points'] = _read_entry_points( epvalue.ep.dist.get_metadata('entry_points.txt') ) activated_extensions[epvalue.ep.dist.project_name] = ActivatedExtension( location=location, **metadata ) return activated_extensions def _load_metadata_from_wheel(extpath, extname=None): wheel_name = extname if extname is None: wheel_name = os.path.basename(extpath) project_name, version, _ = wheel_name.split('-', 2) metadata_path = '{}-{}.dist-info/METADATA'.format(project_name, version) with ZipFile(extpath, 'r') as wheel_file: metadata_file = wheel_file.open(metadata_path, 'r') metadata_lines = metadata_file.read() metadata = _read_metadata(metadata_lines) # classifier framework :: minemeld should be in METADATA # for this to be an extension classifiers = metadata.get_all('Classifier') if classifiers is None: return None for c in classifiers: if c.lower() == 'framework :: minemeld': break else: return None ie_metadata = {} for field in InstalledExtension._fields: if field == 'path' or field == 'entry_points': continue ie_metadata[field] = metadata.get(METADATA_MAP[field], None) entry_points = None try: ep_path = '{}-{}.dist-info/entry_points.txt'.format(project_name, version) with ZipFile(extpath, 'r') as wheel_file: ep_file = wheel_file.open(ep_path, 'r') ep_contents = ep_file.read() entry_points = _read_entry_points(ep_contents) except (IOError, OSError): pass ie_metadata['entry_points'] = entry_points return InstalledExtension( path=extpath, **ie_metadata ) def _load_metadata_from_dir(extpath): with open(os.path.join(extpath, 'minemeld.json'), 'r') as f: metadata = json.load(f) return InstalledExtension( name=metadata['name'], version=metadata['version'], author=metadata['author'], author_email=metadata.get('author_email', None), description=metadata.get('description', None), url=metadata.get('url', None), entry_points=metadata.get('entry_points', None), path=extpath ) def _is_activated(installed_extension, activated): activated_extension = activated.get(installed_extension.name, None) if activated_extension is None: return False if activated_extension.version != installed_extension.version: return False if installed_extension.path == activated_extension.location: return True if activated_extension.location == 'site-packages' and \ installed_extension.path.endswith('.whl'): return True return False def get_metadata_from_wheel(wheelpath, wheelname=None): return _load_metadata_from_wheel(wheelpath, wheelname) def installed_extensions(installation_dir): _installed_extensions = [] entries = os.listdir(installation_dir) for e in entries: epath = os.path.join(installation_dir, e) # check if this is a wheel if e.endswith('.whl'): try: installed_extension = _load_metadata_from_wheel(epath) if installed_extension is None: continue _installed_extensions.append(installed_extension) except (ValueError, IOError, KeyError, OSError) as excpt: LOG.error(u'Error extracting metadata from {}: {}'.format(e, str(excpt))) # check if it is a directory elif os.path.isdir(epath): try: installed_extension = _load_metadata_from_dir(epath) if installed_extension is None: continue _installed_extensions.append(installed_extension) except (IOError, OSError, KeyError) as excpt: LOG.error(u'Error extracting metadata from {}: {}'.format(e, str(excpt))) return _installed_extensions def activated_extensions(): return _activated_extensions() def extensions(installation_dir): _extensions = [] _installed = installed_extensions(installation_dir) _activated = activated_extensions() for installed_extension in _installed: _extension_activated = _is_activated(installed_extension, _activated) _extensions.append(ExternalExtension( installed=True, activated=_extension_activated, **installed_extension._asdict() )) if _extension_activated: _activated.pop(installed_extension.name) for _activated_extension in _activated.values(): _adict = _activated_extension._asdict() _adict.pop('location') _extensions.append(ExternalExtension( installed=False, activated=True, path=None, **_adict )) return _extensions def freeze(installation_dir): _freeze = [] _extensions = extensions(installation_dir) for e in _extensions: if not e.activated: continue if not e.installed: continue if e.path.endswith('.whl'): _freeze.append(e.path) else: _freeze.append('-e {}'.format(e.path)) return _freeze def load_frozen_paths(freeze_file): for l in freeze_file: l = l.strip() if not l.startswith('-e '): continue _, epath = l.split(' ', 1) if epath not in sys.path: LOG.info('Extension path {!r} not in sys.path, adding'.format(epath)) sys.path.append(epath) ================================================ FILE: minemeld/fabric.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ minemeld.fabric This module implements fabric abstraction over communication backend class. Each chassis has an instance of Fabric and nodes request connections to the fabric using this instance. """ from __future__ import absolute_import import logging import minemeld.comm LOG = logging.getLogger(__name__) class Fabric(object): """MineMeld chassis fabric class Args: chassis: MineMeld chassis instance config (dict): communication backend config comm_class (string): communication backend to be used """ def __init__(self, chassis, config, comm_class): self.chassis = chassis self.comm_config = config self.comm_class = comm_class self.comm = minemeld.comm.factory(self.comm_class, self.comm_config) def request_rpc_channel(self, ftname, node, allowed_methods): """Creates a new RPC channel on the communication backend. Args: ftname (str): node name node: node instance allowed_methods (list): list of allowed methods """ self.comm.request_rpc_server_channel(ftname, node, allowed_methods) def request_pub_channel(self, ftname): """Creates a new channel for publishing to a topic with name ftname. Args: ftname (str): node name """ return self.comm.request_pub_channel(ftname) def request_sub_channel(self, ftname, node, subname, allowed_methods): """Creates a subscription channel to topic subname. Args: ftname (str): name of the node node: node instance subname (str): name of the topic to subscribe to allowed_methods (list): list of allowed methods """ _ = ftname # noqa self.comm.request_sub_channel(subname, node, allowed_methods) def send_rpc(self, sftname, dftname, method, params, block=True, timeout=None): """Sends a RPC command to a specific node. Args: sftname (str): source node name dftname (str): destination node name method (str): method name params (dict): parameters block (bool): if call should block timeout (int): timeout in seconds """ params['source'] = sftname self.comm.send_rpc( dftname, method, params, block=block, timeout=timeout ) def _comm_failure(self): self.chassis.fabric_failed() def start(self): LOG.debug("fabric start called") self.comm.add_failure_listener(self._comm_failure) self.comm.start(start_dispatching=False) def start_dispatching(self): self.comm.start_dispatching() def stop(self): LOG.debug("fabric stop called") self.comm.stop() def factory(classname, chassis, config): """Factory for Fabric class. Args: classname (str): communication backend name chassis: chassis instance config (dict): communication backend config """ return Fabric( chassis=chassis, config=config, comm_class=classname ) ================================================ FILE: minemeld/flask/__init__.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import logging import yaml from flask import Flask import minemeld.loader from .logger import LOG REDIS_URL = os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock') def create_app(): yaml.SafeLoader.add_constructor( u'tag:yaml.org,2002:timestamp', yaml.SafeLoader.construct_yaml_str ) app = Flask(__name__) app.config['MAX_CONTENT_LENGTH'] = 5 * 1024 * 1024 # max 5MB for uploads LOG.init_app(app) # extension code from . import config from . import aaa from . import session from . import mmrpc from . import redisclient from . import supervisorclient from . import jobs from . import sns from . import events session.init_app(app, REDIS_URL) aaa.init_app(app) config.init() if config.get('DEBUG', False): logging.getLogger().setLevel(logging.DEBUG) else: logging.getLogger().setLevel(logging.INFO) mmrpc.init_app(app) redisclient.init_app(app) supervisorclient.init_app(app) jobs.init_app(app) sns.init_app() events.init_app(app, REDIS_URL) # entrypoints from . import metricsapi # noqa from . import feedredis # noqa from . import configapi # noqa from . import configdataapi # noqa from . import taxiidiscovery # noqa from . import taxiicollmgmt # noqa from . import taxiipoll # noqa from . import supervisorapi # noqa from . import loginapi # noqa from . import prototypeapi # noqa from . import validateapi # noqa from . import aaaapi # noqa from . import statusapi # noqa from . import tracedapi # noqa from . import logsapi # noqa from . import extensionsapi # noqa from . import jobsapi # noqa configapi.init_app(app) extensionsapi.init_app(app) app.register_blueprint(metricsapi.BLUEPRINT) app.register_blueprint(statusapi.BLUEPRINT) app.register_blueprint(feedredis.BLUEPRINT) app.register_blueprint(configapi.BLUEPRINT) app.register_blueprint(configdataapi.BLUEPRINT) app.register_blueprint(taxiidiscovery.BLUEPRINT) app.register_blueprint(taxiicollmgmt.BLUEPRINT) app.register_blueprint(taxiipoll.BLUEPRINT) app.register_blueprint(supervisorapi.BLUEPRINT) app.register_blueprint(loginapi.BLUEPRINT) app.register_blueprint(prototypeapi.BLUEPRINT) app.register_blueprint(validateapi.BLUEPRINT) app.register_blueprint(aaaapi.BLUEPRINT) app.register_blueprint(tracedapi.BLUEPRINT) app.register_blueprint(logsapi.BLUEPRINT) app.register_blueprint(extensionsapi.BLUEPRINT) app.register_blueprint(jobsapi.BLUEPRINT) # install blueprints from extensions for apiname, apimmep in minemeld.loader.map(minemeld.loader.MM_API_ENTRYPOINT).iteritems(): LOG.info('Loading blueprint from {}'.format(apiname)) if not apimmep.loadable: LOG.info('API entrypoint {} not loadable, ignored'.format(apiname)) continue try: bprint = apimmep.ep.load() app.register_blueprint(bprint()) except (ImportError, RuntimeError): LOG.exception('Error loading API entry point {}'.format(apiname)) # install webui blueprints from extensions for webuiname, webuimmep in minemeld.loader.map(minemeld.loader.MM_WEBUI_ENTRYPOINT).iteritems(): LOG.info('Loading blueprint from {}'.format(webuiname)) if not webuimmep.loadable: LOG.info('API entrypoint {} not loadable, ignored'.format(webuiname)) continue try: bprint = webuimmep.ep.load() app.register_blueprint( bprint(), url_prefix='/extensions/webui/{}'.format(webuiname) ) except (ImportError, RuntimeError): LOG.exception('Error loading WebUI entry point {}'.format(webuiname)) for r in app.url_map.iter_rules(): LOG.debug('app rule: {!r}'.format(r)) return app ================================================ FILE: minemeld/flask/aaa.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import base64 from functools import wraps import gevent import gevent.lock import flask.ext.login from flask import current_app, Blueprint, request from . import config from .logger import LOG ANONYMOUS = 'mm-anonymous' PREVENT_WRITE_GUARD = None PREVENT_WRITE = None def disable_prevent_write(locker): global PREVENT_WRITE with PREVENT_WRITE_GUARD: if PREVENT_WRITE == locker: LOG.info('Disabled prevent write from locker {}'.format(locker)) PREVENT_WRITE = None def enable_prevent_write(locker, timeout=900): global PREVENT_WRITE def _cleanup_prevent_write(): gevent.sleep(timeout) LOG.info('Checking if prevent write still enabled by locker {}'.format(locker)) disable_prevent_write(locker) with PREVENT_WRITE_GUARD: if PREVENT_WRITE is None: PREVENT_WRITE = locker gevent.spawn(_cleanup_prevent_write) return False class MMBlueprint(Blueprint): def __init__(self, *args, **kwargs): super(MMBlueprint, self).__init__(*args, **kwargs) self.send_static_file = self._login_required( super(MMBlueprint, self).send_static_file, login_required=True, read_write=False, feeds=False ) def _audit(self, f, audit_required): if not audit_required: return f @wraps(f) def audited_view(*args, **kwargs): if request and flask.ext.login.current_user: params = [] for key, values in request.values.iterlists(): if key == '_': continue params.append(('value:{}'.format(key), values)) for filename, files in request.files.iterlists(): params.append(('file:{}'.format(filename), [file.filename for file in files])) body = request.get_json(silent=True) if body is not None: params.append(['jsonbody', json.dumps(body)[:1024]]) LOG.audit( user_id=flask.ext.login.current_user.get_id(), action_name='{} {}'.format(request.method, request.path), params=params ) else: LOG.critical('no request or current_user in audited_view') return f(*args, **kwargs) return audited_view def _login_required(self, f, login_required, read_write, feeds): @wraps(f) def decorated_view(*args, **kwargs): if not login_required: return f(*args, **kwargs) if not config.get('API_AUTH_ENABLED', True) and not feeds: return f(*args, **kwargs) if not config.get('FEEDS_AUTH_ENABLED', False) and feeds: return f(*args, **kwargs) if not feeds: if not flask.ext.login.current_user.is_authenticated(): return current_app.login_manager.unauthorized() if flask.ext.login.current_user.get_id().startswith('feeds/'): return current_app.login_manager.unauthorized() if read_write and not flask.ext.login.current_user.is_read_write(): return 'Forbidden', 403 return f(*args, **kwargs) return decorated_view def _write_prevented(self, f, read_write): @wraps(f) def decorated_view(*args, **kwargs): if read_write and PREVENT_WRITE is not None: return 'Changes disabled by {}'.format(PREVENT_WRITE), 403 return f(*args, **kwargs) return decorated_view def route(self, rule, **options): def decorator(f): login_required = options.pop('login_required', True) read_write = options.pop('read_write', True) feeds = options.pop("feeds", False) super_decorator = super(MMBlueprint, self).route(rule, **options) _wp_f = self._write_prevented(f, read_write) _lr_f = self._login_required(_wp_f, login_required, read_write, feeds) _audit_f = self._audit(_lr_f, read_write) return super_decorator(_audit_f) return decorator class MMAnonynmousUser(object): def __init__(self): self._id = ANONYMOUS def get_id(self): return self._id def is_authenticated(self): return False def is_active(self): return True def is_anonymous(self): return True def is_read_write(self): return False def check_feed(self, feedname): if not config.get('FEEDS_AUTH_ENABLED', False): return True fattributes = config.get('FEEDS_ATTRS', None) if fattributes is None or feedname not in fattributes: return False ftags = set(fattributes[feedname].get('tags', [])) if 'anonymous' in ftags: return True return False class MMAuthenticatedUser(object): def __init__(self, _id=None): self._id = unicode(_id) def get_id(self): return self._id def is_authenticated(self): return True def is_active(self): return True def is_anonymous(self): return False class MMAuthenticatedAdminUser(MMAuthenticatedUser): def __init__(self, _id): super(MMAuthenticatedAdminUser, self).__init__(_id=u'admin/{}'.format(_id)) def is_read_write(self): read_write = config.get('READ_WRITE', None) if read_write is None: return True if isinstance(read_write, str) or isinstance(read_write, unicode): read_write = read_write.split(',') elif not isinstance(read_write, list): LOG.error('Unknown READ_WRITE format') return False if self._id[6:] in read_write: return True return False def check_feed(self, feedname): return True class MMAuthenticatedFeedUser(MMAuthenticatedUser): def __init__(self, _id): super(MMAuthenticatedFeedUser, self).__init__(_id=u'feeds/{}'.format(_id)) def is_read_write(self): # this should never be called return False def check_feed(self, feedname): if not config.get('FEEDS_AUTH_ENABLED', False): return True fattributes = config.get('FEEDS_ATTRS', None) if fattributes is None or feedname not in fattributes: return False ftags = set(fattributes[feedname].get('tags', [])) # if 'any' is present, any authenticated user can access # the feed if 'any' in ftags: return True uattributes = config.get('FEEDS_USERS_ATTRS', None) if uattributes is None or self._id[6:] not in uattributes: return False tags = set(uattributes[self._id[6:]].get('tags', [])) return len(tags.intersection(ftags)) != 0 def authenticated_user_factory(_id): if _id.startswith('feeds/'): return MMAuthenticatedFeedUser(_id=_id[6:]) if _id.startswith('admin/'): return MMAuthenticatedAdminUser(_id=_id[6:]) if _id == ANONYMOUS: return MMAnonynmousUser() raise RuntimeError('Unknown user_id prefix: {}'.format(_id)) LOGIN_MANAGER = flask.ext.login.LoginManager() LOGIN_MANAGER.session_protection = None LOGIN_MANAGER.anonymous_user = MMAnonynmousUser @LOGIN_MANAGER.request_loader def request_loader(request): api_key = request.headers.get('Authorization') if api_key is None: return None api_key = api_key.replace('Basic', '', 1) try: api_key = base64.b64decode(api_key) except TypeError: return None try: user, password = api_key.split(':', 1) except ValueError: return None auth_user = check_feeds_user(user, password) if auth_user is not None: return auth_user auth_user = check_admin_user(user, password) if auth_user is not None: return auth_user return None @LOGIN_MANAGER.user_loader def user_loader(_id): return authenticated_user_factory(_id) def check_feeds_user(username, password): if not config.get('FEEDS_USERS_DB').check_password(username, password): return None return MMAuthenticatedFeedUser(_id=username) def check_admin_user(username, password): if not config.get('USERS_DB').check_password(username, password): return None return MMAuthenticatedAdminUser(_id=username) @LOGIN_MANAGER.unauthorized_handler def unauthorized(): return 'Unauthorized', 401 def init_app(app): global PREVENT_WRITE global PREVENT_WRITE_GUARD app.config['REMEMBER_COOKIE_NAME'] = None # to block remember cookie LOGIN_MANAGER.init_app(app) # initialize PREVENT_WRITE PREVENT_WRITE_GUARD = gevent.lock.BoundedSemaphore() PREVENT_WRITE = None ================================================ FILE: minemeld/flask/aaaapi.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import collections from flask import request, jsonify from flask.ext.login import current_user from . import config from .aaa import MMBlueprint from .logger import LOG __all__ = ['BLUEPRINT'] BLUEPRINT = MMBlueprint('aaa', __name__, url_prefix='/aaa') # if you change things here change also backup/import API API_USERS_ATTRS_ATTR = 'API_USERS_ATTRS' FEEDS_USERS_ATTRS_ATTR = 'FEEDS_USERS_ATTRS' FEEDS_ATTRS_ATTR = 'FEEDS_ATTRS' Subsystem = collections.namedtuple( 'Subsystem', ['authdb', 'attrs', 'enabled', 'enabled_default'], verbose=True ) _SUBSYSTEM_MAP = { 'api': Subsystem( authdb='USERS_DB', enabled='API_AUTH_ENABLED', enabled_default=True, attrs=config.APIConfigDict(attribute=API_USERS_ATTRS_ATTR, level=50) ), 'feeds': Subsystem( authdb='FEEDS_USERS_DB', enabled='FEEDS_AUTH_ENABLED', enabled_default=False, attrs=config.APIConfigDict(attribute=FEEDS_USERS_ATTRS_ATTR, level=50) ) } _FEEDS_ATTRS = config.APIConfigDict(attribute=FEEDS_ATTRS_ATTR, level=50) @BLUEPRINT.route('/users/current', methods=['GET'], read_write=False) def get_current_user(): return jsonify(result={ 'id': current_user.get_id(), 'read_write': current_user.is_read_write() }) @BLUEPRINT.route('/users/', methods=['GET'], read_write=False) def get_users(subsystem): subsystem = _SUBSYSTEM_MAP.get(subsystem, None) if subsystem is None: return jsonify(error='Invalid subsystem'), 400 result = { 'enabled': config.get(subsystem.enabled, subsystem.enabled_default), 'users': {} } users = config.get(subsystem.authdb).users() users_attrs = subsystem.attrs.value() for u in users: attrs = {} if u in users_attrs: attrs = users_attrs[u] result['users'][u] = attrs return jsonify(result=result) @BLUEPRINT.route('/users//', methods=['PUT'], read_write=True) def set_user_password(subsystem, username): subsystem = _SUBSYSTEM_MAP.get(subsystem, None) if subsystem is None: return jsonify(error='Invalid subsystem'), 400 with config.lock(): users_db = config.get(subsystem.authdb) if not users_db.path: return jsonify(error='Users database not available'), 500 try: password = request.get_json()['password'] except Exception: return jsonify(error='Invalid request'), 400 users_db.set_password(username, password) users_db.save() return jsonify(result='ok') @BLUEPRINT.route('/users///attributes', methods=['POST'], read_write=True) def set_user_attributes(subsystem, username): subsystem = _SUBSYSTEM_MAP.get(subsystem, None) if subsystem is None: return jsonify(error='Invalid subsystem'), 400 with config.lock(): users_db = config.get(subsystem.authdb) if not users_db.path: return jsonify(error='Users database not available'), 500 if username not in users_db.users(): return jsonify(error='Unknown user'), 400 try: attributes = request.get_json() except Exception: return jsonify(error='Invalid request'), 400 if not isinstance(attributes, dict): return jsonify(error='Attributes should be a dict'), 400 subsystem.attrs.set(username, attributes) return jsonify(result='ok') @BLUEPRINT.route('/users//', methods=['DELETE'], read_write=True) def delete_user(subsystem, username): subsystem = _SUBSYSTEM_MAP.get(subsystem, None) if subsystem is None: return jsonify(error='Invalid subsystem'), 400 with config.lock(): users_db = config.get(subsystem.authdb) if not users_db.path: return jsonify(error='Users database not available'), 500 # delete user from database and tags if users_db.delete(username): users_db.save() subsystem.attrs.delete(username) return jsonify(result='ok') @BLUEPRINT.route('/feeds', methods=['GET'], read_write=False) def get_feeds(): result = { 'enabled': config.get( _SUBSYSTEM_MAP['feeds'].enabled, _SUBSYSTEM_MAP['feeds'].enabled_default ), 'feeds': _FEEDS_ATTRS.value() } return jsonify(result=result) @BLUEPRINT.route('/feeds//attributes', methods=['PUT', 'POST'], read_write=True) def set_feed_attributes(feedname): with config.lock(): try: attributes = request.get_json() except Exception: return jsonify(error='Invalid request'), 400 if not isinstance(attributes, dict): return jsonify(error='Attributes should be a dict'), 400 _FEEDS_ATTRS.set(feedname, attributes) return jsonify(result='ok') @BLUEPRINT.route('/feeds/', methods=['DELETE'], read_write=True) def delete_feed(feedname): with config.lock(): _FEEDS_ATTRS.delete(feedname) return jsonify(result='ok') @BLUEPRINT.route('/tags', methods=['GET'], read_write=False) def get_tags(): tags = set() for _, subsystem in _SUBSYSTEM_MAP.iteritems(): for _, attributes in subsystem.attrs.value().iteritems(): if 'tags' in attributes: for t in attributes['tags']: tags.add(t) for _, attributes in _FEEDS_ATTRS.value().iteritems(): if 'tags' in attributes: for t in attributes['tags']: tags.add(t) return jsonify(result=list(tags - set(['any', 'anonymous']))) ================================================ FILE: minemeld/flask/cbfeed.py ================================================ import json import time class CbFeed(object): def __init__(self, feedinfo, reports): self.data = {'feedinfo': feedinfo, 'reports': reports} def dump(self): return json.dumps(self.data, indent=2) class CbFeedInfo(object): def __init__(self, **kwargs): self.yieldable_atts = ("category", "icon", "icon_small", "version", "provider_url", "display_name", "summary", "tech_data", "name") self.data = kwargs self.data["category"] = self.data.get("category", "MineMeld") self.data["icon"] = self.data.get("icon", MinemeldIcon.MM_icon_png) self.data["icon_small"] = self.data.get("icon_small", MinemeldIcon.MM_icon_small_png) self.data["version"] = self.data.get("version", "0.1") self.data["provider_url"] = self.data.get("provider_url", "https://live.paloaltonetworks.com/t5/MineMeld/ct-p/MineMeld") self.data["display_name"] = self.data.get("display_name", "MineMeld Feed") self.data["summary"] = self.data.get("summary", "Indicators routed through MineMeld") self.data["tech_data"] = self.data.get("tech_data", "Indicators routed through MineMeld") if "name" not in kwargs: raise ValueError("Mandatory 'name' attribute not provided") def dump(self): return self.data def iterate(self): last_element = len(self.yieldable_atts) - 1 for idx, id in enumerate(self.yieldable_atts): final_comma = "," if idx < last_element else "" if isinstance(self.data[id], int): yield "\"{}\": {}{}".format(id, self.data[id], final_comma) else: yield "\"{}\": \"{}\"{}".format(id, self.data[id], final_comma) class CbReport(object): def __init__(self, **kwargs): # these fields are optional self.optional = ("tags", "description") self.yieldable_atts = ("timestamp", "id", "link", "score", "description", "title") if "timestamp" not in kwargs: kwargs["timestamp"] = int(time.mktime(time.gmtime())) if "id" not in kwargs: raise ValueError("Mandatory 'id' attribute not provided") self.data = kwargs self.data["link"] = self.data.get("link", "https://live.paloaltonetworks.com/t5/MineMeld/ct-p/MineMeld") self.data["score"] = self.data.get("score", 100) if not isinstance(self.data["score"], int): self.data["score"] = 100 self.data["description"] = self.data.get("description", "MineMeld Generated Report") self.data["iocs"] = self.data.get("iocs", None) self.data["title"] = self.data.get("title", "MineMeld Generated Report") def dump(self): return self.data def iterate(self): last_element = len(self.yieldable_atts) - 1 for idx, id in enumerate(self.yieldable_atts): final_comma = "," if idx < last_element else "" if isinstance(self.data[id], int): yield "\"{}\": {}{}".format(id, self.data[id], final_comma) else: yield "\"{}\": \"{}\"{}".format(id, self.data[id], final_comma) class MinemeldIcon(object): MM_icon_small_png = ("iVBORw0KGgoAAAANSUhEUgAAADIAAAAyCAYAAAAeP4ixAAAMFmlDQ1BJQ0MgUHJvZmlsZ" "QAASImVVwdYU8kWnltSCAktEAEpoTdBehUIHQQB6WAjJAFCCZAQVOzIooJrQcWCFV0Bsa" "0FkLUiioVFwF4XRFRW1sWCDZU3KaDP1753vm/u/Dlzzpn/zD13MgOAsi07NzcLVQEgW5A" "vjAryZSYkJjFJPUABUAEFGACczRHl+kRGhgEoo/0/y7tbAJH0160lsf51/L+KKpcn4gCA" "REKcwhVxsiE+BgCuyckV5gNAaIN6o9n5uRI8CLG6EBIEgIhLcJoMa0pwigxPkNrERPlBz" "AKATGWzhWkAKEl4Mws4aTCOkoSjrYDLF0C8FWIvTjqbC/EDiCdkZ+dArEyG2Dzluzhp/x" "QzZSwmm502hmW5SIXszxflZrHn/p/L8b8lO0s8OochbNR0YXCUJGe4bjWZOaESTIX4pCA" "lPAJiNYgv8blSewm+ly4OjpXbD3BEfnDNAAMAFHDZ/qEQ60DMEGfG+sixPVso9YX2aDg/" "PyRGjlOEOVHy+GiBICs8TB5neTovZBRv54kCokdtUvmBIRDDSkOPFabHxMt4oi0F/Lhwi" "JUg7hBlRofKfR8VpvuFj9oIxVESzsYQv00VBkbJbDDNbNFoXpgNhy2dC9YCxspPjwmW+W" "IJPFFC2CgHLs8/QMYB4/IEsXJuGKwu3yi5b0luVqTcHtvOywqKkq0zdlhUED3q25UPC0y" "2DtjjDPbkSPlc73LzI2Nk3HAUhAE/4A+YQAxbCsgBGYDfPtAwAH/JRgIBGwhBGuABa7lm" "1CNeOiKAz2hQCP6CiAdEY36+0lEeKID6L2Na2dMapEpHC6QemeApxNm4Nu6Fe+Bh8MmCz" "R53xd1G/ZjKo7MSA4j+xGBiINFijAcHss6CTQj4/0YXCnsezE7CRTCaw7d4hKeETsJjwk" "1CN+EuiANPpFHkVrP4RcIfmDPBFNANowXKs0v5PjvcFLJ2wn1xT8gfcscZuDawxh1hJj6" "4N8zNCWq/Zyge4/ZtLX+cT8L6+3zkeiVLJSc5i5SxN+M3ZvVjFL/v1ogL+9AfLbHl2FGs" "FTuHXcZOYg2AiZ3BGrE27JQEj1XCE2kljM4WJeWWCePwR21s62z7bT//MDdbPr9kvUT5v" "Dn5ko/BLyd3rpCflp7P9IG7MY8ZIuDYTGDa29q5ACDZ22VbxxuGdM9GGFe+6fLOAuBWCp" "Vp33RsIwBOPAWA/u6bzug1LPc1AJzq4IiFBTKdZDsGBPiPoQy/Ci2gB4yAOczHHjgDD8A" "CAWAyiAAxIBHMhCueDrIh59lgPlgCSkAZWAM2gC1gB9gNasABcAQ0gJPgHLgIroIOcBPc" "h3XRB16AQfAODCMIQkJoCB3RQvQRE8QKsUdcES8kAAlDopBEJBlJQwSIGJmPLEXKkHJkC" "7ILqUV+RU4g55DLSCdyF+lB+pHXyCcUQ6moOqqLmqITUVfUBw1FY9AZaBqahxaixegqdB" "Nahe5H69Fz6FX0JtqNvkCHMIApYgzMALPGXDE/LAJLwlIxIbYQK8UqsCrsINYE3/N1rBs" "bwD7iRJyOM3FrWJvBeCzOwfPwhfhKfAteg9fjLfh1vAcfxL8SaAQdghXBnRBCSCCkEWYT" "SggVhL2E44QL8LvpI7wjEokMohnRBX6XicQM4jziSuI24iHiWWInsZc4RCKRtEhWJE9SB" "IlNyieVkDaT9pPOkLpIfaQPZEWyPtmeHEhOIgvIReQK8j7yaXIX+Rl5WEFFwUTBXSFCga" "swV2G1wh6FJoVrCn0KwxRVihnFkxJDyaAsoWyiHKRcoDygvFFUVDRUdFOcqshXXKy4SfG" "w4iXFHsWPVDWqJdWPOp0qpq6iVlPPUu9S39BoNFMai5ZEy6etotXSztMe0T4o0ZVslEKU" "uEqLlCqV6pW6lF4qKyibKPsoz1QuVK5QPqp8TXlARUHFVMVPha2yUKVS5YTKbZUhVbqqn" "WqEarbqStV9qpdVn6uR1EzVAtS4asVqu9XOq/XSMboR3Y/OoS+l76FfoPepE9XN1EPUM9" "TL1A+ot6sPaqhpOGrEaczRqNQ4pdHNwBimjBBGFmM14wjjFuPTON1xPuN441aMOziua9x" "7zfGaLE2eZqnmIc2bmp+0mFoBWplaa7UatB5q49qW2lO1Z2tv176gPTBefbzHeM740vFH" "xt/TQXUsdaJ05uns1mnTGdLV0w3SzdXdrHted0CPocfSy9Bbr3dar1+fru+lz9dfr39G/" "0+mBtOHmcXcxGxhDhroGAQbiA12GbQbDBuaGcYaFhkeMnxoRDFyNUo1Wm/UbDRorG88xX" "i+cZ3xPRMFE1eTdJONJq0m703NTONNl5k2mD430zQLMSs0qzN7YE4z9zbPM68yv2FBtHC" "1yLTYZtFhiVo6WaZbVlpes0KtnK34VtusOicQJrhNEEyomnDbmmrtY11gXWfdY8OwCbMp" "smmweTnReGLSxLUTWyd+tXWyzbLdY3vfTs1usl2RXZPda3tLe459pf0NB5pDoMMih0aHV" "45WjjzH7Y53nOhOU5yWOTU7fXF2cRY6H3TudzF2SXbZ6nLbVd010nWl6yU3gpuv2yK3k2" "4f3Z3d892PuP/tYe2R6bHP4/kks0m8SXsm9XoaerI9d3l2ezG9kr12enV7G3izvau8H7O" "MWFzWXtYzHwufDJ/9Pi99bX2Fvsd93/u5+y3wO+uP+Qf5l/q3B6gFxAZsCXgUaBiYFlgX" "OBjkFDQv6GwwITg0eG3w7RDdEE5IbcjgZJfJCya3hFJDo0O3hD4OswwThjVNQadMnrJuy" "oNwk3BBeEMEiAiJWBfxMNIsMi/yt6nEqZFTK6c+jbKLmh/VGk2PnhW9L/pdjG/M6pj7se" "ax4tjmOOW46XG1ce/j/ePL47sTJiYsSLiaqJ3IT2xMIiXFJe1NGpoWMG3DtL7pTtNLpt+" "aYTZjzozLM7VnZs08NUt5FnvW0WRCcnzyvuTP7Ah2FXsoJSRla8ogx4+zkfOCy+Ku5/bz" "PHnlvGepnqnlqc/TPNPWpfWne6dXpA/w/fhb+K8ygjN2ZLzPjMiszhzJis86lE3OTs4+I" "VATZApacvRy5uR05lrlluR257nnbcgbFIYK94oQ0QxRY746POa0ic3FP4l7CrwKKgs+zI" "6bfXSO6hzBnLa5lnNXzH1WGFj4yzx8Hmde83yD+Uvm9yzwWbBrIbIwZWHzIqNFxYv6Fgc" "trllCWZK55Pci26LyordL45c2FesWLy7u/Snop7oSpRJhye1lHst2LMeX85e3r3BYsXnF" "11Ju6ZUy27KKss8rOSuv/Gz386afR1alrmpf7bx6+xriGsGaW2u919aUq5YXlveum7Kuf" "j1zfen6txtmbbhc4VixYyNlo3hj96awTY2bjTev2fx5S/qWm5W+lYe26mxdsfX9Nu62ru" "2s7Qd36O4o2/FpJ3/nnV1Bu+qrTKsqdhN3F+x+uiduT+svrr/U7tXeW7b3S7Wgursmqqa" "l1qW2dp/OvtV1aJ24rn//9P0dB/wPNB60PrjrEONQ2WFwWHz4z1+Tf711JPRI81HXoweP" "mRzbepx+vLQeqZ9bP9iQ3tDdmNjYeWLyieYmj6bjv9n8Vn3S4GTlKY1Tq09TThefHjlTe" "GbobO7ZgXNp53qbZzXfP59w/kbL1Jb2C6EXLl0MvHi+1af1zCXPSycvu18+ccX1SsNV56" "v1bU5tx393+v14u3N7/TWXa40dbh1NnZM6T3d5d5277n/94o2QG1dvht/svBV7687t6be" "773DvPL+bdffVvYJ7w/cXPyA8KH2o8rDikc6jqj8s/jjU7dx9qse/p+1x9OP7vZzeF09E" "Tz73FT+lPa14pv+s9rn985P9gf0df077s+9F7ovhgZK/VP/a+tL85bG/WX+3DSYM9r0Sv" "hp5vfKN1pvqt45vm4cihx69y343/L70g9aHmo+uH1s/xX96Njz7M+nzpi8WX5q+hn59MJ" "I9MpLLFrKlRwEMNjQ1FYDX1QDQEuHZoQMAipLs7iUVRHZflCLwn7DsfiYVZwCqWQDELgY" "gDJ5RtsNmAjEV9pKjdwwLoA4OY00uolQHe1ksKrzBED6MjLzRBYDUBMAX4cjI8LaRkS97" "INm7AJzNk935JEKE5/udEyWoo++PQfCD/AMf7G3o0obnYAAAAAlwSFlzAAAWJQAAFiUBS" "VIk8AAAAgVpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD" "0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjp" "SREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50" "YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgI" "CAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iCiAgIC" "AgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4" "KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjEwMjI8L2V4aWY6UGl4ZWxZRGlt" "ZW5zaW9uPgogICAgICAgICA8ZXhpZjpQaXhlbFhEaW1lbnNpb24+OTE2PC9leGlmOlBpe" "GVsWERpbWVuc2lvbj4KICAgICAgICAgPHRpZmY6T3JpZW50YXRpb24+MTwvdGlmZjpPcm" "llbnRhdGlvbj4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC9" "4OnhtcG1ldGE+Cj6cgV0AAA+rSURBVGgFzVprcFXHfd/dc859IfGSBAaHGlkgg4TAiZKm" "7bSxnDpN/MhrYhHbsfOa1E0m0/RbP3Xq637px2YmM52J+6GeTp2HNAmNXRtjZxKRh5PJo" "EkISIDAwjgEMEQCrMe9uuec3f5+/z1HlpBkE7dOvXDv2bOP//u1e6XV26g5p/Sg6jeXh9" "eZzqkx19c3lGqt3NuIxDcmxbmqWW5Vtbr8+HJr/9/HBgb6g5yIp0fue++zo/d/4cCxT96" "Tj71lzNAEMuA6R/ZmnzkTT4/037B/9BPPPz92rxs6s9f9+Own3f6RTzx74MJDqwj7jZhZ" "Vp0rEUVgA64/oN2ib8GQWskkVoKxcJwC2bt3MD3kHo6MtvvWtkR3xHGa1qbjeOpKI17bW" "vhgOln7a+7ZdM9T81pbCCPvXzcjlByJ36sH04GRauH5F/vXkCGtq7a6gn3nSFZ6PjbcG3" "Lud6OTd5Ur5k8u/26uobTWkE+Eh6rXElUx7k+55nzvcMrnSu26GCETlNyzI/3rnxnt/2q" "TOjIS19Kx/aP3/veB4/3tVTDzZjTT2dssEcmq9F3awEqdMlCx9BxkZK1TrUW3g7CrWlmY" "wYr0rjiRc05p50xYlQytXR/8XVRQ20ygNqxrje62afotrqVmaCr5vut46j41lElZ714YZ" "AkHHGobW7UqtFtV7eRmgffIyIrw35CRLuU3W23/cX1bqefyxFwtbtjUpi6ZuFS3kOQf/3" "DsgVuJiDngOhiQJRAQrceNjPQXYEw7k8SCdEyRA+oFojFQSXPFrJ54Nb2Fm4aGLr05RsQ" "Z4ROAoZ2zfz47HVNUEd7p8AFwurVNgWovq24iansdRJxf2HIBjWt7E6zophjSBzwRBKkF" "D6ZgVApGVGLj3Qv3Ltd/AwnCMtGeO37fJnRuSmJarvLWDOZSq2xTUaly2Hgn1/X1tYGW6" "2s505EynYWSKUHDFjgEXw6hHCjvMNoJI5curQxfoka+8drn0NAQGbU2adwSRLoV6gcfYr" "8ZRq0iZ8FZKozoTHvYc90MwbH2FIqBatTTFJsQrWhWaEBUCZ2mppSzu5xDgNQ6FZvDGi5" "Z2F5XI2PNU15CRu8ulYXnlAP8EBLgGQO1wMZ3nD376RYCRoTxe/jyOu1g3xAoBAzt9oDI" "RZRZgGTSKAXazM0Rvr55cvLBGz245eG/LiPrxm8WZIiCu4W6BXLAuwtgZiZNKUJElbntR" "JRp0eNc4ZtSBTn20KGHI/R3JpS6l4//hoQKMKuCgUYS54JQrW80YoEPDMvSvOwgoRIZw6" "7H4HaJ4vmSNaKOIDZYRbIKDm9MQyJXc67FfOEyz8FBH92uFKa3ANNWREFqJPc9JAzANYg" "qoA7dtAJnCXTaQ1DDw5mVXAN3RUZyZN8/cv9G7OkQW9WMXrAoqIPISkAWos9kBkC9hN3b" "67V4DZ5Fr239PozGodpRKJhmCEkcHSDnG2jPONPi8HAO8cPxzErmF2adFRlR/X4FDKfDB" "LolVz+R0cz4JDKYl0oSvBknEqPDC7N++7LfY5lUtUp2RUUhgYx4uBQUGl0Sjk/T0An9UL" "kujtNKloO/IiN5eHSh7S5V6HoqAVwKH3B8KwWCVM/BNMDZ9osX+2/gzOhof8SyhghZZF5" "biz2c1U0IgT4/ZACFQMoEiMrIUtC6ZPhGA8HAuY4LFx7aQPi5tbCftxXDbx6zAfxW460X" "shEhSYgRZKHwpKEtBxNZ34gTSu1Cd/dgI0cAtPCzQRBSNVJgol7y5YzTB0b3dqfUpuiAU" "lKOwkcihI8IBAn3NOsoMi2plYBysa1taYZfnhFIglUuQUESu4hM6gmInWjxcRGQExmkZj" "AZr1oVRsmVmBn+B8+d2HtXmqT3gcAtyKAnQFsVxF/IC8tqVannT/ztJuTTrUyyaJrSF23" "jlY4eArbMeDaTVauC8NWplPB/2tfHLYub53vxmKqqqsjoqRMPtyLOb/NRxclaSVgZsiyq" "yG7mAlRH73nq6N5/KJXV05Xm4KFCUfeta4v+BpH2uWdOfmo1NTE8fE7sVLmr28NIr/U1F" "nMdiwYQj2fue6JvzwizIaW2x5O6NMMvq5HbsoxeSGodqbKbYy81b2BEhk+ODOkYPLhw6m" "pDTTSC+4tFF85MJxhUDSw09Xpqm5qinpmZ+P3Y9l/jjcvEGVub9lQQtqcbCReH0J7QTXn" "QP/g+rxIwIQwrX6rkAYXMA5a0ZTUy1twpGnFB0lVeJbx6M8s2cXeJw1jFviyGxV1tuLBW" "Zxmr+ClggkGtMDsTI6jZCYypprZpzmGf7YG20cMXFgkMvGKDKwUBEgvnME4GcUZpSK6x2" "86f72+TicxqfB8BIu8sfPIqRt6t3RPQECAbIuKHEuOzDDtmH10ZoBvVIVsQwlwj2gMQFz" "LRODMZhZVxrv3FE/tjPsFrN8O2mAzfuBaVllPBK0UT/SjE4Ze88IMmjGDtDTDfDg5cW0E" "sYYTE3X77UMLFTrseVKXsSkNPHB3lgy6Fws88gzSxOWQDjpJ0rsViGzIEKfPS+ztvPk8g" "1aqyPz31xQ2Ygu8hSyHfcZxrWTyiBH0xNG5fAG8H4fPIAZcOj6iQSMi+toJYwghQiZR/c" "vzvm+Fc27MDDyGSSmsA36hwIjLBBPtCAr6gDU1XwkLZT+LQg0aAwphjdPQBHKI4/Go82w" "EaN0rEYtTjUgxkxI9XIv0TmhLkIXMULiUT0FC1FUaurSCWMDKYnQhr7uJWbIKjSx0kxIH" "ItAKfiV34I8B8gX0oQrJyHXUxlcdcQML4zU00HZB4RMauKolYxsH3fJIV38M6uIGAwWlN" "j61p3TCSxOoKcodAEt2jx3oPYHdxMHd49tmWMJJndOtiImN2ZukujBBMhEoxToOfQf0/j" "2hj3oxUTUjyCylB/MPDhQ2U4aEpHCay3NFTl96KMz8XCfXCOdIk1wZBCO19bQ42+yKSLL" "dJcMBTZ1rqPHPmgXWcyK2HvSWMHOQoGqNKII4qgMgI8Rn6TKyjX4c6PSJZmWOguA7FySK" "hCiuRPExg9NycqylTHsOImv5lk/geproIB2rwRSh0hr1BvWYbFV06ybUAcDRi1vV4ZUAK" "V+U2RVG9nQMqsx72ljDyCC6OOeFc2sPrmLyhh6SkgpmpVCX16BSEeXJqKmF4ChLQU08QT" "4gXH6yj8hwJwb+XN8UbzxIOC77vHf887oDstsz3ZD2xRZA+aujfNBebXuZaqPMwPmyEyh" "6ZTpubeWVgpYBU6rVSZREjlBCJOHSuWgEyHHiEJ1kDaBYhEedede7iml1n67Z0FuHzYhE" "ENFCP0jeJkSjxn6woOjqC0rHu7mrj64d6SYEq2fgmDN6Y+V6O39JMgf/YO9u/eoXrEEgO" "17y9wn4lehG8ZTgHjXL2wXUHGZSWA5KXQbVX3iemzv0RKNnC0gQ0CVFCGBgBp2Nf2f6Vu" "Q0bBqcB8iTtGPicODpQEXL2kOIJFP2awDtbmsXREzt7S7mMjIdCC+tInOccPWOc+BKH0r" "R4Ej4xBcKB3+sGPXF4ECkOj2I095/FpsXfJQjEuPotvNngLQmwiJY4zvAHA/IRSBaqoxy" "rscBf0MgMjN740iYQRpTaKisCHe6m72EH/YWi5mFDGhL4L3xH6c2bd/wWyF7KHF4kD3aQ" "GCX3dJ6U2k3xDtrT7EH4785en9FTF++JCkAG+4KIRa9Api0tzYa/yveAw18mUAUtINMbp" "UuyXGCAtGbrQaJHub6WvCLEwKnO0ozQWORABE6vay1FM6+mP7yze/BJTjhc9DHvoDsqeU" "jGOKMQPChbtWX16tl2DjyS3T4KNxxgO/jokKgKPtWTOZqf8N9BbQYUR2YkH4xUcvTqdKq" "Q0RnffEMHFMN5Jb6eqc+Vz3Dizm3vldLkQzu/9W+4rfwPzBtkcjBjLC6vnwhLLR/nOn8I" "y5zY6iM0JzYPFkJlLquEOAn5E+Pw8PhijVBFLB8GBgYCZ92ORVEF8RJ1EMFdKOu14wIZX" "wjmp+qJe0VB/MDkJe6RSmAAnLEPv/uxWSCH8qrzd8N3dX33MzbR727U3Ud0EnTf1f3dBz" "/Q8dhVf1nNUn/+GuqIaADc5jjxlGiIiJpleH8RTvVK6+ryd7wtuw68o+GyAw9NGeQhytt" "CIQhQhpx83/avXcq2aL3xyVeOnP74CdRTG01sYxCMe0ex+QQRjj8NvMC1Q0N9UM8QArS4" "rcgYmXkYU/x4LTwKPN6csguMYTi/PlavJ7PwwwovKLBUGGJaQKj3x2RchFNQ84zkx8dY1" "bbDP5rgVGRBNjJosA4CGWJWPI9zPYtLBLbHipXgfQiVRayDJ6pg7fpi+epkfBqR5+sk9G" "BfH4gYYnfel3iWZxXRh7mcAVkgXz4azcw0/6ZcunIGDr+T5xqQAd/BYQZFHZ6dzn22pPX" "jdWpyocoyOHF3AbcK8MH8VpEmI0UbJCwRKGOCrq97O/Y9MXVl7p9gRq4QSdGfgolnorB4" "xz27v3FZfiDKJJ0hkAeP0hTEUiY8s5Rye/vjddjECUYuvOemK9eoCN9bLl2a2uJhjniJ8" "yW/bOBdbFY9Yy8jFv+rsD6bKhMUXwu91E8G+q927nsENVJH2tB/meryzju7vnP3HZ3fGM" "9/IPLIfr9vb45Eog5DgPOb8Y7LTWdR0hdtHO/kBH9uENPCJNb6ywZs7ZJ7KjDhd+McCkd" "PYzdZUuYUx3KmX7P5KvZXT2OKHyLXvLLJbyo59mabMeYwcwcaYzZFR76SIqJePbC8jJCQ" "nflIFURX3ffH798Y1+baJaOz9PESByP4oSJRp2/vevwCNqr+/gHYvOeTzHBv7jdkEkLBf" "C4Y7vj9W/4TBaR/fLaGDIaaDpjo8IKYDoMkJw5PPxNGsmOjbcym22BsG7JbxfxkgYxOV7" "JHSQ4JhkhERHzP2/+F9HNY/ukdfvPmfcfOn/voC62thb+YmGjMQXAR/CNB5MIvGvodXEs" "/E2fPLxsQDxYdeEQn4B/Mgw9fB12+2Scgj+yt+/Zm6/+QIAjd5ycnG6daWopl3J+Fa9ZE" "ZSlIjf53UuAQAUUj63ovU2VkrUdqcUQIAJIGHrTcazktoTf/JdbPvrXf9FtaAArUU6dPf" "+w9brLxJWD8M1COFO3+84Ybn8x+iB1MhZHRRwcpc5rhVj7RkeM3ng0cZ8u1aXu6bDZKcj" "v46Gs5Qda+xV9yaU1zbh9kef/PC9H5IOU9WUzrttv65Inj9zc5XGkKy0VcLq1eUyjPTqf" "1wJjP3d79r9NyIY0/GlgI7A/R13IDXzXO9fG2GTmNff8XGDn+zIDklX23/+i996EQ+DL4" "KYGpQ84E/3L3zm+P/W9yQo7sD/Ykt8shoyaWG39bj2VEC0PsUxNva4Iz4v4H8vqTm++oi" "2AAAAAASUVORK5C") MM_icon_png = ("iVBORw0KGgoAAAANSUhEUgAAASwAAAFLCAYAAABsjLGXAAAMFmlDQ1BJQ0MgUHJvZmlsZ" "QAASImVVwdYU8kWnltSCAktEAEpoTdBehUIHQQB6WAjJAFCCZAQVOzIooJrQcWCFV0Bsa" "0FkLUiioVFwF4XRFRW1sWCDZU3KaDP1753vm/u/Dlzzpn/zD13MgOAsi07NzcLVQEgW5A" "vjAryZSYkJjFJPUABUAEFGACczRHl+kRGhgEoo/0/y7tbAJH0160lsf51/L+KKpcn4gCA" "REKcwhVxsiE+BgCuyckV5gNAaIN6o9n5uRI8CLG6EBIEgIhLcJoMa0pwigxPkNrERPlBz" "AKATGWzhWkAKEl4Mws4aTCOkoSjrYDLF0C8FWIvTjqbC/EDiCdkZ+dArEyG2Dzluzhp/x" "QzZSwmm502hmW5SIXszxflZrHn/p/L8b8lO0s8OochbNR0YXCUJGe4bjWZOaESTIX4pCA" "lPAJiNYgv8blSewm+ly4OjpXbD3BEfnDNAAMAFHDZ/qEQ60DMEGfG+sixPVso9YX2aDg/" "PyRGjlOEOVHy+GiBICs8TB5neTovZBRv54kCokdtUvmBIRDDSkOPFabHxMt4oi0F/Lhwi" "JUg7hBlRofKfR8VpvuFj9oIxVESzsYQv00VBkbJbDDNbNFoXpgNhy2dC9YCxspPjwmW+W" "IJPFFC2CgHLs8/QMYB4/IEsXJuGKwu3yi5b0luVqTcHtvOywqKkq0zdlhUED3q25UPC0y" "2DtjjDPbkSPlc73LzI2Nk3HAUhAE/4A+YQAxbCsgBGYDfPtAwAH/JRgIBGwhBGuABa7lm" "1CNeOiKAz2hQCP6CiAdEY36+0lEeKID6L2Na2dMapEpHC6QemeApxNm4Nu6Fe+Bh8MmCz" "R53xd1G/ZjKo7MSA4j+xGBiINFijAcHss6CTQj4/0YXCnsezE7CRTCaw7d4hKeETsJjwk" "1CN+EuiANPpFHkVrP4RcIfmDPBFNANowXKs0v5PjvcFLJ2wn1xT8gfcscZuDawxh1hJj6" "4N8zNCWq/Zyge4/ZtLX+cT8L6+3zkeiVLJSc5i5SxN+M3ZvVjFL/v1ogL+9AfLbHl2FGs" "FTuHXcZOYg2AiZ3BGrE27JQEj1XCE2kljM4WJeWWCePwR21s62z7bT//MDdbPr9kvUT5v" "Dn5ko/BLyd3rpCflp7P9IG7MY8ZIuDYTGDa29q5ACDZ22VbxxuGdM9GGFe+6fLOAuBWCp" "Vp33RsIwBOPAWA/u6bzug1LPc1AJzq4IiFBTKdZDsGBPiPoQy/Ci2gB4yAOczHHjgDD8A" "CAWAyiAAxIBHMhCueDrIh59lgPlgCSkAZWAM2gC1gB9gNasABcAQ0gJPgHLgIroIOcBPc" "h3XRB16AQfAODCMIQkJoCB3RQvQRE8QKsUdcES8kAAlDopBEJBlJQwSIGJmPLEXKkHJkC" "7ILqUV+RU4g55DLSCdyF+lB+pHXyCcUQ6moOqqLmqITUVfUBw1FY9AZaBqahxaixegqdB" "Nahe5H69Fz6FX0JtqNvkCHMIApYgzMALPGXDE/LAJLwlIxIbYQK8UqsCrsINYE3/N1rBs" "bwD7iRJyOM3FrWJvBeCzOwfPwhfhKfAteg9fjLfh1vAcfxL8SaAQdghXBnRBCSCCkEWYT" "SggVhL2E44QL8LvpI7wjEokMohnRBX6XicQM4jziSuI24iHiWWInsZc4RCKRtEhWJE9SB" "IlNyieVkDaT9pPOkLpIfaQPZEWyPtmeHEhOIgvIReQK8j7yaXIX+Rl5WEFFwUTBXSFCga" "swV2G1wh6FJoVrCn0KwxRVihnFkxJDyaAsoWyiHKRcoDygvFFUVDRUdFOcqshXXKy4SfG" "w4iXFHsWPVDWqJdWPOp0qpq6iVlPPUu9S39BoNFMai5ZEy6etotXSztMe0T4o0ZVslEKU" "uEqLlCqV6pW6lF4qKyibKPsoz1QuVK5QPqp8TXlARUHFVMVPha2yUKVS5YTKbZUhVbqqn" "WqEarbqStV9qpdVn6uR1EzVAtS4asVqu9XOq/XSMboR3Y/OoS+l76FfoPepE9XN1EPUM9" "TL1A+ot6sPaqhpOGrEaczRqNQ4pdHNwBimjBBGFmM14wjjFuPTON1xPuN441aMOziua9x" "7zfGaLE2eZqnmIc2bmp+0mFoBWplaa7UatB5q49qW2lO1Z2tv176gPTBefbzHeM740vFH" "xt/TQXUsdaJ05uns1mnTGdLV0w3SzdXdrHted0CPocfSy9Bbr3dar1+fru+lz9dfr39G/" "0+mBtOHmcXcxGxhDhroGAQbiA12GbQbDBuaGcYaFhkeMnxoRDFyNUo1Wm/UbDRorG88xX" "i+cZ3xPRMFE1eTdJONJq0m703NTONNl5k2mD430zQLMSs0qzN7YE4z9zbPM68yv2FBtHC" "1yLTYZtFhiVo6WaZbVlpes0KtnK34VtusOicQJrhNEEyomnDbmmrtY11gXWfdY8OwCbMp" "smmweTnReGLSxLUTWyd+tXWyzbLdY3vfTs1usl2RXZPda3tLe459pf0NB5pDoMMih0aHV" "45WjjzH7Y53nOhOU5yWOTU7fXF2cRY6H3TudzF2SXbZ6nLbVd010nWl6yU3gpuv2yK3k2" "4f3Z3d892PuP/tYe2R6bHP4/kks0m8SXsm9XoaerI9d3l2ezG9kr12enV7G3izvau8H7O" "MWFzWXtYzHwufDJ/9Pi99bX2Fvsd93/u5+y3wO+uP+Qf5l/q3B6gFxAZsCXgUaBiYFlgX" "OBjkFDQv6GwwITg0eG3w7RDdEE5IbcjgZJfJCya3hFJDo0O3hD4OswwThjVNQadMnrJuy" "oNwk3BBeEMEiAiJWBfxMNIsMi/yt6nEqZFTK6c+jbKLmh/VGk2PnhW9L/pdjG/M6pj7se" "ax4tjmOOW46XG1ce/j/ePL47sTJiYsSLiaqJ3IT2xMIiXFJe1NGpoWMG3DtL7pTtNLpt+" "aYTZjzozLM7VnZs08NUt5FnvW0WRCcnzyvuTP7Ah2FXsoJSRla8ogx4+zkfOCy+Ku5/bz" "PHnlvGepnqnlqc/TPNPWpfWne6dXpA/w/fhb+K8ygjN2ZLzPjMiszhzJis86lE3OTs4+I" "VATZApacvRy5uR05lrlluR257nnbcgbFIYK94oQ0QxRY746POa0ic3FP4l7CrwKKgs+zI" "6bfXSO6hzBnLa5lnNXzH1WGFj4yzx8Hmde83yD+Uvm9yzwWbBrIbIwZWHzIqNFxYv6Fgc" "trllCWZK55Pci26LyordL45c2FesWLy7u/Snop7oSpRJhye1lHst2LMeX85e3r3BYsXnF" "11Ju6ZUy27KKss8rOSuv/Gz386afR1alrmpf7bx6+xriGsGaW2u919aUq5YXlveum7Kuf" "j1zfen6txtmbbhc4VixYyNlo3hj96awTY2bjTev2fx5S/qWm5W+lYe26mxdsfX9Nu62ru" "2s7Qd36O4o2/FpJ3/nnV1Bu+qrTKsqdhN3F+x+uiduT+svrr/U7tXeW7b3S7Wgursmqqa" "l1qW2dp/OvtV1aJ24rn//9P0dB/wPNB60PrjrEONQ2WFwWHz4z1+Tf711JPRI81HXoweP" "mRzbepx+vLQeqZ9bP9iQ3tDdmNjYeWLyieYmj6bjv9n8Vn3S4GTlKY1Tq09TThefHjlTe" "GbobO7ZgXNp53qbZzXfP59w/kbL1Jb2C6EXLl0MvHi+1af1zCXPSycvu18+ccX1SsNV56" "v1bU5tx393+v14u3N7/TWXa40dbh1NnZM6T3d5d5277n/94o2QG1dvht/svBV7687t6be" "773DvPL+bdffVvYJ7w/cXPyA8KH2o8rDikc6jqj8s/jjU7dx9qse/p+1x9OP7vZzeF09E" "Tz73FT+lPa14pv+s9rn985P9gf0df077s+9F7ovhgZK/VP/a+tL85bG/WX+3DSYM9r0Sv" "hp5vfKN1pvqt45vm4cihx69y343/L70g9aHmo+uH1s/xX96Njz7M+nzpi8WX5q+hn59MJ" "I9MpLLFrKlRwEMNjQ1FYDX1QDQEuHZoQMAipLs7iUVRHZflCLwn7DsfiYVZwCqWQDELgY" "gDJ5RtsNmAjEV9pKjdwwLoA4OY00uolQHe1ksKrzBED6MjLzRBYDUBMAX4cjI8LaRkS97" "INm7AJzNk935JEKE5/udEyWoo++PQfCD/AMf7G3o0obnYAAAAAlwSFlzAAAWJQAAFiUBS" "VIk8AAAAgRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD" "0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjp" "SREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50" "YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgI" "CAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iCiAgIC" "AgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4" "KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjk5NjwvZXhpZjpQaXhlbFlEaW1l" "bnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj45MDI8L2V4aWY6UGl4Z" "WxYRGltZW5zaW9uPgogICAgICAgICA8dGlmZjpPcmllbnRhdGlvbj4xPC90aWZmOk9yaW" "VudGF0aW9uPgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g" "6eG1wbWV0YT4KTlGaRAAAQABJREFUeAHsvQmYXNlVJvi2WHPRmirtS2qXajNZA8ZuqJDB" "XxsGA20mhdtuFg9QYMyOh6U/BoWmjcHdA3xA21DFzLDaMMpv+gNc2GCXrZRtDIbKqrJdU" "qm0lfaUlJJyz4h46/z/vfFSKeUWEflexI1U3CplbO/dd+6555577lk1rdVaGGhhoIWBFg" "ZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIW" "BFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgYeCgzoD8UoW4NsYUARDARBgDV3VO/r" "O6l3dR2cXn9DQyeD3t6DgablA13X8NpqLQy0MNDCQJ0xEASafizoNY8fz1n5QDMqeTyv5" "z/eW8n1D9M1LYQ8TLPdGmvdMJAP8saGgU+aP/7UgDPzoZCwrK9c+4P1CTfTYSTc1UEwld" "b1dKFgF+52ZFcMP/bIj92ccb1+/HjePHQo78747qF+22JYD/X0twYfNQbIqJ7u7zcOHeq" "fZjJfPpff5enFb/W94rd6vn3QDSZ3ekExbeiJFI5/hh8E+N8pWXp2zDCSr+pG6gupoOPv" "3rrnw68QvnxeM44cyWu6nvejhrfZ+msxrGabsRa8SmKAx7d+LWce0iWjOnvnLzuv3vnKu" "xxv/D1+UHxrpk3P6oanua6tObamBWA9vu8LZRWYlmbopmaYgZZMmZppWtrkuOaZWuazyc" "Ta//b0rv/z8xz08SBnhf0riYQ6ANViWHVAcusRyxsD1Dcd1vs8jvLkrWPtg8P9P+75Yz+" "ZyjrdWuBpxaKn+R7eaLqvQ06CRt0Q2imhgJ+JG/wEaQvsTNMNzcq2W5pdMPEp/acr/L0f" "/Kb9//kOdWEzpbeZdz8M71sM62GY5dYYY8PAsy/2JEI91Qun3/+fvGD0aLbD7y4Vbc0u+" "WRS4E9gP1pQkcJ9GlAdzC3QPPxvda5M6ZOj+uWEseo9b9v3R/+UP65Z+UPa9JFz+p6H4E" "2LYT0Ek9waYjwYOA7GcQiM4+zg73ddHHv5jxKpqXe5rqPZRd8Fj8LaCswInkwXByeVMZJ" "2UfdSxqp3vW3fH//dwypptRhWBBTV6uLhwgD1VX041h3WNe9L537y0KQz/PFMu7thYtzx" "tIASlWZFjpFAd6yklvCh2Uqaa7/rbXv+8O8fRqZVnZga+Sy0OmxhoLkwQGZ19Kimk1mdO" "PPMeybsW5+zkqUNE2OurQc8+sXArIgiPUh4juYYpqeV3Nt9J87/8mPUZZFpNRcGlwZti2" "EtDX+tux8yDBzt10y4GfjHz/zAT3jm8Mc1w9PtokZfqySVVXGiAxr5hOsEdqZNyxRKV/7" "sxeDFBJlWPp9/aNZxrAiOc/JafbcwUG8MhDqrL5372e8vaYN/bTsO3ROo/K63lON0rEgm" "7ImV//u37Xv2Q8cCDVZKTVgp642Tej/voeHM9UZs63nLCwO0BlLB/qULv/otE/aNv3KgX" "Ncaw6wgx+nG1KStuf74L754/fe2klkdO9YbhYJf+UlrMSzlp6gFYKMxQGZA14WvXfr4qq" "nStT9Pph098HQHR8B6S1YSFUFgeq7mZDv9lePjZ9/HL7t6hx6K01JjEN5oCmw9v4WBSjE" "AJfthrU+ExAwVjv/3dHth+8RYAF91LVlpFzFdZ9i2rcFBtRfxiR/WdTBQOKKi0Uq5bFtL" "wlq2U9saWBQYeHbgGQuq9OALZ3/+e4zkxHvGxxwyr0QUfS+xDwP+XhrCfg7864Vfe4J99" "fcfWvbHwpaEtUSqad2+fDFQllgouVifOf1D/0fCcsC7dDKshq8bxh9C+POybZo5XrzxjY" "DpxeU7E/dG1pKw7uGi9a6FgfswEEos/Wd/5j+lsqXHi1OeBz6hhBQjcmUhdlp4fvnuXgJ" "+Itcvjq73DWKZfWj4TrHM8NkazjLBQFm6cvFqfvb0D33AQrQNvTfj9rWqFn0IqoaxsiCO" "hHkdJ0SIXcs5Y2lLwqqWQlrXPxQY6O8/KiSpL1745ZxhlZ4qTrnMbayEdDU9AdS0u65m6" "enNQXBshfwebGsZtxbDWsaT2xpa7RgYyuWFtc12br8rmYbYopmuatIVbAFGqRRoCb24bX" "j4lS1ytCdbDKv2aW/d2cJA82GAx6rQc9z1x3o8HLvwn3KMgAzUx4kwm9EtvNklMX2vsEX" "zYX5xiFsS1uI4al3xkGGgr69XrIvPvd67KQi8XY4tMoMqt1bIQeF65XW24dUfOsBpGhj4" "pHKMNUryaSnda8SmsNJg3w1LNrEbWabpCLZi0EyrVFONmG38bV1d0mvccYNupLVa47m+n" "NHGgzYXBHoS4Yy6Ye3jjz09A8s6prDFsOYigTm+Y0T8009rxpmOT+qDIApYYmBCFpuZ0H" "XIW5AlScvzrX480KychrBU7bC/nK02HOxybZCrnmhrY3515LlSTeEOgEh4TBPIJMqePy4" "YFumybOGcQZfLZ4ZaDGuRuQyQr7uvr087fDhPor3Pz4Um76/cPdo2OvzVtGms0dfoB0tP" "bv+5cRhvvEM6o/gPi94fxkRri6BV6Z9P5HKY537CeAAMgJyBWnflWplhGboL0tT1Lbduf" "XT9unUfuNHXd5jH12UpaSk4DWrQBerBWYdO5H0ITIJJXbny5cxVv/8bbGf0m31/6smSO7" "QDkKZBzSuCwM4iw4dhGtYUnKDHTCM7lDRWfDGZ7Pj7N2/79QGOKJ/XjAMHevXDh2WxAjV" "G2YLiQQzM9GP61Ml3fTHbZv67qQn4DuiN925/EFYo3DUU2fEPrvQMCwdD3dz/VFfXfxlY" "zhtkS8J6gAoYmX/qVF8QFq986epvPzE2dek9r008+92BVtqX7QCP9x0tDU2VTxMNLUiiv" "gA99myUanI0wyiCeMa+fWRcP/LC6R/5x2RixW9/687f+Zym9WnLmZgeQGVTfjyq5THB+e" "BTZ9/Xpduj2x2GDrLGjWIHLEoaBCkF2CBO+ZmsaRSKE4/iK7FBNiXyKwC6xbBCJIHfPDv" "QYx1+qk9U6v3nSx9+61TxygeHxgfemWn3TNN2tBLKNY2P6DjqgVxIMfwrX8NeQEWCssHO" "PLzzE4kO5ztKxYnv+MxrP/DX7cbWD75l729cI9PK5fqpB1NsGdwbxsP67oAm/ZgMR9uCy" "dnkkmFVWGK+3jgjqaWNQDdBbFbCM7Spyd2EAbS1bOlKOVNtvSedz2O1Xs4wcx69dPkjGz" "//+o/+6UTx1S9ZmaHv9bVJc2K05JaKyICk4TJdZ/Q+GT3/Ud95/z+pnLXQHyL6jWBq3Hd" "su6RlOkvvHvPODrxw5offzrS2zAtO5Sj6aDWFMDA8sEqsCRz7H01nTcbiUBek5DyRZtNl" "kYOSPtI1P05UYiPkbqkkzIRvKe2hl7DEEU3Pu3lg8bOv/+gP3pp4+XezHd7qwhgyOo6bD" "vYvE6xF4IkEUmVDcqIgwRxF4yOOg8Lkj/jexGc+//qPfOBte//vj4G0DDAtENjyzmFUJc" "4aevmenjNimk3D2GMluOrpmxlJua5Ix0UgaSFMcasFh3I9FOzxi9tBT0nQE2tL44rlx7M" "eaglLKNYh7ZCSPnPqB/4wkRr7MzNhr0YFFIfSFJlNWUHFS5bSSDlJpxi4PmprGqmJj/af" "+emfyUOhD0Fr+VHVUjDV4HtzWr+wrgWa8wQqyWPZqykFkx1ZZFiIboSgbqBoK77xd9y69" "aEtRGHZUsi3y6o9tAzreAAr4KG8++L531rxmdPvfSG7wv6JqSnbc2yWFBcJ2qJnJDhKYh" "F4U5NFzTdv/96Jsx88nIfu/niQe+glXRVWFY/o1CuePHkM2USN7XQYRYueDpY4WAKE3VR" "LYPVa+Eem6iLS0TTddsNwtrL7rq7lGaLzUDKs8Bj4tUu/ueqO/crnM+3Ot42NQKsuswvF" "HZGP/g3X84ta0b3+xy8P/u72Q3q/ewz+Xkuk49btS8QAvO3EehhM/AOyH3g7yiE5yjEsD" "pMSFo+DlLKkVkF329oMvL/1GH/P5a4rCTdhW0p76BgW3Rao9L5y5VjmxtSrn850+N8wPu" "qUgESmva3XJFuOrdlglJ13Rk/9N07gYR15w5epopTja4bW1S9DcowgscO09DbPUzckhxJ" "WmltfmWJRSAfiIbQbvr5L4vo5IR42A96rgfGhYlgU+UPHzdMTn/rLTIfzTROjYB3YrKpB" "WkTXJibGbLhwjf8vXzj7q0+jz+B4f74lZUWE3KV0Y3uTsBByaejKWgi5tZJhkV9R2sJfn" "VklNG1yWVsKHyqG1d+vC4bwudefOZLuKLwLkhWU6w0rKAB1iekm055mezc/QJI7kYNnfa" "s1DAMnTvQL/MMXcx8lGDT5t2EQzf1gAsWFmzahvSpfQvWbg5BHfLP5zp1Pdcqv82X5q3z" "RMnh5aBiW0FuhEOYXzv1vb3X9kfzEGE+BOqa8bsfAWeSCZ5vFgouCmIV3/PMbv7kd5OU/" "LAUxZyGjwV9QD0SrLcHwA/sgpHFF2ZUEi7qr5P2rFwyL7Ku0NQi+vInj6O/vv/8Kftnkb" "dkNaK754FGQeqsgOG4VnGu/l4BUg2SN8GgXMTVz3VKX72CQgnVH89NZv6PkXD7Eh4apTe" "oCQOsh0xiQITkoPfN6fi1MhdscGzRiqOlyQl5KZkUrId9LMUoeCZNJLYHvhMf79OCW0Zu" "HgmE9N/CUcBv4/Nm/fn+23espTnowAsPHqsGNtAZXat+AnOdpzrcQnPBY0mDQHrrHhyE5" "U8HwRi9wN7uUVmSQqFK4IHOiGJgCzSAkZ/pIiK/AZ3U3kyHYQ/sl0P3yZRn9XfYMK0DYD" "UNuzp79y07XG//5UgmCFb1tlGmIo4CXsueVRMZIHksoESoD3kMCSJcm/ZZsvXQgnTENTA" "L5gprzAF4qFO73MywwKgbki9ODoKVcbvmF6Cx7hhWe4y+5X/qP6ay3Ax7BmFHE/6nSYI/" "m8cPQjS0n3/joegnWUTUXiio4iwGOM2Fq4cDenUgy2tkgnajZQB2hhfBBAH14Jvv+WJjM" "j2Lig5c09WdxVGrqESwAPCUVism8xA1Gf8gUvEq1uD2hx4IDYHG9b42sA6g3+soZAxYYW" "uuniDEw+LxMLYxAhMeZNgj/KblpkP1wt6XT6GxWpKOKjocIVWPr5OSzG9rafnyQKbxx+e" "xLI8Zfvbpb1gyr7Lnsff619/cUg6FvKhbEpqmOdCVmGelJfc/vbE8abVaGO+PXesXxhOm" "WW60eGChvbNJC6LvdmA5xFlRxlVPJznAcKt35/v4mNz9NK6yfmrpMSyEY1vIq+7Wsj4Td" "5VQhdnDnze0dhgGtBKUtpXZOpsSCP5aXTTsgwuui5PjAQF4pGO9fFMvvUxiS8y9X85uh0" "hYhOWBZyq0NEgW5apIhOWRYeP8AoYh0ONksEozoUwc5UwPlNcD3y6EpNylRIvVCz7DcNQ" "NXTB6Oh7P2pCifV2tf2CmhLXXhweoKL+Xnn1+e+bhrxU/c94UhOa7jgGG5q+gxrpJZZub" "4ScBprFpaCOduhp9MetBjTe3i7z09zylJ83PDvvi3y/ZISCag6zJ/OqSYx0QqY0X1EuCq" "yApIxfvozvB4Un5dVsS2ODk29oqSfeexTJupTU46SNYYMNOUeg1AhRZC5nSfzbfgj8Ua1" "b77RBl4dY0HNWB3GUtY8lj1wmv/YQ2Y13am38Dszp7fGpAW+S1grcjfgDyBXvfw8HNbZP" "8yc0Dkz2p1OAsDQ0P9IA6hE9qlG9g+AiaDVY9WCCQlP/pgLUDIiClkiE5pGwZh4nq8qGl" "AmDURFXyxbCWssjuDb/vGViSNlLm559qQKkBSnJeQCJEURMQIGUZhhavd2YavLveXMwfE" "+exW34JJYbuQR3A/KD4RQGzBvgZ+JXiYUigStAJORQlrfuh0vYRK1dTFjYz8Fje/i8gFQ" "sFkWUhay1jCwhShwUx9EI6AOvxTSIMLbEzy+nr/FUQIRorqJx5KSkGFckvoseoNx8P6vK" "NH84ImguDFLISRLS5ipTAnSq4LcFEtAWjDkJy554whOhiAYXdoTmkrr1lOm5+SEzP3RFT" "37dBQl9iEzMDaa4koUTVThQgixCwkDRoE4IXvu8LpD5VPuE22WswYOHBAmv2/cuFvNnlB" "qRsZZ/HExsaYzjVkclVCJpL2gV74fp7dV1gK27D5edqQSOY3V3/N+t2yPBJS/xAq3APdf" "8JHBaSFZrhRkzdNhBCsTIDoUQjUx8N8RogO4TgE5I0Ccdk/V6YS7tN8w+g2LS2NjAdQ+K" "iJdG5uzOFOCyFTX83DsDBnukjmh0uEpXA5bX5YJsuxcf/RtE+d/ekUXnbQEXCB2RXXNuq" "PIELYo6Adhe4BbmKBv31s7BNrJTz5+WmyUQAvs+d2dHxS4Lho3z6QxqEQxUekeUbBcZKq" "p0NyJInPC6Uo8htMJ/MTm9+8FzfRD8uSYYUVQwz7xhZQ4w6bSkhQoorzco8IA8Nmcaags" "KVYPAN/ILbl5aUsx6TW3wsXBuTRWzf2+fCFo1SrFoQSGtIJ0yEzad/iTSbz0wJvaxAsr2" "R+y/JIOJ1TSte3myZyc7s+7brKeY0+QIQ8h3jQPZjF0jgdXV8peykvC+vO4ous/leQOYU" "WQs+f2M/CbmqyK6nR4FEQxhmpI8D7+Rs3P1KXve3u3S9y8zu1XDY/JaWO+Seisl86OsbF" "dGL9P4bqvbhJV1LMJ0ndR4Sa9FIOggmheF9uXsqVzV49r8oLOrk08vwq0MhWh0dysjAFG" "3kprYMLWwinAdcRhgaP9yDh+6bQY2nl9DnTVzTpm2XJsC70dAsxH+rTPSrPy2wiLBcS8O" "0ny3C3pKtYJ1AeuW8OvbTR86c2eeBXEF+UY1gEiJtbBRbCe9hCTH02i/vKyfwGwvQ5965" "oynfLjmFRzEfJrPJC95UNyZmLCEFBSJmMODDN7g6CsylaCPP5/LKbI1VWSuif5CfcfcmU" "iSK33ELU9AonZFS4V0oMXAczk/n19Mj0OargvlY4Kh1/rf034D4p5v/DyR9ZDel+s8vqv" "YqG5MwmQl2n7kHXnO2jo38tFO9HjrQU73ER0VCun4KLViiN7k6kuMep6asXjj+MIRRAh1" "/O88qDLYO4A21cnDLwGexYTWY8zxDm/HrZMawww2jgO5sxQ1tEbm4oJuYcvQJf3k+EsuR" "4IhFkPW9MFBIIpQAFQF1uIOiHGQiBBl89JO3D20o4QQOwQLBoIZw7ad98AHHzc3m+3TY+" "/udMDKmF1vP57miG75cdwwqRrhvFfekUcnOLkBz1dpZ5iJCM1WMhAd8dFilxwvG0XqPFA" "KQNSt6CRbneRLdgWFzeCjYCKYwztB9V2Dg6RBlBrCqsLxTOMpnfsqjItOwY1lBOhuQg/9" "reZJoEqKaYfx8RimUzTYlYS6xa7wuPd3gpi7PK9K+tNxFh4KjgTq9c+cNNkL+77ZKwECq" "5HshbZ5f1WgQNOP4xvXtb1kLG5MKy2fyUnKBFpmL+n6lw1/rKFkIXCndcej8zmP/eOv8i" "iTCQZur7ns3KJ6C0oIjqw8xJL9KD3HdF68PSMRDmzbftSVoI1woLoYIKd3JVEjSzjFLKq" "oKcxa2JBFLNeDKZHza/Km5fOo7j6GFZMSyREkQucFQb13cKMV/so3GgrvY+CRIph7smK/" "jy/T0wWUiA+hR/5927R4UoXy4kUPsDW3fOwkCYPtvR7jyaaUvQqVhdSRbACQthmVZmDWa" "BL6gRQVC9SOaHzY920HuktsB9qv60rBhWuLA/9dp7t+BYJUNyKrcE13WOKGHdr3APHx+I" "kuOG4axx3clt8tuWpTDETlSv4+MydbDrFbsN04a7gCkk86j6j7ofQSvslPyn4oZEX3STC" "QrbIa2zcDDurqqDip9UrwuXGcOSCzthZDYbprZKpIqdKbzUC6uVPAf73DxEyB3QbW+nhn" "W4nBtr1TKbp0oQFO81h3IyoR183p5UOX022UsYQyhYTVXyEUxPkNZ13e8eHf2QcJNpdkv" "hsloIz5UrhPje5GOZjDCpQJOqnggsiBCAMdXtXESIhMmBAVsBCE2E6Gjac0rv/vGyluh7" "p24QVIGXwPL84hZPKrCqYgXRQzV3j6QPqg1ESjdBLHNfN/e3MnICDg6dtu1t4TUync7cV" "zfDt8uKYe0ZPyOm1DDMXZCwQJPKxTsLmuBxkETIQNa5GwgNugffL4S6B6/ZdQ9zj7Mx3/" "ZpfQLzLw9+dFOgueWkfdDwKNYIEGklCVoWMYT4XCWQuFyHtM4cBzKZXy53vcou1ELKvEt" "GLTArgkY/JF0AcGa3H5devur5XwkixHBIhKK2HAjyQQoic3IcUcxz+8jIxxCYy5Z/8DL5" "detv1Rjo6v+YwGXg2lv9oNTBzWHWJFTdazw3ULSmw2iVFsJpYCitI9kMTI2+CILu75e6u" "+kLmuzNsmFYiLkTYv6L15/NYr1vRUoZToWSizwkwtBCOJtmZIgO0oNs9v1LQvcQevDPvr" "b1Ta0YmCzdfDQLXSE2CNj+1UuLLMYFXjq3cabSUctjYaDLZH6HDiFZaRNbCpcNwzpQjrk" "bGX0RC9zbJkNy1GRYVFwtTIR0+guCTMYwPXvk0UpJs3VdZRg4Uc6Xj7q6OwPNBpEoqjrA" "cHhQncc4U9FgyZwYUA+V3eYg+GobbgL15ZXcyCsZ0LJJ4BdW7zV1o9s3jTSq+JanuxI01" "O8aUIuQ+5g5cgGqwU+6i3xGVmHK2c9bloPTH8fR6MYFTH8kwuF4Y0+wDCEZlpiXRgM3x/" "MpUaRxJAzpZo5LFvmK0jockf3S9tu3+yitv97MyfyWjYQVzlrJm0BZL+wniobkEE7aL6c" "zR86zUrCqAhEGqRdFbiwssqYW5cP5afRrWNZraOh0B3z1NruOTVpRch2Qi1LPmQDB1M5R" "5ZEwmYTa1NR2Ev/NHFCv5ETVQtQnTvSLXdMwdYS0YNcsb0q19BXnPSERUukuiHBeMYsOp" "CK2aOf1659EKja4xZZr6MUJ33LvOyzrda30wgYvKO4QgriiZb1CXSdrEXJfm5dUFp407H" "26y2R+jjN0YOFL1f91WTAsLHw9n5divufZ+1m9V8VGgiMRMi5sMSLEmMxybqwdicRxoXg" "/Uq6hp+LYmgWm0A/JcUu7DMtLgVZgRVMTelIxJXE6ji6lcWNkMj8wrlC9wJ2wKduyYFih" "EvGlwV/tQmj6NiGZIJpQxRkJiZBm6kUaKvj6AUT5jOY5e8W1vQcXv2uRTh/2n6fLejmD+" "9KwJ4MduNzwVMQLGQ0V7mRYhLT2xoB6HDnKtQLAoOk0q+SYFxvjsmBYYeT95NjExsC3Nw" "kLYVBxNtnFcBTt76A8OuFzV1+MCOFD42aojwvGhSi/XPJyR4vQ6nr7SLmsV6Cb+zwPhhl" "l5SvJqFg4VbTFiGVBNOioosNkfvq2mzf/+BF56dGmZFjLwkrYJSqC9Gm2XjiQyiT0QqHk" "Y3KUY8akOTKq+4hwAbLhzs+ME6xeTSJbLnm55YKp/19KFWjiOGS7w/utBGZkAfzXH8J7T" "ySt1B5DeK8f+Y76UL4rbNSNyxvx5mazWgqVW9QSwdX9PVOuCOIFzu5EkvK9oewZnUQ4Xw" "zhHKPmsRBSgL2s8nLPMc46fSWlipHga4weQFkvrGJwsDo9vKrHUA1LPWeFZb0W65tjRCZ" "b1hefKkvrzRlQvywY1uDzsiJI4HvIzc29Sc3zOXUSJEIGsvJ9BU340MBDY+edO78kFO+a" "dnhZzFkFY4/hEpnN48LlL2/w/MmNTCFMn4YYHrSkLgkQyaOqsl6LPtHwUyy04U+KWgEIq" "F/0DhUvaPojYVnMFy4NQeDuYKoQNCWJkGKfIEJARygrABJeylDG6f5K6Ft24JarzexDA/" "gb2kLcBcbEvkRKt4pFbhuKbm7AFFUH3J0ERS8Rc1wnMr7WFSmLnn9eptdZYrd1v73pGVa" "flDi8E+d+YUvBvrbDgVcvJlhJOZ+EJ4iwzLAqmG24YOteW4dhToyNktC+WME9rUvmwUBY" "1muieGt3IuNqxaJQHai5BkAsNM5QhRBR2XIwLIZMFneAeRk4G8KhQ+j0ouCH82A8+q+b/" "ngxHZITWEgV4q3iLqKoWkJslZVaCGdMNagKNGX4QveAEB0hTc74vfW2MgxMl/WC3fUxWW" "S0shvrfRU5iDTOROn9LEN0wKe2F+7+ziY5puZTLzQ9wwqJqejefjzbZiK1jOZTLAm/V+V" "1JhFWC5OwFPrFx3gfCLnp83JXO/4oroc0wTM4p0Gz/THk++cBXTkyEUMlkPTTmz9fmris" "qj9kgEJnp9krC1phC28Oj8hVddTgi5ueYQ0N9QsixEruRuofkiDV7spRItdLLUQIaVEvl" "bC49KD7xo28KIgZOso2mHaa7PFHBU1cHPr/NuBcVC7rZShJ/6SVmWW9okA0j4Ho12vDpu" "6618XmF0W/9e5DzfN7hVjABPD0x60SXKr4pLAQkn0px66k4rQWIiShOQ69lJ1NpjW6FaO" "71aw+NJynRrU+7YCgijtTN1HWa6KLCVdANcpRCgEiCTN8SyR4xPvogNQDw3A1sOld6JYZ" "QJpOvaDkDkNkVtLCYGAGB0Os2sIKIRR9VWshEdZopmbuE49FKQJ3XOyMA+Xc9aqNU2V4u" "gdeELTuasMH020mcBooeyakhBVNSM79M4LdHRlAwBB9mcwPa6Xp1AtNzbDCyPvzBQYHOy" "I3N+Y6DGa4f7Ya/GkpRIgQHd+yIE4avgheDUtUNXhITfX4EGcle2ynYZbgLtAkZb0ixDJ" "o0HCpyAqCrUFwvF12nVdwi59/0E3NsMLIe983t5uWlsaRUOHIMCZiK4v3lPmraJAGhA8N" "jr0iRCcnS1Q1FaFVMdxYLj10QmbzCFBUVAQCR3nSihBikoaIhgiT9kXYNzgVivTCKgXXh" "tu3P1e2FDZXzcumZlhh5L3j3nk0nWV6Bl3JyHsSIY+qDMkRrXpWo7siN5bfHQTH2tEXeF" "hz7YyRrrsqOyPD1/I8/kAf6E5uVbqsF0hEGGdiOCeQBrmpIwNICu+EHksTcbhVIrSBlzc" "1w7pQjrzX9cRuqiQg8lbPCuqAfLIpUdYLRFhmWVU+FT40UvG+9c6dr5RDdJprZ6xywJFe" "Hpb1Onf34xsR5gTVAZ2LuXzVa6SPmcaZKIHk+mAGECbz0/y75ZqXn4zyEbEjtGmthBL50" "kKIuLBHTYbkKIh6gkQ/i+nacjVxLHEkDDJpI1m0Jw+iy9PN6EMTOzXP84CwrNd4YWKLF0" "ytYDy5YFc1zcU8D4nga9IKQVpKWa/FwOC6EZu7rglH5L6+gaayFDaxhCWPRG8M/8lKKK4" "2O7AQgmEpyLJmE2ENQGJkupdKk9pKYmdsFaVYbGnO/r1k3zhI52IsWmXLegG2WCyEITbI" "qEVMYTAhkkIePtxctQKalmH19/cL2K/dvrAJSsQtzMWmajK2SIgQhCyUxYH9DWXigzZCT" "QYdLg5VXj9Wdi52PXtXoJe4rykmW0lMCaDAUBapqLREtKKsKvWhgb59bOzZtbKzfA176B" "LBqPH2pj0STo/XKO1Noh5IqWRjvmlfUavdI8KlnVi5M7osDotUM0FwMqnrB22hTG7xrAU" "nnEwduBPOxZ42+Tid2siwxLwseGdjfuQuzAD5uODj8OmIjGR+61335ga8uV3e/JviaNi0" "ElYYeW+7o3uTOCpxJfMP/inXJBEuNZBVh0kayuLA3XXr1p9u4SD7+poveLXekxM6F4O5w" "7nY3uKoXtYLFFxFvrQa0BkW6UXRL2+M+tCmas3KsKYj78GiDjI4OLYtaYnTya2cFsKlEi" "GlKegegkRCQ/VeW5ike1tFKRadndC5+NUrz23wvKJwLqaIteiNdb6AAFHEScL1RWQZxfu" "YgES3updMekhbM7WTw2wmfWhTMiwsXs4mxSrN8ad2C4YV1/TyIUtokgjLqW7RzxKIEEcb" "3W1rQwJobVyUr28VpVh8YkLnYjdwujXDzjJJAw6ES5iGxZ9Z6xUkaGZoqKCiUq2PmL6P+" "lCk3hZFevFl0+hDm5JhoaSoILiXLv/BRrzZbotsBuoVnSCQkghRZDCCJUL9A6oVo09XxB" "S2ilJMr79533R0XBeYn3Ku78uA2YNdRZQPb95H1vwD9+E4YghnAwSlHpL5aUFpOzZ/JHo" "nmZJS1W9NybDCsl4FZ3iT7xfXeczNrWiqW9IBiZBK86WSBPugSRq6GBFTiM/03hYLUn1S" "awyEFy48J5TJwNJez6eFUF2S5/ySVuJvZX2o5u0aHv5IOUTnaFPQkbqzt8CsdZezFTje8" "KOZtgQZgbIKd54BBcPieJbIsbADo74c9XV+99DQf6aFB62leJd4mP2XEin8jATDctyRA2" "VVwuwLFfiGpEFeRafRJZJJBaMJUCuAl9krHafYVJETTcmwwsh7Tysh8h75fZA/uIJZqvs" "lIRGmQyJc4h5G+7zj+Mja4K01DHs7B9TyeF9oWvPEOPjUUIev+VsdFucjEhVsPA4y/xUj" "Ivg+5kYcuB0dJnJj3Son82uOsl9NybByMlsBJtZ+glVy8J/SRJiIiAix8pA1Uvc62g3Nd" "+4KxXvMhN3k3ct4y1euPr/eCwqbheqAbliKNQLEHZdJ+1gGjvwqbiARUyiS+SGH4U6io7" "//ufjZJB+0xNZ0DIs6G+yReAksz3cQec/TYOzzWzWaQyKssqxXJc8RRSnApAXDasaskZU" "MMoprQulT1yf3mpaf8qhuV3VzA2Shwj2KsS/WB9cRl44fTAhLYVkIiJtPLgbWor83HcMq" "l/XS/uXib+Hs7e3gEUmqtBcda90v4PKg13IUFsIQeHBrZI2kpbAgGBaYd9OYpMMx1Ov1h" "NYvHjVRGtyVSjMbrU7NjZqLEsQi8qUBujqJOtBjuUCGv/nKlS9npBCQVxM3Mwim6RjWvb" "Je3lYEGXQ0jYUwOiqUinct2DU29uE1ci7VJ7QZNFe/t+WkfXjgo5DGVRWu5BEQrCLeGMI" "H0c6YQkpY9o5U6jOb5K/qpyxq2ljCKe/2wUzW1MbHNQ9zDTlGrUb+RPUuFe5RblsQ5cGw" "OFZnk+eN0sJzp1WUgvi4v/HIgyaMMSV3eDeOhJgPFo6Jbue4/4m1fyJEYdK++kFHS6GvJ" "RJBWvPcboBwLjxC1z6S+O9sOgnrxHSlj2A3JCwyg/rNcRXzMYsIo+NacB4NvGzWNG37ti" "g73ipKMdfEHBUYHxz/QhfkCJT1EhZCJemdVkEq20VITv2oGU9EVXEEevnBLZEbay4sqvZ" "dU0lYmFhsmtKvxvUnH8OmCYal6K5JIsTyiIMIWZQCO6OpT/kip1FPT3NYeOpJ/KFz8a3h" "M4+43vh6yFew1PjKMSxyVfIoYSEEdHwvOG19kMW9D4ZCK8yxJiTS+jy6tqcoN4ELDeOol" "hdzefbOpzqhed7q4hCOCVZuDCER0kIYRyArjzsyN1YhzI0lTKUL4e5h+y10Lna10YOpjA" "mLMmKaxCFdPUxQwgothGRY9WvY/RlT6I8JCYvCAIWC+j2/+icpt9gXGsKBPqkUHBn9+gY" "vKG6n0hB7kpJjIBHSQsgMXTEQIRSmtBRqO65cORZaeJTEw0LzGedvA9pzovvJ0lC3mbCh" "IzLB1GOYiQgGQaiiioaoBhwq+WwbhlNd39YsyfyaisjDyHsnsHdbiSAZYD9QdTsgEWbAs" "AR8ka8TEhqld2d7KvVvVLyjqW/hkXDW5+/g8zJpn6b7KOuFjMh1PWlVPkaSBjc1Wgjr3S" "Cplw04hU2FwuBG+Xy16aipdFhhWa+SN7Qvjcoftm3QFTBR74le7HkhEU6X9Vrshqp/p4U" "n0NJpI+16U9RjnW0GC0/Vw6zxBh6Z0YQ+xvbGtplgWNw46s8SFh8AYaKFUORL4+Vih1v8" "voiuYOpVP5NBxl57jAH1XysbcJRVMTQVw/pIWNZLM/Z7PkRZnrfrO8EV0UlIhMxtJBZJ9" "DDqhqG7mUxgjY2IKjrPR5mELZ/PG0x819uLkA3UrTszIFO07Bk/EwwNdQWneg+iKmKeQx" "PDqwgpdbyor6+PJwfvjVt/v/7C3b/dxWNPoCONIs/pijWpOijrOhsAHgw4XjrlGaViYbd" "EjTxKK4amaXCahmGVd03B+R1/bG+CysLoGcE0YpbyhkS4tLJeiz8d+EDSBnAM3RXVoHGH" "kCgWv3P2FcRtGEHQq/X5up6f0Vff7Bs0fIfN4nh/ziIDO3y4T6kdubdXwjw6NbjBCSZXB" "/TUU5BaCBV5FKUrBj7zfb1JmnMPgwQe7AoXmZ6e8lEasKjYmoZhlZP2BVfHXlhz6vontj" "kOlYU4/Su2a4ZEKGIIYyVCXXNFEjZ3TxAct3T9EKpei6MQ6b6ihurRRr/Wb+D4BGTeI9S" "Tt4613x15aZ1hJtbY7s1NU/btjkQiXUyYnUP4N2gkskNv0fN3D2n9vE87hvyEWl+vpgrj" "Co/HRf3Wo/BXM8Yn6M8QKFkqh+RLXSf1WOQb9W50bvZYqDEo7OCzpaWwOjqqJ8xNw7DCy" "h6DIy9v8IOpDQGXiqImWNJdaKaOjwhlEjbsjDuHh/+BCtPLIVNfjIDyYFRPo0waJCliUU" "hTJ87//GOOM/W2QHO+5dqdvz/o+VNb4JebSqQMK4s0JAHqqBVLt7WifbuoF81bnz39Qy9" "ZZvsLmczev32z/jNXURJDO348Z+Fo6oHoG7D07o16qFzWy7Ynd2hpkbSP06AcrQskgWHQ" "moyXBklYugj18gOve3LyIxvb2n75ejnHmlJSczi7yk1iCNh8r65bOJBKmWahIORYzrOSL" "bQQkijjAVIq3k0zWOl5hZ14zOX+fuGTNuM49wBqeIzT8uahMqP68pXfWV2cev3drjf5Hw" "ulq2/OthsWrGowZkDnwzgzoLhUwjqydeGfAz5kQHeWRpjL1mTK36rrpe8dHfuXD33+9fd" "/fHX28d98csv7r3EToeR2/7HyATji/IjnHy4zYdQgfJLnZtXLetFC2CgOTwmLBhxYnFcX" "Cs5mvLkeSqhxTlOtfePQ0hwtLOuFtMi7rCQFA4M7QDy8YAko4dQTqXXIHMmxu+3IjRV4Y" "+VUM5+cFx/HjvViH9c0MqsbN/687XOvP/Mr4+MDL5uZkY8mssV/B0nNmhi3vckxz7GLyD" "zik0npcBtBlzIlPYbFTBHgZY7mTY17zvhY0dPMqZWJtrsfuDnxxZePv/5zP4rLcZ7I+8c" "C+Tw+s55NLH0qCpB+yPFQmh6SIXjYvHipJ2wPPovHwdBC+OBv9fpMNQLg8JnMz/cHlc+x" "1hwMa8auaWge/Gp45q7XlFb3nGkiFOyhunurv5rrkEvUL2eNHJhTujoe5C2hX8JC/vxrz" "7znlbuf+mqybew3jcTU1snxkluY9FwfDAqdca+Hm0jAeu6kDep9Zi52vic7ENeBmZmua/" "jjI7ajm1NdZvbmH79w+sf+MgjeSB/W+1CUrBFM66iA99TgX+CY7O62UcsRECtJ5yTh6ZC" "cxtEz8KX7BvZ/IGkXQNLCIzXfq9aUnMgHkTRj10TJk8ntPpXNam6ago/S6hPGEM5c7Q+O" "a6mfiQLJvKeLUoBJ3MdghF6JUtXXLn181WdP/8jHjczox62Us3N8pIjIJhaf1akWwL/aI" "gbAvQw8MeE6hjc+VvAynWPvfeH1D30aOzeqU/d5oWS31LFWfv8BgXI7KG72vKkVLOsFBh" "vnNFQO2owrCRB3CLq+sG4l+VWjgAQdUSIFLU0KizPy4M+ioxmgN/RtUzCssKzX1679X5u" "A1p22LU6DysE+TYSQP0IijHl2dVaDZvn6wcFfRVYCVoPuncYLleCHDvW7Xzn/oaduTH76" "5XT7+HsmJ0u+XYQTmw5GBUkpOvgQjA2z7cjdkp3pHM999vQPC98CSnY8dkT3nIV7Ghh4Q" "Yy/YF9/lMYCcALqD+r2/IWhu/9XSuNpbBXceBonYAl+btDqjrW1LQhehEs2VtzRvJI4my" "bu+1Gp2qfyrmmPb/T8whrlk/YBq/UgQvAB4fGO8vXrLauwhbNGZ0826pDIrE6c/YXvHrF" "PfdFKT20bH3VsiEMkREpVcTQ4muiJsWHHyXQWv/szr73vKB8S+njF8cAH+3x+XGaugGNx" "d6AV8XOEPPnBhy3xMydCZBllP43kWJCSbZsAFHeMjv6NqMYUVswmaCq1pmBY4a5Z0u4+l" "m2HYCB9hpTcATi59QtkZdYGzW9vt6A4HRV6LFp4jgc5izqkz51+f68dXPtb3SikiwWNoe" "JJ4C5WvMn+dRN6LWjfx3/9M6d/5K2EhdJeHQhfz+ekP5nrj0HXiSfGOtraR0T2EMYQ8n2" "j4US9y8A0tSy0LTsJTm/vQSUx1xQMKyzr5XqFbniNQDlokhSVayERMoawTkRIovItPA9e" "hyKn0ebNGbgt9Lv9Z37x7Y5/5xhS4Gquw2MRlen1agHWou6kMtSxTf5XPpXSXtxHQ/iX0" "aiJxwQp37NRJYc8WkiU9Rp4xc/hcZBqA+o6GytdCZABie61t0MScIZEkV5Nm9/iXPEgY7" "iwKRjWofKuiSrPT7LyMf5TkvsLIgRGqXSvFxHCS51JK7BKp0T1k927P136yvkP7yl51/v" "MhKP5rkjLWg/p5j7yBESJqQnPS2f9t3zutR/7Yf7Yrx2K9Xx2QJOqg1du/OkGXyuhQEmd" "to37Rr74B7HL4LIkNhoyLDUEQXJ6ZLXQ9TLDmtvivPjo4r1CeYYldmW5ayJ8sATLD3Woj" "RagZ0/KNBHCEUBYCOsEJPAD/YMgtF03bnywjZCN2Wf+LN3mrHBsDZqJekpWs/ACZurhjG" "b/FH+h5Cfmc9Zl0XzR1f8xuZG5xe2BbrexZiV2DjU3N0BG/RX9sFRpMpnfuGBYkEuVtBQ" "qz7D6NBF5r71y449gIXS6HZEHSlExn0QIGYJEKPb2+lCiVJgGpe2PPPJI22dP/+yvZVcU" "3zwJx048PlkfEOZ9ilmYRGVuy+75l3NHD/Gq/v54pSw+Y7x4eX+mDbscjjn4qBBLIHTlB" "gIRus760kr49NmvENWF9R3J/EZGPr5KXoAjtmJNeYYV7pqBU9qKGJF24Vej6K5JLtUIIu" "RR1IDf56XBwm/Y3t1fmhwvgMz0WI9fFdIxloEBXRbCfYKJd/KeMGKhwvuruiwsUIL4IQS" "EI4ZQzXhnsZlxy6WuUxmOAFBQphDHwuIm172wXiJevWR+dddtVEWBMy6edG48mmkztLEx" "tct6hYGsM0CP+y1iaEQ4kDXinftRX5tE1SY6qNfmCBo1sNCCI50zmYfzFh5fwcGEXxZ1b" "1E+C0wbXcpAbscbPsjSJCqX9aKkwM0tUiQsCaGsBO0HaUSKejLH+2sqxhQqL2GFuyaUIb" "sgYWFHipbQlzTHD9wsiRBaI35fJ0qkdMUj6ISra9fGRnxEhIml+gBoDfsI8Awa6zxvbPu" "ZwedWS0CORi5YhAVK7gRnO6GL2WbbiCFUNSQHSKGeU+g660QnFRAApWEvnfY1zx7bxeuj" "TApZwfMrukRpCWvmrun6o48bdOoGw1Jnju/hmFLOfUQY+ZK896yZ70Jt3p0itKS+aSRQq" "Ucl/BAaHadT27+rnb/xLwIrR2cOIKL3YYGSKxf+cb0XTG4VGjwF9VdEAOdHxBDiA98LpE" "SEh6V0AwlYhHpB1pqZFDIEeSldR3av0hJWuGsGwekOVGna4sDsBbOrcjCHMyrKetWRCEN" "iL0G1POzokLTUYlb3USlOg15bMTZe2lV2dHRNZ7dh+WlIWUxDoAovuB8VwEKMFZXue1Z1" "H1j2i9WYit28D5sheFhsU1YdaOWrlVv8M0cR7ppfu/4PG32tuE34AcosAjMvU+I95zUkw" "roBhGfSW3oczIohhXyvWiP/Rg4tLILgjtYxMUX4jmj5yFfBmQHp6Fj0bu2mkl8PDDo1KI" "gRKVVRfyXmK3JMLIUCmBSSBTv8nbdvf2iT7Cn64/tSIFSaYYW7JhwAd5mml6KHZHgEWsq" "g47iXdBcSYb1okLjgUXTUjmNEEfUJTmUhYTkW56V3bvykYFhxcJEXygVKQCIHfR9KM0UJ" "hbRBRpXG0V20OJBR89QxKSRvLq1JaD6z2KKpZSlUmmF1lHfNgju4L5UVUw3HQ/V2zVlEK" "Gc61r98JmmdySKnUGSBE8nvlGsQrgzpHTlG2JiNFIBHCiqOLXofUqKwf8cd2a3aMYZwzW" "xEB6XxSJEw8wG1v6eCGCE6ll7yb4rY1HLZr9p7jPhOpRlWWNYLiqv9zBxJBVbE44+sO+6" "a9SZCYoP6K5FhRlHMyOA+Cjz6dSK7L5Yd+6gY/XDw8koIdDscWAj5wMgmN8KOhHEGkClm" "IZwxQh1B0BSz/J38sqdHZr+YcUFD3yprJeSuiSZ2TThD7jMYeq8kCcpjWb2S9s2kFqKDD" "IvKGinEzPxVjfdgIHCJglhspF8lRF3IJhE1ZCETPH/xX2EhnFgfwMUD4kvkz1kq3ASIUh" "VppVFlvSoZg8hy4U+K2FRcL9ZgJffV4xqFJSy5a44GX14d6MZWWdZLvV0zJMKZZb3qMXH" "hM4o4DnIRKLc6JUwCNBHT57t3Qpijfp1mgubIvmTKSMLSBfOguhbCtOUw26GqDXosxKZq" "/pYgOImssVrAwrqqAKsMILMRIpV9Z658FUn7JjaImGcFd03CzVUpFO6zBxHbN6R3Hi+KD" "PVXtGG6oL/SjFJBKyYSm84SzKFcF9EVaQvDfSbtOzutJI6DKFCChyjHEiBtCqg603txeE" "hB8KTwohqYjCnkFDndIyN9QvF+5Ig6indlGVYYFuAH4/uxa1rCr0bRXZN0WG8LIVe8YFi" "gedVInrCJBhWuCeUe0DPRlknfEN/JoszlC6J5OXVU7BkwPASPq1qghA7POuI9TT0RJMwd" "x7HNFUwhZtEhRqXGEJ1AMwy/zXX9HYQsXIsqQKkswwp3zZJ9G2W9HCxKVXdNaaaeTtpXh" "1klhZNJMd2T7StsIQR8JrLUmUb2wjdu+vAwUdPbeyxSmZC6znxexhCWvLEdvoiOV4+FU7" "pCUDZcGkw30BOfwBY3nEgQTuVCzcqWQmw07o0DnDOVmqoMa7oYJs7QjwkiVGwfCieR+6O" "wEBKTdYSR2jwH0pVDW4R663MaPSa0y0gxeAVnIB+MhcHPEWPpqBj9ufH/sQ4+E7sYQwiE" "KEjXGDckrITRMbGq7al/Qm2MG8kkbV6qSVhi6jBHvmaY5l5+QkxhpJuMeEKNfxScWE4h8" "AUGz93T9ka6Vd01iXOuvmkLYY2TUMttXKV0ZyAlqcqv5CyiBpi15gzHeOBI9KD2lbOMjo" "3e2uD6413IsIonKalwDxIJFNbWjRtrO3peC7QV1xgBQGD5R6UGGANWzPX88YOECxuiiKh" "XAUYlGVZY1uvMVSj9Am2nXYJfiA55WrEmlgZgYiBrI8zUwkKoHLnfmyShvYJmCQvgNL8d" "HuiJfA67y2W9iqVbB9NZy4JeT0VNNodPAwTSIq+4yA9AxGkTVR/QlJtByAtGSZTS07aPj" "HxMqWR+kRMQZ2DpTebmngqGNrrBBMp6kTVAWaNgozBYb4W7wAaei0Lx0Yss0eEYEPpmcd" "LUOpIbL7LbPePvjHxxhgVKnGBih2Ei/ZCiSfs0xJVZVprs+xRx4RtrztsousZ3/KxWC1C" "nEFo3FqMt3BJlv1QJ0VEQWZo2XdbLu/toJosSVhrtv+pqasiwBDeNfDnOT8a029OlQV2s" "wGOaYqemD2czm85xJLmcVI7PP6rqfzlxQvYZoEAJzjHKIgR8FHnBdC1prXuNo9SDxKmiT" "F5BMauOlFMRjkX5OCTzSwbGuKjGpGlqlP1SkmFN75reVLduFrErWaBE1eZUQkQ1BANZBX" "R1kAH5HE4ale2OyhZCoMeETtkyO6/sXfvuu3KZHIl0EmdaCF1/aquHonp1mIKKVvzMi0T" "SycA3XdtCBVtTMCzL6r7sealbCRRfo0Fi5vVqvNd9JvPz9YndhGdgIK8EapVkWIfKZb08" "v/QkvaTxnxLIepCQuProSpOs9x6JZ7IWh+IWQj+ZSlACPIcFWYzDQtjXd1jQ78lbf4Ic5" "P5OBwhBYhnlaJoOWBZdGILkzbXtj14iHa1a9QPXQNjXaCmkdelB2lLhs8iNFXiPE5aeHj" "VCdNSbXFgGsU1yDi3Xn9ziChd39Q4+ID8Aeb+FkN/Vo/E51F8pSeUhAgKIf4GJgPAucRw" "8cCAG4adXPsx1Jze6fmEVNzf1KAUwQn9FCyGcRq9vXvO2QUINJu4ZRscp6Twqx6HSX+JR" "OuEWuiW8tBQ2XnBQjmGFZb1ODf4FwgLcbu6aaPXiBVXRDBkGHUYpZdWTefBZJcYQ4o2Si" "AF80NmA5qncM2TQc1cuclDDGMKJycGD2TaT8fHkkpE/pyqimOdigw60esdJHv+Ow6jMy+" "A4cNYw8DZq17R5YKjua4bocO0F3Tdv/sYjvDeUaKvrJ9qrlWNYYVkvxyls9bRCh6KOy2I" "WyDBYDJN6rHoxLK5GhuQUlDZDgFvRQjhlae1paSHM5X4ychQNDfWLPl3EvWmGKFCijL/Q" "/csUITlGGv9MIW1uPvcdVCJoprn264UChoAIJnyMHD/3w1DdJ9A2gqAJkr02mfRETGFvW" "aKtrqdor1aOYYXDm/KvcdekFIHQcTVKVoWw8VVQF7jHtIVw5o8xveczybBoIRQ5sGJ6zp" "K7JbeysAYDYzSRWC0WaV9f5EGE+uFeaSH0g8knOCFCub1k4CPvgNNm2piwhNn1dfZ+1Xq" "E08gj14WSbfhwIGXjdQq1QFgK2yC52vbNRwmYCjGFyjGssKyX73ko61VUlQgFYXFbrGcM" "IR9K3UJoISTVK0blBFEAZVo49+grLu9f+y6RVuZU78FIQRV6aqnrTHi+A10nszQoqMEiI" "4LA6TspL6mZbxA966a2Cz1HMvmWS7pmXU4mBf+KFD98zhIbgIKxwPJglQ52si8Vyn4pxb" "AohuaxOxM5CLN4nEcuRXdNoT8ygb0U/hHOurTyc6haoLSu4PKUaAA3SSSSDE44D8kBpY4" "4r/mIsXRUrPJXr39sva+Vdtsl4VqiFD2XaSJIMHYrMK9uW/HON/jdyZMHcKDXtJUrv2tY" "hyLeojRaPyrioytqoC9E6CAzljYVJvNr+JFbqQk+ehTsCi0IbrV7gbPFsQWtKwVjONPkq" "knD9S2DGqX6NSIoDMkRyKrfoyt+EhXuOA5qKWvN67zpmHQdixRP/f3CHU0ruvZWlKXvFE" "kCFVS40+vZws5mmdnrq1btGCE+ent7ufDJpcDL01+3oJDH9hMpftj30psukvkhbcO2IDi" "bIgMjxI1sSjGDAwdkorBXrv6PDX5Q3MHwACBIKRg5WUzo4geFYHX7k0Y6uQ2l4qewQusA" "JuiaGFHepQF7sqGnNCMwRRhK98AzkSOno+O64NcF+/r+TJtILOWKqeEEKdS4wq0Eshhqq" "a8RrGM0KuOYOFDGiaGZ5+Fei01avdAzMFWU/SLF2TtHRj4uQnQabSmMnJCWQitdXdL9H0" "LoLt1w0oi0oG1cwcZE/ZOoZ7wLZunNI3wPIYIzG2sjKijPkWHJQP9YH1dj50K8MqcmDb+" "zbZtQuI+Pb4wcN8+Py+IIEGCg67RV3NcE/pBVFCSc0pKJzAV+ETLvECeGtuJrhQLldfUs" "hYAJyfzgiqsHHVBCbCf84Rrl+0Y0pRhWuGtO2df2prNAF/AFLq8cy/KhYetoWw9v86ljh" "ua92ta2BnOHRNgxNq54IsIFbTNpH99HzgWigR8hOVAl66nhtN4pFmkuJ/WS0XQve8mXoy" "Fcf1SUo1KPSiSc4FbCQgjG9FV+EzKqXO6kmD6kZL1cKpkl5CJR0FJIrOpuR4euwVIokvn" "lcrJgrRxd/f8qxbA+cuE5bjVUP+5HWA6wpRR407NDBY3vwXPZ2vBFzci8ZllJQj39e1xv" "KKirA7YAAEAASURBVG1KCyHIiBxLwQawEIbCqjDt57d3fedtCeKRSJEjiiJgEtCS+LfFd" "XgaVJBl6QbEE1/3nHSpPb36CoEcKjMqBBMLnKxZ872XYJy4lGKNOOi4eY1ajYSGGE1dJv" "NDVGFDYVSGI4Dw7hXD9Ib34bOKJIjJo5nah5k6g2wJyZPQ05zW9TZ8z3mMn4vQ/0pYCPE" "0FRuOaEEymYGF0HwDIoObhx6EOpsoYT1QLoowcOH310OXuIMOjpiU+JFf5SCQaxjxlPBk" "D7QrB9a/4yJvP1VmVJoGzKDp+lNTWpC+YkIxT5JXrnEHAs0FiiTzU4ZhhUn7hoM3VgI9W" "xUuhulLM7V1+c2bv/OqZ7S/ViySzKiDiJfkuCJVLuslF5uPdapraWutCMnBOSJyRhI6XP" "tGaQc8nNo9MCwIXJE/R46n9r9ko2REycSKS7q+cYo9HdGktAn2Ghw7Ji2Fmm5+TVoKa39" "WbHcCtSyxh01n29DQ30KXhUFp+YbhWiGGJS2EVwb71yPoeZNLKZ90qFqD6Gchb4pltl8l" "aAhgvVQspqGD4IKJVpKYOXQiIlS4z/xepfeUpGCvhz0gQ1OYSIvc1Z+LfA5Dj+sJ5+reL" "IRbkAn1h5E/Z8m4xf5lWSJsUDBvMqiZ0mZXOb7SNLPnXFccCaHJIkNQqelgWJSyiqhT+G" "/IisHWuLJfyjCskAg9d2KflQwSwmNNwV2Ty4OJ6Qwj8TKnrlDIXcb2czlF0V/sPnyJp/H" "QqXKWUTAO6JADozChuR2ZXYJhDcUQQxhGQxi6tZeRW9gmFFvkPOpB3EbSPt+1IG2uEdbS" "kEGF1JHLyXe+n361iGyMuAfnwvBXVV4D+GL5WiplpCy9JIpShGu1ERAqw7DCsl6T9o1dy" "RTFK1PJXVM6RaYQyNomiHDdunUTutZx3hLZNeMhN9IwxQf1LYSI4gUeLDNzN5veKJTMCP" "GPlq7BFfNlKcTxRg5Q18miCdE+ZOm9UTmAOROphv3AFBLW7F5lUdlkcu0Vz0uOSGEsPil" "99vMr+gY2Ad3NZLg1jItkfogqrOjGOC5ShmEdllprrHj/cc8Hw1KOBAX6ee4zSxBzDL1N" "mKnFt7r/KkNRAHRsUFOlHFoIVfXBgvIKTpJwudBTZzd3fpOIIYy6DmFeRkMEt4Lj7WAJ2" "xycV4B0Zej43iJl1WuIfl5mdG12h2Des6VNWaNxxYoP4vfgagIOpnHS0D3YqntHqubGEP" "iOSOYXpqWurpdorlZiooEMOhaJxY5dU+GyXpIIdT87viaz6XI4BYax6lTcBQUoYZUgc8b" "q7BUOqMZXnswsK4Xae22X2EU+DgthORri7Ll/e8TzJ7e5IuZZCKA1Qh3bbYghpJpAv75r" "3fcLfPRqvTzVTzfqs8AH8KK7ht5+rqx4j23Tm35w1W8Q4Is8T4Ff2MVb83moUwF31d1Ec" "IMSDCu0EL5x6+/XAxEo68VimOqV9eI+QyIElV0BEV4GZZUnTYelULwVW2QE8zJnF8rHEA" "If3HsSescrHMDT/dFLPqGntWUVdiFbZ1ZYCPlQ9RqiITQw7xVMEU0dBxnTLGY0MKAJ5ac" "XeKdlELTcuNUajo4QHfBaXdsZJvNrlKVQCYYVFsMcnRrc4Abj6zxk08QJUVEiROUTcwV9" "jJw+SBAkrFTqTRehg7iZxKkQ39+3i0ZBeAIbIHVRJSeKDmPog4sRWg5d87JaKtF+no/o6" "Hgm8jk80yE9rafs27vTWW7zuppuadjVEokU0SBqMj77omRM/GJm6+mBHIpmWStfK5WILv" "ViZwFTOZlfaV0y6TS07JcSDCsshunpwwfTGdiZqN/D0p85sUq8x5nHohu3rkkl6qmDlKi" "0trbvvAlwrzFvN+h01i4aBeyhS4OCWBHDw6BhIcROXAym2qxNYpFe6Pn2yJn3qgvS0xoy" "1QFf6bJevuHYupZNyLJee8qMaTYtyBAdqOFep6UQLVYpffbzK/qGyfy8bNYyvXIyv4GBV" "Q3hHQ156IMoGu+RgawFd6TbsFjWy8TMxbLuH3x0VZ/hpGh4DgorGGuFyb678FayJwgXDP" "jNfD2OggLEAjk3E1cwhpATph5mCBSC5pC0L2l13t61/onr+AZe3aciBZW4PnxYxiV6/tg" "eUYeQD1KsYb6wtQUmaQWHZJFiJ6eFjOlBYGWIThBsu+L7yZsKl/2CxAg9VjmZX095zT44" "mrg/K8GwDh0tE2Fgo6wXUjTINRr32KvqXxChSHUL6AxLpE3pERJEr8ChoZtnobFAnxHrU" "4gMPJyVw+nWoKqEBZuJl0xy/AxVknmfQq/uqhC94MV58u5Aelzr212Hgrjg5wveVe8fwV" "dhLYVSw0veXt2x74p8/nwZV48Ipr527U8N4rbrapf9gg0/mBTJ/I6W12y9cdtwhgURRde" "E1YHn5ImtwqVBQYYFToF0sdw400NrEruuyomiBDHERQQ21fHVEsx4+CJakR698wGskhP5" "+UoOIpK/zEtpmkk4Sa56gx0+6NUdxUP6yh7Wlwqvr/P8QjkaQkUWHvhUD2ATu75z9fcIa" "RNBOYIxzcYDPUyllG5o7Sj7NfsKRb5BqhlsEBAFsWYNaSmMeHOuYKANZ1hhWa8zd/o2wm" "jf7UCU4PxVAHtdL8EkwUKY4IZ+bdemd00TYX9/TsBhmqmLTBMist1iq4wSOHZG9QYIWz1" "xojxQsHJ4aTOyJCkshKdiLOtVdIb3Wgkj7VHkjFqijWjiDOQUtIyO16AuwCqff9pA6Zxe" "sQ6hJjpnGCaoP1LyiWhEGsp+kQDt7rt3j2KtsslCtvJ9ff42nGH1atITuliY2Oz6EyiGC" "ZagHLuSkwGPX5ipBRFypVB3pYVOdLb9zbQcXkkyf7dgLdFMIFFBRiVCchTFC2YMEHqGa6" "e1tpSUsJ7WctEgYEYvJ8rvS/bIzlSGoSzSXWDGJYq8ZdAzMq7qsqwXsowumLECm56YWUt" "b83UG0mO+o5XSI8AKaV2W/fJXQKO6NYIua+qi4QwrjEsquTcOZNstzYPpB5uOgkuTRJgg" "ZELhTiIExoMjR/JiO1y//t9PoujXZYbolHfNmiZk5k3smIhgWS+1YwhhIYTFAU6cE6a+R" "SiZ7+V9mjmiJb7P9XOjQHID7VEIvMD+EvuL4XbMFwEzHaQWRlkvYU2uNAAcdsU3SiUNZb" "/ml8hiALmiLnkMBMq9jg6kB3eGRDK/cO1W1EFEFzWcYYXFMG2vsCvQYSFE0jMgRjWGJYi" "QlVkyyXX3ESGZE+AVzn+6oX89kWBeo+hkeiJC+bJeXKHQ7yXMzsF9699xi7TZO533KRpK" "JU3ky2o8x5vYxTLqSsYQcrgwGrt2Av56UsLSFpE2c7mcYMSJxP/0BjyyIKXzlBG9P18EM" "4GU5QBV10UQNMp+Cbgj6LfiLhrKsEiEoZka1geU9eKWGd1irxgLi11ItwXQkGubqHqkn5" "eX52bclS+/zyBNCCWsCJ1ewbGUL+sFFp1kWa9AP4uFxrxPgPoIJzOydrScg+nkld9ZDQr" "ZKXSdKqakxeQnEE+p6dbV9Su/9SIRkMvBIrFgk1L6ypXvHUaEx6C6Zb8gZGGN+v6EKKxa" "3qy5p9atNZRhlWV6IiHt+MWtrmNj4HqDYZoT94HFXU9LXN2y6ulLvGImEfb394ubEPQLH" "QTcMiLUQZAaGJJDx9G6UoYYUaV/CBz80xJr4dohLIQL6mwq7XXmdQf6ZA6mscLkI55WXO" "8x2EXBiuA8ELLqdUJPD27sfKqiFNHlhS/sg4aWKSfzU2/jBl3DGdalWmR7EBxD8LkWhKX" "5Zs5VnO8byhzCkkGv3/yLDUFgw0IoVqVy61JkIQARggwHt6x4y105IXJX5PuhIZkmJJFY" "ccVxrFGkCcEYoiE4YETor/iqaoOFEGsOFlRdF6XYh7t7Iqerrl5ZUck17u5LJs2U0HUqR" "yliU4H7C6RsIy1w0ftA0r755jAs+wWm8EbUnjHzPbPa7zHJus1g86C0bWzoJZHMLyzNV2" "1ftV4fOWFVA0gYyFpymeq22E43DxE9W00ndbmWZb0wW0ZW1pYTRMi9VLZTp6RT4IoV33U" "Fs3kNCwqLd+kFBbgew5AcVVPKSMbsmcUpS8umtojj8p7xd07jJsTRUl/PDMgYwpJX6E6k" "oL9C2AEWt3Isi/5orMlomSmBi2/vrqwm4/j4GYkzY8VXpZQeUC8aOR6XMg84CbHsFytZZ" "2y9KPRYveWNZCn9VnNvQxlWWNZr0r6CYpikPTUDWSURJpG0z7pA5HY/QIShpRAFBbD/ZM" "6SuWExLbkRI7QQql7WizoXU7OG015WJDXMLaqzqR41z/QMYDtjs6HrxItyrAogyWk3bSg" "dE0GH2Nz2VFiTMZeTUrrl61dKJcNR0VIosW54WSbz8yZ2czYGyhsJ39ejNZRhXQjLemnB" "Hs8vggYbCs6c+CYRYqOzStBNJfXV99WWC28QhCr1VrjUe52FB5baxJrEoqTCnf/wDFUbw" "lAweKPjIhxqhYUwaoU7d3aMXyiuPd/eTguhio1BG9C5G14p5ZjJjkuEsXL3Dimlu8b/fB" "F+pJelP99iyvrGYAHzgZUaiHqQF8rB6PWCZOkrq0ZIKc6HFkLblaluVdw1uZtjsaC2XML" "OJMzLHO5cRBg6/+lm5ylbpgmJxPmPMYSUshTmV6gilAaOzAvgKjZCNiJXuIf50mghRBa5" "XS79PDAnNZJenLf5VAdguq5uS7/9Ih90r6zXYo89wj1KW7fuEFJuJ67EmXJ7MUgW+p0bJ" "4POA10m88Marmux44YxrHICMDDrW+2+74pUt6B6BYkQZuoU0WRc2ph55yVO5lxEmMvxF6" "hLTaQJQbIzDGXJY2EHwkKI1yV3RuDiaBQr8H/SWiX80w4ciR7UMIZwzLv7iK85az1ycOr" "5lWtI0QDpOmG2X+rq2jdO8CoNAAe5SMEF9+BA+XVly36BFHnkxcaxc3QUG4ho+brNRcMY" "Vpi079VbnwIRFrZIM3X0xC4RuoS/ICN4cSMkJ3t13bqDE+xpbiLMiXOK72++jMzmN5gmB" "HMrdk2+q7aRAngzPdzVUr3eG4lYZPA5QxVCHBEM4eFeqVf3vV4Wf9fVLwPMC4XbB1JpM+" "lzTyeXVK0BLLo0YLcSzDu/SEjOg+BDShfrERWZznueMNxQjqyZhh7sP4rPPJ3LEB1nneu" "OCUthPct+NYxhdfV/TBBcYE/tMUxP6UBWKtF9FLvkhM9PhPgFbe3aXxrEYgLDEifCJRGb" "sBDy9KPe0uRQuZJQhDAwC5OB15naKkKWhsrKY3FBRH/CPn3D324luC8sHFAc0WOr6gZzR" "FnT8MFokJFW4OLpMgOqtKNQSkcl8a8VsVOR/6nFrjgSmGcRPpfJIE7Nu3mQ39QzRKdhDI" "sDZRstDe5mICt8BihLqLc0USAVyUK0pJG9QHjnI0JBsNM7avYURXpIIFxdVTdyOSJClPV" "iMDjeL4nzVQ1BxTeIsl6mnhm29FWXxV0RV/Vin6eO9onh+15R5GICPpRDB2U+wAXHSnil" "+YaQsCrG4vSF0lIYBNmrrmuNq1r2CyOFrg65JTS/m6AjRKdu89EwhnXiRL9YzDDfHoTlh" "341dRv0NH0s/gbUp5k28lyZZvu9sl7z3Dcw0EOxCvMZiDQhVErU2ihVUVXATKOq+mAJh1" "qEoRh6+uzuDd85xLFGXdYLKNSZe4l9wwtoKy2E/I6f1WoQNZkvzU2OdCa2XyJss8t6LQa" "xLPvV1fVfLmKMV6nAx6hrJ6LFHlf772XSnnpTuYuaNuZaHt8QhoXRThNhyb4rUt2qGMhK" "ziNqy/mpsZS15hoRvBAR9vR0COLS9ZWvlhAoDWKTDKyGmeGKDC2ENdxel1sEN0fSPstIi" "wWan5Ywo3x8XjCnfzr3wXVB4Oy2YdCAGqUhdLvwqFiTESEOun79sa3vu8xrHyzrtfD9oD" "apeMeL7hh65g2qItCUY1jcMJjMD9XZt2EtwyoMrsoDcR1agyb+qBjc2NhX1sAps1uW9VK" "TCMUu5wdXv2HzT17kfCxMhFKkB91dKBQwq9KxrHqCK99RRAy12jGExAglrPbYynqFFkIY" "ZVACzhcWQmGW5KNVauDeVAOgIjhSRCNPF6cfhFADiGKTA+2cZOYPtFr6qOGx1dwiy35hA" "9lz586vbZB3yjVdTS+1XNsQhhUS4YXhr673/IkNnksW7TcElkWQBgmLpdc73hBEKDbBhY" "hQOv+lUgcvwcEBITrky9XrsUCs0yE5grMvAmQjfuZi9JG6SfMycGnInCcMcZT1Gi5XZyn" "4sBBmjAROhDx+qIgWWJNRNUnzhcL92YEekXKo+rnJy1uMjrOOzSWBNK7KNVn2CyralaDU" "zRI8GZweN6gNQUZ3mQhdbepAIq0nEBQGIuQyVayJXdMk8zhJyI7TarNgOyJ2w87OH78NV" "fn1paQJ4apUOcsojgA4Cmoo6+UXssmtoihHHGW99vTIGDtsHLAQgkoYJ6VggwrWZLKRpL" "WyHJJTazzlSUFDvm+eoj8fmlBkKTZkqC81r70dyevcmyLVjKbVp+xXQxjWgPacwH/JGdp" "hJR1qijAz6km+LOvlolRTOrla7JpwaliQbiB1sIy3wKmvZV6txfmPWCDnFhZCHAn5Xj3M" "EA2YNcgQkD6HNmU2wJUj+rJe7POEJo0zvl94AotEIoc/qNWowDHsIkNWNBFPOVc0RCUg9" "5WtrJ2J9cz8cVdRSyGGgvk3OCHGLo6rv/+5upBpQxjWYA8LpdKj13sSXu4Yu3rCFcATOm" "WHXuueLiSI+WvLcTSiBU8/LZ3/8OmCTNdUvTKS6FDeQoiIoWQKHEszTq9c+S3DHP0Reql" "F2KjIzZdjCMGshIUwwu6j64pJ+8Tx37q2NvP4G+y41oyroZU1veqXr+HULTJ/gBTVlCox" "KUEw8QTHm8vJNc33cba6M6yZRGh7o9t8X/CuOMdYY9+ytpzmJ293ZndclZ1IHdVCHWLiR" "DPNlSj7JeiMOK5qIZN9i7JeVd0ln1uvv+Tmhs40wB1igcqyXtWNczFYw3xpXzj7011Q8O" "5y4eMEkVO93Q1ELTzcA+Pa3o3vqShp3/xjl8ODtO7BgRSB9OoNtww7cvhj7er6liD4VAq" "bLLCQj52fxP6AByemr69PPPPG+D+ugzpil40MhrSnPHhd4z/L2nKQj649uvF9wqWhkiwE" "5eSj8NvyLyFNiM00IWhVsR5eTP0Vj0AKIkZMDRgW3MM4lbKsV1cMZb0gpshnGRZCQILVy" "MWEFwVpBfMkHIUNS0ZDQC1Q7ZzLkXIpcOGXdaWBf4ZhYVXud2FXcb8imR8p1dkxMvJPdQ" "vRqTvD0nrlIX1o5AYthGtY1gtNyXUpasvpHa+D+CgqAcbFwcyVCwqsWvW2N3A9RHoWpSD" "rqazxCbxaxBAu/rjKOo38KjDgwENlmKTWnll9QXafi/wpYQyh7QztT2dNixZCJV0aIPXR" "yI0XoeucoRaoCSfTmT/0ladQRYdNRa4lYgqxIbd53uROAlmPEJ26M6yQCEve4KMgQlZ18" "EH+Ci5NbHMwU4MURVCvrC1XyZEnL5iTrn/POHPAV1P2izcSEaKsV1nhTkJQrQFOOEmaGt" "K8TCS09acJX61K5oXGFsYQYia2cyrQVHRLIzpk0j4jI/zRFhpTJb/lcvIq3QzOMPsopC4" "F1wdh0mkpRDK/UZF9tJKxLfWaujOssKxXwZ3YYRgljBplvSDnL3UgEd8viRC7W8pYJxTu" "lWYhoEh//Lgs+6UZ1qusoALdMXlRRY2IoIWQKZ84ORXfWFHv0VwEGOFzhG3faL92YEOvD" "MmJuKwXIQ1jCFGWXtkYQu5oJhxGUZNxXDfNi4Q7ZLR8X1vLUaLX0ulHL+PIfb3sz6cgKf" "AsAGkj0ESdwnqU/ao3w9IP90o/miAovolJuTnhtU1qvHdR8eSAYZmaJyrBaFqu4geGDpS" "GkTwHfxrcB3Gp0oYrhYUQJKugtqY8igB5vekUpbGsV4E0K/JYVDrGCq4DaUyHb6H/LT70" "V/yuglvrfAlwAWkTsbBXv7X7By/y4b2ajAmsHZC8WBPt7T92EyNWvuxXoBX3cayg19jz7" "NeVYQlVjlAqwsnOn9zieSzBoeCyhM6KTopgOFfXtD2GXQ7sKicZLd8v1sKCAoaR/XrZUl" "ixDoIrUvmkfQIBSNpnrJLHZaCKkuVieKnu97xgTi98/UcfCTR3t80sCFLLX103cV8NsCh" "h6YaJkJynHDJVtCXhgrhEP2Jt6lrbSYQoslMhdcU9nGr6J4wlJAaAlNU9PPy78Hpnk/Mm" "30f/t64MK0x1e+7uxzcgrB2BrMKloc4wLI5EkYVA1OoyB3dvOCyOPNVIEGFBAYTDXikWj" "QKOT1h8lRExKT20EC4OaWOuIOtgWS9MnEij0j3wTORzGIZv4TGPYLGuooRFJDZmxAs8Fc" "yF4VsI1hCqg+dqDsm5/xlh5g84lJ/X4aFbhd3m/o5i/RQYDiyFKNG3ySsMbpCPijdEJ3J" "CWxg/BwTBTRUKm11/cqV0wVJOf4UhUC9BHU2bWJCytlw1EoT011qz5vsuYWVXXPaLyMGu" "pUHPqqLcGU4tJsxH0j7NzyY2iOPyeIWVYcIOKnkdHrggaNPxR5Bl1EiCX1HCUI5hYboQk" "oPSXkZSpB/aMy4zdlQyxoWuCTN/mNoaFOflNlZ75o+FnrOk3zAbNJqlUmZKM4fFsVDTZP" "3IJfW7wM11ZVgDAy+I5xXd6wez7ZCjoV/GPwWJUBAgATtP3FVaW47XyiZ1ELr+lgLU9iJ" "NCBnRYk0gBNdR8CSiKrhlsS5j+J3MnCZ8c9gLkhf4gFwVx+VKAdoTpurRElstGC6AG+WO" "RBiLcEYrTfmuoaXE8XjpCvcQQzLzh6/7FwvQEmLjU48khE5R9zNpyIGat4uQx132SzCQE" "EVxvz4/LuONSo6zE4VTldw0iAMwCrOEc1lCXyuq91ZaWy7EH3UQ9Pwu9/VauQJK+POcr2" "ROuE9YBx3IMGReKjYyDoahmEbmXM+2Z2Ip68Vx57ScYFB+UJIxhCoio4wLXU9e2rD6TYJ" "5z1WgpDbQpZSeTPZcBDleqTXzR23Pru4u+ppogS1CdHqm60dW10elV9eVYeXL8UZ+MPZE" "oKJHjcQadrPAcG3LQz2Bi/yqFh+j0PNb17OvO45gP8T1okITk/bRrYHMS8VGjUUCFTbAU" "lnWy4MsWbNX93zjgzSKrvOCYZl6ehMSxc13aWO/BzISyAqaMNv+9eC6D0xgdvXo4imPCF" "pZufK9w1DoX1tK5o84kUQ6lXUi3R18Dj7DUhifmqduDCufz/McgbEECc93NruqWghFvmp" "x5LnctfKtlzgJteyauRzvpKUxcUqURaogrxF5lEjah1e+V7PBqQz0mLA6hc4GDjgxgJoX" "fQ4HL69MWCt22A6syWBhiuGDByLDw2HQ1Ns/SdiO9+dMQLnoplTJODBc6tnl+gxSNWX+q" "OQ5EVwjyn4FgbtrcPBXu9hfGAMaQd+zuqgbwzpwRFoPTg3+xQZfK3YzkBUzqxoRQgaSZb" "0MI3Vt5+q3jxJjc5f1moXLB76QOohE4pFLrmMN04EUw12QmPmjCMlZ8KoHHlPHjxgBVXG" "oDINUwJomFO6VOtRWB+YRcbk5nkbsbxYmMgUlLGZoSGmGW0pdXpF4+tMEOFcOy6purPNf" "jRAdsT4hYZ0HVeLC+CSX+aFY+BdsXiJEB+hYk077G3l1bzkGdOE7a/u1bgwrHIOrlbYFm" "t3OYpjYjNRjWMCj8KvRLaG/qv3II50HV6z4jqtgzSj7RVRzvc/diAieksmwlJMlyiBjxg" "JY8I3ClGdnrc3lkBzJmOceVc3fCjx1du6/4/nFK8yVvgDqan7Ikm6EH0M6k0S2ilX/75t" "2/IcRRjeEx9gl9Tvj5lwu/NApyn7hU8X+fOGd8b+y7Jfmt7WZZqk0eJDPizOmsG4MKxzE" "ZOny/kwb5Wb1asuJyWX5nsBiYQVhIZyvrNdihFAW6fHylGMYmdPlZH5zMix+SYY1M2nfY" "v035HdghszcNFK3VnSsFxksas37tBD8xF0+L49Dum5eFzGdYFkL3VPP30C7vmH6iamx1P" "iKxL6P8dlDQ70xwCc3A9OzLtu2VRSJGxaR0uuJh/KzxF5rmYj41K2d/C7Osl91Y1gncv1" "Crg+8YA8kLKB94eNRAxDPR5LoTBtijml0iFQhS4MjJy2FvndOKk3nX3SUqhg/yLAcvo+B" "+pc2FN5NJTMkRfx3Zufqw+K4XEnKnVoe/PQReRwyjew/g0WKh9fSTxz3YLv12trTkK46f" "/9NO95/EYHx5uHDfZCNo21Hj0pL4er1vW+AJlD2a2EpPdqnV96b2Jxxua9NCkuheFv57V" "VdWReGhYPQdObIkj/8KLWJHGRVkNblYpgHsVP4bmoqrXde4SNzuZ9cApw5CbXR+apd4tu" "FCwow3x8zNXDLUrVBnwJDQlqkAZZJ++KZx5wm8Z4I0p+bmoC/HvIpAidLmItoMArrqJPJ" "GpCurJPbE9/3IfZ6KuJMqyGkR47kxXghpU/Bn+9yNZk/wj7q88qiFODXvrE9CF5MgLlii" "cej7qkLwzp6NC/WYBDcasfRf6tj26A8Nct6iUBW7GY927/vopzsXiEZ1jbxJ8MFdm6xmE" "IiiFlGqeVSmWEROrB1IX0Od/fESD+HBd6/eXf+JUhYX85mEdxJ/8RGNmRSweATpSLiKFN" "rf2LHjkPF40HeypddMKIGTS58mfkDJT8QU8gQoHg2iKXADpgQU0jCLe28fv0T5WR+R2Mh" "4xgJ7h4KDhyQFsKXrv7dej+Y3I5UHILu712hzDvhxY2QnEu6vgOerQATVFN7kyJ9e/vuN" "xzXvEXnP/Q3JwMkZxMWwtofFved2DR9q1SALd/ICgvhnp5aK8MsDioX6/HjOUpVwFnyj2" "Cc5P+cjXATWLyTSK/QRSaCbFsCB9TsTx/a+XtfInyH9DyjNWJsedE3rNZnXY/LtYrMHzF" "CdX/XlLBENo2ViYSznb/19/fHwlti6fT+wWhaV28YXzS5C4WBMog/ks48D17Y6M/Q0Qiv" "9EA/SVDKea1qXiBHj8oBZbM/dxOM6sZ8zn9kiTwKqmwhBKtA0j5D871gzDJXiDCUHNUWM" "bZc7riQqN6+7//568Kk/nIGCR/BvmJmEHMOCLMT+CtWJWEhtX7n7fv//L/jsGZAuRy7xI" "eFLwDS9QyqiQt0q6XQk+gCeetuZyd2FH9ExBTmcshfGUOrC8PqGPikAH6qdGNPKgukB4a" "aqhrdRwlui4UVhASh5fJLQjl1EFJdR6lKOv/xyDez8SORM9NC+MAlMy9v3Hsyc8b06amL" "G3e89aYE5EisoGIRCCmLrwk988vMTwYNI+vBx8oo70OyfJbf3pmwJkeNZ9+x/xO/KH/PU" "9yLdfx8zr3MH51XbDsxXk3mj/vGEfOHkK59XS8HQQ/EMkd1YVgXLkwDf8D3cR5c0jErHs" "yDaZD4kKccoq2WEhIW5NolPUwSdK/AMdQfCGWRZ5oHOyU6aCFkTn9UdlC1+Qn4QyHy4vw" "O/VAxn48+JGeugR861O9Suf/tB/70s76f+O0Vq1IM/ohdshGwIPQIQmTQuSppTo2bf/zv" "9//VT/B7VoeJS281GwdSrdDV9Y1XIOQh+yhyFbHotpKNS6ggso/ijThCRw1m7AyL1oLDh" "6WytOTe3UuRQ8UGIwAshJBo3dToys4NwkI4NBSFU+SQYEGBvuoVivQY/yyRnhcwQwPFTn" "UbdSf0wWoXCvcDR3rrxlpPncoLzLxj/199cGJE+0wHNCX4wo4XVzoiQD2zvT1lFkaTv/u" "OA3/1DJ937Fgvwm9knGO8zw97l2PX9e8s6Wb2LP35FF1CusNd19d3XL/+bFZu1vnIaSR2" "hhUm7bt798UVgH67YwsJK/KBhNNb+yvKenH30vxrj617+ir7OXVK7m6193nvTsPwLhaLM" "PwyN+ccjVlGSYhz/jjH9fX8isRHqy5jkD3fFfqrsJhIPeBAHKp/LOglo9ey7o53T47pL3" "WuSCTxkUwrajaPrSNw4dycsMx2zSus++Db9//FL/DZlKzi8Ldi3/M1gXv4efF3GD1Qp7A" "OS3Y+YBb+HgwLegPN2ZLJvLZOXhp9Mr/YRx9mjrw89dIjKLKy0WNZL1D/wmNvzK8kBsvo" "RJ7yg2L3PnLkSASLISfE97Vrv/E8Dnxw/qMe6J6ymogIQ3LUxAoXSjkkZ8J3E+aq1zg70" "eV9qmyuD+t9Hq1y3/L4bw2vTb3pbYWx1InOlckklghXyVIV8Zxn9BG4yZRutnWkLLfY8W" "8rUo+99dDeP/ht0msezKq+ktU9vIRlvyy9E4H0Yulw3UZAm/eeEcE7YSmEY3HWtmUVnTC" "6JYK+p7uInWGFO7HjD+9LpowkUt0C0fE4lU2PqqY32Mag0Qx0T0gQ0TlF5gVh6fr7RhBm" "gjQhAuX3EZtgWGBrSnJx4rIckgP47mQ9SxyXNVlesiZM13oT9VlkWk/t/JXR1ft/9u2ly" "dV/1JZtM9Jt8FXShV4LDqaomLzIYpa/w8IimJRgVDqcQa32zrTl2ZlzfmHtT3373j/55m" "/c/itfziNGkByxfjqr2djJ5eR3umW+TrUCdFixr9vZUCz6DdCpe9ksXjRv56JX13hB7AM" "fyvWLxVko3d6ZSOE4iIwc+MNRKdMADE9jyIGlo6xXl5AgwnxWSwVypkiP+sBfZ2gLJRb2" "yz9EBJXtNpYPJ0P8gFelGlYsXRp0PfH6N+3/zTuErbd3qZVhahuhUMLjePgUYjS/bc8fv" "j9p7H5n4Kx7KZPJmNl2FIw3A8Qq8D+hN3XwhkQ3/YpfRP0dHNGNZEYHk0pYlpXUnGLmK/" "5U1zNrVr77iUN7P/pRWCaFRJc/pLm0UtYGbTR39fVJXWoQbLjk+8lblWT+iObJ1fbCZYQ" "WOI/zJY6yX8Ixj53H1U4dDddg8BjO4PdWaVwPrKFfYhmMw3BwEEymNSFh1dDNArfk0H0/" "HmKg7BdZlFRg8wYwNM0BCxd1CPmTos0wRKGFCwSvLH1y42lI4/GQ+iREUGhv2ZZ/HoaMT" "//zhV///oJ9472BN/HN6UywyrSQWAKpqsSOwHVERKOxjgCrNJcKZtErmecC2/x80lr7N0" "/v+Ui/ZEx/oB0PclZOO+7h81KPmpHgR24OurZ69S9dv3nz3Tfh8b4OOdYaykTnGhis4Jg" "KbAnIjcXfqfrAR6Ax5AFz3VXdd7EyLAAPYKXPjO2O7bRE1Qn1ViVwiqBeHFO91M215q7L" "RGE0FkI5Gf398hWlMb9aKk3ygwksTC8jFp3A/4qF+IYw33s1jeRL/CRDcgYaxrAIQ6hP4" "hGxzFg+ga8/8cqVP9w0MnXycd/xdxa8oQNYPrBYmYHvuTrSOk+lrDWn4TV+ecOKnacf2/" "AM6yqWx/Ff4Sics04gp5X0XleHTgEj+QAzu/q3bv2vJ62E8xjf40uhjCc+1GgBQnR4ZNV" "3jY7+/OoVK373LqpNAZH5yJhrrAyrbCEMzg4e67o0+tmddgkbFshGillqoJhQ0AgmSq/7" "+o09m3/wuqb9UPnIEw3Rhs5/UJFdLpW0KWalg/GB4qY4a4QWQiENqIOWEBIQW2CWCijMo" "ftn+WVYICK8oJGvPCJyY+zXDpmH9H7vyS3vZ9obkfpmcbh+XGOmha7+vJ7LHSlLVP2L39" "aAK2TZL/gz+t55ptsJAnjRKtbIVOnagNf1sBg+AvDAsKK1FMbKsEILYcEdXu/5k+tkam7" "1FO6w2/lgIkhD0HEKu4PIxYRXCj0RNbpH9Glr1nzzxVs3n0fZL2N3sQhsYIIpZikdkgPp" "EzmwDCRcHEvo7SJpX04UiOiPCDdL74Zzhl7AuDS9r6/X7OqSvm/Tlsw+WAjKaTBDIxB/O" "4X0LYf1PCSsPG7nP3Vbj4jbHIDiYjWS+V0FoJSuotlQIxy1SOaXThupQvHWo+j3tYGBVV" "TNRiaNx8qwusvATro3DqQyZmJySihvOAClGuVt00zgWOafJ2AbvqvH1PLT3vkRwCpFYlg" "Ki7duHIbzn/b/t3ctwHFd5fme+9inVi9bluTYsWVLduxAEqLwKBC8TlKYSR/DlK5oM9BC" "mCYlbaADocx0OmjVTttpIRAIBUISkiYpBYlJQ2GghQySeTUhUbETbMd2/I5tET+l1Ur7u" "I9+/zl7beNIiXb37u6Rfc6Mdle7d+/9z7f3fvc//7OP7Hl0ulGwKAWN0mu66uQbohU7ZD" "ygRbomhHzBqfhBzlfYSuarSzWXW3Ou94KUKMh9icofuu7uF30Kz/poZGMt9ClE9Yac0UO" "z7+/fHehpXVPyyPSLtl5FO7tGN3nRvgC1luBOBgozKqI8RsTsKLX1CrYKAV1IxIkkMQhq" "J4JHufD0SMb2ApTOmv4Q/GiVPjCXPIRYde3Z3JMupeRIyq2VTnFRfE8EMYdCG/fDgHEEp" "ACpg1wFBAcCrRo8bfYNYo9jgV7zNb1OtghbMoSn3nKYBa5cCQfoAi23C6hE4Dn7Sb5K2n" "q99rySfPK6bj5HEcEYFPPuUYVRmdt64dQTcjMmuuTUMSXntTG9lLYY5CdNc/PtJ2DCODZ" "f5Y9GI0KXOLX9wjL9cpIF/5NzILALv2aERUKmSxHdBTu7Cl4avuxpNKCvOD5sVSaizxFp" "dGRZ7LcO0Oe1qFPuHxe/5QszMzbcp6RUebBfnRf27m8kzzNdJLpt44mhsCaGbwOSR8RLQ" "xIQgJZOC0Xc1SLPl3oEyDh5tP2i0Aanb2Liz3mKTpBtv2pGWL6QL048sgwBsL3ce0Coyz" "bArHS30pl1rK/75uNCPHE3C1bUJFeNbTv6IjhA1OcGWZL9ihZY8gHDZ4+EcKbn0CXHsMO" "csM4asoMFR+3ttRHwNm1K8usVD/s8HuwenOby2odf2BbQU3iKDk7qdgTnLqdvBdn2q2aE" "BTWFj9O5M5c53mybIy5MKa9LeAhRRTPK7VcgD9Bq8GIiboir9CtXPngKAaS7qaEAQv88h" "K1IulLmPx/IHAsQzXppWcdVh+idWmqf/IjqYV4EkknxETOaeeUP/EfXLz+vxCcyPIq2X0" "1NuKrcDC81E2ROYc0Iy1865ItHN0ZjpoEqo6jtJeEdATYapJzgl9f308+dRvdePNXiJIA" "ulyx5ZUPPUE4hzHpoIiTDSTaPDJDPArGG0OX56q5PZGlJ4gdszvMN9XYNEfADkGEjOpzP" "6wVE8tPNVbYziO72CBPCA9PXEhx+HGIQ0NSMsI4fH+NAup69hunwEMKjgQs0eNWlOhSIQ" "fUC1BxEQG+lXW2qbn+v+m3/hGOsaWx2hsKwPCTW1oQcX1WOBX+IG4yLOuKm1vxt+o7fem" "vB31cbBopAElH4tMNly67fCy39pXCYbnq4qiQbPom6bv4aIdoIlzsIMWtGWFi3CpvN2V5" "l0t0JaC2Gfh+4KItWETerQwLQZBC4zrkP/4Rj7LKf5QtsXzRCGToCpzm/0NA3mWOh1Eph" "1jp0+bLXccLawgNGGyrUJX7wNCcnxgamdc06TOWQYL2QjrBwD+Z2LCgpl3vP3ldq+xWMs" "lITwgLpg2Q5+ZsoiXy5LdrkyKZd0cnPlzz41Q+v6RnYR29sSdaOQGg5hfuhsWxZelpjoc" "cjYUNDmXT5bpHAAWK5kUgIAbWJB3vaPngmjTy7RpZYod/mUh90TY2i3A3hgDTCX51f+UM" "mbHBCl3IK7d5TPc9Qig5GOpDrvyaE5VcZffrIvWjrVegtws2JUaNj0a4rHGBWXrSPRY92" "sCsytJfBABM155ZKeCOKWuSxzJRto90eloWSBQAiIRiB/1Y2Y57oaLkKLbaAS2k5Mvec1" "Lv1QiCZTPNDwav9oiNt2y/EBXDjLGtxnMJqEjiotl81IpGNnE1DmrMShNXqUBxPKQCRhJ" "dnoHUVL6hn8jrlVKyt1io246VRNH1F59e2wTr0eGebSUxOPlR5BjwkpF1FjPZ/ubrrwy+" "LigjpwOwQ8kx08Unit/1ytPDzeR4TI2WRD7r+7UQC5e/dfKBtv2pCWOPjT/L9ZmYPb4g1" "8TwUqisUiEoY6CnmUSF1qBJGE18Odiduq4uMY2NJjo9pRj/rzDpaSPcsCnEIdG4V7gx2h" "2IUhfDy2dj/ber7wudoN1u2JBVZVYhn0F/zPW6GEUbbL2MawQM4Z+WzD/t2DthA1gsM1g" "RyDtWEsL6bETmE0Kr6RLntmhymqnMBvzL5LI0C7lLMjXANa11md11IQ5RE0fSWpY8+DSv" "RIyuWmCg1wOy6sOWroUa1oZhr5WdMLx5e9WEYInlJYmoC8WpfU5/VEwGRU7h06U0HYchC" "2y8yvMvc9su5UqAzEkiUQE2YZDApljhFN4MSE3XhgLLPGLgF4CHEWrsYmm1uaj5MO6hvF" "Ldok9URafmUmbPPRMKMWlc1cmkIi56roa65Fja6PvHWNelfkIGXyLVscNUXaohAml9QvO" "0Xi7xIQc++NlPDg1ay61IXHXcVqsNGhKmlesN74ISFuzFP6sXZH8IfPIT8fG+48nAh4qj" "h6FJTSsj4Ulv3ew/S5zu04Np6XXi8C/8nW9azz/ZbrO3Bg60R85PdCZ2SoMGjDWF4XASe" "3dIaNvLZpq9uXv/5uyEvitrxxg4Xiq7+byACdH6AoBCWSZU/nBdKTU0aKNG8hxZ9CjVv1" "bFjh0uewuqL+QVOWBsHhVDPHbl3mevleiiigbJd5p1Wgz6g2xTy5DTLTOxfyVbOkhiD2i" "C/e9VLpP7+cc7mnZ0jX7Xy7uNL2y0Ttiw08KzroDnbzW1hKz/d/sRN6x+4nY6OuyJWhPL" "ZRuqKjKQHgw2UnyIIQN5Z4A3puAe+rufuAqBBLJanIbUrruvZXto+iBSdwAmrlEIIbWG6" "B/eAhAO1ATeFOl+DC4ATtylKesavzJN6g2vrtYBjlzahuyV1EqZ/r0gs/6Ax42yPxk302" "qt1V2NfRrJZeS51Ui5klnzzhnVfeg99Qk1LVQqOj5F8z8mkkElH5Q9RQ13CkCHhZHPicf" "QBYR4nrCCQDJywfBadmp1YH2sinuJF/qUiLK5WI9/ZsQ247lt3E5BBtfUq90ehTsKUY8i" "W3Du1pjX+B2beOYXqrCHsR9w7y93hArbHjwFOZEXDdNGKPWbkMy1fumH9l/4IGhXvsExd" "aRawG7VJgxDw234Z4eUHXNdA2y8SREptmGt9nlfkhnc/06Ma2AInLD+HUGf6eo9syJK46" "88HiYyUuGj1Ii+VoG8//7NGvGZszCZ7VkfHI7tXxiM3GgX3eDRucE1LkEtgUtEJBEXdc+" "MJ3dKcWNbNLb/1pivu/ws6AmlWiqwCw7pmO0qlRG5ee/vgUfyWE5ZFSjqd1XINMimQwQ1" "Uuo4kK2V6VKW8BE1YbKCUQ5h3z2wkYZGkKR2QdDdC4xFctuFT7dHeQwTm8eN3NFTO664b" "LxJp9az4+ta+ROv1Rt7Y1YJW7OQLBulTyEMV8iGSnuJL4AWMxnUzFo8axdm27y9pvfa65" "LrPPITPeCt2RVZ0Jsg/SoZ33vaLsch2XgII2rFskuP616mYH0Jl1p46lWoR8lXnKQyUsO" "AhpOJJ3oS3LY6ggdXFIrcpB3qMYH4U0VgBUB67ousDPKQhlUo1/Acn0roPpHXZZV/bdf3" "St71Jn2GPNsUtFolj8QbCoqBO4EvLNSIv+ptr8M/AQSWS8mz0vNFjCcOMRCOsOBt9Riuu" "GLhp/f03X7f8rhfoePQFlSc4F5TyvifafuEk8Ny9osntfKdD4+ZAXptS26/lth0vVR+tz" "lMIPSO4sXGjEObgwf9ahqJ9q3iTcE5hwR0jkD1Bj0brKiyOmnYBU4c0DAwpfvHbQVo8FW" "bJR6Yw1z/55f73PX4mn/+bWNR8o2GhUA8CXYsoBA9hSfmiFtLnyU0vefsnFgppzAoZOvW" "wy07pBScXe9LSWx66YcPd3/IxLC0Bi9wt6L+pnhcFAv39Cf67W3oCzXlPkszceSOT8NCw" "KKfQQ/hQzClmKEVnTyolWrBVKmeghNXRcSWunxGsPPJ9uqHFiV3RvhrvSTeQ9EzR5S43u" "N833m/ero0jAEOOQcGa5D0c2DHivaHnsScg1RNP7b3l5uxMMYUGrJuga/VEooZuotQDBY" "zwlTehDK4izIsw17u2ebTohJ93WfQHrfGe/37Lqru4N5RmOOqlzaTGG4cq4zoBsihHByc" "slzn7Z1BbDecB9bik92S63iALs6NRw5wssjUE8/h4pir5AiWs3YnvcGFmiqd6IzEYTaYY" "rQkDPQZNuuqho0sO6ClidXCD+7oMtfVCk0qJBnkPSRzStojA3rL269/Dv9/bs+fO5sPe9" "AZWYL2zucwGx9ORrSnshNFQZzZiLN0esjKHY7H1u67tvrVUo15MjPaVTI6WOhynxZvqcZ" "EiIJrzUtuvnP1z9AhwV+TzPEVHQk0LEHv21QT0vn2UU1j5tRYombTtO9t89EpucJeK7M+" "elxCN+hAiaFTPv0jvJpOiSeXZLSR6Uco7ZAgWNI4f7/D6+u6lpeLTpb9XlxRL3VEN39M6" "vJQ2ghLVlGZT1Q3u1Y+nPq0jAoO4yaa1lpaPnZo49odHEPEOwpKv4LZYAXBlsIfAEWE8Z" "II535SxcNgCIyyooyQEN1wXncw63oN94XLUb0t4UyyTGZ5jTVzWdt0BcWBxt6qfEOUdSf" "y4IqePUp82bRrjjgye+zhC+8LDb7RiT4KEYd9CwcDNmsoFLA/txbE12VxxzVEanKuzENp" "+5d8so+TECwVuc/V6T5/+QGtb28NnRDG/NGexcmUOjLBKQrgve8NNW1/47mq7iBWNlAZ3" "0dYLtqBjqztSvxaAibtVueA1YvtS5YQ5PJqcuUoijTVCNHXMOiMArRs3rjG0uvT2uuR+k" "VN95ik6kK7DtrOUUwjCqtxTyO/UQeA8UhLi4KG98BDmVlDRPqqUHMS+A90HeJ08hDqLbe" "eeQdwB5PydA5212tlFiAC0aD50L7qNtBgYiuh6rkhzEXuqySP3FEajehjl0jfQEfxsmEq" "OFhhhnWvrdWI9ak1HbElzCD0EiDEGxZKxvQTYsJbi1SUqAU99RyHQWASEpxBxOQfzeW1G" "0mJ+ZCpywiF0+HFYD+HlFyGsBLvACMs/eM7JrAlFQPKIVsR7smlYvK0XpeSEjLZtJLNPt" "L786lkhsHgQGOamga6u1+/HDZgX84OCJZuGBThhb6NH5nFPoVZFGafACGtLckzYVTzv9a" "SVcpOgbL88EKOifTYarxuefkCIl5RNSiWPQmBBCJDqQoOxj80yT99bqo0lHWHBO8DQJoD" "WqquEvLx7VEXKTCCERZ6AdMlD6Lgza6lyJQJGpQMOYHkW9dXS9COdS96xj8DbUsO2XrR/" "NRQCNUQAXJDkjjNcbzvp3MaQ8brjbb+gxaybmvrjpQIPMEYFIxDCGir1HHvu4D+1gfXXU" "idlrL0C2XcFc5r/KxSAhRpYhh5+aW37b0/ShrVv6zW/OOoThUBQCHjM2EnB0BjyXXdQAn" "nbL09bMjOT6xJzrsxTGMjkNo6Ig085WXgI8928rRcytYVgMj1SWy9iUuN5kqoebb1kmr2" "S5eJDYGxMzIk0LCrmh0HXnWxaFhGWixSdMHSGjSTk+Pi+ivihoi/RAc8fHSnKIdS0ontm" "AxIdI1ivImjx/C0keQ2xEIWl6XqYewjr1dZLktkrMS5CBCj7gaZlGM0HUC75JJaFdOXJR" "lgkohuydLApW0X/+Mnb9LqcEQhh7R4XOYS2U+ixwsTyLJCWPuVMZEHboq1XseAgBita17" "ZeC5JNbaQQqACBHTtE45T29uRR8NQxal8PyhIOsAr2V8uvEIvC/H6NOEayIhkDIazb+sc" "phAFVGgpXy+hUFQAxV3gIzdmQFeY1sPy7k/hcPSoEFh8Cg4NprLLAUWwA16DJi/nJeA2S" "3keeQhRxWkEoV1p9tGrCEmAJRve8wmryEMo44BtEWy8Er3nakRWrbj5AMu5I1a+tl4yYK" "JkWPwIgAiguKXEdM28PhToQOcg2iCd4HTem9U1OptpJvpGRktxlCFs1YYkcQk0jDyE4tJ" "eEQriodJCROqobumbqkb0r2Vsb0tarjN9FbaoQKAMBURQPaf1byfBOznB8mU55mQazqZi" "Ep7VnszlefbSUr1+WjFUTlp9DKDyEdodwX8rI8ZRDCA1L03YSQo1o61XWL6M2VggsEIGz" "nkLT3gvCKqJkMhQtOSronjeFUk6hEYbqUHFOYdWE5ae25Iu/3ogqmCEXJVHJtHaeoA1/C" "WEglKc7aFEa0hMvkECNauvVcDCUABcdAn77LMPo2oegaBTzwzqRLFtyDeIEbpYxGbusUt" "GqJixek4kk0dzVJm9nIF8fQvrlgJZOy1VUsOMaVqWAqe8pBGRDwDdgL0FvS5zme8hWiyE" "bYZ2VCcFFV9E/PtHS64WOqglrhzbCgXGdWe6uJG1moQev33YIGEXagutaJ1qaug/QcZWH" "sH7oqyPVA4EkL42s66EfURcdYYyvx3EXfgxfJgRDck8h7N/cw7nwPYCOy9n4wm3J8o+MI" "O4WRNmWy10qJYb3Ltyu8f+Lon3oyj7R3/3JIyRPKiUy3Rsvm5JAIRAEAiKAVNPi/zM5id" "pOmkb9oKVSHsAN6FNIaXv6yomJd8Z9Aitn9lURlp9D+OM9d3ag2M26ktuyqn2WI/xCt4X" "T0qWifYYe2wljpJNOU2lZ6YySC52O2k4hMAcCohv0smX3b3U9/WfxOFe4eHzkHBs36i1e" "fRRaTZeumwkhBFSeMkZV5LKxVGXUYeFOZOMspRxCHL0sAcqQteJNqbCFgZAG17P30E66f" "7ef/5oV71B9USEgGQKkrZyr3BB5mMcWyRdehPZ6JBQ7OTnZNkMQDg2VB2RVhHV6vI1/37" "Yn4CHUQw4s7zi8jISFlBxNC1ttpZQc0YSyPKjU1goBuREYGkpy84zr9n0jO63tikTQHVT" "jncJlEVyUd2LW0b6+f4eDANVSEKlfjnBVEda6/t38YNBf4CHkTMUBK0eAOmzLq4zmc1g7" "O8V9dDzfs1mHY6tDKATqhgA1KCEta/ny9AzyZT+DQgR07LIIoZbCkhnGQF9lqIKHuGCi6" "09Z8lVFWFvQsYMO7Lqz10gX9XEOedeCmxcWrCNNiTX76e1UFSVaz+1WvVIIyIjAKLdbLe" "189MHJSe3ppiZYbxkTlbIaLC6ISscqTDNY9Cckyhjv+lOeUBUT1vkeQjgbV5KHUNKBon2" "YpseOXLf8rhNCxsGyWF3SeSmxFAKvQIC0GOrwTc+mlfhoNku9qzzyGDb6AnVRi87IZNwZ" "j7WOkeCVNKOomLD8xMXRXbctRVP6XrsIDpDPyMcVYjL06Zr1KwJJeQgJBTUuZgREt/Cku" "XTpQ097WvhvW1pCFJdFmlcjb9RuLIYlKjOe7Oy8dy8pPEh/LptEKyYsrKv48GyrE4vSJQ" "6FfsjYh5BIFMVPUVHwRRJ406Zk5XMWU1aPCoFFgIBYGnZ1feMfz5z2Hm9ttUjLQqJHQwa" "I0jNyOfAVi39RSJA06hqH5ecQFo2TG8Ix3cKKkDNWQ+CY/6DUzRUeQlez9Cbe1mv+TdUn" "CoGLBwFaEg4Pp7jVvWC/9X2nT3tPEWlBs4G/vO6jCC0P5WXMxzs7H/6h0K7GKooRq1jb8" "D1tsPuvojrpGKTeSRbSgOZHkA3F+WdN3T5AQvpy02s1FAIXMwIDAyMoRZA0V6782Gw01v" "o7MMJvbWuzQpgzGeHrtTy0kdsYmpx0Mq7W9nGBd+XNiysmLD+HEE0nSjmEMkaOe4j7IA+" "hdqSn98b9BFZKUyk54qRRj5cCAoyN2URaLS0PngpHOm6YmmSj7e0h3whfkZZTBm60f5OC" "tnUj/Gfd3V85QLIwNlLxcXlPszIE4JuSSof1Z8lgxuAhpJUXwp0k068gLPc5wq7KAAAJP" "0lEQVQQQhVF0b4BXrSPBFdDIXApIeCTFmNfPo3r9MZfT7z3gaYm69ZCwdYKBY+0LeKBoC" "8MG5ea2dxsaVOT+l2d3d/4Ji1RQVZV2dEq0rD8HMLv7flghydxDiGUXrT1gobliZIy5O7" "FD1MvVfhSuibUXCVHgEjLD3fo6h7+UGY6cmuxaJxpaaGQb3jLeKwWKykhVU2Grq8CanKZ" "4bAJsmJ/3dk9fDcpOalU+V7BCyWpiLD8PoRG0e7EDttlzSHEPUO3wec6M3fwiScvnL76X" "yFw6SAgwh3SOpFHd/djD5nWqqumJqOPGEZIa242LMPgvURJA6K/cm/scMTzAFXW1mqFig" "XjdC4ff3dn98inhZGdPIRl7/MVP05FhHV6jWiC6OqzG8IRPewg8xl7DlqlfIWwZb7BU3I" "KeSyXdUsU7Rsrcw9qc4XARYYAFfujKT37bL+1ZMlnD3d2P/anurf8bVOZ2IjjmPmWFstE" "pQeTl9RiHpZ1rAiiIQIju5OD1/yZvwaxlUjKgeNNb242LROd1U9PGt8MRTqu7ep65NueJ" "zyVQZAVjsnXrvRc1ljXL5KHmaNfbsU0LZ+jgHvEZso0YGNDUoLh2OxEyAhxg7uocDgmk5" "RKFoVA3REQ5DFeJDIZGhrxlnR+4ecQ4ucnTvzVhqnMSURY5n4fq8SrEgkdbVk93p6L7NT" "UrwHaEjdgGwiYQOl4VEEBmyFoPJt1Tk9Oed/X9cR9XV3/9mOaVMnATmQX2KjI6J7UkiCo" "Meh39tVeZZwX2ATm3VHJfuUU3SMnt90woWlfKbuUxbz7Vh8oBC4CBHxvHRELrmcs6e6hl" "cjfed7wP7z88k9el53OvtnVnGs0d/pKpptxGOxb0Xs0ojE9j1XlJJStSbx+HtmBT7le/K" "fLux85RLCA1KC8pKB8VWdgnwvispdxtB711bvvb3/Pj6JN+uaZabIUScdcdjxhmrlp87F" "3bfyP93O5CYEA1tFzAaneUwgsdgTOLd9eGXZAicunTg015fPjoUiku9jWdtO0aN56btZ0" "jSGh2SBb2bl3g31VgYaVBsmlvf/85btbocSsPZtDWK6JLth5vGJvpLnSKtXQ47vpwyEAq" "dUQyFcIoN5QCCwyBM5pXHTtpPTx8X16f/84NC+yXXEPIq9hJab1Vf5EJCe2+z1sk8ZlVz" "uyogOWTVh+H8L26Iq2rHOsg3IIz2ovYiYyPMLg7hqFnK41Wav+lwTaJINUSgaFwCJAQKy" "guJbFAzzp+hZOwyE8+2OQ6wQlksN24/4HNX0um7D8HEJPN3ot3YwWi3nMBlOUaeBugHQA" "w7PDe9f2vf0pEm1LUtjdZBJTyaIQWAwICALjlzhIyh9p/0Vdn8smLF+6kBHpMJCWVCjmX" "BCWXB5Cz3PDkZBhzyaeWMY2T4+OauZmlq7ZutrHRD0rBBQCtUWgbKLxi25Z+tKjszOOMB" "QJfbG2ki5077TWZq41kzGdFmP1o/S148nUeXeGhe5IbacQUAjIhkDZhDVSmkHRMV6CIjO" "rIw4DS1xpCAFNGp14IqRZRvvwtWs+vo2EG2DVpwTI9sMpeRQClyICZROWXw99zdobjxl6" "9CiVlgFlSUJY0K5018plQsXWWN8/0w/qacM0R0nkuxRPMTVnhUBwCJRNWBTan0ZgWBe7O" "su08C+sEMxgiG8ITqQq9gTtKtEc1cJm5+euXfGRbWnYri6MFali7+qrCgGFQIMRKJuwSN" "5NWpp/L2QmvuM6VKeZ0vYaO6DnFcMxZs1mYjs39d0yRNIMJkXeVGMlU0dXCCgEgkKgIsL" "aMqTxBMr2xBu/U8yF9oTCjW7YyBxmuFYxH3JaQ2s+wNh1M6OjaWhXirCCOlHUfhQCMiBQ" "sWZEtXUoBH909x0fNaMn78lM5m2EY1UcJlEpGLCpI6zCYbF4DHnlnR/a1HfP10aRG7UZ9" "X8q3af6nkJAISAnAhVpWDSVLVsoEFPTkn3/+sVsRn8uGufm9/o2bGTQrEBWTYkoc3Mtny" "KyIvvaZq2yAvc0HzUUAgoBeRGoWMOiKfla1tiuO6/PORM/9pitoTIWhfPzbh01nTZKRWi" "6ZzYlIloh2/T3N13xwKdg+kd8BR6oJoYaCgGFwEWHQMUaFiFBS8I0lobJ9ff+xNBid8Tj" "1JADVQuh9tQKKTAskVEhFNFRK8zSijPNd3KywpvpoTS4SpFVrbBX+1UINBqBqjWh0YcPU" "PtU/UOpbc/ccls/iyW0ZCFvgwihbgVY1E/kM2koIaaxRLNl5mbZcUtvTd24/v6vp9Oank" "ymWTqtjOyNPqHU8RUCtUSgqiWhLxh1zPE1mx/sfP/HzXDuM47jaMWCR00bqaVQxcchjQo" "kBY3NY+EoGgYxQ4NncqQ10v2RN/V8emLY0wyUSKQSGGoZ6P8g6lkhcJEiUNWS0MeEyAra" "DS9u/84Nj97tOi0pp2ieTLRQ00ZSwBiKqJI3b8GkQuTjYGubbFKRGModJiKGnY/+ys23D" "rxzw2MDRFb3oS71AK/Vs+D9+iKrZ4WAQmARIlCx5jPXXKluzsiIpg8MaM5PD6WXZ7N7P6" "1ps7dE4kzLzdqabXsuLEzQhnQiJAo3ReofbOTEarxzNNETvfTMUNjQQmgTVMjhAzv8TNh" "c8pV39L7vUcRYkSeSDXspHTmCNbOVzTU/9Z5CQCHQWAQCJSx/KufHQY3uuuPtBW/6LzXX" "eVck7rUyHZ5EuBJtFP5zqag9vmSAvwx029ARfwpzmDabBUm57KCuh34YMVq/dX3vPU9Ci" "+PkxPdNYQtqCejDrZ4VApcMAjUhLEKPurxS40TftrT18JcvyxReekfePvF218326iy8xP" "PsDjQNhL4VOm17uRNho3lCN5qeDZmRZxLRjVuvXDYw7f8SFEJBsV/KsO4jop4VAgqBwBE" "goknzLhq/uWvP2xPePvlA+86XvgjiOhz9zU/Ff8PDmjHqpU0y6s/1uXpPIaAQUAjUBAHS" "uIi8YLOa19BP4Qnk9RPbpefdriYCqp0qBBQC0iPQEM2FDO3kMBzShvjxB7VBmLJggVd2K" "elPGCWgQkAhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQk" "AhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQkAhoBCoOQL/D8pAub21P/M1AAAAAEl" "FTkSuQmCC") ================================================ FILE: minemeld/flask/config.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import gevent import yaml import filelock import passlib.apache from . import utils from .logger import LOG CONFIG = {} API_CONFIG_PATH = None API_CONFIG_LOCK = None CONFIG_FILES_RE = '^(?:(?:[0-9]+.*\.yml)|(?:.*\.htpasswd))$' # if you change things here change also backup/import API _AUTH_DBS = { 'USERS_DB': 'wsgi.htpasswd', 'FEEDS_USERS_DB': 'feeds.htpasswd' } def get(key, default=None): try: result = CONFIG[key] except KeyError: pass else: return result try: result = os.environ[key] except KeyError: pass else: if result == 'False': result = False if result == 'True': result = True return result return default def store(file, value): with API_CONFIG_LOCK.acquire(): with open(os.path.join(API_CONFIG_PATH, file), 'w+') as f: yaml.safe_dump(value, stream=f) def lock(): return API_CONFIG_LOCK.acquire() class APIConfigDict(object): def __init__(self, attribute, level=50): self.attribute = attribute self.filename = '%d-%s.yml' % (level, attribute.lower().replace('_', '-')) def set(self, key, value): curvalues = get(self.attribute, {}) curvalues[key] = value store(self.filename, {self.attribute: curvalues}) def delete(self, key): curvalues = get(self.attribute, {}) curvalues.pop(key, None) store(self.filename, {self.attribute: curvalues}) def value(self): return get(self.attribute, {}) def _load_config(config_path): global CONFIG new_config = {} # comptaibilty early releases where all the config # was store in a single file old_config_file = os.path.join(config_path, 'wsgi.yml') if os.path.exists(old_config_file): try: with open(old_config_file, 'r') as f: add_config = yaml.safe_load(f) if add_config is not None: new_config.update(add_config) except OSError: pass with API_CONFIG_LOCK.acquire(): api_config_path = os.path.join(config_path, 'api') if os.path.exists(api_config_path): config_files = sorted(os.listdir(api_config_path)) for cf in config_files: if not cf.endswith('.yml'): continue try: with open(os.path.join(api_config_path, cf), 'r') as f: add_config = yaml.safe_load(f) if add_config is not None: new_config.update(add_config) except (OSError, IOError, ValueError): LOG.exception('Error loading config file %s' % cf) CONFIG = new_config LOG.info('Config loaded: %r', new_config) def _load_auth_dbs(config_path): with API_CONFIG_LOCK.acquire(): api_config_path = os.path.join(config_path, 'api') for env, default in _AUTH_DBS.iteritems(): dbname = get(env, default) new_db = False dbpath = os.path.join( api_config_path, dbname ) # for compatibility with old releases if not os.path.exists(dbpath): old_dbpath = os.path.join( config_path, dbname ) if os.path.exists(old_dbpath): dbpath = old_dbpath else: new_db = True CONFIG[env] = passlib.apache.HtpasswdFile( path=dbpath, new=new_db ) LOG.info('%s loaded from %s', env, dbpath) def _config_monitor(config_path): api_config_path = os.path.join(config_path, 'api') dirsnapshot = utils.DirSnapshot(api_config_path, CONFIG_FILES_RE) while True: try: with API_CONFIG_LOCK.acquire(timeout=600): new_snapshot = utils.DirSnapshot(api_config_path, CONFIG_FILES_RE) if new_snapshot != dirsnapshot: try: _load_config(config_path) _load_auth_dbs(config_path) except gevent.GreenletExit: break except: LOG.exception('Error loading config') dirsnapshot = new_snapshot except filelock.Timeout: LOG.error('Timeout locking config in config monitor') gevent.sleep(1) # initialization def init(): global API_CONFIG_PATH global API_CONFIG_LOCK config_path = os.environ.get('MM_CONFIG', None) if config_path is None: LOG.critical('MM_CONFIG environment variable not set') raise RuntimeError('MM_CONFIG environment variable not set') if not os.path.isdir(config_path): config_path = os.path.dirname(config_path) # init global vars API_CONFIG_PATH = os.path.join(config_path, 'api') API_CONFIG_LOCK = filelock.FileLock( os.environ.get('API_CONFIG_LOCK', '/var/run/minemeld/api-config.lock') ) _load_config(config_path) _load_auth_dbs(config_path) if config_path is not None: gevent.spawn(_config_monitor, config_path) ================================================ FILE: minemeld/flask/configapi.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import yaml import uuid import time import json import copy import minemeld.run.config from flask import request, jsonify from .redisclient import SR from .aaa import MMBlueprint from .logger import LOG from . import utils __all__ = ['BLUEPRINT'] FEED_INTERVAL = 100 # these should be in sync with restore.py REDIS_KEY_PREFIX = 'mm:config:' REDIS_KEY_CONFIG = REDIS_KEY_PREFIX+'candidate' REDIS_NODES_LIST = 'nodes' LOCK_TIMEOUT = 3000 BLUEPRINT = MMBlueprint('config', __name__, url_prefix='/config') class VersionMismatchError(Exception): pass class MMConfigVersion(object): def __init__(self, version=None): if version is None: self.config = str(uuid.uuid4()) self.counter = 0 return LOG.error('version: %s', version) self.config, self.counter = version.split('+', 1) self.counter = int(self.counter) def __str__(self): return '%s+%d' % (self.config, self.counter) def __repr__(self): return 'MMConfigVersion(%s+%d)' % (self.config, self.counter) def __eq__(self, other): return self.config == other.config and self.counter == other.counter def __ne__(self, other): return not self.__eq__(other) def __iadd__(self, y): self.counter += y return self def _lock(resource): resname = resource+':lock' value = str(uuid.uuid4()) result = SR.set(resname, value, nx=True, px=LOCK_TIMEOUT) if result is None: return None return value def _lock_timeout(resource, timeout=30): t1 = time.time() tt = t1+timeout while t1 < tt: result = _lock(resource) if result is not None: return result t1 = time.sleep(0.01) return None def _unlock(resource, value): resname = resource+':lock' result = SR.get(resname) if result == value: SR.delete(resname) return True LOG.error('lost lock %s - %s', value, result) return False def _redlock(f): def _redlocked(*args, **kwargs): lock = kwargs.pop('lock', False) timeout = kwargs.pop('timeout', 30) if lock: clock = _lock_timeout(REDIS_KEY_CONFIG, timeout=timeout) if clock is None: raise ValueError('Unable to lock config') LOG.info('lock set %s', clock) result = f(*args, **kwargs) if lock: _unlock(REDIS_KEY_CONFIG, clock) LOG.info('lock cleared %s', clock) return result return _redlocked def _set_stanza(stanza, value, version, config_key=REDIS_KEY_CONFIG): version_key = stanza+':version' cversion = SR.hget(config_key, version_key) if cversion is not None: if version != MMConfigVersion(version=cversion): raise VersionMismatchError('version mismatch, current version %s' % cversion) version += 1 SR.hset(config_key, version_key, str(version)) SR.hset(config_key, stanza, json.dumps(value)) return version @_redlock def _get_stanza(stanza, config_key=REDIS_KEY_CONFIG): version_key = stanza+':version' version = SR.hget(config_key, version_key) if version is None: return None value = SR.hget(config_key, stanza) if value is None: return None value = json.loads(value) value['version'] = version return value def _load_running_config(): return _load_config_from_file(utils.running_config_path()) def _load_committed_config(): return _load_config_from_file(utils.committed_config_path()) def _load_config_from_file(rcpath): with open(rcpath, 'r') as f: rcconfig = yaml.safe_load(f) if rcconfig is None: rcconfig = {} version = MMConfigVersion() tempconfigkey = REDIS_KEY_PREFIX+str(version) SR.hset(tempconfigkey, 'version', version.config) SR.hset(tempconfigkey, 'changed', 0) if 'fabric' in rcconfig: _set_stanza( 'fabric', {'name': 'fabric', 'properties': rcconfig['fabric']}, config_key=tempconfigkey, version=version ) if 'mgmtbus' in rcconfig: _set_stanza( 'mgmtbus', {'name': 'mgmtbus', 'properties': rcconfig['mgmtbus']}, config_key=tempconfigkey, version=version ) nodes = rcconfig.get('nodes', {}) for idx, (nodename, nodevalue) in enumerate(nodes.iteritems()): _set_stanza( 'node%d' % idx, {'name': nodename, 'properties': nodevalue}, config_key=tempconfigkey, version=version ) SR.hset(tempconfigkey, 'next_node_id', len(nodes)) clock = _lock_timeout(REDIS_KEY_CONFIG) if clock is None: SR.delete(tempconfigkey) raise ValueError('Unable to lock config') SR.delete(REDIS_KEY_CONFIG) SR.rename(tempconfigkey, REDIS_KEY_CONFIG) _unlock(REDIS_KEY_CONFIG, clock) return version.config def _commit_config(version): ccpath = utils.committed_config_path() clock = _lock_timeout(REDIS_KEY_CONFIG) if clock is None: raise ValueError('Unable to lock config') config_info = _config_info() if version != config_info['version']: raise VersionMismatchError('Versions mismatch') newconfig = {} fabric = _get_stanza('fabric') if fabric is not None: newconfig['fabric'] = json.loads(fabric)['properties'] mgmtbus = _get_stanza('mgmtbus') if mgmtbus is not None: newconfig['mgmtbus'] = json.loads(mgmtbus)['properties'] newconfig['nodes'] = {} for n in range(config_info['next_node_id']): node = _get_stanza('node%d' % n) if node is None: continue if node['name'] in newconfig: raise ValueError('Error in config: duplicate node name - %s' % node['name']) if 'properties' not in node: raise ValueError('Error in config: no properties for node %s' % node['name']) newconfig['nodes'][node['name']] = node['properties'] _unlock(REDIS_KEY_CONFIG, clock) # we build a copy of the config for validation # original config is not used because it could be modified # during validation temp_config = minemeld.run.config.MineMeldConfig.from_dict(copy.deepcopy(newconfig)) valid = minemeld.run.config.resolve_prototypes(temp_config) if not valid: raise ValueError('Error resolving prototypes') messages = minemeld.run.config.validate_config(temp_config) if len(messages) != 0: return messages with open(ccpath, 'w') as f: yaml.safe_dump( newconfig, f, encoding='utf-8', default_flow_style=False ) SR.hset(REDIS_KEY_CONFIG, 'changed', 0) return 'OK' @_redlock def _config_full(): cinfo = _config_info(lock=False) cinfo['nodes'] = [] nnid = cinfo['next_node_id'] for n in range(nnid): nc = _get_stanza('node%d' % n, lock=False) cinfo['nodes'].append(nc) return cinfo @_redlock def _config_info(): version = SR.hget(REDIS_KEY_CONFIG, 'version') if version is None: raise ValueError('candidate config not initialized') fabric = SR.hget(REDIS_KEY_CONFIG, 'fabric') is not None mgmtbus = SR.hget(REDIS_KEY_CONFIG, 'mgmtbus') is not None changed = SR.hget(REDIS_KEY_CONFIG, 'changed') == "1" next_node_id = int(SR.hget(REDIS_KEY_CONFIG, 'next_node_id')) return { 'fabric': fabric, 'mgmtbus': mgmtbus, 'version': version, 'next_node_id': next_node_id, 'changed': changed } @_redlock def _create_node(nodebody): info = _config_info() version = nodebody.pop('version', None) if version != info['version']: raise ValueError('version mismatch') cversion = MMConfigVersion(version=info['version']+'+0') _set_stanza( 'node%d' % info['next_node_id'], nodebody, cversion ) SR.hset(REDIS_KEY_CONFIG, 'changed', 1) SR.hset(REDIS_KEY_CONFIG, 'next_node_id', info['next_node_id']+1) return { 'version': str(cversion), 'id': info['next_node_id'] } @_redlock def _delete_node(nodenum, version): node = _get_stanza('node%d' % nodenum) if node is None: raise ValueError('node %d does not exist' % nodenum) if MMConfigVersion(version=version) != MMConfigVersion(node['version']): raise VersionMismatchError('version mismatch') SR.hdel(REDIS_KEY_CONFIG, 'node%d' % nodenum) SR.hdel(REDIS_KEY_CONFIG, 'node%d:version' % nodenum) SR.hset(REDIS_KEY_CONFIG, 'changed', 1) return 'OK' @_redlock def _set_node(nodenum, nodebody): if 'version' not in nodebody: raise ValueError('version is required') version = MMConfigVersion(version=nodebody.pop('version')) result = _set_stanza( 'node%d' % nodenum, nodebody, version, ) SR.hset(REDIS_KEY_CONFIG, 'changed', 1) return str(result) @BLUEPRINT.route('/running', methods=['GET'], read_write=False) def get_running_config(): return jsonify(result=utils.running_config()) @BLUEPRINT.route('/committed', methods=['GET'], read_write=False) def get_committed_config(): return jsonify(result=utils.committed_config()) # API for manipulating candidate config @BLUEPRINT.route('/reload', methods=['GET'], read_write=False) def reload_running_config(): cname = request.args.get('c', 'running') try: if cname == 'running': version = _load_running_config() elif cname == 'committed': version = _load_committed_config() else: return jsonify(error={'message': 'Unknown config'}), 400 except Exception as e: LOG.exception('Error in loading config') return jsonify(error={'message': str(e)}), 500 return jsonify(result=str(version)) @BLUEPRINT.route('/commit', methods=['POST'], read_write=True) def commit(): try: body = request.get_json() except Exception as e: return jsonify(error={'message': str(e)}), 400 version = body.get('version', None) if version is None: return jsonify(error={'message': 'version required'}), 400 try: result = _commit_config(version) except VersionMismatchError: return jsonify(error={'message': 'version mismatch'}), 409 except Exception as e: LOG.exception('exception in commit') return jsonify(error={'message': str(e)}), 400 if result != 'OK': return jsonify(error={'message': result}), 402 return jsonify(result='OK') @BLUEPRINT.route('/info', methods=['GET'], read_write=False) def get_config_info(): try: result = _config_info(lock=True) except Exception as e: return jsonify(error={'message': str(e)}), 500 return jsonify(result=result) @BLUEPRINT.route('/full', methods=['GET'], read_write=False) def get_config_full(): try: result = _config_full(lock=True) except Exception as e: return jsonify(error={'message': str(e)}), 500 return jsonify(result=result) @BLUEPRINT.route('/fabric', methods=['GET'], read_write=False) def get_fabric(): try: result = _get_stanza('fabric', lock=True) except Exception as e: return jsonify(error={'message': str(e)}), 500 if result is None: return jsonify(error={'message': 'Not Found'}), 404 return jsonify(result=result) @BLUEPRINT.route('/mgmtbus', methods=['GET'], read_write=False) def get_mgmtbus(): try: result = _get_stanza('mgmtbus', lock=True) except Exception as e: return jsonify(error={'message': str(e)}), 500 if result is None: return jsonify(error={'message': 'Not Found'}), 404 return jsonify(result=result) @BLUEPRINT.route('/node', methods=['POST'], read_write=False) def create_node(): try: body = request.get_json() except Exception as e: return jsonify(error={'message': str(e)}), 400 try: result = _create_node(body, lock=True) except VersionMismatchError: return jsonify(error={'message': 'version mismatch'}), 409 except Exception as e: return jsonify(error={'message': str(e)}), 500 return jsonify(result=result) @BLUEPRINT.route('/node/', methods=['GET'], read_write=False) def get_node(nodenum): try: nodenum = int(nodenum) except ValueError: return jsonify(error='invalid node number'), 400 try: result = _get_stanza('node%d' % nodenum, lock=True) except Exception as e: LOG.exception('error in get_node') return jsonify(error={'message': str(e)}), 500 if result is None: return jsonify(error={'message': 'Not Found'}), 404 return jsonify(result=result) @BLUEPRINT.route('/node/', methods=['PUT'], read_write=False) def set_node(nodenum): try: nodenum = int(nodenum) except ValueError: return jsonify(error='invalid node number'), 400 try: body = request.get_json() except Exception as e: return jsonify(error={'message': str(e)}), 400 try: result = _set_node(nodenum, body, lock=True) except VersionMismatchError: return jsonify(error={'message': 'version mismatch'}), 409 except Exception as e: LOG.exception('exception is _set_node') return jsonify(error={'message': str(e)}), 500 return jsonify(result=result) @BLUEPRINT.route('/node/', methods=['DELETE'], read_write=False) def delete_node(nodenum): try: nodenum = int(nodenum) except ValueError: return jsonify(error='invalid node number'), 400 version = request.args.get('version', None) if version is None: return jsonify(error={'message': 'version required'}), 400 try: result = _delete_node(nodenum, version, lock=True) except VersionMismatchError: return jsonify(error={'message': 'version mismatch'}), 409 except Exception as e: return jsonify(error={'message': str(e)}), 500 return jsonify(result=result) @_redlock def _init_config(): try: _config_info(lock=False) except ValueError: LOG.info('Loading running config in memory') try: _load_running_config() except OSError: LOG.exception('Error loading running config') def init_app(app): app.before_first_request(_init_config) ================================================ FILE: minemeld/flask/configdataapi.py ================================================ # Copyright 2015-present Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os.path import os import shutil import sqlite3 import time from tempfile import NamedTemporaryFile import yaml import filelock import ujson as json from gevent import sleep from flask import request, jsonify from .mmrpc import MMRpcClient from .aaa import MMBlueprint from .logger import LOG __all__ = ['BLUEPRINT'] LOCK_TIMEOUT = 3000 BLUEPRINT = MMBlueprint('configdata', __name__, url_prefix='/config/data') def _safe_remove(path, g=None): try: os.remove(path) except: LOG.exception('Exception removing {}'.format(path)) class _CDataYaml(object): def __init__(self, cpath, datafilename): self.cpath = cpath self.datafilename = datafilename def read(self): fdfname = self.datafilename+'.yml' lockfname = os.path.join(self.cpath, fdfname+'.lock') lock = filelock.FileLock(lockfname) os.listdir(self.cpath) if fdfname not in os.listdir(self.cpath): return jsonify(error={ 'message': 'Unknown config data file' }), 400 try: with lock.acquire(timeout=10): with open(os.path.join(self.cpath, fdfname), 'r') as f: result = yaml.safe_load(f) except Exception as e: return jsonify(error={ 'message': 'Error loading config data file: %s' % str(e) }), 500 return jsonify(result=result) def create(self): tdir = os.path.dirname(os.path.join(self.cpath, self.datafilename)) if not os.path.samefile(self.cpath, tdir): return jsonify(error={'msg': 'Wrong config data filename'}), 400 fdfname = os.path.join(self.cpath, self.datafilename+'.yml') lockfname = fdfname+'.lock' lock = filelock.FileLock(lockfname) try: body = request.get_json() except Exception as e: return jsonify(error={'message': str(e)}), 400 try: with lock.acquire(timeout=10): with open(fdfname, 'w') as f: yaml.safe_dump(body, stream=f) except Exception as e: return jsonify(error={ 'message': str(e) }), 500 def append(self): tdir = os.path.dirname(os.path.join(self.cpath, self.datafilename)) if not os.path.samefile(self.cpath, tdir): return jsonify(error={'msg': 'Wrong config data filename'}), 400 cdfname = os.path.join(self.cpath, self.datafilename+'.yml') lockfname = cdfname+'.lock' lock = filelock.FileLock(lockfname) try: with lock.acquire(timeout=10): if not os.path.isfile(cdfname): config_data_file = [] else: with open(cdfname, 'r') as f: config_data_file = yaml.safe_load(f) if type(config_data_file) != list: raise RuntimeError('Config data file is not a list') body = request.get_json() if body is None: return jsonify(error={ 'message': 'No record in request' }), 400 config_data_file.append(body) with open(cdfname, 'w') as f: yaml.safe_dump(config_data_file, stream=f) except Exception as e: return jsonify(error={ 'message': 'Error appending to config data file: %s' % str(e) }), 500 class _CDataLocalDB(object): def __init__(self, cpath, datafilename): self.cpath = cpath self.datafilename = datafilename self.full_path = os.path.join(self.cpath, self.datafilename) def read(self): tdir = os.path.dirname(self.full_path) if not os.path.samefile(self.cpath, tdir): return jsonify(error={'msg': 'Wrong config data filename'}), 400 result = [] if not os.path.isfile(self.full_path+'.db'): return jsonify(result=[]) try: conn = sqlite3.connect(self.full_path+'.db') for row in conn.execute('select * from indicators'): indicator = json.loads(row[2]) indicator['indicator'] = row[0] indicator['type'] = row[1] indicator['_expiration_ts'] = row[3] indicator['_update_ts'] = row[4] result.append(indicator) sleep(0) finally: conn.close() return jsonify(result=result) def create(self): return jsonify(error=dict(message='Method not allowed on localdb files')), 400 def _parse_text_data(self, data): result = [] state = 'INIT' indicator = {} attribute = None for line in iter(data.splitlines()): if len(line) > 0 and line[0] == '#': continue if state == 'INIT': line = line.strip() if len(line) == 0: continue indicator['type'] = line state = 'TYPE' continue if state == 'TYPE': line = line.strip() if len(line) == 0: continue indicator['indicator'] = line state = 'INDICATOR' continue if state == 'INDICATOR': line = line.strip() if len(line) == 0: result.append(indicator) indicator = {} state = 'INIT' continue attribute = line state = 'ATTRIBUTE' continue if state == 'ATTRIBUTE': line = line.strip() indicator[attribute] = line if attribute == 'confidence': if not line.isdigit(): LOG.error('Invalid confidence value: {!r}'.format(line)) return None indicator[attribute] = int(line) elif attribute == 'ttl': if line.isdigit(): indicator[attribute] = int(line) else: indicator[attribute] = 'disabled' state = 'INDICATOR' continue if state == 'INDICATOR': result.append(indicator) state = 'INIT' if state != 'INIT': LOG.error('Error parsing indicators, state: {}'.format(state)) return None if len(result) == 0: return None return result def append(self): tdir = os.path.dirname(self.full_path) if not os.path.samefile(self.cpath, tdir): return jsonify(error={'msg': 'Wrong config data filename'}), 400 record = request.get_json() if record is None: record = self._parse_text_data(request.data) if record is None: return jsonify(error={ 'message': 'No valid record in request' }), 400 indicators = [record] if isinstance(record, list): indicators = record now = int(time.time()*1000) updates = [] for en, entry in enumerate(indicators): indicator = entry.pop('indicator', None) if indicator is None: return jsonify(error={ 'message': 'entry {}: indicator field is missing'.format(en) }), 400 type_ = entry.pop('type', None) if type_ is None: return jsonify(error={ 'message': 'entry {}: type field is missing'.format(en) }), 400 expiration_ts = entry.pop('ttl', None) if expiration_ts is not None: if isinstance(expiration_ts, int): expiration_ts = (expiration_ts*1000+now) else: expiration_ts = 'disabled' updates.append(( indicator, type_, json.dumps(entry), expiration_ts, now, json.dumps([indicator, type_, entry]) )) try: conn = sqlite3.connect(self.full_path+'.db') with conn: conn.execute('''create table if not exists indicators (indicator text, type text, attributes text, expiration_ts integer, update_ts integer, content text, primary key(indicator, type));''') conn.execute('''create index if not exists updateIndex on indicators(update_ts);''') conn.executemany('''insert or replace into indicators (indicator, type, attributes, expiration_ts, update_ts, content) values (?, ?, ?, ?, ?, ?); ''', updates) finally: conn.close() class _CDataUploadOnly(object): def __init__(self, extension, cpath, datafilename): self.extension = extension self.cpath = cpath self.datafilename = datafilename def read(self): fdfname = '{}.{}'.format(self.datafilename, self.extension) os.listdir(self.cpath) if fdfname not in os.listdir(self.cpath): return jsonify(error={ 'message': 'Unknown config data file' }), 400 return jsonify(result='ok') def create(self): tdir = os.path.dirname(os.path.join(self.cpath, self.datafilename)) if not os.path.samefile(self.cpath, tdir): return jsonify(error={'msg': 'Wrong config data filename'}), 400 fdfname = os.path.join(self.cpath, '{}.{}'.format(self.datafilename, self.extension)) if 'file' not in request.files: return jsonify(error={'messsage': 'No file'}), 400 file = request.files['file'] if file.filename == '': return jsonify(error={'message': 'No file'}), 400 tf = NamedTemporaryFile(prefix='mm-extension-upload', delete=False) try: file.save(tf) tf.close() shutil.move(tf.name, fdfname) finally: _safe_remove(tf.name) class _CDataCertificate(_CDataUploadOnly): def __init__(self, cpath, datafilename): super(_CDataCertificate, self).__init__( extension='crt', cpath=cpath, datafilename=datafilename ) class _CDataPrivateKey(_CDataUploadOnly): def __init__(self, cpath, datafilename): super(_CDataPrivateKey, self).__init__( extension='pem', cpath=cpath, datafilename=datafilename ) # API for working with side configs and dynamic data files @BLUEPRINT.route('/', methods=['GET'], read_write=False) def get_config_data(datafilename): cpath = os.path.dirname(os.environ.get('MM_CONFIG')) datafiletype = request.values.get('t', 'yaml') if datafiletype == 'yaml': return _CDataYaml(cpath, datafilename).read() elif datafiletype == 'cert': return _CDataCertificate(cpath, datafilename).read() elif datafiletype == 'pkey': return _CDataPrivateKey(cpath, datafilename).read() elif datafiletype == 'localdb': return _CDataLocalDB(cpath, datafilename).read() return jsonify(error=dict(message='Unknown data file type')), 400 @BLUEPRINT.route('/', methods=['PUT'], read_write=True) def save_config_data(datafilename): cpath = os.path.dirname(os.environ.get('MM_CONFIG')) datafiletype = request.values.get('t', 'yaml') if datafiletype == 'yaml': result = _CDataYaml(cpath, datafilename).create() elif datafiletype == 'cert': result = _CDataCertificate(cpath, datafilename).create() elif datafiletype == 'pkey': result = _CDataPrivateKey(cpath, datafilename).create() elif datafiletype == 'localdb': result = _CDataLocalDB(cpath, datafilename).create() else: return jsonify(error=dict(message='Unknown data file type')), 400 if result is None: hup = request.args.get('h', None) if hup is not None: MMRpcClient.send_cmd(hup, 'hup', {'source': 'minemeld-web'}) return jsonify(result='ok'), 200 return result @BLUEPRINT.route('//append', methods=['POST'], read_write=True) def append_config_data(datafilename): cpath = os.path.dirname(os.environ.get('MM_CONFIG')) datafiletype = request.values.get('t', 'yaml') if datafiletype == 'yaml': result = _CDataYaml(cpath, datafilename).append() elif datafiletype == 'localdb': result = _CDataLocalDB(cpath, datafilename).append() else: return jsonify(error=dict(message='Unknown data file type')), 400 if result is None: hup = request.args.get('h', None) if hup is not None: MMRpcClient.send_cmd(hup, 'hup', {'source': 'minemeld-web'}) return jsonify(result='ok'), 200 return result ================================================ FILE: minemeld/flask/events.py ================================================ import gevent import gevent.queue import redis import werkzeug.local import ujson as json from blinker import signal from flask import g from .logger import LOG STATUS_EVENTS_SUBSCRIBER = None class StatusEventsSubscriber(object): """Subscribes to mm-status events from engine """ def __init__(self, connection_pool): self.SR = redis.StrictRedis(connection_pool=connection_pool) self._g = None self._signal = signal('mm-status') def _retry_wrap(self): while True: try: self._listen() except gevent.GreenletExit: break except: LOG.exception('Exception in event listener') def _listen(self): pubsub = self.SR.pubsub(ignore_subscribe_messages=True) pubsub.psubscribe('mm-engine-status.*') while pubsub.subscribed: response = pubsub.get_message(timeout=30.0) if response is None: continue if not bool(self._signal.receivers): LOG.info('no receivers') continue data = json.loads(response['data']) source = data.pop('source', '') self._signal.send(source, data=data) def start(self): if self._g is not None: return self._g = gevent.spawn(self._retry_wrap) class EventsReceiver(object): def __init__(self): self._signal = signal('mm-status') self._signal.connect(self._signal_receiver) self._q = gevent.queue.Queue() self._iterator = self._generator() def _signal_receiver(self, sender, data): message = { 'source': sender } message.update(data) self._q.put(message) def _generator(self): yield 'data: ok\n\n' while True: try: message = self._q.get(timeout=5.0) yield 'data: '+json.dumps(message)+'\n\n' except gevent.queue.Empty: yield 'data: ping\n\n' continue yield 'data: { "msg": "" }\n\n' def __iter__(self): return self def next(self): result = next(self._iterator) return result def close(self): self._signal.disconnect(self._signal_receiver) def get_EventsGenerator(): result = getattr(g, '_events_generator', None) if result is None: result = EventsReceiver() g._events_generator = result return result EventsGenerator = werkzeug.local.LocalProxy(get_EventsGenerator) def teardown(exception): eg = getattr(g, '_events_generator', None) if eg is not None: g._events_generator.close() g._events_generator = None def init_app(app, redis_url): """Initalize event generator in the app Args: app (object): Flask App redis_url (str): Redis URL for communicating with engine """ global STATUS_EVENTS_SUBSCRIBER redis_cp = redis.ConnectionPool.from_url( redis_url, max_connections=1 ) STATUS_EVENTS_SUBSCRIBER = StatusEventsSubscriber(connection_pool=redis_cp) STATUS_EVENTS_SUBSCRIBER.start() app.teardown_appcontext(teardown) ================================================ FILE: minemeld/flask/extensionsapi.py ================================================ # Copyright 2015-2017 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import os import os.path import shutil import functools import subprocess import uuid import stat from tempfile import NamedTemporaryFile import filelock from gevent import Timeout from gevent.subprocess import Popen from flask import jsonify, request from werkzeug.utils import secure_filename import minemeld.extensions import minemeld.loader from . import config from .jobs import JOBS_MANAGER from .prototypeapi import reset_prototype_paths from .aaa import MMBlueprint from .logger import LOG __all__ = ['BLUEPRINT'] BLUEPRINT = MMBlueprint('extensions', __name__, url_prefix='') DISABLE_NEW_EXTENSIONS = config.get('DISABLE_NEW_EXTENSIONS', False) def _get_extensions(): library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None) if library_directory is None: raise RuntimeError('MINEMELD_LOCAL_LIBRARY_PATH not set') return minemeld.extensions.extensions(library_directory) def _build_activate_args(ext_path): pip_path = config.get('MINEMELD_PIP_PATH', None) if pip_path is None: raise RuntimeError('MINEMELD_PIP_PATH not set') library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None) if library_directory is None: raise RuntimeError('MINEMELD_LOCAL_LIBRARY_PATH not set') constraints_file = os.path.join(library_directory, 'constraints.txt') args = [ pip_path, 'install', '-c', constraints_file ] if ext_path.endswith('.whl'): args.append(ext_path) else: args.append('-e') args.append(ext_path) return args def _build_deactivate_args(extension_name): pip_path = config.get('MINEMELD_PIP_PATH', None) if pip_path is None: raise RuntimeError('MINEMELD_PIP_PATH not set') args = [ pip_path, 'uninstall', '-y', extension_name ] return args def _find_running_job(extension, jobs): for jobid, job in jobs.iteritems(): if job['status'] != 'RUNNING': continue if job['name'] == extension.name: return jobid return None def _load_frozen_paths(): library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None) if library_directory is None: LOG.error('freeze not updated - MINEMELD_LOCAL_LIBRARY_PATH not set') return freeze_path = os.path.join(library_directory, 'freeze.txt') if not os.path.isfile(freeze_path): LOG.info('Extensions frigidaire not found, paths not loaded') return freeze_lock = filelock.FileLock('{}.lock'.format(freeze_path)) with freeze_lock.acquire(timeout=30): with open(freeze_path, 'r') as ff: minemeld.extensions.load_frozen_paths(ff) def _update_freeze_file(): library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None) if library_directory is None: LOG.error('freeze not updated - MINEMELD_LOCAL_LIBRARY_PATH not set') return freeze_path = os.path.join(library_directory, 'freeze.txt') freeze_lock = filelock.FileLock('{}.lock'.format(freeze_path)) with freeze_lock.acquire(timeout=30): with open(freeze_path, 'w+') as ff: frozen = minemeld.extensions.freeze(library_directory) for frozen_ext in frozen: ff.write('{}\n'.format(frozen_ext)) def _extensions_changed(activated_path, deactivated_path, g): if g is not None and g.value == 0: # process was successful if deactivated_path is not None: try: sys.path.remove(deactivated_path) except ValueError: LOG.error('extensions_changed: Error removing {}'.format(deactivated_path)) if activated_path is not None: if activated_path not in sys.path: sys.path.append(activated_path) minemeld.loader.bump_workingset() reset_prototype_paths() _update_freeze_file() def _safe_remove(path, g=None): try: os.remove(path) except: LOG.exception('Exception removing {}'.format(path)) @BLUEPRINT.route('/extensions', methods=['GET'], read_write=False) def list_extensions(): extensions = _get_extensions() jobs = JOBS_MANAGER.get_jobs('extensions') result = [] for e in extensions: edict = e._asdict() rjobid = _find_running_job(e, jobs) if rjobid is not None: edict['running_job'] = rjobid result.append(edict) return jsonify(result=result) @BLUEPRINT.route('/extensions//activate', methods=['POST'], read_write=True) def activate_extension(extension): if DISABLE_NEW_EXTENSIONS: return 'Disabled', 403 params = request.get_json(silent=True) if params is None: return jsonify(error={'message': 'no params'}), 400 ext_path = params.get('path', None) if ext_path is None: return jsonify(error={'message': 'path not specified'}), 400 ext_version = params.get('version', None) if ext_version is None: return jsonify(error={'message': 'version not specified'}), 400 extensions = _get_extensions() for e in extensions: if e.name != extension: continue if e.version != ext_version: continue if e.path == ext_path: break else: return jsonify(error={'message': 'extension not found'}), 400 if not e.installed: return jsonify(error={'message': 'extension not installed'}), 400 if e.activated: return jsonify(error={'messsage': 'extension already activated'}), 400 jobs = JOBS_MANAGER.get_jobs('extensions') if _find_running_job(e, jobs) is not None: return jsonify(error={'message': 'pending job'}), 400 jobid = JOBS_MANAGER.exec_job( job_group='extensions', description='activate {} v{}'.format(e.name, e.version), args=_build_activate_args(e.path), data={ 'name': extension, 'version': ext_version, 'path': ext_path }, callback=functools.partial(_extensions_changed, e.path, None) ) return jsonify(result=jobid) @BLUEPRINT.route('/extensions//deactivate', methods=['GET', 'POST'], read_write=True) def deactivate_extension(extension): if DISABLE_NEW_EXTENSIONS: return 'Disabled', 403 params = request.get_json(silent=True) if params is None: return jsonify(error={'message': 'no params'}), 400 ext_path = params.get('path', None) if ext_path is None: return jsonify(error={'message': 'path not specified'}), 400 ext_version = params.get('version', None) if ext_version is None: return jsonify(error={'message': 'version not specified'}), 400 extensions = _get_extensions() for e in extensions: if e.name != extension: continue if e.version != ext_version: continue if e.path == ext_path: break else: return jsonify(error={'message': 'extension not found'}), 400 if not e.installed: return jsonify(error={'message': 'extension not installed'}), 400 if not e.activated: return jsonify(error={'messsage': 'extension not activated'}), 400 jobs = JOBS_MANAGER.get_jobs('extensions') if _find_running_job(e, jobs) is not None: return jsonify(error={'message': 'pending job'}), 400 jobid = JOBS_MANAGER.exec_job( job_group='extensions', description='deactivate {} v{}'.format(e.name, e.version), args=_build_deactivate_args(extension), data={ 'name': extension, 'version': ext_version, 'path': ext_path }, callback=functools.partial(_extensions_changed, None, e.path) ) return jsonify(result=jobid) @BLUEPRINT.route('/extensions//uninstall', methods=['GET', 'POST'], read_write=True) def uninstall_extension(extension): if DISABLE_NEW_EXTENSIONS: return 'Disabled', 403 params = request.get_json(silent=True) if params is None: return jsonify(error={'message': 'no params'}), 400 ext_path = params.get('path', None) if ext_path is None: return jsonify(error={'message': 'path not specified'}), 400 ext_version = params.get('version', None) if ext_version is None: return jsonify(error={'message': 'version not specified'}), 400 extensions = _get_extensions() for e in extensions: if e.name != extension: continue if e.version != ext_version: continue if e.path == ext_path: break else: return jsonify(error={'message': 'extension not found'}), 400 if not e.installed: return jsonify(error={'message': 'extension not installed'}), 400 if e.activated: return jsonify(error={'messsage': 'extension activated'}), 400 jobs = JOBS_MANAGER.get_jobs('extensions') if _find_running_job(e, jobs) is not None: return jsonify(error={'message': 'pending job'}), 400 if e.path.endswith('.whl'): os.remove(e.path) else: shutil.rmtree(e.path) _extensions_changed(None, None, None) return jsonify(result='ok') @BLUEPRINT.route('/extensions', methods=['POST'], read_write=True) def upload_extension(): if DISABLE_NEW_EXTENSIONS: return 'Disabled', 403 library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None) if library_directory is None: raise RuntimeError('MINEMELD_LOCAL_LIBRARY_PATH not set') if 'file' not in request.files: return jsonify(error={'messsage': 'No file'}), 400 file = request.files['file'] if file.filename == '': return jsonify(error={'message': 'No file'}), 400 if not file or not file.filename.endswith('.whl'): return jsonify(error={'message': 'Incorrect filename'}), 400 filename = secure_filename(file.filename) if filename != file.filename: return jsonify(error={'message': 'Incorrect filename'}), 400 if os.path.basename(filename) != filename: return jsonify(error={'message': 'Incorrect filename'}), 400 toks = filename.split('-', 5) if len(toks) != 4 and len(toks) != 5: return jsonify(error={'message': 'Invalid wheel filename'}), 400 if toks[-1] != 'any.whl' and toks[-1] != 'linux_x86_64.whl': return jsonify(error={'message': 'Invalid wheel platform'}), 400 if not toks[-3].startswith('py2') and not toks[-3].startswith('cp2'): return jsonify(error={'message': 'Invalid wheel python version'}), 400 tf = NamedTemporaryFile(prefix='mm-extension-upload', delete=False) try: file.save(tf) tf.close() metadata = minemeld.extensions.get_metadata_from_wheel(tf.name, filename) if metadata is None: return jsonify(error={'message': 'Invalid MineMeld extension'}), 400 full_filename = os.path.join(library_directory, filename) shutil.move(tf.name, full_filename) except (KeyError, ValueError) as e: LOG.error('Invalid extension: {}'.format(str(e))) return jsonify(error={'message': 'Invalid python wheel'}), 400 finally: _safe_remove(tf.name) return jsonify(result='OK') @BLUEPRINT.route('/extensions/git-refs', methods=['GET'], read_write=False) def get_git_refs(): if DISABLE_NEW_EXTENSIONS: return 'Disabled', 403 git_endpoint = request.values.get('ep', None) if git_endpoint is None: return jsonify(error={'message': 'Missing endpoint'}), 400 if not git_endpoint.endswith('.git'): return jsonify(error={'message': 'Invalid git endpoint'}), 400 git_path = config.get('MINEMELD_GIT_PATH', None) if git_path is None: return jsonify(error={'message': 'MINEMELD_GIT_PATH not set'}), 500 git_args = [git_path, 'ls-remote', '-t', '-h', git_endpoint] git_process = Popen( args=git_args, close_fds=True, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) timeout = Timeout(20.0) timeout.start() try: git_stdout, git_stderr = git_process.communicate() except Timeout: git_process.kill() return jsonify(error={'message': 'Timeout accessing git repo'}), 400 finally: timeout.cancel() if git_process.returncode != 0: LOG.error('Error running {}: {}'.format(git_args, git_stderr)) return jsonify(error={'message': 'Error running git: {}'.format(git_stderr)}), 400 return jsonify(result=[line.rsplit('/', 1)[-1] for line in git_stdout.splitlines()]) @BLUEPRINT.route('/extensions/git-install', methods=['POST'], read_write=True) def install_from_git(): if DISABLE_NEW_EXTENSIONS: return 'Disabled', 403 library_directory = config.get('MINEMELD_LOCAL_LIBRARY_PATH', None) if library_directory is None: raise RuntimeError('MINEMELD_LOCAL_LIBRARY_PATH not set') params = request.get_json(silent=True) if params is None: return jsonify(error={'message': 'no params'}), 400 git_endpoint = params.get('ep', None) if git_endpoint is None: return jsonify(error={'message': 'Missing endpoint'}), 400 if not git_endpoint.endswith('.git'): return jsonify(error={'message': 'Invalid git endpoint'}), 400 git_ref = params.get('ref', None) if git_ref is None: return jsonify(error={'message': 'Missing git ref'}), 400 git_path = config.get('MINEMELD_GIT_PATH', None) if git_path is None: return jsonify(error={'message': 'MINEMELD_GIT_PATH not set'}), 500 install_directory = os.path.join( library_directory, str(uuid.uuid4()) ) job_args = [ "mm-extension-from-git", git_path, git_ref, git_endpoint, install_directory ] jobid = JOBS_MANAGER.exec_job( job_group='extensions-git', description='install from git {} branch {}'.format(git_endpoint, git_ref), args=job_args, data={ 'endpoint': git_endpoint, 'ref': git_ref, 'path': install_directory } ) return jsonify(result=jobid) def init_app(app): _load_frozen_paths() ================================================ FILE: minemeld/flask/feedredis.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import cStringIO import json import re from collections import defaultdict from contextlib import contextmanager import unicodecsv from flask import request, jsonify, Response, stream_with_context from flask.ext.login import current_user from gevent import sleep from netaddr import IPRange, IPNetwork, IPSet, iprange_to_cidrs from .aaa import MMBlueprint from .cbfeed import CbFeedInfo, CbReport from .logger import LOG from .mmrpc import MMMaster from .redisclient import SR __all__ = ['BLUEPRINT'] FEED_INTERVAL = 100 _PROTOCOL_RE = re.compile('^(?:[a-z]+:)*//') _PORT_RE = re.compile('^([a-z0-9\-\.]+)(?:\:[0-9]+)*') _INVALID_TOKEN_RE = re.compile('(?:[^\./+=\?&]+\*[^\./+=\?&]*)|(?:[^\./+=\?&]*\*[^\./+=\?&]+)') _BROAD_PATTERN = re.compile('^(?:\*\.)+[a-zA-Z]+(?::[0-9]+)?$') _IPV4_MASK_RE = re.compile('^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(\\/[0-9]+)?$') _IPV4_RANGE_RE = re.compile( '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}-[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$') BLUEPRINT = MMBlueprint('feeds', __name__, url_prefix='/feeds') def _translate_ip_ranges(indicator, value=None): if value is not None and value['type'] != 'IPv4': return [indicator] try: ip_range = IPRange(*indicator.split('-', 1)) except (AddrFormatError, ValueError, TypeError): return [indicator] return [str(x) if x.size != 1 else str(x.network) for x in ip_range.cidrs()] def _extract_cidrs(indicator): try: parsed = iprange_to_cidrs(*indicator.split('-', 1)) if '-' in indicator else [IPNetwork(indicator)] except (AddrFormatError, ValueError, TypeError): raise Exception('Invalid IP Address in summarization: {!r}'.format(indicator)) return parsed @contextmanager def _buffer(): result = cStringIO.StringIO() try: yield result finally: result.close() def generate_panosurl_feed(feed, start, num, desc, value, **kwargs): zrange = SR.zrange if desc: zrange = SR.zrevrange if num is None: num = (1 << 32) - 1 cstart = start while cstart < (start + num): ilist = zrange(feed, cstart, cstart - 1 + min(start + num - cstart, FEED_INTERVAL)) for i in ilist: i = i.lower() i = _PROTOCOL_RE.sub('', i) withport = i i = _PORT_RE.sub('\g<1>', i) LOG.debug('{} => {}'.format(withport, i)) if withport != i and 'sp' not in kwargs: # port removed, but strip port not enabled # ignore entry continue old = i i = _INVALID_TOKEN_RE.sub('*', i) if old != i: # url changed, invalid tokens detected if 'di' in kwargs: # drop invalid in params, drop entry continue # check if the pattern is now too broad hostname = i if '/' in hostname: hostname, _ = hostname.split('/', 1) if _BROAD_PATTERN.match(hostname) is not None: continue if '/' not in i and not 'nsl' in kwargs: i = i + '/' # add a slash at the end of the hostname # for PAN-OS *.domain.com does not match domain.com # we should provide both # this could generate more than num entries in the egress feed if i.startswith('*.'): yield i[2:] + '\n' yield i + '\n' if len(ilist) < 100: break cstart += 100 def generate_plain_feed(feed, start, num, desc, value, **kwargs): zrange = SR.zrange if desc: zrange = SR.zrevrange if num is None: num = (1 << 32) - 1 translate_ip_ranges = kwargs.pop('translate_ip_ranges', False) should_aggregate = 'sum' in kwargs if should_aggregate: translate_ip_ranges = False temp_set = set() cstart = start while cstart < (start + num): ilist = zrange(feed, cstart, cstart - 1 + min(start + num - cstart, FEED_INTERVAL)) if should_aggregate: for i in ilist: for n in _extract_cidrs(i): temp_set.add(n) else: if translate_ip_ranges: ilist = [xi for i in ilist for xi in _translate_ip_ranges(i)] yield '\n'.join(ilist) + '\n' if len(ilist) < 100: break cstart += 100 if should_aggregate: ip_set = IPSet(temp_set) for cidr in ip_set.iter_cidrs(): yield str(cidr) + '\n' def generate_json_feed(feed, start, num, desc, value, **kwargs): zrange = SR.zrange if desc: zrange = SR.zrevrange if num is None: num = (1 << 32) - 1 translate_ip_ranges = kwargs.pop('translate_ip_ranges', False) if value == 'json': yield '[\n' cstart = start firstelement = True while cstart < (start + num): ilist = zrange(feed, cstart, cstart - 1 + min(start + num - cstart, FEED_INTERVAL)) result = cStringIO.StringIO() for indicator in ilist: v = SR.hget(feed + '.value', indicator) xindicators = [indicator] if translate_ip_ranges and '-' in indicator: xindicators = _translate_ip_ranges(indicator, None if v is None else json.loads(v)) if v is None: v = 'null' for i in xindicators: if value == 'json' and not firstelement: result.write(',\n') if value == 'json-seq': result.write('\x1E') result.write('{"indicator":') result.write(json.dumps(i)) result.write(',"value":') result.write(v) result.write('}') if value == 'json-seq': result.write('\n') firstelement = False yield result.getvalue() result.close() if len(ilist) < 100: break cstart += 100 if value == 'json': yield ']\n' def generate_csv_feed(feed, start, num, desc, value, **kwargs): def _is_atomic_type(fv): return (isinstance(fv, unicode) or isinstance(fv, str) or isinstance(fv, int) or isinstance(fv, bool)) def _format_field_value(fv): if _is_atomic_type(fv): return fv if isinstance(fv, list): ok = True for fve in fv: ok &= _is_atomic_type(fve) if ok: return ','.join(fv) return json.dumps(fv) zrange = SR.zrange if desc: zrange = SR.zrevrange if num is None: num = (1 << 32) - 1 translate_ip_ranges = kwargs.pop('translate_ip_ranges', False) # extract name of fields and column names columns = [] fields = [] for addf in kwargs.pop('f', []): if '|' in addf: fname, cname = addf.rsplit('|', 1) else: fname = addf cname = addf columns.append(cname) fields.append(fname) # if no fields are specified, only indicator is generated if len(fields) == 0: fields = ['indicator'] columns = ['indicator'] # check if header should be generated header = kwargs.pop('h', None) if header is None: header = True else: header = int(header[0]) # check if bom should be generated ubom = kwargs.pop('ubom', None) if ubom is None: ubom = False else: ubom = int(ubom[0]) cstart = start if ubom: LOG.debug('BOM') yield '\xef\xbb\xbf' with _buffer() as current_line: w = unicodecsv.DictWriter( current_line, fieldnames=columns, encoding='utf-8' ) if header: w.writeheader() yield current_line.getvalue() while cstart < (start + num): ilist = zrange(feed, cstart, cstart - 1 + min(start + num - cstart, FEED_INTERVAL)) for indicator in ilist: v = SR.hget(feed + '.value', indicator) v = None if v is None else json.loads(v) xindicators = [indicator] if translate_ip_ranges and '-' in indicator: xindicators = _translate_ip_ranges(indicator, v) for i in xindicators: fieldvalues = {} for f, c in zip(fields, columns): if f == 'indicator': fieldvalues[c] = i continue if v is not None and f in v: fieldvalues[c] = _format_field_value(v[f]) current_line.truncate(0) w.writerow(fieldvalues) yield current_line.getvalue() if len(ilist) < FEED_INTERVAL: break cstart += FEED_INTERVAL def generate_mwg_feed(feed, start, num, desc, value, **kwargs): zrange = SR.zrange if desc: zrange = SR.zrevrange if num is None: num = (1 << 32) - 1 translate_ip_ranges = kwargs.pop('translate_ip_ranges', False) type_ = kwargs.get('t', None) if type_ is None: type_ = 'string' else: type_ = type_[0] translate_ip_ranges |= type_ == 'ip' yield 'type={}\n'.format(type_) cstart = start while cstart < (start + num): ilist = zrange(feed, cstart, cstart - 1 + min(start + num - cstart, FEED_INTERVAL)) for indicator in ilist: v = SR.hget(feed + '.value', indicator) v = None if v is None else json.loads(v) xindicators = [indicator] if translate_ip_ranges and '-' in indicator: xindicators = _translate_ip_ranges(indicator, v) sources = 'from minemeld' if v is not None: sources = v.get('sources', 'from minemeld') if isinstance(sources, list): sources = ','.join(sources) for i in xindicators: yield '"{}" "{}"\n'.format( i.replace('"', '\\"'), sources.replace('"', '\\"') ) if len(ilist) < 100: break cstart += 100 # This formatter implements BlueCoat custom URL format as described at # https://www.bluecoat.com/documents/download/a366dc73-d455-4859-b92a-c96bd034cb4c/f849f1e3-a906-4ee8-924e-a2061dfe3cdf # It expects the value 'bc_category' in the indicator. The value can be either a single string or a list of strings. # Optional feed arguments: # ca : Indicator's attribute that hosts the BlueCoat category. Defaults to 'bc_category' # cd : Default BlueCoat category for indicators that do not have 'catattr'. This argument can appear multiple # times and it will be handled as a list of categories the indicator belongs to. If not present then # indicators without 'catattr' will be discarded. def generate_bluecoat_feed(feed, start, num, desc, value, **kwargs): zrange = SR.zrange ilist = zrange(feed, 0, (1 << 32) - 1) bc_dict = defaultdict(list) flag_category_default = kwargs.get('cd', None) flag_category_attr = kwargs.get('ca', ['bc_category'])[0] for i in ilist: sleep(0) v = SR.hget(feed + '.value', i) v = None if v is None else json.loads(v) i = i.lower() i = _PROTOCOL_RE.sub('', i) i = _INVALID_TOKEN_RE.sub('*', i) if v is None: if flag_category_default is None: continue else: bc_cat_list = flag_category_default else: bc_cat_attr = v.get(flag_category_attr, None) if isinstance(bc_cat_attr, list): bc_cat_list = bc_cat_attr elif isinstance(bc_cat_attr, basestring): bc_cat_list = [bc_cat_attr] elif flag_category_default is not None: bc_cat_list = flag_category_default else: continue for bc_cat in bc_cat_list: bc_dict[bc_cat].append(i) for key, value in bc_dict.iteritems(): yield 'define category {}\n'.format(key) for ind in value: yield ind + '\n' yield 'end\n' def generate_carbon_black(feed, start, num, desc, value, **kwargs): zrange = SR.zrange ilist = zrange(feed, 0, (1 << 32) - 1) mm_to_cb = {"IPv4": "ipv4", "domain": "dns", "md5": "md5"} ind_by_type = {"dns": [], "md5": []} # Let's stream the information as soon as we have it yield "{\n\"feedinfo\": {\n" cb_feed_info = CbFeedInfo(name=feed) for cb_info_parts in cb_feed_info.iterate(): yield " " + cb_info_parts yield "\n},\n\"reports\": [{" report_args = dict() report_args["id"] = feed + "_report" report_title = kwargs.get('rt', ["MieneMeld Generated Report"]) if report_title is not None: report_title = report_title[0] report_args["title"] = report_title report_score = kwargs.get('rs', None) if report_score is not None: try: report_score = int(report_score[0]) except ValueError: report_score = None report_args["score"] = report_score cb_report = CbReport(**report_args) for cb_report_parts in cb_report.iterate(): yield " " + cb_report_parts yield ", \"iocs\": {" yield " \"ipv4\": [" # Loop though all indicators # Only indicators of type IPv4, domain and md5 can be exported to Carbon Black ipv4_line = None for i in ilist: sleep(0) v = SR.hget(feed + '.value', i) v = None if v is None else json.loads(v) if v is None: continue v_type = v.get("type", None) if v_type not in mm_to_cb: continue if v_type in ("domain", "md5"): ind_by_type[mm_to_cb[v_type]].append(i.lower()) continue # Carbon Black do not support IPv4 networks not ranges. We must expand them. ip_range = None if _IPV4_MASK_RE.match(i): ip_range = IPSet(IPNetwork(i)) elif _IPV4_RANGE_RE.match(i): range_parts = i.split("-") ip_range = IPRange(range_parts[0], range_parts[1]) for ip_addr in ip_range: if ipv4_line is not None: yield ipv4_line + "," ipv4_line = "\"{}\"".format(str(ip_addr)) yield ("" if ipv4_line is None else ipv4_line) + "]," yield "\"dns\": {},".format(json.dumps(ind_by_type["dns"])) yield "\"md5\": {}".format(json.dumps(ind_by_type["md5"])) yield "}}]}" _FEED_FORMATS = { 'json': { 'formatter': generate_json_feed, 'mimetype': 'application/json' }, 'json-seq': { 'formatter': generate_json_feed, 'mimetype': 'application/json-seq' }, 'panosurl': { 'formatter': generate_panosurl_feed, 'mimetype': 'text/plain' }, 'mwg': { 'formatter': generate_mwg_feed, 'mimetype': 'text/plain' }, 'bluecoat': { 'formatter': generate_bluecoat_feed, 'mimetype': 'text/plain' }, 'carbonblack': { 'formatter': generate_carbon_black, 'mimetype': 'application/json' }, 'csv': { 'formatter': generate_csv_feed, 'mimetype': 'text/csv' } } @BLUEPRINT.route('/', methods=['GET'], feeds=True, read_write=False) def get_feed_content(feed): if not current_user.check_feed(feed): return 'Unauthorized', 401 # check if feed exists status = MMMaster.status() tr = status.get('result', None) if tr is None: LOG.error("Error retrieving status from MMMaster: {!r}".format(status.get('error', 'error'))) return 'Internal error', 500 nname = 'mbus:slave:' + feed if nname not in tr: return 'Unknown feed', 404 nclass = tr[nname].get('class', None) if nclass != 'minemeld.ft.redis.RedisSet': return 'Unknown feed', 404 start = request.values.get('s') if start is None: start = 0 try: start = int(start) if start < 0: raise ValueError() except ValueError: LOG.error("Invalid request, s not a non-negative integer: %s", start) return 's should be a positive integer', 400 num = request.values.get('n') if num is not None: try: num = int(num) if num <= 0: raise ValueError() except ValueError: LOG.error("Invalid request, n not a positive integer: %s", num) return 'n should be a positive integer', 400 else: num = None desc = request.values.get('d') desc = (False if desc is None else True) value = request.values.get('v') if value is not None and value not in _FEED_FORMATS: return 'unknown format', 400 kwargs = {} kwargs['translate_ip_ranges'] = int(request.values.get('tr', 0)) # generate IP ranges # move to kwargs all the additional parameters, pop the predefined kwargs.update(request.values.to_dict(flat=False)) kwargs.pop('s', None) kwargs.pop('n', None) kwargs.pop('d', None) kwargs.pop('v', None) kwargs.pop('tr', None) mimetype = 'text/plain' formatter = generate_plain_feed if value in _FEED_FORMATS: formatter = _FEED_FORMATS[value]['formatter'] mimetype = _FEED_FORMATS[value]['mimetype'] return Response( stream_with_context( formatter(feed, start, num, desc, value, **kwargs) ), mimetype=mimetype ) ================================================ FILE: minemeld/flask/jobs.py ================================================ # Copyright 2017 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import uuid import tempfile import subprocess import shutil import json import time import signal from collections import namedtuple, defaultdict import redis import psutil import werkzeug.local import gevent from gevent.subprocess import Popen from flask import g from . import REDIS_URL from . import config from .logger import LOG __all__ = ['init_app', 'JOBS_MANAGER'] REDIS_CP = redis.ConnectionPool.from_url( REDIS_URL, max_connections=int(os.environ.get('REDIS_MAX_CONNECTIONS', 5)) ) REDIS_JOBS_GROUP_PREFIX = 'mm-jobs-{}' _Job = namedtuple('_Job', field_names=['glet', 'timeout_glet']) class JobsManager(object): def __init__(self, connection_pool): self.SR = redis.StrictRedis(connection_pool=connection_pool) self.running_jobs = defaultdict(dict) def _safe_rmtree(self, path): shutil.rmtree(path, ignore_errors=True) def _safe_remove(self, path): try: os.remove(path) except: pass def _get_job_status(self, jobpid, jobhash): try: jobprocess = psutil.Process(pid=jobpid) except psutil.NoSuchProcess: return { 'status': 'DONE', 'returncode': None } if hash(jobprocess) != jobhash: return { 'status': 'DONE', 'returncode': None } return { 'status': 'RUNNING' } def _collect_job(self, jobdata): if 'collected' in jobdata: return tempdir = jobdata.get('cwd', None) if tempdir is not None: self._safe_rmtree(tempdir) jobdata['collected'] = True def _job_monitor_glet(self, job_group, jobid, description, args, data): jobname = (REDIS_JOBS_GROUP_PREFIX+'-{}').format(job_group, jobid) joblogfile = os.path.join( config.get('MINEMELD_LOG_DIRECTORY_PATH', '/tmp'), '{}.log'.format(jobname) ) jobtempdir = tempfile.mkdtemp(prefix=jobname) LOG.info('Executing job {} - {} cwd: {} logfile: {}'.format(jobname, args, jobtempdir, joblogfile)) try: with open(joblogfile, 'w+') as logfile: jobprocess = Popen( args=args, close_fds=True, cwd=jobtempdir, shell=False, stdout=logfile, stderr=subprocess.STDOUT ) except OSError: self._safe_remove(joblogfile) self._safe_rmtree(jobtempdir) LOG.exception('Error starting job {}'.format(jobname)) return jobpsproc = psutil.Process(pid=jobprocess.pid) jobdata = data if jobdata is None: jobdata = {} jobdata['create_time'] = int(time.time()*1000) jobdata['description'] = description jobdata['job_id'] = jobid jobdata['pid'] = jobpsproc.pid jobdata['hash'] = hash(jobpsproc) jobdata['logfile'] = joblogfile jobdata['cwd'] = jobtempdir jobdata['status'] = 'RUNNING' self.SR.hset( REDIS_JOBS_GROUP_PREFIX.format(job_group), jobid, json.dumps(jobdata) ) jobprocess.wait() if jobprocess.returncode != 0: jobdata['status'] = 'ERROR' else: jobdata['status'] = 'DONE' jobdata['returncode'] = jobprocess.returncode jobdata['end_time'] = int(time.time()*1000) self._collect_job(jobdata) self.SR.hset( REDIS_JOBS_GROUP_PREFIX.format(job_group), jobid, json.dumps(jobdata) ) job = self.running_jobs[job_group].pop(jobid, None) if job is not None and job.timeout_glet is not None: job.timeout_glet.kill() return jobprocess.returncode def _job_timeout_glet(self, job_group, jobid, timeout): gevent.sleep(timeout) prefix = REDIS_JOBS_GROUP_PREFIX.format(job_group) jobdata = self.SR.hget(prefix, jobid) if jobdata is None: return jobdata = json.loads(jobdata) status = jobdata.get('status', None) if status != 'RUNNING': LOG.info('Timeout for job {}-{} triggered but status not running'.format(prefix, jobid)) return pid = jobdata.get('pid', None) if pid is None: LOG.error('Timeout for job {}-{} triggered but no pid available'.format(prefix, jobid)) return LOG.error('Timeout for job {}-{} triggered, sending TERM signal'.format(prefix, jobid)) os.kill(pid, signal.SIGTERM) def delete_job(self, job_group, jobid): prefix = REDIS_JOBS_GROUP_PREFIX.format(job_group) jobdata = self.SR.hget(prefix, jobid) if jobdata is None: return jobdata = json.loads(jobdata) logfile = jobdata.get('logfile', None) if logfile is not None: self._safe_remove(logfile) self._collect_job(jobdata) self.SR.hdel(prefix, jobid) def get_jobs(self, job_group): prefix = REDIS_JOBS_GROUP_PREFIX.format(job_group) result = {} jobs_map = self.SR.hgetall(prefix) for jobid, jobdata in jobs_map.iteritems(): try: jobdata = json.loads(jobdata) if jobdata['status'] == 'RUNNING': jobpid = jobdata['pid'] job_status = self._get_job_status(jobpid, jobdata['hash']) jobdata.update(job_status) result[jobid] = jobdata except (ValueError, KeyError, psutil.ZombieProcess, psutil.AccessDenied): LOG.error('Invalid job value - deleting job {}::{}'.format(job_group, jobid)) self.delete_job(job_group, jobid) continue if jobdata['status'] == 'DONE' and 'collected' not in jobdata: if jobid not in self.running_jobs[job_group]: self._collect_job(jobdata) self.SR.hset(job_group, jobid, json.dumps(jobdata)) return result def exec_job(self, job_group, description, args, data=None, callback=None, timeout=None): jobid = str(uuid.uuid4()) glet = gevent.spawn( self._job_monitor_glet, job_group, jobid, description, args, data ) if callback is not None: glet.link(callback) timeout_glet = None if timeout is not None: timeout_glet = gevent.spawn(self._job_timeout_glet, job_group, jobid, timeout) self.running_jobs[job_group][jobid] = _Job(glet=glet, timeout_glet=timeout_glet) return jobid def get_JobsManager(): jobsmgr = getattr(g, '_jobs_manager', None) if jobsmgr is None: jobsmgr = JobsManager(connection_pool=REDIS_CP) g._jobs_manager = jobsmgr return jobsmgr def teardown(exception): jobsmgr = getattr(g, '_jobs_manager', None) if jobsmgr is not None: g._jobs_manager = None LOG.info( 'redis connection pool: in use: {} available: {}'.format( len(REDIS_CP._in_use_connections), len(REDIS_CP._available_connections) ) ) JOBS_MANAGER = werkzeug.local.LocalProxy(get_JobsManager) def init_app(app): app.teardown_appcontext(teardown) ================================================ FILE: minemeld/flask/jobsapi.py ================================================ # Copyright 2015-2017 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import os.path from flask import send_from_directory, jsonify from .jobs import JOBS_MANAGER from .aaa import MMBlueprint from .logger import LOG __all__ = ['BLUEPRINT'] BLUEPRINT = MMBlueprint('jobs', __name__, url_prefix='/jobs') @BLUEPRINT.route('/', methods=['GET'], read_write=False) def get_jobs(job_group): jobs = JOBS_MANAGER.get_jobs(job_group) return jsonify(result=jobs) @BLUEPRINT.route('//', methods=['GET'], read_write=False) def get_job(job_group, jobid): jobs = JOBS_MANAGER.get_jobs(job_group) if jobid not in jobs: return jsonify(error={'message': 'job unknown'}), 400 return jsonify(result=jobs[jobid]) @BLUEPRINT.route('///log', methods=['GET'], read_write=False) def get_job_log(job_group, jobid): jobs = JOBS_MANAGER.get_jobs(job_group) if jobid not in jobs: return jsonify(error={'message': 'job unknown'}), 400 job = jobs[jobid] return send_from_directory( os.path.dirname(job['logfile']), os.path.basename(job['logfile']), as_attachment=True ) ================================================ FILE: minemeld/flask/logger.py ================================================ import logging import json from flask import current_app # [2017-01-16 20:32:07 +0000] [15997] [INFO] LOG_FORMAT = '[%(asctime)s] [%(process)d] [%(levelname)s] %(message)s' LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S %Z' class MMLogger(object): def __init__(self): self.system_logger = logging.getLogger('minemeld') self._init_logger(self.system_logger) self.system_logger.info('MMLogger started') def init_app(self, app): del app.logger.handlers[:] self._init_logger(app.logger) def _init_logger(self, logger): logger.propagate = False logger.setLevel(logging.DEBUG) handler = logging.StreamHandler() handler.setFormatter(logging.Formatter( fmt=LOG_FORMAT, datefmt=LOG_DATE_FORMAT )) logger.addHandler(handler) def debug(self, *args, **kwargs): if current_app: current_app.logger.debug(*args, **kwargs) else: self.system_logger.debug(*args, **kwargs) def info(self, *args, **kwargs): if current_app: current_app.logger.info(*args, **kwargs) else: self.system_logger.info(*args, **kwargs) def warning(self, *args, **kwargs): if current_app: current_app.logger.warning(*args, **kwargs) else: self.system_logger.warning(*args, **kwargs) def error(self, *args, **kwargs): if current_app: current_app.logger.error(*args, **kwargs) else: self.system_logger.error(*args, **kwargs) def critical(self, *args, **kwargs): if current_app: current_app.logger.critical(*args, **kwargs) else: self.system_logger.critical(*args, **kwargs) def exception(self, *args, **kwargs): if current_app: current_app.logger.exception(*args, **kwargs) else: self.system_logger.exception(*args, **kwargs) def audit(self, user_id, action_name, params, msg=None): audit_params = dict(user=user_id, action=action_name, params=params, msg=msg) self.info('AUDIT - {}'.format(json.dumps(audit_params))) LOG = MMLogger() ================================================ FILE: minemeld/flask/loginapi.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from flask import request, jsonify import flask.ext.login from . import aaa from .logger import LOG __all__ = ['BLUEPRINT'] BLUEPRINT = aaa.MMBlueprint('login', __name__, url_prefix='') @BLUEPRINT.route('/login', methods=['GET', 'POST'], login_required=False, read_write=False) def login(): username = request.values.get('u') if username is None: return jsonify(error='Missing username'), 400 password = request.values.get('p') if password is None: return jsonify(error='Missing password'), 400 user = aaa.check_admin_user(username, password) if user is None: return jsonify(error="Wrong credentials"), 401 flask.ext.login.login_user(user) return 'OK' @BLUEPRINT.route('/logout', methods=['GET'], login_required=False, read_write=False) def logout(): flask.ext.login.logout_user() return 'OK' ================================================ FILE: minemeld/flask/logsapi.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from flask import send_from_directory, jsonify from . import config from .aaa import MMBlueprint from .logger import LOG __all__ = ['BLUEPRINT'] BLUEPRINT = MMBlueprint('logs', __name__, url_prefix='/logs') @BLUEPRINT.route('/minemeld-engine.log', methods=['GET'], read_write=True) def get_minemeld_engine_log(): log_directory = config.get('MINEMELD_LOG_DIRECTORY_PATH', None) if log_directory is None: return jsonify(error={'message': 'LOG_DIRECTORY not set'}), 500 return send_from_directory(log_directory, 'minemeld-engine.log', as_attachment=True) @BLUEPRINT.route('/minemeld-web.log', methods=['GET'], read_write=True) def get_minemeld_web_log(): log_directory = config.get('MINEMELD_LOG_DIRECTORY_PATH', None) if log_directory is None: return jsonify(error={'message': 'LOG_DIRECTORY not set'}), 500 return send_from_directory(log_directory, 'minemeld-web.log', as_attachment=True) ================================================ FILE: minemeld/flask/main.py ================================================ from . import create_app app = create_app() ================================================ FILE: minemeld/flask/metricsapi.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import os.path import hashlib import rrdtool from flask import request, jsonify import minemeld.collectd from . import config from .aaa import MMBlueprint from .logger import LOG __all__ = ['BLUEPRINT'] RRD_PATH = config.get('RRD_PATH', '/var/lib/collectd/rrd/minemeld/') RRD_SOCKET_PATH = config.get('RRD_SOCKET_PATH', '/var/run/collectd.sock') ALLOWED_CF = ['MAX', 'MIN', 'AVERAGE'] BLUEPRINT = MMBlueprint('metrics', __name__, url_prefix='/metrics') def _list_metrics(prefix=None): result = os.listdir(RRD_PATH) if prefix is not None: result = [m for m in result if m.startswith(prefix)] return result def _fetch_metric(cc, metric, type_=None, cf='MAX', dt=86400, r=1800): dirname = os.path.join(RRD_PATH, metric) if type_ is None: rrdname = os.listdir(dirname)[0] type_ = rrdname.replace('.rrd', '') else: rrdname = type_+'.rrd' if rrdname not in os.listdir(dirname): raise RuntimeError('Unknown metric type') cc.flush(identifier='minemeld/%s/%s' % (metric, type_)) (start, end, step), metrics, data = rrdtool.fetch( str(os.path.join(dirname, rrdname)), cf, '--start', '-%d' % dt, '--resolution', '%d' % r ) result = [] if type_ != 'minemeld_delta': curts = start for v in data: result.append([curts, v[0]]) curts += step else: curts = start+step ov = data[0][0] for v in data[1:]: cv = v[0] if cv is not None and ov is not None: if cv >= ov: cv = cv - ov result.append([curts, cv]) ov = v[0] curts += step return result @BLUEPRINT.route('/', read_write=False) def get_metrics(): return jsonify(result=_list_metrics()) @BLUEPRINT.route('/minemeld/', read_write=False) def get_node_type_metrics(nodetype): cf = str(request.args.get('cf', 'MAX')).upper() if cf not in ALLOWED_CF: return jsonify(error={'message': 'Unknown function'}), 400 try: dt = int(request.args.get('dt', '86400')) except ValueError: return jsonify(error={'message': 'Invalid delta'}), 400 if dt < 0: return jsonify(error={'message': 'Invalid delta'}), 400 try: resolution = int(request.args.get('r', '1800')) except ValueError: return jsonify(error={'message': 'Invalid resolution'}), 400 if resolution < 0: return jsonify(error={'message': 'Invalid resolution'}), 400 type_ = request.args.get('t', None) metrics = _list_metrics(prefix='minemeld.'+nodetype+'.') cc = minemeld.collectd.CollectdClient(RRD_SOCKET_PATH) result = [] for m in metrics: v = _fetch_metric(cc, m, cf=cf, dt=dt, r=resolution, type_=type_) _, _, mname = m.split('.', 2) result.append({ 'metric': mname, 'values': v }) return jsonify(result=result) @BLUEPRINT.route('/minemeld', read_write=False) def get_global_metrics(): cf = str(request.args.get('cf', 'MAX')).upper() if cf not in ALLOWED_CF: return jsonify(error={'message': 'Unknown function'}), 400 try: dt = int(request.args.get('dt', '86400')) except ValueError: return jsonify(error={'message': 'Invalid delta'}), 400 if dt < 0: return jsonify(error={'message': 'Invalid delta'}), 400 try: resolution = int(request.args.get('r', '1800')) except ValueError: return jsonify(error={'message': 'Invalid resolution'}), 400 if resolution < 0: return jsonify(error={'message': 'Invalid resolution'}), 400 type_ = request.args.get('t', None) metrics = _list_metrics(prefix='minemeld.') metrics = [m for m in metrics if 'minemeld.sources' not in m] metrics = [m for m in metrics if 'minemeld.outputs' not in m] metrics = [m for m in metrics if 'minemeld.transits' not in m] cc = minemeld.collectd.CollectdClient(RRD_SOCKET_PATH) result = [] for m in metrics: v = _fetch_metric(cc, m, cf=cf, dt=dt, r=resolution, type_=type_) _, mname = m.split('.', 1) result.append({ 'metric': mname, 'values': v }) return jsonify(result=result) @BLUEPRINT.route('/', read_write=False) def get_node_metrics(node): cf = str(request.args.get('cf', 'MAX')).upper() if cf not in ALLOWED_CF: return jsonify(error={'message': 'Unknown function'}), 400 try: dt = int(request.args.get('dt', '86400')) except ValueError: return jsonify(error={'message': 'Invalid delta'}), 400 if dt < 0: return jsonify(error={'message': 'Invalid delta'}), 400 try: resolution = int(request.args.get('r', '1800')) except ValueError: return jsonify(error={'message': 'Invalid resolution'}), 400 if resolution < 0: return jsonify(error={'message': 'Invalid resolution'}), 400 type_ = request.args.get('t', None) node = hashlib.md5(node).hexdigest()[:10] metrics = _list_metrics(prefix=node+'.') cc = minemeld.collectd.CollectdClient(RRD_SOCKET_PATH) result = [] for m in metrics: v = _fetch_metric(cc, m, cf=cf, dt=dt, r=resolution, type_=type_) _, mname = m.split('.', 1) result.append({ 'metric': mname, 'values': v }) return jsonify(result=result) @BLUEPRINT.route('//', methods=['GET'], read_write=False) def get_metric(node, metric): cf = str(request.args.get('cf', 'MAX')).upper() if cf not in ALLOWED_CF: return jsonify(error={'message': 'Unknown function'}), 400 try: dt = int(request.args.get('dt', '86400')) except ValueError: return jsonify(error={'message': 'Invalid delta'}), 400 if dt < 0: return jsonify(error={'message': 'Invalid delta'}), 400 try: resolution = int(request.args.get('r', '1800')) except ValueError: return jsonify(error={'message': 'Invalid resolution'}), 400 if resolution < 0: return jsonify(error={'message': 'Invalid resolution'}), 400 type_ = request.args.get('t', 'minemeld_counter') node = hashlib.md5(node).hexdigest()[:10] metric = node+'.'+metric if metric not in _list_metrics(): return jsonify(error={'message': 'Unknown metric'}), 404 cc = minemeld.collectd.CollectdClient(RRD_SOCKET_PATH) try: result = _fetch_metric(cc, metric, type_=type_, cf=cf, dt=dt, r=resolution) except RuntimeError as e: return jsonify(error={'message': str(e)}), 400 return jsonify(result=result) ================================================ FILE: minemeld/flask/mmrpc.py ================================================ import json import gevent import gevent.event import gevent.queue import werkzeug.local from flask import g import minemeld.comm from minemeld.mgmtbus import MGMTBUS_PREFIX, MGMTBUS_MASTER from . import config from .logger import LOG __all__ = ['init_app', 'MMMaster', 'MMRpcClient'] class _MMMasterConnection(object): def __init__(self): self.comm = None tconfig = config.get('MGMTBUS', {}) self.comm_class = tconfig.get('class', 'ZMQRedis') self.comm_config = tconfig.get('config', {}) def _open_channel(self): if self.comm is not None: return self.comm = minemeld.comm.factory( self.comm_class, self.comm_config ) self.comm.start() def _send_cmd(self, method, params={}): self._open_channel() return self.comm.send_rpc( MGMTBUS_MASTER, method, params, timeout=10.0 ) def status(self): return self._send_cmd('status') def stop(self): if self.comm is not None: self.comm.stop() self.comm = None class _MMRpcClient(object): def __init__(self): self.comm = None tconfig = config.get('MGMTBUS', {}) self.comm_class = tconfig.get('class', 'ZMQRedis') self.comm_config = tconfig.get('config', {}) def _open_channel(self): if self.comm is not None: return self.comm = minemeld.comm.factory( self.comm_class, self.comm_config ) self.comm.start() def send_raw_cmd(self, target, method, params={}, timeout=10): self._open_channel() return self.comm.send_rpc(target, method, params, timeout=timeout) def send_cmd(self, target, method, params={}, timeout=10): target = '{}directslave:{}'.format(MGMTBUS_PREFIX, target) return self.send_raw_cmd(target, method, params=params, timeout=timeout) def stop(self): if self.comm is not None: self.comm.stop() self.comm = None def get_mmmaster(): r = getattr(g, 'MMMaster', None) if r is None: r = _MMMasterConnection() g.MMMaster = r return r MMMaster = werkzeug.LocalProxy(get_mmmaster) # pylint:disable=E1101 def get_mmrpcclient(): r = getattr(g, 'MMRpcClient', None) if r is None: r = _MMRpcClient() g.MMRpcClient = r return r MMRpcClient = werkzeug.LocalProxy(get_mmrpcclient) # pylint:disable=E1101 def teardown(exception): r = getattr(g, 'MMMaster', None) if r is not None: g.MMMaster.stop() g.MMMaster = None r = getattr(g, 'MMRpcClient', None) if r is not None: g.MMRpcClient.stop() g.MMRpcClient = None def init_app(app): app.teardown_appcontext(teardown) ================================================ FILE: minemeld/flask/prototypeapi.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import os import os.path import json import yaml import filelock from flask import jsonify, request import minemeld.loader from . import config from .utils import running_config, committed_config from .aaa import MMBlueprint from .logger import LOG __all__ = ['BLUEPRINT'] PROTOTYPE_ENV = 'MINEMELD_PROTOTYPE_PATH' LOCAL_PROTOTYPE_PATH = 'MINEMELD_LOCAL_PROTOTYPE_PATH' BLUEPRINT = MMBlueprint('prototype', __name__, url_prefix='') PROTOTYPE_PATHS = None def _prototype_paths(): global PROTOTYPE_PATHS if PROTOTYPE_PATHS is not None: return PROTOTYPE_PATHS paths = config.get(PROTOTYPE_ENV, None) if paths is None: raise RuntimeError('{} environment variable not set'.format(PROTOTYPE_ENV)) paths = paths.split(':') prototype_eps = minemeld.loader.map(minemeld.loader.MM_PROTOTYPES_ENTRYPOINT) for pname, mmep in prototype_eps.iteritems(): if not mmep.loadable: LOG.info('Prototype entry point {} not loadable, ignored'.format(pname)) continue try: # even if old dist is no longer available, old module could be cached cmodule = sys.modules.get(mmep.ep.module_name, None) cmodule_path = getattr(cmodule, '__path__', None) if cmodule is not None and cmodule_path is not None: if not cmodule_path[0].startswith(mmep.ep.dist.location): LOG.info('Invalidating cache for {}'.format(mmep.ep.module_name)) sys.modules.pop(mmep.ep.module_name) ep = mmep.ep.load() # we add prototype paths in front, to let extensions override default protos paths.insert(0, ep()) except: LOG.exception('Exception loading paths from {}'.format(pname)) PROTOTYPE_PATHS = paths return paths def _local_library_path(prototypename): toks = prototypename.split('.', 1) if len(toks) != 2: raise ValueError('bad prototype name') library, prototype = toks if os.path.basename(library) != library: raise ValueError('bad library name, nice try') if library != 'minemeldlocal': raise ValueError('invalid library') library_filename = library+'.yml' local_path = config.get(LOCAL_PROTOTYPE_PATH) if local_path is None: paths = os.getenv(PROTOTYPE_ENV, None) if paths is None: raise RuntimeError( '%s environment variable not set' % (PROTOTYPE_ENV) ) paths = paths.split(':') for p in paths: if '/local/' in p: local_path = p break if local_path is None: raise RuntimeError( 'No local path in %s' % PROTOTYPE_ENV ) library_path = os.path.join(local_path, library_filename) return library_path, prototype @BLUEPRINT.route('/prototype', methods=['GET'], read_write=False) def list_prototypes(): paths = _prototype_paths() prototypes = {} for p in paths: try: for plibrary in os.listdir(p): if not plibrary.endswith('.yml'): continue plibraryname, _ = plibrary.rsplit('.', 1) with open(os.path.join(p, plibrary), 'r') as f: pcontents = yaml.safe_load(f) if plibraryname not in prototypes: prototypes[plibraryname] = pcontents continue # oldest has precedence newprotos = pcontents.get('prototypes', {}) currprotos = prototypes[plibraryname].get('prototypes', {}) newprotos.update(currprotos) prototypes[plibraryname]['prototypes'] = newprotos except: LOG.exception('Error loading libraries from %s', p) return jsonify(result=prototypes) @BLUEPRINT.route('/prototype/', methods=['GET'], read_write=False) def get_prototype(prototypename): toks = prototypename.split('.', 1) if len(toks) != 2: return jsonify(error={'message': 'bad prototype name'}), 400 library, prototype = toks if os.path.basename(library) != library: return jsonify(error={'message': 'bad library name, nice try'}), 400 library_filename = library+'.yml' paths = _prototype_paths() for path in paths: full_library_name = os.path.join(path, library_filename) if not os.path.isfile(full_library_name): continue with open(full_library_name, 'r') as f: library_contents = yaml.safe_load(f) prototypes = library_contents.get('prototypes', None) if prototypes is None: continue if prototype not in prototypes: continue curr_prototype = prototypes[prototype] result = { 'class': curr_prototype['class'], 'developmentStatus': None, 'config': None, 'nodeType': None, 'description': None, 'indicatorTypes': None, 'tags': None } if 'config' in curr_prototype: result['config'] = yaml.dump( curr_prototype['config'], indent=4, default_flow_style=False ) if 'development_status' in curr_prototype: result['developmentStatus'] = curr_prototype['development_status'] if 'node_type' in curr_prototype: result['nodeType'] = curr_prototype['node_type'] if 'description' in curr_prototype: result['description'] = curr_prototype['description'] if 'indicator_types' in curr_prototype: result['indicatorTypes'] = curr_prototype['indicator_types'] if 'tags' in curr_prototype: result['tags'] = curr_prototype['tags'] return jsonify(result=result), 200 @BLUEPRINT.route('/prototype/', methods=['POST'], read_write=True) def add_local_prototype(prototypename): AUTHOR_ = 'minemeld-web' DESCRIPTION_ = 'Local prototype library managed via MineMeld WebUI' try: library_path, prototype = _local_library_path(prototypename) except ValueError as e: return jsonify(error={'message': str(e)}), 400 lock = filelock.FileLock('{}.lock'.format(library_path)) with lock.acquire(timeout=10): if os.path.isfile(library_path): with open(library_path, 'r') as f: library_contents = yaml.safe_load(f) if not isinstance(library_contents, dict): library_contents = {} if 'description' not in library_contents: library_contents['description'] = DESCRIPTION_ if 'prototypes' not in library_contents: library_contents['prototypes'] = {} if 'author' not in library_contents: library_contents['author'] = AUTHOR_ else: library_contents = { 'author': AUTHOR_, 'description': DESCRIPTION_, 'prototypes': {} } try: incoming_prototype = request.get_json() except Exception as e: return jsonify(error={'message': str(e)}), 400 new_prototype = { 'class': incoming_prototype['class'], } if 'config' in incoming_prototype: try: new_prototype['config'] = yaml.safe_load( incoming_prototype['config'] ) except Exception as e: return jsonify(error={'message': 'invalid YAML in config'}), 400 if 'developmentStatus' in incoming_prototype: new_prototype['development_status'] = \ incoming_prototype['developmentStatus'] if 'nodeType' in incoming_prototype: new_prototype['node_type'] = incoming_prototype['nodeType'] if 'description' in incoming_prototype: new_prototype['description'] = incoming_prototype['description'] if 'indicatorTypes' in incoming_prototype: new_prototype['indicator_types'] = incoming_prototype['indicatorTypes'] if 'tags' in incoming_prototype: new_prototype['tags'] = incoming_prototype['tags'] library_contents['prototypes'][prototype] = new_prototype with open(library_path, 'w') as f: yaml.safe_dump(library_contents, f, indent=4, default_flow_style=False) return jsonify(result='OK'), 200 @BLUEPRINT.route('/prototype/', methods=['DELETE'], read_write=True) def delete_local_prototype(prototypename): try: library_path, prototype = _local_library_path(prototypename) except ValueError as e: return jsonify(error={'message': str(e)}), 400 if not os.path.isfile(library_path): return jsonify(error={'message': 'missing local prototype library'}), 400 # check if the proto is in use in running or committed config rcconfig = running_config() for nodename, nodevalue in rcconfig.get('nodes', {}).iteritems(): if 'prototype' not in nodevalue: continue if nodevalue['prototype'] == prototypename: return jsonify(error={'message': 'prototype in use in running config'}), 400 ccconfig = committed_config() for nodename, nodevalue in ccconfig.get('nodes', {}).iteritems(): if 'prototype' not in nodevalue: continue if nodevalue['prototype'] == prototypename: return jsonify(error={'message': 'prototype in use in committed config'}), 400 lock = filelock.FileLock('{}.lock'.format(library_path)) with lock.acquire(timeout=10): with open(library_path, 'r') as f: library_contents = yaml.safe_load(f) if not isinstance(library_contents, dict): return jsonify(error={'message': 'invalid local prototype library'}), 400 library_contents['prototypes'].pop(prototype, None) with open(library_path, 'w') as f: yaml.safe_dump(library_contents, f, indent=4, default_flow_style=False) return jsonify(result='OK'), 200 def reset_prototype_paths(): global PROTOTYPE_PATHS PROTOTYPE_PATHS = None ================================================ FILE: minemeld/flask/redisclient.py ================================================ import os import redis import werkzeug.local from flask import g from . import REDIS_URL from .logger import LOG __all__ = ['init_app', 'SR'] REDIS_CP = redis.ConnectionPool.from_url( REDIS_URL, max_connections=int(os.environ.get('REDIS_MAX_CONNECTIONS', 200)) ) def get_SR(): SR = getattr(g, '_redis_client', None) if SR is None: SR = redis.StrictRedis(connection_pool=REDIS_CP) g._redis_client = SR return SR def teardown(exception): SR = getattr(g, '_redis_client', None) if SR is not None: g._redis_client = None LOG.debug( 'redis connection pool: in use: {} available: {}'.format( len(REDIS_CP._in_use_connections), len(REDIS_CP._available_connections) ) ) SR = werkzeug.local.LocalProxy(get_SR) def init_app(app): app.teardown_appcontext(teardown) ================================================ FILE: minemeld/flask/session.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from datetime import timedelta from uuid import uuid4 import ujson import redis import werkzeug.datastructures import flask.sessions from .logger import LOG SESSION_EXPIRATION_ENV = 'SESSION_EXPIRATION' DEFAULT_SESSION_EXPIRATION = 10 class RedisSession(werkzeug.datastructures.CallbackDict, flask.sessions.SessionMixin): def __init__(self, initial=None, sid=None, new=False): def on_update(self): self.modified = True werkzeug.datastructures.CallbackDict.__init__(self, initial, on_update) self.sid = sid self.new = new self.modified = False class RedisSessionInterface(flask.sessions.SessionInterface): serializer = ujson session_class = RedisSession def __init__(self, redis_=None, prefix='mm-session:'): if redis_ is None: redis_ = redis.StrictRedis() self.redis = redis_ self.prefix = prefix self.expirtaion_delta = timedelta( minutes=int(os.environ.get( SESSION_EXPIRATION_ENV, DEFAULT_SESSION_EXPIRATION )) ) def generate_sid(self): return str(uuid4()) def get_redis_expiration_time(self, app, session): return timedelta(minutes=10) def open_session(self, app, request): LOG.debug( 'redis session connection pool: in use: {} available: {}'.format( len(self.redis.connection_pool._in_use_connections), len(self.redis.connection_pool._available_connections) ) ) sid = request.cookies.get(app.session_cookie_name) if not sid: sid = self.generate_sid() return self.session_class(sid=sid, new=True) val = self.redis.get(self.prefix + sid) if val is not None: data = self.serializer.loads(val) return self.session_class(data, sid=sid) return self.session_class(sid=sid, new=True) def save_session(self, app, session, response): domain = self.get_cookie_domain(app) if 'user_id' not in session: self.redis.delete(self.prefix + session.sid) if session.modified: response.delete_cookie( app.session_cookie_name, domain=domain ) return redis_exp = self.get_redis_expiration_time(app, session) cookie_exp = self.get_expiration_time(app, session) val = self.serializer.dumps(dict(session)) self.redis.setex( self.prefix + session.sid, int(redis_exp.total_seconds()), val ) response.set_cookie( app.session_cookie_name, session.sid, expires=cookie_exp, httponly=True, domain=domain ) def init_app(app, redis_url): redis_cp = redis.ConnectionPool.from_url( redis_url, max_connections=int(os.environ.get('REDIS_SESSIONS_MAX_CONNECTIONS', 20)) ) app.session_interface = RedisSessionInterface( redis_=redis.StrictRedis(connection_pool=redis_cp) ) app.config.update( SESSION_COOKIE_NAME='mm-session', SESSION_COOKIE_SECURE=True ) ================================================ FILE: minemeld/flask/sns.py ================================================ import requests import json import os.path import uuid import minemeld from .logger import LOG from . import config TYPEHELLO = 'hello' TYPEMKWISH = 'mkwish' TYPESTATS = 'stats' SNS_API_URL = None SNS_ENABLED = None UUID_FILENAME = None SNS_OBJ = None SNS_AVAILABLE = False class Sns(object): def __init__(self, path): self.api_url = SNS_API_URL self.filename = os.path.join(path, UUID_FILENAME) self.mm_version = minemeld.__version__ self.init_ok = self._init_uuid() def get_status(self): return self.init_ok def _init_uuid(self): uuid_error = uuid.UUID(bytes='\x01' * 16).hex # Test case 1: UUIDFILENAME file does not exist. We try to create a new uuid and store in the filesystem if not os.path.isfile(self.filename): self.uuid = uuid.uuid4().hex # If we've failed to send the one-in-a-lifetime hello we just don't store the uuid to try again next boot if self._hello_world(): try: with open(self.filename, 'w') as f: f.write(self.uuid) LOG.debug('New uuid file created.') except Exception as e: # Let the caller know uuid was not saved meaning sns might not be ready LOG.exception('Something went wrong creating the uuid file: {}'.format(self.filename)) return False LOG.debug('Instance uuid = {}'.format(self.uuid)) LOG.debug('MineMeld cloud notification service is ready.') return True LOG.info('MineMeld cloud notification service is not available.') return False # Test case 2: UUIDFILENAME exists but we can't open it (permissions issues?) try: f = open(self.filename) except IOError: self.uuid = uuid_error LOG.exception('Failure opening the uuid file {}'.format(self.filename)) return True r_uuid = f.readline().strip() f.close() # Test case 3: We can read UUIDFILENAME but the content is not a valid UUID4 try: val = uuid.UUID(r_uuid, version=4) except ValueError: self.uuid = uuid_error LOG.info('Invalid uuid value in the file: {}'.format(r_uuid)) return True self.uuid = val.hex if val.hex == r_uuid else uuid_error LOG.debug('Instance uuid = {}'.format(self.uuid)) return True def _send_message(self, kvmessage): kvmessage['uuid'] = self.uuid kvmessage['version'] = self.mm_version try: r = requests.post(self.api_url, data=json.dumps(kvmessage), timeout=5, headers={'Content-Type': 'application/json'}) except Exception as e: LOG.exception('Failure sending the message to the sns cloud provider') return False if r.status_code == requests.codes.ok: response = r.json() if response.get('response', '') == 'ok': return True return False def _hello_world(self): return self._send_message({'type': TYPEHELLO, 'message': 'Hello world!'}) def make_wish(self, message): LOG.debug('Sending new wish message to SNS') return self._send_message({'type': TYPEMKWISH, 'message': message}) def send_stats(self, stats): stats['type'] = TYPESTATS return self._send_message(stats) def init_app(): global SNS_OBJ global SNS_AVAILABLE global UUID_FILENAME global SNS_API_URL global SNS_ENABLED SNS_API_URL = config.get('SNS_URL', 'https://minemeld-notifications.panw.io/0.9/') SNS_ENABLED = config.get('SNS_ENABLED', False) UUID_FILENAME = config.get('UUID_FILE', 'uu.id4') if not SNS_ENABLED: return SNS_OBJ = Sns(config.API_CONFIG_PATH) SNS_AVAILABLE = SNS_OBJ.get_status() ================================================ FILE: minemeld/flask/statusapi.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import os.path import uuid import functools import time from zipfile import ZipFile from tempfile import NamedTemporaryFile, gettempdir import gevent import psutil import yaml from flask import Response, stream_with_context, jsonify, request, send_file from . import config from .mmrpc import MMMaster from .mmrpc import MMRpcClient from .events import EventsGenerator from .redisclient import SR from .aaa import MMBlueprint, enable_prevent_write, disable_prevent_write from .logger import LOG from .jobs import JOBS_MANAGER from .utils import safe_remove, committed_config_path from .sns import SNS_OBJ, SNS_AVAILABLE from minemeld import __version__ __all__ = ['BLUEPRINT'] BLUEPRINT = MMBlueprint('status', __name__, url_prefix='/status') class _PubSubWrapper(object): def __init__(self, subscription, pattern=False): self.subscription = subscription self.pattern = pattern self.pubsub = SR.pubsub(ignore_subscribe_messages=True) if pattern: self.pubsub.psubscribe(subscription) else: self.pubsub.subscribe(subscription) self.generator = self._msg_generator() def _listen(self): while self.pubsub.subscribed: response = self.pubsub.get_message(timeout=5.0) yield response def _msg_generator(self): yield 'data: ok\n\n' for message in self._listen(): if message is None: yield 'data: ping\n\n' continue message = message['data'] if message == '': break yield 'data: ' + message + '\n\n' yield 'data: { "msg": "" }\n\n' def __iter__(self): return self def next(self): return next(self.generator) def close(self): if self.pattern: self.pubsub.punsubscribe(self.subscription) else: self.pubsub.unsubscribe(self.subscription) self.pubsub.close() self.pubsub = None @BLUEPRINT.route('/events/query/', read_write=False) def get_query_events(quuid): try: uuid.UUID(quuid) except ValueError: return jsonify(error={'message': 'Bad query uuid'}), 400 swc_response = stream_with_context( _PubSubWrapper('mm-traced-q.' + quuid) ) r = Response(swc_response, mimetype='text/event-stream') return r @BLUEPRINT.route('/events/status', read_write=False) def get_status_events(): swc_response = stream_with_context(EventsGenerator) r = Response(swc_response, mimetype='text/event-stream') return r @BLUEPRINT.route('/system', methods=['GET'], read_write=False) def get_system_status(): data_path = config.get('MINEMELD_LOCAL_PATH', None) if data_path is None: jsonify(error={'message': 'MINEMELD_LOCAL_PATH not set'}), 500 res = {} res['cpu'] = psutil.cpu_percent(interval=1, percpu=True) res['memory'] = psutil.virtual_memory().percent res['swap'] = psutil.swap_memory().percent res['disk'] = psutil.disk_usage(data_path).percent res['sns'] = SNS_AVAILABLE return jsonify(result=res, timestamp=int(time.time() * 1000)) @BLUEPRINT.route('/info', methods=['GET'], read_write=False) def get_system_info(): res = {} res['sns'] = SNS_AVAILABLE res['version'] = __version__ return jsonify(result=res) @BLUEPRINT.route('/minemeld', methods=['GET'], read_write=False) def get_minemeld_status(): status = MMMaster.status() tr = status.get('result', None) if tr is None: return jsonify(error={'message': status.get('error', 'error')}), 400 result = [] for f, v in tr.iteritems(): _, _, v['name'] = f.split(':', 2) result.append(v) return jsonify(result=result) @BLUEPRINT.route('/config', methods=['GET'], read_write=False) def get_minemeld_running_config(): rcpath = os.path.join( os.path.dirname(os.environ.get('MM_CONFIG')), 'running-config.yml' ) with open(rcpath, 'r') as f: rcconfig = yaml.safe_load(f) return jsonify(result=rcconfig) # XXX this should be moved to a different endpoint @BLUEPRINT.route('//hup', methods=['GET', 'POST'], read_write=False) def hup_node(nodename): status = MMMaster.status() tr = status.get('result', None) if tr is None: return jsonify(error={'message': status.get('error', 'error')}), 400 nname = 'mbus:slave:' + nodename if nname not in tr: return jsonify(error={'message': 'Unknown node'}), 404 MMRpcClient.send_cmd(nodename, 'hup', {'source': 'minemeld-web'}) return jsonify(result='ok'), 200 # XXX this should be moved to a different endpoint @BLUEPRINT.route('//signal/', methods=['GET', 'POST'], read_write=False) def signal_node(nodename, signalname): status = MMMaster.status() tr = status.get('result', None) if tr is None: return jsonify(error={'message': status.get('error', 'error')}), 400 nname = 'mbus:slave:' + nodename if nname not in tr: return jsonify(error={'message': 'Unknown node'}), 404 params = request.get_json(silent=True) if params is None: params = {} params.update({ 'source': 'minemeld-web', 'signal': signalname }) MMRpcClient.send_cmd( target=nodename, method='signal', params=params ) return jsonify(result='ok'), 200 def _clean_local_backup(local_backup_file, g): def _safe_remove(path): LOG.info('Removing backup {}'.format(local_backup_file)) try: os.remove(path) except: pass if g.value != 0: _safe_remove(local_backup_file) return LOG.info('Removing backup {} in 300s'.format(local_backup_file)) gevent.spawn_later(300, _safe_remove, local_backup_file) # XXX this should be moved to a different endpoint @BLUEPRINT.route('/backup', methods=['POST'], read_write=True) def generate_local_backup(): params = request.get_json(silent=True) if params is None: return jsonify(error={'message': 'missing request body'}), 400 password = params.get('p', None) if password is None: return jsonify(error={'message': 'missing p paramater in request body'}), 400 sevenz_path = config.get('MINEMELD_7Z_PATH', None) if sevenz_path is None: return jsonify(error={'message': 'MINEMELD_7Z_PATH not set'}), 500 # create temp zip file tf = NamedTemporaryFile(prefix='mm-local-backup', suffix='.zip', delete=False) tf.close() # initialize the zip structure inside the file ZipFile(tf.name, 'w').close() # build args args = [sevenz_path, 'a', '-p{}'.format(password), '-y', tf.name] proto_path = config.get('MINEMELD_LOCAL_PROTOTYPE_PATH', None) if proto_path is not None: args.append(proto_path) certs_path = config.get('MINEMELD_LOCAL_CERTS_PATH', None) if certs_path is not None: args.append(certs_path) config_path = os.path.dirname(os.environ.get('MM_CONFIG')) args.append(config_path) jobs = JOBS_MANAGER.get_jobs(job_group='status-backup') for jobid, jobdata in jobs.iteritems(): if jobdata == 'RUNNING': return jsonify(error={'message': 'a backup job is already running'}), 400 jobid = JOBS_MANAGER.exec_job( job_group='status-backup', description='local backup', args=args, data={ 'result-file': tf.name }, callback=functools.partial(_clean_local_backup, tf.name) ) return jsonify(result=jobid) # XXX this should be moved to a different endpoint @BLUEPRINT.route('/backup/', methods=['GET'], read_write=True) def get_local_backup(jobid): jobs = JOBS_MANAGER.get_jobs(job_group='status-backup') if jobid not in jobs: return jsonify(error={'message': 'unknown job'}), 404 return send_file(jobs[jobid]['result-file']) @BLUEPRINT.route('/backup/import', methods=['POST'], read_write=True) def import_local_backup(): if 'file' not in request.files: return jsonify(error={'messsage': 'No file in request'}), 400 file = request.files['file'] if file.filename == '': return jsonify(error={'message': 'No file'}), 400 tf = NamedTemporaryFile(prefix='mm-import-backup', delete=False) try: file.save(tf) tf.close() with ZipFile(tf.name, 'r') as zf: contents = zf.namelist() except Exception, e: safe_remove(tf.name) raise e ibid = os.path.basename(tf.name)[16:] result = { 'id': ibid, 'configuration': 'config/committed-config.yml' in contents, 'localPrototypes': 'prototypes/minemeldlocal.yml' in contents, 'feedsAAA': True, 'localCertificates': False } # check for feeds AAA files testfile = os.path.join( 'config/api', config.APIConfigDict(attribute='FEEDS_USERS_ATTRS', level=50).filename ) result['feedsAAA'] &= testfile in contents testfile = os.path.join( 'config/api', config.APIConfigDict(attribute='FEEDS_ATTRS', level=50).filename ) result['feedsAAA'] &= testfile in contents testfile = os.path.join( 'config/api', 'feeds.htpasswd' ) result['feedsAAA'] &= testfile in contents # check for local certificates, there should be at least one # to flag certs as available for fname in contents: if fname.startswith('certs/site/') and not fname.endswith('/'): result['localCertificates'] = True break if not (result['configuration'] or result['localPrototypes'] or result['feedsAAA']): safe_remove(tf.name) return jsonify(error={'message': 'Invalid MineMeld backup'}), 400 return jsonify(result=result) def _cleanup_after_restore(backup_file, locker, g): disable_prevent_write(locker) safe_remove(backup_file) @BLUEPRINT.route('/backup/import//restore', methods=['POST'], read_write=True) def restore_local_backup(backup_id): restore_path = config.get('MINEMELD_RESTORE_PATH', None) if restore_path is None: return jsonify(error={'message': 'MINEMELD_RESTORE_PATH not set'}), 500 params = request.get_json(silent=True) if params is None: return jsonify(error={'message': 'missing request body'}), 400 password = params.get('p', None) restore_configuration = params.get('configuration', False) restore_prototypes = params.get('localPrototypes', False) restore_feeds_aaa = params.get('feedsAAA', False) restore_certificates = params.get('localCertificates', False) if not (restore_configuration or restore_prototypes or restore_feeds_aaa): return jsonify(error={'message': 'Nothing to do'}), 400 backup_file = os.path.join(gettempdir(), 'mm-import-backup{}'.format(backup_id)) if not os.path.samefile(os.path.dirname(backup_file), gettempdir()): return jsonify(error={'message': 'Invalid backup id'}), 400 if not os.path.exists(backup_file): return jsonify(error={'message': 'Invalid backup id'}), 404 locker = 'restore-backup-{}-{}'.format(backup_id, int(time.time())) enable_prevent_write(locker) try: jobs = JOBS_MANAGER.get_jobs(job_group='status-backup') for jobid, jobdata in jobs.iteritems(): if jobdata == 'RUNNING': disable_prevent_write(locker) return jsonify(error={'message': 'a backup job is running'}), 400 jobs = JOBS_MANAGER.get_jobs(job_group='restore-backup') for jobid, jobdata in jobs.iteritems(): if jobdata == 'RUNNING': disable_prevent_write(locker) return jsonify(error={'message': 'a restore job is running'}), 400 args = [restore_path] if restore_configuration: p = os.path.dirname(committed_config_path()) args.extend(['--configuration-path', p]) if restore_prototypes: p = config.get('MINEMELD_LOCAL_PROTOTYPE_PATH', None) if p is None: return jsonify(error={'message': 'MINEMELD_LOCAL_PROTOTYPE_PATH not set'}), 500 args.extend(['--prototypes-path', p]) if restore_feeds_aaa: args.extend([ '--feeds-aaa-path', os.path.join(os.path.dirname(committed_config_path()), 'api') ]) args.extend([ '--feeds-aaa', 'feeds.htpasswd' ]) args.extend([ '--feeds-aaa', config.APIConfigDict(attribute='FEEDS_ATTRS', level=50).filename ]) args.extend([ '--feeds-aaa', config.APIConfigDict(attribute='FEEDS_USERS_ATTRS', level=50).filename ]) if restore_certificates: p = config.get('MINEMELD_LOCAL_CERTS_PATH', None) if p is None: LOG.error('MINEMELD_LOCAL_CERTS_PATH not set, local certificates not restored') else: args.extend([ '--certificates-path', p ]) if password is not None: args.extend(['--password', password]) args.append(backup_file) jobid = JOBS_MANAGER.exec_job( job_group='restore-backup', description='restore backup', args=args, callback=functools.partial(_cleanup_after_restore, backup_file, locker), timeout=200 ) except: disable_prevent_write(locker) raise return jsonify(result=jobid) @BLUEPRINT.route('/mkwish', methods=['POST'], read_write=False) def sns_wish(): request.get_data() message = request.data success = SNS_OBJ.make_wish(message) if success: return jsonify(result='ok') return jsonify(error={'messsage': 'Error sending the message'}), 400 ================================================ FILE: minemeld/flask/supervisorapi.py ================================================ # Copyright 2015-2017 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import time from signal import SIGHUP import psutil import gevent import xmlrpclib import supervisor.xmlrpc from flask import jsonify from . import config from .supervisorclient import MMSupervisor from .aaa import MMBlueprint from .logger import LOG __all__ = ['BLUEPRINT'] BLUEPRINT = MMBlueprint('supervisor', __name__, url_prefix='') def _restart_engine(): LOG.info('Restarting minemeld-engine') supervisorurl = config.get('SUPERVISOR_URL', 'unix:///var/run/supervisor.sock') sserver = xmlrpclib.ServerProxy( 'http://127.0.0.1', transport=supervisor.xmlrpc.SupervisorTransport( None, None, supervisorurl ) ) try: result = sserver.supervisor.stopProcess('minemeld-engine', False) if not result: LOG.error('Stop minemeld-engine returned False') return except xmlrpclib.Fault as e: LOG.error('Error stopping minemeld-engine: {!r}'.format(e)) LOG.info('Stopped minemeld-engine for API request') now = time.time() info = None while (time.time()-now) < 60*10*1000: info = sserver.supervisor.getProcessInfo('minemeld-engine') if info['statename'] in ('FATAL', 'STOPPED', 'UNKNOWN', 'EXITED'): break gevent.sleep(5) else: LOG.error('Timeout during minemeld-engine restart') return sserver.supervisor.startProcess('minemeld-engine', False) LOG.info('Started minemeld-engine') @BLUEPRINT.route('/supervisor', methods=['GET'], read_write=False) def service_status(): try: supervisorstate = MMSupervisor.supervisor.getState() except: LOG.exception("Exception connecting to supervisor") return jsonify(result={'statename': 'STOPPED'}) supervisorstate['processes'] = {} pinfo = MMSupervisor.supervisor.getAllProcessInfo() for p in pinfo: process = { 'statename': p['statename'], 'start': p['start'], 'children': None } try: ps = psutil.Process(pid=p['pid']) process['children'] = len(ps.children()) except: LOG.exception("Error retrieving childen of %d" % p['pid']) supervisorstate['processes'][p['name']] = process return jsonify(result=supervisorstate) @BLUEPRINT.route('/supervisor/minemeld-engine/start', methods=['GET'], read_write=True) def start_minemeld_engine(): result = MMSupervisor.supervisor.startProcess('minemeld-engine', False) return jsonify(result=result) @BLUEPRINT.route('/supervisor/minemeld-engine/stop', methods=['GET'], read_write=True) def stop_minemeld_engine(): result = MMSupervisor.supervisor.stopProcess('minemeld-engine', False) return jsonify(result=result) @BLUEPRINT.route('/supervisor/minemeld-engine/restart', methods=['GET'], read_write=True) def restart_minemeld_engine(): info = MMSupervisor.supervisor.getProcessInfo('minemeld-engine') if info['statename'] == 'STARTING' or info['statename'] == 'STOPPING': return jsonify(error={ 'message': ('minemeld-engine not in RUNNING state: %s' % info['statename']) }), 400 gevent.spawn(_restart_engine) return jsonify(result='OK') @BLUEPRINT.route('/supervisor/minemeld-web/hup', methods=['GET'], read_write=True) def hup_minemeld_web(): info = MMSupervisor.supervisor.getProcessInfo('minemeld-web') apipid = info['pid'] os.kill(apipid, SIGHUP) return jsonify(result='OK') ================================================ FILE: minemeld/flask/supervisorclient.py ================================================ from flask import g import psutil # noqa import xmlrpclib import supervisor.xmlrpc import werkzeug.local from . import config __all__ = ['init_app', 'MMSupervisor'] def get_Supervisor(): sserver = getattr(g, '_supervisor', None) if sserver is None: supervisorurl = config.get('SUPERVISOR_URL', 'unix:///var/run/supervisor.sock') sserver = xmlrpclib.ServerProxy( 'http://127.0.0.1', transport=supervisor.xmlrpc.SupervisorTransport( None, None, supervisorurl ) ) g._supervisor = sserver return sserver MMSupervisor = werkzeug.local.LocalProxy(get_Supervisor) def teardown(exception): SR = getattr(g, '_supervisor', None) if SR is not None: g._supervisor = None def init_app(app): app.teardown_appcontext(teardown) ================================================ FILE: minemeld/flask/taxiicollmgmt.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import re import libtaxii import libtaxii.messages_11 import libtaxii.constants from flask import request from flask.ext.login import current_user from . import config from .taxiiutils import taxii_check, taxii_make_response, get_taxii_feeds from .aaa import MMBlueprint from .logger import LOG __all__ = ['BLUEPRINT'] HOST_RE = re.compile('^[a-zA-Z\d-]{1,63}(?:\.[a-zA-Z\d-]{1,63})*(?::[0-9]{1,5})*$') BLUEPRINT = MMBlueprint('taxiicollmgmt', __name__, url_prefix='') @BLUEPRINT.route('/taxii-collection-management-service', methods=['POST'], feeds=True, read_write=False) @taxii_check def taxii_collection_mgmt_service(): taxii_feeds = get_taxii_feeds() authorized_feeds = filter( current_user.check_feed, taxii_feeds ) if len(authorized_feeds) == 0: return 'Unauthorized', 401 server_host = config.get('TAXII_HOST', None) if server_host is None: server_host = request.headers.get('Host', None) if server_host is None: return 'Missing Host header', 400 if HOST_RE.match(server_host) is None: return 'Invalid Host header', 400 tm = libtaxii.messages_11.get_message_from_xml(request.data) if tm.message_type != \ libtaxii.constants.MSG_COLLECTION_INFORMATION_REQUEST: return 'Invalid message, invalid Message Type', 400 cir = libtaxii.messages_11.CollectionInformationResponse( libtaxii.messages_11.generate_message_id(), tm.message_id ) for feed in authorized_feeds: cii = libtaxii.messages_11.CollectionInformation( feed, '{} Data Feed'.format(feed), ['urn:stix.mitre.org:xml:1.1.1'], True ) si = libtaxii.messages_11.PollingServiceInstance( 'urn:taxii.mitre.org:protocol:http:1.0', 'https://{}/taxii-poll-service'.format(server_host), ['urn:taxii.mitre.org:message:xml:1.1'] ) cii.polling_service_instances.append(si) cir.collection_informations.append(cii) return taxii_make_response(cir) ================================================ FILE: minemeld/flask/taxiidiscovery.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import re import libtaxii import libtaxii.messages_11 import libtaxii.constants from flask import request from flask.ext.login import current_user from . import config from .taxiiutils import get_taxii_feeds, taxii_check, taxii_make_response from .aaa import MMBlueprint from .logger import LOG __all__ = ['BLUEPRINT'] BLUEPRINT = MMBlueprint('taxiidiscovery', __name__, url_prefix='') HOST_RE = re.compile('^[a-zA-Z\d-]{1,63}(?:\.[a-zA-Z\d-]{1,63})*(?::[0-9]{1,5})*$') _SERVICE_INSTANCES = [ { 'type': libtaxii.constants.SVC_DISCOVERY, 'path': 'taxii-discovery-service' }, { 'type': libtaxii.constants.SVC_COLLECTION_MANAGEMENT, 'path': 'taxii-collection-management-service' }, { 'type': libtaxii.constants.SVC_POLL, 'path': 'taxii-poll-service' } ] @BLUEPRINT.route('/taxii-discovery-service', methods=['POST'], feeds=True, read_write=False) @taxii_check def taxii_discovery_service(): taxii_feeds = get_taxii_feeds() authorized = next( (tf for tf in taxii_feeds if current_user.check_feed(tf)), None ) if authorized is None: return 'Unauthorized', 401 server_host = config.get('TAXII_HOST', None) if server_host is None: server_host = request.headers.get('Host', None) if server_host is None: return 'Missing Host header', 400 if HOST_RE.match(server_host) is None: return 'Invalid Host header', 400 tm = libtaxii.messages_11.get_message_from_xml(request.data) if tm.message_type != libtaxii.constants.MSG_DISCOVERY_REQUEST: return 'Invalid message, invalid Message Type', 400 dresp = libtaxii.messages_11.DiscoveryResponse( libtaxii.messages_11.generate_message_id(), tm.message_id ) for si in _SERVICE_INSTANCES: sii = libtaxii.messages_11.ServiceInstance( si['type'], 'urn:taxii.mitre.org:services:1.1', 'urn:taxii.mitre.org:protocol:http:1.0', "https://{}/{}".format(server_host, si['path']), ['urn:taxii.mitre.org:message:xml:1.1'], available=True ) dresp.service_instances.append(sii) return taxii_make_response(dresp) ================================================ FILE: minemeld/flask/taxiipoll.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import datetime import pytz import lz4.frame import libtaxii import libtaxii.constants import stix.core from flask import request, Response, stream_with_context from flask.ext.login import current_user from .redisclient import SR from .taxiiutils import taxii_check, get_taxii_feeds from .aaa import MMBlueprint from .logger import LOG from minemeld.ft.utils import dt_to_millisec __all__ = ['BLUEPRINT'] BLUEPRINT = MMBlueprint('taxiipoll', __name__, url_prefix='') _TAXII_POLL_RESPONSE_HEADER = """ %(inclusive_end_timestamp_label)s """ def _oldest_indicator_timestamp(feed): olist = SR.zrevrange( feed, 0, 0, withscores=True ) if len(olist) == 0: return None ots = int(olist[0][1])/1000 return datetime.datetime.fromtimestamp(ots, pytz.utc) def _indicators_feed(feed, excbegtime, incendtime): if excbegtime is None: excbegtime = 0 else: excbegtime = dt_to_millisec(excbegtime) + 1 incendtime = dt_to_millisec(incendtime) cstart = 0 while True: indicators = SR.zrangebyscore( feed, excbegtime, incendtime, start=cstart, num=100 ) if indicators is None: break for i in indicators: value = SR.hget(feed + '.value', i) if value.startswith('lz4'): try: value = lz4.frame.decompress(value[3:]) value = stix.core.STIXPackage.from_json(value) value = value.to_xml( ns_dict={'https://go.paloaltonetworks.com/minemeld': 'minemeld'} ) except ValueError: continue yield value if len(indicators) < 100: break cstart += 100 def data_feed_11(rmsgid, cname, excbegtime, incendtime): tfeeds = get_taxii_feeds() if cname not in tfeeds: return 'Invalid message, unknown feed', 400 if not incendtime: incendtime = datetime.datetime.utcnow().replace(tzinfo=pytz.utc) def _resp_generator(): # yield the opening tag of the Poll Response resp_header = _TAXII_POLL_RESPONSE_HEADER % { 'collection_name': cname, 'message_id': libtaxii.messages_11.generate_message_id(), 'in_response_to': rmsgid, 'inclusive_end_timestamp_label': incendtime.isoformat() } if excbegtime is not None: resp_header += ( '' + excbegtime.isoformat() + '' ) yield resp_header # yield the content blocks for i in _indicators_feed(cname, excbegtime, incendtime): cb1 = libtaxii.messages_11.ContentBlock( content_binding=libtaxii.constants.CB_STIX_XML_11, content=i ) yield cb1.to_xml()+'\n' # yield the closing tag yield '' return Response( response=stream_with_context(_resp_generator()), status=200, headers={ 'X-TAXII-Content-Type': 'urn:taxii.mitre.org:message:xml:1.1', 'X-TAXII-Protocol': 'urn:taxii.mitre.org:protocol:http:1.0' }, mimetype='application/xml' ) @BLUEPRINT.route('/taxii-poll-service', methods=['POST'], feeds=True, read_write=False) @taxii_check def taxii_poll_service(): taxiict = request.headers['X-TAXII-Content-Type'] if taxiict == 'urn:taxii.mitre.org:message:xml:1.1': tm = libtaxii.messages_11.get_message_from_xml(request.data) if tm.message_type != libtaxii.constants.MSG_POLL_REQUEST: return 'Invalid message', 400 cname = tm.collection_name excbegtime = tm.exclusive_begin_timestamp_label incendtime = tm.inclusive_end_timestamp_label if not current_user.check_feed(cname): return 'Unauthorized', 401 return data_feed_11(tm.message_id, cname, excbegtime, incendtime) elif taxiict == 'urn:taxii.mitre.org:message:xml:1.0': # old TAXII 1.0 not supported yet return 'Invalid message', 400 else: return 'Invalid message', 400 ================================================ FILE: minemeld/flask/taxiiutils.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import functools from flask import request from flask import make_response from .mmrpc import MMMaster from .logger import LOG def taxii_make_response(m11): h = { 'Content-Type': "application/xml", 'X-TAXII-Content-Type': 'urn:taxii.mitre.org:message:xml:1.1', 'X-TAXII-Protocol': 'urn:taxii.mitre.org:protocol:http:1.0' } r = make_response((m11.to_xml(pretty_print=True), 200, h)) return r def taxii_make_response_10(m10): h = { 'Content-Type': "application/xml", 'X-TAXII-Content-Type': 'urn:taxii.mitre.org:message:xml:1.0', 'X-TAXII-Protocol': 'urn:taxii.mitre.org:protocol:http:1.0' } r = make_response((m10.to_xml(pretty_print=True), 200, h)) return r def taxii_check(f): @functools.wraps(f) def check(*args, **kwargs): tct = request.headers.get('X-TAXII-Content-Type', None) if tct not in [ 'urn:taxii.mitre.org:message:xml:1.1', 'urn:taxii.mitre.org:message:xml:1.0' ]: return 'Invalid TAXII Headers', 400 tct = request.headers.get('X-TAXII-Protocol', None) if tct not in [ 'urn:taxii.mitre.org:protocol:http:1.0', 'urn:taxii.mitre.org:protocol:https:1.0' ]: return 'Invalid TAXII Headers', 400 tct = request.headers.get('X-TAXII-Services', None) if tct not in [ 'urn:taxii.mitre.org:services:1.1', 'urn:taxii.mitre.org:services:1.0' ]: return 'Invalid TAXII Headers', 400 return f(*args, **kwargs) return check def get_taxii_feeds(): # check if feed exists status = MMMaster.status() status = status.get('result', None) if status is None: raise RuntimeError('Error retrieving engine status') result = [] for node, node_status in status.iteritems(): class_ = node_status.get('class', None) if class_ != 'minemeld.ft.taxii.DataFeed': continue _, _, feedname = node.split(':', 2) result.append(feedname) return result ================================================ FILE: minemeld/flask/tracedapi.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import uuid from flask import request, jsonify from . import config from .mmrpc import MMRpcClient from .jobs import JOBS_MANAGER from .aaa import MMBlueprint from .logger import LOG import minemeld.traced __all__ = ['BLUEPRINT'] BLUEPRINT = MMBlueprint('traced', __name__, url_prefix='/traced') @BLUEPRINT.route('/query', read_write=False) def traced_query(): query_uuid = request.args.get('uuid', None) if query_uuid is None: return jsonify(error={'message': 'query UUID missing'}), 400 try: uuid.UUID(query_uuid) except ValueError: return jsonify(error={'message': 'invalid query UUID'}), 400 timestamp = request.args.get('ts', None) if timestamp is not None: try: timestamp = int(timestamp) except ValueError: return jsonify(error={'message': 'invalid timestamp'}), 400 counter = request.args.get('c', None) if counter is not None: try: counter = int(counter) except ValueError: return jsonify(error={'message': 'invalid counter'}), 400 num_lines = request.args.get('nl', None) if num_lines is not None: try: num_lines = int(num_lines) except ValueError: return jsonify(error={'message': 'invalid num_lines'}), 400 query = request.args.get('q', "") result = MMRpcClient.send_raw_cmd(minemeld.traced.QUERY_QUEUE, 'query', { 'uuid': query_uuid, 'timestamp': timestamp, 'counter': counter, 'num_lines': num_lines, 'query': query }) return jsonify(result=result), 200 @BLUEPRINT.route('/query//kill', read_write=False) def traced_kill_query(query_uuid): try: uuid.UUID(query_uuid) except ValueError: return jsonify(error={'message': 'invalid query UUID'}), 400 result = MMRpcClient.send_raw_cmd(minemeld.traced.QUERY_QUEUE, 'kill_query', { 'uuid': query_uuid }) return jsonify(result=result), 200 @BLUEPRINT.route('/purge-all', read_write=True) def traced_purge_all(): traced_purge_path = config.get('MINEMELD_TRACED_PURGE_PATH', None) if traced_purge_path is None: return jsonify(error={'message': 'MINEMELD_TRACED_PURGE_PATH not set'}), 500 jobs = JOBS_MANAGER.get_jobs(job_group='traced-purge') for jobid, jobdata in jobs.iteritems(): if jobdata == 'RUNNING': return jsonify(error={'message': 'a trace purge job is already running'}), 400 jobid = JOBS_MANAGER.exec_job( job_group='traced-purge', description='purge all traces', args=[traced_purge_path, '--all'], data={} ) return jsonify(result=jobid) ================================================ FILE: minemeld/flask/utils.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import re import yaml from .logger import LOG class DirSnapshot(object): def __init__(self, path, regex=None): self._entries = self._init_snapshot(path, regex) def _init_snapshot(self, path, regex): result = set() files = os.listdir(path) pattern = re.compile(regex) if regex is not None else None for f in files: if pattern is not None: if pattern.match(f) is None: continue mtime = os.stat(os.path.join(path, f)).st_mtime result.add('%s_%d' % (f, int(mtime))) return result def __eq__(self, other): return self._entries == other._entries def __ne__(self, other): return self._entries != other._entries def running_config_path(): rcpath = os.path.join( os.path.dirname(os.environ.get('MM_CONFIG')), 'running-config.yml' ) return rcpath def committed_config_path(): ccpath = os.path.join( os.path.dirname(os.environ.get('MM_CONFIG')), 'committed-config.yml' ) return ccpath def running_config(): with open(running_config_path(), 'r') as f: rcconfig = yaml.safe_load(f) return rcconfig def committed_config(): with open(committed_config_path(), 'r') as f: ccconfig = yaml.safe_load(f) return ccconfig def safe_remove(path): try: os.remove(path) except: LOG.exception('Exception removing {}'.format(path)) ================================================ FILE: minemeld/flask/validateapi.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import yaml from flask import jsonify, request import minemeld.ft.condition from .aaa import MMBlueprint from .logger import LOG __all__ = ['BLUEPRINT'] BLUEPRINT = MMBlueprint('validate', __name__, url_prefix='/validate') def _return_validation_error(msg): return jsonify(error={ 'message': msg }), 400 @BLUEPRINT.route('/syslogminerrule', methods=['POST'], read_write=False) def validate_syslogminerrule(): try: crule = request.data except Exception as e: return _return_validation_error( 'Error accessing request body: %s' % str(e) ) try: crule = yaml.safe_load(crule) except Exception as e: return _return_validation_error( 'YAML not valid: %s' % str(e) ) if 'name' not in crule: return _return_validation_error('"name" is required') conditions = crule.get('conditions', None) if conditions is None or len(conditions) == 0: return _return_validation_error( 'no "conditions" in rule' ) for c in conditions: try: minemeld.ft.condition.Condition(c) except Exception as e: return _return_validation_error( 'Condition %s is not valid' % c ) indicators = crule.get('indicators', None) if type(indicators) != list: return _return_validation_error( 'no "indicators" in rule' ) for i in indicators: if type(i) != str: return _return_validation_error( 'wrong indicator format: %s' % i ) return jsonify(result='ok') ================================================ FILE: minemeld/ft/__init__.py ================================================ from minemeld.loader import load, MM_NODES_ENTRYPOINT def factory(classname, name, chassis, config): node_class = load(MM_NODES_ENTRYPOINT, classname) return node_class( name=name, chassis=chassis, config=config ) class ft_states(object): READY = 0 CONNECTED = 1 REBUILDING = 2 RESET = 3 INIT = 4 STARTED = 5 CHECKPOINT = 6 IDLE = 7 STOPPED = 8 ================================================ FILE: minemeld/ft/actorbase.py ================================================ import logging from collections import namedtuple import gevent from gevent.queue import Queue from minemeld.ft.base import BaseFT, _counting LOG = logging.getLogger(__name__) ActorCommand = namedtuple('ActorCommand', ['command', 'kwargs_']) class ActorBaseFT(BaseFT): def __init__(self, *args, **kwargs): super(ActorBaseFT, self).__init__(*args, **kwargs) self._actor_queue = Queue(maxsize=1) self._actor_glet = None @_counting('rebuild.queued') def command_rebuild(self): pass @_counting('checkpoint.queued') def checkpoint(self, **kwargs): self._actor_queue.put(ActorCommand(command='checkpoint', kwargs_=kwargs)) @_counting('update.queued') def update(self, **kwargs): self._actor_queue.put(ActorCommand(command='update', kwargs_=kwargs)) @_counting('withdraw.queued') def withdraw(self, **kwargs): self._actor_queue.put(ActorCommand(command='withdraw', kwargs_=kwargs)) def _actor_loop(self): while True: acommand = self._actor_queue.get() if acommand.command == 'checkpoint': method = super(ActorBaseFT, self).checkpoint elif acommand.command == 'update': method = super(ActorBaseFT, self).update elif acommand.command == 'withdraw': method = super(ActorBaseFT, self).withdraw elif acommand.command == 'rebuild': method = self._rebuild else: LOG.error('{} - unknown command {}'.format(self.name, acommand.command)) try: method(**acommand.kwargs_) except gevent.GreenletExit: break except: LOG.exception('{} - error executing {!r}'.format(self.name, acommand)) def start(self): super(ActorBaseFT, self).start() if self._actor_glet is not None: return self._actor_glet = gevent.spawn(self._actor_loop) def stop(self): super(ActorBaseFT, self).stop() if self._actor_glet is None: return self._actor_glet.kill() self._actor_glet = None ================================================ FILE: minemeld/ft/anomali.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements minemeld.ft.anomali.Intelligence, the Miner node for Anomali Intelligence API. """ import os import yaml import netaddr import pytz import datetime import requests import logging from . import basepoller from .utils import interval_in_sec, dt_to_millisec LOG = logging.getLogger(__name__) _API_BASE = 'https://api.threatstream.com' _API_ENDPOINT = '/api/v2/intelligence/' class Intelligence(basepoller.BasePollerFT): def __init__(self, name, chassis, config): super(Intelligence, self).__init__(name, chassis, config) self.last_run = None def configure(self): super(Intelligence, self).configure() self.url = self.config.get('url', None) self.polling_timeout = self.config.get('polling_timeout', 20) self.verify_cert = self.config.get('verify_cert', True) self.prefix = self.config.get('prefix', 'anomali') self.fields = self.config.get('fields', None) self.query = self.config.get('query', None) initial_interval = self.config.get('initial_interval', '3600') self.initial_interval = interval_in_sec(initial_interval) if self.initial_interval is None: LOG.error( '%s - wrong initial_interval format: %s', self.name, initial_interval ) self.initial_interval = 3600 self.api_key = None self.username = None self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.api_key = sconfig.get('api_key', None) if self.api_key is not None: LOG.info('%s - API Key set', self.name) self.username = sconfig.get('username', None) if self.username is not None: LOG.info('%s - username set', self.name) def _calc_age_out(self, indicator, attributes): etsattribute = self.prefix+'_expiration_ts' if etsattribute in attributes: original_ets = attributes[etsattribute] LOG.debug('%s - original_ets: %s', self.name, original_ets) original_ets = original_ets[:19] ets = datetime.datetime.strptime( original_ets, '%Y-%m-%dT%H:%M:%S' ).replace(tzinfo=pytz.UTC) LOG.debug('%s - expiration_ts set for %s', self.name, indicator) return dt_to_millisec(ets) return super(Intelligence, self)._calc_age_out(indicator, attributes) def _process_item(self, item): if 'value' not in item: LOG.debug('%s - value not in %s', self.name, item) return [[None, None]] indicator = item['value'] if not (isinstance(indicator, str) or isinstance(indicator, unicode)): LOG.error( '%s - Wrong indicator type: %s - %s', self.name, indicator, type(indicator) ) return [[None, None]] fields = self.fields if fields is None: fields = item.keys() fields.remove('value') attributes = {} for field in fields: if field not in item: continue attributes['%s_%s' % (self.prefix, field)] = item[field] if 'confidence' in item: attributes['confidence'] = item['confidence'] if item['type'] == 'domain': attributes['type'] = 'domain' elif item['type'] == 'url': attributes['type'] = 'URL' elif item['type'] == 'ip': try: n = netaddr.IPNetwork(indicator) except: LOG.error('%s - Invald IP address: %s', self.name, indicator) return [[None, None]] if n.version == 4: attributes['type'] = 'IPv4' elif n.version == 6: attributes['type'] = 'IPv6' else: LOG.error('%s - Unknown ip version: %d', self.name, n.version) return [[None, None]] else: LOG.info( '%s - indicator type %s not supported', self.name, item['type'] ) return [[None, None]] return [[indicator, attributes]] def _build_iterator(self, now): if self.api_key is None or self.username is None: raise RuntimeError('%s - credentials not set' % self.name) if self.last_run is None: now = datetime.datetime.fromtimestamp(now/1000.0, pytz.UTC) dtinterval = datetime.timedelta(seconds=self.initial_interval) origin = now - dtinterval else: origin = datetime.datetime.fromtimestamp( self.last_run/1000.0, pytz.UTC ) q = '(modified_ts>=%s)' % origin.strftime('%Y-%m-%dT%H:%M:%S') if self.query: q = '(%s AND %s)' % (q, self.query) params = dict( username=self.username, api_key=self.api_key, limit=100, q=q ) LOG.debug('%s - query params: %s', self.name, params) rkwargs = dict( stream=True, verify=self.verify_cert, timeout=self.polling_timeout, params=params ) r = requests.get( _API_BASE+_API_ENDPOINT, **rkwargs ) while True: try: r.raise_for_status() except: LOG.error( '%s - exception in request: %s %s', self.name, r.status_code, r.content ) raise cjson = r.json() if 'objects' not in cjson: LOG.error('%s - no objects in response', self.name) return objects = cjson['objects'] for o in objects: yield o if 'meta' not in cjson: return if 'next' not in cjson['meta']: return next_url = cjson['meta']['next'] if next_url is None: return LOG.debug('%s - requesting next items', self.name) rkwargs.pop('params', None) r = requests.get( _API_BASE+cjson['meta']['next'], **rkwargs ) def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(Intelligence, self).hup(source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass ================================================ FILE: minemeld/ft/auscert.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import requests import logging import itertools import os import yaml from . import http LOG = logging.getLogger(__name__) class MaliciousURLFeed(http.HttpFT): def configure(self): super(MaliciousURLFeed, self).configure() self.api_key = None self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.api_key = sconfig.get('api_key', None) if self.api_key is not None: LOG.info('%s - api_key set', self.name) def _build_iterator(self, now): if self.api_key is None: raise RuntimeError( '{} - API Key not set, ' 'poll not performed'.format(self.name) ) rkwargs = dict( stream=True, verify=self.verify_cert, timeout=self.polling_timeout ) rkwargs["headers"] = { 'API-Key': self.api_key } session = requests.Session() r = session.get( self.url, **rkwargs ) # if api_key is wrong we'll get a 403 response code if r.status_code == 403: raise RuntimeError( '{} - not authorized (Invalid API Key?)'.format(self.name) ) try: r.raise_for_status() except: LOG.debug('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise result = r.iter_lines() if self.ignore_regex is not None: result = itertools.ifilter( lambda x: self.ignore_regex.match(x) is None, result ) return result def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(MaliciousURLFeed, self).hup(source) ================================================ FILE: minemeld/ft/autofocus.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import os import yaml import netaddr import netaddr.core import pan.afapi import ujson import re from . import basepoller LOG = logging.getLogger(__name__) DOMAIN_RE = re.compile('^[a-zA-Z\d-]{,63}(\.[a-zA-Z\d-]{,63})*$') class ExportList(basepoller.BasePollerFT): def configure(self): super(ExportList, self).configure() self.api_key = None self.label = None self.hostname = self.config.get('autofocus_hostname', None) self.verify_cert = self.config.get('verify_cert', None) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.api_key = sconfig.get('api_key', None) if self.api_key is not None: LOG.info('%s - api key set', self.name) self.label = sconfig.get('label', None) def _process_item(self, row): indicator = row result = {} result['type'] = self._type_of_indicator(indicator) result['autofocus_label'] = self.label return [[indicator, result]] def _check_for_ip(self, indicator): if '-' in indicator: # check for address range a1, a2 = indicator.split('-', 1) try: a1 = netaddr.IPAddress(a1) a2 = netaddr.IPAddress(a2) if a1.version == a2.version: if a1.version == 6: return 'IPv6' if a1.version == 4: return 'IPv4' except: return None return None if '/' in indicator: # check for network try: ip = netaddr.IPNetwork(indicator) except: return None if ip.version == 4: return 'IPv4' if ip.version == 6: return 'IPv6' return None try: ip = netaddr.IPAddress(indicator) except: return None if ip.version == 4: return 'IPv4' if ip.version == 6: return 'IPv6' return None def _type_of_indicator(self, indicator): ipversion = self._check_for_ip(indicator) if ipversion is not None: return ipversion if DOMAIN_RE.match(indicator): return 'domain' return 'URL' def _build_iterator(self, now): if self.api_key is None or self.label is None: raise RuntimeError( '%s - api_key or label not set, poll not performed' % self.name ) body = { 'label': self.label, 'panosFormatted': True } af = pan.afapi.PanAFapi( hostname=self.hostname, verify_cert=self.verify_cert, api_key=self.api_key ) r = af.export(data=ujson.dumps(body)) r.raise_for_status() return r.json.get('export_list', []) def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(ExportList, self).hup(source=source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass ================================================ FILE: minemeld/ft/azure.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import itertools import functools from collections import defaultdict import requests import netaddr import lxml.etree import bs4 from . import basepoller LOG = logging.getLogger(__name__) AZUREXML_URL = \ 'https://www.microsoft.com/EN-US/DOWNLOAD/confirmation.aspx?id=41653' AZURE_CLOUD_TO_URL = { 'public': 'https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519', 'usgov': 'http://www.microsoft.com/en-us/download/confirmation.aspx?id=57063', 'china': 'https://www.microsoft.com/en-us/download/confirmation.aspx?id=57062', 'germany': 'https://www.microsoft.com/en-us/download/confirmation.aspx?id=57064' } REGIONS_XPATH = '/AzurePublicIpAddresses/Region' def _build_IPv4(nodename, region, iprange): iprange = iprange.get('Subnet', None) if iprange is None: LOG.error('%s - No Subnet', nodename) return {} try: netaddr.IPNetwork(iprange) except: LOG.exception('%s - Invalid ip range: %s', nodename, iprange) return {} item = { 'indicator': iprange, 'type': 'IPv4', 'confidence': 100, 'azure_region': region, 'sources': ['azure.xml'] } return item def _build_IP(nodename, address_prefix, **keywords): try: ap = netaddr.IPNetwork(address_prefix) except Exception: LOG.exception('%s - Invalid ip range: %s', nodename, address_prefix) return {} if ap.version == 4: type_ = 'IPv4' elif ap.version == 6: type_ = 'IPv6' else: LOG.error('{} - Unknown IP version: {}'.format(nodename, ap.version)) return {} item = { 'indicator': address_prefix, 'type': type_, 'confidence': 100, 'sources': [nodename] } item.update(keywords) return item class AzureXML(basepoller.BasePollerFT): def configure(self): super(AzureXML, self).configure() self.polling_timeout = self.config.get('polling_timeout', 20) self.verify_cert = self.config.get('verify_cert', True) def _process_item(self, item): indicator = item.pop('indicator', None) return [[indicator, item]] def _build_request(self, now): r = requests.Request( 'GET', AZUREXML_URL ) return r.prepare() def _build_iterator(self, now): _iterators = [] rkwargs = dict( stream=False, verify=self.verify_cert, timeout=self.polling_timeout ) r = requests.get( AZUREXML_URL, **rkwargs ) try: r.raise_for_status() except: LOG.error('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise html_soup = bs4.BeautifulSoup(r.content, "lxml") a = html_soup.find('a', class_='failoverLink') if a is None: LOG.error('%s - failoverLink not found', self.name) raise RuntimeError('{} - failoverLink not found'.format(self.name)) LOG.debug('%s - download link: %s', self.name, a['href']) rkwargs = dict( stream=True, verify=self.verify_cert, timeout=self.polling_timeout ) r = requests.get( a['href'], **rkwargs ) try: r.raise_for_status() except: LOG.error('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise parser = lxml.etree.XMLParser() for chunk in r.iter_content(chunk_size=10 * 1024): parser.feed(chunk) rtree = parser.close() regions = rtree.xpath(REGIONS_XPATH) for r in regions: LOG.debug('%s - Extracting region: %s', self.name, r.get('Name')) ipranges = r.xpath('IpRange') _iterators.append(itertools.imap( functools.partial(_build_IPv4, self.name, r.get('Name')), ipranges )) return itertools.chain(*_iterators) class AzureJSON(basepoller.BasePollerFT): def configure(self): super(AzureJSON, self).configure() self.polling_timeout = self.config.get('polling_timeout', 20) self.verify_cert = self.config.get('verify_cert', True) self.cloud = self.config.get('cloud', 'public') self.url = AZURE_CLOUD_TO_URL[self.cloud] def _process_item(self, item): indicator = item.pop('indicator', None) return [[indicator, item]] def _build_iterator(self, now): _iterators = [] rkwargs = dict( stream=False, verify=self.verify_cert, timeout=self.polling_timeout ) r = requests.get( self.url, **rkwargs ) try: r.raise_for_status() except: LOG.error('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise html_soup = bs4.BeautifulSoup(r.content, "lxml") a = html_soup.find('a', class_='failoverLink') if a is None: LOG.error('%s - failoverLink not found', self.name) raise RuntimeError('{} - failoverLink not found'.format(self.name)) LOG.debug('%s - download link: %s', self.name, a['href']) rkwargs = dict( stream=True, verify=self.verify_cert, timeout=self.polling_timeout ) r = requests.get( a['href'], **rkwargs ) try: r.raise_for_status() except: LOG.error('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise rtree = r.json() values = rtree.get('values', None) if values is None: LOG.error('{} - no values in JSON response'.format(self.name)) return [] for v in values: LOG.debug('{} - Extracting value: {!r}'.format(self.name, v.get('id', None))) id_ = v.get('id', None) name = v.get('name', None) props = v.get('properties', None) if props is None: LOG.error('{} - no properties in value'.format(self.name)) continue region = props.get('region', None) platform = props.get('platform', None) system_service = props.get('systemService', None) address_prefixes = props.get('addressPrefixes', []) _iterators.append(itertools.imap( functools.partial( _build_IP, self.name, azure_name=name, azure_id=id_, azure_region=region, azure_platform=platform, azure_system_service=system_service ), address_prefixes )) # aggregate indicators aggregated_indicators = defaultdict(lambda: dict( azure_name_list=set([]), azure_id_list=set(([])), azure_region_list=set([]), azure_platform_list=set([]), azure_system_service_list=set([]) )) for i in itertools.chain(*_iterators): cv = aggregated_indicators[i['indicator']] cv.update(i) for k, v in i.iteritems(): cv[k] = v if k.startswith('azure_'): cv['{}_list'.format(k)].add(str(v).lower()) # convert sets into lists for iv in aggregated_indicators.values(): for k in iv.keys(): if isinstance(iv[k], set): iv[k] = list(iv[k]) return iter(aggregated_indicators.values()) ================================================ FILE: minemeld/ft/bambenek.py ================================================ """ This module implements a thin wrapper class around minemeld.ft.csv.CSVFT to mine Bambenek Consulting feeds """ from __future__ import absolute_import import logging from . import csv LOG = logging.getLogger(__name__) class Miner(csv.CSVFT): pass ================================================ FILE: minemeld/ft/base.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements minemeld.ft.base.BaseFT, the base class for nodes. """ from __future__ import absolute_import import logging import copy import os import collections import json import gevent from . import condition from . import ft_states from . import utils LOG = logging.getLogger(__name__) class _Filters(object): """Implements a set of filters to be applied to indicators. Used by mineneld.ft.base.BaseFT for ingress and egress filters. Args: filters (list): list of filters. """ def __init__(self, filters): self.filters = [] for f in filters: cf = { 'name': f.get('name', 'filter_%d' % len(self.filters)), 'conditions': [], 'actions': [] } fconditions = f.get('conditions', None) if fconditions is None: fconditions = [] for c in fconditions: cf['conditions'].append(condition.Condition(c)) for a in f.get('actions'): cf['actions'].append(a) self.filters.append(cf) def apply(self, origin=None, method=None, indicator=None, value=None): if value is None: d = {} else: d = copy.copy(value) if indicator is not None: d['__indicator'] = indicator if method is not None: d['__method'] = method if origin is not None: d['__origin'] = origin for f in self.filters: LOG.debug("evaluating filter %s", f['name']) r = True for c in f['conditions']: r &= c.eval(d) if not r: continue for a in f['actions']: if a == 'accept': if value is None: return indicator, None d.pop('__indicator') d.pop('__origin', None) d.pop('__method', None) return indicator, d elif a == 'drop': return None, None LOG.debug("no matching filter, default accept") if value is None: return indicator, None d.pop('__indicator') d.pop('__origin', None) d.pop('__method', None) return indicator, d def _counting(statsname): """Decorator for counting calls to decorated instance methods. Counters are stored in statistics attribute of the instance. Args: statsname (str): name of the counter to increment """ def _counter_out(f): def _counter(self, *args, **kwargs): self.statistics[statsname] += 1 f(self, *args, **kwargs) self.publish_status() return _counter return _counter_out class BaseFT(object): """Implements base class of MineMeld engine nodes. **Config parameters** :infilters: inbound filter set. Filters to be applied to received indicators. :outfilters: outbound filter set. Filters to be applied to transmitted indicators. **Filter set** Each filter set is a list of filters. Filters are verified from top to bottom, and the first matching filter is applied. Default action is **accept**. Each filter is a dictionary with 3 keys: :name: name of the filter. :conditions: list of boolean expressions to match on the indicator. :actions: list of actions to be applied to the indicator. Currently the only supported actions are **accept** and **drop** In addition to the atttributes in the indicator value, filters can match on 3 special attributes: :__indicator: the indicator itself. :__method: the method of the message, **update** or **withdraw**. :__origin: the name of the node who sent the indicator. **Condition** A condition in the filter, is boolean expression composed by a JMESPath expression, an operator (<, <=, ==, >=, >, !=) and a value. Example: Example config in YAML:: infilters: - name: accept withdraws conditions: - __method == 'withdraw' actions: - accept - name: accept URL conditions: - type == 'URL' actions: - accept - name: drop all actions: - drop outfilters: - name: accept all (default) actions: - accept Args: name (str): node name, should be unique inside the graph chassis (object): parent chassis instance config (dict): node config. """ def __init__(self, name, chassis, config): self.name = name self.chassis = chassis self._original_config = copy.deepcopy(config) self.config = config self.configure() self.inputs = [] self.output = None self.statistics = collections.defaultdict(int) self.read_checkpoint() self.chassis.request_mgmtbus_channel(self) self._state = ft_states.READY self._last_status_publish = None self._throttled_publish_status = utils.GThrottled(self._internal_publish_status, 3000) self._clock = 0 self._disable_full_trace = 'MM_DISABLE_FULL_TRACE' in os.environ self._disable_full_trace_glet = None @property def state(self): return self._state @state.setter def state(self, value): LOG.info("%s - transitioning to state %d", self.name, value) self._state = value if value >= ft_states.INIT and value <= ft_states.STOPPED: self.publish_status(force=True) def read_checkpoint(self): """Reads checkpoint file from disk. First line of the checkpoint file is a UUID, the *checkpoint* received before stopping. The second line is a dictionary in JSON with the class of the node and the config. The third line is a dictionary in JSON with the persistent state of the node. Checkpoint files are used to check if the saved state on disk is consistent with the current running config. If the state is not consistent `last_checkpoint` is set to None, to indicate that the state stored on disk is not valid or inexistent. Called by `__init__`. """ self.last_checkpoint = None config = { 'class': (self.__class__.__module__+'.'+self.__class__.__name__), 'config': self._original_config } config = json.dumps(config, sort_keys=True) try: with open(self.name+'.chkp', 'r') as f: contents = f.read() if contents[0] == '{': # new format contents = json.loads(contents) self.last_checkpoint = contents['checkpoint'] saved_config = contents['config'] saved_state = contents['state'] else: # old format lines = contents.splitlines() self.last_checkpoint = lines[0] saved_config = '' if len(lines) > 1: # this to support a really old format # where only checkpoint value was saved saved_config = lines[1] saved_state = None LOG.debug('%s - restored checkpoint: %s', self.name, self.last_checkpoint) # old_status is missing in old releases # stick to the old behavior if saved_config and saved_config != config: LOG.info( '%s - saved config does not match new config', self.name ) self.last_checkpoint = None return LOG.info( '%s - saved config matches new config', self.name ) if saved_state is not None: self._saved_state_restore(saved_state) except (ValueError, IOError): LOG.exception('%s - Error reading last checkpoint', self.name) self.last_checkpoint = None def create_checkpoint(self, value): """Saves checkpoint file to disk. Called by `checkpoint`. Args: value (str): received *checkpoint* """ config = { 'class': (self.__class__.__module__+'.'+self.__class__.__name__), 'config': self._original_config } contents = { 'checkpoint': value, 'config': json.dumps(config, sort_keys=True), 'state': self._saved_state_create() } with open(self.name+'.chkp', 'w') as f: f.write(json.dumps(contents)) f.write('\n') def remove_checkpoint(self): try: os.remove('{}.chkp'.format(self.name)) except (IOError, OSError): pass def _saved_state_restore(self, saved_state): pass def _saved_state_create(self): return {} def configure(self): """Applies the config settings stored in `self.config`. Called by `__init__`. When this method is changed to add/remove new parameters, the class docstring should be updated. """ self.infilters = _Filters(self.config.get('infilters', [])) self.outfilters = _Filters(self.config.get('outfilters', [])) def connect(self, inputs, output): if self.state != ft_states.READY: LOG.error('connect called in non ready FT') raise AssertionError('connect called in non ready FT') for i in inputs: LOG.info("%s - requesting fabric sub channel for %s", self.name, i) self.chassis.request_sub_channel( self.name, self, i, allowed_methods=['update', 'withdraw', 'checkpoint'] ) self.inputs = inputs self.inputs_checkpoint = {} if output: self.output = self.chassis.request_pub_channel(self.name) self.chassis.request_rpc_channel( self.name, self, allowed_methods=[ 'update', 'withdraw', 'checkpoint', 'get', 'get_all', 'get_range', 'length' ] ) self.state = ft_states.CONNECTED def apply_infilters(self, origin, method, indicator, value): return self.infilters.apply( origin=origin, method=method, indicator=indicator, value=value ) def apply_outfilters(self, origin, method, indicator, value): return self.outfilters.apply( origin=origin, method=method, indicator=indicator, value=value ) def do_rpc(self, dftname, method, block=True, timeout=30, **kwargs): return self.chassis.send_rpc(self.name, dftname, method, kwargs, block=block, timeout=timeout) @_counting('update.tx') def emit_update(self, indicator, value): if self.output is None: return self.trace('EMIT_UPDATE', indicator, value=value) indicator, value = self.apply_outfilters( origin=self.name, method='update', indicator=indicator, value=value ) if indicator is None: return if value is not None: for k in value.keys(): if k[0] in ['_', '$']: value.pop(k) self.output.publish("update", { 'source': self.name, 'indicator': indicator, 'value': value }) @_counting('withdraw.tx') def emit_withdraw(self, indicator, value=None): if self.output is None: return self.trace('EMIT_WITHDRAW', indicator, value=value) indicator, value = self.apply_outfilters( origin=self.name, method='withdraw', indicator=indicator, value=value ) if indicator is None: return if value is not None: for k in value.keys(): if k[0] in ['_', '$']: value.pop(k) self.output.publish("withdraw", { 'source': self.name, 'indicator': indicator, 'value': value }) @_counting('checkpoint.tx') def emit_checkpoint(self, value): if self.output is None: return self.output.publish('checkpoint', { 'source': self.name, 'value': value }) @_counting('update.rx') def update(self, source=None, indicator=None, value=None): LOG.debug('%s {%s} - update from %s value %s', self.name, self.state, source, value) if not self._disable_full_trace: self.trace('RECVD_UPDATE', indicator, source_node=source, value=value) if self.state not in [ft_states.STARTED, ft_states.CHECKPOINT]: self.statistics['error.wrong_state'] += 1 return if source in self.inputs_checkpoint: LOG.error("update received from checkpointed source") raise AssertionError("update received from checkpointed source") if value is not None: for k in value.keys(): if k.startswith("_"): value.pop(k) fltindicator, fltvalue = self.apply_infilters( origin=source, method='update', indicator=indicator, value=value ) if fltindicator is None: if not self._disable_full_trace: self.trace('DROP_UPDATE', indicator, source_node=source, value=value) self.filtered_withdraw( source=source, indicator=indicator, value=value ) return self.trace('ACCEPT_UPDATE', indicator, source_node=source, value=value) self.filtered_update( source=source, indicator=fltindicator, value=fltvalue ) @_counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): raise NotImplementedError('%s: update' % self.name) @_counting('withdraw.rx') def withdraw(self, source=None, indicator=None, value=None): LOG.debug('%s {%s} - withdraw from %s value %s', self.name, self.state, source, value) if not self._disable_full_trace: self.trace('RECVD_WITHDRAW', indicator, source_node=source, value=value) if self.state not in [ft_states.STARTED, ft_states.CHECKPOINT]: self.statistics['error.wrong_state'] += 1 return if source in self.inputs_checkpoint: LOG.error("withdraw received from checkpointed source") raise AssertionError("withdraw received from checkpointed source") fltindicator, fltvalue = self.apply_infilters( origin=source, method='withdraw', indicator=indicator, value=value ) if fltindicator is None: if not self._disable_full_trace: self.trace('DROP_WITHDRAW', indicator, source_node=source, value=value) return if fltvalue is not None: for k in fltvalue.keys(): if k.startswith("_"): fltvalue.pop(k) self.trace('ACCEPT_WITHDRAW', indicator, source_node=source, value=value) self.filtered_withdraw( source=source, indicator=indicator, value=value ) @_counting('withdraw.processed') def filtered_withdraw(self, source=None, indicator=None, value=None): raise NotImplementedError('%s: withdraw' % self.name) @_counting('checkpoint.rx') def checkpoint(self, source=None, value=None): LOG.debug('%s {%s} - checkpoint from %s value %s', self.name, self.state, source, value) if self.state not in [ft_states.STARTED, ft_states.CHECKPOINT]: LOG.error("%s {%s} - checkpoint received with state not STARTED " "or CHECKPOINT", self.name, self.state) raise AssertionError("checkpoint received with state not STARTED " "or CHECKPOINT") for v in self.inputs_checkpoint.values(): if v != value: LOG.error("different checkpoint value received") raise AssertionError("different checkpoint value received") self.inputs_checkpoint[source] = value if len(self.inputs_checkpoint) != len(self.inputs): self.state = ft_states.CHECKPOINT return self.state = ft_states.IDLE self.create_checkpoint(value) self.last_checkpoint = value self.emit_checkpoint(value) def _full_trace_timeout(self, timeout): """To be used as greenlet for disabling full trace after a specific time """ gevent.sleep(timeout) self._disable_full_trace = True LOG.debug('{} - full trace disabled'.format(self.name)) def enable_full_trace(self, timeout=600): """Enables full trace """ # if full trace is already enabled, do nothing if self._disable_full_trace is False: return self._disable_full_trace_glet = gevent.spawn( self._full_trace_timeout, timeout ) self._disable_full_trace = False LOG.debug('{} - full trace enabled'.format(self.name)) def publish_status(self, force=False): if force: self._internal_publish_status() self._throttled_publish_status() def _internal_publish_status(self): self._last_status_publish = utils.utc_millisec() status = self.mgmtbus_status() self.chassis.publish_status( timestamp=utils.utc_millisec(), nodename=self.name, status=status ) def mgmtbus_state_info(self): return { 'checkpoint': self.last_checkpoint, 'state': self.state, 'is_source': len(self.inputs) == 0 } def mgmtbus_initialize(self): self.state = ft_states.INIT self.remove_checkpoint() self.initialize() return 'OK' def mgmtbus_rebuild(self): self.state = ft_states.REBUILDING self.remove_checkpoint() self.rebuild() self.state = ft_states.INIT return 'OK' def mgmtbus_reset(self): self.state = ft_states.RESET self.remove_checkpoint() self.reset() self.state = ft_states.INIT return 'OK' def mgmtbus_status(self): try: # if node is not ready yet to publish the length length = self.length() except: length = None result = { 'clock': self._clock, 'class': (self.__class__.__module__+'.'+self.__class__.__name__), 'state': self.state, 'statistics': self.statistics, 'length': length, 'inputs': self.inputs, 'output': (self.output is not None), 'trace': not self._disable_full_trace } self._clock += 1 return result def mgmtbus_checkpoint(self, value=None): if len(self.inputs) != 0: return 'ignored' self.state = ft_states.IDLE self.create_checkpoint(value) self.last_checkpoint = value self.emit_checkpoint(value) return 'OK' def mgmtbus_hup(self, source=None): self.hup(source=source) def mgmtbus_signal(self, source=None, signal=None, **kwargs): if signal == 'trace': self.enable_full_trace() return self._disable_full_trace raise NotImplementedError('{}: signal - not implemented'.format(self.name)) def initialize(self): pass def rebuild(self): pass def reset(self): pass def get_state(self): return self.state def get(self, source=None, indicator=None): raise NotImplementedError('%s: get - not implemented' % self.name) def get_all(self, source=None): raise NotImplementedError('%s: get_all - not implemented' % self.name) def get_range(self, source=None, index=None, from_key=None, to_key=None): raise NotImplementedError('%s: get_range - not implemented' % self.name) def length(self, source=None): raise NotImplementedError('%s: length - not implemented' % self.name) def hup(self, source=None): raise NotImplementedError('%s: hup - not implemented' % self.name) def trace(self, action, indicator, **kwargs): if self.state not in [ft_states.STARTED, ft_states.CHECKPOINT]: LOG.debug( "%s - trace called in wrong state %s", self.name, self.state ) return trace = { 'indicator': indicator, 'op': action, } trace.update(kwargs) self.chassis.log( timestamp=utils.utc_millisec(), nodename=self.name, log_type='TRACE', value=trace ) def start(self): LOG.debug("%s - start called", self.name) if self.state != ft_states.INIT: LOG.error("start on not INIT FT") raise AssertionError("start on not INIT FT") self.state = ft_states.STARTED def stop(self): LOG.debug("%s - stop called", self.name) if self._disable_full_trace_glet is not None: self._disable_full_trace_glet.kill() self._disable_full_trace_glet = None if self.state not in [ft_states.IDLE, ft_states.STARTED]: LOG.error("stop on not IDLE or STARTED FT") raise AssertionError("stop on not IDLE or STARTED FT") self._throttled_publish_status.cancel() self.state = ft_states.STOPPED @staticmethod def gc(name, config=None): try: os.remove('{}.chkp'.format(name)) except: pass ================================================ FILE: minemeld/ft/basepoller.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements minemeld.ft.basepoller.BasePollerFT, a base class for miners retrieving indicators by periodically polling an external source. """ import logging import copy import random import collections import sys import shutil import gevent import gevent.event import gevent.queue from . import base from . import ft_states from .table import Table from .utils import utc_millisec from .utils import RWLock from .utils import parse_age_out LOG = logging.getLogger(__name__) _MAX_AGE_OUT = ((1 << 32)-1)*1000 # 2106-02-07 6:28:15 class _BaseBPTable(object): def __init__(self, table): self.table = table def get(self, indicator, itype=None): return self.table.get(indicator) def delete(self, indicator, itype=None): self.table.delete(indicator) def put(self, indicator, value): self.table.put(indicator, value) def query(self, *args, **kwargs): return self.table.query(*args, **kwargs) def length(self): return self.table.num_indicators def close(self): self.table.close() def __del__(self): self.close() class _BPTable_v0(_BaseBPTable): def __init__(self, table): super(_BPTable_v0, self).__init__(table) self.table.create_index('_age_out') self.table.create_index('_withdrawn') self.table.create_index('_last_run') class _BPTable_v1(_BaseBPTable): def __init__(self, table, type_in_key): super(_BPTable_v1, self).__init__(table) self.table.create_index('_age_out') self.table.create_index('_withdrawn') self.table.create_index('_last_run') self.type_in_key = type_in_key cmetadata = self.table.get_custom_metadata() if cmetadata is None: _custom_metadata = dict(version=1, type_in_key=type_in_key) self.table.set_custom_metadata(_custom_metadata) else: if cmetadata.get('type_in_key', None) != self.type_in_key: raise RuntimeError('Can\'t change type in key of an existing table') def get(self, indicator, itype=None): if self.type_in_key: indicator = self._type_key(indicator, itype) return self.table.get(indicator) def delete(self, indicator, itype=None): if self.type_in_key: indicator = self._type_key(indicator, itype) return self.table.delete(indicator) def put(self, indicator, value): if self.type_in_key: itype = value.get('type', None) indicator = self._type_key(indicator, itype) return self.table.put(indicator, value) def query(self, *args, **kwargs): if not self.type_in_key: return self.table.query(*args, **kwargs) if kwargs.get('include_value', False): return self._type_key_query_with_value(*args, **kwargs) return self._type_key_query(*args, **kwargs) def _type_key_query(self, *args, **kwargs): for key in self.table.query(*args, **kwargs): yield self._type_key_indicator(key) def _type_key_query_with_value(self, *args, **kwargs): for key, value in self.table.query(*args, **kwargs): yield self._type_key_indicator(key), value def _type_key(self, indicator, itype): if itype is None: raise RuntimeError('Type None in table with type in key') return u'{}::{}'.format(itype, indicator) def _type_key_indicator(self, key): return key.split('::', 1)[1] def _bptable_factory(name, truncate=False, type_in_key=False): table = Table(name, truncate=truncate) metadata = table.get_custom_metadata() if metadata is not None: version = metadata.get('version', None) if version is None: raise RuntimeError('{} - table with metadata but no version'.format(name)) if version == 1: return _BPTable_v1(table, type_in_key=type_in_key) raise RuntimeError('{} - table with unknown version: {}'.format(name, version)) # no metadata, could be a new table or an old one if table.num_indicators > 0: if type_in_key: raise RuntimeError('Old BPtable0 can\'t be used with multiple indicator types') return _BPTable_v0(table) # new table return _BPTable_v1(table, type_in_key=type_in_key) class IndicatorStatus(object): D_MASK = 1 F_MASK = 2 A_MASK = 4 W_MASK = 8 NX = 0 NFNANW = D_MASK XFNANW = D_MASK | F_MASK NFXANW = D_MASK | A_MASK XFXANW = D_MASK | F_MASK | A_MASK NFNAXW = D_MASK | W_MASK XFNAXW = D_MASK | F_MASK | W_MASK NFXAXW = D_MASK | A_MASK | W_MASK XFXAXW = D_MASK | F_MASK | A_MASK | W_MASK def __init__(self, indicator, attributes, itable, now, in_feed_threshold): self.state = 0 self.cv = itable.get(indicator, itype=attributes.get('type', None)) if self.cv is None: return self.state = self.state | IndicatorStatus.D_MASK if self.cv['_age_out'] < now: self.state = self.state | IndicatorStatus.A_MASK if self.cv['_last_run'] >= in_feed_threshold: self.state = self.state | IndicatorStatus.F_MASK if self.cv.get('_withdrawn', None) is not None: self.state = self.state | IndicatorStatus.W_MASK class BasePollerFT(base.BaseFT): """Implements base class for polling miners. **Config parameters** :source_name: name of the source. This is placed in the *sources* attribute of the generated indicators. Default: name of the node. :attributes: dictionary of attributes for the generated indicators. This dictionary is used as template for the value of the generated indicators. Default: empty :interval: polling interval in seconds. Default: 3600. :num_retries: in case of failure, how many times the miner should try to reach the source. If this number is exceeded, the miner waits until the next polling time to try again. Default: 2 :age_out: age out policies to apply to the indicators. Default: age out check interval 3600 seconds, sudden death enabled, default age out interval 30 days. **Age out policy** Age out policy is described by a dictionary with at least 3 keys: :interval: number of seconds between successive age out checks. :sudden_death: boolean, if *true* indicators are immediately aged out when they disappear from the feed. :default: age out interval. After this interval an indicator is aged out even if it is still present in the feed. If *null*, no age out interval is applied. Additional keys can be used to specify age out interval per indicator *type*. **Age out interval** Age out intervals have the following format:: + *base attribute* can be *last_seen*, if the age out interval should be calculated based on the last time the indicator was found in the feed, or *first_seen*, if instead the age out interval should be based on the time the indicator was first seen in the feed. If not specified *first_seen* is used. *interval* is the length of the interval expressed in seconds. Suffixes *d*, *h* and *m* can be used to specify days, hours or minutes. Example: Example config in YAML for a feed where indicators should be aged out only when they are removed from the feed:: source_name: example.persistent_feed interval: 600 age_out: default: null sudden_death: true interval: 300 attributes: type: IPv4 confidence: 100 share_level: green direction: inbound Example config in YAML for a feed where indicators are aged out when they disappear from the feed and 30 days after they have seen for the first time in the feed:: source_name: example.long_running_feed interval: 3600 age_out: default: first_seen+30d sudden_death: true interval: 1800 attributes: type: URL confidence: 50 share_level: green Example config in YAML for a feed where indicators are aged 30 days after they have seen for the last time in the feed:: source_name: example.delta_feed interval: 3600 age_out: default: last_seen+30d sudden_death: false interval: 1800 attributes: type: URL confidence: 50 share_level: green Args: name (str): node name, should be unique inside the graph chassis (object): parent chassis instance config (dict): node config. """ _AGE_OUT_BASES = None _DEFAULT_AGE_OUT_BASE = None def __init__(self, name, chassis, config): self.table = None self.agg_table = None self._actor_queue = gevent.queue.Queue(maxsize=128) self._actor_glet = None self._actor_commands_ts = collections.defaultdict(int) self._poll_glet = None self._age_out_glet = None self._emit_counter = 0 self.last_run = None self.last_successful_run = None self.last_ageout_run = None self._sub_state = None self._sub_state_message = None self.poll_event = gevent.event.Event() self.state_lock = RWLock() super(BasePollerFT, self).__init__(name, chassis, config) def configure(self): super(BasePollerFT, self).configure() self.source_name = self.config.get('source_name', self.name) self.attributes = self.config.get('attributes', {}) self.multiple_indicator_types = self.config.get('multiple_indicator_types', False) self.interval = self.config.get('interval', 3600) self.num_retries = self.config.get('num_retries', 2) self.aggregate_indicators = self.config.get('aggregate_indicators', False) self.aggregate_use_partial = self.config.get('aggregate_use_partial', False) _age_out = self.config.get('age_out', {}) self.age_out = { 'interval': _age_out.get('interval', 3600), 'sudden_death': _age_out.get('sudden_death', True), 'default': parse_age_out( _age_out.get('default', '30d'), age_out_bases=self._AGE_OUT_BASES, default_base=self._DEFAULT_AGE_OUT_BASE ) } for k, v in _age_out.iteritems(): if k in self.age_out: continue self.age_out[k] = parse_age_out(v) def _saved_state_restore(self, saved_state): self.last_run = saved_state.get('last_run', None) self.last_successful_run = saved_state.get( 'last_successful_run', None ) def _saved_state_create(self): return { 'last_run': self.last_run, 'last_successful_run': self.last_successful_run } def _saved_state_reset(self): self.last_successful_run = None self.last_run = None def _initialize_table(self, truncate=False): self.table = _bptable_factory( self.name, truncate=truncate, type_in_key=self.multiple_indicator_types ) def initialize(self): self._initialize_table() def rebuild(self): self._actor_queue.put( (utc_millisec(), 'rebuild') ) self._initialize_table(truncate=(self.last_checkpoint is None)) def reset(self): self._saved_state_reset() self._initialize_table(truncate=True) @base.BaseFT.state.setter def state(self, value): LOG.debug("%s - acquiring state write lock", self.name) self.state_lock.lock() # this is weird ! from stackoverflow 10810369 super(BasePollerFT, self.__class__).state.fset(self, value) self.state_lock.unlock() LOG.debug("%s - releasing state write lock", self.name) def _controlled_emit_update(self, indicator, value): self._emit_counter += 1 if self._emit_counter == 15937: gevent.sleep(0.001) self._emit_counter = 0 self.emit_update(indicator, value) def _controlled_emit_withdraw(self, indicator, value): self._emit_counter += 1 if self._emit_counter == 15937: gevent.sleep(0.001) self._emit_counter = 0 self.emit_withdraw(indicator=indicator, value=value) def _age_out(self): with self.state_lock: if self.state != ft_states.STARTED: return try: now = utc_millisec() for i, v in self.table.query(index='_age_out', to_key=now-1, include_value=True): LOG.debug('%s - %s %s aged out', self.name, i, v) if v.get('_withdrawn', None) is not None: continue self._controlled_emit_withdraw( indicator=i, value=v ) v['_withdrawn'] = now self.table.put(i, v) self.statistics['aged_out'] += 1 self.last_ageout_run = now except gevent.GreenletExit: raise except: LOG.exception('Exception in _age_out') def _flush(self): with self.state_lock: if self.state != ft_states.STARTED: return try: now = utc_millisec() for i, v in self.table.query(include_value=True): if v.get('_withdrawn', None) is not None: continue self._controlled_emit_withdraw( indicator=i, value=v ) v['_withdrawn'] = now v['_last_run'] = 0 self.table.put(i, v) self.statistics['flushed'] += 1 except gevent.GreenletExit: raise except: LOG.exception('Exception in _flush') def _sudden_death(self): if self.last_successful_run is None: return with self.state_lock: if self.state != ft_states.STARTED: return LOG.debug('checking sudden death for %d', self.last_successful_run) for i, v in self.table.query(index='_last_run', to_key=self.last_successful_run-1, include_value=True): LOG.debug('%s - %s %s sudden death', self.name, i, v) v['_age_out'] = self.last_successful_run-1 self.table.put(i, v) self.statistics['removed'] += 1 def _collect_garbage(self): now = utc_millisec() with self.state_lock: if self.state != ft_states.STARTED: return for i, v in self.table.query(index='_withdrawn', to_key=now, include_value=True): if v.get('_last_run', 0) >= (self.last_successful_run-1): continue LOG.debug('%s - %s collected', self.name, i) self.table.delete(i, itype=v.get('type', None)) self.statistics['garbage_collected'] += 1 def _compare_attributes(self, oa, na): default_attrs = ['sources', 'last_seen', 'first_seen'] default_attrs.extend(self.attributes.keys()) for k in oa: if k in default_attrs or k[0] in ('_', '$'): continue if k not in na: return False for k in na: if oa.get(k, None) != na[k]: return False return True def _update_attributes(self, current, _new, current_run, new_run): x = {k:v for k,v in current.iteritems() if k in _new or k in self.attributes or k in ['sources', 'last_seen', 'first_seen'] or k[0] in ('_', '$')} x.update(_new) return x def _aggregate_iterator(self, iterator): self.agg_table = _bptable_factory( '{}.aggregate-temp'.format(self.name), truncate=True, type_in_key=True ) for nitem, item in enumerate(iterator): if nitem != 0 and nitem % 1024 == 0: gevent.sleep(0.001) with self.state_lock: if self.state != ft_states.STARTED: LOG.info( '%s - state not STARTED, aggregation not performed', self.name ) self.agg_table.close() return False try: ipairs = self._process_item(item) except gevent.GreenletExit: raise except: self.statistics['error.parsing'] += 1 LOG.exception('%s - Exception parsing %s', self.name, item) continue for indicator, attributes in ipairs: self.agg_table.put(indicator, attributes) return True def _aggregate_process_item(self, item): return [item] def _polling_loop(self): LOG.info("Polling %s", self.name) now = utc_millisec() with self.state_lock: if self.state != ft_states.STARTED: LOG.info( '%s - state not STARTED, polling not performed', self.name ) return False iterator = self._build_iterator(now) if iterator is None: return False process_item = self._process_item aggregation_exc = None if self.aggregate_indicators: if self.agg_table is not None: self.agg_table.close() self.agg_table = None try: if not self._aggregate_iterator(iterator): return False except: # if aggregate_use_partial is True, we store exception # and handle partial results if not self.aggregate_use_partial: raise LOG.info('{} - Exception during aggregation, storing'.format(self.name)) aggregation_exc = sys.exc_info() process_item = self._aggregate_process_item iterator = self.agg_table.query(include_value=True) for nitem, item in enumerate(iterator): if nitem != 0 and nitem % 1024 == 0: gevent.sleep(0.001) with self.state_lock: if self.state != ft_states.STARTED: break try: ipairs = process_item(item) except gevent.GreenletExit: raise except: self.statistics['error.parsing'] += 1 LOG.exception('%s - Exception parsing %s', self.name, item) continue for indicator, attributes in ipairs: if indicator is None: LOG.debug('%s - indicator is None for item %s', self.name, item) continue in_feed_threshold = self.last_successful_run if in_feed_threshold is None: in_feed_threshold = now - self.interval*1000 istatus = IndicatorStatus( indicator=indicator, attributes=attributes, itable=self.table, now=now, in_feed_threshold=in_feed_threshold ) if istatus.state in [IndicatorStatus.NX, IndicatorStatus.NFNANW, IndicatorStatus.NFXANW, IndicatorStatus.NFXAXW, IndicatorStatus.NFNAXW]: v = copy.copy(self.attributes) v['sources'] = [self.source_name] v['last_seen'] = now v['first_seen'] = now v['_last_run'] = now v.update(attributes) v['_age_out'] = self._calc_age_out(indicator, v) self.statistics['added'] += 1 self.table.put(indicator, v) if v['_age_out'] >= now: self._controlled_emit_update(indicator, v) LOG.debug('%s - added %s %s', self.name, indicator, v) elif istatus.state == IndicatorStatus.XFNANW: v = istatus.cv eq = self._compare_attributes(v, attributes) old_last_run = v['_last_run'] v['_last_run'] = now v = self._update_attributes( v, attributes, old_last_run, now ) v['_age_out'] = self._calc_age_out(indicator, v) self.table.put(indicator, v) # emit updates if different and not aged out if not eq and v['_age_out'] >= now: self._controlled_emit_update(indicator, v) elif istatus.state == IndicatorStatus.XFXANW: v = istatus.cv v['_last_run'] = now self.table.put(indicator, v) elif istatus.state in [IndicatorStatus.XFXAXW, IndicatorStatus.XFNAXW]: v = istatus.cv v['_last_run'] = now v['_withdrawn'] = now self.table.put(indicator, v) else: LOG.error('%s - indicator state unhandled: %s', self.name, istatus.state) continue if self.agg_table is not None: iterator.close() self.agg_table.close() self.agg_table = None shutil.rmtree('{}.aggregate-temp'.format(self.name)) if aggregation_exc is not None: LOG.info('{} - Reraising exception happened during aggregation'.format(self.name)) raise aggregation_exc[0], aggregation_exc[1], aggregation_exc[2] return True def _rebuild(self): with self.state_lock: if self.state != ft_states.STARTED: return self.sub_state = 'REBUILDING' for i, v in self.table.query(include_value=True): self._controlled_emit_update(i, v) def _poll(self): tryn = 0 while tryn < self.num_retries: lastrun = utc_millisec() try: self.sub_state = 'POLLING' performed = self._polling_loop() if performed: self.last_successful_run = lastrun _result = 'SUCCESS' break except gevent.GreenletExit: raise except Exception as e: try: _error_msg = str(e) except UnicodeDecodeError: _error_msg = repr(e) _result = ('ERROR', _error_msg) self.statistics['error.polling'] += 1 LOG.exception("Exception in polling loop for %s: %s", self.name, str(e)) tryn += 1 gevent.sleep(random.randint(1, 5)) LOG.debug("%s - End of polling - #indicators: %d", self.name, self.table.length()) self.last_run = lastrun self.sub_state = _result def _actor_loop(self): while True: timestamp, command = self._actor_queue.get() LOG.info('%s - command: %d %s', self.name, timestamp, command) try: last_ts = self._actor_commands_ts[command] if timestamp < last_ts: LOG.info( '%s - command %s, old timestamp - ignored', self.name, command ) continue if command == 'poll': self._poll() elif command == 'age_out': self._age_out() elif command == 'sudden_death': self._sudden_death() elif command == 'gc': self._collect_garbage() elif command == 'rebuild': self._rebuild() elif command == 'flush': self._flush() else: LOG.error('%s - unknown command: %s', self.name, command) except gevent.GreenletExit: raise except: LOG.exception( '%s - exception executing command %s', self.name, command ) self._actor_commands_ts[command] = utc_millisec() def _poll_loop(self): # wait to poll until after the first ageout run while self.last_ageout_run is None: gevent.sleep(1) # if last_run is not None it means we have restored # a previous state, wait until poll time if self.last_run is not None: self.sub_state = 'WAITING' LOG.info( '%s - restored last run, waiting until the next poll time', self.name ) try: self._huppable_wait( (self.last_run+self.interval*1000)-utc_millisec() ) except gevent.GreenletExit: return while True: with self.state_lock: if self.state != ft_states.STARTED: break self._actor_queue.put( (utc_millisec(), 'poll') ) if self.age_out['sudden_death']: self._actor_queue.put( (utc_millisec(), 'sudden_death') ) self._actor_queue.put( (utc_millisec(), 'age_out') ) self._actor_queue.put( (utc_millisec(), 'gc') ) try: self._huppable_wait(self.interval*1000) except gevent.GreenletExit: break def _age_out_loop(self): while True: with self.state_lock: if self.state != ft_states.STARTED: break self._actor_queue.put( (utc_millisec(), 'age_out') ) if self.age_out['interval'] is None: break try: gevent.sleep(self.age_out['interval']) except gevent.GreenletExit: break def _calc_age_out(self, indicator, attributes): t = attributes.get('type', None) if t is None or t not in self.age_out: sel = self.age_out['default'] else: sel = self.age_out[t] if sel is None: return _MAX_AGE_OUT b = attributes[sel['base']] return b + sel['offset'] def _huppable_wait(self, deltat): while deltat < 0: LOG.warning( 'Time for processing exceeded interval for %s', self.name ) deltat += self.interval*1000 LOG.info('hup is clear: %r', self.poll_event.is_set()) hup_called = self.poll_event.wait(timeout=deltat/1000.0) if hup_called: LOG.debug('%s - clearing poll event', self.name) self.poll_event.clear() def mgmtbus_status(self): result = super(BasePollerFT, self).mgmtbus_status() result['last_run'] = self.last_run result['last_successful_run'] = self.last_successful_run result['sub_state'] = self.sub_state[0] if self.sub_state[1] is not None: result['sub_state_message'] = self.sub_state[1] return result def mgmtbus_signal(self, source=None, signal=None, **kwargs): if signal != 'flush': super(BasePollerFT, self).mgmtbus_signal( source=source, signal=signal, **kwargs ) self._actor_queue.put( (utc_millisec(), 'flush') ) self._actor_queue.put( (utc_millisec(), 'gc') ) @property def sub_state(self): return (self._sub_state, self._sub_state_message) @sub_state.setter def sub_state(self, value): if (type(value) == tuple): self._sub_state = value[0] self._sub_state_message = value[1] else: self._sub_state = value self._sub_state_message = None self.publish_status(force=True) def hup(self, source=None): LOG.info('%s - hup received, force polling', self.name) self.poll_event.set() def length(self, source=None): return self.table.length() def start(self): super(BasePollerFT, self).start() if self._actor_glet is not None: return self._actor_glet = gevent.spawn( self._actor_loop ) self._poll_glet = gevent.spawn_later( random.randint(0, 2), self._poll_loop ) self._age_out_glet = gevent.spawn( self._age_out_loop ) def stop(self): super(BasePollerFT, self).stop() if self._actor_glet is None: return self._actor_glet.kill() self._poll_glet.kill() self._age_out_glet.kill() self.table.close() LOG.info("%s - # indicators: %d", self.name, self.table.length()) @staticmethod def gc(name, config=None): base.BaseFT.gc(name, config=config) shutil.rmtree(name, ignore_errors=True) shutil.rmtree('{}.aggregate-temp'.format(name), ignore_errors=True) ================================================ FILE: minemeld/ft/cif.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import os import arrow import ujson import yaml import cifsdk.client import cifsdk.constants from . import basepoller LOG = logging.getLogger(__name__) class Feed(basepoller.BasePollerFT): def configure(self): super(Feed, self).configure() self.token = None self.remote = self.config.get('remote', None) self.verify_cert = self.config.get('verify_cert', True) self.filters = self.config.get('filters', None) self.initial_days = self.config.get('initial_days', 7) self.prefix = self.config.get('prefix', 'cif') self.fields = self.config.get('fields', cifsdk.constants.FIELDS) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.token = sconfig.get('token', None) if self.token is not None: LOG.info('%s - token set', self.name) self.remote = sconfig.get('remote', self.remote) self.verify_cert = sconfig.get('verify_cert', self.verify_cert) filters = sconfig.get('filters', self.filters) if filters is not None: if self.filters is not None: self.filters.update(filters) else: self.filters = filters def _process_item(self, item): indicator = item.get('observable', None) if indicator is None: LOG.error('%s - no observable in item', self.name) return [[None, None]] otype = item.get('otype', None) if otype is None: LOG.error('%s - no otype in item', self.name) return [[None, None]] if otype == 'ipv4': type_ = 'IPv4' elif otype == 'ipv6': type_ = 'IPv6' elif otype == 'fqdn': type_ = 'domain' elif otype == 'url': type_ = 'URL' else: LOG.error('%s - unahndled otype %s', self.name, otype) return [[None, None]] attributes = { 'type': type_ } for field in self.fields: if field in ['observable', 'otype', 'confidence']: continue if field not in item: continue attributes['%s_%s' % (self.prefix, field)] = item[field] if 'confidence' in item: attributes['confidence'] = item['confidence'] LOG.debug('%s - %s: %s', self.name, indicator, attributes) return [[indicator, attributes]] def _build_iterator(self, now): if self.token is None or self.remote is None or self.filters is None: LOG.info( '%s - token, remote or filters not set, poll not performed', self.name ) raise RuntimeError( '%s - token, remote or filters not set, poll not performed' % self.name ) filters = {} filters.update(self.filters) days = filters.pop('days', self.initial_days) now = arrow.get(now/1000.0) filters['reporttimeend'] = '{0}Z'.format( now.format('YYYY-MM-DDTHH:mm:ss') ) if self.last_successful_run is None: filters['reporttime'] = '{0}Z'.format( now.shift(days=-days).format('YYYY-MM-DDTHH:mm:ss') ) else: filters['reporttime'] = '{0}Z'.format( arrow.get(self.last_successful_run/1000.0).format('YYYY-MM-DDTHH:mm:ss') ) LOG.debug('%s - filters: %s', self.name, filters) cifclient = cifsdk.client.Client( token=self.token, remote=self.remote, verify_ssl=self.verify_cert, timeout=900 ) try: ret = cifclient.search(filters=filters, decode=False) except SystemExit as e: raise RuntimeError(str(e)) return ujson.loads(ret) def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(Feed, self).hup(source=source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass ================================================ FILE: minemeld/ft/ciscoise.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import os import yaml import minemeld.packages.ise.ers from . import basepoller LOG = logging.getLogger(__name__) class ErsSgt(basepoller.BasePollerFT): def configure(self): super(ErsSgt, self).configure() self.kwargs = {} for x in ['hostname', 'username', 'password', 'verify_cert', 'timeout']: if x == 'verify_cert': self.kwargs['verify'] = self.config.get(x, None) else: self.kwargs[x] = self.config.get(x, None) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() self.prefix = self.config.get('prefix', 'ise_sgt') d = self.kwargs.copy() if d['password']: d['password'] = '*' * 6 LOG.debug('%s prefix: %s', d, self.prefix) def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except IOError as e: LOG.info('%s - No side config: %s', self.name, e) return if sconfig is None: LOG.info('%s - Empty side config: %s', self.name, self.side_config_path) return for x in ['hostname', 'username', 'password', 'verify_cert', 'timeout']: v = sconfig.get(x, None) if v is not None and x == 'verify_cert': self.kwargs['verify'] = v elif v is not None: self.kwargs[x] = v def _process_item(self, item): return [[item['ip'], {'type': 'IPv4', self.prefix: item['sgt']}]] def _build_iterator(self, now): def indicators(ips_sgts_map): LOG.debug('SGT indicators #%d %s', len(api.ips_sgts_map), api.ips_sgts_map) for item in ips_sgts_map: yield {'ip': item, 'sgt': ips_sgts_map[item]} try: api = minemeld.packages.ise.ers.IseErs(**self.kwargs) except minemeld.packages.ise.ers.IseErsError as e: # missing arguments x = '%s: poll not performed: %s' % (self.name, e) LOG.info('%s', x) raise RuntimeError(x) api.sgts_ips_map() return indicators(api.ips_sgts_map) def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(ErsSgt, self).hup(source=source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass ================================================ FILE: minemeld/ft/cofense.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements minemeld.ft.cofense.Triage, the Miner node for Cofense Triage API. """ import os import yaml import requests import itertools import logging import math import pytz from datetime import timedelta from urlparse import urljoin from . import basepoller from .utils import interval_in_sec, EPOCH LOG = logging.getLogger(__name__) _TRIAGE_API_CALL_PATH = '/api/public/v1/triage_threat_indicators' _API_USER_AGENT = 'Cofense Intelligence (minemeld)' _RESULTS_PER_PAGE = 50 class Triage(basepoller.BasePollerFT): def configure(self): super(Triage, self).configure() self.polling_timeout = self.config.get('polling_timeout', 20) self.prefix = self.config.get('prefix', 'cofense') initial_interval = self.config.get('initial_interval', '30d') self.initial_interval = interval_in_sec(initial_interval) if self.initial_interval is None: LOG.error( '%s - wrong initial_interval format: %s', self.name, initial_interval ) self.initial_interval = interval_in_sec('30d') self.source_name = self.config.get('source_name', 'cofense.triage') self.headers = {'user-agent': _API_USER_AGENT} self.confidence_map = self.config.get('confidence_map', { 'Malicious': 100, 'Suspicious': 50 }) self.verify_cert = self.config.get('verify_cert', True) self.api_domain = self.config.get('api_domain', None) self.api_account = self.config.get('api_account', None) self.api_token = self.config.get('api_token', None) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return api_domain = sconfig.get('api_domain', None) if api_domain is not None: self.api_domain = api_domain LOG.info('%s - API Domain set', self.name) api_account = sconfig.get('api_account', None) if api_account is not None: self.api_account = api_account LOG.info('%s - API Account set', self.name) api_token = sconfig.get('api_token', None) if api_token is not None: self.api_token = api_token LOG.info('%s - API Token set', self.name) verify_cert = sconfig.get('verify_cert', None) if verify_cert is not None: self.verify_cert = verify_cert LOG.info('%s - Verify Cert set', self.name) def _process_item(self, item): LOG.debug('{} - item: {}'.format(self.name, item)) report_id = item.get('report_id', None) type_ = item.get('threat_key', None) indicator = item.get('threat_value', None) level = item.get('threat_level', None) if type_ is None or indicator is None: LOG.error('{} - entry with no value or type: {!r}'.format(self.name, item)) return [] if level not in self.confidence_map: LOG.info('{} - threat_level {} not in cofidence map: indicator ignored'.format(self.name, level)) return [] if type_ == 'URL': type_ = 'URL' elif type_ == 'Domain': type_ = 'Domain' elif type_ == 'MD5': type_ = 'md5' elif type_ == 'SHA256': type_ = 'sha256' else: LOG.error('{} - unknown indicator type: {!r}'.format(self.name, item)) return [] value = dict(type=type_) if report_id is not None: value['{}_report_id'.format(self.prefix)] = report_id if level is not None: value['{}_threat_level'.format(self.prefix)] = level value['confidence'] = self.confidence_map[level] return [[indicator, value]] def _build_iterator(self, now): if self.api_domain is None or self.api_account is None or self.api_token is None: raise RuntimeError('%s - credentials not set' % self.name) poll_start = self.last_successful_run if self.last_successful_run is None: poll_start = now - (self.initial_interval * 1000) dt_poll_start = EPOCH + timedelta(milliseconds=poll_start) LOG.debug('{} - polling start: {}'.format(self.name, dt_poll_start)) num_of_pages = self._check_number_of_pages(dt_poll_start) LOG.info("{} - polling: start date: {!r} number of pages: {!r}".format( self.name, dt_poll_start, num_of_pages )) return self._iterate_over_pages(dt_poll_start, num_of_pages) def _check_number_of_pages(self, dt_poll_start): r = self._perform_api_call(dt_poll_start) total_entries = r.headers['Total'] LOG.info('{} - polling total entries: {}'.format(self.name, total_entries)) return int(math.ceil(int(total_entries)/float(_RESULTS_PER_PAGE))) def _iterate_over_pages(self, start_date, num_of_pages): for page_num in xrange(1, num_of_pages+1): r = self._perform_api_call(start_date, page_num) processed_data = r.json() for entry in processed_data: yield entry def _perform_api_call(self, start_date, page=None): headers = self.headers.copy() headers['Authorization'] = 'Token token={}:{}'.format(self.api_account, self.api_token) params = { "per_page": _RESULTS_PER_PAGE, "start_date": start_date.strftime('%Y-%m-%dT%H:%M') } if page is not None: params['page'] = page request_url = urljoin(self.api_domain, _TRIAGE_API_CALL_PATH) r = requests.get(request_url, params=params, verify=self.verify_cert, headers=headers ) r.raise_for_status() return r def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(Triage, self).hup(source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass ================================================ FILE: minemeld/ft/condition/BoolExpr.g4 ================================================ grammar BoolExpr; @header { # flake8: noqa } booleanExpression : expression comparator value ; expression : JAVASCRIPTIDENTIFIER | functionExpression ; functionExpression : JAVASCRIPTIDENTIFIER ( noArgs | oneOrMoreArgs ) ; noArgs : '(' ')' ; oneOrMoreArgs : '(' expression (',' (expression|value))* ')' ; JAVASCRIPTIDENTIFIER : [a-zA-Z_$][a-zA-Z_$0-9]* ; comparator : '<' | '<=' | '==' | '>=' | '>' | '!=' ; value : STRING | NUMBER | 'true' | 'false' | 'null' ; STRING : '"' (~["\\])* '"' | '\'' (~['\\])* '\'' ; NUMBER : '-'? INT ; fragment INT : '0' | [1-9] [0-9]* ; // no leading zeros WS : [ \t\n\r]+ -> skip ; ================================================ FILE: minemeld/ft/condition/BoolExpr.tokens ================================================ T__0=1 T__1=2 T__2=3 T__3=4 T__4=5 T__5=6 T__6=7 T__7=8 T__8=9 T__9=10 T__10=11 T__11=12 JAVASCRIPTIDENTIFIER=13 STRING=14 NUMBER=15 WS=16 '('=1 ')'=2 ','=3 '<'=4 '<='=5 '=='=6 '>='=7 '>'=8 '!='=9 'true'=10 'false'=11 'null'=12 ================================================ FILE: minemeld/ft/condition/BoolExprLexer.py ================================================ # Generated from BoolExpr.g4 by ANTLR 4.7.1 # encoding: utf-8 from __future__ import print_function from antlr4 import * from io import StringIO import sys # flake8: noqa def serializedATN(): with StringIO() as buf: buf.write(u"\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\2") buf.write(u"\22z\b\1\4\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7") buf.write(u"\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t\13\4\f\t\f\4\r\t") buf.write(u"\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22") buf.write(u"\3\2\3\2\3\3\3\3\3\4\3\4\3\5\3\5\3\6\3\6\3\6\3\7\3\7") buf.write(u"\3\7\3\b\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\13\3\13\3\13\3") buf.write(u"\13\3\13\3\f\3\f\3\f\3\f\3\f\3\f\3\r\3\r\3\r\3\r\3\r") buf.write(u"\3\16\3\16\7\16N\n\16\f\16\16\16Q\13\16\3\17\3\17\7\17") buf.write(u"U\n\17\f\17\16\17X\13\17\3\17\3\17\3\17\7\17]\n\17\f") buf.write(u"\17\16\17`\13\17\3\17\5\17c\n\17\3\20\5\20f\n\20\3\20") buf.write(u"\3\20\3\21\3\21\3\21\7\21m\n\21\f\21\16\21p\13\21\5\21") buf.write(u"r\n\21\3\22\6\22u\n\22\r\22\16\22v\3\22\3\22\2\2\23\3") buf.write(u"\3\5\4\7\5\t\6\13\7\r\b\17\t\21\n\23\13\25\f\27\r\31") buf.write(u"\16\33\17\35\20\37\21!\2#\22\3\2\t\6\2&&C\\aac|\7\2&") buf.write(u"&\62;C\\aac|\4\2$$^^\4\2))^^\3\2\63;\3\2\62;\5\2\13\f") buf.write(u"\17\17\"\"\2\u0080\2\3\3\2\2\2\2\5\3\2\2\2\2\7\3\2\2") buf.write(u"\2\2\t\3\2\2\2\2\13\3\2\2\2\2\r\3\2\2\2\2\17\3\2\2\2") buf.write(u"\2\21\3\2\2\2\2\23\3\2\2\2\2\25\3\2\2\2\2\27\3\2\2\2") buf.write(u"\2\31\3\2\2\2\2\33\3\2\2\2\2\35\3\2\2\2\2\37\3\2\2\2") buf.write(u"\2#\3\2\2\2\3%\3\2\2\2\5\'\3\2\2\2\7)\3\2\2\2\t+\3\2") buf.write(u"\2\2\13-\3\2\2\2\r\60\3\2\2\2\17\63\3\2\2\2\21\66\3\2") buf.write(u"\2\2\238\3\2\2\2\25;\3\2\2\2\27@\3\2\2\2\31F\3\2\2\2") buf.write(u"\33K\3\2\2\2\35b\3\2\2\2\37e\3\2\2\2!q\3\2\2\2#t\3\2") buf.write(u"\2\2%&\7*\2\2&\4\3\2\2\2\'(\7+\2\2(\6\3\2\2\2)*\7.\2") buf.write(u"\2*\b\3\2\2\2+,\7>\2\2,\n\3\2\2\2-.\7>\2\2./\7?\2\2/") buf.write(u"\f\3\2\2\2\60\61\7?\2\2\61\62\7?\2\2\62\16\3\2\2\2\63") buf.write(u"\64\7@\2\2\64\65\7?\2\2\65\20\3\2\2\2\66\67\7@\2\2\67") buf.write(u"\22\3\2\2\289\7#\2\29:\7?\2\2:\24\3\2\2\2;<\7v\2\2<=") buf.write(u"\7t\2\2=>\7w\2\2>?\7g\2\2?\26\3\2\2\2@A\7h\2\2AB\7c\2") buf.write(u"\2BC\7n\2\2CD\7u\2\2DE\7g\2\2E\30\3\2\2\2FG\7p\2\2GH") buf.write(u"\7w\2\2HI\7n\2\2IJ\7n\2\2J\32\3\2\2\2KO\t\2\2\2LN\t\3") buf.write(u"\2\2ML\3\2\2\2NQ\3\2\2\2OM\3\2\2\2OP\3\2\2\2P\34\3\2") buf.write(u"\2\2QO\3\2\2\2RV\7$\2\2SU\n\4\2\2TS\3\2\2\2UX\3\2\2\2") buf.write(u"VT\3\2\2\2VW\3\2\2\2WY\3\2\2\2XV\3\2\2\2Yc\7$\2\2Z^\7") buf.write(u")\2\2[]\n\5\2\2\\[\3\2\2\2]`\3\2\2\2^\\\3\2\2\2^_\3\2") buf.write(u"\2\2_a\3\2\2\2`^\3\2\2\2ac\7)\2\2bR\3\2\2\2bZ\3\2\2\2") buf.write(u"c\36\3\2\2\2df\7/\2\2ed\3\2\2\2ef\3\2\2\2fg\3\2\2\2g") buf.write(u"h\5!\21\2h \3\2\2\2ir\7\62\2\2jn\t\6\2\2km\t\7\2\2lk") buf.write(u"\3\2\2\2mp\3\2\2\2nl\3\2\2\2no\3\2\2\2or\3\2\2\2pn\3") buf.write(u"\2\2\2qi\3\2\2\2qj\3\2\2\2r\"\3\2\2\2su\t\b\2\2ts\3\2") buf.write(u"\2\2uv\3\2\2\2vt\3\2\2\2vw\3\2\2\2wx\3\2\2\2xy\b\22\2") buf.write(u"\2y$\3\2\2\2\13\2OV^benqv\3\b\2\2") return buf.getvalue() class BoolExprLexer(Lexer): atn = ATNDeserializer().deserialize(serializedATN()) decisionsToDFA = [ DFA(ds, i) for i, ds in enumerate(atn.decisionToState) ] T__0 = 1 T__1 = 2 T__2 = 3 T__3 = 4 T__4 = 5 T__5 = 6 T__6 = 7 T__7 = 8 T__8 = 9 T__9 = 10 T__10 = 11 T__11 = 12 JAVASCRIPTIDENTIFIER = 13 STRING = 14 NUMBER = 15 WS = 16 channelNames = [ u"DEFAULT_TOKEN_CHANNEL", u"HIDDEN" ] modeNames = [ u"DEFAULT_MODE" ] literalNames = [ u"", u"'('", u"')'", u"','", u"'<'", u"'<='", u"'=='", u"'>='", u"'>'", u"'!='", u"'true'", u"'false'", u"'null'" ] symbolicNames = [ u"", u"JAVASCRIPTIDENTIFIER", u"STRING", u"NUMBER", u"WS" ] ruleNames = [ u"T__0", u"T__1", u"T__2", u"T__3", u"T__4", u"T__5", u"T__6", u"T__7", u"T__8", u"T__9", u"T__10", u"T__11", u"JAVASCRIPTIDENTIFIER", u"STRING", u"NUMBER", u"INT", u"WS" ] grammarFileName = u"BoolExpr.g4" def __init__(self, input=None, output=sys.stdout): super(BoolExprLexer, self).__init__(input, output=output) self.checkVersion("4.7.1") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None ================================================ FILE: minemeld/ft/condition/BoolExprLexer.tokens ================================================ T__0=1 T__1=2 T__2=3 T__3=4 T__4=5 T__5=6 T__6=7 T__7=8 T__8=9 T__9=10 T__10=11 T__11=12 JAVASCRIPTIDENTIFIER=13 STRING=14 NUMBER=15 WS=16 '('=1 ')'=2 ','=3 '<'=4 '<='=5 '=='=6 '>='=7 '>'=8 '!='=9 'true'=10 'false'=11 'null'=12 ================================================ FILE: minemeld/ft/condition/BoolExprListener.py ================================================ # Generated from BoolExpr.g4 by ANTLR 4.7.1 from antlr4 import * # flake8: noqa # This class defines a complete listener for a parse tree produced by BoolExprParser. class BoolExprListener(ParseTreeListener): # Enter a parse tree produced by BoolExprParser#booleanExpression. def enterBooleanExpression(self, ctx): pass # Exit a parse tree produced by BoolExprParser#booleanExpression. def exitBooleanExpression(self, ctx): pass # Enter a parse tree produced by BoolExprParser#expression. def enterExpression(self, ctx): pass # Exit a parse tree produced by BoolExprParser#expression. def exitExpression(self, ctx): pass # Enter a parse tree produced by BoolExprParser#functionExpression. def enterFunctionExpression(self, ctx): pass # Exit a parse tree produced by BoolExprParser#functionExpression. def exitFunctionExpression(self, ctx): pass # Enter a parse tree produced by BoolExprParser#noArgs. def enterNoArgs(self, ctx): pass # Exit a parse tree produced by BoolExprParser#noArgs. def exitNoArgs(self, ctx): pass # Enter a parse tree produced by BoolExprParser#oneOrMoreArgs. def enterOneOrMoreArgs(self, ctx): pass # Exit a parse tree produced by BoolExprParser#oneOrMoreArgs. def exitOneOrMoreArgs(self, ctx): pass # Enter a parse tree produced by BoolExprParser#comparator. def enterComparator(self, ctx): pass # Exit a parse tree produced by BoolExprParser#comparator. def exitComparator(self, ctx): pass # Enter a parse tree produced by BoolExprParser#value. def enterValue(self, ctx): pass # Exit a parse tree produced by BoolExprParser#value. def exitValue(self, ctx): pass ================================================ FILE: minemeld/ft/condition/BoolExprParser.py ================================================ # Generated from BoolExpr.g4 by ANTLR 4.7.1 # encoding: utf-8 from __future__ import print_function from antlr4 import * from io import StringIO import sys # flake8: noqa def serializedATN(): with StringIO() as buf: buf.write(u"\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\3") buf.write(u"\22\63\4\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t") buf.write(u"\7\4\b\t\b\3\2\3\2\3\2\3\2\3\3\3\3\5\3\27\n\3\3\4\3\4") buf.write(u"\3\4\5\4\34\n\4\3\5\3\5\3\5\3\6\3\6\3\6\3\6\3\6\5\6&") buf.write(u"\n\6\7\6(\n\6\f\6\16\6+\13\6\3\6\3\6\3\7\3\7\3\b\3\b") buf.write(u"\3\b\2\2\t\2\4\6\b\n\f\16\2\4\3\2\6\13\4\2\f\16\20\21") buf.write(u"\2/\2\20\3\2\2\2\4\26\3\2\2\2\6\30\3\2\2\2\b\35\3\2\2") buf.write(u"\2\n \3\2\2\2\f.\3\2\2\2\16\60\3\2\2\2\20\21\5\4\3\2") buf.write(u"\21\22\5\f\7\2\22\23\5\16\b\2\23\3\3\2\2\2\24\27\7\17") buf.write(u"\2\2\25\27\5\6\4\2\26\24\3\2\2\2\26\25\3\2\2\2\27\5\3") buf.write(u"\2\2\2\30\33\7\17\2\2\31\34\5\b\5\2\32\34\5\n\6\2\33") buf.write(u"\31\3\2\2\2\33\32\3\2\2\2\34\7\3\2\2\2\35\36\7\3\2\2") buf.write(u"\36\37\7\4\2\2\37\t\3\2\2\2 !\7\3\2\2!)\5\4\3\2\"%\7") buf.write(u"\5\2\2#&\5\4\3\2$&\5\16\b\2%#\3\2\2\2%$\3\2\2\2&(\3\2") buf.write(u"\2\2\'\"\3\2\2\2(+\3\2\2\2)\'\3\2\2\2)*\3\2\2\2*,\3\2") buf.write(u"\2\2+)\3\2\2\2,-\7\4\2\2-\13\3\2\2\2./\t\2\2\2/\r\3\2") buf.write(u"\2\2\60\61\t\3\2\2\61\17\3\2\2\2\6\26\33%)") return buf.getvalue() class BoolExprParser ( Parser ): grammarFileName = "BoolExpr.g4" atn = ATNDeserializer().deserialize(serializedATN()) decisionsToDFA = [ DFA(ds, i) for i, ds in enumerate(atn.decisionToState) ] sharedContextCache = PredictionContextCache() literalNames = [ u"", u"'('", u"')'", u"','", u"'<'", u"'<='", u"'=='", u"'>='", u"'>'", u"'!='", u"'true'", u"'false'", u"'null'" ] symbolicNames = [ u"", u"", u"", u"", u"", u"", u"", u"", u"", u"", u"", u"", u"", u"JAVASCRIPTIDENTIFIER", u"STRING", u"NUMBER", u"WS" ] RULE_booleanExpression = 0 RULE_expression = 1 RULE_functionExpression = 2 RULE_noArgs = 3 RULE_oneOrMoreArgs = 4 RULE_comparator = 5 RULE_value = 6 ruleNames = [ u"booleanExpression", u"expression", u"functionExpression", u"noArgs", u"oneOrMoreArgs", u"comparator", u"value" ] EOF = Token.EOF T__0=1 T__1=2 T__2=3 T__3=4 T__4=5 T__5=6 T__6=7 T__7=8 T__8=9 T__9=10 T__10=11 T__11=12 JAVASCRIPTIDENTIFIER=13 STRING=14 NUMBER=15 WS=16 def __init__(self, input, output=sys.stdout): super(BoolExprParser, self).__init__(input, output=output) self.checkVersion("4.7.1") self._interp = ParserATNSimulator(self, self.atn, self.decisionsToDFA, self.sharedContextCache) self._predicates = None class BooleanExpressionContext(ParserRuleContext): def __init__(self, parser, parent=None, invokingState=-1): super(BoolExprParser.BooleanExpressionContext, self).__init__(parent, invokingState) self.parser = parser def expression(self): return self.getTypedRuleContext(BoolExprParser.ExpressionContext,0) def comparator(self): return self.getTypedRuleContext(BoolExprParser.ComparatorContext,0) def value(self): return self.getTypedRuleContext(BoolExprParser.ValueContext,0) def getRuleIndex(self): return BoolExprParser.RULE_booleanExpression def enterRule(self, listener): if hasattr(listener, "enterBooleanExpression"): listener.enterBooleanExpression(self) def exitRule(self, listener): if hasattr(listener, "exitBooleanExpression"): listener.exitBooleanExpression(self) def booleanExpression(self): localctx = BoolExprParser.BooleanExpressionContext(self, self._ctx, self.state) self.enterRule(localctx, 0, self.RULE_booleanExpression) try: self.enterOuterAlt(localctx, 1) self.state = 14 self.expression() self.state = 15 self.comparator() self.state = 16 self.value() except RecognitionException as re: localctx.exception = re self._errHandler.reportError(self, re) self._errHandler.recover(self, re) finally: self.exitRule() return localctx class ExpressionContext(ParserRuleContext): def __init__(self, parser, parent=None, invokingState=-1): super(BoolExprParser.ExpressionContext, self).__init__(parent, invokingState) self.parser = parser def JAVASCRIPTIDENTIFIER(self): return self.getToken(BoolExprParser.JAVASCRIPTIDENTIFIER, 0) def functionExpression(self): return self.getTypedRuleContext(BoolExprParser.FunctionExpressionContext,0) def getRuleIndex(self): return BoolExprParser.RULE_expression def enterRule(self, listener): if hasattr(listener, "enterExpression"): listener.enterExpression(self) def exitRule(self, listener): if hasattr(listener, "exitExpression"): listener.exitExpression(self) def expression(self): localctx = BoolExprParser.ExpressionContext(self, self._ctx, self.state) self.enterRule(localctx, 2, self.RULE_expression) try: self.state = 20 self._errHandler.sync(self) la_ = self._interp.adaptivePredict(self._input,0,self._ctx) if la_ == 1: self.enterOuterAlt(localctx, 1) self.state = 18 self.match(BoolExprParser.JAVASCRIPTIDENTIFIER) pass elif la_ == 2: self.enterOuterAlt(localctx, 2) self.state = 19 self.functionExpression() pass except RecognitionException as re: localctx.exception = re self._errHandler.reportError(self, re) self._errHandler.recover(self, re) finally: self.exitRule() return localctx class FunctionExpressionContext(ParserRuleContext): def __init__(self, parser, parent=None, invokingState=-1): super(BoolExprParser.FunctionExpressionContext, self).__init__(parent, invokingState) self.parser = parser def JAVASCRIPTIDENTIFIER(self): return self.getToken(BoolExprParser.JAVASCRIPTIDENTIFIER, 0) def noArgs(self): return self.getTypedRuleContext(BoolExprParser.NoArgsContext,0) def oneOrMoreArgs(self): return self.getTypedRuleContext(BoolExprParser.OneOrMoreArgsContext,0) def getRuleIndex(self): return BoolExprParser.RULE_functionExpression def enterRule(self, listener): if hasattr(listener, "enterFunctionExpression"): listener.enterFunctionExpression(self) def exitRule(self, listener): if hasattr(listener, "exitFunctionExpression"): listener.exitFunctionExpression(self) def functionExpression(self): localctx = BoolExprParser.FunctionExpressionContext(self, self._ctx, self.state) self.enterRule(localctx, 4, self.RULE_functionExpression) try: self.enterOuterAlt(localctx, 1) self.state = 22 self.match(BoolExprParser.JAVASCRIPTIDENTIFIER) self.state = 25 self._errHandler.sync(self) la_ = self._interp.adaptivePredict(self._input,1,self._ctx) if la_ == 1: self.state = 23 self.noArgs() pass elif la_ == 2: self.state = 24 self.oneOrMoreArgs() pass except RecognitionException as re: localctx.exception = re self._errHandler.reportError(self, re) self._errHandler.recover(self, re) finally: self.exitRule() return localctx class NoArgsContext(ParserRuleContext): def __init__(self, parser, parent=None, invokingState=-1): super(BoolExprParser.NoArgsContext, self).__init__(parent, invokingState) self.parser = parser def getRuleIndex(self): return BoolExprParser.RULE_noArgs def enterRule(self, listener): if hasattr(listener, "enterNoArgs"): listener.enterNoArgs(self) def exitRule(self, listener): if hasattr(listener, "exitNoArgs"): listener.exitNoArgs(self) def noArgs(self): localctx = BoolExprParser.NoArgsContext(self, self._ctx, self.state) self.enterRule(localctx, 6, self.RULE_noArgs) try: self.enterOuterAlt(localctx, 1) self.state = 27 self.match(BoolExprParser.T__0) self.state = 28 self.match(BoolExprParser.T__1) except RecognitionException as re: localctx.exception = re self._errHandler.reportError(self, re) self._errHandler.recover(self, re) finally: self.exitRule() return localctx class OneOrMoreArgsContext(ParserRuleContext): def __init__(self, parser, parent=None, invokingState=-1): super(BoolExprParser.OneOrMoreArgsContext, self).__init__(parent, invokingState) self.parser = parser def expression(self, i=None): if i is None: return self.getTypedRuleContexts(BoolExprParser.ExpressionContext) else: return self.getTypedRuleContext(BoolExprParser.ExpressionContext,i) def value(self, i=None): if i is None: return self.getTypedRuleContexts(BoolExprParser.ValueContext) else: return self.getTypedRuleContext(BoolExprParser.ValueContext,i) def getRuleIndex(self): return BoolExprParser.RULE_oneOrMoreArgs def enterRule(self, listener): if hasattr(listener, "enterOneOrMoreArgs"): listener.enterOneOrMoreArgs(self) def exitRule(self, listener): if hasattr(listener, "exitOneOrMoreArgs"): listener.exitOneOrMoreArgs(self) def oneOrMoreArgs(self): localctx = BoolExprParser.OneOrMoreArgsContext(self, self._ctx, self.state) self.enterRule(localctx, 8, self.RULE_oneOrMoreArgs) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) self.state = 30 self.match(BoolExprParser.T__0) self.state = 31 self.expression() self.state = 39 self._errHandler.sync(self) _la = self._input.LA(1) while _la==BoolExprParser.T__2: self.state = 32 self.match(BoolExprParser.T__2) self.state = 35 self._errHandler.sync(self) token = self._input.LA(1) if token in [BoolExprParser.JAVASCRIPTIDENTIFIER]: self.state = 33 self.expression() pass elif token in [BoolExprParser.T__9, BoolExprParser.T__10, BoolExprParser.T__11, BoolExprParser.STRING, BoolExprParser.NUMBER]: self.state = 34 self.value() pass else: raise NoViableAltException(self) self.state = 41 self._errHandler.sync(self) _la = self._input.LA(1) self.state = 42 self.match(BoolExprParser.T__1) except RecognitionException as re: localctx.exception = re self._errHandler.reportError(self, re) self._errHandler.recover(self, re) finally: self.exitRule() return localctx class ComparatorContext(ParserRuleContext): def __init__(self, parser, parent=None, invokingState=-1): super(BoolExprParser.ComparatorContext, self).__init__(parent, invokingState) self.parser = parser def getRuleIndex(self): return BoolExprParser.RULE_comparator def enterRule(self, listener): if hasattr(listener, "enterComparator"): listener.enterComparator(self) def exitRule(self, listener): if hasattr(listener, "exitComparator"): listener.exitComparator(self) def comparator(self): localctx = BoolExprParser.ComparatorContext(self, self._ctx, self.state) self.enterRule(localctx, 10, self.RULE_comparator) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) self.state = 44 _la = self._input.LA(1) if not((((_la) & ~0x3f) == 0 and ((1 << _la) & ((1 << BoolExprParser.T__3) | (1 << BoolExprParser.T__4) | (1 << BoolExprParser.T__5) | (1 << BoolExprParser.T__6) | (1 << BoolExprParser.T__7) | (1 << BoolExprParser.T__8))) != 0)): self._errHandler.recoverInline(self) else: self._errHandler.reportMatch(self) self.consume() except RecognitionException as re: localctx.exception = re self._errHandler.reportError(self, re) self._errHandler.recover(self, re) finally: self.exitRule() return localctx class ValueContext(ParserRuleContext): def __init__(self, parser, parent=None, invokingState=-1): super(BoolExprParser.ValueContext, self).__init__(parent, invokingState) self.parser = parser def STRING(self): return self.getToken(BoolExprParser.STRING, 0) def NUMBER(self): return self.getToken(BoolExprParser.NUMBER, 0) def getRuleIndex(self): return BoolExprParser.RULE_value def enterRule(self, listener): if hasattr(listener, "enterValue"): listener.enterValue(self) def exitRule(self, listener): if hasattr(listener, "exitValue"): listener.exitValue(self) def value(self): localctx = BoolExprParser.ValueContext(self, self._ctx, self.state) self.enterRule(localctx, 12, self.RULE_value) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) self.state = 46 _la = self._input.LA(1) if not((((_la) & ~0x3f) == 0 and ((1 << _la) & ((1 << BoolExprParser.T__9) | (1 << BoolExprParser.T__10) | (1 << BoolExprParser.T__11) | (1 << BoolExprParser.STRING) | (1 << BoolExprParser.NUMBER))) != 0)): self._errHandler.recoverInline(self) else: self._errHandler.reportMatch(self) self.consume() except RecognitionException as re: localctx.exception = re self._errHandler.reportError(self, re) self._errHandler.recover(self, re) finally: self.exitRule() return localctx ================================================ FILE: minemeld/ft/condition/__init__.py ================================================ from .BoolExprParser import BoolExprParser # noqa from .BoolExprLexer import BoolExprLexer # noqa from .BoolExprListener import BoolExprListener # noqa from .interface import Condition # noqa ================================================ FILE: minemeld/ft/condition/interface.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import jmespath import logging import antlr4 import operator from .BoolExprParser import BoolExprParser # noqa from .BoolExprLexer import BoolExprLexer # noqa from .BoolExprListener import BoolExprListener # noqa LOG = logging.getLogger(__name__) class _BECompiler(BoolExprListener): def exitExpression(self, ctx): self.expression = jmespath.compile(ctx.getText()) def exitComparator(self, ctx): comparator = ctx.getText() if comparator == '==': self.comparator = operator.eq elif comparator == '<': self.comparator = operator.lt elif comparator == '<=': self.comparator = operator.le elif comparator == '>': self.comparator = operator.gt elif comparator == '>=': self.comparator = operator.ge elif comparator == '!=': self.comparator = operator.ne def exitValue(self, ctx): if ctx.STRING() is not None: self.value = ctx.STRING().getText()[1:-1] elif ctx.NUMBER() is not None: self.value = int(ctx.NUMBER().getText()) elif ctx.getText() == 'null': self.value = None elif ctx.getText() == 'false': self.value = False elif ctx.getText() == 'true': self.value = True class Condition(object): def __init__(self, s): self.expression, self.comparator, self.value = self._parse_boolexpr(s) def _parse_boolexpr(self, s): lexer = BoolExprLexer( antlr4.InputStream(s) ) stream = antlr4.CommonTokenStream(lexer) parser = BoolExprParser(stream) tree = parser.booleanExpression() eb = _BECompiler() walker = antlr4.ParseTreeWalker() walker.walk(eb, tree) return eb.expression, eb.comparator, eb.value def eval(self, i): try: r = self.expression.search(i) except jmespath.exceptions.JMESPathError: LOG.debug("Exception in eval: ", exc_info=True) r = None # XXX this is a workaround for a bug in JMESPath if r == 'null': r = None return self.comparator(r, self.value) ================================================ FILE: minemeld/ft/csv.py ================================================ # Copyright 2015-2020 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements minemeld.ft.csv.CSVFT, the Miner node for csv feeds over HTTP/HTTPS. """ from __future__ import absolute_import import logging import re import os.path import itertools import csv import requests import yaml import shutil from urllib3.response import GzipDecoder from . import basepoller LOG = logging.getLogger(__name__) class CSVFT(basepoller.BasePollerFT): """Implements class for miners of csv feeds over http/https. **Config parameters** :url: URL of the feed. :polling_timeout: timeout of the polling request in seconds. Default: 20 :verify_cert: boolean, if *true* feed HTTPS server certificate is verified. Default: *true* :ignore_regex: Python regular expression for lines that should be ignored. Default: *null* :fieldnames: list of field names in the file. If *null* the values in the first row of the file are used as names. Default: *null* :delimiter: see `csv Python module `_. Default: , :doublequote: see `csv Python module `_. Default: true :escapechar: see `csv Python module `_. Default: null :quotechar: see `csv Python module `_. Default: " :skipinitialspace: see `csv Python module `_. Default: false Example: Example config in YAML:: url: https://sslbl.abuse.ch/blacklist/sslipblacklist.csv ignore_regex: '^#' fieldnames: - indicator - port - sslblabusech_type Args: name (str): node name, should be unique inside the graph chassis (object): parent chassis instance config (dict): node config. """ def configure(self): super(CSVFT, self).configure() self.polling_timeout = self.config.get('polling_timeout', 20) self.url = self.config.get('url', None) self.verify_cert = self.config.get('verify_cert', True) self.username = self.config.get('username', None) self.password = self.config.get('password', None) self.ignore_regex = self.config.get('ignore_regex', None) if self.ignore_regex is not None: self.ignore_regex = re.compile(self.ignore_regex) self.fieldnames = self.config.get('fieldnames', None) self.dialect = { 'delimiter': self.config.get('delimiter', ','), 'doublequote': self.config.get('doublequote', True), 'escapechar': self.config.get('escapechar', None), 'quotechar': self.config.get('quotechar', '"'), 'skipinitialspace': self.config.get('skipinitialspace', False) } self.decode_gzip = self.config.get('decode_gzip', False) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return username = sconfig.get('username', None) if username is not None: self.username = username LOG.info('%s - username set', self.name) password = sconfig.get('password', None) if password is not None: self.password = password LOG.info('%s - password set', self.name) def _process_item(self, item): item.pop(None, None) # I love this indicator = item.pop('indicator', None) return [[indicator, item]] def _build_request(self, now): auth = None if self.username is not None and self.password is not None: auth = (self.username, self.password) r = requests.Request( 'GET', self.url, auth=auth ) return r.prepare() def _build_iterator(self, now): def _debug(x): LOG.info('{!r}'.format(x)) return x _session = requests.Session() prepreq = self._build_request(now) # this is to honour the proxy environment variables rkwargs = _session.merge_environment_settings( prepreq.url, {}, None, None, None # defaults ) rkwargs['stream'] = True rkwargs['verify'] = self.verify_cert rkwargs['timeout'] = self.polling_timeout r = _session.send(prepreq, **rkwargs) try: r.raise_for_status() except: LOG.debug('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise response = r.raw if self.decode_gzip: response = self._gzipped_line_splitter(r) if self.ignore_regex is not None: response = itertools.ifilter( lambda x: self.ignore_regex.match(x) is None, response ) csvreader = csv.DictReader( response, fieldnames=self.fieldnames, **self.dialect ) return csvreader def _gzipped_line_splitter(self, response): # same logic used in urllib32.response.iter_lines pending = None decoder = GzipDecoder() chunks = itertools.imap( decoder.decompress, response.iter_content(chunk_size=1024*1024) ) for chunk in chunks: if pending is not None: chunk = pending + chunk lines = chunk.splitlines() if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: pending = lines.pop() else: pending = None for line in lines: yield line if pending is not None: yield pending def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(CSVFT, self).hup(source=source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) shutil.rmtree('{}_temp'.format(name), ignore_errors=True) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass ================================================ FILE: minemeld/ft/dag.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import yaml import netaddr import os import re import collections import itertools import shutil import gevent import gevent.queue import gevent.event import pan.xapi from . import base from . import actorbase from . import table from .utils import utc_millisec LOG = logging.getLogger(__name__) SUBRE = re.compile("[^A-Za-z0-9_]") class DevicePusher(gevent.Greenlet): def __init__(self, device, prefix, watermark, attributes, persistent): super(DevicePusher, self).__init__() self.device = device self.xapi = pan.xapi.PanXapi( tag=self.device.get('tag', None), api_username=self.device.get('api_username', None), api_password=self.device.get('api_password', None), api_key=self.device.get('api_key', None), port=self.device.get('port', None), hostname=self.device.get('hostname', None), serial=self.device.get('serial', None) ) self.prefix = prefix self.attributes = attributes self.watermark = watermark self.persistent = persistent self.q = gevent.queue.Queue() def put(self, op, address, value): LOG.debug('adding %s:%s to device queue', op, address) self.q.put([op, address, value]) def _get_registered_ip_tags(self, ip): self.xapi.op( cmd='%s' % ip, vsys=self.device.get('vsys', None), cmd_xml=False ) entries = self.xapi.element_root.findall('./result/entry') if entries is None or len(entries) == 0: LOG.warning('%s: ip %s has no tags', self.device.get('hostname', None), ip) return None tags = [member.text for member in entries[0].findall('./tag/member') if member.text and member.text.startswith(self.prefix)] return tags def _get_all_registered_ips(self): cmd = ( '' % (self.prefix, self.watermark) ) self.xapi.op( cmd=cmd, vsys=self.device.get('vsys', None), cmd_xml=False ) entries = self.xapi.element_root.findall('./result/entry') if not entries: return for entry in entries: ip = entry.get("ip") yield ip, self._get_registered_ip_tags(ip) def _dag_message(self, type_, addresses): message = [ "", "1.0", "update", "" ] persistent = '' if type_ == 'register': persistent = ' persistent="%d"' % (1 if self.persistent else 0) message.append('<%s>' % type_) if addresses is not None and len(addresses) != 0: akeys = sorted(addresses.keys()) for a in akeys: message.append( '' % (a, persistent) ) tags = sorted(addresses[a]) if tags is not None: message.append('') for t in tags: message.append('%s' % t) message.append('') message.append('') message.append('' % type_) message.append('') return ''.join(message) def _user_id(self, cmd=None): try: self.xapi.user_id(cmd=cmd, vsys=self.device.get('vsys', None)) except gevent.GreenletExit: raise except pan.xapi.PanXapiError as e: LOG.debug('%s', e) if 'already exists, ignore' in str(e): pass elif 'does not exist, ignore unreg' in str(e): pass elif 'Failed to register' in str(e): pass else: LOG.exception('XAPI exception in pusher for device %s: %s', self.device.get('hostname', None), str(e)) raise def _tags_from_value(self, value): result = [] def _tag(t, v): if type(v) == unicode: v = v.encode('ascii', 'replace') else: v = str(v) v = SUBRE.sub('_', v) tag = '%s%s_%s' % (self.prefix, t, v) return tag for t in self.attributes: if t in value: if t == 'confidence': confidence = value[t] if confidence < 50: tag = '%s%s_low' % (self.prefix, t) elif confidence < 75: tag = '%s%s_medium' % (self.prefix, t) else: tag = '%s%s_high' % (self.prefix, t) result.append(tag) else: LOG.debug('%s %s %s', t, value[t], type(value[t])) if isinstance(value[t], list): for v in value[t]: LOG.debug('%s', v) result.append(_tag(t, v)) else: result.append(_tag(t, value[t])) else: # XXX noop for this case? result.append('%s%s_unknown' % (self.prefix, t)) LOG.debug('%s', result) return set(result) # XXX eliminate duplicates def _push(self, op, address, value): tags = [] tags.append('%s%s' % (self.prefix, self.watermark)) tags += self._tags_from_value(value) if len(tags) == 0: tags = None msg = self._dag_message(op, {address: tags}) self._user_id(cmd=msg) def _init_resync(self): ctags = collections.defaultdict(set) while True: op, address, value = self.q.get() if op == 'EOI': break if op != 'init': raise RuntimeError( 'DevicePusher %s - wrong op %s received in init phase' % (self.device.get('hostname', None), op) ) ctags[address].add('%s%s' % (self.prefix, self.watermark)) for t in self._tags_from_value(value): ctags[address].add(t) LOG.debug('%s', ctags) register = collections.defaultdict(list) unregister = collections.defaultdict(list) for a, atags in self._get_all_registered_ips(): regtags = set() if atags is not None: for t in atags: regtags.add(t) added = ctags[a] - regtags removed = regtags - ctags[a] for t in added: register[a].append(t) for t in removed: unregister[a].append(t) ctags.pop(a) # ips not in firewall for a, atags in ctags.iteritems(): register[a] = atags LOG.debug('register %s', register) LOG.debug('unregister %s', unregister) # XXX use constant for chunk size if len(register) != 0: addrs = iter(register) for i in xrange(0, len(register), 1000): rmsg = self._dag_message( 'register', {k: register[k] for k in itertools.islice(addrs, 1000)} ) self._user_id(cmd=rmsg) if len(unregister) != 0: addrs = iter(unregister) for i in xrange(0, len(unregister), 1000): urmsg = self._dag_message( 'unregister', {k: unregister[k] for k in itertools.islice(addrs, 1000)} ) self._user_id(cmd=urmsg) def _run(self): self._init_resync() while True: try: op, address, value = self.q.peek() self._push(op, address, value) self.q.get() # discard processed message except gevent.GreenletExit: break except pan.xapi.PanXapiError as e: LOG.exception('XAPI exception in pusher for device %s: %s', self.device.get('hostname', None), str(e)) raise class DagPusher(actorbase.ActorBaseFT): def __init__(self, name, chassis, config): self.devices = [] self.device_pushers = [] self.device_list_glet = None self.device_list_mtime = None self.ageout_glet = None self.last_ageout_run = None self.hup_event = gevent.event.Event() super(DagPusher, self).__init__(name, chassis, config) def configure(self): super(DagPusher, self).configure() self.device_list_path = self.config.get('device_list', None) if self.device_list_path is None: self.device_list_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_device_list.yml' % self.name ) self.age_out = self.config.get('age_out', 3600) self.age_out_interval = self.config.get('age_out_interval', None) self.tag_prefix = self.config.get('tag_prefix', 'mmld_') self.tag_watermark = self.config.get('tag_watermark', 'pushed') self.tag_attributes = self.config.get( 'tag_attributes', ['confidence', 'direction'] ) self.persistent_registered_ips = self.config.get( 'persistent_registered_ips', True ) def _initialize_table(self, truncate=False): self.table = table.Table(self.name, truncate=truncate) self.table.create_index('_age_out') def initialize(self): self._initialize_table() def rebuild(self): self.rebuild_flag = True self._initialize_table(truncate=True) def reset(self): self._initialize_table(truncate=True) def _validate_ip(self, indicator, value): type_ = value.get('type', None) if type_ not in ['IPv4', 'IPv6']: LOG.error('%s - invalid indicator type, ignored: %s', self.name, type_) self.statistics['ignored'] += 1 return if '-' in indicator: i1, i2 = indicator.split('-', 1) if i1 != i2: LOG.error('%s - indicator range must be equal, ignored: %s', self.name, indicator) self.statistics['ignored'] += 1 return indicator = i1 try: address = netaddr.IPNetwork(indicator) except netaddr.core.AddrFormatError as e: LOG.error('%s - invalid IP address received, ignored: %s', self.name, e) self.statistics['ignored'] += 1 return if address.size != 1: LOG.error('%s - IP network received, ignored: %s', self.name, address) self.statistics['ignored'] += 1 return if type_ == 'IPv4' and address.version != 4 or \ type_ == 'IPv6' and address.version != 6: LOG.error('%s - IP version mismatch, ignored', self.name) self.statistics['ignored'] += 1 return return address @base._counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): address = self._validate_ip(indicator, value) if address is None: return current_value = self.table.get(str(address)) now = utc_millisec() age_out = now+self.age_out*1000 value['_age_out'] = age_out self.statistics['added'] += 1 self.table.put(str(address), value) LOG.debug('%s - #indicators: %d', self.name, self.length()) value.pop('_age_out') uflag = False if current_value is not None: for t in self.tag_attributes: cv = current_value.get(t, None) nv = value.get(t, None) if isinstance(cv, list) or isinstance(nv, list): uflag |= set(cv) != set(nv) else: uflag |= cv != nv LOG.debug('uflag %s current %s new %s', uflag, current_value, value) for p in self.device_pushers: if uflag: p.put('unregister', str(address), current_value) p.put('register', str(address), value) @base._counting('withdraw.processed') def filtered_withdraw(self, source=None, indicator=None, value=None): address = self._validate_ip(indicator, value) if address is None: return current_value = self.table.get(str(address)) if current_value is None: LOG.warning('%s - unknown indicator received, ignored: %s', self.name, address) self.statistics['ignored'] += 1 return current_value.pop('_age_out', None) self.statistics['removed'] += 1 self.table.delete(str(address)) LOG.debug('%s - #indicators: %d', self.name, self.length()) for p in self.device_pushers: p.put('unregister', str(address), current_value) def _age_out_run(self): while True: try: now = utc_millisec() LOG.debug('now: %s', now) for i, v in self.table.query(index='_age_out', to_key=now-1, include_value=True): LOG.debug('%s - %s %s aged out', self.name, i, v) for dp in self.device_pushers: dp.put( op='unregister', address=i, value=v ) self.statistics['aged_out'] += 1 self.table.delete(i) self.last_ageout_run = now LOG.debug('%s - #indicators: %d', self.name, self.length()) except gevent.GreenletExit: break except Exception: LOG.exception('Exception in _age_out_loop') try: gevent.sleep(self.age_out_interval) except gevent.GreenletExit: break def _spawn_device_pusher(self, device): dp = DevicePusher( device, self.tag_prefix, self.tag_watermark, self.tag_attributes, self.persistent_registered_ips ) dp.link_exception(self._device_pusher_died) for i, v in self.table.query(include_value=True): LOG.debug('%s - addding %s to init', self.name, i) dp.put('init', i, v) dp.put('EOI', None, None) return dp def _device_pusher_died(self, g): try: g.get() except gevent.GreenletExit: pass except Exception: LOG.exception('%s - exception in greenlet for %s, ' 'respawning in 60 seconds', self.name, g.device['hostname']) for idx in range(len(self.device_pushers)): if self.device_pushers[idx].device == g.device: break else: LOG.info('%s - device pusher for %s removed,' + ' respawning aborted', self.name, g.device['hostname']) g = None return dp = self._spawn_device_pusher(g.device) self.device_pushers[idx] = dp dp.start_later(60) def _load_device_list(self): with open(self.device_list_path, 'r') as dlf: dlist = yaml.safe_load(dlf) added = [d for i, d in enumerate(dlist) if d not in self.devices] removed = [i for i, d in enumerate(self.devices) if d not in dlist] dpushers = [] for d in dlist: if d in added: dp = self._spawn_device_pusher(d) dpushers.append(dp) else: idx = self.devices.index(d) dpushers.append(self.device_pushers[idx]) for idx in removed: self.device_pushers[idx].kill() self.device_pushers = dpushers self.devices = dlist for g in self.device_pushers: if g.value is None and not g.started: g.start() def _huppable_wait(self, wait_time): hup_called = self.hup_event.wait(timeout=wait_time) if hup_called: LOG.debug('%s - clearing poll event', self.name) self.hup_event.clear() def _device_list_monitor(self): if self.device_list_path is None: LOG.warning('%s - no device_list path configured', self.name) return while True: try: mtime = os.stat(self.device_list_path).st_mtime except OSError: LOG.debug('%s - error checking mtime of %s', self.name, self.device_list_path) self._huppable_wait(5) continue if mtime != self.device_list_mtime: self.device_list_mtime = mtime try: self._load_device_list() LOG.info('%s - device list loaded', self.name) except Exception: LOG.exception('%s - exception loading device list', self.name) self._huppable_wait(5) def mgmtbus_status(self): result = super(DagPusher, self).mgmtbus_status() result['devices'] = len(self.devices) return result def length(self, source=None): return self.table.num_indicators def start(self): super(DagPusher, self).start() if self.device_list_glet is not None: return self.device_list_glet = gevent.spawn_later( 2, self._device_list_monitor ) if self.age_out_interval is not None: self.ageout_glet = gevent.spawn(self._age_out_run) def stop(self): super(DagPusher, self).stop() if self.device_list_glet is None: return for g in self.device_pushers: g.kill() self.device_list_glet.kill() if self.ageout_glet is not None: self.ageout_glet.kill() self.table.close() def hup(self, source=None): LOG.info('%s - hup received, reload device list', self.name) self.hup_event.set() @staticmethod def gc(name, config=None): actorbase.ActorBaseFT.gc(name, config=config) shutil.rmtree(name, ignore_errors=True) device_list_path = None if config is not None: device_list_path = config.get('device_list', None) if device_list_path is None: device_list_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_device_list.yml'.format(name) ) try: os.remove(device_list_path) except OSError: pass ================================================ FILE: minemeld/ft/dag_ng.py ================================================ # Copyright 2015-2019 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import yaml import netaddr import os import re import collections import itertools import shutil import time import gevent import gevent.queue import gevent.event import pan.xapi from . import base from . import actorbase from . import table from .utils import utc_millisec LOG = logging.getLogger(__name__) # The tag name cannot contain the following: # - single quote # - double quote # - greater than one consecutive space # And cannot be the case insensitive words: # - and, or, not SUBRE = re.compile("['\"]| +") CANARY_INDICATOR = '::/128' # RFC 4291: The Unspecified Address CANARY_TAG = 'canary_for_resync' CANARY_CHECK_SECONDS = 60*5 MAX_CHUNK_SIZE = 512 # max IPs in register/unregister message def _api_wrapper(x): from functools import wraps @wraps(x) def wrapper(self, *args, **kwargs): if self.valid_device_version is None: valid, version = self._valid_device_version() if not valid: self.valid_device_version = False LOG.error('%s: PAN-OS %s: must be 8.0 or greater', self.device.get('hostname', None), version) else: self.valid_device_version = True LOG.info('%s: PAN-OS %s', self.device.get('hostname', None), version) if self.valid_device_version: r = x(self, *args, **kwargs) return r else: return '' # iterable return wrapper class DevicePusher(gevent.Greenlet): def __init__(self, device, prefix, watermark, attributes, persistent): super(DevicePusher, self).__init__() self.device = device self.xapi = pan.xapi.PanXapi( tag=self.device.get('tag', None), api_username=self.device.get('api_username', None), api_password=self.device.get('api_password', None), api_key=self.device.get('api_key', None), port=self.device.get('port', None), hostname=self.device.get('hostname', None), serial=self.device.get('serial', None) ) self.valid_device_version = None self.prefix = prefix self.attributes = attributes self.watermark = watermark self.persistent = persistent self.q = gevent.queue.Queue() def _valid_device_version(self): try: self.xapi.ad_hoc({'type': 'version'}, modify_qs=True) except pan.xapi.PanXapiError as e: return False, None x = self.xapi.element_root.find('./result/sw-version') if x is not None: version = x.text else: LOG.error('%s: type=version request: no sw-version', self.device.get('hostname', None)) return False, None major = version.split('.', 1) valid = False try: if int(major[0]) >= 8: valid = True except ValueError: pass return valid, version @_api_wrapper def _set_canary(self): addresses = { CANARY_INDICATOR: ['%s%s' % (self.prefix, CANARY_TAG)] } cmd = self._dag_message('register', addresses, persistent=False, timeout=CANARY_CHECK_SECONDS+60*2) self._user_id(cmd) def _test_canary(self): cmd = '''\ ''' self.xapi.op(cmd=cmd % (self.prefix, CANARY_TAG), vsys=self.device.get('vsys', None)) if self.xapi.element_root is None: return False x = self.xapi.element_root.find('./result/count') if x is None: LOG.error('%s: no count element in registered-ip response', self.device.get('hostname', None)) return False try: count = int(x.text) except ValueError as e: LOG.error('%s: count invalid: %s: %s', self.device.get('hostname', None), x.text, e) return False # XXX sufficient check? if count != 1: return False self._set_canary() # reset timeout return True def put(self, op, address, value): LOG.debug('adding %s:%s to device queue', op, address) self.q.put([op, address, value]) @_api_wrapper def _get_all_registered_ips(self): cmd = '''\ %d %d ''' start = 1 LIMIT = 500 while True: self.xapi.op(cmd=cmd % (start, LIMIT), vsys=self.device.get('vsys', None)) if self.xapi.element_root is None: return x = self.xapi.element_root.find('./result/count') if x is None: LOG.error('%s: no count element in registered-ip response', self.device.get('hostname', None)) return try: count = int(x.text) except ValueError as e: LOG.error('%s: count invalid: %s: %s', self.device.get('hostname', None), x.text, e) return LOG.info('%s: count %d', self.device.get('hostname', None), count) entries = self.xapi.element_root.findall('./result/entry') for entry in entries: ip = entry.get('ip') members = entry.findall('./tag/member') tags = [] for member in members: tags.append(member.text) if '%s%s' % (self.prefix, self.watermark) in tags: _tags = [x for x in tags if x.startswith(self.prefix)] try: _ip = netaddr.IPNetwork(ip) except netaddr.core.AddrFormatError as e: LOG.error('%s: invalid IP address from firewall: %s', self.device.get('hostname', None), e) yield ip, _tags # canonize host length address with prefix yield str(_ip), _tags if count < LIMIT: break start += LIMIT def _dag_message(self, type_, addresses, persistent=None, timeout=None): message = [ "", # version element ignored "1.0", "update", "" ] _persistent = '' if type_ == 'register': if persistent is not None: _persistent = \ ' persistent="%d"' % (1 if persistent else 0) else: _persistent = \ ' persistent="%d"' % (1 if self.persistent else 0) message.append('<%s>' % type_) if addresses is not None and len(addresses) != 0: akeys = sorted(addresses.keys()) for a in akeys: message.append( '' % (a, _persistent) ) tags = sorted(addresses[a]) if tags is not None: message.append('') for t in tags: if timeout is not None: # PAN-OS 9.0 and greater message.append('%s' % (timeout, t)) else: message.append('%s' % t) message.append('') message.append('') message.append('' % type_) message.append('') return ''.join(message) @_api_wrapper def _user_id(self, cmd=None): try: self.xapi.user_id(cmd=cmd, vsys=self.device.get('vsys', None)) except gevent.GreenletExit: raise except pan.xapi.PanXapiError as e: LOG.debug('%s', e) if 'already exists, ignore' in str(e): pass elif 'does not exist, ignore unreg' in str(e): pass elif 'Failed to register' in str(e): pass else: LOG.exception('XAPI exception in pusher for device %s: %s', self.device.get('hostname', None), str(e)) raise def _tags_from_value(self, value): result = [] def _tag(t, v): if type(v) == unicode: v = v.encode('ascii', 'replace') else: v = str(v) m = re.match('^(and|or|not)$', v, flags=re.IGNORECASE) if m: v = '_%s_' % m.group(0) else: v = SUBRE.sub('_', v) tag = '%s%s_%s' % (self.prefix, t, v) return tag for t in self.attributes: if t in value: if t == 'confidence': confidence = value[t] if confidence < 50: tag = '%s%s_low' % (self.prefix, t) elif confidence < 75: tag = '%s%s_medium' % (self.prefix, t) else: tag = '%s%s_high' % (self.prefix, t) result.append(tag) else: LOG.debug('%s %s %s', t, value[t], type(value[t])) if isinstance(value[t], list): for v in value[t]: LOG.debug('%s', v) result.append(_tag(t, v)) else: result.append(_tag(t, value[t])) else: # XXX noop for this case? result.append('%s%s_unknown' % (self.prefix, t)) LOG.debug('%s', result) return set(result) # XXX eliminate duplicates def _push(self, op, addresses): x = {} for address in addresses: x[address] = [] x[address].append('%s%s' % (self.prefix, self.watermark)) x[address] += self._tags_from_value(addresses[address]) if len(x[address]) == 0: x[address] = None LOG.debug('%s', x) msg = self._dag_message(op, x) self._user_id(cmd=msg) def _init_resync(self): ctags = collections.defaultdict(set) while True: op, address, value = self.q.get() if op == 'EOI': break if op != 'init': raise RuntimeError( 'DevicePusher %s - wrong op %s received in init phase' % (self.device.get('hostname', None), op) ) ctags[address].add('%s%s' % (self.prefix, self.watermark)) for t in self._tags_from_value(value): ctags[address].add(t) LOG.debug('%s', ctags) register = collections.defaultdict(list) unregister = collections.defaultdict(list) for a, atags in self._get_all_registered_ips(): regtags = set() if atags is not None: for t in atags: regtags.add(t) added = ctags[a] - regtags removed = regtags - ctags[a] for t in added: register[a].append(t) for t in removed: unregister[a].append(t) ctags.pop(a) # ips not in firewall for a, atags in ctags.iteritems(): register[a] = atags LOG.debug('register %s', register) LOG.debug('unregister %s', unregister) if len(register) != 0: addrs = iter(register) for i in xrange(0, len(register), MAX_CHUNK_SIZE): rmsg = self._dag_message( 'register', {k: register[k] for k in itertools.islice( addrs, MAX_CHUNK_SIZE)} ) self._user_id(cmd=rmsg) if len(unregister) != 0: addrs = iter(unregister) for i in xrange(0, len(unregister), MAX_CHUNK_SIZE): urmsg = self._dag_message( 'unregister', {k: unregister[k] for k in itertools.islice( addrs, MAX_CHUNK_SIZE)} ) self._user_id(cmd=urmsg) self._set_canary() def _run(self): def _chunk(op, addresses): MAX_TIME = 1 self.q.get() # discard processed message end = time.time() + MAX_TIME while True: try: _op, address, value = self.q.peek_nowait() except gevent.queue.Empty: break if _op != op: break addresses.update({address: value}) self.q.get() if time.time() > end or len(addresses) == MAX_CHUNK_SIZE: break LOG.debug('%s chunks %d', op, len(addresses)) self._init_resync() last_check = int(time.time()) while True: now = int(time.time()) elapsed = now - last_check LOG.debug('%s: elapsed %d', self.device.get('hostname', None), elapsed) if elapsed >= CANARY_CHECK_SECONDS: if self.valid_device_version and not self._test_canary(): raise RuntimeError('%s: out of sync detected' % self.device.get('hostname', None)) last_check = int(time.time()) try: try: LOG.debug('%s: peek %d', self.device.get('hostname', None), CANARY_CHECK_SECONDS-elapsed) op, address, value = self.q.peek( timeout=CANARY_CHECK_SECONDS-elapsed) except gevent.queue.Empty: if self.valid_device_version and not self._test_canary(): raise RuntimeError('%s: out of sync detected' % self.device.get('hostname', None)) last_check = int(time.time()) continue addresses = {address: value} _chunk(op, addresses) self._push(op, addresses) except gevent.GreenletExit: break except pan.xapi.PanXapiError as e: LOG.exception('XAPI exception in pusher for device %s: %s', self.device.get('hostname', None), str(e)) raise class DagPusher(actorbase.ActorBaseFT): def __init__(self, name, chassis, config): self.devices = [] self.device_pushers = [] self.device_list_glet = None self.device_list_mtime = None self.ageout_glet = None self.last_ageout_run = None self.hup_event = gevent.event.Event() super(DagPusher, self).__init__(name, chassis, config) def configure(self): super(DagPusher, self).configure() self.device_list_path = self.config.get('device_list', None) if self.device_list_path is None: self.device_list_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_device_list.yml' % self.name ) self.age_out = self.config.get('age_out', 3600) self.age_out_interval = self.config.get('age_out_interval', None) self.tag_prefix = self.config.get('tag_prefix', 'mmld_') self.tag_watermark = self.config.get('tag_watermark', 'pushed') self.tag_attributes = self.config.get( 'tag_attributes', ['confidence', 'direction'] ) self.persistent_registered_ips = self.config.get( 'persistent_registered_ips', True ) def _initialize_table(self, truncate=False): self.table = table.Table(self.name, truncate=truncate) self.table.create_index('_age_out') def initialize(self): self._initialize_table() def rebuild(self): self.rebuild_flag = True self._initialize_table(truncate=True) def reset(self): self._initialize_table(truncate=True) def _validate_ip(self, indicator, value): type_ = value.get('type', None) if type_ not in ['IPv4', 'IPv6']: LOG.error('%s - invalid indicator type, ignored: %s', self.name, type_) self.statistics['ignored'] += 1 return if '-' in indicator: i1, i2 = indicator.split('-', 1) if i1 != i2: LOG.error('%s - indicator range must be equal, ignored: %s', self.name, indicator) self.statistics['ignored'] += 1 return indicator = i1 try: address = netaddr.IPNetwork(indicator) except netaddr.core.AddrFormatError as e: LOG.error('%s - invalid IP address received, ignored: %s', self.name, e) self.statistics['ignored'] += 1 return if address.size != 1: LOG.error('%s - IP network received, ignored: %s', self.name, address) self.statistics['ignored'] += 1 return if type_ == 'IPv4' and address.version != 4 or \ type_ == 'IPv6' and address.version != 6: LOG.error('%s - IP version mismatch, ignored', self.name) self.statistics['ignored'] += 1 return return address @base._counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): address = self._validate_ip(indicator, value) if address is None: return current_value = self.table.get(str(address)) now = utc_millisec() age_out = now+self.age_out*1000 value['_age_out'] = age_out self.statistics['added'] += 1 self.table.put(str(address), value) LOG.debug('%s - #indicators: %d', self.name, self.length()) value.pop('_age_out') uflag = False if current_value is not None: for t in self.tag_attributes: cv = current_value.get(t, None) nv = value.get(t, None) if isinstance(cv, list) or isinstance(nv, list): uflag |= set(cv) != set(nv) else: uflag |= cv != nv LOG.debug('uflag %s current %s new %s', uflag, current_value, value) for p in self.device_pushers: if uflag: p.put('unregister', str(address), current_value) p.put('register', str(address), value) @base._counting('withdraw.processed') def filtered_withdraw(self, source=None, indicator=None, value=None): address = self._validate_ip(indicator, value) if address is None: return current_value = self.table.get(str(address)) if current_value is None: LOG.warning('%s - unknown indicator received, ignored: %s', self.name, address) self.statistics['ignored'] += 1 return current_value.pop('_age_out', None) self.statistics['removed'] += 1 self.table.delete(str(address)) LOG.debug('%s - #indicators: %d', self.name, self.length()) for p in self.device_pushers: p.put('unregister', str(address), current_value) def _age_out_run(self): while True: try: now = utc_millisec() LOG.debug('now: %s', now) for i, v in self.table.query(index='_age_out', to_key=now-1, include_value=True): LOG.debug('%s - %s %s aged out', self.name, i, v) for dp in self.device_pushers: dp.put( op='unregister', address=i, value=v ) self.statistics['aged_out'] += 1 self.table.delete(i) self.last_ageout_run = now LOG.debug('%s - #indicators: %d', self.name, self.length()) except gevent.GreenletExit: break except Exception: LOG.exception('Exception in _age_out_loop') try: gevent.sleep(self.age_out_interval) except gevent.GreenletExit: break def _spawn_device_pusher(self, device): dp = DevicePusher( device, self.tag_prefix, self.tag_watermark, self.tag_attributes, self.persistent_registered_ips ) dp.link_exception(self._device_pusher_died) for i, v in self.table.query(include_value=True): LOG.debug('%s - addding %s to init', self.name, i) dp.put('init', i, v) dp.put('EOI', None, None) return dp def _device_pusher_died(self, g): def _restart(g): for idx in range(len(self.device_pushers)): if self.device_pushers[idx].device == g.device: break else: LOG.info('%s - device pusher for %s removed,' + ' respawning aborted', self.name, g.device['hostname']) g = None return dp = self._spawn_device_pusher(g.device) self.device_pushers[idx] = dp dp.start_later(60) try: g.get() except gevent.GreenletExit: pass except RuntimeError as e: LOG.error('%s: %s, respawning in 60 seconds', self.name, e) _restart(g) except Exception: LOG.exception('%s - exception in greenlet for %s, ' 'respawning in 60 seconds', self.name, g.device['hostname']) _restart(g) def _load_device_list(self): with open(self.device_list_path, 'r') as dlf: dlist = yaml.safe_load(dlf) added = [d for i, d in enumerate(dlist) if d not in self.devices] removed = [i for i, d in enumerate(self.devices) if d not in dlist] dpushers = [] for d in dlist: if d in added: dp = self._spawn_device_pusher(d) dpushers.append(dp) else: idx = self.devices.index(d) dpushers.append(self.device_pushers[idx]) for idx in removed: self.device_pushers[idx].kill() self.device_pushers = dpushers self.devices = dlist for g in self.device_pushers: if g.value is None and not g.started: g.start() def _huppable_wait(self, wait_time): hup_called = self.hup_event.wait(timeout=wait_time) if hup_called: LOG.debug('%s - clearing poll event', self.name) self.hup_event.clear() def _device_list_monitor(self): if self.device_list_path is None: LOG.warning('%s - no device_list path configured', self.name) return while True: try: mtime = os.stat(self.device_list_path).st_mtime except OSError: LOG.debug('%s - error checking mtime of %s', self.name, self.device_list_path) self._huppable_wait(5) continue if mtime != self.device_list_mtime: self.device_list_mtime = mtime try: self._load_device_list() LOG.info('%s - device list loaded', self.name) except Exception: LOG.exception('%s - exception loading device list', self.name) self._huppable_wait(5) def mgmtbus_status(self): result = super(DagPusher, self).mgmtbus_status() result['devices'] = len(self.devices) return result def length(self, source=None): return self.table.num_indicators def start(self): super(DagPusher, self).start() if self.device_list_glet is not None: return self.device_list_glet = gevent.spawn_later( 2, self._device_list_monitor ) if self.age_out_interval is not None: self.ageout_glet = gevent.spawn(self._age_out_run) def stop(self): super(DagPusher, self).stop() if self.device_list_glet is None: return for g in self.device_pushers: g.kill() self.device_list_glet.kill() if self.ageout_glet is not None: self.ageout_glet.kill() self.table.close() def hup(self, source=None): LOG.info('%s - hup received, reload device list', self.name) self.hup_event.set() @staticmethod def gc(name, config=None): actorbase.ActorBaseFT.gc(name, config=config) shutil.rmtree(name, ignore_errors=True) device_list_path = None if config is not None: device_list_path = config.get('device_list', None) if device_list_path is None: device_list_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_device_list.yml'.format(name) ) try: os.remove(device_list_path) except OSError: pass ================================================ FILE: minemeld/ft/google.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import itertools import functools import collections import netaddr import minemeld.packages.gdns.dig from . import basepoller LOG = logging.getLogger(__name__) _GOOGLE_DNS_SERVER = '8.8.8.8' class GoogleSPF(basepoller.BasePollerFT): def configure(self): super(GoogleSPF, self).configure() self.polling_timeout = self.config.get('polling_timeout', 20) self.tries = self.config.get('tries', 3) self.verify_cert = self.config.get('verify_cert', True) self.udp_port = self.config.get('udp_port', None) self.tcp_port = self.config.get('tcp_port', 53) self.source_name = self.config.get('source_name', self.SOURCE_NAME) def _process_item(self, item): indicator = item.pop('indicator', None) return [[indicator, item]] def _resolve_spf(self, dig, name): LOG.debug('%s - Resolving SPF for %s', self.name, name) reply = dig.query(name, dig.NS_C_IN, dig.NS_T_TXT) spf = dig.parse_txt_reply(reply) if len(spf) > 1: raise RuntimeError( '%s - TXT record for %s has more than 1 block' % (self.name, name) ) spf = spf[0] result = collections.defaultdict(list) spftoks = spf.split() if spftoks[0] != 'v=spf1': raise RuntimeError( '%s - Wrong SPF signature in SPF for %s' % (self.name, name) ) for t in spftoks[1:]: toks = t.split(':', 1) if toks[0] in ['include', 'ip4', 'ip6']: result[toks[0]].append(toks[1]) return result def _build_IPv4(self, netblock, ipnetwork): try: n = netaddr.IPNetwork(ipnetwork) if n.version != 4: raise ValueError('invalid ip4 network: %d' % n.version) except: LOG.exception('%s - Invalid ip4 network: %s', self.name, ipnetwork) return {} item = { 'indicator': ipnetwork, 'type': 'IPv4', 'confidence': 100, self.BLOCK_ATTRIBUTE: netblock, 'sources': [self.SOURCE_NAME] } return item def _build_IPv6(self, netblock, ipnetwork): try: n = netaddr.IPNetwork(ipnetwork) if n.version != 6: raise ValueError('invalid ip6 network: %d' % n.version) except: LOG.exception('%s - Invalid ip6 network: %s', self.name, ipnetwork) return {} item = { 'indicator': ipnetwork, 'type': 'IPv6', 'confidence': 100, self.BLOCK_ATTRIBUTE: netblock, 'sources': [self.SOURCE_NAME] } return item def _build_iterator(self, now): _iterators = [] dig = minemeld.packages.gdns.dig.Dig( servers=[_GOOGLE_DNS_SERVER], udp_port=self.udp_port, tcp_port=self.tcp_port, tries=self.tries, timeout=self.polling_timeout ) mainspf = self._resolve_spf(dig, self.ROOT_SPF) if 'include' not in mainspf: LOG.error( '%s - No includes in SPF' % self.name ) return [] for idomain in mainspf['include']: ispf = self._resolve_spf(dig, idomain) _iterators.append(itertools.imap( functools.partial(self._build_IPv4, idomain), ispf.get('ip4', []) )) _iterators.append(itertools.imap( functools.partial(self._build_IPv6, idomain), ispf.get('ip6', []) )) return itertools.chain(*_iterators) class GoogleNetBlocks(GoogleSPF): ROOT_SPF = '_spf.google.com' SOURCE_NAME = 'google.netblocks' BLOCK_ATTRIBUTE = 'google_netblock' class GoogleCloudNetBlocks(GoogleSPF): ROOT_SPF = '_cloud-netblocks.googleusercontent.com' SOURCE_NAME = 'google.cloudnetblocks' BLOCK_ATTRIBUTE = 'google_cloudnetblock' ================================================ FILE: minemeld/ft/http.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements minemeld.ft.http.HttpFT, the Miner node for plain text feeds over HTTP/HTTPS. """ import requests import logging import re import itertools from minemeld import __version__ as MM_VERSION from . import basepoller LOG = logging.getLogger(__name__) class HttpFT(basepoller.BasePollerFT): """Implements class for miners of plain text feeds over http/https. **Config parameters** :url: URL of the feed. :polling_timeout: timeout of the polling request in seconds. Default: 20 :verify_cert: boolean, if *true* feed HTTPS server certificate is verified. Default: *true* :user_agent: string, value for the User-Agent header in HTTP request. If ``MineMeld``, MineMeld/ is used. Default: python ``requests`` default. :ignore_regex: Python regular expression for lines that should be ignored. Default: *null* :indicator: an *extraction dictionary* to extract the indicator from the line. If *null*, the text until the first whitespace or newline character is used as indicator. Default: *null* :fields: a dicionary of *extraction dictionaries* to extract additional attributes from each line. Default: {} :encoding: encoding of the feed, if not UTF-8. See ``str.decode`` for options. Default: *null*, meaning do nothing, (Assumes UTF-8). **Extraction dictionary** Extraction dictionaries contain the following keys: :regex: Python regular expression for searching the text. :transform: template to generate the final value from the result of the regular expression. Default: the entire match of the regex is used as extracted value. See Python `re `_ module for details about Python regular expressions and templates. Example: Example config in YAML where extraction dictionaries are used to extract the indicator and additional fields:: url: https://www.dshield.org/block.txt ignore_regex: "[#S].*" indicator: regex: '^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})\\t([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})' transform: '\\1-\\2' fields: dshield_nattacks: regex: '^.*\\t.*\\t[0-9]+\\t([0-9]+)' transform: '\\1' dshield_name: regex: '^.*\\t.*\\t[0-9]+\\t[0-9]+\\t([^\\t]+)' transform: '\\1' dshield_country: regex: '^.*\\t.*\\t[0-9]+\\t[0-9]+\\t[^\\t]+\\t([A-Z]+)' transform: '\\1' dshield_email: regex: '^.*\\t.*\\t[0-9]+\\t[0-9]+\\t[^\\t]+\\t[A-Z]+\\t(\\S+)' transform: '\\1' Example config in YAML where the text in each line until the first whitespace is used as indicator:: url: https://ransomwaretracker.abuse.ch/downloads/CW_C2_URLBL.txt ignore_regex: '^#' Args: name (str): node name, should be unique inside the graph chassis (object): parent chassis instance config (dict): node config. """ def configure(self): super(HttpFT, self).configure() self.url = self.config.get('url', None) self.polling_timeout = self.config.get('polling_timeout', 20) self.verify_cert = self.config.get('verify_cert', True) self.user_agent = self.config.get('user_agent', None) self.encoding = self.config.get('encoding', None) self.username = self.config.get('username', None) self.password = self.config.get('password', None) self.ignore_regex = self.config.get('ignore_regex', None) if self.ignore_regex is not None: self.ignore_regex = re.compile(self.ignore_regex) self.indicator = self.config.get('indicator', None) if self.indicator is not None: if 'regex' in self.indicator: self.indicator['regex'] = re.compile(self.indicator['regex']) else: raise ValueError('%s - indicator stanza should have a regex', self.name) if 'transform' not in self.indicator: if self.indicator['regex'].groups > 0: LOG.warning('%s - no transform string for indicator' ' but pattern contains groups', self.name) self.indicator['transform'] = '\g<0>' self.fields = self.config.get('fields', {}) for f, fattrs in self.fields.iteritems(): if 'regex' in fattrs: fattrs['regex'] = re.compile(fattrs['regex']) else: raise ValueError('%s - %s field does not have a regex', self.name, f) if 'transform' not in fattrs: if fattrs['regex'].groups > 0: LOG.warning('%s - no transform string for field %s' ' but pattern contains groups', self.name, f) fattrs['transform'] = '\g<0>' def _process_item(self, line): line = line.strip() if not line: return [[None, None]] if self.indicator is None: indicator = line.split()[0] else: indicator = self.indicator['regex'].search(line) if indicator is None: return [[None, None]] indicator = indicator.expand(self.indicator['transform']) attributes = {} for f, fattrs in self.fields.iteritems(): m = fattrs['regex'].search(line) if m is None: continue attributes[f] = m.expand(fattrs['transform']) try: i = int(attributes[f]) except: pass else: attributes[f] = i return [[indicator, attributes]] def _build_iterator(self, now): rkwargs = dict( stream=True, verify=self.verify_cert, timeout=self.polling_timeout ) if self.user_agent is not None: if self.user_agent == 'MineMeld': rkwargs['headers'] = { 'User-Agent': 'MineMeld/%s' % MM_VERSION } else: rkwargs['headers'] = { 'User-Agent': self.user_agent } if self.username is not None and self.password is not None: rkwargs['auth'] = (self.username, self.password) r = requests.get( self.url, **rkwargs ) try: r.raise_for_status() except: LOG.debug('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise result = r.iter_lines() if self.ignore_regex is not None: result = itertools.ifilter( lambda x: self.ignore_regex.match(x) is None, result ) if self.encoding is not None: result = itertools.imap( lambda x: x.decode(self.encoding).encode('utf_8'), result ) return result ================================================ FILE: minemeld/ft/ipop.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import netaddr import uuid import shutil from . import base from . import actorbase from . import table from . import st from .utils import utc_millisec from .utils import RESERVED_ATTRIBUTES LOG = logging.getLogger(__name__) WL_LEVEL = st.MAX_LEVEL class MWUpdate(object): def __init__(self, start, end, uuids): self.start = start self.end = end self.uuids = set(uuids) s = netaddr.IPAddress(start) e = netaddr.IPAddress(end) self._indicator = '%s-%s' % (s, e) def indicator(self): return self._indicator def __repr__(self): return 'MWUpdate('+self._indicator+', %r)' % self.uuids def __hash__(self): return hash(self._indicator) def __eq__(self, other): return self.start == other.start and \ self.end == other.end class AggregateIPv4FT(actorbase.ActorBaseFT): def __init__(self, name, chassis, config): self.active_requests = [] super(AggregateIPv4FT, self).__init__(name, chassis, config) def configure(self): super(AggregateIPv4FT, self).configure() self.whitelist_prefixes = self.config.get('whitelist_prefixes', []) self.enable_list_merge = self.config.get('enable_list_merge', False) def _initialize_tables(self, truncate=False): self.table = table.Table( self.name, bloom_filter_bits=10, truncate=truncate ) self.table.create_index('_id') self.st = st.ST(self.name+'_st', 32, truncate=truncate) def initialize(self): self._initialize_tables() def rebuild(self): self._initialize_tables(truncate=True) def reset(self): self._initialize_tables(truncate=True) def _indicator_key(self, indicator, source): return indicator+'\x00'+source def _calc_indicator_value(self, uuids, additional_uuid=None, additional_value=None): mv = {'sources': []} for uuid_ in uuids: if uuid_ == additional_uuid: v = additional_value else: # uuid_ = str(uuid.UUID(bytes=uuid_)) k, v = next( self.table.query('_id', from_key=uuid_, to_key=uuid_, include_value=True), (None, None) ) if k is None: LOG.error("Unable to find key associated with uuid: %s", uuid_) for vk in v: if vk in mv and vk in RESERVED_ATTRIBUTES: mv[vk] = RESERVED_ATTRIBUTES[vk](mv[vk], v[vk]) else: if self.enable_list_merge and vk in mv and isinstance(mv[vk], list): if not isinstance(v[vk], list): mv[vk] = v[vk] else: mv[vk].extend(v[vk]) else: mv[vk] = v[vk] return mv def _merge_values(self, origin, ov, nv): result = {'sources': []} result['_added'] = ov['_added'] result['_id'] = ov['_id'] for k in nv.keys(): result[k] = nv[k] return result def _add_indicator(self, origin, indicator, value): added = False now = utc_millisec() ik = self._indicator_key(indicator, origin) v = self.table.get(ik) if v is None: v = { '_id': str(uuid.uuid4()), '_added': now } added = True self.statistics['added'] += 1 v = self._merge_values(origin, v, value) v['_updated'] = now self.table.put(ik, v) return v, added def _calc_ipranges(self, start, end): """Calc IP Ranges overlapping the range between start and end Args: start (int): start of the range end (int): end of the range Returns: set: set of ranges """ result = set() # collect the endpoint between start and end eps = set() for epaddr, _, _, _ in self.st.query_endpoints(start=start, stop=end): eps.add(epaddr) eps = sorted(eps) if len(eps) == 0: return result # walk thru the endpoints, tracking last endpoint # current level, active segments and segments levels oep = None oeplevel = -1 live_ids = set() slevels = {} for epaddr in eps: # for each endpoint we track which segments are starting # and which ones are ending with that specific endpoint end_ids = set() start_ids = set() eplevel = 0 for cuuid, clevel, cstart, cend in self.st.cover(epaddr): slevels[cuuid] = clevel if clevel > eplevel: eplevel = clevel if cstart == epaddr: start_ids.add(cuuid) if cend == epaddr: end_ids.add(cuuid) if cend != epaddr and cstart != epaddr: if cuuid not in live_ids: assert epaddr == eps[0] live_ids.add(cuuid) assert len(end_ids) + len(start_ids) > 0 if len(start_ids) != 0: if oep is not None and oep != epaddr and len(live_ids) != 0: if oeplevel != WL_LEVEL: result.add(MWUpdate(oep, epaddr-1, live_ids)) oep = epaddr oeplevel = eplevel live_ids = live_ids | start_ids if len(end_ids) != 0: if oep is not None and len(live_ids) != 0: if eplevel < WL_LEVEL: result.add(MWUpdate(oep, epaddr, live_ids)) oep = epaddr+1 live_ids = live_ids - end_ids oeplevel = eplevel if len(live_ids) != 0: oeplevel = max([slevels[id_] for id_ in live_ids]) return result def _range_from_indicator(self, indicator): if '-' in indicator: start, end = map( lambda x: int(netaddr.IPAddress(x)), indicator.split('-', 1) ) elif '/' in indicator: ipnet = netaddr.IPNetwork(indicator) start = int(ipnet.ip) end = start+ipnet.size-1 else: start = int(netaddr.IPAddress(indicator)) end = start if (not (start >= 0 and start <= 0xFFFFFFFF)) or \ (not (end >= 0 and end <= 0xFFFFFFFF)): LOG.error('%s - {%s} invalid IPv4 indicator', self.name, indicator) return None, None return start, end def _endpoints_from_range(self, start, end): """Return last endpoint before range and first endpoint after range Args: start (int): range start end (int): range stop Returns: tuple: (last endpoint before, first endpoint after) """ rangestart = next( self.st.query_endpoints(start=0, stop=max(start-1, 0), reverse=True), None ) if rangestart is not None: rangestart = rangestart[0] LOG.debug('%s - range start: %s', self.name, rangestart) rangestop = next( self.st.query_endpoints(reverse=False, start=min(end+1, self.st.max_endpoint), stop=self.st.max_endpoint, include_start=False), None ) if rangestop is not None: rangestop = rangestop[0] LOG.debug('%s - range stop: %s', self.name, rangestop) return rangestart, rangestop @base._counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): vtype = value.get('type', None) if vtype != 'IPv4': self.statistics['update.ignored'] += 1 return v, newindicator = self._add_indicator(source, indicator, value) start, end = self._range_from_indicator(indicator) if start is None or end is None: return level = 1 for p in self.whitelist_prefixes: if source.startswith(p): level = WL_LEVEL break LOG.debug("%s - update: indicator: (%s) %s %s level: %s", self.name, indicator, start, end, level) rangestart, rangestop = self._endpoints_from_range(start, end) rangesb = set(self._calc_ipranges(rangestart, rangestop)) LOG.debug('%s - ranges before update: %s', self.name, rangesb) if not newindicator and level != WL_LEVEL: for u in rangesb: self.emit_update( u.indicator(), self._calc_indicator_value(u.uuids) ) return uuidbytes = v['_id'] self.st.put(uuidbytes, start, end, level=level) rangesa = set(self._calc_ipranges(rangestart, rangestop)) LOG.debug('%s - ranges after update: %s', self.name, rangesa) added = rangesa-rangesb LOG.debug("%s - IP ranges added: %s", self.name, added) removed = rangesb-rangesa LOG.debug("%s - IP ranges removed: %s", self.name, removed) for u in added: self.emit_update( u.indicator(), self._calc_indicator_value(u.uuids) ) for u in rangesa - added: for ou in rangesb: if u == ou and len(u.uuids ^ ou.uuids) != 0: LOG.debug("IP range updated: %s", repr(u)) self.emit_update( u.indicator(), self._calc_indicator_value(u.uuids) ) for u in removed: self.emit_withdraw( u.indicator(), value=self._calc_indicator_value(u.uuids) ) @base._counting('withdraw.processed') def filtered_withdraw(self, source=None, indicator=None, value=None): LOG.debug("%s - withdraw from %s - %s", self.name, source, indicator) if value is not None and value.get('type', None) != 'IPv4': self.statistics['withdraw.ignored'] += 1 return ik = self._indicator_key(indicator, source) v = self.table.get(ik) LOG.debug("%s - v: %s", self.name, v) if v is None: return self.table.delete(ik) self.statistics['removed'] += 1 start, end = self._range_from_indicator(indicator) if start is None or end is None: return level = 1 for p in self.whitelist_prefixes: if source.startswith(p): level = WL_LEVEL break rangestart, rangestop = self._endpoints_from_range(start, end) rangesb = set(self._calc_ipranges(rangestart, rangestop)) LOG.debug("ranges before: %s", rangesb) uuidbytes = v['_id'] self.st.delete(uuidbytes, start, end, level=level) rangesa = set(self._calc_ipranges(rangestart, rangestop)) LOG.debug("ranges after: %s", rangesa) added = rangesa-rangesb LOG.debug("IP ranges added: %s", added) removed = rangesb-rangesa LOG.debug("IP ranges removed: %s", removed) for u in added: self.emit_update( u.indicator(), self._calc_indicator_value(u.uuids) ) for u in rangesa - added: for ou in rangesb: if u == ou and len(u.uuids ^ ou.uuids) != 0: LOG.debug("IP range updated: %s", repr(u)) self.emit_update( u.indicator(), self._calc_indicator_value(u.uuids) ) for u in removed: self.emit_withdraw( u.indicator(), value=self._calc_indicator_value( u.uuids, additional_uuid=v['_id'], additional_value=v ) ) def _send_indicators(self, source=None, from_key=None, to_key=None): if from_key is None: from_key = 0 if to_key is None: to_key = 0xFFFFFFFF result = self._calc_ipranges(from_key, to_key) for u in result: self.do_rpc( source, "update", indicator=u.indicator(), value=self._calc_indicator_value(u.uuids) ) def get(self, source=None, indicator=None): if not type(indicator) in [str, unicode]: raise ValueError("Invalid indicator type") indicator = int(netaddr.IPAddress(indicator)) result = self._calc_ipranges(indicator, indicator) if len(result) == 0: return None u = result.pop() return self._calc_indicator_value(u.uuids) def get_all(self, source=None): self._send_indicators(source=source) return 'OK' def get_range(self, source=None, index=None, from_key=None, to_key=None): if index is not None: raise ValueError('Index not found') if from_key is not None: from_key = int(netaddr.IPAddress(from_key)) if to_key is not None: to_key = int(netaddr.IPAddress(to_key)) self._send_indicators( source=source, from_key=from_key, to_key=to_key ) return 'OK' def length(self, source=None): return self.table.num_indicators def stop(self): super(AggregateIPv4FT, self).stop() for g in self.active_requests: g.kill() self.active_requests = [] self.table.close() LOG.info("%s - # indicators: %d", self.name, self.table.num_indicators) @staticmethod def gc(name, config=None): actorbase.ActorBaseFT.gc(name, config=config) shutil.rmtree(name, ignore_errors=True) shutil.rmtree('{}_st'.format(name), ignore_errors=True) ================================================ FILE: minemeld/ft/json.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements minemeld.ft.json.SimpleJSON, the Miner node for JSON feeds over HTTP/HTTPS. """ import requests import logging import jmespath import os import yaml from . import basepoller LOG = logging.getLogger(__name__) class SimpleJSON(basepoller.BasePollerFT): """Implements class for miners of JSON feeds over http/https. **Config parameters** :url: URL of the feed. :polling_timeout: timeout of the polling request in seconds. Default: 20 :verify_cert: boolean, if *true* feed HTTPS server certificate is verified. Default: *true* :username: string, for BasicAuth authentication (*password* required) :password: string, for BasicAuth authentication (*username* required) :client_cert_required: boolean, triggers client certificate authentication (requires *cert_file* and *key_file*) :cert_file: string, path to the client certificate :key_file: string, path to the private key of the client certificate :extractor: JMESPath expression for extracting the indicators from the JSON document. Default: @ :indicator: the JSON attribute to use as indicator. Default: indicator :fields: list of JSON attributes to include in the indicator value. If *null* no additional attributes are extracted. Default: *null* :prefix: prefix to add to field names. Default: json :headers: Header parameters are optional to sepcify a user-agent or an api-token Example: headers = {'user-agent': 'my-app/0.0.1'} or Authorization: Bearer (curl -H "Authorization: Bearer " "https://api-url.com/api/v1/iocs?first_seen_since=2016-1-1") Example: Example config in YAML:: url: https://ip-ranges.amazonaws.com/ip-ranges.json extractor: "prefixes[?service=='AMAZON']" prefix: aws indicator: ip_prefix headers: {'Authorization': '12345668900', 'user-agent': 'my-app/0.0.1'} fields: - region - service Args: name (str): node name, should be unique inside the graph chassis (object): parent chassis instance config (dict): node config. """ def configure(self): super(SimpleJSON, self).configure() self.url = self.config.get('url', None) self.polling_timeout = self.config.get('polling_timeout', 20) self.verify_cert = self.config.get('verify_cert', True) self.compile_error = None try: self.extractor = jmespath.compile(self.config.get('extractor', '@')) except Exception as e: LOG.debug('%s - exception in jmespath: %s', self.name, e) self.compile_error = "{}".format(e) self.indicator = self.config.get('indicator', 'indicator') self.prefix = self.config.get('prefix', 'json') self.fields = self.config.get('fields', None) self.username = self.config.get('username', None) self.password = self.config.get('password', None) self.headers = self.config.get('headers', None) # option for enabling client cert, default disabled self.client_cert_required = self.config.get('client_cert_required', False) self.key_file = self.config.get('key_file', None) if self.key_file is None and self.client_cert_required: self.key_file = os.path.join( os.environ['MM_CONFIG_DIR'], '%s.pem' % self.name ) self.cert_file = self.config.get('cert_file', None) if self.cert_file is None and self.client_cert_required: self.cert_file = os.path.join( os.environ['MM_CONFIG_DIR'], '%s.crt' % self.name ) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return username = sconfig.get('username', None) password = sconfig.get('password', None) if username is not None and password is not None: self.username = username self.password = password LOG.info('{} - Loaded credentials from side config'.format(self.name)) def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(SimpleJSON, self).hup(source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass client_cert_required = False if config is not None: client_cert_required = config.get('client_cert_required', False) if config is not None: cert_path = config.get('cert_file', None) if cert_path is None and client_cert_required: cert_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}.crt'.format(name) ) if cert_path is not None: try: os.remove(cert_path) except: pass if config is not None: key_path = config.get('key_file', None) if key_path is None and client_cert_required: key_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}.pem'.format(name) ) if key_path is not None: try: os.remove(key_path) except: pass def _process_item(self, item): if self.indicator not in item: LOG.debug('%s not in %s', self.indicator, item) return [[None, None]] indicator = item[self.indicator] if not (isinstance(indicator, str) or isinstance(indicator, unicode)): LOG.error( 'Wrong indicator type: %s - %s', indicator, type(indicator) ) return [[None, None]] fields = self.fields if fields is None: fields = item.keys() fields.remove(self.indicator) attributes = {} for field in fields: if field not in item: continue attributes['%s_%s' % (self.prefix, field)] = item[field] return [[indicator, attributes]] def _build_iterator(self, now): if self.compile_error is not None: raise RuntimeError(self.compile_error) rkwargs = dict( stream=True, verify=self.verify_cert, timeout=self.polling_timeout ) if self.username is not None and self.password is not None: rkwargs['auth'] = (self.username, self.password) if self.headers is not None: rkwargs['headers'] = self.headers if self.client_cert_required and self.key_file is not None and self.cert_file is not None: rkwargs['cert'] = (self.cert_file, self.key_file) r = requests.get( self.url, **rkwargs ) try: r.raise_for_status() except: LOG.debug('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise result = self.extractor.search(r.json()) return result ================================================ FILE: minemeld/ft/local.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import yaml import filelock import os from . import basepoller LOG = logging.getLogger(__name__) class YamlFT(basepoller.BasePollerFT): def __init__(self, name, chassis, config): self.file_monitor_mtime = None super(YamlFT, self).__init__(name, chassis, config) def configure(self): super(YamlFT, self).configure() self.path = self.config.get('path', None) if self.path is None: self.path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_indicators.yml' % self.name ) self.lock_path = self.path+'.lock' def _flush(self): self.file_monitor_mtime = None super(YamlFT, self)._flush() def _process_item(self, item): indicator = item.pop('indicator', None) if indicator is None: return [[None, None]] item['sources'] = [self.name] return [[indicator, item]] def _load_yaml(self): with filelock.FileLock(self.lock_path).acquire(timeout=10): with open(self.path, 'r') as f: result = yaml.safe_load(f) if type(result) != list: raise RuntimeError( '%s - %s should be a list of indicators' % (self.name, self.path) ) return result def _build_iterator(self, now): if self.path is None: LOG.warning('%s - no path configured', self.name) raise RuntimeError('%s - no path configured' % self.name) try: mtime = os.stat(self.path).st_mtime except OSError as e: if e.errno == 2: # no such file return None LOG.exception('%s - error checking mtime of %s', self.name, self.path) raise RuntimeError( '%s - error checking indicators list' % self.name ) if mtime == self.file_monitor_mtime: return None self.file_monitor_mtime = mtime try: return self._load_yaml() except: LOG.exception('%s - exception loading indicators list', self.name) raise @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) path = None if config is not None: path = config.get('path', None) if path is None: path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_indicators.yml'.format(name) ) lock_path = '{}.lock'.format(path) try: os.remove(path) except: pass try: os.remove(lock_path) except: pass class YamlIPv4FT(YamlFT): def _process_item(self, item): item['type'] = 'IPv4' return super(YamlIPv4FT, self)._process_item(item) class YamlURLFT(YamlFT): def _process_item(self, item): item['type'] = 'URL' return super(YamlURLFT, self)._process_item(item) class YamlDomainFT(YamlFT): def _process_item(self, item): item['type'] = 'domain' return super(YamlDomainFT, self)._process_item(item) class YamlIPv6FT(YamlFT): def _process_item(self, item): item['type'] = 'IPv6' return super(YamlIPv6FT, self)._process_item(item) ================================================ FILE: minemeld/ft/localdb.py ================================================ # Copyright 2017-present Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import os.path import logging import sqlite3 from contextlib import contextmanager import ujson as json from . import basepoller from . import ft_states from .utils import interval_in_sec, dt_to_millisec, utc_millisec LOG = logging.getLogger(__name__) _MAX_AGE_OUT = ((1 << 32)-1)*1000 # 2106-02-07 6:28:15 @contextmanager def dbconnection(path): conn = sqlite3.connect(path) yield conn conn.close() class Miner(basepoller.BasePollerFT): def __init__(self, name, chassis, config): super(Miner, self).__init__(name, chassis, config) self.last_run = None def configure(self): if not 'age_out' in self.config: self.config['age_out'] = { 'interval': 1800, 'sudden_death': False, 'default': None } super(Miner, self).configure() self.default_ttl = self.config.get('default_ttl', 86400) self.path = self.config.get('path', None) if self.path is None: self.path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_indicators.db' % self.name ) def _collect_garbage(self): if not os.path.isfile(self.path): return now = utc_millisec() with self.state_lock, dbconnection(self.path) as conn: if self.state != ft_states.STARTED: return with conn: for i, v in self.table.query(index='_withdrawn', to_key=now, include_value=True): # if v.get('_last_run', 0) >= (self.last_successful_run-1): # continue itype = v.get('type', None) conn.execute('delete from indicators where indicator=? and type=?;', (i, itype)) self.table.delete(i, itype=itype) self.statistics['garbage_collected'] += 1 def _calc_age_out(self, indicator, attributes): if isinstance(attributes['_expiration_ts'], int): return attributes['_expiration_ts'] return _MAX_AGE_OUT def _process_item(self, item): indicator = item[0] value = json.loads(item[2]) value['type'] = item[1] value['_expiration_ts'] = item[3] if value['_expiration_ts'] is None: # if none, expiration is set to update_ts+default_ttl value['_expiration_ts'] = item[4]+self.default_ttl*1000 return [[indicator, value]] def _updates_iterator(self, last_successful_run): with dbconnection(self.path) as conn: for row in conn.execute('select * from indicators where update_ts >= ?', (last_successful_run,)): yield row def _build_iterator(self, now): if not os.path.isfile(self.path): return [] last_successful_run = 0 if self.last_successful_run is not None: last_successful_run = self.last_successful_run return self._updates_iterator(last_successful_run) def hup(self, source=None): super(Miner, self).hup(source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) path = None if config is not None: path = config.get('path', None) if path is None: path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_indicators.db'.format(name) ) try: os.remove(path) except: pass ================================================ FILE: minemeld/ft/logstash.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import ujson import datetime import socket from . import base from . import actorbase LOG = logging.getLogger(__name__) class LogstashOutput(actorbase.ActorBaseFT): def __init__(self, name, chassis, config): super(LogstashOutput, self).__init__(name, chassis, config) self._ls_socket = None def configure(self): super(LogstashOutput, self).configure() self.logstash_host = self.config.get('logstash_host', '127.0.0.1') self.logstash_port = int(self.config.get('logstash_port', '5514')) def connect(self, inputs, output): output = False super(LogstashOutput, self).connect(inputs, output) def _connect_logstash(self): if self._ls_socket is not None: return _ls_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) _ls_socket.connect((self.logstash_host, self.logstash_port)) self._ls_socket = _ls_socket def initialize(self): pass def rebuild(self): pass def reset(self): pass def _send_logstash(self, message, source=None, indicator=None, value=None): now = datetime.datetime.now() fields = { '@timestamp': now.isoformat()+'Z', '@version': 1, 'logstash_output_node': self.name, 'message': message } if indicator is not None: fields['@indicator'] = indicator if source is not None: fields['@origin'] = source if value is not None: fields.update(value) if 'last_seen' in fields: last_seen = datetime.datetime.fromtimestamp( float(fields['last_seen'])/1000.0 ) fields['last_seen'] = last_seen.isoformat()+'Z' if 'first_seen' in fields: first_seen = datetime.datetime.fromtimestamp( float(fields['first_seen'])/1000.0 ) fields['first_seen'] = first_seen.isoformat()+'Z' try: self._connect_logstash() self._ls_socket.sendall(ujson.dumps(fields)+'\n') except: self._ls_socket = None raise self.statistics['message.sent'] += 1 @base._counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): self._send_logstash( 'update', source=source, indicator=indicator, value=value ) @base._counting('withdraw.processed') def filtered_withdraw(self, source=None, indicator=None, value=None): self._send_logstash( 'withdraw', source=source, indicator=indicator, value=value ) def length(self, source=None): return 0 ================================================ FILE: minemeld/ft/mm.py ================================================ # Copyright 2017-present Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements minemeld.ft.mm.JSONSEQMiner, the Miner node for MineMeld JSON-SEQ feeds over HTTP/HTTPS. """ import os.path import logging import requests import yaml import ujson from . import basepoller LOG = logging.getLogger(__name__) class JSONSEQMiner(basepoller.BasePollerFT): """Implements class for miners of MineMeld JSON-SEQ feeds over http/https. **Config parameters** :url: URL of the feed. :polling_timeout: timeout of the polling request in seconds. Default: 20 :verify_cert: boolean, if *true* feed HTTPS server certificate is verified. Default: *true* :side_config_path: path to the side config with credentials for the feed Args: name (str): node name, should be unique inside the graph chassis (object): parent chassis instance config (dict): node config. """ def configure(self): super(JSONSEQMiner, self).configure() self.polling_timeout = self.config.get('polling_timeout', 20) self.verify_cert = self.config.get('verify_cert', True) self.url = self.config.get('url', None) self.username = None self.password = None self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return username = sconfig.get('username', None) password = sconfig.get('password', None) if username is not None and password is not None: self.username = username self.password = password LOG.info('{} - Loaded credentials from side config'.format(self.name)) def _process_item(self, item): return [[item['indicator'], item['value']]] def _json_seq_iterator(self, r): for line in r.iter_lines(decode_unicode=True, delimiter='\x1E'): if line: try: yield ujson.loads(line) except ValueError: LOG.error('{} - Error parsing {!r}'.format(self.name, line)) def _build_iterator(self, now): if self.url is None: raise RuntimeError( '{} - feed url not set'.format(self.name) ) rkwargs = dict( stream=True, verify=self.verify_cert, timeout=self.polling_timeout, params={'v': 'json-seq'} ) if self.username is not None and self.password is not None: rkwargs['auth'] = (self.username, self.password) r = requests.get( self.url, **rkwargs ) try: r.raise_for_status() except: LOG.debug('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise return self._json_seq_iterator(r) def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(JSONSEQMiner, self).hup(source) ================================================ FILE: minemeld/ft/o365.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import itertools import functools import uuid import os import json from collections import defaultdict import yaml import netaddr import requests import lxml.etree from . import basepoller LOG = logging.getLogger(__name__) O365_URL = \ 'https://support.content.office.net/en-us/static/O365IPAddresses.xml' XPATH_FUNS_NS = 'http://minemeld.panw.io/o365functions' XPATH_FUNS_PREFIX = 'o365f' XPATH_PRODUCTS = "/products/product/@name" BASE_XPATH = "/products/product[" + XPATH_FUNS_PREFIX + ":lower-case(@name)='%s']" O365_API_BASE_URL = 'https://endpoints.office.com' def _build_IPv4(source, address): item = { 'indicator': address.text, 'type': 'IPv4', 'confidence': 100, 'sources': [source] } return item def _build_IPv6(source, address): item = { 'indicator': address.text, 'type': 'IPv6', 'confidence': 100, 'sources': [source] } return item def _build_URL(source, url): item = { 'indicator': url.text, 'type': 'URL', 'confidence': 100, 'sources': [source] } return item def _xpath_lower_case(context, a): return [e.lower() for e in a] class O365XML(basepoller.BasePollerFT): def configure(self): super(O365XML, self).configure() # register lower-case ns = lxml.etree.FunctionNamespace(XPATH_FUNS_NS) ns['lower-case'] = _xpath_lower_case self.prefixmap = {XPATH_FUNS_PREFIX: XPATH_FUNS_NS} self.polling_timeout = self.config.get('polling_timeout', 20) self.verify_cert = self.config.get('verify_cert', True) self.products = self.config.get('products', []) self.url = self.config.get('url', O365_URL) def _process_item(self, item): indicator = item.pop('indicator', None) return [[indicator, item]] def _build_request(self, now): r = requests.Request( 'GET', self.url ) return r.prepare() def _o365_iterator(self, now): _iterators = [] _session = requests.Session() _adapter = requests.adapters.HTTPAdapter( pool_connections=10, pool_maxsize=10, max_retries=3 ) _session.mount('https://', _adapter) prepreq = self._build_request(now) # this is to honour the proxy environment variables rkwargs = _session.merge_environment_settings( prepreq.url, {}, None, None, None # defaults ) rkwargs['stream'] = True rkwargs['verify'] = self.verify_cert rkwargs['timeout'] = self.polling_timeout r = _session.send(prepreq, **rkwargs) try: r.raise_for_status() except: LOG.debug('%s - exception in request: %s %s', self.name, r.status_code, r.text) raise parser = lxml.etree.XMLParser() for chunk in r.iter_content(chunk_size=10 * 1024): parser.feed(chunk) rtree = parser.close() products = self.products if len(products) == 0: products = self._extract_products(rtree) for p in products: xpath = BASE_XPATH % p.lower() pIPv4s = rtree.xpath( xpath + "/addresslist[@type='IPv4']/address", namespaces=self.prefixmap ) _iterators.append(itertools.imap( functools.partial(_build_IPv4, 'office365.%s' % p.lower()), pIPv4s )) pIPv6s = rtree.xpath( xpath + "/addresslist[@type='IPv6']/address", namespaces=self.prefixmap ) _iterators.append(itertools.imap( functools.partial(_build_IPv6, 'office365.%s' % p.lower()), pIPv6s )) pURLs = rtree.xpath( xpath + "/addresslist[@type='URL']/address", namespaces=self.prefixmap ) _iterators.append(itertools.imap( functools.partial(_build_URL, 'office365.%s' % p.lower()), pURLs )) return itertools.chain(*_iterators) def _build_iterator(self, now): oiterator = self._o365_iterator(now) idict = {} for i in oiterator: indicator = i['indicator'] cvalue = idict.get(indicator, None) if cvalue is not None: i['sources'] = list(set(i['sources']) | set(cvalue['sources'])) idict[indicator] = i return itertools.imap(lambda i: i[1], idict.iteritems()) def _extract_products(self, rtree): products = rtree.xpath(XPATH_PRODUCTS) LOG.info('%s - found products: %r', self.name, products) return products O365_API_FIELDS = [ 'id', 'expressRoute', 'notes', 'serviceArea', 'tcpPorts', 'udpPorts', 'category', 'required' ] class O365API(basepoller.BasePollerFT): def __init__(self, name, chassis, config): self.client_request_id = str(uuid.uuid4()) self.latest_version = '0000000000' super(O365API, self).__init__(name, chassis, config) def configure(self): super(O365API, self).configure() self.polling_timeout = self.config.get('polling_timeout', 20) self.verify_cert = self.config.get('verify_cert', True) self.instance = self.config.get('instance', 'O365Worldwide') self.service_areas = self.config.get('service_areas', None) self.tenant_name = self.config.get('tenant_name', None) self.disable_integrations = self.config.get('disable_integrations', False) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return disable_integrations = sconfig.get('disable_integrations', None) if disable_integrations is not None: self.disable_integrations = disable_integrations LOG.info('{} - Loaded side config'.format(self.name)) def _saved_state_restore(self, saved_state): super(O365API, self)._saved_state_restore(saved_state) self.client_request_id = saved_state.get('client_request_id', None) self.latest_version = saved_state.get('latest_version', None) LOG.info('saved state: client_request_id: {} latest_version: {}'.format( self.client_request_id, self.latest_version )) def _saved_state_create(self): sstate = super(O365API, self)._saved_state_create() sstate['latest_version'] = self.latest_version sstate['client_request_id'] = self.client_request_id return sstate def _saved_state_reset(self): super(O365API, self)._saved_state_reset() self.client_request_id = str(uuid.uuid4()) self.latest_version = '0000000000' def _check_version(self): rkwargs = dict( stream=False, verify=self.verify_cert, timeout=self.polling_timeout, params={ 'clientrequestid': self.client_request_id } ) url = '{}/version/{}'.format( O365_API_BASE_URL, self.instance ) r = requests.get( url, **rkwargs ) try: r.raise_for_status() except: LOG.debug('{} - exception in request: {} {!r}'.format( self.name, r.status_code, r.content )) raise version = r.json() LOG.debug('{} - version: {}'.format(self.name, version)) if version['latest'] > self.latest_version: return version['latest'] return def _process_item(self, item): return [item] def _analyze_item(self, item): result = [] base_value = {} for wka in O365_API_FIELDS: if wka in item: base_value['o365_{}'.format(wka)] = item[wka] if self.disable_integrations and 'o365_notes' in base_value: if 'integration' in base_value['o365_notes'].lower(): return result for url in item.get('urls', []): value = base_value.copy() value['type'] = 'URL' result.append([url, value]) for ip in item.get('ips', []): try: parsed = netaddr.IPNetwork(ip) except (netaddr.AddrFormatError, ValueError): LOG.error('{} - Unknown IP version: {}'.format(self.name, ip)) continue value = base_value.copy() if parsed.version == 4: value['type'] = 'IPv4' elif parsed.version == 6: value['type'] = 'IPv6' result.append([ip, value]) return result def _iterator(self, array, latest_version): indicators = defaultdict(lambda: {'o365_{}_list'.format(f): set() for f in O365_API_FIELDS}) for i in array: for ci, cv in self._analyze_item(i): oldv = indicators[ci] oldv.update(cv) for fn in O365_API_FIELDS: label = 'o365_{}'.format(fn) if label in cv: val = str(cv[label]).lower() if label in ['o365_tcpPorts', 'o365_udpPorts']: ports = val.split(',') for p in ports: oldv['{}_list'.format(label)].add(str(p)) else: oldv['{}_list'.format(label)].add(str(val)) for i, v in indicators.iteritems(): for fn in O365_API_FIELDS: label = 'o365_{}_list'.format(fn) v[label] = list(v[label]) yield [i, v] self.latest_version = latest_version def _build_iterator(self, now): latest_version = self._check_version() if latest_version is None: LOG.info('{} - Already latest version, polling not performed'.format( self.name )) return None rkwargs = dict( stream=False, verify=self.verify_cert, timeout=self.polling_timeout, params={ 'clientrequestid': self.client_request_id } ) if self.tenant_name is not None: rkwargs['params']['tenantname'] = self.tenant_name if self.service_areas is not None: rkwargs['params']['serviceareas'] = ','.join(self.service_areas) url = '{}/endpoints/{}'.format( O365_API_BASE_URL, self.instance ) r = requests.get( url, **rkwargs ) try: r.raise_for_status() except: LOG.debug('{} - exception in request: {} {!r}'.format( self.name, r.status_code, r.content )) raise return self._iterator(r.json(), latest_version) def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() self.latest_version = None super(O365API, self).hup(source=source) ================================================ FILE: minemeld/ft/op.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import shutil from . import base from . import actorbase from . import table from .utils import utc_millisec from .utils import RESERVED_ATTRIBUTES LOG = logging.getLogger(__name__) class AggregateFT(actorbase.ActorBaseFT): _ftclass = 'AggregateFT' def __init__(self, name, chassis, config): self.active_requests = [] self.table = None super(AggregateFT, self).__init__(name, chassis, config) def configure(self): super(AggregateFT, self).configure() self.whitelist_prefixes = self.config.get('whitelist_prefixes', []) self.ignore_cases = self.config.get('ignore_cases', False) def _initialize_table(self, truncate=False): self.table = table.Table(self.name, truncate=truncate) def initialize(self): self._initialize_table() def rebuild(self): self._initialize_table(truncate=True) def reset(self): self._initialize_table(truncate=True) def _indicator_key(self, indicator, source): return indicator+'\x00'+source def _is_whitelist(self, s): for p in self.whitelist_prefixes: if s.startswith(p): return True return False def _emit_update_indicator(self, indicator): LOG.debug("%s - emitting update: %s", self.name, indicator) mv = {'sources': []} for s in self.inputs: if self._is_whitelist(s): continue v = self.table.get(self._indicator_key(indicator, s)) if v is None: continue for k in v.keys(): if k in mv and k in RESERVED_ATTRIBUTES: mv[k] = RESERVED_ATTRIBUTES[k](mv[k], v[k]) else: mv[k] = v[k] if len(mv) > 1: self.emit_update(indicator, mv) def _merge_values(self, source, ov, nv): result = {'sources': []} result['_added'] = ov['_added'] for k in nv.keys(): result[k] = nv[k] return result def _add_indicator(self, source, indicator, value): now = utc_millisec() v = self.table.get(self._indicator_key(indicator, source)) if v is None: v = { '_added': now, } v = self._merge_values(source, v, value) v['_updated'] = now self.table.put(self._indicator_key(indicator, source), v) return v @base._counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): if self.ignore_cases: indicator = indicator.lower() ebl = False ewl = False for i in self.inputs: v = self.table.exists(self._indicator_key(indicator, i)) if self._is_whitelist(i): ewl |= v else: ebl |= v v = self._add_indicator(source, indicator, value) if self._is_whitelist(source): # update from whitelist if ewl: # already whitelisted, no updates return if ebl: self.emit_withdraw(indicator) else: if ewl: return self._emit_update_indicator(indicator) @base._counting('withdraw.processed') def filtered_withdraw(self, source=None, indicator=None, value=None): if self.ignore_cases: indicator = indicator.lower() ikey = self._indicator_key(indicator, source) cvalue = self.table.get(ikey) e = (cvalue is not None) if value is not None and cvalue is not None: if value.get('type', None) != cvalue.get('type', None): self.statistics['withdraw.ignored'] += 1 return ebl = 0 ewl = 0 for i in self.inputs: v = int(self.table.exists(self._indicator_key(indicator, i))) if self._is_whitelist(i): ewl += v else: ebl += v self.table.delete(ikey) if self._is_whitelist(source): # withdraw from whitelist if e and ewl > 1: return if ebl != 0: self._emit_update_indicator(indicator) else: if ewl > 0: return if e: if ebl > 1: self._emit_update_indicator(indicator) else: self.emit_withdraw(indicator, value=cvalue) def get(self, source=None, indicator=None): mv = {} for s in self.inputs: v = self.table.get(self._indicator_key(indicator, s)) if v is None: continue for k in v.keys(): if k in mv and k in RESERVED_ATTRIBUTES: mv[k] = RESERVED_ATTRIBUTES[k](mv[k], v[k]) else: mv[k] = v[k] return mv def get_all(self, source=None): return self.get_range(source=source) def get_range(self, source=None, index=None, from_key=None, to_key=None): if index is not None: raise ValueError("Index not found") if to_key is not None: to_key = self._indicator_key(to_key, '\x7F') cindicator = None cvalue = {} for k, v in self.table.query(index=index, from_key=from_key, to_key=to_key, include_value=True): indicator, _ = k.split('\x00') if indicator == cindicator: for vk in v.keys(): if vk in cvalue and vk in RESERVED_ATTRIBUTES: cvalue[vk] = RESERVED_ATTRIBUTES[vk](cvalue[vk], v[vk]) else: cvalue[vk] = v[vk] else: if cindicator is not None: self.do_rpc(source, "update", indicator=cindicator, value=cvalue) cindicator = indicator cvalue = v if cindicator is not None: self.do_rpc(source, "update", indicator=cindicator, value=cvalue) return 'OK' def length(self, source=None): return self.table.num_indicators def stop(self): super(AggregateFT, self).stop() for g in self.active_requests: g.kill() self.active_requests = [] self.table.close() LOG.info("%s - # indicators: %d", self.name, self.table.num_indicators) @staticmethod def gc(name, config=None): actorbase.ActorBaseFT.gc(name, config=config) shutil.rmtree(name, ignore_errors=True) ================================================ FILE: minemeld/ft/panos.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import gevent import gevent.event import minemeld.packages.panforest import random import copy import re import pan.xapi from . import base from . import table from .utils import utc_millisec LOG = logging.getLogger(__name__) class CheckpointSet(Exception): pass class InterruptablePanForest(minemeld.packages.panforest.PanForest): def __init__(self, wobject, xapi=None, log_type=None, filter=None, nlogs=None, format=None): super(InterruptablePanForest, self).__init__( xapi=xapi, log_type=log_type, filter=filter, nlogs=nlogs, format=format ) self.wobject = wobject def sleep(self, t): value = self.wobject.wait(timeout=t) LOG.debug('value %s', value) if value is not None: raise CheckpointSet() def _age_out_in_usecs(val): multipliers = { '': 1000, 'm': 60000, 'h': 3600000, 'd': 86400000 } mo = re.match("([0-9]+)([dmh]?)", val) if mo is None: return None return int(mo.group(1))*multipliers[mo.group(2)] def _sleeper(slot, maxretries): c = 0 while c < maxretries: yield slot*random.uniform(0, (2**c-1)) c = c+1 yield maxretries*slot class PanOSLogsAPIFT(base.BaseFT): def __init__(self, name, chassis, config): self.glet = None self.age_out_glets = [] self.tables = [] self.active_requests = [] self.rebuild_flag = False self.last_log = None self.idle_waitobject = gevent.event.AsyncResult() super(PanOSLogsAPIFT, self).__init__(name, chassis, config) def configure(self): super(PanOSLogsAPIFT, self).configure() self.source_name = self.config.get('source_name', self.name) self.tag = self.config.get('tag', None) self.hostname = self.config.get('hostname', None) self.api_key = self.config.get('api_key', None) self.api_username = self.config.get('api_username', None) self.api_password = self.config.get('api_password', None) self.log_type = self.config.get('log_type', None) self.filter = self.config.get('filter', None) self.sleeper_slot = int(self.config.get('sleeper_slot', '10')) self.maxretries = int(self.config.get('maxretries', '16')) self.fields = self.config.get('fields', []) self.age_out_interval = int(self.config.get('age_out_interval', '3600')) def _initialize_tables(self, truncate=False): for idx, field in enumerate(self.fields): t = table.Table( self.name+'_%d' % idx, truncate=truncate ) t.create_index('last_seen') self.tables.append(t) def initialize(self): self._initialize_tables() def rebuild(self): self._initialize_tables() self.rebuild_flag = True def reset(self): self._initialize_tables(truncate=True) def emit_checkpoint(self, value): LOG.debug("%s - checkpoint set to %s", self.name, value) self.idle_waitobject.set(value) def _age_out_loop(self, fieldidx): interval = self.fields[fieldidx].get('age_out', '30d') interval = _age_out_in_usecs(interval) t = self.tables[fieldidx] while True: try: now = utc_millisec() for i, v in t.query(index='last_seen', to_key=now-interval, include_value=True): LOG.debug('%s - %s %s aged out', self.name, i, v) self.emit_withdraw(indicator=i) t.delete(i) except gevent.GreenletExit: break except: LOG.exception('Exception in _age_out_loop') gevent.sleep(self.age_out_interval) def _run(self): if self.rebuild_flag: LOG.debug("rebuild flag set, resending current indicators") # reinit flag is set, emit update for all the known indicators for t in self.tables: for i, v in t.query('last_seen', include_value=True): self.emit_update(i, v) sleeper = _sleeper(self.sleeper_slot, self.maxretries) checkpoint = None while True: try: xapi = pan.xapi.PanXapi( api_username=self.api_username, api_password=self.api_password, api_key=self.api_key, hostname=self.hostname, tag=self.tag, timeout=60 ) pf = InterruptablePanForest( self.idle_waitobject, xapi=xapi, log_type=self.log_type, filter=self.filter, format='python' ) for log in pf.follow(): sleeper = _sleeper(self.sleeper_slot, self.maxretries) self.statistics['log.processed'] += 1 now = utc_millisec() for idx, field in enumerate(self.fields): if field['name'] in log: v = copy.copy(field['attributes']) v['last_seen'] = now self.tables[idx].put(log[field['name']], v) self.emit_update(indicator=log[field['name']], value=v) else: LOG.debug('%s - field %s not found', self.name, field['name']) if self.idle_waitobject.ready(): break except gevent.GreenletExit: pass except CheckpointSet: LOG.debug('%s - CheckpointSet catched') pass except: LOG.exception("%s - exception in log loop", self.name) try: checkpoint = self.idle_waitobject.get(timeout=next(sleeper)) except gevent.Timeout: pass LOG.debug('%s - checkpoint: %s', self.name, checkpoint) if checkpoint is not None: super(PanOSLogsAPIFT, self).emit_checkpoint(checkpoint) break def length(self, source=None): return sum([t.num_indicators for t in self.tables]) def start(self): super(PanOSLogsAPIFT, self).start() if self.glet is not None: return self.glet = gevent.spawn_later(random.randint(0, 2), self._run) for idx in range(len(self.fields)): self.age_out_glets.append( gevent.spawn(self._age_out_loop, idx) ) def stop(self): super(PanOSLogsAPIFT, self).stop() if self.glet is None: return for g in self.active_requests: g.kill() self.glet.kill() for g in self.age_out_glets: g.kill() self.age_out_glets = None for t in self.tables: LOG.info("%s - # indicators: %d", self.name, t.num_indicators) ================================================ FILE: minemeld/ft/phishme.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements minemeld.ft.phishme.Intelligence, the Miner node for PhishMe Intelligence API. """ import os import yaml import requests import itertools import logging from . import basepoller from .utils import interval_in_sec LOG = logging.getLogger(__name__) _API_BASE = 'https://www.threathq.com/apiv1' _API_THREAT_SEARCH = '/threat/search' _API_THREAT_UPDATE = '/threat/updates' _API_USER_AGENT = 'PhishMe Intelligence (minemeld)' _RESULTS_PER_PAGE = 10 class Intelligence(basepoller.BasePollerFT): def __init__(self, name, chassis, config): self.position = None super(Intelligence, self).__init__(name, chassis, config) def configure(self): super(Intelligence, self).configure() self.polling_timeout = self.config.get('polling_timeout', 20) self.verify_cert = self.config.get('verify_cert', True) self.prefix = self.config.get('prefix', 'phishme') initial_interval = self.config.get('initial_interval', '30d') self.initial_interval = interval_in_sec(initial_interval) if self.initial_interval is None: LOG.error( '%s - wrong initial_interval format: %s', self.name, initial_interval ) self.initial_interval = interval_in_sec('30d') self.fields = self.config.get('fields', [ 'threatDetailURL', 'label', 'threatType' ]) self.confidence_map = self.config.get('confidence_map', { 'Major': 100, 'Moderate': 70, 'Minor': 34, 'None': 0 }) self.product = self.config.get('product', 'malware') self.source_name = self.config.get('source_name', 'phishme.intelligence') self.headers = {'user-agent': _API_USER_AGENT} self.api_key = None self.username = None self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.api_key = sconfig.get('api_key', None) if self.api_key is not None: LOG.info('%s - API Key set', self.name) self.username = sconfig.get('username', None) if self.username is not None: LOG.info('%s - username set', self.name) def _saved_state_restore(self, saved_state): super(Intelligence, self)._saved_state_restore(saved_state) self.position = saved_state.get('position', None) LOG.info('position from sstate: %s', self.position) def _saved_state_create(self): sstate = super(Intelligence, self)._saved_state_create() sstate['position'] = self.position return sstate def _saved_state_reset(self): super(Intelligence, self)._saved_state_reset() self.position = None def _update_attributes(self, current, _new, current_run, new_run): LOG.debug('current: %r', current) LOG.debug('_new: %r', _new) # create temp store for phishme spec values phishme_values = {} # loop over the phishme fields for f in self.fields: field_name = self.prefix+'_'+f newv = _new.get(field_name, None) if newv is None: continue phishme_values[field_name] = current.get(field_name, []) if newv[0] not in phishme_values[field_name]: phishme_values[field_name].append(newv[0]) # add role field_name = self.prefix+'_role' newrole = _new.get(field_name, None) if newrole is not None: phishme_values[field_name] = current.get(field_name, []) if newrole[0] not in phishme_values[field_name]: phishme_values[field_name].append(newrole[0]) # impact and confidence if _new['confidence'] < current['confidence']: phishme_values['confidence'] = current['confidence'] phishme_values[self.prefix+'_impact'] = current[self.prefix+'_impact'] LOG.debug(phishme_values) current.update(_new) current.update(phishme_values) return current def _convert_block(self, block): v = {} impact = block.get('impact', None) if impact is not None: v[self.prefix+'_impact'] = impact if impact in self.confidence_map: v['confidence'] = self.confidence_map[impact] role = block.get('role', None) if role is not None: v[self.prefix+'_role'] = [role] type_ = block.get('blockType', None) if type_ is None: LOG.error( '%s - no "blockType" attribute in block', self.name ) return None, None if type_ == 'IPv4 Address': v['type'] = 'IPv4' elif type_ == 'Domain Name': v['type'] = 'domain' elif type_ == 'URL': v['type'] = 'URL' else: LOG.error('%s - unknown blockType: %s', self.name, type_) return None, None indicator = block.get('data', None) if indicator is None: LOG.error('%s - no "data" attribute in block', self.name) return None, None return indicator, v def _process_item(self, item): result = [] block_set = item.get('blockSet', None) value = {} for f in self.fields: fv = item.get(f, None) if fv is None: continue value[self.prefix+'_'+f] = [fv] if block_set is not None: for block in block_set: indicator, v = self._convert_block(block) if indicator is not None: v.update(value) result.append([indicator, v]) else: LOG.error('%s - no "blockSet" in item', self.name) result = [[None, None]] return result def _build_iterator(self, now): LOG.info('position: %s', self.position) if self.api_key is None or self.username is None: raise RuntimeError('%s - credentials not set' % self.name) if self.position is None: # backfill return itertools.chain( self._threathq_backfill(now), self._threathq_update(now) ) # update return self._threathq_update(now) def _threathq_backfill(self, now): payload = { 'beginTimestamp': int(now/1000.0 - self.initial_interval), 'endTimestamp': int(now/1000.0), 'threatType': self.product, 'resultsPerPage': _RESULTS_PER_PAGE } cur_page = 0 total_pages = 1 while cur_page < total_pages: LOG.debug('%s - polling backfill %d/%d', self.name, cur_page, total_pages) payload['page'] = cur_page rkwargs = dict( verify=self.verify_cert, timeout=self.polling_timeout, params=payload, auth=(self.username, self.api_key), headers=self.headers ) r = requests.post( _API_BASE+_API_THREAT_SEARCH, **rkwargs ) try: r.raise_for_status() except: LOG.error( '%s - exception in request: %s %s', self.name, r.status_code, r.content ) raise cjson = r.json() data = cjson.get('data', None) if 'data' is None: LOG.error('%s - no "data" in response', self.name) return page = data.get('page', None) if page is None: LOG.error('%s - no "page" in response', self.name) return total_pages = page.get('totalPages', None) if total_pages is None: LOG.error('%s - no "totalPages" in response', self.name) return LOG.debug('%s - total_pages set to %d', self.name, total_pages) threats = data.get('threats', []) for t in threats: yield t cur_page += 1 def _threathq_update(self, now): changelog_size = 1000 while changelog_size == 1000: if self.position is not None: payload = dict(position=self.position) else: payload = dict(timestamp=int(now/1000.0)) rkwargs = dict( stream=True, verify=self.verify_cert, timeout=self.polling_timeout, params=payload, auth=(self.username, self.api_key), headers=self.headers ) r = requests.post( _API_BASE+_API_THREAT_UPDATE, **rkwargs ) try: r.raise_for_status() except: LOG.error( '%s - exception in request: %s %s', self.name, r.status_code, r.content ) raise cjson = r.json() data = cjson.get('data', None) if data is None: LOG.error('%s - no "data" in update request', self.name) return changelog = data.get('changelog', None) if changelog is not None: changelog_size = len(changelog) else: LOG.info('%s - no "changelog" in update request', self.name) changelog_size = 0 changelog = [] thgen = self._retrieve_threats( self._group_changes_in_pages( itertools.ifilter(self._filter_changes, changelog) ) ) for t in thgen: yield t next_position = data.get('nextPosition', None) if next_position is None: LOG.error('%s - no nextPosition in update request', self.name) else: self.position = next_position def _group_changes_in_pages(self, ichanges): # I know I could use izip with *n, but really ? threatids = [] for c in ichanges: id_ = str(c.get('threatId', None)) if id_ is None: LOG.error('%s - change with no threatId', self.name) continue type_ = c.get('threatType', None) if type_ is None: LOG.error('%s - change with no threatType', self.name) continue if type_ == 'malware': id_ = 'm_' + id_ elif type_ == 'phish': id_ = 'p_' + id_ else: LOG.error('%s - unknown threatType: %s', self.name, type_) continue threatids.append(id_) if len(threatids) == _RESULTS_PER_PAGE: yield threatids threatids = [] if len(threatids) != 0: yield threatids def _retrieve_threats(self, pages): for p in pages: payload = { 'resultsPerPage': _RESULTS_PER_PAGE, 'threatId': p } rkwargs = dict( verify=self.verify_cert, timeout=self.polling_timeout, params=payload, auth=(self.username, self.api_key), headers=self.headers ) r = requests.post( _API_BASE+_API_THREAT_SEARCH, **rkwargs ) try: r.raise_for_status() except: LOG.error( '%s - exception in request: %s %s', self.name, r.status_code, r.content ) raise cjson = r.json() data = cjson.get('data', None) if data is None: LOG.error('%s - no "data" in search request', self.name) continue threats = data.get('threats', None) if threats is None: LOG.error('%s - no "threats" in search request', self.name) continue for t in threats: yield t def _filter_changes(self, change): if change.get('deleted', None): LOG.debug('%s - deleted change', self.name) return False if self.product == 'all': return True threat_type = change.get('threatType', None) if threat_type is None: LOG.error('%s - change with no threatType', self.name) return False if threat_type == 'malware' and self.product == 'malware': return True if threat_type == 'phish' and self.product == 'phish': return True return False def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(Intelligence, self).hup(source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass ================================================ FILE: minemeld/ft/proofpoint.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import requests import os import shutil import yaml import datetime import pytz import netaddr import netaddr.core from minemeld import __version__ as MM_VERSION from . import basepoller from . import table from .utils import dt_to_millisec LOG = logging.getLogger(__name__) _CATNAME = [ "CnC", "Bot", "Spam", "Drop", "SpywareCnC", "OnlineGaming", "DriveBySrc", "ChatServer", "TorNode", "Compromised", "P2P", "Proxy", "IPCheck", "Utility", "DDoSTarget", "Scanner", "Brute_Forcer", "FakeAV", "DynDNS", "Undesirable", "AbusedTLD", "SelfSignedSSL", "Blackhole", "RemoteAccessService", "P2PCnC", "Parking", "VPN", "EXE_Source", "Mobile_CnC", "Mobile_Spyware_CnC", "Skype_SuperNode", "Bitcoin_Related", "DDoSAttacker" ] class ETIntelligence(basepoller.BasePollerFT): _FILE = None def __init__(self, name, chassis, config): self.ttable = None super(ETIntelligence, self).__init__(name, chassis, config) def configure(self): super(ETIntelligence, self).configure() self.polling_timeout = self.config.get('polling_timeout', 20) self.verify_cert = self.config.get('verify_cert', True) self.score_threshold = self.config.get('score_threshold', 50) self.source_name = 'proofpoint.etintelligence' self.auth_code = None self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.auth_code = sconfig.get('auth_code', None) if self.auth_code is not None: LOG.info('%s - authorization code set', self.name) monitored_categories = sconfig.get('monitored_categories', []) if type(monitored_categories) != list: LOG.error('%s - wrong monitored_categories format, should ' 'be a list of ints', self.name) self.monitored_categories = [] else: self.monitored_categories = monitored_categories def _process_row(self, row): indicator, category, score, first_seen, last_seen, ports = \ row.split(',') if indicator == 'ip' or indicator == 'domain': return None try: category = int(category) except ValueError: LOG.error('%s - wrong category format, ignored', self.name) return None if category not in self.monitored_categories: return None if category > 0 and category <= len(_CATNAME): category_name = _CATNAME[category-1] else: category_name = '%d' % category try: score = int(score) if score < 0 or score > 127: raise ValueError('wrong score format') except ValueError: LOG.error('%s - wrong score format, ignored', self.name) return None if score <= self.score_threshold: LOG.debug('%s - score below threshold, ignored', self.name) return None try: fs = datetime.datetime.strptime(first_seen, '%Y-%m-%d') fs = fs.replace(tzinfo=pytz.UTC) fs = dt_to_millisec(fs) except: LOG.exception('%s - wrong first_seen format, ignored', self.name) return None try: ls = datetime.datetime.strptime(last_seen, '%Y-%m-%d') ls = ls.replace(tzinfo=pytz.UTC) ls = dt_to_millisec(ls) except: LOG.exception('%s - wrong last_seen format, ignored', self.name) return None ports = ports.split() value = { 'proofpoint_etintelligence_max_score': score, 'proofpoint_etintelligence_last_seen': ls, 'proofpoint_etintelligence_first_seen': fs, 'proofpoint_etintelligence_ports': ports, 'proofpoint_etintelligence_categories': [category_name] } return [indicator, value] def _process_item(self, item): return [item] def _build_iterator(self, now): if self.auth_code is None or len(self.monitored_categories) == 0: raise RuntimeError( '%s - authorization code or categories not set, poll not performed' % self.name ) LOG.info('%s - categories: %s', self.name, self.monitored_categories) if self.ttable is not None: self.ttable.close() self.ttable = None self.ttable = table.Table(self.name+'_temp', truncate=True) url = ('https://rules.emergingthreats.net/' + self.auth_code + '/reputation/' + self._FILE) rkwargs = dict( stream=True, verify=self.verify_cert, timeout=self.polling_timeout, headers={ 'User-Agent': 'MineMeld/%s' % MM_VERSION } ) r = requests.get( url, **rkwargs ) try: r.raise_for_status() except: LOG.debug('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise for line in r.iter_lines(): p = self._process_row(line) if p is None: continue i, nv = p ov = self.ttable.get(i) if ov is None: self.ttable.put(i, nv) else: if (ov['proofpoint_etintelligence_max_score'] < nv['proofpoint_etintelligence_max_score']): ov['proofpoint_etintelligence_max_score'] = \ nv['proofpoint_etintelligence_max_score'] if (ov['proofpoint_etintelligence_first_seen'] > nv['proofpoint_etintelligence_first_seen']): ov['proofpoint_etintelligence_first_seen'] = \ nv['proofpoint_etintelligence_first_seen'] if (ov['proofpoint_etintelligence_last_seen'] > nv['proofpoint_etintelligence_last_seen']): ov['proofpoint_etintelligence_last_seen'] = \ nv['proofpoint_etintelligence_last_seen'] ov['proofpoint_etintelligence_ports'] += \ nv['proofpoint_etintelligence_ports'] ov['proofpoint_etintelligence_categories'] += \ nv['proofpoint_etintelligence_categories'] self.ttable.put(i, ov) return self.ttable.query(include_value=True) def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(ETIntelligence, self).hup(source=source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) shutil.rmtree('{}_temp'.format(name), ignore_errors=True) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass class EmergingThreatsIP(ETIntelligence): _FILE = 'detailed-iprepdata.txt' def _process_item(self, row): ipairs = super(EmergingThreatsIP, self)._process_item(row) result = [] for i, v in ipairs: try: parsed_ip = netaddr.IPAddress(i) except: LOG.error('%s - invalid IP %s, ignored', self.name, i) continue if parsed_ip.version == 4: v['type'] = 'IPv4' elif parsed_ip.version == 6: v['type'] = 'IPv6' else: LOG.error('%s - unknown IP version %s, ignored', self.name, i) continue result.append([i, v]) return result class EmergingThreatsDomain(ETIntelligence): _FILE = 'detailed-domainrepdata.txt' def _process_item(self, row): ipairs = super(EmergingThreatsDomain, self)._process_item(row) result = [] for i, v in ipairs: v['type'] = 'domain' result.append([i, v]) return result ================================================ FILE: minemeld/ft/recordedfuture.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import requests import os import ujson import yaml import netaddr import netaddr.core from minemeld.ft import csv # changed from . to minemeld.ft LOG = logging.getLogger(__name__) class IPRiskList(csv.CSVFT): def configure(self): super(IPRiskList, self).configure() self.source_name = 'recordedfuture.iprisklist' self.confidence = self.config.get('confidence', 80) self.token = None self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.token = sconfig.get('token', None) if self.token is not None: LOG.info('%s - token set', self.name) def _process_item(self, row): row.pop(None, None) # I love this result = {} indicator = row.get('Name', '') if indicator == '': return [] try: if '/' in indicator: ip = netaddr.IPNetwork(indicator) else: ip = netaddr.IPAddress(indicator) except netaddr.core.AddrFormatError: LOG.exception("%s - failed parsing indicator", self.name) return [] if ip.version == 4: result['type'] = 'IPv4' elif ip.version == 6: result['type'] = 'IPv6' else: LOG.debug("%s - unknown IP version %d", self.name, ip.version) return [] risk = row.get('Risk', '') if risk != '': try: result['recordedfuture_risk'] = int(risk) result['confidence'] = (int(risk) * self.confidence) / 100 except: LOG.debug("%s - invalid risk string: %s", self.name, risk) riskstring = row.get('RiskString', '') if riskstring != '': result['recordedfuture_riskstring'] = riskstring edetails = row.get('EvidenceDetails', '') if edetails != '': try: edetails = ujson.loads(edetails) except: LOG.debug("%s - invalid JSON string in EvidenceDetails: %s", self.name, edetails) else: edetails = edetails.get('EvidenceDetails', []) result['recordedfuture_evidencedetails'] = \ [ed['Rule'] for ed in edetails] result['recordedfuture_entityurl'] = \ 'https://app.recordedfuture.com/live/sc/entity/ip:' + indicator return [[indicator, result]] def _build_iterator(self, now): if self.token is None: raise RuntimeError( '%s - token not set, poll not performed' % self.name ) return super(IPRiskList, self)._build_iterator(now) def _build_request(self, now): params = {'output_format': 'csv/splunk'} headers = {'X-RFToken': self.token} r = requests.Request( 'GET', 'https://api.recordedfuture.com/v2/ip/risklist', headers=headers, params=params, ) return r.prepare() def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(IPRiskList, self).hup(source) @staticmethod def gc(name, config=None): csv.CSVFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass class DomainRiskList(csv.CSVFT): def configure(self): super(DomainRiskList, self).configure() self.source_name = 'recordedfuture.domainriskList' self.confidence = self.config.get('confidence', 80) self.token = None self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.token = sconfig.get('token', None) if self.token is not None: LOG.info('%s - token set', self.name) def _process_item(self, row): row.pop(None, None) # I love this result = {} indicator = row.get('Name', '') if indicator == '': return [] risk = row.get('Risk', '') if risk != '': try: result['recordedfuture_risk'] = int(risk) result['confidence'] = (int(risk) * self.confidence) / 100 except: LOG.debug("%s - invalid risk string: %s", self.name, risk) riskstring = row.get('RiskString', '') if riskstring != '': result['recordedfuture_riskstring'] = riskstring edetails = row.get('EvidenceDetails', '') if edetails != '': try: edetails = ujson.loads(edetails) except: LOG.debug("%s - invalid JSON string in EvidenceDetails: %s", self.name, edetails) else: edetails = edetails.get('EvidenceDetails', []) result['recordedfuture_evidencedetails'] = \ [ed['Rule'] for ed in edetails] result['recordedfuture_entityurl'] = \ 'https://app.recordedfuture.com/live/sc/entity/idn:' + indicator return [[indicator, result]] def _build_iterator(self, now): if self.token is None: raise RuntimeError( '%s - token not set, poll not performed' % self.name ) return super(DomainRiskList, self)._build_iterator(now) def _build_request(self, now): params = {'output_format': 'csv/splunk'} headers = {'X-RFToken': self.token} r = requests.Request( 'GET', 'https://api.recordedfuture.com/v2/domain/risklist', headers=headers, params=params, ) return r.prepare() def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(DomainRiskList, self).hup(source) @staticmethod def gc(name, config=None): csv.CSVFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass class MasterRiskList(csv.CSVFT): def configure(self): super(MasterRiskList, self).configure() self.source_name = 'recordedfuture.masterrisklist' self.confidence = self.config.get('confidence', 80) self.entity = None ## entity added self.token = None self.path = None ## fusion/ risklist path added self.api = None ## api type added self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.token = sconfig.get('token', None) if self.token is not None: LOG.info('%s - token set', self.name) self.path = sconfig.get('path', None) if self.path is not None: LOG.info('%s - path set', self.name) self.entity = sconfig.get('entity', None) if self.entity is not None: LOG.info('%s - entity set', self.name) self.api = sconfig.get('api', None) if self.api is not None: LOG.info('%s - API set', self.name) def _process_item(self, row): row.pop(None, None) url_key = 'recordedfuture_entityurl' base_url = 'https://app.recordedfuture.com/live/sc/entity/' result = {} indicator = row.get('Name', '') if indicator == '': return [] if self.entity == 'ip': try: if '/' in indicator: ip = netaddr.IPNetwork(indicator) else: ip = netaddr.IPAddress(indicator) except netaddr.core.AddrFormatError: LOG.exception("%s - failed parsing indicator", self.name) return [] if ip.version == 4: result['type'] = 'IPv4' elif ip.version == 6: result['type'] = 'IPv6' else: LOG.debug("%s - unknown IP version %d", self.name, ip.version) return [] result[url_key] = '{}ip:{}'.format(base_url, indicator) elif self.entity == 'domain': result['type'] = 'domain' result[url_key] = '{}idn:{}'.format(base_url, indicator) elif self.entity == 'url': result['type'] = 'URL' result[url_key] = '{}url:{}'.format(base_url, indicator) elif self.entity == 'hash': algo = row.get('Algorithm', '') if algo != '': result['recordedfuture_algorithm'] = algo result['type'] = self._check_hash_type(indicator) result[url_key] = '{}hash:{}'.format(base_url, indicator) risk = row.get('Risk', '') if risk != '': try: result['recordedfuture_risk'] = int(risk) result['confidence'] = (int(risk) * self.confidence) / 100 except: LOG.debug("%s - invalid risk string: %s", self.name, risk) riskstring = row.get('RiskString', '') if riskstring != '': result['recordedfuture_riskstring'] = riskstring edetails = row.get('EvidenceDetails', '') if edetails != '': try: edetails = ujson.loads(edetails) except: LOG.debug("%s - invalid JSON string in EvidenceDetails: %s", self.name, edetails) else: edetails = edetails.get('EvidenceDetails', []) result['recordedfuture_evidencedetails'] = \ [ed['Rule'] for ed in edetails] return [[indicator, result]] @staticmethod def _check_hash_type(entity): if len(entity) == 64: return 'sha256' elif len(entity) == 40: return 'sha1' elif len(entity) == 32: return 'md5' else: return '' def _build_iterator(self, now): if self.token is None: raise RuntimeError( '%s - token not set, poll not performed' % self.name ) if self.entity is None: raise RuntimeError( '%s - entity not set, poll not performed' % self.name ) if self.api is None: raise RuntimeError( '%s - api not set, poll not performed' % self.name ) if self.api == 'fusion': if self.entity == 'ip': if self.path != None: if self.path.find('ip') == -1: raise RuntimeError( '%s - wrong file path for the given miner' % self.name ) if self.entity == 'url': if self.path != None: if self.path.find('url') == -1: raise RuntimeError( '%s - wrong file path for the given miner' % self.name ) if self.entity == 'hash': if self.path != None: if self.path.find('hash') == -1: raise RuntimeError( '%s - wrong file path for the given miner' % self.name ) if self.entity == 'domain': if self.path != None: if self.path.find('domain') == -1: raise RuntimeError( '%s - wrong file path for the given miner' % self.name ) return super(MasterRiskList, self)._build_iterator(now) def _build_request(self, now): if self.api == 'connectApi': if self.path is None: url = 'https://api.recordedfuture.com/v2/' + str(self.entity) + '/risklist' else: url = 'https://api.recordedfuture.com/v2/' + str(self.entity) + '/risklist?list=' + self.path params = {'output_format': 'csv/splunk'} headers = {'X-RFToken': self.token, 'X-RF-User-Agent': 'Minemeld v1.2', 'content-type': 'application/json'} r = requests.Request('GET', url, headers=headers, params=params) return r.prepare() if self.api == 'fusion': if self.path is None: url = '/public/risklists/default_' + str(self.entity) + '_risklist.csv' else: url = self.path url = url.replace('/', '%2F') params = {'output_format': 'csv/splunk'} headers = {'X-RFToken': self.token, 'X-RF-User-Agent': 'Minemeld v1.2', 'content-type': 'application/json'} re = requests.Request('GET', 'https://api.recordedfuture.com/v2/fusion/files/?path=' + url, headers=headers, params=params) return re.prepare() def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(MasterRiskList, self).hup(source) @staticmethod def gc(name, config=None): csvhelper.CSVFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass ================================================ FILE: minemeld/ft/redis.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import redis import os import ujson as json from . import base from . import actorbase LOG = logging.getLogger(__name__) class RedisSet(actorbase.ActorBaseFT): def __init__(self, name, chassis, config): self.redis_skey = name self.redis_skey_value = name+'.value' self.redis_skey_chkp = name+'.chkp' self.SR = None super(RedisSet, self).__init__(name, chassis, config) def configure(self): super(RedisSet, self).configure() self.redis_url = self.config.get('redis_url', os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock') ) self.scoring_attribute = self.config.get( 'scoring_attribute', 'last_seen' ) self.store_value = self.config.get('store_value', False) self.max_entries = self.config.get('max_entries', 1000 * 1000) def connect(self, inputs, output): output = False super(RedisSet, self).connect(inputs, output) def read_checkpoint(self): self._connect_redis() self.last_checkpoint = None config = { 'class': (self.__class__.__module__+'.'+self.__class__.__name__), 'config': self._original_config } config = json.dumps(config, sort_keys=True) try: contents = self.SR.get(self.redis_skey_chkp) if contents is None: raise ValueError('{} - last checkpoint not found'.format(self.name)) if contents[0] == '{': # new format contents = json.loads(contents) self.last_checkpoint = contents['checkpoint'] saved_config = contents['config'] saved_state = contents['state'] else: self.last_checkpoint = contents saved_config = '' saved_state = None LOG.debug('%s - restored checkpoint: %s', self.name, self.last_checkpoint) # old_status is missing in old releases # stick to the old behavior if saved_config and saved_config != config: LOG.info( '%s - saved config does not match new config', self.name ) self.last_checkpoint = None return LOG.info( '%s - saved config matches new config', self.name ) if saved_state is not None: self._saved_state_restore(saved_state) except (ValueError, IOError): LOG.exception('{} - Error reading last checkpoint'.format(self.name)) self.last_checkpoint = None def create_checkpoint(self, value): self._connect_redis() config = { 'class': (self.__class__.__module__+'.'+self.__class__.__name__), 'config': self._original_config } contents = { 'checkpoint': value, 'config': json.dumps(config, sort_keys=True), 'state': self._saved_state_create() } self.SR.set(self.redis_skey_chkp, json.dumps(contents)) def remove_checkpoint(self): self._connect_redis() self.SR.delete(self.redis_skey_chkp) def _connect_redis(self): if self.SR is not None: return self.SR = redis.StrictRedis.from_url( self.redis_url ) def initialize(self): self._connect_redis() def rebuild(self): self._connect_redis() self.SR.delete(self.redis_skey) self.SR.delete(self.redis_skey_value) def reset(self): self._connect_redis() self.SR.delete(self.redis_skey) self.SR.delete(self.redis_skey_value) def _add_indicator(self, score, indicator, value): if self.length() >= self.max_entries: self.statistics['drop.overflow'] += 1 return with self.SR.pipeline() as p: p.multi() p.zadd(self.redis_skey, score, indicator) if self.store_value: p.hset(self.redis_skey_value, indicator, json.dumps(value)) result = p.execute()[0] self.statistics['added'] += result def _delete_indicator(self, indicator): with self.SR.pipeline() as p: p.multi() p.zrem(self.redis_skey, indicator) p.hdel(self.redis_skey_value, indicator) result = p.execute()[0] self.statistics['removed'] += result @base._counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): score = 0 if self.scoring_attribute is not None: av = value.get(self.scoring_attribute, None) if type(av) == int or type(av) == long: score = av else: LOG.error("scoring_attribute is not int: %s", type(av)) score = 0 self._add_indicator(score, indicator, value) @base._counting('withdraw.processed') def filtered_withdraw(self, source=None, indicator=None, value=None): self._delete_indicator(indicator) def length(self, source=None): return self.SR.zcard(self.redis_skey) @staticmethod def gc(name, config=None): actorbase.ActorBaseFT.gc(name, config=config) if config is None: config = {} redis_skey = name redis_skey_value = '{}.value'.format(name) redis_skey_chkp = '{}.chkp'.format(name) redis_url = config.get('redis_url', os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock') ) cp = None try: cp = redis.ConnectionPool.from_url( url=redis_url ) SR = redis.StrictRedis(connection_pool=cp) SR.delete(redis_skey) SR.delete(redis_skey_value) SR.delete(redis_skey_chkp) except Exception as e: raise RuntimeError(str(e)) finally: if cp is not None: cp.disconnect() ================================================ FILE: minemeld/ft/st.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Simple segment tree implementation based on LevelDB. **KEYS** Numbers are 8-bit unsigned. - Segment key: (1, , , , ) - Endpoint key: (1, , , , ) **ENDPOINT** - Type: 0: START, 1: END """ import plyvel import struct import logging import shutil import array LOG = logging.getLogger(__name__) MAX_LEVEL = 0xFE TYPE_START = 0x00 TYPE_END = 0x1 class ST(object): def __init__(self, name, epsize, truncate=False, bloom_filter_bits=10, write_buffer_size=(4 << 20)): if truncate: try: shutil.rmtree(name) except: pass self.db = plyvel.DB( name, create_if_missing=True, write_buffer_size=write_buffer_size, bloom_filter_bits=bloom_filter_bits ) self.epsize = epsize self.max_endpoint = (1 << epsize)-1 self.num_endpoints = 0 self.num_segments = 0 def _split_interval(self, start, end, lower, upper): if start <= lower and upper <= end: return [(lower, upper)] mid = (lower+upper)/2 result = [] if start <= mid: result += self._split_interval(start, end, lower, mid) if end > mid: result += self._split_interval(start, end, mid+1, upper) return result def _segment_key(self, start, end, uuid_=None, level=None): res = array.array('B', [ 1, (start >> 56) & 0xFF, (start >> 48) & 0xFF, (start >> 40) & 0xFF, (start >> 32) & 0xFF, (start >> 24) & 0xFF, (start >> 16) & 0xFF, (start >> 8) & 0xFF, start & 0xFF, (end >> 56) & 0xFF, (end >> 48) & 0xFF, (end >> 40) & 0xFF, (end >> 32) & 0xFF, (end >> 24) & 0xFF, (end >> 16) & 0xFF, (end >> 8) & 0xFF, end & 0xFF, ]) if level is not None: res.append(level) if uuid_ is not None: for c in uuid_: res.append(ord(c)) return res.tostring() def _split_segment_key(self, key): _, start, end, level = struct.unpack(">BQQB", key[:18]) return start, end, level, key[18:] def _endpoint_key(self, endpoint, level=None, type_=None, uuid_=None): res = array.array('B', [ 2, (endpoint >> 56) & 0xFF, (endpoint >> 48) & 0xFF, (endpoint >> 40) & 0xFF, (endpoint >> 32) & 0xFF, (endpoint >> 24) & 0xFF, (endpoint >> 16) & 0xFF, (endpoint >> 8) & 0xFF, endpoint & 0xFF ]) if level is not None: res.append(level) if type_ is not None: res.append(type_) if uuid_ is not None: for c in uuid_: res.append(ord(c)) return res.tostring() def _split_endpoint_key(self, k): _, endpoint, level, type_ = struct.unpack(">BQBB", k[:11]) type_ = (True if type_ == TYPE_START else False) return endpoint, level, type_, k[11:] def close(self): self.db.close() def put(self, uuid_, start, end, level=0): si = self._split_interval(start, end, 0, self.max_endpoint) value = struct.pack(">QQ", start, end) batch = self.db.write_batch() for i in si: k = self._segment_key(i[0], i[1], uuid_=uuid_, level=level) batch.put(k, value) ks = self._endpoint_key( start, level=level, type_=TYPE_START, uuid_=uuid_ ) batch.put(ks, "\x00") ke = self._endpoint_key( end, level=level, type_=TYPE_END, uuid_=uuid_ ) batch.put(ke, "\x00") batch.write() self.num_endpoints += 2 self.num_segments += len(si) def delete(self, uuid_, start, end, level=0): batch = self.db.write_batch() si = self._split_interval(start, end, 0, self.max_endpoint) for i in si: k = self._segment_key(i[0], i[1], uuid_=uuid_, level=level) batch.delete(k) ks = self._endpoint_key( start, level=level, type_=TYPE_START, uuid_=uuid_ ) batch.delete(ks) ke = self._endpoint_key( end, level=level, type_=TYPE_END, uuid_=uuid_ ) batch.delete(ke) batch.write() self.num_endpoints -= 2 self.num_segments -= len(si) def cover(self, value): """Iterate over segments covering value. Segment format: (uuid, level, start, end). Args: value (int): Address """ lower = 0 upper = self.max_endpoint*2 while True: mid = (lower+upper)/2 if value <= mid: upper = mid else: lower = mid+1 ks = self._segment_key(lower, upper) ke = self._segment_key(lower, upper, level=MAX_LEVEL+1) for k, v in self.db.iterator(start=ks, stop=ke, include_value=True, reverse=True, include_start=False, include_stop=False): _, _, level, uuid_ = self._split_segment_key(k) start, end = struct.unpack(">QQ", v) yield uuid_, level, start, end if lower == upper: break def query_endpoints(self, start=None, stop=None, reverse=False, include_start=True, include_stop=True): """Iterate over endpoints between start and end. endpoints have the format (endpoint, level, type, uuid). Type: 0 - start, 1 - end start (int, optional): Defaults to None. stop (int, optional): Defaults to None. reverse (bool, optional): Defaults to False. include_start (bool, optional): Defaults to True. include_stop (bool, optional): Defaults to True. """ if start is None: start = self._endpoint_key(0) else: start = self._endpoint_key(start) if stop is None: stop = self._endpoint_key(self.max_endpoint, level=MAX_LEVEL+1) else: stop = self._endpoint_key(stop, level=MAX_LEVEL+1) di = self.db.iterator( start=start, stop=stop, reverse=reverse, include_value=False, include_start=include_start, include_stop=include_stop ) for k in di: yield self._split_endpoint_key(k) ================================================ FILE: minemeld/ft/syslog.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import shutil import gevent import gevent.queue import gevent.event import amqp import ujson import netaddr import datetime import socket import random import os import yaml import copy import re from . import base from . import actorbase from . import table from . import ft_states from . import condition from .utils import utc_millisec from .utils import RWLock from .utils import parse_age_out LOG = logging.getLogger(__name__) _MAX_AGE_OUT = ((1 << 32)-1)*1000 class SyslogMatcher(actorbase.ActorBaseFT): def __init__(self, name, chassis, config): self.amqp_glet = None super(SyslogMatcher, self).__init__(name, chassis, config) self._ls_socket = None def configure(self): super(SyslogMatcher, self).configure() self.exchange = self.config.get('exchange', 'mmeld-syslog') self.rabbitmq_username = self.config.get('rabbitmq_username', 'guest') self.rabbitmq_password = self.config.get('rabbitmq_password', 'guest') self.input_types = self.config.get('input_types', {}) self.logstash_host = self.config.get('logstash_host', None) self.logstash_port = self.config.get('logstash_port', 5514) def _initialize_tables(self, truncate=False): self.table_ipv4 = table.Table(self.name+'_ipv4', truncate=truncate) self.table_ipv4.create_index('_start') self.table_indicators = table.Table( self.name+'_indicators', truncate=truncate ) self.table = table.Table(self.name, truncate=truncate) self.table.create_index('syslog_original_indicator') def initialize(self): self._initialize_tables() def rebuild(self): self._initialize_tables(truncate=True) def reset(self): self._initialize_tables(truncate=True) @base._counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): type_ = value.get('type', None) if type_ is None: LOG.error("%s - received update with no type, ignored", self.name) return itype = self.input_types.get(source, None) if itype is None: LOG.debug('%s - no type associated to %s, added %s', self.name, source, type_) self.input_types[source] = type_ itype = type_ if itype != type_: LOG.error("%s - indicator of type %s received from " "source %s with type %s, ignored", self.name, type_, source, itype) return if type_ == 'IPv4': start, end = map(netaddr.IPAddress, indicator.split('-', 1)) LOG.debug('start: %d', start.value) value['_start'] = start.value value['_end'] = end.value self.table_ipv4.put(indicator, value) else: self.table_indicators.put(type_+indicator, value) @base._counting('withdraw.processed') def filtered_withdraw(self, source=None, indicator=None, value=None): itype = self.input_types.get(source, None) if itype is None: LOG.error('%s - withdraw from unknown source', self.name) return if itype == 'IPv4': v = self.table_ipv4.get(indicator) if v is not None: self.table_ipv4.delete(indicator) else: v = self.table_indicators.get(itype+indicator) if v is not None: self.table_indicators.delete(itype+indicator) if v is not None: for i, v in self.table.query(index='syslog_original_indicator', from_key=itype+indicator, to_key=itype+indicator, include_value=True): self.emit_withdraw(i, value=v) self.table.delete(i) def _handle_ip(self, ip, source=True, message=None): try: ipv = netaddr.IPAddress(ip) except: return if ipv.version != 4: return ipv = ipv.value iv = next( (self.table_ipv4.query(index='_start', to_key=ipv, include_value=True, include_start=True, reverse=True)), None ) if iv is None: return i, v = iv if v['_end'] < ipv: return for s in v.get('sources', []): self.statistics['source.'+s] += 1 self.statistics['total_matches'] += 1 v['syslog_original_indicator'] = 'IPv4'+i self.table.put(ip, v) self.emit_update(ip, v) if message is not None: self._send_logstash( message='matched IPv4', indicator=i, value=v, session=message ) def _handle_url(self, url, message=None): domain = url.split('/', 1)[0] v = self.table_indicators.get('domain'+domain) if v is None: return v['syslog_original_indicator'] = 'domain'+domain for s in v.get('sources', []): self.statistics[s] += 1 self.statistics['total_matches'] += 1 self.table.put(domain, v) self.emit_update(domain, v) if message is not None: self._send_logstash( message='matched domain', indicator=domain, value=v, session=message ) @base._counting('syslog.processed') def _handle_syslog_message(self, message): src_ip = message.get('src_ip', None) if src_ip is not None: self._handle_ip(src_ip, message=message) dst_ip = message.get('dest_ip', None) if dst_ip is not None: self._handle_ip(dst_ip, source=False, message=message) url = message.get('url', None) if url is not None: self._handle_url(url, message=message) def _amqp_callback(self, msg): try: message = ujson.loads(msg.body) self._handle_syslog_message(message) except gevent.GreenletExit: raise except: LOG.exception( "%s - exception handling syslog message", self.name ) def _amqp_consumer(self): while True: try: conn = amqp.connection.Connection( userid=self.rabbitmq_username, password=self.rabbitmq_password ) channel = conn.channel() channel.exchange_declare( self.exchange, 'fanout', durable=False, auto_delete=False ) q = channel.queue_declare( exclusive=False ) channel.queue_bind( queue=q.queue, exchange=self.exchange, ) channel.basic_consume( callback=self._amqp_callback, no_ack=True, exclusive=True ) while True: conn.drain_events() except gevent.GreenletExit: break except: LOG.exception('%s - Exception in consumer glet', self.name) gevent.sleep(30) def _connect_logstash(self): if self._ls_socket is not None: return if self.logstash_host is None: return _ls_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) _ls_socket.connect((self.logstash_host, self.logstash_port)) self._ls_socket = _ls_socket def _send_logstash(self, message=None, indicator=None, value=None, session=None): now = datetime.datetime.now() fields = { '@timestamp': now.isoformat()+'Z', '@version': 1, 'syslog_node': self.name, 'message': message } if indicator is not None: fields['indicator'] = indicator if value is not None: if 'last_seen' in value: last_seen = datetime.datetime.fromtimestamp( float(value['last_seen'])/1000.0 ) value['last_seen'] = last_seen.isoformat()+'Z' if 'first_seen' in fields: first_seen = datetime.datetime.fromtimestamp( float(value['first_seen'])/1000.0 ) value['first_seen'] = first_seen.isoformat()+'Z' fields['indicator_value'] = value if session is not None: session.pop('event.tags', None) fields['session'] = session try: self._connect_logstash() if self._ls_socket is not None: self._ls_socket.sendall(ujson.dumps(fields)+'\n') except: self._ls_socket = None raise self.statistics['logstash.sent'] += 1 def mgmtbus_status(self): result = super(SyslogMatcher, self).mgmtbus_status() return result def length(self, source=None): return (self.table_ipv4.num_indicators + self.table_indicators.num_indicators) def start(self): super(SyslogMatcher, self).start() self.amqp_glet = gevent.spawn_later( 2, self._amqp_consumer ) def stop(self): super(SyslogMatcher, self).stop() if self.amqp_glet is None: return self.table.close() self.table_indicators.close() self.table_ipv4.close() @staticmethod def gc(name, config=None): actorbase.ActorBaseFT.gc(name, config=config) shutil.rmtree(name, ignore_errors=True) shutil.rmtree('{}_indicators'.format(name), ignore_errors=True) shutil.rmtree('{}_ipv4'.format(name), ignore_errors=True) class SyslogMiner(base.BaseFT): def __init__(self, name, chassis, config): self.amqp_glet = None self.ageout_glet = None self._actor_glet = None self._actor_queue = gevent.queue.Queue(maxsize=128) self._msg_queue = gevent.queue.Queue(maxsize=1) self._do_process = gevent.event.Event() self.active_requests = [] self.rebuild_flag = False self.last_ageout_run = None self.state_lock = RWLock() super(SyslogMiner, self).__init__(name, chassis, config) def configure(self): super(SyslogMiner, self).configure() self.source_name = self.config.get('source_name', self.name) self.attributes = self.config.get('attributes', {}) _age_out = self.config.get('age_out', {}) self.age_out = { 'interval': _age_out.get('interval', 3600), 'default': parse_age_out(_age_out.get('default', 'last_seen+1h')) } for k, v in _age_out.iteritems(): if k in self.age_out: continue self.age_out[k] = parse_age_out(v) self.exchange = self.config.get('exchange', 'mmeld-syslog') self.rabbitmq_username = self.config.get('rabbitmq_username', 'guest') self.rabbitmq_password = self.config.get('rabbitmq_password', 'guest') self.indicator_mapping = self.config.get('indicator_mapping', { 'src_ip': 'IP', 'dest_ip': 'IP', 'misc': 'URL' }) self.prefix = self.config.get('prefix', 'panossyslog') self.rules = [] self.side_config_path = self.config.get('rules', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_rules.yml' % self.name ) self._load_side_config() def _initialize_table(self, truncate=False): self.table = table.Table(self.name, truncate=truncate) self.table.create_index('_age_out') self.table.create_index('_withdrawn') def initialize(self): self._initialize_table() def rebuild(self): self._actor_queue.put( (utc_millisec(), 'rebuild') ) self._initialize_table(truncate=(self.last_checkpoint is None)) def reset(self): self._initialize_table(truncate=True) def _compile_rule(self, name, f): LOG.debug('%s - compiling rule %s: %s', self.name, name, f) result = { 'name': name, 'metric': 'rule.%s' % re.sub('[^a-zA-Z0-9]', '_', name), 'conditions': [], 'indicators': [], 'fields': [] } conditions = f.get('conditions', None) if conditions is None or len(conditions) == 0: LOG.error('%s - no conditions in rule %s, ignored', self.name, name) return None for c in conditions: result['conditions'].append(condition.Condition(c)) indicators = f.get('indicators', None) if type(indicators) != list: LOG.error('%s - no indicators list in rule %s, ignored', self.name, name) return None for i in indicators: if i not in self.indicator_mapping: LOG.error('%s - rule %s unknown type indicator %s, ignored', self.name, name, i) continue result['indicators'].append(i) if len(result['indicators']) == 0: LOG.error('%s - no valid indicators in rule %s, ignored', self.name, name) return None fields = f.get('fields', None) if fields is not None: if type(fields) != list: LOG.error('%s - wrong fields format in rule %s, ignored', self.name, name) return None result['fields'] = [fld for fld in fields if type(fld) == str] return result def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: rules = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading rules: %s', self.name, str(e)) return if type(rules) != list: LOG.error('%s - Error loading rules: not a list', self.name) return newrules = [] for idx, f in enumerate(rules): fname = f.get('name', None) if fname is None: LOG.error('%s - rule %d does not have a name, ignored', self.name, idx) continue cf = self._compile_rule(fname, f) if cf is not None: newrules.append(cf) self.rules = newrules @base.BaseFT.state.setter def state(self, value): self.state_lock.lock() # this is weird ! from stackoverflow 10810369 super(SyslogMiner, self.__class__).state.fset(self, value) self.state_lock.unlock() def _command_rebuild(self): with self.state_lock: if self.state != ft_states.STARTED: return for i, v in self.table.query(include_value=True): indicator, _ = i.split('\0', 1) self.emit_update(indicator=indicator, value=v) def _command_age_out(self): with self.state_lock: if self.state != ft_states.STARTED: return try: now = utc_millisec() for i, v in self.table.query(index='_age_out', to_key=now-1, include_value=True): indicator, _ = i.split('\0', 1) self.emit_withdraw(indicator=indicator, value=v) self.table.delete(i) self.statistics['aged_out'] += 1 self.last_ageout_run = now except gevent.GreenletExit: raise except: LOG.exception('Exception in _age_out_loop') def _calc_age_out(self, indicator, attributes): t = attributes.get('type', None) if t is None or t not in self.age_out: sel = self.age_out['default'] else: sel = self.age_out[t] if sel is None: return _MAX_AGE_OUT b = attributes[sel['base']] return b + sel['offset'] def _apply_rule(self, f, message): r = True for c in f['conditions']: r &= c.eval(message) if not r: return for i in f['indicators']: indicator = message.get(i, None) if indicator is None: continue value = {} for fld in f['fields']: fv = message.get(fld, None) if fv is not None: value['%s_%s' % (self.prefix, fld)] = fv type_ = self.indicator_mapping[i] if type_ == 'IP': pi = netaddr.IPAddress(indicator) if pi.version == 6: type_ = 'IPv6' elif pi.version == 4: type_ = 'IPv4' else: continue value['type'] = type_ device = message.get('serial_number', 'unknown') yield [indicator, value, device] @base._counting('syslog.processed') def _handle_syslog_message(self, message): devices_attribute = '%s_devices' % self.prefix now = utc_millisec() for f in self.rules: for indicator, value, device in self._apply_rule(f, message): if indicator is None: continue self.statistics[f['metric']] += 1 type_ = value.get('type', None) if type_ is None: LOG.error('%s - no type for indicator %s, ignored', self.name, indicator) continue ikey = indicator+'\0'+type_ cv = self.table.get(ikey) if cv is None: cv = copy.copy(self.attributes) cv['sources'] = [self.source_name] cv['last_seen'] = now cv['first_seen'] = now cv[devices_attribute] = [device] cv.update(value) cv['_age_out'] = self._calc_age_out(indicator, cv) self.statistics['added'] += 1 self.table.put(ikey, cv) self.emit_update(indicator, cv) else: cv['last_seen'] = now cv.update(value) cv['_age_out'] = self._calc_age_out(indicator, cv) if device not in cv[devices_attribute]: cv[devices_attribute].append(device) self.table.put(ikey, cv) self.emit_update(indicator, cv) def _actor_loop(self): while True: msg = None try: msg = self._actor_queue.get(block=False) except gevent.queue.Empty: msg = None if msg is not None: _, command = msg if command == 'age_out': self._command_age_out() elif command == 'rebuild': self._command_rebuild() else: LOG.error('{} - unknown command {} - ignored'.format( self.name, command )) msg = None try: while self._msg_queue.qsize() != 0: msg = self._msg_queue.get(block=False) try: self._handle_syslog_message(msg) except gevent.GreenletExit: raise except: LOG.exception('{} - exception handling message'.format(self.name)) except gevent.queue.Empty: pass self._do_process.wait() self._do_process.clear() def _age_out_loop(self): while True: self._actor_queue.put( (utc_millisec(), 'age_out') ) self._do_process.set() try: gevent.sleep(self.age_out['interval']) except gevent.GreenletExit: break def _amqp_callback(self, msg): try: LOG.info(u'{}'.format(msg.body)) message = ujson.loads(msg.body) self._msg_queue.put(message) self._do_process.set() except gevent.GreenletExit: raise except: LOG.exception( "%s - exception handling syslog message", self.name ) def _amqp_consumer(self): while self.last_ageout_run is None: gevent.sleep(1) with self.state_lock: if self.state != ft_states.STARTED: LOG.error('{} - wrong state in amqp_consumer'.format(self.name)) return while True: try: conn = amqp.connection.Connection( userid=self.rabbitmq_username, password=self.rabbitmq_password ) channel = conn.channel() channel.exchange_declare( self.exchange, 'fanout', durable=False, auto_delete=False ) q = channel.queue_declare( exclusive=False ) channel.queue_bind( queue=q.queue, exchange=self.exchange, ) channel.basic_consume( callback=self._amqp_callback, no_ack=True, exclusive=True ) while True: conn.drain_events() except gevent.GreenletExit: break except: LOG.exception('%s - Exception in consumer glet', self.name) gevent.sleep(30) def length(self, source=None): return self.table.num_indicators def start(self): super(SyslogMiner, self).start() if self.amqp_glet is not None: return self.amqp_glet = gevent.spawn_later( random.randint(0, 2), self._amqp_consumer ) self.ageout_glet = gevent.spawn(self._age_out_loop) self._actor_glet = gevent.spawn(self._actor_loop) def stop(self): super(SyslogMiner, self).stop() if self.amqp_glet is None: return for g in self.active_requests: g.kill() self.amqp_glet.kill() self.ageout_glet.kill() self._actor_glet.kill() self.table.close() LOG.info("%s - # indicators: %d", self.name, self.table.num_indicators) def hup(self, source=None): LOG.info('%s - hup received, reload filters', self.name) self._load_side_config() @staticmethod def gc(name, config=None): base.BaseFT.gc(name, config=config) shutil.rmtree(name, ignore_errors=True) side_config_path = None if config is not None: side_config_path = config.get('rules', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_rules.yml'.format(name) ) try: os.remove(side_config_path) except: pass ================================================ FILE: minemeld/ft/table.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Table implementation based on LevelDB (https://github.com/google/leveldb). This is a sort of poor, lazy man implementation of IndexedDB schema. **KEYS** Numbers are 8-bit unsigned. - Schema Version: (0) - Index Last Global Id: (0,1, ) - Last Update Key: (0,2) - Number of Indicators: (0,3) - Table Last Global ID: (0,4) - Custom Metadata: (0,5) - Indicator Version: (1,0,) - Indicator: (1,1,) **INDICATORS** Each indicators has 2 entries associated in the DB: a version and a value. The version number is used to track indicator existance and versioning. When an indicator value is updated, its version number is incremented. The version number is a 64-bit LSB unsigned int. The value of an indicator is a 64-bit unsigned int LSB followed by a dump of a dictionary of attributes in JSON format. To iterate over all the indicators versions iterate from key (1,0) to key (1,1) excluded. NULL indicators are not allowed. **INDEXES** Indicators are stored in alphabetical order. Indexes are secondary indexes on indicators attributes. Each index has an associated id in the range 0 - 255. The attribute associated to the index is stored at (0,1,), if the key does not exist the index does not exist. There is also a Last Global Id per index, used to index indicators with the same attribute value. Each time a new indicator is added to the index, the Last Global Id is incremented. The Last Global Id of an index is stored at (2,,0) as a 64-bit LSB unsigned int. Each entry in the index is stored with a key (2,,0xF0,,) and value (,). depends on the type of attribute. When iterating over an index, the value of an index entry is loaded and if the version does not match with current indicator version the index entry is deleted. This permits a sort of lazy garbage collection. To retrieve all the indicators with a specific attribute value just iterate over the keys (2,,0xF0,) and (2,,0xF0,,0xFF..FF) """ import os import plyvel import struct import ujson import time import logging import shutil import gevent SCHEMAVERSION_KEY = struct.pack("B", 0) START_INDEX_KEY = struct.pack("BBB", 0, 1, 0) END_INDEX_KEY = struct.pack("BBB", 0, 1, 0xFF) LAST_UPDATE_KEY = struct.pack("BB", 0, 2) NUM_INDICATORS_KEY = struct.pack("BB", 0, 3) TABLE_LAST_GLOBAL_ID = struct.pack("BB", 0, 4) CUSTOM_METADATA = struct.pack("BB", 0, 5) LOG = logging.getLogger(__name__) class InvalidTableException(Exception): pass class Table(object): def __init__(self, name, truncate=False, bloom_filter_bits=0): if truncate: try: shutil.rmtree(name) except: pass self.db = None self._compact_glet = None self.db = plyvel.DB( name, create_if_missing=True, bloom_filter_bits=bloom_filter_bits ) self._read_metadata() self.compact_interval = int(os.environ.get('MM_TABLE_COMPACT_INTERVAL', 3600 * 6)) self.compact_delay = int(os.environ.get('MM_TABLE_COMPACT_DELAY', 3600)) self._compact_glet = gevent.spawn(self._compact_loop) def _init_db(self): self.last_update = 0 self.indexes = {} self.num_indicators = 0 self.last_global_id = 0 batch = self.db.write_batch() batch.put(SCHEMAVERSION_KEY, struct.pack("B", 1)) batch.put(LAST_UPDATE_KEY, struct.pack(">Q", self.last_update)) batch.put(NUM_INDICATORS_KEY, struct.pack(">Q", self.num_indicators)) batch.put(TABLE_LAST_GLOBAL_ID, struct.pack(">Q", self.last_global_id)) batch.write() def _read_metadata(self): sv = self._get(SCHEMAVERSION_KEY) if sv is None: return self._init_db() sv = struct.unpack("B", sv)[0] if sv == 0: # add table last global id self._upgrade_from_s0() elif sv == 1: pass else: raise InvalidTableException("Schema version not supported") self.indexes = {} ri = self.db.iterator( start=START_INDEX_KEY, stop=END_INDEX_KEY ) with ri: for k, v in ri: _, _, indexid = struct.unpack("BBB", k) if v in self.indexes: raise InvalidTableException("2 indexes with the same name") self.indexes[v] = { 'id': indexid, 'last_global_id': 0 } for i in self.indexes: lgi = self._get(self._last_global_id_key(self.indexes[i]['id'])) if lgi is not None: self.indexes[i]['last_global_id'] = struct.unpack(">Q", lgi)[0] else: self.indexes[i]['last_global_id'] = -1 t = self._get(LAST_UPDATE_KEY) if t is None: raise InvalidTableException("LAST_UPDATE_KEY not found") self.last_update = struct.unpack(">Q", t)[0] t = self._get(NUM_INDICATORS_KEY) if t is None: raise InvalidTableException("NUM_INDICATORS_KEY not found") self.num_indicators = struct.unpack(">Q", t)[0] t = self._get(TABLE_LAST_GLOBAL_ID) if t is None: raise InvalidTableException("TABLE_LAST_GLOBAL_ID not found") self.last_global_id = struct.unpack(">Q", t)[0] def _get(self, key): try: result = self.db.get(key) except KeyError: return None return result def __del__(self): self.close() def get_custom_metadata(self): cmetadata = self._get(CUSTOM_METADATA) if cmetadata is None: return None return ujson.loads(cmetadata) def set_custom_metadata(self, metadata=None): if metadata is None: self.db.delete(CUSTOM_METADATA) return cmetadata = ujson.dumps(metadata) self.db.put(CUSTOM_METADATA, cmetadata) def close(self): if self.db is not None: self.db.close() if self._compact_glet is not None: self._compact_glet.kill() self.db = None self._compact_glet = None def exists(self, key): if type(key) == unicode: key = key.encode('utf8') ikeyv = self._indicator_key_version(key) return (self._get(ikeyv) is not None) def get(self, key): if type(key) == unicode: key = key.encode('utf8') ikey = self._indicator_key(key) value = self._get(ikey) if value is None: return None # skip version return ujson.loads(value[8:]) def delete(self, key): if type(key) == unicode: key = key.encode('utf8') ikey = self._indicator_key(key) ikeyv = self._indicator_key_version(key) if self._get(ikeyv) is None: return batch = self.db.write_batch() batch.delete(ikey) batch.delete(ikeyv) self.num_indicators -= 1 batch.put(NUM_INDICATORS_KEY, struct.pack(">Q", self.num_indicators)) batch.write() def _indicator_key(self, key): return struct.pack("BB", 1, 1) + key def _indicator_key_version(self, key): return struct.pack("BB", 1, 0) + key def _index_key(self, idxid, value, lastidxid=None): key = struct.pack("BBB", 2, idxid, 0xF0) if type(value) == unicode: value = value.encode('utf8') if type(value) == str: key += struct.pack(">BL", 0x0, len(value))+value elif type(value) == int or type(value) == long: key += struct.pack(">BQ", 0x1, value) else: raise ValueError("Unhandled value type: %s" % type(value)) if lastidxid is not None: key += struct.pack(">Q", lastidxid) return key def _last_global_id_key(self, idxid): return struct.pack("BBB", 2, idxid, 0) def create_index(self, attribute): if attribute in self.indexes: return if len(self.indexes) == 0: idxid = 0 else: idxid = max([i['id'] for i in self.indexes.values()])+1 self.indexes[attribute] = { 'id': idxid, 'last_global_id': -1 } batch = self.db.write_batch() batch.put(struct.pack("BBB", 0, 1, idxid), attribute) batch.write() def put(self, key, value): if type(key) == unicode: key = key.encode('utf8') if type(value) != dict: raise ValueError() ikey = self._indicator_key(key) ikeyv = self._indicator_key_version(key) exists = self._get(ikeyv) self.last_global_id += 1 cversion = self.last_global_id now = time.time() self.last_update = now batch = self.db.write_batch() batch.put(ikey, struct.pack(">Q", cversion)+ujson.dumps(value)) batch.put(ikeyv, struct.pack(">Q", cversion)) batch.put(LAST_UPDATE_KEY, struct.pack(">Q", self.last_update)) batch.put(TABLE_LAST_GLOBAL_ID, struct.pack(">Q", self.last_global_id)) if exists is None: self.num_indicators += 1 batch.put( NUM_INDICATORS_KEY, struct.pack(">Q", self.num_indicators) ) for iattr, index in self.indexes.iteritems(): v = value.get(iattr, None) if v is None: continue index['last_global_id'] += 1 idxkey = self._index_key(index['id'], v, index['last_global_id']) batch.put(idxkey, struct.pack(">Q", cversion) + key) batch.put( self._last_global_id_key(index['id']), struct.pack(">Q", index['last_global_id']) ) batch.write() def query(self, index=None, from_key=None, to_key=None, include_value=False, include_stop=True, include_start=True, reverse=False): if type(from_key) is unicode: from_key = from_key.encode('ascii', 'replace') if type(to_key) is unicode: to_key = to_key.encode('ascii', 'replace') if index is None: return self._query_by_indicator( from_key=from_key, to_key=to_key, include_value=include_value, include_stop=include_stop, include_start=include_start, reverse=reverse ) return self._query_by_index( index, from_key=from_key, to_key=to_key, include_value=include_value, include_stop=include_stop, include_start=include_start, reverse=reverse ) def _query_by_indicator(self, from_key=None, to_key=None, include_value=False, include_stop=True, include_start=True, reverse=False): if from_key is None: from_key = struct.pack("BB", 1, 1) include_stop = False else: from_key = self._indicator_key(from_key) if to_key is None: to_key = struct.pack("BB", 1, 2) include_start = False else: to_key = self._indicator_key(to_key) ri = self.db.iterator( start=from_key, stop=to_key, include_stop=include_stop, include_start=include_start, reverse=reverse, include_value=False ) with ri: for ekey in ri: ekey = ekey[2:] if include_value: yield ekey.decode('utf8', 'ignore'), self.get(ekey) else: yield ekey.decode('utf8', 'ignore') def _query_by_index(self, index, from_key=None, to_key=None, include_value=False, include_stop=True, include_start=True, reverse=False): if index not in self.indexes: raise ValueError() idxid = self.indexes[index]['id'] if from_key is None: from_key = struct.pack("BBB", 2, idxid, 0xF0) include_start = False else: from_key = self._index_key(idxid, from_key) if to_key is None: to_key = struct.pack("BBB", 2, idxid, 0xF1) include_stop = False else: to_key = self._index_key( idxid, to_key, lastidxid=0xFFFFFFFFFFFFFFFF ) ldeleted = 0 ri = self.db.iterator( start=from_key, stop=to_key, include_value=True, include_start=include_start, include_stop=include_stop, reverse=reverse ) with ri: for ikey, ekey in ri: iversion = struct.unpack(">Q", ekey[:8])[0] ekey = ekey[8:] evalue = self._get(self._indicator_key_version(ekey)) if evalue is None: # LOG.debug("Key does not exist") # key does not exist self.db.delete(ikey) ldeleted += 1 continue cversion = struct.unpack(">Q", evalue)[0] if iversion != cversion: # index value is old # LOG.debug("Version mismatch") self.db.delete(ikey) ldeleted += 1 continue if include_value: yield ekey.decode('utf8', 'ignore'), self.get(ekey) else: yield ekey.decode('utf8', 'ignore') LOG.info('Deleted in scan of {}: {}'.format(index, ldeleted)) def _compact_loop(self): gevent.sleep(self.compact_delay) while True: try: gevent.idle() counter = 0 for idx in self.indexes.keys(): for i in self.query(index=idx, include_value=False): if counter % 512 == 0: gevent.sleep(0.001) # yield to other greenlets counter += 1 except gevent.GreenletExit: break except: LOG.exception('Exception in _compact_loop') try: gevent.sleep(self.compact_interval) except gevent.GreenletExit: break def _upgrade_from_s0(self): LOG.info('Upgrading from schema version 0 to schema version 1') LOG.info('Loading indexes...') indexes = {} ri = self.db.iterator( start=START_INDEX_KEY, stop=END_INDEX_KEY ) with ri: for k, v in ri: _, _, indexid = struct.unpack("BBB", k) if v in indexes: raise InvalidTableException("2 indexes with the same name") indexes[v] = { 'id': indexid, 'last_global_id': 0 } for i in indexes: lgi = self._get(self._last_global_id_key(indexes[i]['id'])) if lgi is not None: indexes[i]['last_global_id'] = struct.unpack(">Q", lgi)[0] else: indexes[i]['last_global_id'] = -1 LOG.info('Scanning indexes...') last_global_id = 0 for i, idata in indexes.iteritems(): from_key = struct.pack("BBB", 2, idata['id'], 0xF0) include_start = False to_key = struct.pack("BBB", 2, idata['id'], 0xF1) include_stop = False ri = self.db.iterator( start=from_key, stop=to_key, include_value=True, include_start=include_start, include_stop=include_stop, reverse=False ) with ri: for ikey, ekey in ri: iversion = struct.unpack(">Q", ekey[:8])[0] if iversion > last_global_id: last_global_id = iversion+1 LOG.info('Last global id: {}'.format(last_global_id)) batch = self.db.write_batch() batch.put(SCHEMAVERSION_KEY, struct.pack("B", 1)) batch.put(TABLE_LAST_GLOBAL_ID, struct.pack(">Q", last_global_id)) batch.write() ================================================ FILE: minemeld/ft/taxii.py ================================================ # Copyright 2015-present Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import copy import urlparse import uuid import os.path from datetime import datetime, timedelta import pytz import lz4.frame import lxml.etree import yaml import redis import gevent import gevent.event import netaddr import werkzeug.urls from six import string_types import libtaxii import libtaxii.clients import libtaxii.messages_11 from libtaxii.constants import MSG_STATUS_MESSAGE, ST_SUCCESS import stix.core.stix_package import stix.core.stix_header import stix.indicator import stix.common.vocabs import stix.common.information_source import stix.common.identity import stix.extensions.marking.ais import stix.data_marking import stix.extensions.marking.tlp import stix_edh import cybox.core import cybox.objects.address_object import cybox.objects.domain_name_object import cybox.objects.uri_object import cybox.objects.file_object import mixbox.idgen import mixbox.namespaces from . import basepoller from . import base from . import actorbase from .utils import dt_to_millisec, interval_in_sec, utc_millisec # stix_edh is imported to register the EDH data marking extensions, but it is not directly used. # Delete the symbol to silence the warning about the import being unnecessary and prevent the # PyCharm 'Optimize Imports' operation from removing the import. del stix_edh LOG = logging.getLogger(__name__) _STIX_MINEMELD_HASHES = [ 'ssdeep', 'md5', 'sha1', 'sha256', 'sha512' ] def set_id_namespace(uri, name): # maec and cybox NS = mixbox.namespaces.Namespace(uri, name) mixbox.idgen.set_id_namespace(NS) class TaxiiClient(basepoller.BasePollerFT): def __init__(self, name, chassis, config): self.poll_service = None self.collection_mgmt_service = None self.last_taxii_run = None self.last_stix_package_ts = None super(TaxiiClient, self).__init__(name, chassis, config) def configure(self): super(TaxiiClient, self).configure() self.initial_interval = self.config.get('initial_interval', '1d') self.initial_interval = interval_in_sec(self.initial_interval) if self.initial_interval is None: LOG.error( '%s - wrong initial_interval format: %s', self.name, self.initial_interval ) self.initial_interval = 86400 self.max_poll_dt = self.config.get( 'max_poll_dt', 86400 ) # options for processing self.ip_version_auto_detect = self.config.get('ip_version_auto_detect', True) self.ignore_composition_operator = self.config.get('ignore_composition_operator', False) self.create_fake_indicator = self.config.get('create_fake_indicator', False) self.hash_priority = self.config.get('hash_priority', _STIX_MINEMELD_HASHES) self.lower_timestamp_precision = self.config.get('lower_timestamp_precision', False) self.discovery_service = self.config.get('discovery_service', None) self.collection = self.config.get('collection', None) # option for enabling client authentication self.client_credentials_required = self.config.get( 'client_credentials_required', True ) self.username = self.config.get('username', None) self.password = self.config.get('password', None) if self.username is not None or self.password is not None: self.client_credentials_required = False # option for enabling client cert, default disabled self.client_cert_required = self.config.get('client_cert_required', False) self.key_file = self.config.get('key_file', None) if self.key_file is None and self.client_cert_required: self.key_file = os.path.join( os.environ['MM_CONFIG_DIR'], '%s.pem' % self.name ) self.cert_file = self.config.get('cert_file', None) if self.cert_file is None and self.client_cert_required: self.cert_file = os.path.join( os.environ['MM_CONFIG_DIR'], '%s.crt' % self.name ) self.subscription_id = None self.subscription_id_required = self.config.get('subscription_id_required', False) self.ca_file = self.config.get('ca_file', None) if self.ca_file is None: self.ca_file = os.path.join( os.environ['MM_CONFIG_DIR'], '%s-ca.crt' % self.name ) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self.prefix = self.config.get('prefix', self.name) self.confidence_map = self.config.get('confidence_map', { 'low': 40, 'medium': 60, 'high': 80 }) self._load_side_config() def _load_side_config(self): if not self.client_credentials_required and not self.subscription_id_required: LOG.info('{} - side config not needed'.format(self.name)) return try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return if self.client_credentials_required: username = sconfig.get('username', None) password = sconfig.get('password', None) if username is not None and password is not None: self.username = username self.password = password LOG.info('{} - Loaded credentials from side config'.format(self.name)) if self.subscription_id_required: subscription_id = sconfig.get('subscription_id', None) if subscription_id is not None: self.subscription_id = subscription_id LOG.info('{} - Loaded subscription id from side config'.format(self.name)) def _saved_state_restore(self, saved_state): super(TaxiiClient, self)._saved_state_restore(saved_state) self.last_taxii_run = saved_state.get('last_taxii_run', None) LOG.info('last_taxii_run from sstate: %s', self.last_taxii_run) def _saved_state_create(self): sstate = super(TaxiiClient, self)._saved_state_create() sstate['last_taxii_run'] = self.last_taxii_run return sstate def _saved_state_reset(self): super(TaxiiClient, self)._saved_state_reset() self.last_taxii_run = None def _build_taxii_client(self): result = libtaxii.clients.HttpClient() up = urlparse.urlparse(self.discovery_service) if up.scheme == 'https': result.set_use_https(True) if self.username and self.password: if self.key_file and self.cert_file: result.set_auth_type( libtaxii.clients.HttpClient.AUTH_CERT_BASIC ) result.set_auth_credentials({ 'username': self.username, 'password': self.password, 'key_file': self.key_file, 'cert_file': self.cert_file }) else: result.set_auth_type( libtaxii.clients.HttpClient.AUTH_BASIC ) result.set_auth_credentials({ 'username': self.username, 'password': self.password }) else: if self.key_file and self.cert_file: result.set_auth_type( libtaxii.clients.HttpClient.AUTH_CERT ) result.set_auth_credentials({ 'key_file': self.key_file, 'cert_file': self.cert_file }) else: result.set_auth_type( libtaxii.clients.HttpClient.AUTH_NONE ) if self.ca_file is not None and os.path.isfile(self.ca_file): result.set_verify_server( verify_server=True, ca_file=self.ca_file ) return result def _call_taxii_service(self, service_url, tc, request): up = urlparse.urlparse(service_url) hostname = up.hostname path = up.path port = up.port resp = tc.call_taxii_service2( hostname, path, libtaxii.constants.VID_TAXII_XML_11, request, port=port ) return resp def _discover_services(self, tc): msg_id = libtaxii.messages_11.generate_message_id() request = libtaxii.messages_11.DiscoveryRequest(msg_id) request = request.to_xml() resp = self._call_taxii_service(self.discovery_service, tc, request) tm = libtaxii.get_message_from_http_response(resp, msg_id) LOG.debug('Discovery_Response {%s} %s', type(tm), tm.to_xml(pretty_print=True)) if tm.message_type == MSG_STATUS_MESSAGE: raise RuntimeError('{} - Error retrieving collections: {} - {}'.format( self.name, tm.status_type, tm.message )) self.collection_mgmt_service = None for si in tm.service_instances: if si.service_type != libtaxii.constants.SVC_COLLECTION_MANAGEMENT: continue self.collection_mgmt_service = si.service_address break if self.collection_mgmt_service is None: raise RuntimeError('%s - collection management service not found' % self.name) def _check_collections(self, tc): msg_id = libtaxii.messages_11.generate_message_id() request = libtaxii.messages_11.CollectionInformationRequest(msg_id) request = request.to_xml() resp = self._call_taxii_service(self.collection_mgmt_service, tc, request) tm = libtaxii.get_message_from_http_response(resp, msg_id) LOG.debug('Collection_Information_Response {%s} %s', type(tm), tm.to_xml(pretty_print=True)) if tm.message_type == MSG_STATUS_MESSAGE: raise RuntimeError('{} - Error retrieving collections: {} - {}'.format( self.name, tm.status_type, tm.message )) tci = None for ci in tm.collection_informations: if ci.collection_name != self.collection: continue tci = ci break if tci is None: raise RuntimeError('%s - collection %s not found' % (self.name, self.collection)) if tci.polling_service_instances is None or \ len(tci.polling_service_instances) == 0: raise RuntimeError('%s - collection %s doesn\'t support polling' % (self.name, self.collection)) if tci.collection_type != libtaxii.constants.CT_DATA_FEED: raise RuntimeError( '%s - collection %s is not a data feed (%s)' % (self.name, self.collection, tci.collection_type) ) for pi in tci.polling_service_instances: LOG.info('{} - message binding: {}'.format( self.name, pi.poll_message_bindings )) if pi.poll_message_bindings[0] == libtaxii.constants.VID_TAXII_XML_11: self.poll_service = pi.poll_address LOG.info('{} - poll service found'.format(self.name)) break else: raise RuntimeError( '%s - collection %s does not support TAXII 1.1 message binding (%s)' % (self.name, self.collection, tci.collection_type) ) LOG.debug('%s - poll service: %s', self.name, self.poll_service) def _poll_fulfillment_request(self, tc, result_id, result_part_number): msg_id = libtaxii.messages_11.generate_message_id() request = libtaxii.messages_11.PollFulfillmentRequest( message_id=msg_id, result_id=result_id, result_part_number=result_part_number, collection_name=self.collection ) request = request.to_xml() resp = self._call_taxii_service(self.poll_service, tc, request) return libtaxii.get_message_from_http_response(resp, msg_id) def _poll_collection(self, tc, begin=None, end=None): msg_id = libtaxii.messages_11.generate_message_id() prargs = dict( message_id=msg_id, collection_name=self.collection, exclusive_begin_timestamp_label=begin, inclusive_end_timestamp_label=end, ) if self.subscription_id_required: prargs['subscription_id'] = self.subscription_id else: pps = libtaxii.messages_11.PollParameters( response_type='FULL', allow_asynch=False ) prargs['poll_parameters'] = pps request = libtaxii.messages_11.PollRequest(**prargs) LOG.debug('%s - first poll request %s', self.name, request.to_xml(pretty_print=True)) request = request.to_xml() resp = self._call_taxii_service(self.poll_service, tc, request) tm = libtaxii.get_message_from_http_response(resp, msg_id) LOG.debug('%s - Poll_Response {%s} %s', self.name, type(tm), tm.to_xml(pretty_print=True)) if tm.message_type == MSG_STATUS_MESSAGE: if tm.status_type == ST_SUCCESS: LOG.info('{} - TAXII Server returned success with no STIX packages'.format( self.name )) return [] raise RuntimeError('{} - Error polling: {} - {}'.format( self.name, tm.status_type, tm.message )) stix_objects = { 'observables': {}, 'indicators': {}, 'ttps': {} } self._handle_content_blocks( tm.content_blocks, stix_objects ) while tm.more: tm = self._poll_fulfillment_request( tc, result_id=tm.result_id, result_part_number=tm.result_part_number+1 ) LOG.debug('{} - Poll_Response {!r}'.format( self.name, tm.to_xml(pretty_print=True) )) if tm.message_type == MSG_STATUS_MESSAGE: if tm.status_type == ST_SUCCESS: break raise RuntimeError('{} - Error polling: {} - {}'.format( self.name, tm.status_type, tm.message )) self._handle_content_blocks( tm.content_blocks, stix_objects ) LOG.debug('%s - stix_objects: %s', self.name, stix_objects) params = { 'ttps': stix_objects['ttps'], 'observables': stix_objects['observables'] } if len(stix_objects['indicators']) == 0 and len(stix_objects['observables']) != 0: LOG.info('{} - TAXII Content contains observables but no indicators'.format(self.name)) if self.create_fake_indicator: stix_objects['indicators']['minemeld:00000000-0000-0000-0000-000000000000'] = { 'observables': stix_objects['observables'].values(), 'ttps': [] } return [[iid, iv, params] for iid, iv in stix_objects['indicators'].iteritems()] def _incremental_poll_collection(self, taxii_client, begin, end): cbegin = begin dt = timedelta(seconds=self.max_poll_dt) self.last_stix_package_ts = None while cbegin < end: cend = min(end, cbegin+dt) LOG.info('{} - polling {!r} to {!r}'.format(self.name, cbegin, cend)) result = self._poll_collection( taxii_client, begin=cbegin, end=cend ) for i in result: yield i if self.last_stix_package_ts is not None: self.last_taxii_run = self.last_stix_package_ts cbegin = cend def _handle_content_blocks(self, content_blocks, objects): try: for cb in content_blocks: if cb.content_binding.binding_id != \ libtaxii.constants.CB_STIX_XML_111: LOG.error('%s - Unsupported content binding: %s', self.name, cb.content_binding.binding_id) continue try: stixpackage = stix.core.stix_package.STIXPackage.from_xml( lxml.etree.fromstring(cb.content) ) except Exception: LOG.exception( '%s - Exception parsing content block', self.name ) continue if stixpackage.indicators: for i in stixpackage.indicators: ci = {} if i.timestamp is not None: ci = { 'timestamp': dt_to_millisec(i.timestamp), } if i.description is not None and i.description.structuring_format is None: # copy description only if there is no markup to avoid side-effects ci['description'] = i.description.value if i.confidence is not None: confidence = str(i.confidence.value).lower() if confidence in self.confidence_map: ci['confidence'] = \ self.confidence_map[confidence] os = [] ttps = [] if i.observables: for o in i.observables: os.append(self._decode_observable(o)) if i.observable and len(os) == 0: os.append(self._decode_observable(i.observable)) if i.indicated_ttps: for t in i.indicated_ttps: ttps.append(self._decode_ttp(t)) ci['observables'] = os ci['ttps'] = ttps objects['indicators'][i.id_] = ci if stixpackage.observables: for o in stixpackage.observables: co = self._decode_observable(o) objects['observables'][o.id_] = co if stixpackage.ttps: for t in stixpackage.ttps: ct = self._decode_ttp(t) objects['ttps'][t.id_] = ct timestamp = stixpackage.timestamp if isinstance(timestamp, datetime): timestamp = dt_to_millisec(timestamp) if self.last_stix_package_ts is None or timestamp > self.last_stix_package_ts: LOG.debug('{} - last STIX package timestamp set to {!r}'.format(self.name, timestamp)) self.last_stix_package_ts = timestamp except: LOG.exception("%s - exception in _handle_content_blocks" % self.name) raise def _decode_observable(self, o): LOG.debug('observable: %s', o.to_dict()) if o.idref: return {'idref': o.idref} odict = o.to_dict() result = {} oc = odict.get('observable_composition', None) if oc: ocoperator = oc.get('operator', None) if ocoperator != 'OR' and not self.ignore_composition_operator: LOG.error( '%s - Observable composition with %s not supported yet: %s', self.name, ocoperator, odict ) return None result['type'] = '_cyboxOR' result['observables'] = [] for nestedo in oc.get('observables', []): if 'idref' not in nestedo: LOG.error( '%s - only Observable references are supported in Observable Composition: %s', self.name, odict ) return None result['observables'].append(nestedo['idref']) return result oo = odict.get('object', None) if oo is None: LOG.error('%s - no object in observable', self.name) return None op = oo.get('properties', None) if op is None: LOG.error('%s - no properties in observable object', self.name) return None return self._decode_object_properties(op, odict=odict) def _decode_object_properties(self, op, odict=None): result = {} ot = op.get('xsi:type', None) if ot is None: LOG.error('%s - no type in observable props', self.name) return None if ot == 'DomainNameObjectType': result['type'] = 'domain' ov = op.get('value', None) if ov is None: LOG.error('%s - no value in observable props', self.name) return None if not isinstance(ov, string_types): ov = ov.get('value', None) if ov is None: LOG.error('%s - no value in observable value', self.name) return None elif ot == 'FileObjectType': ov = '' if 'file_name' in op.keys(): file_name = op.get('file_name') if isinstance(file_name, dict): ov = op['file_name'].get('value', None) result['type'] = 'file.name' else: ov = op['file_name'] result['type'] = 'file.name' hashes = op.get('hashes', []) if not isinstance(hashes, list) or len(hashes) == 0: LOG.error('{} - FileObjectType with unhandled structure: {!r}'.format( self.name, op )) return None indicator_type = None cprio = -1 indicator_hashes = {} for h in hashes: hvalue = h.get('simple_hash_value', None) if hvalue is None: continue if not isinstance(hvalue, string_types): if not isinstance(hvalue, dict): continue hvalue = hvalue.get('value', None) if hvalue is None: continue htype = h.get('type', None) if htype is None: continue elif isinstance(htype, string_types): htype = htype.lower() elif isinstance(htype, dict): htype = htype.get('value', None) if htype is None or not isinstance(htype, string_types): continue htype = htype.lower() if htype not in self.hash_priority: continue prio = self.hash_priority.index(htype) if prio > cprio: indicator_type = htype cprio = prio indicator_hashes[htype] = hvalue if indicator_type is None: LOG.error('{} - No valid hash found in FileObjectType: {!r}'.format( self.name, op )) return None if ov == '': ov = indicator_hashes[indicator_type] result['type'] = indicator_type for h, v in indicator_hashes.iteritems(): if h == indicator_type: continue result['{}_{}'.format(self.prefix, h)] = v elif ot == 'SocketAddressObjectType': ip_address = op.get('ip_address', None) if ip_address is None: return None return self._decode_object_properties(ip_address) elif ot == 'AddressObjectType': ov = op.get('address_value', None) if ov is None: LOG.error('%s - no value in observable props', self.name) return None if not isinstance(ov, string_types): ov = ov.get('value', None) if ov is None: LOG.error('%s - no value in observable value', self.name) return None # set the IP Address type if not self.ip_version_auto_detect: addrcat = op.get('category', None) if addrcat == 'ipv6-addr': result['type'] = 'IPv6' elif addrcat == 'ipv4-addr': result['type'] = 'IPv4' elif addrcat == 'e-mail': result['type'] = 'email-addr' else: LOG.error('{} - unknown address category: {}'.format(self.name, addrcat)) return None else: # some feeds do not set the IP Address type and it # defaults to ipv4-addr even if the IP is IPv6 # this is to auto detect the type if type(ov) == list: address = ov[0] else: address = ov try: parsed = netaddr.IPNetwork(address) except (netaddr.AddrFormatError, ValueError): LOG.error('{} - Unknown IP version: {}'.format(self.name, address)) return None if parsed.version == 4: result['type'] = 'IPv4' elif parsed.version == 6: result['type'] = 'IPv6' if result['type'] in ['IPv4', 'IPv6']: source = op.get('is_source', None) if source is True: result['direction'] = 'inbound' elif source is False: result['direction'] = 'outbound' if 'type' not in result: LOG.error('%s - no IP category and unknown version') return None elif ot == 'URIObjectType': result['type'] = 'URL' ov = op.get('value', None) if ov is None: LOG.error('%s - no value in observable props', self.name) return None if not isinstance(ov, string_types): ov = ov.get('value', None) if ov is None: LOG.error('%s - no value in observable value', self.name) return None elif ot == 'LinkObjectType': if op.get('type', 'URL') != 'URL': LOG.error('{} - Unhandled LinkObjectType type: {!r}'.format(self.name, op)) return None result['type'] = 'URL' ov = op.get('value', None) if ov is None: LOG.error('%s - no value in observable props', self.name) return None if not isinstance(ov, string_types): ov = ov.get('value', None) if ov is None: LOG.error('%s - no value in observable value', self.name) return None elif ot == 'EmailMessageObjectType': result['type'] = 'email-message' ov = '' LOG.debug('EmailMessageObjectType OP: {!r}'.format(op)) body = op.get('raw_body', None) if body is not None: result['body'] = body LOG.debug('EmailMessage Body: {!r}'.format(body)) header = op.get('header', None) if header is not None: result['header'] = header try: ov = header.get('from').get('address_value').get('value') except Exception: LOG.error('{} - no email address listed'.format(self.name)) subject = op.get('subject', None) if subject is not None: result['subject'] = subject if ov == '': ov = subject elif ot == 'ArtifactObjectType': ov = '' result['type'] = 'artifact' LOG.debug('ArtifactObjectType OV: {!r}'.format(ov)) title = odict.get('title', None) if title is not None: ov = title result['title'] = title description = odict.get('description', None) if description is not None: result['description'] = description if ov == '': ov = description artifact = op['raw_artifact'] if artifact is not None: result['artifact'] = artifact elif ot == 'PDFFileObjectType': ov = '' result['type'] = 'pdf-file' if 'file_name' in op.keys(): file_name = op.get('file_name') if type(file_name) == dict: if file_name.get('value', None) is not None: ov = op['file_name'].get('value', None) else: ov = op['file_name'] else: ov = file_name LOG.debug('PDFObjectType OV: {!r}'.format(ov)) if 'file_path' in op.keys(): result['file_path'] = op['file_path'].get('value', None) if 'file_size' in op.keys(): result['file_size'] = op['file_size'].get('value', None) if 'metadata' in op.keys(): result['metadata'] = op['metadata'] if 'file_format' in op.keys(): result['file_format'] = op['file_format'] hashes = op.get('hashes', None) if hashes is not None: for i in hashes: if 'type' in i.keys(): if isinstance(i['type'], string_types): hash_type = i['type'] else: hash_type = i['type'].get('value', None) if 'simple_hash_value' in i.keys(): if isinstance(i['simple_hash_value'], string_types): result[hash_type] = i['simple_hash_value'] else: result[hash_type] = i['simple_hash_value'].get('value', None) elif ot == 'WhoisObjectType': ov = '' result['type'] = 'whois' LOG.debug('WhoisObjectType OV: {!r}'.format(ov)) remarks = op.get('remarks', None) if remarks is not None: result['remarks'] = op['remarks'] ov = remarks.split('\n')[0] elif ot == 'HTTPSessionObjectType': ov = '' result['type'] = 'http-session' if 'http_request_response' in op.keys(): tmp = op['http_request_response'] if len(tmp) == 1: item = tmp[0] LOG.debug('HTTPSessionObjectType item: {!r}'.format(item)) http_client_request = item.get('http_client_request', None) if http_client_request is not None: http_request_header = http_client_request.get('http_request_header', None) if http_request_header is not None: raw_header = http_request_header.get('raw_header', None) if raw_header is not None: result['header'] = raw_header ov = raw_header.split('\n')[0] else: LOG.error('{} - multiple HTTPSessionObjectTypes not supported'.format(self.name)) elif ot == 'PortObjectType': result['type'] = 'port' LOG.debug('PortObjectType OP: {!r}'.format(op)) protocol = op.get('layer4_protocol', None) port = op.get('port_value', None) ov = '{}:{}'.format(protocol, port) elif ot == 'WindowsExecutableFileObjectType': ov = '' result['type'] = 'windows-executable' LOG.debug('WindowsExecutableFileObjectType OP: {!r}'.format(op)) if 'file_name' in op.keys(): if isinstance(op['file_name'], string_types): ov = op['file_name'] else: ov = op['file_name'].get('value', None) if 'size_in_bytes' in op.keys(): result['file_size'] = op['size_in_bytes'] if 'file_format' in op.keys(): result['file_format'] = op['file_format'] hashes = op.get('hashes', None) if hashes is not None: for i in hashes: if 'type' in i.keys(): if isinstance(i['type'], string_types): hash_type = i['type'] else: hash_type = i['type'].get('value', None) if 'simple_hash_value' in i.keys(): if isinstance(i['simple_hash_value'], string_types): result[hash_type] = i['simple_hash_value'] else: result[hash_type] = i['simple_hash_value'].get('value', None) elif ot == 'CISCP:IndicatorTypeVocab-0.0': result['type'] = op['xsi:type'] LOG.debug('CISCP:IndicatorTypeVocab-0.0 OP: {!r}'.format(op)) ov = None LOG.error('{} - CISCP:IndicatorTypeVocab-0.0 Type not currently supported'.format(self.name)) return None elif ot == 'WindowsRegistryKeyObjectType': result['type'] = op['xsi:type'] LOG.debug('WindowsRegistryKeyObjectType OP: {!r}'.format(op)) ov = None LOG.error('{} - WindowsRegistryKeyObjectType Type not currently supported'.format(self.name)) return None elif ot == 'stixVocabs:IndicatorTypeVocab-1.0': result['type'] = op['xsi:type'] LOG.debug('stixVocabs:IndicatorTypeVocab-1.0 OP: {!r}'.format(op)) ov = None LOG.error('{} - stixVocabs:IndicatorTypeVocab-1.0 Type not currently supported'.format(self.name)) return None elif ot == 'NetworkConnectionObjectType': result['type'] = 'NetworkConnection' LOG.debug('NetworkConnectionObjectType OP: {!r}'.format(op)) ov = None LOG.error('{} - NetworkConnectionObjectType Type not currently supported'.format(self.name)) return None else: LOG.error('{} - unknown type {} {!r}'.format(self.name, ot, op)) return None result['indicator'] = ov LOG.debug('{!r}'.format(result)) return result def _decode_ttp(self, t): tdict = t.to_dict() if 'ttp' in tdict: tdict = tdict['ttp'] if 'idref' in tdict: return {'idref': tdict['idref']} if 'description' in tdict: return {'description': tdict['description']} if 'title' in tdict: return {'description': tdict['title']} return {'description': ''} def _process_item(self, item): result = [] value = {} iid, iv, stix_objects = item value['%s_indicator' % self.prefix] = iid if 'description' in iv: value['{}_indicator_description'.format(self.prefix)] = iv['description'] if 'confidence' in iv: value['confidence'] = iv['confidence'] if len(iv['ttps']) != 0: ttp = iv['ttps'][0] if 'idref' in ttp: ttp = stix_objects['ttps'].get(ttp['idref']) if ttp is not None and 'description' in ttp: value['%s_ttp' % self.prefix] = ttp['description'] composed_observables = [] for o in iv['observables']: if o is None: continue v = copy.copy(value) ob = o if 'idref' in o: ob = stix_objects['observables'].get(o['idref'], None) v['%s_observable' % self.prefix] = o['idref'] if ob is None: continue if ob['type'] == '_cyboxOR': for o in ob['observables']: composed_observables.append(o) continue v['type'] = ob['type'] if type(ob['indicator']) == list: indicator = ob['indicator'] else: indicator = [ob['indicator']] for i in indicator: result.append([i, v]) for o in composed_observables: v = copy.copy(value) ob = stix_objects['observables'].get(o, None) v['%s_observable' % self.prefix] = o if ob is None: continue if ob['type'] == '_cyboxOR': LOG.error( '%s - Nested Observable Composition not supported', self.name ) continue v['type'] = ob['type'] if type(ob['indicator']) == list: indicator = ob['indicator'] else: indicator = [ob['indicator']] for i in indicator: result.append([i, v]) return result def _build_iterator(self, now): if self.client_credentials_required: if self.username is None or self.password is None: raise RuntimeError( '%s - username or password required and not set, poll not performed' % self.name ) if self.cert_file is not None and not os.path.isfile(self.cert_file): raise RuntimeError( '%s - client cert required and not set, poll not performed' % self.name ) if self.key_file is not None and not os.path.isfile(self.key_file): raise RuntimeError( '%s - client cert key required and not set, poll not performed' % self.name ) if self.subscription_id_required and self.subscription_id is None: raise RuntimeError( '%s - subscription id required and not set, poll not performed' % self.name ) tc = self._build_taxii_client() self._discover_services(tc) self._check_collections(tc) last_run = self.last_taxii_run max_back = now-(self.initial_interval*1000) if last_run is None or last_run < max_back: last_run = max_back begin = datetime.utcfromtimestamp(last_run/1000) begin = begin.replace(tzinfo=pytz.UTC) end = datetime.utcfromtimestamp(now/1000) end = end.replace(tzinfo=pytz.UTC) if self.lower_timestamp_precision: end = end.replace(second=0, microsecond=0) begin = begin.replace(second=0, microsecond=0) return self._incremental_poll_collection( taxii_client=tc, begin=begin, end=end ) def _flush(self): self.last_taxii_run = None super(TaxiiClient, self)._flush() def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(TaxiiClient, self).hup(source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass client_cert_required = False if config is not None: client_cert_required = config.get('client_cert_required', False) cert_path = None if config is not None: cert_path = config.get('cert_file', None) if cert_path is None and client_cert_required: cert_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}.crt'.format(name) ) if cert_path is not None: try: os.remove(cert_path) except: pass key_path = None if config is not None: key_path = config.get('key_file', None) if key_path is None and client_cert_required: key_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}.pem'.format(name) ) if key_path is not None: try: os.remove(key_path) except: pass def _stix_ip_observable(namespace, indicator, value): category = cybox.objects.address_object.Address.CAT_IPV4 if value['type'] == 'IPv6': category = cybox.objects.address_object.Address.CAT_IPV6 indicators = [indicator] if '-' in indicator: # looks like an IP Range, let's try to make it a CIDR a1, a2 = indicator.split('-', 1) if a1 == a2: # same IP indicators = [a1] else: # use netaddr builtin algo to summarize range into CIDR iprange = netaddr.IPRange(a1, a2) cidrs = iprange.cidrs() indicators = map(str, cidrs) observables = [] for i in indicators: id_ = '{}:observable-{}'.format( namespace, uuid.uuid4() ) ao = cybox.objects.address_object.Address( address_value=i, category=category ) o = cybox.core.Observable( title='{}: {}'.format(value['type'], i), id_=id_, item=ao ) observables.append(o) return observables def _stix_email_addr_observable(namespace, indicator, value): category = cybox.objects.address_object.Address.CAT_EMAIL id_ = '{}:observable-{}'.format( namespace, uuid.uuid4() ) ao = cybox.objects.address_object.Address( address_value=indicator, category=category ) o = cybox.core.Observable( title='{}: {}'.format(value['type'], indicator), id_=id_, item=ao ) return [o] def _stix_domain_observable(namespace, indicator, value): id_ = '{}:observable-{}'.format( namespace, uuid.uuid4() ) do = cybox.objects.domain_name_object.DomainName() do.value = indicator do.type_ = 'FQDN' o = cybox.core.Observable( title='FQDN: ' + indicator, id_=id_, item=do ) return [o] def _stix_url_observable(namespace, indicator, value): id_ = '{}:observable-{}'.format( namespace, uuid.uuid4() ) uo = cybox.objects.uri_object.URI( value=indicator, type_=cybox.objects.uri_object.URI.TYPE_URL ) o = cybox.core.Observable( title='URL: ' + indicator, id_=id_, item=uo ) return [o] def _stix_hash_observable(namespace, indicator, value): id_ = '{}:observable-{}'.format( namespace, uuid.uuid4() ) uo = cybox.objects.file_object.File() uo.add_hash(indicator) o = cybox.core.Observable( title='{}: {}'.format(value['type'], indicator), id_=id_, item=uo ) return [o] _TYPE_MAPPING = { 'IPv4': { 'indicator_type': stix.common.vocabs.IndicatorType.TERM_IP_WATCHLIST, 'mapper': _stix_ip_observable }, 'IPv6': { 'indicator_type': stix.common.vocabs.IndicatorType.TERM_IP_WATCHLIST, 'mapper': _stix_ip_observable }, 'URL': { 'indicator_type': stix.common.vocabs.IndicatorType.TERM_URL_WATCHLIST, 'mapper': _stix_url_observable }, 'domain': { 'indicator_type': stix.common.vocabs.IndicatorType.TERM_DOMAIN_WATCHLIST, 'mapper': _stix_domain_observable }, 'sha256': { 'indicator_type': stix.common.vocabs.IndicatorType.TERM_FILE_HASH_WATCHLIST, 'mapper': _stix_hash_observable }, 'sha1': { 'indicator_type': stix.common.vocabs.IndicatorType.TERM_FILE_HASH_WATCHLIST, 'mapper': _stix_hash_observable }, 'md5': { 'indicator_type': stix.common.vocabs.IndicatorType.TERM_FILE_HASH_WATCHLIST, 'mapper': _stix_hash_observable }, 'email-addr': { 'indicator_type': stix.common.vocabs.IndicatorType.TERM_MALICIOUS_EMAIL, 'mapper': _stix_email_addr_observable } } class DataFeed(actorbase.ActorBaseFT): def __init__(self, name, chassis, config): self.redis_skey = name self.redis_skey_value = name+'.value' self.redis_skey_chkp = name+'.chkp' self.SR = None self.ageout_glet = None super(DataFeed, self).__init__(name, chassis, config) def configure(self): super(DataFeed, self).configure() self.redis_url = self.config.get('redis_url', os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock') ) self.namespace = self.config.get('namespace', 'minemeld') self.namespaceuri = self.config.get( 'namespaceuri', 'https://go.paloaltonetworks.com/minemeld' ) self.age_out_interval = self.config.get('age_out_interval', '24h') self.age_out_interval = interval_in_sec(self.age_out_interval) if self.age_out_interval < 60: LOG.info('%s - age out interval too small, forced to 60 seconds') self.age_out_interval = 60 self.max_entries = self.config.get('max_entries', 1000 * 1000) self.attributes_package_title = self.config.get('attributes_package_title', []) if not isinstance(self.attributes_package_title, list): LOG.error('{} - attributes_package_title should be a list - ignored') self.attributes_package_title = [] self.attributes_package_description = self.config.get('attributes_package_description', []) if not isinstance(self.attributes_package_description, list): LOG.error('{} - attributes_package_description should be a list - ignored') self.attributes_package_description = [] self.attributes_package_sdescription = self.config.get('attributes_package_short_description', []) if not isinstance(self.attributes_package_sdescription, list): LOG.error('{} - attributes_package_sdescription should be a list - ignored') self.attributes_package_sdescription = [] self.attributes_package_information_source = self.config.get('attributes_package_information_source', []) if not isinstance(self.attributes_package_information_source, list): LOG.error('{} - attributes_package_information_source should be a list - ignored') self.attributes_package_information_source = [] def connect(self, inputs, output): output = False super(DataFeed, self).connect(inputs, output) def read_checkpoint(self): self._connect_redis() self.last_checkpoint = self.SR.get(self.redis_skey_chkp) def create_checkpoint(self, value): self._connect_redis() self.SR.set(self.redis_skey_chkp, value) def remove_checkpoint(self): self._connect_redis() self.SR.delete(self.redis_skey_chkp) def _connect_redis(self): if self.SR is not None: return self.SR = redis.StrictRedis.from_url( self.redis_url ) def _read_oldest_indicator(self): olist = self.SR.zrange( self.redis_skey, 0, 0, withscores=True ) LOG.debug('%s - oldest: %s', self.name, olist) if len(olist) == 0: return None, None return int(olist[0][1]), olist[0][0] def initialize(self): self._connect_redis() def rebuild(self): self._connect_redis() self.SR.delete(self.redis_skey) self.SR.delete(self.redis_skey_value) def reset(self): self._connect_redis() self.SR.delete(self.redis_skey) self.SR.delete(self.redis_skey_value) def _add_indicator(self, score, indicator, value): if self.length() >= self.max_entries: LOG.info('dropped overflow') self.statistics['drop.overflow'] += 1 return type_ = value['type'] type_mapper = _TYPE_MAPPING.get(type_, None) if type_mapper is None: self.statistics['drop.unknown_type'] += 1 LOG.error('%s - Unsupported indicator type: %s', self.name, type_) return set_id_namespace(self.namespaceuri, self.namespace) title = None if len(self.attributes_package_title) != 0: for pt in self.attributes_package_title: if pt not in value: continue title = '{}'.format(value[pt]) break description = None if len(self.attributes_package_description) != 0: for pd in self.attributes_package_description: if pd not in value: continue description = '{}'.format(value[pd]) break sdescription = None if len(self.attributes_package_sdescription) != 0: for pd in self.attributes_package_sdescription: if pd not in value: continue sdescription = '{}'.format(value[pd]) break information_source = None if len(self.attributes_package_information_source) != 0: for isource in self.attributes_package_information_source: if isource not in value: continue information_source = '{}'.format(value[isource]) break if information_source is not None: identity = stix.common.identity.Identity(name=information_source) information_source = stix.common.information_source.InformationSource(identity=identity) handling = None share_level = value.get('share_level', None) if share_level in ['white', 'green', 'amber', 'red']: marking_specification = stix.data_marking.MarkingSpecification() marking_specification.controlled_structure = "//node() | //@*" tlp = stix.extensions.marking.tlp.TLPMarkingStructure() tlp.color = share_level.upper() marking_specification.marking_structures.append(tlp) handling = stix.data_marking.Marking() handling.add_marking(marking_specification) header = None if (title is not None or description is not None or handling is not None or sdescription is not None or information_source is not None): header = stix.core.STIXHeader( title=title, description=description, handling=handling, short_description=sdescription, information_source=information_source ) spid = '{}:indicator-{}'.format( self.namespace, uuid.uuid4() ) sp = stix.core.STIXPackage(id_=spid, stix_header=header) observables = type_mapper['mapper'](self.namespace, indicator, value) for o in observables: id_ = '{}:indicator-{}'.format( self.namespace, uuid.uuid4() ) if value['type'] == 'URL': eindicator = werkzeug.urls.iri_to_uri(indicator, safe_conversion=True) else: eindicator = indicator sindicator = stix.indicator.indicator.Indicator( id_=id_, title='{}: {}'.format( value['type'], eindicator ), description='{} indicator from {}'.format( value['type'], ', '.join(value['sources']) ), timestamp=datetime.utcnow().replace(tzinfo=pytz.utc) ) confidence = value.get('confidence', None) if confidence is None: LOG.error('%s - indicator without confidence', self.name) sindicator.confidence = "Unknown" # We shouldn't be here elif confidence < 50: sindicator.confidence = "Low" elif confidence < 75: sindicator.confidence = "Medium" else: sindicator.confidence = "High" sindicator.add_indicator_type(type_mapper['indicator_type']) sindicator.add_observable(o) sp.add_indicator(sindicator) spackage = 'lz4'+lz4.frame.compress( sp.to_json(), compression_level=lz4.frame.COMPRESSIONLEVEL_MINHC ) with self.SR.pipeline() as p: p.multi() p.zadd(self.redis_skey, score, spid) p.hset(self.redis_skey_value, spid, spackage) result = p.execute()[0] self.statistics['added'] += result def _delete_indicator(self, indicator_id): with self.SR.pipeline() as p: p.multi() p.zrem(self.redis_skey, indicator_id) p.hdel(self.redis_skey_value, indicator_id) result = p.execute()[0] self.statistics['removed'] += result def _age_out_run(self): while True: now = utc_millisec() low_watermark = now - self.age_out_interval*1000 otimestamp, oindicator = self._read_oldest_indicator() LOG.debug( '{} - low watermark: {} otimestamp: {}'.format( self.name, low_watermark, otimestamp ) ) while otimestamp is not None and otimestamp < low_watermark: self._delete_indicator(oindicator) otimestamp, oindicator = self._read_oldest_indicator() wait_time = 30 if otimestamp is not None: next_expiration = ( (otimestamp + self.age_out_interval*1000) - now ) wait_time = max(wait_time, next_expiration/1000 + 1) LOG.debug('%s - sleeping for %d secs', self.name, wait_time) gevent.sleep(wait_time) @base._counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): now = utc_millisec() self._add_indicator(now, indicator, value) @base._counting('withdraw.ignored') def filtered_withdraw(self, source=None, indicator=None, value=None): # this is a TAXII data feed, old indicators never expire pass def length(self, source=None): return self.SR.zcard(self.redis_skey) def start(self): super(DataFeed, self).start() self.ageout_glet = gevent.spawn(self._age_out_run) def stop(self): super(DataFeed, self).stop() self.ageout_glet.kill() LOG.info( "%s - # indicators: %d", self.name, self.SR.zcard(self.redis_skey) ) @staticmethod def gc(name, config=None): actorbase.ActorBaseFT.gc(name, config=config) if config is None: config = {} redis_skey = name redis_skey_value = '{}.value'.format(name) redis_skey_chkp = '{}.chkp'.format(name) redis_url = config.get('redis_url', os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock') ) cp = None try: cp = redis.ConnectionPool.from_url( redis_url ) SR = redis.StrictRedis(connection_pool=cp) SR.delete(redis_skey) SR.delete(redis_skey_value) SR.delete(redis_skey_chkp) except Exception as e: raise RuntimeError(str(e)) finally: if cp is not None: cp.disconnect() ================================================ FILE: minemeld/ft/taxii2.py ================================================ # Copyright 2015-present Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import os import yaml from datetime import datetime, timedelta from uuid import UUID import pytz import requests import requests.structures import ujson from stix2patterns.pattern import Pattern, ParseException from . import basepoller from .utils import dt_to_millisec, interval_in_sec LOG = logging.getLogger(__name__) _STIX2_TYPES_TO_MM_TYPES = { 'ipv4-addr': 'IPv4', 'ipv6-addr': 'IPv6', 'domain': 'domain', 'domain-name': 'domain', 'url': 'URL', 'file': None, 'md5': 'md5', 'sha-1': 'sha1', 'sha-256': 'sha256' } class Taxii2Client(basepoller.BasePollerFT): def __init__(self, name, chassis, config): self.poll_service = None self.last_taxii2_run = None self.last_stix2_package_ts = None super(Taxii2Client, self).__init__(name, chassis, config) def configure(self): super(Taxii2Client, self).configure() self.initial_interval = self.config.get('initial_interval', '1d') self.initial_interval = interval_in_sec(self.initial_interval) if self.initial_interval is None: LOG.error('%s - wrong initial_interval format: %s', self.name, self.initial_interval) self.initial_interval = 86400 self.max_poll_dt = self.config.get('max_poll_dt', 86400) # options for processing self.lower_timestamp_precision = self.config.get('lower_timestamp_precision', False) self.auth_type = self.config.get('auth_type', 'none') self.username = self.config.get('username', None) self.password = self.config.get('password', None) self.api_key = self.config.get('api_key', None) self.discovery_service = self.config.get('discovery_service', None) self.api_root = self.config.get('api_root', None) self.collection = self.config.get('collection', None) self.verify_cert = self.config.get('verify_cert', True) self.enabled = self.config.get('enabled', 'no') self.client = None self.taxii_collection = None self.taxii_version = None self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) # self.prefix = self.config.get('prefix', self.name) # self.confidence_map = self.config.get('confidence_map', {'low': 40, 'medium': 60, 'high': 80}) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return auth_type = sconfig.get('auth_type', None) username = sconfig.get('username', None) password = sconfig.get('password', None) api_key = sconfig.get('api_key', None) if api_key is not None: self.api_key = api_key LOG.info('{} - Loaded credentials from side config'.format(self.name)) elif username is not None and password is not None: self.username = username self.password = password LOG.info('{} - Loaded credentials from side config'.format(self.name)) discovery_service = sconfig.get('discovery_service', None) api_root = sconfig.get('api_root', None) collection = sconfig.get('collection', None) verify_cert = sconfig.get('verify_cert', None) enabled = sconfig.get('enabled', None) if discovery_service is not None: self.discovery_service = discovery_service LOG.info('{} - Loaded discovery service from side config'.format(self.name)) if api_root is not None: self.api_root = api_root LOG.info('{} - Loaded api root from side config'.format(self.name)) if collection is not None: self.collection = collection LOG.info('{} - Loaded collection from side config'.format(self.name)) if verify_cert is not None: self.verify_cert = verify_cert LOG.info('{} - Loaded collection from side config'.format(self.name)) if auth_type is not None: self.auth_type = auth_type LOG.info('{} - Loaded collection from side config'.format(self.name)) if enabled is not None: self.enabled = enabled LOG.info('{} - Loaded collection from side config'.format(self.name)) def _saved_state_restore(self, saved_state): super(Taxii2Client, self)._saved_state_restore(saved_state) self.last_taxii2_run = saved_state.get('last_taxii2_run', None) LOG.info('last_taxii2_run from sstate: %s', self.last_taxii2_run) def _saved_state_create(self): sstate = super(Taxii2Client, self)._saved_state_create() sstate['last_taxii2_run'] = self.last_taxii2_run return sstate def _saved_state_reset(self): super(Taxii2Client, self)._saved_state_reset() self.last_taxii2_run = None def _set_accept_header(self, session): content_types = { 'stix20': 'application/vnd.oasis.stix+json; version=2.0', 'taxii20': 'application/vnd.oasis.taxii+json; version=2.0', 'stix21': 'application/taxii+json; version=2.1', 'taxii21': 'application/taxii+json; version=2.1' } try: # Assume the server is TAXII 2.0 self.taxii_version = '2.0' session.headers.update({'Accept': content_types['taxii20']}) r1 = session.get(self.discovery_service) if r1.status_code == 406: # If the server is not TAXII 2.0, assume it is TAXII 2.1 self.taxii_version = '2.1' session.headers.update({'Accept': content_types['taxii21']}) r2 = session.get(self.discovery_service) if r2.status_code == 406: # The server supports neither raise RuntimeError('server does not support TAXII 2.0 nor TAXII 2.1') except Exception as e: raise RuntimeError('error contacting server. {}'.format(e)) def _build_taxii2_client(self): session = requests.Session() session.verify = True if self.verify_cert == 'yes' else False if self.api_key: session.headers.update({'Authorization': 'Token {}'.format(self.api_key)}) elif self.username and self.password: session.auth = (self.username, self.password) # session.auth = requests.auth.HTTPBasicAuth(self.username, self.password) else: pass # Check the TAXII server to ensure the correct Accept header is set self._set_accept_header(session) self.client = session def _get_api_root(self): if self.client: r = self.client.get(self.discovery_service) if r.status_code == requests.codes.ok: try: discovery = r.json() if 'api_roots' in discovery: api_roots = discovery['api_roots'] for url in api_roots: # strip the trailing slash if url[:-1].endswith(self.api_root): self.api_root = url break else: raise RuntimeError('error getting api_root.'.format(r.status_code)) except Exception as e: raise RuntimeError('error getting api_root. {}'.format(e)) else: raise RuntimeError('error getting api_root. received code {}'.format(r.status_code)) else: raise RuntimeError('client does not exist {}'.format(self.collection)) def _is_uuid(self, val, ver): n = len(val) if n == 32 or n == 36: try: uuid_val = UUID(val, version=ver) except Exception: return False return str(uuid_val) == val else: return False def _get_collection(self): try: if self.client: collection_url = '{}collections/'.format(self.api_root) r = self.client.get(collection_url) if r.status_code == requests.codes.ok: collections = r.json()['collections'] if self._is_uuid(self.collection, 3) or self._is_uuid(self.collection, 4): for c in collections: if c['id'] == self.collection: self.taxii_collection = c break else: self.taxii_collection = collections[0] else: msg = 'error getting collection {}. received code {}'.format(self.collection, r.status_code) raise RuntimeError(msg) else: raise RuntimeError('client does not exist {}'.format(self.collection)) except RuntimeError as e: LOG.exception(e) except Exception as e: LOG.exception('collection {} was not found - {}'.format(self.collection, e)) # noinspection PyMethodMayBeStatic def _clean_indicator(self, sub_pattern_value): indicator = str(sub_pattern_value) if indicator[0] == "'" and indicator[-1] == "'": return indicator[1:-1] else: return indicator # noinspection PyMethodMayBeStatic def _detect_and_map_type(self, i_type, sub_pattern_type): if i_type == 'file': sub_pattern_type = sub_pattern_type[-1].lower() return _STIX2_TYPES_TO_MM_TYPES.get(sub_pattern_type, None) else: return _STIX2_TYPES_TO_MM_TYPES.get(i_type, None) def _convert_stix2_obj_to_mm_obj(self, obj, rels, ttps): # Inspect the STIX2 Pattern # result # comparisons # type_dict # foo # 0 # 0 (type) # 1 (op) # 2 (value) # bar # 0 # 0 (type) # 1 (op) # 2 (value) # ... # noinspection PyBroadException try: pattern = obj['pattern'] inspected_pattern = Pattern(pattern).inspect() comparisons = inspected_pattern.comparisons indicators = [] # noinspection PyCompatibility for i_type, i_patterns in comparisons.iteritems(): # The Pattern Inspector buckets each comparison expression in the observable expression based on type if i_type in _STIX2_TYPES_TO_MM_TYPES: # The Pattern Inspector reduces the observable expression into a flat list of comparison expressions for sub_pattern in i_patterns: (sub_pattern_type, sub_pattern_op, sub_pattern_value) = sub_pattern mm_type = self._detect_and_map_type(i_type, sub_pattern_type) if mm_type: indicator = self._clean_indicator(sub_pattern_value) value = { "type": mm_type } if 'confidence' in obj: value['confidence'] = obj['confidence'] descriptions = [r["description"].strip() for r in rels if "description" in r] if len(descriptions): value["description"] = ", ".join(descriptions) techniques = [t["name"].strip() for t in ttps] if len(techniques): value["techniques"] = ", ".join(techniques) i = [indicator, value] indicators.append(i) return indicators except ParseException as e: LOG.warning('error parsing indicator pattern {}'.format(e)) except Exception as e: LOG.error('exception parsing indicator pattern {}'.format(e)) def _explore(self, root, types): objs = [root] while objs: obj = objs.pop() if isinstance(obj, dict): if 'type' in obj and obj['type'] in types: yield obj else: objs.extend(obj.values()) elif isinstance(obj, list): objs.extend(obj) def _poll_taxii21_server(self): """ TAXII 2.1 uses a limit url query parameter and a 'more' true/false key in the returned data https://docs.oasis-open.org/cti/taxii/v2.1/csprd01/taxii-v2.1-csprd01.html#_Toc532988055 :return: list of objects """ data = [] params = {'limit': '100'} if self.last_stix2_package_ts: params['added_after'] = self.last_stix2_package_ts fetch_more = True while fetch_more: # Poll the server # Check the 'more' field in the response json to see if there is more data # Poll until there is no data url = '{}collections/{}/objects/'.format(self.api_root, self.taxii_collection['id']) r = self.client.get(url, params=params) if r.status_code in [200, 201, 206]: try: r_json = ujson.loads(r.text) # Filter objects by type in the data returned by the TAXII 2.x server types = ['indicator', 'attack-pattern', 'relationship'] objs = self._explore(r_json, types) data.extend(objs) # Sort the objs in data by timestamp to find the most recent timestamp data.sort(key=lambda x: datetime.strptime(x['modified'], '%Y-%m-%dT%H:%M:%S.%fZ')) if len(data): ts = data[-1]['modified'] params['added_after'] = ts self.last_stix2_package_ts = ts if 'more' in r_json and r_json['more'] is True: pass else: break except Exception as e: LOG.exception(e) break else: break return data def _poll_taxii20_server(self): """ TAXII 2.0 uses Range and Content-Range headers for pagination http://docs.oasis-open.org/cti/taxii/v2.0/cs01/taxii-v2.0-cs01.html#_Toc496542715 :return: list of objects """ data = [] size = 100 params = {} if self.last_stix2_package_ts: params['added_after'] = self.last_stix2_package_ts fetch_more = True while fetch_more: # Poll the server # Check the response headers to see if there is paginated data # Poll until there is no data url = '{}collections/{}/objects/'.format(self.api_root, self.taxii_collection['id']) r = self.client.get(url, params=params) if r.status_code in [200, 201, 206]: try: r_json = ujson.loads(r.text) # Filter objects by type in the data returned by the TAXII 2.x server types = ['indicator', 'attack-pattern', 'relationship'] objs = self._explore(r_json, types) data.extend(objs) # Sort the objs in data by timestamp to find the most recent timestamp data.sort(key=lambda x: datetime.strptime(x['modified'], '%Y-%m-%dT%H:%M:%S.%fZ')) if len(data): self.last_stix2_package_ts = data[-1]['modified'] content_range = r.headers.get('Content-Range', None) if content_range and content_range.startswith('items '): content_range_start_end_size = content_range[6:] content_range_start_end, content_range_size = content_range_start_end_size.split('/') content_range_start, content_range_end = content_range_start_end.split('-') next_start = int(content_range_end) + 1 # next_end = next_start + size next_end = next_start + (int(content_range_end) - int(content_range_start)) + 1 if next_start < int(content_range_size): updated_content_range = 'items {}-{}'.format(next_start, next_end) self.client.headers.update({'Range': updated_content_range}) else: break else: break except Exception as e: LOG.exception(e) break else: break return data def _poll_and_filter_collection(self, begin=None, end=None): if self.client: if self.taxii_collection: if self.taxii_version == '2.0': data = self._poll_taxii20_server() elif self.taxii_version == '2.1': data = self._poll_taxii21_server() else: # Unsupported data = [] raw_objs = data # Sort objects by type in the data returned by the TAXII 2.x server objs = {} types = ['indicator', 'attack-pattern', 'relationship'] for k in types: objs[k] = [] for obj in raw_objs: if obj['type'] == 'indicator' and 'pattern' in obj: objs[obj['type']].append(obj) else: objs[obj['type']].append(obj) ids_to_ttps = {} for t in objs['attack-pattern']: ids_to_ttps[t['id']] = t indicators = [] for i in objs['indicator']: i_rels = [x for x in objs['relationship'] if i['id'] == x['source_ref']] i_ttp_rels = [x for x in i_rels if x['target_ref'].startswith('attack-pattern')] i_ttps = [ids_to_ttps[x['target_ref']] for x in i_ttp_rels] mm_is = self._convert_stix2_obj_to_mm_obj(i, i_rels, i_ttps) if mm_is: # The indicator pattern is valid and was parsed indicators.extend(mm_is) return indicators else: raise RuntimeError('no collection {}'.format(self.collection)) else: raise RuntimeError('client does not exist {}'.format(self.collection)) def _incremental_poll_collection(self, begin, end): cbegin = begin dt = timedelta(seconds=self.max_poll_dt) # self.last_stix2_package_ts = None while cbegin < end: cend = min(end, cbegin + dt) LOG.info('{} - polling {!r} to {!r}'.format(self.name, cbegin, cend)) result = self._poll_and_filter_collection(begin=cbegin, end=cend) for i in result: yield i if self.last_stix2_package_ts is not None: self.last_taxii2_run = self.last_stix2_package_ts cbegin = cend def _process_item(self, item): return [item] def _manage_time(self, now): last_run = self.last_taxii2_run if last_run: last_run = dt_to_millisec(datetime.strptime(self.last_taxii2_run, '%Y-%m-%dT%H:%M:%S.%fZ')) max_back = now - (self.initial_interval * 1000) if last_run is None or last_run < max_back: last_run = max_back begin = datetime.utcfromtimestamp(last_run / 1000) begin = begin.replace(tzinfo=pytz.UTC) end = datetime.utcfromtimestamp(now / 1000) end = end.replace(tzinfo=pytz.UTC) if self.lower_timestamp_precision: end = end.replace(second=0, microsecond=0) begin = begin.replace(second=0, microsecond=0) return begin, end def _check_args(self): if (self.username or self.password) and self.api_key: raise RuntimeError( '%s - username, password, and api_key cannot all be set, poll not performed' % self.name ) if not self.discovery_service: raise RuntimeError( '%s - discovery_service required and not set, poll not performed' % self.name ) if not self.api_root: raise RuntimeError( '%s - api_root required and not set, poll not performed' % self.name ) if not self.collection: raise RuntimeError( '%s - collection required and not set, poll not performed' % self.name ) if not self.enabled: raise RuntimeError( '%s - node is disabled, poll not performed' % self.name ) def _build_iterator(self, now): self._check_args() self._build_taxii2_client() self._get_api_root() self._get_collection() self._check_args() (begin, end) = self._manage_time(now) return self._incremental_poll_collection(begin=begin, end=end) def _flush(self): self.last_taxii2_run = None super(Taxii2Client, self)._flush() def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(Taxii2Client, self).hup(source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except Exception: pass ================================================ FILE: minemeld/ft/test.py ================================================ from __future__ import absolute_import import logging import gevent from . import base from .utils import utc_millisec import netaddr LOG = logging.getLogger(__name__) class TestMiner(base.BaseFT): def __init__(self, name, chassis, config): super(TestMiner, self).__init__(name, chassis, config) self._glet = None def configure(self): super(TestMiner, self).configure() self.num_messages = self.config.get('num_messages', 100000) self.mps = self.config.get('mps', 1000) def initialize(self): pass def rebuild(self): pass def reset(self): pass def _run(self): cip = 0x0A000000 v = { 'type': 'IPv4', 'confidence': 0, 'share_level': 'red' } LOG.info('%s - start sending messages: %d', self.name, utc_millisec()) t1 = utc_millisec() for i in xrange(self.num_messages): ip = str(netaddr.IPAddress(i+cip)) self.emit_update(ip, v) if ((i+1) % self.mps) == 0: now = utc_millisec() LOG.info('%d: %d', i+1, now - t1) if now - t1 < 1000: gevent.sleep((1000 - now + t1)/1000.0) t1 = now LOG.info('%s - all messages sent: %d', self.name, utc_millisec()) def length(self, source=None): return 0 def start(self): super(TestMiner, self).start() self._glet = gevent.spawn_later( 2, self._run ) def stop(self): super(TestMiner, self).stop() if self._glet is None: return self._glet.kill() class TestFeed(base.BaseFT): def __init__(self, name, chassis, config): super(TestFeed, self).__init__(name, chassis, config) def configure(self): super(TestFeed, self).configure() self.num_messages = self.config.get('num_messages', 100000) def read_checkpoint(self): self.last_checkpoint = None def create_checkpoint(self, value): pass def initialize(self): pass def rebuild(self): pass def reset(self): pass @base._counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): if self.statistics['update.processed'] == 1: LOG.info('%s - first message: %d', self.name, utc_millisec()) elif self.statistics['update.processed'] == self.num_messages: LOG.info('%s - last message: %d', self.name, utc_millisec()) @base._counting('withdraw.processed') def filtered_withdraw(self, source=None, indicator=None, value=None): pass def length(self, source=None): pass class FaultyConfig(base.BaseFT): def configure(self): super(FaultyConfig, self).configure() raise RuntimeError('fault !') def initialize(self): pass def rebuild(self): pass def reset(self): pass def length(self, source=None): return 0 class FaultyInit(base.BaseFT): def __init__(self, name, chassis, config): raise RuntimeError('fault !') def configure(self): pass def initialize(self): pass def rebuild(self): pass def reset(self): pass def length(self, source=None): return 0 ================================================ FILE: minemeld/ft/threatconnect.py ================================================ import logging import hmac import hashlib import base64 import time import requests import pytz import os import yaml import re from netaddr import IPNetwork, AddrFormatError from urllib import quote from basepoller import BasePollerFT from utils import utc_millisec, dt_to_millisec from datetime import datetime LOG = logging.getLogger(__name__) GENERIC_INDICATOR_MAP = [ {"apiBranch": "emailAddresses", "apiEntity": "emailAddress", "indicator": {"address": "email-addr"}}, {"apiBranch": "hosts", "apiEntity": "host", "indicator": {"hostName": "domain"}}, {"apiBranch": "urls", "apiEntity": "url", "indicator": {"text": "URL"}}, {"apiBranch": "files", "apiEntity": "file", "indicator": {"md5": "md5", "sha1": "sha1", "sha256": "sha256"}}, {"apiBranch": "registryKeys", "apiEntity": "registryKey", "indicator": None}, {"apiBranch": "userAgents", "apiEntity": "userAgent", "indicator": None} ] IP_INDICATOR_MAP = [ {"apiBranch": "addresses", "apiEntity": "address", "indicator": ["ip"]}, {"apiBranch": "ipPorts", "apiEntity": "ipPort", "indicator": None} ] GROUP_TYPES = ["adversaries", "campaigns", "documents", "emails", "incidents", "signatures", "threats"] SHA256_PATTERN = "[A-Fa-f0-9]{64}" SHA1_PATTERN = "[A-Fa-f0-9]{40}" MD5_PATTERN = "[A-Fa-f0-9]{32}" class ThreatConnect(object): api_secret = None api_key = None api_url = None api_base_uri = None signature = None api_timestamp = None owner = None hash_patterns = {"sha256": re.compile(SHA256_PATTERN), "sha1": re.compile(SHA1_PATTERN), "md5": re.compile(MD5_PATTERN)} def __init__(self, api_secret, api_key, api_url, api_base_uri, owner): self.api_secret = api_secret self.api_key = api_key self.api_url = api_url self.api_base_uri = api_base_uri self.owner = None if owner is None else quote(owner) def prepare_get(self, uri): self.api_timestamp = str(int(time.time())) message = '{}:GET:{}'.format(uri, self.api_timestamp) digest = hmac.new(self.api_secret, msg=message, digestmod=hashlib.sha256).digest() self.signature = 'TC {}:{}'.format(self.api_key, base64.b64encode(digest).decode()) def __call__(self, r): r.headers['Authorization'] = self.signature r.headers['Timestamp'] = self.api_timestamp return r def _detect_ip_version(self, ip_addr): try: parsed = IPNetwork(ip_addr) except (AddrFormatError, ValueError): LOG.error('{} - Unknown IP version: {}'.format(self.name, ip_addr)) return None if parsed.version == 4: return 'IPv4' if parsed.version == 6: return 'IPv6' return None def _detect_sha_version(self, hash): for hash_type, re_obj in self.hash_patterns.iteritems(): if re_obj.match(hash) is not None: return hash_type return None def group_indicator_processing(self, item, group_type, group_id, f_seen, l_seen): attributes = {'tc_group_type': group_type, 'tc_group_id': group_id, 'first_seen': f_seen, 'last_seen': l_seen} indicator = item.get("summary", None) confidence = item.get('threatAssessConfidence', None) if confidence is not None: attributes['confidence'] = int(confidence) tc_indicator_type = item.get("type", None) if tc_indicator_type == "Address": attributes['type'] = self._detect_ip_version(indicator) elif tc_indicator_type == "File": attributes['type'] = self._detect_sha_version(indicator) elif tc_indicator_type == "EmailAddress": attributes['type'] = "email-addr" elif tc_indicator_type == "URL": attributes['type'] = "URL" elif tc_indicator_type == "Host": attributes['type'] = "domain" if tc_indicator_type is None or indicator is None: return [] return [indicator, attributes] def general_processing(self, item, indicator_map, f_seen, l_seen): result = [] for tc_indicator, mm_indicator in indicator_map.iteritems(): indicator = item.get(tc_indicator, None) if indicator is None: continue attributes = {'type': mm_indicator, 'first_seen': f_seen, 'last_seen': l_seen} confidence = item.get('threatAssessConfidence', None) if confidence is not None: attributes['confidence'] = int(confidence) add_attributes = dict(indicator_map) add_attributes.pop(tc_indicator) for tc_attribute, mm_attribute in add_attributes.iteritems(): value = item.get(tc_attribute, None) if value is None: continue attributes[mm_attribute] = value result.append([indicator, attributes]) return result def ip_processing(self, item, indicator_list, f_seen, l_seen): result = [] for tc_indicator in indicator_list: indicator = item.get(tc_indicator, None) if indicator is None: continue ip_type = self._detect_ip_version(indicator) if ip_type is None: continue attributes = {'type': ip_type, 'first_seen': f_seen, 'last_seen': l_seen} confidence = item.get('threatAssessConfidence', None) if confidence is not None: attributes['confidence'] = int(confidence) result.append([indicator, attributes]) return result def _paginate_request(self, entry_point, entity, from_timestamp=None): if from_timestamp is not None: isotime = datetime.fromtimestamp(from_timestamp / 1000).replace(tzinfo=pytz.utc).isoformat() def do_call(start): api_request = entry_point + '?resultStart={}&resultLimit=100'.format(start) if from_timestamp is not None: api_request += "&modifiedSince={}".format(isotime) if self.owner is not None: api_request += '&owner={}'.format(self.owner) self.prepare_get(api_request) final_url = self.api_url + api_request response = requests.get(final_url, auth=self) doc = response.json() if doc["status"] != "Success": raise RuntimeError("ThreatConnectAPI - {}".format(doc.get("message", "unknown error"))) return doc r_data = do_call(0) pointer = 0 if "data" not in r_data: return if "resultCount" not in r_data["data"]: return result_count = r_data["data"]["resultCount"] while True: items = r_data["data"][entity] for item in items: yield item pointer += len(items) if result_count <= pointer: break r_data = do_call(pointer) def indicator_iterator(self, last_tc_run): from_timestamp = last_tc_run for a in IP_INDICATOR_MAP: indicator_list = a.get("indicator", None) if indicator_list is None: continue for item in self._paginate_request(self.api_base_uri + "/v2/indicators/" + a["apiBranch"], a["apiEntity"], from_timestamp): yield ("IP", item, indicator_list) for a in GENERIC_INDICATOR_MAP: indicator_map = a.get("indicator", None) if indicator_map is None: continue for item in self._paginate_request(self.api_base_uri + "/v2/indicators/" + a["apiBranch"], a["apiEntity"], from_timestamp): yield ("GENERAL", item, indicator_map) def groups_iterator(self, groups): for group_type, group_ids in groups.iteritems(): for group_id in group_ids: for item in self._paginate_request( self.api_base_uri + "/v2/groups/{}/{}/indicators".format(group_type, group_id), "indicator"): yield (item, group_type, group_id) class TCMiner(BasePollerFT): tc = None api_secret = None api_key = None api_url = None api_base_uri = None owner = None side_config_path = None last_tc_run = None def configure(self): super(TCMiner, self).configure() self.api_key = self.config.get('apikey', None) self.api_secret = self.config.get('apisecret', None) self.sndbox_fqdn = self.config.get('sndbox_fqdn', 'sandbox.threatconnect.com') sandbox = self.config.get('sandbox', False) if sandbox: self.api_url = 'https://{}'.format(self.sndbox_fqdn) self.api_base_uri = '/api' else: self.api_url = 'https://api.threatconnect.com' self.api_base_uri = '' self.owner = self.config.get('owner', None) if not (None in [self.api_key, self.api_secret]): self.tc = ThreatConnect(self.api_secret, self.api_key, self.api_url, self.api_base_uri, self.owner) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.tc = None data_owner = sconfig.get('owner', self.owner) side_api_key = sconfig.get('apikey', self.api_key) side_api_secret = sconfig.get('apisecret', self.api_secret) if not (None in [side_api_key, side_api_secret]): self.tc = ThreatConnect(side_api_secret, side_api_key, self.api_url, self.api_base_uri, data_owner) def _saved_state_restore(self, saved_state): super(TCMiner, self)._saved_state_restore(saved_state) self.last_tc_run = saved_state.get('last_tc_run', None) LOG.info('last_tc_run from sstate: %s', self.last_tc_run) def _saved_state_create(self): sstate = super(TCMiner, self)._saved_state_create() sstate['last_tc_run'] = self.last_tc_run return sstate def _saved_state_reset(self): super(TCMiner, self)._saved_state_reset() self.last_tc_run = None def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(TCMiner, self).hup(source=source) @staticmethod def gc(name, config=None): BasePollerFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) finally: pass class IndicatorsMiner(TCMiner): initial_interval = None def configure(self): super(IndicatorsMiner, self).configure() self.initial_interval = self.config.get('initial_interval', 30) def _build_iterator(self, now): if self.tc is None: raise RuntimeError( '{} - API Key or API Secret not set, ' 'poll not performed'.format(self.name) ) if self.last_successful_run is None: self.last_successful_run = utc_millisec() - self.initial_interval * 86400000.0 if self.last_tc_run is None: self.last_tc_run = self.last_successful_run return self.tc.indicator_iterator(self.last_tc_run) def _process_item(self, item): tc_date_added = item[1].get('dateAdded', None) tc_last_modified = item[1].get('lastModified', None) f_seen = utc_millisec() if tc_date_added is None else dt_to_millisec( datetime.strptime(tc_date_added, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.utc)) l_seen = utc_millisec() if tc_last_modified is None else dt_to_millisec( datetime.strptime(tc_last_modified, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.utc)) if l_seen > self.last_tc_run: self.last_tc_run = l_seen if item[0] == "IP": return self.tc.ip_processing(item[1], item[2], f_seen, l_seen) if item[0] == "GENERAL": return self.tc.general_processing(item[1], item[2], f_seen, l_seen) return [] class GroupsMiner(TCMiner): groups = {} def configure(self): super(GroupsMiner, self).configure() groups = self.config.get('groups', None) if groups is not None and isinstance(groups, dict): for group_type in GROUP_TYPES: group_ids = groups.get(group_type, None) if group_ids is not None and isinstance(group_ids, list): self.groups[group_type] = group_ids def _build_iterator(self, now): if self.tc is None: raise RuntimeError( '{} - API Key or API Secret not set, ' 'poll not performed'.format(self.name) ) return self.tc.groups_iterator(self.groups) def _process_item(self, item): tc_date_added = item[0].get('dateAdded', None) tc_last_modified = item[0].get('lastModified', None) f_seen = utc_millisec() if tc_date_added is None else dt_to_millisec( datetime.strptime(tc_date_added, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.utc)) l_seen = utc_millisec() if tc_last_modified is None else dt_to_millisec( datetime.strptime(tc_last_modified, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=pytz.utc)) return [self.tc.group_indicator_processing(item[0], item[1], item[2], f_seen, l_seen)] ================================================ FILE: minemeld/ft/threatq.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements minemeld.ft.threatq.Export, the Miner node for ThreatQ export API. """ import requests import logging import os import yaml import netaddr from . import basepoller LOG = logging.getLogger(__name__) class Export(basepoller.BasePollerFT): """Implements class for Miners of ThreatQ Export API. **Config parameters** :side_config (str): path to the side config file, defaults to CONFIGDIR/_side_config.yml :polling_timeout: timeout of the polling request in seconds. Default: 20 **Side Config parameters** :url: URL of the feed. :polling_timeout: timeout of the polling request in seconds. Default: 20 :verify_cert: boolean, if *true* feed HTTPS server certificate is verified. Default: *true* Example: Example side config in YAML:: url: https://10.5.172.225/api/export/6e472a434efe34ceb5a99ff6c9a8124e/?token=xoZjB4ypoNQdnbQhVi0B verify_cert: false Args: name (str): node name, should be unique inside the graph chassis (object): parent chassis instance config (dict): node config. """ def configure(self): super(Export, self).configure() self.polling_timeout = self.config.get('polling_timeout', 20) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.url = sconfig.get('url', None) if self.url is not None: LOG.info('%s - url set', self.name) self.verify_cert = sconfig.get('verify_cert', True) def _process_item(self, line): line = line.strip() if not line: return [[None, None]] itype, indicator = line.split(',', 1) attributes = {} if itype == 'IP Address': ipaddr = netaddr.IPAddress(indicator) if ipaddr.version == 4: attributes['type'] = 'IPv4' elif ipaddr.version == 6: attributes['type'] = 'IPv6' else: LOG.error( '%s - %s: unknown IP version %s', line, self.name, ipaddr.version ) return [[None, None]] elif itype == 'CIDR Block': ipaddr = netaddr.IPNetwork(indicator) if ipaddr.version == 4: attributes['type'] = 'IPv4' elif ipaddr.version == 6: attributes['type'] = 'IPv6' else: LOG.error( '%s - %s: unknown IP version %s', line, self.name, ipaddr.version ) return [[None, None]] elif itype == 'FQDN': attributes['type'] = 'domain' elif itype == 'URL': attributes['type'] = 'URL' else: LOG.error( '%s - unknown indicator type %s - ignored', self.name, itype ) return [[None, None]] return [[indicator, attributes]] def _build_iterator(self, now): if self.url is None: raise RuntimeError( '%s - url not set, poll not performed' % self.name ) rkwargs = dict( stream=True, verify=self.verify_cert, timeout=self.polling_timeout ) r = requests.get( self.url, **rkwargs ) try: r.raise_for_status() except: LOG.debug('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise result = r.iter_lines() return result def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(Export, self).hup(source=source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass ================================================ FILE: minemeld/ft/tmt.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import requests import os import yaml import itertools import csv import gevent import shutil from . import basepoller from . import table from .utils import interval_in_sec LOG = logging.getLogger(__name__) class DTIAPI(basepoller.BasePollerFT): _AGE_OUT_BASES = ['first_seen', 'last_seen', 'tmt_last_sample_timestamp'] _DEFAULT_AGE_OUT_BASE = 'tmt_last_sample_timestamp' def __init__(self, name, chassis, config): self.ttable = None super(DTIAPI, self).__init__(name, chassis, config) def configure(self): super(DTIAPI, self).configure() self.polling_timeout = self.config.get('polling_timeout', 120) self.verify_cert = self.config.get('verify_cert', True) self.dialect = { 'delimiter': self.config.get('delimiter', ','), 'doublequote': self.config.get('doublequote', True), 'escapechar': self.config.get('escapechar', None), 'quotechar': self.config.get('quotechar', '"'), 'skipinitialspace': self.config.get('skipinitialspace', False) } self.include_suspicious = self.config.get('include_suspicious', True) initial_interval = self.config.get('initial_interval', '2d') self.initial_interval = interval_in_sec(initial_interval) if self.initial_interval is None: LOG.error( '%s - wrong initial_interval format: %s', self.name, initial_interval ) self.initial_interval = interval_in_sec('2d') self.source_name = 'themediatrust.dti' self.api_key = None self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.api_key = sconfig.get('api_key', None) if self.api_key is not None: LOG.info('%s - authorization code set', self.name) def _process_row(self, row): ip = row.pop('ip_addres', None) if ip == '0.0.0.0': ip = None domain = row.pop('host_name', None) value = {} for k, v in row.iteritems(): if k == 'last_sample_timestamp': value['tmt_last_sample_timestamp'] = int(v)*1000 continue key = k if not k.startswith('tmt'): key = 'tmt_%s' % k value[key] = [v] return ip, domain, value def _process_item(self, item): type_, indicator = item[0].split(':', 1) value = {} for k, v in item[1].iteritems(): value[k] = v value['type'] = type_ return [[indicator, value]] def _tmerge(self, indicator, value): ov = self.ttable.get(indicator) if ov is None: self.ttable.put(indicator, value) return for k, v in value.iteritems(): if k == 'tmt_last_sample_timestamp': if v > ov[k]: # confusing, this is just for PEP8 sake ov[k] = v continue if v[0] not in ov[k]: ov[k].append(v) self.ttable.put(indicator, ov) def _build_iterator(self, now): if self.api_key is None: raise RuntimeError('%s - api_key not set' % self.name) if self.ttable is not None: self.ttable.close() self.ttable = None self.ttable = table.Table(self.name+'_temp', truncate=True) last_fetch = self.last_run if last_fetch is None: last_fetch = int(now/1000) - self.initial_interval params = dict( key=self.api_key, action='fjord_base', include_suspicious=(1 if self.include_suspicious else 0), last_fetch=last_fetch ) rkwargs = dict( stream=True, verify=self.verify_cert, timeout=self.polling_timeout, params=params ) r = requests.get( 'https://www.themediatrust.com/api', **rkwargs ) try: r.raise_for_status() except: LOG.debug('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise response = itertools.ifilter( lambda x: not x.startswith('got commandoptions'), r.raw ) csvreader = csv.DictReader( response, **self.dialect ) for row in csvreader: gevent.sleep(0) ip, domain, value = self._process_row(row) if ip is None and domain is None: continue if ip is not None: self._tmerge('IPv4:%s' % ip, value) if domain is not None: self._tmerge('domain:%s' % domain, value) return self.ttable.query(include_value=True) def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(DTIAPI, self).hup(source=source) @staticmethod def gc(name, config=None): basepoller.BasePollerFT.gc(name, config=config) shutil.rmtree('{}_temp'.format(name), ignore_errors=True) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass ================================================ FILE: minemeld/ft/utils.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import time import operator import functools import datetime import pytz import re import gevent import gevent.lock import gevent.event EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.UTC) def utc_millisec(): return int(time.time()*1000) def dt_to_millisec(dt): if dt.tzinfo == None: dt = dt.replace(tzinfo=pytz.UTC) delta = dt - EPOCH return int(delta.total_seconds()*1000) def interval_in_sec(val): if isinstance(val, int): return val multipliers = { '': 1, 'm': 60, 'h': 3600, 'd': 86400 } mo = re.match("([0-9]+)([dmh]?)", val) if mo is None: return None return int(mo.group(1))*multipliers[mo.group(2)] def age_out_in_millisec(val): multipliers = { '': 1000, 'm': 60000, 'h': 3600000, 'd': 86400000 } mo = re.match("([0-9]+)([dmh]?)", val) if mo is None: return None return int(mo.group(1))*multipliers[mo.group(2)] def _merge_atomic_values(op, v1, v2): if op(v1, v2): return v2 return v1 def _merge_array(v1, v2): for e in v2: if e not in v1: v1.append(e) return v1 RESERVED_ATTRIBUTES = { 'sources': _merge_array, 'first_seen': functools.partial(_merge_atomic_values, operator.gt), 'last_seen': functools.partial(_merge_atomic_values, operator.lt), 'type': functools.partial(_merge_atomic_values, operator.eq), 'direction': functools.partial(_merge_atomic_values, operator.eq), 'confidence': functools.partial(_merge_atomic_values, operator.lt), 'country': functools.partial(_merge_atomic_values, operator.eq), 'AS': functools.partial(_merge_atomic_values, operator.eq) } class RWLock(object): def __init__(self): self.num_readers = 0 self.num_writers = 0 self.m1 = gevent.lock.Semaphore(1) self.m2 = gevent.lock.Semaphore(1) self.m3 = gevent.lock.Semaphore(1) self.w = gevent.lock.Semaphore(1) self.r = gevent.lock.Semaphore(1) def lock(self): self.m2.acquire() self.num_writers += 1 if self.num_writers == 1: self.r.acquire() self.m2.release() self.w.acquire() def unlock(self): self.w.release() self.m2.acquire() self.num_writers -= 1 if self.num_writers == 0: self.r.release() self.m2.release() def rlock(self): self.m3.acquire() self.r.acquire() self.m1.acquire() self.num_readers += 1 if self.num_readers == 1: self.w.acquire() self.m1.release() self.r.release() self.m3.release() def runlock(self): self.m1.acquire() self.num_readers -= 1 if self.num_readers == 0: self.w.release() self.m1.release() def __enter__(self): self.rlock() def __exit__(self, type, value, traceback): self.runlock() _AGE_OUT_BASES = ['last_seen', 'first_seen'] def parse_age_out(s, age_out_bases=None, default_base=None): if s is None: return None if age_out_bases is None: age_out_bases = _AGE_OUT_BASES if default_base is None: default_base = 'first_seen' if default_base not in age_out_bases: raise ValueError('%s not in %s' % (default_base, age_out_bases)) result = {} toks = s.split('+', 1) if len(toks) == 1: t = toks[0].strip() if t in age_out_bases: result['base'] = t result['offset'] = 0 else: result['base'] = default_base result['offset'] = age_out_in_millisec(t) if result['offset'] is None: raise ValueError('Invalid age out offset %s' % t) else: base = toks[0].strip() if base not in age_out_bases: raise ValueError('Invalid age out base %s' % base) result['base'] = base result['offset'] = age_out_in_millisec(toks[1].strip()) if result['offset'] is None: raise ValueError('Invalid age out offset %s' % t) return result class GThrottled(object): def __init__(self, f, wait): self._timeout = None self._previous = 0 self._cancelled = False self._args = [] self._kwargs = {} self.f = f self.wait = wait def later(self): self._previous = utc_millisec() self._timeout = None self.f(*self._args, **self._kwargs) def __call__(self, *args, **kwargs): now = utc_millisec() remaining = self.wait - (now - self._previous) if self._cancelled: return if remaining <= 0 or remaining > self.wait: if self._timeout is not None: self._timeout.join(timeout=5) self._timeout = None self._previous = now self.f(*args, **kwargs) elif self._timeout is None: self._args = args self._kwargs = kwargs self._timeout = gevent.spawn_later(remaining/1000.0, self.later) else: self._args = args self._kwargs = kwargs def cancel(self): self._cancelled = True if self._timeout: self._timeout.join(timeout=5) if self._timeout is not None: self._timeout.kill() self._previous = 0 self._timeout = None self._args = [] self._kwargs = {} ================================================ FILE: minemeld/ft/visa.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements minemeld.ft.visa.VTI, the Miner node for Visa Threat Intelligence API. """ import logging import requests import re from . import json from utils import utc_millisec, dt_to_millisec from datetime import datetime from netaddr import IPNetwork, AddrFormatError LOG = logging.getLogger(__name__) VTI_INDICATOR_TYPES = {'Hash': 'HASH', 'IP': 'IP', 'Email': 'email-addr', 'URL': 'URL', 'FQDN': 'domain'} VTI_VICTIM_TYPES = ('Restaurant', 'Retail', 'Hospitality and Lodging', 'QSR', 'B2B', 'Supermarket', 'POS Integrator', 'Financial Institution', 'Cinema', 'Parking', 'Pharmacy', 'Telecommunications', 'Other Retail') SHA256_PATTERN = "[A-Fa-f0-9]{64}" SHA1_PATTERN = "[A-Fa-f0-9]{40}" MD5_PATTERN = "[A-Fa-f0-9]{32}" class VTI(json.SimpleJSON): initial_interval = None indicator_type = None victim_type = None hash_patterns = {"sha256": re.compile(SHA256_PATTERN), "sha1": re.compile(SHA1_PATTERN), "md5": re.compile(MD5_PATTERN)} def configure(self): super(VTI, self).configure() self.initial_interval = self.config.get('initial_interval', '30') self.indicator_type = self.config.get('indicator_type', None) if self.indicator_type is not None and self.indicator_type not in VTI_INDICATOR_TYPES: self.indicator_type = None self.victim_type = self.config.get('victim_type', None) if self.victim_type is not None and self.victim_type not in VTI_VICTIM_TYPES: self.victim_type = None def _process_item(self, item): if self.indicator not in item: LOG.debug('%s not in %s', self.indicator, item) return [[None, None]] indicator = item[self.indicator] if not (isinstance(indicator, str) or isinstance(indicator, unicode)): LOG.error( 'Wrong indicator type: %s - %s', indicator, type(indicator) ) return [[None, None]] indicator_type = item.get('indicatorType', None) if indicator_type is not None: indicator_type = VTI_INDICATOR_TYPES.get(indicator_type, None) if indicator_type == 'HASH': indicator_type = self._detect_sha_version(indicator) if indicator_type == 'IP': indicator_type = self._detect_ip_version(indicator) upload_date = item.get('uploadDate', None) if upload_date is None: upload_date = utc_millisec() else: try: dt = datetime.strptime(upload_date, '%Y-%m-%d') upload_date = dt_to_millisec(dt) except ValueError: upload_date = utc_millisec() if upload_date > self.last_vti_run: self.last_vti_run = upload_date fields = self.fields if fields is None: fields = item.keys() fields.remove(self.indicator) if 'indicatorType' in fields: fields.remove('indicatorType') if 'uploadDate' in fields: fields.remove('uploadDate') attributes = {'type': indicator_type, 'first_seen': upload_date, 'last_seen': upload_date} for field in fields: if field not in item: continue attributes['%s_%s' % (self.prefix, field)] = item[field] return [[indicator, attributes]] def _build_iterator(self, now): rkwargs = dict( stream=True, verify=self.verify_cert, timeout=self.polling_timeout ) if self.headers is not None: rkwargs['headers'] = self.headers if self.username is not None and self.password is not None: rkwargs['auth'] = (self.username, self.password) else: raise RuntimeError('%s - credentials not set' % self.name) if self.client_cert_required and self.key_file is not None and self.cert_file is not None: rkwargs['cert'] = (self.cert_file, self.key_file) else: raise RuntimeError('%s - client certificate/key not set' % self.name) if self.last_successful_run is None: self.last_successful_run = utc_millisec() - self.initial_interval * 86400000.0 if self.last_vti_run is None: self.last_vti_run = self.last_successful_run start_date = datetime.fromtimestamp(self.last_vti_run / 1000) end_date = datetime.fromtimestamp(utc_millisec() / 1000) payload = {'startDate': start_date.strftime('%Y-%m-%d'), 'endDate': end_date.strftime('%Y-%m-%d')} if self.indicator_type is not None: payload['indicatorType'] = self.indicator_type if self.victim_type is not None: payload['victimType'] = self.victim_type r = requests.get( self.url, params=payload, **rkwargs ) try: r.raise_for_status() except: LOG.debug('%s - exception in request: %s %s', self.name, r.status_code, r.content) raise result = self.extractor.search(r.json()) if result is None: result = [] return result def _detect_ip_version(self, ip_addr): try: parsed = IPNetwork(ip_addr) except (AddrFormatError, ValueError): LOG.error('{} - Unknown IP version: {}'.format(self.name, ip_addr)) return None if parsed.version == 4: return 'IPv4' if parsed.version == 6: return 'IPv6' return None def _detect_sha_version(self, hash_value): for hash_type, re_obj in self.hash_patterns.iteritems(): if re_obj.match(hash_value) is not None: return hash_type return None def _saved_state_restore(self, saved_state): super(VTI, self)._saved_state_restore(saved_state) self.last_vti_run = saved_state.get('last_vti_run', None) LOG.info('last_vti_run from sstate: %s', self.last_vti_run) def _saved_state_create(self): sstate = super(VTI, self)._saved_state_create() sstate['last_vti_run'] = self.last_vti_run return sstate def _saved_state_reset(self): super(VTI, self)._saved_state_reset() self.last_vti_run = None ================================================ FILE: minemeld/ft/vt.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements: - minemeld.ft.vt.Notifications, the Miner node for VirusTotal Notifications feed """ import logging import os import yaml from . import json LOG = logging.getLogger(__name__) _VT_NOTIFICATIONS = 'https://www.virustotal.com/intelligence/hunting/notifications-feed/?key=' class Notifications(json.SimpleJSON): def __init__(self, name, chassis, config): super(Notifications, self).__init__(name, chassis, config) self.api_key = None def configure(self): self.config['url'] = None self.config['extractor'] = 'notifications' self.config['prefix'] = 'vt' super(Notifications, self).configure() self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.api_key = sconfig.get('api_key', None) if self.api_key is not None: LOG.info('%s - api key set', self.name) self.url = _VT_NOTIFICATIONS + self.api_key def _process_item(self, item): result = [] for htype in ['md5', 'sha256', 'sha1']: value = {self.prefix+'_'+k: v for k, v in item.iteritems()} indicator = value.pop(self.prefix+'_'+htype, None) value['type'] = htype if indicator is not None: result.append([indicator, value]) return result def _build_iterator(self, now): if self.api_key is None: LOG.info('%s - API key not set', self.name) raise RuntimeError( '%s - API Key not set' % self.name ) return super(Notifications, self)._build_iterator(now) def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() super(Notifications, self).hup(source=source) @staticmethod def gc(name, config=None): json.SimpleJSON.gc(name, config=config) side_config_path = None if config is not None: side_config_path = config.get('side_config', None) if side_config_path is None: side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '{}_side_config.yml'.format(name) ) try: os.remove(side_config_path) except: pass ================================================ FILE: minemeld/ft/xmpp.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import import logging import ujson import random import gevent import gevent.event import gevent.queue import yaml import os import sleekxmpp import sleekxmpp.xmlstream from . import base from . import op LOG = logging.getLogger(__name__) class XMPPOutput(base.BaseFT): def __init__(self, name, chassis, config): super(XMPPOutput, self).__init__(name, chassis, config) self._xmpp_client = None self._xmpp_glet = None self._publisher_glet = None self.q = gevent.queue.Queue() self._read_sequence_number() self._load_event = gevent.event.Event() self._xmpp_client_ready = gevent.event.Event() def configure(self): super(XMPPOutput, self).configure() self.server = self.config.get('server', None) self.port = self.config.get('port', 5222) self.pubsub_service = self.config.get('pubsub_service', None) self.node = self.config.get('node', None) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): self.jid = None self.password = None try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.jid = sconfig.get('jid', None) if self.jid is not None: LOG.info('%s - jid set', self.name) self.password = sconfig.get('password', None) if self.password is not None: LOG.info('%s - password set', self.name) def connect(self, inputs, output): output = False super(XMPPOutput, self).connect(inputs, output) def initialize(self): pass def rebuild(self): self.sequence_number = None def reset(self): self.sequence_number = None def _read_sequence_number(self): self.sequence_number = None try: with open(self.name+'.seqn', 'r') as f: self.sequence_number = int(f.read().strip()) os.remove(self.name+'.seqn') except IOError: pass def _write_sequence_number(self): if self.sequence_number is None: return with open(self.name+'.seqn', 'w') as f: f.write('%s' % self.sequence_number) @base._counting('update.processed') def filtered_update(self, source=None, indicator=None, value=None): self.q.put(['UPDATE', indicator, value]) @base._counting('withdraw.processed') def filtered_withdraw(self, source=None, indicator=None, value=None): self.q.put(['WITHDRAW', indicator, value]) def _xmpp_publish(self, cmd, data=None): if data is None: data = '' payload_xml = sleekxmpp.xmlstream.ET.Element('mm-command') command_xml = sleekxmpp.xmlstream.ET.SubElement(payload_xml, 'command') command_xml.text = cmd seqno_xml = sleekxmpp.xmlstream.ET.SubElement(payload_xml, 'seqno') seqno_xml.text = '%s' % self.sequence_number data_xml = sleekxmpp.xmlstream.ET.SubElement(payload_xml, 'data') data_xml.text = ujson.dumps(data) result = self._xmpp_client['xep_0060'].publish( self.pubsub_service, self.node, payload=payload_xml ) LOG.debug('%s - xmpp publish: %s', self.name, result) self.sequence_number += 1 self.statistics['xmpp.published'] += 1 def _xmpp_session_start(self, event): LOG.debug('%s - _xmpp_session_start', self.name) self._xmpp_client.get_roster() self._xmpp_client.send_presence() if self.sequence_number is None: self.sequence_number = random.getrandbits(64) self._xmpp_publish('INIT') self._xmpp_client_ready.set() def _xmpp_disconnected(self, event): LOG.debug('%s - _xmpp_disconnected', self.name) self._xmpp_client_ready.clear() def _start_xmpp_client(self): if self._xmpp_client is not None: return if self.jid is None or self.password is None: raise RuntimeError('%s - jid or password not set', self.name) if self.server is None or self.port is None: raise RuntimeError('%s - server or port not set', self.name) if self.node is None or self.pubsub_service is None: raise RuntimeError( '%s - node or pubsub_service not set', self.name ) self._xmpp_client = sleekxmpp.ClientXMPP( jid=self.jid, password=self.password ) self._xmpp_client.register_plugin('xep_0030') self._xmpp_client.register_plugin('xep_0059') self._xmpp_client.register_plugin('xep_0060') self._xmpp_client.add_event_handler( 'session_start', self._xmpp_session_start ) self._xmpp_client.add_event_handler( 'disconnected', self._xmpp_disconnected ) if not self._xmpp_client.connect((self.server, self.port)): raise RuntimeError( '%s - error connecting to XMPP server', self.name ) self._xmpp_client.process(block=True) def _publisher(self): while True: self._xmpp_client_ready.wait() try: while True: cmd, indicator, value = self.q.peek() if value is None: value = {} value['origins'] = [self.jid] self._xmpp_publish(cmd, { 'indicator': indicator, 'value': value }) _ = self.q.get() except gevent.GreenletExit: break except Exception as e: LOG.exception('%s - Exception in publishing message', self.name) gevent.sleep(30) self.statistics['xmpp.publish_error'] += 1 def _run(self): while True: try: self._start_xmpp_client() except RuntimeError() as e: LOG.error('%s - %s', self.name, str(e)) self.statistics['xmpp.error'] += 1 except gevent.GreenletExit: if self._xmpp_client is not None: self._xmpp_client.disconnect() break except Exception as e: LOG.exception('%s - error in starting XMPP client', self.name) try: if self._xmpp_client is not None: self._xmpp_client.disconnect() self._xmpp_client = None hup_called = self._load_event.wait(timeout=60) if hup_called: LOG.debug('%s - clearing load event', self.name) self._load_event.clear() except gevent.GreenletExit: break def length(self, source=None): return 0 def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() self._load_event.set() def mgmtbus_checkpoint(self, value=None): self._write_sequence_number() return super(XMPPOutput, self).mgmtbus_checkpoint(value=value) def start(self): super(XMPPOutput, self).start() if self._xmpp_glet is not None: return self._xmpp_glet = gevent.spawn_later(random.randint(0, 2), self._run) self._publisher_glet = gevent.spawn_later(random.randint(0, 3), self._publisher) def stop(self): super(XMPPOutput, self).stop() if self._xmpp_client is None: return self._xmpp_glet.kill() self._publisher_glet.kill() class XMPPMiner(op.AggregateFT): def __init__(self, name, chassis, config): super(XMPPMiner, self).__init__(name, chassis, config) self._xmpp_client = None self._xmpp_glet = None self._load_event = gevent.event.Event() def configure(self): super(XMPPMiner, self).configure() self.server = self.config.get('server', None) self.port = self.config.get('port', 5222) self.pubsub_service = self.config.get('pubsub_service', None) self.node = self.config.get('node', None) self.side_config_path = self.config.get('side_config', None) if self.side_config_path is None: self.side_config_path = os.path.join( os.environ['MM_CONFIG_DIR'], '%s_side_config.yml' % self.name ) self._load_side_config() def _load_side_config(self): self.jid = None self.password = None try: with open(self.side_config_path, 'r') as f: sconfig = yaml.safe_load(f) except Exception as e: LOG.error('%s - Error loading side config: %s', self.name, str(e)) return self.jid = sconfig.get('jid', None) if self.jid is not None: LOG.info('%s - jid set', self.name) self.password = sconfig.get('password', None) if self.password is not None: LOG.info('%s - password set', self.name) def _xmpp_session_start(self, event): LOG.debug('%s - _xmpp_session_start', self.name) self._xmpp_client.get_roster() self._xmpp_client.send_presence() result = self._xmpp_client['xep_0060'].subscribe( self.pubsub_service, self.node ) LOG.debug('%s - subscribe result: %s', self.name, result) def _xmpp_publish(self, msg): LOG.debug('%s - _publish %s', self.name, msg) node_ = msg['pubsub_event']['items']['node'] if node_ != self.node: return payload = msg['pubsub_event']['items']['item']['payload'] if payload is None: return command = payload.find('{http://jabber.org/protocol/pubsub#event}command') if command is None: LOG.error( '%s - pubsub event received with no commands', self.name ) return command = command.text if command == 'INIT': return data = payload.find('{http://jabber.org/protocol/pubsub#event}data') if data is None: LOG.error( '%s - pubsub event received with no data', self.name ) return data = data.text data = ujson.loads(data) indicator = data.get('indicator', None) if indicator is None: LOG.error('%s - received command with no indicator', self.name) return value = data.get('value', None) if value is None: LOG.error('%s - received command with no value', self.name) return origins = value.get('origins', None) if origins is None: LOG.error('%s - received indicator with no origin', self.name) return if self.jid in origins: LOG.debug('%s - indicator already known, ignored', self.name) return if command == 'UPDATE': for o in origins: if o not in self.inputs: self.inputs.append(o) self.update( source=o, indicator=indicator, value=value ) elif command == 'WITHDRAW': for o in origins: if o not in self.inputs: self.inputs.append(o) self.withdraw( source=o, indicator=indicator, value=value ) else: LOG.error('%s - unknown command %s', self.name, command) def _start_xmpp_client(self): if self._xmpp_client is not None: return if self.jid is None or self.password is None: raise RuntimeError('%s - jid or password not set', self.name) if self.server is None or self.port is None: raise RuntimeError('%s - server or port not set', self.name) if self.node is None or self.pubsub_service is None: raise RuntimeError( '%s - node or pubsub_service not set', self.name ) self._xmpp_client = sleekxmpp.ClientXMPP( jid=self.jid, password=self.password ) self._xmpp_client.register_plugin('xep_0030') self._xmpp_client.register_plugin('xep_0059') self._xmpp_client.register_plugin('xep_0060') self._xmpp_client.add_event_handler( 'session_start', self._xmpp_session_start ) self._xmpp_client.add_event_handler( 'pubsub_publish', self._xmpp_publish ) if not self._xmpp_client.connect((self.server, self.port)): raise RuntimeError( '%s - error connecting to XMPP server', self.name ) self._xmpp_client.process(block=True) def _run(self): while True: try: self._start_xmpp_client() except RuntimeError() as e: LOG.error('%s - %s', self.name, str(e)) self.statistics['xmpp.error'] += 1 except gevent.GreenletExit: if self._xmpp_client is not None: self._xmpp_client.disconnect() break except Exception as e: LOG.exception('%s - error in starting XMPP client', self.name) try: if self._xmpp_client is not None: self._xmpp_client.disconnect() self._xmpp_client = None hup_called = self._load_event.wait(timeout=60) if hup_called: LOG.debug('%s - clearing load event', self.name) self._load_event.clear() except gevent.GreenletExit: break def start(self): super(XMPPMiner, self).start() if self._xmpp_glet is not None: return self._xmpp_glet = gevent.spawn_later(random.randint(0, 2), self._run) def stop(self): super(XMPPMiner, self).stop() if self._xmpp_client is None: return self._xmpp_glet.kill() def hup(self, source=None): LOG.info('%s - hup received, reload side config', self.name) self._load_side_config() self._load_event.set() ================================================ FILE: minemeld/loader.py ================================================ import logging import pip from pkg_resources import working_set, WorkingSet from collections import namedtuple LOG = logging.getLogger(__name__) MM_NODES_ENTRYPOINT = 'minemeld_nodes' MM_NODES_GCS_ENTRYPOINT = 'minemeld_nodes_gcs' MM_NODES_VALIDATORS_ENTRYPOINT = 'minemeld_nodes_validators' MM_PROTOTYPES_ENTRYPOINT = 'minemeld_prototypes' MM_API_ENTRYPOINT = 'minemeld_api' MM_WEBUI_ENTRYPOINT = 'minemeld_webui' MMEntryPoint = namedtuple( 'MMEntryPoint', ['ep', 'name', 'loadable', 'conflicts'] ) _ENTRYPOINT_GROUPS = {} _WS = None def _conflicts(requirements, installed): result = [] for r in requirements: installed_dist = installed.get(r.project_name, None) if installed_dist is None: result.append('{} not installed'.format(r.project_name)) continue if installed_dist.version not in r: result.append('{}=={} not compatible with {}'.format( installed_dist.project_name, installed_dist.version, str(r) )) return result def _initialize_entry_point_group(entrypoint_group): global _WS installed = {d.project_name: d for d in working_set} if _WS is None: _WS = WorkingSet() cache = {} result = {} for ep in _WS.iter_entry_points(entrypoint_group): egg_name = ep.dist.egg_name() conflicts = cache.get(egg_name, None) if conflicts is None: conflicts = _conflicts( ep.dist.requires(), installed ) cache[egg_name] = conflicts if len(conflicts) != 0: LOG.error('{} not loadable: {}'.format( ep.name, ', '.join(conflicts) )) result[ep.name] = MMEntryPoint( ep=ep, name=ep.name, conflicts=conflicts, loadable=(len(conflicts) == 0) ) _ENTRYPOINT_GROUPS[entrypoint_group] = result def bump_workingset(): global _WS, _ENTRYPOINT_GROUPS _WS = None _ENTRYPOINT_GROUPS = {} def list(entrypoint_group): if entrypoint_group not in _ENTRYPOINT_GROUPS: _initialize_entry_point_group(entrypoint_group) eg = _ENTRYPOINT_GROUPS[entrypoint_group] return eg.keys() def map(entrypoint_group): if entrypoint_group not in _ENTRYPOINT_GROUPS: _initialize_entry_point_group(entrypoint_group) eg = _ENTRYPOINT_GROUPS[entrypoint_group] return eg def load(entrypoint_group, entrypoint_name): LOG.info('Loading %s:%s', entrypoint_group, entrypoint_name) if entrypoint_group not in _ENTRYPOINT_GROUPS: _initialize_entry_point_group(entrypoint_group) eg = _ENTRYPOINT_GROUPS[entrypoint_group] mmep = eg.get(entrypoint_name, None) if mmep is None: raise RuntimeError('Unknown entry point: {}:{}'.format(entrypoint_group, entrypoint_name)) if not mmep.loadable: raise RuntimeError('Entry point {}:{} not loadable: {}'.format( entrypoint_group, entrypoint_name, ', '.join(mmep.conflicts) )) return mmep.ep.load() ================================================ FILE: minemeld/mgmtbus.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements master and slave hub classes for MineMeld engine management bus. Management bus master sends commands to all managemnt bus slaves by posting a message to a specific topic (MGMTBUS_PREFIX+'bus'). Slaves subscribe to the topic, and when a command is received they reply back to the master by sending the answer to the queue MGMTBUS_PREFIX+'master'. Slaves connections are multiplexed via slave hub class. Management bus is used to control the MineMeld engine graph and to periodically retrieve metrics from all the nodes. """ from __future__ import absolute_import import logging import uuid import collections import time import hashlib import os import gevent import gevent.event import gevent.lock import gevent.timeout import redis import ujson import minemeld.comm import minemeld.ft from .collectd import CollectdClient from .startupplanner import plan LOG = logging.getLogger(__name__) MGMTBUS_PREFIX = "mbus:" MGMTBUS_TOPIC = MGMTBUS_PREFIX+'bus' MGMTBUS_CHASSIS_TOPIC = MGMTBUS_PREFIX+'chassisbus' MGMTBUS_MASTER = '@'+MGMTBUS_PREFIX+'master' MGMTBUS_LOG_TOPIC = MGMTBUS_PREFIX+'log' MGMTBUS_STATUS_TOPIC = MGMTBUS_PREFIX+'status' class MgmtbusMaster(object): """MineMeld engine management bus master Args: ftlist (list): list of nodes config (dict): config comm_class (string): communication backend to be used comm_config (dict): config for the communication backend """ def __init__(self, ftlist, config, comm_class, comm_config, num_chassis): super(MgmtbusMaster, self).__init__() self.ftlist = ftlist self.config = config self.comm_config = comm_config self.comm_class = comm_class self.num_chassis = num_chassis self._chassis = [] self._all_chassis_ready = gevent.event.Event() self.graph_status = None self._start_timestamp = int(time.time())*1000 self._status_lock = gevent.lock.Semaphore() self.status_glet = None self._status = {} self.SR = redis.StrictRedis.from_url( os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock') ) self.comm = minemeld.comm.factory(self.comm_class, self.comm_config) self._out_channel = self.comm.request_pub_channel(MGMTBUS_TOPIC) self.comm.request_rpc_server_channel( name=MGMTBUS_MASTER, obj=self, allowed_methods=['rpc_status', 'rpc_chassis_ready'], method_prefix='rpc_' ) self._slaves_rpc_client = self.comm.request_rpc_fanout_client_channel( MGMTBUS_TOPIC ) self._chassis_rpc_client = self.comm.request_rpc_fanout_client_channel( MGMTBUS_CHASSIS_TOPIC ) self.comm.request_rpc_server_channel( name=MGMTBUS_STATUS_TOPIC, obj=self, allowed_methods=['status'] ) def rpc_status(self): """Returns collected status via RPC """ return self._status def rpc_chassis_ready(self, chassis_id=None): """Chassis signal ready state via this RPC """ if chassis_id in self._chassis: LOG.error('duplicate chassis_id received in rpc_chassis_ready') return 'ok' self._chassis.append(chassis_id) if len(self._chassis) == self.num_chassis: self._all_chassis_ready.set() return 'ok' def wait_for_chassis(self, timeout=60): """Wait for all the chassis signal ready state """ if self.num_chassis == 0: # empty config return if not self._all_chassis_ready.wait(timeout=timeout): raise RuntimeError('Timeout waiting for chassis') def start_chassis(self): self._send_cmd_and_wait( 'start', to_slaves=False, # chassis timeout=60 ) def _send_cmd(self, command, to_slaves=True, params=None, and_discard=False): """Sends command to slaves or chassis over mgmt bus. Args: command (str): command params (dict): params of the command and_discard (bool): discard answer, don't wait to_slaves (bool): send command to nodes, otherwise to chassis Returns: returns a gevent.event.AsyncResult that is signaled when all the answers are collected """ if params is None: params = {} rpc_client = self._slaves_rpc_client num_results = len(self.ftlist) if not to_slaves: rpc_client = self._chassis_rpc_client num_results = self.num_chassis return rpc_client.send_rpc( command, params=params, and_discard=and_discard, num_results=num_results ) def _send_cmd_and_wait(self, command, to_slaves=True, timeout=60): """Simple wrapper around _send_cmd for raising exceptions """ revt = self._send_cmd(command, to_slaves=to_slaves) success = revt.wait(timeout=timeout) if success is None: LOG.critical('Timeout in {}'.format(command)) raise RuntimeError('Timeout in {}'.format(command)) result = revt.get(block=False) if result['errors'] > 0: LOG.critical('Errors reported in {}'.format(command)) raise RuntimeError('Errors reported in {}'.format(command)) return result def _send_node_cmd(self, nodename, command, params=None): """Send command to a single node """ if params is None: params = {} try: result = self.comm.send_rpc( dest='{}directslave:{}'.format(MGMTBUS_PREFIX, nodename), method=command, params=params, timeout=60 ) except gevent.timeout.Timeout: msg = 'Timeout in {} to node {}'.format(command, nodename) LOG.error(msg) raise RuntimeError(msg) if result.get('result', None) is None: raise RuntimeError('Error in {} to node {}: {}'.format( command, nodename, result.get('error', '') )) return result['result'] def init_graph(self, config): """Initalizes graph by sending startup messages. Args: config (MineMeldConfig): config """ result = self._send_cmd_and_wait('state_info', timeout=60) LOG.info('state: {}'.format(result['answers'])) LOG.info('changes: {!r}'.format(config.changes)) state_info = {k.split(':', 2)[-1]: v for k, v in result['answers'].iteritems()} startup_plan = plan(config, state_info) for node, command in startup_plan.iteritems(): LOG.info('{} <= {}'.format(node, command)) self._send_node_cmd(node, command) self.graph_status = 'INIT' def checkpoint_graph(self, max_tries=60): """Checkpoints the graph. Args: max_tries (int): number of minutes before giving up """ LOG.info('checkpoint_graph called, checking current state') if self.graph_status != 'INIT': LOG.info('graph status {}, checkpoint_graph ignored'.format(self.graph_status)) return while True: revt = self._send_cmd('state_info') success = revt.wait(timeout=30) if success is None: LOG.error('timeout in state_info') gevent.sleep(60) continue result = revt.get(block=False) if result['errors'] > 0: LOG.critical('errors reported from nodes in ' + 'checkpoint_graph: %s', result['errors']) gevent.sleep(60) continue all_started = True for answer in result['answers'].values(): if answer.get('state', None) != minemeld.ft.ft_states.STARTED: all_started = False break if not all_started: LOG.error('some nodes not started yet, waiting') gevent.sleep(60) continue break chkp = str(uuid.uuid4()) LOG.info('Sending checkpoint {} to nodes'.format(chkp)) for nodename in self.ftlist: self._send_node_cmd(nodename, 'checkpoint', params={'value': chkp}) ntries = 0 while ntries < max_tries: revt = self._send_cmd('state_info') success = revt.wait(timeout=60) if success is None: LOG.error("Error retrieving nodes states after checkpoint") gevent.sleep(30) continue result = revt.get(block=False) cgraphok = True for answer in result['answers'].values(): cgraphok &= (answer['checkpoint'] == chkp) if cgraphok: LOG.info('checkpoint graph - all good') break gevent.sleep(2) ntries += 1 if ntries == max_tries: LOG.error('checkpoint_graph: nodes still not in ' 'checkpoint state after max_tries') self.graph_status = 'CHECKPOINT' def _send_collectd_metrics(self, answers, interval): """Send collected metrics from nodes to collectd. Args: answers (list): list of metrics interval (int): collection interval """ collectd_socket = self.config.get( 'COLLECTD_SOCKET', '/var/run/collectd.sock' ) cc = CollectdClient(collectd_socket) gstats = collections.defaultdict(lambda: 0) for source, a in answers.iteritems(): ntype = 'processors' if len(a.get('inputs', [])) == 0: ntype = 'miners' elif not a.get('output', False): ntype = 'outputs' stats = a.get('statistics', {}) length = a.get('length', None) _, _, source = source.split(':', 2) source = hashlib.md5(source).hexdigest()[:10] for m, v in stats.iteritems(): gstats[ntype+'.'+m] += v cc.putval(source+'.'+m, v, interval=interval, type_='minemeld_delta') if length is not None: gstats['length'] += length gstats[ntype+'.length'] += length cc.putval( source+'.length', length, type_='minemeld_counter', interval=interval ) for gs, v in gstats.iteritems(): type_ = 'minemeld_delta' if gs.endswith('length'): type_ = 'minemeld_counter' cc.putval('minemeld.'+gs, v, type_=type_, interval=interval) def _merge_status(self, nodename, status): currstatus = self._status.get(nodename, None) if currstatus is not None: if currstatus.get('clock', -1) > status.get('clock', -2): LOG.error('old clock: {} > {} - dropped'.format( currstatus.get('clock', -1), status.get('clock', -2) )) return self._status[nodename] = status try: source = nodename.split(':', 2)[2] self.SR.publish( 'mm-engine-status.'+source, ujson.dumps({ 'source': source, 'timestamp': int(time.time())*1000, 'status': status }) ) except: LOG.exception('Error publishing status') def _status_loop(self): """Greenlet that periodically retrieves metrics from nodes and sends them to collected. """ loop_interval = self.config.get('STATUS_INTERVAL', '60') try: loop_interval = int(loop_interval) except ValueError: LOG.error('invalid STATUS_INTERVAL settings, ' 'reverting to default') loop_interval = 60 while True: revt = self._send_cmd('status') success = revt.wait(timeout=30) if success is None: LOG.error('timeout in waiting for status updates from nodes') else: result = revt.get(block=False) with self._status_lock: for nodename, nodestatus in result['answers'].iteritems(): self._merge_status(nodename, nodestatus) try: self._send_collectd_metrics( result['answers'], loop_interval ) except: LOG.exception('Exception in _status_loop') gevent.sleep(loop_interval) def status(self, timestamp, **kwargs): source = kwargs.get('source', None) if source is None: LOG.error('no source in status report - dropped') return status = kwargs.get('status', None) if status is None: LOG.error('no status in status report - dropped') return if self._status_lock.locked(): return with self._status_lock: if timestamp < self._start_timestamp: return self._merge_status('mbus:slave:'+source, status) def start_status_monitor(self): """Starts status monitor greenlet. """ if self.status_glet is not None: LOG.error('double call to start_status') return self.status_glet = gevent.spawn(self._status_loop) def stop_status_monitor(self): """Stops status monitor greenlet. """ if self.status_glet is None: return self.status_glet.kill() self.status_glet = None def start(self): self.comm.start() def stop(self): self.comm.stop() class MgmtbusSlaveHub(object): """Hub MineMeld engine management bus slaves. Each chassis has an instance of this class, and each node in the chassis request a channel to the management bus via this instance. Args: config (dict): config comm_class (string): communication backend to be used comm_config (dict): config for the communication backend """ def __init__(self, config, comm_class, comm_config): self.config = config self.comm_config = comm_config self.comm_class = comm_class self.comm = minemeld.comm.factory(self.comm_class, self.comm_config) def request_log_channel(self): LOG.debug("Adding log channel") return self.comm.request_pub_channel( topic=MGMTBUS_LOG_TOPIC, multi_write=True ) def send_status(self, params): self.comm.send_rpc( dest=MGMTBUS_STATUS_TOPIC, method='status', params=params, block=True ) def request_chassis_rpc_channel(self, chassis): self.comm.request_rpc_server_channel( '{}chassis:{}'.format(MGMTBUS_PREFIX, chassis.chassis_id), chassis, allowed_methods=[ 'mgmtbus_start' ], method_prefix='mgmtbus_', fanout=MGMTBUS_CHASSIS_TOPIC ) def request_channel(self, node): self.comm.request_rpc_server_channel( '{}directslave:{}'.format(MGMTBUS_PREFIX, node.name), node, allowed_methods=[ 'mgmtbus_state_info', 'mgmtbus_initialize', 'mgmtbus_rebuild', 'mgmtbus_reset', 'mgmtbus_status', 'mgmtbus_checkpoint', 'mgmtbus_hup', 'mgmtbus_signal' ], method_prefix='mgmtbus_' ) self.comm.request_rpc_server_channel( '{}slave:{}'.format(MGMTBUS_PREFIX, node.name), node, allowed_methods=[ 'mgmtbus_state_info', 'mgmtbus_initialize', 'mgmtbus_rebuild', 'mgmtbus_reset', 'mgmtbus_status', 'mgmtbus_checkpoint' ], method_prefix='mgmtbus_', fanout=MGMTBUS_TOPIC ) def add_failure_listener(self, f): self.comm.add_failure_listener(f) def send_master_rpc(self, command, params=None, timeout=None): return self.comm.send_rpc( MGMTBUS_MASTER, command, params, timeout=timeout ) def start(self): LOG.debug('mgmtbus start called') self.comm.start() def stop(self): self.comm.stop() def master_factory(config, comm_class, comm_config, nodes, num_chassis): """Factory of management bus master instances Args: config (dict): management bus master config comm_class (string): communication backend. Unused, ZMQRedis is always used comm_config (dict): config of the communication backend fts (list): list of nodes Returns: Instance of minemeld.mgmtbus.MgmtbusMaster class """ _ = comm_class # noqa return MgmtbusMaster( ftlist=nodes, config=config, comm_class='ZMQRedis', comm_config=comm_config, num_chassis=num_chassis ) def slave_hub_factory(config, comm_class, comm_config): """Factory of management bus slave hub instances Args: config (dict): management bus master config comm_class (string): communication backend. Unused, ZMQRedis is always used comm_config (dict): config of the communication backend. Returns: Instance of minemeld.mgmtbus.MgmtbusSlaveHub class """ _ = comm_class # noqa return MgmtbusSlaveHub( config, 'ZMQRedis', comm_config ) ================================================ FILE: minemeld/packages/__init__.py ================================================ ================================================ FILE: minemeld/packages/gdns/LICENSE ================================================ Except when otherwise stated (look at the beginning of each file) the software and the documentation in this project are copyrighted by: Denis Bilenko and the contributors, http://www.gevent.org and licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: minemeld/packages/gdns/__init__.py ================================================ ================================================ FILE: minemeld/packages/gdns/_ares.pyx ================================================ # Copyright (c) 2011-2012 Denis Bilenko. See LICENSE for details. # 2016 - Modified by Palo Alto Networks cimport cares import sys from cpython cimport * from libc.string cimport memset from libc.stdlib cimport malloc, free from _socket import gaierror __all__ = ['channel'] TIMEOUT = 1 DEF EV_READ = 1 DEF EV_WRITE = 2 cdef extern from "dnshelper.c": int AF_INET int AF_INET6 struct hostent: char* h_name int h_addrtype struct sockaddr_t "sockaddr": pass struct ares_channeldata: pass object parse_h_aliases(hostent*) object parse_h_addr_list(hostent*) void* create_object_from_hostent(void*) # this imports _socket lazily struct sockaddr_in6: pass int gevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, sockaddr_in6* sa6) ARES_SUCCESS = cares.ARES_SUCCESS ARES_ENODATA = cares.ARES_ENODATA ARES_EFORMERR = cares.ARES_EFORMERR ARES_ESERVFAIL = cares.ARES_ESERVFAIL ARES_ENOTFOUND = cares.ARES_ENOTFOUND ARES_ENOTIMP = cares.ARES_ENOTIMP ARES_EREFUSED = cares.ARES_EREFUSED ARES_EBADQUERY = cares.ARES_EBADQUERY ARES_EBADNAME = cares.ARES_EBADNAME ARES_EBADFAMILY = cares.ARES_EBADFAMILY ARES_EBADRESP = cares.ARES_EBADRESP ARES_ECONNREFUSED = cares.ARES_ECONNREFUSED ARES_ETIMEOUT = cares.ARES_ETIMEOUT ARES_EOF = cares.ARES_EOF ARES_EFILE = cares.ARES_EFILE ARES_ENOMEM = cares.ARES_ENOMEM ARES_EDESTRUCTION = cares.ARES_EDESTRUCTION ARES_EBADSTR = cares.ARES_EBADSTR ARES_EBADFLAGS = cares.ARES_EBADFLAGS ARES_ENONAME = cares.ARES_ENONAME ARES_EBADHINTS = cares.ARES_EBADHINTS ARES_ENOTINITIALIZED = cares.ARES_ENOTINITIALIZED ARES_ELOADIPHLPAPI = cares.ARES_ELOADIPHLPAPI ARES_EADDRGETNETWORKPARAMS = cares.ARES_EADDRGETNETWORKPARAMS ARES_ECANCELLED = cares.ARES_ECANCELLED ARES_FLAG_USEVC = cares.ARES_FLAG_USEVC ARES_FLAG_PRIMARY = cares.ARES_FLAG_PRIMARY ARES_FLAG_IGNTC = cares.ARES_FLAG_IGNTC ARES_FLAG_NORECURSE = cares.ARES_FLAG_NORECURSE ARES_FLAG_STAYOPEN = cares.ARES_FLAG_STAYOPEN ARES_FLAG_NOSEARCH = cares.ARES_FLAG_NOSEARCH ARES_FLAG_NOALIASES = cares.ARES_FLAG_NOALIASES ARES_FLAG_NOCHECKRESP = cares.ARES_FLAG_NOCHECKRESP _ares_errors = dict([ (cares.ARES_SUCCESS, 'ARES_SUCCESS'), (cares.ARES_ENODATA, 'ARES_ENODATA'), (cares.ARES_EFORMERR, 'ARES_EFORMERR'), (cares.ARES_ESERVFAIL, 'ARES_ESERVFAIL'), (cares.ARES_ENOTFOUND, 'ARES_ENOTFOUND'), (cares.ARES_ENOTIMP, 'ARES_ENOTIMP'), (cares.ARES_EREFUSED, 'ARES_EREFUSED'), (cares.ARES_EBADQUERY, 'ARES_EBADQUERY'), (cares.ARES_EBADNAME, 'ARES_EBADNAME'), (cares.ARES_EBADFAMILY, 'ARES_EBADFAMILY'), (cares.ARES_EBADRESP, 'ARES_EBADRESP'), (cares.ARES_ECONNREFUSED, 'ARES_ECONNREFUSED'), (cares.ARES_ETIMEOUT, 'ARES_ETIMEOUT'), (cares.ARES_EOF, 'ARES_EOF'), (cares.ARES_EFILE, 'ARES_EFILE'), (cares.ARES_ENOMEM, 'ARES_ENOMEM'), (cares.ARES_EDESTRUCTION, 'ARES_EDESTRUCTION'), (cares.ARES_EBADSTR, 'ARES_EBADSTR'), (cares.ARES_EBADFLAGS, 'ARES_EBADFLAGS'), (cares.ARES_ENONAME, 'ARES_ENONAME'), (cares.ARES_EBADHINTS, 'ARES_EBADHINTS'), (cares.ARES_ENOTINITIALIZED, 'ARES_ENOTINITIALIZED'), (cares.ARES_ELOADIPHLPAPI, 'ARES_ELOADIPHLPAPI'), (cares.ARES_EADDRGETNETWORKPARAMS, 'ARES_EADDRGETNETWORKPARAMS'), (cares.ARES_ECANCELLED, 'ARES_ECANCELLED')]) # maps c-ares flag to _socket module flag _cares_flag_map = None cdef _prepare_cares_flag_map(): global _cares_flag_map import _socket _cares_flag_map = [ (getattr(_socket, 'NI_NUMERICHOST', 1), cares.ARES_NI_NUMERICHOST), (getattr(_socket, 'NI_NUMERICSERV', 2), cares.ARES_NI_NUMERICSERV), (getattr(_socket, 'NI_NOFQDN', 4), cares.ARES_NI_NOFQDN), (getattr(_socket, 'NI_NAMEREQD', 8), cares.ARES_NI_NAMEREQD), (getattr(_socket, 'NI_DGRAM', 16), cares.ARES_NI_DGRAM)] cpdef _convert_cares_flags(int flags, int default=cares.ARES_NI_LOOKUPHOST|cares.ARES_NI_LOOKUPSERVICE): if _cares_flag_map is None: _prepare_cares_flag_map() for socket_flag, cares_flag in _cares_flag_map: if socket_flag & flags: default |= cares_flag flags &= ~socket_flag if not flags: return default raise gaierror(-1, "Bad value for ai_flags: 0x%x" % flags) cpdef strerror(code): return '%s: %s' % (_ares_errors.get(code) or code, cares.ares_strerror(code)) class InvalidIP(ValueError): pass cdef void gevent_sock_state_callback(void *data, int s, int read, int write): if not data: return cdef channel ch = data ch._sock_state_callback(s, read, write) cdef class result: cdef public object value cdef public object exception def __init__(self, object value=None, object exception=None): self.value = value self.exception = exception def __repr__(self): if self.exception is None: return '%s(%r)' % (self.__class__.__name__, self.value) elif self.value is None: return '%s(exception=%r)' % (self.__class__.__name__, self.exception) else: return '%s(value=%r, exception=%r)' % (self.__class__.__name__, self.value, self.exception) # add repr_recursive precaution def successful(self): return self.exception is None def get(self): if self.exception is not None: raise self.exception return self.value class ares_host_result(tuple): def __new__(cls, family, iterable): cdef object self = tuple.__new__(cls, iterable) self.family = family return self def __getnewargs__(self): return (self.family, tuple(self)) cdef void gevent_ares_host_callback(void *arg, int status, int timeouts, hostent* host): cdef channel channel cdef object callback channel, callback = arg Py_DECREF(arg) cdef object host_result try: if status or not host: callback(result(None, gaierror(status, strerror(status)))) else: try: host_result = ares_host_result(host.h_addrtype, (host.h_name, parse_h_aliases(host), parse_h_addr_list(host))) except: callback(result(None, sys.exc_info()[1])) else: callback(result(host_result)) except: channel.loop.handle_error(callback, *sys.exc_info()) cdef void gevent_ares_generic_callback(void *arg, int status, int timeouts, unsigned char *abuf, int alen): cdef channel channel cdef object callback channel, callback = arg Py_DECREF(arg) cdef bytes generic_result try: if status or not abuf: callback(result(None, gaierror(status, strerror(status)))) else: try: generic_result = abuf[:alen] except: callback(result(None, sys.exc_info()[1])) else: callback(result(generic_result)) except: channel.loop.handle_error(callback, *sys.exc_info()) cdef void gevent_ares_nameinfo_callback(void *arg, int status, int timeouts, char *c_node, char *c_service): cdef channel channel cdef object callback channel, callback = arg Py_DECREF(arg) cdef object node cdef object service try: if status: callback(result(None, gaierror(status, strerror(status)))) else: if c_node: node = PyBytes_FromString(c_node) else: node = None if c_service: service = PyBytes_FromString(c_service) else: service = None callback(result((node, service))) except: channel.loop.handle_error(callback, *sys.exc_info()) cdef public class channel [object PyGeventAresChannelObject, type PyGeventAresChannel_Type]: cdef public object loop cdef ares_channeldata* channel cdef public dict _watchers cdef public object _timer def __init__(self, object loop, flags=None, timeout=None, tries=None, ndots=None, udp_port=None, tcp_port=None, servers=None): cdef ares_channeldata* channel = NULL cdef cares.ares_options options memset(&options, 0, sizeof(cares.ares_options)) cdef int optmask = cares.ARES_OPT_SOCK_STATE_CB options.sock_state_cb = gevent_sock_state_callback options.sock_state_cb_data = self if flags is not None: options.flags = int(flags) optmask |= cares.ARES_OPT_FLAGS if timeout is not None: options.timeout = int(float(timeout) * 1000) optmask |= cares.ARES_OPT_TIMEOUTMS if tries is not None: options.tries = int(tries) optmask |= cares.ARES_OPT_TRIES if ndots is not None: options.ndots = int(ndots) optmask |= cares.ARES_OPT_NDOTS if udp_port is not None: options.udp_port = int(udp_port) optmask |= cares.ARES_OPT_UDP_PORT if tcp_port is not None: options.tcp_port = int(tcp_port) optmask |= cares.ARES_OPT_TCP_PORT cdef int result = cares.ares_library_init(cares.ARES_LIB_INIT_ALL) # ARES_LIB_INIT_WIN32 -DUSE_WINSOCK? if result: raise gaierror(result, strerror(result)) result = cares.ares_init_options(&channel, &options, optmask) if result: raise gaierror(result, strerror(result)) self._timer = loop.timer(TIMEOUT, TIMEOUT) self._watchers = {} self.channel = channel try: if servers is not None: self.set_servers(servers) self.loop = loop except: self.destroy() raise def __repr__(self): args = (self.__class__.__name__, id(self), self._timer, len(self._watchers)) return '<%s at 0x%x _timer=%r _watchers[%s]>' % args def destroy(self): if self.channel: # XXX ares_library_cleanup? cares.ares_destroy(self.channel) self.channel = NULL self._watchers.clear() self._timer.stop() self.loop = None def __dealloc__(self): if self.channel: # XXX ares_library_cleanup? cares.ares_destroy(self.channel) self.channel = NULL def set_servers(self, servers=None): if not self.channel: raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed') if not servers: servers = [] if isinstance(servers, basestring): servers = servers.split(',') cdef int length = len(servers) cdef int result, index cdef char* string cdef cares.ares_addr_node* c_servers if length <= 0: result = cares.ares_set_servers(self.channel, NULL) else: c_servers = malloc(sizeof(cares.ares_addr_node) * length) if not c_servers: raise MemoryError try: index = 0 for server in servers: string = server if cares.ares_inet_pton(AF_INET, string, &c_servers[index].addr) > 0: c_servers[index].family = AF_INET elif cares.ares_inet_pton(AF_INET6, string, &c_servers[index].addr) > 0: c_servers[index].family = AF_INET6 else: raise InvalidIP(repr(string)) c_servers[index].next = &c_servers[index] + 1 index += 1 if index >= length: break c_servers[length - 1].next = NULL index = cares.ares_set_servers(self.channel, c_servers) if index: raise ValueError(strerror(index)) finally: free(c_servers) def num_servers(self): if not self.channel: return None cdef cares.ares_addr_node *servers cdef cares.ares_addr_node *curserver cdef int result cdef int num = 0 result = cares.ares_get_servers(self.channel, &servers) if result != cares.ARES_SUCCESS: raise RuntimeError('Error retrieving channel servers: %d' % result) curserver = servers while curserver != NULL: num += 1 curserver = curserver.next cares.ares_free_data(servers) return num # this crashes c-ares #def cancel(self): # cares.ares_cancel(self.channel) cdef _sock_state_callback(self, int socket, int read, int write): if not self.channel: return cdef object watcher = self._watchers.get(socket) cdef int events = 0 if read: events |= EV_READ if write: events |= EV_WRITE if watcher is None: if not events: return watcher = self.loop.io(socket, events) self._watchers[socket] = watcher elif events: if watcher.events == events: return watcher.stop() watcher.events = events else: watcher.stop() self._watchers.pop(socket, None) if not self._watchers: self._timer.stop() return watcher.start(self._process_fd, watcher, pass_events=True) self._timer.again(self._on_timer) def _on_timer(self): cares.ares_process_fd(self.channel, cares.ARES_SOCKET_BAD, cares.ARES_SOCKET_BAD) def _process_fd(self, int events, object watcher): if not self.channel: return cdef int read_fd = watcher.fd cdef int write_fd = read_fd if not (events & EV_READ): read_fd = cares.ARES_SOCKET_BAD if not (events & EV_WRITE): write_fd = cares.ARES_SOCKET_BAD cares.ares_process_fd(self.channel, read_fd, write_fd) def gethostbyname(self, object callback, char* name, int family=AF_INET): if not self.channel: raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed') # note that for file lookups still AF_INET can be returned for AF_INET6 request cdef object arg = (self, callback) Py_INCREF(arg) cares.ares_gethostbyname(self.channel, name, family, gevent_ares_host_callback, arg) def query(self, object callback, char *name, int dnsclass, int type_): if not self.channel: raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed') cdef object arg = (self, callback) Py_INCREF(arg) cares.ares_query( self.channel, name, dnsclass, type_, gevent_ares_generic_callback, arg ) def gethostbyaddr(self, object callback, char* addr): if not self.channel: raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed') # will guess the family cdef char addr_packed[16] cdef int family cdef int length if cares.ares_inet_pton(AF_INET, addr, addr_packed) > 0: family = AF_INET length = 4 elif cares.ares_inet_pton(AF_INET6, addr, addr_packed) > 0: family = AF_INET6 length = 16 else: raise InvalidIP(repr(addr)) cdef object arg = (self, callback) Py_INCREF(arg) cares.ares_gethostbyaddr(self.channel, addr_packed, length, family, gevent_ares_host_callback, arg) def parse_txt_reply(self, bytes reply): cdef cares.ares_txt_reply *txtout cdef cares.ares_txt_reply *curtxt cdef int ok cdef list result = [] ok = cares.ares_parse_txt_reply(reply, len(reply), &txtout) if ok != cares.ARES_SUCCESS: raise RuntimeError('Error parsing txt reply: %d' % ok) curtxt = txtout while curtxt: result.append(curtxt.txt[:curtxt.length]) curtxt = txtout.next return result cpdef _getnameinfo(self, object callback, tuple sockaddr, int flags): if not self.channel: raise gaierror(cares.ARES_EDESTRUCTION, 'this ares channel has been destroyed') cdef char* hostp = NULL cdef int port = 0 cdef int flowinfo = 0 cdef int scope_id = 0 cdef sockaddr_in6 sa6 if not PyTuple_Check(sockaddr): raise TypeError('expected a tuple, got %r' % (sockaddr, )) PyArg_ParseTuple(sockaddr, "si|ii", &hostp, &port, &flowinfo, &scope_id) if port < 0 or port > 65535: raise gaierror(-8, 'Invalid value for port: %r' % port) cdef int length = gevent_make_sockaddr(hostp, port, flowinfo, scope_id, &sa6) if length <= 0: raise InvalidIP(repr(hostp)) cdef object arg = (self, callback) Py_INCREF(arg) cdef sockaddr_t* x = &sa6 cares.ares_getnameinfo(self.channel, x, length, flags, gevent_ares_nameinfo_callback, arg) def getnameinfo(self, object callback, tuple sockaddr, int flags): return self._getnameinfo(callback, sockaddr, _convert_cares_flags(flags)) ================================================ FILE: minemeld/packages/gdns/cares.pxd ================================================ cdef extern from "ares.h": struct ares_options: int flags void* sock_state_cb void* sock_state_cb_data int timeout int tries int ndots unsigned short udp_port unsigned short tcp_port char **domains int ndomains char* lookups int ARES_OPT_FLAGS int ARES_OPT_SOCK_STATE_CB int ARES_OPT_TIMEOUTMS int ARES_OPT_TRIES int ARES_OPT_NDOTS int ARES_OPT_TCP_PORT int ARES_OPT_UDP_PORT int ARES_OPT_SERVERS int ARES_OPT_DOMAINS int ARES_OPT_LOOKUPS int ARES_FLAG_USEVC int ARES_FLAG_PRIMARY int ARES_FLAG_IGNTC int ARES_FLAG_NORECURSE int ARES_FLAG_STAYOPEN int ARES_FLAG_NOSEARCH int ARES_FLAG_NOALIASES int ARES_FLAG_NOCHECKRESP int ARES_LIB_INIT_ALL int ARES_SOCKET_BAD int ARES_SUCCESS int ARES_ENODATA int ARES_EFORMERR int ARES_ESERVFAIL int ARES_ENOTFOUND int ARES_ENOTIMP int ARES_EREFUSED int ARES_EBADQUERY int ARES_EBADNAME int ARES_EBADFAMILY int ARES_EBADRESP int ARES_ECONNREFUSED int ARES_ETIMEOUT int ARES_EOF int ARES_EFILE int ARES_ENOMEM int ARES_EDESTRUCTION int ARES_EBADSTR int ARES_EBADFLAGS int ARES_ENONAME int ARES_EBADHINTS int ARES_ENOTINITIALIZED int ARES_ELOADIPHLPAPI int ARES_EADDRGETNETWORKPARAMS int ARES_ECANCELLED int ARES_NI_NOFQDN int ARES_NI_NUMERICHOST int ARES_NI_NAMEREQD int ARES_NI_NUMERICSERV int ARES_NI_DGRAM int ARES_NI_TCP int ARES_NI_UDP int ARES_NI_SCTP int ARES_NI_DCCP int ARES_NI_NUMERICSCOPE int ARES_NI_LOOKUPHOST int ARES_NI_LOOKUPSERVICE int ares_library_init(int flags) void ares_library_cleanup() int ares_init_options(void *channelptr, ares_options *options, int) int ares_init(void *channelptr) void ares_destroy(void *channelptr) void ares_gethostbyname(void* channel, char *name, int family, void* callback, void *arg) void ares_gethostbyaddr(void* channel, void *addr, int addrlen, int family, void* callback, void *arg) void ares_process_fd(void* channel, int read_fd, int write_fd) char* ares_strerror(int code) void ares_cancel(void* channel) void ares_getnameinfo(void* channel, void* sa, int salen, int flags, void* callback, void *arg) void ares_query(void* channel, const char *name, int dnsclass, int type, void* callback, void *arg) struct in_addr: pass struct ares_in6_addr: pass struct addr_union: in_addr addr4 ares_in6_addr addr6 struct ares_addr_node: ares_addr_node *next int family addr_union addr int ares_set_servers(void* channel, ares_addr_node *servers) int ares_get_servers(void* channel, ares_addr_node **servers) struct ares_txt_reply: ares_txt_reply *next unsigned char *txt int length int ares_parse_txt_reply(const unsigned char* abuf, int alen, ares_txt_reply **txt_out) void ares_free_data(void *dataptr) cdef extern from "cares_pton.h": int ares_inet_pton(int af, char *src, void *dst) ================================================ FILE: minemeld/packages/gdns/cares_ntop.h ================================================ #include #define ares_inet_ntop(w,x,y,z) inet_ntop(w,x,y,z) ================================================ FILE: minemeld/packages/gdns/cares_pton.h ================================================ #include #define ares_inet_pton(x,y,z) inet_pton(x,y,z) #define ares_inet_net_pton(w,x,y,z) inet_net_pton(w,x,y,z) ================================================ FILE: minemeld/packages/gdns/dig.py ================================================ # Based on resolver_ares.py from gevent # Copyright (c) 2011 Denis Bilenko. See LICENSE for details. from __future__ import absolute_import import os from _socket import gaierror from gevent.hub import Waiter, get_hub from minemeld.packages.gdns._ares import channel class Dig(object): ares_class = channel # from arpa/nameser.h NS_C_INVALID = 0 # Cookie NS_C_IN = 1 # Internet NS_C_2 = 2 # unallocated/unsupported NS_C_CHAOS = 3 # MIT Chaos-net NS_C_HS = 4 # MIT Hesiod NS_C_NONE = 254 # prereq. sections in update requests NS_C_ANY = 255 # Wildcard match NS_C_MAX = 65536 NS_T_INVALID = 0 # Cookie NS_T_A = 1 # Host address NS_T_NS = 2 # Authoritative server NS_T_MD = 3 # Mail destination NS_T_MF = 4 # Mail forwarder NS_T_CNAME = 5 # Canonical name NS_T_SOA = 6 # Start of authority zone NS_T_MB = 7 # Mailbox domain name NS_T_MG = 8 # Mail group member NS_T_MR = 9 # Mail rename name NS_T_NULL = 10 # Null resource record NS_T_WKS = 11 # Well known service NS_T_PTR = 12 # Domain name pointer NS_T_HINFO = 13 # Host information NS_T_MINFO = 14 # Mailbox information NS_T_MX = 15 # Mail routing information NS_T_TXT = 16 # Text strings NS_T_RP = 17 # Responsible person NS_T_AFSDB = 18 # AFS cell database NS_T_X25 = 19 # X_25 calling address NS_T_ISDN = 20 # ISDN calling address NS_T_RT = 21 # Router NS_T_NSAP = 22 # NSAP address NS_T_NSAP_PTR = 23 # Reverse NSAP lookup (deprecated) NS_T_SIG = 24 # Security signature NS_T_KEY = 25 # Security key NS_T_PX = 26 # X.400 mail mapping NS_T_GPOS = 27 # Geographical position (withdrawn) NS_T_AAAA = 28 # Ip6 Address NS_T_LOC = 29 # Location Information NS_T_NXT = 30 # Next domain (security) NS_T_EID = 31 # Endpoint identifier NS_T_NIMLOC = 32 # Nimrod Locator NS_T_SRV = 33 # Server Selection NS_T_ATMA = 34 # ATM Address NS_T_NAPTR = 35 # Naming Authority PoinTeR NS_T_KX = 36 # Key Exchange NS_T_CERT = 37 # Certification record NS_T_A6 = 38 # IPv6 address (deprecated, use NS_T_AAAA) NS_T_DNAME = 39 # Non-terminal DNAME (for IPv6) NS_T_SINK = 40 # Kitchen sink (experimentatl) NS_T_OPT = 41 # EDNS0 option (meta-RR) NS_T_APL = 42 # Address prefix list (RFC3123) NS_T_TKEY = 249 # Transaction key NS_T_TSIG = 250 # Transaction signature NS_T_IXFR = 251 # Incremental zone transfer NS_T_AXFR = 252 # Transfer zone of authority NS_T_MAILB = 253 # Transfer mailbox records NS_T_MAILA = 254 # Transfer mail agent records NS_T_ANY = 255 # Wildcard match NS_T_ZXFR = 256 # BIND-specific, nonstandard NS_T_MAX = 65536 def __init__(self, hub=None, **kwargs): if hub is None: hub = get_hub() self.hub = hub self.ares = self.ares_class(hub.loop, **kwargs) self.pid = os.getpid() self.params = kwargs self.fork_watcher = hub.loop.fork(ref=False) self.fork_watcher.start(self._on_fork) def __repr__(self): return ( '' % (id(self), self.ares) ) def _on_fork(self): pid = os.getpid() if pid != self.pid: self.hub.loop.run_callback(self.ares.destroy) self.ares = self.ares_class(self.hub.loop, **self.params) self.pid = pid def close(self): if self.ares is not None: self.hub.loop.run_callback(self.ares.destroy) self.ares = None self.fork_watcher.stop() def query(self, name, dnsclass, type_): if isinstance(name, unicode): name = name.encode('ascii') elif not isinstance(name, str): raise TypeError('Expected string, not %s' % type(name).__name__) while True: ares = self.ares try: waiter = Waiter(self.hub) ares.query(waiter, name, dnsclass, type_) result = waiter.get() return result except gaierror: if ares is self.ares: raise def parse_txt_reply(self, reply): return self.ares.parse_txt_reply(reply) ================================================ FILE: minemeld/packages/gdns/dnshelper.c ================================================ /* Copyright (c) 2011 Denis Bilenko. See LICENSE for details. */ #include "Python.h" #ifdef HAVE_NETDB_H #include #endif #include "ares.h" #include "cares_ntop.h" #include "cares_pton.h" #if PY_VERSION_HEX < 0x02060000 #define PyBytes_FromString PyString_FromString #endif static PyObject* _socket_error = 0; static PyObject* get_socket_object(PyObject** pobject, const char* name) { if (!*pobject) { PyObject* _socket; _socket = PyImport_ImportModule("_socket"); if (_socket) { *pobject = PyObject_GetAttrString(_socket, name); if (!*pobject) { PyErr_WriteUnraisable(Py_None); } Py_DECREF(_socket); } else { PyErr_WriteUnraisable(Py_None); } if (!*pobject) { *pobject = PyExc_IOError; } } return *pobject; } static int gevent_append_addr(PyObject* list, int family, void* src, char* tmpbuf, size_t tmpsize) { int status = -1; PyObject* tmp; if (ares_inet_ntop(family, src, tmpbuf, tmpsize)) { tmp = PyBytes_FromString(tmpbuf); if (tmp) { status = PyList_Append(list, tmp); Py_DECREF(tmp); } } return status; } static PyObject* parse_h_aliases(struct hostent *h) { char **pch; PyObject *result = NULL; PyObject *tmp; result = PyList_New(0); if (result && h->h_aliases) { for (pch = h->h_aliases; *pch != NULL; pch++) { if (*pch != h->h_name && strcmp(*pch, h->h_name)) { int status; tmp = PyBytes_FromString(*pch); if (tmp == NULL) { break; } status = PyList_Append(result, tmp); Py_DECREF(tmp); if (status) { break; } } } } return result; } static PyObject * parse_h_addr_list(struct hostent *h) { char **pch; PyObject *result = NULL; result = PyList_New(0); if (result) { switch (h->h_addrtype) { case AF_INET: { char tmpbuf[sizeof "255.255.255.255"]; for (pch = h->h_addr_list; *pch != NULL; pch++) { if (gevent_append_addr(result, AF_INET, *pch, tmpbuf, sizeof(tmpbuf))) { break; } } break; } case AF_INET6: { char tmpbuf[sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")]; for (pch = h->h_addr_list; *pch != NULL; pch++) { if (gevent_append_addr(result, AF_INET6, *pch, tmpbuf, sizeof(tmpbuf))) { break; } } break; } default: PyErr_SetString(get_socket_object(&_socket_error, "error"), "unsupported address family"); Py_DECREF(result); result = NULL; } } return result; } static int gevent_make_sockaddr(char* hostp, int port, int flowinfo, int scope_id, struct sockaddr_in6* sa6) { if ( ares_inet_pton(AF_INET, hostp, &((struct sockaddr_in*)sa6)->sin_addr.s_addr) > 0 ) { ((struct sockaddr_in*)sa6)->sin_family = AF_INET; ((struct sockaddr_in*)sa6)->sin_port = htons(port); return sizeof(struct sockaddr_in); } else if ( ares_inet_pton(AF_INET6, hostp, &sa6->sin6_addr.s6_addr) > 0 ) { sa6->sin6_family = AF_INET6; sa6->sin6_port = htons(port); sa6->sin6_flowinfo = flowinfo; sa6->sin6_scope_id = scope_id; return sizeof(struct sockaddr_in6); } return -1; } ================================================ FILE: minemeld/packages/gevent_openssl/COPYING ================================================ Copyright (c) 2015, Menno Finlay-Smits All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Menno Finlay-Smits nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL MENNO SMITS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ================================================ FILE: minemeld/packages/gevent_openssl/SSL.py ================================================ """gevent_openssl.SSL - gevent compatibility with OpenSSL.SSL (pyOpenSSL) """ import logging import OpenSSL.SSL from gevent.socket import wait_read, wait_write _real_connection = OpenSSL.SSL.Connection LOG = logging.getLogger(__name__) class Connection(object): """OpenSSL Connection wrapper """ _reverse_mapping = _real_connection._reverse_mapping def __init__(self, context, sock): self._context = context self._sock = sock self._connection = _real_connection(context, sock) def __getattr__(self, attr): return getattr(self._connection, attr) def __iowait(self, io_func, *args, **kwargs): fd = self._sock.fileno() timeout = self._sock.gettimeout() while True: try: return io_func(*args, **kwargs) except (OpenSSL.SSL.WantReadError, OpenSSL.SSL.WantX509LookupError): wait_read(fd, timeout=timeout) except OpenSSL.SSL.WantWriteError: wait_write(fd, timeout=timeout) def accept(self): sock, addr = self._sock.accept() return Connection(self._context, sock), addr def do_handshake(self): # even if some sites are super sensible # to handshake timeouts (to avoid DDoS), # we have to make handshake not blocking to avoid issues # with firewalls or other middle boxes dropping the connection return self.__iowait(self._connection.do_handshake) def connect(self, *args, **kwargs): return self.__iowait(self._connection.connect, *args, **kwargs) def send(self, data, flags=0): return self.__send(self._connection.send, data, flags) def sendall(self, data, flags=0): # see https://github.com/mjs/gevent_openssl/issues/12 # Note: all of the types supported by OpenSSL's Connection.sendall, # basestring, memoryview, and buffer, support len(...) and slicing, # so they are safe to use here. while len(data) > 0: res = self.send(data, flags) data = data[res:] def __send(self, send_method, data, flags=0): try: return self.__iowait(send_method, data, flags) except OpenSSL.SSL.SysCallError as e: if e[0] == -1 and not data: # errors when writing empty strings are expected and can be # ignored return 0 raise def recv(self, bufsiz, flags=0): pending = self._connection.pending() if pending: return self._connection.recv(min(pending, bufsiz)) try: return self.__iowait(self._connection.recv, bufsiz, flags) except OpenSSL.SSL.ZeroReturnError: return '' except OpenSSL.SSL.SysCallError as e: if e[0] == -1 and 'Unexpected EOF' in e[1]: # errors when reading empty strings are expected and can be # ignored return '' raise def shutdown(self): return self.__iowait(self._connection.shutdown) ================================================ FILE: minemeld/packages/gevent_openssl/__init__.py ================================================ """gevent_openssl - gevent compatibility with pyOpenSSL. Usage ----- Instead of importing OpenSSL directly, do so in the following manner: .. import gevent_openssl as OpenSSL or .. import gevent_openssl; gevent_openssl.monkey_patch() Any calls that would have blocked the current thread will now only block the current green thread. This compatibility is accomplished by ensuring the nonblocking flag is set before any blocking operation and the OpenSSL file descriptor is polled internally to trigger needed events. """ from . import SSL as MySSL from OpenSSL import * def monkey_patch(): """ Monkey patches `OpenSSL.SSL.Connection` """ mod = __import__('OpenSSL').SSL mod.Connection = MySSL.Connection ================================================ FILE: minemeld/packages/ise/__init__.py ================================================ import logging DEBUG1 = logging.DEBUG DEBUG2 = DEBUG1 - 1 DEBUG3 = DEBUG2 - 1 logging.addLevelName(DEBUG2, 'DEBUG2') logging.addLevelName(DEBUG3, 'DEBUG3') ================================================ FILE: minemeld/packages/ise/ers.py ================================================ # # Copyright (c) 2016 Palo Alto Networks, Inc. # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # '''Interface to the Cisco ISE ERS (External RESTful Services) API The interface is specific to requirements for creating SGT mappings on PAN-OS. See ERS SDK page at: https://ise:9060/ers/sdk ''' from collections import defaultdict import inspect import logging import pprint import xml.etree.ElementTree as etree from . import DEBUG1, DEBUG2, DEBUG3 try: import requests except ImportError: raise ValueError('Install requests library: ' 'http://docs.python-requests.org/') # https://github.com/shazow/urllib3/issues/655 # Requests treats None as forever _None = object() class IseErsRequest: def __init__(self, name=None): self.name = name # python-requests self.response = None self.status_code = None self.reason = None self.headers = None self.encoding = None self.content = None self.text = None # self.xml_root = None self.obj = None def raise_for_status(self): if self.response is not None: try: self.response.raise_for_status() except requests.exceptions.HTTPError as e: raise IseErsError(e) class IseErsError(Exception): pass class IseErs: def __init__(self, hostname=None, username=None, password=None, verify=None, timeout=_None): if hostname is None: raise IseErsError('no hostname') if username is None: raise IseErsError('no username') if password is None: raise IseErsError('no password') self._log = logging.getLogger(__name__).log self.verify = verify if self.verify is False: requests.packages.urllib3.disable_warnings() self.timeout = timeout self._log(DEBUG2, 'timeout: %s', repr(timeout)) self.uri = 'https://' + hostname + ':9060' self.auth = requests.auth.HTTPBasicAuth(username, password) def _request(self, uri, headers): kwargs = {} if self.verify is not None: kwargs['verify'] = self.verify if self.timeout is not _None: kwargs['timeout'] = self.timeout try: r = requests.get(url=uri, headers=headers, auth=self.auth, **kwargs) except (requests.exceptions.RequestException, ValueError) as e: raise IseErsError(e) return r def _set_attributes(self, r): x = IseErsRequest(inspect.stack()[1][3]) # http://docs.python-requests.org/en/master/api/#requests.Response x.response = r x.status_code = r.status_code x.reason = r.reason x.headers = r.headers x.encoding = r.encoding self._log(DEBUG2, r.encoding) self._log(DEBUG2, r.request.headers) # XXX authorization header self._log(DEBUG2, r.headers) x.content = r.content # bytes x.text = r.text # Unicode self._log(DEBUG3, r.text) try: x.xml_root = etree.fromstring(r.content) except etree.ParseError as e: self._log(DEBUG1, 'ElementTree.fromstring ParseError: %s', e) if x.xml_root is not None: self._log(DEBUG1, 'root tag: %s', x.xml_root.tag) if x.xml_root.tag == '{ers.ise.cisco.com}ersResponse': message = x.xml_root.find('messages/message') if message is not None: x.obj = {} x.obj['error'] = {} for k in ['type', 'code']: if k in message.attrib: x.obj['error'][k] = message.attrib[k] title = message.findall('title') if title is not None: x.obj['error']['title'] = [] for elem in title: x.obj['error']['title'].append(elem.text) self._log(DEBUG2, x.obj) return x def sgts_ips_map(self): r = self.sgt() r.raise_for_status() if 'sgt' not in r.obj: raise IseErsError('no response data for sgt()') _sgt = r.obj['sgt'] r = self.sgmapping() r.raise_for_status() if 'sgmapping' not in r.obj: raise IseErsError('no response data for sgmapping()') _sgmapping = r.obj['sgmapping'] _sgt_name_desc_map = {} _sgt_id_name_map = {} _sgts_ips_map = {} for x in _sgt: _sgt_name_desc_map[x['name']] = x['description'] _sgt_id_name_map[x['id']] = x['name'] _sgts_ips_map[x['name']] = [] self._log(DEBUG2, pprint.pformat(_sgt_name_desc_map)) self._log(DEBUG2, pprint.pformat(_sgt_id_name_map)) for x in _sgmapping: r = self.sgmapping(id=x['id']) r.raise_for_status() if 'sgmapping_id' not in r.obj: raise IseErsError('no response data for sgmapping(%s)' % x['id']) _sgmapping_id = r.obj['sgmapping_id'] if 'hostIp' in _sgmapping_id: if _sgmapping_id['sgt'] not in _sgt_id_name_map: pass # XXX refresh _sgt_id_name_map else: _sgts_ips_map[_sgt_id_name_map[_sgmapping_id['sgt']]].\ append(_sgmapping_id['hostIp']) self._log(DEBUG2, pprint.pformat(_sgts_ips_map)) d = defaultdict(list) [d[v].append(k) for k in _sgts_ips_map for v in _sgts_ips_map[k]] self.sgt_name_desc_map = _sgt_name_desc_map self.sgts_ips_map = _sgts_ips_map self.ips_sgts_map = dict(d) return self.sgt_name_desc_map, self.sgts_ips_map, self.ips_sgts_map def sgt(self, id=None): '''Security Groups: Get-By-Id Get-All ''' headers = { 'accept': 'application/vnd.com.cisco.ise.trustsec.sgt.1.0+xml' } path = '/ers/config/sgt' if id is not None: path += '/' + str(id) uri = self.uri + path r = self._request(uri=uri, headers=headers) x = self._set_attributes(r) if x.xml_root is not None: rk = 'sgt' # root key if x.xml_root.tag == '{ers.ise.cisco.com}searchResult': x.obj = {} x.obj[rk] = [] for elem in x.xml_root.findall('resources/resource'): for k in ['name', 'id', 'description']: if k not in elem.attrib: raise IseErsError('missing attribute \"%s\": %s' % (k, elem.attrib)) x.obj[rk].append(elem.attrib) elif x.xml_root.tag == '{trustsec.ers.ise.cisco.com}sgt': rk = 'sgt_id' x.obj = {} x.obj[rk] = {} for k in ['id', 'name', 'description']: if k not in x.xml_root.attrib: raise IseErsError('missing attribute \"%s\": %s' % (k, elem.attrib)) else: x.obj[rk][k] = x.xml_root.attrib[k] for elem in x.xml_root.findall('*'): x.obj[rk][elem.tag] = elem.text self._log(DEBUG2, pprint.pformat(x.obj)) return x def sgmapping(self, id=None): '''IP To SGT Mapping: Get-All Get-By-Id ''' headers = { 'accept': 'application/vnd.com.cisco.ise.trustsec.sgmapping.1.0+xml' } path = '/ers/config/sgmapping' if id is not None: path += '/' + str(id) uri = self.uri + path r = self._request(uri=uri, headers=headers) x = self._set_attributes(r) if x.xml_root is not None: if x.xml_root.tag == '{ers.ise.cisco.com}searchResult': rk = 'sgmapping' x.obj = {} x.obj[rk] = [] for elem in x.xml_root.findall('resources/resource'): for k in ['name', 'id']: if k not in elem.attrib: raise IseErsError('missing attribute \"%s\": %s' % (k, elem.attrib)) x.obj[rk].append(elem.attrib) elif x.xml_root.tag == '{trustsec.ers.ise.cisco.com}sgMapping': rk = 'sgmapping_id' x.obj = {} x.obj[rk] = {} for elem in x.xml_root.findall('*'): x.obj[rk][elem.tag] = elem.text self._log(DEBUG2, pprint.pformat(x.obj)) return x ================================================ FILE: minemeld/packages/panforest/__init__.py ================================================ from . import forest PanForest = forest.PanForest PanForestError = forest.PanForestError ================================================ FILE: minemeld/packages/panforest/forest.py ================================================ # # Copyright (c) 2015 Palo Alto Networks, Inc. # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # import pprint import random import sys import time import logging import pan.config _NLOGS = 100 LOG = logging.getLogger(__name__) class PanForestError(Exception): def __init__(self, msg): self.msg = msg def __str__(self): if self.msg is None: return '' return self.msg class PanForest(object): def __init__(self, xapi=None, log_type=None, filter=None, nlogs=None, format=None): self.xapi = xapi self.log_type = log_type self.filter = filter self.nlogs = _NLOGS if nlogs is None else nlogs if format not in ['xml', 'python', None]: raise PanForestError('Invalid format: %s' % format) self.format = 'python' if format is None else format def __iter__(self): return self.follow() def sleep(self, t): time.sleep(t) def follow(self): """\ generator function to return log entries matching optional filter """ # filter time < now so we start at logs in current second now = int(time.time()) - 1 filter = '(receive_time leq %d)' % now LOG.debug('filter: %s', filter) _, obj = self._log_get(nlogs=1, filter=filter) if not obj: raise PanForestError("Can't get last log") count = self._log_count(obj) if count != 1: seqno = 0 else: seqno = self._log_seqno(obj) LOG.debug('starting seqno: %s', seqno) filter = self._filter(seqno) nlogs = self.nlogs skip = 0 sleeper = self._sleeper() while True: LOG.debug('skip: %d nlogs: %d filter: "%s"' % (skip, nlogs, filter)) elem, obj = self._log_get(nlogs=nlogs, filter=filter, skip=skip) if not obj: raise PanForestError("Can't get log chunk") count = self._log_count(obj) if count == 0: skip = 0 filter = self._filter(seqno) try: wait = next(sleeper) except StopIteration: pass x = random.uniform(0, 0.5) LOG.debug('sleep: %d %d', wait, x) self.sleep(wait+x) continue elif count == nlogs: sleeper = self._sleeper() skip += nlogs t = self._log_seqno(obj) if t > seqno: seqno = t # don't update filter self.sleep(0) elif count < nlogs: sleeper = self._sleeper() t = self._log_seqno(obj) if t > seqno: seqno = t filter = self._filter(seqno) self.sleep(0) else: assert False, 'NOTREACHED' if self.format == 'python': for entry in self._log_entry(obj['logs'], 'entry'): yield entry elif self.format == 'xml': nodes = elem.findall('.') for node in nodes: yield node def tail(self, lines): elem, obj = self._log_get(nlogs=lines, filter=self.filter) if not obj: raise PanForestError("Can't get log") count = self._log_count(obj) if count == 0: return None if self.format == 'python': return obj elif self.format == 'xml': return elem def _log_get(self, nlogs=None, skip=None, filter=None): try: self.xapi.log(log_type=self.log_type, skip=skip, nlogs=nlogs, filter=filter) except pan.xapi.PanXapiError as e: raise PanForestError('pan.xapi.PanXapi: %s' % e) path = './result/log/logs' elem = self.xapi.element_root.find(path) if elem is None: raise PanForestError('No %s in element_root' % path) obj = self._xml_python(elem) LOG.debug(pprint.pformat(obj)) return elem, obj def _sleeper(self): """return iterator of seconds to sleep until log match""" try: xrange(1) except NameError: _range = range else: _range = xrange x = _range(1, 60, 3) return iter(x) @staticmethod def _xml_python(elem, path=None): try: conf = pan.config.PanConfig(config=elem) except pan.config.PanConfigError as e: raise PanForestError('pan.config.PanConfigError: %s' % e) obj = conf.python(path) return obj def _log_count(self, obj): if not ('logs' in obj and 'count' in obj['logs']): raise PanForestError('logs count not found') try: count = int(obj['logs']['count']) except ValueError: raise PanForestError('count not numeric: %s' % obj['logs']['count']) LOG.debug('count: %d' % count) return count def _log_seqno(self, obj): x = self._log_entry(obj['logs']['entry'][0], 'seqno') try: n = int(x) except ValueError: raise PanForestError('seqno not numeric: %s' % x) return n @staticmethod def _log_entry(obj, key): if key not in obj: raise PanForestError('key not in entry: %s' % key) return obj[key] def _filter(self, seqno): s = 'not (seqno leq %d)' % seqno if self.filter: s = self.filter + ' and ' + s return s ================================================ FILE: minemeld/run/__init__.py ================================================ ================================================ FILE: minemeld/run/cacert_merge.py ================================================ # Copyright 2015-2017 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import argparse import sys import os import os.path import yaml try: from ssl import create_default_context except ImportError: def create_default_context(cafile, cadata): print('WARNING: old python version (< 2.7.9) - certificate verification not performed') try: import certifi CERTIFI_WHERE = certifi.where() except ImportError: # XXX Error? CERTIFI_WHERE = None LOG = logging.getLogger(__name__) def main(): logging.basicConfig( level=logging.DEBUG, format="%(asctime)s (%(process)d)%(module)s.%(funcName)s" " %(levelname)s: %(message)s", datefmt="%Y-%m-%dT%H:%M:%S" ) parser = argparse.ArgumentParser(usage='%(prog)s [options] [cafile ...]') parser.add_argument('--no-merge-certifi', action='store_const', const=True, help='do not merge certifi CA bundle ' '(default: merge "%s")' % CERTIFI_WHERE) parser.add_argument('--config', help='configuration file path (default: no config)') parser.add_argument('--dst', required=True, help='destination CA bundle path') parser.add_argument('cafile', nargs='*', help='local CA bundle file(s) ' '(default: stdin)') args = parser.parse_args() try: certs = open(args.dst, 'w') except IOError as e: LOG.error('open: %s: %s' % (args.dst, e)) return 1 config = { 'no_merge_certifi': False } if args.config: with open(args.config, 'r') as f: loaded_config = yaml.safe_load(f) if loaded_config is not None: config.update(loaded_config) config.update({k: v for k, v in vars(args).iteritems() if v is not None}) LOG.info('config: {}'.format(config)) if not config['no_merge_certifi'] and CERTIFI_WHERE: try: with open(CERTIFI_WHERE) as f: buf = f.read() except IOError as e: LOG.error('%s: %s' % (CERTIFI_WHERE, e)) return 1 try: certs.write(buf) except IOError as e: LOG.error('%s: %s' % (args.dst, e)) return 1 if args.cafile: for x in args.cafile: files = [x] if os.path.isdir(x): files = [os.path.join(x, e) for e in os.listdir(x)] for fname in files: verify_cafile(cafile=fname) try: with open(fname) as f: buf = f.read() except IOError as e: LOG.error('%s: %s' % (fname, e)) return 1 try: certs.write(buf) except IOError as e: LOG.error('%s: %s' % (args.dst, e)) return 1 else: x = sys.stdin.read() try: x = unicode(x) except NameError: # 3.x pass verify_cafile(cadata=x) try: certs.write(x) except IOError as e: LOG.error('%s: %s' % (args.dst, e)) return 1 certs.close() verify_cafile(cafile=args.dst) return 0 def verify_cafile(cafile=None, cadata=None): try: create_default_context(cafile=cafile, cadata=cadata) except IOError as e: if cafile: LOG.error('Invalid cafile %s: %s' % (cafile, e)) else: LOG.error('Invalid cadata: %s' % e) sys.exit(1) ================================================ FILE: minemeld/run/config.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import print_function import sys import time import os import os.path import logging import shutil import re import json import multiprocessing import functools from collections import namedtuple import yaml import gevent.core import minemeld.loader __all__ = ['load_config', 'validate_config', 'resolve_prototypes'] # disables construction of timestamp objects yaml.SafeLoader.add_constructor( u'tag:yaml.org,2002:timestamp', yaml.SafeLoader.construct_yaml_str ) LOG = logging.getLogger(__name__) COMMITTED_CONFIG = 'committed-config.yml' RUNNING_CONFIG = 'running-config.yml' PROTOTYPE_ENV = 'MINEMELD_PROTOTYPE_PATH' MGMTBUS_NUM_CONNS_ENV = 'MGMTBUS_NUM_CONNS' FABRIC_NUM_CONNS_ENV = 'FABRIC_NUM_CONNS' CHANGE_ADDED = 0 CHANGE_DELETED = 1 CHANGE_INPUT_ADDED = 2 CHANGE_INPUT_DELETED = 3 CHANGE_OUTPUT_ENABLED = 4 CHANGE_OUTPUT_DISABLED = 5 _ConfigChange = namedtuple( '_ConfigChange', ['nodename', 'nodeclass', 'change', 'detail'] ) _Config = namedtuple( '_Config', ['nodes', 'fabric', 'mgmtbus', 'changes'] ) class MineMeldConfigChange(_ConfigChange): def __new__(_cls, nodename, nodeclass, change, detail=None): return _ConfigChange.__new__( _cls, nodename=nodename, nodeclass=nodeclass, change=change, detail=detail ) class MineMeldConfig(_Config): def as_nset(self): result = set() for nname, nvalue in self.nodes.iteritems(): result.add( json.dumps( [nname, nvalue.get('class', None)], sort_keys=True ) ) return result def compute_changes(self, oconfig): if oconfig is None: # oconfig is None, mark everything as added for nodename, nodeattrs in self.nodes.iteritems(): self.changes.append( MineMeldConfigChange(nodename=nodename, nodeclass=nodeattrs['class'], change=CHANGE_ADDED) ) return my_nset = self.as_nset() other_nset = oconfig.as_nset() deleted = other_nset - my_nset added = my_nset - other_nset untouched = my_nset & other_nset # mark delted as deleted for snode in deleted: nodename, nodeclass = json.loads(snode) change = MineMeldConfigChange( nodename=nodename, nodeclass=nodeclass, change=CHANGE_DELETED, detail=oconfig.nodes[nodename] ) self.changes.append(change) # mark added as added for snode in added: nodename, nodeclass = json.loads(snode) change = MineMeldConfigChange( nodename=nodename, nodeclass=nodeclass, change=CHANGE_ADDED ) self.changes.append(change) # check inputs/output for untouched for snode in untouched: nodename, nodeclass = json.loads(snode) my_inputs = set(self.nodes[nodename].get('inputs', [])) other_inputs = set(oconfig.nodes[nodename].get('inputs', [])) iadded = my_inputs - other_inputs ideleted = other_inputs - my_inputs for i in iadded: change = MineMeldConfigChange( nodename=nodename, nodeclass=nodeclass, change=CHANGE_INPUT_ADDED, detail=i ) self.changes.append(change) for i in ideleted: change = MineMeldConfigChange( nodename=nodename, nodeclass=nodeclass, change=CHANGE_INPUT_DELETED, detail=i ) self.changes.append(change) my_output = self.nodes[nodename].get('output', False) other_output = oconfig.nodes[nodename].get('output', False) if my_output == other_output: continue change_type = CHANGE_OUTPUT_DISABLED if my_output: change_type = CHANGE_OUTPUT_ENABLED change = MineMeldConfigChange( nodename=nodename, nodeclass=nodeclass, change=change_type ) self.changes.append(change) @classmethod def from_dict(cls, dconfig=None): if dconfig is None: dconfig = {} fabric = dconfig.get('fabric', None) if fabric is None: fabric_num_conns = int( os.getenv(FABRIC_NUM_CONNS_ENV, 50) ) fabric = { 'class': 'ZMQRedis', 'config': { 'num_connections': fabric_num_conns, 'priority': gevent.core.MINPRI # pylint:disable=E1101 } } mgmtbus = dconfig.get('mgmtbus', None) if mgmtbus is None: mgmtbus_num_conns = int( os.getenv(MGMTBUS_NUM_CONNS_ENV, 10) ) mgmtbus = { 'transport': { 'class': 'ZMQRedis', 'config': { 'num_connections': mgmtbus_num_conns, 'priority': gevent.core.MAXPRI # pylint:disable=E1101 } }, 'master': {}, 'slave': {} } nodes = dconfig.get('nodes', None) if nodes is None: nodes = {} return cls(nodes=nodes, fabric=fabric, mgmtbus=mgmtbus, changes=[]) def _load_node_prototype(protoname, paths): proto_module, proto_name = protoname.rsplit('.', 1) pmodule = None pmprotos = {} for p in paths: pmpath = os.path.join(p, proto_module+'.yml') try: with open(pmpath, 'r') as pf: pmodule = yaml.safe_load(pf) if pmodule is None: pmodule = {} except IOError: pmodule = None continue pmprotos = pmodule.get('prototypes', {}) if proto_name not in pmprotos: pmodule = None continue if 'class' not in pmprotos[proto_name]: pmodule = None continue return pmprotos[proto_name] raise RuntimeError('Unable to load prototype %s: ' ' not found' % (protoname)) def _load_config_from_file(f): valid = True config = yaml.safe_load(f) if not isinstance(config, dict) and config is not None: raise ValueError('Invalid config YAML type') return valid, MineMeldConfig.from_dict(config) def _load_and_validate_config_from_file(path): valid = False config = None if os.path.isfile(path): try: with open(path, 'r') as cf: valid, config = _load_config_from_file(cf) if not valid: LOG.error('Invalid config file {}'.format(path)) except (RuntimeError, IOError): LOG.exception( 'Error loading config {}, config ignored'.format(path) ) valid, config = False, None if valid and config is not None: valid = resolve_prototypes(config) if valid and config is not None: vresults = validate_config(config) if len(vresults) != 0: LOG.error('Invalid config {}: {}'.format( path, ', '.join(vresults) )) valid = False return valid, config def _destroy_node(change, installed_nodes=None, installed_nodes_gcs=None): LOG.info('Destroying {!r}'.format(change)) destroyed_name = change.nodename destroyed_class = change.nodeclass if destroyed_class is None: LOG.error('Node {} with no class destroyed'.format(destroyed_name)) return 1 # load node class GC from entry_point or from "gc" staticmethod of class node_gc = None mmep = installed_nodes_gcs.get(destroyed_class, None) if mmep is None: mmep = installed_nodes.get(destroyed_class, None) try: nodep = mmep.ep.load() if hasattr(nodep, 'gc'): node_gc = nodep.gc except ImportError: LOG.exception("Error checking node class {} for gc method".format(destroyed_class)) else: try: node_gc = mmep.ep.load() except ImportError: LOG.exception("Error resolving gc for class {}".format(destroyed_class)) if node_gc is None: LOG.error('Node {} with class {} with no garbage collector destroyed'.format( destroyed_name, destroyed_class )) return 1 try: node_gc( destroyed_name, config=change.detail.get('config', None) ) except: LOG.exception('Exception destroying old node {} of class {}'.format( destroyed_name, destroyed_class )) return 1 return 0 def _destroy_old_nodes(config): # this destroys resources used by destroyed nodes # a nodes has been destroyed if a node with same # name & config does not exist in the new config # the case of different node config but same and name # and class is handled by node itself destroyed_nodes = [c for c in config.changes if c.change == CHANGE_DELETED] LOG.info('Destroyed nodes: {!r}'.format(destroyed_nodes)) if len(destroyed_nodes) == 0: return installed_nodes = minemeld.loader.map(minemeld.loader.MM_NODES_ENTRYPOINT) installed_nodes_gcs = minemeld.loader.map(minemeld.loader.MM_NODES_GCS_ENTRYPOINT) dpool = multiprocessing.Pool() _bound_destroy_node = functools.partial( _destroy_node, installed_nodes=installed_nodes, installed_nodes_gcs=installed_nodes_gcs ) dpool.imap_unordered( _bound_destroy_node, destroyed_nodes ) dpool.close() dpool.join() dpool = None def _load_config_from_dir(path): ccpath = os.path.join( path, COMMITTED_CONFIG ) rcpath = os.path.join( path, RUNNING_CONFIG ) ccvalid, cconfig = _load_and_validate_config_from_file(ccpath) rcvalid, rcconfig = _load_and_validate_config_from_file(rcpath) if not rcvalid and not ccvalid: # both running and canidate are not valid print( "At least one of", RUNNING_CONFIG, "or", COMMITTED_CONFIG, "should exist in", path, file=sys.stderr ) sys.exit(1) elif rcvalid and not ccvalid: # running is valid but candidate is not return rcconfig elif not rcvalid and ccvalid: # candidate is valid while running is not LOG.info('Switching to candidate config') cconfig.compute_changes(rcconfig) LOG.info('Changes in config: {!r}'.format(cconfig.changes)) _destroy_old_nodes(cconfig) if rcconfig is not None: shutil.copyfile( rcpath, '{}.{}'.format(rcpath, int(time.time())) ) shutil.copyfile(ccpath, rcpath) return cconfig elif rcvalid and ccvalid: LOG.info('Switching to candidate config') cconfig.compute_changes(rcconfig) LOG.info('Changes in config: {!r}'.format(cconfig.changes)) _destroy_old_nodes(cconfig) shutil.copyfile( rcpath, '{}.{}'.format(rcpath, int(time.time())) ) shutil.copyfile(ccpath, rcpath) return cconfig def _detect_cycles(nodes): # using Topoligical Sorting to detect cycles in graph, see Wikipedia graph = {} S = set() L = [] for n in nodes: graph[n] = { 'inputs': [], 'outputs': [] } for n, v in nodes.iteritems(): for i in v.get('inputs', []): if i in graph: graph[i]['outputs'].append(n) graph[n]['inputs'].append(i) for n, v in graph.iteritems(): if len(v['inputs']) == 0: S.add(n) while len(S) != 0: n = S.pop() L.append(n) for m in graph[n]['outputs']: graph[m]['inputs'].remove(n) if len(graph[m]['inputs']) == 0: S.add(m) graph[n]['outputs'] = [] nedges = 0 for n, v in graph.iteritems(): nedges += len(v['inputs']) nedges += len(v['outputs']) return nedges == 0 def resolve_prototypes(config): # retrieve prototype dir from environment # used for main library and local library paths = os.getenv(PROTOTYPE_ENV, None) if paths is None: raise RuntimeError('Unable to load prototypes: %s ' 'environment variable not set' % (PROTOTYPE_ENV)) paths = paths.split(':') # add prototype dirs from extension to paths prototypes_entrypoints = minemeld.loader.map(minemeld.loader.MM_PROTOTYPES_ENTRYPOINT) for epname, mmep in prototypes_entrypoints.iteritems(): if not mmep.loadable: LOG.info('Prototypes entrypoint {} not loadable'.format(epname)) continue try: ep = mmep.ep.load() # we add prototype paths in front, to let extensions override default protos paths.insert(0, ep()) except: LOG.exception( 'Exception retrieving path from prototype entrypoint {}'.format(epname) ) # resolve all prototypes valid = True nodes_config = config.nodes for _, nconfig in nodes_config.iteritems(): if 'prototype' in nconfig: try: nproto = _load_node_prototype(nconfig['prototype'], paths) except RuntimeError as e: LOG.error('Error loading prototype {}: {}'.format( nconfig['prototype'], str(e) )) valid = False continue nconfig.pop('prototype') nconfig['class'] = nproto['class'] nproto_config = nproto.get('config', {}) nproto_config.update( nconfig.get('config', {}) ) nconfig['config'] = nproto_config return valid def validate_config(config): result = [] nodes = config.nodes for n in nodes.keys(): if re.match('^[a-zA-Z0-9_\-]+$', n) is None: # pylint:disable=W1401 result.append('%s node name is invalid' % n) for n, v in nodes.iteritems(): for i in v.get('inputs', []): if i not in nodes: result.append('%s -> %s is unknown' % (n, i)) continue if not nodes[i].get('output', False): result.append('%s -> %s output disabled' % (n, i)) installed_nodes = minemeld.loader.map(minemeld.loader.MM_NODES_ENTRYPOINT) for n, v in nodes.iteritems(): nclass = v.get('class', None) if nclass is None: result.append('No class in {}'.format(n)) continue mmep = installed_nodes.get(nclass, None) if mmep is None: result.append( 'Unknown node class {} in {}'.format(nclass, n) ) continue if not mmep.loadable: result.append( 'Class {} in {} not safe to load'.format(nclass, n) ) if not _detect_cycles(nodes): result.append('loop detected') return result def load_config(config_path): if os.path.isdir(config_path): return _load_config_from_dir(config_path) # this is just a file, as we can't do a delta # we just load it and mark all the nodes as added valid, config = _load_and_validate_config_from_file(config_path) if not valid: raise RuntimeError('Invalid config') config.compute_changes(None) return config ================================================ FILE: minemeld/run/console.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import gevent import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import logging import signal import time import uuid import json import click import minemeld.comm import minemeld.mgmtbus import minemeld.traced LOG = logging.getLogger(__name__) def _send_cmd(ctx, target, command, params=None, source=True): if params is None: params = {} if source: params['source'] = ctx.obj['SOURCE'] return ctx.obj['COMM'].send_rpc( target, command, params ) def _print_json(obj): print json.dumps( obj, indent=4, sort_keys=True ) @click.group() @click.option('--comm-class', default='ZMQRedis', metavar='CLASSNAME') @click.option('--verbose', count=True) @click.pass_context def cli(ctx, verbose, comm_class): comm_class = str(comm_class) source = 'console-%d' % int(time.time()) loglevel = logging.WARNING if verbose > 0: loglevel = logging.INFO if verbose > 1: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, format="%(asctime)s (%(process)d)%(module)s.%(funcName)s" " %(levelname)s: %(message)s", datefmt="%Y-%m-%dT%H:%M:%S" ) comm = minemeld.comm.factory(comm_class, {}) # XXX should support config gevent.signal(signal.SIGTERM, comm.stop) gevent.signal(signal.SIGQUIT, comm.stop) gevent.signal(signal.SIGINT, comm.stop) comm.start() ctx.obj['COMM'] = comm ctx.obj['SOURCE'] = source @cli.command() @click.argument('target') @click.pass_context def length(ctx, target): if target is None: raise click.UsageError(message='target required') _print_json(_send_cmd(ctx, target, 'length')) ctx.obj['COMM'].stop() @cli.command() @click.argument('target') @click.pass_context def hup(ctx, target): if target is None: raise click.UsageError(message='target required') target = 'mbus:directslave:'+target _print_json(_send_cmd(ctx, target, 'hup')) ctx.obj['COMM'].stop() @cli.command() @click.argument('target', required=False, default=None) @click.pass_context def status(ctx, target): if target is None: target = minemeld.mgmtbus.MGMTBUS_MASTER else: target = 'mbus:directslave:'+target _print_json(_send_cmd(ctx, target, 'status', source=False)) ctx.obj['COMM'].stop() @cli.command(name='signal') @click.argument('signal') @click.argument('target') @click.pass_context def mm_signal(ctx, signal, target): target = 'mbus:directslave:'+target _print_json(_send_cmd(ctx, target, 'signal', source=False, params={'signal': signal})) ctx.obj['COMM'].stop() # XXX query should subscribe to the Redis topic to dump the # query results @cli.command() @click.argument('query') @click.option('--from-counter', default=None, type=int) @click.option('--from-timestamp', default=None, type=int) @click.option('--num-lines', default=100, type=int) @click.option('--query-uuid', default=None) @click.pass_context def query(ctx, query, from_counter, from_timestamp, num_lines, query_uuid): if query_uuid is None: query_uuid = str(uuid.uuid4()) _print_json( _send_cmd( ctx, minemeld.traced.QUERY_QUEUE, 'query', source=False, params={ 'uuid': query_uuid, 'timestamp': from_timestamp, 'counter': from_counter, 'num_lines': num_lines, 'query': query } ) ) ctx.obj['COMM'].stop() def main(): cli(obj={}) # pylint:disable=E1123,E1120 ================================================ FILE: minemeld/run/extgit.py ================================================ # Copyright 2019 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import logging import argparse import subprocess import os.path import shutil from minemeld import __version__ def _parse_args(): parser = argparse.ArgumentParser( description="Install MineMeld extension from git repo" ) parser.add_argument( '--version', action='version', version=__version__ ) parser.add_argument( 'git_path', action='store', metavar='GIT_PATH', help='path of git executable' ) parser.add_argument( 'git_ref', action='store', metavar='GIT_REF', help='git reference' ) parser.add_argument( 'git_endpoint', action='store', metavar='GIT_ENDPOINT', help='git endpoint' ) parser.add_argument( 'install_directory', action='store', metavar='INSTALL_DIRECTORY', help='directory to install the extension into' ) return parser.parse_args() def main(): logging.basicConfig(level=logging.DEBUG) args = _parse_args() git_args = [ args.git_path, 'clone', '-b', args.git_ref, '--depth', '1', args.git_endpoint, args.install_directory ] logging.info('Calling git: {!r}'.format(git_args)) subprocess.check_call(git_args) if not os.path.exists(os.path.join(args.install_directory, 'minemeld.json')): logging.error('minemeld.json does not exists in install directory - invalid extension') try: shutil.rmtree(args.install_directory) except Exception as _: logging.exception('Error removing install directory') sys.exit(1) sys.exit(0) ================================================ FILE: minemeld/run/freeze.py ================================================ # Copyright 2017 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import logging import argparse import minemeld.extensions from minemeld import __version__ def _parse_args(): parser = argparse.ArgumentParser( description="Freeze MineMeld extensions" ) parser.add_argument( '--version', action='version', version=__version__ ) parser.add_argument( 'library', action='store', metavar='LIBRARY', help='path of the MineMeld library directory' ) parser.add_argument( 'outfile', action='store', metavar='OUTFILE', default='-', nargs='?', help='path of the file to write the output to. (default: stdout)' ) return parser.parse_args() def main(): logging.basicConfig(level=logging.DEBUG) args = _parse_args() if args.outfile == '-': of = sys.stdout else: of = open(args.outfile, 'w+') frozenext = minemeld.extensions.freeze(args.library) for e in frozenext: of.write('{}\n'.format(e)) of.close() ================================================ FILE: minemeld/run/launcher.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import print_function import gevent import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import os.path import logging import signal import multiprocessing import argparse import os import math import psutil import minemeld.chassis import minemeld.mgmtbus import minemeld.comm import minemeld.run.config from minemeld import __version__ LOG = logging.getLogger(__name__) def _run_chassis(fabricconfig, mgmtbusconfig, fts): try: # lower priority to make master and web # more "responsive" os.nice(5) c = minemeld.chassis.Chassis( fabricconfig['class'], fabricconfig['config'], mgmtbusconfig ) c.configure(fts) gevent.signal(signal.SIGUSR1, c.stop) while not c.fts_init(): if c.poweroff.wait(timeout=0.1) is not None: break gevent.sleep(1) LOG.info('Nodes initialized') try: c.poweroff.wait() LOG.info('power off') except KeyboardInterrupt: LOG.error("We should not be here !") c.stop() except: LOG.exception('Exception in chassis main procedure') raise def _check_disk_space(num_nodes): free_disk_per_node = int(os.environ.get( 'MM_DISK_SPACE_PER_NODE', 10*1024 # default: 10MB per node )) needed_disk = free_disk_per_node*num_nodes*1024 free_disk = psutil.disk_usage('.').free LOG.debug('Disk space - needed: {} available: {}'.format(needed_disk, free_disk)) if free_disk <= needed_disk: LOG.critical( ('Not enough space left on the device, available: {} needed: {}' ' - please delete traces, logs and old engine versions and restart').format( free_disk, needed_disk ) ) return None return free_disk def _parse_args(): parser = argparse.ArgumentParser( description="Low-latency threat indicators processor" ) parser.add_argument( '--version', action='version', version=__version__ ) parser.add_argument( '--multiprocessing', default=0, type=int, action='store', metavar='NP', help='enable multiprocessing. NP is the number of chassis, ' '0 to use two chassis per machine core (default)' ) parser.add_argument( '--nodes-per-chassis', default=15.0, type=float, action='store', metavar='NPC', help='number of nodes per chassis (default 15)' ) parser.add_argument( '--verbose', action='store_true', help='verbose' ) parser.add_argument( 'config', action='store', metavar='CONFIG', help='path of the config file or of the config directory' ) return parser.parse_args() def _setup_environment(config): # make config dir available to nodes cdir = config if not os.path.isdir(cdir): cdir = os.path.dirname(config) os.environ['MM_CONFIG_DIR'] = cdir if not 'REQUESTS_CA_BUNDLE' in os.environ and 'MM_CA_BUNDLE' in os.environ: os.environ['REQUESTS_CA_BUNDLE'] = os.environ['MM_CA_BUNDLE'] def main(): mbusmaster = None processes_lock = None processes = None disk_space_monitor_glet = None def _cleanup(): if mbusmaster is not None: mbusmaster.checkpoint_graph() if processes_lock is None: return with processes_lock: if processes is None: return for p in processes: if not p.is_alive(): continue try: os.kill(p.pid, signal.SIGUSR1) except OSError: continue while sum([int(t.is_alive()) for t in processes]) != 0: gevent.sleep(1) def _sigint_handler(): LOG.info('SIGINT received') _cleanup() signal_received.set() def _sigterm_handler(): LOG.info('SIGTERM received') _cleanup() signal_received.set() def _disk_space_monitor(num_nodes): while True: if _check_disk_space(num_nodes=num_nodes) is None: _cleanup() signal_received.set() break gevent.sleep(60) args = _parse_args() # logging loglevel = logging.INFO if args.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, format="%(asctime)s (%(process)d)%(module)s.%(funcName)s" " %(levelname)s: %(message)s", datefmt="%Y-%m-%dT%H:%M:%S" ) LOG.info("Starting mm-run.py version %s", __version__) LOG.info("mm-run.py arguments: %s", args) _setup_environment(args.config) # load and validate config config = minemeld.run.config.load_config(args.config) LOG.info("mm-run.py config: %s", config) if _check_disk_space(num_nodes=len(config.nodes)) is None: LOG.critical('Not enough disk space available, exit') return 2 np = args.multiprocessing if np == 0: np = multiprocessing.cpu_count() LOG.info('multiprocessing: #cores: %d', multiprocessing.cpu_count()) LOG.info("multiprocessing: max #chassis: %d", np) npc = args.nodes_per_chassis if npc <= 0: LOG.critical('nodes-per-chassis should be a positive integer') return 2 np = min( int(math.ceil(len(config.nodes)/npc)), np ) LOG.info("Number of chassis: %d", np) ftlists = [{} for j in range(np)] j = 0 for ft in config.nodes: pn = j % len(ftlists) ftlists[pn][ft] = config.nodes[ft] j += 1 # cleanup if config.mgmtbus['transport']['class'] != config.fabric['class']: raise ValueError('mgmtbus class and fabric class should match') minemeld.comm.cleanup(config.fabric['class'], config.fabric['config']) signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) processes = [] for g in ftlists: if len(g) == 0: continue p = multiprocessing.Process( target=_run_chassis, args=( config.fabric, config.mgmtbus, g ) ) processes.append(p) p.start() processes_lock = gevent.lock.BoundedSemaphore() signal_received = gevent.event.Event() gevent.signal(signal.SIGINT, _sigint_handler) gevent.signal(signal.SIGTERM, _sigterm_handler) try: mbusmaster = minemeld.mgmtbus.master_factory( config=config.mgmtbus['master'], comm_class=config.mgmtbus['transport']['class'], comm_config=config.mgmtbus['transport']['config'], nodes=config.nodes.keys(), num_chassis=len(processes) ) mbusmaster.start() mbusmaster.wait_for_chassis(timeout=10) # here nodes are all CONNECTED, fabric and mgmtbus up, with mgmtbus # dispatching and fabric not dispatching mbusmaster.start_status_monitor() mbusmaster.init_graph(config) # here nodes are all INIT mbusmaster.start_chassis() # here nodes should all be starting except Exception: LOG.exception('Exception initializing graph') _cleanup() raise disk_space_monitor_glet = gevent.spawn(_disk_space_monitor, len(config.nodes)) try: while not signal_received.wait(timeout=1.0): with processes_lock: r = [int(t.is_alive()) for t in processes] if sum(r) != len(processes): LOG.info("One of the chassis has stopped, exit") break except KeyboardInterrupt: LOG.info("Ctrl-C received, exiting") except: LOG.exception("Exception in main loop") if disk_space_monitor_glet is not None: disk_space_monitor_glet.kill() ================================================ FILE: minemeld/run/restore.py ================================================ #!/usr/bin/env python import os import signal import time import logging import argparse import xmlrpclib from zipfile import ZipFile from collections import deque from contextlib import contextmanager import yaml import redis import supervisor.xmlrpc REDIS_KEY_PREFIX = 'mm:config:' LOG = logging.getLogger(__name__) class BFile(object): def __init__(self, zip_path, type_, extracted_path=None, target_path=None): self.zip_path = zip_path self.type_ = type_ self.extracted_path = extracted_path self.target_path = target_path def __repr__(self): return '{}({}) => {}({})'.format( self.zip_path, self.type_, self.target_path, self.extracted_path ) def _parse_args(): parser = argparse.ArgumentParser( description="Restore full MineMeld backup" ) parser.add_argument( '--dry-run', action='store_true', help='Dry run' ) parser.add_argument( '--configuration-path', action='store', help='Path to restore configuration files to' ) parser.add_argument( '--prototypes-path', action='store', help='Path to restore prototypes to' ) parser.add_argument( '--feeds-aaa-path', action='store', help='Path to restore feeds AAA configuration to' ) parser.add_argument( '--feeds-aaa', action='append', help='Restore feeds AAA configuration' ) parser.add_argument( '--certificates-path', action='store', help='Path to restore certificates to' ) parser.add_argument( '--password', action='store', help='Password for the backup file' ) parser.add_argument( 'backup', action='store', help='path of the backup file' ) return parser.parse_args() class ContextManagerStack(object): def __init__(self): self._stack = deque() def enter(self, cm): result = cm.__enter__() self._stack.append(cm.__exit__) return result def __enter__(self): return self def __exit__(self, *exc_info): while self._stack: cb = self._stack.pop() cb(*exc_info) @contextmanager def handle_minemeld_engine(supervisor_url): sserver = xmlrpclib.ServerProxy( 'http://127.0.0.1', transport=supervisor.xmlrpc.SupervisorTransport( None, None, supervisor_url ) ) # check supervisor state sstate = sserver.supervisor.getState() if sstate['statecode'] == 2: # FATAL raise RuntimeError('Supervisor state: 2') if sstate['statecode'] != 1: raise RuntimeError( "Supervisor transitioning to a new state, restore not performed" ) # check minemeld-engine state pstate = sserver.supervisor.getProcessInfo('minemeld-engine')['statename'] if pstate not in ['STOPPED', 'EXITED', 'FATAL', 'RUNNING']: raise RuntimeError( ("minemeld-engine transitioning to a new state, " + "restore not performed") ) # if minemeld-engine state is running, stop it if pstate == 'RUNNING': result = sserver.supervisor.stopProcess('minemeld-engine', False) if not result: raise RuntimeError('Stop minemeld-engine returned False') LOG.info('Stopping minemeld-engine') now = time.time() info = None while (time.time()-now) < 60*15*1000: info = sserver.supervisor.getProcessInfo('minemeld-engine') if info['statename'] == 'STOPPED': break time.sleep(5) if info is not None and info['statename'] != 'STOPPED': raise RuntimeError('Timeout during minemeld-engine stop') else: LOG.info('minemeld-engine not running: {}'.format(pstate)) yield # we restart only if no Exception have been raised by other tasks sserver.supervisor.startProcess('minemeld-engine', False) started_at = time.time() # check minemeld-engine state pstate = sserver.supervisor.getProcessInfo('minemeld-engine')['statename'] while pstate != 'RUNNING': LOG.info('minemeld-engine state: {}'.format(pstate)) if pstate == 'FATAL': raise RuntimeError('minemeld-engine failed to start') if (time.time() - started_at) > 40: raise RuntimeError('minemeld-engine didn\'t start in 40 seconds') time.sleep(1) pstate = sserver.supervisor.getProcessInfo('minemeld-engine')['statename'] LOG.info('Started minemeld-engine') @contextmanager def extract_file(backup_id, bfile, efile, configuration_path, prototypes_path, feeds_aaa_path, certificates_path): if efile.type_ == 'configuration': new_path = configuration_path elif efile.type_ == 'prototypes': new_path = prototypes_path elif efile.type_ == 'feeds_aaa': new_path = feeds_aaa_path elif efile.type_ == 'certificates': new_path = certificates_path if efile.zip_path.startswith('certs/site'): new_path = os.path.join(certificates_path, '/site') else: raise RuntimeError('Unknown file type: {!r}'.format(efile)) new_path = os.path.join( new_path, '{}'.format(os.path.basename(efile.zip_path)) ) extracted_path = '{}.{}'.format(new_path, backup_id) LOG.info('Extracting {} to {}'.format(efile.zip_path, extracted_path)) efile.extracted_path = extracted_path efile.target_path = new_path fin = bfile.open(efile.zip_path, 'r') with open(extracted_path, 'w') as fout: while True: b = fin.read(1024 * 1024) if not b: break fout.write(b) try: yield except: try: os.remove(extracted_path) LOG.info('Removed temporary file {}'.format(extracted_path)) except: pass @contextmanager def backup_file(old_file_path): new_path = '{}.bak'.format(old_file_path) os.rename(old_file_path, new_path) try: yield except: try: os.rename(new_path, old_file_path) except: LOG.error('Error restoring {} to {}'.format(new_path, old_file_path)) else: try: os.remove(new_path) except: LOG.error('Error removing backup {}'.format(new_path)) @contextmanager def restore_file(f): LOG.info('Moving {} to {}'.format(f.extracted_path, f.target_path)) os.rename(f.extracted_path, f.target_path) try: yield except: try: os.remove(f.target_path) except: LOG.error('Error removing extracted file during recovery: {}'.format(f.target_path)) def _list_of_configuration_files(bfile, flist): dir_prefix = 'config/' committed_config_path = os.path.join(dir_prefix, 'committed-config.yml') if committed_config_path not in flist: raise RuntimeError('No committed-config in backup') committed_config = yaml.safe_load(bfile.open(committed_config_path, 'r')) result = [BFile(zip_path=committed_config_path, type_='configuration')] prefixes = [os.path.join(dir_prefix, nname) for nname in committed_config['nodes']] for c in flist: if not c.startswith(dir_prefix): continue for p in prefixes: if c.startswith(p): result.append(BFile(zip_path=c, type_='configuration')) break return result def _list_of_prototypes_files(bfile, flist): dir_prefix = 'prototypes/' result = [] for c in flist: if c == dir_prefix: continue if not c.startswith(dir_prefix): continue result.append(BFile(zip_path=c, type_='prototypes')) return result def _list_of_certificates_files(bfile, flist): # from the backup file we restore certs/config.yml # and all the files in certs/site/ result = [] for c in flist: if c == 'certs/site/': continue if not c == 'certs/config.yml' and not c.startswith('certs/site/'): continue result.append(BFile(zip_path=c, type_='certificates')) return result def _list_of_feeds_aaa_files(faaf, bfile, flist): faaf_path = os.path.join('config/api/', faaf) if faaf_path in flist: return [BFile(zip_path=faaf_path, type_='feeds_aaa')] return [] def _reload_candidate_config(supervisor_url): SR = redis.StrictRedis() ckeys = SR.keys('{}*'.format(REDIS_KEY_PREFIX)) if ckeys: for ck in ckeys: LOG.info('Deleting {}'.format(ck)) SR.delete(ck) LOG.info('Candidate config keys deleted') sserver = xmlrpclib.ServerProxy( 'http://127.0.0.1', transport=supervisor.xmlrpc.SupervisorTransport( None, None, supervisor_url ) ) # check supervisor state sstate = sserver.supervisor.getState() if sstate['statecode'] == 2: # FATAL raise RuntimeError('Supervisor state: 2') if sstate['statecode'] != 1: raise RuntimeError( "Supervisor transitioning to a new state, restore not performed" ) # check minemeld-engine state pinfo = sserver.supervisor.getProcessInfo('minemeld-web') if pinfo['statename'] != 'RUNNING': raise RuntimeError('minemeld-web not running, reload not sent') os.kill(pinfo['pid'], signal.SIGHUP) LOG.info('API process reloaded') def main(): supervisor_url = 'unix:///var/run/minemeld/minemeld.sock' logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(levelname)s: %(message)s' ) args = _parse_args() supervisor_url = os.environ.get( 'SUPERVISOR_URL', supervisor_url ) LOG.info('restore started: {!r}'.format(args)) with ContextManagerStack() as cmstack: backup_id = os.path.basename(args.backup) bfile = cmstack.enter(ZipFile(args.backup, 'r')) if args.password: bfile.setpassword(args.password) contents = bfile.namelist() files = [] if args.configuration_path: files.extend(_list_of_configuration_files(bfile, contents)) if args.prototypes_path: files.extend(_list_of_prototypes_files(bfile, contents)) if args.certificates_path: files.extend(_list_of_certificates_files(bfile, contents)) if args.feeds_aaa_path: if not args.feeds_aaa: LOG.warning('No feeds AAA config file specified') else: for faaf in args.feeds_aaa: files.extend(_list_of_feeds_aaa_files(faaf, bfile, contents)) LOG.info('List of files to be restored: {}'.format(files)) # stop/start minemeld-engine cmstack.enter(handle_minemeld_engine(supervisor_url)) # extract files for f in files: cmstack.enter(extract_file( backup_id=backup_id, bfile=bfile, efile=f, configuration_path=args.configuration_path, prototypes_path=args.prototypes_path, feeds_aaa_path=args.feeds_aaa_path, certificates_path=args.certificates_path )) LOG.info('Extracted files: {}'.format(files)) # check if I can move old files for f in files: LOG.info('Checking {} for write permissions'.format(f.target_path)) if not os.path.exists(f.target_path): continue if not os.path.isfile(f.target_path): raise RuntimeError('{} is not a file !'.format(f.target_path)) if not os.access(f.target_path, os.W_OK): raise RuntimeError('No permission to write to {}'.format(f.target_path)) if not os.access(os.path.dirname(f.target_path), os.W_OK): raise RuntimeError('No permission to write to {}'.format(os.path.dirname(f.target_path))) # backup old files for f in files: if os.path.exists(f.target_path): cmstack.enter(backup_file( f.target_path )) # move new files for f in files: cmstack.enter(restore_file(f)) try: _reload_candidate_config(supervisor_url) except: LOG.exception('Error reverting candidate config') ================================================ FILE: minemeld/startupplanner.py ================================================ import logging from operator import itemgetter from collections import defaultdict import networkx as nx from minemeld.run.config import CHANGE_INPUT_DELETED, CHANGE_ADDED, CHANGE_INPUT_ADDED LOG = logging.getLogger(__name__) class CheckpointNodes(object): def __init__(self): self.nodes = set() self.num_sources = 0 def _build_graph(config): graph = nx.DiGraph() # nodes for nodename, _ in config.nodes.iteritems(): graph.add_node(nodename) # edges for nodename, nodevalue in config.nodes.iteritems(): inputs = nodevalue.get('inputs', []) graph.add_edges_from([(i, nodename) for i in inputs]) return graph def _plan_subgraph(sg, config, state_info): LOG.info('state_info: {!r}'.format(state_info)) LOG.info('planning for subgraph {!r}'.format(sg.nodes())) plan = {} checkpoints = defaultdict(CheckpointNodes) for nodename in sg: chkp = state_info[nodename].get('checkpoint', None) checkpoints[chkp].nodes.add(nodename) if state_info[nodename].get('is_source', False): checkpoints[chkp].num_sources += 1 changes = defaultdict(list) for c in config.changes: if c.nodename in sg: changes[c.nodename].append(c) # if there are no checkpoints => reset if len(checkpoints) == 1 and None in checkpoints: LOG.info('No checkpoints, new graph: reset') for nodename in sg: plan[nodename] = 'reset' return plan # if there are no changes and all the nodes are at the same # checkpoint => initialize if len(checkpoints) == 1 and len(changes) == 0: LOG.info('No changes and all nodes have the same checkpoint: initialize') for nodename in sg: plan[nodename] = 'initialize' return plan # pick the most common checkpoint among sources as reference point scheckpoints = sorted( [(c, cn.num_sources) for c, cn in checkpoints.iteritems() if c is not None], key=itemgetter(1), reverse=True ) quorum_checkpoint = None if len(scheckpoints) > 0: quorum_checkpoint = scheckpoints[0][0] LOG.info('Quorum checkpoint: {}'.format(quorum_checkpoint)) # invalid nodes are nodes whose current state is not up to # date # - nodes with an old checkpoint # - nodes with no checkpoint but not added # - nodes that had an input deleted invalid_nodes = [] for nodename in sg: if nodename not in checkpoints[quorum_checkpoint].nodes and nodename not in checkpoints[None].nodes: invalid_nodes.append(nodename) continue added = next((c for c in changes[nodename] if c.change == CHANGE_ADDED), None) if added is None and nodename in checkpoints[None].nodes: invalid_nodes.append(nodename) continue ideleted = next((c for c in changes[nodename] if c.change == CHANGE_INPUT_DELETED), None) if ideleted is not None: invalid_nodes.append(nodename) continue # there is at least one invalid node, we reset all the nodes except for the # sources with checkpoint == quorum_checkpoint # XXX this can be improved if len(invalid_nodes) > 0: for nodename in sg: if nodename in invalid_nodes: plan[nodename] = 'reset' continue if not state_info[nodename].get('is_source', False): plan[nodename] = 'reset' continue if nodename not in checkpoints[quorum_checkpoint].nodes: plan[nodename] = 'reset' continue plan[nodename] = 'rebuild' LOG.info('Invalid nodes detected ({}): {}'.format(invalid_nodes, plan)) return plan # let's check added nodes and nodes added as inputs, if they have no ancestors # we can just initialize init_flag = True added_nodes = [] for nodename, clist in changes.iteritems(): added = next((c for c in clist if c.change == CHANGE_ADDED), None) if added is not None: if not state_info[nodename].get('is_source', False): init_flag = False break added_nodes.append(nodename) added_input_nodes = set() for nodename, clist in changes.iteritems(): input_added = [c.detail for c in clist if c.change == CHANGE_INPUT_ADDED] for ainode in input_added: if not state_info[ainode].get('is_source', False): init_flag = False added_input_nodes.add(ainode) if init_flag: LOG.info('Only source nodes have been added: initialize') for nodename in sg: if nodename in added_nodes: plan[nodename] = 'reset' elif nodename in added_input_nodes: plan[nodename] = 'rebuild' else: plan[nodename] = 'initialize' return plan for nodename in sg: if not state_info[nodename].get('is_source', False): plan[nodename] = 'reset' continue if nodename not in checkpoints[quorum_checkpoint].nodes: plan[nodename] = 'reset' continue plan[nodename] = 'rebuild' LOG.info('Non-source nodes added ({}): {}'.format(added_nodes, plan)) return plan def plan(config, state_info): """Defines a startup plan for the MineMeld graph. Args: config (MineMeldConfig): config state_info (dict): state_info for each node Returns a dictionary where keys are node names and values the satrtup command for the node. """ plan = {} graph = _build_graph(config) for subgraph in nx.weakly_connected_component_subgraphs(graph, copy=True): plan.update(_plan_subgraph(subgraph, config, state_info)) return plan ================================================ FILE: minemeld/supervisord/__init__.py ================================================ ================================================ FILE: minemeld/supervisord/listener.py ================================================ import sys import os import logging import time import redis import ujson from supervisor import childutils LOG = logging.getLogger(__name__) def _handle_event(SR, engine_process_name, hdrs, payload): event = hdrs.get('eventname', None) if not event.startswith('PROCESS_STATE'): return event = event.split('_', 2)[-1] processname = None pkvs = payload.split() for pkv in pkvs: pkey, pvalue = pkv.split(':', 1) if pkey == 'processname': processname = pvalue break else: LOG.error('processname key not found in payload') return if processname != engine_process_name: return SR.publish( 'mm-engine-status.', ujson.dumps({ 'source': '', 'timestamp': int(time.time())*1000, 'status': event }) ) def main(): logging.basicConfig(level=logging.DEBUG) engine_process_name = os.environ.get('MM_ENGINE_PROCESSNAME', 'minemeld-engine') SR = redis.StrictRedis.from_url( os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock') ) while True: hdrs, payload = childutils.listener.wait(sys.stdin, sys.stdout) LOG.info('hdr: {!r} payload: {!r}'.format(hdrs, payload)) try: _handle_event(SR, engine_process_name, hdrs, payload) except: LOG.exception('Exception in handling event') finally: childutils.listener.ok(sys.stdout) ================================================ FILE: minemeld/traced/__init__.py ================================================ from minemeld.traced.queryprocessor import QUERY_QUEUE # noqa ================================================ FILE: minemeld/traced/main.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements the main entry point to the mm-traced daemon """ from __future__ import print_function import gevent import gevent.event import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import argparse import logging import yaml import functools import signal from minemeld import __version__ import minemeld.comm import minemeld.traced.storage import minemeld.traced.writer import minemeld.traced.queryprocessor LOG = logging.getLogger(__name__) def _parse_args(): parser = argparse.ArgumentParser( description="Tracing daemon for MineMeld engine" ) parser.add_argument( '--version', action='version', version=__version__ ) parser.add_argument( '--verbose', action='store_true', help='verbose' ) parser.add_argument( 'config', action='store', metavar='CONFIG', help='path of the config file or of the config directory' ) return parser.parse_args() def _ioloop_failure(event): LOG.debug("loop failure") event.set() def main(): def _sigint_handler(): raise KeyboardInterrupt('Ctrl-C from _sigint_handler') def _sigterm_handler(): raise KeyboardInterrupt('Ctrl-C from _sigterm_handler') def _cleanup(): trace_writer.stop() query_processor.stop() store.stop() comm.stop() args = _parse_args() loglevel = logging.INFO if args.verbose: loglevel = logging.DEBUG logging.basicConfig( level=loglevel, format="%(asctime)s (%(process)d)%(module)s.%(funcName)s" " %(levelname)s: %(message)s", datefmt="%Y-%m-%dT%H:%M:%S" ) LOG.info('Starting mm-traced version %s', __version__) LOG.info('mm-traced arguments: %s', args) with open(args.config, 'r') as f: config = yaml.safe_load(f) if config is None: config = {} LOG.info('mm-traced config: %s', config) store = minemeld.traced.storage.Store(config.get('store', None)) transport_config = config.get('transport', { 'class': 'ZMQRedis', 'config': { 'num_connections': 1 } }) comm = minemeld.comm.factory( transport_config['class'], transport_config['config'] ) trace_writer = minemeld.traced.writer.Writer( comm, store, topic=config.get('topic', 'mbus:log'), config=config.get('writer', {}) ) query_processor = minemeld.traced.queryprocessor.QueryProcessor( comm, store, config=config.get('queryprocessor', {}) ) shutdown_event = gevent.event.Event() comm.add_failure_listener( functools.partial(_ioloop_failure, shutdown_event) ) comm.start() gevent.signal(signal.SIGINT, _sigint_handler) gevent.signal(signal.SIGTERM, _sigterm_handler) try: shutdown_event.wait() except KeyboardInterrupt: pass except: LOG.exception('Exception') finally: _cleanup() ================================================ FILE: minemeld/traced/purge.py ================================================ #!/usr/bin/env python import logging import os import time import sys import argparse import shutil import json import xmlrpclib import supervisor.xmlrpc LOG = logging.getLogger(__name__) def _parse_args(): parser = argparse.ArgumentParser( description="Purge utility for old MineMeld traces" ) parser.add_argument( '--dry-run', action='store_true', help='Dry run' ) parser.add_argument( '--all', action='store_true', help='Delete all traces' ) parser.add_argument( 'config', action='store', metavar='CONFIG', nargs='?', help='path of the config file or of the config directory' ) return parser.parse_args() def stop_minemeld_traced(supervisor_url): sserver = xmlrpclib.ServerProxy( 'http://127.0.0.1', transport=supervisor.xmlrpc.SupervisorTransport( None, None, supervisor_url ) ) sstate = sserver.supervisor.getState() if sstate['statecode'] == 2: # FATAL return False if sstate['statecode'] != 1: LOG.critical( "Supervisor transitioning to a new state, we will purge next time" ) sys.exit(1) pstate = sserver.supervisor.getProcessInfo('minemeld-traced')['statename'] if pstate in ['STOPPED', 'EXITED', 'FATAL']: return False if pstate != 'RUNNING': LOG.critical( ("minemeld-traced transitioning to a new state, " + "we will purge next time") ) sys.exit(1) result = sserver.supervisor.stopProcess('minemeld-traced', False) if not result: LOG.critical('Stop minemeld-traced returned False') sys.exit(1) LOG.info('Stopping minemeld-traced') now = time.time() info = None while (time.time()-now) < 60*15*1000: info = sserver.supervisor.getProcessInfo('minemeld-traced') if info['statename'] == 'STOPPED': break time.sleep(5) if info is not None and info['statename'] != 'STOPPED': LOG.critical('Timeout during minemeld-traced stop') sys.exit(1) return True def start_minemeld_traced(supervisor_url): sserver = xmlrpclib.ServerProxy( 'http://127.0.0.1', transport=supervisor.xmlrpc.SupervisorTransport( None, None, supervisor_url ) ) sserver.supervisor.startProcess('minemeld-traced', False) LOG.info('Started minemeld-traced') def main(): trace_directory = '/opt/minemeld/local/trace' supervisor_url = 'unix:///var/run/minemeld/minemeld.sock' num_days = 30 logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(levelname)s: %(message)s' ) args = _parse_args() if args.config is not None: try: with open(args.config, 'r') as f: config = json.load(f) except (IOError, ValueError) as e: LOG.critical( 'Error loading config file %s: %s' % (args.config, str(e)) ) sys.exit(1) trace_directory = config.get('trace_directory', trace_directory) supervisor_url = config.get('supervisor_url', supervisor_url) num_days = config.get('num_days', num_days) trace_directory = os.environ.get( 'MINEMELD_TRACE_DIRECTORY', trace_directory ) if not os.path.isdir(trace_directory): LOG.critical("%s is not a directory", trace_directory) sys.exit(1) num_days = int(os.environ.get('MINEMELD_TRACE_NUM_DAYS', num_days)) if num_days < 1: LOG.critical( 'MINEMELD_TRACE_NUM_DAYS should be greater than 1: %d', num_days ) sys.exit(1) supervisor_url = os.environ.get( 'SUPERVISOR_URL', supervisor_url ) LOG.info( "mm-traced-purge started, #days: %d directory: %s", num_days, trace_directory ) now = time.time() today = now - (now % 86400) oldest = today - (num_days-1)*86400 tables = os.listdir(trace_directory) tobe_removed = [] for t in tables: try: d = int(t, 16) except ValueError: LOG.debug("Invalid table name: %s", t) continue if d < oldest or args.all: LOG.info('Marking table %s for removal', t) tobe_removed.append(t) if len(tobe_removed) > 0 and not args.dry_run: trunning = stop_minemeld_traced(supervisor_url) for tr in tobe_removed: LOG.info("Removing %s", tr) try: shutil.rmtree(os.path.join(trace_directory, tr)) except: LOG.exception("Error removing table %s", tr) if trunning: start_minemeld_traced(supervisor_url) ================================================ FILE: minemeld/traced/queryprocessor.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements the query processor for mm-traced daemon """ import logging import calendar import time import ujson import re import os import gevent import greenlet import gevent.lock import gevent.event import redis LOG = logging.getLogger(__name__) QUERY_QUEUE = 'mmtraced:query' _REGEX_SPECIAL_CHARS = [ '[', '\\', '^', '$', '.', '|', '?', '*', '+', '(', ')' ] class Query(gevent.Greenlet): def __init__(self, store, query, timestamp, counter, num_lines, uuid, redis_config): self.uuid = uuid self.store = store self.query = query self.starting_timestamp = timestamp self.starting_counter = counter self.num_lines = num_lines self.redis_url = redis_config.get('redis_url', os.environ.get('REDIS_URL', 'unix:///var/run/redis/redis.sock') ) super(Query, self).__init__() LOG.info("Query %s - %s", uuid, query) self._parse_query(query) def _parse_query(self, query): query = query.strip() components = query.lower().split() field_specific = re.compile('^[\w$]+:.*$') self.parsed_query = [] for c in components: negate = False if c[0] == '-': negate = True c = c[1:] matching_re = c if field_specific.match(c) is not None: field, value = c.split(':', 1) efield = [] for c in field: if c in _REGEX_SPECIAL_CHARS: efield.append('\\') efield.append(c) efield = ''.join(efield) evalue = [] for c in value: if c in _REGEX_SPECIAL_CHARS: evalue.append('\\') evalue.append(c) evalue = ''.join(evalue) matching_re = ( '"%(field)s":(?:\[(?:".*",)*)?"*[^"]*%(value)s' % { 'field': efield, 'value': evalue } ) LOG.debug(matching_re) self.parsed_query.append({ 're': re.compile(matching_re, re.IGNORECASE), 'negate': negate }) def _check_query(self, log): for q in self.parsed_query: occ = q['re'].search(log) if not ((occ is not None) ^ q['negate']): return False return True def _core_run(self): LOG.debug("Query %s started", self.uuid) SR = redis.StrictRedis.from_url( self.redis_url ) line_generator = self.store.iterate_backwards( self.uuid, self.starting_timestamp, self.starting_counter ) try: num_generated_lines = 0 while num_generated_lines < self.num_lines: line = next(line_generator, None) if not line: break gevent.sleep(0) if 'log' not in line: SR.publish('mm-traced-q.'+self.uuid, ujson.dumps(line)) continue if self._check_query(line['log']): SR.publish('mm-traced-q.'+self.uuid, ujson.dumps(line)) num_generated_lines += 1 SR.publish( 'mm-traced-q.'+self.uuid, '{"msg": "Loaded %d lines"}' % num_generated_lines ) finally: SR.publish('mm-traced-q.'+self.uuid, '') LOG.info("Query %s finished - %d", self.uuid, num_generated_lines) def _run(self): try: self._core_run() finally: # make sure we release the tables if we stop in the middle # of an iteration self.store.release_all(self.uuid) class QueryProcessor(object): def __init__(self, comm, store, config=None): if config is None: config = {} self._stop = gevent.event.Event() self.max_concurrency = config.get('max_concurrency', 10) self.redis_config = config.get('redis', {}) self.store = store self.queries_lock = gevent.lock.BoundedSemaphore() self.queries = {} comm.request_rpc_server_channel( QUERY_QUEUE, self, allowed_methods=['query', 'kill_query'] ) def _query_finished(self, gquery): self.queries_lock.acquire() self.queries.pop(gquery.uuid, None) self.queries_lock.release() try: result = gquery.get() except: self.store.release_all(gquery.uuid) LOG.exception('Query finished with exception') return if isinstance(result, greenlet.GreenletExit): self.store.release_all(gquery.uuid) def query(self, uuid, query, timestamp=None, counter=None, num_lines=None): LOG.debug('Query called: {!r}'.format(query)) if self._stop.is_set(): raise RuntimeError('stopping') if timestamp is None: timestamp = int(calendar.timegm(time.gmtime())*1000) if counter is None: counter = 0xFFFFFFFFFFFFFFFF if num_lines is None: num_lines = 100 self.queries_lock.acquire() LOG.debug('Locked') if len(self.queries) >= self.max_concurrency: self.queries_lock.release() raise RuntimeError('max number of concurrent queries reached') if uuid in self.queries: self.queries_lock.release() raise RuntimeError('UUID not unique') try: gquery = Query( self.store, query, timestamp, counter, num_lines, uuid, self.redis_config ) gquery.link(self._query_finished) self.queries[uuid] = gquery gquery.start() finally: self.queries_lock.release() return 'OK' def kill_query(self, uuid): if self._stop.is_set(): raise RuntimeError('stopping') self.queries_lock.acquire() if uuid in self.queries: self.queries[uuid].kill() self.queries_lock.release() return 'OK' def stop(self): LOG.info('QueryProcessor - stop called') if self._stop.is_set(): return self._stop.set() self.queries_lock.acquire() for _, gquery in self.queries.iteritems(): gquery.kill() self.queries_lock.release() ================================================ FILE: minemeld/traced/storage.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements the storage mechansim for the mm-traced daemon """ import logging import datetime import time import Queue import os import os.path import gevent.queue import gevent.event import gevent.lock import plyvel import pytz LOG = logging.getLogger(__name__) START_KEY = '%016x%015x' % (0, 0) TABLE_MAX_COUNTER_KEY = 'MAX_COUNTER' class TableNotFound(Exception): pass class Table(object): def __init__(self, name, create_if_missing=True): LOG.debug('New table: %s %s', name, create_if_missing) self.name = name self.last_used = None self.refs = [] if not create_if_missing and not os.path.exists(name): raise TableNotFound('Table does not exists') try: self.db = plyvel.DB( name, create_if_missing=create_if_missing ) except plyvel.Error as e: if not create_if_missing: raise TableNotFound(str(e)) raise self.max_counter = None try: self.max_counter = self.db.get(TABLE_MAX_COUNTER_KEY) except KeyError: pass if self.max_counter is None: LOG.warning( 'MAX_ID key not found in %s', self.name ) self.max_counter = -1 else: self.max_counter = int(self.max_counter, 16) LOG.debug('Table %s - max id: %d', self.name, self.max_counter) def add_reference(self, refid): self.refs.append(refid) def remove_reference(self, refid): try: self.refs.remove(refid) except ValueError: LOG.warning( 'Attempt to remove non existing reference: %s - %s', refid, self.name ) LOG.debug('{} #refs: {}'.format(self.name, self.ref_count())) def ref_count(self): return len(self.refs) def put(self, key, value): self.last_used = time.time() self.max_counter += 1 new_max_counter = '%016x' % self.max_counter batch = self.db.write_batch() batch.put(key+new_max_counter, value) batch.put(TABLE_MAX_COUNTER_KEY, new_max_counter) batch.write() def backwards_iterator(self, timestamp, counter): return self.db.iterator( start=START_KEY, stop=('%016x%016x' % (timestamp, counter)), include_start=False, include_stop=True, reverse=True ) def close(self): LOG.debug('{} - close'.format(self.name)) self.db.close() @staticmethod def oldest_table(): # XXX we should switch to something iterative entries = os.listdir('.') if len(entries) == 0: return None tables = [] for e in entries: try: int(e, 16) except: continue tables.append(e) if len(tables) == 0: return None tables = sorted(tables) return tables[0] def _lock_current_tables(): """Decorator for locking current_tables """ def _lock_out(f): def _lock_in(self, *args, **kwargs): self.current_tables_lock.acquire() try: result = f(self, *args, **kwargs) finally: self.current_tables_lock.release() return result return _lock_in return _lock_out class Store(object): def __init__(self, config=None): if config is None: config = {} self._stop = gevent.event.Event() self.max_tables = config.get('max_tables', 5) self.current_tables = {} self.current_tables_lock = gevent.lock.BoundedSemaphore() self.max_written_timestamp = None self.max_written_counter = 0 self.add_queue = gevent.queue.PriorityQueue() def _open_table(self, name, create_if_missing): table = Table(name, create_if_missing=create_if_missing) self.current_tables[name] = table return table def _close_table(self, table): table.close() self.current_tables.pop(table.name) def _add_table(self, name, priority, create_if_missing=True): self.current_tables_lock.acquire() if len(self.current_tables) < self.max_tables: try: result = self._open_table( name, create_if_missing=create_if_missing ) finally: self.current_tables_lock.release() return result self.current_tables_lock.release() future_table = gevent.event.AsyncResult() self.add_queue.put(( priority, (future_table, name, create_if_missing) )) self._process_queue() return future_table.get() def _get_table(self, day, ref, create_if_missing=True): table = self.current_tables.get(day, None) if table is None: prio = 99 if ref != 'write' else 1 table = self._add_table( day, prio, create_if_missing=create_if_missing ) table.add_reference(ref) return table @_lock_current_tables() def _process_queue_element(self, name, ftable, create_if_missing): if name in self.current_tables: ftable.set(self.current_tables[name]) return True if len(self.current_tables) < self.max_tables: new_table = self._open_table( name, create_if_missing=create_if_missing ) ftable.set(new_table) return True # garbage collect candidate = None for _, table in self.current_tables.iteritems(): if table.ref_count() != 0: continue if candidate is None or candidate.last_used > table.last_used: candidate = table if candidate is None: return False self._close_table(candidate) new_table = self._open_table( name, create_if_missing=create_if_missing ) ftable.set(new_table) return True def _process_queue(self): # this is for perf improvement if self.add_queue.empty(): return try: while True: prio, (ftable, name, create_if_missing) = \ self.add_queue.get_nowait() result = self._process_queue_element( name, ftable, create_if_missing ) if not result: prio = (prio - 1) if prio > 2 else prio self.add_queue.put(( prio, (ftable, name, create_if_missing) )) return except IndexError: return except Queue.Empty: return def _release(self, table, ref): table.remove_reference(ref) self._process_queue() def write(self, timestamp, log): if self._stop.is_set(): raise RuntimeError('stopping') tssec = timestamp/1000 day = '%016x' % (tssec - (tssec % 86400)) table = self._get_table(day, 'write') try: table.put( '%016x' % timestamp, log ) finally: self._release(table, 'write') def iterate_backwards(self, ref, timestamp, counter): if self._stop.is_set(): raise RuntimeError('stopping') tssec = timestamp/1000 current_day = (tssec - (tssec % 86400)) oldest_table = Table.oldest_table() if oldest_table is None: yield {'msg': 'No more logs to check'} return while True: table_name = '%016x' % current_day if table_name < oldest_table: yield {'msg': 'No more logs to check'} return day = datetime.datetime.fromtimestamp( current_day, pytz.UTC ) day = '%04d-%02d-%02d' % ( day.year, day.month, day.day ) yield {'msg': 'Checking %s' % day} try: table = self._get_table( table_name, ref, create_if_missing=False ) except TableNotFound: if current_day == 0: # XXX this is unreachable yield {'msg': 'This should be unreachable'} return current_day -= 86400 continue table_iterator = table.backwards_iterator( timestamp=timestamp, counter=counter ) for linets, line in table_iterator: yield { 'timestamp': int(linets[:16], 16), 'counter': int(linets[16:], 16), 'log': line } self._release(table, ref) if current_day == 0: yield {'msg': 'We haved reached the origins of time'} return current_day -= 86400 def release_all(self, ref): if self._stop.is_set(): raise RuntimeError('stopping') self.current_tables_lock.acquire() for t in self.current_tables.values(): t.remove_reference(ref) self.current_tables_lock.release() self._process_queue() def stop(self): LOG.info('Store - stop called') if self._stop.is_set(): return self._stop.set() self.current_tables_lock.acquire() for t in self.current_tables.keys(): self.current_tables[t].close() self.current_tables.pop(t, None) self.current_tables_lock.release() ================================================ FILE: minemeld/traced/writer.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements the writer class for logs """ import logging import psutil import ujson import gevent import gevent.event LOG = logging.getLogger(__name__) class DiskSpaceMonitor(gevent.Greenlet): def __init__(self, threshold, low_disk): self._threshold = threshold self._low_disk = low_disk super(DiskSpaceMonitor, self).__init__() def _run(self): while True: perc_used = psutil.disk_usage('.').percent if perc_used >= self._threshold: if not self._low_disk.is_set(): self._low_disk.set() LOG.critical( 'Disk space used above threshold ({}%), writing disabled'.format(self._threshold) ) else: if self._low_disk.is_set(): self._low_disk.clear() LOG.info('Disk space used below threshold, writing restored') gevent.sleep(60) class Writer(object): def __init__(self, comm, store, topic, config): self._stop = gevent.event.Event() self._low_disk = gevent.event.Event() self.store = store self.comm = comm self.comm.request_sub_channel( topic, self, allowed_methods=['log'], name='mbus:log:writer', multi_write=True ) self._disk_monitor_glet = DiskSpaceMonitor( threshold=config.get('threshold', 70), low_disk=self._low_disk ) self._disk_monitor_glet.start() def log(self, timestamp, **kwargs): if self._stop.is_set(): return if self._low_disk.is_set(): return self.store.write(timestamp, ujson.dumps(kwargs)) def stop(self): LOG.info('Writer - stop called') if self._stop.is_set(): return self._stop.set() self._disk_monitor_glet.kill() ================================================ FILE: nodes.json ================================================ { "minemeld.ft.anomali.Intelligence": { "class": "minemeld.ft.anomali:Intelligence" }, "minemeld.ft.auscert.MaliciousURLFeed": { "class": "minemeld.ft.auscert:MaliciousURLFeed" }, "minemeld.ft.autofocus.ExportList": { "class": "minemeld.ft.autofocus:ExportList" }, "minemeld.ft.azure.AzureXML": { "class": "minemeld.ft.azure:AzureXML" }, "minemeld.ft.azure.AzureJSON": { "class": "minemeld.ft.azure:AzureJSON" }, "minemeld.ft.cif.Feed": { "class": "minemeld.ft.cif:Feed" }, "minemeld.ft.ciscoise.ErsSgt": { "class": "minemeld.ft.ciscoise:ErsSgt" }, "minemeld.ft.csv.CSVFT": { "class": "minemeld.ft.csv:CSVFT" }, "minemeld.ft.dag.DagPusher": { "class": "minemeld.ft.dag:DagPusher" }, "minemeld.ft.dag_ng.DagPusher": { "class": "minemeld.ft.dag_ng:DagPusher" }, "minemeld.ft.google.GoogleNetBlocks": { "class": "minemeld.ft.google:GoogleNetBlocks" }, "minemeld.ft.google.GoogleCloudNetBlocks": { "class": "minemeld.ft.google:GoogleCloudNetBlocks" }, "minemeld.ft.google.GoogleSPF": { "class": "minemeld.ft.google:GoogleSPF" }, "minemeld.ft.http.HttpFT": { "class": "minemeld.ft.http:HttpFT" }, "minemeld.ft.ipop.AggregateIPv4FT": { "class": "minemeld.ft.ipop:AggregateIPv4FT" }, "minemeld.ft.json.SimpleJSON": { "class": "minemeld.ft.json:SimpleJSON" }, "minemeld.ft.local.YamlFT": { "class": "minemeld.ft.local:YamlFT" }, "minemeld.ft.local.YamlIPv4FT": { "class": "minemeld.ft.local:YamlIPv4FT" }, "minemeld.ft.local.YamlURLFT": { "class": "minemeld.ft.local:YamlURLFT" }, "minemeld.ft.local.YamlDomainFT": { "class": "minemeld.ft.local:YamlDomainFT" }, "minemeld.ft.local.YamlIPv6FT": { "class": "minemeld.ft.local:YamlIPv6FT" }, "minemeld.ft.logstash.LogstashOutput": { "class": "minemeld.ft.logstash:LogstashOutput" }, "minemeld.ft.o365.O365XML": { "class": "minemeld.ft.o365:O365XML" }, "minemeld.ft.o365.O365API": { "class": "minemeld.ft.o365:O365API" }, "minemeld.ft.op.AggregateFT": { "class": "minemeld.ft.op:AggregateFT" }, "minemeld.ft.phishme.Intelligence": { "class": "minemeld.ft.phishme:Intelligence" }, "minemeld.ft.proofpoint.ETIntelligence": { "class": "minemeld.ft.proofpoint:ETIntelligence" }, "minemeld.ft.proofpoint.EmergingThreatsIP": { "class": "minemeld.ft.proofpoint:EmergingThreatsIP" }, "minemeld.ft.proofpoint.EmergingThreatsDomain": { "class": "minemeld.ft.proofpoint:EmergingThreatsDomain" }, "minemeld.ft.recordedfuture.IPRiskList": { "class": "minemeld.ft.recordedfuture:IPRiskList" }, "minemeld.ft.recordedfuture.DomainRiskList": { "class": "minemeld.ft.recordedfuture:DomainRiskList" }, "minemeld.ft.recordedfuture.MasterRiskList": { "class": "minemeld.ft.recordedfuture:MasterRiskList" }, "minemeld.ft.redis.RedisSet": { "class": "minemeld.ft.redis:RedisSet" }, "minemeld.ft.syslog.SyslogMatcher": { "class": "minemeld.ft.syslog:SyslogMatcher" }, "minemeld.ft.syslog.SyslogMiner": { "class": "minemeld.ft.syslog:SyslogMiner" }, "minemeld.ft.taxii.TaxiiClient": { "class": "minemeld.ft.taxii:TaxiiClient" }, "minemeld.ft.taxii.DataFeed": { "class": "minemeld.ft.taxii:DataFeed" }, "minemeld.ft.threatq.Export": { "class": "minemeld.ft.threatq:Export" }, "minemeld.ft.tmt.DTIAPI": { "class": "minemeld.ft.tmt:DTIAPI" }, "minemeld.ft.vt.Notifications": { "class": "minemeld.ft.vt:Notifications" }, "minemeld.ft.mm.JSONSEQMiner": { "class": "minemeld.ft.mm:JSONSEQMiner" }, "minemeld.ft.localdb.Miner": { "class": "minemeld.ft.localdb:Miner" }, "minemeld.ft.threatconnect.IndicatorsMiner": { "class": "minemeld.ft.threatconnect:IndicatorsMiner" }, "minemeld.ft.threatconnect.GroupsMiner": { "class": "minemeld.ft.threatconnect:GroupsMiner" }, "minemeld.ft.visa.VTI": { "class": "minemeld.ft.visa:VTI" }, "minemeld.ft.taxii2.Taxii2Client": { "class": "minemeld.ft.taxii2:Taxii2Client" }, "minemeld.ft.cofense.Triage": { "class": "minemeld.ft.cofense:Triage" }, "minemeld.ft.bambenek.Miner": { "class": "minemeld.ft.bambenek:Miner" } } ================================================ FILE: requirements-dev.txt ================================================ Sphinx==1.3.1 sphinx-rtd-theme==0.1.8 tox==2.1.1 mock==1.3.0 git+https://github.com/PaloAltoNetworks/platter#egg=platter cython==0.24 virtualenv==16.7.9 ================================================ FILE: requirements-web.txt ================================================ Flask==0.12.4 gunicorn==19.5.0 psutil==5.6.6 Flask-Login==0.2.11 passlib==1.6.5 rrdtool==0.1.2 supervisor==3.1.3 blinker==1.4 ================================================ FILE: requirements.txt ================================================ pip>=9.0.1 amqp==1.4.6 gevent==1.0.2 greenlet==0.4.7 hiredis==0.2.0 PyYAML==5.4 redis==2.10.5 requests==2.20.0 plyvel==0.9 netaddr==0.7.18 jmespath==0.7.1 click==4.1 pan-python==0.10.0 stix==1.1.1.8 cybox==2.1.0.17 six==1.11.0 lxml==4.6.3 stix-edh==1.0.0 libtaxii==1.1.107 pytz==2015.4 certifi ujson==1.34 filelock==2.0.4 sleekxmpp==1.3.1 beautifulsoup4==4.4.1 cifsdk==2.0.14 lz4==2.2.1 networkx==1.11 unicodecsv==0.14.1 Werkzeug==0.12.2 pyzmq==16.0.3 stix2-patterns==1.1.0 ================================================ FILE: scripts/prebuild-script.sh ================================================ #!/bin/bash pip install cython==0.24 ================================================ FILE: setup.py ================================================ # Copyright 2015-2017 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json from setuptools import Extension, setup, find_packages try: from Cython.Build import cythonize except ImportError: # this is for platter cythonize = lambda x: x import sys import os.path sys.path.insert(0, os.path.abspath('.')) from minemeld import __version__ with open('nodes.json') as f: _entry_points = { 'minemeld_nodes': [], 'minemeld_nodes_gcs': [], 'minemeld_nodes_validators': [] } _nodes = json.load(f) for node, v in _nodes.iteritems(): _entry_points['minemeld_nodes'].append( '{} = {}'.format(node, v['class']) ) if 'gc' in v: _entry_points['minemeld_nodes_gcs'].append( '{} = {}'.format(node, v['gc']) ) if 'validator' in v: _entry_points['minemeld_nodes_validators'].append( '{} = {}'.format(node, v['validator']) ) with open('requirements.txt') as f: _requirements = f.read().splitlines() with open('README.md') as f: _long_description = f.read() _packages = find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]) GDNS = Extension( name='minemeld.packages.gdns._ares', sources=['minemeld/packages/gdns/_ares.pyx'], include_dirs=[], libraries=['cares'], define_macros=[('HAVE_NETDB_H', '')], depends=['minemeld/packages/gdns/dnshelper.c'] ) setup( name='minemeld-core', version=__version__, url='https://github.com/PaloAltoNetworks-BD/minemeld-core', author='Palo Alto Networks', author_email='techbizdev@paloaltonetworks.com', description='An extensible indicator processing framework', classifiers=[ 'Development Status :: 3 - Alpha', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python :: 2.7', 'Topic :: Security', 'Topic :: Internet' ], long_description=_long_description, packages=_packages, provides=['minemeld'], install_requires=_requirements, ext_modules=cythonize([GDNS]), entry_points={ 'console_scripts': [ 'mm-run = minemeld.run.launcher:main', 'mm-console = minemeld.run.console:main', 'mm-traced = minemeld.traced.main:main', 'mm-traced-purge = minemeld.traced.purge:main', 'mm-supervisord-listener = minemeld.supervisord.listener:main', 'mm-extensions-freeze = minemeld.run.freeze:main', 'mm-cacert-merge = minemeld.run.cacert_merge:main', 'mm-restore = minemeld.run.restore:main', 'mm-extension-from-git = minemeld.run.extgit:main' ], 'minemeld_nodes': _entry_points['minemeld_nodes'], 'minemeld_nodes_gcs': _entry_points['minemeld_nodes_gcs'], 'minemeld_nodes_validators': _entry_points['minemeld_nodes_validators'] } ) ================================================ FILE: tests/comm_mock.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements mock classes for minemed.comm tests """ class MockComm(object): def __init__(self, config): self.config = config self.sub_channels = [] self.rpc_server_channels = [] def request_sub_channel(self, topic, obj=None, allowed_methods=None, name=None): self.sub_channels.append({ 'topic': topic, 'obj': obj, 'allowed_methods': allowed_methods, 'name': name }) def request_rpc_server_channel(self, name, obj=None, allowed_methods=[], method_prefix='', fanout=None): self.rpc_server_channels.append({ 'name': name, 'obj': obj, 'allowed_methods': allowed_methods, 'method_prefix': method_prefix, 'fanout': fanout }) def comm_factory(config): return MockComm(config) ================================================ FILE: tests/empty.yml ================================================ ================================================ FILE: tests/feeds.htpasswd ================================================ user1:$apr1$SdhtTFdb$br1vVDVDr3r/ZYTo0aj.L0 guest:$apr1$fUbzwJlK$tXcBNLcN8zhXNGjgB7M6./ ================================================ FILE: tests/integration/basic/DomainHC%3Fv%3Dcarbonblack.result ================================================ { "feedinfo": { "category": "MineMeld", "icon": "iVBORw0KGgoAAAANSUhEUgAAASwAAAFLCAYAAABsjLGXAAAMFmlDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnltSCAktEAEpoTdBehUIHQQB6WAjJAFCCZAQVOzIooJrQcWCFV0Bsa0FkLUiioVFwF4XRFRW1sWCDZU3KaDP1753vm/u/Dlzzpn/zD13MgOAsi07NzcLVQEgW5AvjAryZSYkJjFJPUABUAEFGACczRHl+kRGhgEoo/0/y7tbAJH0160lsf51/L+KKpcn4gCAREKcwhVxsiE+BgCuyckV5gNAaIN6o9n5uRI8CLG6EBIEgIhLcJoMa0pwigxPkNrERPlBzAKATGWzhWkAKEl4Mws4aTCOkoSjrYDLF0C8FWIvTjqbC/EDiCdkZ+dArEyG2Dzluzhp/xQzZSwmm502hmW5SIXszxflZrHn/p/L8b8lO0s8OochbNR0YXCUJGe4bjWZOaESTIX4pCAlPAJiNYgv8blSewm+ly4OjpXbD3BEfnDNAAMAFHDZ/qEQ60DMEGfG+sixPVso9YX2aDg/PyRGjlOEOVHy+GiBICs8TB5neTovZBRv54kCokdtUvmBIRDDSkOPFabHxMt4oi0F/LhwiJUg7hBlRofKfR8VpvuFj9oIxVESzsYQv00VBkbJbDDNbNFoXpgNhy2dC9YCxspPjwmW+WIJPFFC2CgHLs8/QMYB4/IEsXJuGKwu3yi5b0luVqTcHtvOywqKkq0zdlhUED3q25UPC0y2DtjjDPbkSPlc73LzI2Nk3HAUhAE/4A+YQAxbCsgBGYDfPtAwAH/JRgIBGwhBGuABa7lm1CNeOiKAz2hQCP6CiAdEY36+0lEeKID6L2Na2dMapEpHC6QemeApxNm4Nu6Fe+Bh8MmCzR53xd1G/ZjKo7MSA4j+xGBiINFijAcHss6CTQj4/0YXCnsezE7CRTCaw7d4hKeETsJjwk1CN+EuiANPpFHkVrP4RcIfmDPBFNANowXKs0v5PjvcFLJ2wn1xT8gfcscZuDawxh1hJj64N8zNCWq/Zyge4/ZtLX+cT8L6+3zkeiVLJSc5i5SxN+M3ZvVjFL/v1ogL+9AfLbHl2FGsFTuHXcZOYg2AiZ3BGrE27JQEj1XCE2kljM4WJeWWCePwR21s62z7bT//MDdbPr9kvUT5vDn5ko/BLyd3rpCflp7P9IG7MY8ZIuDYTGDa29q5ACDZ22VbxxuGdM9GGFe+6fLOAuBWCpVp33RsIwBOPAWA/u6bzug1LPc1AJzq4IiFBTKdZDsGBPiPoQy/Ci2gB4yAOczHHjgDD8ACAWAyiAAxIBHMhCueDrIh59lgPlgCSkAZWAM2gC1gB9gNasABcAQ0gJPgHLgIroIOcBPch3XRB16AQfAODCMIQkJoCB3RQvQRE8QKsUdcES8kAAlDopBEJBlJQwSIGJmPLEXKkHJkC7ILqUV+RU4g55DLSCdyF+lB+pHXyCcUQ6moOqqLmqITUVfUBw1FY9AZaBqahxaixegqdBNahe5H69Fz6FX0JtqNvkCHMIApYgzMALPGXDE/LAJLwlIxIbYQK8UqsCrsINYE3/N1rBsbwD7iRJyOM3FrWJvBeCzOwfPwhfhKfAteg9fjLfh1vAcfxL8SaAQdghXBnRBCSCCkEWYTSggVhL2E44QL8LvpI7wjEokMohnRBX6XicQM4jziSuI24iHiWWInsZc4RCKRtEhWJE9SBIlNyieVkDaT9pPOkLpIfaQPZEWyPtmeHEhOIgvIReQK8j7yaXIX+Rl5WEFFwUTBXSFCgaswV2G1wh6FJoVrCn0KwxRVihnFkxJDyaAsoWyiHKRcoDygvFFUVDRUdFOcqshXXKy4SfGw4iXFHsWPVDWqJdWPOp0qpq6iVlPPUu9S39BoNFMai5ZEy6etotXSztMe0T4o0ZVslEKUuEqLlCqV6pW6lF4qKyibKPsoz1QuVK5QPqp8TXlARUHFVMVPha2yUKVS5YTKbZUhVbqqnWqEarbqStV9qpdVn6uR1EzVAtS4asVqu9XOq/XSMboR3Y/OoS+l76FfoPepE9XN1EPUM9TL1A+ot6sPaqhpOGrEaczRqNQ4pdHNwBimjBBGFmM14wjjFuPTON1xPuN441aMOziua9x7zfGaLE2eZqnmIc2bmp+0mFoBWplaa7UatB5q49qW2lO1Z2tv176gPTBefbzHeM740vFHxt/TQXUsdaJ05uns1mnTGdLV0w3SzdXdrHted0CPocfSy9Bbr3dar1+fru+lz9dfr39G/0+mBtOHmcXcxGxhDhroGAQbiA12GbQbDBuaGcYaFhkeMnxoRDFyNUo1Wm/UbDRorG88xXi+cZ3xPRMFE1eTdJONJq0m703NTONNl5k2mD430zQLMSs0qzN7YE4z9zbPM68yv2FBtHC1yLTYZtFhiVo6WaZbVlpes0KtnK34VtusOicQJrhNEEyomnDbmmrtY11gXWfdY8OwCbMpsmmweTnReGLSxLUTWyd+tXWyzbLdY3vfTs1usl2RXZPda3tLe459pf0NB5pDoMMih0aHV45WjjzH7Y53nOhOU5yWOTU7fXF2cRY6H3TudzF2SXbZ6nLbVd010nWl6yU3gpuv2yK3k24f3Z3d892PuP/tYe2R6bHP4/kks0m8SXsm9XoaerI9d3l2ezG9kr12enV7G3izvau8H7OMWFzWXtYzHwufDJ/9Pi99bX2Fvsd93/u5+y3wO+uP+Qf5l/q3B6gFxAZsCXgUaBiYFlgXOBjkFDQv6GwwITg0eG3w7RDdEE5IbcjgZJfJCya3hFJDo0O3hD4OswwThjVNQadMnrJuyoNwk3BBeEMEiAiJWBfxMNIsMi/yt6nEqZFTK6c+jbKLmh/VGk2PnhW9L/pdjG/M6pj7seax4tjmOOW46XG1ce/j/ePL47sTJiYsSLiaqJ3IT2xMIiXFJe1NGpoWMG3DtL7pTtNLpt+aYTZjzozLM7VnZs08NUt5FnvW0WRCcnzyvuTP7Ah2FXsoJSRla8ogx4+zkfOCy+Ku5/bzPHnlvGepnqnlqc/TPNPWpfWne6dXpA/w/fhb+K8ygjN2ZLzPjMiszhzJis86lE3OTs4+IVATZApacvRy5uR05lrlluR257nnbcgbFIYK94oQ0QxRY746POa0ic3FP4l7CrwKKgs+zI6bfXSO6hzBnLa5lnNXzH1WGFj4yzx8Hmde83yD+Uvm9yzwWbBrIbIwZWHzIqNFxYv6FgctrllCWZK55Pci26LyordL45c2FesWLy7u/Snop7oSpRJhye1lHst2LMeX85e3r3BYsXnF11Ju6ZUy27KKss8rOSuv/Gz386afR1alrmpf7bx6+xriGsGaW2u919aUq5YXlveum7Kufj1zfen6txtmbbhc4VixYyNlo3hj96awTY2bjTev2fx5S/qWm5W+lYe26mxdsfX9Nu62ru2s7Qd36O4o2/FpJ3/nnV1Bu+qrTKsqdhN3F+x+uiduT+svrr/U7tXeW7b3S7Wgursmqqal1qW2dp/OvtV1aJ24rn//9P0dB/wPNB60PrjrEONQ2WFwWHz4z1+Tf711JPRI81HXowePmRzbepx+vLQeqZ9bP9iQ3tDdmNjYeWLyieYmj6bjv9n8Vn3S4GTlKY1Tq09TThefHjlTeGbobO7ZgXNp53qbZzXfP59w/kbL1Jb2C6EXLl0MvHi+1af1zCXPSycvu18+ccX1SsNV56v1bU5tx393+v14u3N7/TWXa40dbh1NnZM6T3d5d5277n/94o2QG1dvht/svBV7687t6be773DvPL+bdffVvYJ7w/cXPyA8KH2o8rDikc6jqj8s/jjU7dx9qse/p+1x9OP7vZzeF09ETz73FT+lPa14pv+s9rn985P9gf0df077s+9F7ovhgZK/VP/a+tL85bG/WX+3DSYM9r0Svhp5vfKN1pvqt45vm4cihx69y343/L70g9aHmo+uH1s/xX96Njz7M+nzpi8WX5q+hn59MJI9MpLLFrKlRwEMNjQ1FYDX1QDQEuHZoQMAipLs7iUVRHZflCLwn7DsfiYVZwCqWQDELgYgDJ5RtsNmAjEV9pKjdwwLoA4OY00uolQHe1ksKrzBED6MjLzRBYDUBMAX4cjI8LaRkS97INm7AJzNk935JEKE5/udEyWoo++PQfCD/AMf7G3o0obnYAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAAgRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iCiAgICAgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjk5NjwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj45MDI8L2V4aWY6UGl4ZWxYRGltZW5zaW9uPgogICAgICAgICA8dGlmZjpPcmllbnRhdGlvbj4xPC90aWZmOk9yaWVudGF0aW9uPgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4KTlGaRAAAQABJREFUeAHsvQmYXNlVJvi2WHPRmirtS2qXajNZA8ZuqJDBXxsGA20mhdtuFg9QYMyOh6U/BoWmjcHdA3xA21DFzLDaMMpv+gNc2GCXrZRtDIbKqrJdUqm0lfaUlJJyz4h46/z/vfFSKeUWEflexI1U3CplbO/dd+6555577lk1rdVaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgZaGGhhoIWBFgYeCgzoD8UoW4NsYUARDARBgDV3VO/rO6l3dR2cXn9DQyeD3t6DgablA13X8NpqLQy0MNDCQJ0xEASafizoNY8fz1n5QDMqeTyv5z/eW8n1D9M1LYQ8TLPdGmvdMJAP8saGgU+aP/7UgDPzoZCwrK9c+4P1CTfTYSTc1UEwldb1dKFgF+52ZFcMP/bIj92ccb1+/HjePHQo78747qF+22JYD/X0twYfNQbIqJ7u7zcOHeqfZjJfPpff5enFb/W94rd6vn3QDSZ3ekExbeiJFI5/hh8E+N8pWXp2zDCSr+pG6gupoOPv3rrnw68QvnxeM44cyWu6nvejhrfZ+msxrGabsRa8SmKAx7d+LWce0iWjOnvnLzuv3vnKuxxv/D1+UHxrpk3P6oanua6tObamBWA9vu8LZRWYlmbopmaYgZZMmZppWtrkuOaZWuazycTa//b0rv/z8xz08SBnhf0riYQ6ANViWHVAcusRyxsD1Dcd1vs8jvLkrWPtg8P9P+75Yz+ZyjrdWuBpxaKn+R7eaLqvQ06CRt0Q2imhgJ+JG/wEaQvsTNMNzcq2W5pdMPEp/acr/L0f/Kb9//kOdWEzpbeZdz8M71sM62GY5dYYY8PAsy/2JEI91Qun3/+fvGD0aLbD7y4Vbc0u+WRS4E9gP1pQkcJ9GlAdzC3QPPxvda5M6ZOj+uWEseo9b9v3R/+UP65Z+UPa9JFz+p6H4E2LYT0Ek9waYjwYOA7GcQiM4+zg73ddHHv5jxKpqXe5rqPZRd8Fj8LaCswInkwXByeVMZJ2UfdSxqp3vW3fH//dwypptRhWBBTV6uLhwgD1VX041h3WNe9L537y0KQz/PFMu7thYtzxtIASlWZFjpFAd6yklvCh2Uqaa7/rbXv+8O8fRqZVnZga+Sy0OmxhoLkwQGZ19Kimk1mdOPPMeybsW5+zkqUNE2OurQc8+sXArIgiPUh4juYYpqeV3Nt9J87/8mPUZZFpNRcGlwZti2EtDX+tux8yDBzt10y4GfjHz/zAT3jm8Mc1w9PtokZfqySVVXGiAxr5hOsEdqZNyxRKV/7sxeDFBJlWPp9/aNZxrAiOc/JafbcwUG8MhDqrL5372e8vaYN/bTsO3ROo/K63lON0rEgm7ImV//u37Xv2Q8cCDVZKTVgp642Tej/voeHM9UZs63nLCwO0BlLB/qULv/otE/aNv3KgXNcaw6wgx+nG1KStuf74L754/fe2klkdO9YbhYJf+UlrMSzlp6gFYKMxQGZA14WvXfr4qqnStT9Pph098HQHR8B6S1YSFUFgeq7mZDv9lePjZ9/HL7t6hx6K01JjEN5oCmw9v4WBSjEAJfthrU+ExAwVjv/3dHth+8RYAF91LVlpFzFdZ9i2rcFBtRfxiR/WdTBQOKKi0Uq5bFtLwlq2U9saWBQYeHbgGQuq9OALZ3/+e4zkxHvGxxwyr0QUfS+xDwP+XhrCfg7864Vfe4J99fcfWvbHwpaEtUSqad2+fDFQllgouVifOf1D/0fCcsC7dDKshq8bxh9C+POybZo5XrzxjYDpxeU7E/dG1pKw7uGi9a6FgfswEEos/Wd/5j+lsqXHi1OeBz6hhBQjcmUhdlp4fvnuXgJ+Itcvjq73DWKZfWj4TrHM8NkazjLBQFm6cvFqfvb0D33AQrQNvTfj9rWqFn0IqoaxsiCOhHkdJ0SIXcs5Y2lLwqqWQlrXPxQY6O8/KiSpL1745ZxhlZ4qTrnMbayEdDU9AdS0u65m6enNQXBshfwebGsZtxbDWsaT2xpa7RgYyuWFtc12br8rmYbYopmuatIVbAFGqRRoCb24bXj4lS1ytCdbDKv2aW/d2cJA82GAx6rQc9z1x3o8HLvwn3KMgAzUx4kwm9EtvNklMX2vsEXzYX5xiFsS1uI4al3xkGGgr69XrIvPvd67KQi8XY4tMoMqt1bIQeF65XW24dUfOsBpGhj4pHKMNUryaSnda8SmsNJg3w1LNrEbWabpCLZi0EyrVFONmG38bV1d0mvccYNupLVa47m+nNHGgzYXBHoS4Yy6Ye3jjz09A8s6prDFsOYigTm+Y0T8009rxpmOT+qDIApYYmBCFpuZ0HXIW5AlScvzrX480KychrBU7bC/nK02HOxybZCrnmhrY3515LlSTeEOgEh4TBPIJMqePy4YFumybOGcQZfLZ4ZaDGuRuQyQr7uvr087fDhPor3Pz4Um76/cPdo2OvzVtGms0dfoB0tPbv+5cRhvvEM6o/gPi94fxkRri6BV6Z9P5HKY537CeAAMgJyBWnflWplhGboL0tT1LbdufXT9unUfuNHXd5jH12UpaSk4DWrQBerBWYdO5H0ITIJJXbny5cxVv/8bbGf0m31/6smSO7QDkKZBzSuCwM4iw4dhGtYUnKDHTCM7lDRWfDGZ7Pj7N2/79QGOKJ/XjAMHevXDh2WxAjVG2YLiQQzM9GP61Ml3fTHbZv67qQn4DuiN925/EFYo3DUU2fEPrvQMCwdD3dz/VFfXfxlYzhtkS8J6gAoYmX/qVF8QFq986epvPzE2dek9r008+92BVtqX7QCP9x0tDU2VTxMNLUiivgA99myUanI0wyiCeMa+fWRcP/LC6R/5x2RixW9/687f+Zym9WnLmZgeQGVTfjyq5THB+eBTZ9/Xpduj2x2GDrLGjWIHLEoaBCkF2CBO+ZmsaRSKE4/iK7FBNiXyKwC6xbBCJIHfPDvQYx1+qk9U6v3nSx9+61TxygeHxgfemWn3TNN2tBLKNY2P6DjqgVxIMfwrX8NeQEWCssHOPLzzE4kO5ztKxYnv+MxrP/DX7cbWD75l729cI9PK5fqpB1NsGdwbxsP67oAm/ZgMR9uCydnkkmFVWGK+3jgjqaWNQDdBbFbCM7Spyd2EAbS1bOlKOVNtvSedz2O1Xs4wcx69dPkjGz//+o/+6UTx1S9ZmaHv9bVJc2K05JaKyICk4TJdZ/Q+GT3/Ud95/z+pnLXQHyL6jWBq3Hdsu6RlOkvvHvPODrxw5offzrS2zAtO5Sj6aDWFMDA8sEqsCRz7H01nTcbiUBek5DyRZtNlkYOSPtI1P05UYiPkbqkkzIRvKe2hl7DEEU3Pu3lg8bOv/+gP3pp4+XezHd7qwhgyOo6bDvYvE6xF4IkEUmVDcqIgwRxF4yOOg8Lkj/jexGc+//qPfOBte//vj4G0DDAtENjyzmFUJc4aevmenjNimk3D2GMluOrpmxlJua5Ix0UgaSFMcasFh3I9FOzxi9tBT0nQE2tL44rlx7MeaglLKNYh7ZCSPnPqB/4wkRr7MzNhr0YFFIfSFJlNWUHFS5bSSDlJpxi4PmprGqmJj/af+emfyUOhD0Fr+VHVUjDV4HtzWr+wrgWa8wQqyWPZqykFkx1ZZFiIboSgbqBoK77xd9y69aEtRGHZUsi3y6o9tAzreAAr4KG8++L531rxmdPvfSG7wv6JqSnbc2yWFBcJ2qJnJDhKYhF4U5NFzTdv/96Jsx88nIfu/niQe+glXRVWFY/o1CuePHkM2USN7XQYRYueDpY4WAKE3VRLYPVa+Eem6iLS0TTddsNwtrL7rq7lGaLzUDKs8Bj4tUu/ueqO/crnM+3Ot42NQKsuswvFHZGP/g3X84ta0b3+xy8P/u72Q3q/ewz+Xkuk49btS8QAvO3EehhM/AOyH3g7yiE5yjEsDpMSFo+DlLKkVkF329oMvL/1GH/P5a4rCTdhW0p76BgW3Rao9L5y5VjmxtSrn850+N8wPuqUgESmva3XJFuOrdlglJ13Rk/9N07gYR15w5epopTja4bW1S9DcowgscO09DbPUzckhxJWmltfmWJRSAfiIbQbvr5L4vo5IR42A96rgfGhYlgU+UPHzdMTn/rLTIfzTROjYB3YrKpBWkTXJibGbLhwjf8vXzj7q0+jz+B4f74lZUWE3KV0Y3uTsBByaejKWgi5tZJhkV9R2sJfnVklNG1yWVsKHyqG1d+vC4bwudefOZLuKLwLkhWU6w0rKAB1iekm055mezc/QJI7kYNnfas1DAMnTvQL/MMXcx8lGDT5t2EQzf1gAsWFmzahvSpfQvWbg5BHfLP5zp1Pdcqv82X5q3zRMnh5aBiW0FuhEOYXzv1vb3X9kfzEGE+BOqa8bsfAWeSCZ5vFgouCmIV3/PMbv7kd5OU/LAUxZyGjwV9QD0SrLcHwA/sgpHFF2ZUEi7qr5P2rFwyL7Ku0NQi+vInj6O/vv/8KftnkbdkNaK754FGQeqsgOG4VnGu/l4BUg2SN8GgXMTVz3VKX72CQgnVH89NZv6PkXD7Eh4apTeoCQOsh0xiQITkoPfN6fi1MhdscGzRiqOlyQl5KZkUrId9LMUoeCZNJLYHvhMf79OCW0ZuHgmE9N/CUcBv4/Nm/fn+23espTnowAsPHqsGNtAZXat+AnOdpzrcQnPBY0mDQHrrHhyE5U8HwRi9wN7uUVmSQqFK4IHOiGJgCzSAkZ/pIiK/AZ3U3kyHYQ/sl0P3yZRn9XfYMK0DYDUNuzp79y07XG//5UgmCFb1tlGmIo4CXsueVRMZIHksoESoD3kMCSJcm/ZZsvXQgnTENTAL5gprzAF4qFO73MywwKgbki9ODoKVcbvmF6Cx7hhWe4y+5X/qP6ay3Ax7BmFHE/6nSYI/m8cPQjS0n3/joegnWUTUXiio4iwGOM2Fq4cDenUgy2tkgnajZQB2hhfBBAH14Jvv+WJjMj2Lig5c09WdxVGrqESwAPCUVism8xA1Gf8gUvEq1uD2hx4IDYHG9b42sA6g3+soZAxYYWuuniDEw+LxMLYxAhMeZNgj/KblpkP1wt6XT6GxWpKOKjocIVWPr5OSzG9rafnyQKbxx+exLI8Zfvbpb1gyr7Lnsff619/cUg6FvKhbEpqmOdCVmGelJfc/vbE8abVaGO+PXesXxhOmWW60eGChvbNJC6LvdmA5xFlRxlVPJznAcKt35/v4mNz9NK6yfmrpMSyEY1vIq+7Wsj4Td5VQhdnDnze0dhgGtBKUtpXZOpsSCP5aXTTsgwuui5PjAQF4pGO9fFMvvUxiS8y9X85uh0hYhOWBZyq0NEgW5apIhOWRYeP8AoYh0ONksEozoUwc5UwPlNcD3y6EpNylRIvVCz7DcNQNXTB6Oh7P2pCifV2tf2CmhLXXhweoKL+Xnn1+e+bhrxU/c94UhOa7jgGG5q+gxrpJZZub4ScBprFpaCOduhp9MetBjTe3i7z09zylJ83PDvvi3y/ZISCag6zJ/OqSYx0QqY0X1EuCqyApIxfvozvB4Un5dVsS2ODk29oqSfeexTJupTU46SNYYMNOUeg1AhRZC5nSfzbfgj8Ua1b77RBl4dY0HNWB3GUtY8lj1wmv/YQ2Y13am38Dszp7fGpAW+S1grcjfgDyBXvfw8HNbZP8yc0Dkz2p1OAsDQ0P9IA6hE9qlG9g+AiaDVY9WCCQlP/pgLUDIiClkiE5pGwZh4nq8qGlAmDURFXyxbCWssjuDb/vGViSNlLm559qQKkBSnJeQCJEURMQIGUZhhavd2YavLveXMwfE+exW34JJYbuQR3A/KD4RQGzBvgZ+JXiYUigStAJORQlrfuh0vYRK1dTFjYz8Fje/i8gFQsFkWUhay1jCwhShwUx9EI6AOvxTSIMLbEzy+nr/FUQIRorqJx5KSkGFckvoseoNx8P6vKNH84ImguDFLISRLS5ipTAnSq4LcFEtAWjDkJy554whOhiAYXdoTmkrr1lOm5+SEzP3RFT37dBQl9iEzMDaa4koUTVThQgixCwkDRoE4IXvu8LpD5VPuE22WswYOHBAmv2/cuFvNnlBqRsZZ/HExsaYzjVkclVCJpL2gV74fp7dV1gK27D5edqQSOY3V3/N+t2yPBJS/xAq3APdf8JHBaSFZrhRkzdNhBCsTIDoUQjUx8N8RogO4TgE5I0Ccdk/V6YS7tN8w+g2LS2NjAdQ+KiJdG5uzOFOCyFTX83DsDBnukjmh0uEpXA5bX5YJsuxcf/RtE+d/ekUXnbQEXCB2RXXNuqPIELYo6Adhe4BbmKBv31s7BNrJTz5+WmyUQAvs+d2dHxS4Lho3z6QxqEQxUekeUbBcZKqp0NyJInPC6Uo8htMJ/MTm9+8FzfRD8uSYYUVQwz7xhZQ4w6bSkhQoorzco8IA8NmcaagsKVYPAN/ILbl5aUsx6TW3wsXBuTRWzf2+fCFo1SrFoQSGtIJ0yEzad/iTSbz0wJvaxAsr2R+y/JIOJ1TSte3myZyc7s+7brKeY0+QIQ8h3jQPZjF0jgdXV8peykvC+vO4ous/leQOYUWQs+f2M/CbmqyK6nR4FEQxhmpI8D7+Rs3P1KXve3u3S9y8zu1XDY/JaWO+Seisl86OsbFdGL9P4bqvbhJV1LMJ0ndR4Sa9FIOggmheF9uXsqVzV49r8oLOrk08vwq0MhWh0dysjAFG3kprYMLWwinAdcRhgaP9yDh+6bQY2nl9DnTVzTpm2XJsC70dAsxH+rTPSrPy2wiLBcS8O0ny3C3pKtYJ1AeuW8OvbTR86c2eeBXEF+UY1gEiJtbBRbCe9hCTH02i/vKyfwGwvQ5965oynfLjmFRzEfJrPJC95UNyZmLCEFBSJmMODDN7g6CsylaCPP5/LKbI1VWSuif5CfcfcmUiSK33ELU9AonZFS4V0oMXAczk/n19Mj0OargvlY4Kh1/rf034D4p5v/DyR9ZDel+s8vqvYqG5MwmQl2n7kHXnO2jo38tFO9HjrQU73ER0VCun4KLViiN7k6kuMep6asXjj+MIRRAh1/O88qDLYO4A21cnDLwGexYTWY8zxDm/HrZMawww2jgO5sxQ1tEbm4oJuYcvQJf3k+EsuR4IhFkPW9MFBIIpQAFQF1uIOiHGQiBBl89JO3D20o4QQOwQLBoIZw7ad98AHHzc3m+3TY+/udMDKmF1vP57miG75cdwwqRrhvFfekUcnOLkBz1dpZ5iJCM1WMhAd8dFilxwvG0XqPFAKQNSt6CRbneRLdgWFzeCjYCKYwztB9V2Dg6RBlBrCqsLxTOMpnfsqjItOwY1lBOhuQg/9reZJoEqKaYfx8RimUzTYlYS6xa7wuPd3gpi7PK9K+tNxFh4KjgTq9c+cNNkL+77ZKwECq5HshbZ5f1WgQNOP4xvXtb1kLG5MKy2fyUnKBFpmL+n6lw1/rKFkIXCndcej8zmP/eOv8iiTCQZur7ns3KJ6C0oIjqw8xJL9KD3HdF68PSMRDmzbftSVoI1woLoYIKd3JVEjSzjFLKqoKcxa2JBFLNeDKZHza/Km5fOo7j6GFZMSyREkQucFQb13cKMV/so3GgrvY+CRIph7smK/jy/T0wWUiA+hR/5927R4UoXy4kUPsDW3fOwkCYPtvR7jyaaUvQqVhdSRbACQthmVZmDWaBL6gRQVC9SOaHzY920HuktsB9qv60rBhWuLA/9dp7t+BYJUNyKrcE13WOKGHdr3APHx+IkuOG4axx3clt8tuWpTDETlSv4+MydbDrFbsN04a7gCkk86j6j7ofQSvslPyn4oZEX3STCQrbIa2zcDDurqqDip9UrwuXGcOSCzthZDYbprZKpIqdKbzUC6uVPAf73DxEyB3QbW+nhnW4nBtr1TKbp0oQFO81h3IyoR183p5UOX022UsYQyhYTVXyEUxPkNZ13e8eHf2QcJNpdkvhsloIz5UrhPje5GOZjDCpQJOqnggsiBCAMdXtXESIhMmBAVsBCE2E6Gjac0rv/vGyluh7p24QVIGXwPL84hZPKrCqYgXRQzV3j6QPqg1ESjdBLHNfN/e3MnICDg6dtu1t4TUync7cVzfDt8uKYe0ZPyOm1DDMXZCwQJPKxTsLmuBxkETIQNa5GwgNugffL4S6B6/ZdQ9zj7Mx3/ZpfQLzLw9+dFOgueWkfdDwKNYIEGklCVoWMYT4XCWQuFyHtM4cBzKZXy53vcou1ELKvEtGLTArgkY/JF0AcGa3H5devur5XwkixHBIhKK2HAjyQQoic3IcUcxz+8jIxxCYy5Z/8DL5detv1Rjo6v+YwGXg2lv9oNTBzWHWJFTdazw3ULSmw2iVFsJpYCitI9kMTI2+CILu75e6u+kLmuzNsmFYiLkTYv6L15/NYr1vRUoZToWSizwkwtBCOJtmZIgO0oNs9v1LQvcQevDPvrb1Ta0YmCzdfDQLXSE2CNj+1UuLLMYFXjq3cabSUctjYaDLZH6HDiFZaRNbCpcNwzpQjrkbGX0RC9zbJkNy1GRYVFwtTIR0+guCTMYwPXvk0UpJs3VdZRg4Uc6Xj7q6OwPNBpEoqjrAcHhQncc4U9FgyZwYUA+V3eYg+GobbgL15ZXcyCsZ0LJJ4BdW7zV1o9s3jTSq+JanuxI01O8aUIuQ+5g5cgGqwU+6i3xGVmHK2c9bloPTH8fR6MYFTH8kwuF4Y0+wDCEZlpiXRgM3x/MpUaRxJAzpZo5LFvmK0jockf3S9tu3+yitv97MyfyWjYQVzlrJm0BZL+wniobkEE7aL6czR86zUrCqAhEGqRdFbiwssqYW5cP5afRrWNZraOh0B3z1NruOTVpRch2Qi1LPmQDB1M5R5ZEwmYTa1NR2Ev/NHFCv5ETVQtQnTvSLXdMwdYS0YNcsb0q19BXnPSERUukuiHBeMYsOpCK2aOf1659EKja4xZZr6MUJ33LvOyzrda30wgYvKO4QgriiZb1CXSdrEXJfm5dUFp407H26y2R+jjN0YOFL1f91WTAsLHw9n5divufZ+1m9V8VGgiMRMi5sMSLEmMxybqwdicRxoXg/Uq6hp+LYmgWm0A/JcUu7DMtLgVZgRVMTelIxJXE6ji6lcWNkMj8wrlC9wJ2wKduyYFihEvGlwV/tQmj6NiGZIJpQxRkJiZBm6kUaKvj6AUT5jOY5e8W1vQcXv2uRTh/2n6fLejmD+9KwJ4MduNzwVMQLGQ0V7mRYhLT2xoB6HDnKtQLAoOk0q+SYFxvjsmBYYeT95NjExsC3NwkLYVBxNtnFcBTt76A8OuFzV1+MCOFD42aojwvGhSi/XPJyR4vQ6nr7SLmsV6Cb+zwPhhll5SvJqFg4VbTFiGVBNOioosNkfvq2mzf/+BF56dGmZFjLwkrYJSqC9Gm2XjiQyiT0QqHkY3KUY8akOTKq+4hwAbLhzs+ME6xeTSJbLnm55YKp/19KFWjiOGS7w/utBGZkAfzXH8J7TySt1B5DeK8f+Y76UL4rbNSNyxvx5mazWgqVW9QSwdX9PVOuCOIFzu5EkvK9oewZnUQ4XwzhHKPmsRBSgL2s8nLPMc46fSWlipHga4weQFkvrGJwsDo9vKrHUA1LPWeFZb0W65tjRCZb1hefKkvrzRlQvywY1uDzsiJI4HvIzc29Sc3zOXUSJEIGsvJ9BU340MBDY+edO78kFO+adnhZzFkFY4/hEpnN48LlL2/w/MmNTCFMn4YYHrSkLgkQyaOqsl6LPtHwUyy04U+KWgEIqF/0DhUvaPojYVnMFy4NQeDuYKoQNCWJkGKfIEJARygrABJeylDG6f5K6Ft24JarzexDA/gb2kLcBcbEvkRKt4pFbhuKbm7AFFUH3J0ERS8Rc1wnMr7WFSmLnn9eptdZYrd1v73pGVaflDi8E+d+YUvBvrbDgVcvJlhJOZ+EJ4iwzLAqmG24YOteW4dhToyNktC+WME9rUvmwUBY1muieGt3IuNqxaJQHai5BkAsNM5QhRBR2XIwLIZMFneAeRk4G8KhQ+j0ouCH82A8+q+b/ngxHZITWEgV4q3iLqKoWkJslZVaCGdMNagKNGX4QveAEB0hTc74vfW2MgxMl/WC3fUxWWS0shvrfRU5iDTOROn9LEN0wKe2F+7+ziY5puZTLzQ9wwqJqejefjzbZiK1jOZTLAm/V+V1JhFWC5OwFPrFx3gfCLnp83JXO/4oroc0wTM4p0Gz/THk++cBXTkyEUMlkPTTmz9fmrisqj9kgEJnp9krC1phC28Oj8hVddTgi5ueYQ0N9QsixEruRuofkiDV7spRItdLLUQIaVEvlbC49KD7xo28KIgZOso2mHaa7PFHBU1cHPr/NuBcVC7rZShJ/6SVmWW9okA0j4Ho12vDpu6618XmF0W/9e5DzfN7hVjABPD0x60SXKr4pLAQkn0px66k4rQWIiShOQ69lJ1NpjW6FaO71aw+NJynRrU+7YCgijtTN1HWa6KLCVdANcpRCgEiCTN8SyR4xPvogNQDw3A1sOld6JYZQJpOvaDkDkNkVtLCYGAGB0Os2sIKIRR9VWshEdZopmbuE49FKQJ3XOyMA+Xc9aqNU2V4ugdeELTuasMH020mcBooeyakhBVNSM79M4LdHRlAwBB9mcwPa6Xp1AtNzbDCyPvzBQYHOyI3N+Y6DGa4f7Ya/GkpRIgQHd+yIE4avgheDUtUNXhITfX4EGcle2ynYZbgLtAkZb0ixDJo0HCpyAqCrUFwvF12nVdwi59/0E3NsMLIe983t5uWlsaRUOHIMCZiK4v3lPmraJAGhA8Njr0iRCcnS1Q1FaFVMdxYLj10QmbzCFBUVAQCR3nSihBikoaIhgiT9kXYNzgVivTCKgXXhtu3P1e2FDZXzcumZlhh5L3j3nk0nWV6Bl3JyHsSIY+qDMkRrXpWo7siN5bfHQTH2tEXeFhz7YyRrrsqOyPD1/I8/kAf6E5uVbqsF0hEGGdiOCeQBrmpIwNICu+EHksTcbhVIrSBlzc1w7pQjrzX9cRuqiQg8lbPCuqAfLIpUdYLRFhmWVU+FT40UvG+9c6dr5RDdJprZ6xywJFeHpb1Onf34xsR5gTVAZ2LuXzVa6SPmcaZKIHk+mAGECbz0/y75ZqXn4zyEbEjtGmthBL50kKIuLBHTYbkKIh6gkQ/i+nacjVxLHEkDDJpI1m0Jw+iy9PN6EMTOzXP84CwrNd4YWKLF0ytYDy5YFc1zcU8D4nga9IKQVpKWa/FwOC6EZu7rglH5L6+gaayFDaxhCWPRG8M/8lKKK42O7AQgmEpyLJmE2ENQGJkupdKk9pKYmdsFaVYbGnO/r1k3zhI52IsWmXLegG2WCyEITbIqEVMYTAhkkIePtxctQKalmH19/cL2K/dvrAJSsQtzMWmajK2SIgQhCyUxYH9DWXigzZCTQYdLg5VXj9Wdi52PXtXoJe4rykmW0lMCaDAUBapqLREtKKsKvWhgb59bOzZtbKzfA176BLBqPH2pj0STo/XKO1Noh5IqWRjvmlfUavdI8KlnVi5M7osDotUM0FwMqnrB22hTG7xrAUnnEwduBPOxZ42+Tid2siwxLwseGdjfuQuzAD5uODj8OmIjGR+61335ga8uV3e/JviaNi0ElYYeW+7o3uTOCpxJfMP/inXJBEuNZBVh0kayuLA3XXr1p9u4SD7+poveLXekxM6F4O5w7nY3uKoXtYLFFxFvrQa0BkW6UXRL2+M+tCmas3KsKYj78GiDjI4OLYtaYnTya2cFsKlEiGlKegegkRCQ/VeW5ike1tFKRadndC5+NUrz23wvKJwLqaIteiNdb6AAFHEScL1RWQZxfuYgES3updMekhbM7WTw2wmfWhTMiwsXs4mxSrN8ad2C4YV1/TyIUtokgjLqW7RzxKIEEcb3W1rQwJobVyUr28VpVh8YkLnYjdwujXDzjJJAw6ES5iGxZ9Z6xUkaGZoqKCiUq2PmL6P+lCk3hZFevFl0+hDm5JhoaSoILiXLv/BRrzZbotsBuoVnSCQkghRZDCCJUL9A6oVo09XxBS2ilJMr79533R0XBeYn3Ku78uA2YNdRZQPb95H1vwD9+E4YghnAwSlHpL5aUFpOzZ/JHonmZJS1W9NybDCsl4FZ3iT7xfXeczNrWiqW9IBiZBK86WSBPugSRq6GBFTiM/03hYLUn1SawyEFy48J5TJwNJez6eFUF2S5/ySVuJvZX2o5u0aHv5IOUTnaFPQkbqzt8CsdZezFTje8KOZtgQZgbIKd54BBcPieJbIsbADo74c9XV+99DQf6aFB62leJd4mP2XEin8jATDctyRA2VVwuwLFfiGpEFeRafRJZJJBaMJUCuAl9krHafYVJETTcmwwsh7Tysh8h75fZA/uIJZqvslIRGmQyJc4h5G+7zj+Mja4K01DHs7B9TyeF9oWvPEOPjUUIev+VsdFucjEhVsPA4y/xUjIvg+5kYcuB0dJnJj3Son82uOsl9NybByMlsBJtZ+glVy8J/SRJiIiAix8pA1Uvc62g3Nd+4KxXvMhN3k3ct4y1euPr/eCwqbheqAbliKNQLEHZdJ+1gGjvwqbiARUyiS+SGH4U6io7//ufjZJB+0xNZ0DIs6G+yReAksz3cQec/TYOzzWzWaQyKssqxXJc8RRSnApAXDasaskZUMMoprQulT1yf3mpaf8qhuV3VzA2Shwj2KsS/WB9cRl44fTAhLYVkIiJtPLgbWor83HcMql/XS/uXib+Hs7e3gEUmqtBcda90v4PKg13IUFsIQeHBrZI2kpbAgGBaYd9OYpMMx1Ov1hNYvHjVRGtyVSjMbrU7NjZqLEsQi8qUBujqJOtBjuUCGv/nKlS9npBCQVxM3Mwim6RjWvbJe3lYEGXQ0jYUwOiqUinct2DU29uE1ci7VJ7QZNFe/t+WkfXjgo5DGVRWu5BEQrCLeGMIH0c6YQkpY9o5U6jOb5K/qpyxq2ljCKe/2wUzW1MbHNQ9zDTlGrUb+RPUuFe5RblsQ5cGwOFZnk+eN0sJzp1WUgvi4v/HIgyaMMSV3eDeOhJgPFo6Jbue4/4m1fyJEYdK++kFHS6GvJRJBWvPcboBwLjxC1z6S+O9sOgnrxHSlj2A3JCwyg/rNcRXzMYsIo+NacB4NvGzWNG37tig73ipKMdfEHBUYHxz/QhfkCJT1EhZCJemdVkEq20VITv2oGU9EVXEEevnBLZEbay4sqvZdU0lYmFhsmtKvxvUnH8OmCYal6K5JIsTyiIMIWZQCO6OpT/kip1FPT3NYeOpJ/KFz8a3hM4+43vh6yFew1PjKMSxyVfIoYSEEdHwvOG19kMW9D4ZCK8yxJiTS+jy6tqcoN4ELDeOolhdzefbOpzqhed7q4hCOCVZuDCER0kIYRyArjzsyN1YhzI0lTKUL4e5h+y10Lna10YOpjAmLMmKaxCFdPUxQwgothGRY9WvY/RlT6I8JCYvCAIWC+j2/+icpt9gXGsKBPqkUHBn9+gYvKG6n0hB7kpJjIBHSQsgMXTEQIRSmtBRqO65cORZaeJTEw0LzGedvA9pzovvJ0lC3mbChIzLB1GOYiQgGQaiiioaoBhwq+WwbhlNd39YsyfyaisjDyHsnsHdbiSAZYD9QdTsgEWbAsAR8ka8TEhqld2d7KvVvVLyjqW/hkXDW5+/g8zJpn6b7KOuFjMh1PWlVPkaSBjc1Wgjr3SCplw04hU2FwuBG+Xy16aipdFhhWa+SN7Qvjcoftm3QFTBR74le7HkhEU6X9Vrshqp/p4Un0NJpI+16U9RjnW0GC0/Vw6zxBh6Z0YQ+xvbGtplgWNw46s8SFh8AYaKFUORL4+Vih1v8voiuYOpVP5NBxl57jAH1XysbcJRVMTQVw/pIWNZLM/Z7PkRZnrfrO8EV0UlIhMxtJBZJ9DDqhqG7mUxgjY2IKjrPR5mELZ/PG0x819uLkA3UrTszIFO07Bk/EwwNdQWneg+iKmKeQxPDqwgpdbyor6+PJwfvjVt/v/7C3b/dxWNPoCONIs/pijWpOijrOhsAHgw4XjrlGaViYbdEjTxKK4amaXCahmGVd03B+R1/bG+CysLoGcE0YpbyhkS4tLJeiz8d+EDSBnAM3RXVoHGHkCgWv3P2FcRtGEHQq/X5up6f0Vff7Bs0fIfN4nh/ziIDO3y4T6kdubdXwjw6NbjBCSZXB/TUU5BaCBV5FKUrBj7zfb1JmnMPgwQe7AoXmZ6e8lEasKjYmoZhlZP2BVfHXlhz6vontjkOlYU4/Su2a4ZEKGIIYyVCXXNFEjZ3TxAct3T9EKpei6MQ6b6ihurRRr/Wb+D4BGTeI9STt4613x15aZ1hJtbY7s1NU/btjkQiXUyYnUP4N2gkskNv0fN3D2n9vE87hvyEWl+vpgrjCo/HRf3Wo/BXM8Yn6M8QKFkqh+RLXSf1WOQb9W50bvZYqDEo7OCzpaWwOjqqJ8xNw7DCyh6DIy9v8IOpDQGXiqImWNJdaKaOjwhlEjbsjDuHh/+BCtPLIVNfjIDyYFRPo0waJCliUUhTJ87//GOOM/W2QHO+5dqdvz/o+VNb4JebSqQMK4s0JAHqqBVLt7WifbuoF81bnz39Qy9ZZvsLmczev32z/jNXURJDO348Z+Fo6oHoG7D07o16qFzWy7Ynd2hpkbSP06AcrQskgWHQmoyXBklYugj18gOve3LyIxvb2n75ejnHmlJSczi7yk1iCNh8r65bOJBKmWahIORYzrOSLbQQkijjAVIq3k0zWOl5hZ14zOX+fuGTNuM49wBqeIzT8uahMqP68pXfWV2cev3drjf5Hwulq2/OthsWrGowZkDnwzgzoLhUwjqydeGfAz5kQHeWRpjL1mTK36rrpe8dHfuXD33+9fd/fHX28d98csv7r3EToeR2/7HyATji/IjnHy4zYdQgfJLnZtXLetFC2CgOTwmLBhxYnFcXCs5mvLkeSqhxTlOtfePQ0hwtLOuFtMi7rCQFA4M7QDy8YAko4dQTqXXIHMmxu+3IjRV4Y+VUM5+cFx/HjvViH9c0MqsbN/687XOvP/Mr4+MDL5uZkY8mssV/B0nNmhi3vckxz7GLyDzik0npcBtBlzIlPYbFTBHgZY7mTY17zvhY0dPMqZWJtrsfuDnxxZePv/5zP4rLcZ7I+8cC+Tw+s55NLH0qCpB+yPFQmh6SIXjYvHipJ2wPPovHwdBC+OBv9fpMNQLg8JnMz/cHlc+x1hwMa8auaWge/Gp45q7XlFb3nGkiFOyhunurv5rrkEvUL2eNHJhTujoe5C2hX8JC/vxrz7znlbuf+mqybew3jcTU1snxkluY9FwfDAqdca+Hm0jAeu6kDep9Zi52vic7ENeBmZmua/jjI7ajm1NdZvbmH79w+sf+MgjeSB/W+1CUrBFM66iA99TgX+CY7O62UcsRECtJ5yTh6ZCcxtEz8KX7BvZ/IGkXQNLCIzXfq9aUnMgHkTRj10TJk8ntPpXNam6ago/S6hPGEM5c7Q+Oa6mfiQLJvKeLUoBJ3MdghF6JUtXXLn181WdP/8jHjczox62Us3N8pIjIJhaf1akWwL/aIgbAvQw8MeE6hjc+VvAynWPvfeH1D30aOzeqU/d5oWS31LFWfv8BgXI7KG72vKkVLOsFBhvnNFQO2owrCRB3CLq+sG4l+VWjgAQdUSIFLU0KizPy4M+ioxmgN/RtUzCssKzX1679X5uA1p22LU6DysE+TYSQP0IijHl2dVaDZvn6wcFfRVYCVoPuncYLleCHDvW7Xzn/oaduTH765XT7+HsmJ0u+XYQTmw5GBUkpOvgQjA2z7cjdkp3pHM999vQPC98CSnY8dkT3nIV7Ghh4QYy/YF9/lMYCcALqD+r2/IWhu/9XSuNpbBXceBonYAl+btDqjrW1LQhehEs2VtzRvJI4mybu+1Gp2qfyrmmPb/T8whrlk/YBq/UgQvAB4fGO8vXrLauwhbNGZ0826pDIrE6c/YXvHrFPfdFKT20bH3VsiEMkREpVcTQ4muiJsWHHyXQWv/szr73vKB8S+njF8cAH+3x+XGaugGNxd6AV8XOEPPnBhy3xMydCZBllP43kWJCSbZsAFHeMjv6NqMYUVswmaCq1pmBY4a5Z0u4+lm2HYCB9hpTcATi59QtkZdYGzW9vt6A4HRV6LFp4jgc5izqkz51+f68dXPtb3SikiwWNoeJJ4C5WvMn+dRN6LWjfx3/9M6d/5K2EhdJeHQhfz+ekP5nrj0HXiSfGOtraR0T2EMYQ8n2j4US9y8A0tSy0LTsJTm/vQSUx1xQMKyzr5XqFbniNQDlokhSVayERMoawTkRIovItPA9ehyKn0ebNGbgt9Lv9Z37x7Y5/5xhS4Gquw2MRlen1agHWou6kMtSxTf5XPpXSXtxHQ/iX0aiJxwQp37NRJYc8WkiU9Rp4xc/hcZBqA+o6GytdCZABie61t0MScIZEkV5Nm9/iXPEgY7iwKRjWofKuiSrPT7LyMf5TkvsLIgRGqXSvFxHCS51JK7BKp0T1k927P136yvkP7yl51/vMhKP5rkjLWg/p5j7yBESJqQnPS2f9t3zutR/7Yf7Yrx2K9Xx2QJOqg1du/OkGXyuhQEmdto37Rr74B7HL4LIkNhoyLDUEQXJ6ZLXQ9TLDmtvivPjo4r1CeYYldmW5ayJ8sATLD3WojRagZ0/KNBHCEUBYCOsEJPAD/YMgtF03bnywjZCN2Wf+LN3mrHBsDZqJekpWs/ACZurhjGb/FH+h5Cfmc9Zl0XzR1f8xuZG5xe2BbrexZiV2DjU3N0BG/RX9sFRpMpnfuGBYkEuVtBQqz7D6NBF5r71y449gIXS6HZEHSlExn0QIGYJEKPb2+lCiVJgGpe2PPPJI22dP/+yvZVcU3zwJx048PlkfEOZ9ilmYRGVuy+75l3NHD/Gq/v54pSw+Y7x4eX+mDbscjjn4qBBLIHTlBgIRus760kr49NmvENWF9R3J/EZGPr5KXoAjtmJNeYYV7pqBU9qKGJF24Vej6K5JLtUIIuRR1IDf56XBwm/Y3t1fmhwvgMz0WI9fFdIxloEBXRbCfYKJd/KeMGKhwvuruiwsUIL4IQSEI4ZQzXhnsZlxy6WuUxmOAFBQphDHwuIm172wXiJevWR+dddtVEWBMy6edG48mmkztLExtct6hYGsM0CP+y1iaEQ4kDXinftRX5tE1SY6qNfmCBo1sNCCI50zmYfzFh5fwcGEXxZ1b1E+C0wbXcpAbscbPsjSJCqX9aKkwM0tUiQsCaGsBO0HaUSKejLH+2sqxhQqL2GFuyaUIbsgYWFHipbQlzTHD9wsiRBaI35fJ0qkdMUj6ISra9fGRnxEhIml+gBoDfsI8Awa6zxvbPuZwedWS0CORi5YhAVK7gRnO6GL2WbbiCFUNSQHSKGeU+g660QnFRAApWEvnfY1zx7bxeujTApZwfMrukRpCWvmrun6o48bdOoGw1Jnju/hmFLOfUQY+ZK896yZ70Jt3p0itKS+aSRQqUcl/BAaHadT27+rnb/xLwIrR2cOIKL3YYGSKxf+cb0XTG4VGjwF9VdEAOdHxBDiA98LpESEh6V0AwlYhHpB1pqZFDIEeSldR3av0hJWuGsGwekOVGna4sDsBbOrcjCHMyrKetWRCENiL0G1POzokLTUYlb3USlOg15bMTZe2lV2dHRNZ7dh+WlIWUxDoAovuB8VwEKMFZXue1Z1H1j2i9WYit28D5sheFhsU1YdaOWrlVv8M0cR7ppfu/4PG32tuE34AcosAjMvU+I95zUkwroBhGfSW3oczIohhXyvWiP/Rg4tLILgjtYxMUX4jmj5yFfBmQHp6Fj0bu2mkl8PDDo1KIgRKVVRfyXmK3JMLIUCmBSSBTv8nbdvf2iT7Cn64/tSIFSaYYW7JhwAd5mml6KHZHgEWsqg47iXdBcSYb1okLjgUXTUjmNEEfUJTmUhYTkW56V3bvykYFhxcJEXygVKQCIHfR9KM0UJhbRBRpXG0V20OJBR89QxKSRvLq1JaD6z2KKpZSlUmmF1lHfNgju4L5UVUw3HQ/V2zVlEKGc61r98JmmdySKnUGSBE8nvlGsQrgzpHTlG2JiNFIBHCiqOLXofUqKwf8cd2a3aMYZwzWxEB6XxSJEw8wG1v6eCGCE6ll7yb4rY1HLZr9p7jPhOpRlWWNYLiqv9zBxJBVbE44+sO+6a9SZCYoP6K5FhRlHMyOA+Cjz6dSK7L5Yd+6gY/XDw8koIdDscWAj5wMgmN8KOhHEGkClmIZwxQh1B0BSz/J38sqdHZr+YcUFD3yprJeSuiSZ2TThD7jMYeq8kCcpjWb2S9s2kFqKDDIvKGinEzPxVjfdgIHCJglhspF8lRF3IJhE1ZCETPH/xX2EhnFgfwMUD4kvkz1kq3ASIUhVppVFlvSoZg8hy4U+K2FRcL9ZgJffV4xqFJSy5a44GX14d6MZWWdZLvV0zJMKZZb3qMXHhM4o4DnIRKLc6JUwCNBHT57t3Qpijfp1mgubIvmTKSMLSBfOguhbCtOUw26GqDXosxKZq/pYgOImssVrAwrqqAKsMILMRIpV9Z658FUn7JjaImGcFd03CzVUpFO6zBxHbN6R3Hi+KDPVXtGG6oL/SjFJBKyYSm84SzKFcF9EVaQvDfSbtOzutJI6DKFCChyjHEiBtCqg603txeEhB8KTwohqYjCnkFDndIyN9QvF+5Ig6indlGVYYFuAH4/uxa1rCr0bRXZN0WG8LIVe8YFigedVInrCJBhWuCeUe0DPRlknfEN/JoszlC6J5OXVU7BkwPASPq1qghA7POuI9TT0RJMwdx7HNFUwhZtEhRqXGEJ1AMwy/zXX9HYQsXIsqQKkswwp3zZJ9G2W9HCxKVXdNaaaeTtpXh1klhZNJMd2T7StsIQR8JrLUmUb2wjdu+vAwUdPbeyxSmZC6znxexhCWvLEdvoiOV4+FU7pCUDZcGkw30BOfwBY3nEgQTuVCzcqWQmw07o0DnDOVmqoMa7oYJs7QjwkiVGwfCieR+6OwEBKTdYSR2jwH0pVDW4R663MaPSa0y0gxeAVnIB+MhcHPEWPpqBj9ufH/sQ4+E7sYQwiEKEjXGDckrITRMbGq7al/Qm2MG8kkbV6qSVhi6jBHvmaY5l5+QkxhpJuMeEKNfxScWE4h8AUGz93T9ka6Vd01iXOuvmkLYY2TUMttXKV0ZyAlqcqv5CyiBpi15gzHeOBI9KD2lbOMjo3e2uD6413IsIonKalwDxIJFNbWjRtrO3peC7QV1xgBQGD5R6UGGANWzPX88YOECxuiiKhXAUYlGVZY1uvMVSj9Am2nXYJfiA55WrEmlgZgYiBrI8zUwkKoHLnfmyShvYJmCQvgNL8dHuiJfA67y2W9iqVbB9NZy4JeT0VNNodPAwTSIq+4yA9AxGkTVR/QlJtByAtGSZTS07aPjHxMqWR+kRMQZ2DpTebmngqGNrrBBMp6kTVAWaNgozBYb4W7wAaei0Lx0Yss0eEYEPpmcdLUOpIbL7LbPePvjHxxhgVKnGBih2Ei/ZCiSfs0xJVZVprs+xRx4RtrztsousZ3/KxWC1CnEFo3FqMt3BJlv1QJ0VEQWZo2XdbLu/toJosSVhrtv+pqasiwBDeNfDnOT8a029OlQV2swGOaYqemD2czm85xJLmcVI7PP6rqfzlxQvYZoEAJzjHKIgR8FHnBdC1prXuNo9SDxKmiTF5BMauOlFMRjkX5OCTzSwbGuKjGpGlqlP1SkmFN75reVLduFrErWaBE1eZUQkQ1BANZBXR1kAH5HE4ale2OyhZCoMeETtkyO6/sXfvuu3KZHIl0EmdaCF1/aquHonp1mIKKVvzMi0TSycA3XdtCBVtTMCzL6r7sealbCRRfo0Fi5vVqvNd9JvPz9YndhGdgIK8EapVkWIfKZb08v/QkvaTxnxLIepCQuProSpOs9x6JZ7IWh+IWQj+ZSlACPIcFWYzDQtjXd1jQ78lbf4Ic5P5OBwhBYhnlaJoOWBZdGILkzbXtj14iHa1a9QPXQNjXaCmkdelB2lLhs8iNFXiPE5aeHjVCdNSbXFgGsU1yDi3Xn9ziChd39Q4+ID8Aeb+FkN/Vo/E51F8pSeUhAgKIf4GJgPAucRw8cCAG4adXPsx1Jze6fmEVNzf1KAUwQn9FCyGcRq9vXvO2QUINJu4ZRscp6Twqx6HSX+JROuEWuiW8tBQ2XnBQjmGFZb1ODf4FwgLcbu6aaPXiBVXRDBkGHUYpZdWTefBZJcYQ4o2SiAF80NmA5qncM2TQc1cuclDDGMKJycGD2TaT8fHkkpE/pyqimOdigw60esdJHv+Ow6jMy+A4cNYw8DZq17R5YKjua4bocO0F3Tdv/sYjvDeUaKvrJ9qrlWNYYVkvxyls9bRCh6KOy2IWyDBYDJN6rHoxLK5GhuQUlDZDgFvRQjhlae1paSHM5X4ychQNDfWLPl3EvWmGKFCijL/Q/csUITlGGv9MIW1uPvcdVCJoprn264UChoAIJnyMHD/3w1DdJ9A2gqAJkr02mfRETGFvWaKtrqdor1aOYYXDm/KvcdekFIHQcTVKVoWw8VVQF7jHtIVw5o8xveczybBoIRQ5sGJ6zpK7JbeysAYDYzSRWC0WaV9f5EGE+uFeaSH0g8knOCFCub1k4CPvgNNm2piwhNn1dfZ+1XqE08gj14WSbfhwIGXjdQq1QFgK2yC52vbNRwmYCjGFyjGssKyX73ko61VUlQgFYXFbrGcMIR9K3UJoISTVK0blBFEAZVo49+grLu9f+y6RVuZU78FIQRV6aqnrTHi+A10nszQoqMEiI4LA6TspL6mZbxA966a2Cz1HMvmWS7pmXU4mBf+KFD98zhIbgIKxwPJglQ52si8Vyn4pxbAohuaxOxM5CLN4nEcuRXdNoT8ygb0U/hHOurTyc6haoLSu4PKUaAA3SSSSDE44D8kBpY44r/mIsXRUrPJXr39sva+Vdtsl4VqiFD2XaSJIMHYrMK9uW/HON/jdyZMHcKDXtJUrv2tYhyLeojRaPyrioytqoC9E6CAzljYVJvNr+JFbqQk+ehTsCi0IbrV7gbPFsQWtKwVjONPkqknD9S2DGqX6NSIoDMkRyKrfoyt+EhXuOA5qKWvN67zpmHQdixRP/f3CHU0ruvZWlKXvFEkCFVS40+vZws5mmdnrq1btGCE+ent7ufDJpcDL01+3oJDH9hMpftj30psukvkhbcO2IDibIgMjxI1sSjGDAwdkorBXrv6PDX5Q3MHwACBIKRg5WUzo4geFYHX7k0Y6uQ2l4qewQusAJuiaGFHepQF7sqGnNCMwRRhK98AzkSOno+O64NcF+/r+TJtILOWKqeEEKdS4wq0Eshhqqa8RrGM0KuOYOFDGiaGZ5+Fei01avdAzMFWU/SLF2TtHRj4uQnQabSmMnJCWQitdXdL9H0LoLt1w0oi0oG1cwcZE/ZOoZ7wLZunNI3wPIYIzG2sjKijPkWHJQP9YH1dj50K8MqcmDb+zbZtQuI+Pb4wcN8+Py+IIEGCg67RV3NcE/pBVFCSc0pKJzAV+ETLvECeGtuJrhQLldfUshYAJyfzgiqsHHVBCbCf84Rrl+0Y0pRhWuGtO2df2prNAF/AFLq8cy/KhYetoWw9v86ljhua92ta2BnOHRNgxNq54IsIFbTNpH99HzgWigR8hOVAl66nhtN4pFmkuJ/WS0XQve8mXoyFcf1SUo1KPSiSc4FbCQgjG9FV+EzKqXO6kmD6kZL1cKpkl5CJR0FJIrOpuR4euwVIokvnlcrJgrRxd/f8qxbA+cuE5bjVUP+5HWA6wpRR407NDBY3vwXPZ2vBFzci8ZllJQj39e1xvKKirA7YAAEAASURBVG1KCyHIiBxLwQawEIbCqjDt57d3fedtCeKRSJEjiiJgEtCS+LfFdXgaVJBl6QbEE1/3nHSpPb36CoEcKjMqBBMLnKxZ872XYJy4lGKNOOi4eY1ajYSGGE1dJvNDVGFDYVSGI4Dw7hXD9Ib34bOKJIjJo5nah5k6g2wJyZPQ05zW9TZ8z3mMn4vQ/0pYCPE0FRuOaEEymYGF0HwDIoObhx6EOpsoYT1QLoowcOH310OXuIMOjpiU+JFf5SCQaxjxlPBkD7QrB9a/4yJvP1VmVJoGzKDp+lNTWpC+YkIxT5JXrnEHAs0FiiTzU4ZhhUn7hoM3VgI9WxUuhulLM7V1+c2bv/OqZ7S/ViySzKiDiJfkuCJVLuslF5uPdapraWutCMnBOSJyRhI6XPtGaQc8nNo9MCwIXJE/R46n9r9ko2REycSKS7q+cYo9HdGktAn2Ghw7Ji2Fmm5+TVoKa39WbHcCtSyxh01n29DQ30KXhUFp+YbhWiGGJS2EVwb71yPoeZNLKZ90qFqD6Gchb4pltl8laAhgvVQspqGD4IKJVpKYOXQiIlS4z/xepfeUpGCvhz0gQ1OYSIvc1Z+LfA5Dj+sJ5+reLIRbkAn1h5E/Z8m4xf5lWSJsUDBvMqiZ0mZXOb7SNLPnXFccCaHJIkNQqelgWJSyiqhT+G/IisHWuLJfyjCskAg9d2KflQwSwmNNwV2Ty4OJ6Qwj8TKnrlDIXcb2czlF0V/sPnyJp/HQqXKWUTAO6JADozChuR2ZXYJhDcUQQxhGQxi6tZeRW9gmFFvkPOpB3EbSPt+1IG2uEdbSkEGF1JHLyXe+n361iGyMuAfnwvBXVV4D+GL5WiplpCy9JIpShGu1ERAqw7DCsl6T9o1dyRTFK1PJXVM6RaYQyNomiHDdunUTutZx3hLZNeMhN9IwxQf1LYSI4gUeLDNzN5veKJTMCPGPlq7BFfNlKcTxRg5Q18miCdE+ZOm9UTmAOROphv3AFBLW7F5lUdlkcu0Vz0uOSGEsPil99vMr+gY2Ad3NZLg1jItkfogqrOjGOC5ShmEdllprrHj/cc8Hw1KOBAX6ee4zSxBzDL1NmKnFt7r/KkNRAHRsUFOlHFoIVfXBgvIKTpJwudBTZzd3fpOIIYy6DmFeRkMEt4Lj7WAJ2xycV4B0Zej43iJl1WuIfl5mdG12h2Des6VNWaNxxYoP4vfgagIOpnHS0D3YqntHqubGEPiOSOYXpqWurpdorlZiooEMOhaJxY5dU+GyXpIIdT87viaz6XI4BYax6lTcBQUoYZUgc8bq7BUOqMZXnswsK4Xae22X2EU+DgthORri7Ll/e8TzJ7e5IuZZCKA1Qh3bbYghpJpAv75r3fcLfPRqvTzVTzfqs8AH8KK7ht5+rqx4j23Tm35w1W8Q4Is8T4Ff2MVb83moUwF31d1EcIMSDCu0EL5x6+/XAxEo68VimOqV9eI+QyIElV0BEV4GZZUnTYelULwVW2QE8zJnF8rHEAIf3HsSescrHMDT/dFLPqGntWUVdiFbZ1ZYCPlQ9RqiITQw7xVMEU0dBxnTLGY0MKAJ5acXeKdlELTcuNUajo4QHfBaXdsZJvNrlKVQCYYVFsMcnRrc4Abj6zxk08QJUVEiROUTcwV9jJw+SBAkrFTqTRehg7iZxKkQ39+3i0ZBeAIbIHVRJSeKDmPog4sRWg5d87JaKtF+no/o6Hgm8jk80yE9rafs27vTWW7zuppuadjVEokU0SBqMj77omRM/GJm6+mBHIpmWStfK5WILvViZwFTOZlfaV0y6TS07JcSDCsshunpwwfTGdiZqN/D0p85sUq8x5nHohu3rkkl6qmDlKi0trbvvAlwrzFvN+h01i4aBeyhS4OCWBHDw6BhIcROXAym2qxNYpFe6Pn2yJn3qgvS0xoy1QFf6bJevuHYupZNyLJee8qMaTYtyBAdqOFep6UQLVYpffbzK/qGyfy8bNYyvXIyv4GBVQ3hHQ156IMoGu+RgawFd6TbsFjWy8TMxbLuH3x0VZ/hpGh4DgorGGuFyb678FayJwgXDPjNfD2OggLEAjk3E1cwhpATph5mCBSC5pC0L2l13t61/onr+AZe3aciBZW4PnxYxiV6/tgeUYeQD1KsYb6wtQUmaQWHZJFiJ6eFjOlBYGWIThBsu+L7yZsKl/2CxAg9VjmZX095zT44mrg/K8GwDh0tE2Fgo6wXUjTINRr32KvqXxChSHUL6AxLpE3pERJEr8ChoZtnobFAnxHrU4gMPJyVw+nWoKqEBZuJl0xy/AxVknmfQq/uqhC94MV58u5Aelzr212Hgrjg5wveVe8fwVdhLYVSw0veXt2x74p8/nwZV48Ipr527U8N4rbrapf9gg0/mBTJ/I6W12y9cdtwhgURRdeE1YHn5ImtwqVBQYYFToF0sdw400NrEruuyomiBDHERQQ21fHVEsx4+CJakR698wGskhP5+UoOIpK/zEtpmkk4Sa56gx0+6NUdxUP6yh7Wlwqvr/P8QjkaQkUWHvhUD2ATu75z9fcIaRNBOYIxzcYDPUyllG5o7Sj7NfsKRb5BqhlsEBAFsWYNaSmMeHOuYKANZ1hhWa8zd/o2wmjf7UCU4PxVAHtdL8EkwUKY4IZ+bdemd00TYX9/TsBhmqmLTBMist1iq4wSOHZG9QYIWz1xojxQsHJ4aTOyJCkshKdiLOtVdIb3Wgkj7VHkjFqijWjiDOQUtIyO16AuwCqff9pA6ZxesQ6hJjpnGCaoP1LyiWhEGsp+kQDt7rt3j2KtsslCtvJ9ff42nGH1atITuliY2Oz6EyiGCZagHLuSkwGPX5ipBRFypVB3pYVOdLb9zbQcXkkyf7dgLdFMIFFBRiVCchTFC2YMEHqGa6e1tpSUsJ7WctEgYEYvJ8rvS/bIzlSGoSzSXWDGJYq8ZdAzMq7qsqwXsowumLECm56YWUtb83UG0mO+o5XSI8AKaV2W/fJXQKO6NYIua+qi4QwrjEsquTcOZNstzYPpB5uOgkuTRJggZELhTiIExoMjR/JiO1y//t9PoujXZYbolHfNmiZk5k3smIhgWS+1YwhhIYTFAU6cE6a+RSiZ7+V9mjmiJb7P9XOjQHID7VEIvMD+EvuL4XbMFwEzHaQWRlkvYU2uNAAcdsU3SiUNZb/ml8hiALmiLnkMBMq9jg6kB3eGRDK/cO1W1EFEFzWcYYXFMG2vsCvQYSFE0jMgRjWGJYiQlVkyyXX3ESGZE+AVzn+6oX89kWBeo+hkeiJC+bJeXKHQ7yXMzsF9699xi7TZO533KRpKJU3ky2o8x5vYxTLqSsYQcrgwGrt2Av56UsLSFpE2c7mcYMSJxP/0BjyyIKXzlBG9P18EM4GU5QBV10UQNMp+Cbgj6LfiLhrKsEiEoZka1geU9eKWGd1irxgLi11ItwXQkGubqHqkn5eX52bclS+/zyBNCCWsCJ1ewbGUL+sFFp1kWa9AP4uFxrxPgPoIJzOydrScg+nkld9ZDQrZKXSdKqakxeQnEE+p6dbV9Su/9SIRkMvBIrFgk1L6ypXvHUaEx6C6Zb8gZGGN+v6EKKxa3qy5p9atNZRhlWV6IiHt+MWtrmNj4HqDYZoT94HFXU9LXN2y6ulLvGImEfb394ubEPQLHQTcMiLUQZAaGJJDx9G6UoYYUaV/CBz80xJr4dohLIQL6mwq7XXmdQf6ZA6mscLkI55WXO8x2EXBiuA8ELLqdUJPD27sfKqiFNHlhS/sg4aWKSfzU2/jBl3DGdalWmR7EBxD8LkWhKX5Zs5VnO8byhzCkkGv3/yLDUFgw0IoVqVy61JkIQARggwHt6x4y105IXJX5PuhIZkmJJFYccVxrFGkCcEYoiE4YETor/iqaoOFEGsOFlRdF6XYh7t7Iqerrl5ZUck17u5LJs2U0HUqRyliU4H7C6RsIy1w0ftA0r755jAs+wWm8EbUnjHzPbPa7zHJus1g86C0bWzoJZHMLyzNV21ftV4fOWFVA0gYyFpymeq22E43DxE9W00ndbmWZb0wW0ZW1pYTRMi9VLZTp6RT4IoV33UFs3kNCwqLd+kFBbgew5AcVVPKSMbsmcUpS8umtojj8p7xd07jJsTRUl/PDMgYwpJX6E6koL9C2AEWt3Isi/5orMlomSmBi2/vrqwm4/j4GYkzY8VXpZQeUC8aOR6XMg84CbHsFytZZ2y9KPRYveWNZCn9VnNvQxlWWNZr0r6CYpikPTUDWSURJpG0z7pA5HY/QIShpRAFBbD/ZM6SuWExLbkRI7QQql7WizoXU7OG015WJDXMLaqzqR41z/QMYDtjs6HrxItyrAogyWk3bSgdE0GH2Nz2VFiTMZeTUrrl61dKJcNR0VIosW54WSbz8yZ2czYGyhsJ39ejNZRhXQjLemnBHs8vggYbCs6c+CYRYqOzStBNJfXV99WWC28QhCr1VrjUe52FB5baxJrEoqTCnf/wDFUbwlAweKPjIhxqhYUwaoU7d3aMXyiuPd/eTguhio1BG9C5G14p5ZjJjkuEsXL3Dimlu8b/fBF+pJelP99iyvrGYAHzgZUaiHqQF8rB6PWCZOkrq0ZIKc6HFkLblaluVdw1uZtjsaC2XMLOJMzLHO5cRBg6/+lm5ylbpgmJxPmPMYSUshTmV6gilAaOzAvgKjZCNiJXuIf50mghRBa5XS79PDAnNZJenLf5VAdguq5uS7/9Ih90r6zXYo89wj1KW7fuEFJuJ67EmXJ7MUgW+p0bJ4POA10m88Marmux44YxrHICMDDrW+2+74pUt6B6BYkQZuoU0WRc2ph55yVO5lxEmMvxF6hLTaQJQbIzDGXJY2EHwkKI1yV3RuDiaBQr8H/SWiX80w4ciR7UMIZwzLv7iK85az1ycOr5lWtI0QDpOmG2X+rq2jdO8CoNAAe5SMEF9+BA+XVly36BFHnkxcaxc3QUG4ho+brNRcMYVpi079VbnwIRFrZIM3X0xC4RuoS/ICN4cSMkJ3t13bqDE+xpbiLMiXOK72++jMzmN5gmBHMrdk2+q7aRAngzPdzVUr3eG4lYZPA5QxVCHBEM4eFeqVf3vV4Wf9fVLwPMC4XbB1JpM+lzTyeXVK0BLLo0YLcSzDu/SEjOg+BDShfrERWZznueMNxQjqyZhh7sP4rPPJ3LEB1nneuOCUthPct+NYxhdfV/TBBcYE/tMUxP6UBWKtF9FLvkhM9PhPgFbe3aXxrEYgLDEifCJRGbsBDy9KPe0uRQuZJQhDAwC5OB15naKkKWhsrKY3FBRH/CPn3D324luC8sHFAc0WOr6gZzRFnT8MFokJFW4OLpMgOqtKNQSkcl8a8VsVOR/6nFrjgSmGcRPpfJIE7Nu3mQ39QzRKdhDIsDZRstDe5mICt8BihLqLc0USAVyUK0pJG9QHjnI0JBsNM7avYURXpIIFxdVTdyOSJClPViMDjeL4nzVQ1BxTeIsl6mnhm29FWXxV0RV/Vin6eO9onh+15R5GICPpRDB2U+wAXHSnil+YaQsCrG4vSF0lIYBNmrrmuNq1r2CyOFrg65JTS/m6AjRKdu89EwhnXiRL9YzDDfHoTlh341dRv0NH0s/gbUp5k28lyZZvu9sl7z3Dcw0EOxCvMZiDQhVErU2ihVUVXATKOq+mAJh1qEoRh6+uzuDd85xLFGXdYLKNSZe4l9wwtoKy2E/I6f1WoQNZkvzU2OdCa2XyJss8t6LQaxLPvV1fVfLmKMV6nAx6hrJ6LFHlf772XSnnpTuYuaNuZaHt8QhoXRThNhyb4rUt2qGMhKziNqy/mpsZS15hoRvBAR9vR0COLS9ZWvlhAoDWKTDKyGmeGKDC2ENdxel1sEN0fSPstIiwWan5Ywo3x8XjCnfzr3wXVB4Oy2YdCAGqUhdLvwqFiTESEOun79sa3vu8xrHyzrtfD9oDapeMeL7hh65g2qItCUY1jcMJjMD9XZt2EtwyoMrsoDcR1agyb+qBjc2NhX1sAps1uW9VKTCMUu5wdXv2HzT17kfCxMhFKkB91dKBQwq9KxrHqCK99RRAy12jGExAglrPbYynqFFkIYZVACzhcWQmGW5KNVauDeVAOgIjhSRCNPF6cfhFADiGKTA+2cZOYPtFr6qOGx1dwiy35hA9lz586vbZB3yjVdTS+1XNsQhhUS4YXhr673/IkNnksW7TcElkWQBgmLpdc73hBEKDbBhYhQOv+lUgcvwcEBITrky9XrsUCs0yE5grMvAmQjfuZi9JG6SfMycGnInCcMcZT1Gi5XZyn4sBBmjAROhDx+qIgWWJNRNUnzhcL92YEekXKo+rnJy1uMjrOOzSWBNK7KNVn2CyralaDUzRI8GZweN6gNQUZ3mQhdbepAIq0nEBQGIuQyVayJXdMk8zhJyI7TarNgOyJ2w87OH78NVfn1paQJ4apUOcsojgA4Cmoo6+UXssmtoihHHGW99vTIGDtsHLAQgkoYJ6VggwrWZLKRpLWyHJJTazzlSUFDvm+eoj8fmlBkKTZkqC81r70dyevcmyLVjKbVp+xXQxjWgPacwH/JGdphJR1qijAz6km+LOvlolRTOrla7JpwaliQbiB1sIy3wKmvZV6txfmPWCDnFhZCHAn5Xj3MEA2YNcgQkD6HNmU2wJUj+rJe7POEJo0zvl94AotEIoc/qNWowDHsIkNWNBFPOVc0RCUg95WtrJ2J9cz8cVdRSyGGgvk3OCHGLo6rv/+5upBpQxjWYA8LpdKj13sSXu4Yu3rCFcATOmWHXuueLiSI+WvLcTSiBU8/LZ3/8OmCTNdUvTKS6FDeQoiIoWQKHEszTq9c+S3DHP0ReqlF2KjIzZdjCMGshIUwwu6j64pJ+8Tx37q2NvP4G+y41oyroZU1veqXr+HULTJ/gBTVlCoxKUEw8QTHm8vJNc33cba6M6yZRGh7o9t8X/CuOMdYY9+ytpzmJ293ZndclZ1IHdVCHWLiRDPNlSj7JeiMOK5qIZN9i7JeVd0ln1uvv+Tmhs40wB1igcqyXtWNczFYw3xpXzj7011Q8O5y4eMEkVO93Q1ELTzcA+Pa3o3vqShp3/xjl8ODtO7BgRSB9OoNtww7cvhj7er6liD4VAqbLLCQj52fxP6AByemr69PPPPG+D+ugzpil40MhrSnPHhd4z/L2nKQj649uvF9wqWhkiwE5eSj8NvyLyFNiM00IWhVsR5eTP0Vj0AKIkZMDRgW3MM4lbKsV1cMZb0gpshnGRZCQILVyMWEFwVpBfMkHIUNS0ZDQC1Q7ZzLkXIpcOGXdaWBf4ZhYVXud2FXcb8imR8p1dkxMvJPdQvRqTvD0nrlIX1o5AYthGtY1gtNyXUpasvpHa+D+CgqAcbFwcyVCwqsWvW2N3A9RHoWpSDrqazxCbxaxBAu/rjKOo38KjDgwENlmKTWnll9QXafi/wpYQyh7QztT2dNixZCJV0aIPXRyI0XoeucoRaoCSfTmT/0ladQRYdNRa4lYgqxIbd53uROAlmPEJ26M6yQCEve4KMgQlZ18EH+Ci5NbHMwU4MURVCvrC1XyZEnL5iTrn/POHPAV1P2izcSEaKsV1nhTkJQrQFOOEmaGtK8TCS09acJX61K5oXGFsYQYia2cyrQVHRLIzpk0j4jI/zRFhpTJb/lcvIq3QzOMPsopC4F1wdh0mkpRDK/UZF9tJKxLfWaujOssKxXwZ3YYRgljBplvSDnL3UgEd8viRC7W8pYJxTulWYhoEh//Lgs+6UZ1qusoALdMXlRRY2IoIWQKZ84ORXfWFHv0VwEGOFzhG3faL92YEOvDMmJuKwXIQ1jCFGWXtkYQu5oJhxGUZNxXDfNi4Q7ZLR8X1vLUaLX0ulHL+PIfb3sz6cgKfAsAGkj0ESdwnqU/ao3w9IP90o/miAovolJuTnhtU1qvHdR8eSAYZmaJyrBaFqu4geGDpSGkTwHfxrcB3Gp0oYrhYUQJKugtqY8igB5vekUpbGsV4E0K/JYVDrGCq4DaUyHb6H/LT70V/yuglvrfAlwAWkTsbBXv7X7By/y4b2ajAmsHZC8WBPt7T92EyNWvuxXoBX3cayg19jz7NeVYQlVjlAqwsnOn9zieSzBoeCyhM6KTopgOFfXtD2GXQ7sKicZLd8v1sKCAoaR/XrZUlixDoIrUvmkfQIBSNpnrJLHZaCKkuVieKnu97xgTi98/UcfCTR3t80sCFLLX103cV8NsChh6YaJkJynHDJVtCXhgrhEP2Jt6lrbSYQoslMhdcU9nGr6J4wlJAaAlNU9PPy78Hpnk/Mm30f/t64MK0x1e+7uxzcgrB2BrMKloc4wLI5EkYVA1OoyB3dvOCyOPNVIEGFBAYTDXikWjQKOT1h8lRExKT20EC4OaWOuIOtgWS9MnEij0j3wTORzGIZv4TGPYLGuooRFJDZmxAs8FcyF4VsI1hCqg+dqDsm5/xlh5g84lJ/X4aFbhd3m/o5i/RQYDiyFKNG3ySsMbpCPijdEJ3JCWxg/BwTBTRUKm11/cqV0wVJOf4UhUC9BHU2bWJCytlw1EoT011qz5vsuYWVXXPaLyMGupUHPqqLcGU4tJsxH0j7NzyY2iOPyeIWVYcIOKnkdHrggaNPxR5Bl1EiCX1HCUI5hYboQkoPSXkZSpB/aMy4zdlQyxoWuCTN/mNoaFOflNlZ75o+FnrOk3zAbNJqlUmZKM4fFsVDTZP3IJfW7wM11ZVgDAy+I5xXd6wez7ZCjoV/GPwWJUBAgATtP3FVaW47XyiZ1ELr+lgLU9iJNCBnRYk0gBNdR8CSiKrhlsS5j+J3MnCZ8c9gLkhf4gFwVx+VKAdoTpurRElstGC6AG+WORBiLcEYrTfmuoaXE8XjpCvcQQzLzh6/7FwvQEmLjU48khE5R9zNpyIGat4uQx132SzCQEEVxvz4/LuONSo6zE4VTldw0iAMwCrOEc1lCXyuq91ZaWy7EH3UQ9Pwu9/VauQJK+POcr2ROuE9YBx3IMGReKjYyDoahmEbmXM+2Z2Ip68Vx57ScYFB+UJIxhCoio4wLXU9e2rD6TYJ5z1WgpDbQpZSeTPZcBDleqTXzR23Pru4u+ppogS1CdHqm60dW10elV9eVYeXL8UZ+MPZEoKJHjcQadrPAcG3LQz2Bi/yqFh+j0PNb17OvO45gP8T1okITk/bRrYHMS8VGjUUCFTbAUlnWy4MsWbNX93zjgzSKrvOCYZl6ehMSxc13aWO/BzISyAqaMNv+9eC6D0xgdvXo4imPCFpZufK9w1DoX1tK5o84kUQ6lXUi3R18Dj7DUhifmqduDCufz/McgbEECc93NruqWghFvmpx5LnctfKtlzgJteyauRzvpKUxcUqURaogrxF5lEjah1e+V7PBqQz0mLA6hc4GDjgxgJoXfQ4HL69MWCt22A6syWBhiuGDByLDw2HQ1Ns/SdiO9+dMQLnoplTJODBc6tnl+gxSNWX+qOQ5EVwjyn4FgbtrcPBXu9hfGAMaQd+zuqgbwzpwRFoPTg3+xQZfK3YzkBUzqxoRQgaSZb0MI3Vt5+q3jxJjc5f1moXLB76QOohE4pFLrmMN04EUw12QmPmjCMlZ8KoHHlPHjxgBVXGoDINUwJomFO6VOtRWB+YRcbk5nkbsbxYmMgUlLGZoSGmGW0pdXpF4+tMEOFcOy6purPNfjRAdsT4hYZ0HVeLC+CSX+aFY+BdsXiJEB+hYk077G3l1bzkGdOE7a/u1bgwrHIOrlbYFmt3OYpjYjNRjWMCj8KvRLaG/qv3II50HV6z4jqtgzSj7RVRzvc/diAieksmwlJMlyiBjxgJY8I3ClGdnrc3lkBzJmOceVc3fCjx1du6/4/nFK8yVvgDqan7Ikm6EH0M6k0S2ilX/75t2/IcRRjeEx9gl9Tvj5lwu/NApyn7hU8X+fOGd8b+y7Jfmt7WZZqk0eJDPizOmsG4MKxzEZOny/kwb5Wb1asuJyWX5nsBiYQVhIZyvrNdihFAW6fHylGMYmdPlZH5zMix+SYY1M2nfYv035HdghszcNFK3VnSsFxksas37tBD8xF0+L49Dum5eFzGdYFkL3VPP30C7vmH6iamx1PiKxL6P8dlDQ70xwCc3A9OzLtu2VRSJGxaR0uuJh/KzxF5rmYj41K2d/C7Osl91Y1gncv1Crg+8YA8kLKB94eNRAxDPR5LoTBtijml0iFQhS4MjJy2FvndOKk3nX3SUqhg/yLAcvo+B+pc2FN5NJTMkRfx3Zufqw+K4XEnKnVoe/PQReRwyjew/g0WKh9fSTxz3YLv12trTkK46f/9NO95/EYHx5uHDfZCNo21Hj0pL4er1vW+AJlD2a2EpPdqnV96b2Jxxua9NCkuheFv57VVdWReGhYPQdObIkj/8KLWJHGRVkNblYpgHsVP4bmoqrXde4SNzuZ9cApw5CbXR+apd4tuFCwow3x8zNXDLUrVBnwJDQlqkAZZJ++KZx5wm8Z4I0p+bmoC/HvIpAidLmItoMArrqJPJGpCurJPbE9/3IfZ6KuJMqyGkR47kxXghpU/Bn+9yNZk/wj7q88qiFODXvrE9CF5MgLliicej7qkLwzp6NC/WYBDcasfRf6tj26A8Nct6iUBW7GY927/vopzsXiEZ1jbxJ8MFdm6xmEIiiFlGqeVSmWEROrB1IX0Od/fESD+HBd6/eXf+JUhYX85mEdxJ/8RGNmRSweATpSLiKFNrf2LHjkPF40HeypddMKIGTS58mfkDJT8QU8gQoHg2iKXADpgQU0jCLe28fv0T5WR+R2Mh4xgJ7h4KDhyQFsKXrv7dej+Y3I5UHILu712hzDvhxY2QnEu6vgOerQATVFN7kyJ9e/vuNxzXvEXnP/Q3JwMkZxMWwtofFved2DR9q1SALd/ICgvhnp5aK8MsDioX6/HjOUpVwFnyj2Cc5P+cjXATWLyTSK/QRSaCbFsCB9TsTx/a+XtfInyH9DyjNWJsedE3rNZnXY/LtYrMHzFCdX/XlLBENo2ViYSznb/19/fHwlti6fT+wWhaV28YXzS5C4WBMog/ks48D17Y6M/Q0Qiv9EA/SVDKea1qXiBHj8oBZbM/dxOM6sZ8zn9kiTwKqmwhBKtA0j5D871gzDJXiDCUHNUWMbZc7riQqN6+7//568Kk/nIGCR/BvmJmEHMOCLMT+CtWJWEhtX7n7fv//L/jsGZAuRy7xIeFLwDS9QyqiQt0q6XQk+gCeetuZyd2FH9ExBTmcshfGUOrC8PqGPikAH6qdGNPKgukB4aaqhrdRwlui4UVhASh5fJLQjl1EFJdR6lKOv/xyDez8SORM9NC+MAlMy9v3Hsyc8b06amLG3e89aYE5EisoGIRCCmLrwk988vMTwYNI+vBx8oo70OyfJbf3pmwJkeNZ9+x/xO/KH/PU9yLdfx8zr3MH51XbDsxXk3mj/vGEfOHkK59XS8HQQ/EMkd1YVgXLkwDf8D3cR5c0jErHsyDaZD4kKccoq2WEhIW5NolPUwSdK/AMdQfCGWRZ5oHOyU6aCFkTn9UdlC1+Qn4QyHy4vwO/VAxn48+JGeugR861O9Suf/tB/70s76f+O0Vq1IM/ohdshGwIPQIQmTQuSppTo2bf/zv9//VT/B7VoeJS281GwdSrdDV9Y1XIOQh+yhyFbHotpKNS6ggso/ijThCRw1m7AyL1oLDh6WytOTe3UuRQ8UGIwAshJBo3dToys4NwkI4NBSFU+SQYEGBvuoVivQY/yyRnhcwQwPFTnUbdSf0wWoXCvcDR3rrxlpPncoLzLxj/199cGJE+0wHNCX4wo4XVzoiQD2zvT1lFkaTv/uOA3/1DJ937Fgvwm9knGO8zw97l2PX9e8s6Wb2LP35FF1CusNd19d3XL/+bFZu1vnIaSR2hhUm7bt798UVgH67YwsJK/KBhNNb+yvKenH30vxrj617+ir7OXVK7m6193nvTsPwLhaLMPwyN+ccjVlGSYhz/jjH9fX8isRHqy5jkD3fFfqrsJhIPeBAHKp/LOglo9ey7o53T47pL3WuSCTxkUwrajaPrSNw4dycsMx2zSus++Db9//FL/DZlKzi8Ldi3/M1gXv4efF3GD1Qp7AOS3Y+YBb+HgwLegPN2ZLJvLZOXhp9Mr/YRx9mjrw89dIjKLKy0WNZL1D/wmNvzK8kBsvoRJ7yg2L3PnLkSASLISfE97Vrv/E8Dnxw/qMe6J6ymogIQ3LUxAoXSjkkZ8J3E+aq1zg70eV9qmyuD+t9Hq1y3/L4bw2vTb3pbYWx1InOlckklghXyVIV8Zxn9BG4yZRutnWkLLfY8W8rUo+99dDeP/ht0msezKq+ktU9vIRlvyy9E4H0Yulw3UZAm/eeEcE7YSmEY3HWtmUVnTC6JYK+p7uInWGFO7HjD+9LpowkUt0C0fE4lU2PqqY32Mag0Qx0T0gQ0TlF5gVh6fr7RhBmgjQhAuX3EZtgWGBrSnJx4rIckgP47mQ9SxyXNVlesiZM13oT9VlkWk/t/JXR1ft/9u2lydV/1JZtM9Jt8FXShV4LDqaomLzIYpa/w8IimJRgVDqcQa32zrTl2ZlzfmHtT3373j/55m/c/itfziNGkByxfjqr2djJ5eR3umW+TrUCdFixr9vZUCz6DdCpe9ksXjRv56JX13hB7AMfyvWLxVko3d6ZSOE4iIwc+MNRKdMADE9jyIGlo6xXl5AgwnxWSwVypkiP+sBfZ2gLJRb2yz9EBJXtNpYPJ0P8gFelGlYsXRp0PfH6N+3/zTuErbd3qZVhahuhUMLjePgUYjS/bc8fvj9p7H5n4Kx7KZPJmNl2FIw3A8Qq8D+hN3XwhkQ3/YpfRP0dHNGNZEYHk0pYlpXUnGLmK/5U1zNrVr77iUN7P/pRWCaFRJc/pLm0UtYGbTR39fVJXWoQbLjk+8lblWT+iObJ1fbCZYQWOI/zJY6yX8Ixj53H1U4dDddg8BjO4PdWaVwPrKFfYhmMw3BwEEymNSFh1dDNArfk0H0/HmKg7BdZlFRg8wYwNM0BCxd1CPmTos0wRKGFCwSvLH1y42lI4/GQ+iREUGhv2ZZ/HoaMT//zhV///oJ9472BN/HN6UywyrSQWAKpqsSOwHVERKOxjgCrNJcKZtErmecC2/x80lr7N0/v+Ui/ZEx/oB0PclZOO+7h81KPmpHgR24OurZ69S9dv3nz3Tfh8b4OOdYaykTnGhis4JgKbAnIjcXfqfrAR6Ax5AFz3VXdd7EyLAAPYKXPjO2O7bRE1Qn1ViVwiqBeHFO91M215q7LRGE0FkI5Gf398hWlMb9aKk3ygwksTC8jFp3A/4qF+IYw33s1jeRL/CRDcgYaxrAIQ6hP4hGxzFg+ga8/8cqVP9w0MnXycd/xdxa8oQNYPrBYmYHvuTrSOk+lrDWn4TV+ecOKnacf2/AM6yqWx/Ff4Sics04gp5X0XleHTgEj+QAzu/q3bv2vJ62E8xjf40uhjCc+1GgBQnR4ZNV3jY7+/OoVK373LqpNAZH5yJhrrAyrbCEMzg4e67o0+tmddgkbFshGillqoJhQ0AgmSq/7+o09m3/wuqb9UPnIEw3Rhs5/UJFdLpW0KWalg/GB4qY4a4QWQiENqIOWEBIQW2CWCijMoftn+WVYICK8oJGvPCJyY+zXDpmH9H7vyS3vZ9obkfpmcbh+XGOmha7+vJ7LHSlLVP2L39aAK2TZL/gz+t55ptsJAnjRKtbIVOnagNf1sBg+AvDAsKK1FMbKsEILYcEdXu/5k+tkam71FO6w2/lgIkhD0HEKu4PIxYRXCj0RNbpH9Glr1nzzxVs3n0fZL2N3sQhsYIIpZikdkgPpEzmwDCRcHEvo7SJpX04UiOiPCDdL74Zzhl7AuDS9r6/X7OqSvm/Tlsw+WAjKaTBDIxB/O4X0LYf1PCSsPG7nP3Vbj4jbHIDiYjWS+V0FoJSuotlQIxy1SOaXThupQvHWo+j3tYGBVVTNRiaNx8qwusvATro3DqQyZmJySihvOAClGuVt00zgWOafJ2AbvqvH1PLT3vkRwCpFYlgKi7duHIbzn/b/t3ctwHFd5fme+9inVi9bluTYsWVLduxAEqLwKBC8TlKYSR/DlK5oM9BCmCYlbaADocx0OmjVTttpIRAIBUISkiYpBYlJQ2GghQySeTUhUbETbMd2/I5tET+l1Ur7uI9+/zl7beNIiXb37u6Rfc6Mdle7d+/9z7f3fvc//7OP7Hl0ulGwKAWN0mu66uQbohU7ZDygRbomhHzBqfhBzlfYSuarSzWXW3Ou94KUKMh9icofuu7uF30Kz/poZGMt9ClE9Yac0UOz7+/fHehpXVPyyPSLtl5FO7tGN3nRvgC1luBOBgozKqI8RsTsKLX1CrYKAV1IxIkkMQhqJ4JHufD0SMb2ApTOmv4Q/GiVPjCXPIRYde3Z3JMupeRIyq2VTnFRfE8EMYdCG/fDgHEEpACpg1wFBAcCrRo8bfYNYo9jgV7zNb1OtghbMoSn3nKYBa5cCQfoAi23C6hE4Dn7Sb5K2nq99rySfPK6bj5HEcEYFPPuUYVRmdt64dQTcjMmuuTUMSXntTG9lLYY5CdNc/PtJ2DCODZf5Y9GI0KXOLX9wjL9cpIF/5NzILALv2aERUKmSxHdBTu7Cl4avuxpNKCvOD5sVSaizxFpdGRZ7LcO0Oe1qFPuHxe/5QszMzbcp6RUebBfnRf27m8kzzNdJLpt44mhsCaGbwOSR8RLQxIQgJZOC0Xc1SLPl3oEyDh5tP2i0Aanb2Liz3mKTpBtv2pGWL6QL048sgwBsL3ce0CoyzbArHS30pl1rK/75uNCPHE3C1bUJFeNbTv6IjhA1OcGWZL9ihZY8gHDZ4+EcKbn0CXHsMOcsM4asoMFR+3ttRHwNm1K8usVD/s8HuwenOby2odf2BbQU3iKDk7qdgTnLqdvBdn2q2aEBTWFj9O5M5c53mybIy5MKa9LeAhRRTPK7VcgD9Bq8GIiboir9CtXPngKAaS7qaEAQv88hK1IulLmPx/IHAsQzXppWcdVh+idWmqf/IjqYV4EkknxETOaeeUP/EfXLz+vxCcyPIq2X01NuKrcDC81E2ROYc0Iy1865ItHN0ZjpoEqo6jtJeEdATYapJzgl9f308+dRvdePNXiJIAulyx5ZUPPUE4hzHpoIiTDSTaPDJDPArGG0OX56q5PZGlJ4gdszvMN9XYNEfADkGEjOpzP6wVE8tPNVbYziO72CBPCA9PXEhx+HGIQ0NSMsI4fH+NAup69hunwEMKjgQs0eNWlOhSIQfUC1BxEQG+lXW2qbn+v+m3/hGOsaWx2hsKwPCTW1oQcX1WOBX+IG4yLOuKm1vxt+o7femvB31cbBopAElH4tMNly67fCy39pXCYbnq4qiQbPom6bv4aIdoIlzsIMWtGWFi3CpvN2V5l0t0JaC2Gfh+4KItWETerQwLQZBC4zrkP/4Rj7LKf5QtsXzRCGToCpzm/0NA3mWOh1Eph1jp0+bLXccLawgNGGyrUJX7wNCcnxgamdc06TOWQYL2QjrBwD+Z2LCgpl3vP3ldq+xWMslITwgLpg2Q5+ZsoiXy5LdrkyKZd0cnPlzz41Q+v6RnYR29sSdaOQGg5hfuhsWxZelpjoccjYUNDmXT5bpHAAWK5kUgIAbWJB3vaPngmjTy7RpZYod/mUh90TY2i3A3hgDTCX51f+UMmbHBCl3IK7d5TPc9Qig5GOpDrvyaE5VcZffrIvWjrVegtws2JUaNj0a4rHGBWXrSPRY92sCsytJfBABM155ZKeCOKWuSxzJRto90eloWSBQAiIRiB/1Y2Y57oaLkKLbaAS2k5Mvec1Lv1QiCZTPNDwav9oiNt2y/EBXDjLGtxnMJqEjiotl81IpGNnE1DmrMShNXqUBxPKQCRhJdnoHUVL6hn8jrlVKyt1io246VRNH1F59e2wTr0eGebSUxOPlR5BjwkpF1FjPZ/ubrrwy+LigjpwOwQ8kx08Unit/1ytPDzeR4TI2WRD7r+7UQC5e/dfKBtv2pCWOPjT/L9ZmYPb4g18TwUqisUiEoY6CnmUSF1qBJGE18Odiduq4uMY2NJjo9pRj/rzDpaSPcsCnEIdG4V7gx2h2IUhfDy2dj/ber7wudoN1u2JBVZVYhn0F/zPW6GEUbbL2MawQM4Z+WzD/t2DthA1gsM1gRyDtWEsL6bETmE0Kr6RLntmhymqnMBvzL5LI0C7lLMjXANa11md11IQ5RE0fSWpY8+DSvRIyuWmCg1wOy6sOWroUa1oZhr5WdMLx5e9WEYInlJYmoC8WpfU5/VEwGRU7h06U0HYchC2y8yvMvc9su5UqAzEkiUQE2YZDApljhFN4MSE3XhgLLPGLgF4CHEWrsYmm1uaj5MO6hvFLdok9URafmUmbPPRMKMWlc1cmkIi56roa65Fja6PvHWNelfkIGXyLVscNUXaohAml9QvO0Xi7xIQc++NlPDg1ay61IXHXcVqsNGhKmlesN74ISFuzFP6sXZH8IfPIT8fG+48nAh4qjh6FJTSsj4Ulv3ew/S5zu04Np6XXi8C/8nW9azz/ZbrO3Bg60R85PdCZ2SoMGjDWF4XASe3dIaNvLZpq9uXv/5uyEvitrxxg4Xiq7+byACdH6AoBCWSZU/nBdKTU0aKNG8hxZ9CjVv1bFjh0uewuqL+QVOWBsHhVDPHbl3mevleiiigbJd5p1Wgz6g2xTy5DTLTOxfyVbOkhiD2iC/e9VLpP7+cc7mnZ0jX7Xy7uNL2y0Ttiw08KzroDnbzW1hKz/d/sRN6x+4nY6OuyJWhPLZRuqKjKQHgw2UnyIIQN5Z4A3puAe+rufuAqBBLJanIbUrruvZXto+iBSdwAmrlEIIbWG6B/eAhAO1ATeFOl+DC4ATtylKesavzJN6g2vrtYBjlzahuyV1EqZ/r0gs/6Ax42yPxk302qt1V2NfRrJZeS51Ui5klnzzhnVfeg99Qk1LVQqOj5F8z8mkkElH5Q9RQ13CkCHhZHPicfQBYR4nrCCQDJywfBadmp1YH2sinuJF/qUiLK5WI9/ZsQ247lt3E5BBtfUq90ehTsKUY8iW3Du1pjX+B2beOYXqrCHsR9w7y93hArbHjwFOZEXDdNGKPWbkMy1fumH9l/4IGhXvsExdaRawG7VJgxDw234Z4eUHXNdA2y8SREptmGt9nlfkhnc/06Ma2AInLD+HUGf6eo9syJK4688HiYyUuGj1Ii+VoG8//7NGvGZszCZ7VkfHI7tXxiM3GgX3eDRucE1LkEtgUtEJBEXdc+MJ3dKcWNbNLb/1pivu/ws6AmlWiqwCw7pmO0qlRG5ee/vgUfyWE5ZFSjqd1XINMimQwQ1Uuo4kK2V6VKW8BE1YbKCUQ5h3z2wkYZGkKR2QdDdC4xFctuFT7dHeQwTm8eN3NFTO664bLxJp9az4+ta+ROv1Rt7Y1YJW7OQLBulTyEMV8iGSnuJL4AWMxnUzFo8axdm27y9pvfa65LrPPITPeCt2RVZ0Jsg/SoZ33vaLsch2XgII2rFskuP616mYH0Jl1p46lWoR8lXnKQyUsOAhpOJJ3oS3LY6ggdXFIrcpB3qMYH4U0VgBUB67ousDPKQhlUo1/Acn0roPpHXZZV/bdf3St71Jn2GPNsUtFolj8QbCoqBO4EvLNSIv+ptr8M/AQSWS8mz0vNFjCcOMRCOsOBt9RiuuGLhp/f03X7f8rhfoePQFlSc4F5TyvifafuEk8Ny9osntfKdD4+ZAXptS26/lth0vVR+tzlMIPSO4sXGjEObgwf9ahqJ9q3iTcE5hwR0jkD1Bj0brKiyOmnYBU4c0DAwpfvHbQVo8FWbJR6Yw1z/55f73PX4mn/+bWNR8o2GhUA8CXYsoBA9hSfmiFtLnyU0vefsnFgppzAoZOvWwy07pBScXe9LSWx66YcPd3/IxLC0Bi9wt6L+pnhcFAv39Cf67W3oCzXlPkszceSOT8NCwKKfQQ/hQzClmKEVnTyolWrBVKmeghNXRcSWunxGsPPJ9uqHFiV3RvhrvSTeQ9EzR5S43uN833m/ero0jAEOOQcGa5D0c2DHivaHnsScg1RNP7b3l5uxMMYUGrJuga/VEooZuotQDBYzwlTehDK4izIsw17u2ebTohJ93WfQHrfGe/37Lqru4N5RmOOqlzaTGG4cq4zoBsihHBycslzn7Z1BbDecB9bik92S63iALs6NRw5wssjUE8/h4pir5AiWs3YnvcGFmiqd6IzEYTaYYrQkDPQZNuuqho0sO6ClidXCD+7oMtfVCk0qJBnkPSRzStojA3rL269/Dv9/bs+fO5sPe9AZWYL2zucwGx9ORrSnshNFQZzZiLN0esjKHY7H1u67tvrVUo15MjPaVTI6WOhynxZvqcZEiIJrzUtuvnP1z9AhwV+TzPEVHQk0LEHv21QT0vn2UU1j5tRYombTtO9t89EpucJeK7M+elxCN+hAiaFTPv0jvJpOiSeXZLSR6Uco7ZAgWNI4f7/D6+u6lpeLTpb9XlxRL3VEN39M6vJQ2ghLVlGZT1Q3u1Y+nPq0jAoO4yaa1lpaPnZo49odHEPEOwpKv4LZYAXBlsIfAEWE8ZII535SxcNgCIyyooyQEN1wXncw63oN94XLUb0t4UyyTGZ5jTVzWdt0BcWBxt6qfEOUdSfy4IqePUp82bRrjjgye+zhC+8LDb7RiT4KEYd9CwcDNmsoFLA/txbE12VxxzVEanKuzENp+5d8so+TECwVuc/V6T5/+QGtb28NnRDG/NGexcmUOjLBKQrgve8NNW1/47mq7iBWNlAZ30dYLtqBjqztSvxaAibtVueA1YvtS5YQ5PJqcuUoijTVCNHXMOiMArRs3rjG0uvT2uuR+kVN95ik6kK7DtrOUUwjCqtxTyO/UQeA8UhLi4KG98BDmVlDRPqqUHMS+A90HeJ08hDqLbeeeQdwB5PydA5212tlFiAC0aD50L7qNtBgYiuh6rkhzEXuqySP3FEajehjl0jfQEfxsmEqOFhhhnWvrdWI9ak1HbElzCD0EiDEGxZKxvQTYsJbi1SUqAU99RyHQWASEpxBxOQfzeW1G0mJ+ZCpywiF0+HFYD+HlFyGsBLvACMs/eM7JrAlFQPKIVsR7smlYvK0XpeSEjLZtJLNPtL786lkhsHgQGOamga6u1+/HDZgX84OCJZuGBThhb6NH5nFPoVZFGafACGtLckzYVTzv9aSVcpOgbL88EKOifTYarxuefkCIl5RNSiWPQmBBCJDqQoOxj80yT99bqo0lHWHBO8DQJoDWqquEvLx7VEXKTCCERZ6AdMlD6Lgza6lyJQJGpQMOYHkW9dXS9COdS96xj8DbUsO2XrR/NRQCNUQAXJDkjjNcbzvp3MaQ8brjbb+gxaybmvrjpQIPMEYFIxDCGir1HHvu4D+1gfXXUidlrL0C2XcFc5r/KxSAhRpYhh5+aW37b0/ShrVv6zW/OOoThUBQCHjM2EnB0BjyXXdQAnnbL09bMjOT6xJzrsxTGMjkNo6Ig085WXgI8928rRcytYVgMj1SWy9iUuN5kqoebb1kmr2S5eJDYGxMzIk0LCrmh0HXnWxaFhGWixSdMHSGjSTk+Pi+ivihoi/RAc8fHSnKIdS0ontmAxIdI1ivImjx/C0keQ2xEIWl6XqYewjr1dZLktkrMS5CBCj7gaZlGM0HUC75JJaFdOXJRlgkohuydLApW0X/+Mnb9LqcEQhh7R4XOYS2U+ixwsTyLJCWPuVMZEHboq1XseAgBita17ZeC5JNbaQQqACBHTtE45T29uRR8NQxal8PyhIOsAr2V8uvEIvC/H6NOEayIhkDIazb+scphAFVGgpXy+hUFQAxV3gIzdmQFeY1sPy7k/hcPSoEFh8Cg4NprLLAUWwA16DJi/nJeA2S3keeQhRxWkEoV1p9tGrCEmAJRve8wmryEMo44BtEWy8Er3nakRWrbj5AMu5I1a+tl4yYKJkWPwIgAiguKXEdM28PhToQOcg2iCd4HTem9U1OptpJvpGRktxlCFs1YYkcQk0jDyE4tJeEQriodJCROqobumbqkb0r2Vsb0tarjN9FbaoQKAMBURQPaf1byfBOznB8mU55mQazqZiEp7VnszlefbSUr1+WjFUTlp9DKDyEdodwX8rI8ZRDCA1L03YSQo1o61XWL6M2VggsEIGznkLT3gvCKqJkMhQtOSronjeFUk6hEYbqUHFOYdWE5ae25Iu/3ogqmCEXJVHJtHaeoA1/CWEglKc7aFEa0hMvkECNauvVcDCUABcdAn77LMPo2oegaBTzwzqRLFtyDeIEbpYxGbusUtGqJixek4kk0dzVJm9nIF8fQvrlgJZOy1VUsOMaVqWAqe8pBGRDwDdgL0FvS5zme8hWiyEbYZ2VCcFFV9E/PtHS64WOqglrhzbCgXGdWe6uJG1moQev33YIGEXagutaJ1qaug/QcZWHsH7oqyPVA4EkL42s66EfURcdYYyvx3EXfgxfJgRDck8h7N/cw7nwPYCOy9n4wm3J8o+MIO4WRNmWy10qJYb3Ltyu8f+Lon3oyj7R3/3JIyRPKiUy3Rsvm5JAIRAEAiKAVNPi/zM5idpOmkb9oKVSHsAN6FNIaXv6yomJd8Z9Aitn9lURlp9D+OM9d3ag2M26ktuyqn2WI/xCt4XT0qWifYYe2wljpJNOU2lZ6YySC52O2k4hMAcCohv0smX3b3U9/WfxOFe4eHzkHBs36i1efRRaTZeumwkhBFSeMkZV5LKxVGXUYeFOZOMspRxCHL0sAcqQteJNqbCFgZAG17P30E66f7ef/5oV71B9USEgGQKkrZyr3BB5mMcWyRdehPZ6JBQ7OTnZNkMQDg2VB2RVhHV6vI1/37Yn4CHUQw4s7zi8jISFlBxNC1ttpZQc0YSyPKjU1goBuREYGkpy84zr9n0jO63tikTQHVTjncJlEVyUd2LW0b6+f4eDANVSEKlfjnBVEda6/t38YNBf4CHkTMUBK0eAOmzLq4zmc1g7O8V9dDzfs1mHY6tDKATqhgA1KCEta/ny9AzyZT+DQgR07LIIoZbCkhnGQF9lqIKHuGCi609Z8lVFWFvQsYMO7Lqz10gX9XEOedeCmxcWrCNNiTX76e1UFSVaz+1WvVIIyIjAKLdbLe189MHJSe3ppiZYbxkTlbIaLC6ISscqTDNY9Cckyhjv+lOeUBUT1vkeQjgbV5KHUNKBon2YpseOXLf8rhNCxsGyWF3SeSmxFAKvQIC0GOrwTc+mlfhoNku9qzzyGDb6AnVRi87IZNwZj7WOkeCVNKOomLD8xMXRXbctRVP6XrsIDpDPyMcVYjL06Zr1KwJJeQgJBTUuZgREt/CkuXTpQ097WvhvW1pCFJdFmlcjb9RuLIYlKjOe7Oy8dy8pPEh/LptEKyYsrKv48GyrE4vSJQ6FfsjYh5BIFMVPUVHwRRJ406Zk5XMWU1aPCoFFgIBYGnZ1feMfz5z2Hm9ttUjLQqJHQwaI0jNyOfAVi39RSJA06hqH5ecQFo2TG8Ix3cKKkDNWQ+CY/6DUzRUeQlez9Cbe1mv+TdUnCoGLBwFaEg4Pp7jVvWC/9X2nT3tPEWlBs4G/vO6jCC0P5WXMxzs7H/6h0K7GKooRq1jb8D1tsPuvojrpGKTeSRbSgOZHkA3F+WdN3T5AQvpy02s1FAIXMwIDAyMoRZA0V6782Gw01vo7MMJvbWuzQpgzGeHrtTy0kdsYmpx0Mq7W9nGBd+XNiysmLD+HEE0nSjmEMkaOe4j7IA+hdqSn98b9BFZKUyk54qRRj5cCAoyN2URaLS0PngpHOm6YmmSj7e0h3whfkZZTBm60f5OCtnUj/Gfd3V85QLIwNlLxcXlPszIE4JuSSof1Z8lgxuAhpJUXwp0k068gLPc5wq7KAAAJP0lEQVQQQhVF0b4BXrSPBFdDIXApIeCTFmNfPo3r9MZfT7z3gaYm69ZCwdYKBY+0LeKBoC8MG5ea2dxsaVOT+l2d3d/4Ji1RQVZV2dEq0rD8HMLv7flghydxDiGUXrT1gobliZIy5O7FD1MvVfhSuibUXCVHgEjLD3fo6h7+UGY6cmuxaJxpaaGQb3jLeKwWKykhVU2Grq8CanKZ4bAJsmJ/3dk9fDcpOalU+V7BCyWpiLD8PoRG0e7EDttlzSHEPUO3wec6M3fwiScvnL76XyFw6SAgwh3SOpFHd/djD5nWqqumJqOPGEZIa242LMPgvURJA6K/cm/scMTzAFXW1mqFigXjdC4ff3dn98inhZGdPIRl7/MVP05FhHV6jWiC6OqzG8IRPewg8xl7DlqlfIWwZb7BU3IKeSyXdUsU7Rsrcw9qc4XARYYAFfujKT37bL+1ZMlnD3d2P/anurf8bVOZ2IjjmPmWFstEpQeTl9RiHpZ1rAiiIQIju5OD1/yZvwaxlUjKgeNNb242LROd1U9PGt8MRTqu7ep65NueJzyVQZAVjsnXrvRc1ljXL5KHmaNfbsU0LZ+jgHvEZso0YGNDUoLh2OxEyAhxg7uocDgmk5RKFoVA3REQ5DFeJDIZGhrxlnR+4ecQ4ucnTvzVhqnMSURY5n4fq8SrEgkdbVk93p6L7NTUrwHaEjdgGwiYQOl4VEEBmyFoPJt1Tk9Oed/X9cR9XV3/9mOaVMnATmQX2KjI6J7UkiCoMeh39tVeZZwX2ATm3VHJfuUU3SMnt90woWlfKbuUxbz7Vh8oBC4CBHxvHRELrmcs6e6hlcjfed7wP7z88k9el53OvtnVnGs0d/pKpptxGOxb0Xs0ojE9j1XlJJStSbx+HtmBT7le/KfLux85RLCA1KC8pKB8VWdgnwvispdxtB711bvvb3/Pj6JN+uaZabIUScdcdjxhmrlp87F3bfyP93O5CYEA1tFzAaneUwgsdgTOLd9eGXZAicunTg015fPjoUiku9jWdtO0aN56btZ0jSGh2SBb2bl3g31VgYaVBsmlvf/85btbocSsPZtDWK6JLth5vGJvpLnSKtXQ47vpwyEAqdUQyFcIoN5QCCwyBM5pXHTtpPTx8X16f/84NC+yXXEPIq9hJab1Vf5EJCe2+z1sk8ZlVzuyogOWTVh+H8L26Iq2rHOsg3IIz2ovYiYyPMLg7hqFnK41Wav+lwTaJINUSgaFwCJAQKyguJbFAzzp+hZOwyE8+2OQ6wQlksN24/4HNX0um7D8HEJPN3ot3YwWi3nMBlOUaeBugHQAw7PDe9f2vf0pEm1LUtjdZBJTyaIQWAwICALjlzhIyh9p/0Vdn8smLF+6kBHpMJCWVCjmXBCWXB5Cz3PDkZBhzyaeWMY2T4+OauZmlq7ZutrHRD0rBBQCtUWgbKLxi25Z+tKjszOOMBQJfbG2ki5077TWZq41kzGdFmP1o/S148nUeXeGhe5IbacQUAjIhkDZhDVSmkHRMV6CIjOrIw4DS1xpCAFNGp14IqRZRvvwtWs+vo2EG2DVpwTI9sMpeRQClyICZROWXw99zdobjxl69CiVlgFlSUJY0K5018plQsXWWN8/0w/qacM0R0nkuxRPMTVnhUBwCJRNWBTan0ZgWBe7Osu08C+sEMxgiG8ITqQq9gTtKtEc1cJm5+euXfGRbWnYri6MFali7+qrCgGFQIMRKJuwSN5NWpp/L2QmvuM6VKeZ0vYaO6DnFcMxZs1mYjs39d0yRNIMJkXeVGMlU0dXCCgEgkKgIsLaMqTxBMr2xBu/U8yF9oTCjW7YyBxmuFYxH3JaQ2s+wNh1M6OjaWhXirCCOlHUfhQCMiBQsWZEtXUoBH909x0fNaMn78lM5m2EY1UcJlEpGLCpI6zCYbF4DHnlnR/a1HfP10aRG7UZ9X8q3af6nkJAISAnAhVpWDSVLVsoEFPTkn3/+sVsRn8uGufm9/o2bGTQrEBWTYkoc3MtnyKyIvvaZq2yAvc0HzUUAgoBeRGoWMOiKfla1tiuO6/PORM/9pitoTIWhfPzbh01nTZKRWi6ZzYlIloh2/T3N13xwKdg+kd8BR6oJoYaCgGFwEWHQMUaFiFBS8I0lobJ9ff+xNBid8Tj1JADVQuh9tQKKTAskVEhFNFRK8zSijPNd3KywpvpoTS4SpFVrbBX+1UINBqBqjWh0YcPUPtU/UOpbc/ccls/iyW0ZCFvgwihbgVY1E/kM2koIaaxRLNl5mbZcUtvTd24/v6vp9OankymWTqtjOyNPqHU8RUCtUSgqiWhLxh1zPE1mx/sfP/HzXDuM47jaMWCR00bqaVQxcchjQokBY3NY+EoGgYxQ4NncqQ10v2RN/V8emLY0wyUSKQSGGoZ6P8g6lkhcJEiUNWS0MeEyAraDS9u/84Nj97tOi0pp2ieTLRQ00ZSwBiKqJI3b8GkQuTjYGubbFKRGModJiKGnY/+ys23Drxzw2MDRFb3oS71AK/Vs+D9+iKrZ4WAQmARIlCx5jPXXKluzsiIpg8MaM5PD6WXZ7N7P61ps7dE4kzLzdqabXsuLEzQhnQiJAo3ReofbOTEarxzNNETvfTMUNjQQmgTVMjhAzv8TNhc8pV39L7vUcRYkSeSDXspHTmCNbOVzTU/9Z5CQCHQWAQCJSx/KufHQY3uuuPtBW/6LzXXeVck7rUyHZ5EuBJtFP5zqag9vmSAvwx029ARfwpzmDabBUm57KCuh34YMVq/dX3vPU9Ci+PkxPdNYQtqCejDrZ4VApcMAjUhLEKPurxS40TftrT18JcvyxReekfePvF218326iy8xPPsDjQNhL4VOm17uRNho3lCN5qeDZmRZxLRjVuvXDYw7f8SFEJBsV/KsO4jop4VAgqBwBEgoknzLhq/uWvP2xPePvlA+86XvgjiOhz9zU/Ff8PDmjHqpU0y6s/1uXpPIaAQUAjUBAHSuIi8YLOa19BP4Qnk9RPbpefdriYCqp0qBBQC0iPQEM2FDO3kMBzShvjxB7VBmLJggVd2KelPGCWgQkAhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQkAhoBBQCCgEFAIKAYWAQkAhoBCoOQL/D8pAub21P/M1AAAAAElFTkSuQmCC", "icon_small": "iVBORw0KGgoAAAANSUhEUgAAADIAAAAyCAYAAAAeP4ixAAAMFmlDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnltSCAktEAEpoTdBehUIHQQB6WAjJAFCCZAQVOzIooJrQcWCFV0Bsa0FkLUiioVFwF4XRFRW1sWCDZU3KaDP1753vm/u/Dlzzpn/zD13MgOAsi07NzcLVQEgW5AvjAryZSYkJjFJPUABUAEFGACczRHl+kRGhgEoo/0/y7tbAJH0160lsf51/L+KKpcn4gCAREKcwhVxsiE+BgCuyckV5gNAaIN6o9n5uRI8CLG6EBIEgIhLcJoMa0pwigxPkNrERPlBzAKATGWzhWkAKEl4Mws4aTCOkoSjrYDLF0C8FWIvTjqbC/EDiCdkZ+dArEyG2Dzluzhp/xQzZSwmm502hmW5SIXszxflZrHn/p/L8b8lO0s8OochbNR0YXCUJGe4bjWZOaESTIX4pCAlPAJiNYgv8blSewm+ly4OjpXbD3BEfnDNAAMAFHDZ/qEQ60DMEGfG+sixPVso9YX2aDg/PyRGjlOEOVHy+GiBICs8TB5neTovZBRv54kCokdtUvmBIRDDSkOPFabHxMt4oi0F/LhwiJUg7hBlRofKfR8VpvuFj9oIxVESzsYQv00VBkbJbDDNbNFoXpgNhy2dC9YCxspPjwmW+WIJPFFC2CgHLs8/QMYB4/IEsXJuGKwu3yi5b0luVqTcHtvOywqKkq0zdlhUED3q25UPC0y2DtjjDPbkSPlc73LzI2Nk3HAUhAE/4A+YQAxbCsgBGYDfPtAwAH/JRgIBGwhBGuABa7lm1CNeOiKAz2hQCP6CiAdEY36+0lEeKID6L2Na2dMapEpHC6QemeApxNm4Nu6Fe+Bh8MmCzR53xd1G/ZjKo7MSA4j+xGBiINFijAcHss6CTQj4/0YXCnsezE7CRTCaw7d4hKeETsJjwk1CN+EuiANPpFHkVrP4RcIfmDPBFNANowXKs0v5PjvcFLJ2wn1xT8gfcscZuDawxh1hJj64N8zNCWq/Zyge4/ZtLX+cT8L6+3zkeiVLJSc5i5SxN+M3ZvVjFL/v1ogL+9AfLbHl2FGsFTuHXcZOYg2AiZ3BGrE27JQEj1XCE2kljM4WJeWWCePwR21s62z7bT//MDdbPr9kvUT5vDn5ko/BLyd3rpCflp7P9IG7MY8ZIuDYTGDa29q5ACDZ22VbxxuGdM9GGFe+6fLOAuBWCpVp33RsIwBOPAWA/u6bzug1LPc1AJzq4IiFBTKdZDsGBPiPoQy/Ci2gB4yAOczHHjgDD8ACAWAyiAAxIBHMhCueDrIh59lgPlgCSkAZWAM2gC1gB9gNasABcAQ0gJPgHLgIroIOcBPch3XRB16AQfAODCMIQkJoCB3RQvQRE8QKsUdcES8kAAlDopBEJBlJQwSIGJmPLEXKkHJkC7ILqUV+RU4g55DLSCdyF+lB+pHXyCcUQ6moOqqLmqITUVfUBw1FY9AZaBqahxaixegqdBNahe5H69Fz6FX0JtqNvkCHMIApYgzMALPGXDE/LAJLwlIxIbYQK8UqsCrsINYE3/N1rBsbwD7iRJyOM3FrWJvBeCzOwfPwhfhKfAteg9fjLfh1vAcfxL8SaAQdghXBnRBCSCCkEWYTSggVhL2E44QL8LvpI7wjEokMohnRBX6XicQM4jziSuI24iHiWWInsZc4RCKRtEhWJE9SBIlNyieVkDaT9pPOkLpIfaQPZEWyPtmeHEhOIgvIReQK8j7yaXIX+Rl5WEFFwUTBXSFCgaswV2G1wh6FJoVrCn0KwxRVihnFkxJDyaAsoWyiHKRcoDygvFFUVDRUdFOcqshXXKy4SfGw4iXFHsWPVDWqJdWPOp0qpq6iVlPPUu9S39BoNFMai5ZEy6etotXSztMe0T4o0ZVslEKUuEqLlCqV6pW6lF4qKyibKPsoz1QuVK5QPqp8TXlARUHFVMVPha2yUKVS5YTKbZUhVbqqnWqEarbqStV9qpdVn6uR1EzVAtS4asVqu9XOq/XSMboR3Y/OoS+l76FfoPepE9XN1EPUM9TL1A+ot6sPaqhpOGrEaczRqNQ4pdHNwBimjBBGFmM14wjjFuPTON1xPuN441aMOziua9x7zfGaLE2eZqnmIc2bmp+0mFoBWplaa7UatB5q49qW2lO1Z2tv176gPTBefbzHeM740vFHxt/TQXUsdaJ05uns1mnTGdLV0w3SzdXdrHted0CPocfSy9Bbr3dar1+fru+lz9dfr39G/0+mBtOHmcXcxGxhDhroGAQbiA12GbQbDBuaGcYaFhkeMnxoRDFyNUo1Wm/UbDRorG88xXi+cZ3xPRMFE1eTdJONJq0m703NTONNl5k2mD430zQLMSs0qzN7YE4z9zbPM68yv2FBtHC1yLTYZtFhiVo6WaZbVlpes0KtnK34VtusOicQJrhNEEyomnDbmmrtY11gXWfdY8OwCbMpsmmweTnReGLSxLUTWyd+tXWyzbLdY3vfTs1usl2RXZPda3tLe459pf0NB5pDoMMih0aHV45WjjzH7Y53nOhOU5yWOTU7fXF2cRY6H3TudzF2SXbZ6nLbVd010nWl6yU3gpuv2yK3k24f3Z3d892PuP/tYe2R6bHP4/kks0m8SXsm9XoaerI9d3l2ezG9kr12enV7G3izvau8H7OMWFzWXtYzHwufDJ/9Pi99bX2Fvsd93/u5+y3wO+uP+Qf5l/q3B6gFxAZsCXgUaBiYFlgXOBjkFDQv6GwwITg0eG3w7RDdEE5IbcjgZJfJCya3hFJDo0O3hD4OswwThjVNQadMnrJuyoNwk3BBeEMEiAiJWBfxMNIsMi/yt6nEqZFTK6c+jbKLmh/VGk2PnhW9L/pdjG/M6pj7seax4tjmOOW46XG1ce/j/ePL47sTJiYsSLiaqJ3IT2xMIiXFJe1NGpoWMG3DtL7pTtNLpt+aYTZjzozLM7VnZs08NUt5FnvW0WRCcnzyvuTP7Ah2FXsoJSRla8ogx4+zkfOCy+Ku5/bzPHnlvGepnqnlqc/TPNPWpfWne6dXpA/w/fhb+K8ygjN2ZLzPjMiszhzJis86lE3OTs4+IVATZApacvRy5uR05lrlluR257nnbcgbFIYK94oQ0QxRY746POa0ic3FP4l7CrwKKgs+zI6bfXSO6hzBnLa5lnNXzH1WGFj4yzx8Hmde83yD+Uvm9yzwWbBrIbIwZWHzIqNFxYv6FgctrllCWZK55Pci26LyordL45c2FesWLy7u/Snop7oSpRJhye1lHst2LMeX85e3r3BYsXnF11Ju6ZUy27KKss8rOSuv/Gz386afR1alrmpf7bx6+xriGsGaW2u919aUq5YXlveum7Kufj1zfen6txtmbbhc4VixYyNlo3hj96awTY2bjTev2fx5S/qWm5W+lYe26mxdsfX9Nu62ru2s7Qd36O4o2/FpJ3/nnV1Bu+qrTKsqdhN3F+x+uiduT+svrr/U7tXeW7b3S7Wgursmqqal1qW2dp/OvtV1aJ24rn//9P0dB/wPNB60PrjrEONQ2WFwWHz4z1+Tf711JPRI81HXowePmRzbepx+vLQeqZ9bP9iQ3tDdmNjYeWLyieYmj6bjv9n8Vn3S4GTlKY1Tq09TThefHjlTeGbobO7ZgXNp53qbZzXfP59w/kbL1Jb2C6EXLl0MvHi+1af1zCXPSycvu18+ccX1SsNV56v1bU5tx393+v14u3N7/TWXa40dbh1NnZM6T3d5d5277n/94o2QG1dvht/svBV7687t6be773DvPL+bdffVvYJ7w/cXPyA8KH2o8rDikc6jqj8s/jjU7dx9qse/p+1x9OP7vZzeF09ETz73FT+lPa14pv+s9rn985P9gf0df077s+9F7ovhgZK/VP/a+tL85bG/WX+3DSYM9r0Svhp5vfKN1pvqt45vm4cihx69y343/L70g9aHmo+uH1s/xX96Njz7M+nzpi8WX5q+hn59MJI9MpLLFrKlRwEMNjQ1FYDX1QDQEuHZoQMAipLs7iUVRHZflCLwn7DsfiYVZwCqWQDELgYgDJ5RtsNmAjEV9pKjdwwLoA4OY00uolQHe1ksKrzBED6MjLzRBYDUBMAX4cjI8LaRkS97INm7AJzNk935JEKE5/udEyWoo++PQfCD/AMf7G3o0obnYAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAAgVpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6ZXhpZj0iaHR0cDovL25zLmFkb2JlLmNvbS9leGlmLzEuMC8iCiAgICAgICAgICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjEwMjI8L2V4aWY6UGl4ZWxZRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpQaXhlbFhEaW1lbnNpb24+OTE2PC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPHRpZmY6T3JpZW50YXRpb24+MTwvdGlmZjpPcmllbnRhdGlvbj4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvcmRmOlJERj4KPC94OnhtcG1ldGE+Cj6cgV0AAA+rSURBVGgFzVprcFXHfd/dc859IfGSBAaHGlkgg4TAiZKm7bSxnDpN/MhrYhHbsfOa1E0m0/RbP3Xq637px2YmM52J+6GeTp2HNAmNXRtjZxKRh5PJoEkISIDAwjgEMEQCrMe9uuec3f5+/z1HlpBkE7dOvXDv2bOP//u1e6XV26g5p/Sg6jeXh9eZzqkx19c3lGqt3NuIxDcmxbmqWW5Vtbr8+HJr/9/HBgb6g5yIp0fue++zo/d/4cCxT96Tj71lzNAEMuA6R/ZmnzkTT4/037B/9BPPPz92rxs6s9f9+Own3f6RTzx74MJDqwj7jZhZVp0rEUVgA64/oN2ib8GQWskkVoKxcJwC2bt3MD3kHo6MtvvWtkR3xHGa1qbjeOpKI17bWvhgOln7a+7ZdM9T81pbCCPvXzcjlByJ36sH04GRauH5F/vXkCGtq7a6gn3nSFZ6PjbcG3Lud6OTd5Ur5k8u/26uobTWkE+Eh6rXElUx7k+55nzvcMrnSu26GCETlNyzI/3rnxnt/2qTOjIS19Kx/aP3/veB4/3tVTDzZjTT2dssEcmq9F3awEqdMlCx9BxkZK1TrUW3g7CrWlmYwYr0rjiRc05p50xYlQytXR/8XVRQ20ygNqxrje62afotrqVmaCr5vut46j41lElZ714YZAkHHGobW7UqtFtV7eRmgffIyIrw35CRLuU3W23/cX1bqefyxFwtbtjUpi6ZuFS3kOQf/3DsgVuJiDngOhiQJRAQrceNjPQXYEw7k8SCdEyRA+oFojFQSXPFrJ54Nb2Fm4aGLr05RsQZ4ROAoZ2zfz47HVNUEd7p8AFwurVNgWovq24iansdRJxf2HIBjWt7E6zophjSBzwRBKkFD6ZgVApGVGLj3Qv3Ltd/AwnCMtGeO37fJnRuSmJarvLWDOZSq2xTUaly2Hgn1/X1tYGW62s505EynYWSKUHDFjgEXw6hHCjvMNoJI5curQxfoka+8drn0NAQGbU2adwSRLoV6gcfYr8ZRq0iZ8FZKozoTHvYc90MwbH2FIqBatTTFJsQrWhWaEBUCZ2mppSzu5xDgNQ6FZvDGi5Z2F5XI2PNU15CRu8ulYXnlAP8EBLgGQO1wMZ3nD376RYCRoTxe/jyOu1g3xAoBAzt9oDIRZRZgGTSKAXazM0Rvr55cvLBGz245eG/LiPrxm8WZIiCu4W6BXLAuwtgZiZNKUJElbntRJRp0eNc4ZtSBTn20KGHI/R3JpS6l4//hoQKMKuCgUYS54JQrW80YoEPDMvSvOwgoRIZw67H4HaJ4vmSNaKOIDZYRbIKDm9MQyJXc67FfOEyz8FBH92uFKa3ANNWREFqJPc9JAzANYgqoA7dtAJnCXTaQ1DDw5mVXAN3RUZyZN8/cv9G7OkQW9WMXrAoqIPISkAWos9kBkC9hN3b67V4DZ5Fr239PozGodpRKJhmCEkcHSDnG2jPONPi8HAO8cPxzErmF2adFRlR/X4FDKfDBLolVz+R0cz4JDKYl0oSvBknEqPDC7N++7LfY5lUtUp2RUUhgYx4uBQUGl0Sjk/T0An9ULkujtNKloO/IiN5eHSh7S5V6HoqAVwKH3B8KwWCVM/BNMDZ9osX+2/gzOhof8SyhghZZF5biz2c1U0IgT4/ZACFQMoEiMrIUtC6ZPhGA8HAuY4LFx7aQPi5tbCftxXDbx6zAfxW460XshEhSYgRZKHwpKEtBxNZ34gTSu1Cd/dgI0cAtPCzQRBSNVJgol7y5YzTB0b3dqfUpuiAUlKOwkcihI8IBAn3NOsoMi2plYBysa1taYZfnhFIglUuQUESu4hM6gmInWjxcRGQExmkZjAZr1oVRsmVmBn+B8+d2HtXmqT3gcAtyKAnQFsVxF/IC8tqVannT/ztJuTTrUyyaJrSF23jlY4eArbMeDaTVauC8NWplPB/2tfHLYub53vxmKqqqsjoqRMPtyLOb/NRxclaSVgZsiyqyG7mAlRH73nq6N5/KJXV05Xm4KFCUfeta4v+BpH2uWdOfmo1NTE8fE7sVLmr28NIr/U1FnMdiwYQj2fue6JvzwizIaW2x5O6NMMvq5HbsoxeSGodqbKbYy81b2BEhk+ODOkYPLhw6mpDTTSC+4tFF85MJxhUDSw09Xpqm5qinpmZ+P3Y9l/jjcvEGVub9lQQtqcbCReH0J7QTXnQP/g+rxIwIQwrX6rkAYXMA5a0ZTUy1twpGnFB0lVeJbx6M8s2cXeJw1jFviyGxV1tuLBWZxmr+ClggkGtMDsTI6jZCYypprZpzmGf7YG20cMXFgkMvGKDKwUBEgvnME4GcUZpSK6x286f72+TicxqfB8BIu8sfPIqRt6t3RPQECAbIuKHEuOzDDtmH10ZoBvVIVsQwlwj2gMQFzLRODMZhZVxrv3FE/tjPsFrN8O2mAzfuBaVllPBK0UT/SjE4Ze88IMmjGDtDTDfDg5cW0EsYYTE3X77UMLFTrseVKXsSkNPHB3lgy6Fws88gzSxOWQDjpJ0rsViGzIEKfPS+ztvPk8g1aqyPz31xQ2Ygu8hSyHfcZxrWTyiBH0xNG5fAG8H4fPIAZcOj6iQSMi+toJYwghQiZR/cvzvm+Fc27MDDyGSSmsA36hwIjLBBPtCAr6gDU1XwkLZT+LQg0aAwphjdPQBHKI4/Go82wEaN0rEYtTjUgxkxI9XIv0TmhLkIXMULiUT0FC1FUaurSCWMDKYnQhr7uJWbIKjSx0kxIHItAKfiV34I8B8gX0oQrJyHXUxlcdcQML4zU00HZB4RMauKolYxsH3fJIV38M6uIGAwWlNj61p3TCSxOoKcodAEt2jx3oPYHdxMHd49tmWMJJndOtiImN2ZukujBBMhEoxToOfQf0/j2hj3oxUTUjyCylB/MPDhQ2U4aEpHCay3NFTl96KMz8XCfXCOdIk1wZBCO19bQ42+yKSLLdJcMBTZ1rqPHPmgXWcyK2HvSWMHOQoGqNKII4qgMgI8Rn6TKyjX4c6PSJZmWOguA7FySKhCiuRPExg9NycqylTHsOImv5lk/geproIB2rwRSh0hr1BvWYbFV06ybUAcDRi1vV4ZUAKV+U2RVG9nQMqsx72ljDyCC6OOeFc2sPrmLyhh6SkgpmpVCX16BSEeXJqKmF4ChLQU08QT4gXH6yj8hwJwb+XN8UbzxIOC77vHf887oDstsz3ZD2xRZA+aujfNBebXuZaqPMwPmyEyh6ZTpubeWVgpYBU6rVSZREjlBCJOHSuWgEyHHiEJ1kDaBYhEedede7iml1n67Z0FuHzYhEENFCP0jeJkSjxn6woOjqC0rHu7mrj64d6SYEq2fgmDN6Y+V6O39JMgf/YO9u/eoXrEEgO17y9wn4lehG8ZTgHjXL2wXUHGZSWA5KXQbVX3iemzv0RKNnC0gQ0CVFCGBgBp2Nf2f6VuQ0bBqcB8iTtGPicODpQEXL2kOIJFP2awDtbmsXREzt7S7mMjIdCC+tInOccPWOc+BKH0rR4Ej4xBcKB3+sGPXF4ECkOj2I095/FpsXfJQjEuPotvNngLQmwiJY4zvAHA/IRSBaqoxyrscBf0MgMjN740iYQRpTaKisCHe6m72EH/YWi5mFDGhL4L3xH6c2bd/wWyF7KHF4kD3aQGCX3dJ6U2k3xDtrT7EH4785en9FTF++JCkAG+4KIRa9Api0tzYa/yveAw18mUAUtINMbpUuyXGCAtGbrQaJHub6WvCLEwKnO0ozQWORABE6vay1FM6+mP7yze/BJTjhc9DHvoDsqeUjGOKMQPChbtWX16tl2DjyS3T4KNxxgO/jokKgKPtWTOZqf8N9BbQYUR2YkH4xUcvTqdKqQ0RnffEMHFMN5Jb6eqc+Vz3Dizm3vldLkQzu/9W+4rfwPzBtkcjBjLC6vnwhLLR/nOn8Iy5zY6iM0JzYPFkJlLquEOAn5E+Pw8PhijVBFLB8GBgYCZ92ORVEF8RJ1EMFdKOu14wIZXwjmp+qJe0VB/MDkJe6RSmAAnLEPv/uxWSCH8qrzd8N3dX33MzbR727U3Ud0EnTf1f3dBz/Q8dhVf1nNUn/+GuqIaADc5jjxlGiIiJpleH8RTvVK6+ryd7wtuw68o+GyAw9NGeQhyttCIQhQhpx83/avXcq2aL3xyVeOnP74CdRTG01sYxCMe0ex+QQRjj8NvMC1Q0N9UM8QArS4rcgYmXkYU/x4LTwKPN6csguMYTi/PlavJ7PwwwovKLBUGGJaQKj3x2RchFNQ84zkx8dY1bbDP5rgVGRBNjJosA4CGWJWPI9zPYtLBLbHipXgfQiVRayDJ6pg7fpi+epkfBqR5+sk9GBfH4gYYnfel3iWZxXRh7mcAVkgXz4azcw0/6ZcunIGDr+T5xqQAd/BYQZFHZ6dzn22pPXjdWpyocoyOHF3AbcK8MH8VpEmI0UbJCwRKGOCrq97O/Y9MXVl7p9gRq4QSdGfgolnorB4xz27v3FZfiDKJJ0hkAeP0hTEUiY8s5Rye/vjddjECUYuvOemK9eoCN9bLl2a2uJhjniJ8yW/bOBdbFY9Yy8jFv+rsD6bKhMUXwu91E8G+q927nsENVJH2tB/meryzju7vnP3HZ3fGM9/IPLIfr9vb45Eog5DgPOb8Y7LTWdR0hdtHO/kBH9uENPCJNb6ywZs7ZJ7KjDhd+McCkdPYzdZUuYUx3KmX7P5KvZXT2OKHyLXvLLJbyo59mabMeYwcwcaYzZFR76SIqJePbC8jJCQnflIFURX3ffH798Y1+baJaOz9PESByP4oSJRp2/vevwCNqr+/gHYvOeTzHBv7jdkEkLBfC4Y7vj9W/4TBaR/fLaGDIaaDpjo8IKYDoMkJw5PPxNGsmOjbcym22BsG7JbxfxkgYxOV7JHSQ4JhkhERHzP2/+F9HNY/ukdfvPmfcfOn/voC62thb+YmGjMQXAR/CNB5MIvGvodXEs/E2fPLxsQDxYdeEQn4B/Mgw9fB12+2Scgj+yt+/Zm6/+QIAjd5ycnG6daWopl3J+Fa9ZEZSlIjf53UuAQAUUj63ovU2VkrUdqcUQIAJIGHrTcazktoTf/JdbPvrXf9FtaAArUU6dPf+w9brLxJWD8M1COFO3+84Ybn8x+iB1MhZHRRwcpc5rhVj7RkeM3ng0cZ8u1aXu6bDZKcjv46Gs5Qda+xV9yaU1zbh9kef/PC9H5IOU9WUzrttv65Inj9zc5XGkKy0VcLq1eUyjPTqf1wJjP3d79r9NyIY0/GlgI7A/R13IDXzXO9fG2GTmNff8XGDn+zIDklX23/+i996EQ+DL4KYGpQ84E/3L3zm+P/W9yQo7sD/Ykt8shoyaWG39bj2VEC0PsUxNva4Iz4v4H8vqTm++oi2AAAAAASUVORK5C", "version": "0.1", "provider_url": "https://live.paloaltonetworks.com/t5/MineMeld/ct-p/MineMeld", "display_name": "MineMeld Feed", "summary": "Indicators routed through MineMeld", "tech_data": "Indicators routed through MineMeld", "name": "DomainHC" }, "reports": [{ "timestamp": 1550566791, "id": "DomainHC_report", "link": "https://live.paloaltonetworks.com/t5/MineMeld/ct-p/MineMeld", "score": 100, "description": "MineMeld Generated Report", "title": "MieneMeld Generated Report", "iocs": { "ipv4": [],"dns": ["25z5g623wpqpdwis.onion.to", "27c73bq66y4xqoh7.dorfact.at", "27lelchgcvs2wpm7.3lhjyx.top", "27lelchgcvs2wpm7.7jiff7.top", "27lelchgcvs2wpm7.7zv8o2.top", "27lelchgcvs2wpm7.9ildst.top", "27lelchgcvs2wpm7.adevf4.top", "27lelchgcvs2wpm7.ag082d.top", "27lelchgcvs2wpm7.apperloads.win", "27lelchgcvs2wpm7.asd3r3.top", "27lelchgcvs2wpm7.b7mciu.top", "27lelchgcvs2wpm7.bedrastic.bid", "27lelchgcvs2wpm7.bestfordownload.click", "27lelchgcvs2wpm7.bonbestal.asia", "27lelchgcvs2wpm7.fm0cga.top", "27lelchgcvs2wpm7.h9ihx3.top", "27lelchgcvs2wpm7.laverhants.link", "27lelchgcvs2wpm7.liopakerb.black", "27lelchgcvs2wpm7.marksgain.kim", "27lelchgcvs2wpm7.nfgpeb.top", "27lelchgcvs2wpm7.redefined.click", "27lelchgcvs2wpm7.rt4e34.win", "27lelchgcvs2wpm7.tankbe.pro", "27lelchgcvs2wpm7.thyx30.top", "27lelchgcvs2wpm7.uboys5.top", "27lelchgcvs2wpm7.vrid8l.top", "27lelchgcvs2wpm7.wins4n.win", "27lelchgcvs2wpm7.wishsends.mobi", "27lelchgcvs2wpm7.xkfi59.top", "27lelchgcvs2wpm7.xmvr54.top", "2bdfb.spinakrosa.at", "2gdb4.leoraorage.at", "2ymh2gnnbg6pgq2r.gremsot.pl", "2ymh2gnnbg6pgq2r.winregion.tw", "32kl2rwsjvqjeui7.onion.cab", "32kl2rwsjvqjeui7.onion.to", "32kl2rwsjvqjeui7.tor2web.org", "37kddsserrt.xyz", "3qbyaoohkcqkzrz6.bestxprice.ch", "3qbyaoohkcqkzrz6.livecamshow.ch", "3qbyaoohkcqkzrz6.torclassik.li", "3qbyaoohkcqkzrz6.torcommunity.ch", "3qbyaoohkcqkzrz6.tordonator.li", "3qbyaoohkcqkzrz6.tordoor.li", "3qbyaoohkcqkzrz6.torgate.es", "3qbyaoohkcqkzrz6.torgateway.li", "3qbyaoohkcqkzrz6.tormain.li", "3qbyaoohkcqkzrz6.tormaster.ch", "3qbyaoohkcqkzrz6.tormaster.fr", "3qbyaoohkcqkzrz6.torplanet.eu", "3qbyaoohkcqkzrz6.torprovider.li", "3qbyaoohkcqkzrz6.torreactor.li", "3qbyaoohkcqkzrz6.torstation.li", "4kqd3hmqgptupi3p.0vgu64.top", "4kqd3hmqgptupi3p.143h2a.top", "4kqd3hmqgptupi3p.1tvjk1.top", "4kqd3hmqgptupi3p.1zp109.bid", "4kqd3hmqgptupi3p.249isv.bid", "4kqd3hmqgptupi3p.2y4t6f.bid", "4kqd3hmqgptupi3p.3arvfd.top", "4kqd3hmqgptupi3p.3lhjyx.top", "4kqd3hmqgptupi3p.43wjor.top", "4kqd3hmqgptupi3p.4j11jt.bid", "4kqd3hmqgptupi3p.4k9xlx.top", "4kqd3hmqgptupi3p.5b4ej6.bid", "4kqd3hmqgptupi3p.5ctoeb.bid", "4kqd3hmqgptupi3p.62er3d.top", "4kqd3hmqgptupi3p.6h03gw.top", "4kqd3hmqgptupi3p.6j7jcn.bid", "4kqd3hmqgptupi3p.6ntrb6.top", "4kqd3hmqgptupi3p.6ogy3i.top", "4kqd3hmqgptupi3p.7w9p1n.bid", "4kqd3hmqgptupi3p.859rkn.top", "4kqd3hmqgptupi3p.8kcfnk.bid", "4kqd3hmqgptupi3p.91006j.bid", "4kqd3hmqgptupi3p.9ildst.top", "4kqd3hmqgptupi3p.a0g0o7.bid", "4kqd3hmqgptupi3p.adevf4.top", "4kqd3hmqgptupi3p.anypicked.red", "4kqd3hmqgptupi3p.as5su5.top", "4kqd3hmqgptupi3p.asfall.in", "4kqd3hmqgptupi3p.athere.in", "4kqd3hmqgptupi3p.b7mciu.top", "4kqd3hmqgptupi3p.barberryshin.casa", "4kqd3hmqgptupi3p.bestergo.pw", "4kqd3hmqgptupi3p.bigfooters.loan", "4kqd3hmqgptupi3p.bnctf6.top", "4kqd3hmqgptupi3p.bookjumps.us", "4kqd3hmqgptupi3p.boxsame.kim", "4kqd3hmqgptupi3p.boxtimed.gdn", "4kqd3hmqgptupi3p.breakown.loan", "4kqd3hmqgptupi3p.byeraser.lol", "4kqd3hmqgptupi3p.carrygain.kim", "4kqd3hmqgptupi3p.cfu46r.bid", "4kqd3hmqgptupi3p.chargecar.vip", "4kqd3hmqgptupi3p.choiceher.win", "4kqd3hmqgptupi3p.clockhate.loan", "4kqd3hmqgptupi3p.cm5ohx.bid", "4kqd3hmqgptupi3p.csv7o6.bid", "4kqd3hmqgptupi3p.cutslifes.bid", "4kqd3hmqgptupi3p.dd4xo3.top", "4kqd3hmqgptupi3p.dkrie7.top", "4kqd3hmqgptupi3p.dmvute.top", "4kqd3hmqgptupi3p.dozensby.loan", "4kqd3hmqgptupi3p.easyits.black", "4kqd3hmqgptupi3p.effortany.win", "4kqd3hmqgptupi3p.endsdoubt.loan", "4kqd3hmqgptupi3p.eventeach.gdn", "4kqd3hmqgptupi3p.ezm0r5.top", "4kqd3hmqgptupi3p.f0jlbj.bid", "4kqd3hmqgptupi3p.fairlies.link", "4kqd3hmqgptupi3p.foodtopic.mobi", "4kqd3hmqgptupi3p.g7kcux.bid", "4kqd3hmqgptupi3p.gameswarm.loan", "4kqd3hmqgptupi3p.gapplayed.link", "4kqd3hmqgptupi3p.getsbug.kim", "4kqd3hmqgptupi3p.gg4dgp.bid", "4kqd3hmqgptupi3p.gio6f6.bid", "4kqd3hmqgptupi3p.gletterstan.trade", "4kqd3hmqgptupi3p.goodslet.win", "4kqd3hmqgptupi3p.goshare.red", "4kqd3hmqgptupi3p.gs2ka7.top", "4kqd3hmqgptupi3p.he81tz.bid", "4kqd3hmqgptupi3p.heardbids.date", "4kqd3hmqgptupi3p.heldbegun.kim", "4kqd3hmqgptupi3p.hessale.pw", "4kqd3hmqgptupi3p.holescase.pw", "4kqd3hmqgptupi3p.homehuge.top", "4kqd3hmqgptupi3p.hotcopies.bid", "4kqd3hmqgptupi3p.inforcing.pw", "4kqd3hmqgptupi3p.insystem.men", "4kqd3hmqgptupi3p.itdrink.club", "4kqd3hmqgptupi3p.ix1upt.bid", "4kqd3hmqgptupi3p.jal9lk.bid", "4kqd3hmqgptupi3p.k7oud1.top", "4kqd3hmqgptupi3p.kml2o2.top", "4kqd3hmqgptupi3p.l6k4x7.bid", "4kqd3hmqgptupi3p.laterugly.win", "4kqd3hmqgptupi3p.liescale.in", "4kqd3hmqgptupi3p.liesshall.bid", "4kqd3hmqgptupi3p.lobulz.bid", "4kqd3hmqgptupi3p.lorrydo.lol", "4kqd3hmqgptupi3p.masterany.red", "4kqd3hmqgptupi3p.meetbinds.pw", "4kqd3hmqgptupi3p.metmet.win", "4kqd3hmqgptupi3p.metpast.site", "4kqd3hmqgptupi3p.mi3596.bid", "4kqd3hmqgptupi3p.mtxtul.top", "4kqd3hmqgptupi3p.mustspace.us", "4kqd3hmqgptupi3p.myaddress.link", "4kqd3hmqgptupi3p.namefalls.pro", "4kqd3hmqgptupi3p.nameuser.site", "4kqd3hmqgptupi3p.nearlybut.us", "4kqd3hmqgptupi3p.needmight.win", "4kqd3hmqgptupi3p.newrange.link", "4kqd3hmqgptupi3p.nextask.loan", "4kqd3hmqgptupi3p.nh47ri.bid", "4kqd3hmqgptupi3p.nxmu0x.bid", "4kqd3hmqgptupi3p.o8hpwj.top", "4kqd3hmqgptupi3p.outputon.asia", "4kqd3hmqgptupi3p.ownamount.pro", "4kqd3hmqgptupi3p.p79b8l.bid", "4kqd3hmqgptupi3p.pairsraw.loan", "4kqd3hmqgptupi3p.pap44w.top", "4kqd3hmqgptupi3p.powersno.link", "4kqd3hmqgptupi3p.pushstory.bid", "4kqd3hmqgptupi3p.r21wmw.top", "4kqd3hmqgptupi3p.rsi6gn.top", "4kqd3hmqgptupi3p.salethe.gdn", "4kqd3hmqgptupi3p.sayssales.bid", "4kqd3hmqgptupi3p.scoreable.bid", "4kqd3hmqgptupi3p.seemby.loan", "4kqd3hmqgptupi3p.sel7rg.bid", "4kqd3hmqgptupi3p.selfcrash.site", "4kqd3hmqgptupi3p.sentowing.trade", "4kqd3hmqgptupi3p.sitcalls.us", "4kqd3hmqgptupi3p.sk8r54.top", "4kqd3hmqgptupi3p.somegave.info", "4kqd3hmqgptupi3p.stageend.link", "4kqd3hmqgptupi3p.stopsage.gdn", "4kqd3hmqgptupi3p.storingus.gdn", "4kqd3hmqgptupi3p.tankplain.date", "4kqd3hmqgptupi3p.termprior.men", "4kqd3hmqgptupi3p.themevery.win", "4kqd3hmqgptupi3p.thyx30.top", "4kqd3hmqgptupi3p.tieslaws.link", "4kqd3hmqgptupi3p.todaynine.loan", "4kqd3hmqgptupi3p.twz1ga.top", "4kqd3hmqgptupi3p.uwckha.top", "4kqd3hmqgptupi3p.v11z5e.top", "4kqd3hmqgptupi3p.valueshes.bid", "4kqd3hmqgptupi3p.variedtax.kim", "4kqd3hmqgptupi3p.vkm4l6.top", "4kqd3hmqgptupi3p.wallluck.date", "4kqd3hmqgptupi3p.whmykv.bid", "4kqd3hmqgptupi3p.wins4n.top", "4kqd3hmqgptupi3p.wz139z.top", "4kqd3hmqgptupi3p.xmfru5.top", "4kqd3hmqgptupi3p.y12acl.bid", "4kqd3hmqgptupi3p.y5j7e6.top", "4kqd3hmqgptupi3p.yg767p.bid", "4kqd3hmqgptupi3p.yoursdoor.lol", "4kqd3hmqgptupi3p.z8ijgn.bid", "4kqd3hmqgptupi3p.z97f9v.bid", "4rebaopfgrewe.top", "4w5wihkwyhsav2ha.dreamtest.at", "4w5wihkwyhsav2ha.fastdances.at", "4w5wihkwyhsav2ha.grandhaus.at", "4w5wihkwyhsav2ha.payfactor.at", "52uo5k3t73ypjije.01fake.bid", "52uo5k3t73ypjije.086ux2.top", "52uo5k3t73ypjije.0n5joc.top", "52uo5k3t73ypjije.0nyi6l.bid", "52uo5k3t73ypjije.0vgu64.top", "52uo5k3t73ypjije.11pmnz.top", "52uo5k3t73ypjije.1bipa9.top", "52uo5k3t73ypjije.1de02r.top", "52uo5k3t73ypjije.1f1dw3.bid", "52uo5k3t73ypjije.1g0vo2.bid", "52uo5k3t73ypjije.1pma4t.bid", "52uo5k3t73ypjije.1ufr2v.bid", "52uo5k3t73ypjije.209kai.bid", "52uo5k3t73ypjije.249isv.bid", "52uo5k3t73ypjije.26lpul.bid", "52uo5k3t73ypjije.2gbbja.top", "52uo5k3t73ypjije.2llgoy.bid", "52uo5k3t73ypjije.2y4t6f.bid", "52uo5k3t73ypjije.2ym6om.bid", "52uo5k3t73ypjije.31wkhu.top", "52uo5k3t73ypjije.33dofy.top", "52uo5k3t73ypjije.35u068.bid", "52uo5k3t73ypjije.3di24a.top", "52uo5k3t73ypjije.3gpdgx.bid", "52uo5k3t73ypjije.3lhjyx.top", "52uo5k3t73ypjije.3rr6ao.top", "52uo5k3t73ypjije.3zotov.bid", "52uo5k3t73ypjije.40wiai.top", "52uo5k3t73ypjije.43l7lm.bid", "52uo5k3t73ypjije.43wjor.top", "52uo5k3t73ypjije.495iru.top", "52uo5k3t73ypjije.4jub4e.bid", "52uo5k3t73ypjije.4k9xlx.top", "52uo5k3t73ypjije.4n592s.top", "52uo5k3t73ypjije.4nf7ij.top", "52uo5k3t73ypjije.4oyhvh.top", "52uo5k3t73ypjije.4pjetv.bid", "52uo5k3t73ypjije.4xiiup.bid", "52uo5k3t73ypjije.4yl1hr.bid", "52uo5k3t73ypjije.4ynpjd.top", "52uo5k3t73ypjije.50cs7p.bid", "52uo5k3t73ypjije.56185u.bid", "52uo5k3t73ypjije.5ctoeb.bid", "52uo5k3t73ypjije.5ittco.bid", "52uo5k3t73ypjije.5kb3dl.top", "52uo5k3t73ypjije.5o4bjf.bid", "52uo5k3t73ypjije.5tb8hy.bid", "52uo5k3t73ypjije.5vhk5r.bid", "52uo5k3t73ypjije.5zxii2.bid", "52uo5k3t73ypjije.62er3d.top", "52uo5k3t73ypjije.68xmf9.bid", "52uo5k3t73ypjije.6ec2xb.bid", "52uo5k3t73ypjije.6j7jcn.bid", "52uo5k3t73ypjije.6w3rkc.bid", "52uo5k3t73ypjije.7156et.bid", "52uo5k3t73ypjije.7asel7.top", "52uo5k3t73ypjije.7j6htz.bid", "52uo5k3t73ypjije.7jiff7.top", "52uo5k3t73ypjije.7ud98m.bid", "52uo5k3t73ypjije.7wrwp4.top", "52uo5k3t73ypjije.80yabh.bid", "52uo5k3t73ypjije.86rhzr.bid", "52uo5k3t73ypjije.8a0sf6.top", "52uo5k3t73ypjije.8cjlyt.bid", "52uo5k3t73ypjije.8hphyr.top", "52uo5k3t73ypjije.8i8dt4.top", "52uo5k3t73ypjije.8kcfnk.bid", "52uo5k3t73ypjije.8rrxd9.bid", "52uo5k3t73ypjije.8rxv74.bid", "52uo5k3t73ypjije.91006j.bid", "52uo5k3t73ypjije.94ycl8.bid", "52uo5k3t73ypjije.95ovzy.top", "52uo5k3t73ypjije.9bjnlk.bid", "52uo5k3t73ypjije.9cd81s.bid", "52uo5k3t73ypjije.9ildst.top", "52uo5k3t73ypjije.9kxz23.bid", "52uo5k3t73ypjije.9nj8ex.top", "52uo5k3t73ypjije.9sfrr0.bid", "52uo5k3t73ypjije.9tftgh.bid", "52uo5k3t73ypjije.a0g0o7.bid", "52uo5k3t73ypjije.a2uzpe.top", "52uo5k3t73ypjije.aclox4.bid", "52uo5k3t73ypjije.ahvshc.top", "52uo5k3t73ypjije.ai7hur.bid", "52uo5k3t73ypjije.ajolkg.bid", "52uo5k3t73ypjije.aryh7f.bid", "52uo5k3t73ypjije.asxjdp.top", "52uo5k3t73ypjije.b2s4ch.bid", "52uo5k3t73ypjije.b7mciu.top", "52uo5k3t73ypjije.b8ll6n.top", "52uo5k3t73ypjije.bar8sc.bid", "52uo5k3t73ypjije.bcjl1h.top", "52uo5k3t73ypjije.bipa9k.bid", "52uo5k3t73ypjije.bipnnp.bid", "52uo5k3t73ypjije.bj9eea.bid", "52uo5k3t73ypjije.bnctf6.top", "52uo5k3t73ypjije.bp9mn8.bid", "52uo5k3t73ypjije.bt7r70.top", "52uo5k3t73ypjije.c3fz3z.bid", "52uo5k3t73ypjije.c7ex9n.top", "52uo5k3t73ypjije.catfills.mobi", "52uo5k3t73ypjije.cc0r87.bid", "52uo5k3t73ypjije.cfu46r.bid", "52uo5k3t73ypjije.cjc2jn.top", "52uo5k3t73ypjije.cm5ohx.bid", "52uo5k3t73ypjije.cm898n.bid", "52uo5k3t73ypjije.cmfkru.top", "52uo5k3t73ypjije.cpvwgx.bid", "52uo5k3t73ypjije.csdbnk.bid", "52uo5k3t73ypjije.csj0k5.top", "52uo5k3t73ypjije.csv7o6.bid", "52uo5k3t73ypjije.cto5ee.bid", "52uo5k3t73ypjije.czzg7f.bid", "52uo5k3t73ypjije.daigy0.top", "52uo5k3t73ypjije.das34.com", "52uo5k3t73ypjije.dd4xo3.top", "52uo5k3t73ypjije.ddwub3.top", "52uo5k3t73ypjije.deg5xr.top", "52uo5k3t73ypjije.dkrie7.top", "52uo5k3t73ypjije.dkriur.top", "52uo5k3t73ypjije.dkro3u.top", "52uo5k3t73ypjije.dmrueo.top", "52uo5k3t73ypjije.dmvute.top", "52uo5k3t73ypjije.dsv023.bid", "52uo5k3t73ypjije.dvuybv.bid", "52uo5k3t73ypjije.e32d1o.bid", "52uo5k3t73ypjije.e6in0v.top", "52uo5k3t73ypjije.e78hjo.bid", "52uo5k3t73ypjije.e8hua8.top", "52uo5k3t73ypjije.ei9evn.top", "52uo5k3t73ypjije.en3oyw.bid", "52uo5k3t73ypjije.eoivrm.bid", "52uo5k3t73ypjije.ep493u.top", "52uo5k3t73ypjije.er05vm.bid", "52uo5k3t73ypjije.ezm0r5.top", "52uo5k3t73ypjije.f0jlbj.bid", "52uo5k3t73ypjije.f242v5.bid", "52uo5k3t73ypjije.f3z72p.bid", "52uo5k3t73ypjije.fe98iy.top", "52uo5k3t73ypjije.fi50le.bid", "52uo5k3t73ypjije.fkgrie.top", "52uo5k3t73ypjije.g0ots2.top", "52uo5k3t73ypjije.g0spln.bid", "52uo5k3t73ypjije.g5196b.bid", "52uo5k3t73ypjije.gg4dgp.bid", "52uo5k3t73ypjije.gio6f6.bid", "52uo5k3t73ypjije.givxuf.bid", "52uo5k3t73ypjije.gmnjz7.bid", "52uo5k3t73ypjije.gnee6i.top", "52uo5k3t73ypjije.gnuvaw.bid", "52uo5k3t73ypjije.goztus.bid", "52uo5k3t73ypjije.gpy3tc.top", "52uo5k3t73ypjije.gtnfgj.top", "52uo5k3t73ypjije.gu7eao.bid", "52uo5k3t73ypjije.gvoafg.bid", "52uo5k3t73ypjije.h3ss4t.bid", "52uo5k3t73ypjije.hawtzr.bid", "52uo5k3t73ypjije.hbd7m4.bid", "52uo5k3t73ypjije.hhc366.bid", "52uo5k3t73ypjije.hlu8yz.top", "52uo5k3t73ypjije.hossy3.bid", "52uo5k3t73ypjije.hv42mo.bid", "52uo5k3t73ypjije.i5cgcw.top", "52uo5k3t73ypjije.i6gn9s.bid", "52uo5k3t73ypjije.i8zh1k.bid", "52uo5k3t73ypjije.iait3w.bid", "52uo5k3t73ypjije.ibngww.top", "52uo5k3t73ypjije.ie7t8k.top", "52uo5k3t73ypjije.ih9te2.bid", "52uo5k3t73ypjije.ij0cia.bid", "52uo5k3t73ypjije.imhhwm.top", "52uo5k3t73ypjije.insystem.men", "52uo5k3t73ypjije.izyclz.bid", "52uo5k3t73ypjije.j8873f.bid", "52uo5k3t73ypjije.j92msu.top", "52uo5k3t73ypjije.jal9lk.bid", "52uo5k3t73ypjije.jg6jtw.top", "52uo5k3t73ypjije.js43vy.bid", "52uo5k3t73ypjije.k0dcd2.bid", "52uo5k3t73ypjije.k21zey.bid", "52uo5k3t73ypjije.k56185.top", "52uo5k3t73ypjije.k7oud1.top", "52uo5k3t73ypjije.k8ytej.bid", "52uo5k3t73ypjije.k9z7pm.top", "52uo5k3t73ypjije.ka0te8.top", "52uo5k3t73ypjije.kas17.com", "52uo5k3t73ypjije.kcufx4.top", "52uo5k3t73ypjije.kml2o2.top", "52uo5k3t73ypjije.kswcuk.top", "52uo5k3t73ypjije.kt70uk.bid", "52uo5k3t73ypjije.ku824r.bid", "52uo5k3t73ypjije.kwnw1b.bid", "52uo5k3t73ypjije.kyjw0g.bid", "52uo5k3t73ypjije.kzhzuc.top", "52uo5k3t73ypjije.kzo8mc.top", "52uo5k3t73ypjije.kzwor6.top", "52uo5k3t73ypjije.l6ry3h.bid", "52uo5k3t73ypjije.laugk2.top", "52uo5k3t73ypjije.lba61x.top", "52uo5k3t73ypjije.ldsl8m.bid", "52uo5k3t73ypjije.lethints.date", "52uo5k3t73ypjije.lh9ax3.bid", "52uo5k3t73ypjije.li8wfu.bid", "52uo5k3t73ypjije.lib2vi.top", "52uo5k3t73ypjije.lio2wr.bid", "52uo5k3t73ypjije.loanshown.info", "52uo5k3t73ypjije.lrraca.bid", "52uo5k3t73ypjije.lwbi59.top", "52uo5k3t73ypjije.m33d4b.bid", "52uo5k3t73ypjije.m5fgoi.top", "52uo5k3t73ypjije.m6j75a.bid", "52uo5k3t73ypjije.mbwxyg.bid", "52uo5k3t73ypjije.mfgb1h.top", "52uo5k3t73ypjije.mn1kms.bid", "52uo5k3t73ypjije.msu96b.top", "52uo5k3t73ypjije.mtxtul.top", "52uo5k3t73ypjije.myurv5.bid", "52uo5k3t73ypjije.n41n1a.top", "52uo5k3t73ypjije.n6kswi.top", "52uo5k3t73ypjije.n8niwa.bid", "52uo5k3t73ypjije.nb83bp.bid", "52uo5k3t73ypjije.neekll.bid", "52uo5k3t73ypjije.nh47ri.bid", "52uo5k3t73ypjije.nmapwy.bid", "52uo5k3t73ypjije.nxmu0x.bid", "52uo5k3t73ypjije.o08a6d.top", "52uo5k3t73ypjije.o0hwme.bid", "52uo5k3t73ypjije.o5xcnd.bid", "52uo5k3t73ypjije.o6fa2g.bid", "52uo5k3t73ypjije.o8hpwj.bid", "52uo5k3t73ypjije.o8hpwj.top", "52uo5k3t73ypjije.o9w43w.bid", "52uo5k3t73ypjije.oef1sh.bid", "52uo5k3t73ypjije.ojesoa.bid", "52uo5k3t73ypjije.ojx58b.bid", "52uo5k3t73ypjije.omrexj.top", "52uo5k3t73ypjije.ooulp2.bid", "52uo5k3t73ypjije.ovpgod.top", "52uo5k3t73ypjije.p0lxvm.bid", "52uo5k3t73ypjije.p2lsgr.top", "52uo5k3t73ypjije.p5dxeh.bid", "52uo5k3t73ypjije.pap44w.top", "52uo5k3t73ypjije.pfija1.bid", "52uo5k3t73ypjije.pop81.com", "52uo5k3t73ypjije.poplenjohs.review", "52uo5k3t73ypjije.pr2zwz.bid", "52uo5k3t73ypjije.r21wmw.top", "52uo5k3t73ypjije.r2ok0b.bid", "52uo5k3t73ypjije.r4z3o5.bid", "52uo5k3t73ypjije.rdmwha.bid", "52uo5k3t73ypjije.red4is.top", "52uo5k3t73ypjije.rexjyp.bid", "52uo5k3t73ypjije.rgdk0u.top", "52uo5k3t73ypjije.rl0bdw.top", "52uo5k3t73ypjije.rnkj09.top", "52uo5k3t73ypjije.rv50gt.bid", "52uo5k3t73ypjije.s2xb1s.bid", "52uo5k3t73ypjije.sdfztr.bid", "52uo5k3t73ypjije.self56.top", "52uo5k3t73ypjije.sg62es.top", "52uo5k3t73ypjije.skri59.top", "52uo5k3t73ypjije.snwy26.top", "52uo5k3t73ypjije.sotn58.bid", "52uo5k3t73ypjije.srmlzh.bid", "52uo5k3t73ypjije.ssh3ln.bid", "52uo5k3t73ypjije.sx90yk.bid", "52uo5k3t73ypjije.sxjdpg.bid", "52uo5k3t73ypjije.thyx30.top", "52uo5k3t73ypjije.ti4wic.top", "52uo5k3t73ypjije.to6maq.top", "52uo5k3t73ypjije.twz1ga.top", "52uo5k3t73ypjije.txszfs.top", "52uo5k3t73ypjije.tzgwdf.top", "52uo5k3t73ypjije.u2r7tm.bid", "52uo5k3t73ypjije.u36ik0.bid", "52uo5k3t73ypjije.u50s89.bid", "52uo5k3t73ypjije.ujtwhg.top", "52uo5k3t73ypjije.ul8ib9.bid", "52uo5k3t73ypjije.un8niw.top", "52uo5k3t73ypjije.uv39h5.bid", "52uo5k3t73ypjije.uw3r6a.top", "52uo5k3t73ypjije.uw7w05.bid", "52uo5k3t73ypjije.uwazu7.bid", "52uo5k3t73ypjije.uwckha.bid", "52uo5k3t73ypjije.uwckha.top", "52uo5k3t73ypjije.ux93ip.top", "52uo5k3t73ypjije.v11z5e.top", "52uo5k3t73ypjije.v9y6z8.bid", "52uo5k3t73ypjije.veupl2.top", "52uo5k3t73ypjije.vkm4l6.top", "52uo5k3t73ypjije.vkslju.bid", "52uo5k3t73ypjije.vlo18w.bid", "52uo5k3t73ypjije.vmotsf.bid", "52uo5k3t73ypjije.vor28o.bid", "52uo5k3t73ypjije.vt3dg6.bid", "52uo5k3t73ypjije.w6sj06.bid", "52uo5k3t73ypjije.w8yolm.bid", "52uo5k3t73ypjije.wg00sp.bid", "52uo5k3t73ypjije.whmykv.bid", "52uo5k3t73ypjije.whosewine.lol", "52uo5k3t73ypjije.wht5py.top", "52uo5k3t73ypjije.wins4n.win", "52uo5k3t73ypjije.wl52rt.bid", "52uo5k3t73ypjije.wrd4fo.top", "52uo5k3t73ypjije.ws1uet.top", "52uo5k3t73ypjije.wz139z.top", "52uo5k3t73ypjije.x2kl7t.top", "52uo5k3t73ypjije.x3nnbd.top", "52uo5k3t73ypjije.x7fylp.bid", "52uo5k3t73ypjije.x9a6yb.bid", "52uo5k3t73ypjije.x9kjcn.bid", "52uo5k3t73ypjije.x9le66.top", "52uo5k3t73ypjije.xab7m0.top", "52uo5k3t73ypjije.xglk6h.bid", "52uo5k3t73ypjije.xjb384.bid", "52uo5k3t73ypjije.xmfru5.top", "52uo5k3t73ypjije.xtppp8.bid", "52uo5k3t73ypjije.y12acl.bid", "52uo5k3t73ypjije.y5j7e6.top", "52uo5k3t73ypjije.ye42cp.bid", "52uo5k3t73ypjije.yg767p.bid", "52uo5k3t73ypjije.yn8krm.bid", "52uo5k3t73ypjije.yrd7v5.bid", "52uo5k3t73ypjije.yty0gm.bid", "52uo5k3t73ypjije.yv7l4b.top", "52uo5k3t73ypjije.yw4629.top", "52uo5k3t73ypjije.ywszbe.bid", "52uo5k3t73ypjije.z6a7f1.bid", "52uo5k3t73ypjije.z8ijgn.bid", "52uo5k3t73ypjije.z97f9v.bid", "52uo5k3t73ypjije.zclw5i.top", "52uo5k3t73ypjije.zcwrhe.bid", "52uo5k3t73ypjije.zd3p2g.top", "52uo5k3t73ypjije.zda7bk.top", "52uo5k3t73ypjije.zed84j.bid", "52uo5k3t73ypjije.zhvlh1.bid", "52uo5k3t73ypjije.zxtezv.bid", "52uo5k3t73ypjije.zzis8p.bid", "5rport45vcdef345adfkksawe.bematvocal.at", "6dtxgqam4crv6rr6.onion.cab", "6g4ds.froekuge.com", "74nfnjhlq45nkgws4hbdbk45wekfjhqw4talefgnv.curryfort.at", "88fga.ketteaero.com", "8b4bb47tiaolhy4uhhlfaqerg.sofarany.at", "94dbhbj3l4blaeyfgl7q45glbaer.giponfeste.at", "974gfbjhb23hbfkyfaby3byqlyuebvly5q254y.mendilobo.com", "9hrds.wolfcrap.at", "a64gfdsjhb4htbiwaysbdvukyft5q.zobodine.at", "aa12111.top", "aarnknthc.xyz", "abvtqhwodwjmi.work", "acbstypdrijslr.ru", "accemfsqovkd.pw", "acjhwpdjhlhbncf.click", "aechjic.pw", "ahsqbeospcdrngfv.info", "ahuqfrqk54v3vnzj.1vcxfn.bid", "ahuqfrqk54v3vnzj.45yu0p.bid", "ahuqfrqk54v3vnzj.4h16v3.top", "ahuqfrqk54v3vnzj.6avw2a.bid", "ahuqfrqk54v3vnzj.7y1266.top", "ahuqfrqk54v3vnzj.8kiec2.top", "ahuqfrqk54v3vnzj.9sfk22.bid", "ahuqfrqk54v3vnzj.bds4sn.top", "ahuqfrqk54v3vnzj.bz7k7l.top", "ahuqfrqk54v3vnzj.c8jxpp.top", "ahuqfrqk54v3vnzj.cb3pul.top", "ahuqfrqk54v3vnzj.dxzr2l.top", "ahuqfrqk54v3vnzj.ewg6uf.bid", "ahuqfrqk54v3vnzj.g4dc5s.bid", "ahuqfrqk54v3vnzj.h4lu4i.bid", "ahuqfrqk54v3vnzj.i81wik.bid", "ahuqfrqk54v3vnzj.kj3f52.bid", "ahuqfrqk54v3vnzj.l7g2sv.bid", "ahuqfrqk54v3vnzj.n3oyw7.bid", "ahuqfrqk54v3vnzj.roep3o.top", "ahuqfrqk54v3vnzj.sg9lxh.bid", "ahuqfrqk54v3vnzj.tjubo1.top", "ahuqfrqk54v3vnzj.u9fcji.bid", "ahuqfrqk54v3vnzj.uzeb6r.bid", "ahuqfrqk54v3vnzj.v5neyw.bid", "ahuqfrqk54v3vnzj.vgxcci.top", "ahuqfrqk54v3vnzj.x90yk1.bid", "ahuqfrqk54v3vnzj.xs2xeh.bid", "ahuqfrqk54v3vnzj.zn90h4.bid", "ampjsppmftmfdblpt.info", "anbqjdoyw6wkmpeu.oldtrees.at", "applesnoutsthings.bid", "aqmip.fr", "arddxjkwrp.xyz", "as3ws.fopyirr.com", "avsxrcoq2q5fgrw2.13inb1.top", "avsxrcoq2q5fgrw2.17vj7b.top", "avsxrcoq2q5fgrw2.199ovv.top", "avsxrcoq2q5fgrw2.1gtx3p.top", "avsxrcoq2q5fgrw2.1mwipu.top", "avsxrcoq2q5fgrw2.1nsnuh.top", "avsxrcoq2q5fgrw2.2wfe60.top", "avsxrcoq2q5fgrw2.5m2n7x.top", "avsxrcoq2q5fgrw2.5s96fr.top", "avsxrcoq2q5fgrw2.79j8fm.top", "avsxrcoq2q5fgrw2.8l4jpw.top", "avsxrcoq2q5fgrw2.9c431m.bid", "avsxrcoq2q5fgrw2.arpbxw.top", "avsxrcoq2q5fgrw2.ayjy5d.top", "avsxrcoq2q5fgrw2.dgjpgy.top", "avsxrcoq2q5fgrw2.et7izd.top", "avsxrcoq2q5fgrw2.ewg6uf.bid", "avsxrcoq2q5fgrw2.h44l3d.bid", "avsxrcoq2q5fgrw2.ihuk7s.top", "avsxrcoq2q5fgrw2.j4cser.bid", "avsxrcoq2q5fgrw2.lbxvhk.top", "avsxrcoq2q5fgrw2.lxvmhm.top", "avsxrcoq2q5fgrw2.nbz4dn.top", "avsxrcoq2q5fgrw2.p93w1x.bid", "avsxrcoq2q5fgrw2.r1sjrp.top", "avsxrcoq2q5fgrw2.rys9pj.top", "avsxrcoq2q5fgrw2.tjdup0.top", "avsxrcoq2q5fgrw2.uunmkj.top", "avsxrcoq2q5fgrw2.vestjb.top", "avsxrcoq2q5fgrw2.vofy7f.top", "avsxrcoq2q5fgrw2.w22p3v.top", "avsxrcoq2q5fgrw2.w5hilw.top", "avsxrcoq2q5fgrw2.wgx4go.top", "avsxrcoq2q5fgrw2.y1fx4w.top", "avsxrcoq2q5fgrw2.y9kxz2.bid", "avsxrcoq2q5fgrw2.yr1h37.top", "avsxrcoq2q5fgrw2.z0mkoc.top", "avsxrcoq2q5fgrw2.zi842m.bid", "avxdypmdbo.pw", "axnemuevqnstqyflb.work", "b4youfred5485jgsa3453f.italazudda.com", "barjhxoye.info", "bciuemfaapyf.biz", "bddadevlpkwrrmud.xyz", "bfd45u8ehdklrfqwlhbhjbgqw.niptana.at", "bkdjvmmkwgkvgw.su", "blxbymhjva.info", "bnjhx.eu", "bqbbsfdw.be", "bqukfjfv.org", "bwcfinnt.work", "bwpegsfa.info", "bxlrywuuobje.pw", "cdxbbpngq.pw", "cerberhhyed5frqa.305iot.top", "cerberhhyed5frqa.305iot.win", "cerberhhyed5frqa.45gf4t.win", "cerberhhyed5frqa.45kgok.win", "cerberhhyed5frqa.5kti58.win", "cerberhhyed5frqa.ad34ft.win", "cerberhhyed5frqa.adevf4.win", "cerberhhyed5frqa.alri58.win", "cerberhhyed5frqa.as13fd.win", "cerberhhyed5frqa.asxce4.win", "cerberhhyed5frqa.azlto5.win", "cerberhhyed5frqa.cmr95i.top", "cerberhhyed5frqa.cmr95i.win", "cerberhhyed5frqa.cmti5o.win", "cerberhhyed5frqa.cneo59.top", "cerberhhyed5frqa.cneo59.win", "cerberhhyed5frqa.dk59jg.win", "cerberhhyed5frqa.dkrti5.top", "cerberhhyed5frqa.er48rt.win", "cerberhhyed5frqa.fgfid6.win", "cerberhhyed5frqa.fkr84i.win", "cerberhhyed5frqa.fkri48.win", "cerberhhyed5frqa.gkfit9.top", "cerberhhyed5frqa.gkfit9.win", "cerberhhyed5frqa.kipfgs65s.com", "cerberhhyed5frqa.lfotp5.top", "cerberhhyed5frqa.li4loi.win", "cerberhhyed5frqa.lib2vi.win", "cerberhhyed5frqa.m5fgoi.win", "cerberhhyed5frqa.m5gid4.top", "cerberhhyed5frqa.m5gid4.win", "cerberhhyed5frqa.m5gips.win", "cerberhhyed5frqa.mix3hi.win", "cerberhhyed5frqa.moneu5.win", "cerberhhyed5frqa.oneswi.win", "cerberhhyed5frqa.qor499.top", "cerberhhyed5frqa.raress.win", "cerberhhyed5frqa.sdfiso.win", "cerberhhyed5frqa.sims6n.win", "cerberhhyed5frqa.ti4wic.win", "cerberhhyed5frqa.to6maq.win", "cerberhhyed5frqa.vmfu48.win", "cerberhhyed5frqa.we34re.top", "cerberhhyed5frqa.we34re.win", "cerberhhyed5frqa.werti4.win", "cerberhhyed5frqa.wet4io.win", "cerberhhyed5frqa.wewiso.win", "cerberhhyed5frqa.workju.win", "cerberhhyed5frqa.xltnet.win", "cerberhhyed5frqa.xmfhr6.win", "cerberhhyed5frqa.xmfir0.top", "cerberhhyed5frqa.xmfir0.win", "cerberhhyed5frqa.xmfjr7.top", "cerberhhyed5frqa.xmfkr8.top", "cerberhhyed5frqa.xmfu59.win", "cerberhhyed5frqa.xo59ok.win", "cerberhhyed5frqa.xtrvb4.win", "cerberhhyed5frqa.zgf48j.win", "chromebewfk.top", "citointechnologiesalefor.top", "clhyelmwnuqhigecp.pw", "corefitness.info", "cpawdrtxfjkwrkkl.pw", "cpyrltela.pw", "crosseunity.top", "cudcfybkk.pw", "cwprfpjtmjb.biz", "cxlgwofgrjfoaa.info", "d34fa.lasmeio.com", "dd7bsndhr45nfksdnkferfer.javakale.at", "de2nuvwegoo32oqv.torbook.li", "de2nuvwegoo32oqv.tordrims.li", "de2nuvwegoo32oqv.torfigth.li", "de2nuvwegoo32oqv.tormilki.li", "de2nuvwegoo32oqv.torminimals.li", "de2nuvwegoo32oqv.torspaces.li", "de2nuvwegoo32oqv.tortelevision.li", "de2nuvwegoo32oqv.tortodorf.li", "de2nuvwegoo32oqv.torworks.li", "dkoipg.pw", "dltvwp.it", "dmwajvm.fr", "dolfexalto.com", "domainstop.top", "dqtfhkgskushlum.org", "dtojlhpasjk.pw", "dvmbtgoobxcc.pw", "dwytqrgblrynsgtew.org", "dyoravdkiavfkbkx.pw", "earthspiruitr.top", "eaxpifdtwsv.biz", "ecjfdaqmmyusxntwl.work", "egerdpkvutvodmtsy.pw", "egovrxvuspxck.be", "eoalsoub.pw", "eppilxqwyqdhmpdsn.pw", "eqtrtdavtnr.pw", "euduudaehipk.pw", "exnqhgk.xyz", "eypdxikxsufj.pw", "eywlmqugxx.info", "f4dsbjhb45wfiuqeib4fkqeg.meccaledgy.at", "f5xraa2y2ybtrefz.onion.to", "fdehgchykmiqwdg.info", "ffoqr3ug7m726zou.04hyxg.top", "ffoqr3ug7m726zou.0v7hry.bid", "ffoqr3ug7m726zou.1321z6.top", "ffoqr3ug7m726zou.13inb1.top", "ffoqr3ug7m726zou.14gmtu.top", "ffoqr3ug7m726zou.17vj7b.top", "ffoqr3ug7m726zou.1967qy.top", "ffoqr3ug7m726zou.1feasu.top", "ffoqr3ug7m726zou.1gtx3p.top", "ffoqr3ug7m726zou.1mwipu.top", "ffoqr3ug7m726zou.1nsnuh.top", "ffoqr3ug7m726zou.2fu7bc.top", "ffoqr3ug7m726zou.2msuuj.top", "ffoqr3ug7m726zou.2rl0pv.top", "ffoqr3ug7m726zou.4tkb0d.top", "ffoqr3ug7m726zou.5e4u7d.bid", "ffoqr3ug7m726zou.5hmjh7.bid", "ffoqr3ug7m726zou.5m2n7x.top", "ffoqr3ug7m726zou.735giv.top", "ffoqr3ug7m726zou.8uvtsg.top", "ffoqr3ug7m726zou.9yim37.top", "ffoqr3ug7m726zou.ac7zvz.top", "ffoqr3ug7m726zou.b31wkh.bid", "ffoqr3ug7m726zou.b4abvx.top", "ffoqr3ug7m726zou.bd7tlu.top", "ffoqr3ug7m726zou.bdlvdy.top", "ffoqr3ug7m726zou.bpuhab.top", "ffoqr3ug7m726zou.bwei9h.top", "ffoqr3ug7m726zou.ca15sj.top", "ffoqr3ug7m726zou.do9wwg.top", "ffoqr3ug7m726zou.e1e7w2.top", "ffoqr3ug7m726zou.efebgv.top", "ffoqr3ug7m726zou.f5x6ws.top", "ffoqr3ug7m726zou.ffsm1a.bid", "ffoqr3ug7m726zou.gwz8gh.top", "ffoqr3ug7m726zou.hajw7w.bid", "ffoqr3ug7m726zou.hpwom3.top", "ffoqr3ug7m726zou.hy6dxo.bid", "ffoqr3ug7m726zou.hzrekn.top", "ffoqr3ug7m726zou.i4ucg2.bid", "ffoqr3ug7m726zou.iocvou.top", "ffoqr3ug7m726zou.jye7lt.top", "ffoqr3ug7m726zou.kfymbh.top", "ffoqr3ug7m726zou.l4dlll.bid", "ffoqr3ug7m726zou.l6r7i9.top", "ffoqr3ug7m726zou.lc1xfc.top", "ffoqr3ug7m726zou.le6611.bid", "ffoqr3ug7m726zou.lruwth.top", "ffoqr3ug7m726zou.m3cvi8.top", "ffoqr3ug7m726zou.momg04.top", "ffoqr3ug7m726zou.ndnmuk.top", "ffoqr3ug7m726zou.ptnbfm.top", "ffoqr3ug7m726zou.rssh3l.bid", "ffoqr3ug7m726zou.rxmbsm.top", "ffoqr3ug7m726zou.rzt69n.top", "ffoqr3ug7m726zou.rzvhne.top", "ffoqr3ug7m726zou.s611js.top", "ffoqr3ug7m726zou.sg9lxh.bid", "ffoqr3ug7m726zou.smd95z.top", "ffoqr3ug7m726zou.tsrwj3.top", "ffoqr3ug7m726zou.tx0igu.bid", "ffoqr3ug7m726zou.u9fcji.bid", "ffoqr3ug7m726zou.ud9z0v.top", "ffoqr3ug7m726zou.ukswcu.bid", "ffoqr3ug7m726zou.umvv28.top", "ffoqr3ug7m726zou.utebcd.top", "ffoqr3ug7m726zou.v0xn1i.bid", "ffoqr3ug7m726zou.vjso7r.top", "ffoqr3ug7m726zou.w22p3v.top", "ffoqr3ug7m726zou.w67y8u.bid", "ffoqr3ug7m726zou.wf912u.bid", "ffoqr3ug7m726zou.wmvsh0.top", "ffoqr3ug7m726zou.wwa4tu.top", "ffoqr3ug7m726zou.wx2n44.top", "ffoqr3ug7m726zou.x8p2m7.bid", "ffoqr3ug7m726zou.x9ap4h.top", "ffoqr3ug7m726zou.xe1ws1.top", "ffoqr3ug7m726zou.y9kxz2.bid", "ffoqr3ug7m726zou.yjo0z9.top", "ffoqr3ug7m726zou.yur4j5.top", "ffoqr3ug7m726zou.yv3uwa.bid", "ffoqr3ug7m726zou.zee0xr.top", "ffoqr3ug7m726zou.zio9yg.bid", "ffoqr3ug7m726zou.zjfbxy.top", "ffoqr3ug7m726zou.zkxb17.top", "ffoqr3ug7m726zou.zn90h4.bid", "ffoqr3ug7m726zou.zpjpsf.top", "ffoqr3ug7m726zou.zu3fzc.bid", "fhvjsmtkirihxh.xyz", "fitga.ru", "fmirgordkhig.xyz", "fnarsipfqe.pw", "fnjyygovdjyemga.xyz", "fnmi62725zfti2vy.13inb1.top", "fnmi62725zfti2vy.17vj7b.top", "fnmi62725zfti2vy.1gtx3p.top", "fnmi62725zfti2vy.o08ra6.top", "fnmi62725zfti2vy.p9wol3.top", "fnmi62725zfti2vy.vwgxhm.bid", "fooplodanx.top", "fpashgkepwtoqdjg.pw", "fqoapcjolfwwenqx.pw", "fqtdrnqmeofknd.biz", "ftoxmpdipwobp4qy.10nzk9.top", "ftoxmpdipwobp4qy.17vj7b.top", "ftoxmpdipwobp4qy.199ovv.top", "ftoxmpdipwobp4qy.1gtx3p.top", "ftoxmpdipwobp4qy.1nsnuh.top", "ftoxmpdipwobp4qy.7pnxn9.top", "ftoxmpdipwobp4qy.lxvmhm.top", "fuuasvhpsvuihlnje.pw", "fuuwnsv.pw", "fyqtguo.biz", "g4dhhg53jsdjnnkjwjrfyiouh3o4u4th.vinerteen.com", "gccxqpuuylioxoip.pw", "gfcuxnaek.ru", "gfkuwflbhsjdabnu4nfukerfqwlfwr4rw.ringbalor.com", "gfwncoyhbdvggns.pw", "gguaxufrt.pw", "gitybdjgbxd.nl", "glhxgchhfemcjgr.pw", "gnsquwmgukkpgpt.pw", "govementruystd.top", "gsebqsi.ru", "gsmdqrmqddqtuv.xyz", "gvludcvhcrjwmgq.in", "gwbak.nickymaru.com", "gwe32fdr74bhfsyujb34gfszfv.zatcurr.com", "h3ds4.maconslab.com", "h54dc.leverdaze.at", "h5nuwefkuh134ljngkasdbasfg.corolbugan.com", "hjhqmbxyinislkkt.11bwgu.top", "hjhqmbxyinislkkt.127axt.top", "hjhqmbxyinislkkt.12bsy8.top", "hjhqmbxyinislkkt.12bxp9.top", "hjhqmbxyinislkkt.12ct4c.top", "hjhqmbxyinislkkt.12gsjz.top", "hjhqmbxyinislkkt.12m58x.top", "hjhqmbxyinislkkt.12zucf.top", "hjhqmbxyinislkkt.13bcem.top", "hjhqmbxyinislkkt.13eymq.top", "hjhqmbxyinislkkt.13fmby.top", "hjhqmbxyinislkkt.13khiv.top", "hjhqmbxyinislkkt.13kn4l.top", "hjhqmbxyinislkkt.13qgdd.top", "hjhqmbxyinislkkt.13ydzv.top", "hjhqmbxyinislkkt.142djp.top", "hjhqmbxyinislkkt.14dr1s.top", "hjhqmbxyinislkkt.14klmz.top", "hjhqmbxyinislkkt.14o2wp.top", "hjhqmbxyinislkkt.14stvt.top", "hjhqmbxyinislkkt.14yppf.top", "hjhqmbxyinislkkt.15e8hv.top", "hjhqmbxyinislkkt.15mwt4.top", "hjhqmbxyinislkkt.15u3kg.top", "hjhqmbxyinislkkt.16ke1t.top", "hjhqmbxyinislkkt.16l1zt.top", "hjhqmbxyinislkkt.17kc8y.top", "hjhqmbxyinislkkt.17rm9b.top", "hjhqmbxyinislkkt.18f5bw.top", "hjhqmbxyinislkkt.18lmhb.top", "hjhqmbxyinislkkt.18nepv.top", "hjhqmbxyinislkkt.18yzmj.top", "hjhqmbxyinislkkt.18zrup.top", "hjhqmbxyinislkkt.19b6nk.top", "hjhqmbxyinislkkt.19hj4f.top", "hjhqmbxyinislkkt.19s7gy.top", "hjhqmbxyinislkkt.19xdpm.top", "hjhqmbxyinislkkt.19xvyd.top", "hjhqmbxyinislkkt.1a2xx3.top", "hjhqmbxyinislkkt.1a8u1r.top", "hjhqmbxyinislkkt.1aajb7.top", "hjhqmbxyinislkkt.1aamtz.top", "hjhqmbxyinislkkt.1accfa.top", "hjhqmbxyinislkkt.1acfka.top", "hjhqmbxyinislkkt.1adh2r.top", "hjhqmbxyinislkkt.1aq4sz.top", "hjhqmbxyinislkkt.1aqq5k.top", "hjhqmbxyinislkkt.1b8tmn.top", "hjhqmbxyinislkkt.1bas8q.top", "hjhqmbxyinislkkt.1bcnad.top", "hjhqmbxyinislkkt.1bcxcs.top", "hjhqmbxyinislkkt.1bu9xu.top", "hjhqmbxyinislkkt.1c1ajf.top", "hjhqmbxyinislkkt.1cdqfv.top", "hjhqmbxyinislkkt.1cnkik.top", "hjhqmbxyinislkkt.1csesc.top", "hjhqmbxyinislkkt.1dq6nd.top", "hjhqmbxyinislkkt.1dvqvh.top", "hjhqmbxyinislkkt.1e47tj.top", "hjhqmbxyinislkkt.1eagrj.top", "hjhqmbxyinislkkt.1eeyaj.top", "hjhqmbxyinislkkt.1efxa8.top", "hjhqmbxyinislkkt.1fgsmc.top", "hjhqmbxyinislkkt.1fnjrj.top", "hjhqmbxyinislkkt.1fttxm.top", "hjhqmbxyinislkkt.1fy93v.top", "hjhqmbxyinislkkt.1fygsg.top", "hjhqmbxyinislkkt.1fzjn3.top", "hjhqmbxyinislkkt.1fzz7a.top", "hjhqmbxyinislkkt.1gjpzp.top", "hjhqmbxyinislkkt.1gqrpq.top", "hjhqmbxyinislkkt.1gredn.top", "hjhqmbxyinislkkt.1grvue.top", "hjhqmbxyinislkkt.1gswwp.top", "hjhqmbxyinislkkt.1gu5um.top", "hjhqmbxyinislkkt.1gunao.top", "hjhqmbxyinislkkt.1gvyo8.top", "hjhqmbxyinislkkt.1gxfxt.top", "hjhqmbxyinislkkt.1gzjuc.top", "hjhqmbxyinislkkt.1hapca.top", "hjhqmbxyinislkkt.1j43kf.top", "hjhqmbxyinislkkt.1jmip6.top", "hjhqmbxyinislkkt.1jnhdc.top", "hjhqmbxyinislkkt.1jwuaa.top", "hjhqmbxyinislkkt.1k6bas.top", "hjhqmbxyinislkkt.1kge5a.top", "hjhqmbxyinislkkt.1khwro.top", "hjhqmbxyinislkkt.1kjhhf.top", "hjhqmbxyinislkkt.1kraqn.top", "hjhqmbxyinislkkt.1kw51p.top", "hjhqmbxyinislkkt.1lqrja.top", "hjhqmbxyinislkkt.1ltyev.top", "hjhqmbxyinislkkt.1mat7v.top", "hjhqmbxyinislkkt.1mee2x.top", "hjhqmbxyinislkkt.1mqvsc.top", "hjhqmbxyinislkkt.1mswjm.top", "hjhqmbxyinislkkt.1mvku2.top", "hjhqmbxyinislkkt.1mwvgh.top", "hjhqmbxyinislkkt.1nm62r.top", "hjhqmbxyinislkkt.1npg9s.top", "hjhqmbxyinislkkt.1ntyds.top", "hjhqmbxyinislkkt.1pcvko.top", "hjhqmbxyinislkkt.1ppto6.top", "hjhqmbxyinislkkt.1pxbfh.top", "hjhqmbxyinislkkt.1q7pwb.top", "hjhqmbxyinislkkt.1qjl23.top", "hjhqmbxyinislkkt.1qk2un.top", "hjhqmbxyinislkkt.1w5iy8.top", "hjhqmbxyinislkkt.1xynaz.top", "hmndhdbscgru.pw", "honourableud.top", "hppfsslyeyseudg.biz", "hrfgd74nfksjdcnnklnwefvdsf.materdunst.com", "htankds.info", "hycninyxuaa.xyz", "i01001.dgn.vn", "i3ezlvkoi7fwyood.onion.to", "i3ezlvkoi7fwyood.tor2web.org", "i5ndw.titlecorta.at", "ibjgnqsthdyp.pw", "ibtfqftkgi.pw", "ifohvkxmyp.biz", "igoodsnd.wang", "ik4dm.mazerunci.at", "iqfyujpvubwawc.pw", "irhng84nfaslbv243ljtblwqjrb.pinnafaon.at", "irudhkunrlfu25fhkaqw34blr5qlby4tgq43t.orrisbirth.com", "iuieylpvfurcvmpk.pw", "jfmiondv.xyz", "jghbktqepe.pw", "jhdgh.club", "jhomitevd2abj3fk.onion.to", "juhacjacjckclqf.pw", "jxqdry.ru", "jymhmkdaxfbl.click", "k234s.ascotsprue.com", "k34ew.keyedgell.com", "k3cxd.pileanoted.com", "k47d3.proporr.com", "k4restportgonst34d23r.oftpony.at", "kbv5s.kylepasse.at", "kcdfajaxngiff.info", "kciylimohteftc.pw", "kh5jfnvkk5twerfnku5twuilrnglnuw45yhlw.vealsithe.com", "kjkwjqvqrjocpi.xyz", "kkd47eh4hdjshb5t.angortra.at", "kkr4hbwdklf234bfl84uoqleflqwrfqwuelfh.brazabaya.com", "kpybuhnosdrm.in", "kqlxtqptsmys.in", "ks-davis.com", "ktlgpiilbj.biz", "kwontdmplpnbl.pw", "kypsuw.pw", "l123d.feustude.at", "lcrdceiajmiar.org", "lfdachijzuwx4bc4.0ndl3j.bid", "lfdachijzuwx4bc4.6szfn7.top", "lfdachijzuwx4bc4.83zw1f.bid", "lfdachijzuwx4bc4.8dlgyg.bid", "lfdachijzuwx4bc4.af38vz.top", "lfdachijzuwx4bc4.ci221p.top", "lfdachijzuwx4bc4.djintc.bid", "lfdachijzuwx4bc4.e6cf2t.bid", "lfdachijzuwx4bc4.eujvrw.bid", "lfdachijzuwx4bc4.ev99l6.bid", "lfdachijzuwx4bc4.ex9n9v.top", "lfdachijzuwx4bc4.fe6cf2.top", "lfdachijzuwx4bc4.fwzxnb.bid", "lfdachijzuwx4bc4.iuzppd.top", "lfdachijzuwx4bc4.le2brr.bid", "lfdachijzuwx4bc4.m7f27y.bid", "lfdachijzuwx4bc4.twyjdx.bid", "lfdachijzuwx4bc4.tx0igu.bid", "lfdachijzuwx4bc4.u9fcji.bid", "lfdachijzuwx4bc4.vrgdrs.top", "lfdachijzuwx4bc4.w4629d.top", "lfdachijzuwx4bc4.x4tk5c.bid", "lfdachijzuwx4bc4.zreknv.bid", "lollyoff.info", "lookingpersonals.top", "lpholfnvwbukqwye.onion.cab", "lpholfnvwbukqwye.onion.to", "lrmficvqs.pw", "ltpwqva.xyz", "luvenxj.uk", "lvanwwbyabcfevyi.pw", "lyrnvane.pw", "macooptwafkwchtpo.pw", "mmhmtea.pw", "mphtadhci5mrdlju.onion.to", "mphtadhci5mrdlju.tor2web.org", "muuojcu.xyz", "mwqwverayognn.pw", "mxyfasm.pw", "mz7oyb3v32vshcvk.bidobject.li", "mz7oyb3v32vshcvk.getstar.li", "mz7oyb3v32vshcvk.torapples.li", "mz7oyb3v32vshcvk.torlongor.li", "mz7oyb3v32vshcvk.tormidle.at", "mz7oyb3v32vshcvk.toysworlds.at", "newgiftnd.wang", "newgiftst.top", "nhhyxorxbxarxe.org", "nikessysleys.top", "nlpqflkbvkdde.eu", "nn54djhfnrnm4dnjnerfsd.replylaten.at", "nnrtsdf34dsjhb23rsdf.spannflow.com", "nwcpgymgh.work", "o4dm3.leaama.at", "odgtnkmq.pw", "oehknf74ohqlfnpq9rhfgcq93g.hateflux.com", "ohpbdikmrrhr.pw", "ohplsuljopekq.biz", "ojmekzw4mujvqeju.bioserv.at", "ojmekzw4mujvqeju.dreamtest.at", "ojmekzw4mujvqeju.fineboy.at", "ojmekzw4mujvqeju.minitili.at", "omeaswslhgdw.xyz", "oqwygprskqv65j72.12kb9j.top", "oqwygprskqv65j72.13gpqd.top", "oqwygprskqv65j72.13rdvu.top", "oqwygprskqv65j72.14jqyo.top", "oqwygprskqv65j72.17q8f6.top", "oqwygprskqv65j72.1aj1bb.top", "oqwygprskqv65j72.1d88b8.top", "oqwygprskqv65j72.1dofqx.top", "oqwygprskqv65j72.1fdlhn.top", "oqwygprskqv65j72.1fs9pz.top", "oqwygprskqv65j72.1gam57.top", "oqwygprskqv65j72.1gqj8x.top", "oqwygprskqv65j72.1hbdbx.top", "oqwygprskqv65j72.1j1x2b.top", "oqwygprskqv65j72.1jquw7.top", "oqwygprskqv65j72.1kh9ct.top", "oqwygprskqv65j72.1mudaw.top", "oqwygprskqv65j72.1nzpby.top", "ozfin.ru", "p27dokhpz2n7nvgr.12a63k.top", "p27dokhpz2n7nvgr.12c8ff.top", "p27dokhpz2n7nvgr.12gzrv.top", "p27dokhpz2n7nvgr.12hxjv.top", "p27dokhpz2n7nvgr.12nwsv.top", "p27dokhpz2n7nvgr.12smak.top", "p27dokhpz2n7nvgr.12t3rn.top", "p27dokhpz2n7nvgr.12ulcz.top", "p27dokhpz2n7nvgr.12umzf.top", "p27dokhpz2n7nvgr.12uzfa.top", "p27dokhpz2n7nvgr.12vpkc.top", "p27dokhpz2n7nvgr.1321z6.top", "p27dokhpz2n7nvgr.133chr.top", "p27dokhpz2n7nvgr.135nt3.top", "p27dokhpz2n7nvgr.13g2v9.top", "p27dokhpz2n7nvgr.13gmvm.top", "p27dokhpz2n7nvgr.13ixv2.top", "p27dokhpz2n7nvgr.13upky.top", "p27dokhpz2n7nvgr.13upnc.top", "p27dokhpz2n7nvgr.13wm9b.top", "p27dokhpz2n7nvgr.13xwn9.top", "p27dokhpz2n7nvgr.14ewqv.top", "p27dokhpz2n7nvgr.14gmtu.top", "p27dokhpz2n7nvgr.14kfoz.top", "p27dokhpz2n7nvgr.14udep.top", "p27dokhpz2n7nvgr.15jznv.top", "p27dokhpz2n7nvgr.15l2ub.top", "p27dokhpz2n7nvgr.15nhsf.top", "p27dokhpz2n7nvgr.15oqwp.top", "p27dokhpz2n7nvgr.15rnwa.top", "p27dokhpz2n7nvgr.15wmdx.top", "p27dokhpz2n7nvgr.168w5y.top", "p27dokhpz2n7nvgr.16ay2s.top", "p27dokhpz2n7nvgr.16bwhs.top", "p27dokhpz2n7nvgr.16fohp.top", "p27dokhpz2n7nvgr.16nxpn.top", "p27dokhpz2n7nvgr.16qpet.top", "p27dokhpz2n7nvgr.173w9w.top", "p27dokhpz2n7nvgr.17g6gc.top", "p27dokhpz2n7nvgr.17gvad.top", "p27dokhpz2n7nvgr.17m14u.top", "p27dokhpz2n7nvgr.17ryrs.top", "p27dokhpz2n7nvgr.17u2yg.top", "p27dokhpz2n7nvgr.18dawg.top", "p27dokhpz2n7nvgr.18kkhl.top", "p27dokhpz2n7nvgr.18kmtt.top", "p27dokhpz2n7nvgr.195heb.top", "p27dokhpz2n7nvgr.1967qy.top", "p27dokhpz2n7nvgr.1a7ivn.top", "p27dokhpz2n7nvgr.1a7wnt.top", "p27dokhpz2n7nvgr.1aghep.top", "p27dokhpz2n7nvgr.1ajohk.top", "p27dokhpz2n7nvgr.1apgrn.top", "p27dokhpz2n7nvgr.1apkjn.top", "p27dokhpz2n7nvgr.1aweql.top", "p27dokhpz2n7nvgr.1axzcw.top", "p27dokhpz2n7nvgr.1azkux.top", "p27dokhpz2n7nvgr.1b3qjy.top", "p27dokhpz2n7nvgr.1bj4k9.top", "p27dokhpz2n7nvgr.1bniyw.top", "p27dokhpz2n7nvgr.1bvadx.top", "p27dokhpz2n7nvgr.1bywu2.top", "p27dokhpz2n7nvgr.1bzolk.top", "p27dokhpz2n7nvgr.1cauz3.top", "p27dokhpz2n7nvgr.1cb19l.top", "p27dokhpz2n7nvgr.1cbcpy.top", "p27dokhpz2n7nvgr.1cewld.top", "p27dokhpz2n7nvgr.1cggqc.top", "p27dokhpz2n7nvgr.1cglxz.top", "p27dokhpz2n7nvgr.1chy1m.top", "p27dokhpz2n7nvgr.1cknbd.top", "p27dokhpz2n7nvgr.1cpb4z.top", "p27dokhpz2n7nvgr.1cpy1q.top", "p27dokhpz2n7nvgr.1cq7gd.top", "p27dokhpz2n7nvgr.1cvmb4.top", "p27dokhpz2n7nvgr.1cw65b.top", "p27dokhpz2n7nvgr.1czh7o.top", "p27dokhpz2n7nvgr.1d8d9w.top", "p27dokhpz2n7nvgr.1d8m97.top", "p27dokhpz2n7nvgr.1daq6h.top", "p27dokhpz2n7nvgr.1dlcbk.top", "p27dokhpz2n7nvgr.1dp6un.top", "p27dokhpz2n7nvgr.1dsdm4.top", "p27dokhpz2n7nvgr.1dyzdh.top", "p27dokhpz2n7nvgr.1dz7gk.top", "p27dokhpz2n7nvgr.1ebvqb.top", "p27dokhpz2n7nvgr.1eeb86.top", "p27dokhpz2n7nvgr.1em2j4.top", "p27dokhpz2n7nvgr.1enbyr.top", "p27dokhpz2n7nvgr.1evjph.top", "p27dokhpz2n7nvgr.1fel3k.top", "p27dokhpz2n7nvgr.1fgywm.top", "p27dokhpz2n7nvgr.1fqwek.top", "p27dokhpz2n7nvgr.1fu8p3.top", "p27dokhpz2n7nvgr.1gnlsi.top", "p27dokhpz2n7nvgr.1gqqsc.top", "p27dokhpz2n7nvgr.1gvql3.top", "p27dokhpz2n7nvgr.1gy9bo.top", "p27dokhpz2n7nvgr.1h23cc.top", "p27dokhpz2n7nvgr.1hkjl3.top", "p27dokhpz2n7nvgr.1hpvzl.top", "p27dokhpz2n7nvgr.1hw36d.top", "p27dokhpz2n7nvgr.1jemdr.top", "p27dokhpz2n7nvgr.1jh5kv.top", "p27dokhpz2n7nvgr.1jhnvt.top", "p27dokhpz2n7nvgr.1jpb8w.top", "p27dokhpz2n7nvgr.1js3tl.top", "p27dokhpz2n7nvgr.1jw2lx.top", "p27dokhpz2n7nvgr.1jyhqc.top", "p27dokhpz2n7nvgr.1jzmjr.top", "p27dokhpz2n7nvgr.1kja1j.top", "p27dokhpz2n7nvgr.1kq4l8.top", "p27dokhpz2n7nvgr.1ktjse.top", "p27dokhpz2n7nvgr.1kyjw7.top", "p27dokhpz2n7nvgr.1l4zyd.top", "p27dokhpz2n7nvgr.1lcteo.top", "p27dokhpz2n7nvgr.1lfyy4.top", "p27dokhpz2n7nvgr.1lt2pn.top", "p27dokhpz2n7nvgr.1m3xsy.top", "p27dokhpz2n7nvgr.1mfakx.top", "p27dokhpz2n7nvgr.1mfdt8.top", "p27dokhpz2n7nvgr.1mir1h.top", "p27dokhpz2n7nvgr.1ms2rx.top", "p27dokhpz2n7nvgr.1mwipu.top", "p27dokhpz2n7nvgr.1nhkou.top", "p27dokhpz2n7nvgr.1nmrtq.top", "p27dokhpz2n7nvgr.1nprob.top", "p27dokhpz2n7nvgr.1p5fwl.top", "p27dokhpz2n7nvgr.1pbfky.top", "p27dokhpz2n7nvgr.1pbu64.top", "p27dokhpz2n7nvgr.1pglcs.top", "p27dokhpz2n7nvgr.1plugt.top", "p27dokhpz2n7nvgr.1psts4.top", "p27dokhpz2n7nvgr.1pymg3.top", "p27dokhpz2n7nvgr.1vjnyh.top", "p27dokhpz2n7nvgr.1wmvk2.top", "p54dhkus4tlkfashdb6vjetgsdfg.greetingshere.at", "pagaldaily.com", "pdlbtnfhtoxghb.org", "pe2cku7pebkpgeko.13inb1.top", "pe2cku7pebkpgeko.199ovv.top", "pe2cku7pebkpgeko.1cb19l.top", "pe2cku7pebkpgeko.1gtx3p.top", "pe2cku7pebkpgeko.1mwipu.top", "pe2cku7pebkpgeko.1plugt.top", "pe2cku7pebkpgeko.1pr21c.top", "pe2cku7pebkpgeko.582h0n.top", "pe2cku7pebkpgeko.5hmjh7.bid", "pe2cku7pebkpgeko.ahovbr.top", "pe2cku7pebkpgeko.bw9e2z.top", "pe2cku7pebkpgeko.dj68hn.top", "pe2cku7pebkpgeko.hclz73.top", "pe2cku7pebkpgeko.kwrd4f.bid", "pe2cku7pebkpgeko.p93w1x.bid", "pe2cku7pebkpgeko.pkx86a.top", "pe2cku7pebkpgeko.prbuoi.top", "pe2cku7pebkpgeko.r1sjrp.top", "pe2cku7pebkpgeko.reu88i.top", "pe2cku7pebkpgeko.rjf9yn.top", "pe2cku7pebkpgeko.tsrwj3.top", "pe2cku7pebkpgeko.ttx0ig.top", "pe2cku7pebkpgeko.utebcd.top", "pe2cku7pebkpgeko.va3ibn.top", "pe2cku7pebkpgeko.vfe2f1.top", "pe2cku7pebkpgeko.yjo0z9.top", "pe2cku7pebkpgeko.z5xfkc.top", "pennysgoods.top", "plfbvdrpvsm.pw", "pmenboeqhyrpvomq.0nyi6l.bid", "pmenboeqhyrpvomq.0vgu64.top", "pmenboeqhyrpvomq.2agglf.top", "pmenboeqhyrpvomq.4pzclh.top", "pmenboeqhyrpvomq.58na23.top", "pmenboeqhyrpvomq.5b1s82.top", "pmenboeqhyrpvomq.7s0g3v.top", "pmenboeqhyrpvomq.89m6y8.bid", "pmenboeqhyrpvomq.8kcfnk.bid", "pmenboeqhyrpvomq.9ildst.top", "pmenboeqhyrpvomq.9nkxd3.top", "pmenboeqhyrpvomq.a4coac.top", "pmenboeqhyrpvomq.afteghonte.lol", "pmenboeqhyrpvomq.as5su5.top", "pmenboeqhyrpvomq.asxjdp.top", "pmenboeqhyrpvomq.azwsxe.top", "pmenboeqhyrpvomq.b7mciu.top", "pmenboeqhyrpvomq.bnctf6.top", "pmenboeqhyrpvomq.cmri58.top", "pmenboeqhyrpvomq.e6in0v.top", "pmenboeqhyrpvomq.enanhb.bid", "pmenboeqhyrpvomq.factordo.site", "pmenboeqhyrpvomq.fm0cga.top", "pmenboeqhyrpvomq.g0ots2.top", "pmenboeqhyrpvomq.gletterstan.trade", "pmenboeqhyrpvomq.gnuvaw.bid", "pmenboeqhyrpvomq.hasterlyston.cloud", "pmenboeqhyrpvomq.hwh75t.top", "pmenboeqhyrpvomq.ibngww.top", "pmenboeqhyrpvomq.k7oud1.top", "pmenboeqhyrpvomq.ka0te8.top", "pmenboeqhyrpvomq.kswcuk.top", "pmenboeqhyrpvomq.li4loi.top", "pmenboeqhyrpvomq.loopsay.link", "pmenboeqhyrpvomq.m54tkp.bid", "pmenboeqhyrpvomq.mtxtul.top", "pmenboeqhyrpvomq.n41n1a.top", "pmenboeqhyrpvomq.n80yab.top", "pmenboeqhyrpvomq.nh47ri.bid", "pmenboeqhyrpvomq.o08a6d.top", "pmenboeqhyrpvomq.o8hpwj.top", "pmenboeqhyrpvomq.p8rruv.top", "pmenboeqhyrpvomq.pap44w.top", "pmenboeqhyrpvomq.paypoints.red", "pmenboeqhyrpvomq.r21wmw.top", "pmenboeqhyrpvomq.rnkj09.top", "pmenboeqhyrpvomq.s71vsc.top", "pmenboeqhyrpvomq.self56.top", "pmenboeqhyrpvomq.shutlazy.casa", "pmenboeqhyrpvomq.swissprogramms.bid", "pmenboeqhyrpvomq.t4hvl4.bid", "pmenboeqhyrpvomq.thyx30.top", "pmenboeqhyrpvomq.txszfs.top", "pmenboeqhyrpvomq.v11z5e.top", "pmenboeqhyrpvomq.viceled.pw", "pmenboeqhyrpvomq.vkm4l6.top", "pmenboeqhyrpvomq.wn4h1k.top", "pmenboeqhyrpvomq.wrd4fo.top", "pmenboeqhyrpvomq.x1kofw.top", "pmenboeqhyrpvomq.xneyvm.top", "pmenboeqhyrpvomq.xx6jck.top", "pmenboeqhyrpvomq.y5j7e6.top", "pmenboeqhyrpvomq.y7fjr4.bid", "pmenboeqhyrpvomq.yw4629.top", "pnyviolg.eu", "po4dbsjbneljhrlbvaueqrgveatv.bonmawp.at", "poimoiyreque5.pw", "polaerunity.top", "ponmaredimare.top", "pornohd24.com", "preeqlultgfifg.pw", "prest54538hnksjn4kjfwdbhwere.hotchunman.com", "pts764gt354fder34fsqw45gdfsavadfgsfg.kraskula.com", "pvwinlrmwvccuo.eu", "qbqrfyeqqvcvv.pw", "qcwbrevxrotoepsp.pw", "qdesslfdcmd.pw", "qdvkdyvrtpjc.pw", "qfjhpgbefuhenjp7.1225wj.top", "qfjhpgbefuhenjp7.12efwa.top", "qfjhpgbefuhenjp7.12f53x.top", "qfjhpgbefuhenjp7.12u5fl.top", "qfjhpgbefuhenjp7.13iuvw.top", "qfjhpgbefuhenjp7.143kzi.top", "qfjhpgbefuhenjp7.158ugp.top", "qfjhpgbefuhenjp7.16g9ub.top", "qfjhpgbefuhenjp7.17cwdi.top", "qfjhpgbefuhenjp7.17ipn9.top", "qfjhpgbefuhenjp7.17xukb.top", "qfjhpgbefuhenjp7.18dwag.top", "qfjhpgbefuhenjp7.18ggbf.top", "qfjhpgbefuhenjp7.18rkju.top", "qfjhpgbefuhenjp7.19ckzf.top", "qfjhpgbefuhenjp7.1a2jzy.top", "qfjhpgbefuhenjp7.1cosak.top", "qfjhpgbefuhenjp7.1e1jbc.top", "qfjhpgbefuhenjp7.1e1y8p.top", "qfjhpgbefuhenjp7.1fcfjn.top", "qfjhpgbefuhenjp7.1jfjhb.top", "qfjhpgbefuhenjp7.1jrkyn.top", "qfjhpgbefuhenjp7.1mkwry.top", "qfjhpgbefuhenjp7.1mnsg6.top", "qfuxosx.eu", "qlwnvdjwro.pw", "qqonof.info", "qqtphtlhny.pw", "qsbfwgtedexirbyoq.pw", "qvdgqayo.pw", "rastypasty34.top", "rbg4hfbilrf7to452p89hrfq.boonmower.com", "rbwubtpsyokqn.info", "real346real.top", "remoteunityrety.top", "renaulrtcenturytrick.top", "rkiywansamtu.top", "rolerxunitywsto.top", "rootaleyz.top", "rowerpovertort.top", "rqfsctpgpuani.pw", "rrcspgfghsjnklts.pw", "rzss2zfue73dfvmj.onlinerpgame.ch", "rzss2zfue73dfvmj.truewargame.ch", "sdwempsovemtr.yt", "seelkqtkkqxvq.click", "semiconductry.top", "sgowntfjwkybawi.pw", "sgrnhwyqxdk.pw", "sondr5344ygfweyjbfkw4fhsefv.heliofetch.at", "sonicfopase.top", "sqrgvbgfyya.org", "sqsigig.pw", "ssvylrn.pw", "stevnxwq.pw", "stgg5jv6mqiibmax.toradmin.li", "stgg5jv6mqiibmax.toranimals.li", "stgg5jv6mqiibmax.torbrouke.li", "stgg5jv6mqiibmax.torclasses.li", "stgg5jv6mqiibmax.torclever.li", "stgg5jv6mqiibmax.torcreator.li", "stgg5jv6mqiibmax.torking.li", "stgg5jv6mqiibmax.torpice.li", "stgg5jv6mqiibmax.torpoint.ch", "stgg5jv6mqiibmax.torshop.li", "sumnitdomains.top", "svkjhguk.ru", "svvgyjweurxn.click", "swfqg.in", "sxflmtgxerkpgwlnp.pw", "t54ndnku456ngkwsudqer.wallymac.com", "tdhyjfxltpj.pw", "tes543berda73i48fsdfsd.keratadze.at", "topgearspoilytyrdc.top", "toxnwbkoulii.pw", "toytyaclucomunit.top", "tqlcjh.fr", "tregretryfaltervipo.top", "trxswbwxhr.xyz", "tswsgajtwhqkosd.su", "tt54rfdjhb34rfbnknaerg.milerteddy.com", "ttoyqvq.pw", "tuouyunittyewr.top", "twbers4hmi6dc65f.onion.cab", "twbers4hmi6dc65f.onion.to", "twbers4hmi6dc65f.tor2web.org", "u24er.ovaarmor.com", "u54bbnhf354fbkh254tbkhjbgy8258gnkwerg.tahaplap.com", "ubisortdasert.top", "uetwvrlnee.fr", "uhgmnigjpf.biz", "uhhvhjqowpgopq.xyz", "uhjxayhpisr.pw", "uhufnlsad7bhf4ykqfbevmxergwrth.himfinn.com", "uiredn4njfsa4234bafb32ygjdawfvs.frascuft.com", "uj5nj.onanwhit.com", "umjjvccteg.biz", "unintyregullyar.top", "unittogreas.top", "unityharerteraz.top", "unityrulesyur.top", "unixbroungs.top", "unocl45trpuoefft.054t69.bid", "unocl45trpuoefft.06j7o0.top", "unocl45trpuoefft.086ux2.top", "unocl45trpuoefft.0evktl.top", "unocl45trpuoefft.0kousz.bid", "unocl45trpuoefft.0kv6tw.bid", "unocl45trpuoefft.0vgu64.top", "unocl45trpuoefft.18xhww.bid", "unocl45trpuoefft.1cn41a.bid", "unocl45trpuoefft.1de02r.top", "unocl45trpuoefft.1v3bnu.top", "unocl45trpuoefft.249isv.bid", "unocl45trpuoefft.2y4t6f.bid", "unocl45trpuoefft.308an1.top", "unocl45trpuoefft.31wkhu.top", "unocl45trpuoefft.36u6mp.bid", "unocl45trpuoefft.3n9lut.bid", "unocl45trpuoefft.42wunw.bid", "unocl45trpuoefft.4bb9vz.bid", "unocl45trpuoefft.4k98id.top", "unocl45trpuoefft.54drms.bid", "unocl45trpuoefft.54m2k3.bid", "unocl45trpuoefft.5o3euy.bid", "unocl45trpuoefft.5v3uvc.bid", "unocl45trpuoefft.60c61d.bid", "unocl45trpuoefft.6w3rkc.bid", "unocl45trpuoefft.75tdcj.bid", "unocl45trpuoefft.78of7m.bid", "unocl45trpuoefft.791sd5.bid", "unocl45trpuoefft.7cevps.bid", "unocl45trpuoefft.7eup7k.bid", "unocl45trpuoefft.7tooul.bid", "unocl45trpuoefft.88wz5p.bid", "unocl45trpuoefft.8kcfnk.bid", "unocl45trpuoefft.8uwckh.top", "unocl45trpuoefft.9bjnlk.bid", "unocl45trpuoefft.9lnito.top", "unocl45trpuoefft.9lx4s6.bid", "unocl45trpuoefft.9u3iy1.top", "unocl45trpuoefft.a3migu.bid", "unocl45trpuoefft.a4v4c3.bid", "unocl45trpuoefft.ageshere.club", "unocl45trpuoefft.ahhc36.top", "unocl45trpuoefft.at593l.bid", "unocl45trpuoefft.at9gwv.bid", "unocl45trpuoefft.awspm2.top", "unocl45trpuoefft.barzc4.bid", "unocl45trpuoefft.bjahwh.bid", "unocl45trpuoefft.c3fz3z.bid", "unocl45trpuoefft.c4issd.bid", "unocl45trpuoefft.c9kp0o.bid", "unocl45trpuoefft.ceikto.bid", "unocl45trpuoefft.cgf59i.top", "unocl45trpuoefft.cifbp9.bid", "unocl45trpuoefft.ckw9fm.top", "unocl45trpuoefft.cm5ohx.bid", "unocl45trpuoefft.csdbnk.bid", "unocl45trpuoefft.csv7o6.bid", "unocl45trpuoefft.cypz3w.top", "unocl45trpuoefft.czzg7f.bid", "unocl45trpuoefft.dwkofh.top", "unocl45trpuoefft.dyo7c9.top", "unocl45trpuoefft.efebgv.bid", "unocl45trpuoefft.eloppu.bid", "unocl45trpuoefft.emogew.bid", "unocl45trpuoefft.eo6rzt.bid", "unocl45trpuoefft.ev6i0x.bid", "unocl45trpuoefft.eyohd2.top", "unocl45trpuoefft.f17bam.bid", "unocl45trpuoefft.freshsdog.loan", "unocl45trpuoefft.frn62e.top", "unocl45trpuoefft.gg4dgp.bid", "unocl45trpuoefft.gio6f6.bid", "unocl45trpuoefft.givxuf.bid", "unocl45trpuoefft.hawtzr.bid", "unocl45trpuoefft.he81tz.bid", "unocl45trpuoefft.hur45z.bid", "unocl45trpuoefft.hvh2gb.bid", "unocl45trpuoefft.hxrd02.bid", "unocl45trpuoefft.hynwbs.top", "unocl45trpuoefft.hyr1h3.bid", "unocl45trpuoefft.i1wcrl.bid", "unocl45trpuoefft.i561zy.bid", "unocl45trpuoefft.ibngww.top", "unocl45trpuoefft.idw6s5.bid", "unocl45trpuoefft.igpfcu.bid", "unocl45trpuoefft.igrj6t.bid", "unocl45trpuoefft.ih301a.bid", "unocl45trpuoefft.ii2yoh.bid", "unocl45trpuoefft.ilm071.bid", "unocl45trpuoefft.j0cia7.bid", "unocl45trpuoefft.j404oy.bid", "unocl45trpuoefft.j8exy2.bid", "unocl45trpuoefft.jcife9.bid", "unocl45trpuoefft.jdf4je.bid", "unocl45trpuoefft.jjogbj.top", "unocl45trpuoefft.jnd0bj.bid", "unocl45trpuoefft.jsotn5.top", "unocl45trpuoefft.jvrh8g.bid", "unocl45trpuoefft.k56185.top", "unocl45trpuoefft.kf1gxm.bid", "unocl45trpuoefft.kg5bof.bid", "unocl45trpuoefft.kml2o2.top", "unocl45trpuoefft.knowhands.us", "unocl45trpuoefft.ks3ghp.bid", "unocl45trpuoefft.kswcuk.top", "unocl45trpuoefft.l05l27.top", "unocl45trpuoefft.l69xgc.bid", "unocl45trpuoefft.l97i5a.bid", "unocl45trpuoefft.lak8wd.bid", "unocl45trpuoefft.larebg.bid", "unocl45trpuoefft.lcyznu.bid", "unocl45trpuoefft.lio2wr.bid", "unocl45trpuoefft.lk0bzc.top", "unocl45trpuoefft.ll3zot.bid", "unocl45trpuoefft.lzskva.bid", "unocl45trpuoefft.m03t72.bid", "unocl45trpuoefft.m33d4b.bid", "unocl45trpuoefft.m9a225.top", "unocl45trpuoefft.mbwxyg.bid", "unocl45trpuoefft.md9eyv.bid", "unocl45trpuoefft.meetsface.win", "unocl45trpuoefft.metpast.date", "unocl45trpuoefft.mezy7j.bid", "unocl45trpuoefft.moonsides.faith", "unocl45trpuoefft.n20b1c.top", "unocl45trpuoefft.n41n1a.top", "unocl45trpuoefft.n94lrn.bid", "unocl45trpuoefft.na2iuz.bid", "unocl45trpuoefft.nmit4p.bid", "unocl45trpuoefft.noyl9o.bid", "unocl45trpuoefft.nz6emv.bid", "unocl45trpuoefft.o2dval.top", "unocl45trpuoefft.o8hpwj.top", "unocl45trpuoefft.og5ezh.top", "unocl45trpuoefft.on2420.bid", "unocl45trpuoefft.ozlrnx.bid", "unocl45trpuoefft.p1gneb.bid", "unocl45trpuoefft.p2ix1u.bid", "unocl45trpuoefft.p4sr76.top", "unocl45trpuoefft.pap44w.top", "unocl45trpuoefft.pbprju.bid", "unocl45trpuoefft.piy4l3.bid", "unocl45trpuoefft.ptneek.bid", "unocl45trpuoefft.r21wmw.top", "unocl45trpuoefft.r2vai7.bid", "unocl45trpuoefft.rgbb50.bid", "unocl45trpuoefft.rie9py.bid", "unocl45trpuoefft.rslh9a.top", "unocl45trpuoefft.s7b63k.bid", "unocl45trpuoefft.sirchi.bid", "unocl45trpuoefft.sp4o1t.bid", "unocl45trpuoefft.tcly4s.bid", "unocl45trpuoefft.tfmmby.bid", "unocl45trpuoefft.thanreal.link", "unocl45trpuoefft.ttabop.bid", "unocl45trpuoefft.u64rj2.top", "unocl45trpuoefft.uaol08.bid", "unocl45trpuoefft.ukwnvw.bid", "unocl45trpuoefft.um1x6z.bid", "unocl45trpuoefft.uog1ky.bid", "unocl45trpuoefft.uso3z0.bid", "unocl45trpuoefft.uw3r6a.top", "unocl45trpuoefft.uwckha.top", "unocl45trpuoefft.v4kx51.bid", "unocl45trpuoefft.v50gtu.bid", "unocl45trpuoefft.vfuvsv.bid", "unocl45trpuoefft.vi5iko.bid", "unocl45trpuoefft.vkm4l6.top", "unocl45trpuoefft.vkslju.bid", "unocl45trpuoefft.vlwbcz.bid", "unocl45trpuoefft.vmomcc.bid", "unocl45trpuoefft.whmykv.bid", "unocl45trpuoefft.wl8t6k.bid", "unocl45trpuoefft.wlvxd6.bid", "unocl45trpuoefft.wz139z.top", "unocl45trpuoefft.x9kjcn.bid", "unocl45trpuoefft.x9le66.top", "unocl45trpuoefft.xf38wp.bid", "unocl45trpuoefft.xlxd92.bid", "unocl45trpuoefft.y721yz.top", "unocl45trpuoefft.ye4f7k.bid", "unocl45trpuoefft.yky1uf.bid", "unocl45trpuoefft.ytbyhs.bid", "unocl45trpuoefft.yty0gm.bid", "unocl45trpuoefft.zbj2kc.bid", "unocl45trpuoefft.zdamew.bid", "unocl45trpuoefft.zgheyh.bid", "unocl45trpuoefft.zjems2.bid", "unocl45trpuoefft.zn9cme.bid", "urulvtffwoq.xyz", "uuwflbmjmi.eu", "uvcmlfca.biz", "uxvvm.us", "uxwavkmttywsuynt.pw", "vcabbvhrqhot.pw", "vewrb.italisumo.at", "vpuroeit.pw", "vrvis6ndra5jeggj.livegaming.ch", "vrvis6ndra5jeggj.livewargaming.ch", "vrvis6ndra5jeggj.onlinebattlefield.ch", "vrympoqs5ra34nfo.bigbird.at", "vrympoqs5ra34nfo.bigclear.at", "vrympoqs5ra34nfo.smartbus.at", "vrympoqs5ra34nfo.torhelper.pl", "vujqbcditgsqxe.fr", "vyohacxzoue32vvk.0ayn1s.top", "vyohacxzoue32vvk.0ot7em.bid", "vyohacxzoue32vvk.0vtwzy.top", "vyohacxzoue32vvk.1m47ka.bid", "vyohacxzoue32vvk.23fvxw.bid", "vyohacxzoue32vvk.2hr4fs.top", "vyohacxzoue32vvk.34o9h1.bid", "vyohacxzoue32vvk.3buvlc.bid", "vyohacxzoue32vvk.3m370u.top", "vyohacxzoue32vvk.3peyo3.bid", "vyohacxzoue32vvk.3t3hyf.top", "vyohacxzoue32vvk.5a5vmh.top", "vyohacxzoue32vvk.5i0ukv.bid", "vyohacxzoue32vvk.5m2n7x.top", "vyohacxzoue32vvk.5s96fr.top", "vyohacxzoue32vvk.6wkz70.bid", "vyohacxzoue32vvk.79j8fm.top", "vyohacxzoue32vvk.7a07br.bid", "vyohacxzoue32vvk.7jrv53.bid", "vyohacxzoue32vvk.7m7ujm.bid", "vyohacxzoue32vvk.8g1k17.bid", "vyohacxzoue32vvk.ac7zvz.top", "vyohacxzoue32vvk.axu3u8.bid", "vyohacxzoue32vvk.b14kkk.bid", "vyohacxzoue32vvk.c4cwr4.bid", "vyohacxzoue32vvk.c8jxpp.top", "vyohacxzoue32vvk.chnbyl.bid", "vyohacxzoue32vvk.cp3yme.top", "vyohacxzoue32vvk.d7h6yx.top", "vyohacxzoue32vvk.dgjpgy.top", "vyohacxzoue32vvk.dks71o.bid", "vyohacxzoue32vvk.ean5e7.top", "vyohacxzoue32vvk.ewfp5y.bid", "vyohacxzoue32vvk.ezb568.top", "vyohacxzoue32vvk.fp6fj6.top", "vyohacxzoue32vvk.fsly47.top", "vyohacxzoue32vvk.g7rst5.bid", "vyohacxzoue32vvk.gjbmis.top", "vyohacxzoue32vvk.h2xun1.top", "vyohacxzoue32vvk.ibar8s.top", "vyohacxzoue32vvk.jb4uh0.top", "vyohacxzoue32vvk.jnv1df.top", "vyohacxzoue32vvk.joco7r.top", "vyohacxzoue32vvk.jwi2ek.bid", "vyohacxzoue32vvk.k9p80d.top", "vyohacxzoue32vvk.kfymbh.top", "vyohacxzoue32vvk.kwrd4f.bid", "vyohacxzoue32vvk.l4dlll.bid", "vyohacxzoue32vvk.mayrwf.top", "vyohacxzoue32vvk.mpduf5.bid", "vyohacxzoue32vvk.ncw0rp.top", "vyohacxzoue32vvk.nta934.top", "vyohacxzoue32vvk.o08ra6.top", "vyohacxzoue32vvk.o5b17o.top", "vyohacxzoue32vvk.p9su2u.top", "vyohacxzoue32vvk.pr52ni.top", "vyohacxzoue32vvk.r31sot.top", "vyohacxzoue32vvk.r3b2sh.top", "vyohacxzoue32vvk.roep3o.top", "vyohacxzoue32vvk.ss8doe.top", "vyohacxzoue32vvk.t6ueop.bid", "vyohacxzoue32vvk.u8e2dz.top", "vyohacxzoue32vvk.ug6ewx.top", "vyohacxzoue32vvk.vjso7r.top", "vyohacxzoue32vvk.w22p3v.top", "vyohacxzoue32vvk.w67y8u.bid", "vyohacxzoue32vvk.x83zw1.top", "vyohacxzoue32vvk.xsf5a8.top", "vyohacxzoue32vvk.xy2rlg.bid", "vyohacxzoue32vvk.zmn16h.top", "vyohacxzoue32vvk.zn90h4.bid", "vyohacxzoue32vvk.zp9i1l.bid", "vyohacxzoue32vvk.zu3fzc.bid", "vyohacxzoue32vvk.zz3w5l.bid", "w6bfg4hahn5bfnlsafgchkvg5fwsfvrt.hareuna.at", "waduavfijwkanvf.xyz", "wbaskcsxiffiax.info", "wdvxeval.ru", "wersalitrestyws.top", "wjfkoqueatxdmqw.biz", "wjtqjleommc4z46i.249isv.bid", "wjtqjleommc4z46i.2y4t6f.bid", "wjtqjleommc4z46i.35rof4.bid", "wjtqjleommc4z46i.35u068.bid", "wjtqjleommc4z46i.44vva6.bid", "wjtqjleommc4z46i.4bb9vz.bid", "wjtqjleommc4z46i.54vw9b.bid", "wjtqjleommc4z46i.5n5y6v.bid", "wjtqjleommc4z46i.5r1sol.bid", "wjtqjleommc4z46i.7hu6og.bid", "wjtqjleommc4z46i.8a9r2h.bid", "wjtqjleommc4z46i.993hev.bid", "wjtqjleommc4z46i.9sellg.bid", "wjtqjleommc4z46i.9ule2e.bid", "wjtqjleommc4z46i.au6d1d.bid", "wjtqjleommc4z46i.bipa9k.bid", "wjtqjleommc4z46i.c3fz3z.bid", "wjtqjleommc4z46i.cc0r87.bid", "wjtqjleommc4z46i.cdyd2z.bid", "wjtqjleommc4z46i.cgab48.bid", "wjtqjleommc4z46i.cm5ohx.bid", "wjtqjleommc4z46i.csv7o6.bid", "wjtqjleommc4z46i.cto5ee.bid", "wjtqjleommc4z46i.d11zjd.bid", "wjtqjleommc4z46i.e53rg4.bid", "wjtqjleommc4z46i.eag72x.top", "wjtqjleommc4z46i.efyh72.bid", "wjtqjleommc4z46i.f0jlbj.bid", "wjtqjleommc4z46i.fw1bwy.bid", "wjtqjleommc4z46i.fwfu4t.bid", "wjtqjleommc4z46i.gg4dgp.bid", "wjtqjleommc4z46i.h8prbu.top", "wjtqjleommc4z46i.hom07d.bid", "wjtqjleommc4z46i.i8zh1k.bid", "wjtqjleommc4z46i.idw6s5.bid", "wjtqjleommc4z46i.ilmgcl.bid", "wjtqjleommc4z46i.izyclz.bid", "wjtqjleommc4z46i.j0n83w.bid", "wjtqjleommc4z46i.jal9lk.bid", "wjtqjleommc4z46i.jujthy.bid", "wjtqjleommc4z46i.kt70uk.bid", "wjtqjleommc4z46i.kyjw0g.bid", "wjtqjleommc4z46i.kzhzuc.top", "wjtqjleommc4z46i.ldsl8m.bid", "wjtqjleommc4z46i.m33d4b.bid", "wjtqjleommc4z46i.n8ln0w.bid", "wjtqjleommc4z46i.nh47ri.bid", "wjtqjleommc4z46i.nnbdlh.bid", "wjtqjleommc4z46i.nxmu0x.bid", "wjtqjleommc4z46i.o8hpwj.top", "wjtqjleommc4z46i.obx4vo.bid", "wjtqjleommc4z46i.oodvxp.bid", "wjtqjleommc4z46i.p41khf.bid", "wjtqjleommc4z46i.pmnz7a.bid", "wjtqjleommc4z46i.salethe.gdn", "wjtqjleommc4z46i.srmlzh.bid", "wjtqjleommc4z46i.srtos7.bid", "wjtqjleommc4z46i.t4jp3w.bid", "wjtqjleommc4z46i.u36ik0.bid", "wjtqjleommc4z46i.uv39h5.bid", "wjtqjleommc4z46i.uwckha.top", "wjtqjleommc4z46i.vh6vss.bid", "wjtqjleommc4z46i.w3r6a4.bid", "wjtqjleommc4z46i.whmykv.bid", "wjtqjleommc4z46i.xjwlms.bid", "wjtqjleommc4z46i.y12acl.bid", "wjtqjleommc4z46i.y2ijlz.bid", "wjtqjleommc4z46i.y7603i.bid", "wjtqjleommc4z46i.yfr0o1.bid", "wjtqjleommc4z46i.z7uxzg.bid", "wjtqjleommc4z46i.z97f9v.bid", "wjtqjleommc4z46i.zclhx9.bid", "wor4d.slewirk.at", "wpvvusso.xyz", "wqxvsxppjivs.pw", "wrubyjtvqhxaqkh.pw", "wtxvmsikbmtbq.pw", "wvltrlrnf.xyz", "www.1axb.com", "www.chromebewfk.top", "www.chromefastl.top", "www.chromehakc.top", "www.cleverdotl.top", "www.ddiopoola.top", "www.dealkolld.top", "www.dokjasura.top", "www.fkauueeepla.top", "www.flowerxpo.top", "www.foolalexas.top", "www.googlefoad.top", "www.newsectorbs.top", "www.newtonpaiva.br", "www.watherfka.top", "www.weekendlk.top", "x5sbb5gesp6kzwsh.frontmain.pl", "x5sbb5gesp6kzwsh.frontymen.pl", "x5sbb5gesp6kzwsh.homewind.pl", "x5sbb5gesp6kzwsh.mailteam.pl", "x5sbb5gesp6kzwsh.questpul.pl", "xfyubqmldwvuyar.yt", "xhrnfffaixawpuob.pw", "xmniabhrfafptwx.pw", "xofguhypjgvxrm.pw", "xpcx6erilkjced3j.16hwwh.top", "xpcx6erilkjced3j.16umxg.top", "xpcx6erilkjced3j.17gcun.top", "xpcx6erilkjced3j.18ey8e.top", "xpcx6erilkjced3j.19kdeh.top", "xpcx6erilkjced3j.1blery.top", "xpcx6erilkjced3j.1cgbcv.top", "xpcx6erilkjced3j.1ebjjq.top", "xpcx6erilkjced3j.1j9jad.top", "xpcx6erilkjced3j.1jyrty.top", "xpcx6erilkjced3j.1mfmkz.top", "xpcx6erilkjced3j.1mpsnr.top", "xpcx6erilkjced3j.1n5mod.top", "xrhwryizf5mui7a5.50mb1c.bid", "xrhwryizf5mui7a5.djintc.bid", "xrhwryizf5mui7a5.g72xh8.top", "xrhwryizf5mui7a5.h44l3d.bid", "xrhwryizf5mui7a5.j4cser.bid", "xrhwryizf5mui7a5.jhrb5a.top", "xrhwryizf5mui7a5.r8c85p.top", "xrhwryizf5mui7a5.rt01jw.top", "xrhwryizf5mui7a5.uw9x7z.bid", "xrhwryizf5mui7a5.vgxcci.top", "xvchcbeqxkd.pw", "xyhhuxa.be", "y4bxj.adozeuds.com", "yavmxpiqfwmubk.pw", "yaynawvtuqcarjwc.pw", "ycvcjbhgkmsiyhdd.info", "yofkhfskdyiqo.biz", "ytcijiooxdtlbevrh.info", "ytrest84y5i456hghadefdsd.pontogrot.com", "yuertao.pw", "yuysikankhqvdwdv.xyz", "ywjgjvpuyitnbiw.info", "yyre45dbvn2nhbefbmh.begumvelic.at", "zjfq4lnfbs7pncr5.onion.to", "zjfq4lnfbs7pncr5.tor2web.org", "ztuw5bvuuapzdfya.klimbim.pl", "zutzt67dcxr6mxcn.onion.to"],"md5": []}}]} ================================================ FILE: tests/integration/basic/IPv4.lst ================================================ 103.249.88.244-103.249.88.244 103.89.88.88-103.89.88.88 104.18.34.162-104.18.34.162 108.170.60.189-108.170.60.189 109.230.199.159-109.230.199.159 109.230.199.169-109.230.199.169 109.230.199.30-109.230.199.30 125.209.82.158-125.209.82.158 136.25.2.43-136.25.2.43 137.74.131.18-137.74.131.18 138.197.148.53-138.197.148.53 140.82.48.224-140.82.48.224 144.76.215.117-144.76.215.117 162.244.32.180-162.244.32.180 173.254.223.115-173.254.223.115 173.46.85.161-173.46.85.161 173.46.85.168-173.46.85.168 173.46.85.19-173.46.85.19 173.46.85.205-173.46.85.205 173.46.85.22-173.46.85.22 173.46.85.234-173.46.85.234 173.46.85.60-173.46.85.60 173.46.85.68-173.46.85.68 173.46.85.71-173.46.85.71 173.46.85.86-173.46.85.86 173.46.85.98-173.46.85.98 178.162.132.90-178.162.132.90 178.239.21.106-178.239.21.106 179.43.176.148-179.43.176.148 18.221.114.76-18.221.114.76 181.129.146.34-181.129.146.34 181.129.171.34-181.129.171.34 181.129.93.226-181.129.93.226 181.215.247.164-181.215.247.164 181.215.47.171-181.215.47.171 185.125.205.69-185.125.205.69 185.125.205.73-185.125.205.73 185.125.205.75-185.125.205.75 185.125.205.78-185.125.205.78 185.125.205.79-185.125.205.79 185.125.205.91-185.125.205.91 185.127.27.238-185.127.27.238 185.141.62.213-185.141.62.213 185.156.174.115-185.156.174.115 185.158.248.92-185.158.248.92 185.158.249.233-185.158.249.233 185.158.251.60-185.158.251.60 185.174.173.128-185.174.173.128 185.189.149.187-185.189.149.187 185.202.174.91-185.202.174.91 185.203.118.6-185.203.118.6 185.205.210.139-185.205.210.139 185.212.47.103-185.212.47.103 185.22.65.5-185.22.65.5 185.223.163.26-185.223.163.26 185.231.153.46-185.231.153.46 185.236.203.53-185.236.203.53 185.236.203.60-185.236.203.60 185.244.30.101-185.244.30.101 185.244.30.105-185.244.30.105 185.244.30.106-185.244.30.106 185.244.30.109-185.244.30.109 185.244.30.111-185.244.30.111 185.244.30.113-185.244.30.113 185.244.30.114-185.244.30.114 185.244.30.120-185.244.30.120 185.244.30.121-185.244.30.121 185.244.30.124-185.244.30.124 185.244.30.93-185.244.30.93 185.77.129.11-185.77.129.11 186.147.161.204-186.147.161.204 186.167.66.51-186.167.66.51 187.19.17.132-187.19.17.132 192.227.248.175-192.227.248.175 192.99.212.140-192.99.212.140 193.29.56.44-193.29.56.44 194.5.98.104-194.5.98.104 194.5.98.139-194.5.98.139 194.5.98.148-194.5.98.148 194.5.98.193-194.5.98.193 194.5.98.194-194.5.98.194 194.5.98.226-194.5.98.226 194.5.98.38-194.5.98.38 194.5.98.56-194.5.98.56 194.5.99.119-194.5.99.119 194.5.99.136-194.5.99.136 194.5.99.158-194.5.99.158 194.5.99.159-194.5.99.159 194.5.99.175-194.5.99.175 194.5.99.2-194.5.99.2 194.5.99.207-194.5.99.207 194.5.99.226-194.5.99.226 194.5.99.250-194.5.99.250 194.5.99.59-194.5.99.59 194.5.99.63-194.5.99.63 194.5.99.67-194.5.99.67 194.5.99.7-194.5.99.7 194.5.99.97-194.5.99.97 194.68.225.63-194.68.225.63 194.76.225.59-194.76.225.59 194.99.20.254-194.99.20.254 195.123.212.149-195.123.212.149 195.123.213.169-195.123.213.169 195.123.227.20-195.123.227.20 195.123.245.214-195.123.245.214 195.123.245.90-195.123.245.90 199.21.106.189-199.21.106.189 202.63.242.48-202.63.242.48 204.95.99.204-204.95.99.204 209.58.186.245-209.58.186.245 212.47.194.15-212.47.194.15 212.73.150.215-212.73.150.215 213.152.161.138-213.152.161.138 24.217.192.131-24.217.192.131 24.247.181.155-24.247.181.155 24.247.182.169-24.247.182.169 24.247.182.240-24.247.182.240 24.247.182.253-24.247.182.253 31.171.152.103-31.171.152.103 31.171.152.105-31.171.152.105 31.171.152.106-31.171.152.106 31.171.152.107-31.171.152.107 31.7.188.40-31.7.188.40 35.198.61.54-35.198.61.54 37.59.134.55-37.59.134.55 45.249.90.124-45.249.90.124 45.55.36.231-45.55.36.231 46.166.173.109-46.166.173.109 46.17.45.29-46.17.45.29 46.17.47.216-46.17.47.216 46.183.223.10-46.183.223.10 47.44.54.70-47.44.54.70 5.2.64.188-5.2.64.188 5.2.67.66-5.2.67.66 5.206.225.115-5.206.225.115 5.8.88.125-5.8.88.125 54.37.86.44-54.37.86.44 54.38.146.43-54.38.146.43 68.111.123.100-68.111.123.100 68.183.249.84-68.183.249.84 72.189.124.41-72.189.124.41 76.107.90.235-76.107.90.235 78.155.220.198-78.155.220.198 81.177.141.211-81.177.141.211 81.177.180.174-81.177.180.174 82.199.134.139-82.199.134.139 82.199.134.156-82.199.134.156 83.166.245.213-83.166.245.213 85.217.170.62-85.217.170.62 87.236.22.142-87.236.22.142 91.192.100.16-91.192.100.16 91.192.100.27-91.192.100.27 91.192.100.3-91.192.100.3 91.192.100.40-91.192.100.40 91.192.100.44-91.192.100.44 91.192.100.48-91.192.100.48 91.192.100.52-91.192.100.52 92.222.10.99-92.222.10.99 93.115.26.171-93.115.26.171 94.103.83.137-94.103.83.137 94.185.86.56-94.185.86.56 94.237.44.31-94.237.44.31 95.213.251.165-95.213.251.165 95.47.161.68-95.47.161.68 97.87.175.152-97.87.175.152 1.1.1.0-1.1.1.33 1.1.2.0-1.1.2.255 1.1.3.10-1.1.3.44 ================================================ FILE: tests/integration/basic/IPv4HC%3Fs%3D5%26n%3D10.result ================================================ 104.18.34.162-104.18.34.162 108.170.60.189-108.170.60.189 109.230.199.159-109.230.199.159 109.230.199.169-109.230.199.169 109.230.199.30-109.230.199.30 125.209.82.158-125.209.82.158 136.25.2.43-136.25.2.43 137.74.131.18-137.74.131.18 138.197.148.53-138.197.148.53 140.82.48.224-140.82.48.224 ================================================ FILE: tests/integration/basic/IPv4HC%3Fv%3Dcsv%26f%3Dconfidence%26f%3Dsources%7Cfeeds%26f%3Dindicator%7Cclientip%26tr%3D1.result ================================================ confidence,feeds,clientip 100,localdb,1.1.1.0/27 100,localdb,1.1.1.32/31 100,localdb,1.1.2.0/24 100,localdb,1.1.3.10/31 100,localdb,1.1.3.12/30 100,localdb,1.1.3.16/28 100,localdb,1.1.3.32/29 100,localdb,1.1.3.40/30 100,localdb,1.1.3.44 100,localdb,103.249.88.244 100,localdb,103.89.88.88 100,localdb,104.18.34.162 100,localdb,108.170.60.189 100,localdb,109.230.199.159 100,localdb,109.230.199.169 100,localdb,109.230.199.30 100,localdb,125.209.82.158 100,localdb,136.25.2.43 100,localdb,137.74.131.18 100,localdb,138.197.148.53 100,localdb,140.82.48.224 100,localdb,144.76.215.117 100,localdb,162.244.32.180 100,localdb,173.254.223.115 100,localdb,173.46.85.161 100,localdb,173.46.85.168 100,localdb,173.46.85.19 100,localdb,173.46.85.205 100,localdb,173.46.85.22 100,localdb,173.46.85.234 100,localdb,173.46.85.60 100,localdb,173.46.85.68 100,localdb,173.46.85.71 100,localdb,173.46.85.86 100,localdb,173.46.85.98 100,localdb,178.162.132.90 100,localdb,178.239.21.106 100,localdb,179.43.176.148 100,localdb,18.221.114.76 100,localdb,181.129.146.34 100,localdb,181.129.171.34 100,localdb,181.129.93.226 100,localdb,181.215.247.164 100,localdb,181.215.47.171 100,localdb,185.125.205.69 100,localdb,185.125.205.73 100,localdb,185.125.205.75 100,localdb,185.125.205.78 100,localdb,185.125.205.79 100,localdb,185.125.205.91 100,localdb,185.127.27.238 100,localdb,185.141.62.213 100,localdb,185.156.174.115 100,localdb,185.158.248.92 100,localdb,185.158.249.233 100,localdb,185.158.251.60 100,localdb,185.174.173.128 100,localdb,185.189.149.187 100,localdb,185.202.174.91 100,localdb,185.203.118.6 100,localdb,185.205.210.139 100,localdb,185.212.47.103 100,localdb,185.22.65.5 100,localdb,185.223.163.26 100,localdb,185.231.153.46 100,localdb,185.236.203.53 100,localdb,185.236.203.60 100,localdb,185.244.30.101 100,localdb,185.244.30.105 100,localdb,185.244.30.106 100,localdb,185.244.30.109 100,localdb,185.244.30.111 100,localdb,185.244.30.113 100,localdb,185.244.30.114 100,localdb,185.244.30.120 100,localdb,185.244.30.121 100,localdb,185.244.30.124 100,localdb,185.244.30.93 100,localdb,185.77.129.11 100,localdb,186.147.161.204 100,localdb,186.167.66.51 100,localdb,187.19.17.132 100,localdb,192.227.248.175 100,localdb,192.99.212.140 100,localdb,193.29.56.44 100,localdb,194.5.98.104 100,localdb,194.5.98.139 100,localdb,194.5.98.148 100,localdb,194.5.98.193 100,localdb,194.5.98.194 100,localdb,194.5.98.226 100,localdb,194.5.98.38 100,localdb,194.5.98.56 100,localdb,194.5.99.119 100,localdb,194.5.99.136 100,localdb,194.5.99.158 100,localdb,194.5.99.159 100,localdb,194.5.99.175 100,localdb,194.5.99.2 100,localdb,194.5.99.207 100,localdb,194.5.99.226 100,localdb,194.5.99.250 100,localdb,194.5.99.59 100,localdb,194.5.99.63 100,localdb,194.5.99.67 100,localdb,194.5.99.7 100,localdb,194.5.99.97 100,localdb,194.68.225.63 100,localdb,194.76.225.59 100,localdb,194.99.20.254 100,localdb,195.123.212.149 100,localdb,195.123.213.169 100,localdb,195.123.227.20 100,localdb,195.123.245.214 100,localdb,195.123.245.90 100,localdb,199.21.106.189 100,localdb,202.63.242.48 100,localdb,204.95.99.204 100,localdb,209.58.186.245 100,localdb,212.47.194.15 100,localdb,212.73.150.215 100,localdb,213.152.161.138 100,localdb,24.217.192.131 100,localdb,24.247.181.155 100,localdb,24.247.182.169 100,localdb,24.247.182.240 100,localdb,24.247.182.253 100,localdb,31.171.152.103 100,localdb,31.171.152.105 100,localdb,31.171.152.106 100,localdb,31.171.152.107 100,localdb,31.7.188.40 100,localdb,35.198.61.54 100,localdb,37.59.134.55 100,localdb,45.249.90.124 100,localdb,45.55.36.231 100,localdb,46.166.173.109 100,localdb,46.17.45.29 100,localdb,46.17.47.216 100,localdb,46.183.223.10 100,localdb,47.44.54.70 100,localdb,5.2.64.188 100,localdb,5.2.67.66 100,localdb,5.206.225.115 100,localdb,5.8.88.125 100,localdb,54.37.86.44 100,localdb,54.38.146.43 100,localdb,68.111.123.100 100,localdb,68.183.249.84 100,localdb,72.189.124.41 100,localdb,76.107.90.235 100,localdb,78.155.220.198 100,localdb,81.177.141.211 100,localdb,81.177.180.174 100,localdb,82.199.134.139 100,localdb,82.199.134.156 100,localdb,83.166.245.213 100,localdb,85.217.170.62 100,localdb,87.236.22.142 100,localdb,91.192.100.16 100,localdb,91.192.100.27 100,localdb,91.192.100.3 100,localdb,91.192.100.40 100,localdb,91.192.100.44 100,localdb,91.192.100.48 100,localdb,91.192.100.52 100,localdb,92.222.10.99 100,localdb,93.115.26.171 100,localdb,94.103.83.137 100,localdb,94.185.86.56 100,localdb,94.237.44.31 100,localdb,95.213.251.165 100,localdb,95.47.161.68 100,localdb,97.87.175.152 ================================================ FILE: tests/integration/basic/IPv4HC%3Fv%3Djson%26tr%3D1.result ================================================ [ {"indicator":"1.1.1.0/27","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"1.1.1.32/31","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"1.1.2.0/24","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"1.1.3.10/31","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"1.1.3.12/30","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"1.1.3.16/28","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"1.1.3.32/29","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"1.1.3.40/30","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"1.1.3.44","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"103.249.88.244","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"103.89.88.88","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"104.18.34.162","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"108.170.60.189","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"109.230.199.159","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"109.230.199.169","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"109.230.199.30","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"125.209.82.158","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"136.25.2.43","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"137.74.131.18","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"138.197.148.53","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"140.82.48.224","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"144.76.215.117","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"162.244.32.180","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"173.254.223.115","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"173.46.85.161","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"173.46.85.168","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"173.46.85.19","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"173.46.85.205","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"173.46.85.22","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"173.46.85.234","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"173.46.85.60","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"173.46.85.68","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"173.46.85.71","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"173.46.85.86","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"173.46.85.98","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"178.162.132.90","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"178.239.21.106","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"179.43.176.148","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"18.221.114.76","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"181.129.146.34","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"181.129.171.34","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"181.129.93.226","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"181.215.247.164","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"181.215.47.171","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.125.205.69","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.125.205.73","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.125.205.75","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.125.205.78","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.125.205.79","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.125.205.91","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.127.27.238","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.141.62.213","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.156.174.115","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.158.248.92","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.158.249.233","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.158.251.60","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.174.173.128","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.189.149.187","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.202.174.91","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.203.118.6","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.205.210.139","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.212.47.103","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.22.65.5","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.223.163.26","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.231.153.46","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.236.203.53","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.236.203.60","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.244.30.101","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.244.30.105","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.244.30.106","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.244.30.109","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.244.30.111","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.244.30.113","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.244.30.114","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.244.30.120","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.244.30.121","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.244.30.124","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.244.30.93","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"185.77.129.11","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"186.147.161.204","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"186.167.66.51","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"187.19.17.132","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"192.227.248.175","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"192.99.212.140","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"193.29.56.44","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.98.104","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.98.139","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.98.148","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.98.193","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.98.194","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.98.226","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.98.38","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.98.56","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.119","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.136","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.158","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.159","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.175","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.2","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.207","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.226","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.250","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.59","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.63","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.67","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.7","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.5.99.97","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.68.225.63","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.76.225.59","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"194.99.20.254","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"195.123.212.149","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"195.123.213.169","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"195.123.227.20","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"195.123.245.214","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"195.123.245.90","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"199.21.106.189","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"202.63.242.48","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"204.95.99.204","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"209.58.186.245","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"212.47.194.15","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"212.73.150.215","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"213.152.161.138","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"24.217.192.131","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"24.247.181.155","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"24.247.182.169","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"24.247.182.240","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"24.247.182.253","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"31.171.152.103","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"31.171.152.105","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"31.171.152.106","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"31.171.152.107","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"31.7.188.40","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"35.198.61.54","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"37.59.134.55","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"45.249.90.124","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"45.55.36.231","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"46.166.173.109","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"46.17.45.29","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"46.17.47.216","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"46.183.223.10","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"47.44.54.70","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"5.2.64.188","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"5.2.67.66","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"5.206.225.115","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"5.8.88.125","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"54.37.86.44","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"54.38.146.43","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"68.111.123.100","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"68.183.249.84","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"72.189.124.41","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"76.107.90.235","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"78.155.220.198","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"81.177.141.211","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"81.177.180.174","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"82.199.134.139","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"82.199.134.156","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"83.166.245.213","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"85.217.170.62","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"87.236.22.142","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"91.192.100.16","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"91.192.100.27","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"91.192.100.3","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"91.192.100.40","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"91.192.100.44","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"91.192.100.48","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"91.192.100.52","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"92.222.10.99","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"93.115.26.171","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"94.103.83.137","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"94.185.86.56","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"94.237.44.31","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"95.213.251.165","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"95.47.161.68","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}, {"indicator":"97.87.175.152","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}}] ================================================ FILE: tests/integration/basic/IPv4HC%3Fv%3Djson-seq.result ================================================ {"indicator":"1.1.1.0-1.1.1.33","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"1.1.2.0-1.1.2.255","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"1.1.3.10-1.1.3.44","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"103.249.88.244-103.249.88.244","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"103.89.88.88-103.89.88.88","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"104.18.34.162-104.18.34.162","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"108.170.60.189-108.170.60.189","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"109.230.199.159-109.230.199.159","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"109.230.199.169-109.230.199.169","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"109.230.199.30-109.230.199.30","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"125.209.82.158-125.209.82.158","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"136.25.2.43-136.25.2.43","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"137.74.131.18-137.74.131.18","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"138.197.148.53-138.197.148.53","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"140.82.48.224-140.82.48.224","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"144.76.215.117-144.76.215.117","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"162.244.32.180-162.244.32.180","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"173.254.223.115-173.254.223.115","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"173.46.85.161-173.46.85.161","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"173.46.85.168-173.46.85.168","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"173.46.85.19-173.46.85.19","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"173.46.85.205-173.46.85.205","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"173.46.85.22-173.46.85.22","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"173.46.85.234-173.46.85.234","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"173.46.85.60-173.46.85.60","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"173.46.85.68-173.46.85.68","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"173.46.85.71-173.46.85.71","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"173.46.85.86-173.46.85.86","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"173.46.85.98-173.46.85.98","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"178.162.132.90-178.162.132.90","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"178.239.21.106-178.239.21.106","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"179.43.176.148-179.43.176.148","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"18.221.114.76-18.221.114.76","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"181.129.146.34-181.129.146.34","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"181.129.171.34-181.129.171.34","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"181.129.93.226-181.129.93.226","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"181.215.247.164-181.215.247.164","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"181.215.47.171-181.215.47.171","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.125.205.69-185.125.205.69","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.125.205.73-185.125.205.73","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.125.205.75-185.125.205.75","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.125.205.78-185.125.205.78","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.125.205.79-185.125.205.79","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.125.205.91-185.125.205.91","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.127.27.238-185.127.27.238","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.141.62.213-185.141.62.213","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.156.174.115-185.156.174.115","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.158.248.92-185.158.248.92","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.158.249.233-185.158.249.233","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.158.251.60-185.158.251.60","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.174.173.128-185.174.173.128","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.189.149.187-185.189.149.187","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.202.174.91-185.202.174.91","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.203.118.6-185.203.118.6","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.205.210.139-185.205.210.139","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.212.47.103-185.212.47.103","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.22.65.5-185.22.65.5","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.223.163.26-185.223.163.26","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.231.153.46-185.231.153.46","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.236.203.53-185.236.203.53","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.236.203.60-185.236.203.60","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.244.30.101-185.244.30.101","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.244.30.105-185.244.30.105","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.244.30.106-185.244.30.106","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.244.30.109-185.244.30.109","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.244.30.111-185.244.30.111","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.244.30.113-185.244.30.113","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.244.30.114-185.244.30.114","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.244.30.120-185.244.30.120","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.244.30.121-185.244.30.121","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.244.30.124-185.244.30.124","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.244.30.93-185.244.30.93","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"185.77.129.11-185.77.129.11","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"186.147.161.204-186.147.161.204","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"186.167.66.51-186.167.66.51","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"187.19.17.132-187.19.17.132","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"192.227.248.175-192.227.248.175","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"192.99.212.140-192.99.212.140","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"193.29.56.44-193.29.56.44","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.98.104-194.5.98.104","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.98.139-194.5.98.139","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.98.148-194.5.98.148","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.98.193-194.5.98.193","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.98.194-194.5.98.194","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.98.226-194.5.98.226","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.98.38-194.5.98.38","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.98.56-194.5.98.56","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.119-194.5.99.119","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.136-194.5.99.136","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.158-194.5.99.158","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.159-194.5.99.159","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.175-194.5.99.175","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.2-194.5.99.2","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.207-194.5.99.207","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.226-194.5.99.226","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.250-194.5.99.250","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.59-194.5.99.59","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.63-194.5.99.63","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.67-194.5.99.67","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.7-194.5.99.7","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.5.99.97-194.5.99.97","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.68.225.63-194.68.225.63","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.76.225.59-194.76.225.59","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"194.99.20.254-194.99.20.254","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"195.123.212.149-195.123.212.149","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"195.123.213.169-195.123.213.169","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"195.123.227.20-195.123.227.20","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"195.123.245.214-195.123.245.214","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"195.123.245.90-195.123.245.90","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"199.21.106.189-199.21.106.189","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"202.63.242.48-202.63.242.48","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"204.95.99.204-204.95.99.204","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"209.58.186.245-209.58.186.245","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"212.47.194.15-212.47.194.15","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"212.73.150.215-212.73.150.215","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"213.152.161.138-213.152.161.138","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"24.217.192.131-24.217.192.131","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"24.247.181.155-24.247.181.155","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"24.247.182.169-24.247.182.169","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"24.247.182.240-24.247.182.240","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"24.247.182.253-24.247.182.253","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"31.171.152.103-31.171.152.103","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"31.171.152.105-31.171.152.105","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"31.171.152.106-31.171.152.106","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"31.171.152.107-31.171.152.107","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"31.7.188.40-31.7.188.40","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"35.198.61.54-35.198.61.54","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"37.59.134.55-37.59.134.55","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"45.249.90.124-45.249.90.124","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"45.55.36.231-45.55.36.231","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"46.166.173.109-46.166.173.109","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"46.17.45.29-46.17.45.29","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"46.17.47.216-46.17.47.216","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"46.183.223.10-46.183.223.10","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"47.44.54.70-47.44.54.70","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"5.2.64.188-5.2.64.188","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"5.2.67.66-5.2.67.66","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"5.206.225.115-5.206.225.115","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"5.8.88.125-5.8.88.125","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"54.37.86.44-54.37.86.44","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"54.38.146.43-54.38.146.43","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"68.111.123.100-68.111.123.100","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"68.183.249.84-68.183.249.84","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"72.189.124.41-72.189.124.41","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"76.107.90.235-76.107.90.235","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"78.155.220.198-78.155.220.198","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"81.177.141.211-81.177.141.211","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"81.177.180.174-81.177.180.174","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"82.199.134.139-82.199.134.139","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"82.199.134.156-82.199.134.156","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"83.166.245.213-83.166.245.213","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"85.217.170.62-85.217.170.62","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"87.236.22.142-87.236.22.142","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"91.192.100.16-91.192.100.16","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"91.192.100.27-91.192.100.27","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"91.192.100.3-91.192.100.3","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"91.192.100.40-91.192.100.40","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"91.192.100.44-91.192.100.44","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"91.192.100.48-91.192.100.48","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"91.192.100.52-91.192.100.52","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"92.222.10.99-92.222.10.99","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"93.115.26.171-93.115.26.171","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"94.103.83.137-94.103.83.137","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"94.185.86.56-94.185.86.56","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"94.237.44.31-94.237.44.31","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"95.213.251.165-95.213.251.165","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"95.47.161.68-95.47.161.68","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} {"indicator":"97.87.175.152-97.87.175.152","value":{"sources":["localdb"],"confidence":100,"first_seen":1550570114477,"type":"IPv4","share_level":"red","last_seen":1550570114477}} ================================================ FILE: tests/integration/basic/IPv4HC%3Fv%3Dmwg.result ================================================ type=string "1.1.1.0-1.1.1.33" "localdb" "1.1.2.0-1.1.2.255" "localdb" "1.1.3.10-1.1.3.44" "localdb" "103.249.88.244-103.249.88.244" "localdb" "103.89.88.88-103.89.88.88" "localdb" "104.18.34.162-104.18.34.162" "localdb" "108.170.60.189-108.170.60.189" "localdb" "109.230.199.159-109.230.199.159" "localdb" "109.230.199.169-109.230.199.169" "localdb" "109.230.199.30-109.230.199.30" "localdb" "125.209.82.158-125.209.82.158" "localdb" "136.25.2.43-136.25.2.43" "localdb" "137.74.131.18-137.74.131.18" "localdb" "138.197.148.53-138.197.148.53" "localdb" "140.82.48.224-140.82.48.224" "localdb" "144.76.215.117-144.76.215.117" "localdb" "162.244.32.180-162.244.32.180" "localdb" "173.254.223.115-173.254.223.115" "localdb" "173.46.85.161-173.46.85.161" "localdb" "173.46.85.168-173.46.85.168" "localdb" "173.46.85.19-173.46.85.19" "localdb" "173.46.85.205-173.46.85.205" "localdb" "173.46.85.22-173.46.85.22" "localdb" "173.46.85.234-173.46.85.234" "localdb" "173.46.85.60-173.46.85.60" "localdb" "173.46.85.68-173.46.85.68" "localdb" "173.46.85.71-173.46.85.71" "localdb" "173.46.85.86-173.46.85.86" "localdb" "173.46.85.98-173.46.85.98" "localdb" "178.162.132.90-178.162.132.90" "localdb" "178.239.21.106-178.239.21.106" "localdb" "179.43.176.148-179.43.176.148" "localdb" "18.221.114.76-18.221.114.76" "localdb" "181.129.146.34-181.129.146.34" "localdb" "181.129.171.34-181.129.171.34" "localdb" "181.129.93.226-181.129.93.226" "localdb" "181.215.247.164-181.215.247.164" "localdb" "181.215.47.171-181.215.47.171" "localdb" "185.125.205.69-185.125.205.69" "localdb" "185.125.205.73-185.125.205.73" "localdb" "185.125.205.75-185.125.205.75" "localdb" "185.125.205.78-185.125.205.78" "localdb" "185.125.205.79-185.125.205.79" "localdb" "185.125.205.91-185.125.205.91" "localdb" "185.127.27.238-185.127.27.238" "localdb" "185.141.62.213-185.141.62.213" "localdb" "185.156.174.115-185.156.174.115" "localdb" "185.158.248.92-185.158.248.92" "localdb" "185.158.249.233-185.158.249.233" "localdb" "185.158.251.60-185.158.251.60" "localdb" "185.174.173.128-185.174.173.128" "localdb" "185.189.149.187-185.189.149.187" "localdb" "185.202.174.91-185.202.174.91" "localdb" "185.203.118.6-185.203.118.6" "localdb" "185.205.210.139-185.205.210.139" "localdb" "185.212.47.103-185.212.47.103" "localdb" "185.22.65.5-185.22.65.5" "localdb" "185.223.163.26-185.223.163.26" "localdb" "185.231.153.46-185.231.153.46" "localdb" "185.236.203.53-185.236.203.53" "localdb" "185.236.203.60-185.236.203.60" "localdb" "185.244.30.101-185.244.30.101" "localdb" "185.244.30.105-185.244.30.105" "localdb" "185.244.30.106-185.244.30.106" "localdb" "185.244.30.109-185.244.30.109" "localdb" "185.244.30.111-185.244.30.111" "localdb" "185.244.30.113-185.244.30.113" "localdb" "185.244.30.114-185.244.30.114" "localdb" "185.244.30.120-185.244.30.120" "localdb" "185.244.30.121-185.244.30.121" "localdb" "185.244.30.124-185.244.30.124" "localdb" "185.244.30.93-185.244.30.93" "localdb" "185.77.129.11-185.77.129.11" "localdb" "186.147.161.204-186.147.161.204" "localdb" "186.167.66.51-186.167.66.51" "localdb" "187.19.17.132-187.19.17.132" "localdb" "192.227.248.175-192.227.248.175" "localdb" "192.99.212.140-192.99.212.140" "localdb" "193.29.56.44-193.29.56.44" "localdb" "194.5.98.104-194.5.98.104" "localdb" "194.5.98.139-194.5.98.139" "localdb" "194.5.98.148-194.5.98.148" "localdb" "194.5.98.193-194.5.98.193" "localdb" "194.5.98.194-194.5.98.194" "localdb" "194.5.98.226-194.5.98.226" "localdb" "194.5.98.38-194.5.98.38" "localdb" "194.5.98.56-194.5.98.56" "localdb" "194.5.99.119-194.5.99.119" "localdb" "194.5.99.136-194.5.99.136" "localdb" "194.5.99.158-194.5.99.158" "localdb" "194.5.99.159-194.5.99.159" "localdb" "194.5.99.175-194.5.99.175" "localdb" "194.5.99.2-194.5.99.2" "localdb" "194.5.99.207-194.5.99.207" "localdb" "194.5.99.226-194.5.99.226" "localdb" "194.5.99.250-194.5.99.250" "localdb" "194.5.99.59-194.5.99.59" "localdb" "194.5.99.63-194.5.99.63" "localdb" "194.5.99.67-194.5.99.67" "localdb" "194.5.99.7-194.5.99.7" "localdb" "194.5.99.97-194.5.99.97" "localdb" "194.68.225.63-194.68.225.63" "localdb" "194.76.225.59-194.76.225.59" "localdb" "194.99.20.254-194.99.20.254" "localdb" "195.123.212.149-195.123.212.149" "localdb" "195.123.213.169-195.123.213.169" "localdb" "195.123.227.20-195.123.227.20" "localdb" "195.123.245.214-195.123.245.214" "localdb" "195.123.245.90-195.123.245.90" "localdb" "199.21.106.189-199.21.106.189" "localdb" "202.63.242.48-202.63.242.48" "localdb" "204.95.99.204-204.95.99.204" "localdb" "209.58.186.245-209.58.186.245" "localdb" "212.47.194.15-212.47.194.15" "localdb" "212.73.150.215-212.73.150.215" "localdb" "213.152.161.138-213.152.161.138" "localdb" "24.217.192.131-24.217.192.131" "localdb" "24.247.181.155-24.247.181.155" "localdb" "24.247.182.169-24.247.182.169" "localdb" "24.247.182.240-24.247.182.240" "localdb" "24.247.182.253-24.247.182.253" "localdb" "31.171.152.103-31.171.152.103" "localdb" "31.171.152.105-31.171.152.105" "localdb" "31.171.152.106-31.171.152.106" "localdb" "31.171.152.107-31.171.152.107" "localdb" "31.7.188.40-31.7.188.40" "localdb" "35.198.61.54-35.198.61.54" "localdb" "37.59.134.55-37.59.134.55" "localdb" "45.249.90.124-45.249.90.124" "localdb" "45.55.36.231-45.55.36.231" "localdb" "46.166.173.109-46.166.173.109" "localdb" "46.17.45.29-46.17.45.29" "localdb" "46.17.47.216-46.17.47.216" "localdb" "46.183.223.10-46.183.223.10" "localdb" "47.44.54.70-47.44.54.70" "localdb" "5.2.64.188-5.2.64.188" "localdb" "5.2.67.66-5.2.67.66" "localdb" "5.206.225.115-5.206.225.115" "localdb" "5.8.88.125-5.8.88.125" "localdb" "54.37.86.44-54.37.86.44" "localdb" "54.38.146.43-54.38.146.43" "localdb" "68.111.123.100-68.111.123.100" "localdb" "68.183.249.84-68.183.249.84" "localdb" "72.189.124.41-72.189.124.41" "localdb" "76.107.90.235-76.107.90.235" "localdb" "78.155.220.198-78.155.220.198" "localdb" "81.177.141.211-81.177.141.211" "localdb" "81.177.180.174-81.177.180.174" "localdb" "82.199.134.139-82.199.134.139" "localdb" "82.199.134.156-82.199.134.156" "localdb" "83.166.245.213-83.166.245.213" "localdb" "85.217.170.62-85.217.170.62" "localdb" "87.236.22.142-87.236.22.142" "localdb" "91.192.100.16-91.192.100.16" "localdb" "91.192.100.27-91.192.100.27" "localdb" "91.192.100.3-91.192.100.3" "localdb" "91.192.100.40-91.192.100.40" "localdb" "91.192.100.44-91.192.100.44" "localdb" "91.192.100.48-91.192.100.48" "localdb" "91.192.100.52-91.192.100.52" "localdb" "92.222.10.99-92.222.10.99" "localdb" "93.115.26.171-93.115.26.171" "localdb" "94.103.83.137-94.103.83.137" "localdb" "94.185.86.56-94.185.86.56" "localdb" "94.237.44.31-94.237.44.31" "localdb" "95.213.251.165-95.213.251.165" "localdb" "95.47.161.68-95.47.161.68" "localdb" "97.87.175.152-97.87.175.152" "localdb" ================================================ FILE: tests/integration/basic/IPv4HC.result ================================================ 1.1.1.0-1.1.1.33 1.1.2.0-1.1.2.255 1.1.3.10-1.1.3.44 103.249.88.244-103.249.88.244 103.89.88.88-103.89.88.88 104.18.34.162-104.18.34.162 108.170.60.189-108.170.60.189 109.230.199.159-109.230.199.159 109.230.199.169-109.230.199.169 109.230.199.30-109.230.199.30 125.209.82.158-125.209.82.158 136.25.2.43-136.25.2.43 137.74.131.18-137.74.131.18 138.197.148.53-138.197.148.53 140.82.48.224-140.82.48.224 144.76.215.117-144.76.215.117 162.244.32.180-162.244.32.180 173.254.223.115-173.254.223.115 173.46.85.161-173.46.85.161 173.46.85.168-173.46.85.168 173.46.85.19-173.46.85.19 173.46.85.205-173.46.85.205 173.46.85.22-173.46.85.22 173.46.85.234-173.46.85.234 173.46.85.60-173.46.85.60 173.46.85.68-173.46.85.68 173.46.85.71-173.46.85.71 173.46.85.86-173.46.85.86 173.46.85.98-173.46.85.98 178.162.132.90-178.162.132.90 178.239.21.106-178.239.21.106 179.43.176.148-179.43.176.148 18.221.114.76-18.221.114.76 181.129.146.34-181.129.146.34 181.129.171.34-181.129.171.34 181.129.93.226-181.129.93.226 181.215.247.164-181.215.247.164 181.215.47.171-181.215.47.171 185.125.205.69-185.125.205.69 185.125.205.73-185.125.205.73 185.125.205.75-185.125.205.75 185.125.205.78-185.125.205.78 185.125.205.79-185.125.205.79 185.125.205.91-185.125.205.91 185.127.27.238-185.127.27.238 185.141.62.213-185.141.62.213 185.156.174.115-185.156.174.115 185.158.248.92-185.158.248.92 185.158.249.233-185.158.249.233 185.158.251.60-185.158.251.60 185.174.173.128-185.174.173.128 185.189.149.187-185.189.149.187 185.202.174.91-185.202.174.91 185.203.118.6-185.203.118.6 185.205.210.139-185.205.210.139 185.212.47.103-185.212.47.103 185.22.65.5-185.22.65.5 185.223.163.26-185.223.163.26 185.231.153.46-185.231.153.46 185.236.203.53-185.236.203.53 185.236.203.60-185.236.203.60 185.244.30.101-185.244.30.101 185.244.30.105-185.244.30.105 185.244.30.106-185.244.30.106 185.244.30.109-185.244.30.109 185.244.30.111-185.244.30.111 185.244.30.113-185.244.30.113 185.244.30.114-185.244.30.114 185.244.30.120-185.244.30.120 185.244.30.121-185.244.30.121 185.244.30.124-185.244.30.124 185.244.30.93-185.244.30.93 185.77.129.11-185.77.129.11 186.147.161.204-186.147.161.204 186.167.66.51-186.167.66.51 187.19.17.132-187.19.17.132 192.227.248.175-192.227.248.175 192.99.212.140-192.99.212.140 193.29.56.44-193.29.56.44 194.5.98.104-194.5.98.104 194.5.98.139-194.5.98.139 194.5.98.148-194.5.98.148 194.5.98.193-194.5.98.193 194.5.98.194-194.5.98.194 194.5.98.226-194.5.98.226 194.5.98.38-194.5.98.38 194.5.98.56-194.5.98.56 194.5.99.119-194.5.99.119 194.5.99.136-194.5.99.136 194.5.99.158-194.5.99.158 194.5.99.159-194.5.99.159 194.5.99.175-194.5.99.175 194.5.99.2-194.5.99.2 194.5.99.207-194.5.99.207 194.5.99.226-194.5.99.226 194.5.99.250-194.5.99.250 194.5.99.59-194.5.99.59 194.5.99.63-194.5.99.63 194.5.99.67-194.5.99.67 194.5.99.7-194.5.99.7 194.5.99.97-194.5.99.97 194.68.225.63-194.68.225.63 194.76.225.59-194.76.225.59 194.99.20.254-194.99.20.254 195.123.212.149-195.123.212.149 195.123.213.169-195.123.213.169 195.123.227.20-195.123.227.20 195.123.245.214-195.123.245.214 195.123.245.90-195.123.245.90 199.21.106.189-199.21.106.189 202.63.242.48-202.63.242.48 204.95.99.204-204.95.99.204 209.58.186.245-209.58.186.245 212.47.194.15-212.47.194.15 212.73.150.215-212.73.150.215 213.152.161.138-213.152.161.138 24.217.192.131-24.217.192.131 24.247.181.155-24.247.181.155 24.247.182.169-24.247.182.169 24.247.182.240-24.247.182.240 24.247.182.253-24.247.182.253 31.171.152.103-31.171.152.103 31.171.152.105-31.171.152.105 31.171.152.106-31.171.152.106 31.171.152.107-31.171.152.107 31.7.188.40-31.7.188.40 35.198.61.54-35.198.61.54 37.59.134.55-37.59.134.55 45.249.90.124-45.249.90.124 45.55.36.231-45.55.36.231 46.166.173.109-46.166.173.109 46.17.45.29-46.17.45.29 46.17.47.216-46.17.47.216 46.183.223.10-46.183.223.10 47.44.54.70-47.44.54.70 5.2.64.188-5.2.64.188 5.2.67.66-5.2.67.66 5.206.225.115-5.206.225.115 5.8.88.125-5.8.88.125 54.37.86.44-54.37.86.44 54.38.146.43-54.38.146.43 68.111.123.100-68.111.123.100 68.183.249.84-68.183.249.84 72.189.124.41-72.189.124.41 76.107.90.235-76.107.90.235 78.155.220.198-78.155.220.198 81.177.141.211-81.177.141.211 81.177.180.174-81.177.180.174 82.199.134.139-82.199.134.139 82.199.134.156-82.199.134.156 83.166.245.213-83.166.245.213 85.217.170.62-85.217.170.62 87.236.22.142-87.236.22.142 91.192.100.16-91.192.100.16 91.192.100.27-91.192.100.27 91.192.100.3-91.192.100.3 91.192.100.40-91.192.100.40 91.192.100.44-91.192.100.44 91.192.100.48-91.192.100.48 91.192.100.52-91.192.100.52 92.222.10.99-92.222.10.99 93.115.26.171-93.115.26.171 94.103.83.137-94.103.83.137 94.185.86.56-94.185.86.56 94.237.44.31-94.237.44.31 95.213.251.165-95.213.251.165 95.47.161.68-95.47.161.68 97.87.175.152-97.87.175.152 ================================================ FILE: tests/integration/basic/README.md ================================================ # Simple script to exercise and test feed output formats. ## Usage ``` export MM_URL=https://192.168.55.169 export MM_USERNAME=admin export MM_PASSWORD=minemeld ./test.py ``` ## Warning URL.lst contains a list of malicious URLs, do not click ================================================ FILE: tests/integration/basic/URL.lst ================================================ http://7-eleven-handbags.com/X1rZYp.php http://8vs.com/6jezbr.php http://abdal.com.ua/7_jzay.php http://aditaborai.com.br/WgNGXe.php http://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php http://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php http://allstarpaintbody.com/lrQ2bG.php http://americancorner.udp.cl/etloxW.php http://ample-sun.eu/4BKEt7.php http://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php http://anoukdelecluse.nl/lGZLB1.php http://appeum.com/wp-content/themes/cc.php http://arot.altervista.org/KHTUdq.php http://arttoday.sk/mE8MKJ.php http://ascortimisoara.ro/kWIH5V.php http://aspectdesigns.com.au/0rTVlG.php http://audetlaw.com/LnVAdF.php http://autohaus-seevetal.com/9x6UwK.php http://avancarvisual.com.br/wp-content/themes/twentytwelve/VzkgnX.php http://babylicious.ie/s1GHUZ.php http://balkanium.altervista.org/p3er4s.php http://beachhouseplans.com/wp-admin/js/5d8gMe.php http://best-service.jp/olxu2Y.php http://beyondthedog.net/edHDvf.php http://bigboattravel.com/uploads/3/5/4/5/3545341/header_images/NthjHz.php http://bisofit.com/QXwm4I.php http://bktrade.kiev.ua/76b3ZQ.php http://boilersandfurnaces.com/uploads/3/5/1/6/3516773/RPyH2q.php http://bolizarsospos.com/0l0vp1va6b2 http://bolizarsospos.com/1cslstk2qv121 http://bolizarsospos.com/1xb81c28qs2db http://bolizarsospos.com/22o1210hbpw http://bolizarsospos.com/2h6t511wpuvnych http://bolizarsospos.com/379gz635s3j946 http://bolizarsospos.com/4kpy8ju42x137 http://bolizarsospos.com/503qu7boexyk http://bolizarsospos.com/574xl5yme0gdz http://bolizarsospos.com/5gpf7ecxhf http://bolizarsospos.com/5hmwl5qvpz2f3gc http://bolizarsospos.com/6m50uk8ty1031 http://bolizarsospos.com/6tvpgu93q4wx5t http://bolizarsospos.com/703hjdr3ez72 http://bolizarsospos.com/73075bdj8meb http://bolizarsospos.com/7gr904pzv6 http://bolizarsospos.com/7ms68qsdfj0jt http://bolizarsospos.com/89e8f40k8zcn38 http://bolizarsospos.com/8eo5zwhh4zndwwa http://bolizarsospos.com/8tsdhjccoxz6c http://bolizarsospos.com/94g2mr36b4 http://bolizarsospos.com/9bqdnk2h58ty2l http://bolizarsospos.com/9hul78mtg1n63 http://bolizarsospos.com/b0slgvfxvyf http://bolizarsospos.com/b3amhlkiar2c http://bolizarsospos.com/b8g7g560612 http://bolizarsospos.com/bo5ha9ild1zjukv http://bolizarsospos.com/cannzqzrum14o4c http://bolizarsospos.com/ch3eq62ad8k http://bolizarsospos.com/ci72o4ruf2y87 http://bolizarsospos.com/d5i52z8cgv5 http://bolizarsospos.com/d65v4fx21f http://bolizarsospos.com/d7jly5f09tqj http://bolizarsospos.com/di53su4z7uqvj http://bolizarsospos.com/dypi31624z http://bolizarsospos.com/e5tkclwq9w0 http://bolizarsospos.com/e887nn5k9pb6 http://bolizarsospos.com/f1s0y87wrwo http://bolizarsospos.com/fgivit1drjuh http://bolizarsospos.com/fpkirizbrzxc5 http://bolizarsospos.com/fxoztyxp320q http://bolizarsospos.com/g5k4uvxghygg7r http://bolizarsospos.com/gvi00me81aabu http://bolizarsospos.com/hq5drme48h http://bolizarsospos.com/hzpz767vze9 http://bolizarsospos.com/ifkhfc5369az88 http://bolizarsospos.com/iijoama0ynrowtp http://bolizarsospos.com/j4hzoz8cgdeza http://bolizarsospos.com/kka7641ov7 http://bolizarsospos.com/ko679ybid6ys58 http://bolizarsospos.com/kxdmlkhmuyf9 http://bolizarsospos.com/kzqnheutxkjwhr http://bolizarsospos.com/mi5b67bilrfu http://bolizarsospos.com/n2csus3eo1tyg http://bolizarsospos.com/nve4m67l83 http://bolizarsospos.com/o0nyjlre41o3 http://bolizarsospos.com/pgjcokoi2kisu http://bolizarsospos.com/pl36lz43r6r7 http://bolizarsospos.com/q3xryv3mh1 http://bolizarsospos.com/qely217wcjdl7b http://bolizarsospos.com/qo9ux20lo1 http://bolizarsospos.com/qu9ajlxsiw http://bolizarsospos.com/r45byxsjhz http://bolizarsospos.com/raph9xccgxt http://bolizarsospos.com/rb05hez1r044 http://bolizarsospos.com/rdjg0eb5r0qs http://bolizarsospos.com/ri86nx23dhqbmch http://bolizarsospos.com/rjotoddb4n7hl http://bolizarsospos.com/rof06587c1x2y3t http://bolizarsospos.com/s40o542jt7v http://bolizarsospos.com/sb2zarf5vy http://bolizarsospos.com/uamuxps7y98 http://bolizarsospos.com/uiyi9dkf5bs http://bolizarsospos.com/v13rw8n8w2 http://bolizarsospos.com/vzum6ywdedxjtd http://bolizarsospos.com/walqb5xzunmr http://bolizarsospos.com/wilqkaz24rnqli http://bolizarsospos.com/x1tg111bara5 http://bolizarsospos.com/x753k2s01gnd5b http://bolizarsospos.com/x7lfazpjuuiel http://bolizarsospos.com/xgw1o6gt9h8k9g http://bolizarsospos.com/xjp3zmw6glginuq http://bolizarsospos.com/yias364ajr http://bolizarsospos.com/zyayxp2kpay http://bucksmedia.go2cloud.org/aff_c http://building.msu.ac.th/q3Bslr.php http://businessaviators.com/r1doyF.php http://challengestrata.com.au/fP_BXS.php http://cheapshirts.us/zVnMrG.php http://chong.joelle.free.fr/_L43PH.php http://connectao.com/wp-content/themes/twentyeleven/cc.php http://*.feyda.net/hOeDr4.php http://d3mpd.fe.uns.ac.id/XPgmur.php http://daffamedia.com/wp-content/plugins/wp_module/img5.php http://dechehang.com/GZ2QRn.php http://definitionen.de/v7GVES.php http://dichiro.com/WaIrd6.php http://dillardvideo.com/wp-admin/network/2.php http://dining-bar.com/BQ_Ln4.php http://domaine-cassillac.com/4q3esU.php http://double-wing.de/DZkCLR.php http://drdigitalmd.com/img1.php http://eatside.es/xZQGXV.php http://ecocalsots.com/N79GTA.php http://ecolux-comfort.com/nPAbsy.php http://elcoachingempresarial.com/wp-admin/user/2.php http://emprende21.es/oTIq7A.php http://estudiobarco.com.ar/5TFv7E.php http://event-travel.co.uk/3K6Psd.php http://feuerwehr-stadt-riesa.de/UFiPOq.php http://foundersomaha.net/wp-includes/Text/Diff/Renderer/ap3.php http://frame3d.de/ItGJKd.php http://fun-pop.com/Ks1rCc.php http://genedillardart.com/wp-admin/network/3.php http://gibdd.ws/J7D65p.php http://glitchygaming.com/r07QZu.php http://grochowina.net/UnvPso.php http://haarsaloncindy.nl/XzF03r.php http://hamilton150.co.nz/LmfuMZ.php http://icsot.na.its.ac.id/8vwRUX.php http://igatha.com/h4MeKJ.php http://ilovesport.kiev.ua/z8X9T7.php http://imagescameraclub.com/j7b5kK.php http://inspirenetworks.in/vAqu3L.php http://italyprego.com/Lf2dcA.php http://jadwalpialadunia.in/rG4Rdi.php http://jambola.com/LuylWV.php http://jauregia.net/img5.php http://konyavakfi.nl/Zmje1r.php http://kuruyaprak.com/OTLuKo.php http://kvnysoho.com/eHafFT.php http://lazymoosestamping.com/YRfbgB.php http://london-escorts-agency.org.uk/fdnmyD.php http://lzclient.com/img4.php http://madisonbootcamps.com/gWQ3wp.php http://mangohills.net/RxIoCE.php http://marciogerhardtsouza.com.br/mPCsDz.php http://marcortes.com/img5.php http://markossolomon.com/F1q7QX.php http://maternalserenity.co.uk/I_NwPg.php http://millsmanagement.nl/AnOgVK.php http://mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php http://naimselmonaj.com/QoYx31.php http://nonnuoccaobang.com/BRdoDL.php http://nupleta.com.br/KoHV09.php http://openroadsolutions.com/FJ2dOw.php http://oregonreversemortgage.com/Rwafp_.php http://p237996.mybestmv.com/adServe/domainClick http://paintituppottery.com/6cmeb2.php http://patrianossa.com.br/u8LkzD.php http://portalmaismidia.com.br/tnSmIb.php http://portret-tekening.nl/mNQVts.php http://procrediti.com.ua/d6yGOX.php http://rajsima87.com/img2.php http://recaswine.ro/dXlq0Y.php http://silstop.pl/Si0cCJ.php http://smartnote.co/2NxVzA.php http://studiolegalecsb.it/iQcNfC.php http://takaram.ir/gjOREZ.php http://takatei.com/rfYI4L.php http://tcblog.de/mXdVTh.php http://theassemblyguy.co.nz/vpFAbQ.php http://trion.com.ph/jdKAap.php http://tusrecetas.net/JbElN7.php http://tutorialswalk.info/wp-content/themes/Defne/img2.php http://viralcrazies.com/iFHt4C.php http://vsedveri-33.ru/ http://weberteam.hu/WCTdO5.php http://www.001edizioni.com/NZwt_a.php http://www.bishopbell.co.uk/enRmcC.php http://www.chemes.eu/wp-content/themes/decoy2/redux-framework/ReduxCore/inc/fields/info/2.php http://www.decorandoimoveis.com/QEO5yh.php http://www.feddoctor.com/Oe1LMr.php http://www.gjscomputerservices.com.au/S1_rvm.php http://www.granmarquise.com.br/6f_8ei.php http://www.hanecaklaw.com/ http://www.hanoiguidedtours.com/iQ2q1f.php http://www.healthstafftravel.com.au/oyBbUs.php http://www.plexipr.com/vAHzWX.php http://www.rippedknees.co.uk/TXmJcq.php http://www.taoblu.com/wp-content/plugins/wp_module/sbML0j.php http://www.vishvagujarat.com/5of9dt.php http://www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkG.php *abc.example.com/foobar?a=:80 https://*abc.*test.com http://www.example.com:1000 ================================================ FILE: tests/integration/basic/URLHC%3Fs%3D5%26n%3D10.result ================================================ http://aditaborai.com.br/WgNGXe.php http://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php http://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php http://allstarpaintbody.com/lrQ2bG.php http://americancorner.udp.cl/etloxW.php http://ample-sun.eu/4BKEt7.php http://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php http://anoukdelecluse.nl/lGZLB1.php http://appeum.com/wp-content/themes/cc.php http://arot.altervista.org/KHTUdq.php ================================================ FILE: tests/integration/basic/URLHC%3Fv%3Dbluecoat%26cd%3Dtest.result ================================================ define category test *.example.com/foobar?a=:80 *.feyda.net/hoedr4.php 7-eleven-handbags.com/x1rzyp.php 8vs.com/6jezbr.php abdal.com.ua/7_jzay.php aditaborai.com.br/wgngxe.php airconditioning12601.com/uploads/3/5/7/6/3576233/v5k3za.php allreadytravel.com/uploads/3/5/4/9/3549731/header_images/tomae1.php allstarpaintbody.com/lrq2bg.php americancorner.udp.cl/etloxw.php ample-sun.eu/4bket7.php anime-tuner.square7.ch/wp-content/themes/twentyeleven/mstgk_.php anoukdelecluse.nl/lgzlb1.php appeum.com/wp-content/themes/cc.php arot.altervista.org/khtudq.php arttoday.sk/me8mkj.php ascortimisoara.ro/kwih5v.php aspectdesigns.com.au/0rtvlg.php audetlaw.com/lnvadf.php autohaus-seevetal.com/9x6uwk.php avancarvisual.com.br/wp-content/themes/twentytwelve/vzkgnx.php babylicious.ie/s1ghuz.php balkanium.altervista.org/p3er4s.php beachhouseplans.com/wp-admin/js/5d8gme.php best-service.jp/olxu2y.php beyondthedog.net/edhdvf.php bigboattravel.com/uploads/3/5/4/5/3545341/header_images/nthjhz.php bisofit.com/qxwm4i.php bktrade.kiev.ua/76b3zq.php boilersandfurnaces.com/uploads/3/5/1/6/3516773/rpyh2q.php bolizarsospos.com/0l0vp1va6b2 bolizarsospos.com/1cslstk2qv121 bolizarsospos.com/1xb81c28qs2db bolizarsospos.com/22o1210hbpw bolizarsospos.com/2h6t511wpuvnych bolizarsospos.com/379gz635s3j946 bolizarsospos.com/4kpy8ju42x137 bolizarsospos.com/503qu7boexyk bolizarsospos.com/574xl5yme0gdz bolizarsospos.com/5gpf7ecxhf bolizarsospos.com/5hmwl5qvpz2f3gc bolizarsospos.com/6m50uk8ty1031 bolizarsospos.com/6tvpgu93q4wx5t bolizarsospos.com/703hjdr3ez72 bolizarsospos.com/73075bdj8meb bolizarsospos.com/7gr904pzv6 bolizarsospos.com/7ms68qsdfj0jt bolizarsospos.com/89e8f40k8zcn38 bolizarsospos.com/8eo5zwhh4zndwwa bolizarsospos.com/8tsdhjccoxz6c bolizarsospos.com/94g2mr36b4 bolizarsospos.com/9bqdnk2h58ty2l bolizarsospos.com/9hul78mtg1n63 bolizarsospos.com/b0slgvfxvyf bolizarsospos.com/b3amhlkiar2c bolizarsospos.com/b8g7g560612 bolizarsospos.com/bo5ha9ild1zjukv bolizarsospos.com/cannzqzrum14o4c bolizarsospos.com/ch3eq62ad8k bolizarsospos.com/ci72o4ruf2y87 bolizarsospos.com/d5i52z8cgv5 bolizarsospos.com/d65v4fx21f bolizarsospos.com/d7jly5f09tqj bolizarsospos.com/di53su4z7uqvj bolizarsospos.com/dypi31624z bolizarsospos.com/e5tkclwq9w0 bolizarsospos.com/e887nn5k9pb6 bolizarsospos.com/f1s0y87wrwo bolizarsospos.com/fgivit1drjuh bolizarsospos.com/fpkirizbrzxc5 bolizarsospos.com/fxoztyxp320q bolizarsospos.com/g5k4uvxghygg7r bolizarsospos.com/gvi00me81aabu bolizarsospos.com/hq5drme48h bolizarsospos.com/hzpz767vze9 bolizarsospos.com/ifkhfc5369az88 bolizarsospos.com/iijoama0ynrowtp bolizarsospos.com/j4hzoz8cgdeza bolizarsospos.com/kka7641ov7 bolizarsospos.com/ko679ybid6ys58 bolizarsospos.com/kxdmlkhmuyf9 bolizarsospos.com/kzqnheutxkjwhr bolizarsospos.com/mi5b67bilrfu bolizarsospos.com/n2csus3eo1tyg bolizarsospos.com/nve4m67l83 bolizarsospos.com/o0nyjlre41o3 bolizarsospos.com/pgjcokoi2kisu bolizarsospos.com/pl36lz43r6r7 bolizarsospos.com/q3xryv3mh1 bolizarsospos.com/qely217wcjdl7b bolizarsospos.com/qo9ux20lo1 bolizarsospos.com/qu9ajlxsiw bolizarsospos.com/r45byxsjhz bolizarsospos.com/raph9xccgxt bolizarsospos.com/rb05hez1r044 bolizarsospos.com/rdjg0eb5r0qs bolizarsospos.com/ri86nx23dhqbmch bolizarsospos.com/rjotoddb4n7hl bolizarsospos.com/rof06587c1x2y3t bolizarsospos.com/s40o542jt7v bolizarsospos.com/sb2zarf5vy bolizarsospos.com/uamuxps7y98 bolizarsospos.com/uiyi9dkf5bs bolizarsospos.com/v13rw8n8w2 bolizarsospos.com/vzum6ywdedxjtd bolizarsospos.com/walqb5xzunmr bolizarsospos.com/wilqkaz24rnqli bolizarsospos.com/x1tg111bara5 bolizarsospos.com/x753k2s01gnd5b bolizarsospos.com/x7lfazpjuuiel bolizarsospos.com/xgw1o6gt9h8k9g bolizarsospos.com/xjp3zmw6glginuq bolizarsospos.com/yias364ajr bolizarsospos.com/zyayxp2kpay bucksmedia.go2cloud.org/aff_c building.msu.ac.th/q3bslr.php businessaviators.com/r1doyf.php challengestrata.com.au/fp_bxs.php cheapshirts.us/zvnmrg.php chong.joelle.free.fr/_l43ph.php connectao.com/wp-content/themes/twentyeleven/cc.php d3mpd.fe.uns.ac.id/xpgmur.php daffamedia.com/wp-content/plugins/wp_module/img5.php dechehang.com/gz2qrn.php definitionen.de/v7gves.php dichiro.com/waird6.php dillardvideo.com/wp-admin/network/2.php dining-bar.com/bq_ln4.php domaine-cassillac.com/4q3esu.php double-wing.de/dzkclr.php drdigitalmd.com/img1.php eatside.es/xzqgxv.php ecocalsots.com/n79gta.php ecolux-comfort.com/npabsy.php elcoachingempresarial.com/wp-admin/user/2.php emprende21.es/otiq7a.php estudiobarco.com.ar/5tfv7e.php event-travel.co.uk/3k6psd.php feuerwehr-stadt-riesa.de/ufipoq.php foundersomaha.net/wp-includes/text/diff/renderer/ap3.php frame3d.de/itgjkd.php fun-pop.com/ks1rcc.php genedillardart.com/wp-admin/network/3.php gibdd.ws/j7d65p.php glitchygaming.com/r07qzu.php grochowina.net/unvpso.php haarsaloncindy.nl/xzf03r.php hamilton150.co.nz/lmfumz.php icsot.na.its.ac.id/8vwrux.php igatha.com/h4mekj.php ilovesport.kiev.ua/z8x9t7.php imagescameraclub.com/j7b5kk.php inspirenetworks.in/vaqu3l.php italyprego.com/lf2dca.php jadwalpialadunia.in/rg4rdi.php jambola.com/luylwv.php jauregia.net/img5.php konyavakfi.nl/zmje1r.php kuruyaprak.com/otluko.php kvnysoho.com/ehafft.php lazymoosestamping.com/yrfbgb.php london-escorts-agency.org.uk/fdnmyd.php lzclient.com/img4.php madisonbootcamps.com/gwq3wp.php mangohills.net/rxioce.php marciogerhardtsouza.com.br/mpcsdz.php marcortes.com/img5.php markossolomon.com/f1q7qx.php maternalserenity.co.uk/i_nwpg.php millsmanagement.nl/anogvk.php mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php naimselmonaj.com/qoyx31.php nonnuoccaobang.com/brdodl.php nupleta.com.br/kohv09.php openroadsolutions.com/fj2dow.php oregonreversemortgage.com/rwafp_.php p237996.mybestmv.com/adserve/domainclick paintituppottery.com/6cmeb2.php patrianossa.com.br/u8lkzd.php portalmaismidia.com.br/tnsmib.php portret-tekening.nl/mnqvts.php procrediti.com.ua/d6ygox.php rajsima87.com/img2.php recaswine.ro/dxlq0y.php silstop.pl/si0ccj.php smartnote.co/2nxvza.php studiolegalecsb.it/iqcnfc.php takaram.ir/gjorez.php takatei.com/rfyi4l.php tcblog.de/mxdvth.php theassemblyguy.co.nz/vpfabq.php trion.com.ph/jdkaap.php tusrecetas.net/jbeln7.php tutorialswalk.info/wp-content/themes/defne/img2.php viralcrazies.com/ifht4c.php vsedveri-33.ru/ weberteam.hu/wctdo5.php www.001edizioni.com/nzwt_a.php www.bishopbell.co.uk/enrmcc.php www.chemes.eu/wp-content/themes/decoy2/redux-framework/reduxcore/inc/fields/info/2.php www.decorandoimoveis.com/qeo5yh.php www.example.com:1000 www.feddoctor.com/oe1lmr.php www.gjscomputerservices.com.au/s1_rvm.php www.granmarquise.com.br/6f_8ei.php www.hanecaklaw.com/ www.hanoiguidedtours.com/iq2q1f.php www.healthstafftravel.com.au/oybbus.php www.plexipr.com/vahzwx.php www.rippedknees.co.uk/txmjcq.php www.taoblu.com/wp-content/plugins/wp_module/sbml0j.php www.vishvagujarat.com/5of9dt.php www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkg.php *.*.com end ================================================ FILE: tests/integration/basic/URLHC%3Fv%3Dbluecoat.result ================================================ ================================================ FILE: tests/integration/basic/URLHC%3Fv%3Dcsv%26f%3Dconfidence%26f%3Dsources%7Cfeeds%26f%3Dindicator%7Curl.result ================================================ confidence,feeds,url 100,localdb,*abc.example.com/foobar?a=:80 100,localdb,http://*.feyda.net/hOeDr4.php 100,localdb,http://7-eleven-handbags.com/X1rZYp.php 100,localdb,http://8vs.com/6jezbr.php 100,localdb,http://abdal.com.ua/7_jzay.php 100,localdb,http://aditaborai.com.br/WgNGXe.php 100,localdb,http://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php 100,localdb,http://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php 100,localdb,http://allstarpaintbody.com/lrQ2bG.php 100,localdb,http://americancorner.udp.cl/etloxW.php 100,localdb,http://ample-sun.eu/4BKEt7.php 100,localdb,http://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php 100,localdb,http://anoukdelecluse.nl/lGZLB1.php 100,localdb,http://appeum.com/wp-content/themes/cc.php 100,localdb,http://arot.altervista.org/KHTUdq.php 100,localdb,http://arttoday.sk/mE8MKJ.php 100,localdb,http://ascortimisoara.ro/kWIH5V.php 100,localdb,http://aspectdesigns.com.au/0rTVlG.php 100,localdb,http://audetlaw.com/LnVAdF.php 100,localdb,http://autohaus-seevetal.com/9x6UwK.php 100,localdb,http://avancarvisual.com.br/wp-content/themes/twentytwelve/VzkgnX.php 100,localdb,http://babylicious.ie/s1GHUZ.php 100,localdb,http://balkanium.altervista.org/p3er4s.php 100,localdb,http://beachhouseplans.com/wp-admin/js/5d8gMe.php 100,localdb,http://best-service.jp/olxu2Y.php 100,localdb,http://beyondthedog.net/edHDvf.php 100,localdb,http://bigboattravel.com/uploads/3/5/4/5/3545341/header_images/NthjHz.php 100,localdb,http://bisofit.com/QXwm4I.php 100,localdb,http://bktrade.kiev.ua/76b3ZQ.php 100,localdb,http://boilersandfurnaces.com/uploads/3/5/1/6/3516773/RPyH2q.php 100,localdb,http://bolizarsospos.com/0l0vp1va6b2 100,localdb,http://bolizarsospos.com/1cslstk2qv121 100,localdb,http://bolizarsospos.com/1xb81c28qs2db 100,localdb,http://bolizarsospos.com/22o1210hbpw 100,localdb,http://bolizarsospos.com/2h6t511wpuvnych 100,localdb,http://bolizarsospos.com/379gz635s3j946 100,localdb,http://bolizarsospos.com/4kpy8ju42x137 100,localdb,http://bolizarsospos.com/503qu7boexyk 100,localdb,http://bolizarsospos.com/574xl5yme0gdz 100,localdb,http://bolizarsospos.com/5gpf7ecxhf 100,localdb,http://bolizarsospos.com/5hmwl5qvpz2f3gc 100,localdb,http://bolizarsospos.com/6m50uk8ty1031 100,localdb,http://bolizarsospos.com/6tvpgu93q4wx5t 100,localdb,http://bolizarsospos.com/703hjdr3ez72 100,localdb,http://bolizarsospos.com/73075bdj8meb 100,localdb,http://bolizarsospos.com/7gr904pzv6 100,localdb,http://bolizarsospos.com/7ms68qsdfj0jt 100,localdb,http://bolizarsospos.com/89e8f40k8zcn38 100,localdb,http://bolizarsospos.com/8eo5zwhh4zndwwa 100,localdb,http://bolizarsospos.com/8tsdhjccoxz6c 100,localdb,http://bolizarsospos.com/94g2mr36b4 100,localdb,http://bolizarsospos.com/9bqdnk2h58ty2l 100,localdb,http://bolizarsospos.com/9hul78mtg1n63 100,localdb,http://bolizarsospos.com/b0slgvfxvyf 100,localdb,http://bolizarsospos.com/b3amhlkiar2c 100,localdb,http://bolizarsospos.com/b8g7g560612 100,localdb,http://bolizarsospos.com/bo5ha9ild1zjukv 100,localdb,http://bolizarsospos.com/cannzqzrum14o4c 100,localdb,http://bolizarsospos.com/ch3eq62ad8k 100,localdb,http://bolizarsospos.com/ci72o4ruf2y87 100,localdb,http://bolizarsospos.com/d5i52z8cgv5 100,localdb,http://bolizarsospos.com/d65v4fx21f 100,localdb,http://bolizarsospos.com/d7jly5f09tqj 100,localdb,http://bolizarsospos.com/di53su4z7uqvj 100,localdb,http://bolizarsospos.com/dypi31624z 100,localdb,http://bolizarsospos.com/e5tkclwq9w0 100,localdb,http://bolizarsospos.com/e887nn5k9pb6 100,localdb,http://bolizarsospos.com/f1s0y87wrwo 100,localdb,http://bolizarsospos.com/fgivit1drjuh 100,localdb,http://bolizarsospos.com/fpkirizbrzxc5 100,localdb,http://bolizarsospos.com/fxoztyxp320q 100,localdb,http://bolizarsospos.com/g5k4uvxghygg7r 100,localdb,http://bolizarsospos.com/gvi00me81aabu 100,localdb,http://bolizarsospos.com/hq5drme48h 100,localdb,http://bolizarsospos.com/hzpz767vze9 100,localdb,http://bolizarsospos.com/ifkhfc5369az88 100,localdb,http://bolizarsospos.com/iijoama0ynrowtp 100,localdb,http://bolizarsospos.com/j4hzoz8cgdeza 100,localdb,http://bolizarsospos.com/kka7641ov7 100,localdb,http://bolizarsospos.com/ko679ybid6ys58 100,localdb,http://bolizarsospos.com/kxdmlkhmuyf9 100,localdb,http://bolizarsospos.com/kzqnheutxkjwhr 100,localdb,http://bolizarsospos.com/mi5b67bilrfu 100,localdb,http://bolizarsospos.com/n2csus3eo1tyg 100,localdb,http://bolizarsospos.com/nve4m67l83 100,localdb,http://bolizarsospos.com/o0nyjlre41o3 100,localdb,http://bolizarsospos.com/pgjcokoi2kisu 100,localdb,http://bolizarsospos.com/pl36lz43r6r7 100,localdb,http://bolizarsospos.com/q3xryv3mh1 100,localdb,http://bolizarsospos.com/qely217wcjdl7b 100,localdb,http://bolizarsospos.com/qo9ux20lo1 100,localdb,http://bolizarsospos.com/qu9ajlxsiw 100,localdb,http://bolizarsospos.com/r45byxsjhz 100,localdb,http://bolizarsospos.com/raph9xccgxt 100,localdb,http://bolizarsospos.com/rb05hez1r044 100,localdb,http://bolizarsospos.com/rdjg0eb5r0qs 100,localdb,http://bolizarsospos.com/ri86nx23dhqbmch 100,localdb,http://bolizarsospos.com/rjotoddb4n7hl 100,localdb,http://bolizarsospos.com/rof06587c1x2y3t 100,localdb,http://bolizarsospos.com/s40o542jt7v 100,localdb,http://bolizarsospos.com/sb2zarf5vy 100,localdb,http://bolizarsospos.com/uamuxps7y98 100,localdb,http://bolizarsospos.com/uiyi9dkf5bs 100,localdb,http://bolizarsospos.com/v13rw8n8w2 100,localdb,http://bolizarsospos.com/vzum6ywdedxjtd 100,localdb,http://bolizarsospos.com/walqb5xzunmr 100,localdb,http://bolizarsospos.com/wilqkaz24rnqli 100,localdb,http://bolizarsospos.com/x1tg111bara5 100,localdb,http://bolizarsospos.com/x753k2s01gnd5b 100,localdb,http://bolizarsospos.com/x7lfazpjuuiel 100,localdb,http://bolizarsospos.com/xgw1o6gt9h8k9g 100,localdb,http://bolizarsospos.com/xjp3zmw6glginuq 100,localdb,http://bolizarsospos.com/yias364ajr 100,localdb,http://bolizarsospos.com/zyayxp2kpay 100,localdb,http://bucksmedia.go2cloud.org/aff_c 100,localdb,http://building.msu.ac.th/q3Bslr.php 100,localdb,http://businessaviators.com/r1doyF.php 100,localdb,http://challengestrata.com.au/fP_BXS.php 100,localdb,http://cheapshirts.us/zVnMrG.php 100,localdb,http://chong.joelle.free.fr/_L43PH.php 100,localdb,http://connectao.com/wp-content/themes/twentyeleven/cc.php 100,localdb,http://d3mpd.fe.uns.ac.id/XPgmur.php 100,localdb,http://daffamedia.com/wp-content/plugins/wp_module/img5.php 100,localdb,http://dechehang.com/GZ2QRn.php 100,localdb,http://definitionen.de/v7GVES.php 100,localdb,http://dichiro.com/WaIrd6.php 100,localdb,http://dillardvideo.com/wp-admin/network/2.php 100,localdb,http://dining-bar.com/BQ_Ln4.php 100,localdb,http://domaine-cassillac.com/4q3esU.php 100,localdb,http://double-wing.de/DZkCLR.php 100,localdb,http://drdigitalmd.com/img1.php 100,localdb,http://eatside.es/xZQGXV.php 100,localdb,http://ecocalsots.com/N79GTA.php 100,localdb,http://ecolux-comfort.com/nPAbsy.php 100,localdb,http://elcoachingempresarial.com/wp-admin/user/2.php 100,localdb,http://emprende21.es/oTIq7A.php 100,localdb,http://estudiobarco.com.ar/5TFv7E.php 100,localdb,http://event-travel.co.uk/3K6Psd.php 100,localdb,http://feuerwehr-stadt-riesa.de/UFiPOq.php 100,localdb,http://foundersomaha.net/wp-includes/Text/Diff/Renderer/ap3.php 100,localdb,http://frame3d.de/ItGJKd.php 100,localdb,http://fun-pop.com/Ks1rCc.php 100,localdb,http://genedillardart.com/wp-admin/network/3.php 100,localdb,http://gibdd.ws/J7D65p.php 100,localdb,http://glitchygaming.com/r07QZu.php 100,localdb,http://grochowina.net/UnvPso.php 100,localdb,http://haarsaloncindy.nl/XzF03r.php 100,localdb,http://hamilton150.co.nz/LmfuMZ.php 100,localdb,http://icsot.na.its.ac.id/8vwRUX.php 100,localdb,http://igatha.com/h4MeKJ.php 100,localdb,http://ilovesport.kiev.ua/z8X9T7.php 100,localdb,http://imagescameraclub.com/j7b5kK.php 100,localdb,http://inspirenetworks.in/vAqu3L.php 100,localdb,http://italyprego.com/Lf2dcA.php 100,localdb,http://jadwalpialadunia.in/rG4Rdi.php 100,localdb,http://jambola.com/LuylWV.php 100,localdb,http://jauregia.net/img5.php 100,localdb,http://konyavakfi.nl/Zmje1r.php 100,localdb,http://kuruyaprak.com/OTLuKo.php 100,localdb,http://kvnysoho.com/eHafFT.php 100,localdb,http://lazymoosestamping.com/YRfbgB.php 100,localdb,http://london-escorts-agency.org.uk/fdnmyD.php 100,localdb,http://lzclient.com/img4.php 100,localdb,http://madisonbootcamps.com/gWQ3wp.php 100,localdb,http://mangohills.net/RxIoCE.php 100,localdb,http://marciogerhardtsouza.com.br/mPCsDz.php 100,localdb,http://marcortes.com/img5.php 100,localdb,http://markossolomon.com/F1q7QX.php 100,localdb,http://maternalserenity.co.uk/I_NwPg.php 100,localdb,http://millsmanagement.nl/AnOgVK.php 100,localdb,http://mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php 100,localdb,http://naimselmonaj.com/QoYx31.php 100,localdb,http://nonnuoccaobang.com/BRdoDL.php 100,localdb,http://nupleta.com.br/KoHV09.php 100,localdb,http://openroadsolutions.com/FJ2dOw.php 100,localdb,http://oregonreversemortgage.com/Rwafp_.php 100,localdb,http://p237996.mybestmv.com/adServe/domainClick 100,localdb,http://paintituppottery.com/6cmeb2.php 100,localdb,http://patrianossa.com.br/u8LkzD.php 100,localdb,http://portalmaismidia.com.br/tnSmIb.php 100,localdb,http://portret-tekening.nl/mNQVts.php 100,localdb,http://procrediti.com.ua/d6yGOX.php 100,localdb,http://rajsima87.com/img2.php 100,localdb,http://recaswine.ro/dXlq0Y.php 100,localdb,http://silstop.pl/Si0cCJ.php 100,localdb,http://smartnote.co/2NxVzA.php 100,localdb,http://studiolegalecsb.it/iQcNfC.php 100,localdb,http://takaram.ir/gjOREZ.php 100,localdb,http://takatei.com/rfYI4L.php 100,localdb,http://tcblog.de/mXdVTh.php 100,localdb,http://theassemblyguy.co.nz/vpFAbQ.php 100,localdb,http://trion.com.ph/jdKAap.php 100,localdb,http://tusrecetas.net/JbElN7.php 100,localdb,http://tutorialswalk.info/wp-content/themes/Defne/img2.php 100,localdb,http://viralcrazies.com/iFHt4C.php 100,localdb,http://vsedveri-33.ru/ 100,localdb,http://weberteam.hu/WCTdO5.php 100,localdb,http://www.001edizioni.com/NZwt_a.php 100,localdb,http://www.bishopbell.co.uk/enRmcC.php 100,localdb,http://www.chemes.eu/wp-content/themes/decoy2/redux-framework/ReduxCore/inc/fields/info/2.php 100,localdb,http://www.decorandoimoveis.com/QEO5yh.php 100,localdb,http://www.example.com:1000 100,localdb,http://www.feddoctor.com/Oe1LMr.php 100,localdb,http://www.gjscomputerservices.com.au/S1_rvm.php 100,localdb,http://www.granmarquise.com.br/6f_8ei.php 100,localdb,http://www.hanecaklaw.com/ 100,localdb,http://www.hanoiguidedtours.com/iQ2q1f.php 100,localdb,http://www.healthstafftravel.com.au/oyBbUs.php 100,localdb,http://www.plexipr.com/vAHzWX.php 100,localdb,http://www.rippedknees.co.uk/TXmJcq.php 100,localdb,http://www.taoblu.com/wp-content/plugins/wp_module/sbML0j.php 100,localdb,http://www.vishvagujarat.com/5of9dt.php 100,localdb,http://www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkG.php 100,localdb,https://*abc.*test.com ================================================ FILE: tests/integration/basic/URLHC%3Fv%3Djson%26tr%3D1.result ================================================ [ {"indicator":"*abc.example.com/foobar?a=:80","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://*.feyda.net/hOeDr4.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://7-eleven-handbags.com/X1rZYp.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://8vs.com/6jezbr.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://abdal.com.ua/7_jzay.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://aditaborai.com.br/WgNGXe.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://allstarpaintbody.com/lrQ2bG.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://americancorner.udp.cl/etloxW.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://ample-sun.eu/4BKEt7.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://anoukdelecluse.nl/lGZLB1.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://appeum.com/wp-content/themes/cc.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://arot.altervista.org/KHTUdq.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://arttoday.sk/mE8MKJ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://ascortimisoara.ro/kWIH5V.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://aspectdesigns.com.au/0rTVlG.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://audetlaw.com/LnVAdF.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://autohaus-seevetal.com/9x6UwK.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://avancarvisual.com.br/wp-content/themes/twentytwelve/VzkgnX.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://babylicious.ie/s1GHUZ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://balkanium.altervista.org/p3er4s.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://beachhouseplans.com/wp-admin/js/5d8gMe.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://best-service.jp/olxu2Y.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://beyondthedog.net/edHDvf.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bigboattravel.com/uploads/3/5/4/5/3545341/header_images/NthjHz.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bisofit.com/QXwm4I.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bktrade.kiev.ua/76b3ZQ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://boilersandfurnaces.com/uploads/3/5/1/6/3516773/RPyH2q.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/0l0vp1va6b2","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/1cslstk2qv121","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/1xb81c28qs2db","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/22o1210hbpw","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/2h6t511wpuvnych","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/379gz635s3j946","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/4kpy8ju42x137","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/503qu7boexyk","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/574xl5yme0gdz","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/5gpf7ecxhf","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/5hmwl5qvpz2f3gc","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/6m50uk8ty1031","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/6tvpgu93q4wx5t","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/703hjdr3ez72","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/73075bdj8meb","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/7gr904pzv6","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/7ms68qsdfj0jt","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/89e8f40k8zcn38","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/8eo5zwhh4zndwwa","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/8tsdhjccoxz6c","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/94g2mr36b4","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/9bqdnk2h58ty2l","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/9hul78mtg1n63","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/b0slgvfxvyf","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/b3amhlkiar2c","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/b8g7g560612","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/bo5ha9ild1zjukv","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/cannzqzrum14o4c","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/ch3eq62ad8k","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/ci72o4ruf2y87","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/d5i52z8cgv5","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/d65v4fx21f","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/d7jly5f09tqj","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/di53su4z7uqvj","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/dypi31624z","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/e5tkclwq9w0","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/e887nn5k9pb6","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/f1s0y87wrwo","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/fgivit1drjuh","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/fpkirizbrzxc5","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/fxoztyxp320q","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/g5k4uvxghygg7r","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/gvi00me81aabu","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/hq5drme48h","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/hzpz767vze9","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/ifkhfc5369az88","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/iijoama0ynrowtp","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/j4hzoz8cgdeza","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/kka7641ov7","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/ko679ybid6ys58","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/kxdmlkhmuyf9","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/kzqnheutxkjwhr","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/mi5b67bilrfu","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/n2csus3eo1tyg","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/nve4m67l83","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/o0nyjlre41o3","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/pgjcokoi2kisu","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/pl36lz43r6r7","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/q3xryv3mh1","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/qely217wcjdl7b","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/qo9ux20lo1","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/qu9ajlxsiw","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/r45byxsjhz","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/raph9xccgxt","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/rb05hez1r044","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/rdjg0eb5r0qs","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/ri86nx23dhqbmch","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/rjotoddb4n7hl","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/rof06587c1x2y3t","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/s40o542jt7v","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/sb2zarf5vy","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/uamuxps7y98","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/uiyi9dkf5bs","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/v13rw8n8w2","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/vzum6ywdedxjtd","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/walqb5xzunmr","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/wilqkaz24rnqli","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/x1tg111bara5","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/x753k2s01gnd5b","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/x7lfazpjuuiel","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/xgw1o6gt9h8k9g","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/xjp3zmw6glginuq","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/yias364ajr","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bolizarsospos.com/zyayxp2kpay","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://bucksmedia.go2cloud.org/aff_c","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://building.msu.ac.th/q3Bslr.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://businessaviators.com/r1doyF.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://challengestrata.com.au/fP_BXS.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://cheapshirts.us/zVnMrG.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://chong.joelle.free.fr/_L43PH.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://connectao.com/wp-content/themes/twentyeleven/cc.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://d3mpd.fe.uns.ac.id/XPgmur.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://daffamedia.com/wp-content/plugins/wp_module/img5.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://dechehang.com/GZ2QRn.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://definitionen.de/v7GVES.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://dichiro.com/WaIrd6.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://dillardvideo.com/wp-admin/network/2.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://dining-bar.com/BQ_Ln4.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://domaine-cassillac.com/4q3esU.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://double-wing.de/DZkCLR.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://drdigitalmd.com/img1.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://eatside.es/xZQGXV.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://ecocalsots.com/N79GTA.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://ecolux-comfort.com/nPAbsy.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://elcoachingempresarial.com/wp-admin/user/2.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://emprende21.es/oTIq7A.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://estudiobarco.com.ar/5TFv7E.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://event-travel.co.uk/3K6Psd.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://feuerwehr-stadt-riesa.de/UFiPOq.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://foundersomaha.net/wp-includes/Text/Diff/Renderer/ap3.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://frame3d.de/ItGJKd.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://fun-pop.com/Ks1rCc.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://genedillardart.com/wp-admin/network/3.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://gibdd.ws/J7D65p.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://glitchygaming.com/r07QZu.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://grochowina.net/UnvPso.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://haarsaloncindy.nl/XzF03r.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://hamilton150.co.nz/LmfuMZ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://icsot.na.its.ac.id/8vwRUX.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://igatha.com/h4MeKJ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://ilovesport.kiev.ua/z8X9T7.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://imagescameraclub.com/j7b5kK.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://inspirenetworks.in/vAqu3L.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://italyprego.com/Lf2dcA.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://jadwalpialadunia.in/rG4Rdi.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://jambola.com/LuylWV.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://jauregia.net/img5.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://konyavakfi.nl/Zmje1r.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://kuruyaprak.com/OTLuKo.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://kvnysoho.com/eHafFT.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://lazymoosestamping.com/YRfbgB.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://london-escorts-agency.org.uk/fdnmyD.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://lzclient.com/img4.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://madisonbootcamps.com/gWQ3wp.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://mangohills.net/RxIoCE.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://marciogerhardtsouza.com.br/mPCsDz.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://marcortes.com/img5.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://markossolomon.com/F1q7QX.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://maternalserenity.co.uk/I_NwPg.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://millsmanagement.nl/AnOgVK.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://naimselmonaj.com/QoYx31.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://nonnuoccaobang.com/BRdoDL.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://nupleta.com.br/KoHV09.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://openroadsolutions.com/FJ2dOw.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://oregonreversemortgage.com/Rwafp_.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://p237996.mybestmv.com/adServe/domainClick","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://paintituppottery.com/6cmeb2.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://patrianossa.com.br/u8LkzD.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://portalmaismidia.com.br/tnSmIb.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://portret-tekening.nl/mNQVts.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://procrediti.com.ua/d6yGOX.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://rajsima87.com/img2.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://recaswine.ro/dXlq0Y.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://silstop.pl/Si0cCJ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://smartnote.co/2NxVzA.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://studiolegalecsb.it/iQcNfC.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://takaram.ir/gjOREZ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://takatei.com/rfYI4L.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://tcblog.de/mXdVTh.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://theassemblyguy.co.nz/vpFAbQ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://trion.com.ph/jdKAap.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://tusrecetas.net/JbElN7.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://tutorialswalk.info/wp-content/themes/Defne/img2.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://viralcrazies.com/iFHt4C.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://vsedveri-33.ru/","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://weberteam.hu/WCTdO5.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.001edizioni.com/NZwt_a.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.bishopbell.co.uk/enRmcC.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.chemes.eu/wp-content/themes/decoy2/redux-framework/ReduxCore/inc/fields/info/2.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.decorandoimoveis.com/QEO5yh.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.example.com:1000","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.feddoctor.com/Oe1LMr.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.gjscomputerservices.com.au/S1_rvm.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.granmarquise.com.br/6f_8ei.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.hanecaklaw.com/","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.hanoiguidedtours.com/iQ2q1f.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.healthstafftravel.com.au/oyBbUs.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.plexipr.com/vAHzWX.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.rippedknees.co.uk/TXmJcq.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.taoblu.com/wp-content/plugins/wp_module/sbML0j.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.vishvagujarat.com/5of9dt.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"http://www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkG.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}, {"indicator":"https://*abc.*test.com","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}}] ================================================ FILE: tests/integration/basic/URLHC%3Fv%3Djson-seq.result ================================================ {"indicator":"*abc.example.com/foobar?a=:80","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://*.feyda.net/hOeDr4.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://7-eleven-handbags.com/X1rZYp.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://8vs.com/6jezbr.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://abdal.com.ua/7_jzay.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://aditaborai.com.br/WgNGXe.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://allstarpaintbody.com/lrQ2bG.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://americancorner.udp.cl/etloxW.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://ample-sun.eu/4BKEt7.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://anoukdelecluse.nl/lGZLB1.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://appeum.com/wp-content/themes/cc.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://arot.altervista.org/KHTUdq.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://arttoday.sk/mE8MKJ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://ascortimisoara.ro/kWIH5V.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://aspectdesigns.com.au/0rTVlG.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://audetlaw.com/LnVAdF.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://autohaus-seevetal.com/9x6UwK.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://avancarvisual.com.br/wp-content/themes/twentytwelve/VzkgnX.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://babylicious.ie/s1GHUZ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://balkanium.altervista.org/p3er4s.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://beachhouseplans.com/wp-admin/js/5d8gMe.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://best-service.jp/olxu2Y.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://beyondthedog.net/edHDvf.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bigboattravel.com/uploads/3/5/4/5/3545341/header_images/NthjHz.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bisofit.com/QXwm4I.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bktrade.kiev.ua/76b3ZQ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://boilersandfurnaces.com/uploads/3/5/1/6/3516773/RPyH2q.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/0l0vp1va6b2","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/1cslstk2qv121","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/1xb81c28qs2db","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/22o1210hbpw","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/2h6t511wpuvnych","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/379gz635s3j946","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/4kpy8ju42x137","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/503qu7boexyk","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/574xl5yme0gdz","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/5gpf7ecxhf","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/5hmwl5qvpz2f3gc","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/6m50uk8ty1031","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/6tvpgu93q4wx5t","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/703hjdr3ez72","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/73075bdj8meb","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/7gr904pzv6","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/7ms68qsdfj0jt","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/89e8f40k8zcn38","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/8eo5zwhh4zndwwa","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/8tsdhjccoxz6c","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/94g2mr36b4","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/9bqdnk2h58ty2l","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/9hul78mtg1n63","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/b0slgvfxvyf","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/b3amhlkiar2c","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/b8g7g560612","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/bo5ha9ild1zjukv","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/cannzqzrum14o4c","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/ch3eq62ad8k","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/ci72o4ruf2y87","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/d5i52z8cgv5","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/d65v4fx21f","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/d7jly5f09tqj","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/di53su4z7uqvj","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/dypi31624z","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/e5tkclwq9w0","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/e887nn5k9pb6","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/f1s0y87wrwo","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/fgivit1drjuh","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/fpkirizbrzxc5","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/fxoztyxp320q","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/g5k4uvxghygg7r","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/gvi00me81aabu","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/hq5drme48h","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/hzpz767vze9","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/ifkhfc5369az88","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/iijoama0ynrowtp","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/j4hzoz8cgdeza","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/kka7641ov7","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/ko679ybid6ys58","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/kxdmlkhmuyf9","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/kzqnheutxkjwhr","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/mi5b67bilrfu","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/n2csus3eo1tyg","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/nve4m67l83","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/o0nyjlre41o3","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/pgjcokoi2kisu","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/pl36lz43r6r7","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/q3xryv3mh1","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/qely217wcjdl7b","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/qo9ux20lo1","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/qu9ajlxsiw","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/r45byxsjhz","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/raph9xccgxt","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/rb05hez1r044","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/rdjg0eb5r0qs","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/ri86nx23dhqbmch","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/rjotoddb4n7hl","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/rof06587c1x2y3t","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/s40o542jt7v","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/sb2zarf5vy","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/uamuxps7y98","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/uiyi9dkf5bs","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/v13rw8n8w2","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/vzum6ywdedxjtd","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/walqb5xzunmr","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/wilqkaz24rnqli","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/x1tg111bara5","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/x753k2s01gnd5b","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/x7lfazpjuuiel","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/xgw1o6gt9h8k9g","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/xjp3zmw6glginuq","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/yias364ajr","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bolizarsospos.com/zyayxp2kpay","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://bucksmedia.go2cloud.org/aff_c","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://building.msu.ac.th/q3Bslr.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://businessaviators.com/r1doyF.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://challengestrata.com.au/fP_BXS.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://cheapshirts.us/zVnMrG.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://chong.joelle.free.fr/_L43PH.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://connectao.com/wp-content/themes/twentyeleven/cc.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://d3mpd.fe.uns.ac.id/XPgmur.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://daffamedia.com/wp-content/plugins/wp_module/img5.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://dechehang.com/GZ2QRn.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://definitionen.de/v7GVES.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://dichiro.com/WaIrd6.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://dillardvideo.com/wp-admin/network/2.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://dining-bar.com/BQ_Ln4.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://domaine-cassillac.com/4q3esU.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://double-wing.de/DZkCLR.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://drdigitalmd.com/img1.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://eatside.es/xZQGXV.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://ecocalsots.com/N79GTA.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://ecolux-comfort.com/nPAbsy.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://elcoachingempresarial.com/wp-admin/user/2.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://emprende21.es/oTIq7A.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://estudiobarco.com.ar/5TFv7E.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://event-travel.co.uk/3K6Psd.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://feuerwehr-stadt-riesa.de/UFiPOq.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://foundersomaha.net/wp-includes/Text/Diff/Renderer/ap3.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://frame3d.de/ItGJKd.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://fun-pop.com/Ks1rCc.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://genedillardart.com/wp-admin/network/3.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://gibdd.ws/J7D65p.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://glitchygaming.com/r07QZu.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://grochowina.net/UnvPso.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://haarsaloncindy.nl/XzF03r.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://hamilton150.co.nz/LmfuMZ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://icsot.na.its.ac.id/8vwRUX.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://igatha.com/h4MeKJ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://ilovesport.kiev.ua/z8X9T7.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://imagescameraclub.com/j7b5kK.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://inspirenetworks.in/vAqu3L.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://italyprego.com/Lf2dcA.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://jadwalpialadunia.in/rG4Rdi.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://jambola.com/LuylWV.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://jauregia.net/img5.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://konyavakfi.nl/Zmje1r.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://kuruyaprak.com/OTLuKo.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://kvnysoho.com/eHafFT.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://lazymoosestamping.com/YRfbgB.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://london-escorts-agency.org.uk/fdnmyD.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://lzclient.com/img4.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://madisonbootcamps.com/gWQ3wp.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://mangohills.net/RxIoCE.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://marciogerhardtsouza.com.br/mPCsDz.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://marcortes.com/img5.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://markossolomon.com/F1q7QX.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://maternalserenity.co.uk/I_NwPg.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://millsmanagement.nl/AnOgVK.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://naimselmonaj.com/QoYx31.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://nonnuoccaobang.com/BRdoDL.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://nupleta.com.br/KoHV09.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://openroadsolutions.com/FJ2dOw.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://oregonreversemortgage.com/Rwafp_.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://p237996.mybestmv.com/adServe/domainClick","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://paintituppottery.com/6cmeb2.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://patrianossa.com.br/u8LkzD.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://portalmaismidia.com.br/tnSmIb.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://portret-tekening.nl/mNQVts.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://procrediti.com.ua/d6yGOX.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://rajsima87.com/img2.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://recaswine.ro/dXlq0Y.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://silstop.pl/Si0cCJ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://smartnote.co/2NxVzA.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://studiolegalecsb.it/iQcNfC.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://takaram.ir/gjOREZ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://takatei.com/rfYI4L.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://tcblog.de/mXdVTh.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://theassemblyguy.co.nz/vpFAbQ.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://trion.com.ph/jdKAap.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://tusrecetas.net/JbElN7.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://tutorialswalk.info/wp-content/themes/Defne/img2.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://viralcrazies.com/iFHt4C.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://vsedveri-33.ru/","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://weberteam.hu/WCTdO5.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.001edizioni.com/NZwt_a.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.bishopbell.co.uk/enRmcC.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.chemes.eu/wp-content/themes/decoy2/redux-framework/ReduxCore/inc/fields/info/2.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.decorandoimoveis.com/QEO5yh.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.example.com:1000","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.feddoctor.com/Oe1LMr.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.gjscomputerservices.com.au/S1_rvm.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.granmarquise.com.br/6f_8ei.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.hanecaklaw.com/","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.hanoiguidedtours.com/iQ2q1f.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.healthstafftravel.com.au/oyBbUs.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.plexipr.com/vAHzWX.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.rippedknees.co.uk/TXmJcq.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.taoblu.com/wp-content/plugins/wp_module/sbML0j.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.vishvagujarat.com/5of9dt.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"http://www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkG.php","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} {"indicator":"https://*abc.*test.com","value":{"sources":["localdb"],"confidence":100,"first_seen":1561029985300,"type":"URL","share_level":"red","last_seen":1561029985300}} ================================================ FILE: tests/integration/basic/URLHC%3Fv%3Dmwg.result ================================================ type=string "*abc.example.com/foobar?a=:80" "localdb" "http://*.feyda.net/hOeDr4.php" "localdb" "http://7-eleven-handbags.com/X1rZYp.php" "localdb" "http://8vs.com/6jezbr.php" "localdb" "http://abdal.com.ua/7_jzay.php" "localdb" "http://aditaborai.com.br/WgNGXe.php" "localdb" "http://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php" "localdb" "http://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php" "localdb" "http://allstarpaintbody.com/lrQ2bG.php" "localdb" "http://americancorner.udp.cl/etloxW.php" "localdb" "http://ample-sun.eu/4BKEt7.php" "localdb" "http://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php" "localdb" "http://anoukdelecluse.nl/lGZLB1.php" "localdb" "http://appeum.com/wp-content/themes/cc.php" "localdb" "http://arot.altervista.org/KHTUdq.php" "localdb" "http://arttoday.sk/mE8MKJ.php" "localdb" "http://ascortimisoara.ro/kWIH5V.php" "localdb" "http://aspectdesigns.com.au/0rTVlG.php" "localdb" "http://audetlaw.com/LnVAdF.php" "localdb" "http://autohaus-seevetal.com/9x6UwK.php" "localdb" "http://avancarvisual.com.br/wp-content/themes/twentytwelve/VzkgnX.php" "localdb" "http://babylicious.ie/s1GHUZ.php" "localdb" "http://balkanium.altervista.org/p3er4s.php" "localdb" "http://beachhouseplans.com/wp-admin/js/5d8gMe.php" "localdb" "http://best-service.jp/olxu2Y.php" "localdb" "http://beyondthedog.net/edHDvf.php" "localdb" "http://bigboattravel.com/uploads/3/5/4/5/3545341/header_images/NthjHz.php" "localdb" "http://bisofit.com/QXwm4I.php" "localdb" "http://bktrade.kiev.ua/76b3ZQ.php" "localdb" "http://boilersandfurnaces.com/uploads/3/5/1/6/3516773/RPyH2q.php" "localdb" "http://bolizarsospos.com/0l0vp1va6b2" "localdb" "http://bolizarsospos.com/1cslstk2qv121" "localdb" "http://bolizarsospos.com/1xb81c28qs2db" "localdb" "http://bolizarsospos.com/22o1210hbpw" "localdb" "http://bolizarsospos.com/2h6t511wpuvnych" "localdb" "http://bolizarsospos.com/379gz635s3j946" "localdb" "http://bolizarsospos.com/4kpy8ju42x137" "localdb" "http://bolizarsospos.com/503qu7boexyk" "localdb" "http://bolizarsospos.com/574xl5yme0gdz" "localdb" "http://bolizarsospos.com/5gpf7ecxhf" "localdb" "http://bolizarsospos.com/5hmwl5qvpz2f3gc" "localdb" "http://bolizarsospos.com/6m50uk8ty1031" "localdb" "http://bolizarsospos.com/6tvpgu93q4wx5t" "localdb" "http://bolizarsospos.com/703hjdr3ez72" "localdb" "http://bolizarsospos.com/73075bdj8meb" "localdb" "http://bolizarsospos.com/7gr904pzv6" "localdb" "http://bolizarsospos.com/7ms68qsdfj0jt" "localdb" "http://bolizarsospos.com/89e8f40k8zcn38" "localdb" "http://bolizarsospos.com/8eo5zwhh4zndwwa" "localdb" "http://bolizarsospos.com/8tsdhjccoxz6c" "localdb" "http://bolizarsospos.com/94g2mr36b4" "localdb" "http://bolizarsospos.com/9bqdnk2h58ty2l" "localdb" "http://bolizarsospos.com/9hul78mtg1n63" "localdb" "http://bolizarsospos.com/b0slgvfxvyf" "localdb" "http://bolizarsospos.com/b3amhlkiar2c" "localdb" "http://bolizarsospos.com/b8g7g560612" "localdb" "http://bolizarsospos.com/bo5ha9ild1zjukv" "localdb" "http://bolizarsospos.com/cannzqzrum14o4c" "localdb" "http://bolizarsospos.com/ch3eq62ad8k" "localdb" "http://bolizarsospos.com/ci72o4ruf2y87" "localdb" "http://bolizarsospos.com/d5i52z8cgv5" "localdb" "http://bolizarsospos.com/d65v4fx21f" "localdb" "http://bolizarsospos.com/d7jly5f09tqj" "localdb" "http://bolizarsospos.com/di53su4z7uqvj" "localdb" "http://bolizarsospos.com/dypi31624z" "localdb" "http://bolizarsospos.com/e5tkclwq9w0" "localdb" "http://bolizarsospos.com/e887nn5k9pb6" "localdb" "http://bolizarsospos.com/f1s0y87wrwo" "localdb" "http://bolizarsospos.com/fgivit1drjuh" "localdb" "http://bolizarsospos.com/fpkirizbrzxc5" "localdb" "http://bolizarsospos.com/fxoztyxp320q" "localdb" "http://bolizarsospos.com/g5k4uvxghygg7r" "localdb" "http://bolizarsospos.com/gvi00me81aabu" "localdb" "http://bolizarsospos.com/hq5drme48h" "localdb" "http://bolizarsospos.com/hzpz767vze9" "localdb" "http://bolizarsospos.com/ifkhfc5369az88" "localdb" "http://bolizarsospos.com/iijoama0ynrowtp" "localdb" "http://bolizarsospos.com/j4hzoz8cgdeza" "localdb" "http://bolizarsospos.com/kka7641ov7" "localdb" "http://bolizarsospos.com/ko679ybid6ys58" "localdb" "http://bolizarsospos.com/kxdmlkhmuyf9" "localdb" "http://bolizarsospos.com/kzqnheutxkjwhr" "localdb" "http://bolizarsospos.com/mi5b67bilrfu" "localdb" "http://bolizarsospos.com/n2csus3eo1tyg" "localdb" "http://bolizarsospos.com/nve4m67l83" "localdb" "http://bolizarsospos.com/o0nyjlre41o3" "localdb" "http://bolizarsospos.com/pgjcokoi2kisu" "localdb" "http://bolizarsospos.com/pl36lz43r6r7" "localdb" "http://bolizarsospos.com/q3xryv3mh1" "localdb" "http://bolizarsospos.com/qely217wcjdl7b" "localdb" "http://bolizarsospos.com/qo9ux20lo1" "localdb" "http://bolizarsospos.com/qu9ajlxsiw" "localdb" "http://bolizarsospos.com/r45byxsjhz" "localdb" "http://bolizarsospos.com/raph9xccgxt" "localdb" "http://bolizarsospos.com/rb05hez1r044" "localdb" "http://bolizarsospos.com/rdjg0eb5r0qs" "localdb" "http://bolizarsospos.com/ri86nx23dhqbmch" "localdb" "http://bolizarsospos.com/rjotoddb4n7hl" "localdb" "http://bolizarsospos.com/rof06587c1x2y3t" "localdb" "http://bolizarsospos.com/s40o542jt7v" "localdb" "http://bolizarsospos.com/sb2zarf5vy" "localdb" "http://bolizarsospos.com/uamuxps7y98" "localdb" "http://bolizarsospos.com/uiyi9dkf5bs" "localdb" "http://bolizarsospos.com/v13rw8n8w2" "localdb" "http://bolizarsospos.com/vzum6ywdedxjtd" "localdb" "http://bolizarsospos.com/walqb5xzunmr" "localdb" "http://bolizarsospos.com/wilqkaz24rnqli" "localdb" "http://bolizarsospos.com/x1tg111bara5" "localdb" "http://bolizarsospos.com/x753k2s01gnd5b" "localdb" "http://bolizarsospos.com/x7lfazpjuuiel" "localdb" "http://bolizarsospos.com/xgw1o6gt9h8k9g" "localdb" "http://bolizarsospos.com/xjp3zmw6glginuq" "localdb" "http://bolizarsospos.com/yias364ajr" "localdb" "http://bolizarsospos.com/zyayxp2kpay" "localdb" "http://bucksmedia.go2cloud.org/aff_c" "localdb" "http://building.msu.ac.th/q3Bslr.php" "localdb" "http://businessaviators.com/r1doyF.php" "localdb" "http://challengestrata.com.au/fP_BXS.php" "localdb" "http://cheapshirts.us/zVnMrG.php" "localdb" "http://chong.joelle.free.fr/_L43PH.php" "localdb" "http://connectao.com/wp-content/themes/twentyeleven/cc.php" "localdb" "http://d3mpd.fe.uns.ac.id/XPgmur.php" "localdb" "http://daffamedia.com/wp-content/plugins/wp_module/img5.php" "localdb" "http://dechehang.com/GZ2QRn.php" "localdb" "http://definitionen.de/v7GVES.php" "localdb" "http://dichiro.com/WaIrd6.php" "localdb" "http://dillardvideo.com/wp-admin/network/2.php" "localdb" "http://dining-bar.com/BQ_Ln4.php" "localdb" "http://domaine-cassillac.com/4q3esU.php" "localdb" "http://double-wing.de/DZkCLR.php" "localdb" "http://drdigitalmd.com/img1.php" "localdb" "http://eatside.es/xZQGXV.php" "localdb" "http://ecocalsots.com/N79GTA.php" "localdb" "http://ecolux-comfort.com/nPAbsy.php" "localdb" "http://elcoachingempresarial.com/wp-admin/user/2.php" "localdb" "http://emprende21.es/oTIq7A.php" "localdb" "http://estudiobarco.com.ar/5TFv7E.php" "localdb" "http://event-travel.co.uk/3K6Psd.php" "localdb" "http://feuerwehr-stadt-riesa.de/UFiPOq.php" "localdb" "http://foundersomaha.net/wp-includes/Text/Diff/Renderer/ap3.php" "localdb" "http://frame3d.de/ItGJKd.php" "localdb" "http://fun-pop.com/Ks1rCc.php" "localdb" "http://genedillardart.com/wp-admin/network/3.php" "localdb" "http://gibdd.ws/J7D65p.php" "localdb" "http://glitchygaming.com/r07QZu.php" "localdb" "http://grochowina.net/UnvPso.php" "localdb" "http://haarsaloncindy.nl/XzF03r.php" "localdb" "http://hamilton150.co.nz/LmfuMZ.php" "localdb" "http://icsot.na.its.ac.id/8vwRUX.php" "localdb" "http://igatha.com/h4MeKJ.php" "localdb" "http://ilovesport.kiev.ua/z8X9T7.php" "localdb" "http://imagescameraclub.com/j7b5kK.php" "localdb" "http://inspirenetworks.in/vAqu3L.php" "localdb" "http://italyprego.com/Lf2dcA.php" "localdb" "http://jadwalpialadunia.in/rG4Rdi.php" "localdb" "http://jambola.com/LuylWV.php" "localdb" "http://jauregia.net/img5.php" "localdb" "http://konyavakfi.nl/Zmje1r.php" "localdb" "http://kuruyaprak.com/OTLuKo.php" "localdb" "http://kvnysoho.com/eHafFT.php" "localdb" "http://lazymoosestamping.com/YRfbgB.php" "localdb" "http://london-escorts-agency.org.uk/fdnmyD.php" "localdb" "http://lzclient.com/img4.php" "localdb" "http://madisonbootcamps.com/gWQ3wp.php" "localdb" "http://mangohills.net/RxIoCE.php" "localdb" "http://marciogerhardtsouza.com.br/mPCsDz.php" "localdb" "http://marcortes.com/img5.php" "localdb" "http://markossolomon.com/F1q7QX.php" "localdb" "http://maternalserenity.co.uk/I_NwPg.php" "localdb" "http://millsmanagement.nl/AnOgVK.php" "localdb" "http://mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php" "localdb" "http://naimselmonaj.com/QoYx31.php" "localdb" "http://nonnuoccaobang.com/BRdoDL.php" "localdb" "http://nupleta.com.br/KoHV09.php" "localdb" "http://openroadsolutions.com/FJ2dOw.php" "localdb" "http://oregonreversemortgage.com/Rwafp_.php" "localdb" "http://p237996.mybestmv.com/adServe/domainClick" "localdb" "http://paintituppottery.com/6cmeb2.php" "localdb" "http://patrianossa.com.br/u8LkzD.php" "localdb" "http://portalmaismidia.com.br/tnSmIb.php" "localdb" "http://portret-tekening.nl/mNQVts.php" "localdb" "http://procrediti.com.ua/d6yGOX.php" "localdb" "http://rajsima87.com/img2.php" "localdb" "http://recaswine.ro/dXlq0Y.php" "localdb" "http://silstop.pl/Si0cCJ.php" "localdb" "http://smartnote.co/2NxVzA.php" "localdb" "http://studiolegalecsb.it/iQcNfC.php" "localdb" "http://takaram.ir/gjOREZ.php" "localdb" "http://takatei.com/rfYI4L.php" "localdb" "http://tcblog.de/mXdVTh.php" "localdb" "http://theassemblyguy.co.nz/vpFAbQ.php" "localdb" "http://trion.com.ph/jdKAap.php" "localdb" "http://tusrecetas.net/JbElN7.php" "localdb" "http://tutorialswalk.info/wp-content/themes/Defne/img2.php" "localdb" "http://viralcrazies.com/iFHt4C.php" "localdb" "http://vsedveri-33.ru/" "localdb" "http://weberteam.hu/WCTdO5.php" "localdb" "http://www.001edizioni.com/NZwt_a.php" "localdb" "http://www.bishopbell.co.uk/enRmcC.php" "localdb" "http://www.chemes.eu/wp-content/themes/decoy2/redux-framework/ReduxCore/inc/fields/info/2.php" "localdb" "http://www.decorandoimoveis.com/QEO5yh.php" "localdb" "http://www.example.com:1000" "localdb" "http://www.feddoctor.com/Oe1LMr.php" "localdb" "http://www.gjscomputerservices.com.au/S1_rvm.php" "localdb" "http://www.granmarquise.com.br/6f_8ei.php" "localdb" "http://www.hanecaklaw.com/" "localdb" "http://www.hanoiguidedtours.com/iQ2q1f.php" "localdb" "http://www.healthstafftravel.com.au/oyBbUs.php" "localdb" "http://www.plexipr.com/vAHzWX.php" "localdb" "http://www.rippedknees.co.uk/TXmJcq.php" "localdb" "http://www.taoblu.com/wp-content/plugins/wp_module/sbML0j.php" "localdb" "http://www.vishvagujarat.com/5of9dt.php" "localdb" "http://www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkG.php" "localdb" "https://*abc.*test.com" "localdb" ================================================ FILE: tests/integration/basic/URLHC%3Fv%3Dpanosurl%26di%3D1.result ================================================ feyda.net/hoedr4.php *.feyda.net/hoedr4.php 7-eleven-handbags.com/x1rzyp.php 8vs.com/6jezbr.php abdal.com.ua/7_jzay.php aditaborai.com.br/wgngxe.php airconditioning12601.com/uploads/3/5/7/6/3576233/v5k3za.php allreadytravel.com/uploads/3/5/4/9/3549731/header_images/tomae1.php allstarpaintbody.com/lrq2bg.php americancorner.udp.cl/etloxw.php ample-sun.eu/4bket7.php anime-tuner.square7.ch/wp-content/themes/twentyeleven/mstgk_.php anoukdelecluse.nl/lgzlb1.php appeum.com/wp-content/themes/cc.php arot.altervista.org/khtudq.php arttoday.sk/me8mkj.php ascortimisoara.ro/kwih5v.php aspectdesigns.com.au/0rtvlg.php audetlaw.com/lnvadf.php autohaus-seevetal.com/9x6uwk.php avancarvisual.com.br/wp-content/themes/twentytwelve/vzkgnx.php babylicious.ie/s1ghuz.php balkanium.altervista.org/p3er4s.php beachhouseplans.com/wp-admin/js/5d8gme.php best-service.jp/olxu2y.php beyondthedog.net/edhdvf.php bigboattravel.com/uploads/3/5/4/5/3545341/header_images/nthjhz.php bisofit.com/qxwm4i.php bktrade.kiev.ua/76b3zq.php boilersandfurnaces.com/uploads/3/5/1/6/3516773/rpyh2q.php bolizarsospos.com/0l0vp1va6b2 bolizarsospos.com/1cslstk2qv121 bolizarsospos.com/1xb81c28qs2db bolizarsospos.com/22o1210hbpw bolizarsospos.com/2h6t511wpuvnych bolizarsospos.com/379gz635s3j946 bolizarsospos.com/4kpy8ju42x137 bolizarsospos.com/503qu7boexyk bolizarsospos.com/574xl5yme0gdz bolizarsospos.com/5gpf7ecxhf bolizarsospos.com/5hmwl5qvpz2f3gc bolizarsospos.com/6m50uk8ty1031 bolizarsospos.com/6tvpgu93q4wx5t bolizarsospos.com/703hjdr3ez72 bolizarsospos.com/73075bdj8meb bolizarsospos.com/7gr904pzv6 bolizarsospos.com/7ms68qsdfj0jt bolizarsospos.com/89e8f40k8zcn38 bolizarsospos.com/8eo5zwhh4zndwwa bolizarsospos.com/8tsdhjccoxz6c bolizarsospos.com/94g2mr36b4 bolizarsospos.com/9bqdnk2h58ty2l bolizarsospos.com/9hul78mtg1n63 bolizarsospos.com/b0slgvfxvyf bolizarsospos.com/b3amhlkiar2c bolizarsospos.com/b8g7g560612 bolizarsospos.com/bo5ha9ild1zjukv bolizarsospos.com/cannzqzrum14o4c bolizarsospos.com/ch3eq62ad8k bolizarsospos.com/ci72o4ruf2y87 bolizarsospos.com/d5i52z8cgv5 bolizarsospos.com/d65v4fx21f bolizarsospos.com/d7jly5f09tqj bolizarsospos.com/di53su4z7uqvj bolizarsospos.com/dypi31624z bolizarsospos.com/e5tkclwq9w0 bolizarsospos.com/e887nn5k9pb6 bolizarsospos.com/f1s0y87wrwo bolizarsospos.com/fgivit1drjuh bolizarsospos.com/fpkirizbrzxc5 bolizarsospos.com/fxoztyxp320q bolizarsospos.com/g5k4uvxghygg7r bolizarsospos.com/gvi00me81aabu bolizarsospos.com/hq5drme48h bolizarsospos.com/hzpz767vze9 bolizarsospos.com/ifkhfc5369az88 bolizarsospos.com/iijoama0ynrowtp bolizarsospos.com/j4hzoz8cgdeza bolizarsospos.com/kka7641ov7 bolizarsospos.com/ko679ybid6ys58 bolizarsospos.com/kxdmlkhmuyf9 bolizarsospos.com/kzqnheutxkjwhr bolizarsospos.com/mi5b67bilrfu bolizarsospos.com/n2csus3eo1tyg bolizarsospos.com/nve4m67l83 bolizarsospos.com/o0nyjlre41o3 bolizarsospos.com/pgjcokoi2kisu bolizarsospos.com/pl36lz43r6r7 bolizarsospos.com/q3xryv3mh1 bolizarsospos.com/qely217wcjdl7b bolizarsospos.com/qo9ux20lo1 bolizarsospos.com/qu9ajlxsiw bolizarsospos.com/r45byxsjhz bolizarsospos.com/raph9xccgxt bolizarsospos.com/rb05hez1r044 bolizarsospos.com/rdjg0eb5r0qs bolizarsospos.com/ri86nx23dhqbmch bolizarsospos.com/rjotoddb4n7hl bolizarsospos.com/rof06587c1x2y3t bolizarsospos.com/s40o542jt7v bolizarsospos.com/sb2zarf5vy bolizarsospos.com/uamuxps7y98 bolizarsospos.com/uiyi9dkf5bs bolizarsospos.com/v13rw8n8w2 bolizarsospos.com/vzum6ywdedxjtd bolizarsospos.com/walqb5xzunmr bolizarsospos.com/wilqkaz24rnqli bolizarsospos.com/x1tg111bara5 bolizarsospos.com/x753k2s01gnd5b bolizarsospos.com/x7lfazpjuuiel bolizarsospos.com/xgw1o6gt9h8k9g bolizarsospos.com/xjp3zmw6glginuq bolizarsospos.com/yias364ajr bolizarsospos.com/zyayxp2kpay bucksmedia.go2cloud.org/aff_c building.msu.ac.th/q3bslr.php businessaviators.com/r1doyf.php challengestrata.com.au/fp_bxs.php cheapshirts.us/zvnmrg.php chong.joelle.free.fr/_l43ph.php connectao.com/wp-content/themes/twentyeleven/cc.php d3mpd.fe.uns.ac.id/xpgmur.php daffamedia.com/wp-content/plugins/wp_module/img5.php dechehang.com/gz2qrn.php definitionen.de/v7gves.php dichiro.com/waird6.php dillardvideo.com/wp-admin/network/2.php dining-bar.com/bq_ln4.php domaine-cassillac.com/4q3esu.php double-wing.de/dzkclr.php drdigitalmd.com/img1.php eatside.es/xzqgxv.php ecocalsots.com/n79gta.php ecolux-comfort.com/npabsy.php elcoachingempresarial.com/wp-admin/user/2.php emprende21.es/otiq7a.php estudiobarco.com.ar/5tfv7e.php event-travel.co.uk/3k6psd.php feuerwehr-stadt-riesa.de/ufipoq.php foundersomaha.net/wp-includes/text/diff/renderer/ap3.php frame3d.de/itgjkd.php fun-pop.com/ks1rcc.php genedillardart.com/wp-admin/network/3.php gibdd.ws/j7d65p.php glitchygaming.com/r07qzu.php grochowina.net/unvpso.php haarsaloncindy.nl/xzf03r.php hamilton150.co.nz/lmfumz.php icsot.na.its.ac.id/8vwrux.php igatha.com/h4mekj.php ilovesport.kiev.ua/z8x9t7.php imagescameraclub.com/j7b5kk.php inspirenetworks.in/vaqu3l.php italyprego.com/lf2dca.php jadwalpialadunia.in/rg4rdi.php jambola.com/luylwv.php jauregia.net/img5.php konyavakfi.nl/zmje1r.php kuruyaprak.com/otluko.php kvnysoho.com/ehafft.php lazymoosestamping.com/yrfbgb.php london-escorts-agency.org.uk/fdnmyd.php lzclient.com/img4.php madisonbootcamps.com/gwq3wp.php mangohills.net/rxioce.php marciogerhardtsouza.com.br/mpcsdz.php marcortes.com/img5.php markossolomon.com/f1q7qx.php maternalserenity.co.uk/i_nwpg.php millsmanagement.nl/anogvk.php mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php naimselmonaj.com/qoyx31.php nonnuoccaobang.com/brdodl.php nupleta.com.br/kohv09.php openroadsolutions.com/fj2dow.php oregonreversemortgage.com/rwafp_.php p237996.mybestmv.com/adserve/domainclick paintituppottery.com/6cmeb2.php patrianossa.com.br/u8lkzd.php portalmaismidia.com.br/tnsmib.php portret-tekening.nl/mnqvts.php procrediti.com.ua/d6ygox.php rajsima87.com/img2.php recaswine.ro/dxlq0y.php silstop.pl/si0ccj.php smartnote.co/2nxvza.php studiolegalecsb.it/iqcnfc.php takaram.ir/gjorez.php takatei.com/rfyi4l.php tcblog.de/mxdvth.php theassemblyguy.co.nz/vpfabq.php trion.com.ph/jdkaap.php tusrecetas.net/jbeln7.php tutorialswalk.info/wp-content/themes/defne/img2.php viralcrazies.com/ifht4c.php vsedveri-33.ru/ weberteam.hu/wctdo5.php www.001edizioni.com/nzwt_a.php www.bishopbell.co.uk/enrmcc.php www.chemes.eu/wp-content/themes/decoy2/redux-framework/reduxcore/inc/fields/info/2.php www.decorandoimoveis.com/qeo5yh.php www.feddoctor.com/oe1lmr.php www.gjscomputerservices.com.au/s1_rvm.php www.granmarquise.com.br/6f_8ei.php www.hanecaklaw.com/ www.hanoiguidedtours.com/iq2q1f.php www.healthstafftravel.com.au/oybbus.php www.plexipr.com/vahzwx.php www.rippedknees.co.uk/txmjcq.php www.taoblu.com/wp-content/plugins/wp_module/sbml0j.php www.vishvagujarat.com/5of9dt.php www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkg.php ================================================ FILE: tests/integration/basic/URLHC%3Fv%3Dpanosurl%26sp%3D1%26nsl%3D1.result ================================================ example.com/foobar?a=:80 *.example.com/foobar?a=:80 feyda.net/hoedr4.php *.feyda.net/hoedr4.php 7-eleven-handbags.com/x1rzyp.php 8vs.com/6jezbr.php abdal.com.ua/7_jzay.php aditaborai.com.br/wgngxe.php airconditioning12601.com/uploads/3/5/7/6/3576233/v5k3za.php allreadytravel.com/uploads/3/5/4/9/3549731/header_images/tomae1.php allstarpaintbody.com/lrq2bg.php americancorner.udp.cl/etloxw.php ample-sun.eu/4bket7.php anime-tuner.square7.ch/wp-content/themes/twentyeleven/mstgk_.php anoukdelecluse.nl/lgzlb1.php appeum.com/wp-content/themes/cc.php arot.altervista.org/khtudq.php arttoday.sk/me8mkj.php ascortimisoara.ro/kwih5v.php aspectdesigns.com.au/0rtvlg.php audetlaw.com/lnvadf.php autohaus-seevetal.com/9x6uwk.php avancarvisual.com.br/wp-content/themes/twentytwelve/vzkgnx.php babylicious.ie/s1ghuz.php balkanium.altervista.org/p3er4s.php beachhouseplans.com/wp-admin/js/5d8gme.php best-service.jp/olxu2y.php beyondthedog.net/edhdvf.php bigboattravel.com/uploads/3/5/4/5/3545341/header_images/nthjhz.php bisofit.com/qxwm4i.php bktrade.kiev.ua/76b3zq.php boilersandfurnaces.com/uploads/3/5/1/6/3516773/rpyh2q.php bolizarsospos.com/0l0vp1va6b2 bolizarsospos.com/1cslstk2qv121 bolizarsospos.com/1xb81c28qs2db bolizarsospos.com/22o1210hbpw bolizarsospos.com/2h6t511wpuvnych bolizarsospos.com/379gz635s3j946 bolizarsospos.com/4kpy8ju42x137 bolizarsospos.com/503qu7boexyk bolizarsospos.com/574xl5yme0gdz bolizarsospos.com/5gpf7ecxhf bolizarsospos.com/5hmwl5qvpz2f3gc bolizarsospos.com/6m50uk8ty1031 bolizarsospos.com/6tvpgu93q4wx5t bolizarsospos.com/703hjdr3ez72 bolizarsospos.com/73075bdj8meb bolizarsospos.com/7gr904pzv6 bolizarsospos.com/7ms68qsdfj0jt bolizarsospos.com/89e8f40k8zcn38 bolizarsospos.com/8eo5zwhh4zndwwa bolizarsospos.com/8tsdhjccoxz6c bolizarsospos.com/94g2mr36b4 bolizarsospos.com/9bqdnk2h58ty2l bolizarsospos.com/9hul78mtg1n63 bolizarsospos.com/b0slgvfxvyf bolizarsospos.com/b3amhlkiar2c bolizarsospos.com/b8g7g560612 bolizarsospos.com/bo5ha9ild1zjukv bolizarsospos.com/cannzqzrum14o4c bolizarsospos.com/ch3eq62ad8k bolizarsospos.com/ci72o4ruf2y87 bolizarsospos.com/d5i52z8cgv5 bolizarsospos.com/d65v4fx21f bolizarsospos.com/d7jly5f09tqj bolizarsospos.com/di53su4z7uqvj bolizarsospos.com/dypi31624z bolizarsospos.com/e5tkclwq9w0 bolizarsospos.com/e887nn5k9pb6 bolizarsospos.com/f1s0y87wrwo bolizarsospos.com/fgivit1drjuh bolizarsospos.com/fpkirizbrzxc5 bolizarsospos.com/fxoztyxp320q bolizarsospos.com/g5k4uvxghygg7r bolizarsospos.com/gvi00me81aabu bolizarsospos.com/hq5drme48h bolizarsospos.com/hzpz767vze9 bolizarsospos.com/ifkhfc5369az88 bolizarsospos.com/iijoama0ynrowtp bolizarsospos.com/j4hzoz8cgdeza bolizarsospos.com/kka7641ov7 bolizarsospos.com/ko679ybid6ys58 bolizarsospos.com/kxdmlkhmuyf9 bolizarsospos.com/kzqnheutxkjwhr bolizarsospos.com/mi5b67bilrfu bolizarsospos.com/n2csus3eo1tyg bolizarsospos.com/nve4m67l83 bolizarsospos.com/o0nyjlre41o3 bolizarsospos.com/pgjcokoi2kisu bolizarsospos.com/pl36lz43r6r7 bolizarsospos.com/q3xryv3mh1 bolizarsospos.com/qely217wcjdl7b bolizarsospos.com/qo9ux20lo1 bolizarsospos.com/qu9ajlxsiw bolizarsospos.com/r45byxsjhz bolizarsospos.com/raph9xccgxt bolizarsospos.com/rb05hez1r044 bolizarsospos.com/rdjg0eb5r0qs bolizarsospos.com/ri86nx23dhqbmch bolizarsospos.com/rjotoddb4n7hl bolizarsospos.com/rof06587c1x2y3t bolizarsospos.com/s40o542jt7v bolizarsospos.com/sb2zarf5vy bolizarsospos.com/uamuxps7y98 bolizarsospos.com/uiyi9dkf5bs bolizarsospos.com/v13rw8n8w2 bolizarsospos.com/vzum6ywdedxjtd bolizarsospos.com/walqb5xzunmr bolizarsospos.com/wilqkaz24rnqli bolizarsospos.com/x1tg111bara5 bolizarsospos.com/x753k2s01gnd5b bolizarsospos.com/x7lfazpjuuiel bolizarsospos.com/xgw1o6gt9h8k9g bolizarsospos.com/xjp3zmw6glginuq bolizarsospos.com/yias364ajr bolizarsospos.com/zyayxp2kpay bucksmedia.go2cloud.org/aff_c building.msu.ac.th/q3bslr.php businessaviators.com/r1doyf.php challengestrata.com.au/fp_bxs.php cheapshirts.us/zvnmrg.php chong.joelle.free.fr/_l43ph.php connectao.com/wp-content/themes/twentyeleven/cc.php d3mpd.fe.uns.ac.id/xpgmur.php daffamedia.com/wp-content/plugins/wp_module/img5.php dechehang.com/gz2qrn.php definitionen.de/v7gves.php dichiro.com/waird6.php dillardvideo.com/wp-admin/network/2.php dining-bar.com/bq_ln4.php domaine-cassillac.com/4q3esu.php double-wing.de/dzkclr.php drdigitalmd.com/img1.php eatside.es/xzqgxv.php ecocalsots.com/n79gta.php ecolux-comfort.com/npabsy.php elcoachingempresarial.com/wp-admin/user/2.php emprende21.es/otiq7a.php estudiobarco.com.ar/5tfv7e.php event-travel.co.uk/3k6psd.php feuerwehr-stadt-riesa.de/ufipoq.php foundersomaha.net/wp-includes/text/diff/renderer/ap3.php frame3d.de/itgjkd.php fun-pop.com/ks1rcc.php genedillardart.com/wp-admin/network/3.php gibdd.ws/j7d65p.php glitchygaming.com/r07qzu.php grochowina.net/unvpso.php haarsaloncindy.nl/xzf03r.php hamilton150.co.nz/lmfumz.php icsot.na.its.ac.id/8vwrux.php igatha.com/h4mekj.php ilovesport.kiev.ua/z8x9t7.php imagescameraclub.com/j7b5kk.php inspirenetworks.in/vaqu3l.php italyprego.com/lf2dca.php jadwalpialadunia.in/rg4rdi.php jambola.com/luylwv.php jauregia.net/img5.php konyavakfi.nl/zmje1r.php kuruyaprak.com/otluko.php kvnysoho.com/ehafft.php lazymoosestamping.com/yrfbgb.php london-escorts-agency.org.uk/fdnmyd.php lzclient.com/img4.php madisonbootcamps.com/gwq3wp.php mangohills.net/rxioce.php marciogerhardtsouza.com.br/mpcsdz.php marcortes.com/img5.php markossolomon.com/f1q7qx.php maternalserenity.co.uk/i_nwpg.php millsmanagement.nl/anogvk.php mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php naimselmonaj.com/qoyx31.php nonnuoccaobang.com/brdodl.php nupleta.com.br/kohv09.php openroadsolutions.com/fj2dow.php oregonreversemortgage.com/rwafp_.php p237996.mybestmv.com/adserve/domainclick paintituppottery.com/6cmeb2.php patrianossa.com.br/u8lkzd.php portalmaismidia.com.br/tnsmib.php portret-tekening.nl/mnqvts.php procrediti.com.ua/d6ygox.php rajsima87.com/img2.php recaswine.ro/dxlq0y.php silstop.pl/si0ccj.php smartnote.co/2nxvza.php studiolegalecsb.it/iqcnfc.php takaram.ir/gjorez.php takatei.com/rfyi4l.php tcblog.de/mxdvth.php theassemblyguy.co.nz/vpfabq.php trion.com.ph/jdkaap.php tusrecetas.net/jbeln7.php tutorialswalk.info/wp-content/themes/defne/img2.php viralcrazies.com/ifht4c.php vsedveri-33.ru/ weberteam.hu/wctdo5.php www.001edizioni.com/nzwt_a.php www.bishopbell.co.uk/enrmcc.php www.chemes.eu/wp-content/themes/decoy2/redux-framework/reduxcore/inc/fields/info/2.php www.decorandoimoveis.com/qeo5yh.php www.example.com www.feddoctor.com/oe1lmr.php www.gjscomputerservices.com.au/s1_rvm.php www.granmarquise.com.br/6f_8ei.php www.hanecaklaw.com/ www.hanoiguidedtours.com/iq2q1f.php www.healthstafftravel.com.au/oybbus.php www.plexipr.com/vahzwx.php www.rippedknees.co.uk/txmjcq.php www.taoblu.com/wp-content/plugins/wp_module/sbml0j.php www.vishvagujarat.com/5of9dt.php www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkg.php ================================================ FILE: tests/integration/basic/URLHC%3Fv%3Dpanosurl%26sp%3D1.result ================================================ example.com/foobar?a=:80 *.example.com/foobar?a=:80 feyda.net/hoedr4.php *.feyda.net/hoedr4.php 7-eleven-handbags.com/x1rzyp.php 8vs.com/6jezbr.php abdal.com.ua/7_jzay.php aditaborai.com.br/wgngxe.php airconditioning12601.com/uploads/3/5/7/6/3576233/v5k3za.php allreadytravel.com/uploads/3/5/4/9/3549731/header_images/tomae1.php allstarpaintbody.com/lrq2bg.php americancorner.udp.cl/etloxw.php ample-sun.eu/4bket7.php anime-tuner.square7.ch/wp-content/themes/twentyeleven/mstgk_.php anoukdelecluse.nl/lgzlb1.php appeum.com/wp-content/themes/cc.php arot.altervista.org/khtudq.php arttoday.sk/me8mkj.php ascortimisoara.ro/kwih5v.php aspectdesigns.com.au/0rtvlg.php audetlaw.com/lnvadf.php autohaus-seevetal.com/9x6uwk.php avancarvisual.com.br/wp-content/themes/twentytwelve/vzkgnx.php babylicious.ie/s1ghuz.php balkanium.altervista.org/p3er4s.php beachhouseplans.com/wp-admin/js/5d8gme.php best-service.jp/olxu2y.php beyondthedog.net/edhdvf.php bigboattravel.com/uploads/3/5/4/5/3545341/header_images/nthjhz.php bisofit.com/qxwm4i.php bktrade.kiev.ua/76b3zq.php boilersandfurnaces.com/uploads/3/5/1/6/3516773/rpyh2q.php bolizarsospos.com/0l0vp1va6b2 bolizarsospos.com/1cslstk2qv121 bolizarsospos.com/1xb81c28qs2db bolizarsospos.com/22o1210hbpw bolizarsospos.com/2h6t511wpuvnych bolizarsospos.com/379gz635s3j946 bolizarsospos.com/4kpy8ju42x137 bolizarsospos.com/503qu7boexyk bolizarsospos.com/574xl5yme0gdz bolizarsospos.com/5gpf7ecxhf bolizarsospos.com/5hmwl5qvpz2f3gc bolizarsospos.com/6m50uk8ty1031 bolizarsospos.com/6tvpgu93q4wx5t bolizarsospos.com/703hjdr3ez72 bolizarsospos.com/73075bdj8meb bolizarsospos.com/7gr904pzv6 bolizarsospos.com/7ms68qsdfj0jt bolizarsospos.com/89e8f40k8zcn38 bolizarsospos.com/8eo5zwhh4zndwwa bolizarsospos.com/8tsdhjccoxz6c bolizarsospos.com/94g2mr36b4 bolizarsospos.com/9bqdnk2h58ty2l bolizarsospos.com/9hul78mtg1n63 bolizarsospos.com/b0slgvfxvyf bolizarsospos.com/b3amhlkiar2c bolizarsospos.com/b8g7g560612 bolizarsospos.com/bo5ha9ild1zjukv bolizarsospos.com/cannzqzrum14o4c bolizarsospos.com/ch3eq62ad8k bolizarsospos.com/ci72o4ruf2y87 bolizarsospos.com/d5i52z8cgv5 bolizarsospos.com/d65v4fx21f bolizarsospos.com/d7jly5f09tqj bolizarsospos.com/di53su4z7uqvj bolizarsospos.com/dypi31624z bolizarsospos.com/e5tkclwq9w0 bolizarsospos.com/e887nn5k9pb6 bolizarsospos.com/f1s0y87wrwo bolizarsospos.com/fgivit1drjuh bolizarsospos.com/fpkirizbrzxc5 bolizarsospos.com/fxoztyxp320q bolizarsospos.com/g5k4uvxghygg7r bolizarsospos.com/gvi00me81aabu bolizarsospos.com/hq5drme48h bolizarsospos.com/hzpz767vze9 bolizarsospos.com/ifkhfc5369az88 bolizarsospos.com/iijoama0ynrowtp bolizarsospos.com/j4hzoz8cgdeza bolizarsospos.com/kka7641ov7 bolizarsospos.com/ko679ybid6ys58 bolizarsospos.com/kxdmlkhmuyf9 bolizarsospos.com/kzqnheutxkjwhr bolizarsospos.com/mi5b67bilrfu bolizarsospos.com/n2csus3eo1tyg bolizarsospos.com/nve4m67l83 bolizarsospos.com/o0nyjlre41o3 bolizarsospos.com/pgjcokoi2kisu bolizarsospos.com/pl36lz43r6r7 bolizarsospos.com/q3xryv3mh1 bolizarsospos.com/qely217wcjdl7b bolizarsospos.com/qo9ux20lo1 bolizarsospos.com/qu9ajlxsiw bolizarsospos.com/r45byxsjhz bolizarsospos.com/raph9xccgxt bolizarsospos.com/rb05hez1r044 bolizarsospos.com/rdjg0eb5r0qs bolizarsospos.com/ri86nx23dhqbmch bolizarsospos.com/rjotoddb4n7hl bolizarsospos.com/rof06587c1x2y3t bolizarsospos.com/s40o542jt7v bolizarsospos.com/sb2zarf5vy bolizarsospos.com/uamuxps7y98 bolizarsospos.com/uiyi9dkf5bs bolizarsospos.com/v13rw8n8w2 bolizarsospos.com/vzum6ywdedxjtd bolizarsospos.com/walqb5xzunmr bolizarsospos.com/wilqkaz24rnqli bolizarsospos.com/x1tg111bara5 bolizarsospos.com/x753k2s01gnd5b bolizarsospos.com/x7lfazpjuuiel bolizarsospos.com/xgw1o6gt9h8k9g bolizarsospos.com/xjp3zmw6glginuq bolizarsospos.com/yias364ajr bolizarsospos.com/zyayxp2kpay bucksmedia.go2cloud.org/aff_c building.msu.ac.th/q3bslr.php businessaviators.com/r1doyf.php challengestrata.com.au/fp_bxs.php cheapshirts.us/zvnmrg.php chong.joelle.free.fr/_l43ph.php connectao.com/wp-content/themes/twentyeleven/cc.php d3mpd.fe.uns.ac.id/xpgmur.php daffamedia.com/wp-content/plugins/wp_module/img5.php dechehang.com/gz2qrn.php definitionen.de/v7gves.php dichiro.com/waird6.php dillardvideo.com/wp-admin/network/2.php dining-bar.com/bq_ln4.php domaine-cassillac.com/4q3esu.php double-wing.de/dzkclr.php drdigitalmd.com/img1.php eatside.es/xzqgxv.php ecocalsots.com/n79gta.php ecolux-comfort.com/npabsy.php elcoachingempresarial.com/wp-admin/user/2.php emprende21.es/otiq7a.php estudiobarco.com.ar/5tfv7e.php event-travel.co.uk/3k6psd.php feuerwehr-stadt-riesa.de/ufipoq.php foundersomaha.net/wp-includes/text/diff/renderer/ap3.php frame3d.de/itgjkd.php fun-pop.com/ks1rcc.php genedillardart.com/wp-admin/network/3.php gibdd.ws/j7d65p.php glitchygaming.com/r07qzu.php grochowina.net/unvpso.php haarsaloncindy.nl/xzf03r.php hamilton150.co.nz/lmfumz.php icsot.na.its.ac.id/8vwrux.php igatha.com/h4mekj.php ilovesport.kiev.ua/z8x9t7.php imagescameraclub.com/j7b5kk.php inspirenetworks.in/vaqu3l.php italyprego.com/lf2dca.php jadwalpialadunia.in/rg4rdi.php jambola.com/luylwv.php jauregia.net/img5.php konyavakfi.nl/zmje1r.php kuruyaprak.com/otluko.php kvnysoho.com/ehafft.php lazymoosestamping.com/yrfbgb.php london-escorts-agency.org.uk/fdnmyd.php lzclient.com/img4.php madisonbootcamps.com/gwq3wp.php mangohills.net/rxioce.php marciogerhardtsouza.com.br/mpcsdz.php marcortes.com/img5.php markossolomon.com/f1q7qx.php maternalserenity.co.uk/i_nwpg.php millsmanagement.nl/anogvk.php mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php naimselmonaj.com/qoyx31.php nonnuoccaobang.com/brdodl.php nupleta.com.br/kohv09.php openroadsolutions.com/fj2dow.php oregonreversemortgage.com/rwafp_.php p237996.mybestmv.com/adserve/domainclick paintituppottery.com/6cmeb2.php patrianossa.com.br/u8lkzd.php portalmaismidia.com.br/tnsmib.php portret-tekening.nl/mnqvts.php procrediti.com.ua/d6ygox.php rajsima87.com/img2.php recaswine.ro/dxlq0y.php silstop.pl/si0ccj.php smartnote.co/2nxvza.php studiolegalecsb.it/iqcnfc.php takaram.ir/gjorez.php takatei.com/rfyi4l.php tcblog.de/mxdvth.php theassemblyguy.co.nz/vpfabq.php trion.com.ph/jdkaap.php tusrecetas.net/jbeln7.php tutorialswalk.info/wp-content/themes/defne/img2.php viralcrazies.com/ifht4c.php vsedveri-33.ru/ weberteam.hu/wctdo5.php www.001edizioni.com/nzwt_a.php www.bishopbell.co.uk/enrmcc.php www.chemes.eu/wp-content/themes/decoy2/redux-framework/reduxcore/inc/fields/info/2.php www.decorandoimoveis.com/qeo5yh.php www.example.com/ www.feddoctor.com/oe1lmr.php www.gjscomputerservices.com.au/s1_rvm.php www.granmarquise.com.br/6f_8ei.php www.hanecaklaw.com/ www.hanoiguidedtours.com/iq2q1f.php www.healthstafftravel.com.au/oybbus.php www.plexipr.com/vahzwx.php www.rippedknees.co.uk/txmjcq.php www.taoblu.com/wp-content/plugins/wp_module/sbml0j.php www.vishvagujarat.com/5of9dt.php www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkg.php ================================================ FILE: tests/integration/basic/URLHC%3Fv%3Dpanosurl.result ================================================ example.com/foobar?a=:80 *.example.com/foobar?a=:80 feyda.net/hoedr4.php *.feyda.net/hoedr4.php 7-eleven-handbags.com/x1rzyp.php 8vs.com/6jezbr.php abdal.com.ua/7_jzay.php aditaborai.com.br/wgngxe.php airconditioning12601.com/uploads/3/5/7/6/3576233/v5k3za.php allreadytravel.com/uploads/3/5/4/9/3549731/header_images/tomae1.php allstarpaintbody.com/lrq2bg.php americancorner.udp.cl/etloxw.php ample-sun.eu/4bket7.php anime-tuner.square7.ch/wp-content/themes/twentyeleven/mstgk_.php anoukdelecluse.nl/lgzlb1.php appeum.com/wp-content/themes/cc.php arot.altervista.org/khtudq.php arttoday.sk/me8mkj.php ascortimisoara.ro/kwih5v.php aspectdesigns.com.au/0rtvlg.php audetlaw.com/lnvadf.php autohaus-seevetal.com/9x6uwk.php avancarvisual.com.br/wp-content/themes/twentytwelve/vzkgnx.php babylicious.ie/s1ghuz.php balkanium.altervista.org/p3er4s.php beachhouseplans.com/wp-admin/js/5d8gme.php best-service.jp/olxu2y.php beyondthedog.net/edhdvf.php bigboattravel.com/uploads/3/5/4/5/3545341/header_images/nthjhz.php bisofit.com/qxwm4i.php bktrade.kiev.ua/76b3zq.php boilersandfurnaces.com/uploads/3/5/1/6/3516773/rpyh2q.php bolizarsospos.com/0l0vp1va6b2 bolizarsospos.com/1cslstk2qv121 bolizarsospos.com/1xb81c28qs2db bolizarsospos.com/22o1210hbpw bolizarsospos.com/2h6t511wpuvnych bolizarsospos.com/379gz635s3j946 bolizarsospos.com/4kpy8ju42x137 bolizarsospos.com/503qu7boexyk bolizarsospos.com/574xl5yme0gdz bolizarsospos.com/5gpf7ecxhf bolizarsospos.com/5hmwl5qvpz2f3gc bolizarsospos.com/6m50uk8ty1031 bolizarsospos.com/6tvpgu93q4wx5t bolizarsospos.com/703hjdr3ez72 bolizarsospos.com/73075bdj8meb bolizarsospos.com/7gr904pzv6 bolizarsospos.com/7ms68qsdfj0jt bolizarsospos.com/89e8f40k8zcn38 bolizarsospos.com/8eo5zwhh4zndwwa bolizarsospos.com/8tsdhjccoxz6c bolizarsospos.com/94g2mr36b4 bolizarsospos.com/9bqdnk2h58ty2l bolizarsospos.com/9hul78mtg1n63 bolizarsospos.com/b0slgvfxvyf bolizarsospos.com/b3amhlkiar2c bolizarsospos.com/b8g7g560612 bolizarsospos.com/bo5ha9ild1zjukv bolizarsospos.com/cannzqzrum14o4c bolizarsospos.com/ch3eq62ad8k bolizarsospos.com/ci72o4ruf2y87 bolizarsospos.com/d5i52z8cgv5 bolizarsospos.com/d65v4fx21f bolizarsospos.com/d7jly5f09tqj bolizarsospos.com/di53su4z7uqvj bolizarsospos.com/dypi31624z bolizarsospos.com/e5tkclwq9w0 bolizarsospos.com/e887nn5k9pb6 bolizarsospos.com/f1s0y87wrwo bolizarsospos.com/fgivit1drjuh bolizarsospos.com/fpkirizbrzxc5 bolizarsospos.com/fxoztyxp320q bolizarsospos.com/g5k4uvxghygg7r bolizarsospos.com/gvi00me81aabu bolizarsospos.com/hq5drme48h bolizarsospos.com/hzpz767vze9 bolizarsospos.com/ifkhfc5369az88 bolizarsospos.com/iijoama0ynrowtp bolizarsospos.com/j4hzoz8cgdeza bolizarsospos.com/kka7641ov7 bolizarsospos.com/ko679ybid6ys58 bolizarsospos.com/kxdmlkhmuyf9 bolizarsospos.com/kzqnheutxkjwhr bolizarsospos.com/mi5b67bilrfu bolizarsospos.com/n2csus3eo1tyg bolizarsospos.com/nve4m67l83 bolizarsospos.com/o0nyjlre41o3 bolizarsospos.com/pgjcokoi2kisu bolizarsospos.com/pl36lz43r6r7 bolizarsospos.com/q3xryv3mh1 bolizarsospos.com/qely217wcjdl7b bolizarsospos.com/qo9ux20lo1 bolizarsospos.com/qu9ajlxsiw bolizarsospos.com/r45byxsjhz bolizarsospos.com/raph9xccgxt bolizarsospos.com/rb05hez1r044 bolizarsospos.com/rdjg0eb5r0qs bolizarsospos.com/ri86nx23dhqbmch bolizarsospos.com/rjotoddb4n7hl bolizarsospos.com/rof06587c1x2y3t bolizarsospos.com/s40o542jt7v bolizarsospos.com/sb2zarf5vy bolizarsospos.com/uamuxps7y98 bolizarsospos.com/uiyi9dkf5bs bolizarsospos.com/v13rw8n8w2 bolizarsospos.com/vzum6ywdedxjtd bolizarsospos.com/walqb5xzunmr bolizarsospos.com/wilqkaz24rnqli bolizarsospos.com/x1tg111bara5 bolizarsospos.com/x753k2s01gnd5b bolizarsospos.com/x7lfazpjuuiel bolizarsospos.com/xgw1o6gt9h8k9g bolizarsospos.com/xjp3zmw6glginuq bolizarsospos.com/yias364ajr bolizarsospos.com/zyayxp2kpay bucksmedia.go2cloud.org/aff_c building.msu.ac.th/q3bslr.php businessaviators.com/r1doyf.php challengestrata.com.au/fp_bxs.php cheapshirts.us/zvnmrg.php chong.joelle.free.fr/_l43ph.php connectao.com/wp-content/themes/twentyeleven/cc.php d3mpd.fe.uns.ac.id/xpgmur.php daffamedia.com/wp-content/plugins/wp_module/img5.php dechehang.com/gz2qrn.php definitionen.de/v7gves.php dichiro.com/waird6.php dillardvideo.com/wp-admin/network/2.php dining-bar.com/bq_ln4.php domaine-cassillac.com/4q3esu.php double-wing.de/dzkclr.php drdigitalmd.com/img1.php eatside.es/xzqgxv.php ecocalsots.com/n79gta.php ecolux-comfort.com/npabsy.php elcoachingempresarial.com/wp-admin/user/2.php emprende21.es/otiq7a.php estudiobarco.com.ar/5tfv7e.php event-travel.co.uk/3k6psd.php feuerwehr-stadt-riesa.de/ufipoq.php foundersomaha.net/wp-includes/text/diff/renderer/ap3.php frame3d.de/itgjkd.php fun-pop.com/ks1rcc.php genedillardart.com/wp-admin/network/3.php gibdd.ws/j7d65p.php glitchygaming.com/r07qzu.php grochowina.net/unvpso.php haarsaloncindy.nl/xzf03r.php hamilton150.co.nz/lmfumz.php icsot.na.its.ac.id/8vwrux.php igatha.com/h4mekj.php ilovesport.kiev.ua/z8x9t7.php imagescameraclub.com/j7b5kk.php inspirenetworks.in/vaqu3l.php italyprego.com/lf2dca.php jadwalpialadunia.in/rg4rdi.php jambola.com/luylwv.php jauregia.net/img5.php konyavakfi.nl/zmje1r.php kuruyaprak.com/otluko.php kvnysoho.com/ehafft.php lazymoosestamping.com/yrfbgb.php london-escorts-agency.org.uk/fdnmyd.php lzclient.com/img4.php madisonbootcamps.com/gwq3wp.php mangohills.net/rxioce.php marciogerhardtsouza.com.br/mpcsdz.php marcortes.com/img5.php markossolomon.com/f1q7qx.php maternalserenity.co.uk/i_nwpg.php millsmanagement.nl/anogvk.php mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php naimselmonaj.com/qoyx31.php nonnuoccaobang.com/brdodl.php nupleta.com.br/kohv09.php openroadsolutions.com/fj2dow.php oregonreversemortgage.com/rwafp_.php p237996.mybestmv.com/adserve/domainclick paintituppottery.com/6cmeb2.php patrianossa.com.br/u8lkzd.php portalmaismidia.com.br/tnsmib.php portret-tekening.nl/mnqvts.php procrediti.com.ua/d6ygox.php rajsima87.com/img2.php recaswine.ro/dxlq0y.php silstop.pl/si0ccj.php smartnote.co/2nxvza.php studiolegalecsb.it/iqcnfc.php takaram.ir/gjorez.php takatei.com/rfyi4l.php tcblog.de/mxdvth.php theassemblyguy.co.nz/vpfabq.php trion.com.ph/jdkaap.php tusrecetas.net/jbeln7.php tutorialswalk.info/wp-content/themes/defne/img2.php viralcrazies.com/ifht4c.php vsedveri-33.ru/ weberteam.hu/wctdo5.php www.001edizioni.com/nzwt_a.php www.bishopbell.co.uk/enrmcc.php www.chemes.eu/wp-content/themes/decoy2/redux-framework/reduxcore/inc/fields/info/2.php www.decorandoimoveis.com/qeo5yh.php www.feddoctor.com/oe1lmr.php www.gjscomputerservices.com.au/s1_rvm.php www.granmarquise.com.br/6f_8ei.php www.hanecaklaw.com/ www.hanoiguidedtours.com/iq2q1f.php www.healthstafftravel.com.au/oybbus.php www.plexipr.com/vahzwx.php www.rippedknees.co.uk/txmjcq.php www.taoblu.com/wp-content/plugins/wp_module/sbml0j.php www.vishvagujarat.com/5of9dt.php www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkg.php ================================================ FILE: tests/integration/basic/URLHC.result ================================================ *abc.example.com/foobar?a=:80 http://*.feyda.net/hOeDr4.php http://7-eleven-handbags.com/X1rZYp.php http://8vs.com/6jezbr.php http://abdal.com.ua/7_jzay.php http://aditaborai.com.br/WgNGXe.php http://airconditioning12601.com/uploads/3/5/7/6/3576233/V5k3Za.php http://allreadytravel.com/uploads/3/5/4/9/3549731/header_images/ToMaE1.php http://allstarpaintbody.com/lrQ2bG.php http://americancorner.udp.cl/etloxW.php http://ample-sun.eu/4BKEt7.php http://anime-tuner.square7.ch/wp-content/themes/twentyeleven/MsTGk_.php http://anoukdelecluse.nl/lGZLB1.php http://appeum.com/wp-content/themes/cc.php http://arot.altervista.org/KHTUdq.php http://arttoday.sk/mE8MKJ.php http://ascortimisoara.ro/kWIH5V.php http://aspectdesigns.com.au/0rTVlG.php http://audetlaw.com/LnVAdF.php http://autohaus-seevetal.com/9x6UwK.php http://avancarvisual.com.br/wp-content/themes/twentytwelve/VzkgnX.php http://babylicious.ie/s1GHUZ.php http://balkanium.altervista.org/p3er4s.php http://beachhouseplans.com/wp-admin/js/5d8gMe.php http://best-service.jp/olxu2Y.php http://beyondthedog.net/edHDvf.php http://bigboattravel.com/uploads/3/5/4/5/3545341/header_images/NthjHz.php http://bisofit.com/QXwm4I.php http://bktrade.kiev.ua/76b3ZQ.php http://boilersandfurnaces.com/uploads/3/5/1/6/3516773/RPyH2q.php http://bolizarsospos.com/0l0vp1va6b2 http://bolizarsospos.com/1cslstk2qv121 http://bolizarsospos.com/1xb81c28qs2db http://bolizarsospos.com/22o1210hbpw http://bolizarsospos.com/2h6t511wpuvnych http://bolizarsospos.com/379gz635s3j946 http://bolizarsospos.com/4kpy8ju42x137 http://bolizarsospos.com/503qu7boexyk http://bolizarsospos.com/574xl5yme0gdz http://bolizarsospos.com/5gpf7ecxhf http://bolizarsospos.com/5hmwl5qvpz2f3gc http://bolizarsospos.com/6m50uk8ty1031 http://bolizarsospos.com/6tvpgu93q4wx5t http://bolizarsospos.com/703hjdr3ez72 http://bolizarsospos.com/73075bdj8meb http://bolizarsospos.com/7gr904pzv6 http://bolizarsospos.com/7ms68qsdfj0jt http://bolizarsospos.com/89e8f40k8zcn38 http://bolizarsospos.com/8eo5zwhh4zndwwa http://bolizarsospos.com/8tsdhjccoxz6c http://bolizarsospos.com/94g2mr36b4 http://bolizarsospos.com/9bqdnk2h58ty2l http://bolizarsospos.com/9hul78mtg1n63 http://bolizarsospos.com/b0slgvfxvyf http://bolizarsospos.com/b3amhlkiar2c http://bolizarsospos.com/b8g7g560612 http://bolizarsospos.com/bo5ha9ild1zjukv http://bolizarsospos.com/cannzqzrum14o4c http://bolizarsospos.com/ch3eq62ad8k http://bolizarsospos.com/ci72o4ruf2y87 http://bolizarsospos.com/d5i52z8cgv5 http://bolizarsospos.com/d65v4fx21f http://bolizarsospos.com/d7jly5f09tqj http://bolizarsospos.com/di53su4z7uqvj http://bolizarsospos.com/dypi31624z http://bolizarsospos.com/e5tkclwq9w0 http://bolizarsospos.com/e887nn5k9pb6 http://bolizarsospos.com/f1s0y87wrwo http://bolizarsospos.com/fgivit1drjuh http://bolizarsospos.com/fpkirizbrzxc5 http://bolizarsospos.com/fxoztyxp320q http://bolizarsospos.com/g5k4uvxghygg7r http://bolizarsospos.com/gvi00me81aabu http://bolizarsospos.com/hq5drme48h http://bolizarsospos.com/hzpz767vze9 http://bolizarsospos.com/ifkhfc5369az88 http://bolizarsospos.com/iijoama0ynrowtp http://bolizarsospos.com/j4hzoz8cgdeza http://bolizarsospos.com/kka7641ov7 http://bolizarsospos.com/ko679ybid6ys58 http://bolizarsospos.com/kxdmlkhmuyf9 http://bolizarsospos.com/kzqnheutxkjwhr http://bolizarsospos.com/mi5b67bilrfu http://bolizarsospos.com/n2csus3eo1tyg http://bolizarsospos.com/nve4m67l83 http://bolizarsospos.com/o0nyjlre41o3 http://bolizarsospos.com/pgjcokoi2kisu http://bolizarsospos.com/pl36lz43r6r7 http://bolizarsospos.com/q3xryv3mh1 http://bolizarsospos.com/qely217wcjdl7b http://bolizarsospos.com/qo9ux20lo1 http://bolizarsospos.com/qu9ajlxsiw http://bolizarsospos.com/r45byxsjhz http://bolizarsospos.com/raph9xccgxt http://bolizarsospos.com/rb05hez1r044 http://bolizarsospos.com/rdjg0eb5r0qs http://bolizarsospos.com/ri86nx23dhqbmch http://bolizarsospos.com/rjotoddb4n7hl http://bolizarsospos.com/rof06587c1x2y3t http://bolizarsospos.com/s40o542jt7v http://bolizarsospos.com/sb2zarf5vy http://bolizarsospos.com/uamuxps7y98 http://bolizarsospos.com/uiyi9dkf5bs http://bolizarsospos.com/v13rw8n8w2 http://bolizarsospos.com/vzum6ywdedxjtd http://bolizarsospos.com/walqb5xzunmr http://bolizarsospos.com/wilqkaz24rnqli http://bolizarsospos.com/x1tg111bara5 http://bolizarsospos.com/x753k2s01gnd5b http://bolizarsospos.com/x7lfazpjuuiel http://bolizarsospos.com/xgw1o6gt9h8k9g http://bolizarsospos.com/xjp3zmw6glginuq http://bolizarsospos.com/yias364ajr http://bolizarsospos.com/zyayxp2kpay http://bucksmedia.go2cloud.org/aff_c http://building.msu.ac.th/q3Bslr.php http://businessaviators.com/r1doyF.php http://challengestrata.com.au/fP_BXS.php http://cheapshirts.us/zVnMrG.php http://chong.joelle.free.fr/_L43PH.php http://connectao.com/wp-content/themes/twentyeleven/cc.php http://d3mpd.fe.uns.ac.id/XPgmur.php http://daffamedia.com/wp-content/plugins/wp_module/img5.php http://dechehang.com/GZ2QRn.php http://definitionen.de/v7GVES.php http://dichiro.com/WaIrd6.php http://dillardvideo.com/wp-admin/network/2.php http://dining-bar.com/BQ_Ln4.php http://domaine-cassillac.com/4q3esU.php http://double-wing.de/DZkCLR.php http://drdigitalmd.com/img1.php http://eatside.es/xZQGXV.php http://ecocalsots.com/N79GTA.php http://ecolux-comfort.com/nPAbsy.php http://elcoachingempresarial.com/wp-admin/user/2.php http://emprende21.es/oTIq7A.php http://estudiobarco.com.ar/5TFv7E.php http://event-travel.co.uk/3K6Psd.php http://feuerwehr-stadt-riesa.de/UFiPOq.php http://foundersomaha.net/wp-includes/Text/Diff/Renderer/ap3.php http://frame3d.de/ItGJKd.php http://fun-pop.com/Ks1rCc.php http://genedillardart.com/wp-admin/network/3.php http://gibdd.ws/J7D65p.php http://glitchygaming.com/r07QZu.php http://grochowina.net/UnvPso.php http://haarsaloncindy.nl/XzF03r.php http://hamilton150.co.nz/LmfuMZ.php http://icsot.na.its.ac.id/8vwRUX.php http://igatha.com/h4MeKJ.php http://ilovesport.kiev.ua/z8X9T7.php http://imagescameraclub.com/j7b5kK.php http://inspirenetworks.in/vAqu3L.php http://italyprego.com/Lf2dcA.php http://jadwalpialadunia.in/rG4Rdi.php http://jambola.com/LuylWV.php http://jauregia.net/img5.php http://konyavakfi.nl/Zmje1r.php http://kuruyaprak.com/OTLuKo.php http://kvnysoho.com/eHafFT.php http://lazymoosestamping.com/YRfbgB.php http://london-escorts-agency.org.uk/fdnmyD.php http://lzclient.com/img4.php http://madisonbootcamps.com/gWQ3wp.php http://mangohills.net/RxIoCE.php http://marciogerhardtsouza.com.br/mPCsDz.php http://marcortes.com/img5.php http://markossolomon.com/F1q7QX.php http://maternalserenity.co.uk/I_NwPg.php http://millsmanagement.nl/AnOgVK.php http://mmcomposite.dk/wp-content/plugins/js_composer/assets/lib/prettyphoto/images/1.php http://naimselmonaj.com/QoYx31.php http://nonnuoccaobang.com/BRdoDL.php http://nupleta.com.br/KoHV09.php http://openroadsolutions.com/FJ2dOw.php http://oregonreversemortgage.com/Rwafp_.php http://p237996.mybestmv.com/adServe/domainClick http://paintituppottery.com/6cmeb2.php http://patrianossa.com.br/u8LkzD.php http://portalmaismidia.com.br/tnSmIb.php http://portret-tekening.nl/mNQVts.php http://procrediti.com.ua/d6yGOX.php http://rajsima87.com/img2.php http://recaswine.ro/dXlq0Y.php http://silstop.pl/Si0cCJ.php http://smartnote.co/2NxVzA.php http://studiolegalecsb.it/iQcNfC.php http://takaram.ir/gjOREZ.php http://takatei.com/rfYI4L.php http://tcblog.de/mXdVTh.php http://theassemblyguy.co.nz/vpFAbQ.php http://trion.com.ph/jdKAap.php http://tusrecetas.net/JbElN7.php http://tutorialswalk.info/wp-content/themes/Defne/img2.php http://viralcrazies.com/iFHt4C.php http://vsedveri-33.ru/ http://weberteam.hu/WCTdO5.php http://www.001edizioni.com/NZwt_a.php http://www.bishopbell.co.uk/enRmcC.php http://www.chemes.eu/wp-content/themes/decoy2/redux-framework/ReduxCore/inc/fields/info/2.php http://www.decorandoimoveis.com/QEO5yh.php http://www.example.com:1000 http://www.feddoctor.com/Oe1LMr.php http://www.gjscomputerservices.com.au/S1_rvm.php http://www.granmarquise.com.br/6f_8ei.php http://www.hanecaklaw.com/ http://www.hanoiguidedtours.com/iQ2q1f.php http://www.healthstafftravel.com.au/oyBbUs.php http://www.plexipr.com/vAHzWX.php http://www.rippedknees.co.uk/TXmJcq.php http://www.taoblu.com/wp-content/plugins/wp_module/sbML0j.php http://www.vishvagujarat.com/5of9dt.php http://www.weddingsonthefrenchriviera.com/wp-content/uploads/bcswkG.php https://*abc.*test.com ================================================ FILE: tests/integration/basic/domain.lst ================================================ 25z5g623wpqpdwis.onion.to 27c73bq66y4xqoh7.dorfact.at 27lelchgcvs2wpm7.3lhjyx.top 27lelchgcvs2wpm7.7jiff7.top 27lelchgcvs2wpm7.7zv8o2.top 27lelchgcvs2wpm7.9ildst.top 27lelchgcvs2wpm7.adevf4.top 27lelchgcvs2wpm7.ag082d.top 27lelchgcvs2wpm7.apperloads.win 27lelchgcvs2wpm7.asd3r3.top 27lelchgcvs2wpm7.b7mciu.top 27lelchgcvs2wpm7.bedrastic.bid 27lelchgcvs2wpm7.bestfordownload.click 27lelchgcvs2wpm7.bonbestal.asia 27lelchgcvs2wpm7.fm0cga.top 27lelchgcvs2wpm7.h9ihx3.top 27lelchgcvs2wpm7.laverhants.link 27lelchgcvs2wpm7.liopakerb.black 27lelchgcvs2wpm7.marksgain.kim 27lelchgcvs2wpm7.nfgpeb.top 27lelchgcvs2wpm7.redefined.click 27lelchgcvs2wpm7.rt4e34.win 27lelchgcvs2wpm7.tankbe.pro 27lelchgcvs2wpm7.thyx30.top 27lelchgcvs2wpm7.uboys5.top 27lelchgcvs2wpm7.vrid8l.top 27lelchgcvs2wpm7.wins4n.win 27lelchgcvs2wpm7.wishsends.mobi 27lelchgcvs2wpm7.xkfi59.top 27lelchgcvs2wpm7.xmvr54.top 2bdfb.spinakrosa.at 2gdb4.leoraorage.at 2ymh2gnnbg6pgq2r.gremsot.pl 2ymh2gnnbg6pgq2r.winregion.tw 32kl2rwsjvqjeui7.onion.cab 32kl2rwsjvqjeui7.onion.to 32kl2rwsjvqjeui7.tor2web.org 37kddsserrt.xyz 3qbyaoohkcqkzrz6.bestxprice.ch 3qbyaoohkcqkzrz6.livecamshow.ch 3qbyaoohkcqkzrz6.torclassik.li 3qbyaoohkcqkzrz6.torcommunity.ch 3qbyaoohkcqkzrz6.tordonator.li 3qbyaoohkcqkzrz6.tordoor.li 3qbyaoohkcqkzrz6.torgate.es 3qbyaoohkcqkzrz6.torgateway.li 3qbyaoohkcqkzrz6.tormain.li 3qbyaoohkcqkzrz6.tormaster.ch 3qbyaoohkcqkzrz6.tormaster.fr 3qbyaoohkcqkzrz6.torplanet.eu 3qbyaoohkcqkzrz6.torprovider.li 3qbyaoohkcqkzrz6.torreactor.li 3qbyaoohkcqkzrz6.torstation.li 4kqd3hmqgptupi3p.0vgu64.top 4kqd3hmqgptupi3p.143h2a.top 4kqd3hmqgptupi3p.1tvjk1.top 4kqd3hmqgptupi3p.1zp109.bid 4kqd3hmqgptupi3p.249isv.bid 4kqd3hmqgptupi3p.2y4t6f.bid 4kqd3hmqgptupi3p.3arvfd.top 4kqd3hmqgptupi3p.3lhjyx.top 4kqd3hmqgptupi3p.43wjor.top 4kqd3hmqgptupi3p.4j11jt.bid 4kqd3hmqgptupi3p.4k9xlx.top 4kqd3hmqgptupi3p.5b4ej6.bid 4kqd3hmqgptupi3p.5ctoeb.bid 4kqd3hmqgptupi3p.62er3d.top 4kqd3hmqgptupi3p.6h03gw.top 4kqd3hmqgptupi3p.6j7jcn.bid 4kqd3hmqgptupi3p.6ntrb6.top 4kqd3hmqgptupi3p.6ogy3i.top 4kqd3hmqgptupi3p.7w9p1n.bid 4kqd3hmqgptupi3p.859rkn.top 4kqd3hmqgptupi3p.8kcfnk.bid 4kqd3hmqgptupi3p.91006j.bid 4kqd3hmqgptupi3p.9ildst.top 4kqd3hmqgptupi3p.a0g0o7.bid 4kqd3hmqgptupi3p.adevf4.top 4kqd3hmqgptupi3p.anypicked.red 4kqd3hmqgptupi3p.as5su5.top 4kqd3hmqgptupi3p.asfall.in 4kqd3hmqgptupi3p.athere.in 4kqd3hmqgptupi3p.b7mciu.top 4kqd3hmqgptupi3p.barberryshin.casa 4kqd3hmqgptupi3p.bestergo.pw 4kqd3hmqgptupi3p.bigfooters.loan 4kqd3hmqgptupi3p.bnctf6.top 4kqd3hmqgptupi3p.bookjumps.us 4kqd3hmqgptupi3p.boxsame.kim 4kqd3hmqgptupi3p.boxtimed.gdn 4kqd3hmqgptupi3p.breakown.loan 4kqd3hmqgptupi3p.byeraser.lol 4kqd3hmqgptupi3p.carrygain.kim 4kqd3hmqgptupi3p.cfu46r.bid 4kqd3hmqgptupi3p.chargecar.vip 4kqd3hmqgptupi3p.choiceher.win 4kqd3hmqgptupi3p.clockhate.loan 4kqd3hmqgptupi3p.cm5ohx.bid 4kqd3hmqgptupi3p.csv7o6.bid 4kqd3hmqgptupi3p.cutslifes.bid 4kqd3hmqgptupi3p.dd4xo3.top 4kqd3hmqgptupi3p.dkrie7.top 4kqd3hmqgptupi3p.dmvute.top 4kqd3hmqgptupi3p.dozensby.loan 4kqd3hmqgptupi3p.easyits.black 4kqd3hmqgptupi3p.effortany.win 4kqd3hmqgptupi3p.endsdoubt.loan 4kqd3hmqgptupi3p.eventeach.gdn 4kqd3hmqgptupi3p.ezm0r5.top 4kqd3hmqgptupi3p.f0jlbj.bid 4kqd3hmqgptupi3p.fairlies.link 4kqd3hmqgptupi3p.foodtopic.mobi 4kqd3hmqgptupi3p.g7kcux.bid 4kqd3hmqgptupi3p.gameswarm.loan 4kqd3hmqgptupi3p.gapplayed.link 4kqd3hmqgptupi3p.getsbug.kim 4kqd3hmqgptupi3p.gg4dgp.bid 4kqd3hmqgptupi3p.gio6f6.bid 4kqd3hmqgptupi3p.gletterstan.trade 4kqd3hmqgptupi3p.goodslet.win 4kqd3hmqgptupi3p.goshare.red 4kqd3hmqgptupi3p.gs2ka7.top 4kqd3hmqgptupi3p.he81tz.bid 4kqd3hmqgptupi3p.heardbids.date 4kqd3hmqgptupi3p.heldbegun.kim 4kqd3hmqgptupi3p.hessale.pw 4kqd3hmqgptupi3p.holescase.pw 4kqd3hmqgptupi3p.homehuge.top 4kqd3hmqgptupi3p.hotcopies.bid 4kqd3hmqgptupi3p.inforcing.pw 4kqd3hmqgptupi3p.insystem.men 4kqd3hmqgptupi3p.itdrink.club 4kqd3hmqgptupi3p.ix1upt.bid 4kqd3hmqgptupi3p.jal9lk.bid 4kqd3hmqgptupi3p.k7oud1.top 4kqd3hmqgptupi3p.kml2o2.top 4kqd3hmqgptupi3p.l6k4x7.bid 4kqd3hmqgptupi3p.laterugly.win 4kqd3hmqgptupi3p.liescale.in 4kqd3hmqgptupi3p.liesshall.bid 4kqd3hmqgptupi3p.lobulz.bid 4kqd3hmqgptupi3p.lorrydo.lol 4kqd3hmqgptupi3p.masterany.red 4kqd3hmqgptupi3p.meetbinds.pw 4kqd3hmqgptupi3p.metmet.win 4kqd3hmqgptupi3p.metpast.site 4kqd3hmqgptupi3p.mi3596.bid 4kqd3hmqgptupi3p.mtxtul.top 4kqd3hmqgptupi3p.mustspace.us 4kqd3hmqgptupi3p.myaddress.link 4kqd3hmqgptupi3p.namefalls.pro 4kqd3hmqgptupi3p.nameuser.site 4kqd3hmqgptupi3p.nearlybut.us 4kqd3hmqgptupi3p.needmight.win 4kqd3hmqgptupi3p.newrange.link 4kqd3hmqgptupi3p.nextask.loan 4kqd3hmqgptupi3p.nh47ri.bid 4kqd3hmqgptupi3p.nxmu0x.bid 4kqd3hmqgptupi3p.o8hpwj.top 4kqd3hmqgptupi3p.outputon.asia 4kqd3hmqgptupi3p.ownamount.pro 4kqd3hmqgptupi3p.p79b8l.bid 4kqd3hmqgptupi3p.pairsraw.loan 4kqd3hmqgptupi3p.pap44w.top 4kqd3hmqgptupi3p.powersno.link 4kqd3hmqgptupi3p.pushstory.bid 4kqd3hmqgptupi3p.r21wmw.top 4kqd3hmqgptupi3p.rsi6gn.top 4kqd3hmqgptupi3p.salethe.gdn 4kqd3hmqgptupi3p.sayssales.bid 4kqd3hmqgptupi3p.scoreable.bid 4kqd3hmqgptupi3p.seemby.loan 4kqd3hmqgptupi3p.sel7rg.bid 4kqd3hmqgptupi3p.selfcrash.site 4kqd3hmqgptupi3p.sentowing.trade 4kqd3hmqgptupi3p.sitcalls.us 4kqd3hmqgptupi3p.sk8r54.top 4kqd3hmqgptupi3p.somegave.info 4kqd3hmqgptupi3p.stageend.link 4kqd3hmqgptupi3p.stopsage.gdn 4kqd3hmqgptupi3p.storingus.gdn 4kqd3hmqgptupi3p.tankplain.date 4kqd3hmqgptupi3p.termprior.men 4kqd3hmqgptupi3p.themevery.win 4kqd3hmqgptupi3p.thyx30.top 4kqd3hmqgptupi3p.tieslaws.link 4kqd3hmqgptupi3p.todaynine.loan 4kqd3hmqgptupi3p.twz1ga.top 4kqd3hmqgptupi3p.uwckha.top 4kqd3hmqgptupi3p.v11z5e.top 4kqd3hmqgptupi3p.valueshes.bid 4kqd3hmqgptupi3p.variedtax.kim 4kqd3hmqgptupi3p.vkm4l6.top 4kqd3hmqgptupi3p.wallluck.date 4kqd3hmqgptupi3p.whmykv.bid 4kqd3hmqgptupi3p.wins4n.top 4kqd3hmqgptupi3p.wz139z.top 4kqd3hmqgptupi3p.xmfru5.top 4kqd3hmqgptupi3p.y12acl.bid 4kqd3hmqgptupi3p.y5j7e6.top 4kqd3hmqgptupi3p.yg767p.bid 4kqd3hmqgptupi3p.yoursdoor.lol 4kqd3hmqgptupi3p.z8ijgn.bid 4kqd3hmqgptupi3p.z97f9v.bid 4rebaopfgrewe.top 4w5wihkwyhsav2ha.dreamtest.at 4w5wihkwyhsav2ha.fastdances.at 4w5wihkwyhsav2ha.grandhaus.at 4w5wihkwyhsav2ha.payfactor.at 52uo5k3t73ypjije.01fake.bid 52uo5k3t73ypjije.086ux2.top 52uo5k3t73ypjije.0n5joc.top 52uo5k3t73ypjije.0nyi6l.bid 52uo5k3t73ypjije.0vgu64.top 52uo5k3t73ypjije.11pmnz.top 52uo5k3t73ypjije.1bipa9.top 52uo5k3t73ypjije.1de02r.top 52uo5k3t73ypjije.1f1dw3.bid 52uo5k3t73ypjije.1g0vo2.bid 52uo5k3t73ypjije.1pma4t.bid 52uo5k3t73ypjije.1ufr2v.bid 52uo5k3t73ypjije.209kai.bid 52uo5k3t73ypjije.249isv.bid 52uo5k3t73ypjije.26lpul.bid 52uo5k3t73ypjije.2gbbja.top 52uo5k3t73ypjije.2llgoy.bid 52uo5k3t73ypjije.2y4t6f.bid 52uo5k3t73ypjije.2ym6om.bid 52uo5k3t73ypjije.31wkhu.top 52uo5k3t73ypjije.33dofy.top 52uo5k3t73ypjije.35u068.bid 52uo5k3t73ypjije.3di24a.top 52uo5k3t73ypjije.3gpdgx.bid 52uo5k3t73ypjije.3lhjyx.top 52uo5k3t73ypjije.3rr6ao.top 52uo5k3t73ypjije.3zotov.bid 52uo5k3t73ypjije.40wiai.top 52uo5k3t73ypjije.43l7lm.bid 52uo5k3t73ypjije.43wjor.top 52uo5k3t73ypjije.495iru.top 52uo5k3t73ypjije.4jub4e.bid 52uo5k3t73ypjije.4k9xlx.top 52uo5k3t73ypjije.4n592s.top 52uo5k3t73ypjije.4nf7ij.top 52uo5k3t73ypjije.4oyhvh.top 52uo5k3t73ypjije.4pjetv.bid 52uo5k3t73ypjije.4xiiup.bid 52uo5k3t73ypjije.4yl1hr.bid 52uo5k3t73ypjije.4ynpjd.top 52uo5k3t73ypjije.50cs7p.bid 52uo5k3t73ypjije.56185u.bid 52uo5k3t73ypjije.5ctoeb.bid 52uo5k3t73ypjije.5ittco.bid 52uo5k3t73ypjije.5kb3dl.top 52uo5k3t73ypjije.5o4bjf.bid 52uo5k3t73ypjije.5tb8hy.bid 52uo5k3t73ypjije.5vhk5r.bid 52uo5k3t73ypjije.5zxii2.bid 52uo5k3t73ypjije.62er3d.top 52uo5k3t73ypjije.68xmf9.bid 52uo5k3t73ypjije.6ec2xb.bid 52uo5k3t73ypjije.6j7jcn.bid 52uo5k3t73ypjije.6w3rkc.bid 52uo5k3t73ypjije.7156et.bid 52uo5k3t73ypjije.7asel7.top 52uo5k3t73ypjije.7j6htz.bid 52uo5k3t73ypjije.7jiff7.top 52uo5k3t73ypjije.7ud98m.bid 52uo5k3t73ypjije.7wrwp4.top 52uo5k3t73ypjije.80yabh.bid 52uo5k3t73ypjije.86rhzr.bid 52uo5k3t73ypjije.8a0sf6.top 52uo5k3t73ypjije.8cjlyt.bid 52uo5k3t73ypjije.8hphyr.top 52uo5k3t73ypjije.8i8dt4.top 52uo5k3t73ypjije.8kcfnk.bid 52uo5k3t73ypjije.8rrxd9.bid 52uo5k3t73ypjije.8rxv74.bid 52uo5k3t73ypjije.91006j.bid 52uo5k3t73ypjije.94ycl8.bid 52uo5k3t73ypjije.95ovzy.top 52uo5k3t73ypjije.9bjnlk.bid 52uo5k3t73ypjije.9cd81s.bid 52uo5k3t73ypjije.9ildst.top 52uo5k3t73ypjije.9kxz23.bid 52uo5k3t73ypjije.9nj8ex.top 52uo5k3t73ypjije.9sfrr0.bid 52uo5k3t73ypjije.9tftgh.bid 52uo5k3t73ypjije.a0g0o7.bid 52uo5k3t73ypjije.a2uzpe.top 52uo5k3t73ypjije.aclox4.bid 52uo5k3t73ypjije.ahvshc.top 52uo5k3t73ypjije.ai7hur.bid 52uo5k3t73ypjije.ajolkg.bid 52uo5k3t73ypjije.aryh7f.bid 52uo5k3t73ypjije.asxjdp.top 52uo5k3t73ypjije.b2s4ch.bid 52uo5k3t73ypjije.b7mciu.top 52uo5k3t73ypjije.b8ll6n.top 52uo5k3t73ypjije.bar8sc.bid 52uo5k3t73ypjije.bcjl1h.top 52uo5k3t73ypjije.bipa9k.bid 52uo5k3t73ypjije.bipnnp.bid 52uo5k3t73ypjije.bj9eea.bid 52uo5k3t73ypjije.bnctf6.top 52uo5k3t73ypjije.bp9mn8.bid 52uo5k3t73ypjije.bt7r70.top 52uo5k3t73ypjije.c3fz3z.bid 52uo5k3t73ypjije.c7ex9n.top 52uo5k3t73ypjije.catfills.mobi 52uo5k3t73ypjije.cc0r87.bid 52uo5k3t73ypjije.cfu46r.bid 52uo5k3t73ypjije.cjc2jn.top 52uo5k3t73ypjije.cm5ohx.bid 52uo5k3t73ypjije.cm898n.bid 52uo5k3t73ypjije.cmfkru.top 52uo5k3t73ypjije.cpvwgx.bid 52uo5k3t73ypjije.csdbnk.bid 52uo5k3t73ypjije.csj0k5.top 52uo5k3t73ypjije.csv7o6.bid 52uo5k3t73ypjije.cto5ee.bid 52uo5k3t73ypjije.czzg7f.bid 52uo5k3t73ypjije.daigy0.top 52uo5k3t73ypjije.das34.com 52uo5k3t73ypjije.dd4xo3.top 52uo5k3t73ypjije.ddwub3.top 52uo5k3t73ypjije.deg5xr.top 52uo5k3t73ypjije.dkrie7.top 52uo5k3t73ypjije.dkriur.top 52uo5k3t73ypjije.dkro3u.top 52uo5k3t73ypjije.dmrueo.top 52uo5k3t73ypjije.dmvute.top 52uo5k3t73ypjije.dsv023.bid 52uo5k3t73ypjije.dvuybv.bid 52uo5k3t73ypjije.e32d1o.bid 52uo5k3t73ypjije.e6in0v.top 52uo5k3t73ypjije.e78hjo.bid 52uo5k3t73ypjije.e8hua8.top 52uo5k3t73ypjije.ei9evn.top 52uo5k3t73ypjije.en3oyw.bid 52uo5k3t73ypjije.eoivrm.bid 52uo5k3t73ypjije.ep493u.top 52uo5k3t73ypjije.er05vm.bid 52uo5k3t73ypjije.ezm0r5.top 52uo5k3t73ypjije.f0jlbj.bid 52uo5k3t73ypjije.f242v5.bid 52uo5k3t73ypjije.f3z72p.bid 52uo5k3t73ypjije.fe98iy.top 52uo5k3t73ypjije.fi50le.bid 52uo5k3t73ypjije.fkgrie.top 52uo5k3t73ypjije.g0ots2.top 52uo5k3t73ypjije.g0spln.bid 52uo5k3t73ypjije.g5196b.bid 52uo5k3t73ypjije.gg4dgp.bid 52uo5k3t73ypjije.gio6f6.bid 52uo5k3t73ypjije.givxuf.bid 52uo5k3t73ypjije.gmnjz7.bid 52uo5k3t73ypjije.gnee6i.top 52uo5k3t73ypjije.gnuvaw.bid 52uo5k3t73ypjije.goztus.bid 52uo5k3t73ypjije.gpy3tc.top 52uo5k3t73ypjije.gtnfgj.top 52uo5k3t73ypjije.gu7eao.bid 52uo5k3t73ypjije.gvoafg.bid 52uo5k3t73ypjije.h3ss4t.bid 52uo5k3t73ypjije.hawtzr.bid 52uo5k3t73ypjije.hbd7m4.bid 52uo5k3t73ypjije.hhc366.bid 52uo5k3t73ypjije.hlu8yz.top 52uo5k3t73ypjije.hossy3.bid 52uo5k3t73ypjije.hv42mo.bid 52uo5k3t73ypjije.i5cgcw.top 52uo5k3t73ypjije.i6gn9s.bid 52uo5k3t73ypjije.i8zh1k.bid 52uo5k3t73ypjije.iait3w.bid 52uo5k3t73ypjije.ibngww.top 52uo5k3t73ypjije.ie7t8k.top 52uo5k3t73ypjije.ih9te2.bid 52uo5k3t73ypjije.ij0cia.bid 52uo5k3t73ypjije.imhhwm.top 52uo5k3t73ypjije.insystem.men 52uo5k3t73ypjije.izyclz.bid 52uo5k3t73ypjije.j8873f.bid 52uo5k3t73ypjije.j92msu.top 52uo5k3t73ypjije.jal9lk.bid 52uo5k3t73ypjije.jg6jtw.top 52uo5k3t73ypjije.js43vy.bid 52uo5k3t73ypjije.k0dcd2.bid 52uo5k3t73ypjije.k21zey.bid 52uo5k3t73ypjije.k56185.top 52uo5k3t73ypjije.k7oud1.top 52uo5k3t73ypjije.k8ytej.bid 52uo5k3t73ypjije.k9z7pm.top 52uo5k3t73ypjije.ka0te8.top 52uo5k3t73ypjije.kas17.com 52uo5k3t73ypjije.kcufx4.top 52uo5k3t73ypjije.kml2o2.top 52uo5k3t73ypjije.kswcuk.top 52uo5k3t73ypjije.kt70uk.bid 52uo5k3t73ypjije.ku824r.bid 52uo5k3t73ypjije.kwnw1b.bid 52uo5k3t73ypjije.kyjw0g.bid 52uo5k3t73ypjije.kzhzuc.top 52uo5k3t73ypjije.kzo8mc.top 52uo5k3t73ypjije.kzwor6.top 52uo5k3t73ypjije.l6ry3h.bid 52uo5k3t73ypjije.laugk2.top 52uo5k3t73ypjije.lba61x.top 52uo5k3t73ypjije.ldsl8m.bid 52uo5k3t73ypjije.lethints.date 52uo5k3t73ypjije.lh9ax3.bid 52uo5k3t73ypjije.li8wfu.bid 52uo5k3t73ypjije.lib2vi.top 52uo5k3t73ypjije.lio2wr.bid 52uo5k3t73ypjije.loanshown.info 52uo5k3t73ypjije.lrraca.bid 52uo5k3t73ypjije.lwbi59.top 52uo5k3t73ypjije.m33d4b.bid 52uo5k3t73ypjije.m5fgoi.top 52uo5k3t73ypjije.m6j75a.bid 52uo5k3t73ypjije.mbwxyg.bid 52uo5k3t73ypjije.mfgb1h.top 52uo5k3t73ypjije.mn1kms.bid 52uo5k3t73ypjije.msu96b.top 52uo5k3t73ypjije.mtxtul.top 52uo5k3t73ypjije.myurv5.bid 52uo5k3t73ypjije.n41n1a.top 52uo5k3t73ypjije.n6kswi.top 52uo5k3t73ypjije.n8niwa.bid 52uo5k3t73ypjije.nb83bp.bid 52uo5k3t73ypjije.neekll.bid 52uo5k3t73ypjije.nh47ri.bid 52uo5k3t73ypjije.nmapwy.bid 52uo5k3t73ypjije.nxmu0x.bid 52uo5k3t73ypjije.o08a6d.top 52uo5k3t73ypjije.o0hwme.bid 52uo5k3t73ypjije.o5xcnd.bid 52uo5k3t73ypjije.o6fa2g.bid 52uo5k3t73ypjije.o8hpwj.bid 52uo5k3t73ypjije.o8hpwj.top 52uo5k3t73ypjije.o9w43w.bid 52uo5k3t73ypjije.oef1sh.bid 52uo5k3t73ypjije.ojesoa.bid 52uo5k3t73ypjije.ojx58b.bid 52uo5k3t73ypjije.omrexj.top 52uo5k3t73ypjije.ooulp2.bid 52uo5k3t73ypjije.ovpgod.top 52uo5k3t73ypjije.p0lxvm.bid 52uo5k3t73ypjije.p2lsgr.top 52uo5k3t73ypjije.p5dxeh.bid 52uo5k3t73ypjije.pap44w.top 52uo5k3t73ypjije.pfija1.bid 52uo5k3t73ypjije.pop81.com 52uo5k3t73ypjije.poplenjohs.review 52uo5k3t73ypjije.pr2zwz.bid 52uo5k3t73ypjije.r21wmw.top 52uo5k3t73ypjije.r2ok0b.bid 52uo5k3t73ypjije.r4z3o5.bid 52uo5k3t73ypjije.rdmwha.bid 52uo5k3t73ypjije.red4is.top 52uo5k3t73ypjije.rexjyp.bid 52uo5k3t73ypjije.rgdk0u.top 52uo5k3t73ypjije.rl0bdw.top 52uo5k3t73ypjije.rnkj09.top 52uo5k3t73ypjije.rv50gt.bid 52uo5k3t73ypjije.s2xb1s.bid 52uo5k3t73ypjije.sdfztr.bid 52uo5k3t73ypjije.self56.top 52uo5k3t73ypjije.sg62es.top 52uo5k3t73ypjije.skri59.top 52uo5k3t73ypjije.snwy26.top 52uo5k3t73ypjije.sotn58.bid 52uo5k3t73ypjije.srmlzh.bid 52uo5k3t73ypjije.ssh3ln.bid 52uo5k3t73ypjije.sx90yk.bid 52uo5k3t73ypjije.sxjdpg.bid 52uo5k3t73ypjije.thyx30.top 52uo5k3t73ypjije.ti4wic.top 52uo5k3t73ypjije.to6maq.top 52uo5k3t73ypjije.twz1ga.top 52uo5k3t73ypjije.txszfs.top 52uo5k3t73ypjije.tzgwdf.top 52uo5k3t73ypjije.u2r7tm.bid 52uo5k3t73ypjije.u36ik0.bid 52uo5k3t73ypjije.u50s89.bid 52uo5k3t73ypjije.ujtwhg.top 52uo5k3t73ypjije.ul8ib9.bid 52uo5k3t73ypjije.un8niw.top 52uo5k3t73ypjije.uv39h5.bid 52uo5k3t73ypjije.uw3r6a.top 52uo5k3t73ypjije.uw7w05.bid 52uo5k3t73ypjije.uwazu7.bid 52uo5k3t73ypjije.uwckha.bid 52uo5k3t73ypjije.uwckha.top 52uo5k3t73ypjije.ux93ip.top 52uo5k3t73ypjije.v11z5e.top 52uo5k3t73ypjije.v9y6z8.bid 52uo5k3t73ypjije.veupl2.top 52uo5k3t73ypjije.vkm4l6.top 52uo5k3t73ypjije.vkslju.bid 52uo5k3t73ypjije.vlo18w.bid 52uo5k3t73ypjije.vmotsf.bid 52uo5k3t73ypjije.vor28o.bid 52uo5k3t73ypjije.vt3dg6.bid 52uo5k3t73ypjije.w6sj06.bid 52uo5k3t73ypjije.w8yolm.bid 52uo5k3t73ypjije.wg00sp.bid 52uo5k3t73ypjije.whmykv.bid 52uo5k3t73ypjije.whosewine.lol 52uo5k3t73ypjije.wht5py.top 52uo5k3t73ypjije.wins4n.win 52uo5k3t73ypjije.wl52rt.bid 52uo5k3t73ypjije.wrd4fo.top 52uo5k3t73ypjije.ws1uet.top 52uo5k3t73ypjije.wz139z.top 52uo5k3t73ypjije.x2kl7t.top 52uo5k3t73ypjije.x3nnbd.top 52uo5k3t73ypjije.x7fylp.bid 52uo5k3t73ypjije.x9a6yb.bid 52uo5k3t73ypjije.x9kjcn.bid 52uo5k3t73ypjije.x9le66.top 52uo5k3t73ypjije.xab7m0.top 52uo5k3t73ypjije.xglk6h.bid 52uo5k3t73ypjije.xjb384.bid 52uo5k3t73ypjije.xmfru5.top 52uo5k3t73ypjije.xtppp8.bid 52uo5k3t73ypjije.y12acl.bid 52uo5k3t73ypjije.y5j7e6.top 52uo5k3t73ypjije.ye42cp.bid 52uo5k3t73ypjije.yg767p.bid 52uo5k3t73ypjije.yn8krm.bid 52uo5k3t73ypjije.yrd7v5.bid 52uo5k3t73ypjije.yty0gm.bid 52uo5k3t73ypjije.yv7l4b.top 52uo5k3t73ypjije.yw4629.top 52uo5k3t73ypjije.ywszbe.bid 52uo5k3t73ypjije.z6a7f1.bid 52uo5k3t73ypjije.z8ijgn.bid 52uo5k3t73ypjije.z97f9v.bid 52uo5k3t73ypjije.zclw5i.top 52uo5k3t73ypjije.zcwrhe.bid 52uo5k3t73ypjije.zd3p2g.top 52uo5k3t73ypjije.zda7bk.top 52uo5k3t73ypjije.zed84j.bid 52uo5k3t73ypjije.zhvlh1.bid 52uo5k3t73ypjije.zxtezv.bid 52uo5k3t73ypjije.zzis8p.bid 5rport45vcdef345adfkksawe.bematvocal.at 6dtxgqam4crv6rr6.onion.cab 6g4ds.froekuge.com 74nfnjhlq45nkgws4hbdbk45wekfjhqw4talefgnv.curryfort.at 88fga.ketteaero.com 8b4bb47tiaolhy4uhhlfaqerg.sofarany.at 94dbhbj3l4blaeyfgl7q45glbaer.giponfeste.at 974gfbjhb23hbfkyfaby3byqlyuebvly5q254y.mendilobo.com 9hrds.wolfcrap.at a64gfdsjhb4htbiwaysbdvukyft5q.zobodine.at aa12111.top aarnknthc.xyz abvtqhwodwjmi.work acbstypdrijslr.ru accemfsqovkd.pw acjhwpdjhlhbncf.click aechjic.pw ahsqbeospcdrngfv.info ahuqfrqk54v3vnzj.1vcxfn.bid ahuqfrqk54v3vnzj.45yu0p.bid ahuqfrqk54v3vnzj.4h16v3.top ahuqfrqk54v3vnzj.6avw2a.bid ahuqfrqk54v3vnzj.7y1266.top ahuqfrqk54v3vnzj.8kiec2.top ahuqfrqk54v3vnzj.9sfk22.bid ahuqfrqk54v3vnzj.bds4sn.top ahuqfrqk54v3vnzj.bz7k7l.top ahuqfrqk54v3vnzj.c8jxpp.top ahuqfrqk54v3vnzj.cb3pul.top ahuqfrqk54v3vnzj.dxzr2l.top ahuqfrqk54v3vnzj.ewg6uf.bid ahuqfrqk54v3vnzj.g4dc5s.bid ahuqfrqk54v3vnzj.h4lu4i.bid ahuqfrqk54v3vnzj.i81wik.bid ahuqfrqk54v3vnzj.kj3f52.bid ahuqfrqk54v3vnzj.l7g2sv.bid ahuqfrqk54v3vnzj.n3oyw7.bid ahuqfrqk54v3vnzj.roep3o.top ahuqfrqk54v3vnzj.sg9lxh.bid ahuqfrqk54v3vnzj.tjubo1.top ahuqfrqk54v3vnzj.u9fcji.bid ahuqfrqk54v3vnzj.uzeb6r.bid ahuqfrqk54v3vnzj.v5neyw.bid ahuqfrqk54v3vnzj.vgxcci.top ahuqfrqk54v3vnzj.x90yk1.bid ahuqfrqk54v3vnzj.xs2xeh.bid ahuqfrqk54v3vnzj.zn90h4.bid ampjsppmftmfdblpt.info anbqjdoyw6wkmpeu.oldtrees.at applesnoutsthings.bid aqmip.fr arddxjkwrp.xyz as3ws.fopyirr.com avsxrcoq2q5fgrw2.13inb1.top avsxrcoq2q5fgrw2.17vj7b.top avsxrcoq2q5fgrw2.199ovv.top avsxrcoq2q5fgrw2.1gtx3p.top avsxrcoq2q5fgrw2.1mwipu.top avsxrcoq2q5fgrw2.1nsnuh.top avsxrcoq2q5fgrw2.2wfe60.top avsxrcoq2q5fgrw2.5m2n7x.top avsxrcoq2q5fgrw2.5s96fr.top avsxrcoq2q5fgrw2.79j8fm.top avsxrcoq2q5fgrw2.8l4jpw.top avsxrcoq2q5fgrw2.9c431m.bid avsxrcoq2q5fgrw2.arpbxw.top avsxrcoq2q5fgrw2.ayjy5d.top avsxrcoq2q5fgrw2.dgjpgy.top avsxrcoq2q5fgrw2.et7izd.top avsxrcoq2q5fgrw2.ewg6uf.bid avsxrcoq2q5fgrw2.h44l3d.bid avsxrcoq2q5fgrw2.ihuk7s.top avsxrcoq2q5fgrw2.j4cser.bid avsxrcoq2q5fgrw2.lbxvhk.top avsxrcoq2q5fgrw2.lxvmhm.top avsxrcoq2q5fgrw2.nbz4dn.top avsxrcoq2q5fgrw2.p93w1x.bid avsxrcoq2q5fgrw2.r1sjrp.top avsxrcoq2q5fgrw2.rys9pj.top avsxrcoq2q5fgrw2.tjdup0.top avsxrcoq2q5fgrw2.uunmkj.top avsxrcoq2q5fgrw2.vestjb.top avsxrcoq2q5fgrw2.vofy7f.top avsxrcoq2q5fgrw2.w22p3v.top avsxrcoq2q5fgrw2.w5hilw.top avsxrcoq2q5fgrw2.wgx4go.top avsxrcoq2q5fgrw2.y1fx4w.top avsxrcoq2q5fgrw2.y9kxz2.bid avsxrcoq2q5fgrw2.yr1h37.top avsxrcoq2q5fgrw2.z0mkoc.top avsxrcoq2q5fgrw2.zi842m.bid avxdypmdbo.pw axnemuevqnstqyflb.work b4youfred5485jgsa3453f.italazudda.com barjhxoye.info bciuemfaapyf.biz bddadevlpkwrrmud.xyz bfd45u8ehdklrfqwlhbhjbgqw.niptana.at bkdjvmmkwgkvgw.su blxbymhjva.info bnjhx.eu bqbbsfdw.be bqukfjfv.org bwcfinnt.work bwpegsfa.info bxlrywuuobje.pw cdxbbpngq.pw cerberhhyed5frqa.305iot.top cerberhhyed5frqa.305iot.win cerberhhyed5frqa.45gf4t.win cerberhhyed5frqa.45kgok.win cerberhhyed5frqa.5kti58.win cerberhhyed5frqa.ad34ft.win cerberhhyed5frqa.adevf4.win cerberhhyed5frqa.alri58.win cerberhhyed5frqa.as13fd.win cerberhhyed5frqa.asxce4.win cerberhhyed5frqa.azlto5.win cerberhhyed5frqa.cmr95i.top cerberhhyed5frqa.cmr95i.win cerberhhyed5frqa.cmti5o.win cerberhhyed5frqa.cneo59.top cerberhhyed5frqa.cneo59.win cerberhhyed5frqa.dk59jg.win cerberhhyed5frqa.dkrti5.top cerberhhyed5frqa.er48rt.win cerberhhyed5frqa.fgfid6.win cerberhhyed5frqa.fkr84i.win cerberhhyed5frqa.fkri48.win cerberhhyed5frqa.gkfit9.top cerberhhyed5frqa.gkfit9.win cerberhhyed5frqa.kipfgs65s.com cerberhhyed5frqa.lfotp5.top cerberhhyed5frqa.li4loi.win cerberhhyed5frqa.lib2vi.win cerberhhyed5frqa.m5fgoi.win cerberhhyed5frqa.m5gid4.top cerberhhyed5frqa.m5gid4.win cerberhhyed5frqa.m5gips.win cerberhhyed5frqa.mix3hi.win cerberhhyed5frqa.moneu5.win cerberhhyed5frqa.oneswi.win cerberhhyed5frqa.qor499.top cerberhhyed5frqa.raress.win cerberhhyed5frqa.sdfiso.win cerberhhyed5frqa.sims6n.win cerberhhyed5frqa.ti4wic.win cerberhhyed5frqa.to6maq.win cerberhhyed5frqa.vmfu48.win cerberhhyed5frqa.we34re.top cerberhhyed5frqa.we34re.win cerberhhyed5frqa.werti4.win cerberhhyed5frqa.wet4io.win cerberhhyed5frqa.wewiso.win cerberhhyed5frqa.workju.win cerberhhyed5frqa.xltnet.win cerberhhyed5frqa.xmfhr6.win cerberhhyed5frqa.xmfir0.top cerberhhyed5frqa.xmfir0.win cerberhhyed5frqa.xmfjr7.top cerberhhyed5frqa.xmfkr8.top cerberhhyed5frqa.xmfu59.win cerberhhyed5frqa.xo59ok.win cerberhhyed5frqa.xtrvb4.win cerberhhyed5frqa.zgf48j.win chromebewfk.top citointechnologiesalefor.top clhyelmwnuqhigecp.pw corefitness.info cpawdrtxfjkwrkkl.pw cpyrltela.pw crosseunity.top cudcfybkk.pw cwprfpjtmjb.biz cxlgwofgrjfoaa.info d34fa.lasmeio.com dd7bsndhr45nfksdnkferfer.javakale.at de2nuvwegoo32oqv.torbook.li de2nuvwegoo32oqv.tordrims.li de2nuvwegoo32oqv.torfigth.li de2nuvwegoo32oqv.tormilki.li de2nuvwegoo32oqv.torminimals.li de2nuvwegoo32oqv.torspaces.li de2nuvwegoo32oqv.tortelevision.li de2nuvwegoo32oqv.tortodorf.li de2nuvwegoo32oqv.torworks.li dkoipg.pw dltvwp.it dmwajvm.fr dolfexalto.com domainstop.top dqtfhkgskushlum.org dtojlhpasjk.pw dvmbtgoobxcc.pw dwytqrgblrynsgtew.org dyoravdkiavfkbkx.pw earthspiruitr.top eaxpifdtwsv.biz ecjfdaqmmyusxntwl.work egerdpkvutvodmtsy.pw egovrxvuspxck.be eoalsoub.pw eppilxqwyqdhmpdsn.pw eqtrtdavtnr.pw euduudaehipk.pw exnqhgk.xyz eypdxikxsufj.pw eywlmqugxx.info f4dsbjhb45wfiuqeib4fkqeg.meccaledgy.at f5xraa2y2ybtrefz.onion.to fdehgchykmiqwdg.info ffoqr3ug7m726zou.04hyxg.top ffoqr3ug7m726zou.0v7hry.bid ffoqr3ug7m726zou.1321z6.top ffoqr3ug7m726zou.13inb1.top ffoqr3ug7m726zou.14gmtu.top ffoqr3ug7m726zou.17vj7b.top ffoqr3ug7m726zou.1967qy.top ffoqr3ug7m726zou.1feasu.top ffoqr3ug7m726zou.1gtx3p.top ffoqr3ug7m726zou.1mwipu.top ffoqr3ug7m726zou.1nsnuh.top ffoqr3ug7m726zou.2fu7bc.top ffoqr3ug7m726zou.2msuuj.top ffoqr3ug7m726zou.2rl0pv.top ffoqr3ug7m726zou.4tkb0d.top ffoqr3ug7m726zou.5e4u7d.bid ffoqr3ug7m726zou.5hmjh7.bid ffoqr3ug7m726zou.5m2n7x.top ffoqr3ug7m726zou.735giv.top ffoqr3ug7m726zou.8uvtsg.top ffoqr3ug7m726zou.9yim37.top ffoqr3ug7m726zou.ac7zvz.top ffoqr3ug7m726zou.b31wkh.bid ffoqr3ug7m726zou.b4abvx.top ffoqr3ug7m726zou.bd7tlu.top ffoqr3ug7m726zou.bdlvdy.top ffoqr3ug7m726zou.bpuhab.top ffoqr3ug7m726zou.bwei9h.top ffoqr3ug7m726zou.ca15sj.top ffoqr3ug7m726zou.do9wwg.top ffoqr3ug7m726zou.e1e7w2.top ffoqr3ug7m726zou.efebgv.top ffoqr3ug7m726zou.f5x6ws.top ffoqr3ug7m726zou.ffsm1a.bid ffoqr3ug7m726zou.gwz8gh.top ffoqr3ug7m726zou.hajw7w.bid ffoqr3ug7m726zou.hpwom3.top ffoqr3ug7m726zou.hy6dxo.bid ffoqr3ug7m726zou.hzrekn.top ffoqr3ug7m726zou.i4ucg2.bid ffoqr3ug7m726zou.iocvou.top ffoqr3ug7m726zou.jye7lt.top ffoqr3ug7m726zou.kfymbh.top ffoqr3ug7m726zou.l4dlll.bid ffoqr3ug7m726zou.l6r7i9.top ffoqr3ug7m726zou.lc1xfc.top ffoqr3ug7m726zou.le6611.bid ffoqr3ug7m726zou.lruwth.top ffoqr3ug7m726zou.m3cvi8.top ffoqr3ug7m726zou.momg04.top ffoqr3ug7m726zou.ndnmuk.top ffoqr3ug7m726zou.ptnbfm.top ffoqr3ug7m726zou.rssh3l.bid ffoqr3ug7m726zou.rxmbsm.top ffoqr3ug7m726zou.rzt69n.top ffoqr3ug7m726zou.rzvhne.top ffoqr3ug7m726zou.s611js.top ffoqr3ug7m726zou.sg9lxh.bid ffoqr3ug7m726zou.smd95z.top ffoqr3ug7m726zou.tsrwj3.top ffoqr3ug7m726zou.tx0igu.bid ffoqr3ug7m726zou.u9fcji.bid ffoqr3ug7m726zou.ud9z0v.top ffoqr3ug7m726zou.ukswcu.bid ffoqr3ug7m726zou.umvv28.top ffoqr3ug7m726zou.utebcd.top ffoqr3ug7m726zou.v0xn1i.bid ffoqr3ug7m726zou.vjso7r.top ffoqr3ug7m726zou.w22p3v.top ffoqr3ug7m726zou.w67y8u.bid ffoqr3ug7m726zou.wf912u.bid ffoqr3ug7m726zou.wmvsh0.top ffoqr3ug7m726zou.wwa4tu.top ffoqr3ug7m726zou.wx2n44.top ffoqr3ug7m726zou.x8p2m7.bid ffoqr3ug7m726zou.x9ap4h.top ffoqr3ug7m726zou.xe1ws1.top ffoqr3ug7m726zou.y9kxz2.bid ffoqr3ug7m726zou.yjo0z9.top ffoqr3ug7m726zou.yur4j5.top ffoqr3ug7m726zou.yv3uwa.bid ffoqr3ug7m726zou.zee0xr.top ffoqr3ug7m726zou.zio9yg.bid ffoqr3ug7m726zou.zjfbxy.top ffoqr3ug7m726zou.zkxb17.top ffoqr3ug7m726zou.zn90h4.bid ffoqr3ug7m726zou.zpjpsf.top ffoqr3ug7m726zou.zu3fzc.bid fhvjsmtkirihxh.xyz fitga.ru fmirgordkhig.xyz fnarsipfqe.pw fnjyygovdjyemga.xyz fnmi62725zfti2vy.13inb1.top fnmi62725zfti2vy.17vj7b.top fnmi62725zfti2vy.1gtx3p.top fnmi62725zfti2vy.o08ra6.top fnmi62725zfti2vy.p9wol3.top fnmi62725zfti2vy.vwgxhm.bid fooplodanx.top fpashgkepwtoqdjg.pw fqoapcjolfwwenqx.pw fqtdrnqmeofknd.biz ftoxmpdipwobp4qy.10nzk9.top ftoxmpdipwobp4qy.17vj7b.top ftoxmpdipwobp4qy.199ovv.top ftoxmpdipwobp4qy.1gtx3p.top ftoxmpdipwobp4qy.1nsnuh.top ftoxmpdipwobp4qy.7pnxn9.top ftoxmpdipwobp4qy.lxvmhm.top fuuasvhpsvuihlnje.pw fuuwnsv.pw fyqtguo.biz g4dhhg53jsdjnnkjwjrfyiouh3o4u4th.vinerteen.com gccxqpuuylioxoip.pw gfcuxnaek.ru gfkuwflbhsjdabnu4nfukerfqwlfwr4rw.ringbalor.com gfwncoyhbdvggns.pw gguaxufrt.pw gitybdjgbxd.nl glhxgchhfemcjgr.pw gnsquwmgukkpgpt.pw govementruystd.top gsebqsi.ru gsmdqrmqddqtuv.xyz gvludcvhcrjwmgq.in gwbak.nickymaru.com gwe32fdr74bhfsyujb34gfszfv.zatcurr.com h3ds4.maconslab.com h54dc.leverdaze.at h5nuwefkuh134ljngkasdbasfg.corolbugan.com hjhqmbxyinislkkt.11bwgu.top hjhqmbxyinislkkt.127axt.top hjhqmbxyinislkkt.12bsy8.top hjhqmbxyinislkkt.12bxp9.top hjhqmbxyinislkkt.12ct4c.top hjhqmbxyinislkkt.12gsjz.top hjhqmbxyinislkkt.12m58x.top hjhqmbxyinislkkt.12zucf.top hjhqmbxyinislkkt.13bcem.top hjhqmbxyinislkkt.13eymq.top hjhqmbxyinislkkt.13fmby.top hjhqmbxyinislkkt.13khiv.top hjhqmbxyinislkkt.13kn4l.top hjhqmbxyinislkkt.13qgdd.top hjhqmbxyinislkkt.13ydzv.top hjhqmbxyinislkkt.142djp.top hjhqmbxyinislkkt.14dr1s.top hjhqmbxyinislkkt.14klmz.top hjhqmbxyinislkkt.14o2wp.top hjhqmbxyinislkkt.14stvt.top hjhqmbxyinislkkt.14yppf.top hjhqmbxyinislkkt.15e8hv.top hjhqmbxyinislkkt.15mwt4.top hjhqmbxyinislkkt.15u3kg.top hjhqmbxyinislkkt.16ke1t.top hjhqmbxyinislkkt.16l1zt.top hjhqmbxyinislkkt.17kc8y.top hjhqmbxyinislkkt.17rm9b.top hjhqmbxyinislkkt.18f5bw.top hjhqmbxyinislkkt.18lmhb.top hjhqmbxyinislkkt.18nepv.top hjhqmbxyinislkkt.18yzmj.top hjhqmbxyinislkkt.18zrup.top hjhqmbxyinislkkt.19b6nk.top hjhqmbxyinislkkt.19hj4f.top hjhqmbxyinislkkt.19s7gy.top hjhqmbxyinislkkt.19xdpm.top hjhqmbxyinislkkt.19xvyd.top hjhqmbxyinislkkt.1a2xx3.top hjhqmbxyinislkkt.1a8u1r.top hjhqmbxyinislkkt.1aajb7.top hjhqmbxyinislkkt.1aamtz.top hjhqmbxyinislkkt.1accfa.top hjhqmbxyinislkkt.1acfka.top hjhqmbxyinislkkt.1adh2r.top hjhqmbxyinislkkt.1aq4sz.top hjhqmbxyinislkkt.1aqq5k.top hjhqmbxyinislkkt.1b8tmn.top hjhqmbxyinislkkt.1bas8q.top hjhqmbxyinislkkt.1bcnad.top hjhqmbxyinislkkt.1bcxcs.top hjhqmbxyinislkkt.1bu9xu.top hjhqmbxyinislkkt.1c1ajf.top hjhqmbxyinislkkt.1cdqfv.top hjhqmbxyinislkkt.1cnkik.top hjhqmbxyinislkkt.1csesc.top hjhqmbxyinislkkt.1dq6nd.top hjhqmbxyinislkkt.1dvqvh.top hjhqmbxyinislkkt.1e47tj.top hjhqmbxyinislkkt.1eagrj.top hjhqmbxyinislkkt.1eeyaj.top hjhqmbxyinislkkt.1efxa8.top hjhqmbxyinislkkt.1fgsmc.top hjhqmbxyinislkkt.1fnjrj.top hjhqmbxyinislkkt.1fttxm.top hjhqmbxyinislkkt.1fy93v.top hjhqmbxyinislkkt.1fygsg.top hjhqmbxyinislkkt.1fzjn3.top hjhqmbxyinislkkt.1fzz7a.top hjhqmbxyinislkkt.1gjpzp.top hjhqmbxyinislkkt.1gqrpq.top hjhqmbxyinislkkt.1gredn.top hjhqmbxyinislkkt.1grvue.top hjhqmbxyinislkkt.1gswwp.top hjhqmbxyinislkkt.1gu5um.top hjhqmbxyinislkkt.1gunao.top hjhqmbxyinislkkt.1gvyo8.top hjhqmbxyinislkkt.1gxfxt.top hjhqmbxyinislkkt.1gzjuc.top hjhqmbxyinislkkt.1hapca.top hjhqmbxyinislkkt.1j43kf.top hjhqmbxyinislkkt.1jmip6.top hjhqmbxyinislkkt.1jnhdc.top hjhqmbxyinislkkt.1jwuaa.top hjhqmbxyinislkkt.1k6bas.top hjhqmbxyinislkkt.1kge5a.top hjhqmbxyinislkkt.1khwro.top hjhqmbxyinislkkt.1kjhhf.top hjhqmbxyinislkkt.1kraqn.top hjhqmbxyinislkkt.1kw51p.top hjhqmbxyinislkkt.1lqrja.top hjhqmbxyinislkkt.1ltyev.top hjhqmbxyinislkkt.1mat7v.top hjhqmbxyinislkkt.1mee2x.top hjhqmbxyinislkkt.1mqvsc.top hjhqmbxyinislkkt.1mswjm.top hjhqmbxyinislkkt.1mvku2.top hjhqmbxyinislkkt.1mwvgh.top hjhqmbxyinislkkt.1nm62r.top hjhqmbxyinislkkt.1npg9s.top hjhqmbxyinislkkt.1ntyds.top hjhqmbxyinislkkt.1pcvko.top hjhqmbxyinislkkt.1ppto6.top hjhqmbxyinislkkt.1pxbfh.top hjhqmbxyinislkkt.1q7pwb.top hjhqmbxyinislkkt.1qjl23.top hjhqmbxyinislkkt.1qk2un.top hjhqmbxyinislkkt.1w5iy8.top hjhqmbxyinislkkt.1xynaz.top hmndhdbscgru.pw honourableud.top hppfsslyeyseudg.biz hrfgd74nfksjdcnnklnwefvdsf.materdunst.com htankds.info hycninyxuaa.xyz i01001.dgn.vn i3ezlvkoi7fwyood.onion.to i3ezlvkoi7fwyood.tor2web.org i5ndw.titlecorta.at ibjgnqsthdyp.pw ibtfqftkgi.pw ifohvkxmyp.biz igoodsnd.wang ik4dm.mazerunci.at iqfyujpvubwawc.pw irhng84nfaslbv243ljtblwqjrb.pinnafaon.at irudhkunrlfu25fhkaqw34blr5qlby4tgq43t.orrisbirth.com iuieylpvfurcvmpk.pw jfmiondv.xyz jghbktqepe.pw jhdgh.club jhomitevd2abj3fk.onion.to juhacjacjckclqf.pw jxqdry.ru jymhmkdaxfbl.click k234s.ascotsprue.com k34ew.keyedgell.com k3cxd.pileanoted.com k47d3.proporr.com k4restportgonst34d23r.oftpony.at kbv5s.kylepasse.at kcdfajaxngiff.info kciylimohteftc.pw kh5jfnvkk5twerfnku5twuilrnglnuw45yhlw.vealsithe.com kjkwjqvqrjocpi.xyz kkd47eh4hdjshb5t.angortra.at kkr4hbwdklf234bfl84uoqleflqwrfqwuelfh.brazabaya.com kpybuhnosdrm.in kqlxtqptsmys.in ks-davis.com ktlgpiilbj.biz kwontdmplpnbl.pw kypsuw.pw l123d.feustude.at lcrdceiajmiar.org lfdachijzuwx4bc4.0ndl3j.bid lfdachijzuwx4bc4.6szfn7.top lfdachijzuwx4bc4.83zw1f.bid lfdachijzuwx4bc4.8dlgyg.bid lfdachijzuwx4bc4.af38vz.top lfdachijzuwx4bc4.ci221p.top lfdachijzuwx4bc4.djintc.bid lfdachijzuwx4bc4.e6cf2t.bid lfdachijzuwx4bc4.eujvrw.bid lfdachijzuwx4bc4.ev99l6.bid lfdachijzuwx4bc4.ex9n9v.top lfdachijzuwx4bc4.fe6cf2.top lfdachijzuwx4bc4.fwzxnb.bid lfdachijzuwx4bc4.iuzppd.top lfdachijzuwx4bc4.le2brr.bid lfdachijzuwx4bc4.m7f27y.bid lfdachijzuwx4bc4.twyjdx.bid lfdachijzuwx4bc4.tx0igu.bid lfdachijzuwx4bc4.u9fcji.bid lfdachijzuwx4bc4.vrgdrs.top lfdachijzuwx4bc4.w4629d.top lfdachijzuwx4bc4.x4tk5c.bid lfdachijzuwx4bc4.zreknv.bid lollyoff.info lookingpersonals.top lpholfnvwbukqwye.onion.cab lpholfnvwbukqwye.onion.to lrmficvqs.pw ltpwqva.xyz luvenxj.uk lvanwwbyabcfevyi.pw lyrnvane.pw macooptwafkwchtpo.pw mmhmtea.pw mphtadhci5mrdlju.onion.to mphtadhci5mrdlju.tor2web.org muuojcu.xyz mwqwverayognn.pw mxyfasm.pw mz7oyb3v32vshcvk.bidobject.li mz7oyb3v32vshcvk.getstar.li mz7oyb3v32vshcvk.torapples.li mz7oyb3v32vshcvk.torlongor.li mz7oyb3v32vshcvk.tormidle.at mz7oyb3v32vshcvk.toysworlds.at newgiftnd.wang newgiftst.top nhhyxorxbxarxe.org nikessysleys.top nlpqflkbvkdde.eu nn54djhfnrnm4dnjnerfsd.replylaten.at nnrtsdf34dsjhb23rsdf.spannflow.com nwcpgymgh.work o4dm3.leaama.at odgtnkmq.pw oehknf74ohqlfnpq9rhfgcq93g.hateflux.com ohpbdikmrrhr.pw ohplsuljopekq.biz ojmekzw4mujvqeju.bioserv.at ojmekzw4mujvqeju.dreamtest.at ojmekzw4mujvqeju.fineboy.at ojmekzw4mujvqeju.minitili.at omeaswslhgdw.xyz oqwygprskqv65j72.12kb9j.top oqwygprskqv65j72.13gpqd.top oqwygprskqv65j72.13rdvu.top oqwygprskqv65j72.14jqyo.top oqwygprskqv65j72.17q8f6.top oqwygprskqv65j72.1aj1bb.top oqwygprskqv65j72.1d88b8.top oqwygprskqv65j72.1dofqx.top oqwygprskqv65j72.1fdlhn.top oqwygprskqv65j72.1fs9pz.top oqwygprskqv65j72.1gam57.top oqwygprskqv65j72.1gqj8x.top oqwygprskqv65j72.1hbdbx.top oqwygprskqv65j72.1j1x2b.top oqwygprskqv65j72.1jquw7.top oqwygprskqv65j72.1kh9ct.top oqwygprskqv65j72.1mudaw.top oqwygprskqv65j72.1nzpby.top ozfin.ru p27dokhpz2n7nvgr.12a63k.top p27dokhpz2n7nvgr.12c8ff.top p27dokhpz2n7nvgr.12gzrv.top p27dokhpz2n7nvgr.12hxjv.top p27dokhpz2n7nvgr.12nwsv.top p27dokhpz2n7nvgr.12smak.top p27dokhpz2n7nvgr.12t3rn.top p27dokhpz2n7nvgr.12ulcz.top p27dokhpz2n7nvgr.12umzf.top p27dokhpz2n7nvgr.12uzfa.top p27dokhpz2n7nvgr.12vpkc.top p27dokhpz2n7nvgr.1321z6.top p27dokhpz2n7nvgr.133chr.top p27dokhpz2n7nvgr.135nt3.top p27dokhpz2n7nvgr.13g2v9.top p27dokhpz2n7nvgr.13gmvm.top p27dokhpz2n7nvgr.13ixv2.top p27dokhpz2n7nvgr.13upky.top p27dokhpz2n7nvgr.13upnc.top p27dokhpz2n7nvgr.13wm9b.top p27dokhpz2n7nvgr.13xwn9.top p27dokhpz2n7nvgr.14ewqv.top p27dokhpz2n7nvgr.14gmtu.top p27dokhpz2n7nvgr.14kfoz.top p27dokhpz2n7nvgr.14udep.top p27dokhpz2n7nvgr.15jznv.top p27dokhpz2n7nvgr.15l2ub.top p27dokhpz2n7nvgr.15nhsf.top p27dokhpz2n7nvgr.15oqwp.top p27dokhpz2n7nvgr.15rnwa.top p27dokhpz2n7nvgr.15wmdx.top p27dokhpz2n7nvgr.168w5y.top p27dokhpz2n7nvgr.16ay2s.top p27dokhpz2n7nvgr.16bwhs.top p27dokhpz2n7nvgr.16fohp.top p27dokhpz2n7nvgr.16nxpn.top p27dokhpz2n7nvgr.16qpet.top p27dokhpz2n7nvgr.173w9w.top p27dokhpz2n7nvgr.17g6gc.top p27dokhpz2n7nvgr.17gvad.top p27dokhpz2n7nvgr.17m14u.top p27dokhpz2n7nvgr.17ryrs.top p27dokhpz2n7nvgr.17u2yg.top p27dokhpz2n7nvgr.18dawg.top p27dokhpz2n7nvgr.18kkhl.top p27dokhpz2n7nvgr.18kmtt.top p27dokhpz2n7nvgr.195heb.top p27dokhpz2n7nvgr.1967qy.top p27dokhpz2n7nvgr.1a7ivn.top p27dokhpz2n7nvgr.1a7wnt.top p27dokhpz2n7nvgr.1aghep.top p27dokhpz2n7nvgr.1ajohk.top p27dokhpz2n7nvgr.1apgrn.top p27dokhpz2n7nvgr.1apkjn.top p27dokhpz2n7nvgr.1aweql.top p27dokhpz2n7nvgr.1axzcw.top p27dokhpz2n7nvgr.1azkux.top p27dokhpz2n7nvgr.1b3qjy.top p27dokhpz2n7nvgr.1bj4k9.top p27dokhpz2n7nvgr.1bniyw.top p27dokhpz2n7nvgr.1bvadx.top p27dokhpz2n7nvgr.1bywu2.top p27dokhpz2n7nvgr.1bzolk.top p27dokhpz2n7nvgr.1cauz3.top p27dokhpz2n7nvgr.1cb19l.top p27dokhpz2n7nvgr.1cbcpy.top p27dokhpz2n7nvgr.1cewld.top p27dokhpz2n7nvgr.1cggqc.top p27dokhpz2n7nvgr.1cglxz.top p27dokhpz2n7nvgr.1chy1m.top p27dokhpz2n7nvgr.1cknbd.top p27dokhpz2n7nvgr.1cpb4z.top p27dokhpz2n7nvgr.1cpy1q.top p27dokhpz2n7nvgr.1cq7gd.top p27dokhpz2n7nvgr.1cvmb4.top p27dokhpz2n7nvgr.1cw65b.top p27dokhpz2n7nvgr.1czh7o.top p27dokhpz2n7nvgr.1d8d9w.top p27dokhpz2n7nvgr.1d8m97.top p27dokhpz2n7nvgr.1daq6h.top p27dokhpz2n7nvgr.1dlcbk.top p27dokhpz2n7nvgr.1dp6un.top p27dokhpz2n7nvgr.1dsdm4.top p27dokhpz2n7nvgr.1dyzdh.top p27dokhpz2n7nvgr.1dz7gk.top p27dokhpz2n7nvgr.1ebvqb.top p27dokhpz2n7nvgr.1eeb86.top p27dokhpz2n7nvgr.1em2j4.top p27dokhpz2n7nvgr.1enbyr.top p27dokhpz2n7nvgr.1evjph.top p27dokhpz2n7nvgr.1fel3k.top p27dokhpz2n7nvgr.1fgywm.top p27dokhpz2n7nvgr.1fqwek.top p27dokhpz2n7nvgr.1fu8p3.top p27dokhpz2n7nvgr.1gnlsi.top p27dokhpz2n7nvgr.1gqqsc.top p27dokhpz2n7nvgr.1gvql3.top p27dokhpz2n7nvgr.1gy9bo.top p27dokhpz2n7nvgr.1h23cc.top p27dokhpz2n7nvgr.1hkjl3.top p27dokhpz2n7nvgr.1hpvzl.top p27dokhpz2n7nvgr.1hw36d.top p27dokhpz2n7nvgr.1jemdr.top p27dokhpz2n7nvgr.1jh5kv.top p27dokhpz2n7nvgr.1jhnvt.top p27dokhpz2n7nvgr.1jpb8w.top p27dokhpz2n7nvgr.1js3tl.top p27dokhpz2n7nvgr.1jw2lx.top p27dokhpz2n7nvgr.1jyhqc.top p27dokhpz2n7nvgr.1jzmjr.top p27dokhpz2n7nvgr.1kja1j.top p27dokhpz2n7nvgr.1kq4l8.top p27dokhpz2n7nvgr.1ktjse.top p27dokhpz2n7nvgr.1kyjw7.top p27dokhpz2n7nvgr.1l4zyd.top p27dokhpz2n7nvgr.1lcteo.top p27dokhpz2n7nvgr.1lfyy4.top p27dokhpz2n7nvgr.1lt2pn.top p27dokhpz2n7nvgr.1m3xsy.top p27dokhpz2n7nvgr.1mfakx.top p27dokhpz2n7nvgr.1mfdt8.top p27dokhpz2n7nvgr.1mir1h.top p27dokhpz2n7nvgr.1ms2rx.top p27dokhpz2n7nvgr.1mwipu.top p27dokhpz2n7nvgr.1nhkou.top p27dokhpz2n7nvgr.1nmrtq.top p27dokhpz2n7nvgr.1nprob.top p27dokhpz2n7nvgr.1p5fwl.top p27dokhpz2n7nvgr.1pbfky.top p27dokhpz2n7nvgr.1pbu64.top p27dokhpz2n7nvgr.1pglcs.top p27dokhpz2n7nvgr.1plugt.top p27dokhpz2n7nvgr.1psts4.top p27dokhpz2n7nvgr.1pymg3.top p27dokhpz2n7nvgr.1vjnyh.top p27dokhpz2n7nvgr.1wmvk2.top p54dhkus4tlkfashdb6vjetgsdfg.greetingshere.at pagaldaily.com pdlbtnfhtoxghb.org pe2cku7pebkpgeko.13inb1.top pe2cku7pebkpgeko.199ovv.top pe2cku7pebkpgeko.1cb19l.top pe2cku7pebkpgeko.1gtx3p.top pe2cku7pebkpgeko.1mwipu.top pe2cku7pebkpgeko.1plugt.top pe2cku7pebkpgeko.1pr21c.top pe2cku7pebkpgeko.582h0n.top pe2cku7pebkpgeko.5hmjh7.bid pe2cku7pebkpgeko.ahovbr.top pe2cku7pebkpgeko.bw9e2z.top pe2cku7pebkpgeko.dj68hn.top pe2cku7pebkpgeko.hclz73.top pe2cku7pebkpgeko.kwrd4f.bid pe2cku7pebkpgeko.p93w1x.bid pe2cku7pebkpgeko.pkx86a.top pe2cku7pebkpgeko.prbuoi.top pe2cku7pebkpgeko.r1sjrp.top pe2cku7pebkpgeko.reu88i.top pe2cku7pebkpgeko.rjf9yn.top pe2cku7pebkpgeko.tsrwj3.top pe2cku7pebkpgeko.ttx0ig.top pe2cku7pebkpgeko.utebcd.top pe2cku7pebkpgeko.va3ibn.top pe2cku7pebkpgeko.vfe2f1.top pe2cku7pebkpgeko.yjo0z9.top pe2cku7pebkpgeko.z5xfkc.top pennysgoods.top plfbvdrpvsm.pw pmenboeqhyrpvomq.0nyi6l.bid pmenboeqhyrpvomq.0vgu64.top pmenboeqhyrpvomq.2agglf.top pmenboeqhyrpvomq.4pzclh.top pmenboeqhyrpvomq.58na23.top pmenboeqhyrpvomq.5b1s82.top pmenboeqhyrpvomq.7s0g3v.top pmenboeqhyrpvomq.89m6y8.bid pmenboeqhyrpvomq.8kcfnk.bid pmenboeqhyrpvomq.9ildst.top pmenboeqhyrpvomq.9nkxd3.top pmenboeqhyrpvomq.a4coac.top pmenboeqhyrpvomq.afteghonte.lol pmenboeqhyrpvomq.as5su5.top pmenboeqhyrpvomq.asxjdp.top pmenboeqhyrpvomq.azwsxe.top pmenboeqhyrpvomq.b7mciu.top pmenboeqhyrpvomq.bnctf6.top pmenboeqhyrpvomq.cmri58.top pmenboeqhyrpvomq.e6in0v.top pmenboeqhyrpvomq.enanhb.bid pmenboeqhyrpvomq.factordo.site pmenboeqhyrpvomq.fm0cga.top pmenboeqhyrpvomq.g0ots2.top pmenboeqhyrpvomq.gletterstan.trade pmenboeqhyrpvomq.gnuvaw.bid pmenboeqhyrpvomq.hasterlyston.cloud pmenboeqhyrpvomq.hwh75t.top pmenboeqhyrpvomq.ibngww.top pmenboeqhyrpvomq.k7oud1.top pmenboeqhyrpvomq.ka0te8.top pmenboeqhyrpvomq.kswcuk.top pmenboeqhyrpvomq.li4loi.top pmenboeqhyrpvomq.loopsay.link pmenboeqhyrpvomq.m54tkp.bid pmenboeqhyrpvomq.mtxtul.top pmenboeqhyrpvomq.n41n1a.top pmenboeqhyrpvomq.n80yab.top pmenboeqhyrpvomq.nh47ri.bid pmenboeqhyrpvomq.o08a6d.top pmenboeqhyrpvomq.o8hpwj.top pmenboeqhyrpvomq.p8rruv.top pmenboeqhyrpvomq.pap44w.top pmenboeqhyrpvomq.paypoints.red pmenboeqhyrpvomq.r21wmw.top pmenboeqhyrpvomq.rnkj09.top pmenboeqhyrpvomq.s71vsc.top pmenboeqhyrpvomq.self56.top pmenboeqhyrpvomq.shutlazy.casa pmenboeqhyrpvomq.swissprogramms.bid pmenboeqhyrpvomq.t4hvl4.bid pmenboeqhyrpvomq.thyx30.top pmenboeqhyrpvomq.txszfs.top pmenboeqhyrpvomq.v11z5e.top pmenboeqhyrpvomq.viceled.pw pmenboeqhyrpvomq.vkm4l6.top pmenboeqhyrpvomq.wn4h1k.top pmenboeqhyrpvomq.wrd4fo.top pmenboeqhyrpvomq.x1kofw.top pmenboeqhyrpvomq.xneyvm.top pmenboeqhyrpvomq.xx6jck.top pmenboeqhyrpvomq.y5j7e6.top pmenboeqhyrpvomq.y7fjr4.bid pmenboeqhyrpvomq.yw4629.top pnyviolg.eu po4dbsjbneljhrlbvaueqrgveatv.bonmawp.at poimoiyreque5.pw polaerunity.top ponmaredimare.top pornohd24.com preeqlultgfifg.pw prest54538hnksjn4kjfwdbhwere.hotchunman.com pts764gt354fder34fsqw45gdfsavadfgsfg.kraskula.com pvwinlrmwvccuo.eu qbqrfyeqqvcvv.pw qcwbrevxrotoepsp.pw qdesslfdcmd.pw qdvkdyvrtpjc.pw qfjhpgbefuhenjp7.1225wj.top qfjhpgbefuhenjp7.12efwa.top qfjhpgbefuhenjp7.12f53x.top qfjhpgbefuhenjp7.12u5fl.top qfjhpgbefuhenjp7.13iuvw.top qfjhpgbefuhenjp7.143kzi.top qfjhpgbefuhenjp7.158ugp.top qfjhpgbefuhenjp7.16g9ub.top qfjhpgbefuhenjp7.17cwdi.top qfjhpgbefuhenjp7.17ipn9.top qfjhpgbefuhenjp7.17xukb.top qfjhpgbefuhenjp7.18dwag.top qfjhpgbefuhenjp7.18ggbf.top qfjhpgbefuhenjp7.18rkju.top qfjhpgbefuhenjp7.19ckzf.top qfjhpgbefuhenjp7.1a2jzy.top qfjhpgbefuhenjp7.1cosak.top qfjhpgbefuhenjp7.1e1jbc.top qfjhpgbefuhenjp7.1e1y8p.top qfjhpgbefuhenjp7.1fcfjn.top qfjhpgbefuhenjp7.1jfjhb.top qfjhpgbefuhenjp7.1jrkyn.top qfjhpgbefuhenjp7.1mkwry.top qfjhpgbefuhenjp7.1mnsg6.top qfuxosx.eu qlwnvdjwro.pw qqonof.info qqtphtlhny.pw qsbfwgtedexirbyoq.pw qvdgqayo.pw rastypasty34.top rbg4hfbilrf7to452p89hrfq.boonmower.com rbwubtpsyokqn.info real346real.top remoteunityrety.top renaulrtcenturytrick.top rkiywansamtu.top rolerxunitywsto.top rootaleyz.top rowerpovertort.top rqfsctpgpuani.pw rrcspgfghsjnklts.pw rzss2zfue73dfvmj.onlinerpgame.ch rzss2zfue73dfvmj.truewargame.ch sdwempsovemtr.yt seelkqtkkqxvq.click semiconductry.top sgowntfjwkybawi.pw sgrnhwyqxdk.pw sondr5344ygfweyjbfkw4fhsefv.heliofetch.at sonicfopase.top sqrgvbgfyya.org sqsigig.pw ssvylrn.pw stevnxwq.pw stgg5jv6mqiibmax.toradmin.li stgg5jv6mqiibmax.toranimals.li stgg5jv6mqiibmax.torbrouke.li stgg5jv6mqiibmax.torclasses.li stgg5jv6mqiibmax.torclever.li stgg5jv6mqiibmax.torcreator.li stgg5jv6mqiibmax.torking.li stgg5jv6mqiibmax.torpice.li stgg5jv6mqiibmax.torpoint.ch stgg5jv6mqiibmax.torshop.li sumnitdomains.top svkjhguk.ru svvgyjweurxn.click swfqg.in sxflmtgxerkpgwlnp.pw t54ndnku456ngkwsudqer.wallymac.com tdhyjfxltpj.pw tes543berda73i48fsdfsd.keratadze.at topgearspoilytyrdc.top toxnwbkoulii.pw toytyaclucomunit.top tqlcjh.fr tregretryfaltervipo.top trxswbwxhr.xyz tswsgajtwhqkosd.su tt54rfdjhb34rfbnknaerg.milerteddy.com ttoyqvq.pw tuouyunittyewr.top twbers4hmi6dc65f.onion.cab twbers4hmi6dc65f.onion.to twbers4hmi6dc65f.tor2web.org u24er.ovaarmor.com u54bbnhf354fbkh254tbkhjbgy8258gnkwerg.tahaplap.com ubisortdasert.top uetwvrlnee.fr uhgmnigjpf.biz uhhvhjqowpgopq.xyz uhjxayhpisr.pw uhufnlsad7bhf4ykqfbevmxergwrth.himfinn.com uiredn4njfsa4234bafb32ygjdawfvs.frascuft.com uj5nj.onanwhit.com umjjvccteg.biz unintyregullyar.top unittogreas.top unityharerteraz.top unityrulesyur.top unixbroungs.top unocl45trpuoefft.054t69.bid unocl45trpuoefft.06j7o0.top unocl45trpuoefft.086ux2.top unocl45trpuoefft.0evktl.top unocl45trpuoefft.0kousz.bid unocl45trpuoefft.0kv6tw.bid unocl45trpuoefft.0vgu64.top unocl45trpuoefft.18xhww.bid unocl45trpuoefft.1cn41a.bid unocl45trpuoefft.1de02r.top unocl45trpuoefft.1v3bnu.top unocl45trpuoefft.249isv.bid unocl45trpuoefft.2y4t6f.bid unocl45trpuoefft.308an1.top unocl45trpuoefft.31wkhu.top unocl45trpuoefft.36u6mp.bid unocl45trpuoefft.3n9lut.bid unocl45trpuoefft.42wunw.bid unocl45trpuoefft.4bb9vz.bid unocl45trpuoefft.4k98id.top unocl45trpuoefft.54drms.bid unocl45trpuoefft.54m2k3.bid unocl45trpuoefft.5o3euy.bid unocl45trpuoefft.5v3uvc.bid unocl45trpuoefft.60c61d.bid unocl45trpuoefft.6w3rkc.bid unocl45trpuoefft.75tdcj.bid unocl45trpuoefft.78of7m.bid unocl45trpuoefft.791sd5.bid unocl45trpuoefft.7cevps.bid unocl45trpuoefft.7eup7k.bid unocl45trpuoefft.7tooul.bid unocl45trpuoefft.88wz5p.bid unocl45trpuoefft.8kcfnk.bid unocl45trpuoefft.8uwckh.top unocl45trpuoefft.9bjnlk.bid unocl45trpuoefft.9lnito.top unocl45trpuoefft.9lx4s6.bid unocl45trpuoefft.9u3iy1.top unocl45trpuoefft.a3migu.bid unocl45trpuoefft.a4v4c3.bid unocl45trpuoefft.ageshere.club unocl45trpuoefft.ahhc36.top unocl45trpuoefft.at593l.bid unocl45trpuoefft.at9gwv.bid unocl45trpuoefft.awspm2.top unocl45trpuoefft.barzc4.bid unocl45trpuoefft.bjahwh.bid unocl45trpuoefft.c3fz3z.bid unocl45trpuoefft.c4issd.bid unocl45trpuoefft.c9kp0o.bid unocl45trpuoefft.ceikto.bid unocl45trpuoefft.cgf59i.top unocl45trpuoefft.cifbp9.bid unocl45trpuoefft.ckw9fm.top unocl45trpuoefft.cm5ohx.bid unocl45trpuoefft.csdbnk.bid unocl45trpuoefft.csv7o6.bid unocl45trpuoefft.cypz3w.top unocl45trpuoefft.czzg7f.bid unocl45trpuoefft.dwkofh.top unocl45trpuoefft.dyo7c9.top unocl45trpuoefft.efebgv.bid unocl45trpuoefft.eloppu.bid unocl45trpuoefft.emogew.bid unocl45trpuoefft.eo6rzt.bid unocl45trpuoefft.ev6i0x.bid unocl45trpuoefft.eyohd2.top unocl45trpuoefft.f17bam.bid unocl45trpuoefft.freshsdog.loan unocl45trpuoefft.frn62e.top unocl45trpuoefft.gg4dgp.bid unocl45trpuoefft.gio6f6.bid unocl45trpuoefft.givxuf.bid unocl45trpuoefft.hawtzr.bid unocl45trpuoefft.he81tz.bid unocl45trpuoefft.hur45z.bid unocl45trpuoefft.hvh2gb.bid unocl45trpuoefft.hxrd02.bid unocl45trpuoefft.hynwbs.top unocl45trpuoefft.hyr1h3.bid unocl45trpuoefft.i1wcrl.bid unocl45trpuoefft.i561zy.bid unocl45trpuoefft.ibngww.top unocl45trpuoefft.idw6s5.bid unocl45trpuoefft.igpfcu.bid unocl45trpuoefft.igrj6t.bid unocl45trpuoefft.ih301a.bid unocl45trpuoefft.ii2yoh.bid unocl45trpuoefft.ilm071.bid unocl45trpuoefft.j0cia7.bid unocl45trpuoefft.j404oy.bid unocl45trpuoefft.j8exy2.bid unocl45trpuoefft.jcife9.bid unocl45trpuoefft.jdf4je.bid unocl45trpuoefft.jjogbj.top unocl45trpuoefft.jnd0bj.bid unocl45trpuoefft.jsotn5.top unocl45trpuoefft.jvrh8g.bid unocl45trpuoefft.k56185.top unocl45trpuoefft.kf1gxm.bid unocl45trpuoefft.kg5bof.bid unocl45trpuoefft.kml2o2.top unocl45trpuoefft.knowhands.us unocl45trpuoefft.ks3ghp.bid unocl45trpuoefft.kswcuk.top unocl45trpuoefft.l05l27.top unocl45trpuoefft.l69xgc.bid unocl45trpuoefft.l97i5a.bid unocl45trpuoefft.lak8wd.bid unocl45trpuoefft.larebg.bid unocl45trpuoefft.lcyznu.bid unocl45trpuoefft.lio2wr.bid unocl45trpuoefft.lk0bzc.top unocl45trpuoefft.ll3zot.bid unocl45trpuoefft.lzskva.bid unocl45trpuoefft.m03t72.bid unocl45trpuoefft.m33d4b.bid unocl45trpuoefft.m9a225.top unocl45trpuoefft.mbwxyg.bid unocl45trpuoefft.md9eyv.bid unocl45trpuoefft.meetsface.win unocl45trpuoefft.metpast.date unocl45trpuoefft.mezy7j.bid unocl45trpuoefft.moonsides.faith unocl45trpuoefft.n20b1c.top unocl45trpuoefft.n41n1a.top unocl45trpuoefft.n94lrn.bid unocl45trpuoefft.na2iuz.bid unocl45trpuoefft.nmit4p.bid unocl45trpuoefft.noyl9o.bid unocl45trpuoefft.nz6emv.bid unocl45trpuoefft.o2dval.top unocl45trpuoefft.o8hpwj.top unocl45trpuoefft.og5ezh.top unocl45trpuoefft.on2420.bid unocl45trpuoefft.ozlrnx.bid unocl45trpuoefft.p1gneb.bid unocl45trpuoefft.p2ix1u.bid unocl45trpuoefft.p4sr76.top unocl45trpuoefft.pap44w.top unocl45trpuoefft.pbprju.bid unocl45trpuoefft.piy4l3.bid unocl45trpuoefft.ptneek.bid unocl45trpuoefft.r21wmw.top unocl45trpuoefft.r2vai7.bid unocl45trpuoefft.rgbb50.bid unocl45trpuoefft.rie9py.bid unocl45trpuoefft.rslh9a.top unocl45trpuoefft.s7b63k.bid unocl45trpuoefft.sirchi.bid unocl45trpuoefft.sp4o1t.bid unocl45trpuoefft.tcly4s.bid unocl45trpuoefft.tfmmby.bid unocl45trpuoefft.thanreal.link unocl45trpuoefft.ttabop.bid unocl45trpuoefft.u64rj2.top unocl45trpuoefft.uaol08.bid unocl45trpuoefft.ukwnvw.bid unocl45trpuoefft.um1x6z.bid unocl45trpuoefft.uog1ky.bid unocl45trpuoefft.uso3z0.bid unocl45trpuoefft.uw3r6a.top unocl45trpuoefft.uwckha.top unocl45trpuoefft.v4kx51.bid unocl45trpuoefft.v50gtu.bid unocl45trpuoefft.vfuvsv.bid unocl45trpuoefft.vi5iko.bid unocl45trpuoefft.vkm4l6.top unocl45trpuoefft.vkslju.bid unocl45trpuoefft.vlwbcz.bid unocl45trpuoefft.vmomcc.bid unocl45trpuoefft.whmykv.bid unocl45trpuoefft.wl8t6k.bid unocl45trpuoefft.wlvxd6.bid unocl45trpuoefft.wz139z.top unocl45trpuoefft.x9kjcn.bid unocl45trpuoefft.x9le66.top unocl45trpuoefft.xf38wp.bid unocl45trpuoefft.xlxd92.bid unocl45trpuoefft.y721yz.top unocl45trpuoefft.ye4f7k.bid unocl45trpuoefft.yky1uf.bid unocl45trpuoefft.ytbyhs.bid unocl45trpuoefft.yty0gm.bid unocl45trpuoefft.zbj2kc.bid unocl45trpuoefft.zdamew.bid unocl45trpuoefft.zgheyh.bid unocl45trpuoefft.zjems2.bid unocl45trpuoefft.zn9cme.bid urulvtffwoq.xyz uuwflbmjmi.eu uvcmlfca.biz uxvvm.us uxwavkmttywsuynt.pw vcabbvhrqhot.pw vewrb.italisumo.at vpuroeit.pw vrvis6ndra5jeggj.livegaming.ch vrvis6ndra5jeggj.livewargaming.ch vrvis6ndra5jeggj.onlinebattlefield.ch vrympoqs5ra34nfo.bigbird.at vrympoqs5ra34nfo.bigclear.at vrympoqs5ra34nfo.smartbus.at vrympoqs5ra34nfo.torhelper.pl vujqbcditgsqxe.fr vyohacxzoue32vvk.0ayn1s.top vyohacxzoue32vvk.0ot7em.bid vyohacxzoue32vvk.0vtwzy.top vyohacxzoue32vvk.1m47ka.bid vyohacxzoue32vvk.23fvxw.bid vyohacxzoue32vvk.2hr4fs.top vyohacxzoue32vvk.34o9h1.bid vyohacxzoue32vvk.3buvlc.bid vyohacxzoue32vvk.3m370u.top vyohacxzoue32vvk.3peyo3.bid vyohacxzoue32vvk.3t3hyf.top vyohacxzoue32vvk.5a5vmh.top vyohacxzoue32vvk.5i0ukv.bid vyohacxzoue32vvk.5m2n7x.top vyohacxzoue32vvk.5s96fr.top vyohacxzoue32vvk.6wkz70.bid vyohacxzoue32vvk.79j8fm.top vyohacxzoue32vvk.7a07br.bid vyohacxzoue32vvk.7jrv53.bid vyohacxzoue32vvk.7m7ujm.bid vyohacxzoue32vvk.8g1k17.bid vyohacxzoue32vvk.ac7zvz.top vyohacxzoue32vvk.axu3u8.bid vyohacxzoue32vvk.b14kkk.bid vyohacxzoue32vvk.c4cwr4.bid vyohacxzoue32vvk.c8jxpp.top vyohacxzoue32vvk.chnbyl.bid vyohacxzoue32vvk.cp3yme.top vyohacxzoue32vvk.d7h6yx.top vyohacxzoue32vvk.dgjpgy.top vyohacxzoue32vvk.dks71o.bid vyohacxzoue32vvk.ean5e7.top vyohacxzoue32vvk.ewfp5y.bid vyohacxzoue32vvk.ezb568.top vyohacxzoue32vvk.fp6fj6.top vyohacxzoue32vvk.fsly47.top vyohacxzoue32vvk.g7rst5.bid vyohacxzoue32vvk.gjbmis.top vyohacxzoue32vvk.h2xun1.top vyohacxzoue32vvk.ibar8s.top vyohacxzoue32vvk.jb4uh0.top vyohacxzoue32vvk.jnv1df.top vyohacxzoue32vvk.joco7r.top vyohacxzoue32vvk.jwi2ek.bid vyohacxzoue32vvk.k9p80d.top vyohacxzoue32vvk.kfymbh.top vyohacxzoue32vvk.kwrd4f.bid vyohacxzoue32vvk.l4dlll.bid vyohacxzoue32vvk.mayrwf.top vyohacxzoue32vvk.mpduf5.bid vyohacxzoue32vvk.ncw0rp.top vyohacxzoue32vvk.nta934.top vyohacxzoue32vvk.o08ra6.top vyohacxzoue32vvk.o5b17o.top vyohacxzoue32vvk.p9su2u.top vyohacxzoue32vvk.pr52ni.top vyohacxzoue32vvk.r31sot.top vyohacxzoue32vvk.r3b2sh.top vyohacxzoue32vvk.roep3o.top vyohacxzoue32vvk.ss8doe.top vyohacxzoue32vvk.t6ueop.bid vyohacxzoue32vvk.u8e2dz.top vyohacxzoue32vvk.ug6ewx.top vyohacxzoue32vvk.vjso7r.top vyohacxzoue32vvk.w22p3v.top vyohacxzoue32vvk.w67y8u.bid vyohacxzoue32vvk.x83zw1.top vyohacxzoue32vvk.xsf5a8.top vyohacxzoue32vvk.xy2rlg.bid vyohacxzoue32vvk.zmn16h.top vyohacxzoue32vvk.zn90h4.bid vyohacxzoue32vvk.zp9i1l.bid vyohacxzoue32vvk.zu3fzc.bid vyohacxzoue32vvk.zz3w5l.bid w6bfg4hahn5bfnlsafgchkvg5fwsfvrt.hareuna.at waduavfijwkanvf.xyz wbaskcsxiffiax.info wdvxeval.ru wersalitrestyws.top wjfkoqueatxdmqw.biz wjtqjleommc4z46i.249isv.bid wjtqjleommc4z46i.2y4t6f.bid wjtqjleommc4z46i.35rof4.bid wjtqjleommc4z46i.35u068.bid wjtqjleommc4z46i.44vva6.bid wjtqjleommc4z46i.4bb9vz.bid wjtqjleommc4z46i.54vw9b.bid wjtqjleommc4z46i.5n5y6v.bid wjtqjleommc4z46i.5r1sol.bid wjtqjleommc4z46i.7hu6og.bid wjtqjleommc4z46i.8a9r2h.bid wjtqjleommc4z46i.993hev.bid wjtqjleommc4z46i.9sellg.bid wjtqjleommc4z46i.9ule2e.bid wjtqjleommc4z46i.au6d1d.bid wjtqjleommc4z46i.bipa9k.bid wjtqjleommc4z46i.c3fz3z.bid wjtqjleommc4z46i.cc0r87.bid wjtqjleommc4z46i.cdyd2z.bid wjtqjleommc4z46i.cgab48.bid wjtqjleommc4z46i.cm5ohx.bid wjtqjleommc4z46i.csv7o6.bid wjtqjleommc4z46i.cto5ee.bid wjtqjleommc4z46i.d11zjd.bid wjtqjleommc4z46i.e53rg4.bid wjtqjleommc4z46i.eag72x.top wjtqjleommc4z46i.efyh72.bid wjtqjleommc4z46i.f0jlbj.bid wjtqjleommc4z46i.fw1bwy.bid wjtqjleommc4z46i.fwfu4t.bid wjtqjleommc4z46i.gg4dgp.bid wjtqjleommc4z46i.h8prbu.top wjtqjleommc4z46i.hom07d.bid wjtqjleommc4z46i.i8zh1k.bid wjtqjleommc4z46i.idw6s5.bid wjtqjleommc4z46i.ilmgcl.bid wjtqjleommc4z46i.izyclz.bid wjtqjleommc4z46i.j0n83w.bid wjtqjleommc4z46i.jal9lk.bid wjtqjleommc4z46i.jujthy.bid wjtqjleommc4z46i.kt70uk.bid wjtqjleommc4z46i.kyjw0g.bid wjtqjleommc4z46i.kzhzuc.top wjtqjleommc4z46i.ldsl8m.bid wjtqjleommc4z46i.m33d4b.bid wjtqjleommc4z46i.n8ln0w.bid wjtqjleommc4z46i.nh47ri.bid wjtqjleommc4z46i.nnbdlh.bid wjtqjleommc4z46i.nxmu0x.bid wjtqjleommc4z46i.o8hpwj.top wjtqjleommc4z46i.obx4vo.bid wjtqjleommc4z46i.oodvxp.bid wjtqjleommc4z46i.p41khf.bid wjtqjleommc4z46i.pmnz7a.bid wjtqjleommc4z46i.salethe.gdn wjtqjleommc4z46i.srmlzh.bid wjtqjleommc4z46i.srtos7.bid wjtqjleommc4z46i.t4jp3w.bid wjtqjleommc4z46i.u36ik0.bid wjtqjleommc4z46i.uv39h5.bid wjtqjleommc4z46i.uwckha.top wjtqjleommc4z46i.vh6vss.bid wjtqjleommc4z46i.w3r6a4.bid wjtqjleommc4z46i.whmykv.bid wjtqjleommc4z46i.xjwlms.bid wjtqjleommc4z46i.y12acl.bid wjtqjleommc4z46i.y2ijlz.bid wjtqjleommc4z46i.y7603i.bid wjtqjleommc4z46i.yfr0o1.bid wjtqjleommc4z46i.z7uxzg.bid wjtqjleommc4z46i.z97f9v.bid wjtqjleommc4z46i.zclhx9.bid wor4d.slewirk.at wpvvusso.xyz wqxvsxppjivs.pw wrubyjtvqhxaqkh.pw wtxvmsikbmtbq.pw wvltrlrnf.xyz www.1axb.com www.chromebewfk.top www.chromefastl.top www.chromehakc.top www.cleverdotl.top www.ddiopoola.top www.dealkolld.top www.dokjasura.top www.fkauueeepla.top www.flowerxpo.top www.foolalexas.top www.googlefoad.top www.newsectorbs.top www.newtonpaiva.br www.watherfka.top www.weekendlk.top x5sbb5gesp6kzwsh.frontmain.pl x5sbb5gesp6kzwsh.frontymen.pl x5sbb5gesp6kzwsh.homewind.pl x5sbb5gesp6kzwsh.mailteam.pl x5sbb5gesp6kzwsh.questpul.pl xfyubqmldwvuyar.yt xhrnfffaixawpuob.pw xmniabhrfafptwx.pw xofguhypjgvxrm.pw xpcx6erilkjced3j.16hwwh.top xpcx6erilkjced3j.16umxg.top xpcx6erilkjced3j.17gcun.top xpcx6erilkjced3j.18ey8e.top xpcx6erilkjced3j.19kdeh.top xpcx6erilkjced3j.1blery.top xpcx6erilkjced3j.1cgbcv.top xpcx6erilkjced3j.1ebjjq.top xpcx6erilkjced3j.1j9jad.top xpcx6erilkjced3j.1jyrty.top xpcx6erilkjced3j.1mfmkz.top xpcx6erilkjced3j.1mpsnr.top xpcx6erilkjced3j.1n5mod.top xrhwryizf5mui7a5.50mb1c.bid xrhwryizf5mui7a5.djintc.bid xrhwryizf5mui7a5.g72xh8.top xrhwryizf5mui7a5.h44l3d.bid xrhwryizf5mui7a5.j4cser.bid xrhwryizf5mui7a5.jhrb5a.top xrhwryizf5mui7a5.r8c85p.top xrhwryizf5mui7a5.rt01jw.top xrhwryizf5mui7a5.uw9x7z.bid xrhwryizf5mui7a5.vgxcci.top xvchcbeqxkd.pw xyhhuxa.be y4bxj.adozeuds.com yavmxpiqfwmubk.pw yaynawvtuqcarjwc.pw ycvcjbhgkmsiyhdd.info yofkhfskdyiqo.biz ytcijiooxdtlbevrh.info ytrest84y5i456hghadefdsd.pontogrot.com yuertao.pw yuysikankhqvdwdv.xyz ywjgjvpuyitnbiw.info yyre45dbvn2nhbefbmh.begumvelic.at zjfq4lnfbs7pncr5.onion.to zjfq4lnfbs7pncr5.tor2web.org ztuw5bvuuapzdfya.klimbim.pl zutzt67dcxr6mxcn.onion.to ================================================ FILE: tests/integration/basic/gen-results.sh ================================================ #!/bin/bash curl -# -k -o IPv4HC.result "https://127.0.0.1/feeds/IPv4HC" -o IPv4HC.result curl -# -k -o IPv4HC%3Fs%3D5%26n%3D10.result "https://127.0.0.1/feeds/IPv4HC?s=5&n=10" curl -# -k -o IPv4HC%3Fv%3Djson%26tr%3D1.result "https://127.0.0.1/feeds/IPv4HC?v=json&tr=1" curl -# -k -o IPv4HC%3Fv%3Djson-seq.result "https://127.0.0.1/feeds/IPv4HC?v=json-seq" curl -# -k -o IPv4HC%3Fv%3Dmwg.result "https://127.0.0.1/feeds/IPv4HC?v=mwg" curl -# -k -o IPv4HC%3Fv%3Dcsv%26f%3Dconfidence%26f%3Dsources%7Cfeeds%26f%3Dindicator%7Cclientip%26tr%3D1.result "https://127.0.0.1/feeds/IPv4HC?v=csv&f=confidence&f=sources|feeds&f=indicator|clientip&tr=1" curl -# -k -o URLHC.result "https://127.0.0.1/feeds/URLHC" curl -# -k -o URLHC%3Fs%3D5%26n%3D10.result "https://127.0.0.1/feeds/URLHC?s=5&n=10" curl -# -k -o URLHC%3Fv%3Djson%26tr%3D1.result "https://127.0.0.1/feeds/URLHC?v=json&tr=1" curl -# -k -o URLHC%3Fv%3Djson-seq.result "https://127.0.0.1/feeds/URLHC?v=json-seq" curl -# -k -o URLHC%3Fv%3Dmwg.result "https://127.0.0.1/feeds/URLHC?v=mwg" curl -# -k -o URLHC%3Fv%3Dcsv%26f%3Dconfidence%26f%3Dsources%7Cfeeds%26f%3Dindicator%7Curl.result "https://127.0.0.1/feeds/URLHC?v=csv&f=confidence&f=sources|feeds&f=indicator|url" curl -# -k -o URLHC%3Fv%3Dbluecoat.result "https://127.0.0.1/feeds/URLHC?v=bluecoat" curl -# -k -o URLHC%3Fv%3Dbluecoat%26cd%3Dtest.result "https://127.0.0.1/feeds/URLHC?v=bluecoat&cd=test" curl -# -k -o URLHC%3Fv%3Dpanosurl.result "https://127.0.0.1/feeds/URLHC?v=panosurl" curl -# -k -o URLHC%3Fv%3Dpanosurl%26sp%3D1.result "https://127.0.0.1/feeds/URLHC?v=panosurl&sp=1" curl -# -k -o URLHC%3Fv%3Dpanosurl%26di%3D1.result "https://127.0.0.1/feeds/URLHC?v=panosurl&di=1" curl -# -k -o DomainHC%3Fv%3Dcarbonblack.result "https://127.0.0.1/feeds/DomainHC?v=carbonblack" ================================================ FILE: tests/integration/basic/test.py ================================================ #!/usr/bin/env python2 import logging import time import urllib import os import sys import re from itertools import izip_longest import requests import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) LOG = logging.getLogger(__name__) def get_full_config(mmurl, username, password): response = requests.get( '{}/config/full'.format(mmurl), auth=(username, password), verify=False ) response.raise_for_status() return response.json()['result'] def delete_node(nodeid, version, mmurl, username, password): response = requests.delete( '{}/config/node/{}'.format(mmurl, nodeid), params={'version': version}, auth=(username, password), verify=False ) response.raise_for_status() def create_node(version, nodename, prototype, inputs, output, mmurl, username, password): response = requests.post( '{}/config/node'.format(mmurl), auth=(username, password), verify=False, json={ 'name': nodename, 'version': version, 'properties': { 'prototype': prototype, 'inputs': inputs, 'output': output } }, headers={ 'Content-Type': 'application/json' } ) response.raise_for_status() def commit_and_restart(mmurl, username, password): full_config = get_full_config(mmurl, username, password) response = requests.post( '{}/config/commit'.format(mmurl), auth=(username, password), verify=False, json={ 'version': full_config['version'], }, headers={ 'Content-Type': 'application/json' } ) response.raise_for_status() response = requests.get( '{}/supervisor/minemeld-engine/restart'.format(mmurl), auth=(username, password), verify=False ) response.raise_for_status() def delete_config(mmurl, username, password): LOG.info('Deleting config...') full_config = get_full_config(mmurl, username, password) for idx, n in enumerate(full_config['nodes']): if not n: continue delete_node( idx, n['version'], mmurl, username, password ) full_config = get_full_config(mmurl, username, password) nnodes = len([n for n in full_config['nodes'] if n]) if nnodes != 0: raise RuntimeError('Config not deleted: {!r}'.format(full_config)) def create_config(mmurl, username, password): LOG.info('Creating new config...') full_config = get_full_config(mmurl, username, password) # miner create_node( version=full_config['version'], nodename='localdb', prototype='stdlib.localDB', output=True, inputs=[], mmurl=mmurl, username=username, password=password ) # URL flow create_node( version=full_config['version'], nodename='URLAggregator', prototype='stdlib.aggregatorURL', output=True, inputs=['localdb'], mmurl=mmurl, username=username, password=password ) create_node( version=full_config['version'], nodename='URLHC', prototype='stdlib.feedHCWithValue', output=True, inputs=['URLAggregator'], mmurl=mmurl, username=username, password=password ) # domain flow create_node( version=full_config['version'], nodename='DomainAggregator', prototype='stdlib.aggregatorDomain', output=True, inputs=['localdb'], mmurl=mmurl, username=username, password=password ) create_node( version=full_config['version'], nodename='DomainHC', prototype='stdlib.feedHCWithValue', output=True, inputs=['DomainAggregator'], mmurl=mmurl, username=username, password=password ) # IPv4 create_node( version=full_config['version'], nodename='IPv4Aggregator', prototype='stdlib.aggregatorIPv4Generic', output=True, inputs=['localdb'], mmurl=mmurl, username=username, password=password ) create_node( version=full_config['version'], nodename='IPv4HC', prototype='stdlib.feedHCWithValue', output=True, inputs=['IPv4Aggregator'], mmurl=mmurl, username=username, password=password ) def wait_for_restart(mmurl, username, password): LOG.info('Waiting for restart...') now = time.time() while time.time() < (now + 300): response = requests.get( '{}/supervisor'.format(mmurl), auth=(username, password), verify=False ) response.raise_for_status() supervisor_status = response.json()['result'] engine_status = supervisor_status['processes']['minemeld-engine']['statename'] if engine_status == 'RUNNING': break time.sleep(10) else: raise RuntimeError('engine did not restart in 5 minutes') def push_indicators(mmurl, username, password): LOG.info('Pushing indicators...') with open('IPv4.lst', 'r') as f: ipv4_iocs = f.readlines() with open('URL.lst', 'r') as f: url_iocs = f.readlines() with open('domain.lst', 'r') as f: domain_iocs = f.readlines() num_ipv4_iocs = len(ipv4_iocs) ipv4_iocs = ['IPv4\n{}\n\n'.format(ioc) for ioc in ipv4_iocs] num_url_iocs = len(url_iocs) url_iocs = ['URL\n{}\n\n'.format(ioc) for ioc in url_iocs] num_domain_iocs = len(domain_iocs) domain_iocs = ['domain\n{}\n\n'.format(ioc) for ioc in domain_iocs] response = requests.post( '{}/config/data/localdb_indicators/append'.format(mmurl), params={ 'h': 'localdb', 't': 'localdb' }, auth=(username, password), headers={'Content-Type': 'application/text'}, data=''.join(ipv4_iocs)+''.join(url_iocs)+''.join(domain_iocs), verify=False ) response.raise_for_status() # wait for URLs to propagate LOG.info('Waiting for URLs to propagate...') now = time.time() while time.time() < (now + 300): response = requests.get( '{}/feeds/URLHC'.format(mmurl), verify=False ) response.raise_for_status() if len(response.content.splitlines()) == num_url_iocs: break time.sleep(10) else: raise RuntimeError('URL IOCs did not propagate in 5 minutes') # wait for IPv4s to propagate LOG.info('Waiting for IPv4s to propagate...') now = time.time() while time.time() < (now + 300): response = requests.get( '{}/feeds/IPv4HC'.format(mmurl), verify=False ) response.raise_for_status() if len(response.content.splitlines()) == num_ipv4_iocs: break time.sleep(10) else: raise RuntimeError('IPv4 IOCs did not propagate in 5 minutes') # wait for domains to propagate LOG.info('Waiting for domains to propagate...') now = time.time() while time.time() < (now + 300): response = requests.get( '{}/feeds/DomainHC'.format(mmurl), verify=False ) response.raise_for_status() if len(response.content.splitlines()) == num_domain_iocs: break time.sleep(10) else: raise RuntimeError('domain IOCs did not propagate in 5 minutes') def remove_timestamps(s): s = re.sub(r'\"first_seen\":[0-9]+', '\"first_seen\":0', s) s = re.sub(r'\"last_seen\":[0-9]+', '\"last_seen\":0', s) s = re.sub(r'\"timestamp\": [0-9]+', '\"timestamp\": 0', s) return s def check_feeds(mmurl): check_result = True local_files = os.listdir('.') for fname in local_files: if not fname.endswith('.result'): continue req, _ = fname.split('.', 1) with open(fname, 'r') as f: result = f.readlines() LOG.info('Checking {}...'.format(urllib.unquote(req))) response = requests.get( '{}/feeds/{}'.format(mmurl, urllib.unquote(req)), verify=False ) response.raise_for_status() clines = response.content.splitlines() for idx, (cl, rl) in enumerate(izip_longest(result, clines)): cl = remove_timestamps(cl.strip()) rl = remove_timestamps(rl.strip()) if cl != rl: LOG.error('{} does not match'.format(urllib.unquote(req))) LOG.error('L{} expected: {!r}'.format(idx, rl)) LOG.error('L{} result: {!r}'.format(idx, cl)) check_result = False break return check_result def main(): logging.basicConfig(level=logging.INFO) mmurl = os.environ.get('MM_URL', 'https://127.0.0.1') username = os.environ.get('MM_USERNAME', 'admin') password = os.environ.get('MM_PASSWORD', 'minemeld') delete_config(mmurl, username, password) commit_and_restart(mmurl, username, password) wait_for_restart(mmurl, username, password) create_config(mmurl, username, password) commit_and_restart(mmurl, username, password) wait_for_restart(mmurl, username, password) push_indicators(mmurl, username, password) if not check_feeds(mmurl): sys.exit(1) if __name__ == '__main__': main() ================================================ FILE: tests/panos_mock.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import xml.etree.ElementTree import re import os.path import collections import logging MYDIR = os.path.dirname(__file__) SUBRE = re.compile("[^A-Za-z0-9_]") LOG = logging.getLogger(__name__) class PanXapiMock(object): def __init__(self, **kwargs): self.hostname = kwargs.get('hostname', 'xapi') self.counters = collections.defaultdict(lambda: 0) self.user_id_calls = [] def _parse_answer(self, cmd, arg, counter): file_name = '%s_%s_%s_%d' % (self.hostname, cmd, arg, counter) file_name = SUBRE.sub('_', file_name) file_name = os.path.join(MYDIR, file_name+'.xml') LOG.debug(file_name) try: os.stat(file_name) except OSError: file_name = '%s_%s_%s' % (self.hostname, cmd, arg) file_name = SUBRE.sub('_', file_name) file_name = os.path.join(MYDIR, file_name+'.xml') os.stat(file_name) self.element_root = xml.etree.ElementTree.parse(file_name).getroot() def get(self, **kwargs): xpath = kwargs.get('xpath', None) if xpath is None: raise RuntimeError('no xpath in get') c = self.counters['get:::'+xpath] self.counters['get:::'+xpath] += 1 self._parse_answer('get', xpath, c) def show(self, **kwargs): xpath = kwargs.get('xpath', None) if xpath is None: raise RuntimeError('no xpath in show') c = self.counters['show:::'+xpath] self.counters['show:::'+xpath] += 1 self._parse_answer('show', xpath, c) def op(self, **kwargs): cmd = kwargs.get('cmd', None) if cmd is None: raise RuntimeError('no cmd in op') c = self.counters['op:::'+cmd] self.counters['op:::'+cmd] += 1 self._parse_answer('op', cmd, c) def user_id(self, **kwargs): cmd = kwargs.get('cmd', None) if cmd is None: raise RuntimeError('no cmd in user_id') self.user_id_calls.append(cmd) def factory(**kwargs): return PanXapiMock(**kwargs) ================================================ FILE: tests/st_profile.py ================================================ #!/usr/bin/env python # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import uuid import random import tempfile import time import minemeld.ft.st TABLENAME = tempfile.mktemp(prefix='minemeld.ftsttest') def queries(st): # t1 = time.time() # j = 0 # for j in xrange(num_queries): # q = random.randint(0, 0xFFFFFFFF) # t2 = time.time() # dt = t2-t1 # t1 = time.time() j = 0 for j in xrange(num_queries): q = random.randint(0, 0xFFFFFFFF) next(st.cover(q), None) # t2 = time.time() # print "TIME: Queried %d times in %d" % (num_queries, (t2-t1-dt)) if __name__ == '__main__': num_intervals = 100000 num_queries = num_intervals st = minemeld.ft.st.ST(TABLENAME, 32, truncate=True) t1 = time.time() for j in xrange(num_intervals): end = random.randint(0, 0xFFFFFFFF) if random.randint(0, 1) == 0: end = end & 0xFFFFFF00 start = end + 0xFF else: start = end sid = uuid.uuid4().bytes t2 = time.time() dt = t2-t1 t1 = time.time() for j in xrange(num_intervals): end = random.randint(0, 0xFFFFFFFF) if random.randint(0, 1) == 0: start = end & 0xFFFFFF00 end = start + 0xFF else: start = end sid = uuid.uuid4().bytes st.put(sid, start, end) t2 = time.time() print "TIME: Inserted %d intervals in %d" % (num_intervals, (t2-t1-dt)) queries(st) ================================================ FILE: tests/test-prototype-1.yml ================================================ nodes: testprototype: prototype: testproto.test ================================================ FILE: tests/test_comm_amqp.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import unittest import minemeld.comm.amqp class MineMeldCommAMQP(unittest.TestCase): def test_01_rpc(self): class A(object): def f(self): return 'ok' a = A() ac = minemeld.comm.amqp.AMQP({}) ac.request_rpc_server_channel('a', a, allowed_methods=['f']) ac.start() result = ac.send_rpc('a', 'f', {}, timeout=1) self.assertEqual(result['result'], 'ok') ac.stop() def test_02_pubsub(self): class A(object): counter = 0 def f(self): self.counter += 1 a = A() ac = minemeld.comm.amqp.AMQP({}) ac.request_sub_channel('a', a, allowed_methods=['f']) pc = ac.request_pub_channel('a') ac.start() pc.publish('f') gevent.sleep(0.1) self.assertEqual(a.counter, 1) ac.stop() def test_03_rpc_fanout(self): class A(object): def __init__(self, n): self.n = n def f(self): return self.n a1 = A(1) a2 = A(2) ac = minemeld.comm.amqp.AMQP({}) ac.request_rpc_server_channel('a1', a1, allowed_methods=['f'], fanout='test') ac.request_rpc_server_channel('a2', a2, allowed_methods=['f'], fanout='test') client = ac.request_rpc_fanout_client_channel('test') ac.start() evt = client.send_rpc('f', params={}, num_results=2) success = evt.wait(timeout=5) self.assertNotEqual(success, None) result = evt.get(block=False) self.assertEqual(result['answers'], {'a1': 1, 'a2': 2}) ac.stop() ================================================ FILE: tests/test_device_list.yml ================================================ - name: test hostname: 192.168.55.152 api_username: admin api_password: admin - name: test2 tag: ngfw1 ================================================ FILE: tests/test_device_list2.yml ================================================ - name: test2 tag: ngfw2 - name: test hostname: 192.168.55.152 api_username: admin api_password: admin ================================================ FILE: tests/test_flask_aaa.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT autofocus tests Unit tests for minemeld.ft.autofocus """ import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import unittest import mock import shutil import logging import os import os.path import base64 import passlib.apache import xmltodict os.environ['MM_CONFIG'] = '.' os.environ['API_CONFIG_LOCK'] = os.path.join('.', 'api-config.lock') import minemeld.flask.main import minemeld.flask.feedredis LOG = logging.getLogger(__name__) MYDIR = os.path.dirname(__file__) TAXII_POLL_REQUEST = """ 2014-12-19T00:00:00Z 2014-12-19T12:00:00Z FULL """ def _authorization_header(username, password): return 'Basic '+base64.b64encode(username+':'+password) class MineMeldFlaskAAATests(unittest.TestCase): def setUp(self): try: shutil.rmtree('./api') except OSError: pass os.mkdir('api') minemeld.flask.main.app.config.update( DEBUG=True ) self.app = minemeld.flask.main.app.test_client() def tearDown(self): try: shutil.rmtree('./api') except OSError: pass def _taxii_discovery_request(self, username=None, password=None): headers = { 'X-TAXII-Content-Type': 'urn:taxii.mitre.org:message:xml:1.1', 'X-TAXII-Protocol': 'urn:taxii.mitre.org:protocol:http:1.0', 'X-TAXII-Services': 'urn:taxii.mitre.org:services:1.1' } if username is not None: headers['Authorization'] = _authorization_header(username, password) resp = self.app.post( '/taxii-discovery-service', headers=headers, data='' ) return resp def _taxii_collection_request(self, username=None, password=None): headers = { 'X-TAXII-Content-Type': 'urn:taxii.mitre.org:message:xml:1.1', 'X-TAXII-Protocol': 'urn:taxii.mitre.org:protocol:http:1.0', 'X-TAXII-Services': 'urn:taxii.mitre.org:services:1.1' } if username is not None: headers['Authorization'] = _authorization_header(username, password) resp = self.app.post( '/taxii-collection-management-service', headers=headers, data='' ) return resp def _taxii_poll_request(self, collection, username=None, password=None): headers = { 'X-TAXII-Content-Type': 'urn:taxii.mitre.org:message:xml:1.1', 'X-TAXII-Protocol': 'urn:taxii.mitre.org:protocol:http:1.0', 'X-TAXII-Services': 'urn:taxii.mitre.org:services:1.1' } if username is not None: headers['Authorization'] = _authorization_header(username, password) resp = self.app.post( '/taxii-poll-service', headers=headers, data=TAXII_POLL_REQUEST % collection ) return resp def _num_collections(self, resp): ans = xmltodict.parse(resp.data) collections = ans['taxii_11:Collection_Information_Response']['taxii_11:Collection'] if isinstance(collections, list): return len(collections) return 1 @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.feedredis.MMMaster') def test_feeds_auth_disabled(self, mmmastermock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': False } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) mmmastermock.status.return_value = { 'mbus:slave:feed1': { 'class': 'minemeld.ft.redis.RedisSet' } } resp = self.app.get('/feeds/feed1') self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('guest', 'guest') }) self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('user1', 'password1') }) self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('admin', 'password') }) self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.feedredis.MMMaster') def test_feeds_single_tag(self, mmmastermock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) mmmastermock.status.return_value = { 'mbus:slave:feed1': { 'class': 'minemeld.ft.redis.RedisSet' } } resp = self.app.get('/feeds/feed1') self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('guest', 'guest') }) self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('user1', 'password1') }) self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('admin', 'password') }) self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.feedredis.MMMaster') def test_feeds_two_tags(self, mmmastermock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential', 'open'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) mmmastermock.status.return_value = { 'mbus:slave:feed1': { 'class': 'minemeld.ft.redis.RedisSet' } } resp = self.app.get('/feeds/feed1') self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('guest', 'guest') }) self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('user1', 'password1') }) self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('admin', 'password') }) self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.feedredis.MMMaster') def test_feeds_two_and_two(self, mmmastermock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential', 'open'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) mmmastermock.status.return_value = { 'mbus:slave:feed1': { 'class': 'minemeld.ft.redis.RedisSet' } } resp = self.app.get('/feeds/feed1') self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('guest', 'guest') }) self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('user1', 'password1') }) self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('admin', 'password') }) self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.feedredis.MMMaster') def test_feeds_any(self, mmmastermock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['any'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) mmmastermock.status.return_value = { 'mbus:slave:feed1': { 'class': 'minemeld.ft.redis.RedisSet' } } resp = self.app.get('/feeds/feed1') self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('guest', 'guest') }) self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('user1', 'password1') }) self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('admin', 'password') }) self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.feedredis.MMMaster') def test_feeds_anonymous(self, mmmastermock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['anonymous', 'confidential'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) mmmastermock.status.return_value = { 'mbus:slave:feed1': { 'class': 'minemeld.ft.redis.RedisSet' } } resp = self.app.get('/feeds/feed1') self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('guest', 'guest') }) self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('user1', 'password1') }) self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('admin', 'password') }) self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.feedredis.MMMaster') def test_feeds_anonymous_2(self, mmmastermock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['anonymous'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) mmmastermock.status.return_value = { 'mbus:slave:feed1': { 'class': 'minemeld.ft.redis.RedisSet' } } resp = self.app.get('/feeds/feed1') self.assertEqual(resp.status_code, 200) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('guest', 'guest') }) self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('user1', 'password1') }) self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('admin', 'password') }) self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.feedredis.MMMaster') def test_feeds_no_tags(self, mmmastermock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': {} } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) mmmastermock.status.return_value = { 'mbus:slave:feed1': { 'class': 'minemeld.ft.redis.RedisSet' } } resp = self.app.get('/feeds/feed1') self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('guest', 'guest') }) self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('user1', 'password1') }) self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('admin', 'password') }) self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.feedredis.MMMaster') def test_feeds_no_user_tags(self, mmmastermock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) mmmastermock.status.return_value = { 'mbus:slave:feed1': { 'class': 'minemeld.ft.redis.RedisSet' } } resp = self.app.get('/feeds/feed1') self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('guest', 'guest') }) self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('user1', 'password1') }) self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': _authorization_header('admin', 'password') }) self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.feedredis.MMMaster') def test_feeds_malformed(self, mmmastermock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) mmmastermock.status.return_value = { 'mbus:slave:feed1': { 'class': 'minemeld.ft.redis.RedisSet' } } resp = self.app.get('/feeds/feed1', headers={ 'Authorization': 'invalid authorization' }) self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': 'Basic YWJjZGVmCg' }) self.assertEqual(resp.status_code, 401) resp = self.app.get('/feeds/feed1', headers={ 'Authorization': 'Basic '+base64.b64encode('invalidauth') }) self.assertEqual(resp.status_code, 401) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.taxiidiscovery.get_taxii_feeds', return_value=['feed1']) @mock.patch('minemeld.flask.taxiicollmgmt.get_taxii_feeds', return_value=['feed1']) def test_taxii_auth_disabled(self, gtfmock1, gtfmock2, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': False, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) resp = self._taxii_discovery_request() self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='guest', password='guest') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='user1', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='admin', password='password') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='user1', password='password2') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='admin', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request() self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='guest', password='guest') self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='user1', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='admin', password='password') self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='user1', password='password2') self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='admin', password='password2') self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.taxiidiscovery.get_taxii_feeds', return_value=['feed1', 'feed2']) @mock.patch('minemeld.flask.taxiicollmgmt.get_taxii_feeds', return_value=['feed1', 'feed2']) def test_taxii_services_tag(self, gtfmock1, gtfmock2, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential'] }, 'feed2': { 'tags': ['disabled'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) resp = self._taxii_discovery_request() self.assertEqual(resp.status_code, 401) resp = self._taxii_discovery_request(username='guest', password='guest') self.assertEqual(resp.status_code, 401) resp = self._taxii_discovery_request(username='user1', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='admin', password='password') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='user1', password='password2') self.assertEqual(resp.status_code, 401) resp = self._taxii_discovery_request(username='admin', password='password1') self.assertEqual(resp.status_code, 401) resp = self._taxii_collection_request() self.assertEqual(resp.status_code, 401) resp = self._taxii_collection_request(username='guest', password='guest') self.assertEqual(resp.status_code, 401) resp = self._taxii_collection_request(username='user1', password='password1') self.assertEqual(self._num_collections(resp), 1) self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='admin', password='password') self.assertEqual(self._num_collections(resp), 2) self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='user1', password='password2') self.assertEqual(resp.status_code, 401) resp = self._taxii_collection_request(username='admin', password='password2') self.assertEqual(resp.status_code, 401) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.taxiidiscovery.get_taxii_feeds', return_value=['feed1', 'feed2']) @mock.patch('minemeld.flask.taxiicollmgmt.get_taxii_feeds', return_value=['feed1', 'feed2']) def test_taxii_anonymous(self, gtfmock1, gtfmock2, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential'] }, 'feed2': { 'tags': ['anonymous'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) resp = self._taxii_discovery_request() self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='guest', password='guest') self.assertEqual(resp.status_code, 401) resp = self._taxii_discovery_request(username='user1', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='admin', password='password') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='user1', password='password2') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='admin', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request() self.assertEqual(self._num_collections(resp), 1) self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='guest', password='guest') self.assertEqual(resp.status_code, 401) resp = self._taxii_collection_request(username='user1', password='password1') self.assertEqual(self._num_collections(resp), 1) self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='admin', password='password') self.assertEqual(self._num_collections(resp), 2) self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='user1', password='password2') self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='admin', password='password2') self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.taxiidiscovery.get_taxii_feeds', return_value=['feed1', 'feed2']) @mock.patch('minemeld.flask.taxiicollmgmt.get_taxii_feeds', return_value=['feed1', 'feed2']) def test_taxii_any(self, gtfmock1, gtfmock2, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential'] }, 'feed2': { 'tags': ['any'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) resp = self._taxii_discovery_request() self.assertEqual(resp.status_code, 401) resp = self._taxii_discovery_request(username='guest', password='guest') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='user1', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='admin', password='password') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='user1', password='password2') self.assertEqual(resp.status_code, 401) resp = self._taxii_discovery_request(username='admin', password='password1') self.assertEqual(resp.status_code, 401) resp = self._taxii_collection_request() self.assertEqual(resp.status_code, 401) resp = self._taxii_collection_request(username='guest', password='guest') self.assertEqual(self._num_collections(resp), 1) self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='user1', password='password1') self.assertEqual(self._num_collections(resp), 2) self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='admin', password='password') self.assertEqual(self._num_collections(resp), 2) self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='user1', password='password2') self.assertEqual(resp.status_code, 401) resp = self._taxii_collection_request(username='admin', password='password2') self.assertEqual(resp.status_code, 401) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.taxiidiscovery.get_taxii_feeds', return_value=['feed1', 'feed2']) @mock.patch('minemeld.flask.taxiicollmgmt.get_taxii_feeds', return_value=['feed1', 'feed2']) def test_taxii_any_anonymous(self, gtfmock1, gtfmock2, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential'] }, 'feed2': { 'tags': ['any', 'anonymous'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) resp = self._taxii_discovery_request() self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='guest', password='guest') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='user1', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='admin', password='password') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='user1', password='password2') self.assertEqual(resp.status_code, 200) resp = self._taxii_discovery_request(username='admin', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request() self.assertEqual(self._num_collections(resp), 1) self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='guest', password='guest') self.assertEqual(self._num_collections(resp), 1) self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='user1', password='password1') self.assertEqual(self._num_collections(resp), 2) self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='admin', password='password') self.assertEqual(self._num_collections(resp), 2) self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='user1', password='password2') self.assertEqual(resp.status_code, 200) resp = self._taxii_collection_request(username='admin', password='password2') self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.taxiipoll.get_taxii_feeds', return_value=['feed1', 'feed2']) def test_taxiipoll_single_tag(self, gtfmock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) resp = self._taxii_poll_request('feed1') self.assertEqual(resp.status_code, 401) resp = self._taxii_poll_request('feed1', username='guest', password='guest') self.assertEqual(resp.status_code, 401) resp = self._taxii_poll_request('feed1', username='user1', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='admin', password='password') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='user1', password='password2') self.assertEqual(resp.status_code, 401) resp = self._taxii_poll_request('feed1', username='admin', password='password1') self.assertEqual(resp.status_code, 401) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.taxiipoll.get_taxii_feeds', return_value=['feed1', 'feed2']) def test_taxiipolll_two_tags(self, gtfmock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential', 'open'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) resp = self._taxii_poll_request('feed1') self.assertEqual(resp.status_code, 401) resp = self._taxii_poll_request('feed1', username='guest', password='guest') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='user1', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='admin', password='password') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='user1', password='password2') self.assertEqual(resp.status_code, 401) resp = self._taxii_poll_request('feed1', username='admin', password='password1') self.assertEqual(resp.status_code, 401) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.taxiipoll.get_taxii_feeds', return_value=['feed1', 'feed2']) def test_taxiipoll_two_and_two(self, gtfmock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['confidential', 'open'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) resp = self._taxii_poll_request('feed1') self.assertEqual(resp.status_code, 401) resp = self._taxii_poll_request('feed1', username='guest', password='guest') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='user1', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='admin', password='password') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='user1', password='password2') self.assertEqual(resp.status_code, 401) resp = self._taxii_poll_request('feed1', username='admin', password='password1') self.assertEqual(resp.status_code, 401) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.taxiipoll.get_taxii_feeds', return_value=['feed1', 'feed2']) def test_taxiipoll_any(self, gtfmock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['any'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) resp = self._taxii_poll_request('feed1') self.assertEqual(resp.status_code, 401) resp = self._taxii_poll_request('feed1', username='guest', password='guest') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='user1', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='admin', password='password') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='user1', password='password2') self.assertEqual(resp.status_code, 401) resp = self._taxii_poll_request('feed1', username='admin', password='password1') self.assertEqual(resp.status_code, 401) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.taxiipoll.get_taxii_feeds', return_value=['feed1', 'feed2']) def test_taxiipoll_anonymous(self, gtfmock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['anonymous', 'confidential'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) resp = self._taxii_poll_request('feed1') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='guest', password='guest') self.assertEqual(resp.status_code, 401) resp = self._taxii_poll_request('feed1', username='user1', password='password1') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='admin', password='password') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='user1', password='password2') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='admin', password='password1') self.assertEqual(resp.status_code, 200) @mock.patch.dict('minemeld.flask.config.os.environ', { 'MM_CONFIG': '.', 'API_CONFIG_LOCK': os.path.join('.', 'api-config.lock'), }) @mock.patch('minemeld.flask.config.init') @mock.patch('minemeld.flask.config.get') @mock.patch('minemeld.flask.taxiipoll.get_taxii_feeds', return_value=['feed1', 'feed2']) def test_taxiipoll_anonymous_2(self, gtfmock, configmock, configinitmock): _config_attrs = { 'API_AUTH_ENABLED': True, 'USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'wsgi.htpasswd')), 'FEEDS_USERS_DB': passlib.apache.HtpasswdFile(path=os.path.join(MYDIR, 'feeds.htpasswd')), 'FEEDS_AUTH_ENABLED': True, 'FEEDS_USERS_ATTRS': { 'guest': { 'tags': ['open', 'test'] }, 'user1': { 'tags': ['confidential'] } }, 'FEEDS_ATTRS': { 'feed1': { 'tags': ['anonymous'] } } } def _config_get(attribute, default=None): if attribute in _config_attrs: return _config_attrs[attribute] return default configmock.configure_mock(side_effect=_config_get) resp = self._taxii_poll_request('feed1') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='guest', password='guest') self.assertEqual(resp.status_code, 401) resp = self._taxii_poll_request('feed1', username='user1', password='password1') self.assertEqual(resp.status_code, 401) resp = self._taxii_poll_request('feed1', username='admin', password='password') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='user1', password='password2') self.assertEqual(resp.status_code, 200) resp = self._taxii_poll_request('feed1', username='admin', password='password1') self.assertEqual(resp.status_code, 200) ================================================ FILE: tests/test_ft_autofocus.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT autofocus tests Unit tests for minemeld.ft.autofocus """ import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import unittest import mock import time import shutil import logging import gc import calendar import os.path import minemeld.ft.autofocus FTNAME = 'testft-%d' % int(time.time()) LOG = logging.getLogger(__name__) CUR_LOGICAL_TIME = 0 MYDIR = os.path.dirname(__file__) def logical_millisec(*args): return CUR_LOGICAL_TIME def gevent_event_mock_factory(): result = mock.Mock() result.wait.side_effect = gevent.GreenletExit() return result class MineMeldAutofocusFTTests(unittest.TestCase): def setUp(self): try: shutil.rmtree(FTNAME) except: pass def tearDown(self): try: shutil.rmtree(FTNAME) except: pass @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch.object(calendar, 'timegm', side_effect=logical_millisec) def test_type_of_indicators(self, um_mock, sleep_mock, event_mock, spawnl_mock, spawn_mock): chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock config = { 'side_config': os.path.join(MYDIR, 'dummy.yml') } a = minemeld.ft.autofocus.ExportList(FTNAME, chassis, config) inputs = [] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 1) self.assertEqual(spawn_mock.call_count, 3) self.assertEqual(a._type_of_indicator('1.1.1.1'), 'IPv4') self.assertEqual(a._type_of_indicator('1.1.1.2-1.1.1.5'), 'IPv4') self.assertEqual(a._type_of_indicator('1.1.1.0/24'), 'IPv4') self.assertEqual(a._type_of_indicator('www.google.com'), 'domain') self.assertEqual(a._type_of_indicator('www.google.com/test'), 'URL') self.assertEqual(a._type_of_indicator('https://www.google.com'), 'URL') a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() ================================================ FILE: tests/test_ft_base.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT Table tests Unit tests for minemeld.ft.base """ import unittest import mock import gevent import minemeld.ft.base import minemeld.ft class MineMeldFTBaseTests(unittest.TestCase): @mock.patch.object(minemeld.ft.base.BaseFT, 'configure', return_value=None) def test_init(self, configure_mock): config = {} chassis = mock.Mock() bcls = minemeld.ft.base.BaseFT b = bcls('test', chassis, config) self.assertEqual(b.name, 'test') self.assertEqual(b.chassis, chassis) self.assertEqual(b.config, config) self.assertItemsEqual(b.inputs, []) self.assertEqual(b.output, None) self.assertEqual(b.configure.call_count, 1) self.assertEqual(b.state, minemeld.ft.ft_states.READY) def test_connect_io(self): ftname = 'test' config = {} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None b = minemeld.ft.base.BaseFT(ftname, chassis, config) inputs = ['a', 'b', 'c'] output = True b.connect(inputs, output) self.assertItemsEqual(b.inputs, inputs) self.assertEqual(b.output, ochannel) icalls = [] for i in inputs: icalls.append( mock.call(ftname, b, i, allowed_methods=['update', 'withdraw', 'checkpoint']) ) chassis.request_sub_channel.assert_has_calls( icalls, any_order=True ) chassis.request_rpc_channel.assert_called_once_with( ftname, b, allowed_methods=[ 'update', 'withdraw', 'checkpoint', 'get', 'get_all', 'get_range', 'length' ] ) chassis.request_pub_channel.assert_called_once_with(ftname) def test_rpc(self): ftname = 'test' config = {} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None b = minemeld.ft.base.BaseFT(ftname, chassis, config) inputs = [] output = False b.connect(inputs, output) chassis.request_sub_channel.assert_not_called() chassis.request_pub_channel.assert_not_called() chassis.request_rpc_channel.assert_called_once_with( ftname, b, allowed_methods=[ 'update', 'withdraw', 'checkpoint', 'get', 'get_all', 'get_range', 'length' ] ) b.do_rpc('destft', 'rpcmethod', a=1, b=1) chassis.send_rpc.assert_called_once_with( ftname, 'destft', 'rpcmethod', {'a': 1, 'b': 1}, timeout=30, block=True ) def test_emit(self): ftname = 'test' config = {} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None b = minemeld.ft.base.BaseFT(ftname, chassis, config) inputs = [] output = True b.connect(inputs, output) chassis.request_sub_channel.assert_not_called() chassis.request_pub_channel.assert_called_once_with(ftname) chassis.request_rpc_channel.assert_called_once_with( ftname, b, allowed_methods=[ 'update', 'withdraw', 'checkpoint', 'get', 'get_all', 'get_range', 'length' ] ) b.emit_update('testi', {'test': 'v'}) self.assertEqual(ochannel.publish.call_count, 1) self.assertEqual(ochannel.publish.call_args[0][0], 'update') self.assertEqual(ochannel.publish.call_args[0][1]['indicator'], 'testi') self.assertEqual(ochannel.publish.call_args[0][1]['value']['test'], 'v') b.emit_withdraw('testi', {'test': 'v'}) self.assertEqual(ochannel.publish.call_count, 2) self.assertEqual(ochannel.publish.call_args[0][0], 'withdraw') self.assertEqual(ochannel.publish.call_args[0][1]['indicator'], 'testi') self.assertEqual(ochannel.publish.call_args[0][1]['value']['test'], 'v') def test_emit_filtered(self): ftname = 'test' config = { 'outfilters': [ { 'name': 'rule1', 'conditions': [ "direction == 'inbound'", "type == 'IPv4'" ], 'actions': ['accept'] }, { 'name': 'rule2', 'actions': ['drop'] } ] } chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None b = minemeld.ft.base.BaseFT(ftname, chassis, config) inputs = [] output = True b.connect(inputs, output) chassis.request_sub_channel.assert_not_called() chassis.request_pub_channel.assert_called_once_with(ftname) chassis.request_rpc_channel.assert_called_once_with( ftname, b, allowed_methods=[ 'update', 'withdraw', 'checkpoint', 'get', 'get_all', 'get_range', 'length' ] ) b.emit_update('testi', {'type': 'IPv6', 'direction': 'inbound'}) self.assertEqual(ochannel.publish.call_count, 0) ochannel.publish.reset_mock() b.emit_withdraw('testi', {'type': 'IPv6', 'direction': 'inbound'}) self.assertEqual(ochannel.publish.call_count, 0) ochannel.publish.reset_mock() b.emit_update('testi', {'type': 'IPv4', 'direction': 'inbound'}) self.assertEqual(ochannel.publish.call_count, 1) ochannel.publish.reset_mock() b.emit_update('testi', {'type': 'IPv4', 'direction': 'outbound'}) self.assertEqual(ochannel.publish.call_count, 0) ochannel.publish.reset_mock() b.emit_update('testi', {'type': 'IPv6', 'direction': 'inbound'}) self.assertEqual(ochannel.publish.call_count, 0) ochannel.publish.reset_mock() b.emit_update('testi', {'type': 'IPv6', 'direction': 'outbound'}) self.assertEqual(ochannel.publish.call_count, 0) def test_full_trace(self): config = {} chassis = mock.Mock() bcls = minemeld.ft.base.BaseFT b = bcls('test', chassis, config) b._disable_full_trace = True b.enable_full_trace(timeout=1) self.assertEqual(b._disable_full_trace, False) gevent.sleep(1.5) self.assertEqual(b._disable_full_trace, True) ================================================ FILE: tests/test_ft_basepoller.py ================================================ # -*- coding: utf-8 -*- # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT basepoller tests Unit tests for minemeld.ft.basepoller """ import gevent.hub import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import unittest import mock import time import shutil import logging import gc import minemeld.ft.basepoller import minemeld.ft.table FTNAME = 'testft-%d' % int(time.time()) LOG = logging.getLogger(__name__) CUR_LOGICAL_TIME = 0 def logical_millisec(*args): return CUR_LOGICAL_TIME*1000 def gevent_event_mock_factory(): result = mock.Mock() result.wait.side_effect = gevent.GreenletExit() return result class DeltaFeed(minemeld.ft.basepoller.BasePollerFT): def __init__(self, name, chassis): config = { 'age_out': { 'default': 'last_seen+4', 'sudden_death': False } } super(DeltaFeed, self).__init__(name, chassis, config) self.cur_iterator = 0 self.iterators = [ ['A', 'B', 'C'], ['D', 'E', 'F'] ] def _build_iterator(self, now): r = [] if self.cur_iterator < len(self.iterators): r = self.iterators[self.cur_iterator] self.cur_iterator += 1 return r def _process_item(self, item): return [[item, {'type': 'IPv4'}]] class RollingFeed(minemeld.ft.basepoller.BasePollerFT): def __init__(self, name, chassis): config = { 'age_out': { 'default': 'last_seen+4', 'sudden_death': True } } super(RollingFeed, self).__init__(name, chassis, config) self.cur_iterator = 0 self.iterators = [ ['A', 'B', 'C'], ['B', 'C', 'D'] ] def _build_iterator(self, now): r = [] if self.cur_iterator < len(self.iterators): r = self.iterators[self.cur_iterator] self.cur_iterator += 1 return r def _process_item(self, item): return [[item, {'type': 'IPv4'}]] class RollingFeedFirst(minemeld.ft.basepoller.BasePollerFT): def __init__(self, name, chassis): config = { 'age_out': { 'default': 'first_seen+1', 'sudden_death': True } } super(RollingFeedFirst, self).__init__(name, chassis, config) self.cur_iterator = 0 self.iterators = [ ['A', 'B', 'C'], ['B', 'C', 'D'], ['B', 'E', 'F'], ['E', 'F', 'G'], ['B', 'F', 'G'] ] def _build_iterator(self, now): r = [] if self.cur_iterator < len(self.iterators): r = self.iterators[self.cur_iterator] self.cur_iterator += 1 return r def _process_item(self, item): return [[item, {'type': 'IPv4'}]] class RollingFeedFirst2(minemeld.ft.basepoller.BasePollerFT): def __init__(self, name, chassis): config = { 'age_out': { 'default': 'first_seen+1', 'sudden_death': True } } super(RollingFeedFirst2, self).__init__(name, chassis, config) self.cur_iterator = 0 self.iterators = [ ['A'], ['A'], ['A'], [], ['A'] ] def _build_iterator(self, now): r = [] if self.cur_iterator < len(self.iterators): r = self.iterators[self.cur_iterator] self.cur_iterator += 1 return r def _process_item(self, item): return [[item, {'type': 'IPv4'}]] class PermanentFeed(minemeld.ft.basepoller.BasePollerFT): def __init__(self, name, chassis): config = { 'age_out': { 'default': None, 'sudden_death': True } } super(PermanentFeed, self).__init__(name, chassis, config) self.cur_iterator = 0 self.iterators = [ ['A', 'B', 'C'], ['B', 'C', 'D'] ] def _build_iterator(self, now): r = [] if self.cur_iterator < len(self.iterators): r = self.iterators[self.cur_iterator] self.cur_iterator += 1 return r def _process_item(self, item): return [[item, {'type': 'IPv4'}]] class SuperPermanentFeed(minemeld.ft.basepoller.BasePollerFT): def __init__(self, name, chassis): config = { 'age_out': { 'interval': None, 'default': None, 'sudden_death': True } } super(SuperPermanentFeed, self).__init__(name, chassis, config) self.cur_iterator = 0 self.iterators = [ ['A', 'B', 'C'], ['B', 'C', 'D'] ] def _build_iterator(self, now): r = [] if self.cur_iterator < len(self.iterators): r = self.iterators[self.cur_iterator] self.cur_iterator += 1 return r def _process_item(self, item): return [[item, {'type': 'IPv4'}]] class PermanentFeedWithType(minemeld.ft.basepoller.BasePollerFT): def __init__(self, name, chassis): config = { 'multiple_indicator_types': True, 'age_out': { 'default': None, 'sudden_death': True } } super(PermanentFeedWithType, self).__init__(name, chassis, config) self.cur_iterator = 0 self.iterators = [ ['IPv4@A', 'domain@B', 'domain@C'], ['IPv4@B', 'domain@C', 'IPv4@D'] ] def _build_iterator(self, now): r = [] if self.cur_iterator < len(self.iterators): r = self.iterators[self.cur_iterator] self.cur_iterator += 1 return r def _process_item(self, item): it, i = item.split('@', 1) return [[i, {'type': it}]] class PermanentFeedWithTypeAggregated(minemeld.ft.basepoller.BasePollerFT): def __init__(self, name, chassis): config = { 'multiple_indicator_types': True, 'aggregate_indicators': True, 'age_out': { 'default': None, 'sudden_death': True } } super(PermanentFeedWithTypeAggregated, self).__init__(name, chassis, config) self.cur_iterator = 0 self.iterators = [ ['IPv4@A@1', 'domain@B@1', 'domain@C@1', 'IPv4@A@2'], ['IPv4@B@1', 'domain@C@1', 'IPv4@D@1', 'IPv4@D@2'] ] def _build_iterator(self, now): r = [] if self.cur_iterator < len(self.iterators): r = self.iterators[self.cur_iterator] self.cur_iterator += 1 return r def _process_item(self, item): it, i, v = item.split('@', 2) return [[i, {'type': it, 'attribute': v}]] class DeltaFeedWithTypeAggregatedFaulty(minemeld.ft.basepoller.BasePollerFT): def __init__(self, name, chassis): config = { 'multiple_indicator_types': True, 'aggregate_indicators': True, 'aggregate_use_partial': True, 'age_out': { 'default': 'last_seen+4', 'sudden_death': False } } super(DeltaFeedWithTypeAggregatedFaulty, self).__init__(name, chassis, config) self.cur_iterator = 0 self.cur_iterator = 1 self.last_time = None self.iterators = [ ['IPv4@A@1', 'domain@B@1', 'domain@C@1', 'IPv4@A@2'], ['IPv4@B@1', 'domain@C@1', 'IPv4@D@1', 'IPv4@D@2'] ] def _iterator(self, iterator): for i in iterator: yield i raise RuntimeError('BAM !') def _build_iterator(self, now): if self.last_time is None: self.last_time = CUR_LOGICAL_TIME if self.last_time == CUR_LOGICAL_TIME: self.cur_iterator -= 1 self.last_time = CUR_LOGICAL_TIME LOG.info('cur_iterator: {} time: {}'.format(self.cur_iterator, CUR_LOGICAL_TIME)) r = [] if self.cur_iterator < len(self.iterators): r = self._iterator(self.iterators[self.cur_iterator]) self.cur_iterator += 1 return r def _process_item(self, item): it, i, v = item.split('@', 2) return [[i, {'type': it, 'attribute': v}]] class MineMeldFTBasePollerTests(unittest.TestCase): def setUp(self): try: shutil.rmtree(FTNAME) except: pass def tearDown(self): try: shutil.rmtree(FTNAME) except: pass @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_delta_feed(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): global CUR_LOGICAL_TIME chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = DeltaFeed(FTNAME, chassis) inputs = [] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 2) self.assertEqual(spawn_mock.call_count, 3) CUR_LOGICAL_TIME = 1 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) self.assertEqual(um_mock.call_count, 1) CUR_LOGICAL_TIME = 2 a._poll() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 3) self.assertEqual(a.statistics.get('removed', 0), 0) CUR_LOGICAL_TIME = 3 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) CUR_LOGICAL_TIME = 4 a._poll() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 6) self.assertEqual(a.statistics.get('removed', 0), 0) self.assertEqual(a.statistics.get('garbage_collected', 0), 0) CUR_LOGICAL_TIME = 5 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) CUR_LOGICAL_TIME = 6 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) CUR_LOGICAL_TIME = 7 a._age_out() self.assertEqual(a.statistics['aged_out'], 3) CUR_LOGICAL_TIME = 8 a._poll() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 6) self.assertEqual(a.statistics.get('garbage_collected', 0), 3) self.assertEqual(a.length(), 3) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_rolling_feed(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): global CUR_LOGICAL_TIME chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = RollingFeed(FTNAME, chassis) inputs = [] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 2) self.assertEqual(spawn_mock.call_count, 3) CUR_LOGICAL_TIME = 1 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) self.assertEqual(um_mock.call_count, 1) CUR_LOGICAL_TIME = 2 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 3) self.assertEqual(a.statistics.get('removed', 0), 0) CUR_LOGICAL_TIME = 3 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) CUR_LOGICAL_TIME = 4 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 4) self.assertEqual(a.statistics.get('removed', 0), 1) self.assertEqual(a.statistics.get('garbage_collected', 0), 1) CUR_LOGICAL_TIME = 5 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 1) CUR_LOGICAL_TIME = 6 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 1) CUR_LOGICAL_TIME = 7 a._age_out() self.assertEqual(a.statistics['aged_out'], 3) CUR_LOGICAL_TIME = 8 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 4) self.assertEqual(a.statistics.get('garbage_collected', 0), 4) self.assertEqual(a.length(), 0) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_rolling_feed_first(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): global CUR_LOGICAL_TIME chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = RollingFeedFirst(FTNAME, chassis) inputs = [] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 2) self.assertEqual(spawn_mock.call_count, 3) CUR_LOGICAL_TIME = 1 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) self.assertEqual(um_mock.call_count, 1) CUR_LOGICAL_TIME = 2 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 3) self.assertEqual(a.statistics.get('removed', 0), 0) CUR_LOGICAL_TIME = 3 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 4) self.assertEqual(a.statistics.get('removed', 0), 1) self.assertEqual(a.statistics.get('garbage_collected', 0), 1) self.assertEqual(a.statistics['aged_out'], 1) CUR_LOGICAL_TIME = 4 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 6) self.assertEqual(a.statistics.get('removed', 0), 3) self.assertEqual(a.statistics.get('garbage_collected', 0), 3) self.assertEqual(a.statistics['aged_out'], 4) CUR_LOGICAL_TIME = 5 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 7) self.assertEqual(a.statistics.get('removed', 0), 4) self.assertEqual(a.statistics.get('garbage_collected', 0), 4) self.assertEqual(a.statistics['aged_out'], 4) CUR_LOGICAL_TIME = 6 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 8) self.assertEqual(a.statistics.get('removed', 0), 5) self.assertEqual(a.statistics.get('garbage_collected', 0), 5) self.assertEqual(a.statistics['aged_out'], 6) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_rolling_feed_first2(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): global CUR_LOGICAL_TIME chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = RollingFeedFirst2(FTNAME, chassis) inputs = [] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 2) self.assertEqual(spawn_mock.call_count, 3) CUR_LOGICAL_TIME = 1 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) self.assertEqual(um_mock.call_count, 1) CUR_LOGICAL_TIME = 2 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 1) self.assertEqual(a.statistics.get('aged_out', 0), 0) self.assertEqual(a.statistics.get('garbage_collected', 0), 0) self.assertEqual(a.statistics.get('removed', 0), 0) CUR_LOGICAL_TIME = 3 a._age_out() self.assertEqual(a.statistics['added'], 1) self.assertEqual(a.statistics.get('removed', 0), 0) self.assertEqual(a.statistics.get('garbage_collected', 0), 0) self.assertEqual(a.statistics['aged_out'], 0) CUR_LOGICAL_TIME = 4 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 1) self.assertEqual(a.statistics.get('removed', 0), 0) self.assertEqual(a.statistics.get('garbage_collected', 0), 0) self.assertEqual(a.statistics['aged_out'], 1) CUR_LOGICAL_TIME = 5 a._age_out() self.assertEqual(a.statistics['added'], 1) self.assertEqual(a.statistics.get('removed', 0), 0) self.assertEqual(a.statistics.get('garbage_collected', 0), 0) self.assertEqual(a.statistics['aged_out'], 1) CUR_LOGICAL_TIME = 6 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 1) self.assertEqual(a.statistics.get('removed', 0), 0) self.assertEqual(a.statistics.get('garbage_collected', 0), 0) self.assertEqual(a.statistics['aged_out'], 1) CUR_LOGICAL_TIME = 7 a._age_out() self.assertEqual(a.statistics['added'], 1) self.assertEqual(a.statistics.get('removed', 0), 0) self.assertEqual(a.statistics.get('garbage_collected', 0), 0) self.assertEqual(a.statistics['aged_out'], 1) CUR_LOGICAL_TIME = 8 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 1) self.assertEqual(a.statistics['removed'], 1) self.assertEqual(a.statistics['garbage_collected'], 1) self.assertEqual(a.statistics['aged_out'], 1) CUR_LOGICAL_TIME = 9 a._age_out() self.assertEqual(a.statistics['added'], 1) self.assertEqual(a.statistics['removed'], 1) self.assertEqual(a.statistics['garbage_collected'], 1) self.assertEqual(a.statistics['aged_out'], 1) CUR_LOGICAL_TIME = 10 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 2) self.assertEqual(a.statistics['removed'], 1) self.assertEqual(a.statistics['garbage_collected'], 1) self.assertEqual(a.statistics['aged_out'], 1) CUR_LOGICAL_TIME = 11 a._age_out() self.assertEqual(a.statistics['added'], 2) self.assertEqual(a.statistics['removed'], 1) self.assertEqual(a.statistics['garbage_collected'], 1) self.assertEqual(a.statistics['aged_out'], 1) CUR_LOGICAL_TIME = 12 a._age_out() self.assertEqual(a.statistics['added'], 2) self.assertEqual(a.statistics['removed'], 1) self.assertEqual(a.statistics['garbage_collected'], 1) self.assertEqual(a.statistics['aged_out'], 2) self.assertEqual(a.statistics['withdraw.tx'], 2) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_permanent_feed(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): global CUR_LOGICAL_TIME chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = PermanentFeed(FTNAME, chassis) inputs = [] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 2) self.assertEqual(spawn_mock.call_count, 3) CUR_LOGICAL_TIME = 1 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) self.assertEqual(um_mock.call_count, 1) CUR_LOGICAL_TIME = 2 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 3) self.assertEqual(a.statistics.get('removed', 0), 0) CUR_LOGICAL_TIME = 3 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) CUR_LOGICAL_TIME = 4 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 4) self.assertEqual(a.statistics.get('removed', 0), 1) self.assertEqual(a.statistics.get('garbage_collected', 0), 1) CUR_LOGICAL_TIME = 5 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 1) CUR_LOGICAL_TIME = 6 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 1) CUR_LOGICAL_TIME = 7 a._age_out() self.assertEqual(a.statistics['aged_out'], 1) CUR_LOGICAL_TIME = 8 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 4) self.assertEqual(a.statistics.get('garbage_collected', 0), 4) self.assertEqual(a.length(), 0) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_superpermanent_feed(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): global CUR_LOGICAL_TIME chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = PermanentFeed(FTNAME, chassis) inputs = [] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 2) self.assertEqual(spawn_mock.call_count, 3) CUR_LOGICAL_TIME = 1 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) self.assertEqual(um_mock.call_count, 1) CUR_LOGICAL_TIME = 2 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 3) self.assertEqual(a.statistics.get('removed', 0), 0) CUR_LOGICAL_TIME = 4 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 4) self.assertEqual(a.statistics.get('removed', 0), 1) self.assertEqual(a.statistics.get('garbage_collected', 0), 1) self.assertEqual(a.statistics.get('aged_out', 0), 1) CUR_LOGICAL_TIME = 8 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 4) self.assertEqual(a.statistics.get('garbage_collected', 0), 4) self.assertEqual(a.length(), 0) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_drop_old_ops(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): global CUR_LOGICAL_TIME chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = DeltaFeed(FTNAME, chassis) inputs = [] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 2) self.assertEqual(spawn_mock.call_count, 3) a._actor_queue.put((0, 'age_out')) a._actor_queue.put((999, 'age_out')) CUR_LOGICAL_TIME = 1 try: a._actor_loop() except gevent.hub.LoopExit: pass self.assertEqual(a.last_ageout_run, 1000) self.assertEqual(um_mock.call_count, 2) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_permanentwithtype_feed(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): global CUR_LOGICAL_TIME chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = PermanentFeedWithType(FTNAME, chassis) inputs = [] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 2) self.assertEqual(spawn_mock.call_count, 3) CUR_LOGICAL_TIME = 1 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) self.assertEqual(um_mock.call_count, 1) CUR_LOGICAL_TIME = 2 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 3) self.assertEqual(a.statistics.get('removed', 0), 0) CUR_LOGICAL_TIME = 3 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) CUR_LOGICAL_TIME = 4 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 5) self.assertEqual(a.statistics.get('removed', 0), 2) self.assertEqual(a.statistics.get('garbage_collected', 0), 2) CUR_LOGICAL_TIME = 5 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 2) CUR_LOGICAL_TIME = 6 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 2) CUR_LOGICAL_TIME = 7 a._age_out() self.assertEqual(a.statistics['aged_out'], 2) CUR_LOGICAL_TIME = 8 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 5) self.assertEqual(a.statistics.get('garbage_collected', 0), 5) self.assertEqual(a.length(), 0) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_bptable_1(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): t = minemeld.ft.table.Table(FTNAME, truncate=True) bpt0 = minemeld.ft.basepoller._BPTable_v0(t) bpt0.put('A', {'v': 1}) A = bpt0.get('A') self.assertEqual(A['v'], 1) A, V = next(bpt0.query(include_value=True)) self.assertEqual(V['v'], 1) bpt0.delete('A') A = bpt0.get('A') self.assertEqual(A, None) bpt0.close() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_bptable_2(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): t = minemeld.ft.table.Table(FTNAME, truncate=True) bpt1 = minemeld.ft.basepoller._BPTable_v1(t, type_in_key=True) bpt1.put('A', {'type': 1}) A = next(bpt1.query(include_value=False)) self.assertEqual(A, 'A') bpt1.close() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_bptable_3(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): t = minemeld.ft.table.Table(FTNAME, truncate=True) bpt1 = minemeld.ft.basepoller._BPTable_v1(t, type_in_key=True) bpt1.close() with self.assertRaises(RuntimeError): t = minemeld.ft.table.Table(FTNAME, truncate=False) minemeld.ft.basepoller._BPTable_v1(t, type_in_key=False) @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_bptable_4(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): t = minemeld.ft.table.Table(FTNAME, truncate=True) bpt1 = minemeld.ft.basepoller._BPTable_v1(t, type_in_key=True) with self.assertRaises(RuntimeError): bpt1.put('A', {'a': 1}) bpt1.close() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_bptable_5(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): t = minemeld.ft.table.Table(FTNAME, truncate=True) bpt1 = minemeld.ft.basepoller._BPTable_v1(t, type_in_key=True) bpt1.close() bpt1 = minemeld.ft.basepoller._bptable_factory(FTNAME, truncate=False, type_in_key=True) bpt1.close() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_bptable_6(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): t = minemeld.ft.table.Table(FTNAME, truncate=True) bpt0 = minemeld.ft.basepoller._BPTable_v0(t) bpt0.put('A', {'v': 1}) bpt0.close() bpt1 = minemeld.ft.basepoller._bptable_factory(FTNAME, truncate=False, type_in_key=False) bpt1.close() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_bptable_7(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): t = minemeld.ft.table.Table(FTNAME, truncate=True) bpt0 = minemeld.ft.basepoller._BPTable_v0(t) bpt0.put('A', {'v': 1}) bpt0.delete('A') bpt0.close() bpt1 = minemeld.ft.basepoller._bptable_factory(FTNAME, truncate=False, type_in_key=True) bpt1.close() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_bptable_8(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): t = minemeld.ft.table.Table(FTNAME, truncate=True) bpt1 = minemeld.ft.basepoller._BPTable_v1(t, type_in_key=True) bpt1.put(indicator=u'☃.net/påth', value={u'☃.net/påth': 1, 'type': u'☃.net/påth'}) t = bpt1.get(u'☃.net/påth', itype=u'☃.net/påth') self.assertNotEqual(t, None) k, v = next(bpt1.query(include_value=True)) self.assertEqual(k, u'☃.net/påth') self.assertEqual(v, {u'☃.net/påth': 1, 'type': u'☃.net/påth'}) bpt1.close() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_permanentwithtype_feed_agg(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): global CUR_LOGICAL_TIME chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = PermanentFeedWithTypeAggregated(FTNAME, chassis) inputs = [] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 2) self.assertEqual(spawn_mock.call_count, 3) CUR_LOGICAL_TIME = 1 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) self.assertEqual(um_mock.call_count, 1) CUR_LOGICAL_TIME = 2 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 3) self.assertEqual(a.statistics.get('removed', 0), 0) CUR_LOGICAL_TIME = 3 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) CUR_LOGICAL_TIME = 4 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 5) self.assertEqual(a.statistics.get('removed', 0), 2) self.assertEqual(a.statistics.get('garbage_collected', 0), 2) CUR_LOGICAL_TIME = 5 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 2) CUR_LOGICAL_TIME = 6 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 2) CUR_LOGICAL_TIME = 7 a._age_out() self.assertEqual(a.statistics['aged_out'], 2) CUR_LOGICAL_TIME = 8 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 5) self.assertEqual(a.statistics.get('garbage_collected', 0), 5) self.assertEqual(a.length(), 0) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep') @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_permanentwithtype_feed_agg2(self, um_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): global CUR_LOGICAL_TIME chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = DeltaFeedWithTypeAggregatedFaulty(FTNAME, chassis) inputs = [] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 2) self.assertEqual(spawn_mock.call_count, 3) CUR_LOGICAL_TIME = 1 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) self.assertEqual(um_mock.call_count, 1) CUR_LOGICAL_TIME = 2 a._poll() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 3) self.assertEqual(a.statistics.get('removed', 0), 0) CUR_LOGICAL_TIME = 3 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) CUR_LOGICAL_TIME = 4 a._poll() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 5) self.assertEqual(a.statistics.get('removed', 0), 0) self.assertEqual(a.statistics.get('garbage_collected', 0), 0) CUR_LOGICAL_TIME = 5 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) CUR_LOGICAL_TIME = 6 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) CUR_LOGICAL_TIME = 7 a._age_out() self.assertEqual(a.statistics['aged_out'], 3) CUR_LOGICAL_TIME = 8 a._poll() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 5) self.assertEqual(a.statistics.get('garbage_collected', 0), 3) self.assertEqual(a.length(), 2) CUR_LOGICAL_TIME = 9 a._age_out() a._collect_garbage() self.assertEqual(a.statistics['aged_out'], 5) self.assertEqual(a.length(), 0) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() ================================================ FILE: tests/test_ft_boolexpr.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT Table tests Unit tests for minemeld.ft.boolexpr """ import unittest import jmespath import operator import logging import antlr4 import minemeld.ft.condition LOG = logging.getLogger(__name__) class ExprBuilder(minemeld.ft.condition.BoolExprListener): def __init__(self): self.expression = None self.comparator = None self.value = None def exitExpression(self, ctx): self.expression = ctx.getText() def exitComparator(self, ctx): comparator = ctx.getText() if comparator == '==': self.comparator = operator.eq elif comparator == '<': self.comparator = operator.lt elif comparator == '<=': self.comparator = operator.le elif comparator == '>': self.comparator = operator.gt elif comparator == '>=': self.comparator = operator.ge elif comparator == '!=': self.comparator = operator.ne def exitValue(self, ctx): if ctx.STRING() is not None: self.value = ctx.STRING().getText()[1:-1] elif ctx.NUMBER() is not None: self.value = int(ctx.NUMBER().getText()) elif ctx.getText() == 'null': self.value = None elif ctx.getText() == 'false': self.value = False elif ctx.getText() == 'true': self.value = True def _parse_string(s): lexer = minemeld.ft.condition.BoolExprLexer( antlr4.InputStream(s) ) stream = antlr4.CommonTokenStream(lexer) parser = minemeld.ft.condition.BoolExprParser(stream) tree = parser.booleanExpression() eb = ExprBuilder() walker = antlr4.ParseTreeWalker() walker.walk(eb, tree) return eb def _eval_expression(eb, i): ce = jmespath.compile(eb.expression) try: r = ce.search(i) except jmespath.exceptions.JMESPathError: LOG.exception("Exception in searching") r = None LOG.info("r: %s value: %s", r, eb.value) return eb.comparator(r, eb.value) class MineMeldFTBaseTests(unittest.TestCase): def test_simple(self): eb = _parse_string('sources == "http://dshield.org/blocklist"') self.assertEqual(eb.expression, u'sources') self.assertEqual(eb.comparator, operator.eq) self.assertEqual(eb.value, 'http://dshield.org/blocklist') def test_func(self): eb = _parse_string('length(max(dshield_nattacks)) == ' '"http://dshield.org/blocklist"') self.assertEqual(eb.expression, u'length(max(dshield_nattacks))') self.assertEqual(eb.comparator, operator.eq) self.assertEqual(eb.value, 'http://dshield.org/blocklist') def test_eval(self): i = { 'sources': [1, 2], 'type': 'IPv4' } c = minemeld.ft.condition.Condition('length(sources) > 1') self.assertTrue(c.eval(i)) c = minemeld.ft.condition.Condition('length(b) > 1') self.assertFalse(c.eval(i)) c = minemeld.ft.condition.Condition('type(b) == null') self.assertTrue(c.eval(i)) c = minemeld.ft.condition.Condition('length(b) == null') self.assertTrue(c.eval(i)) c = minemeld.ft.condition.Condition("starts_with(type, 'IP') " "== true") self.assertTrue(c.eval(i)) c = minemeld.ft.condition.Condition("type == 'IPv4'") self.assertTrue(c.eval(i)) ================================================ FILE: tests/test_ft_dag.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT dag tests Unit tests for minemeld.ft.dag """ import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import unittest import mock import time import shutil import logging import gc import calendar import os import yaml import functools import pan.xapi import panos_mock import minemeld.ft.dag FTNAME = 'testft-%d' % int(time.time()) DLIST_NAME = 'dag-dlist-%d.yml' % int(time.time()) LOG = logging.getLogger(__name__) CUR_LOGICAL_TIME = 0 MYDIR = os.path.dirname(__file__) GEVENT_SLEEP = gevent.sleep def logical_millisec(*args): return CUR_LOGICAL_TIME def gevent_event_mock_factory(): result = mock.Mock() result.wait.side_effect = gevent.GreenletExit() return result def device_pusher_mock_factory(device, prefix, watermark, attributes, persistence): def _start_se(x): x.started = True result = mock.MagicMock(started=False, device=device, value=None) result.start = mock.Mock(side_effect=functools.partial(_start_se, result)) return result class MineMeldFTDagPusherTests(unittest.TestCase): def setUp(self): try: shutil.rmtree(FTNAME) except: pass try: os.remove(DLIST_NAME) except: pass def tearDown(self): try: shutil.rmtree(FTNAME) except: pass try: os.remove(DLIST_NAME) except: pass @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch.object(minemeld.ft.dag.DagPusher, '_huppable_wait', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch.object(calendar, 'timegm', side_effect=logical_millisec) @mock.patch('minemeld.ft.dag.DevicePusher', side_effect=device_pusher_mock_factory) def test_device_list_load(self, dp_mock, timegm_mock, event_mock, hw_mock, sleep_mock, spawnl_mock, spawn_mock): device_list_path = os.path.join(MYDIR, 'test_device_list.yml') device_list_path2 = os.path.join(MYDIR, 'test_device_list2.yml') with open(device_list_path, 'r') as f: dlist = yaml.safe_load(f) with open(device_list_path2, 'r') as f: dlist2 = yaml.safe_load(f) shutil.copyfile(device_list_path, DLIST_NAME) config = { 'device_list': DLIST_NAME } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.dag.DagPusher(FTNAME, chassis, config) inputs = [] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 1) self.assertEqual(spawn_mock.call_count, 1) # 1st round try: a._device_list_monitor() except gevent.GreenletExit: pass hw_mock.assert_called_with(5) self.assertEqual(len(a.devices), len(dlist)) self.assertEqual(len(a.device_pushers), len(dlist)) self.assertEqual(dp_mock.call_count, len(dlist)) for i, d in enumerate(dlist): self.assertEqual(a.devices[i], d) self.assertEqual(a.device_pushers[i].start.call_count, 1) self.assertEqual(a.device_pushers[i].device, d) # 2nd round GEVENT_SLEEP(1) shutil.copyfile(device_list_path, DLIST_NAME) hw_mock.reset_mock() dp_mock.reset_mock() try: a._device_list_monitor() except gevent.GreenletExit: pass hw_mock.assert_called_with(5) self.assertEqual(len(a.devices), len(dlist)) self.assertEqual(len(a.device_pushers), len(dlist)) self.assertEqual(dp_mock.call_count, 0) for i, d in enumerate(dlist): self.assertEqual(a.devices[i], d) self.assertEqual(a.device_pushers[i].start.call_count, 1) self.assertEqual(a.device_pushers[i].device, d) # 3rd round GEVENT_SLEEP(1) shutil.copyfile(device_list_path2, DLIST_NAME) hw_mock.reset_mock() dp_mock.reset_mock() try: a._device_list_monitor() except gevent.GreenletExit: pass hw_mock.assert_called_with(5) self.assertEqual(len(a.devices), len(dlist2)) self.assertEqual(len(a.device_pushers), len(dlist2)) self.assertEqual(dp_mock.call_count, 1) for i, d in enumerate(dlist2): self.assertEqual(a.devices[i], d) self.assertEqual(a.device_pushers[i].start.call_count, 1) self.assertEqual(a.device_pushers[i].device, d) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch.object(calendar, 'timegm', side_effect=logical_millisec) @mock.patch('minemeld.ft.dag.DevicePusher', side_effect=device_pusher_mock_factory) def test_uw(self, dp_mock, timegm_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): device_list_path = os.path.join(MYDIR, 'test_device_list.yml') shutil.copyfile(device_list_path, DLIST_NAME) config = { 'device_list': DLIST_NAME } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.dag.DagPusher(FTNAME, chassis, config) inputs = ['a'] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 1) self.assertEqual(spawn_mock.call_count, 1) try: a._device_list_monitor() except gevent.GreenletExit: pass a.update('a', indicator='127.0.0.1', value={ 'type': 'IPv4', 'confidence': 100 }) for d in a.device_pushers: d.put.assert_called_with( 'register', '127.0.0.1', { 'type': 'IPv4', 'confidence': 100 } ) for d in a.device_pushers: d.put.reset_mock() a.withdraw('a', indicator='127.0.0.1') for d in a.device_pushers: d.put.assert_called_with( 'unregister', '127.0.0.1', { 'type': 'IPv4', 'confidence': 100 } ) for d in a.device_pushers: d.put.reset_mock() a.withdraw('a', indicator='127.0.0.1') for d in a.device_pushers: self.assertEqual(d.put.call_count, 0) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch.object(calendar, 'timegm', side_effect=logical_millisec) @mock.patch('minemeld.ft.dag.DevicePusher', side_effect=device_pusher_mock_factory) def test_uinvalid(self, dp_mock, timegm_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): device_list_path = os.path.join(MYDIR, 'test_device_list.yml') shutil.copyfile(device_list_path, DLIST_NAME) config = { 'device_list': DLIST_NAME } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.dag.DagPusher(FTNAME, chassis, config) inputs = ['a'] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 1) self.assertEqual(spawn_mock.call_count, 1) try: a._device_list_monitor() except gevent.GreenletExit: pass a.update('a', indicator='1.1.1.1-1.1.1.3', value={ 'type': 'IPv4', 'confidence': 100 }) self.assertEqual(a.length(), 0) a.update('a', indicator='1.1.1.0/24', value={ 'type': 'IPv4', 'confidence': 100 }) self.assertEqual(a.length(), 0) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch.object(calendar, 'timegm', side_effect=logical_millisec) @mock.patch('minemeld.ft.dag.DevicePusher', side_effect=device_pusher_mock_factory) def test_unicast1(self, dp_mock, timegm_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): device_list_path = os.path.join(MYDIR, 'test_device_list.yml') shutil.copyfile(device_list_path, DLIST_NAME) config = { 'device_list': DLIST_NAME } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.dag.DagPusher(FTNAME, chassis, config) inputs = ['a'] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 1) self.assertEqual(spawn_mock.call_count, 1) try: a._device_list_monitor() except gevent.GreenletExit: pass a.update('a', indicator='1.1.1.1-1.1.1.1', value={ 'type': 'IPv4', 'confidence': 100 }) for d in a.device_pushers: d.put.assert_called_with( 'register', '1.1.1.1', { 'type': 'IPv4', 'confidence': 100 } ) for d in a.device_pushers: d.put.reset_mock() a.withdraw('a', indicator='1.1.1.1-1.1.1.1') for d in a.device_pushers: d.put.assert_called_with( 'unregister', '1.1.1.1', { 'type': 'IPv4', 'confidence': 100 } ) for d in a.device_pushers: d.put.reset_mock() a.withdraw('a', indicator='1.1.1.1-1.1.1.1') for d in a.device_pushers: self.assertEqual(d.put.call_count, 0) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch.object(calendar, 'timegm', side_effect=logical_millisec) @mock.patch('minemeld.ft.dag.DevicePusher', side_effect=device_pusher_mock_factory) def test_unicast2(self, dp_mock, timegm_mock, event_mock, sleep_mock, spawnl_mock, spawn_mock): device_list_path = os.path.join(MYDIR, 'test_device_list.yml') shutil.copyfile(device_list_path, DLIST_NAME) config = { 'device_list': DLIST_NAME } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.dag.DagPusher(FTNAME, chassis, config) inputs = ['a'] output = False a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 1) self.assertEqual(spawn_mock.call_count, 1) try: a._device_list_monitor() except gevent.GreenletExit: pass a.update('a', indicator='1.1.1.1/32', value={ 'type': 'IPv4', 'confidence': 100 }) for d in a.device_pushers: d.put.assert_called_with( 'register', '1.1.1.1', { 'type': 'IPv4', 'confidence': 100 } ) for d in a.device_pushers: d.put.reset_mock() a.withdraw('a', indicator='1.1.1.1/32') for d in a.device_pushers: d.put.assert_called_with( 'unregister', '1.1.1.1', { 'type': 'IPv4', 'confidence': 100 } ) for d in a.device_pushers: d.put.reset_mock() a.withdraw('a', indicator='1.1.1.1/32') for d in a.device_pushers: self.assertEqual(d.put.call_count, 0) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(pan.xapi, 'PanXapi', side_effect=panos_mock.factory) def test_devicepusher_dag_message(self, panxapi_mock): RESULT_REG = '1.0updateab' RESULT_UNREG = '1.0updateab' dp = minemeld.ft.dag.DevicePusher( {'tag': 'test'}, 'mmeld_', 'test', [], False ) reg = dp._dag_message('register', {'192.168.1.1': ['a', 'b']}) self.assertEqual(reg, RESULT_REG) unreg = dp._dag_message('unregister', {'192.168.1.1': ['a', 'b']}) self.assertEqual(unreg, RESULT_UNREG) @mock.patch.object(pan.xapi, 'PanXapi', side_effect=panos_mock.factory) def test_devicepusher_tags_from_value(self, panxapi_mock): dp = minemeld.ft.dag.DevicePusher( {'tag': 'test'}, 'mmeld_', 'test', ['confidence', 'direction'], False ) tags = dp._tags_from_value({'confidence': 49, 'direction': 'inbound'}) self.assertEqual(tags, set(['mmeld_confidence_low', 'mmeld_direction_inbound'])) tags = dp._tags_from_value({'confidence': 50}) self.assertEqual(tags, set(['mmeld_confidence_medium', 'mmeld_direction_unknown'])) tags = dp._tags_from_value({'confidence': 75, 'direction': 'outbound'}) self.assertEqual(tags, set(['mmeld_confidence_high', 'mmeld_direction_outbound'])) @mock.patch.object(pan.xapi, 'PanXapi', side_effect=panos_mock.factory) def test_devicepusher_get_all_registered_ips(self, panxapi_mock): dp = minemeld.ft.dag.DevicePusher( {'hostname': 'test_ft_dag_devicepusher'}, 'mmeld_', 'test', ['confidence', 'direction'], False ) result = dp._get_all_registered_ips() self.assertEqual(next(result), ('192.168.1.1', ['mmeld_test', 'mmeld_confidence_100', 'mmeld_pushed'])) self.assertEqual(next(result), ('192.168.1.2', ['mmeld_test', 'mmeld_confidence_100'])) @mock.patch.object(pan.xapi, 'PanXapi', side_effect=panos_mock.factory) def test_devicepusher_push(self, panxapi_mock): dp = minemeld.ft.dag.DevicePusher( {'hostname': 'test_ft_dag_devicepusher'}, 'mmeld_', 'test', ['confidence', 'direction'], False ) dp._push('register', '192.168.1.10', {'confidence': 40, 'direction': 'inbound'}) self.assertEqual( dp.xapi.user_id_calls[0], '1.0updatemmeld_confidence_lowmmeld_direction_inboundmmeld_test' ) @mock.patch.object(pan.xapi, 'PanXapi', side_effect=panos_mock.factory) def test_devicepusher_init_resync(self, panxapi_mock): dp = minemeld.ft.dag.DevicePusher( {'hostname': 'test_ft_dag_devicepusher'}, 'mmeld_', 'test', ['confidence', 'direction'], False ) dp.put('init', '192.168.1.1', {'confidence': 75, 'direction': 'inbound'}) dp.put('init', '192.168.1.10', {'confidence': 80}) dp.put('EOI', None, None) dp._init_resync() self.assertEqual( dp.xapi.user_id_calls[0], '1.0updatemmeld_confidence_highmmeld_direction_inboundmmeld_confidence_highmmeld_direction_unknownmmeld_test' ) self.assertEqual( dp.xapi.user_id_calls[1], '1.0updatemmeld_confidence_100mmeld_pushedmmeld_confidence_100mmeld_test' ) ================================================ FILE: tests/test_ft_dag_devicepusher_op__show__object__registered_ip__ip_192_168_1_1__ip___registered_ip___object___show__0.xml ================================================ mmeld_test mmeld_confidence_100 mmeld_pushed 1 ================================================ FILE: tests/test_ft_dag_devicepusher_op__show__object__registered_ip__ip_192_168_1_2__ip___registered_ip___object___show__0.xml ================================================ mmeld_test mmeld_confidence_100 1 ================================================ FILE: tests/test_ft_dag_devicepusher_op__show__object__registered_ip__tag__entry_name__mmeld_test_____tag___registered_ip___object___show__0.xml ================================================ mmeld_test mmeld_test 2 ================================================ FILE: tests/test_ft_ipop.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT ipop tests Unit tests for minemeld.ft.ipop """ import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import unittest import mock import time import shutil import logging import netaddr import random import guppy # noqa import pdb # noqa import gc # noqa from nose.plugins.attrib import attr import minemeld.ft.ipop LOG = logging.getLogger(__name__) FTNAME = 'testft-%d' % int(time.time()) def check_for_rpc(call_args_list, check_list, all_here=False): LOG.debug("call_args_list: %s", call_args_list) found = [] for chk in check_list: LOG.debug("checking: %s", chk) for j in xrange(len(call_args_list)): if j in found: continue args = call_args_list[j][0] if args[0] != chk['method']: continue if args[1]['indicator'] != chk['indicator']: continue chkvalue = chk.get('value', None) if chkvalue is None: found.append(j) LOG.debug("found @%d", j) break argsvalue = args[1].get('value', None) if chkvalue is not None and argsvalue is None: continue failed = False for k in chkvalue.keys(): if k not in argsvalue: failed = True break if chkvalue[k] != argsvalue[k]: failed = True break if failed: continue found.append(j) LOG.debug("found @%d", j) break c1 = len(found) == len(check_list) if not all_here: return c1 c2 = len(found) == len(call_args_list) return c1+c2 == 2 class MineMeldFTIPOpTests(unittest.TestCase): def setUp(self): try: shutil.rmtree(FTNAME) except: pass try: shutil.rmtree(FTNAME+"_st") except: pass def tearDown(self): try: shutil.rmtree(FTNAME) except: pass try: shutil.rmtree(FTNAME+"_st") except: pass def test_calc_ipranges(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='192.168.0.1', value={ 'type': 'IPv4', 's1$a': 1, 'sources': ['s1s'] }) self.assertEqual(ochannel.publish.call_count, 1) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'update') self.assertEqual(pargs[1]['indicator'], '192.168.0.1-192.168.0.1') ochannel.publish.reset_mock() a.filtered_update('s1', indicator='192.168.0.1-192.168.0.3', value={ 'type': 'IPv4', 's1$b': 1, 'sources': ['s1s'] }) self.assertTrue(check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.1-192.168.0.1', 'value': { 's1$a': 1, 's1$b': 1, 'sources': ['s1s'] } }, { 'method': 'update', 'indicator': '192.168.0.2-192.168.0.3', 'value': { 's1$b': 1, 'sources': ['s1s'] } } ], all_here=True )) ochannel.publish.reset_mock() a.filtered_update('s1', indicator='192.168.0.2-192.168.0.2', value={ 'type': 'IPv4', 's1$c': 1, 'sources': ['s1s'] }) self.assertTrue(check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.2-192.168.0.2', 'value': { 's1$b': 1, 's1$c': 1, 'sources': ['s1s'] } }, { 'method': 'withdraw', 'indicator': '192.168.0.2-192.168.0.3', 'value': { 's1$b': 1, 'sources': ['s1s'] } }, { 'method': 'update', 'indicator': '192.168.0.3-192.168.0.3', 'value': { 's1$b': 1, 'sources': ['s1s'] } } ], all_here=True )) ochannel.publish.reset_mock() a.filtered_update('s1', indicator='255.255.255.255', value={ 'type': 'IPv4', 's1$e': 1, 'sources': ['s1s'] }) self.assertTrue(check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '255.255.255.255-255.255.255.255', 'value': { 'sources': ['s1s'] } } ], all_here=True )) ochannel.publish.reset_mock() a.filtered_update('s1', indicator='0.0.0.0', value={ 'type': 'IPv4', 's1$e': 1, 'sources': ['s1s'] }) self.assertTrue(check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '0.0.0.0-0.0.0.0', 'value': { 'sources': ['s1s'] } } ], all_here=True )) a.stop() a.st.db.close() a = None def test_uwl(self): config = { 'whitelist_prefixes': ['s2'] } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='192.168.0.0/16', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.0-192.168.255.255', 'value': { 's1$a': 1 } } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='192.168.0.0/24', value={ 'type': 'IPv4', 'sources': ['s2s'], 's1$b': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.1.0-192.168.255.255', 'value': { 's1$a': 1 } }, { 'method': 'withdraw', 'indicator': '192.168.0.0-192.168.255.255', 'value': { 's1$a': 1 } } ], all_here=True ) ) a.stop() a.st.db.close() a = None def test_uwl2(self): config = { 'whitelist_prefixes': ['s2'] } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='192.168.0.0/16', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.0-192.168.255.255', 'value': { 's1$a': 1 } } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='192.168.0.1', value={ 'type': 'IPv4', 'sources': ['s2s'], 's1$b': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.0-192.168.0.0', 'value': { 's1$a': 1 } }, { 'method': 'update', 'indicator': '192.168.0.2-192.168.255.255', 'value': { 's1$a': 1 } }, { 'method': 'withdraw', 'indicator': '192.168.0.0-192.168.255.255', 'value': { 's1$a': 1 } } ], all_here=True ) ) a.stop() a.st.db.close() a = None def test_uwl3(self): config = { 'whitelist_prefixes': ['s2'] } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='192.168.0.0/16', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.0-192.168.255.255', 'value': { 's1$a': 1 } } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='192.168.0.1', value={ 'type': 'IPv4', 'sources': ['s2s'], 's1$b': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.0-192.168.0.0', 'value': { 's1$a': 1 } }, { 'method': 'update', 'indicator': '192.168.0.2-192.168.255.255', 'value': { 's1$a': 1 } }, { 'method': 'withdraw', 'indicator': '192.168.0.0-192.168.255.255', 'value': { 's1$a': 1 } } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='192.168.0.2', value={ 'type': 'IPv4', 'sources': ['s2s'], 's1$b': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.3-192.168.255.255', 'value': { 's1$a': 1 } }, { 'method': 'withdraw', 'indicator': '192.168.0.2-192.168.255.255', 'value': { 's1$a': 1 } } ], all_here=True ) ) a.stop() a.st.db.close() a = None def test_overlap_by_one(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='192.168.0.1-192.168.0.3', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.1-192.168.0.3' } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='192.168.0.3-192.168.0.4', value={ 'type': 'IPv4', 'sources': ['s2s'], 's1$b': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.1-192.168.0.2' }, { 'method': 'update', 'indicator': '192.168.0.3-192.168.0.3' }, { 'method': 'update', 'indicator': '192.168.0.4-192.168.0.4' }, { 'method': 'withdraw', 'indicator': '192.168.0.1-192.168.0.3' } ], all_here=True ) ) a.stop() a.st.db.close() a = None def test_overlap_by_lastrange(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='8.8.0.0/16', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '8.8.0.0-8.8.255.255' } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='8.8.255.0/24', value={ 'type': 'IPv4', 'sources': ['s2s'], 's1$b': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '8.8.0.0-8.8.254.255' }, { 'method': 'update', 'indicator': '8.8.255.0-8.8.255.255' }, { 'method': 'withdraw', 'indicator': '8.8.0.0-8.8.255.255' } ], all_here=True ) ) a.stop() a.st.db.close() a = None def test_3overlaps(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='10.1.0.0/16', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '10.1.0.0-10.1.255.255' } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='10.1.1.0/24', value={ 'type': 'IPv4', 'sources': ['s2s'], 's1$b': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '10.1.0.0-10.1.0.255' }, { 'method': 'update', 'indicator': '10.1.1.0-10.1.1.255' }, { 'method': 'update', 'indicator': '10.1.2.0-10.1.255.255' }, { 'method': 'withdraw', 'indicator': '10.1.0.0-10.1.255.255' } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='10.1.1.128/25', value={ 'type': 'IPv4', 'sources': ['s2s'], 's1$c': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '10.1.1.128-10.1.1.255', 's1$a': 1, 's1$b': 1, 's1$c': 1 }, { 'method': 'update', 'indicator': '10.1.1.0-10.1.1.127' }, { 'method': 'withdraw', 'indicator': '10.1.1.0-10.1.1.255' } ], all_here=True ) ) a.stop() a.st.db.close() a = None def test_2overlaps(self): config = { 'whitelist_prefixes': ['s2'] } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='10.1.0.0/24', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '10.1.0.0-10.1.0.255' } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='10.1.0.10', value={ 'type': 'IPv4', 'sources': ['s2s'] }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '10.1.0.11-10.1.0.255' }, { 'method': 'update', 'indicator': '10.1.0.0-10.1.0.9' }, { 'method': 'withdraw', 'indicator': '10.1.0.0-10.1.0.255' } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='10.1.0.25', value={ 'type': 'IPv4', 'sources': ['s2s'] }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '10.1.0.11-10.1.0.24', 's1$a': 1 }, { 'method': 'update', 'indicator': '10.1.0.26-10.1.0.255' }, { 'method': 'withdraw', 'indicator': '10.1.0.11-10.1.0.255' } ], all_here=True ) ) a.stop() a.st.db.close() a = None def test_attr_override(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1', 's2', 's3'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='10.1.0.0/16', value={ 'type': 'IPv4', 'sources': ['s1s'], 'direction': 'inbound', 'first_seen': 10, 'last_seen': 25, 'confidence': 20 }) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='10.1.0.0/16', value={ 'type': 'IPv4', 'sources': ['s2s'], 'direction': 'inbound', 'first_seen': 5, 'last_seen': 20, 'confidence': 30 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '10.1.0.0-10.1.255.255', 'value': { 'direction': 'inbound', 'first_seen': 5, 'last_seen': 25, 'confidence': 30 } } ], all_here=True ) ) a.stop() a.st.db.close() a = None def test_uw(self): config = { 'whitelist_prefixes': ['s2'] } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='192.168.0.0/16', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.0-192.168.255.255', 'value': { 's1$a': 1 } } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_withdraw('s1', indicator='192.168.0.0/16') self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'withdraw', 'indicator': '192.168.0.0-192.168.255.255', 'value': { 's1$a': 1 } } ], all_here=True ) ) a.stop() a.st.db.close() a = None def test_2uw(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='192.168.0.0', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 }) a.filtered_update('s1', indicator='192.168.1.0', value={ 'type': 'IPv4', 'sources': ['s2s'], 's1$a': 1 }) ochannel.publish.reset_mock() a.filtered_withdraw('s1', indicator='192.168.0.0') self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'withdraw', 'indicator': '192.168.0.0-192.168.0.0', 'value': { 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 } } ], all_here=True ) ) a.filtered_update('s1', indicator='192.168.0.0', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 }) ochannel.publish.reset_mock() a.filtered_withdraw('s1', indicator='192.168.1.0') self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'withdraw', 'indicator': '192.168.1.0-192.168.1.0' } ], all_here=True ) ) a.stop() a.st.db.close() a = None def test_uw_wrongtype(self): config = { 'whitelist_prefixes': ['s2'] } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='192.168.0.0/16', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.0-192.168.255.255', 'value': { 's1$a': 1 } } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_withdraw('s1', indicator='192.168.0.0/16', value={'type': 'domain'}) self.assertEqual(ochannel.publish.call_count, 0) ochannel.publish.reset_mock() a.filtered_withdraw('s1', indicator='192.168.0.0/16', value={'type': 'IPv4'}) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'withdraw', 'indicator': '192.168.0.0-192.168.255.255' } ], all_here=True ) ) a.stop() a.st.db.close() a = None def test_updated_indicator(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() ochannel.publish.reset_mock() a.filtered_update('s1', indicator='192.168.0.0', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 1 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.0-192.168.0.0', 'value': { 's1$a': 1 } } ], all_here=True ) ) ochannel.publish.reset_mock() a.filtered_update('s1', indicator='192.168.0.0', value={ 'type': 'IPv4', 'sources': ['s1s'], 's1$a': 2 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '192.168.0.0-192.168.0.0', 'value': { 's1$a': 2 } } ], all_here=True ) ) a.stop() a.st.db.close() a = None @attr('slow') def test_stress_1(self): num_intervals = 100000 config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() t1 = time.time() for j in xrange(num_intervals): start = random.randint(0, 0xFFFFFFFF) if random.randint(0, 4) == 0: start = start & 0xFFFFFF00 end = start + 255 else: end = start end = netaddr.IPAddress(end) start = netaddr.IPAddress(start) ochannel.publish.reset_mock() t2 = time.time() dt = t2-t1 t1 = time.time() for j in xrange(num_intervals): start = random.randint(0, 0xFFFFFFFF) if random.randint(0, 4) == 0: start = start & 0xFFFFFF00 end = start + 255 else: end = start end = netaddr.IPAddress(end) start = netaddr.IPAddress(start) ochannel.publish.reset_mock() a.filtered_update('s1', indicator='%s-%s' % (start, end), value={ 'type': 'IPv4', 'sources': ['s1s'] }) t2 = time.time() print "TIME: Inserted %d intervals in %d" % (num_intervals, (t2-t1-dt)) t1 = time.time() for j in xrange(num_intervals): ochannel.publish.reset_mock() a.filtered_update('s1', indicator='%s' % (start), value={ 'type': 'IPv4', 'sources': ['s1s'], 'count': j }) t2 = time.time() print "TIME: Updated %d intervals in %d" % (num_intervals, (t2-t1-dt)) a.stop() a.st.db.close() a = None @attr('slow') def test_stress_2(self): num_intervals = 200 config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.ipop.AggregateIPv4FT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() t1 = time.time() for _ in xrange(num_intervals): end = random.randint(0, 0xFFFFFFFF) start = random.randint(0, end) end = netaddr.IPAddress(end) start = netaddr.IPAddress(start) ochannel.publish.reset_mock() t2 = time.time() dt = t2-t1 t1 = time.time() for j in xrange(num_intervals): end = random.randint(0, 0xFFFFFFFF) start = random.randint(0, end) end = netaddr.IPAddress(end) start = netaddr.IPAddress(start) ochannel.publish.reset_mock() a.filtered_update('s1', indicator='%s-%s' % (start, end), value={ 'type': 'IPv4', 'sources': ['s1s'] }) t2 = time.time() print "TIME: Inserted %d intervals in %d" % (num_intervals, (t2-t1-dt)) a.stop() a.st.db.close() a = None ================================================ FILE: tests/test_ft_local.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT local tests Unit tests for minemeld.ft.local """ import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import unittest import mock import time import shutil import logging import gc import calendar import os.path import yaml import minemeld.ft.local FTNAME = 'testft-%d' % int(time.time()) LOCALDB_NAME = 'local-%d.yml' % int(time.time()) LOG = logging.getLogger(__name__) CUR_LOGICAL_TIME = 0 MYDIR = os.path.dirname(__file__) def logical_millisec(*args): return CUR_LOGICAL_TIME*1000 def gevent_event_mock_factory(): result = mock.Mock() result.wait.side_effect = gevent.GreenletExit() return result class MineMeldYamlFTTests(unittest.TestCase): def setUp(self): try: shutil.rmtree(FTNAME) except: pass try: os.remove(LOCALDB_NAME) except: pass def tearDown(self): try: shutil.rmtree(FTNAME) except: pass try: os.remove(LOCALDB_NAME) except: pass @mock.patch.object(gevent, 'spawn') @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(gevent, 'sleep', side_effect=gevent.GreenletExit()) @mock.patch('gevent.event.Event', side_effect=gevent_event_mock_factory) @mock.patch('minemeld.ft.basepoller.utc_millisec', side_effect=logical_millisec) def test_yaml(self, um_mock, sleep_mock, event_mock, spawnl_mock, spawn_mock): global CUR_LOGICAL_TIME localdb_path = os.path.join(MYDIR, 'test_localdb.yml') localdb_path2 = os.path.join(MYDIR, 'test_localdb2.yml') with open(localdb_path, 'r') as f: localdb = yaml.safe_load(f) localdb = [k['indicator'] for k in localdb] with open(localdb_path2, 'r') as f: localdb2 = yaml.safe_load(f) localdb2 = [k['indicator'] for k in localdb2] shutil.copyfile(localdb_path, LOCALDB_NAME) chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock config = { 'path': LOCALDB_NAME, 'age_out': { 'default': None, 'sudden_death': True } } a = minemeld.ft.local.YamlFT(FTNAME, chassis, config) inputs = [] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 1) self.assertEqual(spawn_mock.call_count, 3) CUR_LOGICAL_TIME = 1 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 0) self.assertEqual(um_mock.call_count, 1) CUR_LOGICAL_TIME = 2 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], len(localdb)) self.assertEqual(a.statistics.get('removed', 0), 0) lsp = a.last_successful_run CUR_LOGICAL_TIME = 3 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics.get('aged_out', 0), 0) self.assertEqual(a.statistics.get('removed', 0), 0) self.assertEqual(a.statistics.get('garbage_collected', 0), 0) self.assertEqual(a.last_successful_run, lsp) LOG.debug('%d', a.statistics['added']) shutil.copyfile(localdb_path2, LOCALDB_NAME) CUR_LOGICAL_TIME = 4 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual( a.statistics['added'], len(set(localdb) | set(localdb2)) ) self.assertEqual( a.statistics.get('removed', 0), len(set(localdb2)-set(localdb)) ) self.assertEqual(a.statistics.get('garbage_collected', 0), 1) CUR_LOGICAL_TIME = 5 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 1) CUR_LOGICAL_TIME = 6 a._age_out() self.assertEqual(a.statistics.get('aged_out', 0), 1) CUR_LOGICAL_TIME = 7 a._age_out() self.assertEqual(a.statistics['aged_out'], 1) CUR_LOGICAL_TIME = 8 a._poll() a._sudden_death() a._age_out() a._collect_garbage() self.assertEqual(a.statistics['added'], 3) self.assertEqual(a.statistics.get('garbage_collected', 0), 1) self.assertEqual(a.length(), 2) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() ================================================ FILE: tests/test_ft_logstash.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT Logstash tests Unit tests for minemeld.ft.logstash """ import unittest import mock import time import minemeld.ft.logstash FTNAME = 'testft-%d' % int(time.time()) class MineMeldFTLogstashOutputTests(unittest.TestCase): def test_uw(self): config = { 'logstash_host': '127.0.0.1', 'logstash_port': 5514 } chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock b = minemeld.ft.logstash.LogstashOutput(FTNAME, chassis, config) inputs = ['a', 'b', 'c'] output = False b.connect(inputs, output) b.mgmtbus_initialize() b.start() time.sleep(1) b.update('a', indicator='testi', value={'test': 'v'}) self.assertEqual(b.statistics['message.sent'], 1) b.withdraw('a', indicator='testi') self.assertEqual(b.statistics['message.sent'], 2) b.stop() ================================================ FILE: tests/test_ft_op.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT op tests Unit tests for minemeld.ft.op """ import gevent import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import unittest import mock import time import shutil import logging import guppy # noqa import pdb # noqa import gc # noqa import minemeld.ft.op FTNAME = 'testft-%d' % int(time.time()) LOG = logging.getLogger(__name__) def check_for_rpc(call_args_list, check_list, all_here=False, offset=0): LOG.debug("call_args_list: %s", call_args_list) found = [] for chk in check_list: LOG.debug("checking: %s", chk) for j in xrange(len(call_args_list)): if j in found: continue args = call_args_list[j][0] LOG.debug("args: %s", args[offset+0]) if args[offset+0] != chk['method']: continue if args[offset+1]['indicator'] != chk['indicator']: continue chkvalue = chk.get('value', None) if chkvalue is None: found.append(j) LOG.debug("found @%d", j) break argsvalue = args[offset+1].get('value', None) if chkvalue is not None and argsvalue is None: continue failed = False for k in chkvalue.keys(): if k not in argsvalue: failed = True break if chkvalue[k] != argsvalue[k]: failed = True break if failed: continue found.append(j) LOG.debug("found @%d", j) break c1 = len(found) == len(check_list) if not all_here: return c1 c2 = len(found) == len(call_args_list) return c1+c2 == 2 class MineMeldFTOpTests(unittest.TestCase): def setUp(self): try: shutil.rmtree(FTNAME) except: pass def tearDown(self): try: shutil.rmtree(FTNAME) except: pass def test_aggregate_2u(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']}) self.assertEqual(ochannel.publish.call_count, 1) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'update') self.assertEqual(pargs[1]['indicator'], 'i') self.assertListEqual(pargs[1]['value']['sources'], ['s1s']) self.assertEqual(pargs[1]['value']['s1$a'], 1) a.filtered_update('s2', indicator='i', value={'s2$a': 1, 'sources': ['s2s']}) self.assertEqual(ochannel.publish.call_count, 2) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'update') self.assertEqual(pargs[1]['indicator'], 'i') self.assertListEqual(pargs[1]['value']['sources'], ['s1s', 's2s']) self.assertEqual(pargs[1]['value']['s2$a'], 1) self.assertEqual(pargs[1]['value']['s1$a'], 1) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() def test_aggregate_uwl(self): config = { 'whitelist_prefixes': ['s2'] } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']}) self.assertEqual(ochannel.publish.call_count, 1) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'update') self.assertEqual(pargs[1]['indicator'], 'i') self.assertEqual(pargs[1]['value']['s1$a'], 1) self.assertListEqual(pargs[1]['value']['sources'], ['s1s']) a.filtered_update('s2', indicator='i', value={'s2$a': 1, 'sources': ['s2s']}) self.assertEqual(ochannel.publish.call_count, 2) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'withdraw') self.assertEqual(pargs[1]['indicator'], 'i') self.assertIn('value', pargs[1]) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() def test_aggregate_2uw(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']}) pargs = ochannel.publish.call_args[0] self.assertListEqual(pargs[1]['value']['sources'], ['s1s']) a.filtered_update('s2', indicator='i', value={'s2$a': 1, 'sources': ['s2s']}) pargs = ochannel.publish.call_args[0] self.assertListEqual(pargs[1]['value']['sources'], ['s1s', 's2s']) a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']}) pargs = ochannel.publish.call_args[0] self.assertListEqual(pargs[1]['value']['sources'], ['s1s', 's2s']) a.filtered_withdraw('s2', indicator='i') pargs = ochannel.publish.call_args[0] self.assertListEqual(pargs[1]['value']['sources'], ['s1s']) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() def test_aggregate_2u2w(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']}) self.assertEqual(ochannel.publish.call_count, 1) pargs = ochannel.publish.call_args[0] self.assertListEqual(pargs[1]['value']['sources'], ['s1s']) a.filtered_update('s2', indicator='i', value={'s2$a': 1, 'sources': ['s2s']}) self.assertEqual(ochannel.publish.call_count, 2) pargs = ochannel.publish.call_args[0] self.assertListEqual(pargs[1]['value']['sources'], ['s1s', 's2s']) a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']}) self.assertEqual(ochannel.publish.call_count, 3) pargs = ochannel.publish.call_args[0] self.assertListEqual(pargs[1]['value']['sources'], ['s1s', 's2s']) a.filtered_withdraw('s2', indicator='i') self.assertEqual(ochannel.publish.call_count, 4) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'update') self.assertListEqual(pargs[1]['value']['sources'], ['s1s']) a.filtered_withdraw('s1', indicator='i') self.assertEqual(ochannel.publish.call_count, 5) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'withdraw') self.assertIn('value', pargs[1]) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() def test_aggregate_u2w_difftypes(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config) inputs = ['s1', 's2'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'type': 'a', 'sources': ['s1s']}) self.assertEqual(ochannel.publish.call_count, 1) pargs = ochannel.publish.call_args[0] self.assertListEqual(pargs[1]['value']['sources'], ['s1s']) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='i2', value={'s1$a': 1, 'type': 'a', 'sources': ['s1s']}) self.assertEqual(ochannel.publish.call_count, 1) pargs = ochannel.publish.call_args[0] self.assertListEqual(pargs[1]['value']['sources'], ['s1s']) ochannel.publish.reset_mock() a.filtered_withdraw('s1', indicator='i', value={'type': 'b'}) self.assertEqual(ochannel.publish.call_count, 0) ochannel.publish.reset_mock() a.filtered_withdraw('s1', indicator='i', value={'type': 'a'}) self.assertEqual(ochannel.publish.call_count, 1) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'withdraw') ochannel.publish.reset_mock() a.filtered_withdraw('s2', indicator='i2') self.assertEqual(ochannel.publish.call_count, 1) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'withdraw') self.assertIn('value', pargs[1]) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() def test_aggregate_uwlwl(self): config = { 'whitelist_prefixes': ['s2', 's3'] } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config) inputs = ['s1', 's2', 's3'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='i', value={'s1$a': 1, 'sources': ['s1s']}) self.assertEqual(ochannel.publish.call_count, 1) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'update') self.assertEqual(pargs[1]['indicator'], 'i') self.assertEqual(pargs[1]['value']['s1$a'], 1) self.assertListEqual(pargs[1]['value']['sources'], ['s1s']) a.filtered_update('s2', indicator='i', value={'s2$a': 1, 'sources': ['s2s']}) self.assertEqual(ochannel.publish.call_count, 2) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'withdraw') self.assertEqual(pargs[1]['indicator'], 'i') self.assertIn('value', pargs[1]) a.filtered_update('s3', indicator='i', value={'s3$a': 1, 'sources': ['s3s']}) self.assertEqual(ochannel.publish.call_count, 2) a.filtered_withdraw('s3', indicator='i') self.assertEqual(ochannel.publish.call_count, 2) a.filtered_withdraw('s2', indicator='i') self.assertEqual(ochannel.publish.call_count, 3) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'update') self.assertEqual(pargs[1]['indicator'], 'i') self.assertListEqual(pargs[1]['value']['sources'], ['s1s']) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() def test_infilters(self): config = { 'infilters': [ { 'name': 'rule1', 'conditions': [ 'type(sources) == null' ], 'actions': [ 'drop' ] } ] } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config) inputs = ['s1', 's2', 's3'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.update(source='s1', indicator='i', value={'s1a': 1, 'sources': ['s1s']}) gevent.sleep(0.1) self.assertEqual(ochannel.publish.call_count, 1) ochannel.publish.reset_mock() a.update(source='s2', indicator='i', value={'s2a': 1}) gevent.sleep(0.1) self.assertEqual(ochannel.publish.call_count, 0) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() def test_infilters_2u(self): config = { 'infilters': [ { 'name': 'rule1', 'conditions': [ 'type(sources) == null' ], 'actions': [ 'drop' ] } ] } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config) inputs = ['s1', 's2', 's3'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.update(source='s1', indicator='i', value={'s1a': 1, 'sources': ['s1s']}) gevent.sleep(0.1) self.assertEqual(ochannel.publish.call_count, 1) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'update') self.assertEqual(pargs[1]['indicator'], 'i') self.assertEqual(pargs[1]['value']['s1a'], 1) self.assertListEqual(pargs[1]['value']['sources'], ['s1s']) ochannel.publish.reset_mock() a.update(source='s1', indicator='i', value={'s1a': 1}) gevent.sleep(0.1) self.assertEqual(ochannel.publish.call_count, 1) pargs = ochannel.publish.call_args[0] self.assertEqual(pargs[0], 'withdraw') self.assertEqual(pargs[1]['indicator'], 'i') self.assertIn('value', pargs[1]) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() def test_attr_override(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config) inputs = ['s1', 's2', 's3'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='10.1.0.0/16', value={ 'type': 'IPv4', 'sources': ['s1s'], 'direction': 'inbound', 'first_seen': 10, 'last_seen': 25, 'confidence': 20 }) ochannel.publish.reset_mock() a.filtered_update('s2', indicator='10.1.0.0/16', value={ 'type': 'IPv4', 'sources': ['s2s'], 'direction': 'inbound', 'first_seen': 5, 'last_seen': 20, 'confidence': 30 }) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '10.1.0.0/16', 'value': { 'sources': ['s1s', 's2s'], 'direction': 'inbound', 'first_seen': 5, 'last_seen': 25, 'confidence': 30 } } ], all_here=True ) ) a.stop() a = None gc.collect() def test_get_all(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.send_rpc.return_value = {'error': None, 'result': 'OK'} a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config) inputs = ['s1', 's2', 's3'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='10.1.0.0/16', value={ 'type': 'IPv4', 'sources': ['s2s'], 'direction': 'inbound', 'first_seen': 10, 'last_seen': 25, 'confidence': 20 }) a.filtered_update('s2', indicator='10.1.0.0/16', value={ 'type': 'IPv4', 'sources': ['s2s'], 'direction': 'inbound', 'first_seen': 5, 'last_seen': 20, 'confidence': 30 }) a.filtered_update('s3', indicator='10.1.1.0/24', value={ 'type': 'IPv4', 'sources': ['s1s'], 'direction': 'inbound', 'first_seen': 5, 'last_seen': 20, 'confidence': 30 }) a.get_all(source='test') self.assertTrue( check_for_rpc( chassis.send_rpc.call_args_list, [ { 'method': 'update', 'indicator': '10.1.0.0/16', 'value': { 'sources': ['s2s'], 'direction': 'inbound', 'first_seen': 5, 'last_seen': 25, 'confidence': 30 } }, { 'method': 'update', 'indicator': '10.1.1.0/24', 'value': { 'sources': ['s1s'], 'direction': 'inbound', 'first_seen': 5, 'last_seen': 20, 'confidence': 30 } } ], all_here=True, offset=2 ) ) a.stop() a = None gc.collect() def test_get_range(self): config = {} chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.send_rpc.return_value = {'error': None, 'result': 'OK'} a = minemeld.ft.op.AggregateFT(FTNAME, chassis, config) inputs = ['s1', 's2', 's3'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('s1', indicator='10.1.0.0/16', value={ 'type': 'IPv4', 'sources': ['s2s'], 'direction': 'inbound', 'first_seen': 10, 'last_seen': 25, 'confidence': 20 }) a.filtered_update('s2', indicator='10.1.0.0/16', value={ 'type': 'IPv4', 'sources': ['s2s'], 'direction': 'inbound', 'first_seen': 5, 'last_seen': 20, 'confidence': 30 }) a.filtered_update('s3', indicator='10.1.1.0/24', value={ 'type': 'IPv4', 'sources': ['s1s'], 'direction': 'inbound', 'first_seen': 5, 'last_seen': 20, 'confidence': 30 }) a.get_range(source='test', from_key='10.1.0.0/16', to_key='10.1.1.0/24') self.assertTrue( check_for_rpc( chassis.send_rpc.call_args_list, [ { 'method': 'update', 'indicator': '10.1.0.0/16', 'value': { 'sources': ['s2s'], 'direction': 'inbound', 'first_seen': 5, 'last_seen': 25, 'confidence': 30 } }, { 'method': 'update', 'indicator': '10.1.1.0/24', 'value': { 'sources': ['s1s'], 'direction': 'inbound', 'first_seen': 5, 'last_seen': 20, 'confidence': 30 } } ], all_here=True, offset=2 ) ) a.stop() a = None gc.collect() ================================================ FILE: tests/test_ft_redis.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT Redis tests Unit tests for minemeld.ft.redis """ import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import unittest import mock import redis import time import minemeld.ft.redis FTNAME = 'testft-%d' % int(time.time()) class MineMeldFTRedisTests(unittest.TestCase): def setUp(self): SR = redis.StrictRedis() SR.delete(FTNAME) def tearDown(self): SR = redis.StrictRedis() SR.delete(FTNAME) def test_init(self): config = {} chassis = mock.Mock() b = minemeld.ft.redis.RedisSet(FTNAME, chassis, config) self.assertEqual(b.name, FTNAME) self.assertEqual(b.chassis, chassis) self.assertEqual(b.config, config) self.assertItemsEqual(b.inputs, []) self.assertEqual(b.output, None) self.assertEqual(b.redis_skey, FTNAME) self.assertNotEqual(b.SR, None) self.assertEqual(b.redis_url, 'unix:///var/run/redis/redis.sock') def test_connect_io(self): config = {} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None b = minemeld.ft.redis.RedisSet(FTNAME, chassis, config) inputs = ['a', 'b', 'c'] output = True b.connect(inputs, output) b.mgmtbus_initialize() self.assertItemsEqual(b.inputs, inputs) self.assertEqual(b.output, None) icalls = [] for i in inputs: icalls.append( mock.call( FTNAME, b, i, allowed_methods=[ 'update', 'withdraw', 'checkpoint' ] ) ) chassis.request_sub_channel.assert_has_calls( icalls, any_order=True ) chassis.request_rpc_channel.assert_called_once_with( FTNAME, b, allowed_methods=[ 'update', 'withdraw', 'checkpoint', 'get', 'get_all', 'get_range', 'length' ] ) chassis.request_pub_channel.assert_not_called() def test_uw(self): config = {} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock b = minemeld.ft.redis.RedisSet(FTNAME, chassis, config) inputs = ['a', 'b', 'c'] output = False b.connect(inputs, output) b.mgmtbus_initialize() b.start() time.sleep(1) SR = redis.StrictRedis() b.filtered_update('a', indicator='testi', value={'test': 'v'}) sm = SR.zrange(FTNAME, 0, -1) self.assertEqual(len(sm), 1) self.assertIn('testi', sm) b.filtered_withdraw('a', indicator='testi') sm = SR.zrange(FTNAME, 0, -1) self.assertEqual(len(sm), 0) b.stop() self.assertNotEqual(b.SR, None) def test_stats(self): config = {} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock b = minemeld.ft.redis.RedisSet(FTNAME, chassis, config) inputs = ['a', 'b', 'c'] output = False b.connect(inputs, output) b.mgmtbus_reset() b.start() time.sleep(1) b.filtered_update('a', indicator='testi', value={'test': 'v'}) self.assertEqual(b.length(), 1) status = b.mgmtbus_status() self.assertEqual(status['statistics']['added'], 1) b.filtered_update('a', indicator='testi', value={'test': 'v2'}) self.assertEqual(b.length(), 1) status = b.mgmtbus_status() self.assertEqual(status['statistics']['added'], 1) self.assertEqual(status['statistics']['removed'], 0) b.filtered_withdraw('a', indicator='testi') self.assertEqual(b.length(), 0) status = b.mgmtbus_status() self.assertEqual(status['statistics']['removed'], 1) b.stop() def test_store_value(self): config = {'store_value': True} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock b = minemeld.ft.redis.RedisSet(FTNAME, chassis, config) inputs = ['a', 'b', 'c'] output = False b.connect(inputs, output) b.mgmtbus_reset() b.start() time.sleep(1) SR = redis.StrictRedis() b.filtered_update('a', indicator='testi', value={'test': 'v'}) sm = SR.zrange(FTNAME, 0, -1) self.assertEqual(len(sm), 1) self.assertIn('testi', sm) sm = SR.hlen(FTNAME+'.value') self.assertEqual(sm, 1) b.filtered_withdraw('a', indicator='testi') sm = SR.zrange(FTNAME, 0, -1) self.assertEqual(len(sm), 0) sm = SR.hlen(FTNAME+'.value') self.assertEqual(sm, 0) b.stop() self.assertNotEqual(b.SR, None) def test_store_value_overflow(self): config = {'store_value': True} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock b = minemeld.ft.redis.RedisSet(FTNAME, chassis, config) b.max_entries = 1 inputs = ['a', 'b', 'c'] output = False b.connect(inputs, output) b.mgmtbus_reset() b.start() time.sleep(1) SR = redis.StrictRedis() b.filtered_update('a', indicator='testi', value={'test': 'v'}) sm = SR.zrange(FTNAME, 0, -1) self.assertEqual(len(sm), 1) self.assertIn('testi', sm) sm = SR.hlen(FTNAME+'.value') self.assertEqual(sm, 1) b.filtered_update('a', indicator='testio', value={'test': 'v'}) self.assertEqual(b.statistics['drop.overflow'], 1) sm = SR.zrange(FTNAME, 0, -1) self.assertEqual(len(sm), 1) self.assertIn('testi', sm) sm = SR.hlen(FTNAME+'.value') self.assertEqual(sm, 1) b.filtered_withdraw('a', indicator='testi') sm = SR.zrange(FTNAME, 0, -1) self.assertEqual(len(sm), 0) sm = SR.hlen(FTNAME+'.value') self.assertEqual(sm, 0) b.filtered_update('a', indicator='testio', value={'test': 'v'}) self.assertEqual(b.statistics['drop.overflow'], 1) sm = SR.zrange(FTNAME, 0, -1) self.assertEqual(len(sm), 1) self.assertIn('testio', sm) sm = SR.hlen(FTNAME+'.value') self.assertEqual(sm, 1) b.stop() self.assertNotEqual(b.SR, None) ================================================ FILE: tests/test_ft_st.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT ST tests Unit tests for minemeld.ft.st """ import unittest import tempfile import shutil import random import uuid import time from nose.plugins.attrib import attr import minemeld.ft.st TABLENAME = tempfile.mktemp(prefix='minemeld.ftsttest') NUM_ELEMENTS = 10000 class MineMeldFTSTTests(unittest.TestCase): def setUp(self): try: shutil.rmtree(TABLENAME) except: pass def tearDown(self): try: shutil.rmtree(TABLENAME) except: pass def test_add_delete(self): st = minemeld.ft.st.ST(TABLENAME, 8, truncate=True) sid = uuid.uuid4().bytes st.put(sid, 1, 5, 1) st.delete(sid, 1, 5, 1) st.close() def test_query_endpoints_forward(self): st = minemeld.ft.st.ST(TABLENAME, 8, truncate=True) sid1 = uuid.uuid4().bytes sid2 = uuid.uuid4().bytes st.put(sid1, 1, 70, 1) st.put(sid2, 50, 100, 1) eps = [ep[0] for ep in st.query_endpoints( start=0, stop=st.max_endpoint, reverse=False, include_start=False, include_stop=False )] self.assertEqual(eps, [1, 50, 70, 100]) st.close() def test_query_endpoints_reverse(self): st = minemeld.ft.st.ST(TABLENAME, 8, truncate=True) sid1 = uuid.uuid4().bytes sid2 = uuid.uuid4().bytes st.put(sid1, 1, 70, 1) st.put(sid2, 50, 100, 1) eps = [ep[0] for ep in st.query_endpoints( start=0, stop=st.max_endpoint, reverse=True, include_start=False, include_stop=False )] self.assertEqual(eps, [100, 70, 50, 1]) st.close() def test_basic_cover(self): st = minemeld.ft.st.ST(TABLENAME, 8, truncate=True) sid = uuid.uuid4().bytes st.put(sid, 1, 5, 1) for i in range(1, 6): ci = st.cover(i) interval = next(ci, None) self.assertEqual(interval[0], sid) self.assertEqual(interval[1], 1) self.assertEqual(interval[2], 1) self.assertEqual(interval[3], 5) interval2 = next(ci, None) self.assertEqual(interval2, None) st.close() def test_cover_overlap(self): st = minemeld.ft.st.ST(TABLENAME, 8, truncate=True) sid1 = uuid.uuid4().bytes sid2 = uuid.uuid4().bytes st.put(sid1, 1, 5, 1) st.put(sid2, 3, 7, 2) ci = st.cover(1) interval = next(ci, None) self.assertEqual(interval[0], sid1) self.assertEqual(interval[1], 1) self.assertEqual(interval[2], 1) self.assertEqual(interval[3], 5) interval = next(ci, None) self.assertEqual(interval, None) ci = st.cover(3) intervals = [i for i in st.cover(3)] self.assertEqual(len(intervals), 2) self.assertEqual(intervals[0][0], sid1) self.assertEqual(intervals[0][1], 1) self.assertEqual(intervals[0][2], 1) self.assertEqual(intervals[0][3], 5) self.assertEqual(intervals[1][0], sid2) self.assertEqual(intervals[1][1], 2) self.assertEqual(intervals[1][2], 3) self.assertEqual(intervals[1][3], 7) ci = st.cover(7) interval = next(ci, None) self.assertEqual(interval[0], sid2) self.assertEqual(interval[1], 2) self.assertEqual(interval[2], 3) self.assertEqual(interval[3], 7) interval = next(ci, None) self.assertEqual(interval, None) st.close() def test_cover_overlap2(self): st = minemeld.ft.st.ST(TABLENAME, 8, truncate=True) sid1 = uuid.uuid4().bytes sid2 = uuid.uuid4().bytes st.put(sid1, 3, 7, 1) st.put(sid2, 3, 7, 2) intervals = [i for i in st.cover(3)] self.assertEqual(len(intervals), 2) self.assertEqual(intervals[0][0], sid2) self.assertEqual(intervals[0][1], 2) self.assertEqual(intervals[0][2], 3) self.assertEqual(intervals[0][3], 7) self.assertEqual(intervals[1][0], sid1) self.assertEqual(intervals[1][1], 1) self.assertEqual(intervals[1][2], 3) self.assertEqual(intervals[1][3], 7) st.close() def _random_map(self, nbits=10, nintervals=1000): epmax = (1 << nbits)-1 rmap = [set() for i in xrange(epmax+1)] st = minemeld.ft.st.ST(TABLENAME, nbits, truncate=True) for j in xrange(nintervals): sid = uuid.uuid4().bytes end = random.randint(0, epmax) start = random.randint(0, epmax) if end < start: start, end = end, start st.put(sid, start, end, level=1) for k in xrange(start, end+1): rmap[k].add(sid) eps = [] for ep, lvl, t, id_ in st.query_endpoints(): if ep == 0 or ep == epmax: self.assertTrue(len(rmap[ep]) > 0) else: c = len(rmap[ep] ^ rmap[ep-1]) + len(rmap[ep] ^ rmap[ep+1]) self.assertTrue( c > 0, msg="no change detected @ep %d: " "%r %r %r" % (ep, rmap[ep-1], rmap[ep], rmap[ep+1]) ) eps.append(ep) for e in eps: intervals = [x[0] for x in st.cover(e)] intervals.sort() self.assertListEqual(intervals, sorted(rmap[e])) st.close() def test_random_map_fast(self): self._random_map() @attr('slow') def test_random_map_fast2(self): self._random_map(nintervals=2000) def test_255(self): st = minemeld.ft.st.ST(TABLENAME, 32, truncate=True) sid = uuid.uuid4().bytes st.put(sid, 0, 0xFF) self.assertEqual(st.num_segments, 1) self.assertEqual(st.num_endpoints, 2) st.close() @attr('slow') def test_stress_0(self): num_intervals = 100000 st = minemeld.ft.st.ST(TABLENAME, 32, truncate=True) t1 = time.time() for j in xrange(num_intervals): end = random.randint(0, 0xFFFFFFFF) if random.randint(0, 1) == 0: end = end & 0xFFFFFF00 start = end + 0xFF else: start = end sid = uuid.uuid4().bytes t2 = time.time() dt = t2-t1 t1 = time.time() for j in xrange(num_intervals): end = random.randint(0, 0xFFFFFFFF) if random.randint(0, 1) == 0: start = end & 0xFFFFFF00 end = start + 0xFF else: start = end sid = uuid.uuid4().bytes st.put(sid, start, end) t2 = time.time() print "TIME: Inserted %d intervals in %d" % (num_intervals, (t2-t1-dt)) self.assertEqual(st.num_segments, num_intervals) self.assertEqual(st.num_endpoints, num_intervals*2) st.close() @attr('slow') def test_stress_1(self): num_intervals = 100000 st = minemeld.ft.st.ST(TABLENAME, 32, truncate=True) t1 = time.time() for j in xrange(num_intervals): end = random.randint(0, 0xFFFFFFFF) start = random.randint(0, end) sid = uuid.uuid4().bytes t2 = time.time() dt = t2-t1 t1 = time.time() for j in xrange(num_intervals): end = random.randint(0, 0xFFFFFFFF) start = random.randint(0, end) sid = uuid.uuid4().bytes st.put(sid, start, end) t2 = time.time() print "TIME: Inserted %d intervals in %d" % (num_intervals, (t2-t1-dt)) num_queries = 100000 t1 = time.time() j = 0 for j in xrange(num_queries): q = random.randint(0, 0xFFFFFFFF) next(st.cover(q), None) t2 = time.time() print "TIME: Queried %d times in %d" % (num_queries, (t2-t1)) st.close() ================================================ FILE: tests/test_ft_syslog.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT syslog tests Unit tests for minemeld.ft.syslog """ import unittest import shutil import time import logging import mock import gevent import socket import gc import minemeld.ft.syslog FTNAME = 'testft-%d' % int(time.time()) LOG = logging.getLogger(__name__) def check_for_rpc(call_args_list, check_list, all_here=False): LOG.debug("call_args_list: %s", call_args_list) found = [] for chk in check_list: LOG.debug("checking: %s", chk) for j in xrange(len(call_args_list)): if j in found: continue args = call_args_list[j][0] if args[0] != chk['method']: continue if args[1]['indicator'] != chk['indicator']: continue chkvalue = chk.get('value', None) if chkvalue is None: found.append(j) LOG.debug("found @%d", j) break argsvalue = args[1].get('value', None) if chkvalue is not None and argsvalue is None: continue failed = False for k in chkvalue.keys(): if k not in argsvalue: failed = True break if chkvalue[k] != argsvalue[k]: failed = True break if failed: continue found.append(j) LOG.debug("found @%d", j) break c1 = len(found) == len(check_list) if not all_here: return c1 c2 = len(found) == len(call_args_list) return c1+c2 == 2 class MineMeldFTSyslogMatcherests(unittest.TestCase): def setUp(self): try: shutil.rmtree(FTNAME) except: pass try: shutil.rmtree(FTNAME+"_ipv4") except: pass try: shutil.rmtree(FTNAME+"_indicators") except: pass def tearDown(self): try: shutil.rmtree(FTNAME) except: pass try: shutil.rmtree(FTNAME+"_ipv4") except: pass try: shutil.rmtree(FTNAME+"_indicators") except: pass @mock.patch.object(gevent, 'spawn_later') def test_handle_ip_01(self, spawnl_mock): config = { } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config) inputs = ['a'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 2) a.filtered_update('a', indicator='1.1.1.1-1.1.1.2', value={ 'type': 'IPv4', 'confidence': 100 }) self.assertEqual(a.length(), 1) a._handle_ip('1.1.1.1') self.assertEqual(a.table.num_indicators, 1) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': '1.1.1.1', 'value': { 'syslog_original_indicator': 'IPv4'+'1.1.1.1-1.1.1.2', 'type': 'IPv4', 'confidence': 100 } } ], all_here=True ) ) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn_later') def test_handle_ip_02(self, spawnl_mock): config = { } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config) inputs = ['a'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('a', indicator='1.1.1.1-1.1.1.2', value={ 'type': 'IPv4', 'confidence': 100 }) a._handle_ip('1.1.1.1') ochannel.publish.reset_mock() a.filtered_withdraw('a', indicator='1.1.1.1-1.1.1.2') self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'withdraw', 'indicator': '1.1.1.1', 'value': { 'type': 'IPv4', 'confidence': 100 } } ], all_here=True ) ) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn_later') def test_handle_ip_03(self, spawnl_mock): config = { } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config) inputs = ['a'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('a', indicator='1.1.1.1-1.1.1.2', value={ 'type': 'IPv4', 'confidence': 100 }) a.filtered_update('a', indicator='1.1.1.4-1.1.1.5', value={ 'type': 'IPv4', 'confidence': 100 }) a._handle_ip('1.1.1.3') self.assertEqual(ochannel.publish.call_count, 0) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn_later') def test_handle_url_01(self, spawnl_mock): config = { } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config) inputs = ['a'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() self.assertEqual(spawnl_mock.call_count, 2) a.filtered_update('a', indicator='www.example.com', value={ 'type': 'domain', 'confidence': 100 }) self.assertEqual(a.length(), 1) a._handle_url('www.example.com/cgi/addressbook.php') self.assertEqual(a.table.num_indicators, 1) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'update', 'indicator': 'www.example.com', 'value': { 'syslog_original_indicator': 'domain'+'www.example.com', 'type': 'domain', 'confidence': 100 } } ], all_here=True ) ) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn_later') def test_handle_url_02(self, spawnl_mock): config = { } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config) inputs = ['a'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('a', indicator='www.example.com', value={ 'type': 'domain', 'confidence': 100 }) a._handle_url('www.example.com/cgi/addressbook.php') ochannel.publish.reset_mock() a.filtered_withdraw('a', indicator='www.example.com') self.assertEqual(a.table_indicators.num_indicators, 0) self.assertTrue( check_for_rpc( ochannel.publish.call_args_list, [ { 'method': 'withdraw', 'indicator': 'www.example.com', 'value': { 'type': 'domain', 'confidence': 100 } } ], all_here=True ) ) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(socket, 'socket') def test_logstash_url(self, socket_socket, spawnl_mock): config = { 'logstash_host': '127.0.0.1' } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock mock_socket = mock.Mock() socket_socket.return_value = mock_socket a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config) inputs = ['a'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('a', indicator='www.example.com', value={ 'type': 'domain', 'confidence': 100 }) a._handle_url( 'www.example.com/cgi/addressbook.php', message={ 'session_id': 666 } ) self.assertEqual(mock_socket.connect.call_count, 1) self.assertEqual(mock_socket.sendall.call_count, 1) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(socket, 'socket') def test_logstash_ip(self, socket_socket, spawnl_mock): config = { 'logstash_host': '127.0.0.1' } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock mock_socket = mock.Mock() socket_socket.return_value = mock_socket a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config) inputs = ['a'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('a', indicator='1.1.1.0-1.1.1.20', value={ 'type': 'IPv4', 'confidence': 100 }) a._handle_ip('1.1.1.1', message={ 'session_id': 666 }) self.assertEqual(mock_socket.connect.call_count, 1) self.assertEqual(mock_socket.sendall.call_count, 1) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() @mock.patch.object(gevent, 'spawn_later') @mock.patch.object(socket, 'socket') def test_logstash_event_tags(self, socket_socket, spawnl_mock): config = { 'logstash_host': '127.0.0.1' } chassis = mock.Mock() ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock mock_socket = mock.Mock() socket_socket.return_value = mock_socket a = minemeld.ft.syslog.SyslogMatcher(FTNAME, chassis, config) inputs = ['a'] output = True a.connect(inputs, output) a.mgmtbus_initialize() a.start() a.filtered_update('a', indicator='1.1.1.0-1.1.1.20', value={ 'type': 'IPv4', 'confidence': 100 }) a._handle_ip('1.1.1.1', message={ 'session_id': 666, 'event.tags': [1, 2] }) self.assertEqual(mock_socket.connect.call_count, 1) self.assertEqual(mock_socket.sendall.call_count, 1) self.assertEqual( 'session_id' in mock_socket.sendall.call_args[0][0], True ) self.assertEqual( 'event.tags' in mock_socket.sendall.call_args[0][0], False ) a.stop() a = None chassis = None rpcmock = None ochannel = None gc.collect() ================================================ FILE: tests/test_ft_table.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT Table tests Unit tests for minemeld.ft.table """ import unittest import tempfile import shutil import random import time import minemeld.ft.table from nose.plugins.attrib import attr TABLENAME = tempfile.mktemp(prefix='minemeld.fttabletest') NUM_ELEMENTS = 10000 class MineMeldFTTableTests(unittest.TestCase): def setUp(self): try: shutil.rmtree(TABLENAME) except: pass def tearDown(self): try: shutil.rmtree(TABLENAME) except: pass def test_truncate(self): table = minemeld.ft.table.Table(TABLENAME) table.put('key', {'a': 1}) table.close() table = None table = minemeld.ft.table.Table(TABLENAME) self.assertEqual(table.num_indicators, 1) table.close() table = None table = minemeld.ft.table.Table(TABLENAME, truncate=True) self.assertEqual(table.num_indicators, 0) table.close() def test_insert(self): table = minemeld.ft.table.Table(TABLENAME) table.create_index('a') for i in range(NUM_ELEMENTS): value = {'a': random.randint(0, 500)} key = 'i%d' % i table.put(key, value) self.assertEqual(table.num_indicators, NUM_ELEMENTS) table.close() def test_index_query(self): table = minemeld.ft.table.Table(TABLENAME) table.create_index('a') num_below_500 = 0 for i in xrange(NUM_ELEMENTS): value = {'a': random.randint(0, 1000)} key = 'i%d' % i table.put(key, value) if value['a'] <= 500: num_below_500 += 1 j = 0 for k, v in table.query('a', from_key=0, to_key=500, include_value=True): j += 1 self.assertEqual(j, num_below_500) table.close() def test_index_query_2(self): table = minemeld.ft.table.Table(TABLENAME) table.create_index('a') for i in xrange(NUM_ELEMENTS): value = {'a': 1483184218151+random.randint(600, 1000)} key = 'i%d' % i table.put(key, value) num_below_500 = 0 for i in xrange(NUM_ELEMENTS): value = {'a': 1483184218151+random.randint(0, 1000)} key = 'i%d' % i table.put(key, value) if value['a'] <= 1483184218151+500: num_below_500 += 1 j = 0 for k, v in table.query('a', to_key=1483184218151+500, include_value=True): j += 1 self.assertEqual(j, num_below_500) table.close() def test_index_query_3(self): table = minemeld.ft.table.Table(TABLENAME) table.create_index('a') for i in xrange(NUM_ELEMENTS): value = {'a': 1483184218151+random.randint(0, 500)} key = 'i%d' % i table.put(key, value) for i in xrange(NUM_ELEMENTS): key = 'i%d' % i table.delete(key) num_below_500 = 0 for i in xrange(NUM_ELEMENTS): value = {'a': 1483184218151+random.randint(0, 1000)} key = 'i%d' % i table.put(key, value) if value['a'] <= 1483184218151+500: num_below_500 += 1 j = 0 for k, v in table.query('a', to_key=1483184218151+500, include_value=True): j += 1 self.assertEqual(j, num_below_500) table.close() def test_query(self): table = minemeld.ft.table.Table(TABLENAME) table.create_index('a') for i in range(NUM_ELEMENTS): value = {'a': random.randint(0, 500)} key = 'i%d' % i table.put(key, value) j = 0 for k, v in table.query(include_value=True): j += 1 self.assertEqual(j, NUM_ELEMENTS) table.close() def test_exists(self): table = minemeld.ft.table.Table(TABLENAME) table.create_index('a') for i in range(NUM_ELEMENTS): value = {'a': random.randint(0, 500)} key = 'i%d' % i table.put(key, value) for i in range(NUM_ELEMENTS): j = random.randint(0, NUM_ELEMENTS-1) self.assertTrue( table.exists('i%d' % j), msg="i%d does not exists" % j ) table.close() def test_not_exists(self): table = minemeld.ft.table.Table(TABLENAME) table.create_index('a') for i in range(NUM_ELEMENTS): value = {'a': random.randint(0, 500)} key = 'i%d' % i table.put(key, value) for i in range(NUM_ELEMENTS): j = random.randint(NUM_ELEMENTS, 2*NUM_ELEMENTS) self.assertFalse(table.exists('i%d' % j)) table.close() def test_update(self): table = minemeld.ft.table.Table(TABLENAME) table.create_index('a') table.put('k1', {'a': 1}) table.put('k2', {'a': 1}) table.put('k1', {'a': 2}) ok = 0 rk = None for k in table.query('a', from_key=0, to_key=1): rk = k ok += 1 self.assertEqual(rk, 'k2') self.assertEqual(ok, 1) table.close() @attr('slow') def test_random(self): # create table table = minemeld.ft.table.Table(TABLENAME) table.create_index('a') # local dict d = {} # add 10000 elements to the table # with an 'a' attribute in range 0,500 for i in range(NUM_ELEMENTS): value = {'a': random.randint(0, 500)} key = 'i%d' % i d[key] = value table.put(key, value) # check number of indicators added self.assertEqual(table.num_indicators, len(d.keys())) # check sorted query retrieval flatdict = sorted(d.items(), key=lambda x: x[1]['a']) j = 0 for k, v in table.query('a', from_key=0, to_key=500, include_value=True): de = flatdict[j] self.assertEqual(de[1]['a'], v['a']) j = j+1 # 1000 random add or delete for j in range(1000): op = random.randint(0, 1) if op == 0: # delete i = 'i%d' % random.randint(0, 2000) if i in d: del d[i] table.delete(i) elif op == 1: # add i = 'i%d' % random.randint(0, 2000) v = {'a': random.randint(0, 500)} table.put(i, v) d[i] = v # check num of indicators self.assertEqual(table.num_indicators, len(d.keys())) flatdict = sorted(d.items(), key=lambda x: x[1]['a']) j = 0 for k, v in table.query('a', from_key=0, to_key=500, include_value=True): de = flatdict[j] # check sorting self.assertEqual(de[1]['a'], v['a']) j = j+1 # close table table.close() table = None # reopen table = minemeld.ft.table.Table(TABLENAME) table.create_index('a') self.assertEqual(table.num_indicators, len(d.keys())) # check sort again flatdict = sorted(d.items(), key=lambda x: x[1]['a']) j = 0 for k, v in table.query('a', from_key=0, to_key=500, include_value=True): de = flatdict[j] self.assertEqual(de[1]['a'], v['a']) j = j+1 table.close() @attr('slow') def test_write(self): # create table table = minemeld.ft.table.Table(TABLENAME) # local dict d = {} t1 = time.time() for i in xrange(100000): value = {'a': random.randint(0, 500)} key = 'i%d' % i d[key] = value table.put(key, value) t2 = time.time() print 'TIME: Written %d elements in %s secs' % (100000, t2-t1) # check number of indicators added self.assertEqual(table.num_indicators, len(d.keys())) table.close() table = None ================================================ FILE: tests/test_ft_taxii.py ================================================ # -*- coding: utf-8 -*- # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT TAXII tests Unit tests for minemeld.ft.taxii """ import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import unittest import mock import redis import gevent import greenlet import time import xmltodict import os import libtaxii.constants import re import lz4 import json import minemeld.ft.taxii import minemeld.ft FTNAME = 'testft-%d' % int(time.time()) MYDIR = os.path.dirname(__file__) class MockTaxiiContentBlock(object): def __init__(self, stix_xml): class _Binding(object): def __init__(self, id_): self.binding_id = id_ self.content = stix_xml self.content_binding = _Binding(libtaxii.constants.CB_STIX_XML_111) class MineMeldFTTaxiiTests(unittest.TestCase): @mock.patch.object(gevent, 'Greenlet') def test_taxiiclient_parse(self, glet_mock): config = { 'side_config': 'dummy.yml', 'ca_file': 'dummy.crt' } chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock b = minemeld.ft.taxii.TaxiiClient(FTNAME, chassis, config) inputs = [] output = False b.connect(inputs, output) b.mgmtbus_initialize() b.start() testfiles = os.listdir(MYDIR) testfiles = filter( lambda x: x.startswith('test_ft_taxii_stix_package_'), testfiles ) for t in testfiles: with open(os.path.join(MYDIR, t), 'r') as f: sxml = f.read() mo = re.match('test_ft_taxii_stix_package_([A-Za-z0-9]+)_([0-9]+)_.*', t) self.assertNotEqual(mo, None) type_ = mo.group(1) num_indicators = int(mo.group(2)) stix_objects = { 'observables': {}, 'indicators': {}, 'ttps': {} } content_blocks = [ MockTaxiiContentBlock(sxml) ] b._handle_content_blocks( content_blocks, stix_objects ) params = { 'ttps': stix_objects['ttps'], 'observables': stix_objects['observables'] } indicators = [[iid, iv, params] for iid, iv in stix_objects['indicators'].iteritems()] for i in indicators: result = b._process_item(i) self.assertEqual(len(result), num_indicators) if type_ != 'any': for r in result: self.assertEqual(r[1]['type'], type_) b.stop() @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_datafeed_init(self, glet_mock, SR_mock): config = {} chassis = mock.Mock() b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config) self.assertEqual(b.name, FTNAME) self.assertEqual(b.chassis, chassis) self.assertEqual(b.config, config) self.assertItemsEqual(b.inputs, []) self.assertEqual(b.output, None) self.assertEqual(b.redis_skey, FTNAME) self.assertEqual(b.redis_skey_chkp, FTNAME+'.chkp') self.assertEqual(b.redis_skey_value, FTNAME+'.value') @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_datafeed_update_ip(self, glet_mock, SR_mock): config = {} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config) inputs = ['a'] output = False b.connect(inputs, output) b.mgmtbus_initialize() b.start() # __init__ + get chkp + delete chkp self.assertEqual(len(SR_mock.mock_calls), 6) SR_mock.reset_mock() SR_mock.return_value.zcard.return_value = 1 # unicast b.filtered_update( 'a', indicator='1.1.1.1', value={ 'type': 'IPv4', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': break else: self.fail(msg='hset not found') self.assertEqual(args[2].startswith('lz4'), True) stixdict = json.loads(lz4.uncompress(args[2][3:])) indicator = stixdict['indicators'][0] cyboxprops = indicator['observable']['object']['properties'] self.assertEqual(cyboxprops['address_value'], '1.1.1.1') self.assertEqual(cyboxprops['xsi:type'], 'AddressObjectType') SR_mock.reset_mock() # CIDR b.filtered_update( 'a', indicator='1.1.1.0/24', value={ 'type': 'IPv4', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': break else: self.fail(msg='hset not found') self.assertEqual(args[2].startswith('lz4'), True) stixdict = json.loads(lz4.uncompress(args[2][3:])) indicator = stixdict['indicators'][0] cyboxprops = indicator['observable']['object']['properties'] self.assertEqual(cyboxprops['address_value'], '1.1.1.0/24') self.assertEqual(cyboxprops['xsi:type'], 'AddressObjectType') SR_mock.reset_mock() # fake range b.filtered_update( 'a', indicator='1.1.1.1-1.1.1.1', value={ 'type': 'IPv4', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': break else: self.fail(msg='hset not found') self.assertEqual(args[2].startswith('lz4'), True) stixdict = json.loads(lz4.uncompress(args[2][3:])) indicator = stixdict['indicators'][0] cyboxprops = indicator['observable']['object']['properties'] self.assertEqual(cyboxprops['address_value'], '1.1.1.1') self.assertEqual(cyboxprops['xsi:type'], 'AddressObjectType') SR_mock.reset_mock() # fake range 2 b.filtered_update( 'a', indicator='1.1.1.0-1.1.1.31', value={ 'type': 'IPv4', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': break else: self.fail(msg='hset not found') self.assertEqual(args[2].startswith('lz4'), True) stixdict = json.loads(lz4.uncompress(args[2][3:])) indicator = stixdict['indicators'][0] cyboxprops = indicator['observable']['object']['properties'] self.assertEqual(cyboxprops['address_value'], '1.1.1.0/27') self.assertEqual(cyboxprops['xsi:type'], 'AddressObjectType') SR_mock.reset_mock() SR_mock.return_value.zcard.return_value = 1 # real range b.filtered_update( 'a', indicator='1.1.1.0-1.1.1.33', value={ 'type': 'IPv4', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': break else: self.fail(msg='hset not found') self.assertEqual(args[2].startswith('lz4'), True) stixdict = json.loads(lz4.uncompress(args[2][3:])) indicator = stixdict['indicators'] cyboxprops = indicator[0]['observable']['object']['properties'] self.assertEqual(cyboxprops['address_value'], '1.1.1.0/27') self.assertEqual(cyboxprops['xsi:type'], 'AddressObjectType') cyboxprops = indicator[1]['observable']['object']['properties'] self.assertEqual(cyboxprops['address_value'], '1.1.1.32/31') self.assertEqual(cyboxprops['xsi:type'], 'AddressObjectType') SR_mock.reset_mock() b.stop() @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_datafeed_update_domain(self, glet_mock, SR_mock): config = {} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config) inputs = ['a'] output = False b.connect(inputs, output) b.mgmtbus_initialize() b.start() # __init__ + get chkp + delete chkp self.assertEqual(len(SR_mock.mock_calls), 6) SR_mock.reset_mock() SR_mock.return_value.zcard.return_value = 1 # unicast b.filtered_update( 'a', indicator='example.com', value={ 'type': 'domain', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': break else: self.fail(msg='hset not found') self.assertEqual(args[2].startswith('lz4'), True) stixdict = json.loads(lz4.decompress(args[2][3:])) indicator = stixdict['indicators'][0] cyboxprops = indicator['observable']['object']['properties'] self.assertEqual(cyboxprops['value'], 'example.com') self.assertEqual(cyboxprops['type'], 'FQDN') SR_mock.reset_mock() b.stop() @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_datafeed_update_url(self, glet_mock, SR_mock): config = {} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config) inputs = ['a'] output = False b.connect(inputs, output) b.mgmtbus_initialize() b.start() # __init__ + get chkp + delete chkp self.assertEqual(len(SR_mock.mock_calls), 6) SR_mock.reset_mock() SR_mock.return_value.zcard.return_value = 1 # unicast b.filtered_update( 'a', indicator='www.example.com/admin.php', value={ 'type': 'URL', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': break else: self.fail(msg='hset not found') self.assertEqual(args[2].startswith('lz4'), True) stixdict = json.loads(lz4.decompress(args[2][3:])) indicator = stixdict['indicators'][0] cyboxprops = indicator['observable']['object']['properties'] self.assertEqual(cyboxprops['type'], 'URL') self.assertEqual(cyboxprops['value'], 'www.example.com/admin.php') SR_mock.reset_mock() b.stop() @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_datafeed_unicode_url(self, glet_mock, SR_mock): config = {} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config) inputs = ['a'] output = False b.connect(inputs, output) b.mgmtbus_initialize() b.start() # __init__ + get chkp + delete chkp self.assertEqual(len(SR_mock.mock_calls), 6) SR_mock.reset_mock() SR_mock.return_value.zcard.return_value = 1 # unicast b.filtered_update( 'a', indicator=u'☃.net/påth', value={ 'type': 'URL', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': break else: self.fail(msg='hset not found') self.assertEqual(args[2].startswith('lz4'), True) stixdict = json.loads(lz4.decompress(args[2][3:])) indicator = stixdict['indicators'][0] cyboxprops = indicator['observable']['object']['properties'] self.assertEqual(cyboxprops['type'], 'URL') self.assertEqual(cyboxprops['value'], u'\u2603.net/p\xe5th') SR_mock.reset_mock() b.stop() @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_datafeed_overflow(self, glet_mock, SR_mock): config = {} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config) inputs = ['a'] output = False b.connect(inputs, output) b.mgmtbus_initialize() b.start() # __init__ + get chkp + delete chkp self.assertEqual(len(SR_mock.mock_calls), 6) SR_mock.reset_mock() SR_mock.return_value.zcard.return_value = b.max_entries # unicast b.filtered_update( 'a', indicator=u'☃.net/påth', value={ 'type': 'URL', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': self.fail(msg='hset found') self.assertEqual(b.statistics['drop.overflow'], 1) SR_mock.reset_mock() SR_mock.return_value.zcard.return_value = b.max_entries - 1 # unicast b.filtered_update( 'a', indicator=u'☃.net/påth', value={ 'type': 'URL', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': break else: self.fail(msg='hset not found') self.assertEqual(args[2].startswith('lz4'), True) stixdict = json.loads(lz4.decompress(args[2][3:])) indicator = stixdict['indicators'][0] cyboxprops = indicator['observable']['object']['properties'] self.assertEqual(cyboxprops['type'], 'URL') self.assertEqual(cyboxprops['value'], u'\u2603.net/p\xe5th') b.stop() @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_datafeed_update_hash(self, glet_mock, SR_mock): config = {} chassis = mock.Mock() chassis.request_sub_channel.return_value = None ochannel = mock.Mock() chassis.request_pub_channel.return_value = ochannel chassis.request_rpc_channel.return_value = None rpcmock = mock.Mock() rpcmock.get.return_value = {'error': None, 'result': 'OK'} chassis.send_rpc.return_value = rpcmock b = minemeld.ft.taxii.DataFeed(FTNAME, chassis, config) inputs = ['a'] output = False b.connect(inputs, output) b.mgmtbus_initialize() b.start() # __init__ + get chkp + delete chkp self.assertEqual(len(SR_mock.mock_calls), 6) SR_mock.reset_mock() SR_mock.return_value.zcard.return_value = 1 # sha1 b.filtered_update( 'a', indicator='a6a5418b4d67d9f3a33cbf184b25ac7f9fa87d33', value={ 'type': 'sha1', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': break else: self.fail(msg='hset not found') self.assertEqual(args[2].startswith('lz4'), True) stixdict = json.loads(lz4.decompress(args[2][3:])) indicator = stixdict['indicators'][0] cyboxprops = indicator['observable']['object']['properties'] self.assertEqual(cyboxprops['hashes'][0]['simple_hash_value'], 'a6a5418b4d67d9f3a33cbf184b25ac7f9fa87d33') self.assertEqual(cyboxprops['hashes'][0]['type']['value'], 'SHA1') SR_mock.reset_mock() # md5 b.filtered_update( 'a', indicator='e23fadd6ceef8c618fc1c65191d846fa', value={ 'type': 'md5', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': break else: self.fail(msg='hset not found') self.assertEqual(args[2].startswith('lz4'), True) stixdict = json.loads(lz4.decompress(args[2][3:])) indicator = stixdict['indicators'][0] cyboxprops = indicator['observable']['object']['properties'] self.assertEqual(cyboxprops['hashes'][0]['simple_hash_value'], 'e23fadd6ceef8c618fc1c65191d846fa') self.assertEqual(cyboxprops['hashes'][0]['type']['value'], 'MD5') SR_mock.reset_mock() # sha256 b.filtered_update( 'a', indicator='a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9', value={ 'type': 'sha256', 'confidence': 100, 'share_level': 'green', 'sources': ['test.1'] } ) for call in SR_mock.mock_calls: name, args, kwargs = call if name == '().pipeline().__enter__().hset': break else: self.fail(msg='hset not found') self.assertEqual(args[2].startswith('lz4'), True) stixdict = json.loads(lz4.decompress(args[2][3:])) indicator = stixdict['indicators'][0] cyboxprops = indicator['observable']['object']['properties'] self.assertEqual(cyboxprops['hashes'][0]['simple_hash_value'], 'a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9') self.assertEqual(cyboxprops['hashes'][0]['type']['value'], 'SHA256') SR_mock.reset_mock() b.stop() ================================================ FILE: tests/test_ft_taxii_stix_package_IPv4_1_3.xml ================================================ mmtest1 watchlist that contains IP information. Indicators - Watchlist IP Watchlist Sample IP Address Indicator for this watchlist. This contains one indicator with a set of three IP addresses in the watchlist. 10.0.0.0 ================================================ FILE: tests/test_ft_taxii_stix_package_IPv4_3_1.xml ================================================ mmtest1 watchlist that contains IP information. Indicators - Watchlist IP Watchlist Sample IP Address Indicator for this watchlist. This contains one indicator with a set of three IP addresses in the watchlist. 10.0.0.0##comma##10.0.0.1##comma##10.0.0.2 ================================================ FILE: tests/test_ft_taxii_stix_package_IPv4_3_2.xml ================================================ mmtest1 watchlist that contains IP information. Indicators - Watchlist IP Watchlist Sample IP Address Indicator for this watchlist. This contains one indicator with a set of three IP addresses in the watchlist. 10.0.0.0##comma##10.0.0.1##comma##10.0.0.2 ================================================ FILE: tests/test_ft_taxii_stix_package_IPv6_3_1.xml ================================================ mmtest1 watchlist that contains IP information. Indicators - Watchlist IP Watchlist Sample IP Address Indicator for this watchlist. This contains one indicator with a set of three IP addresses in the watchlist. 2607:f8b0:4004:803::1015##comma##2607:f8b0:4004:803::1016##comma##2607:f8b0:4004:803::1017 ================================================ FILE: tests/test_ft_taxii_stix_package_IPv6_3_2.xml ================================================ mmtest1 watchlist that contains IP information. Indicators - Watchlist IP Watchlist Sample IP Address Indicator for this watchlist. This contains one indicator with a set of three IP addresses in the watchlist. 2607:f8b0:4004:803::1015##comma##2607:f8b0:4004:803::1016##comma##2607:f8b0:4004:803::1017 ================================================ FILE: tests/test_localdb.yml ================================================ - indicator: 1.1.1.1 type: IPv4 - indicator: 1.1.1.2 type: IPv4 ================================================ FILE: tests/test_localdb2.yml ================================================ - indicator: 1.1.1.2 type: IPv4 - indicator: 1.1.1.3 type: IPv4 ================================================ FILE: tests/test_run_config.py ================================================ # Copyright 2015 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT run config tests Unit tests for minemeld.run.config """ import unittest import mock import os import os.path import minemeld.run.config MYDIR = os.path.dirname(__file__) os.environ['MINEMELD_PROTOTYPE_PATH'] = MYDIR class MineMeldRunConfigTests(unittest.TestCase): def test_defaults_from_file(self): emptypath = os.path.join(MYDIR, 'empty.yml') config = minemeld.run.config.load_config(emptypath) self.assertEqual(config.fabric['class'], 'AMQP') self.assertEqual(config.fabric['config'], {'num_connections': 5}) self.assertEqual(config.mgmtbus['transport']['class'], 'AMQP') self.assertEqual(config.mgmtbus['transport']['config'], {'num_connections': 1}) self.assertEqual(config.mgmtbus['master'], {}) self.assertEqual(config.mgmtbus['slave'], {}) def test_prototype_1(self): protopath = os.path.join(MYDIR, 'test-prototype-1.yml') config = minemeld.run.config.load_config(protopath) self.assertEqual(config.nodes['testprototype']['class'], 'minemeld.ft.http.HttpFT') self.assertEqual( config.nodes['testprototype']['config'], {'useless1': 1} ) def test_validate_config_1(self): config = { 'nodes': { 'n1': { 'inputs': ['n2', 'n3'] }, 'n2': { 'output': True } } } msgs = minemeld.run.config.validate_config( minemeld.run.config.MineMeldConfig.from_dict(config) ) self.assertEqual(len(msgs), 3) def test_validate_config_2(self): config = { 'nodes': { 'n1': { 'inputs': ['n2'] }, 'n2': { 'output': False } } } msgs = minemeld.run.config.validate_config( minemeld.run.config.MineMeldConfig.from_dict(config) ) self.assertEqual(len(msgs), 3) def test_validate_config_3(self): config = { 'nodes': { 'n1': { 'output': True, 'inputs': ['n2', 'n4'] }, 'n2': { 'output': True }, 'n3': { 'output': True, 'inputs': ['n1'] }, 'n4': { 'output': True, 'inputs': ['n3'] } } } msgs = minemeld.run.config.validate_config( minemeld.run.config.MineMeldConfig.from_dict(config) ) self.assertEqual(len(msgs), 5) def test_validate_config_4(self): config = { 'nodes': { '../config/running-config.yml': { 'output': True }, '': { 'output': True } } } msgs = minemeld.run.config.validate_config( minemeld.run.config.MineMeldConfig.from_dict(config) ) self.assertEqual(len(msgs), 4) ================================================ FILE: tests/test_startupplanner.py ================================================ # Copyright 2015-2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FT run config tests Unit tests for minemeld.run.config """ import unittest import mock import os import os.path from minemeld.run.config import CHANGE_ADDED, CHANGE_DELETED, CHANGE_INPUT_ADDED, CHANGE_INPUT_DELETED from minemeld.run.config import MineMeldConfig, MineMeldConfigChange import minemeld.startupplanner class MineMeldStartupPlanner(unittest.TestCase): def test_subgraphs_1(self): nodes = { 'm1': { 'inputs': [] }, 'm2': {}, 'm3': {}, 'p1': { 'inputs': ['m1', 'm2'] }, 'p2': { 'inputs': ['m3'] }, 'o1': { 'inputs': ['m1'] }, 'o2': { 'inputs': ['p1'] }, 'o3': { 'inputs': ['p2'] } } state_info = { 'm1': { 'checkpoint': None, 'is_source': True }, 'm2': { 'checkpoint': None, 'is_source': True }, 'm3': { 'checkpoint': 'a', 'is_source': True }, 'p1': { 'checkpoint': None, 'is_source': False }, 'p2': { 'checkpoint': 'a', 'is_source': False }, 'o1': { 'checkpoint': None, 'is_source': False }, 'o2': { 'checkpoint': None, 'is_source': False }, 'o3': { 'checkpoint': 'a', 'is_source': False } } config = MineMeldConfig.from_dict(dict(nodes=nodes)) config.compute_changes(config) self.assertEqual(len(config.changes), 0) plan = minemeld.startupplanner.plan(config, state_info) self.assertEqual(plan['m1'], 'reset') self.assertEqual(plan['m2'], 'reset') self.assertEqual(plan['p1'], 'reset') self.assertEqual(plan['o1'], 'reset') self.assertEqual(plan['o2'], 'reset') self.assertEqual(plan['m3'], 'initialize') self.assertEqual(plan['p2'], 'initialize') self.assertEqual(plan['o3'], 'initialize') def test_new_miner(self): nodes_before = { 'm1': { 'inputs': [] }, 'p1': { 'inputs': ['m1'] }, 'o1': { 'inputs': ['p1'] }, } nodes_after = { 'm1': { 'inputs': [] }, 'm2': {}, 'p1': { 'inputs': ['m1', 'm2'] }, 'o1': { 'inputs': ['p1'] }, } state_info = { 'm1': { 'checkpoint': 'a', 'is_source': True }, 'm2': { 'checkpoint': None, 'is_source': True }, 'p1': { 'checkpoint': 'a', 'is_source': False }, 'o1': { 'checkpoint': 'a', 'is_source': False }, } config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after)) config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before)) config_after.compute_changes(config_before) self.assertEqual(len(config_after.changes), 2) plan = minemeld.startupplanner.plan(config_after, state_info) self.assertEqual(plan['m1'], 'initialize') self.assertEqual(plan['m2'], 'reset') self.assertEqual(plan['p1'], 'initialize') self.assertEqual(plan['o1'], 'initialize') def test_removed_output(self): nodes_before = { 'm1': { 'inputs': [] }, 'p1': { 'inputs': ['m1'] }, 'o1': { 'inputs': ['p1'] }, 'o2': { 'inputs': ['p1'] }, } nodes_after = { 'm1': { 'inputs': [] }, 'p1': { 'inputs': ['m1'] }, 'o1': { 'inputs': ['p1'] }, } state_info = { 'm1': { 'checkpoint': 'a', 'is_source': True }, 'm2': { 'checkpoint': 'a', 'is_source': True }, 'p1': { 'checkpoint': 'a', 'is_source': False }, 'o1': { 'checkpoint': 'a', 'is_source': False }, } config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after)) config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before)) config_after.compute_changes(config_before) self.assertEqual(len(config_after.changes), 1) plan = minemeld.startupplanner.plan(config_after, state_info) self.assertEqual(plan['m1'], 'initialize') self.assertEqual(plan['p1'], 'initialize') self.assertEqual(plan['o1'], 'initialize') def test_added_chain(self): nodes_after = { 'm1': { 'inputs': [] }, 'm2': { 'inputs': [] }, 'p1': { 'inputs': ['m1', 'm2'] }, 'p2': { 'inputs': ['m1'] }, 'o1': { 'inputs': ['p1'] }, 'o2': { 'inputs': ['p2'] } } nodes_before = { 'm1': { 'inputs': [] }, 'p1': { 'inputs': ['m1'] }, 'o1': { 'inputs': ['p1'] } } state_info = { 'm1': { 'checkpoint': 'a', 'is_source': True }, 'm2': { 'checkpoint': None, 'is_source': True }, 'p1': { 'checkpoint': 'a', 'is_source': False }, 'p2': { 'checkpoint': None, 'is_source': False }, 'o1': { 'checkpoint': 'a', 'is_source': False }, 'o2': { 'checkpoint': None, 'is_source': False }, } config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after)) config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before)) config_after.compute_changes(config_before) self.assertEqual(len(config_after.changes), 4) plan = minemeld.startupplanner.plan(config_after, state_info) self.assertEqual(plan['m1'], 'rebuild') self.assertEqual(plan['m2'], 'reset') self.assertEqual(plan['p1'], 'reset') self.assertEqual(plan['p2'], 'reset') self.assertEqual(plan['o1'], 'reset') self.assertEqual(plan['o2'], 'reset') def test_invalid_chain_1(self): nodes_after = { 'm1': { 'inputs': [] }, 'm2': { 'inputs': [] }, 'm3': { 'inputs': [] }, 'p1': { 'inputs': ['m1', 'm2', 'm3'] }, 'p2': { 'inputs': ['m1'] }, 'o1': { 'inputs': ['p1'] }, 'o2': { 'inputs': ['p2'] } } nodes_before = { 'm1': { 'inputs': [] }, 'm2': { 'inputs': [] }, 'm3': { 'inputs': [] }, 'p1': { 'inputs': ['m1', 'm2', 'm3'] }, 'o1': { 'inputs': ['p1'] } } state_info = { 'm1': { 'checkpoint': 'a', 'is_source': True }, 'm2': { 'checkpoint': 'b', 'is_source': True }, 'm3': { 'checkpoint': 'b', 'is_source': True }, 'p1': { 'checkpoint': 'a', 'is_source': False }, 'p2': { 'checkpoint': None, 'is_source': False }, 'o1': { 'checkpoint': 'a', 'is_source': False }, 'o2': { 'checkpoint': None, 'is_source': False }, } config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after)) config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before)) config_after.compute_changes(config_before) self.assertEqual(len(config_after.changes), 2) plan = minemeld.startupplanner.plan(config_after, state_info) self.assertEqual(plan['m1'], 'reset') self.assertEqual(plan['m2'], 'rebuild') self.assertEqual(plan['m3'], 'rebuild') self.assertEqual(plan['p1'], 'reset') self.assertEqual(plan['p2'], 'reset') self.assertEqual(plan['o1'], 'reset') self.assertEqual(plan['o2'], 'reset') def test_invalid_node_1(self): nodes_before = { 'm1': { 'inputs': [] }, 'p1': { 'inputs': ['m1'] }, 'o1': { 'inputs': ['p1'] }, } nodes_after = { 'm1': { 'inputs': [] }, 'p1': { 'inputs': ['m1'] }, 'o1': { 'inputs': ['p1'] }, } state_info = { 'm1': { 'checkpoint': 'a', 'is_source': True }, 'p1': { 'checkpoint': 'b', 'is_source': False }, 'o1': { 'checkpoint': 'a', 'is_source': False }, } config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after)) config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before)) config_after.compute_changes(config_before) self.assertEqual(len(config_after.changes), 0) plan = minemeld.startupplanner.plan(config_after, state_info) self.assertEqual(plan['m1'], 'rebuild') self.assertEqual(plan['p1'], 'reset') self.assertEqual(plan['o1'], 'reset') def test_invalid_node_2(self): nodes_before = { 'm1': { 'inputs': [] }, 'p1': { 'inputs': ['m1'] }, 'o1': { 'inputs': ['p1'] }, } nodes_after = { 'm1': { 'inputs': [] }, 'p1': { 'inputs': ['m1'] }, 'o1': { 'inputs': ['p1'] }, } state_info = { 'm1': { 'checkpoint': 'a', 'is_source': True }, 'p1': { 'checkpoint': None, 'is_source': False }, 'o1': { 'checkpoint': 'a', 'is_source': False }, } config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after)) config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before)) config_after.compute_changes(config_before) self.assertEqual(len(config_after.changes), 0) plan = minemeld.startupplanner.plan(config_after, state_info) self.assertEqual(plan['m1'], 'rebuild') self.assertEqual(plan['p1'], 'reset') self.assertEqual(plan['o1'], 'reset') def test_invalid_node_3(self): nodes_before = { 'm1': { 'inputs': [] }, 'm2': { 'inputs': [] }, 'p1': { 'inputs': ['m1', 'm2'] }, 'p2': { 'inputs': ['m1', 'm2'] }, 'o1': { 'inputs': ['p1'] }, } nodes_after = { 'm1': { 'inputs': [] }, 'm2': { 'inputs': [] }, 'p1': { 'inputs': ['m1', 'm2'] }, 'p2': { 'inputs': ['m2'] }, 'o1': { 'inputs': ['p1'] }, } state_info = { 'm1': { 'checkpoint': 'a', 'is_source': True }, 'm2': { 'checkpoint': 'a', 'is_source': True }, 'p1': { 'checkpoint': 'a', 'is_source': False }, 'p2': { 'checkpoint': 'a', 'is_source': False }, 'o1': { 'checkpoint': 'a', 'is_source': False }, } config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after)) config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before)) config_after.compute_changes(config_before) self.assertEqual(len(config_after.changes), 1) plan = minemeld.startupplanner.plan(config_after, state_info) self.assertEqual(plan['m1'], 'rebuild') self.assertEqual(plan['m2'], 'rebuild') self.assertEqual(plan['p1'], 'reset') self.assertEqual(plan['p2'], 'reset') self.assertEqual(plan['o1'], 'reset') def test_existing_source_added(self): nodes_before = { 'm1': { 'inputs': [] }, 'm2': { 'inputs': [] }, 'p1': { 'inputs': ['m1'] }, 'o1': { 'inputs': ['p1'] }, } nodes_after = { 'm1': { 'inputs': [] }, 'm2': { 'inputs': [] }, 'p1': { 'inputs': ['m1', 'm2'] }, 'o1': { 'inputs': ['p1'] }, } state_info = { 'm1': { 'checkpoint': 'a', 'is_source': True }, 'm2': { 'checkpoint': 'a', 'is_source': True }, 'p1': { 'checkpoint': 'a', 'is_source': False }, 'o1': { 'checkpoint': 'a', 'is_source': False }, } config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after)) config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before)) config_after.compute_changes(config_before) self.assertEqual(len(config_after.changes), 1) plan = minemeld.startupplanner.plan(config_after, state_info) self.assertEqual(plan['m1'], 'initialize') self.assertEqual(plan['m2'], 'rebuild') self.assertEqual(plan['p1'], 'initialize') self.assertEqual(plan['o1'], 'initialize') def test_non_existing_source_added(self): nodes_before = { 'm1': { 'inputs': [] }, 'p1': { 'inputs': ['m1'] }, 'o1': { 'inputs': ['p1'] }, } nodes_after = { 'm1': { 'inputs': [] }, 'm2': { 'inputs': [] }, 'p1': { 'inputs': ['m1', 'm2'] }, 'o1': { 'inputs': ['p1'] }, } state_info = { 'm1': { 'checkpoint': 'a', 'is_source': True }, 'm2': { 'checkpoint': None, 'is_source': True }, 'p1': { 'checkpoint': 'a', 'is_source': False }, 'o1': { 'checkpoint': 'a', 'is_source': False }, } config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after)) config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before)) config_after.compute_changes(config_before) self.assertEqual(len(config_after.changes), 2) plan = minemeld.startupplanner.plan(config_after, state_info) self.assertEqual(plan['m1'], 'initialize') self.assertEqual(plan['m2'], 'reset') self.assertEqual(plan['p1'], 'initialize') self.assertEqual(plan['o1'], 'initialize') def test_non_source_existing_input_added(self): nodes_before = { 'm1': { 'inputs': [] }, 'm2': { 'inputs': [] }, 'p1': { 'inputs': ['m1'] }, 'p2': { 'inputs': ['m2'] }, 'o1': { 'inputs': ['p1'] }, } nodes_after = { 'm1': { 'inputs': [] }, 'm2': { 'inputs': [] }, 'p1': { 'inputs': ['m1', 'p2'] }, 'p2': { 'inputs': ['m2'] }, 'o1': { 'inputs': ['p1'] }, } state_info = { 'm1': { 'checkpoint': 'a', 'is_source': True }, 'm2': { 'checkpoint': 'a', 'is_source': True }, 'p1': { 'checkpoint': 'a', 'is_source': False }, 'p2': { 'checkpoint': 'a', 'is_source': False }, 'o1': { 'checkpoint': 'a', 'is_source': False }, } config_after = MineMeldConfig.from_dict(dict(nodes=nodes_after)) config_before = MineMeldConfig.from_dict(dict(nodes=nodes_before)) config_after.compute_changes(config_before) self.assertEqual(len(config_after.changes), 1) plan = minemeld.startupplanner.plan(config_after, state_info) self.assertEqual(plan['m1'], 'rebuild') self.assertEqual(plan['m2'], 'rebuild') self.assertEqual(plan['p1'], 'reset') self.assertEqual(plan['p2'], 'reset') self.assertEqual(plan['o1'], 'reset') ================================================ FILE: tests/test_traced_queryprocessor.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Unit tests for minemeld.traced.queryprocessor """ import gevent.monkey gevent.monkey.patch_all(thread=False, select=False) import redis import gevent import greenlet import unittest import tempfile import shutil import random import time import mock import json import ujson import logging import guppy import gc import minemeld.traced.queryprocessor import traced_mock import comm_mock TABLENAME = tempfile.mktemp(prefix='minemeld.traced.storagetest') LOG = logging.getLogger(__name__) class MineMeldTracedStorage(unittest.TestCase): def setUp(self): traced_mock.table_cleanup() traced_mock.query_cleanup() def tearDown(self): traced_mock.table_cleanup() traced_mock.query_cleanup() @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_query_1(self, glet_mock, SR_mock): store = traced_mock.store_factory() q = minemeld.traced.queryprocessor.Query( store, "log", 0, 0, 100, 'uuid-test', {} ) q._run() self.assertGreater(len(SR_mock.mock_calls), 1) num_logs = 0 eoq = False for call in SR_mock.mock_calls[1:]: name, args, kwargs = call self.assertEqual(name, '().publish') self.assertEqual(args[0], 'mm-traced-q.uuid-test') if args[1] == '': eoq = True else: line = json.loads(args[1]) if 'log' in line: num_logs += 1 self.assertEqual(num_logs, 0) self.assertEqual(eoq, True) @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_query_pos(self, glet_mock, SR_mock): store = traced_mock.store_factory() store.write(1*86400*1000, 'log0') q = minemeld.traced.queryprocessor.Query( store, "log", 3*86400*1000, 0, 100, 'uuid-test', {} ) q._run() self.assertGreater(len(SR_mock.mock_calls), 1) num_logs = 0 eoq = False for call in SR_mock.mock_calls[1:]: name, args, kwargs = call self.assertEqual(name, '().publish') self.assertEqual(args[0], 'mm-traced-q.uuid-test') if args[1] == '': eoq = True else: line = json.loads(args[1]) if 'log' in line: num_logs += 1 self.assertEqual(num_logs, 1) self.assertEqual(eoq, True) @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_query_and(self, glet_mock, SR_mock): store = traced_mock.store_factory() store.write(1*86400*1000, 'log0') store.write(1*86400*1000, 'log1') store.write(2*86400*1000, 'pog1') q = minemeld.traced.queryprocessor.Query( store, "log -0", 3*86400*1000, 0, 100, 'uuid-test', {} ) q._run() LOG.debug(SR_mock.mock_calls) self.assertGreater(len(SR_mock.mock_calls), 1) num_logs = 0 eoq = False for call in SR_mock.mock_calls[1:]: name, args, kwargs = call self.assertEqual(name, '().publish') self.assertEqual(args[0], 'mm-traced-q.uuid-test') if args[1] == '': eoq = True else: line = json.loads(args[1]) if 'log' in line: num_logs += 1 self.assertEqual(num_logs, 1) self.assertEqual(eoq, True) @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_query_fs1(self, glet_mock, SR_mock): store = traced_mock.store_factory() store.write(1*86400*1000, ujson.dumps({ "field1": "foo", "field2": "bar", "field3": ["foo", "bar"], "field4": 12345678 })) store.write(1*86400*1000, ujson.dumps({ "field1": "bar", "field2": "foo", "field3": ["foo", "bar"], "field4": 12345679 })) q = minemeld.traced.queryprocessor.Query( store, "field1:foo", 3*86400*1000, 0, 100, 'uuid-test', {} ) q._run() LOG.debug(SR_mock.mock_calls) self.assertGreater(len(SR_mock.mock_calls), 1) num_logs = 0 eoq = False for call in SR_mock.mock_calls[1:]: name, args, kwargs = call self.assertEqual(name, '().publish') self.assertEqual(args[0], 'mm-traced-q.uuid-test') if args[1] == '': eoq = True else: line = json.loads(args[1]) if 'log' in line: num_logs += 1 self.assertEqual(num_logs, 1) self.assertEqual(eoq, True) @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_query_fs2(self, glet_mock, SR_mock): store = traced_mock.store_factory() store.write(1*86400*1000, ujson.dumps({ "field1": "foo", "field2": "bar", "field3": ["foo", "bar"], "field4": 12345678 })) store.write(1*86400*1000, ujson.dumps({ "field1": "bar", "field2": "foo", "field3": ["foo", "bar"], "field4": 12345679 })) q = minemeld.traced.queryprocessor.Query( store, "field1:foo field2:foo", 3*86400*1000, 0, 100, 'uuid-test', {} ) q._run() LOG.debug(SR_mock.mock_calls) self.assertGreater(len(SR_mock.mock_calls), 1) num_logs = 0 eoq = False for call in SR_mock.mock_calls[1:]: name, args, kwargs = call self.assertEqual(name, '().publish') self.assertEqual(args[0], 'mm-traced-q.uuid-test') if args[1] == '': eoq = True else: line = json.loads(args[1]) if 'log' in line: num_logs += 1 self.assertEqual(num_logs, 0) self.assertEqual(eoq, True) @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_query_fs3(self, glet_mock, SR_mock): store = traced_mock.store_factory() store.write(1*86400*1000, ujson.dumps({ "field1": "foo", "field2": "bar", "field3": ["foo", "bar"], "field4": 12345678 })) store.write(1*86400*1000, ujson.dumps({ "field1": "bar", "field2": "foo", "field3": ["foo", "bar"], "field4": 12345679 })) q = minemeld.traced.queryprocessor.Query( store, "field3:bar FIELD2:foo", 3*86400*1000, 0, 100, 'uuid-test', {} ) q._run() LOG.debug(SR_mock.mock_calls) self.assertGreater(len(SR_mock.mock_calls), 1) num_logs = 0 eoq = False for call in SR_mock.mock_calls[1:]: name, args, kwargs = call self.assertEqual(name, '().publish') self.assertEqual(args[0], 'mm-traced-q.uuid-test') if args[1] == '': eoq = True else: line = json.loads(args[1]) if 'log' in line: num_logs += 1 self.assertEqual(num_logs, 1) self.assertEqual(eoq, True) @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_query_fs4(self, glet_mock, SR_mock): store = traced_mock.store_factory() store.write(1*86400*1000, ujson.dumps({ "field1": "foo", "field2": "bar", "field3": ["foo", "bar"], "field4": 12345678 })) store.write(1*86400*1000, ujson.dumps({ "field1": "bar", "field2": "foo", "field3": ["foo", "bar"], "field4": 12345679 })) q = minemeld.traced.queryprocessor.Query( store, "field3:foo FIELD3:Bar", 3*86400*1000, 0, 100, 'uuid-test', {} ) q._run() LOG.debug(SR_mock.mock_calls) self.assertGreater(len(SR_mock.mock_calls), 1) num_logs = 0 eoq = False for call in SR_mock.mock_calls[1:]: name, args, kwargs = call self.assertEqual(name, '().publish') self.assertEqual(args[0], 'mm-traced-q.uuid-test') if args[1] == '': eoq = True else: line = json.loads(args[1]) if 'log' in line: num_logs += 1 self.assertEqual(num_logs, 2) self.assertEqual(eoq, True) @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_query_fs5(self, glet_mock, SR_mock): store = traced_mock.store_factory() store.write(1*86400*1000, ujson.dumps({ "field1": "1foo1", "field2": "2bar2", "field3": ["5foo5", "6bar6"], "field4": 12345678 })) store.write(1*86400*1000, ujson.dumps({ "field1": "3bar3", "field2": "4foo4", "field3": ["8foo8", "7bar7"], "field4": 12345679 })) q = minemeld.traced.queryprocessor.Query( store, "field3:foo -field4:679", 3*86400*1000, 0, 100, 'uuid-test', {} ) q._run() LOG.debug(SR_mock.mock_calls) self.assertGreater(len(SR_mock.mock_calls), 1) num_logs = 0 eoq = False for call in SR_mock.mock_calls[1:]: name, args, kwargs = call self.assertEqual(name, '().publish') self.assertEqual(args[0], 'mm-traced-q.uuid-test') if args[1] == '': eoq = True else: line = json.loads(args[1]) if 'log' in line: num_logs += 1 self.assertEqual(num_logs, 1) self.assertEqual(eoq, True) @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_query_wcard(self, glet_mock, SR_mock): store = traced_mock.store_factory() store.write(1*86400*1000, ujson.dumps({ "field1": "1*oo1", "field2": "2*ar2", "field3": ["5foo5", "6bar6"], "field4": 12345678 })) store.write(1*86400*1000, ujson.dumps({ "field1": "3bar3", "field2": "4foo4", "field3": ["8*oo8", "7bar7"], "field4": 12345678 })) q = minemeld.traced.queryprocessor.Query( store, "field3:*oo -field4:67?", 3*86400*1000, 0, 100, 'uuid-test', {} ) q._run() LOG.debug(SR_mock.mock_calls) self.assertGreater(len(SR_mock.mock_calls), 1) num_logs = 0 eoq = False for call in SR_mock.mock_calls[1:]: name, args, kwargs = call self.assertEqual(name, '().publish') self.assertEqual(args[0], 'mm-traced-q.uuid-test') if args[1] == '': eoq = True else: line = json.loads(args[1]) if 'log' in line: num_logs += 1 self.assertEqual(num_logs, 1) self.assertEqual(eoq, True) @mock.patch.object(redis, 'StrictRedis') @mock.patch.object(gevent, 'Greenlet') def test_query_empty(self, glet_mock, SR_mock): store = traced_mock.store_factory() q = minemeld.traced.queryprocessor.Query( store, "field3:foo -field4:679", 3*86400*1000, 0, 100, 'uuid-test', {} ) q._run() num_logs = 0 eoq = False for call in SR_mock.mock_calls[1:]: name, args, kwargs = call self.assertEqual(name, '().publish') self.assertEqual(args[0], 'mm-traced-q.uuid-test') if args[1] == '': eoq = True else: line = json.loads(args[1]) if 'log' in line: num_logs += 1 self.assertEqual(num_logs, 0) self.assertEqual(eoq, True) @mock.patch.object(minemeld.traced.queryprocessor, 'Query', side_effect=traced_mock.query_factory) def test_queryprocessor_1(self, query_mock): comm = comm_mock.comm_factory({}) store = traced_mock.store_factory() qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store) self.assertEqual( comm.rpc_server_channels[0]['name'], minemeld.traced.queryprocessor.QUERY_QUEUE ) self.assertEqual( comm.rpc_server_channels[0]['allowed_methods'], ['query', 'kill_query'] ) qp.query('uuid-test-1', "test query") self.assertEqual(len(traced_mock.MOCK_QUERIES), 1) qp.query('uuid-test-2', "test query") self.assertEqual(len(traced_mock.MOCK_QUERIES), 2) gevent.sleep(0) traced_mock.MOCK_QUERIES[0].finish_event.set() gevent.sleep(0) qp.stop() gevent.sleep(0) gevent.sleep(0) self.assertEqual(traced_mock.MOCK_QUERIES[0].get(), None) self.assertIsInstance( traced_mock.MOCK_QUERIES[1].get(), greenlet.GreenletExit ) self.assertNotIn('uuid-test-1', store.release_alls) self.assertIn('uuid-test-2', store.release_alls) @mock.patch.object(minemeld.traced.queryprocessor, 'Query', side_effect=traced_mock.query_factory) def test_queryprocessor_2(self, query_mock): comm = comm_mock.comm_factory({}) store = traced_mock.store_factory() qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store) self.assertEqual( comm.rpc_server_channels[0]['name'], minemeld.traced.queryprocessor.QUERY_QUEUE ) self.assertEqual( comm.rpc_server_channels[0]['allowed_methods'], ['query', 'kill_query'] ) qp.query('uuid-test-1', "test query") self.assertEqual(len(traced_mock.MOCK_QUERIES), 1) qp.query('uuid-test-2', "bad") self.assertEqual(len(traced_mock.MOCK_QUERIES), 2) gevent.sleep(0) traced_mock.MOCK_QUERIES[0].finish_event.set() traced_mock.MOCK_QUERIES[1].finish_event.set() gevent.sleep(0) qp.stop() gevent.sleep(0) gevent.sleep(0) self.assertEqual(traced_mock.MOCK_QUERIES[0].get(), None) self.assertRaises( RuntimeError, traced_mock.MOCK_QUERIES[1].get ) self.assertNotIn('uuid-test-1', store.release_alls) self.assertIn('uuid-test-2', store.release_alls) @mock.patch.object(minemeld.traced.queryprocessor, 'Query', side_effect=traced_mock.query_factory) def test_queryprocessor_3(self, query_mock): comm = comm_mock.comm_factory({}) store = traced_mock.store_factory() qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store) self.assertEqual( comm.rpc_server_channels[0]['name'], minemeld.traced.queryprocessor.QUERY_QUEUE ) self.assertEqual( comm.rpc_server_channels[0]['allowed_methods'], ['query', 'kill_query'] ) qp.query('uuid-test-1', "test query") gevent.sleep(0) qp.kill_query('uuid-test-1') gevent.sleep(0) self.assertIsInstance( traced_mock.MOCK_QUERIES[0].get(), greenlet.GreenletExit ) self.assertEqual( len(qp.queries), 0 ) self.assertIn('uuid-test-1', store.release_alls) qp.stop() gevent.sleep(0) @mock.patch.object(minemeld.traced.queryprocessor, 'Query', side_effect=traced_mock.query_factory) def test_queryprocessor_4(self, query_mock): comm = comm_mock.comm_factory({}) store = traced_mock.store_factory() qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store) self.assertEqual( comm.rpc_server_channels[0]['name'], minemeld.traced.queryprocessor.QUERY_QUEUE ) self.assertEqual( comm.rpc_server_channels[0]['allowed_methods'], ['query', 'kill_query'] ) qp.stop() gevent.sleep(0) self.assertRaises( RuntimeError, qp.query, 'test-uuid-1', "test" ) self.assertRaises( RuntimeError, qp.kill_query, 'test-uuid-1' ) @mock.patch.object(minemeld.traced.queryprocessor, 'Query', side_effect=traced_mock.query_factory) def test_queryprocessor_5(self, query_mock): comm = comm_mock.comm_factory({}) store = traced_mock.store_factory() qp = minemeld.traced.queryprocessor.QueryProcessor(comm, store, {'max_concurrency': 2}) qp.query('uuid-test-1', "test query") gevent.sleep(0) qp.query('uuid-test-2', "test query") gevent.sleep(0) self.assertRaises( RuntimeError, qp.query, 'uuid-test-3', "test query" ) gevent.sleep(0) traced_mock.MOCK_QUERIES[0].finish_event.set() gevent.sleep(0.2) self.assertEqual(len(qp.queries), 1) self.assertEqual( qp.query('uuid-test-4', "test query"), 'OK' ) gevent.sleep(0) qp.stop() gevent.sleep(0) ================================================ FILE: tests/test_traced_storage.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Unit tests for minemeld.traced.storage """ import unittest import tempfile import shutil import random import time import mock import logging from nose.plugins.attrib import attr import minemeld.traced.storage import traced_mock TABLENAME = tempfile.mktemp(prefix='minemeld.traced.storagetest') LOG = logging.getLogger(__name__) class MineMeldTracedStorage(unittest.TestCase): def setUp(self): traced_mock.table_cleanup() try: shutil.rmtree(TABLENAME) except: pass def tearDown(self): traced_mock.table_cleanup() try: shutil.rmtree(TABLENAME) except: pass def test_table_constructor(self): self.assertRaises( minemeld.traced.storage.TableNotFound, minemeld.traced.storage.Table, TABLENAME, create_if_missing=False ) table = minemeld.traced.storage.Table(TABLENAME, create_if_missing=True) self.assertEqual(table.max_counter, -1) table.close() table = None table = minemeld.traced.storage.Table(TABLENAME, create_if_missing=False) self.assertEqual(table.max_counter, -1) table.close() table = None def test_table_write(self): table = minemeld.traced.storage.Table(TABLENAME, create_if_missing=True) table.put('%016x' % 0, 'value0') self.assertEqual(table.max_counter, 0) table.close() table = None table = minemeld.traced.storage.Table(TABLENAME, create_if_missing=False) iterator = table.backwards_iterator(1, 0xFFFFFFFFFFFFFFFF) ts, line = next(iterator) self.assertEqual(line, 'value0') self.assertEqual(int(ts[:16], 16), 0) self.assertEqual(int(ts[16:], 16), 0) self.assertRaises(StopIteration, next, iterator) table.close() table = None def test_table_references(self): table = minemeld.traced.storage.Table(TABLENAME, create_if_missing=True) self.assertEqual(table.ref_count(), 0) table.add_reference('ref1') self.assertEqual(table.ref_count(), 1) table.add_reference('ref2') self.assertEqual(table.ref_count(), 2) table.remove_reference('ref1') self.assertEqual(table.ref_count(), 1) table.remove_reference('ref1') self.assertEqual(table.ref_count(), 1) table.remove_reference('ref2') self.assertEqual(table.ref_count(), 0) def test_table_oldest(self): old_ = '%016x' % (3*86400) new_ = '%016x' % (4*86400) oldest = minemeld.traced.storage.Table.oldest_table() self.assertEqual(oldest, None) table = minemeld.traced.storage.Table(old_, create_if_missing=True) table.close() table = minemeld.traced.storage.Table(new_, create_if_missing=True) table.close() oldest = minemeld.traced.storage.Table.oldest_table() self.assertEqual(oldest, old_) shutil.rmtree(old_) shutil.rmtree(new_) def test_store_simple(self): store = minemeld.traced.storage.Store() store.stop() self.assertEqual(len(store.current_tables), 0) @mock.patch.object(minemeld.traced.storage, 'Table', side_effect=traced_mock.table_factory) def test_store_write(self, table_mock): store = minemeld.traced.storage.Store() store.write(0*86400*1000, 'log0') self.assertEqual(traced_mock.MOCK_TABLES[0].name, '%016x' % 0) store.write(1*86400*1000, 'log1') self.assertEqual(traced_mock.MOCK_TABLES[1].name, '%016x' % (86400*1)) store.write(2*86400*1000, 'log2') self.assertEqual(traced_mock.MOCK_TABLES[2].name, '%016x' % (86400*2)) store.write(3*86400*1000, 'log3') self.assertEqual(traced_mock.MOCK_TABLES[3].name, '%016x' % (86400*3)) store.write(4*86400*1000, 'log4') self.assertEqual(traced_mock.MOCK_TABLES[4].name, '%016x' % (86400*4)) store.write(5*86400*1000, 'log5') self.assertEqual(traced_mock.MOCK_TABLES[5].name, '%016x' % (86400*5)) self.assertNotIn('%016x' % 0, store.current_tables) store.write(6*86400*1000, 'log6') self.assertEqual(traced_mock.MOCK_TABLES[6].name, '%016x' % (86400*6)) self.assertNotIn('%016x' % 86400, store.current_tables) store.stop() self.assertEqual(len(store.current_tables), 0) @mock.patch.object(minemeld.traced.storage, 'Table', side_effect=traced_mock.table_factory) def test_store_iterate_backwards(self, table_mock): _oldest_table_mock = mock.MagicMock(side_effect=traced_mock.MockTable.oldest_table) table_mock.attach_mock(_oldest_table_mock, 'oldest_table') store = minemeld.traced.storage.Store() store.write(1*86400*1000, 'log0') store.write(2*86400*1000, 'log1') store.write(3*86400*1000, 'log2') store.write(4*86400*1000, 'log3') store.write(5*86400*1000, 'log4') self.assertEqual(minemeld.traced.storage.Table.oldest_table(), '%016x' % 86400) iterator = store.iterate_backwards( ref='test-iter1', timestamp=6*86400*1000, counter=0xFFFFFFFFFFFFFFFF ) self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-07') self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-06') self.assertEqual(next(iterator)['log'], 'log4') self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-05') self.assertEqual(next(iterator)['log'], 'log3') self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-04') self.assertEqual(next(iterator)['log'], 'log2') self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-03') self.assertEqual(next(iterator)['log'], 'log1') self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-02') self.assertEqual(next(iterator)['log'], 'log0') self.assertEqual(next(iterator)['msg'], 'No more logs to check') self.assertRaises(StopIteration, next, iterator) store.stop() store.stop() # just for coverage @mock.patch.object(minemeld.traced.storage, 'Table', side_effect=traced_mock.table_factory) def test_store_iterate_backwards_2(self, table_mock): _oldest_table_mock = mock.MagicMock(side_effect=traced_mock.MockTable.oldest_table) table_mock.attach_mock(_oldest_table_mock, 'oldest_table') store = minemeld.traced.storage.Store() store.write(0*86400*1000, 'log0') store.write(2*86400*1000, 'log1') self.assertEqual(minemeld.traced.storage.Table.oldest_table(), '%016x' % 0) iterator = store.iterate_backwards( ref='test-iter1', timestamp=3*86400*1000, counter=0xFFFFFFFFFFFFFFFF ) self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-04') self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-03') self.assertEqual(next(iterator)['log'], 'log1') self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-02') self.assertEqual(next(iterator)['msg'], 'Checking 1970-01-01') self.assertEqual(next(iterator)['log'], 'log0') self.assertEqual(next(iterator)['msg'], 'We haved reached the origins of time') self.assertRaises(StopIteration, next, iterator) store.stop() @mock.patch.object(minemeld.traced.storage, 'Table', side_effect=traced_mock.table_factory) def test_store_iterate_backwards_empty(self, table_mock): _oldest_table_mock = mock.MagicMock(side_effect=traced_mock.MockTable.oldest_table) table_mock.attach_mock(_oldest_table_mock, 'oldest_table') store = minemeld.traced.storage.Store() self.assertEqual(minemeld.traced.storage.Table.oldest_table(), None) iterator = store.iterate_backwards( ref='test-iter1', timestamp=3*86400*1000, counter=0xFFFFFFFFFFFFFFFF ) self.assertEqual(next(iterator)['msg'], 'No more logs to check') self.assertRaises(StopIteration, next, iterator) table_mock.assert_not_called() store.stop() @attr('slow') def test_stress_1(self): num_lines = 200000 store = minemeld.traced.storage.Store() t1 = time.time() for j in xrange(num_lines): value = '{ "log": %d }' % random.randint(0, 0xFFFFFFFF) t2 = time.time() dt = t2-t1 t1 = time.time() for j in xrange(num_lines): value = '{ "log": %d }' % random.randint(0, 0xFFFFFFFF) store.write(j, value) t2 = time.time() print "TIME: Inserted %d lines in %d sec" % (num_lines, (t2-t1-dt)) store.stop() shutil.rmtree('1970-01-01') ================================================ FILE: tests/test_traced_writer.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Unit tests for minemeld.traced.writer """ import unittest import tempfile import shutil import random import time import mock import logging import ujson import minemeld.traced.writer import comm_mock import traced_mock LOG = logging.getLogger(__name__) class MineMeldTracedStorage(unittest.TestCase): def test_writer(self): config = {} comm = comm_mock.comm_factory(config) store = traced_mock.store_factory() writer = minemeld.traced.writer.Writer(comm, store, 'TESTTOPIC') self.assertEqual(comm.sub_channels[0]['topic'], 'TESTTOPIC') self.assertEqual(comm.sub_channels[0]['allowed_methods'], ['log']) writer.log(0, log='testlog') self.assertEqual(store.writes[0]['timestamp'], 0) self.assertEqual(store.writes[0]['log'], ujson.dumps({'log': 'testlog'})) writer.stop() writer.log(0, log='testlog') self.assertEqual(len(store.writes), 1) writer.stop() # just for coverage ================================================ FILE: tests/testproto.yml ================================================ prototypes: test: config: useless1: 1 class: minemeld.ft.http.HttpFT ================================================ FILE: tests/traced_mock.py ================================================ # Copyright 2016 Palo Alto Networks, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This module implements mock classes for minemed.traced tests """ import gevent import gevent.event import logging from minemeld.traced.storage import TableNotFound LOG = logging.getLogger(__name__) CLOCK = -1 def _get_clock(): global CLOCK CLOCK += 1 return CLOCK MOCK_TABLES = [] class MockTable(object): def __init__(self, name, create_if_missing=True): self.name = name self.create_if_missing = create_if_missing self.last_used = None self.refs = [] self.db_open = True self.db = {} self.max_counter = -1 def add_reference(self, refid): self.refs.append(refid) def remove_reference(self, refid): try: self.refs.remove(refid) except ValueError: pass def ref_count(self): return len(self.refs) def put(self, key, value): self.last_used = _get_clock() self.max_counter += 1 new_max_counter = '%016x' % self.max_counter self.db[key+new_max_counter] = value def backwards_iterator(self, timestamp, counter): starting_key = '%016x%016x' % (timestamp, counter) items = [[k, v] for k, v in self.db.iteritems() if k <= starting_key] items = sorted(items, cmp=lambda x, y: cmp(x[0], y[0]), reverse=True) return items def close(self): self.db_open = False @staticmethod def oldest_table(): tables = [t.name for t in MOCK_TABLES] LOG.debug(tables) if len(tables) == 0: return None return sorted(tables)[0] def table_factory(name, create_if_missing=True): table = next((t for t in MOCK_TABLES if t.name == name), None) if table is not None: return table if not create_if_missing: raise TableNotFound() mt = MockTable(name, create_if_missing=create_if_missing) MOCK_TABLES.append(mt) return mt def table_cleanup(): global MOCK_TABLES MOCK_TABLES = [] class MockStore(object): def __init__(self, config=None): if config is None: config = {} self.config = config self.writes = [] self.db = {} self.counter = 0 self.release_alls = [] def write(self, timestamp, log): self.writes.append({ 'timestamp': timestamp, 'log': log }) self.db['%016x%016x' % (timestamp, self.counter)] = log self.counter += 1 def iterate_backwards(self, ref, timestamp, counter): starting_key = '%016x%016x' % (timestamp, counter) items = [[k, v] for k, v in self.db.iteritems() if k <= starting_key] items = sorted(items, cmp=lambda x, y: cmp(x[0], y[0]), reverse=True) for c, i in enumerate(items): if c % 1 == 0: yield {'msg': 'test message'} yield {'timestamp': i[0], 'log': i[1]} def release_all(self, ref): self.release_alls.append(ref) def store_factory(config=None): return MockStore(config=config) MOCK_QUERIES = [] class MockQuery(gevent.Greenlet): def __init__(self, store, query, timestamp, counter, num_lines, uuid, redis_config): self.store = store self.query = query self.timestamp = timestamp self.counter = counter self.num_lines = num_lines self.uuid = uuid self.redis_config = redis_config self.finish_event = gevent.event.Event() super(MockQuery, self).__init__() def kill(self): LOG.debug("%s killed", self.uuid) super(MockQuery, self).kill() def _run(self): LOG.debug("%s started", self.uuid) self.finish_event.wait() LOG.debug("%s finished", self.uuid) class MockEQuery(gevent.Greenlet): def __init__(self, store, query, timestamp, counter, num_lines, uuid, redis_config): self.store = store self.query = query self.timestamp = timestamp self.counter = counter self.num_lines = num_lines self.uuid = uuid self.redis_config = redis_config self.finish_event = gevent.event.Event() super(MockEQuery, self).__init__() def kill(self): LOG.debug("%s killed", self.uuid) super(MockEQuery, self).kill() def _run(self): LOG.debug("%s started", self.uuid) self.finish_event.wait() raise RuntimeError("BAD BAD QUERY!") def query_factory(store, query, timestamp, counter, num_lines, uuid, redis_config): if query == "bad": mqf = MockEQuery else: mqf = MockQuery mq = mqf(store, query, timestamp, counter, num_lines, uuid, redis_config) MOCK_QUERIES.append(mq) return mq def query_cleanup(): global MOCK_QUERIES MOCK_QUERIES = [] ================================================ FILE: tests/traced_storage_profile.py ================================================ #!/usr/bin/env python import shutil import minemeld.traced.storage import random import time if __name__ == "__main__": num_lines = 200000 store = minemeld.traced.storage.Store() t1 = time.time() for j in xrange(num_lines): value = '{ "log": %d }' % random.randint(0, 0xFFFFFFFF) t2 = time.time() dt = t2-t1 t1 = time.time() for j in xrange(num_lines): value = '{ "log": %d }' % random.randint(0, 0xFFFFFFFF) store.write(j, value) t2 = time.time() print "TIME: Inserted %d lines in %d sec" % (num_lines, (t2-t1-dt)) store.stop() # shutil.rmtree('1970-01-01') ================================================ FILE: tests/wsgi.htpasswd ================================================ admin:$apr1$pNXvvCp5$c4VXxDzt9waMeAEFRH7p8. ================================================ FILE: tox.ini ================================================ [tox] envlist = py27, flake8 skipsdist = True [testenv:py27] basedeps = mock nose coverage guppy xmltodict changedir = {envtmpdir} setenv = PYTHONPATH = {toxinidir} deps = {[testenv:py27]basedeps} -r{toxinidir}/requirements.txt -r{toxinidir}/requirements-web.txt commands = nosetests -a '!slow' -s {posargs} [testenv:flake8] deps = flake8 commands = flake8 --ignore E402,E226 --max-line-length=100 [testenv:stress] basepython = python2.7 basedeps = mock nose guppy changedir = {envtmpdir} setenv = PYTHONPATH = {toxinidir} deps = {[testenv:py27]basedeps} -r{toxinidir}/requirements.txt -r{toxinidir}/requirements-web.txt commands = nosetests -s --logging-level=INFO -a 'slow' {posargs} [testenv:profile] basepython = python2.7 basedeps = mock nose guppy changedir = {envtmpdir} setenv = PYTHONPATH = {toxinidir} deps = {[testenv:py27]basedeps} -r{toxinidir}/requirements.txt -r{toxinidir}/requirements-web.txt commands = nosetests -s --logging-level=INFO --with-profile --profile-stats-file profile.log -a 'slow' {posargs}