Repository: mushorg/tanner Branch: main Commit: 3bc9ae2831db Files: 114 Total size: 54.2 MB Directory structure: gitextract_0cd_mqu1/ ├── .coveragerc ├── .github/ │ ├── FUNDING.yml │ └── workflows/ │ └── test.yml ├── .gitignore ├── LICENSE ├── README.md ├── bin/ │ ├── __init__.py │ ├── tanner │ ├── tannerapi │ └── tannerweb ├── docker/ │ ├── docker-compose.yml │ ├── phpox/ │ │ └── Dockerfile │ ├── redis/ │ │ └── Dockerfile │ └── tanner/ │ ├── Dockerfile │ └── template_injection/ │ └── Dockerfile ├── docs/ │ ├── Makefile │ └── source/ │ ├── api.rst │ ├── conf.py │ ├── config.rst │ ├── db_setup.rst │ ├── dorks.rst │ ├── emulators.rst │ ├── index.rst │ ├── parameters.rst │ ├── quick-start.rst │ ├── sessions.rst │ ├── storage.rst │ └── web.rst ├── requirements.txt ├── setup.py └── tanner/ ├── __init__.py ├── api/ │ ├── __init__.py │ ├── api.py │ └── server.py ├── config.py ├── data/ │ ├── GeoLite2-City.mmdb │ ├── config.yaml │ ├── crawler_user_agents.txt │ ├── db_config.json │ └── dorks.pickle ├── dorks_manager.py ├── emulators/ │ ├── __init__.py │ ├── base.py │ ├── cmd_exec.py │ ├── crlf.py │ ├── lfi.py │ ├── mysqli.py │ ├── php_code_injection.py │ ├── php_object_injection.py │ ├── rfi.py │ ├── sqli.py │ ├── sqlite.py │ ├── template_injection.py │ ├── xss.py │ └── xxe_injection.py ├── files/ │ └── engines/ │ ├── mako.py │ └── tornado.py ├── redis_client.py ├── reporting/ │ ├── __init__.py │ ├── hpfeeds.py │ ├── log_hpfeeds.py │ ├── log_local.py │ └── log_mongodb.py ├── server.py ├── sessions/ │ ├── __init__.py │ ├── session.py │ ├── session_analyzer.py │ └── session_manager.py ├── tests/ │ ├── __init__.py │ ├── test_aiodocker_helper.py │ ├── test_api.py │ ├── test_api_server.py │ ├── test_base.py │ ├── test_cmd_exec_emulation.py │ ├── test_config.py │ ├── test_crlf.py │ ├── test_dorks_manager.py │ ├── test_lfi_emulator.py │ ├── test_mysql_db_helper.py │ ├── test_mysqli.py │ ├── test_php_code_injection.py │ ├── test_php_object_injection.py │ ├── test_rfi_emulation.py │ ├── test_server.py │ ├── test_session_analyzer.py │ ├── test_session_manager.py │ ├── test_sqli.py │ ├── test_sqlite.py │ ├── test_sqlite_db_helper.py │ ├── test_template_injection.py │ ├── test_web_server.py │ ├── test_xss_emulator.py │ └── test_xxe_injection.py ├── utils/ │ ├── __init__.py │ ├── aiodocker_helper.py │ ├── api_key_generator.py │ ├── asyncmock.py │ ├── base_db_helper.py │ ├── logger.py │ ├── mysql_db_helper.py │ ├── patterns.py │ ├── php_sandbox_helper.py │ └── sqlite_db_helper.py └── web/ ├── __init__.py ├── server.py ├── static/ │ ├── css/ │ │ └── styles.css │ └── js/ │ └── site.js └── templates/ ├── base.html ├── index.html ├── session.html ├── sessions.html ├── snare-stats.html ├── snare.html └── snares.html ================================================ FILE CONTENTS ================================================ ================================================ FILE: .coveragerc ================================================ [report] omit = */tests/* ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms github: [mushorg,] ================================================ FILE: .github/workflows/test.yml ================================================ name: Tests on: [push] jobs: test: runs-on: ubuntu-latest strategy: matrix: python-version: ['3.9', '3.10'] services: mysql: image: mysql:5.7 env: MYSQL_ROOT_PASSWORD: user_pass MYSQL_DATABASE: tanner_db ports: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 redis: image: redis ports: - 6379:6379 options: --entrypoint redis-server steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install black pytest coveralls if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint with black run: | black --line-length 120 . - name: Test with pytest run: | python setup.py install docker pull busybox:latest pytest ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *,cover # Translations *.mo *.pot # Django stuff: *.log # Sphinx documentation docs/_build/ # PyBuilder target/ .idea/ .vscode/ venv/ ================================================ FILE: LICENSE ================================================ GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. {one line to give the program's name and a brief idea of what it does.} Copyright (C) {year} {name of author} This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: {project} Copyright (C) {year} {fullname} This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . ================================================ FILE: README.md ================================================ TANNER ====== [![Documentation Status](https://readthedocs.org/projects/tanner/badge/?version=latest)](http://tanner.readthedocs.io/en/latest/?badge=latest) [![Build Status](https://travis-ci.org/mushorg/tanner.svg?branch=master)](https://travis-ci.org/mushorg/tanner) [![Coverage Status](https://coveralls.io/repos/github/mushorg/tanner/badge.svg?branch=master)](https://coveralls.io/github/mushorg/tanner?branch=master) [![Coverage Status](https://coveralls.io/repos/github/mushorg/tanner/badge.svg?branch=develop)](https://coveralls.io/github/mushorg/tanner?branch=develop) He who flays the hide About ----- TANNER is a remote data analysis and classification service to evaluate HTTP requests and composing the response then served by [SNARE](https://github.com/mushorg/snare). TANNER uses multiple application vulnerability type emulation techniques when providing responses for SNARE. In addition, TANNER provides Dorks for SNARE powering its luring capabilities. Documentation ------------- The documentation can be found [here](http://tanner.readthedocs.io). Basic Concept ------------- - Evaluating [SNARE](https://github.com/mushorg/snare) events. - Serve dorks. - Emulate vulnerabilities and provide responses. Getting Started --------------- - You need Python3.7 and above for installing tanner. - This was tested with a recent Ubuntu-based Linux. ### Steps to install TANNER #### Step 1: Setup Redis 1. Install the Redis: ``sudo apt-get install redis-server`` 2. Run ``redis-server`` (to start it on `localhost` with default `port`) #### Step 2: Setup PHP Sandbox 1. For PHP Sandbox setup, see sandbox [manual](https://github.com/mushorg/phpox) 2. In PHP Sandbox directory, run sandbox: ``sudo python3 sandbox.py`` #### Step 3: Setup Docker 1. Run ``sudo apt-get install docker-ce docker-ce-cli containerd.io`` For more info please see the detailed installation guide [here.](https://docs.docker.com/engine/installation/linux/ubuntu/) #### Step 4: Setup and run TANNER 1. Get TANNER: `git clone https://github.com/mushorg/tanner.git` 2. Go to the TANNER source directory: ``cd tanner`` 3. Install requirements: `sudo pip3 install -r requirements.txt` 4. Install TANNER: ``sudo python3 setup.py install`` 5. Run TANNER: ``sudo tanner`` 6. (Optional) For runnning TANNER Api ``sudo tannerapi`` 7. (Optional) For runnning TANNER Web ``sudo tannerweb`` Note:- Make sure you have `python3-dev` incase you are facing problem with installing some requirments. ``` sudo apt-get install python3-dev ``` (Recommended) You should bind to 0.0.0.0 when running in production and on a different host than SNARE. ### Install and run TANNER using docker container In case you want to run the TANNER service using docker or facing any problem in setting up TANNER on your machine, you can follow these steps. #### Docker build instructions 1. Change the current directory to `tanner/docker` 2. `sudo docker-compose build` 3. `sudo docker-compose up` More information about running `docker-compose` can be found [here.](https://docs.docker.com/compose/gettingstarted/) Testing ------- In order to run the tests and receive a test coverage report, we recommend running `pytest`: pip install pytest pytest-cov sudo pytest --cov-report term-missing --cov=tanner tanner/tests/ Sample Output ------------- ```shell # sudo tanner _________ _ ___ ____________ /_ __/ | / | / / | / / ____/ __ \ / / / /| | / |/ / |/ / __/ / /_/ / / / / ___ |/ /| / /| / /___/ _, _/ /_/ /_/ |_/_/ |_/_/ |_/_____/_/ |_| Debug logs will be stored in /opt/tanner/tanner.log Error logs will be stored in /opt/tanner/tanner.err ======== Running on http://0.0.0.0:8090 ======== (Press CTRL+C to quit) ``` ================================================ FILE: bin/__init__.py ================================================ ================================================ FILE: bin/tanner ================================================ #!/usr/bin/python3.7 import argparse from tanner.config import TannerConfig from tanner import server from tanner.utils import logger def main(): print( """ _________ _ ___ ____________ /_ __/ | / | / / | / / ____/ __ \\ / / / /| | / |/ / |/ / __/ / /_/ / / / / ___ |/ /| / /| / /___/ _, _/ /_/ /_/ |_/_/ |_/_/ |_/_____/_/ |_| """ ) parser = argparse.ArgumentParser() parser.add_argument("--config", help="tanner config") args = parser.parse_args() if args.config: TannerConfig.set_config(args.config) debug_log_file_name = TannerConfig.get("LOGGER", "log_debug") error_log_file_name = TannerConfig.get("LOGGER", "log_err") logger.Logger.create_logger(debug_log_file_name, error_log_file_name, __package__) print("Debug logs will be stored in", debug_log_file_name) print("Error logs will be stored in", error_log_file_name) if TannerConfig.get("LOCALLOG", "enabled") is True: print("Data logs will be stored in", TannerConfig.get("LOCALLOG", "PATH")) tanner = server.TannerServer() tanner.start() if __name__ == "__main__": main() ================================================ FILE: bin/tannerapi ================================================ #!/usr/bin/python3.7 import argparse from tanner.api import server from tanner.config import TannerConfig def main(): parser = argparse.ArgumentParser() parser.add_argument("--config", help="tanner config") args = parser.parse_args() if args.config: TannerConfig.set_config(args.config) api = server.ApiServer() api.start() if __name__ == "__main__": main() ================================================ FILE: bin/tannerweb ================================================ #!/usr/bin/python3.7 import argparse from tanner.web import server from tanner.config import TannerConfig def main(): parser = argparse.ArgumentParser() parser.add_argument("--config", help="tanner config") args = parser.parse_args() if args.config: TannerConfig.set_config(args.config) tannerweb = server.TannerWebServer() tannerweb.start() if __name__ == "__main__": main() ================================================ FILE: docker/docker-compose.yml ================================================ version: '2.3' networks: tanner_local: services: # Tanner Redis Service tanner_redis: build: ./redis container_name: tanner_redis restart: always stop_signal: SIGKILL tty: true networks: - tanner_local image: "mushorg/redis:latest" read_only: true # PHP Sandbox service tanner_phpox: build: ./phpox container_name: tanner_phpox restart: always stop_signal: SIGKILL tty: true networks: - tanner_local image: "mushorg/phpox:latest" read_only: true tmpfs: "/tmp" # Tanner API Service tanner_api: build: ./tanner environment: - PATH=/opt/tanner/tanner-env/bin/:$PATH container_name: tanner_api restart: always stop_signal: SIGKILL tmpfs: - /tmp/tanner:uid=65534,gid=65534 - /var/log/tanner:uid=65534,gid=65534 tty: true networks: - tanner_local image: "mushorg/tanner:latest" read_only: true command: tannerapi depends_on: - tanner_redis # Tanner WEB Service tanner_web: build: ./tanner environment: - PATH=/opt/tanner/tanner-env/bin/:$PATH container_name: tanner_web restart: always stop_signal: SIGKILL tmpfs: - /tmp/tanner:uid=65534,gid=65534 - /var/log/tanner:uid=65534,gid=65534 tty: true networks: - tanner_local ports: - "8091:8091" image: "mushorg/tanner:latest" command: tannerweb read_only: true depends_on: - tanner_redis # Tanner Service tanner: build: ./tanner environment: - PATH=/opt/tanner/tanner-env/bin/:$PATH container_name: tanner restart: always stop_signal: SIGKILL tmpfs: - /tmp/tanner:uid=65534,gid=65534 - /var/log/tanner:uid=65534,gid=65534 - /opt/tanner/files:uid=65534,gid=65534 tty: true networks: - tanner_local ports: - "8090:8090" image: "mushorg/tanner:latest" command: tanner read_only: true depends_on: - tanner_api - tanner_web - tanner_phpox ================================================ FILE: docker/phpox/Dockerfile ================================================ FROM alpine:3.18 # # Install packages RUN apk -U --no-cache add \ build-base \ file \ git \ make \ php \ php81-dev \ php-tokenizer \ python3 \ py3-pip \ python3-dev \ re2c && \ # # Install bfr sandbox from git git clone --depth=1 https://github.com/mushorg/BFR /opt/BFR && \ cd /opt/BFR && \ php --version && \ phpize && \ ./configure \ --enable-bfr && \ make && \ make install && \ cd / && \ rm -rf /opt/BFR /tmp/* /var/tmp/* && \ php --ini | grep Loaded && \ echo "zend_extension = "$(find /usr -name bfr.so) >> /etc/php81/php.ini && \ # # Install PHP Sandbox git clone --depth=1 https://github.com/mushorg/phpox /opt/phpox && \ cd /opt/phpox && \ pip install -r requirements.txt && \ make && \ # # Clean up apk del --purge build-base \ git \ php-dev \ python3-dev && \ rm -rf /root/* && \ rm -rf /var/cache/apk/* # # Set workdir and start phpsandbox USER nobody:nobody WORKDIR /opt/phpox CMD ["python3", "sandbox.py"] ================================================ FILE: docker/redis/Dockerfile ================================================ FROM redis:alpine # Include dist ADD dist/ /root/dist/ # Setup apt RUN apk -U --no-cache add redis && \ cp /root/dist/redis.conf /etc && \ # Clean up rm -rf /root/* && \ rm -rf /tmp/* /var/tmp/* && \ rm -rf /var/cache/apk/* # Start redis USER nobody:nobody CMD redis-server /etc/redis.conf ================================================ FILE: docker/tanner/Dockerfile ================================================ FROM alpine:3.15 # Include dist ADD dist/ /root/dist/ # Setup apt RUN apk -U --no-cache add \ build-base \ git \ libcap \ libffi-dev \ libressl-dev \ linux-headers \ py3-yarl \ python3 \ python3-dev \ py3-pip && \ # Setup Tanner git clone --depth=1 https://github.com/mushorg/tanner /opt/tanner && \ cp /root/dist/config.yaml /opt/tanner/tanner/data/ && \ cd /opt/tanner/ && \ python3 -m venv tanner-env && \ source /opt/tanner/tanner-env/bin/activate && \ pip install --upgrade pip && \ pip3 install --no-cache-dir wheel && \ pip3 install --no-cache-dir -r requirements.txt && \ python3 setup.py install && \ cd / && \ # Setup configs, user, groups chown -R nobody:nobody /opt/tanner && \ # Clean up apk del --purge \ build-base \ linux-headers \ python3-dev && \ rm -rf /root/* && \ rm -rf /tmp/* /var/tmp/* && \ rm -rf /var/cache/apk/* # Start tanner USER nobody:nobody WORKDIR /opt/tanner CMD ["/opt/tanner/tanner-env/bin/tanner", "--config", "/opt/tanner/tanner/data/config.yaml"] ================================================ FILE: docker/tanner/template_injection/Dockerfile ================================================ FROM alpine RUN apk -U --no-cache add \ python3 RUN pip3 install --upgrade pip && \ pip3 install --no-cache-dir \ tornado \ jinja2 \ mako ================================================ FILE: docs/Makefile ================================================ # Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source .PHONY: help help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " applehelp to make an Apple Help Book" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " epub3 to make an epub3" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " xml to make Docutils-native XML files" @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" @echo " coverage to run coverage check of the documentation (if enabled)" @echo " dummy to check syntax errors of document sources" .PHONY: clean clean: rm -rf $(BUILDDIR)/* .PHONY: html html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." .PHONY: dirhtml dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." .PHONY: singlehtml singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." .PHONY: pickle pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." .PHONY: json json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." .PHONY: htmlhelp htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." .PHONY: qthelp qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/tanner.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/tanner.qhc" .PHONY: applehelp applehelp: $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp @echo @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." @echo "N.B. You won't be able to view it unless you put it in" \ "~/Library/Documentation/Help or install it in your application" \ "bundle." .PHONY: devhelp devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/tanner" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/tanner" @echo "# devhelp" .PHONY: epub epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." .PHONY: epub3 epub3: $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 @echo @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." .PHONY: latex latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." .PHONY: latexpdf latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." .PHONY: latexpdfja latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." .PHONY: text text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." .PHONY: man man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." .PHONY: texinfo texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." .PHONY: info info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." .PHONY: gettext gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." .PHONY: changes changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." .PHONY: linkcheck linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." .PHONY: doctest doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." .PHONY: coverage coverage: $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage @echo "Testing of coverage in the sources finished, look at the " \ "results in $(BUILDDIR)/coverage/python.txt." .PHONY: xml xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." .PHONY: pseudoxml pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." .PHONY: dummy dummy: $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy @echo @echo "Build finished. Dummy builder generates no files." ================================================ FILE: docs/source/api.rst ================================================ Tanner API ========== Tanner api provides various stats related to traffic captured by snare. It can be accessed at ``locahost:8092/?key=API_KEY``. where, ``API_KEY`` is a JWT-token created by a particular tanner-api, which can be found during tanner-api startup: ``API_KEY for full access: 'API_KEY'`` How to create an API_KEY with desired signature? ~~~~~~~~ * By default tanner's API_KEYs use the signature: 'tanner_api_auth' * This signature is veryfied on all the API requests. * It is highly recommended that every tanner-owner set their own signature. * This can be done by modifying tanner.config['API']['auth_signature'] to the desired one. /?key=API_KEY ~~~~ This is the index page which shows ``tanner api``. /snares ~~~~~~~~~~ This shows all the snares' uuid. /snare/?key=API_KEY ~~~~~~~~~~~~~~~~~~~~~~ Replace ```` with a valid `snare-uuid` and it will show all the sessions related to that ``snare-uuid`` and their details. /snare-stats/?key=API_KEY ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Replace ```` with a valid `snare-uuid` and it will show some stats. * No of sessions in the sanre * Total duration for which snare remains active * Attack frequency, which shows no of sessions which face different attacks. //sessions?filters=&key=API_KEY ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This shows all the sessions' uuid which follow the filters. Filters are sepatated by ``white-space`` and name-value pair are separated by ``:``. E.g ``?filters=filter1:value1 filter2:value2``. It supports 5 filters: * **peer_ip** -- Sessions with given ip. E.g ``peer_ip:127.0.0.1 `` * **user-agent** -- Sessions with given user-agent. E.g ``user-agent:Chrome`` * **attack_types** -- Sessions with given attack type such as lfi, rfi, xss, cmd_exec, sqli. E.g ``attack_types:lfi`` * **possible_owners** -- Sessions with given owner type such as user, tool, crawler, attacker. E.g ``possible_owners:attacker`` * **start_time** -- Sessions which started after `start_time`. E.g ``start_time:1480560`` * **end_time** -- Sessions which ended before `end_time`. E.g ``end_time:1480560`` Multiple filters can be applied as ``peer_ip:127.0.0.1 start_time:1480560 possible_owners:attacker`` /api/session/?key=API_KEY ~~~~~~~~~~~~~~~~~~~~~~~~ It gives all information about the session with given uuid. External hyperlinks, like Python_. .. _Python: http://www.python.org/ ================================================ FILE: docs/source/conf.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # # tanner documentation build configuration file, created by # sphinx-quickstart on Wed Jun 22 22:07:03 2016. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # # import os # import sys # sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = ".rst" # The encoding of source files. # # source_encoding = 'utf-8-sig' # The master toctree document. master_doc = "index" # General information about the project. project = "tanner" copyright = "2016, mushorg" author = "mushorg" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = "1.0" # The full version, including alpha/beta/rc tags. release = "1.0" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # # today = '' # # Else, today_fmt is used as the format for a strftime call. # # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path exclude_patterns = [] # The reST default role (used for this markup: `text`) to use for all # documents. # # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. # # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). # # add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. # keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] # The name for this set of Sphinx documents. # " v documentation" by default. # # html_title = 'tanner v1.0' # A shorter title for the navigation bar. Default is the same as html_title. # # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. # # html_logo = None # The name of an image file (relative to this directory) to use as a favicon of # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. # # html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. # # html_extra_path = [] # If not None, a 'Last updated on:' timestamp is inserted at every page # bottom, using the given strftime format. # The empty string is equivalent to '%b %d, %Y'. # # html_last_updated_fmt = None # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. # # html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. # # html_additional_pages = {} # If false, no module index is generated. # # html_domain_indices = True # If false, no index is generated. # # html_use_index = True # If true, the index is split into individual pages for each letter. # # html_split_index = False # If true, links to the reST sources are added to the pages. # # html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. # # html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. # # html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # # html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' # # html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # 'ja' uses this config value. # 'zh' user can custom change `jieba` dictionary path. # # html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. # # html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. htmlhelp_basename = "tannerdoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, "tanner.tex", "tanner Documentation", "mushorg", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. # # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # # latex_use_parts = False # If true, show page references after internal links. # # latex_show_pagerefs = False # If true, show URL addresses after external links. # # latex_show_urls = False # Documents to append as an appendix to all manuals. # # latex_appendices = [] # If false, no module index is generated. # # latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [(master_doc, "tanner", "tanner Documentation", [author], 1)] # If true, show URL addresses after external links. # # man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ( master_doc, "tanner", "tanner Documentation", author, "tanner", "One line description of project.", "Miscellaneous", ), ] # Documents to append as an appendix to all manuals. # # texinfo_appendices = [] # If false, no module index is generated. # # texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. # # texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. # # texinfo_no_detailmenu = False ================================================ FILE: docs/source/config.rst ================================================ Configuration file ================== Tanner uses ``YAML`` like format for configuration file. It's value can specified by using ``config`` flag. The use of ``INI`` configuration file is obsolete now. There are 8 different sections : * **DATA** # Data configuration :db_config: Location of SQLI database configuration :dorks: Location of dorks :user_dorks: Location of user dorks * **TANNER** :host: The host at which Tanner is running :port: The port at which Tanner is running * **WEB** # Tanner web configuration :host: The host at which Tanner Web UI is running :port: The port at which Tanner Web UI is running * **API** # Tanner API configuration :Host: The host at which Tanner API is running :Port: The port at which Tanner API is running * **PHPOX** :Host: The host at which PHPOX is running :Port: The port at which PHPOX is running * **REDIS** # Configure redis if it's running on some different port or network. :host: The host address at which redis is running :port: The port at which which redis is running :poolsize: The poolsize of redis server :timeout: The duration of timeout for redis server * **EMULATORS** :root_dir: The root directory for emulators that need data storing such as SQLI and LFI. Data will be stored in this directory * **EMULATOR_ENABLED** # Enable or disable emulators by setting value true or false respectively. :sqli: True if this emulator is enabled else False :rfi: True if this emulator is enabled else False :lfi: True if this emulator is enabled else False :xss: True if this emulator is enabled else False :cmd_exec: True if this emulator is enabled else False * **SQLI** :type: Supports two types MySQL/SQLITE :db_name: The name of database used in SQLI emulator :host: This will be used for MySQL to get the host address :user: This is the MySQL user which perform DB queries :password: The password corresponding to the above user * **DOCKER** :host_image: The image which emulates commands in Command Execution Emulator and file system in LFI emulator * **LOGGER** :log_debug: Location of tanner log file :log_err: Location of tanner error file * **MONGO** :enabled: Check whether MONGO database is enabled :URI: URI for connecting to MONGO DB * **HPFEEDS** :enabled: Check whether HPFEEDS logging is enabled :HOST: IP address or name of the hpfeeds server :PORT: Port of the hpfeeds service :IDENT: Identifier of the hpfeeds client :SECRET: Secret of the hpfeeds client :CHANNEL: Channel to which publish messages * **LOCALLOG** :enabled: Check local(temporary) logging is enabled :PATH: Location of file for local(temporary) logging If no file is specified, following YAML will be used as default: .. code-block:: python DATA: db_config: /opt/tanner/db/db_config.json dorks: /opt/tanner/data/dorks.pickle user_dorks: /opt/tanner/data/user_dorks.pickle crawler_stats: /opt/tanner/data/crawler_user_agents.txt geo_db: /opt/tanner/db/GeoLite2-City.mmdb tornado: /opt/tanner/data/tornado.py mako: /opt/tanner/data/mako.py TANNER: host: 0.0.0.0 port: 8090 WEB: host: 0.0.0.0 port: 8091, API: host: 0.0.0.0 port: 8092 auth: False auth_signature: tanner_api_auth PHPOX: host: 0.0.0.0 port: 8088 REDIS: host: localhost port: 6379 poolsize: 80 timeout: 1 EMULATORS: root_dir: /opt/tanner EMULATOR_ENABLED: sqli: True rfi: True lfi: True xss: True cmd_exec: True php_code_injection: True php_object_injection: True crlf: True xxe_injection: True template_injection: True SQLI: type: SQLITE db_name: tanner_db host: localhost user: root password: user_pass XXE_INJECTION: OUT_OF_BAND: False RFI: allow_insecure: False DOCKER: host_image: busybox:latest LOGGER: log_debug: /opt/tanner/tanner.log log_err: /opt/tanner/tanner.err MONGO: enabled: False URI: mongodb://localhost HPFEEDS: enabled: False HOST: localhost PORT: 10000 IDENT: '' SECRET: '' CHANNEL: tanner.events LOCALLOG: enabled: False PATH: /tmp/tanner_report.json CLEANLOG: enabled: False REMOTE_DOCKERFILE: GITHUB: "https://raw.githubusercontent.com/mushorg/tanner/master/docker/tanner/template_injection/Dockerfile" SESSIONS: delete_timeout: 300 ================================================ FILE: docs/source/db_setup.rst ================================================ DB Setup ======== To setup a database for sqli emulation TANNER provides ``db_config.json`` file, which stores the configuration of a database. ``db_config.json`` has the following structure: :: { "name": "db name" "tables":[ { "table name": "name of the table" "schema": "the result of sqlite3 command .schema, create table expression" "data_tokens": "types of data in the columns" } ] } Default ``db_config.json``: :: { "name": "test1", "tables": [ { "table_name": "users", "schema": "CREATE TABLE users (id INTEGER PRIMARY KEY, username text, email text, password text);", "data_tokens": "I,L,E,P" }, { "table_name": "comments", "schema": "CREATE TABLE comments (id INTEGER PRIMARY KEY, comment text);", "data_tokens": "I,T" } ] } You can change default config to make your own db structure. Data tokens ~~~~~~~~~~~ Data tokens are used for filling the database with dummy data. There are 4 default tokens: * **I** -- integer id * **L** -- login/username * **E** -- email * **P** -- password * **T** -- piece of text **Note**: TANNER uses the default linux wordlist (``/usr/share/dict/words``) for data. If you don't have the default wordlist in your system, install it or put it manually in ``/usr/share/dict``. ================================================ FILE: docs/source/dorks.rst ================================================ Dorks ===== There are two types of the dorks: * **Manually added dorks** -- came from the Google Hacking database and other sources * **User dorks** -- all user requests with queries All dorks are stored in the Redis and keys for the dorks are static: * **Manually dorks** -- ``uuid.uuid3(uuid.NAMESPACE_DNS,'dorks').hex``. * **User dorks** -- ``uuid.uuid3(uuid.NAMESPACE_DNS,'user_dorks').hex``. On initializing stage, Dorks Manager loads manually added dorks (``dorks.pickle``) from project directory and push them into redis. Dorks are stored in the Redis as ``set`` to avoid repetition. ================================================ FILE: docs/source/emulators.rst ================================================ Emulators --------- Base emulator ~~~~~~~~~~~~~ This is the heart of emulation. Current emulators follow ``find and emulate`` approach where each emulator has a ``scan`` method which is called by base emulator against each ``GET``, ``POST`` parameter and ``cookie value``. The parameter which is affected, gets emulated by calling the corresponding emulator's ``handle`` method. It returns the ``payload`` along with ``injection page`` which is most recently visited ``text/html`` type page. RFI emulator ~~~~~~~~~~~~ It emulates RFI_ vulnerability. This attack type is detected with pattern: :: .*(.*(http(s){0,1}|ftp(s){0,1}):).* RFI emulation include two steps: * Download file * Downloaded files are storing in the ``opt/tanner/scripts`` directory. * Create filename with ``hashlib.md5()`` from its content. * Execute code from downloaded file with PHPox_ and return the result * Get script body from file * Connect to PHPox server (default 127.0.0.1:8088) and send script body * Get the result of execution in the response * Return the result LFI emulator ~~~~~~~~~~~~ It emulates LFI_ vulnerability. This attack type is detected with pattern: :: .*(\/\.\.)*(home|proc|usr|etc)\/.* It is emualted using a docker container with Linux filesystem (default: ``busybox:latest``). When LFI attack is detected, LFI emulator executes a command ``cat **file_to_be_read**`` within the docker and it returns the contents of file if found else return ``No such file or directory``. XSS emulator ~~~~~~~~~~~~ It emulates XSS_ vulnerability. This attack type is detected with pattern: :: .*<(.|\n)*?> Emulator returns the script body and the page, into which this script must be injected. * Script body can be extracted from data in ``POST`` requests and from query in ``GET`` requests . * To avoid replacing characters in data, we use ``urllib.parse.unquote`` function before analyzing path and post data with ``re``. * Page is selected from the current session paths (see :doc:`sessions`). It's the last page with mime type ``text/html``. * Script is injected into page on SNARE side. SQLi emulator ~~~~~~~~~~~~~ It emulates `SQL injection`_ vulnerability. This attack is detected by ``libinjection``. The emulator copies the original database (see :doc:`db_setup` for more info about db) to a dummy database for every attacker. It uses UUID of the session for the attacker's db name. Every query is executed on the attacker's db. The emulator returns the result of the execution and the page where SNARE should show the result. It supports two types of DBs. * **SQLITE** To enable it, set SQLI type to SQLITE in config * **MySQL** To enable it, set SQLI type to MySQL in config and set other necessary fields - Host, User and Password Command Execution Emulator ~~~~~~~~~~~~~~~~~~~~~~~~~~ It emulates `Command Execution`_ vulnerability. This attack is detected with pattern. :: .*(alias|cat|cd|cp|echo|exec|find|for|grep|ifconfig|ls|man|mkdir|netstat|ping|ps|pwd|uname|wget|touch|while).* * Each param value is checked against the pattern and ``command`` is extracted. * The ``command`` is executed in a docker container safely. * Results from container is injected into the index page. PHP Code Injection Emulator ~~~~~~~~~~~~~~~~~~~~~~~~~~~ It emulates `PHP code injection`_ vuln. Usually, this type of vuln is found where user input is directly passed to functions like eval, assert. To mimic the functionality, user input is converted to the following code ```` and then passed to phpox to get php code emulation results. PHP Object Injection Emulator ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It emulates `PHP object injection`_ vuln. PHP allows object serialization So, this type of vulnerability occurs when not properly sanitized input is passed to ``unserialize()`` PHP function. Exploiting this vulnerability involves Magic methods like ``__construct and __destruct`` which are called automatically when an object is created or destroyed and methods like ``__sleep and __wakeup`` are called when an object is serialized or unserialized. The input serialized object is detected with regex pattern. :: (^|;|{|})O:[0-9]+: To mimic this functionality the user input is injected to a vulnerable custom class with magic methods and then it is passed to php sandbox to get the injection results. **Important Note:** You will need to expose the vulnerable code to the attacker using your own suitable method. The default vulnerable code is `here`_. But you can always add your own custom class if needed. CRLF Emulator ~~~~~~~~~~~~~ It emulates `CRLF`_ vuln. The attack is detected using ``\r\n`` pattern in the input. The parameter which looks suspicious is injected as a header with parameter name as header name and param value as header value. XXE Injection Emulator ~~~~~~~~~~~~~~~~~~~~~~ It emulates `External Entity Injection`_ vulnerability. This type of vulnerability occurs when XML input with reference to an external entity is parsed by a weakly configured parser. It is exploited by putting specially crafted DTDs with malicious entities defined in it. The XML input is detected by regex pattern. :: .*<(\?xml|(!DOCTYPE.*)).*> To mimic this functionality attacker's input will be injected into a vulnerable PHP code which parses the XML data and then it gets the injection results from php sandbox. **Note:** You can customize the vulnerable PHP code and can make it more intuitive. for eg: emulating a submit form with user, password fields. Template Injection Emulator ~~~~~~~~~~~~~~~~~~~~~~~~~~~ This emulates `Template Injection`_ vulnerability. This is exploited by using specially crafted payloads for different template engines. For now we are covering ``tornado`` and ``mako`` python templating engines. The injection formats are different for every engine for ex ``tornado: {{7*7}} -> 49`` and ``mako: <% x=7*7 %>${x} -> 49``. The payload is detected using regex pattern: :: .*({{.*}}).* - Tornado .*(<%.*|\s%>).* - Mako To mimic this functionality vulnerable template renderers are stored in `files/engines` directory for every engine in which the payload will be injected. These vulnerable templates are executed safely using custom docker image to get the injection results. .. _Template Injection: https://portswigger.net/blog/server-side-template-injection .. _RFI: https://en.wikipedia.org/wiki/File_inclusion_vulnerability#Remote_File_Inclusion .. _PHPox: https://github.com/mushorg/phpox .. _LFI: https://en.wikipedia.org/wiki/File_inclusion_vulnerability#Local_File_Inclusion .. _XSS: https://en.wikipedia.org/wiki/Cross-site_scripting .. _SQL injection: https://en.wikipedia.org/wiki/SQL_injection .. _Command Execution: https://www.owasp.org/index.php/Command_Injection .. _PHP Code Injection: https://www.owasp.org/index.php/Code_Injection .. _PHP object injection: https://www.owasp.org/index.php/PHP_Object_Injection .. _CRLF: https://www.owasp.org/index.php/CRLF_Injection .. _External Entity Injection: https://www.owasp.org/index.php/XML_External_Entity_(XXE)_Processing .. _manual: https://github.com/client9/libinjection/wiki/doc-sqli-python .. _here: https://github.com/mushorg/tanner/blob/8ce13d1f7d4423ddaf0e7910781199be9b90ce40/tanner/emulators/php_object_injection.py#L16 ================================================ FILE: docs/source/index.rst ================================================ .. tanner documentation master file, created by sphinx-quickstart on Wed Jun 22 22:07:03 2016. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Welcome to TANNER's documentation! ================================== Contents: .. toctree:: :maxdepth: 2 quick-start parameters emulators sessions storage dorks config api web Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` ================================================ FILE: docs/source/parameters.rst ================================================ TANNER command line parameters ============================= tanner [**--config**] Description ~~~~~~~~~~~ * **config** -- path to tanner config file ================================================ FILE: docs/source/quick-start.rst ================================================ Quick Start =========== TANNER, a remote data analysis and classification service, to evaluate HTTP requests and composing the response then served by SNARE. Basic concept """"""""""""" * Evaluating SNARE events. * Serve dorks. * Adopt and change the responses. Setup Redis """"""""""" #. Install the Redis: ``sudo apt-get install redis-server`` #. Start it on ``localhost`` with default ``port``: ``redis-server`` Setup PHP Sanbox """""""""""""""" #. For PHP Sandbox setup, see sandbox manual_ #. In PHP Sandbox directory, run sandbox: ``sudo python3 sandbox.py`` .. _manual: https://github.com/mushorg/phpox Setup and run TANNER """""""""""""""""""" #. Get TANNER: ``git clone https://github.com/mushorg/tanner.git`` #. Go to the tanner source directory ``cd tanner`` #. Install requirements: ``sudo pip3 install -r requirements.txt`` #. Install tanner ``sudo python3 setup.py install`` #. Run TANNER: ``sudo tanner`` Run Tanner Api """""""""""""" #. Run ``sudo tannerapi`` Run Tanner WebUI """""""""""""""" #. Run ``sudo tannerweb`` Docker build instructions """"""""""""""""""""""""" 1. Change current directory to ``tanner/docker`` 2. ``docker-compose build`` 3. ``docker-compose up`` **Note**: Running docker with default ``docker-compose.yml`` setting will start tanner, tannerapi, tannerweb, tanner redis, tannerphpox but only tanner and tannerweb will be accesible from the outside network. More information about running ``docker-compose`` can be found `here `_. ================================================ FILE: docs/source/sessions.rst ================================================ Sessions ======== .. _session: Session ~~~~~~~ Session class accepts ``data`` as a parameter. The ``data`` came from SNARE and is validated before use (see :ref:`session-manager`). **Attributes:** * **ip** -- peer ip address. * **port** -- peer port. * **user_agent** -- peer user agent. * **snare_uuid** -- SNARE sensor uuid. * **paths** -- list of dictionaries. Contains ``path``, ``timestamp``, ``attack_type`` and SNARE ``response status``. * **sess_uuid** -- randomly generated session uuid. * **start_timestamp** -- session start time. * **timestamp** -- current session timestamp. * **count** -- count of the session's updates (i.e. requested paths). * **cookies** -- dictionary of cookies sent by client or set by server ``KEEP_ALIVE_TIME`` is the constant, which set up the active session time. Default value is 75. After this time, the session is expired and can be deleted. .. _session-manager: Session Manager ~~~~~~~~~~~~~~~ Every session is tracking and recording. The session is determined by peer ``ip``, ``user_agent`` and ``sess_uuid``. Session is unique, if there is no sessions with this ``ip``, ``user_agent`` and ``sess_uuid``. If session exists, it will be updated. Active sessions are kept in the process memory (see :ref:`session`). After expiration, session is pushed into the Redis (see :doc:`storage`) Data validation """"""""""""""" If necessary fields missing in the raw data from SNARE, these fields are created with ``None`` value. Session Evaluation ~~~~~~~~~~~~~~~~~~ When session is deleted from python process memory, it is evaluated by session analyzer. The result contains next fields: * *Session attributes* * **sess_uuid** * **peer_ip** * **peer_port** * **user_agent** * **snare_uuid** * **start_time** * **cookies** * **end_time** -- last session timestamp * **requests_per_second** -- request per second from user * **approx_time_between_requests** -- * **accepted_paths** -- number of accepted paths * **errors** -- counts of errors in SNARE responses * **hidden_links** -- count of accepted dorks hidden links * **attack_types** -- list of attack types * **paths** -- list of all paths * **possible_owners** -- list of possible owners. May be ``user``, ``attacker``, ``tool`` and ``crawler`` ================================================ FILE: docs/source/storage.rst ================================================ Storage ======= We use Redis_ as main storage. TANNER connects to the redis with default values: ``host='localhost', port=6379`` You should install and start the Redis on the server before using TANNER. See :doc:`quick-start` .. _Redis: http://redis.io/ ================================================ FILE: docs/source/web.rst ================================================ Tanner WEB ========== Tanner WEB provides various stats related to traffic captured by snare in UI form. It can be accessed at ``locahost:8091/``. / ~~~~ This is the index page which has a logo (mushorg) with ``Tanner web`` written below it. Below that we can find general info of the tanner instance: * **Tanner version** -- Which shows the version of tanner instance * **No. of snares connected** -- Which shows the number of snares connected to the tanner instance * **Latest session** -- Which will navigate you to the latest session's info page Below that we can find a clickable which states, ``Click here to navigate to snares list``, clicking which will move you to the ``/snares`` page. /snares ~~~~~~~~~~ This shows all the snares' uuid. Each snare object is clickable. Clicking displays the page **/snare/** /snare/ ~~~~~~~~~~~~~~~~~~~~~~ Replace ```` with a valid `snare-uuid` and it will provide two options: * **Snare-Stats** -- It will move you to **/snare-stats/** * **Sessions** -- It will move you to **//sessions** /snare-stats/ ~~~~~~~~~~~~~~~~~~~~~~~~~ This page shows some general stats about the snare * **No of Sessions** - Total no of sessions of the snare * **Total Duration** - Total durations during which sessions remain active * **Attack Frequency** - Frequency of different attacks made on the snare //sessions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This shows all the sessions' uuid. Each is clickable. Clicking displays **/session/** Filters can be on the sessions using the input box and clicking the ``Apply`` button. Filters are sepatated by ``white-space`` and name-value pair are separated by ``:``. E.g ``filter1:value1 filter2:value2``. It supports 6 filters: * **peer_ip** -- Sessions with given ip. E.g ``peer_ip:127.0.0.1 `` * **user-agent** -- Sessions with given user-agent. E.g ``user-agent:Chrome`` * **attack_types** -- Sessions with given attack type such as lfi, rfi, xss, cmd_exec, sqli. E.g ``attack_types:lfi`` * **possible_owners** -- Sessions with given owner type such as user, tool, crawler, attacker. E.g ``possible_owners:attacker`` * **start_time** -- Sessions which started after `start_time`. E.g ``start_time:1480560`` * **end_time** -- Sessions which ended before `end_time`. E.g ``end_time:1480560`` * **location** -- Sessions which have been done from the specified geometric `location`. It can take value of either country, city, country_code or zip_code. E.g ``location:India``, ``location:Mumbai``, ``location:US``, ``location:636005`` etc Multiple filters can be applied as ``peer_ip:127.0.0.1 start_time:1480560 possible_owners:attacker`` /session/ ~~~~~~~~~~~~~~~~~~~~~~~~ It gives all information about the session with given uuid. Here you may find some of the text clickable such as ``peer_ip``,``possible_owners``, ``start_time``, ``end_time``, ``attack_types``. Clicking on them will display all the sessions will same attribute value. ================================================ FILE: requirements.txt ================================================ aiohttp aiomysql aiohttp-jinja2==1.5.1 docker<2.6 mimesis<3.0.0 yarl redis aioredis pymongo pylibinjection jinja2 MarkupSafe<2.1.0 pycodestyle geoip2 aiodocker tornado mako pyjwt pyyaml ================================================ FILE: setup.py ================================================ #!/usr/bin/env python from setuptools import find_packages, setup setup( name="Tanner", version="0.6.0", description="He who flays the hide", author="MushMush Foundation", author_email="glastopf@public.honeynet.org", url="https://github.com/mushorg/tanner", packages=find_packages(exclude=["*.pyc"]), scripts=["bin/tanner", "bin/tannerweb", "bin/tannerapi"], data_files=[ ("/opt/tanner/db/", ["tanner/data/db_config.json", "tanner/data/GeoLite2-City.mmdb"]), ( "/opt/tanner/data/", [ "tanner/data/dorks.pickle", "tanner/data/crawler_user_agents.txt", "tanner/files/engines/mako.py", "tanner/files/engines/tornado.py", "tanner/data/config.yaml", ], ), ], ) ================================================ FILE: tanner/__init__.py ================================================ __version__ = "0.6.0" ================================================ FILE: tanner/api/__init__.py ================================================ ================================================ FILE: tanner/api/api.py ================================================ import json import logging import operator import aioredis class Api: def __init__(self, redis_client): self.logger = logging.getLogger("tanner.api.Api") self.redis_client = redis_client async def return_snares(self): query_res = [] try: query_res = await self.redis_client.smembers("snare_ids") except aioredis.exceptions.ConnectionError as connection_error: self.logger.exception("Can not connect to redis %s", connection_error) return list(query_res) async def return_snare_stats(self, snare_uuid): result = {} result["total_sessions"] = 0 result["total_duration"] = 0 result["attack_frequency"] = {"sqli": 0, "lfi": 0, "xss": 0, "rfi": 0, "cmd_exec": 0} sessions = await self.return_snare_info(snare_uuid) if sessions == "Invalid SNARE UUID": return result result["total_sessions"] = len(sessions) for sess in sessions: result["total_duration"] += sess["end_time"] - sess["start_time"] for attack in sess["attack_types"]: if attack in result["attack_frequency"]: result["attack_frequency"][attack] += 1 return result async def return_snare_info(self, uuid, count=-1): query_res = [] try: query_res = await self.redis_client.zrevrangebyscore(uuid, offset=0, count=count) except aioredis.exceptions.ConnectionError as connection_error: self.logger.exception("Can not connect to redis %s", connection_error) else: if not query_res: return "Invalid SNARE UUID" for (i, val) in enumerate(query_res): query_res[i] = json.loads(val) return query_res async def return_session_info(self, sess_uuid, snare_uuid=None): if snare_uuid: snare_uuids = [snare_uuid] else: snare_uuids = await self.return_snares() for snare_id in snare_uuids: sessions = await self.return_snare_info(snare_id) if sessions == "Invalid SNARE UUID": continue for sess in sessions: if sess["sess_uuid"] == sess_uuid: return sess async def return_sessions(self, filters): snare_uuids = await self.return_snares() matching_sessions = [] for snare_id in snare_uuids: result = await self.return_snare_info(snare_id) if result == "Invalid SNARE UUID": return "Invalid filter : SNARE UUID" sessions = result for sess in sessions: match_count = 0 for filter_name, filter_value in filters.items(): try: if self.apply_filter(filter_name, filter_value, sess): match_count += 1 except KeyError: return "Invalid filter : %s" % filter_name if match_count == len(filters): matching_sessions.append(sess) return matching_sessions async def return_latest_session(self): latest_time = -1 latest_session = None snares = await self.return_snares() try: for snare in snares: filters = {"snare_uuid": snare} sessions = await self.return_sessions(filters) for session in sessions: if latest_time < session["end_time"]: latest_time = session["end_time"] latest_session = session["sess_uuid"] except TypeError: return None return latest_session def apply_filter(self, filter_name, filter_value, sess): available_filters = { "user_agent": operator.contains, "peer_ip": operator.eq, "attack_types": operator.contains, "possible_owners": operator.contains, "start_time": operator.le, "end_time": operator.ge, "snare_uuid": operator.eq, "location": operator.contains, } try: if available_filters[filter_name] is operator.contains: return available_filters[filter_name](sess[filter_name], filter_value) else: return available_filters[filter_name](filter_value, sess[filter_name]) except KeyError: raise ================================================ FILE: tanner/api/server.py ================================================ import asyncio import logging from aiohttp import web from aiohttp.web import middleware from tanner.api import api from tanner import redis_client from tanner.config import TannerConfig from tanner.utils.api_key_generator import generate import jwt from jwt.exceptions import DecodeError, InvalidSignatureError class ApiServer: def __init__(self): self.logger = logging.getLogger("tanner.api.ApiServer") self.api = None @staticmethod def _make_response(msg): response_message = dict(version=1, response=dict(message=msg)) return response_message async def handle_index(self, request): result = "tanner api" response_msg = self._make_response(result) return web.json_response(response_msg) async def handle_snares(self, request): result = await self.api.return_snares() response_msg = self._make_response(result) return web.json_response(response_msg) async def handle_snare_info(self, request): snare_uuid = request.match_info["snare_uuid"] result = await self.api.return_snare_info(snare_uuid, 50) response_msg = self._make_response(result) return web.json_response(response_msg) async def handle_snare_stats(self, request): snare_uuid = request.match_info["snare_uuid"] result = await self.api.return_snare_stats(snare_uuid) response_msg = self._make_response(result) return web.json_response(response_msg) async def handle_sessions(self, request): snare_uuid = request.match_info["snare_uuid"] params = request.url.query applied_filters = {"snare_uuid": snare_uuid} try: if "filters" in params: for filt in params["filters"].split(): applied_filters[filt.split(":")[0]] = filt.split(":")[1] if "start_time" in applied_filters: applied_filters["start_time"] = float(applied_filters["start_time"]) if "end_time" in applied_filters: applied_filters["end_time"] = float(applied_filters["end_time"]) except Exception as e: self.logger.exception("Filter error : %s" % e) result = "Invalid filter definition" else: sessions = await self.api.return_sessions(applied_filters) sess_uuids = [sess["sess_uuid"] for sess in sessions] result = sess_uuids response_msg = self._make_response(result) return web.json_response(response_msg) async def handle_session_info(self, request): sess_uuid = request.match_info["sess_uuid"] result = await self.api.return_session_info(sess_uuid) response_msg = self._make_response(result) return web.json_response(response_msg) async def on_shutdown(self, app): self.redis_client.close() @middleware async def auth(self, request, handler): resp = await handler(request) auth_key = request.query.get("key") try: decoded = jwt.decode(auth_key, TannerConfig.get("API", "auth_signature"), algorithm="HS256") except (DecodeError, InvalidSignatureError): return web.Response(body="401: Unauthorized") return resp def setup_routes(self, app): app.router.add_get("/", self.handle_index) app.router.add_get("/snares", self.handle_snares) app.router.add_resource("/snare/{snare_uuid}").add_route("GET", self.handle_snare_info) app.router.add_resource("/snare-stats/{snare_uuid}").add_route("GET", self.handle_snare_stats) app.router.add_resource("/{snare_uuid}/sessions").add_route("GET", self.handle_sessions) app.router.add_resource("/session/{sess_uuid}").add_route("GET", self.handle_session_info) async def make_app(self, auth=False): if auth: app = web.Application(middlewares=[self.auth]) else: app = web.Application() app.on_shutdown.append(self.on_shutdown) self.setup_routes(app) return app def start(self): loop = asyncio.get_event_loop() self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client(poolsize=20)) self.api = api.Api(self.redis_client) set_auth = TannerConfig.get("API", "auth") host = TannerConfig.get("API", "host") port = int(TannerConfig.get("API", "port")) if set_auth: key = generate() print("API_KEY for full access:", key) web.run_app(self.make_app(auth=set_auth), host=host, port=port) ================================================ FILE: tanner/config.py ================================================ import logging import os import sys import yaml LOGGER = logging.getLogger(__name__) class TannerConfig: config = None @staticmethod def read_config(path): config_values = {} try: with open(path, "r") as f: config_values = yaml.load(f, Loader=yaml.FullLoader) except yaml.parser.ParserError as e: print("Couldn't properly parse the config file. Please use properly formatted YAML config.") sys.exit(1) return config_values @staticmethod def set_config(config_path): if not os.path.exists(config_path): print("Config file {} doesn't exist. Check the config path or use default".format(config_path)) sys.exit(1) TannerConfig.config = TannerConfig.read_config(config_path) @staticmethod def get(section, value): try: res = TannerConfig.config[section][value] except (KeyError, TypeError): res = DEFAULT_CONFIG[section][value] return res DEFAULT_CONFIG = TannerConfig.read_config("/opt/tanner/data/config.yaml") ================================================ FILE: tanner/data/GeoLite2-City.mmdb ================================================ [File too large to display: 53.9 MB] ================================================ FILE: tanner/data/config.yaml ================================================ DATA: db_config: /opt/tanner/db/db_config.json dorks: /opt/tanner/data/dorks.pickle user_dorks: /opt/tanner/data/user_dorks.pickle crawler_stats: /opt/tanner/data/crawler_user_agents.txt geo_db: /opt/tanner/db/GeoLite2-City.mmdb tornado: /opt/tanner/data/tornado.py mako: /opt/tanner/data/mako.py TANNER: host: 0.0.0.0 port: 8090 WEB: host: 0.0.0.0 port: 8091 API: host: 0.0.0.0 port: 8092 auth: False auth_signature: tanner_api_auth PHPOX: host: 0.0.0.0 port: 8088 REDIS: host: localhost port: 6379 poolsize: 80 timeout: 1 EMULATORS: root_dir: /opt/tanner EMULATOR_ENABLED: sqli: True rfi: True lfi: True xss: True cmd_exec: True php_code_injection: True php_object_injection: True crlf: True xxe_injection: True template_injection: True SQLI: type: SQLITE db_name: tanner_db host: 127.0.0.1 user: root password: user_pass XXE_INJECTION: OUT_OF_BAND: False RFI: allow_insecure: False DOCKER: host_image: busybox:latest LOGGER: log_debug: /opt/tanner/tanner.log log_err: /opt/tanner/tanner.err MONGO: enabled: False URI: mongodb://localhost HPFEEDS: enabled: False HOST: localhost PORT: 10000 IDENT: '' SECRET: '' CHANNEL: tanner.events LOCALLOG: enabled: False PATH: /tmp/tanner_report.json CLEANLOG: enabled: False REMOTE_DOCKERFILE: GITHUB: "https://raw.githubusercontent.com/mushorg/tanner/master/docker/tanner/template_injection/Dockerfile" SESSIONS: delete_timeout: 300 ================================================ FILE: tanner/data/crawler_user_agents.txt ================================================ Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php) Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html) Clickagy Intelligence Bot v2 SirdataBot Mozilla/5.0 (compatible; Cliqzbot/1.0 +http://cliqz.com/company/cliqzbot) A6-Indexer/1.0 (http://www.a6corp.com/a6-web-scraping-policy/) Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot) Mozilla/5.0 (compatible; AhrefsBot/5.1; +http://ahrefs.com/robot/) Mozilla/5.0 (compatible; Genieo/1.0 http://www.genieo.com/webfilter.html) Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Screaming Frog SEO Spider/9.2 Mozilla/5.0 (compatible; Cliqzbot/2.0; +http://cliqz.com/company/cliqzbot) CCBot/2.0 (http://commoncrawl.org/faq/) Mozilla/5.0 (iPhone; CPU iPhone OS 60 like Mac OS X) AppleWebKit/536.26 (KHTML like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) YisouSpider Googlebot/2.1 (+http://www.google.com/bot.html) Mozilla/5.0 (compatible; Cliqzbot/1.0; +http://cliqz.com/company/cliqzbot) Googlebot/2.1 (+http://www.googlebot.com/bot.html) Mozilla/5.0 (compatible; 008/0.85; http://www.80legs.com/webcrawler.html) Gecko/2008032620 Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp) Googlebot-Video/1.0 SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html) Mozilla/5.0 (compatible; Googlebot/2.1;+http://www.google.com/bot.html) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; http://www.tropicdesigns.net) Mozilla/5.0 (iPhone; U; CPU iPhone OS 41 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html) Mozilla/5.0 (compatible; URLAppendBot/1.0; +http://www.profound.net/urlappendbot.html) DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html) Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07) Mozilla/5.0 (iPhone; CPU iPhone OS 60 like Mac OS X) AppleWebKit/536.26 (KHTML like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html) Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/) Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) Mozilla/5.0 (compatible; DomainAppender /1.0; +http://www.profound.net/domainappender) msnbot/2.0b (+http://search.msn.com/msnbot.htm) msnbot-media/1.1 (+http://search.msn.com/msnbot.htm) Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp) Mozilla/5.0 (compatible; YandexAntivirus/2.0; +http://yandex.com/bots) Mozilla/5.0 (iPhone; U; CPU iPhone OS 41 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html) Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/) Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36; 360Spider Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm) Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/) Mozilla/5.0 (compatible; proximic; +http://www.proximic.com) Mozilla/5.0 (iPhone; CPU iPhone OS 83 like Mac OS X) AppleWebKit/600.1.4 (KHTML like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php+) DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html) ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com) ================================================ FILE: tanner/data/db_config.json ================================================ { "name": "test1", "tables": [ { "table_name": "users", "schema": "CREATE TABLE users (id INTEGER PRIMARY KEY, username text, email text, password text);", "data_tokens": "I,L,E,P" }, { "table_name": "comments", "schema": "CREATE TABLE comments (comment_id INTEGER PRIMARY KEY, comment text);", "data_tokens": "I,T" } ] } ================================================ FILE: tanner/dorks_manager.py ================================================ import logging import math import os import pickle import random import re import uuid import aioredis from tanner import config from tanner.utils import patterns class DorksManager: dorks_key = uuid.uuid3(uuid.NAMESPACE_DNS, "dorks").hex user_dorks_key = uuid.uuid3(uuid.NAMESPACE_DNS, "user_dorks").hex def __init__(self): self.logger = logging.getLogger("tanner.dorks_manager.DorksManager") self.init_done = False @staticmethod async def push_init_dorks(file_name, redis_key, redis_client): dorks = None if os.path.exists(file_name): with open(file_name, "rb") as dorks_file: dorks = pickle.load(dorks_file) if dorks: if isinstance(dorks, str): dorks = dorks.split() if isinstance(dorks, set): dorks = [x for x in dorks if x is not None] await redis_client.sadd(redis_key, *dorks) async def extract_path(self, path, redis_client): extracted = re.match(patterns.QUERY, path) if extracted: extracted = extracted.group(0) try: await redis_client.sadd(self.user_dorks_key, *[extracted]) except aioredis.ConnectionError as connection_error: self.logger.exception("Problem with redis connection: %s", connection_error) async def init_dorks(self, redis_client): try: transaction = redis_client.multi() dorks_exist = transaction.exists(self.dorks_key) user_dorks_exist = transaction.exists(self.user_dorks_key) await transaction.execute() except (aioredis.RedisError, aioredis.ConnectionError) as redis_error: self.logger.exception("Problem with transaction: %s", redis_error) else: dorks_existed = await dorks_exist user_dorks_existed = await user_dorks_exist if not dorks_existed: await self.push_init_dorks(config.TannerConfig.get("DATA", "dorks"), self.dorks_key, redis_client) if not user_dorks_existed: await self.push_init_dorks( config.TannerConfig.get("DATA", "user_dorks"), self.user_dorks_key, redis_client ) self.init_done = True async def choose_dorks(self, redis_client): if not self.init_done: await self.init_dorks(redis_client) chosen_dorks = [] max_dorks = 50 try: transaction = redis_client.multi() dorks = transaction.smembers(self.dorks_key, encoding="utf-8") user_dorks = transaction.smembers(self.user_dorks_key, encoding="utf-8") await transaction.execute() except (aioredis.RedisError, aioredis.ConnectionError) as redis_error: self.logger.exception("Problem with transaction: %s", redis_error) else: dorks = await dorks user_dorks = await user_dorks chosen_dorks.extend(random.sample(dorks, random.randint(math.floor(0.5 * max_dorks), max_dorks))) try: if max_dorks > len(user_dorks): max_dorks = len(user_dorks) chosen_dorks.extend(random.sample(user_dorks, random.randint(math.floor(0.5 * max_dorks), max_dorks))) except TypeError: pass return chosen_dorks ================================================ FILE: tanner/emulators/__init__.py ================================================ ================================================ FILE: tanner/emulators/base.py ================================================ import mimetypes import re import urllib.parse import yarl from tanner import __version__ as tanner_version from tanner.config import TannerConfig from tanner.emulators import ( lfi, rfi, sqli, xss, cmd_exec, php_code_injection, php_object_injection, crlf, xxe_injection, template_injection, ) # noqa from tanner.utils import patterns class BaseHandler: def __init__(self, base_dir, db_name, loop=None): self.emulator_enabled = { "rfi": TannerConfig.get("EMULATOR_ENABLED", "rfi"), "sqli": TannerConfig.get("EMULATOR_ENABLED", "sqli"), "lfi": TannerConfig.get("EMULATOR_ENABLED", "lfi"), "xss": TannerConfig.get("EMULATOR_ENABLED", "xss"), "cmd_exec": TannerConfig.get("EMULATOR_ENABLED", "cmd_exec"), "php_code_injection": TannerConfig.get("EMULATOR_ENABLED", "php_code_injection"), "php_object_injection": TannerConfig.get("EMULATOR_ENABLED", "php_object_injection"), "crlf": TannerConfig.get("EMULATOR_ENABLED", "crlf"), "xxe_injection": TannerConfig.get("EMULATOR_ENABLED", "xxe_injection"), "template_injection": TannerConfig.get("EMULATOR_ENABLED", "template_injection"), } self.emulators = { "rfi": rfi.RfiEmulator(base_dir, loop=loop, allow_insecure=TannerConfig.get("RFI", "allow_insecure")) if self.emulator_enabled["rfi"] else None, "lfi": lfi.LfiEmulator() if self.emulator_enabled["lfi"] else None, "xss": xss.XssEmulator() if self.emulator_enabled["xss"] else None, "sqli": sqli.SqliEmulator(db_name, base_dir) if self.emulator_enabled["sqli"] else None, "cmd_exec": cmd_exec.CmdExecEmulator() if self.emulator_enabled["cmd_exec"] else None, "php_code_injection": php_code_injection.PHPCodeInjection(loop) if self.emulator_enabled["php_code_injection"] else None, "php_object_injection": php_object_injection.PHPObjectInjection(loop) if self.emulator_enabled["php_object_injection"] else None, "crlf": crlf.CRLFEmulator() if self.emulator_enabled["crlf"] else None, "xxe_injection": xxe_injection.XXEInjection(loop) if self.emulator_enabled["xxe_injection"] else None, "template_injection": template_injection.TemplateInjection(loop) if self.emulator_enabled["template_injection"] else None, } self.get_emulators = [ "sqli", "rfi", "lfi", "xss", "php_code_injection", "php_object_injection", "cmd_exec", "crlf", "xxe_injection", "template_injection", ] self.post_emulators = [ "sqli", "rfi", "lfi", "xss", "php_code_injection", "php_object_injection", "cmd_exec", "crlf", "xxe_injection", "template_injection", ] self.cookie_emulators = ["sqli", "php_object_injection"] def extract_get_data(self, path): """ Return all the GET parameter :param path (str): The URL path from which GET parameters are to be extracted :return: A MultiDictProxy object containg name and value of parameters """ path = urllib.parse.unquote(path) encodings = [("&&", "%26%26"), (";", "%3B")] for value, encoded_value in encodings: path = path.replace(value, encoded_value) get_data = yarl.URL(path).query return get_data async def get_emulation_result(self, session, data, target_emulators): """ Return emulation result for the vulnerabilty of highest order :param session (Session object): Current active session :param data (MultiDictProxy object): Data to be checked :param target_emulator (list): Emulators against which data is to be checked :return: A dict object containing name, order and paylod to be injected for vulnerability """ detection = dict(name="unknown", order=0) attack_params = {} for param_id, param_value in data.items(): for emulator in target_emulators: if TannerConfig.get("EMULATOR_ENABLED", emulator): possible_detection = self.emulators[emulator].scan(param_value) if param_value else None if possible_detection: if detection["order"] < possible_detection["order"]: detection = possible_detection if emulator not in attack_params: attack_params[emulator] = [] attack_params[emulator].append(dict(id=param_id, value=param_value)) if detection["name"] in self.emulators: emulation_result = await self.emulators[detection["name"]].handle(attack_params[detection["name"]], session) if emulation_result: detection["payload"] = emulation_result return detection async def handle_post(self, session, data): post_data = data["post_data"] detection = await self.get_emulation_result(session, post_data, self.post_emulators) return detection async def handle_cookies(self, session, data): cookies = data["cookies"] detection = await self.get_emulation_result(session, cookies, self.cookie_emulators) return detection async def handle_get(self, session, data): path = data["path"] get_data = self.extract_get_data(path) detection = dict(name="unknown", order=0) # dummy for wp-content if re.match(patterns.WORD_PRESS_CONTENT, path): detection = {"name": "wp-content", "order": 1} elif re.match(patterns.INDEX, path): detection = {"name": "index", "order": 1} # check attacks against get parameters possible_get_detection = await self.get_emulation_result(session, get_data, self.get_emulators) if possible_get_detection and detection["order"] < possible_get_detection["order"]: detection = possible_get_detection # check attacks against cookie values possible_cookie_detection = await self.handle_cookies(session, data) if possible_cookie_detection and detection["order"] < possible_cookie_detection["order"]: detection = possible_cookie_detection return detection @staticmethod def set_injectable_page(session): injectable_page = None if session: for page in reversed(session.paths): if mimetypes.guess_type(page["path"])[0] == "text/html": injectable_page = page["path"] return injectable_page async def emulate(self, data, session): if data["method"] == "POST": detection = await self.handle_post(session, data) else: detection = await self.handle_get(session, data) if "payload" not in detection: detection["type"] = 1 elif "payload" in detection: if "status_code" not in detection["payload"]: detection["type"] = 2 if detection["payload"]["page"]: injectable_page = self.set_injectable_page(session) if injectable_page is None: injectable_page = "/index.html" detection["payload"]["page"] = injectable_page else: detection["type"] = 3 detection["version"] = tanner_version return detection async def handle(self, data, session): detection = await self.emulate(data, session) return detection ================================================ FILE: tanner/emulators/cmd_exec.py ================================================ from tanner.utils import aiodocker_helper from tanner.utils import patterns class CmdExecEmulator: def __init__(self): self.helper = aiodocker_helper.AIODockerHelper() async def get_cmd_exec_results(self, payload): cmd = ["sh", "-c", payload] execute_result = await self.helper.execute_cmd(cmd) result = dict(value=execute_result, page=True) return result def scan(self, value): detection = None if patterns.CMD_ATTACK.match(value): detection = dict(name="cmd_exec", order=3) return detection async def handle(self, attack_params, session=None): result = await self.get_cmd_exec_results(attack_params[0]["value"]) return result ================================================ FILE: tanner/emulators/crlf.py ================================================ from tanner.utils import patterns class CRLFEmulator: def scan(self, value): detection = None if patterns.CRLF_ATTACK.match(value): detection = dict(name="crlf", order=2) return detection def get_crlf_results(self, attack_params): headers = {attack_params[0]["id"]: attack_params[0]["value"]} return headers async def handle(self, attack_params, session): result = self.get_crlf_results(attack_params) return dict(value="", page=True, headers=result) ================================================ FILE: tanner/emulators/lfi.py ================================================ import shlex from tanner.utils import aiodocker_helper from tanner.utils import patterns class LfiEmulator: def __init__(self): self.helper = aiodocker_helper.AIODockerHelper() async def get_lfi_result(self, file_path): # Terminate the string with NULL byte if "\x00" in file_path: file_path = file_path[: file_path.find("\x00")] cmd = ["sh", "-c", "cat {file}".format(file=shlex.quote(file_path))] execute_result = await self.helper.execute_cmd(cmd) if execute_result: # Nulls are not printable, so replace it with another line-ender execute_result = execute_result.replace("\x00", "\n") return execute_result def scan(self, value): detection = None if patterns.LFI_ATTACK.match(value): detection = dict(name="lfi", order=2) return detection async def handle(self, attack_params, session=None): lfi_result = await self.get_lfi_result(attack_params[0]["value"]) result = dict(value=lfi_result, page=False) return result ================================================ FILE: tanner/emulators/mysqli.py ================================================ import logging from tanner.utils import mysql_db_helper class MySQLIEmulator: def __init__(self, db_name): self.logger = logging.getLogger("tanner.mysqli_emulator") self.db_name = db_name self.helper = mysql_db_helper.MySQLDBHelper() async def setup_db(self): db_exists = await self.helper.check_db_exists(self.db_name) if not db_exists: await self.helper.setup_db_from_config(self.db_name) query_map = await self.helper.create_query_map(self.db_name) return query_map async def create_attacker_db(self, session): attacker_db_name = "attacker_" + session.sess_uuid.hex attacker_db = await self.helper.copy_db(self.db_name, attacker_db_name) session.associate_db(attacker_db) return attacker_db async def execute_query(self, query, db_name): result = [] conn = await self.helper.connect_to_db() cursor = await conn.cursor() await cursor.execute("USE {db_name}".format(db_name=db_name)) try: await cursor.execute(query) rows = await cursor.fetchall() for row in rows: result.append(list(row)) except Exception as mysql_error: self.logger.debug("Error while executing query: %s", mysql_error) result = str(mysql_error) return result ================================================ FILE: tanner/emulators/php_code_injection.py ================================================ import asyncio import logging from tanner.utils.php_sandbox_helper import PHPSandboxHelper from tanner.utils import patterns class PHPCodeInjection: def __init__(self, loop=None): self._loop = loop if loop is not None else asyncio.get_event_loop() self.logger = logging.getLogger("tanner.php_code_injection") self.helper = PHPSandboxHelper(self._loop) async def get_injection_result(self, code): vul_code = "".format(code=code) self.logger.debug("Getting the code injection results of %s from php sandbox", code) code_injection_result = await self.helper.get_result(vul_code) return code_injection_result def scan(self, value): detection = None if patterns.PHP_CODE_INJECTION.match(value): detection = dict(name="php_code_injection", order=3) return detection async def handle(self, attack_params, session=None): result = await self.get_injection_result(attack_params[0]["value"]) if not result or "stdout" not in result: self.logger.exception("Error while getting the injection results from php sandbox..") return dict(status_code=504) return dict(value=result["stdout"], page=False) ================================================ FILE: tanner/emulators/php_object_injection.py ================================================ import asyncio import logging from tanner.utils.php_sandbox_helper import PHPSandboxHelper from tanner.utils import patterns class PHPObjectInjection: def __init__(self, loop=None): self._loop = loop if loop is not None else asyncio.get_event_loop() self.logger = logging.getLogger("tanner.php_object_injection") self.helper = PHPSandboxHelper(self._loop) async def get_injection_result(self, code): """ Injects the code from attacker to vulnerable code and get emulation results from php sandbox. :param code (str): Input payload from attacker :return: object_injection_result (dict): file_md5 (md5 hash), stdout (injection result) as keys. """ vul_code = ( "insert, $ret);" "print $var[0];" "$this->date = date('d-m-y');" "$this->filename = '/tmp/logs/' . $this->date;" "file_put_contents($this->filename, $var[0], FILE_APPEND);" "}} " "$cmd = unserialize('%s');" "?>" % code ) self.logger.debug("Getting the object injection results of %s from php sandbox", code) object_injection_result = await self.helper.get_result(vul_code) return object_injection_result def scan(self, value): """ Scans the input payload to detect attack using regex :param value (str): code from attacker :return: detection (dict): name (attack name), order (attack order) as keys """ detection = None if patterns.PHP_OBJECT_INJECTION.match(value): detection = dict(name="php_object_injection", order=3) return detection async def handle(self, attack_params): """ Handler of emulator :param attack_params (list): contains dicts as elements with id and value (payload from attacker) as keys :return: (dict): value (result of emulator), page (if set to true the payload will be injected to index.html itself) as keys. """ result = await self.get_injection_result(attack_params[0]["value"]) if not result or "stdout" not in result: self.logger.exception("Error while getting the injection results from php sandbox..") return dict(status_code=504) return dict(value=result["stdout"], page=False) ================================================ FILE: tanner/emulators/rfi.py ================================================ import asyncio import ftplib import hashlib import logging import os import re import ssl import time from concurrent.futures import ThreadPoolExecutor import aiohttp import yarl from tanner.utils.php_sandbox_helper import PHPSandboxHelper from tanner.utils import patterns class RfiEmulator: def __init__(self, root_dir, loop=None, allow_insecure=False): self._loop = loop if loop is not None else asyncio.get_event_loop() self.script_dir = os.path.join(root_dir, "files") self.logger = logging.getLogger("tanner.rfi_emulator.RfiEmulator") self.helper = PHPSandboxHelper(self._loop) self.allow_insecure = allow_insecure async def download_file(self, path): file_name = None url = re.match(patterns.REMOTE_FILE_URL, path) if url is None: return None url = url.group(1) url = yarl.URL(url) if not os.path.exists(self.script_dir): os.makedirs(self.script_dir) if url.scheme == "ftp": pool = ThreadPoolExecutor() ftp_future = self._loop.run_in_executor(pool, self.download_file_ftp, url) file_name = await ftp_future else: ssl_context = False if self.allow_insecure else ssl.create_default_context() try: async with aiohttp.ClientSession(loop=self._loop) as client: async with await client.get(url, ssl=ssl_context) as resp: data = await resp.text() except aiohttp.ClientError as client_error: self.logger.exception("Error during downloading the rfi script %s", client_error) else: tmp_filename = url.name + str(time.time()) file_name = hashlib.md5(tmp_filename.encode("utf-8")).hexdigest() with open(os.path.join(self.script_dir, file_name), "bw") as rfile: self.logger.debug("Saving the RFI script %s", os.path.join(self.script_dir, file_name)) rfile.write(data.encode("utf-8")) return file_name def download_file_ftp(self, url): host = url.host ftp_path = url.path.rsplit("/", 1)[0][1:] name = url.name try: ftp = ftplib.FTP(host) ftp.login() ftp.cwd(ftp_path) tmp_filename = name + str(time.time()) file_name = hashlib.md5(tmp_filename.encode("utf-8")).hexdigest() with open(os.path.join(self.script_dir, file_name), "wb") as ftp_script: self.logger.debug("Saving the FTP file as %s", os.path.join(self.script_dir, file_name)) ftp.retrbinary("RETR %s" % name, ftp_script.write) except ftplib.all_errors as ftp_errors: self.logger.exception("Problem with ftp download %s", ftp_errors) return None else: return file_name async def get_rfi_result(self, path): rfi_result = None await asyncio.sleep(1) self.logger.info("Downloading the file has started from %s", path) file_name = await self.download_file(path) if file_name is None: return rfi_result with open(os.path.join(self.script_dir, file_name), "br") as script: script_data = script.read() rfi_result = await self.helper.get_result(script_data) return rfi_result def scan(self, value): detection = None if patterns.RFI_ATTACK.match(value): detection = dict(name="rfi", order=2) return detection async def handle(self, attack_params, session=None): result = await self.get_rfi_result(attack_params[0]["value"]) if not result or "stdout" not in result: return dict(value="", page=True) else: return dict(value=result["stdout"], page=False) ================================================ FILE: tanner/emulators/sqli.py ================================================ import logging import pylibinjection from tanner.config import TannerConfig from tanner.emulators import mysqli, sqlite class SqliEmulator: def __init__(self, db_name, working_dir): self.logger = logging.getLogger("tanner.sqli_emulator") if TannerConfig.get("SQLI", "type") == "MySQL": self.sqli_emulator = mysqli.MySQLIEmulator(db_name) else: self.sqli_emulator = sqlite.SQLITEEmulator(db_name, working_dir) self.query_map = None def scan(self, value): detection = None payload = bytes(value, "utf-8") sqli = pylibinjection.detect_sqli(payload) if int(sqli["sqli"]): detection = dict(name="sqli", order=2) return detection def map_query(self, attack_value): db_query = None param = attack_value["id"] param_value = attack_value["value"].replace("'", " ") tables = [] for table, columns in self.query_map.items(): for column in columns: if param == column["name"]: tables.append(dict(table_name=table, column=column)) if tables: if tables[0]["column"]["type"] == "INTEGER": db_query = "SELECT * from " + tables[0]["table_name"] + " WHERE " + param + "=" + param_value + ";" else: db_query = "SELECT * from " + tables[0]["table_name"] + " WHERE " + param + '="' + param_value + '";' return db_query async def get_sqli_result(self, attack_value, attacker_db): db_query = self.map_query(attack_value) if db_query is None: if TannerConfig.get("SQLI", "type") == "MySQL": error_result = "You have an error in your SQL syntax; check the manual\ that corresponds to your MySQL server version for the\ right syntax to use near {} at line 1".format( attack_value["id"] ) else: error_result = "SQL ERROR: near {}: syntax error".format(attack_value["id"]) self.logger.debug("Error while executing: %s", error_result) result = dict(value=error_result, page=True) else: execute_result = await self.sqli_emulator.execute_query(db_query, attacker_db) if isinstance(execute_result, list): execute_result = " ".join([str(x) for x in execute_result]) result = dict(value=execute_result, page=True) return result async def handle(self, attack_params, session): if self.query_map is None: self.query_map = await self.sqli_emulator.setup_db() attacker_db = await self.sqli_emulator.create_attacker_db(session) result = await self.get_sqli_result(attack_params[0], attacker_db) return result ================================================ FILE: tanner/emulators/sqlite.py ================================================ import os import sqlite3 import logging from tanner.utils import sqlite_db_helper class SQLITEEmulator: def __init__(self, db_name, working_dir): self.logger = logging.getLogger("tanner.sqlite_emulator") self.db_name = db_name self.working_dir = os.path.join(working_dir, "db/") self.helper = sqlite_db_helper.SQLITEDBHelper() async def setup_db(self): if not os.path.exists(self.working_dir): os.makedirs(self.working_dir) db = os.path.join(self.working_dir, self.db_name) if not os.path.exists(db): await self.helper.setup_db_from_config(self.working_dir, self.db_name) query_map = self.helper.create_query_map(self.working_dir, self.db_name) return query_map async def create_attacker_db(self, session): attacker_db_name = "attacker_" + session.sess_uuid.hex attacker_db = self.helper.copy_db(self.db_name, attacker_db_name, self.working_dir) session.associate_db(attacker_db) return attacker_db async def execute_query(self, query, db): result = [] conn = sqlite3.connect(db) cursor = conn.cursor() try: for row in cursor.execute(query): result.append(list(row)) except sqlite3.OperationalError as sqlite_error: self.logger.debug("Error while executing query: %s", sqlite_error) result = str(sqlite_error) return result ================================================ FILE: tanner/emulators/template_injection.py ================================================ import asyncio import logging from urllib.parse import unquote from tanner.utils import patterns from tanner.config import TannerConfig from tanner.utils.aiodocker_helper import AIODockerHelper class TemplateInjection: def __init__(self, loop=None): self._loop = loop if loop is not None else asyncio.get_event_loop() self.logger = logging.getLogger("tanner.template_injection") self.docker_helper = AIODockerHelper() self.remote_path = TannerConfig.get("REMOTE_DOCKERFILE", "GITHUB") async def get_injection_result(self, payload): execute_result = None # Build the custom image await self.docker_helper.setup_host_image(remote_path=self.remote_path, tag="template_injection:latest") if patterns.TEMPLATE_INJECTION_TORNADO.match(payload): work_dir = TannerConfig.get("DATA", "tornado") with open(work_dir, "r") as f: tornado_template = f.read().format(payload) cmd = ["python3", "-c", tornado_template] execute_result = await self.docker_helper.execute_cmd(cmd, "template_injection:latest") # Removing string "b''" from results if execute_result: execute_result = execute_result[2:-2] elif patterns.TEMPLATE_INJECTION_MAKO.match(payload): work_dir = TannerConfig.get("DATA", "mako") with open(work_dir, "r") as f: mako_template = f.read().format(payload) cmd = ["python3", "-c", mako_template] execute_result = await self.docker_helper.execute_cmd(cmd, "template_injection:latest") result = dict(value=execute_result, page=True) return result def scan(self, value): detection = None value = unquote(value) if patterns.TEMPLATE_INJECTION_TORNADO.match(value) or patterns.TEMPLATE_INJECTION_MAKO.match(value): detection = dict(name="template_injection", order=4) return detection async def handle(self, attack_params, session=None): attack_params[0]["value"] = unquote(attack_params[0]["value"]) result = await self.get_injection_result(attack_params[0]["value"]) return result ================================================ FILE: tanner/emulators/xss.py ================================================ from tanner.utils import patterns class XssEmulator: def scan(self, value): detection = None if patterns.XSS_ATTACK.match(value): detection = dict(name="xss", order=3) return detection def get_xss_result(self, session, attack_params): result = None value = "" for param in attack_params: value += param["value"] if not value else "\n" + param["value"] result = dict(value=value, page=True) return result async def handle(self, attack_params, session): xss_result = None xss_result = self.get_xss_result(session, attack_params) return xss_result ================================================ FILE: tanner/emulators/xxe_injection.py ================================================ import asyncio import logging from tanner.config import TannerConfig from tanner.utils.php_sandbox_helper import PHPSandboxHelper from tanner.utils import patterns class XXEInjection: def __init__(self, loop=None): self._loop = loop if loop is not None else asyncio.get_event_loop() self.logger = logging.getLogger("tanner.xxe_injection") self.helper = PHPSandboxHelper(self._loop) async def get_injection_result(self, code): """ Injects the code from attacker to vulnerable code and get emulation results from php sandbox. :param code (str): Input payload from attacker :return: object_injection_result (dict): file_md5 (md5 hash), stdout (injection result) as keys. """ vul_code = ( """loadXML($xml, LIBXML_NOENT | LIBXML_DTDLOAD); $data = simplexml_import_dom($dom); echo $data; ?>""" % code ) self.logger.debug("Getting the XXE injection results of %s from php sandbox", code) xxe_injection_result = await self.helper.get_result(vul_code) return xxe_injection_result def scan(self, value): """ Scans the input payload to detect attack using regex :param value (str): code from attacker :return: detection (dict): name (attack name), order (attack order) as keys """ detection = None if patterns.XXE_INJECTION.match(value): detection = dict(name="xxe_injection", order=3) return detection async def handle(self, attack_params): """ Handler of emulator :param attack_params (list): contains dicts as elements with id and value (payload from attacker) as keys :return: (dict): value (result of emulator), page (if set to true the payload will be injected to index.html itself) as keys. """ result = await self.get_injection_result(attack_params[0]["value"]) if not result or "stdout" not in result: self.logger.exception("Error while getting the injection results from php sandbox..") return dict(status_code=504) if TannerConfig.get("XXE_INJECTION", "OUT_OF_BAND"): self.logger.debug("Out of Band XXE injection detected..") return dict(value="", page=False) return dict(value=result["stdout"], page=False) ================================================ FILE: tanner/files/engines/mako.py ================================================ from mako.template import Template mako_template = Template("""{}""") template_injection_result = mako_template.render() print(template_injection_result) ================================================ FILE: tanner/files/engines/tornado.py ================================================ import tornado code = "{}" result = tornado.template.Template(code) template_injection_result = result.generate() print(template_injection_result) ================================================ FILE: tanner/redis_client.py ================================================ import asyncio import logging import aioredis from tanner.config import TannerConfig LOGGER = logging.getLogger(__name__) class RedisClient: @staticmethod async def get_redis_client(poolsize=None): redis_client = None try: host = TannerConfig.get("REDIS", "host") port = TannerConfig.get("REDIS", "port") username="" password="" if poolsize is None: poolsize = TannerConfig.get("REDIS", "poolsize") redis_client = aioredis.from_url( f"redis://{username}:{password}@{host}:{port}", encoding="utf-8", decode_responses=True, max_connections=poolsize ) except asyncio.TimeoutError as timeout_error: LOGGER.exception("Problem with redis connection. Please, check your redis server. %s", timeout_error) exit() return redis_client ================================================ FILE: tanner/reporting/__init__.py ================================================ ================================================ FILE: tanner/reporting/hpfeeds.py ================================================ # # hpfeeds.py # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, # MA 02111-1307 USA # # This code is combination of the following source codes: # # https://github.com/buffer/thug/blob/master/hpfeeds/hpfeeds.py # https://github.com/DinoTools/dionaea/blob/master/modules/python/dionaea/hpfeeds.py # import logging import struct import hashlib import sys import socket import time logger = logging.getLogger("pyhpfeeds") BUFSIZ = 16384 OP_ERROR = 0 OP_INFO = 1 OP_AUTH = 2 OP_PUBLISH = 3 OP_SUBSCRIBE = 4 MAXBUF = 1024 ** 2 SIZES = { OP_ERROR: 5 + MAXBUF, OP_INFO: 5 + 256 + 20, OP_AUTH: 5 + 256 + 20, OP_PUBLISH: 5 + MAXBUF, OP_SUBSCRIBE: 5 + 256 * 2, } __all__ = ["new", "FeedException"] class BadClient(Exception): pass class FeedException(Exception): pass class Disconnect(Exception): pass # packs a string with 1 byte length field def strpack8(x): if isinstance(x, str): x = x.encode("latin1") return struct.pack("!B", len(x) % 0xFF) + x # unpacks a string with 1 byte length field def strunpack8(x): lenght = x[0] return x[1 : 1 + lenght], x[1 + lenght :] def msghdr(op, data): return struct.pack("!iB", 5 + len(data), op) + data def msgpublish(ident, chan, data): return msghdr(OP_PUBLISH, strpack8(ident) + strpack8(chan) + data.encode("latin1")) def msgsubscribe(ident, chan): if isinstance(chan, str): chan = chan.encode("latin1") return msghdr(OP_SUBSCRIBE, strpack8(ident) + chan) def msgauth(rand, ident, secret): auth_hash = hashlib.sha1(rand + secret.encode("latin1")).digest() return msghdr(OP_AUTH, strpack8(ident) + auth_hash) class FeedUnpack(object): def __init__(self): self.buf = bytearray() def __iter__(self): return self def __next__(self): return self.unpack() def feed(self, data): self.buf.extend(data) def unpack(self): if len(self.buf) < 5: raise StopIteration("No message.") ml, opcode = struct.unpack("!iB", self.buf[:5]) if ml > SIZES.get(opcode, MAXBUF): raise BadClient("Not respecting MAXBUF.") if len(self.buf) < ml: raise StopIteration("No message.") data = self.buf[5:ml] del self.buf[:ml] return opcode, data class HPC(object): def __init__(self, host, port, ident, secret, timeout=3, reconnect=False, reconnect_attempts=3, sleepwait=20): self.host, self.port = host, port self.ident, self.secret = ident, secret self.timeout = timeout self.reconnect = reconnect self.reconnect_attempts = reconnect_attempts self.sleepwait = sleepwait self.brokername = "unknown" self.connected = False self.stopped = False self.s = None self.unpacker = FeedUnpack() try: self.tryconnect() except Exception: raise def send(self, data): try: self.s.sendall(data) except socket.timeout: logger.warn("Timeout while sending - disconnect.") raise Disconnect() except socket.error as e: logger.warn("Socket error: %s", e) raise Disconnect() def tryconnect(self): if not self.connected: i = 0 while i < self.reconnect_attempts: i += 1 try: self.connect() break except socket.error as e: logger.warn("Socket error while connecting: {0}".format(e)) time.sleep(self.sleepwait) except FeedException as e: logger.warn("FeedException while connecting: {0}".format(e)) time.sleep(self.sleepwait) except Disconnect as e: logger.warn("Disconnect while connecting.") time.sleep(self.sleepwait) if not self.connected: raise Disconnect() else: logger.warning("Trying to connect, but already connected!") def close_old(self): if self.s: try: self.s.close() except Exception: pass def connect(self): self.close_old() logger.info("connecting to %s:%s", self.host, self.port) self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.s.settimeout(self.timeout) try: self.s.connect((self.host, self.port)) except Exception: raise FeedException("Could not connect to broker.") self.connected = True try: d = self.s.recv(BUFSIZ) except socket.timeout: raise FeedException("Connection receive timeout.") self.unpacker.feed(d) for opcode, data in self.unpacker: if opcode == OP_INFO: name, rest = strunpack8(data) rand = bytes(rest) logger.debug("info message name: %s, rand: %s", name, repr(rand)) self.brokername = name self.s.send(msgauth(rand, self.ident, self.secret)) break else: raise FeedException("Expected info message at this point.") self.s.settimeout(None) self.s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) if sys.platform in ("linux2",): self.s.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 60) def publish(self, chaninfo, data): if type(chaninfo) == str: chaninfo = [chaninfo] for c in chaninfo: try: self.send(msgpublish(self.ident, c, data)) except Disconnect: logger.info("Disconnected from broker (in publish).") self.connected = False if self.reconnect: self.tryconnect() else: logger.info("Do not reconnect, aborting.") self.close_old() raise def close(self): try: self.s.close() except Exception: logger.warn("Socket exception when closing.") def new(host=None, port=10000, ident=None, secret=None, reconnect=True): try: return HPC(host, port, ident, secret, reconnect=reconnect) except Exception: raise ================================================ FILE: tanner/reporting/log_hpfeeds.py ================================================ import traceback import json from datetime import datetime import tanner.reporting.hpfeeds as hpfeeds from tanner import config class Reporting: def __init__(self): # Create the connection self.host = config.TannerConfig.get("HPFEEDS", "HOST") self.port = config.TannerConfig.get("HPFEEDS", "PORT") self.ident = config.TannerConfig.get("HPFEEDS", "IDENT") self.secret = config.TannerConfig.get("HPFEEDS", "SECRET") self.channel = config.TannerConfig.get("HPFEEDS", "CHANNEL") self.reconnect = True def connect(self): try: self.hpc = hpfeeds.new(self.host, self.port, self.ident, self.secret, self.reconnect) self.connected_state = True except Exception: self.connected_state = False def connected(self): return self.connected_state def create_session(self, session_data): session_data["timestamp"] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f") event_data = json.dumps(session_data) try: self.hpc.publish(self.channel, event_data) except Exception: self.connected_state = False traceback.print_exc() ================================================ FILE: tanner/reporting/log_local.py ================================================ import json from datetime import datetime from tanner import config class Reporting: @staticmethod def create_session(session_data): report_file = config.TannerConfig.get("LOCALLOG", "PATH") with open(report_file, "a") as out: session_data["timestamp"] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f") json.dump(session_data, out) out.write("\n") ================================================ FILE: tanner/reporting/log_mongodb.py ================================================ try: import pymongo MONGO = True except ImportError: MONGO = False from bson.objectid import ObjectId from gridfs import GridFS from tanner import config class Reporting: def __init__(self): if MONGO: # Create the connection mongo_uri = config.TannerConfig.get("MONGO", "URI") connection = pymongo.MongoClient(mongo_uri) # Connect to Databases. tandb = connection["tanner"] tandbfs = connection["voldbfs"] # Get Collections self.tan_sessions = tandb.sessions self.tan_files = GridFS(tandbfs) # Indexes self.tan_sessions.create_index([("$**", "text")]) else: print("pymongo not found. pip install pymongo") def update_session(self, session_id, new_values): session_id = ObjectId(session_id) self.tan_sessions.update_one({"_id": session_id}, {"$set": new_values}) return True def create_session(self, session_data): session_id = self.tan_sessions.insert_one(session_data).inserted_id return session_id ================================================ FILE: tanner/server.py ================================================ import asyncio import json import logging import yarl from aiohttp import web from tanner import dorks_manager, redis_client from tanner.sessions import session_manager from tanner.config import TannerConfig from tanner.emulators import base from tanner.reporting.log_local import Reporting as local_report from tanner.reporting.log_mongodb import Reporting as mongo_report from tanner.reporting.log_hpfeeds import Reporting as hpfeeds_report from tanner import __version__ as tanner_version class TannerServer: def __init__(self): base_dir = TannerConfig.get("EMULATORS", "root_dir") db_name = TannerConfig.get("SQLI", "db_name") self.session_manager = session_manager.SessionManager() self.delete_timeout = TannerConfig.get("SESSIONS", "delete_timeout") self.dorks = dorks_manager.DorksManager() self.base_handler = base.BaseHandler(base_dir, db_name) self.logger = logging.getLogger(__name__) self.redis_client = None if TannerConfig.get("HPFEEDS", "enabled") is True: self.hpf = hpfeeds_report() self.hpf.connect() if self.hpf.connected() is False: self.logger.warning("hpfeeds not connected - no hpfeeds messages will be created") @staticmethod def _make_response(msg): response_message = dict(version=tanner_version, response=dict(message=msg)) return response_message @staticmethod async def default_handler(request): return web.Response(text="Tanner server") async def handle_event(self, request): data = await request.read() try: data = json.loads(data.decode("utf-8")) path = yarl.URL(data["path"]).human_repr() except (TypeError, ValueError, KeyError) as error: self.logger.exception("error parsing request: %s", data) response_msg = self._make_response(msg=type(error).__name__) else: session, _ = await self.session_manager.add_or_update_session(data, self.redis_client) self.logger.info("Requested path %s", path) await self.dorks.extract_path(path, self.redis_client) detection = await self.base_handler.handle(data, session) session.set_attack_type(path, detection["name"]) response_msg = self._make_response(msg=dict(detection=detection, sess_uuid=session.get_uuid())) self.logger.info("TANNER response %s", response_msg) session_data = data session_data["response_msg"] = response_msg # Log to Mongo if TannerConfig.get("MONGO", "enabled") is True: db = mongo_report() session_id = db.create_session(session_data) self.logger.info("Writing session to DB: {}".format(session_id)) # Log to hpfeeds if TannerConfig.get("HPFEEDS", "enabled") is True: if self.hpf.connected(): self.hpf.create_session(session_data) if TannerConfig.get("LOCALLOG", "enabled") is True: lr = local_report() lr.create_session(session_data) return web.json_response(response_msg) async def handle_dorks(self, request): dorks = await self.dorks.choose_dorks(self.redis_client) response_msg = dict(version=tanner_version, response=dict(dorks=dorks)) return web.json_response(response_msg) async def handle_version(self, request): response_msg = dict(version=tanner_version) return web.json_response(response_msg) async def on_shutdown(self, app): await self.session_manager.delete_sessions_on_shutdown(self.redis_client) await self.redis_client.close() async def delete_sessions(self): try: while True: await self.session_manager.delete_old_sessions(self.redis_client) await asyncio.sleep(self.delete_timeout) except asyncio.CancelledError: pass def setup_routes(self, app): app.router.add_route("*", "/", self.default_handler) app.router.add_post("/event", self.handle_event) app.router.add_get("/dorks", self.handle_dorks) app.router.add_get("/version", self.handle_version) async def make_app(self): app = web.Application() app.on_shutdown.append(self.on_shutdown) self.setup_routes(app) app.on_startup.append(self.start_background_delete) app.on_cleanup.append(self.cleanup_background_tasks) return app async def start_background_delete(self, app): app["session_delete"] = asyncio.ensure_future(self.delete_sessions()) async def cleanup_background_tasks(self, app): app["session_delete"].cancel() await app["session_delete"] def start(self): loop = asyncio.get_event_loop() self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client()) host = TannerConfig.get("TANNER", "host") port = TannerConfig.get("TANNER", "port") web.run_app(self.make_app(), host=host, port=port) ================================================ FILE: tanner/sessions/__init__.py ================================================ ================================================ FILE: tanner/sessions/session.py ================================================ import json import time import uuid from urllib.parse import urlparse from tanner.config import TannerConfig from tanner.utils import aiodocker_helper from tanner.utils.mysql_db_helper import MySQLDBHelper from tanner.utils.sqlite_db_helper import SQLITEDBHelper class Session: KEEP_ALIVE_TIME = 75 def __init__(self, data): try: self.ip = data["peer"]["ip"] self.port = data["peer"]["port"] self.user_agent = data["headers"]["user-agent"] self.snare_uuid = data["uuid"] self.paths = [{"path": data["path"], "timestamp": time.time(), "response_status": data["status"]}] self.referer = None if "referer" in data["headers"]: ref = urlparse(data["headers"]["referer"]) self.referer = ref.path self.cookies = data["cookies"] self.associated_db = None self.associated_env = None except KeyError: raise self.sess_uuid = uuid.uuid4() self.start_timestamp = time.time() self.timestamp = time.time() self.count = 1 def update_session(self, data): self.timestamp = time.time() self.count += 1 self.paths.append({"path": data["path"], "timestamp": time.time(), "response_status": data["status"]}) for (key, value) in data["cookies"].items(): self.cookies.update({key: value}) def is_expired(self): exp_time = self.timestamp + self.KEEP_ALIVE_TIME if time.time() - exp_time > 0: return True def to_json(self): sess = dict( peer=dict(ip=self.ip, port=self.port), user_agent=self.user_agent, snare_uuid=self.snare_uuid, sess_uuid=self.sess_uuid.hex, start_time=self.start_timestamp, end_time=self.timestamp, count=self.count, paths=self.paths, cookies=self.cookies, referer=self.referer, ) return json.dumps(sess) def set_attack_type(self, path, attack_type): for sess_path in self.paths: if sess_path["path"] == path: sess_path.update({"attack_type": attack_type}) def associate_db(self, db_name): self.associated_db = db_name async def remove_associated_db(self): if TannerConfig.get("SQLI", "type") == "MySQL": await MySQLDBHelper().delete_db(self.associated_db) else: SQLITEDBHelper().delete_db(self.associated_db) def associate_env(self, env): self.associated_env = env async def remove_associated_env(self): await aiodocker_helper.AIODockerHelper().delete_container(self.associated_env) def get_uuid(self): return str(self.sess_uuid) ================================================ FILE: tanner/sessions/session_analyzer.py ================================================ import asyncio import json import logging import socket from geoip2.database import Reader import geoip2 import aioredis from tanner.dorks_manager import DorksManager from tanner.config import TannerConfig class SessionAnalyzer: def __init__(self, loop=None): self._loop = loop if loop is not None else asyncio.get_event_loop() self.queue = asyncio.Queue() self.logger = logging.getLogger("tanner.session_analyzer.SessionAnalyzer") self.attacks = ["sqli", "rfi", "lfi", "xss", "php_code_injection", "cmd_exec", "crlf"] async def analyze(self, session_key, redis_client): session = None await asyncio.sleep(1) try: session = await redis_client.get(session_key, encoding="utf-8") session = json.loads(session) except (aioredis.ConnectionError, TypeError, ValueError) as error: self.logger.exception("Can't get session for analyze: %s", error) else: result = await self.create_stats(session, redis_client) await self.queue.put(result) await self.save_session(redis_client) async def save_session(self, redis_client): while not self.queue.empty(): session = await self.queue.get() s_key = session["snare_uuid"] del_key = session["sess_uuid"] try: await redis_client.zadd(s_key, session["start_time"], json.dumps(session)) await redis_client.delete(*[del_key]) except aioredis.ConnectionError as redis_error: self.logger.exception("Error with redis. Session will be returned to the queue: %s", redis_error) self.queue.put(session) async def create_stats(self, session, redis_client): sess_duration = session["end_time"] - session["start_time"] referer = None if sess_duration != 0: rps = session["count"] / sess_duration else: rps = 0 location_info = await self._loop.run_in_executor(None, self.find_location, session["peer"]["ip"]) tbr, errors, hidden_links, attack_types = await self.analyze_paths(session["paths"], redis_client) attack_count = self.set_attack_count(attack_types) stats = dict( sess_uuid=session["sess_uuid"], peer_ip=session["peer"]["ip"], peer_port=session["peer"]["port"], location=location_info, user_agent=session["user_agent"], snare_uuid=session["snare_uuid"], start_time=session["start_time"], end_time=session["end_time"], requests_in_second=rps, approx_time_between_requests=tbr, accepted_paths=session["count"], errors=errors, hidden_links=hidden_links, attack_types=attack_types, attack_count=attack_count, paths=session["paths"], cookies=session["cookies"], referer=session["referer"], ) owner = await self.choose_possible_owner(stats) stats.update(owner) return stats @staticmethod async def analyze_paths(paths, redis_client): tbr = [] attack_types = [] current_path = paths[0] dorks = await redis_client.smembers(DorksManager.dorks_key) for _, path in enumerate(paths, start=1): tbr.append(path["timestamp"] - current_path["timestamp"]) current_path = path tbr_average = sum(tbr) / float(len(tbr)) errors = 0 hidden_links = 0 for path in paths: if path["response_status"] != 200: errors += 1 if path["path"] in dorks: hidden_links += 1 if "attack_type" in path: attack_types.append(path["attack_type"]) return tbr_average, errors, hidden_links, attack_types def set_attack_count(self, attack_types): attacks = self.attacks.copy() attacks.append("index") attack_count = {k: 0 for k in attacks} for attack in attacks: attack_count[attack] = attack_types.count(attack) count = {k: v for k, v in attack_count.items() if v != 0} return count async def choose_possible_owner(self, stats): owner_names = ["user", "tool", "crawler", "attacker", "admin"] possible_owners = {k: 0.0 for k in owner_names} if stats["peer_ip"] == "127.0.0.1" or stats["peer_ip"] == "::1": possible_owners["admin"] = 1.0 with open(TannerConfig.get("DATA", "crawler_stats")) as f: bots_owner = await self._loop.run_in_executor(None, f.read) crawler_hosts = ["googlebot.com", "baiduspider", "search.msn.com", "spider.yandex.com", "crawl.sogou.com"] possible_owners["crawler"], possible_owners["tool"] = await self.detect_crawler( stats, bots_owner, crawler_hosts ) possible_owners["attacker"] = await self.detect_attacker(stats, bots_owner, crawler_hosts) maxcf = max([possible_owners["crawler"], possible_owners["attacker"], possible_owners["tool"]]) possible_owners["user"] = round(1 - maxcf, 2) owners = {k: v for k, v in possible_owners.items() if v != 0} return {"possible_owners": owners} @staticmethod def find_location(ip): reader = Reader(TannerConfig.get("DATA", "geo_db")) try: location = reader.city(ip) info = dict( country=location.country.name, country_code=location.country.iso_code, city=location.city.name, zip_code=location.postal.code, ) except geoip2.errors.AddressNotFoundError: info = "NA" # When IP doesn't exist in the db, set info as "NA - Not Available" return info async def detect_crawler(self, stats, bots_owner, crawler_hosts): for path in stats["paths"]: if path["path"] == "/robots.txt": return (1.0, 0.0) if stats["requests_in_second"] > 10: if stats["referer"] is not None: return (0.0, 0.5) if stats["user_agent"] is not None and stats["user_agent"] in bots_owner: return (0.85, 0.15) return (0.5, 0.85) if stats["user_agent"] is not None and stats["user_agent"] in bots_owner: hostname, _, _ = await self._loop.run_in_executor(None, socket.gethostbyaddr, stats["peer_ip"]) if hostname is not None: for crawler_host in crawler_hosts: if crawler_host in hostname: return (0.75, 0.15) return (0.25, 0.15) return (0.0, 0.0) async def detect_attacker(self, stats, bots_owner, crawler_hosts): if set(stats["attack_types"]).intersection(self.attacks): return 1.0 if stats["requests_in_second"] > 10: return 0.0 if stats["user_agent"] is not None and stats["user_agent"] in bots_owner: hostname, _, _ = await self._loop.run_in_executor(None, socket.gethostbyaddr, stats["peer_ip"]) if hostname is not None: for crawler_host in crawler_hosts: if crawler_host in hostname: return 0.25 return 0.75 if stats["hidden_links"] > 0: return 0.5 return 0.0 ================================================ FILE: tanner/sessions/session_manager.py ================================================ import logging import hashlib import aioredis from tanner.sessions.session import Session from tanner.sessions.session_analyzer import SessionAnalyzer class SessionManager: def __init__(self, loop=None): self.sessions = {} self.analyzer = SessionAnalyzer(loop=loop) self.logger = logging.getLogger(__name__) async def add_or_update_session(self, raw_data, redis_client): # handle raw data valid_data = self.validate_data(raw_data) # push snare uuid into redis. await redis_client.sadd("snare_ids", *[valid_data["uuid"]]) session_id = self.get_session_id(valid_data) if session_id not in self.sessions: try: new_session = Session(valid_data) except KeyError as key_error: self.logger.exception("Error during session creation: %s", key_error) return self.sessions[session_id] = new_session return new_session, session_id else: self.sessions[session_id].update_session(valid_data) # prepare the list of sessions return self.sessions[session_id], session_id @staticmethod def validate_data(data): if "peer" not in data: peer = dict(ip=None, port=None) data["peer"] = peer data["headers"] = dict((k.lower(), v) for k, v in data["headers"].items()) if "user-agent" not in data["headers"]: data["headers"]["user-agent"] = None if "path" not in data: data["path"] = None if "uuid" not in data: data["uuid"] = None if "status" not in data: data["status"] = 200 if "error" not in data else 500 if "cookies" not in data: data["cookies"] = dict(sess_uuid=None) if "cookies" in data and "sess_uuid" not in data["cookies"]: data["cookies"]["sess_uuid"] = None return data def get_session_id(self, data): ip = data["peer"]["ip"] user_agent = data["headers"]["user-agent"] sess_uuid = data["cookies"]["sess_uuid"] sess_id_string = "{ip}{user_agent}{sess_uuid}".format(ip=ip, user_agent=user_agent, sess_uuid=sess_uuid) return hashlib.md5(sess_id_string.encode()).hexdigest() async def delete_old_sessions(self, redis_client): id_for_deletion = [sess_id for sess_id, sess in self.sessions.items() if sess.is_expired()] for sess_id in id_for_deletion: is_deleted = await self.delete_session(self.sessions[sess_id], redis_client) if is_deleted: try: del self.sessions[sess_id] except ValueError: continue async def delete_sessions_on_shutdown(self, redis_client): id_for_deletion = list(self.sessions.keys()) for sess_id in id_for_deletion: is_deleted = await self.delete_session(self.sessions[sess_id], redis_client) if is_deleted: del self.sessions[sess_id] try: assert len(self.sessions) == 0 except AssertionError: self.logger.exception("Not all sessions were moved to the storage!") async def delete_session(self, sess, redis_client): await sess.remove_associated_db() if sess.associated_env is not None: await sess.remove_associated_env() try: await redis_client.set(sess.get_uuid(), sess.to_json()) await self.analyzer.analyze(sess.get_uuid(), redis_client) except aioredis.ConnectionError as redis_error: self.logger.exception("Error connect to redis, session stay in memory. %s", redis_error) return False else: return True ================================================ FILE: tanner/tests/__init__.py ================================================ ================================================ FILE: tanner/tests/test_aiodocker_helper.py ================================================ import asyncio import unittest from tanner.utils.aiodocker_helper import AIODockerHelper class TestAioDockerHelper(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) self.handler = AIODockerHelper() self.image = None self.expected_result = None self.returned_result = None def test_setup_host_image(self): self.image = "busybox:latest" async def test(): await self.handler.setup_host_image() self.returned_result = await self.handler.docker_client.images.list(filter=self.image) self.loop.run_until_complete(test()) self.assertTrue(len(self.returned_result) > 0) def test_get_container(self): container_name = "test_get_container" async def test(): await self.handler.create_container(container_name) self.returned_result = await self.handler.get_container(container_name) await self.handler.delete_container(container_name) self.loop.run_until_complete(test()) self.assertTrue(self.returned_result._id) def test_create_container(self): container_name = "test_create_container" async def test(): container = await self.handler.create_container(container_name=container_name) await container.start() self.returned_result = await container.show() await self.handler.delete_container(container_name) self.loop.run_until_complete(test()) self.assertFalse(self.returned_result["State"]["Running"]) def test_execute_cmd(self): cmd = ["sh", "-c", "echo 'Hello!'"] async def test(): self.returned_result = await self.handler.execute_cmd(cmd) self.loop.run_until_complete(test()) self.expected_result = "Hello!" self.assertIn(self.expected_result, self.returned_result) def test_delete_container(self): container_name = "test_delete_container" async def test(): await self.handler.create_container(container_name) await self.handler.delete_container(container_name) self.returned_result = await self.handler.get_container(container_name) self.loop.run_until_complete(test()) self.assertEqual(self.returned_result, None) def tearDown(self): self.loop.run_until_complete(self.handler.docker_client.close()) self.loop.close() ================================================ FILE: tanner/tests/test_api.py ================================================ import unittest import asyncio import aioredis import itertools from unittest import mock from tanner.api.api import Api from tanner import redis_client from tanner.utils.asyncmock import AsyncMock class TestApi(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() self.redis_client = None self.snare_uuid = "9a631aee-2b52-4108-9831-b495ac8afa80" self.uuid = "da1811cd19d748058bc02ee5bf9029d4" self.returned_content = None self.expected_content = None self.conn = None self.key = None async def connect(): self.redis_client = await redis_client.RedisClient.get_redis_client() self.loop.run_until_complete(connect()) self.handler = Api(self.redis_client) def test_return_snares(self): self.expected_content = ["9a631aee-2b52-4108-9831-b495ac8afa80", "8b901tyg-2b65-3428-9765-b431vhm4fu76"] self.key = b"snare_ids" async def setup(): await self.redis_client.sadd(self.key, self.snare_uuid.encode()) await self.redis_client.sadd(self.key, "8b901tyg-2b65-3428-9765-b431vhm4fu76".encode()) async def test(): self.returned_content = await self.handler.return_snares() self.loop.run_until_complete(setup()) self.loop.run_until_complete(test()) for id in self.returned_content: assert id in self.expected_content def test_return_snares_error(self): self.redis_client.smembers = mock.Mock(side_effect=aioredis.ConnectionError) async def test(): self.returned_content = await self.handler.return_snares() with self.assertLogs(level="ERROR") as log: self.loop.run_until_complete(test()) self.assertIn("Can not connect to redis", log.output[0]) def test_return_snare_stats(self): sessions = [ {"end_time": 2.00, "start_time": 0.00, "attack_types": ["lfi", "xss"]}, {"end_time": 4.00, "start_time": 2.00, "attack_types": ["rfi", "lfi", "cmd_exec"]}, ] self.handler.return_snare_info = AsyncMock(return_value=sessions) self.expected_content = { "attack_frequency": {"cmd_exec": 1, "lfi": 2, "rfi": 1, "sqli": 0, "xss": 1}, "total_duration": 4.0, "total_sessions": 2, } async def test(): self.returned_content = await self.handler.return_snare_stats(self.snare_uuid) self.loop.run_until_complete(test()) self.assertEqual(self.returned_content, self.expected_content) def test_return_snare_info(self): self.member1 = ['{"end_time": 2.00, "start_time": 0.00 }', '{"attack_types": ["rfi"]}'] self.keys = [self.snare_uuid.encode(), "4b901tyg-2b65-3428-9765-b431vhm4fu76".encode()] self.scores = [0, 2] self.pair1 = list(itertools.chain(*zip(self.scores, self.member1))) self.member2 = ['{"user_agent": "Mozilla", "peer_ip": "127.0.0.1"}'] self.pair2 = list(itertools.chain(*zip(self.scores, self.member2))) self.returned_content = [] self.expected_content = [ [{"attack_types": ["rfi"]}, {"end_time": 2.0, "start_time": 0.0}], [{"user_agent": "Mozilla", "peer_ip": "127.0.0.1"}], ] async def setup(): await self.redis_client.zadd(self.snare_uuid.encode(), *self.pair1) await self.redis_client.zadd("4b901tyg-2b65-3428-9765-b431vhm4fu76".encode(), *self.pair2) self.loop.run_until_complete(setup()) async def test(id): result = await self.handler.return_snare_info(id, count=2) self.returned_content.append(result) for key in self.keys: self.loop.run_until_complete(test(key)) self.assertEqual(self.expected_content, self.returned_content) def test_return_snare_info_error(self): self.redis_client.zrevrangebyscore = mock.Mock(side_effect=aioredis.ConnectionError) async def test(): self.returned_content = await self.handler.return_snare_info(self.uuid) with self.assertLogs(level="ERROR") as log: self.loop.run_until_complete(test()) self.assertIn("Can not connect to redis", log.output[0]) def test_return_session_info(self): sessions = [ {"sess_uuid": "c546114f97f548f982756495f963e280"}, {"sess_uuid": "da1811cd19d748058bc02ee5bf9029d4"}, ] self.handler.return_snare_info = AsyncMock(return_value=sessions) self.expected_content = {"sess_uuid": "da1811cd19d748058bc02ee5bf9029d4"} async def test(): self.returned_content = await self.handler.return_session_info(self.uuid, self.snare_uuid) self.loop.run_until_complete(test()) self.assertEqual(self.returned_content, self.expected_content) def test_return_session_info_none(self): self.handler.return_snares = AsyncMock( return_value=["8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4", "6ea6aw67-7821-4085-7u6t-q1io3p0i90b1"] ) async def mock_return_snare_info(snare_uuid): sessions = None if snare_uuid == "8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4": sessions = [{"sess_uuid": "da1811cd19d748058bc02ee5bf9029d4"}] if snare_uuid == "6ea6aw67-7821-4085-7u6t-q1io3p0i90b1": sessions = [{"sess_uuid": "c546114f97f548f982756495f963e280"}] return sessions self.handler.return_snare_info = mock_return_snare_info self.expected_content = {"sess_uuid": "da1811cd19d748058bc02ee5bf9029d4"} async def test(): self.returned_content = await self.handler.return_session_info(self.uuid) self.loop.run_until_complete(test()) self.assertEqual(self.returned_content, self.expected_content) def test_return_sessions(self): self.handler.return_snares = AsyncMock(return_value=["8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4"]) sessions = [ {"user_agent": "Mozilla/5.0", "peer_ip": "10.0.0.3"}, { "attack_types": ["lfi", "xss"], "possible_owners": ["crawler"], "start_time": 148580, "end_time": 148588, "snare_uuid": [self.snare_uuid], }, ] self.handler.return_snare_info = AsyncMock(return_value=sessions) self.filters = { "user_agent": "Mozilla", "peer_ip": "10.0.0.1", "attack_types": "xss", "possible_owners": "crawler", "start_time": 148575, "end_time": 148590, "snare_uuid": self.snare_uuid, } self.handler.apply_filter = mock.Mock() def mock_result(filter_name, filter_value, sess): if sess == {"user_agent": "Mozilla/5.0", "peer_ip": "10.0.0.3"}: return True else: return False self.handler.apply_filter.side_effect = mock_result self.expected_content = [{"user_agent": "Mozilla/5.0", "peer_ip": "10.0.0.3"}] calls = [ mock.call("user_agent", "Mozilla", {"user_agent": "Mozilla/5.0", "peer_ip": "10.0.0.3"}), mock.call( "attack_types", "xss", { "attack_types": ["lfi", "xss"], "possible_owners": ["crawler"], "start_time": 148580, "end_time": 148588, "snare_uuid": [self.snare_uuid], }, ), ] async def test(): self.returned_content = await self.handler.return_sessions(self.filters) self.loop.run_until_complete(test()) self.handler.apply_filter.assert_has_calls(calls, any_order=True) self.assertEqual(self.expected_content, self.returned_content) def test_return_sessions_error(self): self.handler.return_snares = AsyncMock(return_value=["8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4"]) session = [{"attack_types": ["rfi", "lfi"]}] self.handler.return_snare_info = AsyncMock(return_value=session) self.filters = {"attacktypes": "lfi"} self.expected_content = "Invalid filter : attacktypes" async def test(): self.returned_content = await self.handler.return_sessions(self.filters) self.loop.run_until_complete(test()) self.assertEqual(self.expected_content, self.returned_content) def test_apply_filter_user_agent(self): filter_name = "user_agent" filter_value = "Mozilla" session = {"user_agent": "Mozilla/5.0"} self.returned_content = self.handler.apply_filter(filter_name, filter_value, session) self.assertTrue(self.returned_content) def test_apply_filter_user_agent_false(self): filter_name = "user_agent" filter_value = "Mozilla Firefox" session = {"user_agent": "Mozilla/5.0"} self.returned_content = self.handler.apply_filter(filter_name, filter_value, session) self.assertFalse(self.returned_content) def test_apply_filter_possible_owner(self): filter_name = "possible_owners" filter_value = "crawler" session = {"possible_owners": ["user"]} self.returned_content = self.handler.apply_filter(filter_name, filter_value, session) self.assertFalse(self.returned_content) def test_apply_filter_attack_types(self): filter_name = "attack_types" filter_value = "xss" session = {"attack_types": ["rfi", "xss"]} self.returned_content = self.handler.apply_filter(filter_name, filter_value, session) self.assertTrue(self.returned_content) def test_apply_filter_attack_types_false(self): filter_name = "attack_types" filter_value = "lfi" session = {"attack_types": ["rfi", "xss"]} self.returned_content = self.handler.apply_filter(filter_name, filter_value, session) self.assertFalse(self.returned_content) def test_apply_filter_start_time(self): filter_name = "start_time" filter_value = 148560 session = {"start_time": 148570} self.returned_content = self.handler.apply_filter(filter_name, filter_value, session) self.assertTrue(self.returned_content) def test_apply_filter_start_time_false(self): filter_name = "start_time" filter_value = 148560 session = {"start_time": 148555} self.returned_content = self.handler.apply_filter(filter_name, filter_value, session) self.assertFalse(self.returned_content) def test_apply_filter_end_time(self): filter_name = "end_time" filter_value = 148580 session = {"end_time": 148565} self.returned_content = self.handler.apply_filter(filter_name, filter_value, session) self.assertTrue(self.returned_content) def test_apply_filter_end_time_false(self): filter_name = "end_time" filter_value = 148580 session = {"end_time": 148590} self.returned_content = self.handler.apply_filter(filter_name, filter_value, session) self.assertFalse(self.returned_content) def tearDown(self): async def close(): await self.redis_client.flushall() await self.redis_client.close() self.loop.run_until_complete(close()) ================================================ FILE: tanner/tests/test_api_server.py ================================================ from unittest import mock from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop from tanner.api import server, api class TestAPIServer(AioHTTPTestCase): def setUp(self): self.serv = server.ApiServer() redis = mock.Mock() redis.close = mock.Mock() self.serv.redis_client = redis self.serv.api = api.Api(self.serv.redis_client) super(TestAPIServer, self).setUp() async def get_application(self): app = await self.serv.make_app() return app @unittest_run_loop async def test_api_index_request(self): assert_content = {"version": 1, "response": {"message": "tanner api"}} request = await self.client.request( "GET", "/?key=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyIjoidGFubmVyX293bmVyIn0." "NQ7x_iq2t2SUs20Z9G-FmgqeNBOp5duiXr_auNVmzfU", ) assert request.status == 200 detection = await request.json() self.assertDictEqual(detection, assert_content) @unittest_run_loop async def test_api_snares_request(self): async def mock_return_snares(): return ["8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4"] assert_content = {"version": 1, "response": {"message": ["8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4"]}} self.serv.api.return_snares = mock_return_snares request = await self.client.request( "GET", "/snares?key=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9" ".eyJ1c2VyIjoidGFubmVyX293bmVyIn0.NQ7x_iq2t2SUs20Z9G-FmgqeNBOp5duiXr_auNVmzfU", ) assert request.status == 200 detection = await request.json() self.assertDictEqual(detection, assert_content) @unittest_run_loop async def test_api_snare_info_request(self): async def mock_return_snare_info(snare_uuid, count): if snare_uuid == "8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4" and count == 50: return [{"test_sess1": "sess1_info"}, {"test_sess1": "sess2_info"}] assert_content = { "version": 1, "response": {"message": [{"test_sess1": "sess1_info"}, {"test_sess1": "sess2_info"}]}, } self.serv.api.return_snare_info = mock_return_snare_info request = await self.client.request( "GET", "/snare/8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4" "?key=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyIjoidGFubmVyX293bmVyIn0" ".NQ7x_iq2t2SUs20Z9G-FmgqeNBOp5duiXr_auNVmzfU", ) assert request.status == 200 detection = await request.json() self.assertDictEqual(detection, assert_content) @unittest_run_loop async def test_api_snare_stats_request(self): async def mock_return_snare_stats(snare_uuid): if snare_uuid == "8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4": return { "total_sessions": 605, "total_duration": 865.560286283493, "attack_frequency": {"sqli": 0, "lfi": 0, "xss": 0, "rfi": 0, "cmd_exec": 0}, } assert_content = { "version": 1, "response": { "message": { "total_sessions": 605, "total_duration": 865.560286283493, "attack_frequency": {"sqli": 0, "lfi": 0, "xss": 0, "rfi": 0, "cmd_exec": 0}, } }, } self.serv.api.return_snare_stats = mock_return_snare_stats request = await self.client.request( "GET", "/snare-stats/8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4?key=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9" ".eyJ1c2VyIjoidGFubmVyX293bmVyIn0.NQ7x_iq2t2SUs20Z9G-FmgqeNBOp5duiXr_auNVmzfU", ) assert request.status == 200 detection = await request.json() self.assertDictEqual(detection, assert_content) @unittest_run_loop async def test_api_sessions_request(self): async def mock_return_sessions(filters): if ( type(filters) is dict and filters["peer_ip"] == "127.0.0.1" and filters["start_time"] == 1497890400 and filters["user_agent"] == "ngnix" ): return [ {"sess_uuid": "f387d46eaeb1454cadf0713a4a55be49"}, {"sess_uuid": "e85ae767b0bb4b1f91b421b3a28082ef"}, ] assert_content = { "version": 1, "response": {"message": ["f387d46eaeb1454cadf0713a4a55be49", "e85ae767b0bb4b1f91b421b3a28082ef"]}, } self.serv.api.return_sessions = mock_return_sessions request = await self.client.request( "GET", "/8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4/sessions?filters=peer_ip:127.0.0.1 start_time:1497890400" " user_agent:ngnix&key=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyIjoidGFubmVyX293bmVyIn0." "NQ7x_iq2t2SUs20Z9G-FmgqeNBOp5duiXr_auNVmzfU", ) # noqa assert request.status == 200 detection = await request.json() self.assertDictEqual(detection, assert_content) @unittest_run_loop async def test_api_sessions_info_request(self): async def mock_return_session_info(sess_uuid): if sess_uuid == "4afd45d61b994d9eb3ba20faa81a45e1": return {"test_sess1": "sess1_info"} assert_content = {"version": 1, "response": {"message": {"test_sess1": "sess1_info"}}} self.serv.api.return_session_info = mock_return_session_info request = await self.client.request( "GET", "/session/4afd45d61b994d9eb3ba20faa81a45e1?key=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9" ".eyJ1c2VyIjoidGFubmVyX293bmVyIn0.NQ7x_iq2t2SUs20Z9G-FmgqeNBOp5duiXr_auNVmzfU", ) assert request.status == 200 detection = await request.json() self.assertDictEqual(detection, assert_content) ================================================ FILE: tanner/tests/test_base.py ================================================ import asyncio import unittest from unittest import mock from tanner.utils.asyncmock import AsyncMock from tanner.sessions import session from tanner.emulators import base from tanner import __version__ as tanner_version class TestBase(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) self.session = mock.Mock() self.session.associate_db = mock.Mock() self.data = mock.Mock() with mock.patch("tanner.emulators.lfi.LfiEmulator", mock.Mock(), create=True): self.handler = base.BaseHandler("/tmp/", "test.db", self.loop) def mock_lfi_scan(value): return dict(name="lfi", order=0) self.handler.emulators["lfi"].scan = mock_lfi_scan self.detection = None def test_handle_sqli(self): data = dict(path="/index.html?id=1 UNION SELECT 1", cookies={"sess_uuid": "9f82e5d0e6b64047bba996222d45e72c"}) async def mock_sqli_handle(path, session): return "sqli_test_payload" def mock_sqli_scan(value): return dict(name="sqli", order=2) self.handler.emulators["sqli"] = mock.Mock() self.handler.emulators["sqli"].handle = mock_sqli_handle self.handler.emulators["sqli"].scan = mock_sqli_scan detection = self.loop.run_until_complete(self.handler.handle_get(self.session, data)) assert_detection = {"name": "sqli", "order": 2, "payload": "sqli_test_payload"} self.assertDictEqual(detection, assert_detection) def test_handle_xss(self): data = dict( path="/index.html?id=", cookies={"sess_uuid": "9f82e5d0e6b64047bba996222d45e72c"} ) async def mock_xss_handle(path, session): return "xss_test_payload" def mock_xss_scan(value): return dict(name="xss", order=3) self.handler.emulators["xss"] = mock.Mock() self.handler.emulators["xss"].handle = mock_xss_handle self.handler.emulators["xss"].scan = mock_xss_scan detection = self.loop.run_until_complete(self.handler.handle_get(self.session, data)) assert_detection = {"name": "xss", "order": 3, "payload": "xss_test_payload"} self.assertDictEqual(detection, assert_detection) def test_handle_lfi(self): data = dict(path="/index.html?file=/etc/passwd", cookies={"sess_uuid": "9f82e5d0e6b64047bba996222d45e72c"}) async def mock_lfi_handle(attack_value, session): return "lfi_test_payload" def mock_lfi_scan(value): return dict(name="lfi", order=2) self.handler.emulators["lfi"] = mock.Mock() self.handler.emulators["lfi"].handle = mock_lfi_handle self.handler.emulators["lfi"].scan = mock_lfi_scan detection = self.loop.run_until_complete(self.handler.handle_get(self.session, data)) assert_detection = {"name": "lfi", "order": 2, "payload": "lfi_test_payload"} self.assertDictEqual(detection, assert_detection) def test_handle_index(self): data = dict(path="/index.html", cookies={"sess_uuid": "9f82e5d0e6b64047bba996222d45e72c"}) detection = self.loop.run_until_complete(self.handler.handle_get(self.session, data)) assert_detection = {"name": "index", "order": 1} self.assertDictEqual(detection, assert_detection) def test_handle_wp_content(self): data = dict(path="/wp-content/", cookies={"sess_uuid": "9f82e5d0e6b64047bba996222d45e72c"}) detection = self.loop.run_until_complete(self.handler.handle_get(self.session, data)) assert_detection = {"name": "wp-content", "order": 1} self.assertDictEqual(detection, assert_detection) def test_handle_rfi(self): data = dict( path="/index.html?file=http://attack.php", cookies={"sess_uuid": "9f82e5d0e6b64047bba996222d45e72c"} ) async def mock_rfi_handle(path, session): return "rfi_test_payload" def mock_rfi_scan(value): return dict(name="rfi", order=2) self.handler.emulators["rfi"] = mock.Mock() self.handler.emulators["rfi"].handle = mock_rfi_handle self.handler.emulators["rfi"].scan = mock_rfi_scan detection = self.loop.run_until_complete(self.handler.handle_get(self.session, data)) assert_detection = {"name": "rfi", "order": 2, "payload": "rfi_test_payload"} self.assertDictEqual(detection, assert_detection) def test_handle_php_object_injection(self): data = dict( path='/page.php?insert=\'O:15:"ObjectInjection":1:{s:6:"insert";s:2:"id";}\'', cookies={"sess_uuid": "9f82e5d0e6b64047bba996222d45e72c"}, ) async def mock_php_object_injection_handle(path, session): return "php_object_injection_test_payload" def mock_php_object_injection_scan(value): return dict(name="php_object_injection", order=3) self.handler.emulators["php_object_injection"] = mock.Mock() self.handler.emulators["php_object_injection"].handle = mock_php_object_injection_handle self.handler.emulators["php_object_injection"].scan = mock_php_object_injection_scan detection = self.loop.run_until_complete(self.handler.handle_get(self.session, data)) assert_detection = {"name": "php_object_injection", "order": 3, "payload": "php_object_injection_test_payload"} self.assertDictEqual(detection, assert_detection) def test_handle_xxe_injection(self): payload = ( '/tmp/db/file1.sql" ) dump1 = dump1.format( host=TannerConfig.get("SQLI", "host"), user=TannerConfig.get("SQLI", "user"), password=TannerConfig.get("SQLI", "password"), ) dump2 = ( "mysqldump --compact --skip-extended-insert -h {host} -u {user} -p{password}" " attacker_db>/tmp/db/file2.sql" ) dump2 = dump2.format( host=TannerConfig.get("SQLI", "host"), user=TannerConfig.get("SQLI", "user"), password=TannerConfig.get("SQLI", "password"), ) diff_db = "diff /tmp/db/file1.sql /tmp/db/file2.sql" async def setup(): await self.cursor.execute("CREATE DATABASE test_db") # Checking if new DB exists async def test(): self.returned_result = await self.handler.copy_db(self.db_name, "attacker_db") self.result = await self.handler.check_db_exists("attacker_db") self.loop.run_until_complete(setup()) self.loop.run_until_complete(test()) # Checking if new DB is exactly same as original DB try: dump_db_1 = subprocess.Popen(dump1, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True) dump_db_2 = subprocess.Popen(dump2, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True) diff_db = subprocess.Popen(diff_db, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True) self.outs, errs = diff_db.communicate(timeout=15) dump_db_1.wait() dump_db_2.wait() diff_db.wait() except subprocess.CalledProcessError: pass self.assertEqual(self.result, self.expected_result) self.assertEqual(self.outs, self.expected_outs) @mock.patch("tanner.config.TannerConfig.get", side_effect=mock_config) def test_insert_dummy_data(self, m): def mock_generate_dummy_data(data_tokens): return [(1, "test1"), (2, "test2")], ["I", "L"] self.handler.generate_dummy_data = mock_generate_dummy_data self.expected_result = ((0, "test0"), (1, "test1"), (2, "test2")) async def setup(): await self.cursor.execute("CREATE DATABASE test_db") await self.cursor.execute("USE {db_name}".format(db_name="test_db")) await self.cursor.execute("CREATE TABLE IF NOT EXISTS test (id INTEGER PRIMARY KEY, username TEXT)") await self.cursor.execute('INSERT INTO test VALUES(0, "test0")') await self.conn.commit() async def test(): await self.handler.insert_dummy_data("test", "I,L", self.cursor) await self.cursor.execute("SELECT * FROM test;") self.returned_result = await self.cursor.fetchall() await self.cursor.close() self.conn.close() self.loop.run_until_complete(setup()) self.loop.run_until_complete(test()) self.assertEqual(self.returned_result, self.expected_result) @mock.patch("tanner.config.TannerConfig.get", side_effect=mock_config) def test_create_query_map(self, m): self.expected_result_creds = { "COMMON": [{"name": "NUM", "type": "INTEGER"}], "CREDS": [ {"name": "ID", "type": "INTEGER"}, {"name": "EMAIL", "type": "TEXT"}, {"name": "PASSWORD", "type": "TEXT"}, ], } self.expected_result_test = { "COMMON": [{"name": "PARA", "type": "TEXT"}], "TEST": [{"name": "ID", "type": "INTEGER"}, {"name": "USERNAME", "type": "TEXT"}], } self.query = [ ["TEST_DB", "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, USERNAME TEXT)", "CREATE TABLE COMMON (PARA TEXT)"], [ "CREDS_DB", "CREATE TABLE CREDS (ID INTEGER PRIMARY KEY, EMAIL VARCHAR(15), PASSWORD VARCHAR(15))", "CREATE TABLE COMMON (NUM INTEGER )", ], ] async def setup(data): await self.cursor.execute("CREATE DATABASE {db_name}".format(db_name=data[0])) await self.cursor.execute("USE {db_name}".format(db_name=data[0])) await self.cursor.execute(data[1]) await self.cursor.execute(data[2]) async def test(data): result = await self.handler.create_query_map(data[0]) self.query_map.append(result) await self.handler.delete_db(data[0]) for data in self.query: self.loop.run_until_complete(setup(data)) self.loop.run_until_complete(test(data)) self.assertEqual(self.query_map[0], self.expected_result_test) self.assertEqual(self.query_map[1], self.expected_result_creds) ================================================ FILE: tanner/tests/test_mysqli.py ================================================ import unittest import asyncio from unittest import mock from tanner.utils.asyncmock import AsyncMock from tanner.emulators.mysqli import MySQLIEmulator def mock_config(section, value): config = {"host": "127.0.0.1", "user": "root", "password": "user_pass"} return config[value] class TestMySQLi(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) self.db_name = "test_db" self.handler = MySQLIEmulator(self.db_name) self.conn = None self.cursor = None self.attacker_db = None self.query_map = None self.expected_result = None self.returned_result = None async def connect(): self.conn = await self.handler.helper.connect_to_db() self.cursor = await self.conn.cursor() self.returned_result = await self.handler.helper.check_db_exists(self.db_name) if self.returned_result == 1: await self.handler.helper.delete_db(self.db_name) await self.cursor.execute("CREATE DATABASE test_db") await self.cursor.execute("USE {db_name}".format(db_name="test_db")) await self.cursor.execute("CREATE TABLE IF NOT EXISTS test (id INTEGER PRIMARY KEY, username TEXT)") await self.cursor.execute('INSERT INTO test VALUES(0, "test0")') await self.conn.commit() self.loop.run_until_complete(connect()) @mock.patch("tanner.config.TannerConfig.get", side_effect=mock_config) def test_setup_db(self, m): self.expected_result = { "comments": [ {"name": "comment_id", "type": "INTEGER"}, ], "users": [ {"name": "id", "type": "INTEGER"}, ], } self.handler.helper.create_query_map = AsyncMock( return_value={ "comments": [ {"name": "comment_id", "type": "INTEGER"}, ], "users": [ {"name": "id", "type": "INTEGER"}, ], } ) self.handler.helper = AsyncMock() async def test(): self.returned_result = await self.handler.setup_db() self.loop.run_until_complete(test()) self.handler.helper.create_query_map.assert_called_with(self.db_name) assert not self.handler.helper.setup_db_from_config.called @mock.patch("tanner.config.TannerConfig.get", side_effect=mock_config) def test_setup_db_not_exists(self, m): self.expected_result = { "comments": [ {"name": "comment_id", "type": "INTEGER"}, ], "users": [ {"name": "id", "type": "INTEGER"}, ], } self.handler.helper.create_query_map = AsyncMock( return_value={ "comments": [ {"name": "comment_id", "type": "INTEGER"}, ], "users": [ {"name": "id", "type": "INTEGER"}, ], } ) self.handler.helper.setup_db_from_config = AsyncMock() async def test(): await self.handler.helper.delete_db(self.db_name) self.returned_result = await self.handler.setup_db() self.loop.run_until_complete(test()) self.handler.helper.setup_db_from_config.assert_called_with(self.db_name) self.handler.helper.create_query_map.assert_called_with(self.db_name) @mock.patch("tanner.config.TannerConfig.get", side_effect=mock_config) def test_create_attacker_db(self, m): session = mock.Mock() session.sess_uuid.hex = "d877339ec415484987b279469167af3d" attacker_db = "attacker_" + session.sess_uuid.hex self.handler.helper.copy_db = AsyncMock(return_value=attacker_db) self.expected_result = "attacker_d877339ec415484987b279469167af3d" async def test(): self.returned_result = await self.handler.create_attacker_db(session) self.loop.run_until_complete(test()) self.assertEqual(self.returned_result, self.expected_result) session.associate_db.assert_called_with(attacker_db) self.handler.helper.copy_db.assert_called_with(self.db_name, attacker_db) @mock.patch("tanner.config.TannerConfig.get", side_effect=mock_config) def test_execute_query(self, m): self.expected_result = [[[0, "test_name"]], [[0, "test@domain.com", "test_pass"]]] result = [] self.query = [ [ "TEST_DB", "CREATE TABLE TEST (ID INTEGER PRIMARY KEY, USERNAME TEXT)", 'INSERT INTO TEST VALUES(0, "test_name")', ], [ "CREDS_DB", "CREATE TABLE CREDS (ID INTEGER PRIMARY KEY, EMAIL VARCHAR(15), PASSWORD VARCHAR(15))", 'INSERT INTO CREDS VALUES(0, "test@domain.com", "test_pass")', ], ] test_query = [["TEST_DB", "SELECT * FROM TEST"], ["CREDS_DB", "SELECT * FROM CREDS"]] async def setup(data): await self.cursor.execute("CREATE DATABASE {db_name}".format(db_name=data[0])) await self.cursor.execute("USE {db_name}".format(db_name=data[0])) await self.cursor.execute(data[1]) await self.cursor.execute(data[2]) await self.conn.commit() for data in self.query: self.loop.run_until_complete(setup(data)) async def test(data): self.returned_result = await self.handler.execute_query(data[1], data[0]) result.append(self.returned_result) await self.handler.helper.delete_db(data[0]) for query in test_query: self.loop.run_until_complete(test(query)) self.assertEqual(self.expected_result, result) @mock.patch("tanner.config.TannerConfig.get", side_effect=mock_config) def test_execute_query_error(self, m): self.cursor.fetchall = mock.Mock(side_effect=Exception) query = "" self.expected_result = "(1065, 'Query was empty')" async def test(): self.returned_result = await self.handler.execute_query(query, self.db_name) self.loop.run_until_complete(test()) self.assertEqual(self.returned_result, self.expected_result) ================================================ FILE: tanner/tests/test_php_code_injection.py ================================================ import asyncio import unittest from tanner.emulators import php_code_injection class TestPHPCodeInjection(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) self.handler = php_code_injection.PHPCodeInjection(loop=self.loop) def test_scan(self): attack = "; phpinfo();" assert_result = dict(name="php_code_injection", order=3) result = self.handler.scan(attack) self.assertEqual(result, assert_result) def test_handle_status_code(self): async def mock_get_injection_results(code): return None self.handler.get_injection_result = mock_get_injection_results attack_params = [dict(id="foo", value=";sleep(50);")] assert_result = dict(status_code=504) result = self.loop.run_until_complete(self.handler.handle(attack_params)) self.assertEqual(result, assert_result) ================================================ FILE: tanner/tests/test_php_object_injection.py ================================================ import asyncio import unittest from tanner.utils.asyncmock import AsyncMock from tanner.emulators.php_object_injection import PHPObjectInjection class TestPHPCodeInjection(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) self.handler = PHPObjectInjection(loop=self.loop) self.result = None self.expected_result = None self.returned_result = None def test_scan(self): payload = 'O:15:"ObjectInjection":1:{s:6:"insert";s:2:"id";}' self.expected_result = dict(name="php_object_injection", order=3) self.returned_result = self.handler.scan(payload) self.assertEqual(self.returned_result, self.expected_result) def test_scan_negative(self): payload = 'O:"ObjectInjection":1:{s:6:"insert";s:2:"id";}' self.expected_result = None self.returned_result = self.handler.scan(payload) self.assertEqual(self.returned_result, self.expected_result) def test_handle_status_code(self): self.handler.get_injection_result = AsyncMock(return_value=None) attack_params = [dict(id="foo", value="O:15:'ObjectInjection':1:{s:6:'insert';}")] self.expected_result = dict(status_code=504) async def test(): self.returned_result = await self.handler.handle(attack_params) self.loop.run_until_complete(test()) self.assertEqual(self.returned_result, self.expected_result) def test_handle(self): attack_params = [dict(id="foo", value='O:15:"ObjectInjection":1:{s:6:"insert";s:2:"id";}')] self.handler.helper.get_result = AsyncMock( return_value={ "file_md5": "a43deb0f2d7904cbb6c27c02ed7c2593", "stdout": "id=0(root) gid=0(root) groups=0(root)", } ) self.expected_result = "id=0(root) gid=0(root) groups=0(root)" async def test(): self.returned_result = await self.handler.handle(attack_params) self.loop.run_until_complete(test()) self.assertIn(self.expected_result, self.returned_result["value"]) ================================================ FILE: tanner/tests/test_rfi_emulation.py ================================================ import asyncio import unittest from unittest import mock from tanner.emulators import rfi import yarl class TestRfiEmulator(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) self.handler = rfi.RfiEmulator("/tmp/", loop=self.loop) def test_http_download(self): path = "http://example.com" data = self.loop.run_until_complete(self.handler.download_file(path)) self.assertIsNotNone(data) def test_http_download_fail(self): path = "http://foobarfvfd" filename = self.loop.run_until_complete(self.handler.download_file(path)) self.assertIsNone(filename) def test_ftp_download(self): self.handler.download_file_ftp = mock.MagicMock() path = "ftp://mirror.yandex.ru/archlinux/lastupdate" data = self.loop.run_until_complete(self.handler.download_file(path)) self.handler.download_file_ftp.assert_called_with(yarl.URL(path)) def test_ftp_download_fail(self): path = "ftp://mirror.yandex.ru/archlinux/foobar" with self.assertLogs(): self.loop.run_until_complete(self.handler.download_file(path)) def test_get_result_fail(self): data = "test data" result = self.loop.run_until_complete(self.handler.get_rfi_result(data)) self.assertIsNone(result) def test_invalid_scheme(self): path = "file://mirror.yandex.ru/archlinux/foobar" data = self.loop.run_until_complete(self.handler.download_file(path)) self.assertIsNone(data) ================================================ FILE: tanner/tests/test_server.py ================================================ import uuid from unittest import mock import hashlib from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop from tanner import server from tanner.config import TannerConfig from tanner.utils.asyncmock import AsyncMock from tanner import __version__ as tanner_version class TestServer(AioHTTPTestCase): def setUp(self): d = dict( MONGO={"enabled": "False", "URI": "mongodb://localhost"}, LOCALLOG={"enabled": "False", "PATH": "/tmp/tanner_report.json"}, ) m = mock.MagicMock() m.__getitem__.side_effect = d.__getitem__ m.__iter__.side_effect = d.__iter__ with mock.patch("tanner.tests.test_server.TannerConfig") as p: TannerConfig.config = m TannerConfig.get = m.get with mock.patch("tanner.dorks_manager.DorksManager", mock.Mock()): with mock.patch("tanner.emulators.base.BaseHandler", mock.Mock(), create=True): with mock.patch("tanner.sessions.session_manager.SessionManager", mock.Mock(), create=True): self.serv = server.TannerServer() self.test_uuid = uuid.uuid4() async def _add_or_update_mock(data, client): sess = mock.Mock() sess.set_attack_type = mock.Mock() sess_id = hashlib.md5(b"foo") test_uuid = uuid sess.get_uuid = mock.Mock(return_value=str(self.test_uuid)) return sess, sess_id async def _delete_sessions_mock(client): pass self.serv.session_manager.add_or_update_session = _add_or_update_mock self.serv.session_manager.delete_sessions_on_shutdown = _delete_sessions_mock async def choosed(client): return [x for x in range(10)] dorks = mock.Mock() dorks.choose_dorks = choosed dorks.extract_path = self._make_coroutine() redis = AsyncMock() redis.close = AsyncMock() self.serv.dorks = dorks self.serv.redis_client = redis super(TestServer, self).setUp() def _make_coroutine(self): async def coroutine(*args, **kwargs): return mock.Mock(*args, **kwargs) return coroutine async def get_application(self): app = await self.serv.make_app() return app @unittest_run_loop async def test_example(self): request = await self.client.request("GET", "/") assert request.status == 200 text = await request.text() assert "Tanner server" in text def test_make_response(self): msg = "test" content = self.serv._make_response(msg) assert_content = dict(version=tanner_version, response=dict(message=msg)) self.assertDictEqual(content, assert_content) @unittest_run_loop async def test_events_request(self): async def _make_handle_coroutine(*args, **kwargs): return {"name": "index", "order": 1, "payload": None} detection_assert = { "version": tanner_version, "response": { "message": { "detection": {"name": "index", "order": 1, "payload": None}, "sess_uuid": str(self.test_uuid), } }, } self.serv.base_handler.handle = _make_handle_coroutine request = await self.client.request("POST", "/event", data=b'{"path":"/index.html"}') assert request.status == 200 detection = await request.json() self.assertDictEqual(detection, detection_assert) @unittest_run_loop async def test_dorks_request(self): assert_content = dict(version=tanner_version, response=dict(dorks=[x for x in range(10)])) request = await self.client.request("GET", "/dorks") assert request.status == 200 detection = await request.json() self.assertDictEqual(detection, assert_content) @unittest_run_loop async def test_version(self): assert_content = dict(version=tanner_version) request = await self.client.request("GET", "/version") assert request.status == 200 detection = await request.json() self.assertDictEqual(detection, assert_content) ================================================ FILE: tanner/tests/test_session_analyzer.py ================================================ import asyncio import json import unittest from unittest.mock import Mock from unittest.mock import patch import geoip2 import aioredis from tanner.sessions.session_analyzer import SessionAnalyzer session = ( b'{"sess_uuid": "c546114f97f548f982756495f963e280", "start_time": 1466091813.4780173, ' b'"referer": "/",' b'"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' b'Chrome/53.0.2767.4 Safari/537.36", "end_time": 1466091899.9854035, ' b'"snare_uuid": "78e51180-bf0d-4757-8a04-f000e5efa179", "count": 24, ' b'"paths": [{"timestamp": 1466091813.4779778, "path": "/", "attack_type": "index", "response_status": 200},' b'{"timestamp": 1466091842.7088752, "path": "/fluent-python.html", "attack_type": "index", ' b'"response_status": 200}, {"timestamp": 1466091858.214475, "path": "/wow-movie.html?exec=/bin/bash", ' b'"attack_type": "index", "response_status": 200}, {"timestamp": 1466091871.9076045, ' b'"path": "/wow-movie.html?exec=/etc/passwd", "attack_type": "lfi", "response_status": 200},' b'{"timestamp": 1466091885.1003792, "path": "/wow-movie.html?exec=/bin/bash", "attack_type": "index", ' b'"response_status": 200}, {"timestamp": 1466091899.9854052, ' b'"path": "/wow-movie.html?exec=/../../../..///././././.../../../etc/passwd",' b' "attack_type": "lfi", "response_status": 200}], ' b'"peer": {"port": 56970, "ip": "74.217.37.84"}, ' b'"cookies": {"sess_uuid": "c546114f97f548f982756495f963e280"}}' ) def mock_open(): with open("./tanner/data/crawler_user_agents.txt") as f: f.close = Mock() return Mock(return_value=f) class TestSessionAnalyzer(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) self.session = json.loads(session.decode("utf-8")) self.handler = SessionAnalyzer(loop=self.loop) self.res = None geoip2.database.Reader.__init__ = Mock(return_value=None) rvalue = geoip2.models.City( { "city": {"geoname_id": 4223379, "names": {"en": "Smyrna", "ru": "Смирна", "zh-CN": "士麦那"}}, "continent": { "code": "NA", "geoname_id": 6255149, "names": { "de": "Nordamerika", "en": "North America", "es": "Norteamérica", "fr": "Amérique du Nord", "ja": "北アメリカ", "pt-BR": "América do Norte", "ru": "Северная Америка", "zh-CN": "北美洲", }, }, "country": { "geoname_id": 6252001, "iso_code": "US", "names": { "de": "USA", "en": "United States", "es": "Estados Unidos", "fr": "États-Unis", "ja": "アメリカ合衆国", "pt-BR": "Estados Unidos", "ru": "США", "zh-CN": "美国", }, }, "location": { "accuracy_radius": 10, "latitude": 33.8633, "longitude": -84.4984, "metro_code": 524, "time_zone": "America/New_York", }, "postal": {"code": "30080"}, "registered_country": { "geoname_id": 6252001, "iso_code": "US", "names": { "de": "USA", "en": "United States", "es": "Estados Unidos", "fr": "États-Unis", "ja": "アメリカ合衆国", "pt-BR": "Estados Unidos", "ru": "США", "zh-CN": "美国", }, }, "subdivisions": [ { "geoname_id": 4197000, "iso_code": "GA", "names": { "en": "Georgia", "es": "Georgia", "fr": "Géorgie", "ja": "ジョージア州", "pt-BR": "Geórgia", "ru": "Джорджия", "zh-CN": "乔治亚", }, } ], "traits": {"ip_address": "74.217.37.8"}, }, ["en"], ) geoip2.database.Reader.city = Mock(return_value=rvalue) def tests_load_session_fail(self): async def sess_get(key): return aioredis.ConnectionError redis_mock = Mock() redis_mock.get = sess_get res = None with self.assertLogs(): self.loop.run_until_complete(self.handler.analyze(None, redis_mock)) def test_create_stats(self): async def sess_get(): return session async def set_of_members(key): return set() async def set_add(): return "" redis_mock = Mock() redis_mock.get = sess_get redis_mock.smembers = set_of_members redis_mock.zadd = set_add with patch("builtins.open", new_callable=mock_open) as m: stats = self.loop.run_until_complete(self.handler.create_stats(self.session, redis_mock)) self.assertEqual(stats["possible_owners"], {"attacker": 1.0}) def test_choose_owner_crawler(self): stats = dict( paths=[{"path": "/robots.txt", "timestamp": 1.0, "response_status": 200, "attack_type": "index"}], attack_types={"index"}, requests_in_second=11.1, referer=None, peer_ip="ip", ) async def test(): self.res = await self.handler.choose_possible_owner(stats) with patch("builtins.open", new_callable=mock_open) as m: self.loop.run_until_complete(test()) self.assertEqual(self.res["possible_owners"], {"crawler": 1.0}) def test_choose_owner_attacker(self): stats = dict( paths=[{"path": "/", "timestamp": 1.0, "response_status": 200, "attack_type": "rfi"}], attack_types={"rfi", "lfi"}, requests_in_second=2, user_agent="user", peer_ip="ip", ) async def test(): self.res = await self.handler.choose_possible_owner(stats) with patch("builtins.open", new_callable=mock_open) as m: self.loop.run_until_complete(test()) self.assertEqual(self.res["possible_owners"], {"attacker": 1.0}) def test_choose_owner_mixed(self): stats = dict( paths=[{"path": "/", "timestamp": 1.0, "response_status": 200, "attack_type": ""}], attack_types="", requests_in_second=2, user_agent="Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", peer_ip="74.217.37.84", hidden_links=0, referer="/", ) async def test(): self.res = await self.handler.choose_possible_owner(stats) with patch("builtins.open", new_callable=mock_open) as m: self.loop.run_until_complete(test()) self.assertEqual(self.res["possible_owners"], {"attacker": 0.75, "crawler": 0.25, "tool": 0.15, "user": 0.25}) def test_choose_owner_user(self): stats = dict( paths=[{"path": "/", "timestamp": 1.0, "response_status": 200, "attack_type": ""}], attack_types="", requests_in_second=2, user_agent="test_user_agent", peer_ip="74.217.37.84", hidden_links=0, referer="/", ) async def test(): self.res = await self.handler.choose_possible_owner(stats) with patch("builtins.open", new_callable=mock_open) as m: self.loop.run_until_complete(test()) self.assertEqual(self.res["possible_owners"], {"user": 1.0}) def test_find_location(self): location_stats = self.handler.find_location("74.217.37.84") expected_res = dict( country="United States", country_code="US", city="Smyrna", zip_code="30080", ) self.assertEqual(location_stats, expected_res) ================================================ FILE: tanner/tests/test_session_manager.py ================================================ import asyncio import unittest from unittest import mock import hashlib from tanner.sessions import session_manager, session class TestSessions(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) self.handler = session_manager.SessionManager(loop=self.loop) self.handler.analyzer = mock.Mock() self.handler.analyzer.send = mock.Mock() def test_validate_missing_peer(self): data = { "headers": { "USER-AGENT": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/41.0.2228.0 Safari/537.36" }, "path": "/foo", "uuid": None, "cookies": {"sess_uuid": None}, } assertion_data = { "peer": {"ip": None, "port": None}, "headers": { "user-agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/41.0.2228.0 Safari/537.36" }, "path": "/foo", "uuid": None, "status": 200, "cookies": {"sess_uuid": None}, } data = self.handler.validate_data(data) self.assertDictEqual(data, assertion_data) def test_validate_missing_user_agent(self): data = { "peer": {"ip": "127.0.0.1", "port": 80}, "headers": {}, "path": "/foo", "uuid": None, "cookies": {"sess_uuid": None}, } assertion_data = { "peer": {"ip": "127.0.0.1", "port": 80}, "headers": {"user-agent": None}, "path": "/foo", "uuid": None, "status": 200, "cookies": {"sess_uuid": None}, } data = self.handler.validate_data(data) self.assertDictEqual(data, assertion_data) def test_validate_missing_cookies(self): data = {"peer": {"ip": "127.0.0.1", "port": 80}, "headers": {}, "path": "/foo", "uuid": None} assertion_data = { "peer": {"ip": "127.0.0.1", "port": 80}, "headers": {"user-agent": None}, "path": "/foo", "uuid": None, "status": 200, "cookies": {"sess_uuid": None}, } data = self.handler.validate_data(data) self.assertDictEqual(data, assertion_data) def test_adding_new_session(self): data = { "peer": {"ip": None, "port": None}, "headers": {}, "path": "/foo", "uuid": None, "cookies": {"sess_uuid": None}, } async def sess_sadd(key, value): return None redis_mock = mock.Mock() redis_mock.sadd = sess_sadd sess, sess_id = self.loop.run_until_complete(self.handler.add_or_update_session(data, redis_mock)) self.assertDictEqual({sess_id: sess}, self.handler.sessions) def test_updating_session(self): async def sess_sadd(key, value): return None data = { "peer": {"ip": None, "port": None}, "headers": {"user-agent": None}, "path": "/foo", "uuid": None, "status": 200, "cookies": {"sess_uuid": None}, } sess = session.Session(data) ip = data["peer"]["ip"] user_agent = data["headers"]["user-agent"] sess_uuid = data["cookies"]["sess_uuid"] sess_id_string = "{ip}{user_agent}{sess_uuid}".format(ip=ip, user_agent=user_agent, sess_uuid=sess_uuid) sess_id = hashlib.md5(sess_id_string.encode()).hexdigest() redis_mock = mock.Mock() redis_mock.sadd = sess_sadd self.handler.sessions[sess_id] = sess self.loop.run_until_complete(self.handler.add_or_update_session(data, redis_mock)) self.assertEqual(self.handler.sessions[sess_id].count, 2) def test_deleting_sessions(self): async def analyze(session_key, redis_client): return None async def sess_set(key, val): return None self.handler.analyzer.analyze = analyze data = { "peer": {"ip": None, "port": None}, "headers": {"user-agent": None}, "path": "/foo", "uuid": None, "status": 200, "cookies": {"sess_uuid": None}, } sess = session.Session(data) sess.is_expired = mock.MagicMock(name="expired") sess.is_expired.__bool__.reurned_value = True self.handler.sessions[sess] = sess redis_mock = mock.Mock() redis_mock.set = sess_set self.loop.run_until_complete(self.handler.delete_old_sessions(redis_mock)) self.assertDictEqual(self.handler.sessions, {}) def test_get_uuid(self): data = { "peer": {"ip": None, "port": None}, "headers": {"user-agent": None}, "path": "/foo", "uuid": None, "status": 200, "cookies": {"sess_id": None}, } sess = session.Session(data) key = sess.get_uuid() self.assertIsNotNone(key) ================================================ FILE: tanner/tests/test_sqli.py ================================================ import asyncio import unittest import os from unittest import mock from tanner.emulators import sqli class SqliTest(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) query_map = { "users": [ {"name": "id", "type": "INTEGER"}, {"name": "login", "type": "text"}, {"name": "email", "type": "text"}, {"name": "username", "type": "text"}, {"name": "password", "type": "text"}, {"name": "pass", "type": "text"}, {"name": "log", "type": "text"}, ], "comments": [{"name": "comment", "type": "text"}], } self.handler = sqli.SqliEmulator("test_db", "/tmp/") self.filename = "/tmp/db/test_db" os.makedirs(os.path.dirname(self.filename), exist_ok=True) open("/tmp/db/test_db", "a").close() self.handler.query_map = query_map self.sess = mock.Mock() self.sess.sess_uuid.hex = "d877339ec415484987b279469167af3d" def test_scan(self): attack = "1 UNION SELECT 1,2,3,4" assert_result = dict(name="sqli", order=2) result = self.handler.scan(attack) self.assertEqual(result, assert_result) def test_scan_negative(self): attack = "1 UNION 1,2,3,4" assert_result = None result = self.handler.scan(attack) self.assertEqual(result, assert_result) def test_handle(self): attack_params = [dict(id="id", value="1 UNION SELECT 1,2,3,4")] assert_result = dict(value="no such table: users", page=True) result = self.loop.run_until_complete(self.handler.handle(attack_params, self.sess)) self.assertEqual(assert_result, result) def test_map_query_id(self): attack_value = dict(id="id", value="1'UNION SELECT 1,2,3,4") assert_result = "SELECT * from users WHERE id=1 UNION SELECT 1,2,3,4;" result = self.handler.map_query(attack_value) self.assertEqual(assert_result, result) def test_map_query_comments(self): attack_value = dict(id="comment", value='some_comment" UNION SELECT 1,2 AND "1"="1') assert_result = 'SELECT * from comments WHERE comment="some_comment" UNION SELECT 1,2 AND "1"="1";' result = self.handler.map_query(attack_value) self.assertEqual(assert_result, result) def test_map_query_error(self): attack_value = dict(id="foo", value="bar'UNION SELECT 1,2") result = self.handler.map_query(attack_value) self.assertIsNone(result) def test_get_sqli_result(self): attack_value = dict(id="id", value="1 UNION SELECT 1,2,3,4") async def mock_execute_query(query, db_name): return [[1, "name", "email@mail.com", "password"], [1, "2", "3", "4"]] self.handler.sqli_emulator = mock.Mock() self.handler.sqli_emulator.execute_query = mock_execute_query assert_result = dict(value="[1, 'name', 'email@mail.com', 'password'] [1, '2', '3', '4']", page=True) result = self.loop.run_until_complete(self.handler.get_sqli_result(attack_value, "foo.db")) self.assertEqual(assert_result, result) def test_get_sqli_result_error(self): attack_value = dict(id="foo", value="bar'UNION SELECT 1,2") assert_result = "SQL ERROR: near foo: syntax error" result = self.loop.run_until_complete(self.handler.get_sqli_result(attack_value, "foo.db")) self.assertEqual(assert_result, result["value"]) def tearDown(self): if os.path.exists(self.filename): os.remove(self.filename) ================================================ FILE: tanner/tests/test_sqlite.py ================================================ import asyncio import os import sqlite3 import unittest from unittest import mock from tanner.utils.asyncmock import AsyncMock from tanner.emulators import sqlite class SqliteTest(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) self.filename = "/tmp/db/test_db" os.makedirs(os.path.dirname(self.filename), exist_ok=True) open("/tmp/db/test_db", "a").close() # Insert some testing data self.conn = sqlite3.connect(self.filename) self.cursor = self.conn.cursor() self.cursor.execute("CREATE TABLE test (id INTEGER PRIMARY KEY, username TEXT)") self.cursor.execute('INSERT INTO TEST VALUES(0, "test0")') self.conn.commit() self.handler = sqlite.SQLITEEmulator("test_db", "/tmp/") self.returned_result = None self.expected_result = None def test_setup_db(self): self.handler.helper.create_query_map = mock.Mock( return_value={"test": [{"name": "id", "type": "INTEGER"}, {"name": "username", "type": "TEXT"}]} ) self.handler.helper.setup_db_from_config = mock.Mock() async def test(): self.returned_result = await self.handler.setup_db() self.loop.run_until_complete(test()) self.handler.helper.create_query_map.assert_called_with("/tmp/db/", "test_db") assert not self.handler.helper.setup_db_from_config.called def test_setup_db_not_exists(self): self.handler.helper.create_query_map = mock.Mock( return_value={"test": [{"name": "id", "type": "INTEGER"}, {"name": "username", "type": "TEXT"}]} ) self.handler.helper.setup_db_from_config = AsyncMock() async def test(): os.remove(self.filename) self.returned_result = await self.handler.setup_db() self.loop.run_until_complete(test()) self.handler.helper.setup_db_from_config.assert_called_with("/tmp/db/", "test_db") self.handler.helper.create_query_map.assert_called_with("/tmp/db/", "test_db") def test_create_attacker_db(self): session = mock.Mock() session.sess_uuid.hex = "d877339ec415484987b279469167af3d" self.loop.run_until_complete(self.handler.create_attacker_db(session)) self.assertTrue(os.path.exists("/tmp/db/attacker_d877339ec415484987b279469167af3d")) def test_execute_query(self): self.expected_result = [[[1, "test_name"]], [[1, "test@domain.com", "test_pass"]]] result = [] self.query = [ [ "/tmp/db/TEST_DB", "CREATE TABLE IF NOT EXISTS TEST (ID INTEGER PRIMARY KEY, USERNAME TEXT)", 'INSERT INTO TEST VALUES(1, "test_name")', ], [ "/tmp/db/CREDS_DB", "CREATE TABLE IF NOT EXISTS CREDS (ID INTEGER PRIMARY KEY, EMAIL VARCHAR(15), " "PASSWORD VARCHAR(15))", "INSERT INTO CREDS VALUES(1, 'test@domain.com', 'test_pass')", ], ] test_query = [["/tmp/db/TEST_DB", "SELECT * FROM TEST"], ["/tmp/db/CREDS_DB", "SELECT * FROM CREDS"]] def setup(data): os.makedirs(os.path.dirname(data[0]), exist_ok=True) self.conn = sqlite3.connect(data[0]) self.cursor = self.conn.cursor() self.cursor.execute(data[1]) self.cursor.execute(data[2]) self.conn.commit() for data in self.query: setup(data) async def test(data): self.returned_result = await self.handler.execute_query(data[1], data[0]) result.append(self.returned_result) self.handler.helper.delete_db(data[0]) for query in test_query: self.loop.run_until_complete(test(query)) self.assertEqual(self.expected_result, result) def tearDown(self): if os.path.exists(self.filename): os.remove(self.filename) ================================================ FILE: tanner/tests/test_sqlite_db_helper.py ================================================ import asyncio import os import sqlite3 import unittest import subprocess from unittest import mock from tanner.utils.asyncmock import AsyncMock from tanner.utils.sqlite_db_helper import SQLITEDBHelper class TestSQLiteDBHelper(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) self.filename = "/tmp/db/test_db" os.makedirs(os.path.dirname(self.filename), exist_ok=True) open("/tmp/db/test_db", "a").close() # Insert some testing data conn = sqlite3.connect(self.filename) self.cursor = conn.cursor() self.cursor.execute("CREATE TABLE TEST (id INTEGER PRIMARY KEY, username TEXT)") self.cursor.execute('INSERT INTO TEST VALUES(0, "test0")') conn.commit() self.handler = SQLITEDBHelper() self.returned_result = None self.expected_result = None def test_setup_db_from_config(self): config = { "name": "test_db", "tables": [ { "schema": "CREATE TABLE IF NOT EXISTS CREDS (ID INTEGER PRIMARY KEY, EMAIL VARCHAR(15), " "PASSWORD VARCHAR(15))", "table_name": "CREDS", "data_tokens": "I,E,P", } ], } def mock_read_config(): return config self.result = [] self.handler.read_config = mock_read_config self.handler.insert_dummy_data = AsyncMock() calls = [mock.call("CREDS", "I,E,P", mock.ANY)] self.expected_result = [ [ ("CREATE TABLE CREDS (ID INTEGER PRIMARY KEY, EMAIL VARCHAR(15), PASSWORD " "VARCHAR(15))",), ("CREATE TABLE TEST (id INTEGER PRIMARY KEY, username TEXT)",), ] ] async def test(): await self.handler.setup_db_from_config("/tmp/", self.filename) self.cursor.execute("SELECT sql FROM sqlite_master ORDER BY tbl_name") result = self.cursor.fetchall() self.result.append(result) self.handler.delete_db(self.filename) self.loop.run_until_complete(test()) self.assertEqual(self.result, self.expected_result) self.handler.insert_dummy_data.assert_has_calls(calls, any_order=True) def test_get_abs_path(self): self.path = "db/attacker_db" self.returned_result = self.handler.get_abs_path(self.path, "/tmp/") self.expected_result = "/tmp/db/attacker_db" self.assertEqual(self.returned_result, self.expected_result) def test_get_abs_path_2(self): self.path = "../../tmp/db/./test_db" self.returned_result = self.handler.get_abs_path(self.path, "/tmp/") self.expected_result = "/tmp/db/test_db" self.assertEqual(self.returned_result, self.expected_result) def test_copy_db(self): self.attacker_db = "/tmp/db/attacker_db" self.returned_result = self.handler.copy_db(self.filename, self.attacker_db, "/tmp/") self.assertTrue(os.path.exists(self.attacker_db)) diff_db = "diff /tmp/db/test_db /tmp/db/attacker_db" self.result = b"" # Checking if new DB is same as original DB try: diff_db = subprocess.Popen(diff_db, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True) self.outs, errs = diff_db.communicate(timeout=15) diff_db.wait() except subprocess.CalledProcessError: pass self.assertEqual(self.outs, self.result) # Deleting the DB os.remove("/tmp/db/attacker_db") def test_create_query_map(self): self.returned_result = self.handler.create_query_map("/tmp/db", "test_db") self.expected_result = {"TEST": [{"name": "id", "type": "INTEGER"}, {"name": "username", "type": "TEXT"}]} self.assertEqual(self.returned_result, self.expected_result) @mock.patch("tanner.utils.sqlite_db_helper.sqlite3") def test_create_query_map_error(self, sqlite): sqlite.OperationalError = sqlite3.OperationalError sqlite.connect().cursor().execute.side_effect = sqlite3.OperationalError with self.assertLogs(level="ERROR") as log: self.returned_result = self.handler.create_query_map("/tmp/db", "test_db") self.assertIn("Error during query map creation", log.output[0]) def test_insert_dummy_data(self): def mock_generate_dummy_data(data_tokens): return [(1, "test1"), (2, "test2")], ["I", "L"] self.handler.generate_dummy_data = mock_generate_dummy_data self.loop.run_until_complete(self.handler.insert_dummy_data("test", "I,L", self.cursor)) self.expected_result = [[0, "test0"], [1, "test1"], [2, "test2"]] result = [] for row in self.cursor.execute("SELECT * FROM test;"): result.append(list(row)) self.assertEqual(result, self.expected_result) def tearDown(self): if os.path.exists(self.filename): os.remove(self.filename) ================================================ FILE: tanner/tests/test_template_injection.py ================================================ import asyncio import unittest import os from unittest import mock from tanner.utils import patterns from tanner.utils.asyncmock import AsyncMock from tanner.emulators.template_injection import TemplateInjection class TestTemplateInjection(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) self.handler = TemplateInjection(loop=self.loop) self.result = None self.expected_result = None self.returned_result = None self.sess = mock.Mock() self.sess.sess_uuid.hex = "e86d20b858224e239d3991c1a2650bc7" self.handler.remote_path = ( "https://raw.githubusercontent.com/mushorg/tanner/master/docker/" "tanner/template_injection/Dockerfile" ) def test_scan(self): payload = "{{7*7}}" self.expected_result = dict(name="template_injection", order=4) self.returned_result = self.handler.scan(payload) self.assertEqual(self.returned_result, self.expected_result) def test_scan_negative(self): payload = "{7*7}" self.expected_result = None self.returned_result = self.handler.scan(payload) self.assertEqual(self.returned_result, self.expected_result) def test_xss_mako_regex(self): # xss payloads cannot be matched with mako's regex but vice versa is possible test_xss = '' # space bypass xss payload verify_xss = patterns.XSS_ATTACK.match(test_xss) self.returned_result = self.handler.scan(test_xss) self.expected_result = None self.assertEqual(self.returned_result, self.expected_result) self.assertTrue(verify_xss) def test_handle_tornado(self): self.handler.docker_helper.execute_cmd = AsyncMock(return_value='posix.uname_result(sysname="Linux")') payload = "{%import os%}{{os.uname()}}" attack_params = [dict(id="foo", value=payload)] self.returned_result = self.loop.run_until_complete(self.handler.handle(attack_params, self.sess)) self.expected_result = os.uname() self.assertIn(self.expected_result[0], self.returned_result["value"]) def test_handle_mako(self): self.handler.docker_helper.execute_cmd = AsyncMock(return_value='posix.uname_result(sysname="Linux")') payload = "<%\nimport os\nx=os.uname()\n%>\n${x}" attack_params = [dict(id="foo", value=payload)] self.returned_result = self.loop.run_until_complete(self.handler.handle(attack_params, self.sess)) self.expected_result = os.uname() self.assertIn(self.expected_result[0], self.returned_result["value"]) def tearDown(self): self.loop.run_until_complete(self.handler.docker_helper.docker_client.close()) self.loop.close() ================================================ FILE: tanner/tests/test_web_server.py ================================================ import asyncio from aiohttp.test_utils import AioHTTPTestCase, unittest_run_loop from tanner.utils.asyncmock import AsyncMock from tanner.web.server import TannerWebServer class TestWebServer(AioHTTPTestCase): def setUp(self): self.loop = asyncio.new_event_loop() self.handler = TannerWebServer() self.handler.api = AsyncMock() self.handler.redis_client = AsyncMock() self.handler.redis_client.close = AsyncMock() self.returned_content = None self.expected_content = None super(TestWebServer, self).setUp() async def get_application(self): app = await self.handler.make_app() return app @unittest_run_loop async def test_handle_index(self): self.handler.api.return_snares = AsyncMock(return_value=["foo"]) self.handler.api.return_latest_session = AsyncMock() response = await self.client.request("GET", "/") self.returned_content = await response.text() self.assertEqual(response.status, 200) @unittest_run_loop async def test_handle_snares(self): self.handler.api.return_snares = AsyncMock(return_value=["9a631aee-2b52-4108-9831-b495ac8afa80"]) response = await self.client.request("GET", "/snares") self.returned_content = await response.text() self.expected_content = ( '' "9a631aee-2b52-4108-9831-b495ac8afa80" ) self.assertIn(self.expected_content, self.returned_content) @unittest_run_loop async def test_handle_snare(self): response = await self.client.request("GET", "/snare/9a631aee-2b52-4108-9831-b495ac8afa80") self.returned_content = await response.text() self.expected_content = "Snare(9a631aee-2b52-4108-9831-b495ac8afa80) - Tanner Web" self.assertIn(self.expected_content, self.returned_content) @unittest_run_loop async def test_handle_snare_stats(self): content = {"attack_frequency": {"cmd_exec": 1, "lfi": 2, "rfi": 1, "sqli": 0, "xss": 1}} self.handler.api.return_snare_stats = AsyncMock(return_value=content) response = await self.client.request("GET", "/snare-stats/9a631aee-2b52-4108-9831-b495ac8afa80") self.returned_content = await response.text() self.clear_returned_content = "".join(self.returned_content.split()[65:-8]) self.expected_content = ( "AttackFrequencycmd_exec:1
lfi:2" "
rfi:1
sqli:0
xss:1
" ) self.assertEqual(self.expected_content, self.clear_returned_content.strip()) @unittest_run_loop async def test_handle_sessions(self): async def mock_return_sessions(filters): if ( filters["peer_ip"] == "127.0.0.1" and filters["start_time"] == 1497890400 and filters["user_agent"] == "Mozilla/5.0" ): return [ {"sess_uuid": "f387d46eaeb1454cadf0713a4a55be49"}, {"sess_uuid": "e85ae767b0bb4b1f91b421b3a28082ef"}, ] self.handler.api.return_sessions = mock_return_sessions response = await self.client.request( "GET", "/8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4/sessions/page/2?filters=peer_ip:127.0.0.1 start_time:1497890400 user_agent:Mozilla/5.0", ) # noqa self.returned_content = await response.text() self.expected_content = ( "Session-uuid\n IP\n Owner\n \n " '\n\n
\n
\n ' '' ) self.assertIn(self.expected_content, self.returned_content) @unittest_run_loop async def test_handle_sessions_error(self): with self.assertLogs(level="ERROR") as log: response = await self.client.request( "GET", "/8fa6aa98-4283-4085-bfb9-a1cd3a9e56e4/sessions/page/1?filters=start_time:1497890400,peer_ip:127.0.0.1", ) # noqa self.assertIn("Filter error :", log.output[0]) @unittest_run_loop async def test_sessions_info(self): session = dict( cookies={"sess_uuid": "9f82e5d0e6b64047bba996222d45e72c"}, possible_owners={"user": 1.0}, attack_count={"lfi": 1}, ) self.handler.api.return_session_info = AsyncMock(return_value=session) self.expected_content = ( "Cookies\n \n \n " "sess_uuid : 9f82e5d0e6b64047bba996222d45e72c
\n \n \n \n " "\n Referer\n \n \n \n " "Possible Owners\n \n \n " '
user : 1.0
\n' ) response = await self.client.request("GET", "/session/da1811cd19d748058bc02ee5bf9029d4") self.returned_content = await response.text() self.assertIn(self.expected_content, self.returned_content) ================================================ FILE: tanner/tests/test_xss_emulator.py ================================================ import asyncio import unittest from tanner.emulators import xss from tanner.utils import patterns class TestXSSEmulator(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) self.handler = xss.XssEmulator() def test_scan(self): attack = "" assert_result = dict(name="xss", order=3) result = self.handler.scan(attack) self.assertEqual(result, assert_result) def test_scan_negative(self): attack = "alert(1);" assert_result = None result = self.handler.scan(attack) self.assertEqual(result, assert_result) def test_xxs_mako_regex(self): # Mako payloads can be matched with XSS but not vice versa test_mako = "<% x=7*7 %>${x}" # basic mako injection payload verify_mako = patterns.TEMPLATE_INJECTION_MAKO.match(test_mako) assert_result = dict(name="xss", order=3) result = self.handler.scan(test_mako) self.assertEqual(result, assert_result) self.assertTrue(verify_mako) def test_multiple_xss(self): attack_params = [ dict(id="comment", value=""), dict(id="name", value=""), dict(id="email", value=""), ] xss = self.loop.run_until_complete(self.handler.handle(attack_params, None)) assert_result = "" self.assertIn(assert_result, xss["value"]) def test_xss(self): attack_params = [dict(id="foo", value="")] xss = self.loop.run_until_complete(self.handler.handle(attack_params, None)) assert_result = dict(value=attack_params[0]["value"], page=True) self.assertDictEqual(xss, assert_result) ================================================ FILE: tanner/tests/test_xxe_injection.py ================================================ import asyncio import unittest from unittest import mock from tanner import config from tanner.utils.asyncmock import AsyncMock from tanner.emulators.xxe_injection import XXEInjection class TestXXEInjection(unittest.TestCase): def setUp(self): self.loop = asyncio.new_event_loop() asyncio.set_event_loop(None) self.handler = XXEInjection(loop=self.loop) self.result = None self.expected_result = None self.returned_result = None def test_scan(self): payload = '' "" self.expected_result = dict(name="xxe_injection", order=3) self.returned_result = self.handler.scan(payload) self.assertEqual(self.returned_result, self.expected_result) def test_scan_negative(self): payload = ']>' self.expected_result = None self.returned_result = self.handler.scan(payload) self.assertEqual(self.returned_result, self.expected_result) def test_handle_status_code(self): self.handler.get_injection_result = AsyncMock(return_value=None) attack_params = [dict(id="foo", value=']>')] self.expected_result = dict(status_code=504) async def test(): self.returned_result = await self.handler.handle(attack_params) self.loop.run_until_complete(test()) self.assertEqual(self.returned_result, self.expected_result) def test_handle(self): config.TannerConfig.get = mock.MagicMock(return_value=False) code = ( '' "" ']>' "&xxe;" ) attack_params = [dict(id="foo", value=code)] self.handler.helper.get_result = AsyncMock( return_value={ "file_md5": "a43deb0f2d7904cbb6c27c02ed7c2593", "stdout": "root:x:0:0:root:/root:/bin/bash\n\n" "daemon:x:1:1:daemon:/usr/sbin", } ) self.expected_result = "root:x:0:0:root:/root:/bin/bash" async def test(): self.returned_result = await self.handler.handle(attack_params) self.loop.run_until_complete(test()) self.assertIn(self.expected_result, self.returned_result["value"]) def test_handle_oob(self): config.TannerConfig.get = mock.MagicMock(return_value=True) code = ( '' "" '' "" '"http://192.168.1.1:8080/?p=%xxe;">\'> ' ) attack_params = [dict(id="OOB", value=code)] self.handler.helper.get_result = AsyncMock( return_value={ "file_md5": "a43deb0f2d7904cbb6c27c02ed7c2593", "stdout": "root:x:0:0:root:/root:/bin/bash\n\n" "daemon:x:1:1:daemon:/usr/sbin", } ) self.expected_result = "" async def test(): self.returned_result = await self.handler.handle(attack_params) self.loop.run_until_complete(test()) self.assertEqual(self.returned_result["value"], self.expected_result) ================================================ FILE: tanner/utils/__init__.py ================================================ ================================================ FILE: tanner/utils/aiodocker_helper.py ================================================ import aiodocker import logging from tanner.config import TannerConfig class AIODockerHelper: def __init__(self): self.logger = logging.getLogger("tanner.aiodocker_helper.AIODockerHelper") self.docker_client = aiodocker.Docker() self.host_image = TannerConfig.get("DOCKER", "host_image") async def setup_host_image(self, remote_path=None, tag=None): """ Helper to pull host image or build an image with remote Dockerfile :param remote_path (str): remote path of Dockerfile :param tag (str): tag to be given to new image build ex: 'myimage:latest' """ try: if remote_path and tag is not None: params = {"tag": tag, "remote": remote_path} await self.docker_client.images.build(**params) else: image = await self.docker_client.images.list(filter=self.host_image) if not image: await self.docker_client.images.pull(self.host_image) except aiodocker.exceptions.DockerError as docker_error: self.logger.exception("Error while pulling %s image %s", self.host_image, docker_error) async def get_container(self, container_name): """ Gets the container object having specified name :param container_name (str): name of the target container :return: container (object) """ container = None try: container = await self.docker_client.containers.get(container=container_name) except aiodocker.exceptions.DockerError as server_error: self.logger.exception("Error while fetching %s container %s", container_name, server_error) return container async def create_container(self, container_name, cmd=None, image=None): """ Helper to create or replace a container (initially pulls the given image) :param container_name (str): name to be given to new container or replace existing if any :param cmd (list): contains commands to run in the container. ex: ["sh", "-c", "echo 'Hello'"] :param image (str): name of image to be used :return: container (object): newly created container object """ await self.setup_host_image() container = None if image is None: image = self.host_image config = {"Cmd": cmd, "Image": image} try: container = await self.docker_client.containers.create_or_replace(config=config, name=container_name) except (aiodocker.exceptions.DockerError or aiodocker.exceptions.DockerContainerError) as docker_error: self.logger.exception("Error while creating a container %s", docker_error) return container async def execute_cmd(self, cmd, image=None): """ Creates a new container, runs the cmd in it and deletes the used container :param cmd (list): contains commands to run in the container. ex: ["sh", "-c", "echo 'Hello'"] :param image (str): name of image to be used :return: execute_result (str): execution output/errors of cmd from the container """ execute_result = None try: if image is None: image = self.host_image config = {"Cmd": cmd, "Image": image} container = await self.docker_client.containers.run(config=config) await container.wait() result_exists = await container.log(stdout=True, stderr=True) if result_exists: execute_result = "".join(result_exists) # Deleting the used container await container.delete(force=True) except (aiodocker.exceptions.DockerError or aiodocker.exceptions.DockerContainerError) as server_error: self.logger.error("Error while executing command %s in container %s", cmd, server_error) return execute_result async def delete_container(self, container_name): """ Delete an existing container :param container_name (str): name of container to be deleted """ container = await self.get_container(container_name) try: if container: await container.delete(force=True) except aiodocker.exceptions.DockerError as server_error: self.logger.exception("Error while removing %s container %s", container_name, server_error) ================================================ FILE: tanner/utils/api_key_generator.py ================================================ import jwt from tanner.config import TannerConfig def generate(): key = TannerConfig.get("API", "auth_signature") encoded = jwt.encode({"user": "tanner_owner"}, key, algorithm="HS256") return encoded.decode("utf-8") ================================================ FILE: tanner/utils/asyncmock.py ================================================ from unittest.mock import Mock class AsyncMock(Mock): # custom function defined to mock asyncio coroutines def __call__(self, *args, **kwargs): sup = super(AsyncMock, self) async def coro(): return sup.__call__(*args, **kwargs) return coro() def __await__(self): return self().__await__() ================================================ FILE: tanner/utils/base_db_helper.py ================================================ import json import logging import random import mimesis from tanner.config import TannerConfig class BaseDBHelper: def __init__(self): self.logger = logging.getLogger("tanner.base_db_helper.BaseDBHelper") def read_config(self): with open(TannerConfig.get("DATA", "db_config")) as db_config: try: config = json.load(db_config) except json.JSONDecodeError as json_error: self.logger.info("Failed to load json: %s", json_error) else: return config @staticmethod def generate_dummy_data(data_tokens): """ Insert dummy data based on data tokens I - integer id L - login/username E - email P - password T - piece of text :return: """ token_list = data_tokens.split(",") samples_count = random.randint(100, 1000) inserted_data = [] for i in range(samples_count): values = [] person = mimesis.Person("en") for token in token_list: person = mimesis.Person() if token == "I": values.append(i) if token == "L": data = person.username() values.append(data) if token == "E": data = person.email() values.append(data) if token == "P": data = person.password() values.append(data) if token == "T": sample_length = random.randint(1, 10) data = mimesis.Text().text(quantity=sample_length) values.append(data) inserted_data.append(tuple(values)) return inserted_data, token_list ================================================ FILE: tanner/utils/logger.py ================================================ import logging import logging.handlers from tanner.config import TannerConfig class LevelFilter(logging.Filter): """Filters (lets through) all messages with level < LEVEL""" def __init__(self, level): self.level = level def filter(self, record): # "<" instead of "<=": since logger.setLevel is inclusive, this should be exclusive return record.levelno < self.level class Logger: @staticmethod def create_logger(debug_filename, err_filename, logger_name): if TannerConfig.get("CLEANLOG", "enabled") == "True": with open(err_filename, "w"): pass logger = logging.getLogger(logger_name) logger.setLevel(logging.DEBUG) logger.propagate = False formatter = logging.Formatter( fmt="%(asctime)s %(levelname)s:%(name)s:%(funcName)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S" ) # ERROR log to 'tanner.err' error_log_handler = logging.handlers.RotatingFileHandler(err_filename, encoding="utf-8") error_log_handler.setLevel(logging.ERROR) error_log_handler.setFormatter(formatter) logger.addHandler(error_log_handler) # DEBUG log to 'tanner.log' debug_log_handler = logging.handlers.RotatingFileHandler(debug_filename, encoding="utf-8") debug_log_handler.setLevel(logging.DEBUG) debug_log_handler.setFormatter(formatter) max_level_filter = LevelFilter(logging.ERROR) debug_log_handler.addFilter(max_level_filter) logger.addHandler(debug_log_handler) return logger ================================================ FILE: tanner/utils/mysql_db_helper.py ================================================ import logging import subprocess import aiomysql from tanner.config import TannerConfig from tanner.utils.base_db_helper import BaseDBHelper class MySQLDBHelper(BaseDBHelper): # Helper Utility of basic functions for mysqli emulator def __init__(self): super(MySQLDBHelper, self).__init__() self.logger = logging.getLogger("tanner.db_helper.MySQLDBHelper") async def connect_to_db(self): """ Creates a aiomysql connection :return: connection object """ conn = await aiomysql.connect( host=TannerConfig.get("SQLI", "host"), user=TannerConfig.get("SQLI", "user"), password=TannerConfig.get("SQLI", "password"), ) return conn async def check_db_exists(self, db_name): """ Checks if DB exists or not :param db_name (str): mysql db name :return: result (int): 0 if no such database exists else 1 """ conn = await self.connect_to_db() cursor = await conn.cursor() check_DB_exists_query = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA " check_DB_exists_query += "WHERE SCHEMA_NAME='{db_name}'".format(db_name=db_name) await cursor.execute(check_DB_exists_query) result = await cursor.fetchall() return len(result) async def setup_db_from_config(self, name=None): """ Helper function to setup DB from db_config.json and inserts dummy data in the created DB. :param name (str): database name """ config = self.read_config() if name is not None: db_name = name else: db_name = config["name"] conn = await self.connect_to_db() cursor = await conn.cursor() create_db_query = "CREATE DATABASE {db_name}" await cursor.execute(create_db_query.format(db_name=db_name)) await cursor.execute("USE {db_name}".format(db_name=db_name)) for table in config["tables"]: query = table["schema"] await cursor.execute(query) await self.insert_dummy_data(table["table_name"], table["data_tokens"], cursor) await conn.commit() conn.close() async def delete_db(self, db): """ Deletes the database :param db (str): db name to be deleted """ conn = await self.connect_to_db() cursor = await conn.cursor() delete_db_query = "DROP DATABASE {db_name}" await cursor.execute(delete_db_query.format(db_name=db)) await conn.commit() conn.close() async def copy_db(self, user_db, attacker_db): """ Copies the user database to new attacker database :param user_db (str): existing user db :param attacker_db (str): new db to be created :return: new created db (str) """ db_exists = await self.check_db_exists(attacker_db) if db_exists: self.logger.info("Attacker db already exists") else: # create new attacker db conn = await self.connect_to_db() cursor = await conn.cursor() await cursor.execute("CREATE DATABASE {db_name}".format(db_name=attacker_db)) conn.close() # copy user db to attacker db dump_db_cmd = "mysqldump -h {host} -u {user} -p{password} {db_name}" restore_db_cmd = "mysql -h {host} -u {user} -p{password} {db_name}" dump_db_cmd = dump_db_cmd.format( host=TannerConfig.get("SQLI", "host"), user=TannerConfig.get("SQLI", "user"), password=TannerConfig.get("SQLI", "password"), db_name=user_db, ) restore_db_cmd = restore_db_cmd.format( host=TannerConfig.get("SQLI", "host"), user=TannerConfig.get("SQLI", "user"), password=TannerConfig.get("SQLI", "password"), db_name=attacker_db, ) try: dump_db_process = subprocess.Popen(dump_db_cmd, stdout=subprocess.PIPE, shell=True) restore_db_process = subprocess.Popen(restore_db_cmd, stdin=dump_db_process.stdout, shell=True) dump_db_process.stdout.close() dump_db_process.wait() restore_db_process.wait() except subprocess.CalledProcessError as e: self.logger.exception("Error during copying sql database : %s" % e) return attacker_db async def insert_dummy_data(self, table_name, data_tokens, cursor): """ Inserts dummy data in the table based on input data tokens for ex: 'I,L' :param table_name (str): table in which data to be inserted :param data_tokens (str): input data format tokens :param cursor (object): current db cursor """ inserted_data, token_list = self.generate_dummy_data(data_tokens) inserted_string_patt = "%s" if len(token_list) > 1: inserted_string_patt += "," inserted_string_patt *= len(token_list) inserted_string_patt = inserted_string_patt[:-1] await cursor.executemany("INSERT INTO " + table_name + " VALUES(" + inserted_string_patt + ")", inserted_data) async def create_query_map(self, db_name): """ Returns a query map of the tables and its columns present in the database :param db_name (str): current database :return: query_map (dict): Created Query Map """ query_map = {} tables = [] conn = await self.connect_to_db() cursor = await conn.cursor() select_tables = "SELECT table_name FROM INFORMATION_SCHEMA.TABLES WHERE table_schema= '{db_name}'" try: await cursor.execute(select_tables.format(db_name=db_name)) result = await cursor.fetchall() for row in result: tables.append(row[0]) except Exception as e: self.logger.exception("Error during query map creation") else: query_map = dict.fromkeys(tables) for table in tables: query = "SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE \ table_name= '{table_name}' AND table_schema= '{db_name}'" columns = [] try: await cursor.execute(query.format(table_name=table, db_name=db_name)) result = await cursor.fetchall() for row in result: if row[7] == "int": columns.append(dict(name=row[3], type="INTEGER")) else: columns.append(dict(name=row[3], type="TEXT")) query_map[table] = columns except Exception: self.logger.exception("Error during query map creation") return query_map ================================================ FILE: tanner/utils/patterns.py ================================================ import re INDEX = re.compile(r"(/index.html|/)") RFI_ATTACK = re.compile(r".*((http(s){0,1}|ftp(s){0,1}):).*", re.IGNORECASE) LFI_ATTACK = re.compile(r".*(/\.\.)*(home|proc|usr|etc)/.*") LFI_FILEPATH = re.compile(r"((\.\.|/).*)") CMD_ATTACK = re.compile( r".*[^A-z:./]" r"(alias|cat|cd|cp|echo|exec|find|for|grep|ifconfig|ls|man|mkdir|netstat|ping|ps|pwd|uname|wget|touch|while)" r"([^A-z:./]|\b)" ) PHP_CODE_INJECTION = re.compile(r".*(;)*(echo|system|print|phpinfo)(\(.*\)).*") PHP_OBJECT_INJECTION = re.compile(r"(^|;|{|})O:[0-9]+:") CRLF_ATTACK = re.compile(r".*(\r\n).*") XXE_INJECTION = re.compile(r".*<(\?xml|(!DOCTYPE.*)).*>") TEMPLATE_INJECTION_MAKO = re.compile(r".*(<%.*|\s%>).*") TEMPLATE_INJECTION_TORNADO = re.compile(r".*({{.*}}).*") XSS_ATTACK = re.compile(r".*<(.|\n)*?>") REMOTE_FILE_URL = re.compile(r"(.*(http(s){0,1}|ftp(s){0,1}):.*)") WORD_PRESS_CONTENT = re.compile(r"/wp-content/.*") HTML_TAGS = re.compile(r".*<(.*)>.*") QUERY = re.compile(r".*\?.*=") ================================================ FILE: tanner/utils/php_sandbox_helper.py ================================================ import logging import asyncio import aiohttp from tanner import config class PHPSandboxHelper: def __init__(self, loop): self.logger = logging.getLogger("tanner.php_sandbox_helper.PHPSandboxHelper") self._loop = loop if loop is not None else asyncio.get_event_loop() async def get_result(self, code): """ Helper utility to get injection results from PHPOX :param code: Payload from the attacker to be injected in vulnerable code :return: Dict object containing file_md5 and stdout - contains injection results """ result = None phpox_address = "http://{host}:{port}".format( host=config.TannerConfig.get("PHPOX", "host"), port=config.TannerConfig.get("PHPOX", "port") ) try: async with aiohttp.ClientSession(loop=self._loop) as session: async with session.post(phpox_address, data=code) as resp: result = await resp.json() except aiohttp.ClientError as client_error: self.logger.error("Error during connection to php sandbox %s", client_error) finally: await session.close() return result ================================================ FILE: tanner/utils/sqlite_db_helper.py ================================================ import logging import os import shutil import sqlite3 from tanner.utils.base_db_helper import BaseDBHelper class SQLITEDBHelper(BaseDBHelper): def __init__(self): super(SQLITEDBHelper, self).__init__() self.logger = logging.getLogger("tanner.sqlite_db_helper.SQLITEDBHelper") async def setup_db_from_config(self, working_dir, name=None): """ Creates database using config (dict object containing name and tables as keys) :param working_dir: Current working directory :param name: Name of database to be created """ config = self.read_config() if name is not None: db_name = os.path.join(working_dir, name) else: db_name = os.path.join(working_dir, config["name"]) conn = sqlite3.connect(db_name) cursor = conn.cursor() for table in config["tables"]: query = table["schema"] cursor.execute(query) await self.insert_dummy_data(table["table_name"], table["data_tokens"], cursor) conn.commit() conn.close() @staticmethod def get_abs_path(path, working_dir): """ Returns the full path with working directory :param path (str): Current path :param working_dir (str): Current working directory :return: str of full path """ if not os.path.isabs(path): path = os.path.normpath(os.path.join(working_dir, path)) return path @staticmethod def delete_db(db): if db is not None and os.path.exists(db): os.remove(db) def copy_db(self, src, dst, working_dir): src = self.get_abs_path(src, working_dir) dst = self.get_abs_path(dst, working_dir) if os.path.exists(dst): self.logger.info("Attacker db already exists") else: shutil.copy(src, dst) return dst async def insert_dummy_data(self, table_name, data_tokens, cursor): """ Inserts Dummy data in the current sqlite database :param table_name (str): Table to inject dummy data :param data_tokens (str): Tokens to generate dummy data, eg: 'I,L' to get integer ID and username data :param cursor (object): Cursor attached with current DB """ inserted_data, token_list = self.generate_dummy_data(data_tokens) inserted_string_patt = "?" if len(token_list) > 1: inserted_string_patt += "," inserted_string_patt *= len(token_list) inserted_string_patt = inserted_string_patt[:-1] cursor.executemany("INSERT INTO " + table_name + " VALUES(" + inserted_string_patt + ")", inserted_data) def create_query_map( self, working_dir, db_name, ): """ Returns a query map from all the present tables and its columns :param working_dir (str): Current working directory :param db_name (str): Current sqlite database :return: Dict object with tables names as keys and list of columns as its values """ query_map = {} tables = [] db = os.path.join(working_dir, db_name) conn = sqlite3.connect(db) cursor = conn.cursor() select_tables = "SELECT name FROM sqlite_master WHERE type='table'" try: for row in cursor.execute(select_tables): tables.append(row[0]) except sqlite3.OperationalError as sqlite_error: self.logger.exception("Error during query map creation: %s", sqlite_error) else: query_map = dict.fromkeys(tables) for table in tables: query = "PRAGMA table_info(" + table + ")" columns = [] try: for row in cursor.execute(query): columns.append(dict(name=row[1], type=row[2])) query_map[table] = columns except sqlite3.OperationalError as sqlite_error: self.logger.exception("Error during query map creation: %s", sqlite_error) return query_map ================================================ FILE: tanner/web/__init__.py ================================================ ================================================ FILE: tanner/web/server.py ================================================ import asyncio import logging import aiohttp_jinja2 import jinja2 from aiohttp import web from tanner.api import api from tanner import redis_client from tanner.config import TannerConfig from tanner import __version__ as tanner_version class TannerWebServer: def __init__(self): self.logger = logging.getLogger("tanner.web.tannerwebserver") self.api = None self.redis_client = None @aiohttp_jinja2.template("index.html") async def handle_index(self, request): snares = await self.api.return_snares() latest_session = await self.api.return_latest_session() count = len(snares) return {"count": count, "l_session": latest_session, "version": tanner_version} @aiohttp_jinja2.template("snares.html") async def handle_snares(self, request): snares = await self.api.return_snares() return {"snares": snares} @aiohttp_jinja2.template("snare.html") async def handle_snare(self, request): snare_uuid = request.match_info["snare_uuid"] return {"snare": snare_uuid} @aiohttp_jinja2.template("snare-stats.html") async def handle_snare_stats(self, request): snare_uuid = request.match_info["snare_uuid"] snare_stats = await self.api.return_snare_stats(snare_uuid) return {"snare_stats": snare_stats} @aiohttp_jinja2.template("sessions.html") async def handle_sessions(self, request): snare_uuid = request.match_info["snare_uuid"] page_id = int(request.match_info["page_id"]) params = request.url.query applied_filters = {"snare_uuid": snare_uuid} try: if "filters" in params: for filt in params["filters"].split(): applied_filters[filt.split(":")[0]] = filt.split(":")[1] if "start_time" in applied_filters: applied_filters["start_time"] = float(applied_filters["start_time"]) if "end_time" in applied_filters: applied_filters["end_time"] = float(applied_filters["end_time"]) except Exception as e: self.logger.exception("Filter error : %s" % e) result = "Invalid filter definition" else: sessions = await self.api.return_sessions(applied_filters) result = sessions[15 * (page_id - 1) : 15 * page_id] next_val = None pre_val = None if page_id * 15 <= len(sessions): next_val = "/{snare_uuid}/sessions/page/{page_id}".format( snare_uuid=snare_uuid, page_id=str(page_id + 1) ) if len(applied_filters) > 1: next_val += "?filters={filters}".format(filters=params["filters"]) if page_id > 1: pre_val = "/{snare_uuid}/sessions/page/{page_id}".format( snare_uuid=snare_uuid, page_id=str(page_id - 1) ) if len(applied_filters) > 1: pre_val += "?filters={filters}".format(filters=params["filters"]) return {"sessions": result, "next_val": next_val, "pre_val": pre_val} @aiohttp_jinja2.template("session.html") async def handle_session_info(self, request): sess_uuid = request.match_info["sess_uuid"] session = await self.api.return_session_info(sess_uuid) return {"session": session} async def on_shutdown(self, app): await self.redis_client.close() def setup_routes(self, app): app.router.add_get("/", self.handle_index) app.router.add_get("/snares", self.handle_snares) app.router.add_resource("/snare/{snare_uuid}").add_route("GET", self.handle_snare) app.router.add_resource("/snare-stats/{snare_uuid}").add_route("GET", self.handle_snare_stats) app.router.add_resource("/session/{sess_uuid}").add_route("GET", self.handle_session_info) app.router.add_resource("/{snare_uuid}/sessions/page/{page_id}").add_route("GET", self.handle_sessions) app.router.add_static("/static/", path="tanner/web/static") async def make_app(self): app = web.Application() aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader("tanner/web/templates")) app.on_shutdown.append(self.on_shutdown) self.setup_routes(app) return app def start(self): loop = asyncio.get_event_loop() self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client(poolsize=20)) self.api = api.Api(self.redis_client) host = TannerConfig.get("WEB", "host") port = int(TannerConfig.get("WEB", "port")) web.run_app(self.make_app(), host=host, port=port) ================================================ FILE: tanner/web/static/css/styles.css ================================================ table { border-collapse: collapse; width: 80%; } th, td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; } tr:hover{ background-color:#f5f5f5 } ================================================ FILE: tanner/web/static/js/site.js ================================================ function findGetParameter(parameterName) { var result = null, tmp = []; location.search .substr(1) .split("&") .forEach(function (item) { tmp = item.split("="); if (tmp[0] === parameterName) result = decodeURIComponent(tmp[1]); }); return result; } document.getElementById('filters').value = findGetParameter("filters"); ================================================ FILE: tanner/web/templates/base.html ================================================ {% block title %}{% endblock %} - Tanner Web
{% block content%} {% endblock%}
================================================ FILE: tanner/web/templates/index.html ================================================ {% extends "base.html" %} {% block title %}Home{% endblock %} {% block content %}

Tannerweb


Tanner version - {{version}}

No. of snares connected - {{count}}

{% if l_session is not none %}

Latest session - {{l_session}}

{% endif %}

Click here to navigate to the snares list.

{% endblock %} ================================================ FILE: tanner/web/templates/session.html ================================================ {% extends "base.html" %} {% block title %}Session({{session.sess_uuid}}){% endblock %} {% block content %}

SESSION INFO

Key Value
UUID {{session.sess_uuid}}
IP {{session.peer_ip}}
Location {{session.location}}
Port {{session.peer_port}}
User Agents {{session.user_agent}}
Snare UUID {{session.snare_uuid}}
Start time {{session.start_time}}
End time {{session.end_time}}
Requests/sec {{session.requests_in_second}}
Time between requests {{session.approx_time_between_requests}}
Paths {{session.accepted_paths}}
Errors {{session.errors}}
Hidden Links {{session.hidden_links}}
Attacks {% for attack, v in session.attack_count.items() %} {{attack}} - {{v}}
{% endfor %}
Paths {% for path in session.paths %} {{path}}
{% endfor %}
Cookies {% for key, val in session.cookies.items() %} {{key}} : {{val}}
{% endfor %}
Referer {{session.referer}}
Possible Owners {% for owner, cf in session.possible_owners.items() %} {{owner}} : {{cf}}
{% endfor %}
{% endblock %} ================================================ FILE: tanner/web/templates/sessions.html ================================================ {% extends "base.html" %} {% block title %}Snare Sessions{% endblock %} {% block content %}

SNARE-SESSIONS

Filters: How?
{% for sess in sessions %} {% endfor %}
No Session-uuid IP Owner
{{loop.index}} {{sess.sess_uuid}} {{sess.peer_ip}} {{sess.possible_owners|join(', ')}}

{% if pre_val %}Previous{% endif %} {% if next_val %}Next{% endif %}
{% endblock %} ================================================ FILE: tanner/web/templates/snare-stats.html ================================================ {% extends "base.html" %} {% block title %}SNARE-STATS{% endblock %} {% block content %}

SNARE-STATS

{% if 'total_sessions' in snare_stats %} {% endif %} {% if 'total_duration' in snare_stats %} {% endif %}
Key Value
No of Sessions{{snare_stats.total_sessions}}
Total Duration{{snare_stats.total_duration}}
Attack Frequency {% if 'attack_frequency' in snare_stats %} {% for key, val in snare_stats.attack_frequency.items() %} {{key}} : {{val}}
{% endfor %} {% endif %}
{% endblock %} ================================================ FILE: tanner/web/templates/snare.html ================================================ {% extends "base.html" %} {% block title %}Snare({{snare}}){% endblock %} {% block content %}

{{snare}}

Snare-Stats

Sessions

{% endblock %} ================================================ FILE: tanner/web/templates/snares.html ================================================ {% extends "base.html" %} {% block title %}Snares{% endblock %} {% block content %}

SNARE-UUIDS

{% for snare in snares %} {% endfor %}
No Snare-uuid
{{loop.index}} {{snare}}
{% endblock %}