[ { "path": ".gitattributes", "content": "* ident\n" }, { "path": ".gitignore", "content": "" }, { "path": "MANIFEST.in", "content": "include *.py\ninclude *.yapsy-plugin\ninclude docs/*.pdf\ninclude pylint.rc\ninclude Makefile\ninclude README\ninclude README.CREDITS\ninclude README.INSTALL\ninclude README.LICENSE\ninclude README.PLUGINS\ninclude mastiff.conf\ninclude skeleton/*.py\ninclude skeleton/*.yapsy-plugin\ninclude tests/*\ninclude utils/*\nexclude README.RELENG\nrecursive-exclude docs *.odt\nrecursive-include mastiff *.py *.yapsy-plugin\n" }, { "path": "Makefile", "content": "# $Id: 77c80f02785dfc5ef2f764bfe7f487dc0c165278 $\n#\n# Makefile for installation of mastiff.\n#\n\nall: build\n\nbuild::\n\t@ python setup.py build\n\ncheck test:\n\t@ bash tests/import-test.sh `pwd`\n\t@ bash tests/mastiff-test.sh\n\t@ rm -rf work/\n\ncheck-clean test-clean: clean\n\t@ rm -f tests/test-*.txt\n\nclean:\n\t@ rm -f `find . -name \"*.pyc\" -o -name \"*~\"`\n\t@ rm -rf dist build mastiff.egg-info\n\t@ rm -f tests/*.txt\n\nclean-all: check-clean dev-clean\n\ndev:\n\t@ python setup.py develop\n\ndev-clean: clean\n\t@ python setup.py develop --uninstall\n\t@ rm -f /usr/local/bin/mas.py\n\ndist sdist::\n\t@ python setup.py sdist\n\ninstall: build\n\t@ python setup.py install\n\nlint:\n\t@ find . -name \"*.py\" -exec pylint --rcfile=pylint.rc {} \\;\n\nsign: dist\n\t@ version_number=`egrep '^version = 0x' mastiff/__init__.py | awk '{print $$3}'` ; \\\n\tversion_string=`utils/version2string -t tar -v $${version_number}` ; \\\n\tdist_file=\"dist/mastiff-$${version_string}.tar.gz\" ; \\\n\tgpg --default-key 64615D14 -s -b $${dist_file}\n\n" }, { "path": "PKG-INFO", "content": "Metadata-Version: 1.0\nName: mastiff\nVersion: 0.8.0.ds0\nSummary: MASTIFF is a static analysis automation framework.\nHome-page: http://www.korelogic.com\nAuthor: Tyler Hudak\nAuthor-email: mastiff-project@korelogic.com\nLicense: Apache License V2.0\nDescription: MASTIFF is a static analysis framework that automates the\n process of extracting key characteristics from a number of different file\n formats. To ensure the framework remains flexible and extensible, a\n community-driven set of plug-ins is used to perform file analysis and data\n extraction. While originally designed to support malware, intrusion, and\n forensic analysis, the framework is well-suited to support a broader range of\n analytic needs. In a nutshell, MASTIFF allows analysts to focus on analysis\n rather than figuring out how to parse files.\nPlatform: Linux\n" }, { "path": "README", "content": "\nREVISION\n\n $Id: 17f09461545f9d0409f9480a417c3831ae34539d $\n\nOVERVIEW\n\n MASTIFF is a static analysis framework that automates the process of\n extracting key characteristics from a number of different file\n formats. To ensure the framework remains flexible and extensible, a\n community-driven set of plug-ins is used to perform file analysis\n and data extraction. While originally designed to support malware,\n intrusion, and forensic analysis, the framework is well-suited to\n support a broader range of analytic needs. In a nutshell, MASTIFF\n allows analysts to focus on analysis rather than figuring out how to\n parse files.\n\n The MASTIFF Project is hosted at:\n\n https://git.korelogic.com/mastiff.git/\n\nDOCUMENTATION\n\n General documentation is located in the docs directory. See the\n README.INSTALL file for instructions on how to build, test, and\n install the framework.\n\nLICENSE\n\n The terms and conditions under which this software is released are\n set forth in README.LICENSE.\n\n" }, { "path": "README.CREDITS", "content": "\nREVISION\n\n $Id: 02e5406c2bbd4202e46796589395a4611897b806 $\n\nCREDITS\n\n Tyler Hudak (author, maintainer)\n Klayton Monroe (contributor, maintainer)\n\nSPONSORS\n\n DARPA Cyber Fast Track Program (2012)\n KoreLogic (2012-present)\n\n" }, { "path": "README.INSTALL", "content": "\nREVISION\n\n $Id: daec28262cb37c5a4952618675b33e234e48773d $\n\nOVERVIEW\n\n MASTIFF is a static analysis framework that automates the process of\n extracting key characteristics from a number of different file\n formats. To ensure the framework remains flexible and extensible, a\n community-driven set of plug-ins is used to perform file analysis\n and data extraction. While originally designed to support malware,\n intrusion, and forensic analysis, the framework is well-suited to\n support a broader range of analytic needs. In a nutshell, MASTIFF\n allows analysts to focus on analysis rather than figuring out how to\n parse files.\n\n The MASTIFF Project is hosted at:\n\n https://git.korelogic.com/mastiff.git/\n\nTECHNICAL REQUIREMENTS\n\n The following software must be installed for MASTIFF to work\n properly.\n\n - Python 2.6.6 or greater\n - Yapsy 1.10 or greater (http://yapsy.sourceforge.net/)\n - Python sqlite3 (http://docs.python.org/library/sqlite3)\n - Python setuptools (http://pypi.python.org/pypi/setuptools/)\n - Yara, libyara and yara-python (http://code.google.com/p/yara-project)\n\n A Python libmagic library is also required. MASTIFF supports two different\n libmagic libraries:\n\n - libmagic Python extensions (ftp://ftp.astron.com/pub/file/)\n This may be installed through the source code above or is the library\n installed as python-magic in most Linux code repositories.\n\n - Python-magic (https://github.com/ahupp/python-magic/)\n This may be installed through the source code above or via Python\n pip.\n\nPREREQUISITES INSTALLATION\n\n The Python setuptools and magic libraries will need to be installed\n on your own. For Debian/Ubuntu-based distributions, this can be\n accomplished with:\n\n $ sudo aptitude install python-setuptools\n $ sudo aptitude install python-magic\n\n On Gentoo-based distributions, there is no Python magic package.\n However, adding the python USE flag to the sys-apps/file package\n will create the correct Python libraries.\n\n Setuptools can be installed as follows:\n\n $ sudo emerge -av setuptools\n\n Yapsy will automatically download and install when the make program\n is run, or you can download and install it on your own. Yapsy is\n also located in the Gentoo Portage repository.\n\n $ sudo emerge -av yapsy\n\n Note that the plug-ins utilized by MASTIFF may have their own\n prerequisites.\n\nTESTING\n\n MASTIFF comes with a test set suite that can be used to determine if all\n prerequisites have been properly installed and MASTIFF is able to analyze\n files correctly. To run these tests, run:\n\n\t$ make test\n\n Two sets of tests will run.\n\n - Python imports for all MASTIFF core files and plug-ins will be checked to\n ensure they can be imported. Any that cannot will be displayed.\n\n - MASTIFF will examine 4 different files to ensure there are no issues.\n\n All output will go into the tests/ directory.\n\nINSTALLATION\n\n If you wish to only test out MASTIFF, skip to the Development\n Testing section.\n\n MASTIFF utilizes the Python setuptools code for installation of the\n package. The easiest way to install the package is:\n\n $ sudo make install\n\n This will install the package into the appropriate Python\n site-packages directory for your system. It will also install\n mas.py, the main MASTIFF wrapper script into /usr/local/bin.\n\n If you do not have Yapsy installed, it will attempt to download and\n install it for you.\n\n If you install using this method, the only way to uninstall is to\n manually delete files.\n\n After installing MASTIFF, modify the mastiff.conf configuration file\n to ensure the options for plug-ins are correctly set for your analysis\n system.\n\nDEVELOPMENT TESTING\n\n If instead you wish to only test it for development purposes, run\n the following command:\n\n $ sudo make dev\n\n This will install placeholders into the Python dist-packages that\n point to this directory. Any modifications made to the code will\n automatically be reflected when running the software. Additionally,\n mas.py will be placed in /usr/local/bin.\n\n To uninstall the dev environment, run:\n\n $ sudo make dev-clean\n\n This will remove all placeholders as well as /usr/local/bin/mas.py.\n\nPLUG-IN REQUIREMENTS\n\n At the current release, the plug-ins utilized by MASTIFF require a\n number of additional libraries or programs to be installed.\n\n - ssdeep (http://ssdeep.sourceforge.net/)\n - pydeep (https://github.com/kbandla/pydeep)\n - Yara, libyara and yara-python must be installed,\n (http://code.google.com/p/yara-project)\n - simplejson (https://github.com/simplejson/simplejson)\n - Didier Stevens pdf-parser.py\n (http://blog.didierstevens.com/programs/pdf-tools/)\n - Didier Stevens' pdfid.py (http://blog.didierstevens.com/programs/pdf-tools/)\n - exiftool (http://www.sno.phy.queensu.ca/~phil/exiftool/)\n - pefile library (http://code.google.com/p/pefile/)\n NOTE: Do NOT install pefile from the Debian/Ubuntu repository! Install\n from source!\n - disitool.py (http://blog.didierstevens.com/programs/disitool/)\n - openssl binary (http://www.openssl.org/)\n - Giuseppe 'Evilcry' Bonfa's pyOLEScanner.py\n (https://github.com/Evilcry/PythonScripts/raw/master/pyOLEScanner.zip)\n - distorm (http://code.google.com/p/distorm/)\n\n Some of these programs may be able to be installed from your\n distribution's software repository, and some may need to be\n installed from source. After these programs have been installed,\n be sure to check the MASTIFF configuration file and update all\n configuration options to point to the correct locations.\n\nRUNNING MASTIFF\n\n The best way to run MASTIFF is to use the mas.py program. This\n script has been written to provide you with the maximum number of\n options for using MASTIFF. This script will be installed to\n /usr/local/bin when you install the package.\n\n mas.py can be run by only giving it a file or directory to analyze as an\n argument.\n\n $ mas.py /path/to/file2analyze\n\n If MASTIFF is given a directory, it will enumerate all files within that\n directory, and every subdirectory, and analyze them.\n\n Although the only required argument is the filename or directory to be\n analyzed, the following table lists available options.\n\n -c CONFIG_FILE, --conf=CONFIG_FILE\n Use an alternate config file. The default is\n './mastiff.conf'.\n -h, --help Show the help message and exit.\n -l PLUGIN_TYPE, --list=PLUGIN_TYPE\n List all available plug-ins of the specified type and\n exit. Type must be one of 'analysis' or 'cat'.\n -o OVERRIDE, --option=OVERRIDE\n Override a config file option. Configuration options\n should be specified as 'Section.Key=Value' and should\n be quoted if any whitespace is present. Multiple\n overrides can be specified by using multiple '-o'\n options.\n -p PLUGIN_NAME, --plugin=PLUGIN_NAME\n Only run the specified analysis plug-in. Name must be\n quoted if it contains whitespace.\n -q, --quiet Only log errors.\n -t FTYPE, --type=FTYPE\n Force file to be analyzed with plug-ins from the\n specified category (e.g., EXE, PDF, etc.). Run with\n '-l cat' to list all available category plug-ins.\n -V, --verbose Print verbose logs.\n -v, --version Show program's version number and exit.\n\n Queue Options:\n --append-queue Append file or directory to job queue and exit.\n --clear-queue Clear job queue and exit.\n --ignore-queue Ignore the job queue and just process file.\n --list-queue List the contents of the job queue and exit.\n --resume-queue Continue processing the queue.\n" }, { "path": "README.LICENSE", "content": "\nREVISION\n\n $Id: f19abdb0df9b2aadb274fb66a8f813edb7f508a0 $\n\nOVERVIEW\n\n This document contains licensing information for The MASTIFF\n Project, which was established by Tyler Hudak of KoreLogic, Inc.\n in 2012. Unless specifically excluded, all files in this project\n fall under the terms and conditions of the Apache License,\n Version 2.0 as stated below. Excluded files or components that\n fall under other licenses are detailed below as well.\n\nTHE APACHE LICENSE VERSION 2.0 (MASTIFF)\n\n Copyright 2012-2013 The MASTIFF Project\n\n All rights reserved.\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use,\n reproduction, and distribution as defined by Sections 1 through 9\n of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under\n common control with that entity. For the purposes of this\n definition, \"control\" means (i) the power, direct or indirect, to\n cause the direction or management of such entity, whether by\n contract or otherwise, or (ii) ownership of fifty percent (50%)\n or more of the outstanding shares, or (iii) beneficial ownership\n of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making\n modifications, including but not limited to software source code,\n documentation source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but not\n limited to compiled object code, generated documentation, and\n conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work (an\n example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or\n Object form, that is based on (or derived from) the Work and for\n which the editorial revisions, annotations, elaborations, or\n other modifications represent, as a whole, an original work of\n authorship. For the purposes of this License, Derivative Works\n shall not include works that remain separable from, or merely\n link (or bind by name) to the interfaces of, the Work and\n Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including the\n original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright\n owner or by an individual or Legal Entity authorized to submit on\n behalf of the copyright owner. For the purposes of this\n definition, \"submitted\" means any form of electronic, verbal, or\n written communication sent to the Licensor or its\n representatives, including but not limited to communication on\n electronic mailing lists, source code control systems, and issue\n tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work,\n but excluding communication that is conspicuously marked or\n otherwise designated in writing by the copyright owner as \"Not a\n Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal\n Entity on behalf of whom a Contribution has been received by\n Licensor and subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions\n of this License, each Contributor hereby grants to You a\n perpetual, worldwide, non-exclusive, no-charge, royalty-free,\n irrevocable copyright license to reproduce, prepare Derivative\n Works of, publicly display, publicly perform, sublicense, and\n distribute the Work and such Derivative Works in Source or Object\n form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have\n made, use, offer to sell, sell, import, and otherwise transfer\n the Work, where such license applies only to those patent claims\n licensable by such Contributor that are necessarily infringed by\n their Contribution(s) alone or by combination of their\n Contribution(s) with the Work to which such Contribution(s) was\n submitted. If You institute patent litigation against any entity\n (including a cross-claim or counterclaim in a lawsuit) alleging\n that the Work or a Contribution incorporated within the Work\n constitutes direct or contributory patent infringement, then any\n patent licenses granted to You under this License for that Work\n shall terminate as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or Derivative\n Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute\n must include a readable copy of the attribution notices\n contained within such NOTICE file, excluding those notices\n that do not pertain to any part of the Derivative Works, in\n at least one of the following places: within a NOTICE text\n file distributed as part of the Derivative Works; within the\n Source form or documentation, if provided along with the\n Derivative Works; or, within a display generated by the\n Derivative Works, if and wherever such third-party notices\n normally appear. The contents of the NOTICE file are for\n informational purposes only and do not modify the License.\n You may add Your own attribution notices within Derivative\n Works that You distribute, alongside or as an addendum to the\n NOTICE text from the Work, provided that such additional\n attribution notices cannot be construed as modifying the\n License.\n\n You may add Your own copyright statement to Your modifications\n and may provide additional or different license terms and\n conditions for use, reproduction, or distribution of Your\n modifications, or for any such Derivative Works as a whole,\n provided Your use, reproduction, and distribution of the Work\n otherwise complies with the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state\n otherwise, any Contribution intentionally submitted for inclusion\n in the Work by You to the Licensor shall be under the terms and\n conditions of this License, without any additional terms or\n conditions. Notwithstanding the above, nothing herein shall\n supersede or modify the terms of any separate license agreement\n you may have executed with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the\n trade names, trademarks, service marks, or product names of the\n Licensor, except as required for reasonable and customary use in\n describing the origin of the Work and reproducing the content of\n the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or\n conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or\n FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for\n determining the appropriateness of using or redistributing the\n Work and assume any risks associated with Your exercise of\n permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect,\n special, incidental, or consequential damages of any character\n arising as a result of this License or out of the use or\n inability to use the Work (including but not limited to damages\n for loss of goodwill, work stoppage, computer failure or\n malfunction, or any and all other commercial damages or losses),\n even if such Contributor has been advised of the possibility of\n such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act\n only on Your own behalf and on Your sole responsibility, not on\n behalf of any other Contributor, and only if You agree to\n indemnify, defend, and hold each Contributor harmless for any\n liability incurred by, or claims asserted against, such\n Contributor by reason of your accepting any such warranty or\n additional liability.\n\nTHE NEW BSD LICENSE (WebJob)\n\n This project includes software developed for The WebJob Project,\n which is distributed under the following terms and conditions:\n\n Copyright 2006-2013 The WebJob Project\n\n All rights reserved.\n\n Redistribution and use in source and binary forms, with or without\n modification, are permitted provided that the following conditions\n are met:\n\n 1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n\n 2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in\n the documentation and/or other materials provided with the\n distribution.\n\n 3. Neither the names of the copyright holders nor the names of any\n contributors may be used to endorse or promote products derived\n from this software without specific prior written permission.\n\n THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS\n FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE\n COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,\n INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\n SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,\n STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)\n ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED\n OF THE POSSIBILITY OF SUCH DAMAGE.\n\n" }, { "path": "README.PLUGINS", "content": "\nREVISION\n\n $Id: 9a263fb024741bc9fa6fafd3b146d260e9db4d26 $\n\nSKELETON PLUG-INS\n\n The project's skeleton directory contains three types of skeleton\n plug-ins that can be used to start coding your own plug-ins for the\n framework. Just choose the skeleton code for the type of plug-in\n you would like to develop, modify a few lines, and start coding.\n Note that these files are intended to serve as examples and helpful\n hints on how to get started, not as definitive ways to create\n plug-ins.\n\n The three types skeleton plug-ins are:\n\n - category-skel: A skeleton category plug-in to define a new\n file type.\n\n - analysis-skel: A skeleton analysis plug-in to define a new\n type of analysis. This code is for a Generic plug-in, but can be\n easily modified for any file-type category.\n\n - analysis-ext-skel: A skeleton analysis plug-in to define a new\n type of analysis that calls an external program. This type of\n plug-in is excellent for acting as a wrapper script around another\n program.\n\n" }, { "path": "mas.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nMASTIFF - MAlicious Static Inspection File Framework\n\nThis program implements the code necessary to statically analyze files within\na plugin-based framework.\n\n\"\"\"\n\n__version__ = \"$Id: e3288d64e94fb2c155552a6922e77e347081d77f $\"\n\nimport sys\nimport logging\nimport os\nimport os.path\n#import signal\n\nif sys.version_info < (2, 6, 6):\n sys.stderr.write(\"Mastiff requires python version 2.6.6\")\n sys.exit(1)\n\nfrom optparse import OptionParser, OptionGroup\nimport mastiff.core as Mastiff\nfrom mastiff import get_release_string\nimport mastiff.queue as queue\n\ndef add_to_queue(job_queue, fname):\n \"\"\" Add file and/or directory to job queue. \"\"\"\n \n log = logging.getLogger('Mastiff.queue')\n # check to see if we are dealing with a directory or a file and handle correctly\n if os.path.isdir(fname) is True:\n # This is a directory - walk it and add all its files\n log.info('Adding directory %s to queue.' % fname)\n for root, _, files in os.walk(fname):\n for new_file in [ os.path.abspath(root + os.sep + f) for f in files]:\n log.debug('Adding %s to job queue.' % new_file )\n job_queue.append(new_file)\n elif os.path.isfile(fname) is True:\n # dealing with a file - just add it to the queue\n log.debug('Adding file %s to job queue.' % fname)\n job_queue.append(fname)\n else:\n log.error('Submission is neither file or directory. Exiting.')\n sys.exit(1)\n \ndef analyze_file(fname, opts, loglevel):\n \"\"\" Analyze a file with MASTIFF. \"\"\"\n \n log = logging.getLogger('Mastiff.analyze')\n log.info(\"Starting analysis on %s\", fname)\n \n my_analysis = Mastiff.Mastiff(opts.config_file, loglevel=loglevel, override=opts.override)\n if opts.ftype is not None:\n log.info('Forcing file type to include \"%s\"', opts.ftype)\n my_analysis.set_filetype(fname=fname, ftype=opts.ftype)\n \n my_analysis.analyze(fname, opts.plugin_name)\n \ndef main():\n \"\"\"Parse options and analyze file.\"\"\"\n\n usage = \"usage: %prog [options] FILE|DIRECTORY\"\n parser = OptionParser(\n add_help_option = False,\n version = \"%prog \" + get_release_string(),\n usage = usage)\n parser.remove_option(\"--version\")\n \n parser.add_option(\n \"--conf\",\n \"-c\",\n action = \"store\",\n default = \"./mastiff.conf\",\n dest = \"config_file\",\n help = \"Use an alternate config file. The default is './mastiff.conf'.\",\n type = \"string\")\n parser.add_option(\n \"--help\",\n \"-h\",\n action = \"help\",\n help = \"Show the help message and exit.\")\n parser.add_option(\n \"--list\",\n \"-l\",\n action = \"store\",\n dest = \"list_plugins\",\n help = \"List all available plug-ins of the specified type and exit. Type must be one of 'analysis', 'cat', or 'output'.\",\n metavar = \"PLUGIN_TYPE\")\n parser.add_option(\n \"--option\",\n \"-o\",\n action=\"append\",\n default = None,\n dest = \"override\",\n help = \"Override a config file option. Configuration options should be specified as 'Section.Key=Value' and should be quoted if any whitespace is present. Multiple overrides can be specified by using multiple '-o' options.\")\n parser.add_option(\n \"--plugin\",\n \"-p\",\n action = \"store\",\n default = None,\n dest = \"plugin_name\",\n help = \"Only run the specified analysis plug-in. Name must be quoted if it contains whitespace.\")\n parser.add_option(\n \"--quiet\",\n \"-q\",\n action = \"store_true\",\n default = False,\n dest = \"quiet\",\n help = \"Only log errors.\")\n parser.add_option(\n \"--type\",\n \"-t\",\n action = \"store\",\n default = None,\n dest = \"ftype\",\n help = \"Force file to be analyzed with plug-ins from the specified category (e.g., EXE, PDF, etc.). Run with '-l cat' to list all available category plug-ins.\",\n type = \"string\")\n parser.add_option(\n \"--verbose\",\n \"-V\",\n action = \"store_true\",\n dest = \"verbose\",\n default = False,\n help = \"Print verbose logs.\")\n parser.add_option(\n \"--version\",\n \"-v\",\n action = \"version\",\n help = \"Show program's version number and exit.\")\n \n queue_group = OptionGroup(parser, \"Queue Options\")\n queue_group.add_option(\n \"--append-queue\",\n \"\",\n action = \"store_true\",\n dest = \"append_queue\",\n default = False,\n help = \"Append file or directory to job queue and exit.\")\n queue_group.add_option(\n \"--clear-queue\",\n \"\",\n action = \"store_true\",\n dest = \"clear_queue\",\n default = False,\n help = \"Clear job queue and exit.\")\n queue_group.add_option(\n \"--ignore-queue\",\n \"\",\n action = \"store_true\",\n dest = \"ignore_queue\",\n default = False,\n help = \"Ignore the job queue and just process file.\") \n queue_group.add_option(\n \"--list-queue\",\n \"\",\n action = \"store_true\",\n dest = \"list_queue\",\n default = False,\n help = \"List the contents of the job queue and exit.\")\n queue_group.add_option(\n \"--resume-queue\",\n action = \"store_true\",\n default = False,\n dest = \"resume_queue\",\n help = \"Continue processing the queue.\")\n parser.add_option_group(queue_group)\n\n (opts, args) = parser.parse_args()\n\n if (args is None or len(args) < 1) and opts.list_plugins is None \\\n and opts.clear_queue is False and opts.resume_queue is False \\\n and opts.list_queue is False:\n parser.print_help()\n sys.exit(1)\n\n if opts.verbose == True:\n loglevel = logging.DEBUG\n elif opts.quiet == True:\n loglevel = logging.ERROR\n else:\n loglevel = logging.INFO\n \n format_ = '[%(asctime)s] [%(levelname)s] [%(name)s] : %(message)s' \n logging.basicConfig(format=format_)\n log = logging.getLogger(\"Mastiff\")\n log.setLevel(loglevel)\n\n # check to see if we are running as root\n if os.geteuid() == 0:\n log.warning('You are running MASTIFF as ROOT! This may be DANGEROUS!')\n\n if opts.list_plugins is not None:\n plugs = Mastiff.Mastiff(opts.config_file)\n plugs.list_plugins(opts.list_plugins)\n sys.exit(0)\n\n # set up job queue\n job_queue = queue.MastiffQueue(opts.config_file)\n \n # process job queue specific options\n if opts.clear_queue is True:\n log.info('Clearing job queue and exiting.')\n job_queue.clear_queue()\n sys.exit(0)\n elif opts.list_queue is True:\n if len(job_queue) == 0:\n log.info(\"MASTIFF job queue is empty.\")\n else:\n log.info(\"MASTIFF job queue has %d entries.\" % len(job_queue))\n print \"\\nFile Name\\n---------\\n%s\" % (job_queue) \n sys.exit(0)\n \n if len(args) > 0:\n fname = args[0]\n else:\n fname = None\n \n if opts.ignore_queue is True: \n log.info('Ignoring job queue.')\n analyze_file(fname, opts, loglevel)\n sys.exit(0)\n\n # add file or directory to queue\n if fname is not None:\n add_to_queue(job_queue, fname)\n if opts.append_queue is True:\n sys.exit(0) \n\n # Start analysis on the files in the queue until it is empty\n while len(job_queue) > 0:\n fname = job_queue.popleft()\n analyze_file(fname, opts, loglevel) \n log.info('There are %d jobs in the queue.' % len(job_queue))\n\nif __name__ == '__main__':\n\n main()\n\n" }, { "path": "mastiff/__init__.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nMASTIFF - MAlicious Static Inspection File Framework\n\nThis program implements the code necessary to statically analyze files within\na plugin-based framework.\n\n\"\"\"\n\n\"\"\"\nThis file contains package-level variables and functions.\n\"\"\"\n\n__version__ = \"$Id: b55ca3df0a5fa81dea4ab70cfcb713e0759c973b $\"\n\nversion = 0x00800000\n\ndef get_release_number():\n \"\"\" Gets the current release version. \"\"\"\n return version\n\ndef get_release_string():\n \"\"\"Return the current release version.\"\"\"\n major = (version >> 28) & 0x0f\n minor = (version >> 20) & 0xff\n patch = (version >> 12) & 0xff\n state = (version >> 10) & 0x03\n build = version & 0x03ff\n if state == 0:\n state_string = \"ds\"\n elif state == 1:\n state_string = \"rc\"\n elif state == 2:\n state_string = \"sr\"\n elif state == 3:\n state_string = \"xs\"\n if state == 2 and build == 0:\n return '%d.%d.%d' % (major, minor, patch)\n else:\n return '%d.%d.%d.%s%d' % (major, minor, patch, state_string, build)\n\n" }, { "path": "mastiff/conf.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nFunctions to parse and maintain the Mastiff config file.\n\nThe Conf class is used to parse and maintain the Mastiff config file.\n\n_init__(self, config_file=None, override=None): Initializes the config file and\nsets up any overridden options.\n\nget_var(section, var): Return a variable from a specified section.\n\nget_bvar(section, var): Return a boolean variable from a specified section.\n\nset_var(section, var, value): Set a variable in a specified section with a\n given value.\n\nget_section(section): Return a dictionary of items within the section.\n\nlist_config(): Prints all configuration variables read in.\n\ndump_config(): Dump a copy of the config into the Mastiff log dir.\n\noverride_option(): Override an option from the config file.\n\n\"\"\"\n\n__version__ = \"$Id: daa2ace9c5481298f0650b96fe31bb786bbc3c8e $\"\n\nimport os\nimport sys\nimport logging\nimport ConfigParser\n\nclass Conf:\n \"\"\"Parse and maintain the Mastiff configuration.\"\"\"\n\n def __init__(self, config_file=None, override=None):\n \"\"\"Initialize the class parameters.\"\"\"\n\n log = logging.getLogger('Mastiff.Conf')\n\n self.config_file = os.path.abspath(config_file)\n\n self.config = ConfigParser.ConfigParser()\n self.set_defaults()\n\n # read from the default file locations and the file given\n # file given will be read last and will over-write any\n # previously read-in config files\n files_read = self.config.read(['/etc/mastiff/mastiff.conf',\n os.path.expanduser('~/.mastiff.conf'),\n config_file])\n\n if not files_read:\n log.error(\"Could not read any configuration files. Exiting.\")\n sys.exit(1)\n else:\n if self.config.getboolean('Misc', 'verbose') == True:\n log.setLevel(logging.DEBUG)\n log.debug(\"Read config from %s\", str(files_read))\n\n if override is not None:\n for opt in override:\n self.override_option(opt)\n\n def set_defaults(self):\n \"\"\"\n Set default variables.\n If set later in a config file, these will be overwritten.\n Note: This is being done instead of a default config file to\n reduce the number of files needed.\n \"\"\"\n self.config.add_section('Dir')\n self.set_var('Dir', 'log_dir', '/var/log/mastiff')\n #self.set_var('Dir', 'plugin_dir', '/usr/local/mastiff/plugins')\n self.config.add_section('Misc')\n self.set_var('Misc', 'verbose', 'off')\n\n def get_var(self, section, var):\n \"\"\"Return a specified variable.\"\"\"\n try:\n return self.config.get(section, var)\n except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):\n log = logging.getLogger('Mastiff.Conf.GetVar')\n log.error('Could not find \"%s\": \"%s\"', section, var)\n return None\n\n def get_bvar(self, section, var):\n \"\"\"Return a boolean variable.\"\"\"\n try:\n return self.config.getboolean(section, var)\n except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):\n log = logging.getLogger('Mastiff.Conf.GetVar')\n log.error('Could not find \"%s\": \"%s\"', section, var)\n return False\n\n def get_section(self, section):\n \"\"\"Return a dictionary of items within a section.\"\"\"\n try:\n options = self.config.items(section)\n except ConfigParser.NoSectionError:\n log = logging.getLogger('Mastiff.Conf.GetSection')\n log.error('Could not get section \"%s\".', section)\n return None\n\n opt_dict = dict()\n\n for pairs in options:\n opt_dict[pairs[0]] = pairs[1]\n\n return opt_dict\n\n def set_var(self, section, var, value):\n \"\"\"Set a given variable with a specified value.\"\"\"\n try:\n return self.config.set(section, var, value)\n except ConfigParser.NoSectionError:\n log = logging.getLogger('Mastiff.Conf.SetVar')\n log.error('Could not find \"%s\": \"%s\"', section, var)\n return None\n\n def override_option(self, override):\n \"\"\"\n Override an option from the config file. Note that if the option\n does not exist, it will be added.\n \"\"\"\n log = logging.getLogger('Mastiff.Conf.override')\n options = override.split('=')\n section = options[0].split('.')\n\n if len(options) != 2 or len(section) != 2:\n log.error('Invalid override option: %s' % override)\n return False\n\n log.info('Overriding option: %s.%s=%s' % (section[0], section[1], options[1]))\n if self.set_var(section[0], section[1], options[1]) is None:\n return False\n\n def list_config(self):\n \"\"\"Print all variables read in.\"\"\"\n print \"Configuration Options:\"\n for section in self.config.sections():\n print \"%s\" % (section)\n for (name, value) in self.config.items(section):\n print \"\\t%s:\\t%s\" % (name, value)\n return\n\n def dump_config(self):\n \"\"\" Dump a copy of the config into the Mastiff log dir. \"\"\"\n\n log = logging.getLogger('Mastiff.Conf.Dump')\n out_dir = self.get_var('Dir', 'log_dir')\n\n try:\n with open(out_dir + os.sep + 'mastiff-run.config', 'w') as dump_file:\n self.config.write(dump_file)\n except ConfigParser.Error, err:\n log.error('Unable to dump config file: %s', err)\n\n" }, { "path": "mastiff/core.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nMASTIFF - MAlicious Static Inspection File Framework\n\nThis module implements the primary class for static analysis inspection.\n\nMastiff member variables:\n\ncat_paths: List that contains the path to the category plug-ins.\n\nplugin_paths: List that contains the paths to the analysis plug-ins.\n\nfiletype: Dictionary used to store the output from the file-type identification\nfunctions.\n\nfile_name: full path to the file being analyzed.\n\nhashes: Tuple of the MD5, SHA1 and SHA256 hashes of the file being analyzed.\nThis is also stored in the configuration file.\n\ndb: Sqlite3 Connection class to the database file.\n\ncat_list: List that contains all of the category plug-ins to be used during\nanalysis.\n\nactivated_plugins: List that contains all of the plug-ins that have been\nactivated. This order of the plug-ins in this list is the order they will run.\n\ncat_manager: Yapsy PluginManager class that manages the category plug-ins.\n\nplugin_manager: Yapsy PluginManager class that manages the analysis plug-ins.\n\nMastiff member functions:\n\n__init__(self, config_file=None, fname=None, loglevel=logging.INFO, override=None)\nThe initialization function of the class. This function will initialize all of the\nmember variables, set up logging, read in and store the configuration file, and\nfind and load all plug-ins.\n\ninit_file(self, fname)\nThis function validates the filename being analyzed\nto ensure it exists and can be accessed, sets up the directory that all\noutput will be logged into, and adds initial file information into the\ndatabase.\n\nset_filetype(self, fname=None, ftype=None)\nCalls the file-type identification helper functions in mastiff/filetype.py,\nand loops through all of the category plug-ins to determine which ones will\nanalyze the file.\n\nvalidate(self, name, plugin)\nValidates an analysis plug-in to ensure that it contains the correct functions.\n\nactivate_plugins(self, single_plugin=None)\nLoops through all analysis plug-ins for category classes relevant to the file\ntype being examined and ensures they are valid. If validated, the analysis\nplug-in is activated. This function also ensures that any pre-requisite plug-ins\nhave been activated.\n\nanalyze(self, fname=None, single_plugin=None)\nEnsures the file type of the file is set up and loops through all activated\nanalysis plug-ins and calls their analyze() function.\n\nlist_plugins(self, type='analysis')\nHelper function that loops through all available plug-ins and prints out their\nname, path and description. The function can print out analysis or category\nplug-in information.\n\"\"\"\n\n__version__ = \"$Id: ace95027e1cc1f56614eaa0fc86d67b5c4aed8bb $\"\n\nimport sys\nimport os\nimport logging\nimport hashlib\nfrom shutil import copyfile\nfrom operator import attrgetter\n\nimport simplejson\n\nif sys.version_info < (2, 6, 6):\n sys.stderr.write(\"Mastiff requires python version 2.6.6\")\n sys.exit(1)\n\ntry:\n from yapsy.PluginManager import PluginManager\nexcept ImportError, err:\n print \"Yapsy not installed or accessible: %s\" % err\n sys.exit(1)\n\nimport mastiff.conf as Conf\nimport mastiff.filetype as FileType\nimport mastiff.sqlite as DB\nimport mastiff.plugins.category.categories as Cats\nimport mastiff.plugins.analysis as analysis\nimport mastiff.plugins.output as masOutput\n\nclass Mastiff:\n \"\"\"Primary class for the static analysis inspection framework.\"\"\"\n\n def __init__(self, config_file=None, fname=None, loglevel=logging.INFO, override=None):\n \"\"\"Initialize variables.\"\"\"\n\n # configure logging for Mastiff module\n format_ = '[%(asctime)s] [%(levelname)s] [%(name)s] : %(message)s'\n logging.basicConfig(format=format_)\n log = logging.getLogger(\"Mastiff\")\n log.setLevel(loglevel)\n if log.handlers:\n log.handlers = []\n\n # read in config file\n self.config = Conf.Conf(config_file, override=override)\n\n # make sure base logging dir exists\n log_dir = self.config.get_var('Dir','log_dir')\n log_dir = os.path.abspath(os.path.expanduser(log_dir))\n if not os.path.isdir(log_dir):\n try:\n os.makedirs(log_dir)\n except OSError, err:\n log.error('Could not make %s: %s. Exiting.', log_dir, err)\n sys.exit(1)\n self.config.set_var('Dir', 'base_dir', log_dir)\n\n # set up file to log output to\n fh = logging.FileHandler(log_dir + os.sep + 'mastiff.log' )\n fh.setFormatter(logging.Formatter(format_))\n log.addHandler(fh)\n fh.setLevel(loglevel)\n\n # verbose logging set in the config and not command line?\n if self.config.get_bvar('Misc','verbose') == True and \\\n loglevel != logging.ERROR:\n log.setLevel(logging.DEBUG)\n fh.setLevel(logging.DEBUG)\n\n # get path to category plugins\n self.cat_paths = [ os.path.dirname(Cats.__file__) ]\n self.output_paths = [ os.path.dirname(masOutput.__file__) ]\n\n # convert plugin paths to list\n self.plugin_paths = [ os.path.dirname(analysis.__file__)]\n # strip whitespace from dirs\n for tmp in str(self.config.get_var('Dir','plugin_dir')).split(','):\n if tmp:\n self.plugin_paths.append(os.path.expanduser(tmp.lstrip().rstrip()))\n \n # do the same for output plugins\n for tmp in str(self.config.get_var('Dir','output_plugin_dir')).split(','):\n if tmp:\n self.output_paths.append(os.path.expanduser(tmp.lstrip().rstrip()))\n\n self.filetype = dict()\n self.file_name = None\n self.hashes = None\n self.cat_list = list()\n self.activated_plugins = list()\n\n # Build the managers\n self.cat_manager = PluginManager()\n self.plugin_manager = PluginManager()\n self.output_manager = PluginManager()\n\n # Find and load all category plugins\n cat_filter = dict()\n self.cat_manager.setPluginPlaces(self.cat_paths)\n self.cat_manager.collectPlugins()\n\n # Import all of the modules for the categories so we can access\n # their classes.\n for pluginInfo in self.cat_manager.getAllPlugins():\n\n log.debug('Found category: %s', pluginInfo.name)\n try:\n mod_name = \"mastiff.plugins.category.%s\" % \\\n os.path.basename(pluginInfo.path)\n cat_mod = __import__(mod_name,\n fromlist=[\"mastiff.plugins.category\"])\n except ImportError, err:\n log.error(\"Unable to import category %s: %s\",\n pluginInfo.name,\n err)\n self.cat_manager.deactivatePluginByName(pluginInfo.name)\n continue\n else:\n # We were able to import it, activate it\n self.cat_manager.activatePluginByName(pluginInfo.name)\n log.debug(\"Activated category: %s\", pluginInfo.name)\n\n # Cat is imported, add class to the category filter\n # cat_filter will be a dict in the form:\n # { cat_name: cat_class }\n # and contains all the category plugins that have been activated\n cat_class = getattr(cat_mod,\n pluginInfo.plugin_object.__class__.__name__)\n cat_filter.update({pluginInfo.plugin_object.cat_name: cat_class})\n\n #log.debug(\"Category Filters: %s\", cat_filter)\n\n # Now collect and load all analysis plugins\n self.plugin_manager.setPluginPlaces(self.plugin_paths)\n self.plugin_manager.setCategoriesFilter( cat_filter )\n self.plugin_manager.collectPlugins()\n\n # Finally collect all output plugins\n self.output_manager.setPluginPlaces(self.output_paths)\n self.output_manager.collectPlugins()\n\n # set up database\n self.db = DB.open_db_conf(self.config)\n DB.create_mastiff_tables(self.db)\n\n # set up the output object\n self.output = dict()\n\n # init the filename if we have it\n if fname is not None:\n self.init_file(fname)\n\n def __del__(self):\n \"\"\"\n Class destructor.\n \"\"\"\n # Close down all logging file handles so we don't have any open file descriptors\n log = logging.getLogger(\"Mastiff\")\n handles = list(log.handlers)\n for file_handle in handles:\n log.removeHandler(file_handle)\n file_handle.close()\n\n def init_file(self, fname):\n \"\"\"\n Validate the filename to ensure it can be accessed and set\n up class variables.\n\n This function is called when a filename is given or can be\n called directly.\n \"\"\"\n log = logging.getLogger(\"Mastiff.Init_File\")\n\n if fname is None:\n return None\n\n try:\n with open(fname, 'rb') as my_file:\n data = my_file.read()\n except IOError, err:\n log.error(\"Could not open file: %s\", err)\n return None\n\n self.file_name = fname\n\n # create tuple of md5, sha1 and sha256 hashes\n self.hashes = hashlib.md5(data).hexdigest(), \\\n hashlib.sha1(data).hexdigest(), \\\n hashlib.sha256(data).hexdigest()\n self.config.set_var('Misc', 'hashes', self.hashes)\n\n self.output[self.hashes] = dict()\n\n # update log_dir\n log_dir = os.path.abspath(os.path.expanduser(self.config.get_var('Dir','log_dir'))) + \\\n os.sep + \\\n self.hashes[0]\n self.config.set_var('Dir', 'log_dir', log_dir)\n\n # create log dir\n if not os.path.exists(log_dir):\n try:\n os.makedirs(log_dir)\n except OSError, err:\n log.error('Could not make %s: %s. Exiting.', log_dir, err)\n sys.exit(1)\n\n # lets set up the individual log file\n # we may miss out on a couple prior logs, but thats OK\n log = logging.getLogger('Mastiff')\n fh = logging.FileHandler(log_dir + os.sep + 'mastiff.log' )\n format_ = '[%(asctime)s] [%(levelname)s] [%(name)s] : %(message)s'\n fh.setFormatter(logging.Formatter(format_))\n log.addHandler(fh)\n fh.setLevel(logging.INFO)\n\n log = logging.getLogger(\"Mastiff.Init_File\")\n log.info('Analyzing %s.', self.file_name)\n log.info(\"Log Directory: %s\", log_dir)\n\n # copy file to the log directory\n if self.config.get_bvar('Misc', 'copy') is True:\n try:\n copyfile(self.file_name, log_dir + os.sep + os.path.basename(self.file_name) + '.VIR')\n except IOError, err:\n log.error('Unable to copy file: %s', err)\n log.debug('Copied file to log directory.')\n else:\n log.debug('Configuration set to not copy file.')\n\n # add entry to database if it exists\n if self.db is not None:\n log.debug('Adding entry to database.')\n DB.insert_mastiff_item(self.db, self.hashes)\n\n return self.hashes\n\n def activate_plugins(self, single_plugin=None):\n \"\"\"\n Activate all plugins that are in the categories we selected.\n If single_plugin is given, only activate that plug-in.\n Note: File Information plug-in is ALWAYS run.\n \"\"\"\n\n has_prereq = list()\n\n for cats in self.cat_list:\n\n log = logging.getLogger('Mastiff.Plugins.Activate')\n log.debug('Activating plugins for category %s.', cats)\n\n self.output[self.hashes][cats] = dict()\n\n for plugin in self.plugin_manager.getPluginsOfCategory(cats):\n\n # check if we are running a single plugin - file information always gets run\n if single_plugin is not None and single_plugin != plugin.name and plugin.name != 'File Information':\n continue\n\n plugin.plugin_object.set_name(plugin.name)\n log.debug('Validating plugin \"%s\"', plugin.name)\n\n # if the plugin validates, try to activate it\n if self.validate(plugin.name, plugin.plugin_object) == True:\n if plugin.plugin_object.prereq is not None:\n # this plugin has a pre-req, can't activate yet\n has_prereq.append([cats, plugin])\n else:\n log.debug('Activating \"%s\".', plugin.name)\n self.plugin_manager.activatePluginByName(plugin.name, cats)\n self.activated_plugins.append(plugin)\n else:\n log.debug(\"Removing plugin %s %s.\", plugin.name, cats)\n self.plugin_manager.deactivatePluginByName(plugin.name,\n cats)\n\n # now try to activate any plug-ins that have pre-reqs\n flag = True\n while flag is True:\n flag = False\n for plugins in has_prereq:\n # check to see if the pre-req in in the activated list\n inact = [p for p in self.activated_plugins if p.name == plugins[1].plugin_object.prereq]\n\n if len(inact) > 0:\n # our pre-req has been activated, we can activate ourself\n log.debug('Activating \"%s\". Pre-req fulfilled.', plugins[1].name)\n self.plugin_manager.activatePluginByName(plugins[1].name, plugins[0])\n self.activated_plugins.append(plugins[1])\n has_prereq.remove(plugins)\n flag = True\n\n # list out any plugins that were not activated due to missing pre-reqs\n for plugins in has_prereq:\n log.debug(\"Plugin %s not activated due to missing pre-req \\\"%s.\\\"\" % \\\n (plugins[1].name, plugins[1].plugin_object.prereq ))\n\n # finally activate the output plugins\n for plugin in self.output_manager.getAllPlugins():\n plugin.plugin_object.set_name(plugin.name)\n log.debug('Activating Output Plug-in \"{}\"'.format(plugin.name))\n self.output_manager.activatePluginByName(plugin.name)\n #self.activated_plugins.append(plugin)\n\n\n def list_plugins(self, ctype='analysis'):\n \"\"\"Print out a list of analysis or cat plugins.\"\"\"\n\n if ctype == 'analysis':\n # analysis plug-ins\n print \"Analysis Plug-in list:\\n\"\n print \"%-25s\\t%-15s\\t%-25s\\n%-50s\" % \\\n (\"Name\", \"Category\", \"Description\", \"Path\")\n print '-' * 80\n\n for plugin in sorted(self.plugin_manager.getAllPlugins(),\n key=attrgetter('plugin_object.cat_name', 'name')):\n print \"%-25s\\t%-15s\\t%-12s\\n%-80s\\n\" % \\\n (plugin.name, plugin.plugin_object.cat_name, \\\n plugin.description, plugin.path)\n\n elif ctype == 'cat':\n print \"Category Plug-in list:\\n\"\n print \"%-25s\\t%-15s\\t%-s\" % (\"Name\", \"FType\", \"Description\")\n print '-' * 80\n # category plug-ins\n for plugin in sorted(self.cat_manager.getAllPlugins(),\n key=attrgetter('name')):\n print \"%-25s\\t%-15s\\t%-s\" % \\\n (plugin.name, plugin.plugin_object.cat_name,\n plugin.description)\n elif ctype == 'output':\n print \"Output Plug-in list:\\n\"\n print \"%-25s\\t%-s\\n%s\" % (\"Name\", \"Description\", \"Path\")\n print '-' * 80\n # category plug-ins\n for plugin in sorted(self.output_manager.getAllPlugins(),\n key=attrgetter('name')):\n print \"%-25s\\t%-s\\n%-80s\\n\" % \\\n (plugin.name, plugin.description, plugin.path)\n else:\n print \"Unknown plugin type.\"\n\n def set_filetype(self, fname=None, ftype=None):\n \"\"\"\n Calls the filetype functions and loops through the category\n plug-ins to see which ones will handle this file.\n \"\"\"\n\n log = logging.getLogger('Mastiff.FileType')\n\n if fname is None and self.file_name is None:\n log.error(\"No file to analyze has been specified. Exiting.\")\n sys.exit(1)\n elif fname is not None and self.file_name is None:\n if self.init_file(fname) is None:\n log.error(\"ERROR accessing file. Exiting.\")\n sys.exit(1)\n\n if self.cat_list:\n # if self.cat_list is already set, assume that we've already\n # gone through this function\n return self.filetype\n\n if ftype is not None:\n # we are forcing a file type to run\n log.info('Forcing category plug-in \"%s\" to be added.', ftype)\n self.cat_list.append(ftype)\n\n # Grab the magic file type of the file. This is done here so as not\n # to do it in every category plug-in.\n self.filetype['magic'] = FileType.get_magic(self.file_name)\n\n # Grab the TrID type\n trid_opts = self.config.get_section('File ID')\n self.filetype['trid'] = list()\n if trid_opts['trid']:\n self.filetype['trid'] = FileType.get_trid(self.file_name,\n trid_opts['trid'],\n trid_opts['trid_db'])\n\n # Cycle through all of the categories and see if they should be added\n # to the list of categories to be run.\n for pluginInfo in self.cat_manager.getAllPlugins():\n cat_name = pluginInfo.plugin_object.is_my_filetype(self.filetype,\n self.file_name)\n log.debug('Checking cat %s for filetype.', pluginInfo.name)\n if cat_name is not None:\n # cat_list contains analysis plugin categories to be used\n self.cat_list.append(cat_name)\n log.debug('Adding %s to plugin selection list.', cat_name)\n\n # add file type to the DB\n if self.db is not None:\n DB.insert_mastiff_item(self.db, self.hashes, self.cat_list)\n\n return self.filetype\n\n def validate(self, name, plugin):\n \"\"\"Return false if a plugin does not have the correct functions.\"\"\"\n\n log = logging.getLogger('Mastiff.Plugins.Validate')\n\n try:\n callable(plugin.activate)\n except AttributeError:\n log.error(\"%s missing activate function.\", name)\n return False\n\n try:\n callable(plugin.deactivate)\n except AttributeError:\n log.error(\"%s missing deactivate function.\", name)\n return False\n\n try:\n callable(plugin.analyze)\n except AttributeError:\n log.error(\"%s missing analyze function.\", name)\n return False\n \n return True\n\n def analyze(self, fname=None, single_plugin=None):\n \"\"\"Perform analysis on a given filename.\"\"\"\n\n log = logging.getLogger('Mastiff.Analysis')\n\n if fname is None and self.file_name is None:\n log.error(\"No filename specified. Exiting.\")\n sys.exit(1)\n elif fname is not None and self.file_name is None:\n # first time seeing the file, initialize it\n if self.init_file(fname) is None:\n log.error(\"ERROR accessing file. Exiting.\")\n return False\n\n # set the file_type\n ftype = self.set_filetype()\n log.info('File categories are %s.', self.cat_list)\n\n if not self.filetype:\n log.error(\"The file type has not been set. Exiting.\")\n sys.exit(1)\n\n # activate the plugins\n self.activate_plugins(single_plugin)\n\n for plugin in self.activated_plugins:\n # skip if plugin is not activated\n if plugin.is_activated == False:\n continue\n\n log.debug('Calling plugin \"%s\".', plugin.name)\n\n # set the output results to be an attribute of the plugin so it can analyze it\n setattr(plugin.plugin_object, 'results', self.output[self.hashes])\n \n # analyze the plugin - if plugin is compliant with universal output\n # its output will be returned\n plug_out = plugin.plugin_object.analyze(self.config, self.file_name)\n\n if plug_out is not False and plug_out is not None and isinstance(plug_out, masOutput.page):\n # add the plugin output to its own entry\n self.output[self.hashes][plugin.plugin_object.cat_name][plugin.plugin_object.name] = plug_out\n \n # go through output plugins and output the data\n for plugin in self.output_manager.getAllPlugins():\n plugin.plugin_object.output(self.config, self.output)\n\n self.config.dump_config()\n \n log.info('Finished analysis for %s.', self.file_name)\n\n# end class mastiff\n\n" }, { "path": "mastiff/filetype.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nFile Type Analysis Functions\n\nThe functions within this module provide the functionality to help determine\nthe type of file given to it.\n\nThis module now supports the use of two different type of libmagic Python libraries:\n- The libmagic Python library maintained with file (ftp://ftp.astron.com/pub/file/).\n This is the version installed via most Debian-based repositories.\n- ahupp's python-magic repostitory installed via pip.\n (https://github.com/ahupp/python-magic)\n\n\"\"\"\n\n__version__ = \"$Id: 82df116d3435226d15057b63acbed2b77919a52d $\"\n\nimport magic\nimport logging\nimport subprocess\nimport re\nimport os\n\ntry:\n import yara\nexcept ImportError, error:\n print \"Could not import yara: %s\" % error\n\ndef get_magic(file_name):\n \"\"\" Determine the file type of a given file based on its magic result.\"\"\"\n\n log = logging.getLogger('Mastiff.FileType.Magic')\n \n try:\n # try to use magic from the file source code\n magic_ = magic.open(magic.MAGIC_NONE)\n magic_.load()\n try:\n file_type = magic_.file(file_name)\n except:\n log.error('Could not determine magic file type.')\n return None\n magic_.close()\n except AttributeError:\n # Now we are trying ahupps magic library\n try:\n file_type = magic.from_file(file_name)\n except AttributeError:\n log.error('No valid magic libraries installed.')\n return None\n except MagicException:\n log.error('Cound not determing magic file type.')\n return None \n\n log.debug('Magic file type is \"%s\"', file_type)\n\n return file_type\n\ndef get_trid(file_name, trid, trid_db):\n \"\"\" DEPRECATING: RECOMMENDED NOT TO USE\n TrID is a file identification tool created by Marco Pontello.\n Unfortunately, TrID does not have a Linux library we can use, so we\n will run the program and store its results.\n\n file_name: file to analyze\n trid = path to trid binary\n trid_db = path to trid database\n\n Returns a list of the hits from TrID. Each item of the returned list\n will contain a list with [ percentage, description ]\n \"\"\"\n\n log = logging.getLogger('Mastiff.FileType.TrID')\n pattern = '^\\s*([0-9\\.]+)\\% \\([\\w\\.]+\\) ([\\S\\s]+) \\([0-9\\/]+\\)$'\n results = list()\n\n # if files don't exist, return empty list\n if not os.path.isfile(trid) or not os.path.isfile(trid_db):\n log.warning('TrID cannot be found. Skipping TrID file type detection.')\n return results\n\n trid_db = '-d:' + trid_db\n\n # TrID has a bug in it where it can't open a file it it begins with \"./\"\n # remove that\n if file_name.startswith('./'):\n file_name = file_name[2:]\n\n try:\n run = subprocess.Popen([trid] + [trid_db] + [file_name],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n close_fds=True)\n except subprocess.CalledProcessError, err:\n log.error('Could not run TrID: %s', err)\n return results\n except OSError, err:\n log.error('Could not run TrID: %s', err)\n return results \n\n (output, error) = run.communicate()\n if error is not None and len(error) > 0:\n log.error('Error running TrID: %s' % error)\n return results\n \n data = [ re.match(pattern, line) for line in output.split('\\n') ]\n\n # create a list of hits\n # each item in results will be [ percentage, description ]\n results = [ [float(match.group(1)), match.group(2)] \\\n for match in data \\\n if match is not None ]\n\n log.debug('TrID types are: %s', results)\n\n return results\n\ndef yara_typecheck(filename, yara_rule):\n \"\"\" Check for file type based on yara rule.\n Returns True if found, False otherwise.\n \"\"\"\n log = logging.getLogger('Mastiff.FileType.Yara')\n \n if yara_rule is None:\n return False\n \n try:\n rules = yara.compile(source=yara_rule)\n except yara.SyntaxError, err:\n log.error('Rule Error: %s', error)\n return False\n except:\n log.error(\"Error attempting to perform Yara filetype.\")\n return False\n \n try:\n matches = rules.match(filename, timeout=10) \n except yara.Error, err:\n log.error('Yara error: %s', err)\n return False \n \n if len(matches) > 0:\n log.debug('File Type matches rule %s', matches[0].rule)\n return True\n \n return False\n\nif __name__ == '__main__':\n import sys\n\n if len(sys.argv) > 1:\n print get_magic(sys.argv[1])\n\n" }, { "path": "mastiff/plugins/__init__.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\n This file contains a number of helper functions for misc. tasks\n the plug-ins may want to use.\n\"\"\"\n\n__version__ = \"$Id: 3fc4dad80994edc30d0dfd81ecadcca67bb486a9 $\"\n\nimport httplib, mimetypes\nimport binascii\n\n\"\"\"\n The following are taken from\n http://code.activestate.com/recipes/146306/\n and are used to allow the uploading of files to multipart forms.\n\"\"\"\ndef post_multipart(host, method, selector, fields, files):\n \"\"\"\n Post fields and files to an http host as multipart/form-data.\n fields is a sequence of (name, value) elements for regular form fields.\n files is a sequence of (name, filename, value) elements for data to be uploaded as files\n Return the server's response page.\n \"\"\"\n content_type, body = encode_multipart_formdata(fields, files)\n if method.startswith('https') is True:\n h = httplib.HTTPSConnection(host)\n else:\n h = httplib.HTTP(host)\n \n h.putrequest('POST', selector)\n h.putheader(\"User-Agent\", 'MASTIFF Statis Analysis Framework')\n h.putheader('Content-Type', content_type)\n h.putheader('Content-Length', str(len(body)))\n h.endheaders()\n h.send(body)\n myresponse = h.getresponse().read()\n return myresponse\n\ndef encode_multipart_formdata(fields, files):\n \"\"\"\n fields is a sequence of (name, value) elements for regular form fields.\n files is a sequence of (name, filename, value) elements for data to be uploaded as files\n Return (content_type, body) ready for httplib.HTTP instance\n \"\"\"\n BOUNDARY = '----------MASTIFF_STATIC_ANALYSIS_FRAMEWORK$'\n CRLF = '\\r\\n'\n L = []\n for (key, value) in fields:\n L.append('--' + BOUNDARY)\n L.append('Content-Disposition: form-data; name=\"%s\"' % key)\n L.append('')\n L.append(value)\n for (key, filename, value) in files:\n L.append('--' + BOUNDARY)\n L.append('Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"' % (key, filename))\n L.append('Content-Type: %s' % get_content_type(filename))\n L.append('')\n L.append(value)\n L.append('--' + BOUNDARY + '--')\n L.append('')\n body = CRLF.join(L)\n content_type = 'multipart/form-data; boundary=%s' % BOUNDARY\n return content_type, body\n\ndef get_content_type(filename):\n \"\"\" Returns MIME type for the file. \"\"\"\n return mimetypes.guess_type(filename)[0] or 'application/octet-stream'\n\ndef bin2hex(data):\n \"\"\"\n Goes through data and turns any binary characters into its hex\n equivalent.\n \"\"\"\n\n hexstring = ''\n for letter in data:\n if ord(letter) <= 31 or ord(letter) >= 127:\n hexstring += '\\\\x' + binascii.hexlify(letter)\n else:\n hexstring += letter\n\n return hexstring\n\ndef printable_str(string):\n \"\"\" Helper function to convert non-printable chars to its ASCII format \"\"\"\n\n new_str = ''\n for char in string:\n if ord(char) >= 32 and ord(char) <= 126:\n new_str = new_str + char\n else:\n new_str = new_str + (r'\\x%02x' % ord(char))\n\n return new_str\n\n" }, { "path": "mastiff/plugins/analysis/EXE/EXE-peinfo.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nPE Info plugin\n\nPlugin Type: EXE\nPurpose:\n Dump information on the PE structure of the given executable. This is\n done using pefile's dump_info() API. It is not structured in any way.\n\n Sample code from the pefile and Didier Stevens pecheck.py was used or\n referenced for this plug-in.\n\nOutput:\n - peinfo-quick.txt - contains minimal information that analysts may\n find useful.\n\n - peinfo-full.txt - contains full information on the file.\n\nRequirements:\n - pefile library (http://code.google.com/p/pefile/)\n\n\"\"\"\n\n__version__ = \"$Id: 7dd537f22578be78ca7e142ea73a7ebe4e2163d5 $\"\n\nimport logging\nimport os\nimport time\nimport sys\n\ntry:\n import pefile\nexcept ImportError, err:\n print (\"Unable to import pefile: %s\" % err)\n\nfrom mastiff.plugins import printable_str\nimport mastiff.plugins.category.exe as exe\n\nclass PEInfo(exe.EXECat):\n \"\"\"Dumps PE information.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n exe.EXECat.__init__(self)\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n try:\n pe = pefile.PE(filename)\n except:\n log.error('Unable to parse PE file: %s' % sys.exc_info()[1])\n \n return False\n\n if not self.output_file_quick(config.get_var('Dir','log_dir'), pe) or not self.output_file_full(config.get_var('Dir','log_dir'), pe):\n return False\n\n return True\n \n @staticmethod\n def _dump_section_headers(pe):\n \"\"\"\n Small internal function to dump the section headers in a table. \n Returns a string to do so.\n \"\"\"\n section_string = ''\n section_flags = pefile.retrieve_flags(pefile.SECTION_CHARACTERISTICS, 'IMAGE_SCN_')\n section_string += '\\nNumber of Sections: %d\\n' % pe.FILE_HEADER.NumberOfSections\n section_string += '{0:15} {1:8} {2:40}\\n'.format('Section Name', 'Entropy', 'Flags')\n section_string += '-'*65 + '\\n'\n for section in pe.sections:\n # thanks to the pefile example code for this\n flags = []\n for flag in section_flags:\n if getattr(section, flag[0]):\n flags.append(flag[0])\n\n # the following line was taken from Didier Steven's pecheck.py code\n section_string += '{0:15} {1:<8.5} {2:40}\\n'.format(''.join(filter(lambda c:c != '\\0', str(section.Name))), \\\n section.get_entropy(),\n ', '.join(flags))\n section_string += '\\n'\n return section_string \n\n def output_file_quick(self, outdir, pe):\n \"\"\"Output short, useful information on file.\"\"\"\n\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.quick') \n\n try:\n outfile = open(outdir + os.sep + 'peinfo-quick.txt', 'w')\n outfile.write('PE Header Information\\n\\n')\n outfile.write('Quick Info:\\n\\n')\n try:\n outfile.write('TimeDateStamp: %s\\n' % time.asctime(time.gmtime(pe.FILE_HEADER.TimeDateStamp)))\n except ValueError:\n outfile.write('TimeDataStamp: Invalid Time %x\\n' % (pe.FILE_HEADER.TimeDateStamp))\n outfile.write('Subsystem: %s\\n' % pefile.SUBSYSTEM_TYPE[pe.OPTIONAL_HEADER.Subsystem])\n\n outfile.write(self._dump_section_headers(pe))\n\n # any parsing warnings (often related to packers\n outfile.write('\\nParser Warnings:\\n')\n for warning in pe.get_warnings():\n outfile.write('- ' + warning + '\\n')\n\n # file info - thx to Ero Carrera for sample code\n # http://blog.dkbza.org/2007/02/pefile-parsing-version-information-from.html\n outfile.write('\\nFile Information:\\n')\n if hasattr(pe, \"FileInfo\"):\n for fileinfo in pe.FileInfo:\n if fileinfo.Key == 'StringFileInfo':\n for string_entry in fileinfo.StringTable:\n for entry in string_entry.entries.items():\n outfile.write(\"{0:20}:\\t{1:40}\\n\".format(printable_str(entry[0]), \\\n printable_str(entry[1])))\n if fileinfo.Key == 'VarFileInfo':\n try:\n for var in fileinfo.Var:\n outfile.write(\"{0:20}:\\t{1:40}\\n\".format(printable_str(var.entry.items()[0][0]),\n printable_str(var.entry.items()[0][1])))\n except:\n # there are times when a VarFileInfo structure may be present, but empty\n pass\n else:\n outfile.write('No file information present.\\n')\n\n # imports\n outfile.write('\\nImports:\\n')\n if hasattr(pe, \"DIRECTORY_ENTRY_IMPORT\"):\n outfile.write('{0:20}\\t{1:30}\\t{2:10}\\n'.format('DLL', 'API', 'Address'))\n outfile.write('-'*70 + '\\n')\n for entry in pe.DIRECTORY_ENTRY_IMPORT:\n for imp in entry.imports:\n outfile.write('{0:20}\\t{1:30}\\t{2:10}\\n'.format(entry.dll, imp.name, hex(imp.address)))\n else:\n outfile.write('No imports.\\n')\n\n # exports\n outfile.write('\\nExports:\\n')\n if hasattr(pe, \"DIRECTORY_ENTRY_EXPORT\"):\n outfile.write('{0:20}\\t{1:10}\\t{2:10}\\n'.format('Name', 'Address', 'Ordinal'))\n outfile.write('-'*50 + '\\n')\n for exp in pe.DIRECTORY_ENTRY_EXPORT.symbols:\n outfile.write('{0:20}\\t{1:10}\\t{2:10}\\n'.format(exp.name, \\\n hex(pe.OPTIONAL_HEADER.ImageBase + exp.address),\\\n exp.ordinal))\n else:\n outfile.write('No Exports.\\n')\n\n\n outfile.close()\n except IOError, err:\n log.error('Cannot write to peinfo.txt: %s' % err)\n return False\n except pefile.PEFormatError, err:\n log.error('Unable to parse PE file: %s' % err)\n return False\n\n return True\n\n def output_file_full(self, outdir, pe):\n \"\"\"Output full information on file.\"\"\"\n\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.full')\n\n try:\n outfile = open(outdir + os.sep + 'peinfo-full.txt', 'w')\n outfile.write('\\nFull Information Dump:\\n')\n outfile.write(self._dump_section_headers(pe)) \n outfile.write(pe.dump_info())\n outfile.close()\n except IOError, err:\n log.error('Cannot write to peinfo.txt: %s' % err)\n return False\n except:\n log.error('Unable to parse PE file.')\n return False\n\n return True\n\n" }, { "path": "mastiff/plugins/analysis/EXE/EXE-peinfo.yapsy-plugin", "content": "[Core]\nName = PE Info\nModule = EXE-peinfo\n\n[Documentation]\nDescription = Dump information on the PE header and structure of an executable.\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/EXE/EXE-resources.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nPE Resources Plug-in\n\nPlugin Type: EXE\nPurpose:\n This plug-in obtains information on any resources contained within\n the Windows EXE and extracts them.\n\n More information on how resources are stored can be found in the\n Microsoft PE and COFF Specification document.\n http://msdn.microsoft.com/library/windows/hardware/gg463125\n\n Thanks to Ero Carrera for creating the pefile library, whose code helped\n understand how to process resources.\n\nOutput:\n resources.txt - File containing a list of all resources in the EXE and any\n associated information.\n log_dir/resource - Directory containing any extracted resource.\n\nPre-requisites:\n - pefile library (http://code.google.com/p/pefile/)\n\n\"\"\"\n\n__version__ = \"$Id: 519a2014141003f89b18bb5c3de571729a952f8e $\"\n\nimport logging\nimport os\nimport time\n\ntry:\n import pefile\nexcept ImportError, err:\n print (\"Unable to import pefile: %s\" % err)\n\nimport mastiff.plugins.category.exe as exe\n\nclass EXE_Resources(exe.EXECat):\n \"\"\"EXE Resources plugin code.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n exe.EXECat.__init__(self)\n self.resources = list()\n self.pe = None\n self.output = dict()\n\n def analyze_dir(self, directory, prefix='', _type='', timedate=0):\n \"\"\" Analyze a resource directory and obtain all of its items.\"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.analyze')\n\n # save the timedate stamp\n timedate = directory.struct.TimeDateStamp\n\n for top_item in directory.entries:\n\n if hasattr(top_item, 'data'):\n # at the language level that contains all of our information\n resource = dict()\n resource['Id'] = prefix\n resource['Type'] = _type\n # store the offset as the offset within the file, not the RVA!\n try:\n resource['Offset'] = self.pe.get_offset_from_rva(top_item.data.struct.OffsetToData)\n resource['Size'] = top_item.data.struct.Size\n resource['Lang'] = [ pefile.LANG.get(top_item.data.lang, '*unknown*'), \\\n pefile.get_sublang_name_for_lang( top_item.data.lang, top_item.data.sublang ) ]\n resource['TimeDate'] = timedate\n except pefile.PEFormatError, err:\n log.error('Error grabbing resource \\\"%s\\\" info: %s' % (prefix, err))\n return False\n\n self.resources.append(resource)\n log.debug('Adding resource item %s' % resource['Id'])\n elif hasattr(top_item, 'directory'):\n if top_item.name is not None:\n # in a name level\n if len(prefix) == 0:\n newprefix = prefix + str(top_item.name)\n else:\n newprefix = ', '.join([prefix, str(top_item.name)])\n else:\n # if name is blank, we are in a Type level\n if len(prefix) == 0:\n newprefix = 'ID ' + str(top_item.id)\n _type = pefile.RESOURCE_TYPE.get(top_item.id)\n else:\n newprefix = ', '.join([prefix, 'ID ' + str(top_item.id)])\n\n # we aren't at the end, recurse\n self.analyze_dir(top_item.directory, prefix=newprefix, _type=_type)\n\n def extract_resources(self, log_dir, filename):\n \"\"\"\n Extract any resources from the file and put them in\n the resources dir.\n \"\"\"\n\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.extract')\n\n if len(self.resources) == 0:\n # no resources\n return False\n\n # create the dir if it doesn't exist\n log_dir = log_dir + os.sep + 'resources'\n if not os.path.exists(log_dir):\n try:\n os.makedirs(log_dir)\n except IOError, err:\n log.error('Unable to create dir %s: %s' % (log_dir, err))\n return False\n\n try:\n my_file = open(filename, 'rb')\n except IOError, err:\n log.error('Unable to open file.')\n return False\n\n file_size = os.path.getsize(filename)\n\n # cycle through resources and extract them\n for res_item in self.resources:\n\n # check to make sure we won't go past the EOF\n if (res_item['Offset'] + res_item['Size']) > file_size:\n log.error('File is smaller than resource location. Could be a packed file.')\n continue\n\n my_file.seek(res_item['Offset'])\n data = my_file.read(res_item['Size'])\n out_name = res_item['Id'].replace('ID ', '_').replace(', ', '_').lstrip('_')\n\n if res_item['Type'] is not None and len(res_item['Type']) > 0:\n out_name += '_' + res_item['Type']\n\n with open(log_dir + os.sep + out_name, 'wb') as out_file:\n log.debug('Writing %s to %s.' % (res_item['Id'], out_name))\n out_file.write(data)\n out_file.close()\n\n my_file.close()\n return True\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n try:\n self.pe = pefile.PE(filename)\n except pefile.PEFormatError, err:\n log.error('Unable to parse PE file: %s' % err)\n return False\n\n if not hasattr(self.pe, 'DIRECTORY_ENTRY_RESOURCE'):\n log.info('No resources for this file.')\n return False\n\n # parse the directory structure\n self.analyze_dir(self.pe.DIRECTORY_ENTRY_RESOURCE)\n \n self.output['metadata'] = { }\n self.output['data'] = dict()\n\n if len(self.resources) == 0:\n log.info('No resources could be found.') \n else:\n # output data to file and extract resources\n self.gen_output(config.get_var('Dir','log_dir'))\n self.output_file(config.get_var('Dir','log_dir'))\n self.extract_resources(config.get_var('Dir','log_dir'), filename)\n\n return self.output\n \n def gen_output(self, outdir):\n \"\"\" Generate the output to send back. \"\"\"\n \n self.output['data']['resources'] = list()\n self.output['data']['resources'].append([ 'Name/ID', 'Type', 'File Offset', 'Size', 'Language', 'Time Date Stamp'])\n \n for item in sorted(self.resources, key=lambda mydict: mydict['Offset']):\n\n lang = ', '.join(item['Lang']).replace('SUBLANG_', '').replace('LANG_', '')\n my_time = time.asctime(time.gmtime(item['TimeDate']))\n self.output['data']['resources'].append([ item['Id'], item['Type'], hex(item['Offset']), hex(item['Size']), lang, my_time ])\n \n return True\n\n def output_file(self, outdir):\n \"\"\"Print output from analysis to a file.\"\"\"\n\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.output')\n\n try:\n outfile = open(outdir + os.sep + 'resources.txt', 'w')\n outfile.write('Resource Information\\n\\n')\n except IOError, err:\n log.error('Could not open resources.txt: %s' % err)\n return False\n\n outstr = '{0:20} {1:15} {2:15} {3:8} {4:<30} {5:<25}\\n'.format( \\\n 'Name/ID',\n 'Type',\n 'File Offset',\n 'Size',\n 'Language',\n 'Time Date Stamp')\n outfile.write(outstr)\n outfile.write('-' * len(outstr) + '\\n')\n\n for item in sorted(self.resources, key=lambda mydict: mydict['Offset']):\n\n lang = ', '.join(item['Lang']).replace('SUBLANG_', '').replace('LANG_', '')\n my_time = time.asctime(time.gmtime(item['TimeDate']))\n\n outstr = '{0:20} {1:15} {2:<15} {3:<8} {4:30} {5:<25}\\n'.format(item['Id'],\n item['Type'],\n hex(item['Offset']),\n hex(item['Size']),\n lang,\n my_time)\n outfile.write(outstr)\n\n return True\n\n" }, { "path": "mastiff/plugins/analysis/EXE/EXE-resources.yapsy-plugin", "content": "[Core]\nName = Resources\nModule = EXE-resources\n\n[Documentation]\nDescription = Obtain information on and extract PE resources.\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/EXE/EXE-sig.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nPE Digital Signature\n\nPlugin Type: EXE\nPurpose:\n This plug-in extracts any digital signatures from a PE executable and converts\n it to both DER and text format.\n\n Extraction is performed using the disitool.py tool from Didier Stevens. Many\n thanks to him for permission to use it.\n\n Conversion to text is performed using the openssl program.\n\n Validation of the signature is not yet done.\n\nPre-requisites:\n - pefile library (http://code.google.com/p/pefile/)\n - disitool.py (http://blog.didierstevens.com/programs/disitool/)\n - openssl binary (http://www.openssl.org/)\n\nConfiguration file:\n\n[Digital Signatures]\n# Options to extract the digital signatures\n#\n# disitool - path to disitool.py script.\n# openssl - path to openssl binary\ndisitool = /usr/local/bin/disitool.py\nopenssl = /usr/bin/openssl\n\nOutput:\n sig.der - DER version of Authenticode signature.\n sig.txt - Text representation of signature.\n\nTODO:\n - Validate the signature.\n\n\"\"\"\n\n__version__ = \"$Id: c0be897e44fd598577a3739b7b978b52a0e8c997 $\"\n\nimport logging\nimport os\nimport subprocess\nimport sys\nfrom cStringIO import StringIO\n\nimport pefile\n\n# Change the following line to import the category class you for the files\n# you wish to perform analysis on\nimport mastiff.plugins.category.exe as exe\n\nclass EXESig(exe.EXECat):\n \"\"\"PE digital signature analysis plugin.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n exe.EXECat.__init__(self)\n\n def activate(self):\n \"\"\"Activate the plugin.\"\"\"\n exe.EXECat.activate(self)\n\n def deactivate(self):\n \"\"\"Deactivate the plugin.\"\"\"\n exe.EXECat.deactivate(self)\n\n def dump_sig_to_text(self, log_dir, openssl):\n \"\"\" Convert a DER signature to its text format and writes it out.\"\"\"\n\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.output_sig')\n der_file = log_dir + os.sep + 'sig.der'\n\n # check to see if file exists\n if os.path.exists(der_file) == False:\n log.error('Cannot find DER file: %s' % der_file)\n return False\n elif openssl is None or os.path.exists(openssl) is False:\n log.error('Cannot open openssl binary: %s' % openssl)\n return False\n\n cmd = [openssl, 'pkcs7', '-inform', 'DER', '-print_certs', '-text', '-in', der_file] \n\n run = subprocess.Popen(cmd,\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE, \n close_fds=True)\n (output, error) = run.communicate()\n if error is not None and len(error) > 0:\n log.error('Error running openssl: %s' % error)\n return False\n\n if output is not None:\n with open(log_dir + os.sep + 'sig.txt', 'w') as out_file:\n log.debug('Signature converted to text.')\n out_file.write(output)\n out_file.close()\n\n return True\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n # get my config options\n sig_opts = config.get_section(self.name)\n\n # import disitool\n disitool_path = config.get_var(self.name, 'disitool')\n if disitool_path is None:\n log.error('disitool.py path is empty.')\n return False\n elif os.path.exists(disitool_path) == False:\n log.error('disitool.py does not exist: %s' % disitool_path)\n return False\n\n sys.path.append(os.path.dirname(disitool_path))\n try:\n try: \n reload(disitool)\n except:\n import disitool\n except ImportError, err:\n log.error('Unable to import disitool: %s' % err)\n return False\n\n # extract sig\n # turn off stdout bc disitool.ExtractDigitalSignature is noisy\n try:\n old_stdout = sys.stdout\n sys.stdout = StringIO()\n sig = disitool.ExtractDigitalSignature(str(filename), \\\n config.get_var('Dir','log_dir') + os.sep + 'sig.der')\n sys.stdout = old_stdout\n except pefile.PEFormatError, err:\n log.error('Unable to extract signature: %s' %err)\n return False\n\n if sig is None:\n log.info(\"No signature on the file.\")\n else:\n log.info(\"Signature extracted.\")\n if sig_opts['openssl'] is None:\n log.error('openssl binary not present. Not converting signature.')\n else:\n # convert the sig to text\n self.dump_sig_to_text(config.get_var('Dir','log_dir'),\n config.get_var(self.name, 'openssl'))\n return True\n\n" }, { "path": "mastiff/plugins/analysis/EXE/EXE-sig.yapsy-plugin", "content": "[Core]\nName = Digital Signatures\nModule = EXE-sig\n\n[Documentation]\nDescription = Extract PE digital signatures.\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/EXE/EXE-singlestring.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nSingle-byte string plug-in\n\nPlugin Type: EXE\nPurpose:\n\nAttackers have begun to obfuscate embedded strings by moving a single byte\nat a time into a character array. In assembler, it looks like:\n\nmov mem, 0x68\nmov mem+4, 0x69\nmov mem+8, 0x21\n...\n\nUsing a strings program, these strings will not be found. This script looks\nfor any strings embedded in this way and prints them out. It does this by\nlooking through the file for C6 opcodes, which are the start of the\n\"mov mem/reg, imm\" instruction. It will then decode it, grab the value and\ncreate a string from it.\n\nRequirements:\n- distorm3 (http://code.google.com/p/distorm/)\n\nOutput:\n None\n\n\"\"\"\n\n__version__ = \"$Id: 6322146c8d971464c6f726ebdba3a3d7a2540028 $\"\n\nimport logging\nimport re\nimport os\n\ntry:\n from distorm3 import Decode, Decode32Bits\nexcept ImportError, err:\n print \"EXE-SingleString: Could not import distorm3: %s\" % error\n \nimport mastiff.plugins.category.exe as exe\n\n# Change the class name and the base class\nclass SingleString(exe.EXECat):\n \"\"\"Extract single-byte strings from an executable.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n exe.EXECat.__init__(self)\n self.length = 3\n self.raw = False\n\n def activate(self):\n \"\"\"Activate the plugin.\"\"\"\n exe.EXECat.activate(self)\n\n def deactivate(self):\n \"\"\"Deactivate the plugin.\"\"\"\n exe.EXECat.deactivate(self)\n\n def findMov(self, filename):\n \"\"\" look through the file for any c6 opcode (mov reg/mem, imm)\n when it finds one, decode it and put it into a dictionary \"\"\"\n #log = logging.getLogger('Mastiff.Plugins.' + self.name + '.findMov')\n\n f = open(filename,'rb')\n offset = 0 \n instructs = {}\n\n mybyte = f.read(1)\n\n while mybyte:\n if mybyte == \"\\xc6\":\n # found a mov op - decode and record it\n f.seek(offset)\n mybyte = f.read(16)\n # p will come back as list of (offset, size, instruction, hexdump)\n p = Decode(offset, mybyte, Decode32Bits)\n\n # break up the mnemonic\n ma = re.match('(MOV) ([\\S\\s]+), ([x0-9a-fA-F]+)', p[0][2])\n if ma is not None:\n instructs[offset] = [ma.group(1), ma.group(2), ma.group(3), p[0][1]] # mnemonic, size\n\n #log.debug( \"MOV instructions detected: %x %s %d\" % (offset,p[0][2],p[0][1]) )\n\n f.seek(offset+1)\n\n mybyte = f.read(1)\n offset = offset + 1\n\n f.close()\n return instructs\n\n def decodeBytes(self, instructs):\n \"\"\" Take in a dict of instructions - parse through each instruction and grab the strings \"\"\"\n #log = logging.getLogger('Mastiff.Plugins.' + self.name + '.decodeBytes')\n\n curString = \"\"\n curOffset = 0\n strList = []\n usedBytes = []\n\n for off in sorted(instructs.keys()):\n\n if off not in usedBytes:\n # set up the new offset if needed\n if curOffset == 0:\n curOffset = off\n\n while off in instructs:\n usedBytes.append(off)\n hexVal = int(instructs[off][2], 16)\n opLen = instructs[off][3]\n\n # is hexVal out of range?\n if hexVal < 32 or hexVal > 126 and (hexVal != 10 or hexVal != 13 or hexVal != 9):\n # end of string\n #log.debug(\"%x non-string char - new string: %d: %s\" % (curOffset, hexVal,curString))\n strList.append([curOffset, curString])\n curOffset = off + opLen\n curString = \"\"\n else:\n #add to string\n if not self.raw and hexVal == 10:\n # line feed\n curString = curString + \"\\\\r\"\n elif not self.raw and hexVal == 13:\n # return\n curString = curString + \"\\\\n\"\n elif not self.raw and hexVal == 9:\n # tab\n curString = curString + \"\\\\t\"\n else:\n curString = curString + chr(hexVal)\n\n off = off + opLen\n\n strList.append([curOffset, curString])\n curOffset = 0\n curString = \"\"\n\n usedBytes.append(off)\n\n return strList\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n self.length = config.get_var(self.name, 'length')\n if self.length is None:\n self.length = 3\n\n self.raw = config.get_bvar(self.name, 'raw')\n\n # find the bytes in the file\n instructs = self.findMov(filename)\n\n # now lets get the strings\n strlist = self.decodeBytes(instructs)\n\n self.output_file(config.get_var('Dir','log_dir'), strlist)\n\n return True\n\n def output_file(self, outdir, strlist):\n \"\"\"Print output from analysis to a file.\"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.output_file')\n\n # if the string is of the right len, print it\n outstr = \"\"\n for string in strlist:\n if len(string[1]) >= int(self.length):\n outstr = outstr + '0x%x: %s\\n' % (string[0], string[1])\n\n if len(outstr) > 0:\n try:\n outfile = open(outdir + os.sep + 'single-string.txt', 'w')\n except IOError, err:\n log.debug(\"Cannot open single-string.txt: %s\" % err)\n return False\n\n outfile.write(outstr)\n outfile.close()\n else:\n log.debug('No single-byte strings found.')\n\n return True\n\n" }, { "path": "mastiff/plugins/analysis/EXE/EXE-singlestring.yapsy-plugin", "content": "[Core]\nName = Single-Byte Strings\nModule = EXE-singlestring\n\n[Documentation]\nDescription = Extract single-byte strings.\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/EXE/__init__.py", "content": "" }, { "path": "mastiff/plugins/analysis/GEN/GEN-fileinfo.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nFile Info plugin\n\nPlugin Type: Generic\nPurpose:\n This plug-in obtains the file information, such as the name and file size\n and stores it into the database.\n\nDatabase:\n A new table named files will be added to the database. This table contains\n the following fields:\n\n id - Primary Key\n sid - The id # of the file in the mastiff table.\n filename - The filename, including path, of the file being analyzed.\n size - The file size in bytes.\n firstseen - GMT date of when it was first seen (in UNIX timestamp).\n lastseen - GMT date of when it was last seen (in UNIX timestamp).\n times - Number of times this file has been analyzed.\n\nOutput:\n Data is only sent to the database. No files are created.\n\n\"\"\"\n\n__version__ = \"$Id: bc5c3cee7ede3183312b586a2e800bddc31bca1e $\"\n\nimport os\nimport time\nimport logging\nimport sqlite3\n\nimport mastiff.plugins.category.generic as gen\nimport mastiff.sqlite as DB\n\nclass GenFileInfo(gen.GenericCat):\n \"\"\"File Information plugin code.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n gen.GenericCat.__init__(self)\n self.page_data.meta['filename'] = 'file_info'\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n data = dict()\n data['filename'] = filename\n data['size'] = os.stat(filename).st_size\n data['time'] = time.time()\n data['hashes'] = config.get_var('Misc', 'hashes')\n\n self.gen_output(config, data)\n\n self.output_db(config, data)\n return self.page_data\n\n def gen_output(self, config, data):\n \"\"\" Add the output into the local page structure. \"\"\"\n info_table = self.page_data.addTable('File Information')\n info_table.addheader([('name', str), ('info', str)], printHeader=False)\n\n info_table.addrow(['File Name', data['filename']])\n info_table.addrow(['Size', data['size']])\n info_table.addrow(['Time Analyzed', data['time']])\n\n hash_table = self.page_data.addTable('File Hashes')\n hash_table.addheader([('Algorithm', str), ('Hash', str)])\n hash_table.addrow(['MD5', data['hashes'][0]])\n hash_table.addrow(['SHA1', data['hashes'][1]])\n hash_table.addrow(['SHA256', data['hashes'][2]])\n\n def output_db(self, config, data):\n \"\"\"Print output from analysis to a file.\"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n\n db = DB.open_db_conf(config)\n if db is None:\n return False\n \n db.text_factory = str\n\n # If the 'files' table does now exist, add it\n if DB.check_table(db, 'files') == False:\n log.debug('Adding table files')\n fields = [ 'id INTEGER PRIMARY KEY',\n 'sid INTEGER',\n 'filename TEXT',\n 'size INTEGER',\n 'firstseen INTEGER',\n 'lastseen INTEGER',\n 'times INTEGER']\n if DB.add_table(db, 'files', fields) is None:\n return False\n db.commit()\n\n cur = db.cursor()\n sqlid = DB.get_id(db, data['hashes'])\n\n if sqlid is None:\n log.error('%s hashes do not exist in the database', data['filename'])\n return False\n\n # see if the filename already exists in the db\n try:\n cur.execute('SELECT id, times FROM files WHERE filename=? AND sid=?',\n (data['filename'], sqlid, ))\n except sqlite3.Error, err:\n log.error('Could not query filename table: %s', err)\n return None\n results = cur.fetchone()\n if results is not None:\n # filename is already in there. just update the lastseen item\n log.debug('%s is already in the database for hashes. Updating times.',\n data['filename'])\n try:\n cur.execute('UPDATE files SET lastseen=?, times=? WHERE id=?',\n (int(data['time']), results[1]+1, results[0], ))\n db.commit()\n except sqlite3.OperationalError, err:\n log.error('Could not update times: %s', err)\n return False\n return True\n\n # file info is not in the database, add it\n try:\n cur.execute('INSERT INTO files (sid, filename, size, firstseen, lastseen, times) \\\n VALUES (?, ?, ?, ?, ?, ?)',\n (sqlid, data['filename'], data['size'],\n int(data['time']), int(data['time']), 1, ))\n db.commit()\n except sqlite3.Error, err:\n log.error('Could not insert filename into files: %s', err)\n return False\n\n return True\n\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-fileinfo.yapsy-plugin", "content": "[Core]\nName = File Information\nModule = GEN-fileinfo\n\n[Documentation]\nDescription = File Information Retrieval Plug-in\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-fuzzy.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nFuzzy Hashing plug-in\n\nPlugin Type: Generic\nPurpose:\n This plug-in generates the fuzzy hash of the given file.\n Also compares the fuzzy hashes against all of hashes already\n generated in the database.\n\nRequirements:\n - ssdeep (http://ssdeep.sourceforge.net/)\n - pydeep (https://github.com/kbandla/pydeep)\n\nOutput:\n - fuzzy.txt - File listing the fuzzy hash of the file and any files that\n match.\n - The 'fuzzy' field will get added to the files table in the DB to store\n the fuzzy hash.\n\n\"\"\"\n\n__version__ = \"$Id: 1e313a680096a1bea3ff4e5ed5f497a2ca29cd57 $\"\n\nimport logging\n\ntry:\n import pydeep\nexcept ImportError, error:\n print 'Gen-fuzzy: Could not import pydeep: %s'.format(error)\n\nimport mastiff.sqlite as DB\nimport sqlite3\nimport mastiff.plugins.category.generic as gen\n\nclass GenFuzzy(gen.GenericCat):\n \"\"\"Fuzzy hashing plugin.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n gen.GenericCat.__init__(self)\n self.page_data.meta['filename'] = 'fuzzy'\n # we will be adding to the file information hashes, so make sure it runs before us\n self.prereq = 'File Information'\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n log.info('Generating fuzzy hash.')\n\n try:\n my_fuzzy = pydeep.hash_file(filename)\n except pydeep.error, err:\n log.error('Could not generate fuzzy hash: %s', err)\n return False\n\n if self.output_db(config, my_fuzzy) is False:\n return False\n\n fuzz_results = list()\n if config.get_bvar(self.name, 'compare') is True:\n fuzz_results = self.compare_hashes(config, my_fuzzy)\n\n self.output_file(config, my_fuzzy, fuzz_results)\n\n return self.page_data\n\n def compare_hashes(self, config, my_fuzzy):\n \"\"\"\n Compare the current hash to all of the fuzzy\n hashes already collected.\n \"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.compare')\n db = DB.open_db_conf(config)\n conn = db.cursor()\n\n log.info('Comparing fuzzy hashes.')\n\n fuzz_results = list()\n my_md5 = config.get_var('Misc', 'hashes')[0]\n query = 'SELECT md5, fuzzy FROM mastiff WHERE fuzzy NOT NULL'\n try:\n # compare current hash for all fuzzy hashes\n for results in conn.execute(query):\n percent = pydeep.compare(my_fuzzy, results[1])\n if percent > 0 and my_md5 != results[0]:\n fuzz_results.append([results[0], percent])\n except sqlite3.OperationalError, err:\n log.error('Could not grab other fuzzy hashes: %s', err)\n return None\n except pydeep.error, err:\n log.error('pydeep error: %s', err)\n return None\n\n return fuzz_results\n\n def output_file(self, config, my_fuzzy, fuzz_results):\n \"\"\" Writes output to a file. \"\"\"\n\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.output_file')\n\n if self.results['Generic']['File Information'] is None:\n # File Information is not present, cannot continue\n log.error('Missing File Information plug-in output. Aborting.')\n return False\n\n # add fuzzy hashes to the hashes already generated\n if self.results['Generic']['File Information'] is not None:\n # adding a new data onto an existing table\n my_table = self.results['Generic']['File Information']['File Hashes']\n my_table.addrow(['Fuzzy Hash', my_fuzzy])\n\n fuzz_table = self.page_data.addTable('Similar Fuzzy Hashes')\n\n if fuzz_results is not None and len(fuzz_results) > 0:\n fuzz_table.addheader([('MD5', str), ('Percent', str)])\n\n for (md5, percent) in fuzz_results:\n fuzz_table.addrow([md5, percent])\n elif config.get_bvar(self.name, 'compare') is True:\n # This only gets printed if we actually compared\n fuzz_table.addheader([('Data', str)], printHeader=False)\n fuzz_table.addrow(['No other fuzzy hashes were related to this file.'])\n\n return True\n\n def output_db(self, config, my_fuzzy):\n \"\"\" Add fuzzy hash to the DB.\"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.DB_output')\n\n # open up the DB and extend the mastiff table to include fuzzy hashes\n db = DB.open_db_conf(config)\n\n # there is a possibility the mastiff table is not available yet\n # check for that and add it\n if DB.check_table(db, 'files') == False:\n log.debug('Adding table \"files\"')\n fields = [ 'id INTEGER PRIMARY KEY',\n 'sid INTEGER',\n 'filename TEXT',\n 'size INTEGER',\n 'firstseen INTEGER',\n 'lastseen INTEGER',\n 'times INTEGER']\n if DB.add_table(db, 'files', fields) is None:\n return False\n db.commit()\n\n if not DB.add_column(db, 'mastiff', 'fuzzy TEXT DEFAULT NULL'):\n log.error('Unable to add column.')\n return False\n\n conn = db.cursor()\n # update our hash\n sqlid = DB.get_id(db, config.get_var('Misc', 'Hashes'))\n query = 'UPDATE mastiff SET fuzzy=? WHERE id=?'\n try:\n conn.execute(query, (my_fuzzy, sqlid, ))\n db.commit()\n except sqlite3.OperationalError, err:\n log.error('Unable to add fuzzy hash: %s', err)\n return False\n\n db.close()\n return True\n\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-fuzzy.yapsy-plugin", "content": "[Core]\nName = Fuzzy Hashing\nModule = GEN-fuzzy\n\n[Documentation]\nDescription = Fuzzy Hashing Plug-in\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-hex.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nHex Dump plugin\n\nPlugin Type: Generic\nPurpose:\n This plug-in creates a hex view of the file being analyzed.\n \nOutput:\n hexdump.txt - Contents of the file displayed as hex and ASCII characters.\n\n\"\"\"\n\n__version__ = \"$Id: b5381b6505e0ffbd3d2a8beba9fabba187a9b1b2 $\"\n\nimport os\nimport logging\n\n# Change the following line to import the category class you for the files\n# you wish to perform analysis on\nimport mastiff.plugins.category.generic as gen\n\n# Change the class name and the base class\nclass GEN_Hex(gen.GenericCat):\n \"\"\"Hex Plug-in Code.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n gen.GenericCat.__init__(self)\n\n def activate(self):\n \"\"\"Activate the plugin.\"\"\"\n gen.GenericCat.activate(self)\n\n def deactivate(self):\n \"\"\"Deactivate the plugin.\"\"\"\n gen.GenericCat.deactivate(self)\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n \n # make sure we are enabled\n if config.get_bvar(self.name, 'enabled') is False:\n log.info('Disabled. Exiting.')\n return True\n \n try:\n in_file = open(filename, 'rb')\n except IOError, err:\n log.error('Unable to open file.')\n return False\n \n offset = 0\n in_size = os.stat(filename).st_size\n out_string = ''\n \n while offset < in_size:\n try:\n chars = in_file.read(16)\n except IOError, err:\n log.error('Cannot read data from file: %s' % err)\n in_file.close()\n return False\n \n alpha_string = '' \n out_string = out_string + '%08x: ' % offset\n \n for byte in chars:\n out_string = out_string + \"%02x \" % (ord(byte))\n alpha_string = alpha_string + self.is_ascii(byte)\n \n if len(chars) < 16:\n # we are at the end of the file - need to adjust so things line up \n out_string = out_string + ' '*((16-len(chars))*3) \n \n # add on the alpha version of the string\n out_string = out_string + ' |' + alpha_string + '|\\n'\n offset += len(chars) \n \n in_file.close()\n \n return self.output_file(config.get_var('Dir','log_dir'), out_string)\n #return True\n \n def is_ascii(self, letter):\n \"\"\" Returns the letter if it is a printable ascii character, period otherwise. \"\"\"\n if 31 < ord(letter) < 127:\n return letter\n return '.'\n\n def output_file(self, outdir, data):\n \"\"\"Print output from analysis to a file.\"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n \n try: \n outfile = open(outdir + os.sep + 'hexdump.txt', 'w')\n outfile.write(data)\n outfile.close()\n except IOError, err:\n log.error('Could not open resources.txt: %s' % err)\n return False\n \n return True\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-hex.yapsy-plugin", "content": "[Core]\nName = Hex Dump\nModule = GEN-hex\n\n[Documentation]\nDescription = Creates a hex dump of the file.\nAuthor = Tyler Hudak\nVersion = 0.1\nWebsite = www.korelogic.com\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-mastiff-online.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nMASTIFF Online Submission Plug-in\n\nPlugin Type: Generic\nPurpose:\n This plug-in provides an interface to upload a file to MASTIFF Online.\n\nOutput:\n None\n\"\"\"\n\n__version__ = \"$Id: 80ab7046885b0c48bf287c08e87fcb08e78be0df $\"\n\nimport logging\nimport mastiff.plugins as plugins\nimport simplejson as json\nimport os\nimport sys\n\n# Change the following line to import the category class you for the files\n# you wish to perform analysis on\nimport mastiff.plugins.category.generic as gen\n\n# Change the class name and the base class\nclass GenMastiffOnline(gen.GenericCat):\n \"\"\"MASTIFF Online plugin code.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n gen.GenericCat.__init__(self)\n self.page_data.meta['filename'] = 'MASTIFF-online'\n\n def activate(self):\n \"\"\"Activate the plugin.\"\"\"\n gen.GenericCat.activate(self)\n\n def deactivate(self):\n \"\"\"Deactivate the plugin.\"\"\"\n gen.GenericCat.deactivate(self)\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n \n # get terms of service acceptance\n tos = config.get_bvar(self.name, 'accept_terms_of_service')\n if tos is None or tos is False:\n log.info('Terms of service not accepted. Accept to enable MASTIFF Online submission.')\n return self.page_data\n \n myjson = None\n \n submit = config.get_bvar(self.name, 'submit')\n if submit is False:\n log.info('Not configured to send to MASTIFF Online.')\n return self.page_data\n \n # send data to MASTIFF Online server\n host = 'mastiff-online.korelogic.com'\n method = 'https'\n selector=\"/cgi/dispatcher.cgi/UploadMOSample\"\n fields = [('accept_terms_of_service', 'true')]\n file_to_send = open(filename, \"rb\").read() \n files = [(\"upload\", os.path.basename(filename), file_to_send)]\n log.debug('Sending sample to MASTIFF Online.')\n response = plugins.post_multipart(host, method, selector, fields, files)\n\n # what gets returned isn't technically JSON, so we have to manipulate it a little bit\n try:\n myjson = json.loads(response[60:-14].replace('\\'','\\\"'))\n except json.scanner.JSONDecodeError, err:\n log.error('Error processing response: {}'.format(err))\n except:\n e = sys.exc_info()[0]\n log.error('Error processing incoming response: {}.'.format(e)) \n \n if myjson is not None:\n self.gen_output(myjson)\n \n return self.page_data\n\n def gen_output(self, myjson):\n \"\"\"Place the results into a Mastiff Output Page.\"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n\n mytable = self.page_data.addTable('MASTIFF Online')\n mytable.addheader([('name', str), ('data', str)], printHeader=False)\n mytable.addrow(['Sample Uploaded On', myjson['sample_uploaded_on']])\n\n if myjson['sample_state'] == 'todo':\n mytable.addrow(['Status', 'In queue']) \n elif myjson['sample_state'] == 'done':\n mytable.addrow(['Status', 'Completed'])\n else:\n mytable.addrow(['Status', myjson['sample_state']])\n \n mytable.addrow(['URL', 'https://mastiff-online.korelogic.com/index.html?sample_hash_md5=' + myjson['sample_hash_md5']]) \n \n\n return True\n\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-mastiff-online.yapsy-plugin", "content": "[Core]\nName = MASTIFF Online\nModule = GEN-mastiff-online\n\n[Documentation]\nDescription = MASTIFF Online Submission Plug-in\nAuthor = Tyler Hudak\nVersion = 0.1\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-metascan.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nMetascan Online Submission plugin\n\nPlugin Type: Generic\nPurpose:\n This plug-in determines if the file being analyzed has been analyzed on\n www.metascan-online.com previously.\n\n Information on the Metascan Online API can be found at:\n https://www.metascan-online.com/en/public-api\n\nRequirements:\n - A Metascan Online API key is required to be entered into the configuration file.\n This can be obtained from www.metascan-online.com.\n\n - The simplejson module must be present. (https://github.com/simplejson/simplejson)\n\nConfiguration Options:\n\n api_key: Your API key from metascan-online.com. Leave this blank to disable the\n plug-in.\n\n submit [on|off]: Whether you want to submit files to the site or not.\n\nOutput:\n The results from Metascan Online retrieval or submission will be placed into\n metascan-online.txt.\n\n\"\"\"\n\n__version__ = \"$Id: f8b6fe885be9b46a67dd7bc27e74c40d7a9eeff6 $\"\n\nimport logging\nimport simplejson\nimport urllib2\nimport os\nimport socket\n\nimport mastiff.plugins.category.generic as gen\n\nclass GenMetascan(gen.GenericCat):\n \"\"\"MetaScan Online plugin code.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n self.api_key = None\n gen.GenericCat.__init__(self)\n\n def retrieve(self, sha256):\n \"\"\"\n Retrieve results for this hash from Metascan Online.\n \"\"\"\n\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.retrieve')\n\n url = \"https://hashlookup.metascan-online.com/v2/hash/\" + sha256\n headers = { 'apikey' : self.api_key}\n\n # set up request\n log.debug('Submitting request to Metascan Online.')\n\n try:\n req = urllib2.Request(url, headers=headers)\n response = urllib2.urlopen(req, timeout=30)\n except urllib2.HTTPError, err:\n log.error('Unable to contact URL: %s', err)\n return None\n except urllib2.URLError, err:\n log.error('Unable to open connection: %s', err)\n return None\n except socket.timeout, err:\n log.error('Timeout when contacting URL: %s', err)\n return None\n except:\n log.error('Unknown Error when opening connection.')\n return None\n\n json = response.read()\n try:\n response_dict = simplejson.loads(json)\n except simplejson.decoder.JSONDecodeError:\n log.error('Error in Metascan Online JSON response. Are you submitting too fast?')\n return None\n else:\n log.debug('Response received.')\n return response_dict\n\n def submit(self, config, filename):\n \"\"\"\n Submit a file to Metascan Online for analysis.\n\n Note: This function will likely fail if a proxy is used.\n \"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.submit')\n\n try:\n outdir = config.get_var('Dir', 'log_dir')\n mo_file = open(outdir + os.sep + 'metascan-online.txt', 'w')\n except IOError, err:\n log.error('Unable to open %s for writing: %s',\n outdir + 'metascan-online.txt', err)\n return False\n\n # make sure we are allowed to submit\n if config.get_bvar(self.name, 'submit') == False:\n log.info('Submission disabled. Not sending file.')\n mo_file.write('File does not exist on Metascan Online.\\n')\n mo_file.write('Submission is disabled, not sending file.\\n')\n mo_file.close()\n return False\n\n log.info('File had not been analyzed by Metascan Online.')\n log.info('Sending file to Metascan Online.')\n\n # send file to Metascan Online\n url = \"https://scan.metascan-online.com/v2/file\"\n headers = { 'apikey' : self.api_key, 'filename': os.path.basename(filename)}\n\n try:\n req = urllib2.Request(url, headers=headers)\n file_to_send = open(filename, \"rb\").read()\n response = urllib2.urlopen(req, data=file_to_send, timeout=30)\n json = simplejson.loads(response.read())\n except urllib2.HTTPError, err:\n log.error('Unable to contact URL: %s', err)\n return None\n except urllib2.URLError, err:\n log.error('Unable to open connection: %s', err)\n return None\n except socket.timeout, err:\n log.error('Timeout when contacting URL: %s', err)\n return None\n except:\n log.error('Unknown Error when sending file.')\n return None\n\n # write to file\n mo_file.write('File uploaded and processing.\\n')\n mo_file.write('Link: https://www.metascan-online.com/en/scanresult/file/%s\\n' % json['data_id'])\n mo_file.close()\n\n return True\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n self.api_key = config.get_var(self.name, 'api_key')\n if self.api_key is None or len(self.api_key) == 0:\n log.error('No Metascan Online API Key - exiting.')\n return False\n\n sha256 = config.get_var('Misc', 'hashes')[2]\n\n response = self.retrieve(sha256)\n if response is None:\n # error occurred\n log.error('Did not get a response from Metascan Online. Exiting.')\n return False\n\n if sha256.upper() in response and response[sha256.upper()] == \"Not Found\":\n # The file has not been submitted\n self.submit(config, filename)\n else:\n # write response to file\n self.output_file(config.get_var('Dir', 'log_dir'), response)\n\n return True\n\n def output_file(self, outdir, response):\n \"\"\"Format the output from Metascan Online into a file. \"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + 'output_file')\n\n try:\n mo_file = open(outdir + os.sep + 'metascan-online.txt', 'w')\n except IOError, err:\n log.error('Unable to open %s for writing: %s',\n outdir + 'metascan-online.txt', err)\n return False\n\n out_str = ''\n result_str = ''\n\n out_str += 'Metascan Online Results for %s\\n' % response['file_info']['md5']\n out_str += 'Last scan date: %s\\n' % response['scan_results']['start_time']\n\n foundAV = 0\n\n if response['scan_results']['scan_all_result_i'] > 0:\n result_str += '{0:22} {1:24} {2:40}\\n'.format('AV', 'Version', 'Results')\n\n for av_key in sorted(response['scan_results']['scan_details'].keys(), key=lambda s: s.lower()):\n\n # scan_result_i should be 1-9 (10 is engine updating)\n if 10 > response['scan_results']['scan_details'][av_key]['scan_result_i'] > 0 :\n threat_name = response['scan_results']['scan_details'][av_key]['threat_found'].encode('utf-8')\n if threat_name == u'':\n threat_name = u'Unknown Threat'\n\n result_str += '{0:22} {1:24} {2:40}\\n'.format(av_key, \\\n response['scan_results']['scan_details'][av_key]['def_time'], \\\n threat_name)\n foundAV += 1\n\n out_str += 'Total positive results: %d/%d\\n' % (foundAV, response['scan_results']['total_avs'])\n out_str += 'Link to metascan-online.com:\\nhttps://www.metascan-online.com/en/scanresult/file/%s\\n\\n' % response['data_id']\n\n mo_file.write(out_str)\n mo_file.write(result_str)\n\n mo_file.close()\n return True\n\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-metascan.yapsy-plugin", "content": "[Core]\nName = Metascan Online\nModule = GEN-metascan\n\n[Documentation]\nDescription = MetaScan Online Submission Plug-in\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-strings.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nEmbedded Strings Extraction Plugin\n\nPlugin Type: Generic\nPurpose:\n Execute the 'strings' program and obtain embedded ASCII and UNICODE\n strings within the given filename. These will be returned in a\n dictionary where the key is the decimal offset of the string\n within the file and the value is a list of string type (U or A)\n and the string itself.\n\nConfiguration Options:\n\n strcmd = Path to the strings binary\n\n DO NOT CHANGE THE FOLLOWING OPTIONS UNLESS YOU KNOW WHAT YOU ARE DOING.\n str_opts = Options to send to strings every time its called.\n This should be set to \"-a -t d\" (without quotes).\n str_uni = Options to send to strings to obtain UNICODE strings.\n This should be set to \"-e l\" (without quotes).\n\nOutput:\n Output will be put into a file given a directory and the strings\n dictionary.\n\"\"\"\n\n__version__ = \"$Id: 8970ce879282a3479538dd5d159f65ab4ad1092f $\"\n\nimport subprocess\nimport re\nimport logging\nimport os\n\nimport mastiff.plugins.category.generic as gen\n\nclass GenStrings(gen.GenericCat):\n \"\"\"Extract embedded strings.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n gen.GenericCat.__init__(self)\n self.strings = {}\n self.page_data.meta['filename'] = 'strings'\n self.prereq = 'File Information'\n\n def _insert_strings(self, output, str_type):\n \"\"\"Insert output from strings command into self.strings list.\"\"\"\n\n for line in output.split('\\n'):\n m = re.match('\\s*([0-9]+)\\s+(.*)', line)\n if m is not None and m.group(2):\n self.strings[int(m.group(1))] = [str_type, m.group(2)]\n\n def analyze(self, config, filename):\n \"\"\"\n Run the strings command on the given filename and extract ASCII\n and UNICODE strings. The formatted output is stored in self.strings.\n \"\"\"\n # make sure we are activated\n if self.is_activated == False:\n return None\n\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n # get my config options\n str_opts = config.get_section(self.name)\n\n if not str_opts['strcmd'] or \\\n not os.path.isfile(str_opts['strcmd']) or \\\n not os.access(str_opts['strcmd'], os.X_OK):\n log.error('%s is not accessible. Skipping.')\n return None\n\n if not str_opts['str_opts'] or not str_opts['str_uni_opts']:\n log.error('Strings options do not exist. Please check config. Exiting.')\n return None\n\n # obtain ASCII strings\n run = subprocess.Popen([str_opts['strcmd']] + \\\n str_opts['str_opts'].split() + [ filename ],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n close_fds=True)\n (output, error) = run.communicate()\n if error is not None and len(error) > 0:\n log.error('Error running program: %s' % error)\n return False\n\n self._insert_strings(output,'A')\n\n # obtain Unicode strings\n run = subprocess.Popen([str_opts['strcmd']] +\n str_opts['str_opts'].split() + str_opts['str_uni_opts'].split() + [ filename ],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n close_fds=True)\n (output, error) = run.communicate()\n if error is not None and len(error) > 0:\n log.error('Error running program: %s' % error)\n return False\n\n self._insert_strings(output,'U')\n\n #self.gen_output(config.get_var('Dir','log_dir'))\n self.gen_output()\n log.debug ('Successfully grabbed strings.')\n\n return self.page_data\n\n def gen_output(self):\n \"\"\"Place the results into a Mastiff Output Page.\"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n\n # self.page_data was previously initialized\n # add a table to it\n str_table = self.page_data.addTable('Embedded Strings')\n\n if len(self.strings) == 0:\n log.warn(\"No embedded strings detected.\")\n str_table.addheader([('Message', str)], printHeader=False)\n str_table.addrow(['No embedded strings detected.' ])\n return True\n\n str_table.addheader([('Offset', str), ('Type', str), ('String', str)])\n for k in sorted(self.strings.iterkeys()):\n str_table.addrow([ '{:0x}'.format(k), self.strings[k][0], self.strings[k][1] ])\n\n return True\n\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-strings.yapsy-plugin", "content": "[Core]\nName = Embedded Strings Plugin\nModule = GEN-strings\n\n[Documentation]\nDescription = Embedded Strings Plugin\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-virustotal.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nVirusTotal Submission plugin\n\nPlugin Type: Generic\nPurpose:\n This plug-in determines if the file being analyzed has been analyzed on\n www.virustotal.com previously.\n\n Information on the VT API can be found at:\n https://www.virustotal.com/documentation/public-api/\n\nRequirements:\n - A VirusTotal API key is required to be entered into the configuration file.\n This can be obtained from virustotal.com.\n\n - The simplejson module must be present. (https://github.com/simplejson/simplejson)\n\nConfiguration Options:\n\n api_key: Your API key from virustotal.com. Leave this blank to disable the\n plug-in.\n\n submit [on|off]: Whether you want to submit files to VT or not.\n\nOutput:\n The results from VirusTotal retrieval or submission will be placed into\n virustotal.txt.\n\nNote:\n Unless special arrangements are made, VT will not let you send more than 4\n queries in a 1 minute timeframe. You may receive errors if you do.\n\n\"\"\"\n\n__version__ = \"$Id: 8603d09770a593e2a2f9c03f2fa34aa6f6440112 $\"\n\nimport logging\nimport simplejson\nimport urllib\nimport urllib2\nimport os\nimport socket\n\nimport mastiff.plugins as plugins\nimport mastiff.plugins.category.generic as gen\n\nclass GenVT(gen.GenericCat):\n \"\"\"VirusTotal plugin code.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n self.api_key = None\n gen.GenericCat.__init__(self)\n\n def retrieve(self, md5):\n \"\"\"\n Retrieve results for this hash from VT.\n This code based on the code from the VT API documentation.\n \"\"\"\n\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.retrieve')\n\n url = \"https://www.virustotal.com/vtapi/v2/file/report\"\n parameters = dict()\n parameters['apikey'] = self.api_key\n # set resource to the MD5 hash of the file\n parameters['resource'] = md5\n\n # set up request\n log.debug('Submitting request to VT.')\n\n data = urllib.urlencode(parameters)\n try:\n req = urllib2.Request(url, data)\n response = urllib2.urlopen(req)\n except urllib2.HTTPError, err:\n log.error('Unable to contact URL: %s', err)\n return None\n except urllib2.URLError, err:\n log.error('Unable to open connection: %s', err)\n return None\n except:\n log.error('Unknown Error when opening connection.')\n return None\n\n json = response.read()\n try:\n response_dict = simplejson.loads(json)\n except simplejson.decoder.JSONDecodeError:\n log.error('Error in VT JSON response. Are you submitting too fast?')\n return None\n else:\n log.debug('Response received.')\n return response_dict\n\n def submit(self, config, filename):\n \"\"\"\n Submit a file to VT for analysis.\n This code based on the code from the VT API documentation.\n\n Note: This function will likely fail if a proxy is used.\n \"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.submit')\n\n try:\n outdir = config.get_var('Dir', 'log_dir')\n vt_file = open(outdir + os.sep + 'virustotal.txt', 'w')\n except IOError, err:\n log.error('Unable to open %s for writing: %s',\n outdir + 'virustotal.txt', err)\n return False\n\n # make sure we are allowed to submit\n if config.get_bvar(self.name, 'submit') == False:\n log.info('Submission disabled. Not sending file.')\n vt_file.write('File does not exist on VirusTotal.\\n')\n vt_file.write('Submission is disabled, not sending file.\\n')\n vt_file.close()\n return False\n\n log.info('Sending file to VirusTotal')\n\n # send file to VT\n host = \"www.virustotal.com\"\n method = 'https'\n selector = \"/vtapi/v2/file/scan\"\n fields = [(\"apikey\", config.get_var(self.name, 'api_key'))]\n file_to_send = open(filename, \"rb\").read()\n files = [(\"file\", os.path.basename(filename), file_to_send)]\n try:\n json = simplejson.loads(plugins.post_multipart(host, method, selector,\n fields, files))\n except socket.error, err:\n log.error('Unable to send file: %s' % err)\n return False\n\n # check for success\n if json['response_code'] != 1:\n # error\n log.error('Could not submit to VT:\\n%s', json['verbose_msg'])\n return False\n\n # write to file\n vt_file.write(json['verbose_msg'] + '\\n')\n vt_file.write('Link:\\n' + json['permalink'] + '\\n')\n vt_file.close()\n\n return True\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n self.api_key = config.get_var(self.name, 'api_key')\n if self.api_key is None or len(self.api_key) == 0:\n log.error('No VirusTotal API Key - exiting.')\n return False\n\n md5 = config.get_var('Misc', 'hashes')[0]\n\n response = self.retrieve(md5)\n if response is None:\n # error occurred\n log.error('Did not get a response from VT. Exiting.')\n return False\n\n # response of 1 means it has been scanned on VT before\n # response of 0 means that is has not\n if response['response_code'] != 1:\n # The file has not been submitted\n self.submit(config, filename)\n else:\n # write response to file\n self.output_file(config.get_var('Dir', 'log_dir'), response)\n\n return True\n\n def output_file(self, outdir, response):\n \"\"\"Format the output from VT into a file. \"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + 'output_file')\n\n try:\n vt_file = open(outdir + os.sep + 'virustotal.txt', 'w')\n except IOError, err:\n log.error('Unable to open %s for writing: %s',\n outdir + 'virustotal.txt', err)\n return False\n\n vt_file.write('VirusTotal Results for %s\\n' % response['md5'])\n vt_file.write('Last scan date: %s\\n' % response['scan_date'])\n vt_file.write('Total positive results: %d/%d\\n' % \\\n (response['positives'], response['total']))\n vt_file.write('Link to virustotal.com:\\n%s\\n\\n' % response['permalink'])\n\n if response['positives'] > 0:\n vt_file.write('{0:25} {1:15} {2:40}\\n'.format('AV', 'Version', 'Results'))\n\n for av_key in sorted(response['scans'].keys(), key=lambda s: s.lower()):\n\n if response['scans'][av_key]['detected'] == True:\n out_str = '{0:25} {1:15} {2:40}\\n'\n out_str = out_str.format(av_key, \\\n response['scans'][av_key]['version'], \\\n response['scans'][av_key]['result'])\n vt_file.write(out_str)\n\n vt_file.close()\n return True\n\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-virustotal.yapsy-plugin", "content": "[Core]\nName = VirusTotal\nModule = GEN-virustotal\n\n[Documentation]\nDescription = VirusTotal.com Submission Plug-in\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-yara.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nYara Plugin\n\nPlugin Type: Generic\nPurpose:\n This plug-in allows the use of Yara plug-ins to be run on the file being\n analyzed. Yara rules are specified through a configuration option and all\n rules will be applied to the file.\n\nRequirements:\n - Yara, libyara and yara-python must be installed.\n http://code.google.com/p/yara-project\n\nConfiguration Options:\n[yara]\n yara_sigs = Base path to Yara signatures. This path will be recursed\n to find additional signatures. Files with \".yar\" or \".yara\" will\n be used.\n Leave blank to disable the plug-in.\n\nOutput:\n yara.txt - Output listing all matches found. This file will not be present\n if no matches were found.\n\nDatabase:\n\n A new table named 'yara' will be created with the following fields:\n\n id INTEGER PRIMARY KEY = Primary key\n sid INTEGER DEFAULT NULL = ID of file being analyzed\n rule_name TEXT DEFAULT NULL = Name of the Yara rule matched\n meta TEXT DEFAULT NULL = Yara meta information\n tag TEXT DEFAULT NULL = Yara tag information\n rule_file TEXT DEFAULT NULL = Full path to rule file match is from\n file_offset INTEGER DEFAULT NULL = Offset in analyzed file match was found\n string_id TEXT DEFAULT NULL = ID of match variable from Yara rule\n data TEXT DEFAULT NULL = Data Yara rule matched on\n\n Only new information will be added to the database.\n The database is _NOT_ checked to see if old information is present.\n\nNOTE:\n\n Since the Yara output can contain data that is in binary, any potential binary\n data is converted to hex. Within the string, the binary data will be\n represented as \"backslash-xXX\" with the XX being the hex equivalent.\n\n Please ensure all of your rules work in Yara before using them\n in mas.py.\n\n\"\"\"\n\n__version__ = \"$Id: 0f0233e8220e4ca4a6677253006de25ecdb365f6 $\"\n\nimport logging\nimport os\nimport sqlite3\n\ntry:\n import yara\nexcept ImportError, error:\n print \"GenYara: Could not import yara: %s\" % error\n\nimport mastiff.sqlite as DB\nimport mastiff.plugins.category.generic as gen\nimport mastiff.plugins as plugins\n\nclass GenYara(gen.GenericCat):\n \"\"\"Yara signature plug-in.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n gen.GenericCat.__init__(self)\n self.filename = \"\"\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n self.filename = filename\n\n # get my config options\n plug_opts = config.get_section(self.name)\n if plug_opts is None:\n log.error('Could not get %s options.', self.name)\n return False\n elif len(plug_opts['yara_sigs']) == 0:\n log.debug('No yara_sigs parameter. Disabling plug-in.')\n return False\n\n # find all yara signature files\n sig_files = self.get_sigs(plug_opts['yara_sigs'])\n if sig_files is None or len(sig_files) == 0:\n log.debug('No signature files detected. Exiting plug-in.')\n return True\n\n # create sig dict of all files found.\n # namespace is the file name of the rule\n sig_dict = dict()\n for files in sig_files:\n sig_dict[files] = files\n\n # compile rules and run against file\n try:\n rules = yara.compile(filepaths=sig_dict)\n except yara.SyntaxError, err:\n log.error('Rule error: %s', err)\n return False\n\n # generate matches \n try:\n matches = rules.match(self.filename, callback=self._debug_print)\n except yara.Error, err:\n log.error('Yara error: %s', err)\n return False \n\n if len(matches) > 0:\n self.output_file(config.get_var('Dir','log_dir'), matches)\n self.output_db(config, matches)\n\n return True\n\n def get_sigs(self, sig_dir):\n \"\"\"\n Recurse through a directory for Yara signature files.\n Files should end in \".yar\" or \"yara\".\n Returns a list of signature files, None on errors.\n \"\"\"\n # sanity check the path\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.get_sigs')\n if not os.path.isdir(os.path.expanduser(sig_dir)) \\\n or not os.path.exists(os.path.expanduser(sig_dir)):\n log.error('%s is not a directory or does not exist.' % sig_dir)\n return None\n\n sig_files = list()\n\n # walk the directory\n for items in os.walk(os.path.expanduser(sig_dir)):\n # find each yara sig file in the dir\n for files in items[2]:\n if files.endswith('.yar') or \\\n files.endswith('.yara'):\n sig_files.append(items[0] + os.sep + files)\n\n return sig_files\n\n def _debug_print(self, data):\n \"\"\" Debug printing of Yara matches.\"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.match')\n\n if data['matches'] == True:\n for match in data['strings']:\n log.debug('Match: %s: %s' % (data['rule'], plugins.bin2hex(match[2])))\n\n return yara.CALLBACK_CONTINUE\n\n\n def output_file(self, outdir, matches):\n \"\"\"Prints any Yara matches to a file named yara.txt.\"\"\"\n\n out_file = open(outdir + os.sep + 'yara.txt', 'w')\n if len(matches) == 0:\n out_file.write('No Yara matches.')\n else:\n out_file.write('Yara Matches for %s\\n' % self.filename)\n for item in matches:\n out_file.write('\\nRule Name: %s\\n' % item.rule)\n out_file.write('Yara Meta: %s\\n' % item.meta)\n out_file.write('Yara Tags: %s\\n' % item.tags)\n out_file.write('Rule File: %s\\n' % item.namespace)\n out_file.write('Match Info:\\n')\n for y_match in item.strings:\n out_file.write('\\tFile Offset: %d\\n' % y_match[0])\n out_file.write('\\tString ID: %s\\n' % y_match[1])\n out_file.write('\\tData: %s\\n\\n' % plugins.bin2hex(y_match[2]))\n out_file.write('*'*79 + '\\n')\n\n out_file.close()\n\n return True\n\n def output_db(self, config, matches):\n \"\"\" Output any matches to the database. \"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.output_db')\n\n db = DB.open_db_conf(config)\n if db is None:\n return False\n\n # add the table 'yara' if it doesn't exist\n if DB.check_table(db, 'yara') == False:\n fields = ['id INTEGER PRIMARY KEY',\n 'sid INTEGER DEFAULT NULL',\n 'rule_name TEXT DEFAULT NULL',\n 'meta TEXT DEFAULT NULL',\n 'tag TEXT DEFAULT NULL',\n 'rule_file TEXT DEFAULT NULL',\n 'file_offset INTEGER DEFAULT NULL',\n 'string_id TEXT DEFAULT NULL',\n 'data TEXT DEFAULT NULL' ]\n if not DB.add_table(db, 'yara', fields ):\n log.error('Unable to add \"yara\" database table.')\n return False\n\n sqlid = DB.get_id(db, config.get_var('Misc', 'hashes'))\n sel_query = 'SELECT count(*) FROM yara '\n sel_query += 'WHERE sid=? AND rule_name=? AND meta=? AND tag=? AND '\n sel_query += 'rule_file=? AND file_offset=? AND string_id=? AND data=? '\n query = 'INSERT INTO yara '\n query += '(sid, rule_name, meta, tag, rule_file, file_offset, string_id, data) '\n query += 'VALUES (?, ?, ?, ?, ?, ?, ?, ?)'\n\n cur = db.cursor()\n\n # go through all matches and insert into DB if needed\n try:\n for item in matches:\n for y_match in item.strings:\n match_insert = ( sqlid, item.rule, str(item.meta), \\\n str(item.tags), item.namespace, \\\n y_match[0], y_match[1], plugins.bin2hex(y_match[2]), )\n # check to see if its already in there\n cur.execute(sel_query, match_insert)\n if cur.fetchone()[0] == 0:\n # not in the db already, add it in\n log.debug('Adding %s match to database.' % (item.rule))\n cur.execute(query, match_insert)\n db.commit()\n except sqlite3.Error, err:\n log.error('SQL error when adding item to DB: %s' % err)\n return False\n\n\n db.close()\n return True\n\n" }, { "path": "mastiff/plugins/analysis/GEN/GEN-yara.yapsy-plugin", "content": "[Core]\nName = yara\nModule = GEN-yara\n\n[Documentation]\nDescription = Yara Signature Plug-in\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/GEN/__init__.py", "content": "" }, { "path": "mastiff/plugins/analysis/Office/Office-metadata.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nOffice MetaData Plug-in\n\nPlugin Type: PDF\nPurpose:\n Extracts any metadata from an Office document using exiftool\n (http://www.sno.phy.queensu.ca/~phil/exiftool/).\n\nOutput:\n metadata.txt - Contains selected pieces of metadata.\n\nRequirements:\n The exiftool binary is required for this plug-in. The binary can be downloaded\n from http://www.sno.phy.queensu.ca/~phil/exiftool/.\n\nConfiguration Options:\n[Office Metadata]\nexiftool = Path to exiftool program\n\"\"\"\n\n__version__ = \"$Id: 036849ac813bffb3d941d7ec24f8911f0a5f7da0 $\"\n\nimport subprocess\nimport logging\nimport os\n\nimport mastiff.plugins.category.office as office\n\nclass OfficeMetadata(office.OfficeCat):\n \"\"\"Office Metadata plug-in.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n office.OfficeCat.__init__(self)\n self.page_data.meta['filename'] = 'office-metadata'\n\n def analyze(self, config, filename):\n \"\"\"\n Obtain the command and options from the config file and call the\n external program.\n \"\"\"\n # make sure we are activated\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n # get my config options\n plug_opts = config.get_section(self.name)\n if plug_opts is None:\n log.error('Could not get %s options.', self.name)\n return False\n\n # verify external program exists and we can call it\n if not plug_opts['exiftool'] or \\\n not os.path.isfile(plug_opts['exiftool']) or \\\n not os.access(plug_opts['exiftool'], os.X_OK):\n log.error('%s is not accessible. Skipping.', plug_opts['exiftool'])\n return False\n\n # run your external program here\n run = subprocess.Popen([plug_opts['exiftool']] + \\\n [ filename ],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n close_fds=True)\n (output, error) = run.communicate()\n if error is not None and len(error) > 0:\n log.error('Error running program: {}'.format(error))\n return False\n\n metadata = dict()\n keywords = [ 'Author', 'Code Page', 'Comments', 'Company',\n 'Create Date', 'Current User', 'Error',\n 'File Modification Date/Time', 'File Type',\n 'Internal Version Number', 'Keywords',\n 'Last Modified By', 'Last Printed', 'MIME Type',\n 'Modify Date', 'Security', 'Software', 'Subject',\n 'Tag PID GUID', 'Template', 'Title', 'Title Of Parts',\n 'Total Edit Time', 'Warning']\n\n # set up output table\n new_table = self.page_data.addTable(title='Office Document Metadata')\n\n # grab only data we are interested in\n for line in output.split('\\n'):\n if line.split(' :')[0].rstrip() in keywords:\n metadata[line.split(':')[0].rstrip()] = line.split(' :')[1].rstrip().lstrip(' ')\n\n if len(metadata) == 0:\n # no data\n log.warn(\"No PDF metadata detected.\")\n new_table.addheader([('Message', str)], printHeader=False)\n new_table.addrow(['No Office metadata detected.' ])\n else:\n # set up output table\n new_table.addheader([('Data', str), ('Value', str)])\n # sort and add to table\n for key in sorted(metadata.iterkeys()):\n new_table.addrow([key, metadata[key]])\n\n log.debug ('Successfully ran %s.', self.name)\n return self.page_data\n\n\n" }, { "path": "mastiff/plugins/analysis/Office/Office-metadata.yapsy-plugin", "content": "[Core]\nName = Office Metadata\nModule = Office-metadata\n\n[Documentation]\nDescription = Extract Office metadata from document.\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/Office/Office-pyOLEScanner.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\npyOLEScanner.py Plug-in\n\nPlugin Type: Office\nPurpose:\n This plugin runs Giuseppe 'Evilcry' Bonfa's pyOLEScanner.py script.\n pyOLEScanner.py examines an Office document and looks for\n specific instances of malicious code.\n\nPre-requisites:\n - pyOLEScanner.py must be downloaded. It can be found at:\n https://github.com/Evilcry/PythonScripts/raw/master/pyOLEScanner.zip\n\nOutput:\n office-analysis.txt - File containing output from scan.\n deflated_doc/ - If Office document is an Office 2007 or later document,\n it will be deflated and extracted into this directory.\n\nConfiguration Options:\n[Office Metadata]\nexiftool = Path to exiftool program\n\nNOTE:\n- An Error such as \"('An Error Occurred:', 'no such table: BWList')\" in the\n output file is normal and can be ignored.\n- For OfficeX files, an error:\n\n Starting Deflate Procedure\n An error occurred during deflating\n\n may occur when the script is unable to unzip the archive.\n\n\"\"\"\n\n__version__ = \"$Id: 4cff51f78ebe3e9404a8c73b1a0512383d600e1d $\"\n\nimport subprocess\nimport logging\nimport os\nimport sys\n\nimport mastiff.plugins.category.office as office\n\nclass OfficepyOLEScanner(office.OfficeCat):\n \"\"\"\n Wrapper for Giuseppe 'Evilcry' Bonfa's pyOLEScanner.py office analysis\n plug-in.\n \"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n office.OfficeCat.__init__(self)\n\n def analyze(self, config, filename):\n \"\"\"\n Obtain the command and options from the config file and call the\n external program.\n \"\"\"\n # make sure we are activated\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.') \n\n # get my config options\n plug_opts = config.get_section(self.name)\n if plug_opts is None:\n log.error('Could not get %s options.', self.name)\n return False\n\n # verify external program exists and we can call it\n if not plug_opts['olecmd'] or \\\n not os.path.isfile(plug_opts['olecmd']) or \\\n not os.access(plug_opts['olecmd'], os.X_OK):\n log.error('%s is not accessible. Skipping.', plug_opts['olecmd'])\n return False\n\n # we need to change dir to log_dir as pyOLEScanner.py places files in\n # the directory we run in\n my_dir = os.getcwd() \n if os.path.isabs(filename) is False: \n # we need to update the filename to point to the right file\n filename = my_dir + os.sep + filename \n \n os.chdir(config.get_var('Dir','log_dir'))\n\n run = subprocess.Popen([sys.executable] + [plug_opts['olecmd']] + \\\n [ filename ],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n close_fds=True)\n (output, error) = run.communicate()\n if error is not None and len(error) > 0:\n log.error('Error running program: %s' % error)\n os.chdir(my_dir)\n return False\n\n # ole2.sqlite is created by pyOLEScanner.py, but is not usable to us\n # so lets delete it\n try:\n if os.path.isfile('ole2.sqlite'):\n os.remove('ole2.sqlite')\n log.debug('Deleted ole2.sqlite.')\n except OSError, err:\n log.error('Unable to delete ole2.sqlite: %s', err) \n\n # change directories back\n os.chdir(my_dir)\n\n self.output_file(config.get_var('Dir','log_dir'), output)\n log.debug ('Successfully ran %s.', self.name)\n\n return True\n\n def output_file(self, outdir, data):\n \"\"\"Place the data into a file.\"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n\n try:\n out_file = open(outdir + os.sep + \"office-analysis.txt\",'w')\n except IOError, err:\n log.error('Write error: %s', err)\n return False\n\n out_file.write(data)\n out_file.close()\n return True\n\n" }, { "path": "mastiff/plugins/analysis/Office/Office-pyOLEScanner.yapsy-plugin", "content": "[Core]\nName = Office pyOLEScanner\nModule = Office-pyOLEScanner\n\n[Documentation]\nDescription = pyOLEScanner plug-in based on Giuseppe 'Evilcry' Bonfa's code.\nAuthor = Tyler Hudak/Giuseppe 'Evilcry' Bonfa\nVersion = 1.0\nWebsite = www.korelogic.com / https://github.com/Evilcry/PythonScripts/raw/master/pyOLEScanner.zip\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/Office/__init__.py", "content": "" }, { "path": "mastiff/plugins/analysis/PDF/PDF-metadata.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nPDF MetaData Plug-in\n\nPlugin Type: PDF\nPurpose:\n Extracts any metadata from a PDF using exiftool (http://www.sno.phy.queensu.ca/~phil/exiftool/)\n\nOutput:\n metadata.txt - Contains selected pieces of extracted metadata.\n\nRequirements:\n The exiftool binary is required for this plug-in. The binary can be downloaded\n from http://www.sno.phy.queensu.ca/~phil/exiftool/.\n\nTODO:\n Exiftool will miss some metadata, especially if the Info object is present but\n not specified. Future versions of this plug-in will brute force the metadata,\n but PDF-parsing code needs to be written (or import pdf-parser.py).\n\nConfiguration Options:\n[PDF Metadata]\nexiftool = Path to exiftool program\n\"\"\"\n\n__version__ = \"$Id: 0ba78966f263ce6cb3ec0447e392d8c544baa55f $\"\n\nimport subprocess\nimport logging\nimport os\n\nimport mastiff.plugins.category.pdf as pdf\n\nclass PDFMetadata(pdf.PDFCat):\n \"\"\"PDF Metadata plug-in.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n pdf.PDFCat.__init__(self)\n self.page_data.meta['filename'] = 'pdf-metadata'\n\n def analyze(self, config, filename):\n \"\"\"\n Obtain the command and options from the config file and call the\n external program.\n \"\"\"\n # make sure we are activated\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n # get my config options\n plug_opts = config.get_section(self.name)\n if plug_opts is None:\n log.error('Could not get %s options.', self.name)\n return False\n\n # verify external program exists and we can call it\n if not plug_opts['exiftool'] or \\\n not os.path.isfile(plug_opts['exiftool']) or \\\n not os.access(plug_opts['exiftool'], os.X_OK):\n log.error('%s is not accessible. Skipping.', plug_opts['exiftool'])\n return False\n\n # run your external program here\n run = subprocess.Popen([plug_opts['exiftool']] + \\\n [ filename ],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n close_fds=True)\n (output, error) = run.communicate()\n if error is not None and len(error) > 0:\n log.error('Error running program: {}'.format(error))\n return False\n\n metadata = dict()\n keywords = [ 'Creator', 'Create Date', 'Title', 'Author', 'Producer',\n 'Modify Date', 'Creation Date', 'Mod Date', 'Subject',\n 'Keywords', 'Author', 'Metadata Date', 'Description',\n 'Creator Tool', 'Document ID', 'Instance ID', 'Warning']\n\n # grab only data we are interested in\n for line in output.split('\\n'):\n if line.split(' :')[0].rstrip() in keywords:\n metadata[line.split(':')[0].rstrip()] = line.split(' :')[1].rstrip()\n\n new_table = self.page_data.addTable(title='PDF Document Metadata')\n\n if len(metadata) == 0:\n # no data\n log.warn(\"No PDF metadata detected.\")\n new_table.addheader([('Message', str)], printHeader=False)\n new_table.addrow(['No PDF metadata detected.' ])\n else:\n # set up output table\n new_table.addheader([('Data', str), ('Value', str)])\n # sort and add to table\n for key in sorted(metadata.iterkeys()):\n new_table.addrow([key, metadata[key]])\n\n log.debug ('Successfully ran %s.', self.name)\n\n return self.page_data\n" }, { "path": "mastiff/plugins/analysis/PDF/PDF-metadata.yapsy-plugin", "content": "[Core]\nName = PDF Metadata\nModule = PDF-metadata\n\n[Documentation]\nDescription = Extract PDF metadata from document.\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/PDF/PDF-pdfid.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\npdfid.py Plug-in\n\nPlugin Type: PDF\nPurpose:\n Run Didier Stevens' pdfid.py script against a PDF and place the results into\n a file.\n\nOutput:\n pdfid.txt - Output of pdfid.py.\n\nRequirements:\n The pdfid.py script must be installed.\n\nConfiguration Options:\n\n [pdfid]\n pdfid_cmd - Path to the pdfid.py script. Must be executable.\n pdfid_opts - Options to give to the script. Can be empty.\n\n\"\"\"\n\n__version__ = \"$Id: a83e6c90f42bdd7ada3f1393dc749b5b61668c4e $\"\n\nimport subprocess\nimport logging\nimport os\nimport sys\n\nimport mastiff.plugins.category.pdf as pdf\n\nclass PDFid(pdf.PDFCat):\n \"\"\"Run Didier Stevens pdfid.py\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n pdf.PDFCat.__init__(self)\n self.page_data.meta['filename'] = 'pdf-id'\n\n def analyze(self, config, filename):\n \"\"\"\n Obtain the command and options from the config file and call the\n external program.\n \"\"\"\n # make sure we are activated\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n # get my config options\n plug_opts = config.get_section(self.name)\n if plug_opts is None:\n log.error('Could not get %s options.', self.name)\n return False\n\n # verify external program exists and we can call it\n if not plug_opts['pdfid_cmd'] or \\\n not os.path.isfile(plug_opts['pdfid_cmd']) or \\\n not os.access(plug_opts['pdfid_cmd'], os.X_OK):\n log.error('%s is not accessible. Skipping.', plug_opts['pdfid_cmd'])\n return False\n elif len(plug_opts['pdfid_cmd']) == 0:\n log.debug('Plug-in disabled.')\n return False\n\n # options cannot be empty - at least have a blank option\n if 'pdfid_opts' not in plug_opts:\n plug_opts['pdfid_opts'] = ''\n elif len(plug_opts['pdfid_opts']) == 0:\n plug_opts['pdfid_opts'] = ''\n else:\n plug_opts['pdfid_opts'] = plug_opts['pdfid_opts'].split()\n\n # run pdfid.py here\n try:\n run = subprocess.Popen([plug_opts['pdfid_cmd']] + \\\n list(plug_opts['pdfid_opts']) + \\\n [ filename ],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n close_fds=True)\n (output, error) = run.communicate()\n except:\n log.error('Error executing pdfid.py: {}'.format(sys.exc_info()[0]))\n return False\n\n if error is not None and len(error) > 0:\n log.error('Error running program: {}'.format(error))\n return False\n\n # parse through output\n if 'PDF Header' in output.split('\\n')[1]:\n # By default, pdfid.py displays the PDF header as the first. This is different enough from the\n # other data extracted it should be in its own table.\n header_table = self.page_data.addTable(title='PDF Header')\n header_table.addheader([('Name', str), ('Value', str)], printHeader=False)\n header_table.addrow(output.split('\\n')[1].lstrip().split(': '))\n\n\n # grab the rest of the data\n if 'PDF Header' in output.split('\\n')[1]:\n pdf_objects = [ x.lstrip().split() for x in output.split('\\n')[2:] ]\n else:\n pdf_objects = [ x.lstrip().split() for x in output.split('\\n')[1:] ]\n\n new_table = self.page_data.addTable(title='PDF Objects')\n new_table.addheader([('Object___Name', str), ('Count', int)])\n [ new_table.addrow([my_obj[0], my_obj[1]]) for my_obj in pdf_objects if my_obj ]\n\n log.debug ('Successfully ran %s.', self.name)\n\n return self.page_data\n\n\n" }, { "path": "mastiff/plugins/analysis/PDF/PDF-pdfid.yapsy-plugin", "content": "[Core]\nName = pdfid\nModule = PDF-pdfid\n\n[Documentation]\nDescription = Run Didier Stevens' pdfid.py script\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/PDF/PDF-pdfparser.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nPDF-pdfparser\n\nPlugin Type: PDF\nPurpose:\n This plug-in uses Didier Stevens pdf-parser.py code to perform two tasks:\n\n - Writes an uncompressed copy of the PDF to a file named uncompressed-pdf.txt\n - Searches the PDF for keywords in objects, specified by the\n self.interesting_objects list, and writes those objects, and any they\n reference, to a file in pdf-objects/.\n\n All rights for pdf-parser.py belong to Didier Stevens.\n\nRequirements:\n - Didier Stevens pdf-parser.py must be installed.\n (http://blog.didierstevens.com/programs/pdf-tools/)\n\nConfiguration Options:\n\n[pdf-parser]\npdf_cmd = Path to pdf-parser.py\n\n\"\"\"\n\n__version__ = \"$Id: e784c089c5df767e0b92109f46fd67ec540973a3 $\"\n\nimport os\nimport subprocess\nimport logging\nimport re\n\nimport mastiff.queue as queue\nimport mastiff.plugins.category.pdf as pdf\n\nclass PDFparser(pdf.PDFCat):\n \"\"\"Plug-in to run Didier Stevens pdf-parser.py script.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n pdf.PDFCat.__init__(self)\n\n # list of objects we want to search for\n self.interesting_objects = [ 'JavaScript', 'JS', 'OpenAction', 'AA' ]\n\n def analyze(self, config, filename):\n \"\"\"\n Obtain the command and options from the config file and call the\n external program.\n \"\"\"\n # make sure we are activated\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n # get my config options\n plug_opts = config.get_section(self.name)\n if plug_opts is None:\n log.error('Could not get %s options.', self.name)\n return False\n\n # verify external program exists and we can call it\n if not plug_opts['pdf_cmd'] or \\\n not os.path.isfile(plug_opts['pdf_cmd']) or \\\n not os.access(plug_opts['pdf_cmd'], os.X_OK):\n log.error('%s is not accessible. Skipping.', plug_opts['pdf_cmd'])\n return False\n\n self.uncompress(config, plug_opts, filename)\n self.get_objects(config, plug_opts, filename)\n\n log.debug ('Successfully ran %s.', self.name)\n return True\n\n def output_object(self, plug_opts, pdf_file, obj_num, reasons, log_dir):\n \"\"\"\n Run pdf-parser to extract a given obj_num and place\n it into the log_dir directory, in the form obj-#.txt.\n \"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.outobj')\n\n # create the dir if it doesn't exist\n log_dir = log_dir + os.sep + 'pdf-objects'\n if not os.path.exists(log_dir):\n try:\n os.makedirs(log_dir)\n except IOError, err:\n log.error('Unable to create dir %s: %s' % (log_dir, err))\n return False\n\n # if we get the obj_num in the form \"12 0\", remove the gen #\n if ' ' in obj_num:\n # contains whitespace\n obj_num = obj_num.split(' ')[0]\n\n filename = log_dir + os.sep + 'obj-' + obj_num + '.txt'\n\n # have pdf-parser extract the object for us\n options = list(['-o ' + obj_num, '-f', '-w'])\n run = subprocess.Popen([plug_opts['pdf_cmd']] + \\\n options + \\\n [ pdf_file ],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n close_fds=True)\n (output, error) = run.communicate()\n if error is not None and len(error) > 0:\n log.error('Unable to extract object %s.' % obj_num)\n return False\n\n # output the file - we don't use the pdf-parser.py -d option as\n # there are times it errors out when attempting to dump an object\n with open(filename, 'w') as out_file:\n out_file.write('Object %s\\n' % obj_num)\n out_file.write('Flagged due to:\\n')\n for why in reasons:\n out_file.write('\\t%s\\n' % why)\n out_file.write('\\n')\n out_file.write(output)\n\n return True\n\n def get_objects(self, config, plug_opts, filename):\n \"\"\" Search through the PDF for objects associated with malicious\n activity and extract those into their own file.\n \"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.get_objects')\n log.info('Extracting interesting objects.')\n\n #objects = list()\n objects = dict()\n\n for keyword in self.interesting_objects:\n # let pdf-parser.py grab the object containing our keywords\n run = subprocess.Popen([plug_opts['pdf_cmd']] + \\\n ['--search=' + keyword ] +\n [ filename ],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE, \n close_fds=True)\n (output, error) = run.communicate()\n # skip anything that gives us an error\n if error is not None and len(error) > 0:\n continue\n\n # go through pdf-parser output and grab any objects and\n # their referenced objects to dump\n for line in output.split('\\n'):\n obj_match = re.match('obj\\s+([0-9]+\\s+[0-9]+)', line)\n ref_match = re.search('Referencing: ([0-9]+\\s+[0-9\\s,R]+)', line)\n\n if obj_match is not None:\n # obj # #\n cur_obj = obj_match.group(1)\n if cur_obj not in objects.keys():\n objects[cur_obj] = list()\n objects[cur_obj].extend(['Keyword: %s' % keyword ])\n log.debug('Adding object %s for keyword %s' % (cur_obj, keyword))\n elif ref_match is not None:\n # Referenced by: object list\n for ref_obj in \\\n [ x.lstrip()[:-2] for x in ref_match.group(1).split(',')]:\n if ref_obj not in objects.keys():\n # item not created yet\n objects[ref_obj] = list()\n if 'Referenced by %s' % cur_obj not in objects[ref_obj]:\n # make sure we didn't add already\n objects[ref_obj].extend(['Referenced by %s' % cur_obj ])\n log.debug('Adding object %s from reference \"%s\"' % (ref_obj, cur_obj))\n\n # output collected objects to file\n for my_obj in objects.keys():\n self.output_object(plug_opts,\n filename,\n my_obj,\n objects[my_obj],\n config.get_var('Dir', 'log_dir'))\n\n def uncompress(self, config, plug_opts, filename):\n \"\"\" Uncompress the PDF using pdf-parser.py \"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.uncompress')\n log.info('Uncompressing PDF.')\n \n feedback = config.get_bvar(self.name, 'feedback')\n if feedback is True:\n job_queue = queue.MastiffQueue(config.config_file)\n else:\n job_queue = None \n\n # run pdf-parser with -w (raw) and -f (decompress) opts\n run = subprocess.Popen([plug_opts['pdf_cmd']] + \\\n ['-w', '-f' ] +\n [ filename ],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n close_fds=True)\n\n (output, error) = run.communicate()\n if error is not None and len(error) > 0:\n log.error('Unable to uncompress PDF: %s.' % filename)\n return False\n\n self.output_file(config.get_var('Dir', 'log_dir'), output)\n \n if job_queue is not None and feedback is True and not filename.endswith('uncompressed-pdf.txt'):\n log.info('%s' % filename)\n log.info('Adding uncompressed PDF to queue.')\n job_queue.append(config.get_var('Dir', 'log_dir') + os.sep + \"uncompressed-pdf.txt\")\n\n def output_file(self, outdir, data):\n \"\"\"Place the data into a file.\"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n\n try:\n out_file = open(outdir + os.sep + \"uncompressed-pdf.txt\",'w')\n except IOError, err:\n log.error('Write error: %s', err)\n return False\n\n out_file.write(data)\n out_file.close()\n return True\n\n" }, { "path": "mastiff/plugins/analysis/PDF/PDF-pdfparser.yapsy-plugin", "content": "[Core]\nName = pdf-parser\nModule = PDF-pdfparser\n\n[Documentation]\nDescription = Use Didier Stevens pdf-parser.py to uncompress PDF and find interesting objects.\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/PDF/__init__.py", "content": "" }, { "path": "mastiff/plugins/analysis/ZIP/ZIP-extract.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nZip archive extract plug-in.\n\nPlugin Type: ZIP\nPurpose:\n Extract all of the files within the archive into a directory.\n\n If the filename contains an absolute path or '..'s, they are removed before\n extraction occurs.\n\nConfiguration Options:\n\n enabled = [on|off]: Whether you want to submit files to VT or not.\n\nOutput:\n Extracts all of the files in the archive to log_dir/zip_contents.\n\n\"\"\"\n\n__version__ = \"$Id: ed40be29fdba1a1b71bcb47d5c5933a737f2a4b2 $\"\n\nimport logging\nimport os\nimport zipfile\nimport struct\n\nimport mastiff.plugins.category.zip as zip\nimport mastiff.queue as queue\n\nclass ZIP_Extract(zip.ZipCat):\n \"\"\"Zip archive extraction plug-in.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n zip.ZipCat.__init__(self)\n\n def activate(self):\n \"\"\"Activate the plugin.\"\"\"\n zip.ZipCat.activate(self)\n\n def deactivate(self):\n \"\"\"Deactivate the plugin.\"\"\"\n zip.ZipCat.deactivate(self)\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n feedback = config.get_bvar(self.name, 'feedback')\n\n if feedback is True:\n job_queue = queue.MastiffQueue(config.config_file)\n else:\n job_queue = None\n\n # make sure we are enabled\n if config.get_bvar(self.name, 'enabled') is False:\n log.info('Disabled. Exiting.')\n return True\n\n try:\n my_zip = zipfile.ZipFile(filename, 'r', allowZip64=True)\n except (zipfile.BadZipfile, IOError, struct.error), err:\n log.error('Unable to open zip file: {}'.format(err))\n return False\n\n log_dir = config.get_var('Dir', 'log_dir')\n log_dir += os.sep + 'zip_contents'\n try:\n os.mkdir(log_dir)\n except OSError, err:\n # dir already exists, skip\n pass\n\n # grab password if one exists\n pwd = config.get_var(self.name, 'password')\n if pwd is not None and len(pwd) > 0:\n log.info('Password \\\"{}\\\" will be used for this zip.'.format(pwd))\n\n # cycle through files and extract them\n for file_member in my_zip.namelist(): \n\n # if its an absolute directory, remove os.sep\n if file_member[0:1] == os.sep:\n log.info('Zip member \\\"{}\\\" contains absolute path. Stripping.'.format(file_member))\n zipfile_name = os.path.normpath(file_member[1:])\n \n try:\n zipfile_name = unicode(os.path.normpath(file_member))\n except UnicodeDecodeError:\n zipfile_name = unicode(os.path.normpath(file_member), errors='replace')\n\n # warn about the ..'s, normpath above removes them\n if os.pardir in file_member:\n log.warning('File contains ..s: {}'.format(file_member))\n\n # we can't just blindly extract in case there are absolute paths or '..'s\n # so we read in the file, create any directories, and write it out\n try:\n log.debug(u'Creating directory {}.'.format(os.path.dirname(zipfile_name)))\n os.makedirs(log_dir + os.sep + os.path.dirname(zipfile_name))\n except OSError, err:\n log.debug(u'Directory {} already exists.'.format(os.path.dirname(zipfile_name)))\n\n if len(os.path.basename(file_member)) == 0:\n try:\n log.debug('{} is just a directory. Not creating file.'.format(file_member))\n except UnicodeEncodeError:\n log.debug('{} is just a directory. Not creating file.'.format(file_member.encode('utf-8')))\n continue \n\n log.info(u'Extracting {}.'.format(zipfile_name))\n\n try:\n in_file = my_zip.open(file_member, 'r', pwd=pwd)\n data = in_file.read()\n in_file.close()\n except RuntimeError, err:\n log.error('Problem extracting: {}'.format(err.message.encode('utf-8')))\n continue\n except (IOError, zipfile.BadZipfile) as err:\n log.error('Problem extracting {}.'.format(file_member))\n log.error('Possible obfuscation or corruption: {}'.format(err.message))\n continue\n\n try:\n outfile = open(log_dir + os.sep + zipfile_name, 'w')\n outfile.write(data)\n outfile.close()\n\n except IOError, err:\n log.error('Could not write file: {}'.format(err))\n return False\n\n # now feed back to mastiff if asked to\n if job_queue is not None and feedback is True:\n log.info('Adding {} to queue.'.format(zipfile_name.encode('utf-8')))\n job_queue.append(log_dir + os.sep + zipfile_name)\n\n my_zip.close()\n\n return True\n\n" }, { "path": "mastiff/plugins/analysis/ZIP/ZIP-extract.yapsy-plugin", "content": "[Core]\nName = ZipExtract\nModule = ZIP-extract\n\n[Documentation]\nDescription = Extract zip archive contents.\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/ZIP/ZIP-zipinfo.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nZipinfo Analysis Plug-in\n\nPlugin Type: ZIP\nPurpose:\n This plug-in extracts metadata information stored within a zip archive\n for the analysis.\n\n Alot of information was taken from\n http://www.pkware.com/documents/casestudies/APPNOTE.TXT.\n\nTO DO:\n - Decode external attributes.\n - Decode extra data.\n\nOutput:\n zipinfo.txt - File containing all of the metadata.\n\n\"\"\"\n\n__version__ = \"$Id: eabccb2f29d8d5bd52fc2fb77e8e180ed3a4e875 $\"\n\nimport os\nimport logging\nimport zipfile\nimport codecs\nimport struct\n\nimport mastiff.plugins.category.zip as zip\n\nclass ZIP_Info(zip.ZipCat):\n \"\"\"Class to extract zip metadata and place it into a file.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n zip.ZipCat.__init__(self)\n self.page_data.meta['filename'] = 'zipinfo'\n\n def activate(self):\n \"\"\"Activate the plugin.\"\"\"\n zip.ZipCat.activate(self)\n\n def deactivate(self):\n \"\"\"Deactivate the plugin.\"\"\"\n zip.ZipCat.deactivate(self)\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n # grab the info out of the file\n try:\n my_zip = zipfile.ZipFile(filename, 'r')\n info_list = my_zip.infolist()\n except (zipfile.BadZipfile, IOError, struct.error), err:\n log.error('Unable to open or process zip file: {}'.format(err))\n return False\n\n info_table = self.page_data.addTable(title='Zip Archive Information')\n info_table.addheader([('Data', str), ('Value', str)], printHeader=False)\n\n info_table.addrow(['File Name', os.path.basename(filename) ])\n\n if my_zip.comment is None or len(my_zip.comment) == 0:\n info_table.addrow(['Comment', 'This file has no comment.'])\n else:\n # ignore any unprintable unicode characters\n info_table.addrow(['Comment', unicode(\"%s\" % (my_zip.comment), errors='ignore')])\n\n if len(my_zip.filelist) > 0:\n self.quick_info(info_list)\n self.full_info(info_list)\n else:\n info_table.addrow(['Warning', 'Zip archive has no files.'])\n\n my_zip.close()\n\n return self.page_data\n\n def quick_info(self, info_list):\n \"\"\" Obtain quick directory listing of the archive with some information.\"\"\"\n\n quick_table = self.page_data.addTable('Quick Info')\n quick_table.addheader([('Modification___Date', str), ('File___Size', int), ('File___Name', str)])\n\n for file_info in info_list:\n date_str = \"%02d/%02d/%d %02d:%02d:%02d\" % \\\n (file_info.date_time[1], file_info.date_time[2], file_info.date_time[0], \\\n file_info.date_time[3], file_info.date_time[4], file_info.date_time[5])\n\n # if file is encrypted, flag it\n try:\n filename = unicode(file_info.filename)\n except UnicodeDecodeError, err:\n filename = unicode(file_info.filename, 'utf-8', 'replace')\n\n if file_info.flag_bits & 0x1 == 0x1:\n filename = '* ' + filename\n\n quick_table.addrow([date_str, file_info.file_size, filename])\n\n return\n\n def _version_created(self, version):\n \"\"\" Return a string containing the system that created the archive.\n Taken from http://www.pkware.com/documents/casestudies/APPNOTE.TXT\n \"\"\"\n sys_list = [\"MS-DOS, OS/2, FAT/VFAT/FAT32\", \"Amiga\", \"OpenVMS\", \"UNIX\",\n \"VM/CMS\", \"Atari ST\", \"OS/2 H.P.F.S.\", \"Macintosh\",\n \"Z-System\", \"CP/M\", \"Windows NTFS\", \"MVS (OS/390 - Z/OS)\",\n \"VSE\", \"Acorn Risc\", \"VFAT\", \"alternative MVS\", \"BeOS\",\n \"Tandem\", \"OS/400\", \"OS X Darwin\", \"Unknown\"]\n if version > 20:\n version = 19\n\n return sys_list[version]\n\n def _flag_bits(self, flag_bits, method):\n \"\"\" Returns a string containing the explanation of the flag bits. \"\"\"\n\n output = \"\"\n if flag_bits & 0x1 == 0x1:\n output += \" \"*24 + \"- This file is encrypted.\\n\"\n\n if method == 6:\n # Imploding\n if flag_bits & 0x2 == 0x2:\n output += \" \"*24 + \"- 8K sliding dictionary used for compression.\\n\"\n else:\n output += \" \"*24 + \"- 4K sliding dictionary used for compression.\\n\"\n if flag_bits & 0x4 == 0x4:\n output += \" \"*24 + \"- 3 Shannon-Fano trees used for sliding dictionary.\\n\"\n else:\n output += \" \"*24 + \"- 2 Shannon-Fano trees used for sliding dictionary.\\n\"\n elif method == 8 or method == 9:\n # Deflating\n if flag_bits & 0x6 == 0:\n output += \" \"*24 + \"- Normal (-en)\"\n elif flag_bits & 0x6 == 0x2:\n output += \" \"*24 + \"- Maximum (-exx/-ex)\"\n elif flag_bits & 0x6 == 0x4:\n output += \" \"*24 + \"- Fast (-ef)\"\n elif flag_bits & 0x6 == 0x6:\n output += \" \"*24 + \"- Super Fast (-es)\"\n else:\n output += \" \"*24 + \"- UNKNOWN\"\n output += \" compression option was used.\\n\"\n elif method == 14:\n # LZMA\n if flag_bits & 0x02 == 0x02:\n output += \" \"*24 + \"- EOS marker indicates end of compressed data stream.\\n\"\n\n if flag_bits & 8 == 8:\n output += \" \"*24 + \"- Correct values for CRC-32 and sizes are in data descriptor.\\n\"\n\n if flag_bits & 32 == 32:\n output += \" \"*24 + \"- File is compressed patched data.\\n\"\n\n if flag_bits & 64 == 64:\n output += \" \"*24 + \"- Strong encryption is used.\\n\"\n\n if flag_bits & 2048 == 2048:\n output += \" \"*24 + \"- Filename and comments must be encoded in UTF-8.\\n\"\n\n if flag_bits & 8192 == 8192:\n output += \" \"*24 + \"- Central Directory encrypted.\"\n\n return output\n\n def _compression_method(self, method):\n \"\"\" Returns a string describing the compression method used. \"\"\"\n\n methods = [ 'no compression', 'Shrunk',\n 'Reduced with compression factor 1',\n 'Reduced with compression factor 2',\n 'Reduced with compression factor 3',\n 'Reduced with compression factor 4', 'Imploded',\n 'Tokenizing compression algorithm', 'Deflated',\n 'Enhanced Deflating using Deflate64(tm)',\n 'PKWARE Data Compression Library Imploding (old IBM TERSE)',\n 'Reserved by PKWARE', 'BZIP2 algorithm', 'Reserved by PKWARE',\n 'LZMA (EFS)', 'Reserved by PKWARE', 'Reserved by PKWARE',\n 'Reserved by PKWARE', 'IBM TERSE (new)',\n 'IBM LZ77 z Architecture (PFS)', 'WavPack compressed',\n 'PPMd version I, Rev 1', 'UNKNOWN']\n\n if method == 97:\n method = 20\n elif method == 98:\n method = 21\n elif method > 19:\n method = 22\n\n return methods[method]\n\n def _internal_attribs(self, attrib):\n \"\"\" Returns a string describing the internal attributes.\"\"\"\n\n output = \"\"\n if attrib & 0x01 == 0x01:\n output += \" \"*24 + \"- File is apparently ASCII or text.\\n\"\n\n \"\"\" NOTE: bit 0x0002 means that a 4 byte variable record length\n field is present, but this info doesn't seem useful in this case.\n \"\"\"\n return output\n\n def full_info(self, info_list):\n \"\"\" Obtain a full set of information for each file within the archive. \"\"\"\n\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.fileinfo')\n\n full_table = self.page_data.addTable('Zip Archive File Info')\n\n try:\n for file_info in info_list:\n my_headers = list()\n my_output = list()\n\n my_headers.append(('File___Name', str))\n try:\n my_output.append(unicode(file_info.filename))\n except UnicodeDecodeError, err:\n my_output.append(unicode(file_info.filename, errors='replace' ))\n\n\n date_str = \"%02d/%02d/%d %02d:%02d:%02d\" % \\\n (file_info.date_time[1], file_info.date_time[2], file_info.date_time[0], \\\n file_info.date_time[3], file_info.date_time[4], file_info.date_time[5])\n\n my_headers.append(('Last___Modification___Date', str))\n my_output.append(date_str)\n\n #(file_info.compress_type, self._compression_method(file_info.compress_type))\n my_headers.append(('Compression___Type', str))\n my_output.append(\"%d - %s\" % (file_info.compress_type, self._compression_method(file_info.compress_type)))\n\n my_headers.append(('File___Comment', str))\n if file_info.comment is None or len(file_info.comment) == 0:\n my_output.append('None')\n else:\n my_output.append(u\"%s\\n\" % file_info.comment)\n\n #(self._version_created(file_info.create_system), file_info.create_system)\n my_headers.append(('Creation___System', str))\n my_output.append(\"%s (%d)\" % (self._version_created(file_info.create_system), file_info.create_system))\n\n my_headers.append(('PKZIP___creation___version', str))\n my_output.append(file_info.create_version)\n\n my_headers.append(('Version___to___extract', str))\n my_output.append(file_info.extract_version)\n\n my_headers.append(('Flag___bits', str))\n my_output.append(\"0x%x\\n%s\" % (file_info.flag_bits, self._flag_bits(file_info.flag_bits, file_info.compress_type).rstrip('\\n')))\n\n my_headers.append(('Volume___number', str))\n my_output.append(file_info.volume)\n\n my_headers.append(('Internal___attributes', str))\n my_tmpstr = self._internal_attribs(file_info.internal_attr)\n if len(my_tmpstr) > 0:\n my_output.append(\"0x%x\\n%s\" % (file_info.internal_attr, my_tmpstr))\n else:\n my_output.append(\"0x%x\" % (file_info.internal_attr))\n\n my_headers.append(('External___attributes', str))\n my_output.append(\"0x%x\" % file_info.external_attr)\n\n my_headers.append(('CRC32', str))\n my_output.append(file_info.CRC)\n\n my_headers.append(('Header___offset', str))\n my_output.append(file_info.header_offset)\n\n my_headers.append(('Compressed___size', str))\n my_output.append(file_info.compress_size)\n\n my_headers.append(('Uncompress___size', str))\n my_output.append(file_info.file_size)\n\n my_headers.append(('Extra___Data', str))\n\n if file_info.extra is not None:\n my_output.append('This file entry contains extra data. Not supported yet.')\n else:\n my_output.append('No extra data.')\n\n # add the header if necessary\n if full_table.header is None:\n full_table.addheader(my_headers, printVertical=True)\n full_table.addrow(my_output)\n\n except ImportError:\n log.error('Error obtaining file information from archive for {}.'.format(file_info.filename.encode('utf-8','backslashreplace')))\n\n return\n\n def output_file(self, outdir, data):\n \"\"\"Print output from analysis to a file.\"\"\"\n\n log = logging.getLogger('Mastiff.Plugins.' + self.name + '.output')\n\n try:\n outfile = codecs.open(outdir + os.sep + 'zipinfo-old.txt', 'w', encoding='utf-8')\n outfile.write(data)\n outfile.close()\n except IOError, err:\n log.error('Could not open zipinfo.txt: {}'.format(err))\n return False\n\n return True\n\n" }, { "path": "mastiff/plugins/analysis/ZIP/ZIP-zipinfo.yapsy-plugin", "content": "[Core]\nName = ZipInfo\nModule = ZIP-zipinfo\n\n[Documentation]\nDescription = Extract zip metadata and file information.\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/analysis/ZIP/__init__.py", "content": "" }, { "path": "mastiff/plugins/analysis/__init__.py", "content": "" }, { "path": "mastiff/plugins/category/EXE.yapsy-plugin", "content": "[Core]\nName = Windows Executable Category\nModule = exe\n\n[Documentation]\nDescription = Windows Executable Category Plugin\nAuthor = Tyler Hudak\nWebsite = www.korelogic.com\nVersion = 1.0\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/category/PDF.yapsy-plugin", "content": "[Core]\nName = Adobe PDF Category\nModule = pdf\n\n[Documentation]\nDescription = Adobe PDF Category Plugin\nAuthor = Tyler Hudak\nWebsite = www.korelogic.com\nVersion = 1.0\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/category/__init__.py", "content": "" }, { "path": "mastiff/plugins/category/categories.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nThe base category classes for each of the file types analyzed by\nmastiff.\n\"\"\"\n\n__version__ = \"$Id: e7abe9b27e953709d06c590305ce0c16eaa36c34 $\"\n\nfrom yapsy.IPlugin import IPlugin\nimport mastiff.plugins.output as output\n\nclass MastiffPlugin(IPlugin):\n \"\"\"The base plugin class every category class should inherit.\"\"\"\n\n def __init__(self, name=None):\n \"\"\"Initialize the Mastiff plugin class.\"\"\"\n IPlugin.__init__(self)\n self.name = name\n self.prereq = None\n self.yara_filetype = None\n self.page_data = output.page()\n self.page_data.meta['filename'] = 'CHANGEME'\n\n def activate(self):\n \"\"\"Power rings activate! Form of Mastiff Plugin!\"\"\"\n IPlugin.activate(self)\n\n def analyze(self, config, filename, output=None):\n pass\n\n def deactivate(self):\n \"\"\"Deactivate plugin.\"\"\"\n IPlugin.deactivate(self)\n\n def set_name(self, name=None):\n \"\"\"\n Yapsy does not provide an easy way to get or set our own\n name, so here's a function to do so.\n \"\"\"\n self.name = name\n return self.name\n\n" }, { "path": "mastiff/plugins/category/exe.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nWindows Executable File Category Plugin\n\nFile Type: Windows Executable Programs\nPurpose:\n This file contains the code for the category class \"exe\", which\n allows plugins to be created to be run on Windows executable files.\n\nOutput:\n None\n\n__init__(): MANDATORY: Any initialization code the category requires. It must\n also call the __init__ for the MastiffPlugin class.\n\nis_my_filetype(id_dict, file_name): MANDATORY: This function will return\n the cat_name if the given id_dict contains one of the\n file types this category can examine, or the yara rule matches the\n file type. The file_name is also given so additional tests can be\n performed, if required. None should be returned if it does not \n analyze this type.\n\"\"\"\n\n__version__ = \"$Id: 609d6d02a651ff56ef7b7da434603e150b723876 $\"\n\nimport struct\nimport mastiff.plugins.category.categories as categories\nimport mastiff.filetype as FileType\n\nclass EXECat(categories.MastiffPlugin):\n \"\"\"Category class for Windows executables.\"\"\"\n\n def __init__(self, name=None):\n \"\"\"Initialize the category.\"\"\"\n categories.MastiffPlugin.__init__(self, name)\n self.cat_name = 'EXE'\n self.my_types = [ 'PE32 executable',\n 'MS-DOS executable',\n 'Win32 Executable',\n 'Win32 EXE'\n ]\n self.yara_filetype = \"\"\"rule isexe {\n\t strings:\n\t\t $MZ = \"MZ\" \n\t condition:\n\t\t $MZ at 0 and uint32(uint32(0x3C)) == 0x00004550\n }\"\"\"\n\n def is_exe(self, filename):\n \"\"\" Look to see if the filename has the header format we expect,\"\"\"\n\n with open(filename, 'rb') as exe_file:\n header = exe_file.read(2)\n if header != 'MZ':\n return False\n\n exe_file.seek(0x3c)\n offset = struct.unpack(' 1024:\n # seems a bit too far - we'll stop just in case\n return False\n\n exe_file.seek(offset[0])\n pe_header = exe_file.read(2)\n if pe_header != 'PE':\n return False\n\n return True\n\n def is_my_filetype(self, id_dict, file_name):\n \"\"\"Determine if magic string is appropriate for this category.\"\"\"\n\n # check magic string first\n try:\n if [ type_ for type_ in self.my_types if type_ in id_dict['magic']]:\n return self.cat_name\n except:\n return None\n\n # run Yara type check\n if FileType.yara_typecheck(file_name, self.yara_filetype) is True:\n return self.cat_name\n\n # perform a manual check\n if self.is_exe(file_name):\n return self.cat_name\n\n return None\n\n" }, { "path": "mastiff/plugins/category/generic.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nGeneric File Category Plugin\n\nFile Type: Any files\nPurpose:\n This file contains the code for the category class \"generic\", which\n allows plugins to be created to be run on any file.\n\nOutput:\n None\n\n__init__(): MANDATORY: Any initialization code the category requires. It must\n also call the __init__ for the MastiffPlugin class.\n\nis_my_filetype(id_dict, file_name): MANDATORY: This function will return\n the cat_name if the given id_dict pertains to one of the\n file types this category can examine. The file_name is also given\n so additional tests can be performed, if required. None should be\n returned if it does not analyze this type.\n\"\"\"\n\n__version__ = \"$Id: 58d893fbc4b026eb0104912013663e1562446620 $\"\n\nimport mastiff.plugins.category.categories as categories\n\nclass GenericCat(categories.MastiffPlugin):\n \"\"\"Category class for any file.\"\"\"\n\n def __init__(self, name=None):\n \"\"\"Initialize the category.\"\"\"\n categories.MastiffPlugin.__init__(self, name)\n self.cat_name = 'Generic'\n self.my_types = []\n\n def is_my_filetype(self, id_dict, file_name):\n \"\"\"Generic plugins are run against every file, so always return the\n cat_name.\"\"\"\n return self.cat_name\n\n\nif __name__ == '__main__':\n # testing code\n genclass = GenericCat()\n print genclass.cat_name\n\n" }, { "path": "mastiff/plugins/category/generic.yapsy-plugin", "content": "[Core]\nName = Generic Category\nModule = generic\n\n[Documentation]\nDescription = Generic Files Category Plugin\nAuthor = Tyler Hudak\nWebsite = www.korelogic.com\nVersion = 1.0\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/category/office.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nMicrosoft Office File Category Plugin\n\nFile Type: Microsoft Office Documents\nPurpose:\n This file contains the code for the category class \"office\", which\n allows plugins to be created to be run on Microsoft Office documents.\n\nOutput:\n None\n\n__init__(): MANDATORY: Any initialization code the category requires. It must\n also call the __init__ for the MastiffPlugin class.\n\"\"\"\n\n__version__ = \"$Id: 55366bcaec0c51d2372ef988b3eef4141f351416 $\"\n\nimport mastiff.plugins.category.categories as categories\nimport mastiff.filetype as FileType\n\nclass OfficeCat(categories.MastiffPlugin):\n \"\"\"Category class for Microsoft Office files.\"\"\"\n\n def __init__(self, name=None):\n \"\"\"Initialize the category.\"\"\"\n categories.MastiffPlugin.__init__(self, name)\n self.cat_name = 'Office'\n self.my_types = [ 'CDF V2 Document', # PPT, DOC, XLS\n 'Composite Document File V2',\n 'Microsoft Word',\n 'Microsoft Office Word',\n 'Microsoft Excel',\n 'Microsoft PowerPoint',\n 'Microsoft Office Document'\n ]\n self.yara_filetype = \"\"\"rule isOleDoc {\n\t condition:\n\t\t ( uint32(0x0) == 0xe011cfd0 and uint32(0x4) == 0xe11ab1a1 ) or\n\t\t // some old beta versions have this signature\n\t\t ( uint32(0x0) == 0x0dfc110e and uint32(0x4) == 0x0e11cfd0 )\n }\"\"\"\n\n def is_my_filetype(self, id_dict, file_name):\n \"\"\"Determine if magic string is appropriate for this category.\"\"\"\n\n try:\n if [ type_ for type_ in self.my_types if type_ in id_dict['magic']]:\n return self.cat_name\n except:\n return None\n\n # run Yara type check\n if FileType.yara_typecheck(file_name, self.yara_filetype) is True:\n return self.cat_name\n\n return None\n\n" }, { "path": "mastiff/plugins/category/office.yapsy-plugin", "content": "[Core]\nName = Microsoft Office Category\nModule = office\n\n[Documentation]\nDescription = Microsoft Office Category Plugin\nAuthor = Tyler Hudak\nWebsite = www.korelogic.com\nVersion = 1.0\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/category/pdf.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nAdobe PDF Category Plugin\n\nFile Type: Adobe PDF files\nPurpose:\n This file contains the code for the category class \"pdf\", which\n allows plugins to be created to be run on any file.\n\nOutput:\n None\n\n__init__(): MANDATORY: Any initialization code the category requires. It must\n also call the __init__ for the MastiffPlugin class.\n\"\"\"\n\n__version__ = \"$Id: 310cf87b738bb0ecdf968865c63e94ed0af9d83a $\"\n\nimport mastiff.plugins.category.categories as categories\nimport mastiff.filetype as FileType\n\nclass PDFCat(categories.MastiffPlugin):\n \"\"\"Category class for Adobe PDFs.\"\"\"\n\n def __init__(self, name=None):\n \"\"\"Initialize the category.\"\"\"\n categories.MastiffPlugin.__init__(self, name)\n self.cat_name = 'PDF'\n self.my_types = [ 'PDF document',\n 'Adobe Portable Document Format' ]\n self.yara_filetype = \"\"\"rule ispdf {\n\t strings:\n\t\t $PDF = \"%PDF-\"\n\t condition:\n\t\t $PDF in (0..1024)\n }\"\"\"\n\n def is_my_filetype(self, id_dict, file_name):\n \"\"\"Determine if magic string is appropriate for this category.\"\"\"\n\n # check the magic string for our file type\n try:\n if [ type_ for type_ in self.my_types if type_ in id_dict['magic'] ]:\n return self.cat_name\n except:\n return None\n\n # run Yara type check\n if FileType.yara_typecheck(file_name, self.yara_filetype) is True:\n return self.cat_name\n\n # the PDF header may be in the first 1024 bytes of the file\n # libmagic and TrID may not pick this up\n with open(file_name, 'r') as pdf_file:\n data = pdf_file.read(1024)\n\n if '%PDF-' in data:\n return self.cat_name\n\n return None\n\n" }, { "path": "mastiff/plugins/category/zip.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nZip File Category Plugin\n\nFile Type: Zip Archive\nPurpose:\n This file contains the category class to analyze Zip archives.\nOutput:\n None\n\n\"\"\"\n\n__version__ = \"$Id: a59af7dd53c334712c50d1d05787a63da5e448a6 $\"\n\nimport zipfile\nimport mastiff.plugins.category.categories as categories\nimport mastiff.filetype as FileType\n\nclass ZipCat(categories.MastiffPlugin):\n \"\"\" Category class for Zip documents.\"\"\"\n\n def __init__(self, name=None):\n \"\"\"Initialize the category.\"\"\"\n categories.MastiffPlugin.__init__(self, name)\n\n self.cat_name = 'ZIP'\n self.my_types = [ 'Zip archive', 'ZIP compressed archive' ]\n self.yara_filetype = \"\"\"rule iszip {\n\t condition:\n\t\t uint32(0x0) == 0x04034b50\n }\"\"\"\n\n def is_my_filetype(self, id_dict, file_name):\n \"\"\"Determine if the magic string is appropriate for this category\"\"\"\n\n # Use the python library first\n try:\n # there are times where is_zipfile returns true for non-zipfiles\n # so we have to try and open it as well\n if zipfile.is_zipfile(file_name) is True:\n return self.cat_name\n except:\n return None\n\n # check magic string next\n try:\n if [ type_ for type_ in self.my_types if type_ in id_dict['magic']]:\n return self.cat_name\n except TypeError:\n return None\n\n # run Yara type check\n if FileType.yara_typecheck(file_name, self.yara_filetype) is True:\n return self.cat_name\n\n return None\n\n" }, { "path": "mastiff/plugins/category/zip.yapsy-plugin", "content": "[Core]\nName = Zip Archive Category Plugin\nModule = zip\n\n[Documentation]\nDescription = Zip Archive Category Plugin\nAuthor = Tyler Hudak\nWebsite = www.korelogic.com\nVersion = 1.0\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/output/OUTPUT-raw.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nRaw Output Plug-In\n\nThis output plug-in writes the output in its raw repr() state to a file.\n\"\"\"\n\n__version__ = \"$Id: 4c5a3bcd2b75a26af7638c27124b544b3ce3d8f0 $\"\n\nimport logging\nimport mastiff.plugins.output as masOutput\n\nclass OUTPUTRaw(masOutput.MastiffOutputPlugin):\n \"\"\"Raw output plugin..\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n masOutput.MastiffOutputPlugin.__init__(self)\n\n def activate(self):\n \"\"\"Activate the plugin.\"\"\"\n masOutput.MastiffOutputPlugin.activate(self)\n\n def deactivate(self):\n \"\"\"Deactivate the plugin.\"\"\"\n masOutput.MastiffOutputPlugin.deactivate(self)\n\n def output(self, config, output):\n log = logging.getLogger('Mastiff.Plugins.Output.' + self.name)\n if config.get_bvar(self.name, 'enabled') is False:\n log.debug('Disabled. Exiting.')\n return True\n\n log.info('Writing raw output.')\n try:\n raw_file = open(config.get_var('Dir', 'log_dir')+'/output_raw.txt', 'w')\n except IOError, err:\n log.error('Could not open output_raw.txt file for writing: {}'.format(err))\n return False\n\n raw_file.write(repr(output))\n raw_file.close()\n return True\n" }, { "path": "mastiff/plugins/output/OUTPUT-raw.yapsy-plugin", "content": "[Core]\nName = Raw Output\nModule = OUTPUT-raw\n\n[Documentation]\nDescription = Dumps output in its raw structure format.\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/output/OUTPUT-text.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nText Output Plug-In\n\nThis output plug-in writes the output to a text file.\n\"\"\"\n\n__version__ = \"$Id: 3ba469857b0e052b44f17b90268cbfeace7145cf $\"\n\nimport logging\nimport mastiff.plugins.output as masOutput\n\ndef renderText(page_format, logdir, filename, datastring):\n \"\"\" Places the datastring previously created into the appropriate file or files. \"\"\"\n\n log = logging.getLogger('Mastiff.Plugins.Output.OUTPUTtext.renderText')\n # print out the formatted text for the plug-in\n if page_format == 'single':\n # all data is on one page, open up one file for it\n out_filename = logdir + '/output_txt.txt'\n mode = 'a'\n # add a separater between plug-in output\n datastring += '*'*80 + '\\n'\n elif page_format == 'multiple':\n # data should be broken up into individual files.\n # this will be set for each file\n out_filename = logdir + '/' + filename + '.txt'\n mode = 'w'\n else:\n log.error('Invalid format type for output plugin: {}'.format(format))\n return False\n\n try:\n txt_file = open(out_filename, mode)\n except IOError, err:\n log.error('Could not open {} file for writing: {}'.format(out_filename, err))\n return False\n\n txt_file.write(datastring.encode('utf-8', 'replace'))\n txt_file.close()\n\ndef _extend(data, length=0):\n \"\"\" Returns a unicode string that is left justified by the length given. \"\"\"\n if data is None:\n return u\"\"\n\n try:\n outstr = data.ljust(length)\n except AttributeError:\n outstr = str(data).ljust(length)\n except UnicodeEncodeError: \n outstr = data.decode('utf-8').ljust(length)\n\n if isinstance(outstr, unicode):\n return outstr\n else:\n return unicode(outstr, 'utf-8', 'replace')\n\ndef processPage(plugin, page, page_format):\n \"\"\" Processes a page of data and puts it into the correct format. \"\"\"\n\n txtstr = unicode('', 'utf-8')\n if page_format == 'single':\n txtstr += '\\n{} Plug-in Results\\n\\n'.format(plugin)\n\n # loop through each table in the page\n for tabledata in sorted(page, key=lambda page: page[2]):\n (title, mytable, index) = tabledata\n\n # first we need to go through the table and find the max length for each column\n col_widths = [ len(getattr(col_name, 'name').replace(masOutput.SPACE, ' ')) for col_name in mytable.header ]\n\n # check to see if it should be printed like a horizontal or vertical table\n if mytable.printVertical is False:\n outlist = list()\n\n for row in mytable:\n # modify the col_widths to set a maximum length of each column to 60 characters\n row_lens = list()\n\n for col in row[1:]:\n try:\n row_lens.append(min(60, len(col)))\n except TypeError:\n # if this isn't a str or unicode value, explicitly convert it\n row_lens.append(min(60, len(str(col))))\n\n col_widths = map(max, zip(col_widths, row_lens))\n\n # format the header\n if mytable.printHeader is not False:\n txtstr += \" \".join((getattr(val, 'name')).replace(masOutput.SPACE, ' ').ljust(length) for val, length in zip(mytable.header, col_widths)) + '\\n'\n txtstr += ' '.join([ '-'*val for val in col_widths ])\n\n # format the data\n for row in mytable:\n # combine the row values together and extend them as needed\n # this may be a confusing statement, but its fast!\n #outlist.append(\"\".join(map(lambda x: _extend(x[0], x[1]+2), zip(row[1:], col_widths))))\n outlist.append(\"\".join([_extend(x[0], x[1]+2) for x in zip(row[1:], col_widths) ]))\n\n txtstr += '\\n'\n txtstr += \"\\n\".join(outlist)\n txtstr += '\\n\\n'\n\n else:\n outlist = list()\n\n # get max column width + 2\n max_col = max(col_widths) + 2\n\n # pre-justify header\n newheader = [ getattr(data,'name').replace(masOutput.SPACE, ' ').ljust(max_col) for data in mytable.header ]\n\n # this adds a slight speed increase for large output\n myappend = outlist.append\n\n # go through each row of data and join the header and values together\n for row in mytable:\n #myappend(\"\\n\".join(map(lambda x: x[0] + _extend(x[1], 0), zip(newheader, row[1:]))))\n myappend(\"\\n\".join([ x[0] + _extend(x[1], 0) for x in zip(newheader, row[1:])]))\n myappend(\"\\n\\n\")\n\n txtstr += \"\".join(outlist)\n txtstr += '\\n'\n\n return txtstr\n\nclass OUTPUTtext(masOutput.MastiffOutputPlugin):\n \"\"\"Text output plugin..\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n masOutput.MastiffOutputPlugin.__init__(self)\n\n def activate(self):\n \"\"\"Activate the plugin.\"\"\"\n masOutput.MastiffOutputPlugin.activate(self)\n\n def deactivate(self):\n \"\"\"Deactivate the plugin.\"\"\"\n masOutput.MastiffOutputPlugin.deactivate(self)\n\n def output(self, config, data):\n log = logging.getLogger('Mastiff.Plugins.Output.' + self.name)\n if config.get_bvar(self.name, 'enabled') is False:\n log.debug('Disabled. Exiting.')\n return True\n\n log.info('Writing text output.')\n\n txtstr = unicode('', 'utf-8')\n page_format = config.get_var(self.name, 'format')\n\n # we need to output the File Information plugin first as it contains the\n # summary information on the analyzed file\n try:\n log.debug('Writing file information.')\n txtstr += processPage('File Information', data[data.keys()[0]]['Generic']['File Information'], page_format)\n renderText(page_format, config.get_var('Dir', 'log_dir'), data[data.keys()[0]]['Generic']['File Information'].meta['filename'], txtstr)\n txtstr = unicode('', 'utf-8')\n except KeyError:\n log.error('File Information plug-in data missing. Aborting.')\n return False\n\n # loop through category data\n for cats, catdata in data[data.keys()[0]].iteritems():\n if page_format == 'single':\n catstr = '{} Category Analysis Results'.format(cats)\n log.debug('Writing {} results.'.format(cats))\n txtstr += '{}\\n'.format(catstr) + '-'*len(catstr) + '\\n'\n\n # loop through plugin data and generate the output text\n for plugin, pages in catdata.iteritems():\n if cats == 'Generic' and plugin == 'File Information':\n continue\n\n # process the page into its output string\n txtstr += processPage(plugin, pages, page_format)\n\n # render the text into the appropriate location\n renderText(page_format, config.get_var('Dir', 'log_dir'), pages.meta['filename'], txtstr)\n txtstr = ''\n\n return True\n" }, { "path": "mastiff/plugins/output/OUTPUT-text.yapsy-plugin", "content": "[Core]\nName = Text Output\nModule = OUTPUT-text\n\n[Documentation]\nDescription = Dumps output in text format.\nAuthor = Tyler Hudak\nVersion = 1.0\nWebsite = www.korelogic.com\nLicense = Apache License, Version 2.0\n" }, { "path": "mastiff/plugins/output/__init__.py", "content": "#!/usr/bin/env python\n\n__version__ = \"$Id: e4ef370e46aed6093a66918da42c5f2b1665cf83 $\"\n\nimport collections\nimport time\nfrom yapsy.IPlugin import IPlugin\n\nBASEHEADER = collections.namedtuple('BASEHEADER', 'name type')\nBASEROW = collections.namedtuple('BASEROW', 'ROWINDEX')\n\n# the data types we accept for rows.\n# TODO: Extensive testing on time to be able to represent the multitude of time formats\n# Maybe have our own class?\nDATATYPES = [int, str, float, unicode, time.struct_time]\n\n# characters that spaces should be replaced with\nSPACE='___'\n\nclass TableError(Exception):\n \"\"\" Table Exception class \"\"\"\n pass\n\nclass PageError(Exception):\n \"\"\" Page Exception class \"\"\"\n pass\n\nclass table(object):\n \"\"\"\n Base constructor for table of data.\n A table contains a header and rows of data.\n - The header is just a single row that contains the description of the data.\n - You may only add one row of data at a time\n \"\"\"\n def __init__(self, header=None, data=None, title=None):\n \"\"\"\n Initialize the table.\n self.header: List containing the column names in BASEHEADER named tuple type.\n self.rowdef: Named tuple based on BASEROW. Names are based on header def.\n self.title: String describing the contents of the table.\n self.rows: List of self.rowdef named tuples. Contains the table data.\n self.INDEX: Used for row order. Currently automatically generated.\n\n Input:\n - header: List containing the data definition.\n - data: List containing the initial row of data to initialize.\n - title: String containing the title for the table.\n \"\"\"\n self.INDEX = 0\n self.header = None\n self.printHeader = True\n self.printVertical = False\n self.rowdef = None\n self.addheader(header)\n self.title = title\n self.rows = list()\n if data is not None:\n self.addrow(data)\n return\n\n def __str__(self):\n \"\"\" Return a string containing a quickly formatted view of the table. \"\"\"\n outstring = ''\n if self.title is not None and self.title != '':\n outstring += self.title + '\\n'\n if self.header is not None:\n for item in self.header:\n outstring += str(item.name) + '\\t'\n outstring += '\\n'\n if self.rows is not None and len(self.rows) > 0:\n for rows in sorted(self.rows, key=lambda x: x[0]):\n outstring += '\\t'.join([ str(x) for x in rows[1:] ]) + '\\n'\n\n return outstring\n\n def __repr__(self):\n return ''\n\n def __iter__(self):\n \"\"\"\n Generator to go through table rows.\n Returns the row tuple of the item.\n \"\"\"\n for item in self.rows:\n yield item\n\n def addtitle(self, title=None):\n \"\"\" Add a title to the table. \"\"\"\n if title is not None:\n self.title = title\n else:\n self.title = ''\n\n def addheader(self, header=None, printHeader=True, printVertical=False):\n \"\"\" Add a header to the table.\n The header defines the format of the table and should be a list\n composed of the names of the fields in the table, and their type\n\n After created, the header is used to construct the named tuple for\n all the rows in the table.\n \"\"\"\n if header is not None:\n self.header = list()\n if isinstance(header, list):\n rowdef = tuple()\n for (item, itemtype) in header:\n\n # make sure itemtype is a valid data type\n if itemtype not in DATATYPES:\n raise TypeError('Data type is not a valid type for MASTIFF output.')\n\n self.header.append(BASEHEADER(item, str))\n rowdef = rowdef + (item, )\n\n else:\n raise TypeError('Headers must be of type list.')\n\n if printHeader is False:\n self.printHeader = False\n if printVertical is True:\n self.printVertical = True\n\n # if we have a rowdef, create the row def tuple\n if len(rowdef) > 0:\n self.rowdef = collections.namedtuple('ROWTUPLE', BASEROW._fields + (rowdef ))\n\n def addrow(self, row):\n \"\"\"\n Add a row of data to the table.\n A header must be defined prior to adding any rows of data.\n Input:\n - row: Iterable containing row of data to add to the table. (best if list or tuple used)\n Each item in the iterable will be placed into a separate column in the table.\n \"\"\"\n\n # make sure we have a header defined\n if self.header is None:\n raise TableError('Header is needed before rows can be added.')\n\n if self.rows is None:\n self.rows = list()\n\n # go through the data and add to the table\n if row is not None:\n # The data should be an iterable.\n try:\n if len(row) != len(self.header):\n raise TableError('Row length ({0}) does not equal header length ({1}).'.format(len(row), len(self.header)))\n \n # Currently the index (row position in the table) is by the order the data is received\n # TODO: Take in an index\n rowlist = [self.INDEX]\n self.INDEX += 1\n\n for item in row:\n rowlist.append(item)\n\n # create and add named tuple into self.rows\n self.rows.append(self.rowdef._make(rowlist))\n except TypeError:\n raise TypeError('Invalid type given for data.')\n\nclass page(object):\n \"\"\"\n A page is a container for multiple tables of data.\n Tables will be listed in the order they are added, unless an index is specified\n when the table is added.\n \"\"\"\n def __init__(self):\n self.tables = dict()\n self.meta = dict()\n self.meta['filename'] = 'CHANGEME'\n self.counter = 0\n\n def __getitem__(self, title):\n \"\"\" Overload the getitem operator to return a specified table. \"\"\"\n try:\n return self.tables[title]['table']\n except KeyError:\n raise KeyError('Table {} does not exist.'.format(title))\n\n def __iter__(self):\n \"\"\"\n Generator to go through the list of tables, sorted by index.\n Yields a list of [ title, table, index ]\n \"\"\"\n for title in self.tables:\n yield [ title, self.tables[title]['table'], self.tables[title]['index'] ]\n\n def __str__(self):\n outstring = ''\n for mytable in sorted(self.tables.iteritems(), key=lambda (k, v): v['index']):\n outstring += str(mytable[1]['table'])\n\n return outstring\n\n def __repr__(self):\n return ''\n\n def addTable(self, title, header=None, index=None):\n if title is None or title == '':\n raise PageError('New tables must have a title.')\n\n if index is None:\n index = self.counter\n\n newTable = table(header=header, title=title)\n self.tables[title] = { 'table': newTable, 'index': index }\n self.counter += 1\n return newTable\n\nclass MastiffOutputPlugin(IPlugin):\n \"\"\"The base plugin class every output plugin should inherit.\"\"\"\n\n def __init__(self, name=None):\n \"\"\"Initialize the Mastiff plugin class.\"\"\"\n IPlugin.__init__(self)\n self.name = name\n\n def activate(self):\n \"\"\"Power rings activate! Form of Mastiff Plugin!\"\"\"\n IPlugin.activate(self)\n\n def deactivate(self):\n \"\"\"Deactivate plugin.\"\"\"\n IPlugin.deactivate(self)\n\n def output(self, config, data):\n \"\"\" Output function. Should be overwritten by plugins. \"\"\"\n return False\n\n def set_name(self, name=None):\n \"\"\"\n Yapsy does not provide an easy way to get or set our own\n name, so here's a function to do so.\n \"\"\"\n self.name = name\n return self.name\n\n\n" }, { "path": "mastiff/queue.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\n The queue module is used to add a job queue to MASTIFF. The MastiffQueue\n class uses the MASTIFF SQLite database to keep track of any files that are\n required to be analyzed. It works as a LIFO queue and has no priorities.\n\n This module was originally taken from Thiago Arruda's public domain Python\n job queue at http://flask.pocoo.org/snippets/88/ and has had some minor\n modifications made to make it in-line with MASTIFF.\n\"\"\"\n\n__version__ = \"$Id\"\n\nimport os, sqlite3, os.path\nimport sys\nfrom cPickle import loads, dumps\nfrom time import sleep\ntry:\n from thread import get_ident\nexcept ImportError:\n from dummy_thread import get_ident\n\nimport mastiff.conf as Conf\nimport logging\n\nclass MastiffQueue(object):\n \"\"\" Class to implement a LIFO job queue in a SQLite Database. \"\"\"\n\n _create = (\n 'CREATE TABLE IF NOT EXISTS queue '\n '('\n ' id INTEGER PRIMARY KEY AUTOINCREMENT,'\n ' file BLOB'\n ')'\n )\n _count = 'SELECT COUNT(*) FROM queue'\n _iterate = 'SELECT id, file FROM queue'\n _append = 'INSERT INTO queue (file) VALUES (?)'\n _write_lock = 'BEGIN IMMEDIATE'\n _popleft_get = (\n 'SELECT id, file FROM queue '\n 'ORDER BY id LIMIT 1'\n )\n _popleft_del = 'DELETE FROM queue WHERE id = ?'\n _peek = (\n 'SELECT file FROM queue '\n 'ORDER BY id LIMIT 1'\n )\n _peek_all = (\n 'SELECT file FROM queue '\n 'ORDER BY id'\n )\n\n def __init__(self, config):\n \"\"\" Initialize the class. \"\"\"\n\n #Read the config file and find where the DB is\n log = logging.getLogger('Mastiff.Queue.init')\n\n conf = Conf.Conf(config)\n self.path = os.path.abspath(conf.get_var('Dir', 'log_dir') + os.sep + conf.get_var('Sqlite', 'db_file'))\n log.debug('Setting up queue table at %s' % self.path)\n\n # create the dir if it doesn't exist\n if not os.path.isdir(os.path.dirname(self.path)):\n try:\n os.makedirs(os.path.dirname(self.path))\n except OSError, err:\n log.error('Could not make %s: %s. Exiting.', self.path, err)\n sys.exit(1)\n\n if not os.path.exists(self.path) or not os.path.isfile(self.path):\n # does not exist, create\n try:\n sqlite3.connect(self.path)\n except sqlite3.OperationalError, err:\n log.error('Cannot access sqlite DB: %s.', err)\n\n self._connection_cache = {}\n with self._get_conn() as conn:\n # create the database if required\n conn.execute(self._create)\n\n def __len__(self):\n \"\"\" Allows len(queue) to return the number of items to be processed. \"\"\"\n with self._get_conn() as conn:\n my_len = conn.execute(self._count).next()[0]\n return my_len\n\n def __iter__(self):\n \"\"\" Iterable object. \"\"\"\n with self._get_conn() as conn:\n for my_id, obj_buffer in conn.execute(self._iterate):\n yield loads(str(obj_buffer))\n \n def __str__(self):\n \"\"\" Return contents of database. \"\"\"\n return '\\n'.join(self)\n\n def _get_conn(self):\n \"\"\" Returns a connection to the database. \"\"\"\n my_id = get_ident()\n if my_id not in self._connection_cache:\n self._connection_cache[my_id] = sqlite3.Connection(self.path, timeout=60)\n return self._connection_cache[my_id]\n\n def append(self, obj):\n \"\"\" Add a job to the queue. \"\"\"\n obj_buffer = buffer(dumps(obj, 2))\n with self._get_conn() as conn:\n conn.execute(self._append, (obj_buffer,))\n\n def popleft(self, sleep_wait=False):\n \"\"\"\n Pops a job off the queue and returns it. It will return the next item\n in the queue, or None is none exist. By default, the function will not\n wait if it cannot access the queue table or there is nothing.\n \"\"\"\n keep_pooling = True\n wait = 0.1\n max_wait = 2\n tries = 0\n with self._get_conn() as conn:\n my_id = None\n while keep_pooling:\n conn.execute(self._write_lock)\n cursor = conn.execute(self._popleft_get)\n try:\n my_id, obj_buffer = cursor.next()\n keep_pooling = False\n except StopIteration:\n conn.commit() # unlock the database\n if not sleep_wait:\n keep_pooling = False\n continue\n tries += 1\n sleep(wait)\n wait = min(max_wait, tries/10 + wait)\n if id:\n conn.execute(self._popleft_del, (my_id,))\n return loads(str(obj_buffer))\n return None\n\n def peek(self):\n \"\"\" Return the next item in the queue, but do not remove it. \"\"\"\n with self._get_conn() as conn:\n cursor = conn.execute(self._peek)\n try:\n return loads(str(cursor.next()[0]))\n except StopIteration:\n return None\n \n def clear_queue(self):\n \"\"\" Clear the job queue. \"\"\"\n while self.__len__() > 0:\n self.popleft(sleep_wait=False) \n" }, { "path": "mastiff/sqlite.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nsqlite.py\n\nThis file contains helper functions used to assist MASTIFF plug-ins in placing\ndata into a sqlite database.\n\n\"\"\"\n\n__version__ = \"$Id: 1ca5305893915a251426468100fa9688b59332d7 $\"\n\nimport logging\nimport os\nimport re\n\nimport sqlite3\n\ndef open_db(db_name):\n \"\"\" Return a sqlite3 Connection object for the given database name.\n If the file does not exist, it will attempt to create it.\n \"\"\"\n\n log = logging.getLogger('Mastiff.DB.open')\n if not os.path.exists(db_name) or not os.path.isfile(db_name):\n log.warning('%s does not exist. Will attempt to create.', db_name)\n\n try:\n db = sqlite3.connect(db_name)\n except sqlite3.OperationalError, err:\n log.error('Cannot access sqlite DB: %s.', err)\n db = None\n \n db.text_factory = str\n\n return db\n\ndef open_db_conf(config):\n \"\"\"\n Read the DB information from a MASTIFF config file.\n Return a Sqlite Connection or None.\n \"\"\"\n log = logging.getLogger('Mastiff.DB.open_db_conf')\n log_dir = config.get_var('Dir','base_dir')\n mastiff_db = config.get_var('Sqlite', 'db_file')\n\n if mastiff_db is None or log_dir is None or len(mastiff_db) == 0:\n log.error('Unable to open DB.')\n return None\n\n # db_file can be a full path - if it is, then use it\n dirname = os.path.expanduser(os.path.dirname(mastiff_db))\n if len(dirname) > 0 and os.path.exists(dirname) == True:\n return open_db(mastiff_db)\n\n return open_db(os.path.expanduser(log_dir) + os.sep + mastiff_db)\n\ndef sanitize(string):\n \"\"\"\n Sanitize a string that cannot be sent correctly to sqlite3.\n Returns a string only containing letters, numbers, whitespace\n or underscore.\n \"\"\"\n return re.sub(r'[^a-zA-Z0-9_\\s]', '', string)\n\ndef check_table(db, table):\n \"\"\" Return True is a table exists, False otherwise\"\"\"\n conn = db.cursor()\n\n # sqlite3 won't let us use table names as variables, so we have to\n # use string substitution\n query = 'SELECT * FROM ' + sanitize(table)\n try:\n conn.execute(query)\n return True\n except sqlite3.OperationalError:\n # table doesn't exist\n return False\n\ndef add_table(db, table, fields):\n \"\"\"\n Add a table to a database.\n Table is a string of the table name.\n fields is a list of columns in the form 'column_name column_type'\n Returns True if successful, False otherwise.\n \"\"\"\n conn = db.cursor()\n\n if check_table(db, table):\n # Table already exists\n return True\n\n query = 'CREATE TABLE ' + sanitize(table) + '('\n for item in fields:\n query = query + sanitize(item) + ','\n query = query[:-1] + ')'\n\n try:\n conn.execute(query)\n db.commit()\n except sqlite3.OperationalError, err:\n log = logging.getLogger('Mastiff.DB.add_table')\n log.error('Could not add table %s: %s', table, err)\n return False\n\n return True\n\ndef add_column(db, table, col_def):\n \"\"\"\n Alter an existing table by adding a column to it.\n db is a sqlite3 db connection\n table is the table name\n col_def is the column definition\n \"\"\"\n log = logging.getLogger('Mastiff.DB.add_column')\n if check_table(db, table) == False:\n log.error('Table %s does not exist.', table)\n return False\n\n conn = db.cursor()\n\n query = 'ALTER TABLE ' + table + ' ADD COLUMN ' + col_def\n try:\n conn.execute(query)\n db.commit()\n except sqlite3.OperationalError, err:\n # dup column name errors are fine\n if 'duplicate column name' not in str(err):\n log.error('Could not add column: %s', err)\n return False\n else:\n log.debug('Extended %s with column def \"%s\".', table, col_def)\n\n return True\n\ndef create_mastiff_tables(db):\n \"\"\"\n Create the tables in the MASTIFF database to store\n the main analysis information.\n\n db is a sqlite3 db connection\n \"\"\"\n if check_table(db, 'mastiff') == True:\n # table already exists, nothing to do\n return True\n\n fields = ['id INTEGER PRIMARY KEY',\n 'md5 TEXT DEFAULT NULL',\n 'sha1 TEXT DEFAULT NULL',\n 'sha256 TEXT DEFAULT NULL',\n 'type TEXT DEFAULT NULL']\n\n # if we were not successful, return None\n if add_table(db, 'mastiff', fields) is None:\n return False\n db.commit()\n\n return True\n\ndef get_id(db, hashes):\n \"\"\"\n Return the db id number of the given tuple of hashes.\n Returns None if tuple does not exist.\n \"\"\"\n\n log = logging.getLogger('Mastiff.DB.get_id')\n cur = db.cursor()\n try:\n cur.execute('SELECT id FROM mastiff WHERE (md5=? AND \\\n sha1=? AND sha256=?)',\n [ hashes[0], hashes[1], hashes[2], ])\n except sqlite3.OperationalError, err:\n log.error('Could not execute query: %s', err)\n return None\n\n sqlid = cur.fetchone()\n if sqlid is None:\n return sqlid\n else:\n return sqlid[0]\n\ndef insert_mastiff_item(db, hashes, cat_list=None):\n \"\"\"\n Insert info on analyzed file into database.\n hashes tuple and cat_list will be inserted into mastiff table.\n \"\"\"\n\n log = logging.getLogger('Mastiff.DB.Insert')\n\n # we'll create the tables just to be sure they exist\n create_mastiff_tables(db)\n\n cur = db.cursor()\n sqlid = get_id(db, hashes)\n\n if sqlid is not None:\n # already in there, just send back the id\n log.debug('Hashes %s are already in the database.', hashes)\n else:\n try:\n cur.execute('INSERT INTO mastiff (md5, sha1, sha256) \\\n VALUES (?, ?, ?)',\n (hashes[0], hashes[1], hashes[2]))\n db.commit()\n except sqlite3.OperationalError, err:\n log.error('Could not insert item into mastiff: %s', err)\n return None\n sqlid = cur.lastrowid\n\n if cat_list is not None and sqlid is not None:\n try:\n log.info('Adding %s', str(cat_list))\n cur.execute('UPDATE mastiff SET type=? WHERE id=?',\n (str(cat_list), sqlid, ))\n db.commit()\n except sqlite3.OperationalError, err:\n log.error('Could not update file type in DB: %s', err)\n\n if sqlid is None:\n return sqlid\n\n return sqlid\n\n# testing functions\nif __name__ == '__main__':\n\n # configure logging for Mastiff module\n format_ = '[%(asctime)s] [%(levelname)s] [%(name)s] : %(message)s'\n logging.basicConfig(format=format_)\n log = logging.getLogger(\"Mastiff\")\n log.setLevel(logging.DEBUG)\n\n mysql = open_db('/tmp/test.db')\n if mysql is None:\n print \"Was not created\"\n\n create_mastiff_tables(mysql)\n print \"*** TEST: inserting items\"\n insert_mastiff_item(mysql, ('123', '345', '456'), 'filename')\n insert_mastiff_item(mysql, ('135', '790', '246'), 'filename2')\n insert_mastiff_item(mysql, ('111', '333', '555'), 'filename3')\n insert_mastiff_item(mysql, ('444', '666', '888'), 'filename4')\n print \"*** TEST: insert dup hashes\"\n insert_mastiff_item(mysql, ('111', '333', '555'), 'filename5')\n print \"*** TEST: insert dup filename\"\n insert_mastiff_item(mysql, ('111', '333', '555'), 'filename3')\n print \"*** TEST: add column\"\n add_column(mysql, 'mastiff', 'test_col TEXT DEFAULT NULL')\n mysql.close()\n\n" }, { "path": "mastiff.conf", "content": "# This is the configuration file for mastiff.\n#\n# Comments are preceded by a # or ;\n#\n\n[Dir]\n# log_dir is the base directory where the logs generated will\n# be placed in.\n#log_dir = /usr/local/mastiff/log\nlog_dir = ./work/log\n\n# plugin_dir is a list of directories test plugins may be present in.\n# should be comma-separated.\n# This may be left blank.\n# For example:\n#plugin_dir = ./plugins, /etc/mastiff\nplugin_dir = \n\n# output_plugin_dir is a list of directories test output plugins may be present in.\n# should be comma-separated.\n# This may be left blank.\n# For example:\n#output_plugin_dir = ./plugins, /etc/mastiff\noutput_plugin_dir = \n\n[Misc]\n# verbose = [on|off]\nverbose = off\n# Make a copy of the analyzed file in the log directory with a .VIR extension.\n# copy = [on|off]\ncopy = on\n\n[Sqlite]\n# Sqlite database options\n# db_file = Name of the database file\ndb_file = mastiff.db\n\n[File ID]\n# trid is the location of the TrID binary\n# trid_db is the location of the TrID database\n#trid = /usr/local/bin/trid\ntrid = \ntrid_db = \n\n[Fuzzy Hashing]\n# compare decides whether or not to correlate previous fuzzy hashes\n# compare = [on|off]\ncompare = on\n\n[Hex Dump]\n# Options for Hex Dump plug-in\n# enabled = [on|off]\nenabled = off\n\n[Embedded Strings Plugin]\n# Options for the Embedded Strings Plugin.\n# strcmd is the path to the strings command\n# DO NOT CHANGE THE FOLLOWING OPTIONS UNLESS YOU KNOW WHAT YOU ARE DOING!\n# str_opts are the options to use for all strings operations\n# str_uni_opts are the options to use to obtain UNICODE strings\nstrcmd = /usr/bin/strings\nstr_opts = -a -t d\nstr_uni_opts = -e l\n\n[VirusTotal]\n# Options for the VirusTotal Submission Plug-in.\n# api_key is your API key from virustotal.com\n# - Leave this empty if you wish to disable this plug-in\napi_key = \n\n# submit [on|off] - submit binary to VirusTotal\nsubmit = off\n\n[Metascan Online]\n# Options for the Metascan Online Submission Plug-in.\n# api_key is your API key from metascan-online.com\n# - Leave this empty if you wish to disable this plug-in\napi_key = \n\n# submit [on|off] - submit binary to Metascan Online\nsubmit = off\n\n[MASTIFF Online]\n# Options for submission to MASTIFF Online\n# accept_terms_of_service [true|false] - To upload samples to MASTIFF Online,\n# you agree to the terms of service and privacy policy located at \n# https://mastiff-online.korelogic.com. Set the option below to true to \n# indicate you agree to the terms.\naccept_terms_of_service = false\n# submit [on|off] - submit sample to MASTIFF Online\nsubmit = off\n\n[pdfid]\n# Options to run Didier Stevens pdfid.py script\n# pdfid_cmd = Path to the pdfid.py script\n# - Leave blank if you want the script disabled.\n# pdfid_opts = Options for program.\n# - Do not put multiple options in quotes.\n# Note: pdfid.py has bugs that may cause errors when examining\n# malformed PDFs when using the -e option.\npdfid_cmd = /usr/local/bin/pdfid.py\n#pdfid_opts = -e\npdfid_opts =\n\n[pdf-parser]\n# Options to run Didier Stevens pdf-parser.py script\n# pdf_cmd = Path to pdf-parser.py.\n# feedback: [on|off] - Feed extracted files back into the MASTIFF queue.\npdf_cmd = /usr/local/bin/pdf-parser.py\nfeedback = on\n\n[PDF Metadata]\n# Options for PDF Metadata script\n# exiftool = path to exitfool\nexiftool = /usr/bin/exiftool\n\n[yara]\n# Options for the Yara signature plug-in\n# yara_sigs = Base path to Yara signatures. This path will be recursed\n# to find additional signatures.\n# Leave blank to disable the plug-in.\nyara_sigs = /usr/local/yara\n\n[Digital Signatures]\n# Options to extract the digital signatures\n#\n# disitool - path to disitool.py script.\n# openssl - path to openssl binary\ndisitool = /usr/local/bin/disitool.py\nopenssl = /usr/bin/openssl\n\n[Office Metadata]\n# Options for Office Metadata script\n# exiftool = path to exitfool\nexiftool = /usr/bin/exiftool\n\n[Single-Byte Strings]\n# options for single-byte string extraction plug-in\n# length - Minimum length to extract\nlength = 3\n# raw - print raw characters instead of formatted ones (e.g. \\\\n vs. \\n)\nraw = False\n\n[ZipExtract]\n# options for Zip archive file extraction plug-in\n# enabled: [on|off] - Extract files or not\n# password: Password to use for zip file. OK to leave blank.\n# feedback: [on|off] - Feed extracted files back into the MASTIFF queue.\nenabled = on\npassword =\nfeedback = on\n\n[Office pyOLEScanner]\n# olecmd = Path to pyOLEScanner.py\nolecmd=/usr/local/src/pyOLEScanner/pyOLEScanner.py\n\n################################\n# Output Plug-in Configuration\n################################\n\n[Raw Output]\n# enabled: [on|off] - Dump output in raw form or not\nenabled = off\n\n[Text Output]\n# enabled = [on|off] - Dump output in raw form or not\n# format = [multiple|single] - Put text output in individual files or one page.\nenabled = on\nformat = multiple\n" }, { "path": "pylint.rc", "content": "[MASTER]\n\n# Specify a configuration file.\n#rcfile=\n\n# Python code to execute, usually for sys.path manipulation such as\n# pygtk.require().\n#init-hook=\n\n# Profiled execution.\nprofile=no\n\n# Add files or directories to the blacklist. They should be base names, not\n# paths.\nignore=CVS\n\n# Pickle collected data for later comparisons.\npersistent=yes\n\n# List of plugins (as comma separated values of python modules names) to load,\n# usually to register additional checkers.\nload-plugins=\n\n\n[MESSAGES CONTROL]\n\n# Enable the message, report, category or checker with the given id(s). You can\n# either give multiple identifier separated by comma (,) or put this option\n# multiple time.\n#enable=\n\n# Disable the message, report, category or checker with the given id(s). You\n# can either give multiple identifier separated by comma (,) or put this option\n# multiple time (only on the command line, not in the configuration file where\n# it should appear only once).\ndisable=C0301,C0326\n\n\n[REPORTS]\n\n# Set the output format. Available formats are text, parseable, colorized, msvs\n# (visual studio) and html\noutput-format=parseable\n\n# Include message's id in output\ninclude-ids=no\n\n# Put messages in a separate file for each module / package specified on the\n# command line instead of printing them on stdout. Reports (if any) will be\n# written in a file name \"pylint_global.[txt|html]\".\nfiles-output=no\n\n# Tells whether to display a full report or only the messages\nreports=yes\n\n# Python expression which should return a note less than 10 (10 is the highest\n# note). You have access to the variables errors warning, statement which\n# respectively contain the number of errors / warnings messages and the total\n# number of statements analyzed. This is used by the global evaluation report\n# (RP0004).\nevaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)\n\n# Add a comment according to your evaluation note. This is used by the global\n# evaluation report (RP0004).\ncomment=no\n\n\n[BASIC]\n\n# Required attributes for module, separated by a comma\nrequired-attributes=\n\n# List of builtins function names that should not be used, separated by a comma\nbad-functions=map,filter,apply,input\n\n# Regular expression which should only match correct module names\nmodule-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$\n\n# Regular expression which should only match correct module level names\nconst-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$\n\n# Regular expression which should only match correct class names\nclass-rgx=[A-Z_][a-zA-Z0-9]+$\n\n# Regular expression which should only match correct function names\nfunction-rgx=[a-z_][a-z0-9_]{2,30}$\n\n# Regular expression which should only match correct method names\nmethod-rgx=[a-z_][a-z0-9_]{2,30}$\n\n# Regular expression which should only match correct instance attribute names\nattr-rgx=[a-z_][a-z0-9_]{2,30}$\n\n# Regular expression which should only match correct argument names\nargument-rgx=[a-z_][a-z0-9_]{2,30}$\n\n# Regular expression which should only match correct variable names\nvariable-rgx=[a-z_][a-z0-9_]{2,30}$\n\n# Regular expression which should only match correct list comprehension /\n# generator expression variable names\ninlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$\n\n# Good variable names which should always be accepted, separated by a comma\ngood-names=i,j,k,ex,Run,_\n\n# Bad variable names which should always be refused, separated by a comma\nbad-names=foo,bar,baz,toto,tutu,tata\n\n# Regular expression which should only match functions or classes name which do\n# not require a docstring\nno-docstring-rgx=__.*__\n\n\n[FORMAT]\n\n# Maximum number of characters on a single line.\nmax-line-length=80\n\n# Maximum number of lines in a module\nmax-module-lines=1000\n\n# String used as indentation unit. This is usually \" \" (4 spaces) or \"\\t\" (1\n# tab).\nindent-string=' '\n\n\n[TYPECHECK]\n\n# Tells whether missing members accessed in mixin class should be ignored. A\n# mixin class is detected if its name ends with \"mixin\" (case insensitive).\nignore-mixin-members=yes\n\n# List of classes names for which member attributes should not be checked\n# (useful for classes with attributes dynamically set).\nignored-classes=SQLObject\n\n# When zope mode is activated, add a predefined set of Zope acquired attributes\n# to generated-members.\nzope=no\n\n# List of members which are set dynamically and missed by pylint inference\n# system, and so shouldn't trigger E0201 when accessed. Python regular\n# expressions are accepted.\ngenerated-members=REQUEST,acl_users,aq_parent\n\n\n[SIMILARITIES]\n\n# Minimum lines number of a similarity.\nmin-similarity-lines=4\n\n# Ignore comments when computing similarities.\nignore-comments=yes\n\n# Ignore docstrings when computing similarities.\nignore-docstrings=yes\n\n\n[VARIABLES]\n\n# Tells whether we should check for unused import in __init__ files.\ninit-import=no\n\n# A regular expression matching the beginning of the name of dummy variables\n# (i.e. not used).\ndummy-variables-rgx=_|dummy\n\n# List of additional names supposed to be defined in builtins. Remember that\n# you should avoid to define new builtins when possible.\nadditional-builtins=\n\n\n[MISCELLANEOUS]\n\n# List of note tags to take in consideration, separated by a comma.\nnotes=FIXME,XXX,TODO\n\n\n[CLASSES]\n\n# List of interface methods to ignore, separated by a comma. This is used for\n# instance to not check methods defines in Zope's Interface base class.\nignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by\n\n# List of method names used to declare (i.e. assign) instance attributes.\ndefining-attr-methods=__init__,__new__,setUp\n\n# List of valid names for the first argument in a class method.\nvalid-classmethod-first-arg=cls\n\n\n[IMPORTS]\n\n# Deprecated modules which should not be used, separated by a comma\ndeprecated-modules=regsub,string,TERMIOS,Bastion,rexec\n\n# Create a graph of every (i.e. internal and external) dependencies in the\n# given file (report RP0402 must not be disabled)\nimport-graph=\n\n# Create a graph of external dependencies in the given file (report RP0402 must\n# not be disabled)\next-import-graph=\n\n# Create a graph of internal dependencies in the given file (report RP0402 must\n# not be disabled)\nint-import-graph=\n\n\n[DESIGN]\n\n# Maximum number of arguments for function / method\nmax-args=5\n\n# Argument names that match this expression will be ignored. Default to name\n# with leading underscore\nignored-argument-names=_.*\n\n# Maximum number of locals for function / method body\nmax-locals=15\n\n# Maximum number of return / yield for function / method body\nmax-returns=6\n\n# Maximum number of branch for function / method body\nmax-branchs=12\n\n# Maximum number of statements in function / method body\nmax-statements=50\n\n# Maximum number of parents for a class (see R0901).\nmax-parents=7\n\n# Maximum number of attributes for a class (see R0902).\nmax-attributes=7\n\n# Minimum number of public methods for a class (see R0903).\nmin-public-methods=2\n\n# Maximum number of public methods for a class (see R0904).\nmax-public-methods=20\n\n\n[EXCEPTIONS]\n\n# Exceptions that will emit a warning when being caught. Defaults to\n# \"Exception\"\novergeneral-exceptions=Exception\n" }, { "path": "setup.cfg", "content": "[egg_info]\ntag_build = \ntag_date = 0\ntag_svn_revision = 0\n\n" }, { "path": "setup.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nThis file is the setup/install script for MASTIFF.\n\"\"\"\n\nimport sys\nfrom setuptools import setup, find_packages\nfrom mastiff import get_release_string\n\nif sys.version_info < (2, 6, 6):\n sys.stderr.write(\"Mastiff requires python version 2.6.6\")\n sys.exit(1)\n \nsetup(\n author='Tyler Hudak',\n author_email='mastiff-project@korelogic.com',\n data_files=[('/etc/mastiff', ['mastiff.conf'])],\n description=\"\"\"MASTIFF is a static analysis automation framework.\"\"\",\n install_requires=['Yapsy == 1.10, !=1.10-python3'],\n license='Apache License V2.0',\n long_description=\"\"\"MASTIFF is a static analysis framework that automates the\nprocess of extracting key characteristics from a number of different file\nformats. To ensure the framework remains flexible and extensible, a\ncommunity-driven set of plug-ins is used to perform file analysis and data\nextraction. While originally designed to support malware, intrusion, and\nforensic analysis, the framework is well-suited to support a broader range of\nanalytic needs. In a nutshell, MASTIFF allows analysts to focus on analysis\nrather than figuring out how to parse files.\"\"\",\n maintainer='Tyler Hudak',\n maintainer_email='mastiff-project@korelogic.com',\n name='mastiff', \n packages=find_packages(), \n package_data={'': ['*.py', '*.yapsy-plugin'] },\n platforms=['Linux'],\n scripts=['mas.py'],\n url='http://www.korelogic.com',\n version=get_release_string())\n\n" }, { "path": "skeleton/OUTPUT-skel.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nOutput plugin skeleton code\n\nPurpose:\n This file provides the skeleton code for a plugin that formats the data \n generated by the analysis plug-ins. This is an example that shows all \n functions defined.\n \n __init__(): MANDATORY: Any initialization code the plugin requires. It must\n also call the __init__ for masOutput.MastiffOutputPlugin.\n\nactivate(): OPTIONAL: Activation code called by Yapsy to activate the plugin.\n\ndeactivate(): OPTIONAL: Deactivated code called by Yapsy.\n\noutput(config, output): MANDATORY: Function that formats the data from analysis\n plug-ins into a specific format. Receives the MASTIFF configuration\n as the config parameter, and the pages of data in the data \n parameter.\n\"\"\"\n\n__version__ = \"$Id: 960d687e79158fbba349a472f85ff2b75d8c9bb1 $\"\n\nimport logging\nimport mastiff.plugins.output as masOutput\n\nclass OUTPUTSkeleton(masOutput.MastiffOutputPlugin):\n \"\"\"Raw output plugin..\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n masOutput.MastiffOutputPlugin.__init__(self)\n\n def activate(self):\n \"\"\"Activate the plugin.\"\"\"\n masOutput.MastiffOutputPlugin.activate(self)\n\n def deactivate(self):\n \"\"\"Deactivate the plugin.\"\"\"\n masOutput.MastiffOutputPlugin.deactivate(self)\n\n def output(self, config, data):\n log = logging.getLogger('Mastiff.Plugins.Output.' + self.name)\n \n # see if we are enabled\n if config.get_bvar(self.name, 'enabled') is False:\n log.debug('Disabled. Exiting.')\n return True\n\n log.info('Writing FORMAT output.')\n \n # loop through category data\n for cats, catdata in data[data.keys()[0]].iteritems():\n catstr = '{} Category Analysis Results'.format(cats)\n log.debug('Writing {} results.'.format(cats))\n\n # loop through plugin data and generate the output text\n for plugin, pages in catdata.iteritems():\n # process the page data into the specific format and \n # output it to the appropriate file/files\n \n # loop through each table in the page\n for tabledata in sorted(pages, key=lambda page: pages[2]):\n (title, mytable, index) = tabledata\n \n # process table data here\n for row in mytable:\n # act on row data \n # (REMOVE THE NEXT LINE)\n pass\n\n \n \n return True\n" }, { "path": "skeleton/OUTPUT-skel.yapsy-plugin", "content": "[Core]\nName = Generic Output Skeleton Plugin\nModule = OUTPUT-skel\n\n[Documentation]\nDescription = Your Description Here\nAuthor = Your Name Here\nVersion = 0.1\nWebsite = Your Website Here\n" }, { "path": "skeleton/analysis-ext-skel.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nAnalysis Plugin using external program code\n\nPlugin Type: Generic\nPurpose:\n This file provides the skeleton code for a plugin that performs static\n analysis on any file given to the Mastiff framework using an external\n program. This is an example that shows all functions defined.\n\nOutput:\n None.\n\nIn the MASTIFF configuration file, the options for this particular plug-in\nwould be:\n\n[GenSkel Ext Prog]\nplugcmd = /path/to/my_prog\n\"\"\"\n\n__version__ = \"$Id: 042c8a566d07d74c75251d9ab7306f4a8ab71c0d $\"\n\nimport subprocess\nimport logging\nimport os\n\n# Change the following line to import the category class you for the files\n# you wish to perform analysis on\nimport mastiff.plugins.category.generic as gen\n\n# Change the class name and the base class\nclass GenSkelExt(gen.GenericCat):\n \"\"\"Skeleton generic plugin that calls external program.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n gen.GenericCat.__init__(self)\n self.page_data.meta['filename'] = 'CHANGEME'\n\n def analyze(self, config, filename):\n \"\"\"\n Obtain the command and options from the config file and call the\n external program.\n \"\"\"\n # make sure we are activated\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n # get my config options\n plug_opts = config.get_section(self.name)\n if plug_opts is None:\n log.error('Could not get %s options.', self.name)\n return False\n\n # *** plug_opts['plugcmd'] SHOULD BE CHANGED TO THE PLUGIN SPECIFIC OPTIONS\n\n # verify external program exists and we can call it\n if not plug_opts['plugcmd'] or \\\n not os.path.isfile(plug_opts['plugcmd']) or \\\n not os.access(plug_opts['plugcmd'], os.X_OK):\n log.error('%s is not accessible. Skipping.', plug_opts['plugcmd'])\n return False\n\n # run your external program here\n run = subprocess.Popen([plug_opts['plugcmd']] + \\\n [ filename ],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE, \n close_fds=True)\n (output, error) = run.communicate()\n if error is not None and len(error) > 0:\n log.error('Error running program: %s' % error)\n return False\n\n self.gen_output(output)\n log.debug ('Successfully ran %s.', self.name)\n\n return True\n\n def gen_output(self, output):\n \"\"\"Place the results into a Mastiff Output Page.\"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n\n # self.page_data was previously initialized\n # add a table to it\n new_table = self.page_data.addTable('ANALYSIS PLUGIN DESCRIPTION')\n\n # parse through data generated from output here\n \n # add header to table\n # example: new_table.addHeader([('Header 1', str), ('Header 2', int)])\n \n # add rows of data to table\n # example: new_table.addRow(['row1', 1])\n\n return True\n" }, { "path": "skeleton/analysis-ext-skel.yapsy-plugin", "content": "[Core]\nName = GenSkel Ext Prog\nModule = analysis-ext-skel\n\n[Documentation]\nDescription = Your Description Here\nAuthor = Your Name Here\nVersion = 0.1\nWebsite = Your Website Here\n" }, { "path": "skeleton/analysis-skel.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nAnalysis plugin skeleton code\n\nPlugin Type: Generic\nPurpose:\n This file provides the skeleton code for a plugin that performs static\n analysis on any file given to the Mastiff framework. This is an example that\n shows all functions defined.\n\nOutput:\n None\n\n__init__(): MANDATORY: Any initialization code the plugin requires. It must\n also call the __init__ for its category class.\n\nactivate(): OPTIONAL: Activation code called by Yapsy to activate the plugin.\n\ndeactivate(): OPTIONAL: Deactivated code called by Yapsy.\n\nanalyze(config, filename): MANDATORY: The main body of code that performs the\n analysis on the file.\n\ngen_output(outdir): Function that puts the data into self.page_data for the output\n plug-ins.\n\"\"\"\n\n__version__ = \"$Id: 107798be8154ef41517034e77db3b5a95dd4fe6b $\"\n\nimport logging\n\n# Change the following line to import the category class you for the files\n# you wish to perform analysis on\nimport mastiff.plugins.category.generic as gen\n\n# Change the class name and the base class\nclass GenSkeleton(gen.GenericCat):\n \"\"\"Skeleton generic plugin code.\"\"\"\n\n def __init__(self):\n \"\"\"Initialize the plugin.\"\"\"\n gen.GenericCat.__init__(self)\n self.page_data.meta['filename'] = 'CHANGEME'\n\n def activate(self):\n \"\"\"Activate the plugin.\"\"\"\n gen.GenericCat.activate(self)\n\n def deactivate(self):\n \"\"\"Deactivate the plugin.\"\"\"\n gen.GenericCat.deactivate(self)\n\n def analyze(self, config, filename):\n \"\"\"Analyze the file.\"\"\"\n\n # sanity check to make sure we can run\n if self.is_activated == False:\n return False\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n log.info('Starting execution.')\n\n # Add analysis code here. Data can be added to tables or passed into gen_output\n\n self.gen_output()\n \n return self.page_data\n\n def gen_output(self):\n \"\"\"Place the results into a Mastiff Output Page.\"\"\"\n log = logging.getLogger('Mastiff.Plugins.' + self.name)\n\n # self.page_data was previously initialized\n # add a table to it\n new_table = self.page_data.addTable('ANALYSIS PLUGIN DESCRIPTION')\n\n # add header to table\n # example: new_table.addHeader([('Header 1', str), ('Header 2', int)])\n \n # add rows of data to table\n # example: new_table.addRow(['row1', 1])\n\n return True\n\n" }, { "path": "skeleton/analysis-skel.yapsy-plugin", "content": "[Core]\nName = Generic Skeleton Plugin\nModule = analysis-skel\n\n[Documentation]\nDescription = Your Description Here\nAuthor = Your Name Here\nVersion = 0.1\nWebsite = Your Website Here\n" }, { "path": "skeleton/category-skel.py", "content": "#!/usr/bin/env python\n\"\"\"\n Copyright 2012-2013 The MASTIFF Project, All Rights Reserved.\n\n This software, having been partly or wholly developed and/or\n sponsored by KoreLogic, Inc., is hereby released under the terms\n and conditions set forth in the project's \"README.LICENSE\" file.\n For a list of all contributors and sponsors, please refer to the\n project's \"README.CREDITS\" file.\n\"\"\"\n\n__doc__ = \"\"\"\nCategory Skeleton Plugin\n\nFile Type: New File Type\nPurpose:\n This file contains the skeleton code for a new category class to analyze\n a new file type.\n\nOutput:\n None\n\n__init__(): MANDATORY: Any initialization code the category requires. It must\n also call the __init__ for its superclass - in this case OfficeCat.\n\"\"\"\n\n__version__ = \"$Id: 64ee75c4869a530a4030a50ff7add6ab87601a11 $\"\n\nimport mastiff.plugins.category.categories as categories\nimport mastiff.filetype as FileType\n\n# Change the class name to identify the new file type\nclass SkelCat(categories.MastiffPlugin):\n \"\"\" Category class for Word documents.\"\"\"\n\n def __init__(self, name=None):\n \"\"\"Initialize the category.\"\"\"\n categories.MastiffPlugin.__init__(self, name)\n\n # cat_name should be a one word description of the file type\n self.cat_name = 'SkelCat'\n # Add in strings from libmagic and TrID output\n self.my_types = [ 'libmagic string', 'TrID string' ]\n # Add in the Yara rule\n self.yara_filetype = \"\"\"rule istype { } \"\"\"\n\n def is_my_filetype(self, id_dict, file_name):\n \"\"\"Determine if the magic string is appropriate for this category\"\"\"\n\n # check magic string first\n try:\n if [ type_ for type_ in self.my_types if type_ in id_dict['magic']]:\n return self.cat_name\n except:\n return None\n \n # run Yara type check\n if FileType.yara_typecheck(file_name, self.yara_filetype) is True:\n return self.cat_name\n\n # check TrID output, if available\n # this can likely be removed\n for (percent, desc) in id_dict['trid']:\n for type_ in self.my_types:\n # make sure percent is high enough and trid string matches\n if type_ in desc and percent > 50:\n return self.cat_name\n\n # add your own code on additional file type determination here \n\n return None\n\n" }, { "path": "skeleton/category-skel.yapsy-plugin", "content": "[Core]\nName = Category Skeleton Plug-in\nModule = category-skeleton\n\n[Documentation]\nDescription = Your Description Here\nAuthor = Your Name Here\nWebsite = Your Website Here\nVersion = 0.1\n" }, { "path": "skeleton/output-skel.yapsy-plugin", "content": "[Core]\nName = Generic Output Skeleton Plugin\nModule = output-skel\n\n[Documentation]\nDescription = Your Description Here\nAuthor = Your Name Here\nVersion = 0.1\nWebsite = Your Website Here\n" }, { "path": "tests/import-test.sh", "content": "#!/bin/bash\n\n# $Id: 00c702350cf2edd48c2e57517593c5bce6a64781 $\n#\n# Find all imports from the MASTIFF python files and ensure they can be\n# imported.\n#\n# $1 = directory to test\n\nif [ $# -eq 0 ] ; then\n echo \"Need a directory to scan.\"\n exit\nelif [ ! -d $1 ] ; then\n echo \"$1 is not a directory.\"\n exit\nfi\n\nPWD=`pwd`\nSAVEIFS=$IFS\nIFS=$(echo -en \"\\n\\b\")\n\necho \"Checking Python imports in $1 and below.\"\necho\n\ncd $1\nfor FILE in `find . -name \"*.py\"`; do\n for IMPORT in `egrep \"^\\s*import\\s+|^\\s*from \\S+ import\" ${FILE} | sed -e 's/^[ \\t]*//' | sort -u`; do\n ERROR=`python -c \"${IMPORT}\" 2>&1 | grep \"ImportError\" | grep -vi disitool`\n if [ $? -ne 1 ]; then\n echo ERROR: ${FILE}: ${ERROR}\n fi\ndone; done\n\ncd ${PWD}\nIFS=${SAVEIFS}\n\necho\necho \"Done checking imports.\"\necho\n" }, { "path": "tests/mastiff-test.sh", "content": "#!/bin/bash\n\nMASCMD=\"python ./mas.py -c ./mastiff.conf -V \"\n\n# Test mastiff by running it against various file types.\n# $1 = file type\n# $2 = file to test\n# $3 = outfile\nmas_test()\n{\n echo -n \"Testing ${1}: \"\n if [ ! -f $2 ] ; then\n echo \"$2 missing. Unable to test.\"\n return 0\n fi\n\n ${MASCMD} ${2} > ${3} 2>&1\n if [ $? -ne 0 ] ; then\n OUTMSG=\"Failed. See ${3} for details.\"\n else\n OUTMSG=\"Success.\"\n fi\n echo $OUTMSG\n}\n \necho \"Checking for MASTIFF functionality.\"\necho\n\nmas_test EXE tests/test.exe tests/test-EXE.txt\nmas_test Office tests/test.doc tests/test-DOC.txt\nmas_test PDF tests/test.pdf tests/test-PDF.txt\nmas_test ZIP tests/test.zip tests/test-ZIP.txt\n\necho\necho \"Done checking MASTIFF functionality.\"\n" }, { "path": "utils/version2string", "content": "#!/usr/bin/perl -w\n######################################################################\n#\n# $Id: 6c139ab440c14c954b44b9fad19f5c34154259f7 $\n#\n######################################################################\n#\n# Copyright 2008-2013 The WebJob Project, All Rights Reserved.\n#\n######################################################################\n#\n# Purpose: Convert version numbers to a string representation.\n#\n######################################################################\n\nuse strict;\nuse File::Basename;\nuse File::Path;\nuse Getopt::Std;\n\n######################################################################\n#\n# Main Routine\n#\n######################################################################\n\n ####################################################################\n #\n # Punch in and go to work.\n #\n ####################################################################\n\n my ($sProgram);\n\n $sProgram = basename(__FILE__);\n\n ####################################################################\n #\n # Validation expressions.\n #\n ####################################################################\n\n my $sVersionRegex = qq(0x[0-9A-Fa-f]{8});\n my $sTypeRegex = qq((?:cvs|program|tar));\n\n ####################################################################\n #\n # Get Options.\n #\n ####################################################################\n\n my (%hOptions);\n\n if (!getopts('t:v:', \\%hOptions))\n {\n Usage($sProgram);\n }\n\n ####################################################################\n #\n # A type, '-t', is optional.\n #\n ####################################################################\n\n my $sType;\n\n $sType = (exists($hOptions{'t'})) ? $hOptions{'t'} : \"program\";\n\n if (defined($sType) && $sType !~ /^$sTypeRegex$/)\n {\n print STDERR \"$sProgram: Type='$sType' Error='Invalid version type.'\\n\";\n exit(2);\n }\n\n ####################################################################\n #\n # A version, '-v', is required.\n #\n ####################################################################\n\n my $sVersion = (exists($hOptions{'v'})) ? $hOptions{'v'} : undef;\n\n if (!defined($sVersion))\n {\n Usage($sProgram);\n }\n\n if ($sVersion !~ /^$sVersionRegex$/)\n {\n print STDERR \"$sProgram: Version='$sVersion' Error='Invalid version.'\\n\";\n exit(2);\n }\n\n ####################################################################\n #\n # If any arguments remain, it's an error.\n #\n ####################################################################\n\n if (scalar(@ARGV) > 0)\n {\n Usage($sProgram);\n }\n\n ####################################################################\n #\n # Do some work.\n #\n ####################################################################\n\n print VersionToString(hex($sVersion), $sType), \"\\n\";\n\n 1;\n\n######################################################################\n#\n# VersionToString\n#\n######################################################################\n\nsub VersionToString\n{\n my ($sVersion, $sType) = @_;\n\n my $sState = ($sVersion >> 10) & 0x03;\n my $sStateString = \"xx\";\n if ($sState == 0)\n {\n $sStateString = \"ds\";\n }\n elsif ($sState == 1)\n {\n $sStateString = \"rc\";\n }\n elsif ($sState == 2)\n {\n $sStateString = \"sr\";\n }\n elsif ($sState == 3)\n {\n $sStateString = \"xs\";\n }\n\n my $sString = \"\";\n if (($sVersion & 0xfff) == 0x800)\n {\n if ($sType =~ /^cvs$/)\n {\n $sString = sprintf\n (\n \"V%d_%d_%d\",\n ($sVersion >> 28) & 0x0f,\n ($sVersion >> 20) & 0xff,\n ($sVersion >> 12) & 0xff\n );\n }\n elsif ($sType =~ /^tar$/)\n {\n $sString = sprintf\n (\n \"%d.%d.%d\",\n ($sVersion >> 28) & 0x0f,\n ($sVersion >> 20) & 0xff,\n ($sVersion >> 12) & 0xff\n );\n }\n elsif ($sType =~ /^program$/)\n {\n $sString = sprintf\n (\n \"%d.%d.%d\",\n ($sVersion >> 28) & 0x0f,\n ($sVersion >> 20) & 0xff,\n ($sVersion >> 12) & 0xff\n );\n }\n }\n else\n {\n if ($sType =~ /^cvs$/)\n {\n $sString = sprintf\n (\n \"V%d_%d_%d_%s%d\",\n ($sVersion >> 28) & 0x0f,\n ($sVersion >> 20) & 0xff,\n ($sVersion >> 12) & 0xff,\n uc($sStateString),\n $sVersion & 0x3ff\n );\n }\n elsif ($sType =~ /^tar$/)\n {\n $sString = sprintf\n (\n \"%d.%d.%d.%s%d\",\n ($sVersion >> 28) & 0x0f,\n ($sVersion >> 20) & 0xff,\n ($sVersion >> 12) & 0xff,\n $sStateString,\n $sVersion & 0x3ff\n );\n }\n elsif ($sType =~ /^program$/)\n {\n $sString = sprintf\n (\n \"%d.%d.%d (%s%d)\",\n ($sVersion >> 28) & 0x0f,\n ($sVersion >> 20) & 0xff,\n ($sVersion >> 12) & 0xff,\n $sStateString,\n $sVersion & 0x3ff\n );\n }\n }\n\n return $sString;\n}\n\n\n######################################################################\n#\n# Usage\n#\n######################################################################\n\nsub Usage\n{\n my ($sProgram) = @_;\n print STDERR \"\\n\";\n print STDERR \"Usage: $sProgram [-t {cvs|program|tar}] -v version\\n\";\n print STDERR \"\\n\";\n exit(1);\n}\n" }, { "path": "utils/version_helper", "content": "#!/usr/bin/perl -w\n######################################################################\n#\n# $Id: 40c3c9381e39f6934a485d3cde86765789e61f42 $\n#\n######################################################################\n#\n# Copyright 2006-2013 The WebJob Project, All Rights Reserved.\n#\n######################################################################\n#\n# Purpose: Manage version numbers.\n#\n######################################################################\n\nuse strict;\nuse File::Basename;\nuse File::Path;\nuse Getopt::Std;\n\n######################################################################\n#\n# Main Routine\n#\n######################################################################\n\n ####################################################################\n #\n # Punch in and go to work.\n #\n ####################################################################\n\n my ($sProgram);\n\n $sProgram = basename(__FILE__);\n\n ####################################################################\n #\n # Validation expressions.\n #\n ####################################################################\n\n my $sBuildNumberRegex = qq((?:\\\\d+|[+]));\n my $sMajorNumberRegex = qq((?:\\\\d+|[+]));\n my $sMinorNumberRegex = qq((?:\\\\d+|[+]));\n my $sPatchNumberRegex = qq((?:\\\\d+|[+]));\n my $sStateNumberRegex = qq((?:[0-3+]|[dx]s|rc|sr));\n\n ####################################################################\n #\n # Get Options.\n #\n ####################################################################\n\n my (%hOptions);\n\n if (!getopts('b:f:M:m:p:s:', \\%hOptions))\n {\n Usage($sProgram);\n }\n\n ####################################################################\n #\n # A filename is required, and can be '-' or a regular file.\n #\n ####################################################################\n\n my ($sFileHandle, $sFilename);\n\n if (!exists($hOptions{'f'}))\n {\n Usage($sProgram);\n }\n else\n {\n $sFilename = $hOptions{'f'};\n if (!defined($sFilename) || length($sFilename) < 1)\n {\n Usage($sProgram);\n }\n if (-f $sFilename)\n {\n if (!open(FH, \"< $sFilename\"))\n {\n print STDERR \"$sProgram: File='$sFilename' Error='$!'\\n\";\n exit(2);\n }\n $sFileHandle = \\*FH;\n }\n else\n {\n if ($sFilename ne '-')\n {\n print STDERR \"$sProgram: File='$sFilename' Error='File must be regular.'\\n\";\n exit(2);\n }\n $sFileHandle = \\*STDIN;\n }\n }\n\n ####################################################################\n #\n # A MajorNumber, '-M', is optional.\n #\n ####################################################################\n\n my $sMajorNumber;\n\n $sMajorNumber = (exists($hOptions{'M'})) ? $hOptions{'M'} : undef;\n\n if (defined($sMajorNumber) && $sMajorNumber !~ /^$sMajorNumberRegex$/)\n {\n print STDERR \"$sProgram: MajorNumber='$sMajorNumber' Error='Invalid major number.'\\n\";\n exit(2);\n }\n\n ####################################################################\n #\n # A MinorNumber, '-m', is optional.\n #\n ####################################################################\n\n my $sMinorNumber;\n\n $sMinorNumber = (exists($hOptions{'m'})) ? $hOptions{'m'} : undef;\n\n if (defined($sMinorNumber) && $sMinorNumber !~ /^$sMinorNumberRegex$/)\n {\n print STDERR \"$sProgram: MinorNumber='$sMinorNumber' Error='Invalid minor number.'\\n\";\n exit(2);\n }\n\n ####################################################################\n #\n # An PatchNumber, '-p', is optional.\n #\n ####################################################################\n\n my $sPatchNumber;\n\n $sPatchNumber = (exists($hOptions{'p'})) ? $hOptions{'p'} : undef;\n\n if (defined($sPatchNumber) && $sPatchNumber !~ /^$sPatchNumberRegex$/)\n {\n print STDERR \"$sProgram: PatchNumber='$sPatchNumber' Error='Invalid patch number.'\\n\";\n exit(2);\n }\n\n ####################################################################\n #\n # A StateNumber, '-s', is optional.\n #\n ####################################################################\n\n my $sStateNumber;\n\n $sStateNumber = (exists($hOptions{'s'})) ? $hOptions{'s'} : undef;\n\n if (defined($sStateNumber) && $sStateNumber !~ /^$sStateNumberRegex$/)\n {\n print STDERR \"$sProgram: StateNumber='$sStateNumber' Error='Invalid state number.'\\n\";\n exit(2);\n }\n if (defined($sStateNumber) && $sStateNumber eq \"ds\")\n {\n $sStateNumber = 0;\n }\n elsif (defined($sStateNumber) && $sStateNumber eq \"rc\")\n {\n $sStateNumber = 1;\n }\n elsif (defined($sStateNumber) && $sStateNumber eq \"sr\")\n {\n $sStateNumber = 2;\n }\n elsif (defined($sStateNumber) && $sStateNumber eq \"xs\")\n {\n $sStateNumber = 3;\n }\n\n ####################################################################\n #\n # A BuildNumber, '-b', is optional.\n #\n ####################################################################\n\n my $sBuildNumber;\n\n $sBuildNumber = (exists($hOptions{'b'})) ? $hOptions{'b'} : undef;\n\n if (defined($sBuildNumber) && $sBuildNumber !~ /^$sBuildNumberRegex$/)\n {\n print STDERR \"$sProgram: BuildNumber='$sBuildNumber' Error='Invalid build number.'\\n\";\n exit(2);\n }\n\n ####################################################################\n #\n # If any arguments remain, it's an error.\n #\n ####################################################################\n\n if (scalar(@ARGV) > 0)\n {\n Usage($sProgram);\n }\n\n ####################################################################\n #\n # Attempt to locate/identify the current version number.\n #\n ####################################################################\n\n my ($sOldVersion, $sVersionFmt);\n\n while (my $sLine = <$sFileHandle>)\n {\n if ($sLine =~ /^#define VERSION (0x[0-9A-Fa-f]{8})\\s*/)\n {\n $sOldVersion = hex($1);\n $sVersionFmt = \"define\";\n last;\n }\n elsif ($sLine =~ /^\\s*(0x[0-9A-Fa-f]{8})\\s*$/)\n {\n $sOldVersion = hex($1);\n $sVersionFmt = \"string\";\n last;\n }\n elsif ($sLine =~ /^\\s*(?:version\\s+=\\s+)?(0x[0-9A-Fa-f]{8})\\s*$/)\n {\n $sOldVersion = hex($1);\n $sVersionFmt = \"assign\";\n last;\n }\n else\n {\n next;\n }\n }\n close($sFileHandle);\n\n if (!defined($sOldVersion))\n {\n print STDERR \"$sProgram: Error='Failed to locate/identify current version number.'\\n\";\n exit(2);\n }\n\n if (!defined($sVersionFmt))\n {\n print STDERR \"$sProgram: Error='Failed to determine version format.'\\n\";\n exit(2);\n }\n\n ####################################################################\n #\n # Compute the new version number.\n #\n ####################################################################\n\n my ($sNewVersion);\n\n $sNewVersion = $sOldVersion;\n\n if (defined($sMajorNumber))\n {\n if ($sMajorNumber =~ /^\\+$/)\n {\n $sNewVersion += 0x10000000;\n $sNewVersion &= 0xf0000000;\n }\n else\n {\n if ($sMajorNumber < 0 || $sMajorNumber > 15)\n {\n print STDERR \"$sProgram: MajorNumber='$sMajorNumber' Error='Invalid major number.'\\n\";\n exit(2);\n }\n $sNewVersion = (($sMajorNumber & 0xf) << 28) + ($sNewVersion & 0x0fffffff);\n }\n }\n\n if (defined($sMinorNumber))\n {\n if ($sMinorNumber =~ /^\\+$/)\n {\n $sNewVersion += 0x00100000;\n $sNewVersion &= 0xfff00000;\n }\n else\n {\n if ($sMinorNumber < 0 || $sMinorNumber > 255)\n {\n print STDERR \"$sProgram: MinorNumber='$sMinorNumber' Error='Invalid minor number.'\\n\";\n exit(2);\n }\n $sNewVersion = (($sMinorNumber & 0xff) << 20) + ($sNewVersion & 0xf00fffff);\n }\n }\n\n if (defined($sPatchNumber))\n {\n if ($sPatchNumber =~ /^\\+$/)\n {\n $sNewVersion += 0x00001000;\n $sNewVersion &= 0xfffff000;\n }\n else\n {\n if ($sPatchNumber < 0 || $sPatchNumber > 255)\n {\n print STDERR \"$sProgram: PatchNumber='$sPatchNumber' Error='Invalid patch number.'\\n\";\n exit(2);\n }\n $sNewVersion = (($sPatchNumber & 0xff) << 12) + ($sNewVersion & 0xfff00fff);\n }\n }\n\n if (defined($sStateNumber))\n {\n if ($sStateNumber =~ /^\\+$/)\n {\n $sNewVersion += 0x00000400;\n $sNewVersion &= 0xfffffc00;\n }\n else\n {\n if ($sStateNumber < 0 || $sStateNumber > 255)\n {\n print STDERR \"$sProgram: StateNumber='$sStateNumber' Error='Invalid state number.'\\n\";\n exit(2);\n }\n $sNewVersion = (($sStateNumber & 0x3) << 10) + ($sNewVersion & 0xfffff3ff);\n }\n }\n\n if (defined($sBuildNumber))\n {\n if ($sBuildNumber =~ /^\\+$/)\n {\n $sNewVersion += 0x00000001;\n }\n else\n {\n if ($sBuildNumber < 0 || $sBuildNumber > 255)\n {\n print STDERR \"$sProgram: BuildNumber='$sBuildNumber' Error='Invalid build number.'\\n\";\n exit(2);\n }\n $sNewVersion = ($sBuildNumber & 0x3ff) + ($sNewVersion & 0xfffffc00);\n }\n }\n\n ####################################################################\n #\n # Generate update/commit/tag commands the user can run manually.\n #\n ####################################################################\n\n my $sOldVersionString = VersionToString($sOldVersion, \"tar\");\n my $sNewVersionString = VersionToString($sNewVersion, \"tar\");\n my $so = sprintf(\"0x%08x\", $sOldVersion);\n my $sn = sprintf(\"0x%08x\", $sNewVersion);\n my $sCommand = \"perl -p -i.bak \";\n if ($sVersionFmt eq \"macro\")\n {\n $sCommand .= \" -e 's/define VERSION $so/define VERSION $sn/g;' $sFilename\";\n }\n else\n {\n $sCommand .= \" -e 's/$so/$sn/g;' $sFilename\";\n }\n print $sCommand, \"\\n\";\n $sCommand = \"cvs commit -m \\\"Updated version number ($sOldVersionString --> $sNewVersionString).\\\"\";\n print $sCommand, \"\\n\";\n $sCommand = \"cvs tag \" . VersionToString($sNewVersion, \"vcs\");\n print $sCommand, \"\\n\";\n if (((($sNewVersion >> 10) & 0x03) == 2) && (($sNewVersion & 0x3ff) == 0))\n {\n $sCommand = \"cvs tag \" . VersionToString($sNewVersion, \"vcs_sr0\");\n print $sCommand, \"\\n\";\n }\n\n 1;\n\n######################################################################\n#\n# VersionToString\n#\n######################################################################\n\nsub VersionToString\n{\n my ($sVersion, $sType) = @_;\n\n my $sState = ($sVersion >> 10) & 0x03;\n my $sStateString = \"xx\";\n if ($sState == 0)\n {\n $sStateString = \"ds\";\n }\n elsif ($sState == 1)\n {\n $sStateString = \"rc\";\n }\n elsif ($sState == 2)\n {\n $sStateString = \"sr\";\n }\n elsif ($sState == 3)\n {\n $sStateString = \"xs\";\n }\n\n my $sString = \"\";\n if ($sType =~ /^vcs$/)\n {\n $sString = sprintf\n (\n \"V%d_%d_%d_%s%d\",\n ($sVersion >> 28) & 0x0f,\n ($sVersion >> 20) & 0xff,\n ($sVersion >> 12) & 0xff,\n uc($sStateString),\n $sVersion & 0x3ff\n );\n }\n elsif ($sType =~ /^vcs_sr0$/)\n {\n $sString = sprintf\n (\n \"V%d_%d_%d\",\n ($sVersion >> 28) & 0x0f,\n ($sVersion >> 20) & 0xff,\n ($sVersion >> 12) & 0xff\n );\n }\n elsif ($sType =~ /^tar$/)\n {\n $sString = sprintf\n (\n \"%d.%d.%d.%s%d\",\n ($sVersion >> 28) & 0x0f,\n ($sVersion >> 20) & 0xff,\n ($sVersion >> 12) & 0xff,\n $sStateString,\n $sVersion & 0x3ff\n );\n }\n elsif ($sType =~ /^program$/)\n {\n $sString = sprintf\n (\n \"%d.%d.%d (%s%d)\",\n ($sVersion >> 28) & 0x0f,\n ($sVersion >> 20) & 0xff,\n ($sVersion >> 12) & 0xff,\n $sStateString,\n $sVersion & 0x3ff\n );\n }\n\n return $sString;\n}\n\n\n######################################################################\n#\n# Usage\n#\n######################################################################\n\nsub Usage\n{\n my ($sProgram) = @_;\n print STDERR \"\\n\";\n print STDERR \"Usage: $sProgram [-M major] [-m minor] [-p patch] [-s state] [-b build] -f {file|-}\\n\";\n print STDERR \"\\n\";\n exit(1);\n}\n" } ]