[ { "path": ".dockerignore", "content": ".git/\n\nbuild\ndist\n*.egg-info\n*.egg/\n*.pyc\n*.swp\n\nvenv/\ncache/\nlog/\nspiderfoot.db*\nspiderfoot.test.db\n\n__pycache__\nhtmlcov/\ndocs/\n" }, { "path": ".github/ISSUE_TEMPLATE.md", "content": "Please provide a description of the issue and any relevant error messages.\n\nIf you can provide an application stack trace that is even better.\n\nWhat version of Python are you using?\n\nWhat version of SpiderFoot are you using (stable release or Git `master` branch)?\n\nYou may also wish to check if your issue has been posted previously:\n\n* https://github.com/smicallef/spiderfoot/issues\n\n" }, { "path": ".github/workflows/codeql-analysis.yml", "content": "name: \"CodeQL\"\n\non:\n push:\n branches:\n - master\n schedule:\n - cron: \"26 16 * * 0\"\n\njobs:\n analyze:\n name: Analyze\n runs-on: ubuntu-latest\n strategy:\n fail-fast: true\n matrix:\n language: [\"python\"]\n steps:\n - name: Checkout repository\n uses: actions/checkout@v2\n - name: Initialize CodeQL\n uses: github/codeql-action/init@v1\n with:\n languages: ${{ matrix.language }}\n - name: Autobuild\n uses: github/codeql-action/autobuild@v1\n - name: Perform CodeQL Analysis\n uses: github/codeql-action/analyze@v1\n" }, { "path": ".github/workflows/tests.yaml", "content": "name: Tests\n\non:\n push:\n branches: master\n pull_request:\n branches: master\n\njobs:\n build:\n\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n python-version: [\"3.7\", \"3.8\", \"3.9\", \"3.10\"]\n os: [ubuntu-latest, macos-latest]\n\n steps:\n - uses: actions/checkout@v2\n - name: Set up Python ${{ matrix.python-version }}\n uses: actions/setup-python@v2\n with:\n python-version: ${{ matrix.python-version }}\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n pip install flake8 pytest\n pip install -r test/requirements.txt\n - name: Lint with flake8\n run: |\n # stop the build if there are Python Flake8 violations\n flake8 . --count --show-source --statistics\n - name: Test with pytest\n run: |\n python -m pytest -n auto --dist loadfile --ignore=test/integration/modules/ --durations=5 --cov-report term --cov=. .\n - uses: codecov/codecov-action@v1\n with:\n fail_ci_if_error: false\n verbose: true\n" }, { "path": ".gitignore", "content": "# SpiderFoot specific #\n#######################\n/cache/\n/log/\n/spiderfoot.db*\npasswd\nspiderfoot.crt\nspiderfoot.key\n\nspiderfoot/static/node_modules/*\n\nspiderfoot/static/node_modules/alertifyjs/*\nspiderfoot/static/node_modules/alertifyjs/build/*\nspiderfoot/static/node_modules/alertifyjs/build/css/*\n\n!spiderfoot/static/node_modules/alertifyjs/\n!spiderfoot/static/node_modules/alertifyjs/build/\n!spiderfoot/static/node_modules/alertifyjs/build/alertify.min.js\n!spiderfoot/static/node_modules/alertifyjs/build/css/\n!spiderfoot/static/node_modules/alertifyjs/build/css/alertify.min.css\n\nspiderfoot/static/node_modules/bootstrap/*\nspiderfoot/static/node_modules/bootstrap/dist/*\nspiderfoot/static/node_modules/bootstrap/dist/js/*\nspiderfoot/static/node_modules/bootstrap/dist/css/*\nspiderfoot/static/node_modules/bootstrap/dist/fonts/*\n\n!spiderfoot/static/node_modules/bootstrap/\n!spiderfoot/static/node_modules/bootstrap/dist/\n!spiderfoot/static/node_modules/bootstrap/dist/js/\n!spiderfoot/static/node_modules/bootstrap/dist/js/bootstrap.min.js\n!spiderfoot/static/node_modules/bootstrap/dist/css/\n!spiderfoot/static/node_modules/bootstrap/dist/css/bootstrap.min.css\n!spiderfoot/static/node_modules/bootstrap/dist/fonts/\n!spiderfoot/static/node_modules/bootstrap/dist/fonts/glyphicons-halflings-regular.woff\n!spiderfoot/static/node_modules/bootstrap/dist/fonts/glyphicons-halflings-regular.woff2\n\nspiderfoot/static/node_modules/jquery/*\nspiderfoot/static/node_modules/jquery/dist/*\n\n!spiderfoot/static/node_modules/jquery/\n!spiderfoot/static/node_modules/jquery/dist/\n!spiderfoot/static/node_modules/jquery/dist/jquery.min.js\n\nspiderfoot/static/node_modules/sigma/*\nspiderfoot/static/node_modules/sigma/build/*\nspiderfoot/static/node_modules/sigma/build/plugins/*\n\n!spiderfoot/static/node_modules/sigma/\n!spiderfoot/static/node_modules/sigma/build/\n!spiderfoot/static/node_modules/sigma/build/sigma.min.js\n!spiderfoot/static/node_modules/sigma/build/plugins/\n!spiderfoot/static/node_modules/sigma/build/plugins/sigma.parsers.json.min.js\n!spiderfoot/static/node_modules/sigma/build/plugins/sigma.plugins.dragNodes.min.js\n!spiderfoot/static/node_modules/sigma/build/plugins/sigma.layout.forceAtlas2.min.js\n!spiderfoot/static/node_modules/sigma/build/plugins/sigma.renderers.snapshot.min.js\n\nspiderfoot/static/node_modules/tablesorter/*\nspiderfoot/static/node_modules/tablesorter/dist/*\nspiderfoot/static/node_modules/tablesorter/dist/js/*\nspiderfoot/static/node_modules/tablesorter/dist/css/*\nspiderfoot/static/node_modules/tablesorter/dist/js/extras/*\n\n!spiderfoot/static/node_modules/tablesorter/\n!spiderfoot/static/node_modules/tablesorter/dist/\n!spiderfoot/static/node_modules/tablesorter/dist/css/\n!spiderfoot/static/node_modules/tablesorter/dist/css/jquery.tablesorter.pager.min.css\n!spiderfoot/static/node_modules/tablesorter/dist/css/theme.default.min.css\n!spiderfoot/static/node_modules/tablesorter/dist/js/\n!spiderfoot/static/node_modules/tablesorter/dist/js/jquery.tablesorter.min.js\n!spiderfoot/static/node_modules/tablesorter/dist/js/jquery.tablesorter.widgets.min.js\n!spiderfoot/static/node_modules/tablesorter/dist/js/extras/\n!spiderfoot/static/node_modules/tablesorter/dist/js/extras/jquery.tablesorter.pager.min.js\n\nspiderfoot/static/node_modules/d3/*\n\n!spiderfoot/static/node_modules/d3/\n!spiderfoot/static/node_modules/d3/d3.min.js\n\n# Compiled source #\n###################\n__pycache__/\n*.com\n*.class\n*.dll\n*.exe\n*.o\n*.so\n*.py[cod]\n*$py.class\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# Packages #\n############\n# it's better to unpack these files and commit the raw source\n# git has its own built in compression methods\n*.7z\n*.dmg\n*.gz\n*.iso\n*.jar\n*.rar\n*.tar\n*.zip\n\n# Logs and databases #\n######################\n*.log\n*.sqlite\n*.db\n\n# OS generated files #\n######################\n.DS_Store\n.DS_Store?\n._*\n.Spotlight-V100\n.Trashes\n.*~\nIcon?\nehthumbs.db\nThumbs.db\n\n# Virtual environments #\n########################\nvenv/\nvenv3/\n\n# IDE #\n#######\n.idea\n.vscode\n\n# Test & Coverage Reports #\n###########################\n.coverage\ncoverage/\ntest/acceptance/results/\nhtmlcov/\nreport/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\ncover/\n\n# Documentation #\n#################\ndocs/_build/\n" }, { "path": ".pylintrc", "content": "[MASTER]\n\n# A comma-separated list of package or module names from where C extensions may\n# be loaded. Extensions are loading into the active Python interpreter and may\n# run arbitrary code\nextension-pkg-whitelist=\n\n# Add files or directories to the blacklist. They should be base names, not\n# paths.\nignore=CVS\n\n# Add files or directories matching the regex patterns to the blacklist. The\n# regex matches against base names, not paths.\nignore-patterns=\n\n# Python code to execute, usually for sys.path manipulation such as\n# pygtk.require().\n#init-hook=\n\n# Use multiple processes to speed up Pylint.\njobs=4\n\n# List of plugins (as comma separated values of python modules names) to load,\n# usually to register additional checkers.\nload-plugins=\n\n# Pickle collected data for later comparisons.\npersistent=yes\n\n# Specify a configuration file.\n#rcfile=\n\n# When enabled, pylint would attempt to guess common misconfiguration and emit\n# user-friendly hints instead of false-positive error messages\nsuggestion-mode=yes\n\n# Allow loading of arbitrary C extensions. Extensions are imported into the\n# active Python interpreter and may run arbitrary code.\nunsafe-load-any-extension=no\n\n\n[MESSAGES CONTROL]\n\n# Only show warnings with the listed confidence levels. Leave empty to show\n# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED\nconfidence=\n\n# Disable the message, report, category or checker with the given id(s). You\n# can either give multiple identifiers separated by comma (,) or put this\n# option multiple times (only on the command line, not in the configuration\n# file where it should appear only once).You can also use \"--disable=all\" to\n# disable everything first and then reenable specific checks. For example, if\n# you want to run only the similarities checker, you can use \"--disable=all\n# --enable=similarities\". If you want to run only the classes checker, but have\n# no Warning level messages displayed, use\"--disable=all --enable=classes\n# --disable=W\"\ndisable=invalid-name,\n no-self-use,\n too-few-public-methods,\n too-many-ancestors,\n too-many-arguments,\n too-many-boolean-expressions,\n too-many-branches,\n too-many-instance-attributes,\n too-many-locals,\n too-many-nested-blocks,\n too-many-public-methods,\n too-many-return-statements,\n too-many-statements,\n\n# Enable the message, report, category or checker with the given id(s). You can\n# either give multiple identifier separated by comma (,) or put this option\n# multiple time (only on the command line, not in the configuration file where\n# it should appear only once). See also the \"--disable\" option for examples.\nenable=c-extension-no-member\n\n\n[REPORTS]\n\n# Python expression which should return a note less than 10 (10 is the highest\n# note). You have access to the variables errors warning, statement which\n# respectively contain the number of errors / warnings messages and the total\n# number of statements analyzed. This is used by the global evaluation report\n# (RP0004).\nevaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)\n\n# Template used to display messages. This is a python new-style format string\n# used to format the message information. See doc for all details\n#msg-template=\n\n# Set the output format. Available formats are text, parseable, colorized, json\n# and msvs (visual studio).You can also give a reporter class, eg\n# mypackage.mymodule.MyReporterClass.\n#output-format=text\noutput-format=parseable\n\n# Tells whether to display a full report or only the messages\nreports=no\n\n# Activate the evaluation score.\nscore=yes\n\n\n[REFACTORING]\n\n# Maximum number of nested blocks for function / method body\nmax-nested-blocks=5\n\n# Complete name of functions that never returns. When checking for\n# inconsistent-return-statements if a never returning function is called then\n# it will be considered as an explicit return statement and no message will be\n# printed.\nnever-returning-functions=optparse.Values,sys.exit\n\n\n[BASIC]\n\n# Regular expression matching correct argument names. Overrides argument-\n# naming-style\nargument-rgx=^[a-z][a-z0-9_]*$\n\n# Regular expression matching correct attribute names. Overrides attr-naming-\n# style\nattr-rgx=^_{0,2}[a-z][a-z0-9_]*$\n\n# Bad variable names which should always be refused, separated by a comma\nbad-names=\n\n# Regular expression matching correct class attribute names. Overrides class-\n# attribute-naming-style\nclass-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$\n\n# Regular expression matching correct class names. Overrides class-naming-style\nclass-rgx=^_?[A-Z][a-zA-Z0-9]*$\n\n# Naming style matching correct constant names\nconst-naming-style=UPPER_CASE\n\n# Minimum line length for functions/classes that require docstrings, shorter\n# ones are exempt.\ndocstring-min-length=-1\n\n# Regular expression matching correct function names. Overrides function-\n# naming-style\nfunction-rgx=^(?:(?PsetUp|tearDown|setUpModule|tearDownModule)|(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$\n\n\n# Good variable names which should always be accepted, separated by a comma\ngood-names=main,\n sf,\n _\n\n# Include a hint for the correct naming format with invalid-name\ninclude-naming-hint=no\n\n# Regular expression matching correct inline iteration names. Overrides\n# inlinevar-naming-style\ninlinevar-rgx=^[a-z][a-z0-9_]*$\n\n# Regular expression matching correct method names. Overrides method-naming-\n# style\nmethod-rgx=(?x)^(?:(?P_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P_{0,2}[a-z][a-z0-9_]*))$\n\n# Regular expression matching correct module names. Overrides module-naming-\n# style\nmodule-rgx=^(_?[a-z][a-z0-9_]*|__init__)$\n\n# Colon-delimited sets of names that determine each other's naming style when\n# the name regexes allow several styles.\nname-group=\n\n# Regular expression which should only match function or class names that do\n# not require a docstring.\nno-docstring-rgx=^_\n\n# List of decorators that produce properties, such as abc.abstractproperty. Add\n# to this list to register other decorators that produce valid properties.\nproperty-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl\n\n# Regular expression matching correct variable names. Overrides variable-\n# naming-style\nvariable-rgx=^[a-z][a-z0-9_]*$\n\n\n[SIMILARITIES]\n\n# Ignore comments when computing similarities.\nignore-comments=yes\n\n# Ignore docstrings when computing similarities.\nignore-docstrings=yes\n\n# Ignore imports when computing similarities.\nignore-imports=no\n\n# Minimum lines number of a similarity.\nmin-similarity-lines=4\n\n\n[SPELLING]\n\n# Limits count of emitted suggestions for spelling mistakes\nmax-spelling-suggestions=4\n\n# Spelling dictionary name. Available dictionaries: en_US (myspell), en\n# (aspell), en_AU (aspell), en_CA (aspell), en_GB (aspell).\nspelling-dict=\n\n# List of comma separated words that should not be checked.\nspelling-ignore-words=\n\n# A path to a file that contains private dictionary; one word per line.\nspelling-private-dict-file=\n\n# Tells whether to store unknown words to indicated private dictionary in\n# --spelling-private-dict-file option instead of raising a message.\nspelling-store-unknown-words=no\n\n\n[LOGGING]\n\n# Logging modules to check that the string format arguments are in logging\n# function parameter format\nlogging-modules=logging\n\n\n[FORMAT]\n\n# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.\nexpected-line-ending-format=\n\n# Regexp for a line that is allowed to be longer than the limit.\nignore-long-lines=(?x)(\n ^\\s*(\\#\\ )??$|\n ^\\s*(from\\s+\\S+\\s+)?import\\s+.+$)\n\n# Number of spaces of indent required inside a hanging or continued line.\nindent-after-paren=4\n\n# String used as indentation unit. This is usually \" \" (4 spaces) or \"\\t\" (1\n# tab).\nindent-string=' '\n\n# Maximum number of characters on a single line.\nmax-line-length=120\n\n# Maximum number of lines in a module\nmax-module-lines=1000\n\n# List of optional constructs for which whitespace checking is disabled. `dict-\n# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\\n222: 2}.\n# `trailing-comma` allows a space between comma and closing bracket: (a, ).\n# `empty-line` allows space-only lines.\nno-space-check=trailing-comma\n\n# Allow the body of a class to be on the same line as the declaration if body\n# contains single statement.\nsingle-line-class-stmt=no\n\n# Allow the body of an if to be on the same line as the test if there is no\n# else.\nsingle-line-if-stmt=no\n\n\n[MISCELLANEOUS]\n\n# List of note tags to take in consideration, separated by a comma.\nnotes=FIXME,\n XXX,\n TODO\n\n\n[TYPECHECK]\n\n# List of decorators that produce context managers, such as\n# contextlib.contextmanager. Add to this list to register other decorators that\n# produce valid context managers.\ncontextmanager-decorators=contextlib.contextmanager\n\n# List of members which are set dynamically and missed by pylint inference\n# system, and so shouldn't trigger E1101 when accessed. Python regular\n# expressions are accepted.\ngenerated-members=\n\n# Tells whether missing members accessed in mixin class should be ignored. A\n# mixin class is detected if its name ends with \"mixin\" (case insensitive).\nignore-mixin-members=yes\n\n# This flag controls whether pylint should warn about no-member and similar\n# checks whenever an opaque object is returned when inferring. The inference\n# can return multiple potential results while evaluating a Python object, but\n# some branches might not be evaluated, which results in partial inference. In\n# that case, it might be useful to still emit no-member and other checks for\n# the rest of the inferred objects.\nignore-on-opaque-inference=yes\n\n# List of class names for which member attributes should not be checked (useful\n# for classes with dynamically set attributes). This supports the use of\n# qualified names.\nignored-classes=optparse.Values,thread._local,_thread._local\n\n# List of module names for which member attributes should not be checked\n# (useful for modules/projects where namespaces are manipulated during runtime\n# and thus existing member attributes cannot be deduced by static analysis. It\n# supports qualified module names, as well as Unix pattern matching.\nignored-modules=\n\n# Show a hint with possible names when a member name was not found. The aspect\n# of finding the hint is based on edit distance.\nmissing-member-hint=yes\n\n# The minimum edit distance a name should have in order to be considered a\n# similar match for a missing member name.\nmissing-member-hint-distance=1\n\n# The total number of similar names that should be taken in consideration when\n# showing a hint for a missing member.\nmissing-member-max-choices=1\n\n\n[VARIABLES]\n\n# List of additional names supposed to be defined in builtins. Remember that\n# you should avoid to define new builtins when possible.\nadditional-builtins=\n\n# Tells whether unused global variables should be treated as a violation.\nallow-global-unused-variables=yes\n\n# List of strings which can identify a callback function by name. A callback\n# name must start or end with one of those strings.\ncallbacks=cb_,\n _cb\n\n# A regular expression matching the name of dummy variables (i.e. expectedly\n# not used).\ndummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_\n\n# Argument names that match this expression will be ignored. Default to name\n# with leading underscore\nignored-argument-names=_.*|^ignored_|^unused_\n\n# Tells whether we should check for unused import in __init__ files.\ninit-import=no\n\n# List of qualified module names which can have objects that can redefine\n# builtins.\nredefining-builtins-modules=six.moves,past.builtins,future.builtins,io,builtins\n\n\n[DESIGN]\n\n# Maximum number of arguments for function / method\nmax-args=5\n\n# Maximum number of attributes for a class (see R0902).\nmax-attributes=7\n\n# Maximum number of boolean expressions in a if statement\nmax-bool-expr=5\n\n# Maximum number of branch for function / method body\nmax-branches=12\n\n# Maximum number of locals for function / method body\nmax-locals=15\n\n# Maximum number of parents for a class (see R0901).\nmax-parents=7\n\n# Maximum number of public methods for a class (see R0904).\nmax-public-methods=20\n\n# Maximum number of return / yield for function / method body\nmax-returns=6\n\n# Maximum number of statements in function / method body\nmax-statements=50\n\n# Minimum number of public methods for a class (see R0903).\nmin-public-methods=2\n\n\n[CLASSES]\n\n# List of method names used to declare (i.e. assign) instance attributes.\ndefining-attr-methods=__init__,\n __new__,\n setUp\n\n# List of member names, which should be excluded from the protected access\n# warning.\nexclude-protected=_asdict,\n _fields,\n _replace,\n _source,\n _make\n\n# List of valid names for the first argument in a class method.\nvalid-classmethod-first-arg=cls,\n class_\n\n# List of valid names for the first argument in a metaclass class method.\nvalid-metaclass-classmethod-first-arg=mcs\n\n\n[IMPORTS]\n\n# Allow wildcard imports from modules that define __all__.\nallow-wildcard-with-all=no\n\n# Analyse import fallback blocks. This can be used to support both Python 2 and\n# 3 compatible code, which means that the block might have code that exists\n# only in one or another interpreter, leading to false positives when analysed.\nanalyse-fallback-blocks=no\n\n# Deprecated modules which should not be used, separated by a comma\ndeprecated-modules=regsub,\n TERMIOS,\n Bastion,\n rexec,\n sets\n\n# Create a graph of external dependencies in the given file (report RP0402 must\n# not be disabled)\next-import-graph=\n\n# Create a graph of every (i.e. internal and external) dependencies in the\n# given file (report RP0402 must not be disabled)\nimport-graph=\n\n# Create a graph of internal dependencies in the given file (report RP0402 must\n# not be disabled)\nint-import-graph=\n\n# Force import order to recognize a module as part of the standard\n# compatibility libraries.\nknown-standard-library=\n\n# Force import order to recognize a module as part of a third party library.\nknown-third-party=enchant\n\n\n[EXCEPTIONS]\n\n# Exceptions that will emit a warning when being caught. Defaults to\n# \"Exception\"\novergeneral-exceptions=Exception,\n BaseException,\n" }, { "path": "Dockerfile", "content": "#\n# Spiderfoot Dockerfile\n#\n# http://www.spiderfoot.net\n#\n# Written by: Michael Pellon \n# Updated by: Chandrapal \n# Updated by: Steve Micallef \n# Updated by: Steve Bate \n# -> Inspired by https://github.com/combro2k/dockerfiles/tree/master/alpine-spiderfoot\n#\n# Usage:\n#\n# sudo docker build -t spiderfoot .\n# sudo docker run -p 5001:5001 --security-opt no-new-privileges spiderfoot\n#\n# Using Docker volume for spiderfoot data\n#\n# sudo docker run -p 5001:5001 -v /mydir/spiderfoot:/var/lib/spiderfoot spiderfoot\n#\n# Using SpiderFoot remote command line with web server\n#\n# docker run --rm -it spiderfoot sfcli.py -s http://my.spiderfoot.host:5001/\n#\n# Running spiderfoot commands without web server (can optionally specify volume)\n#\n# sudo docker run --rm spiderfoot sf.py -h\n#\n# Running a shell in the container for maintenance\n# sudo docker run -it --entrypoint /bin/sh spiderfoot\n#\n# Running spiderfoot unit tests in container\n#\n# sudo docker build -t spiderfoot-test --build-arg REQUIREMENTS=test/requirements.txt .\n# sudo docker run --rm spiderfoot-test -m pytest --flake8 .\n\nFROM alpine:3.12.4 AS build\nARG REQUIREMENTS=requirements.txt\nRUN apk add --no-cache gcc git curl python3 python3-dev py3-pip swig tinyxml-dev \\\n python3-dev musl-dev openssl-dev libffi-dev libxslt-dev libxml2-dev jpeg-dev \\\n openjpeg-dev zlib-dev cargo rust\nRUN python3 -m venv /opt/venv\nENV PATH=\"/opt/venv/bin\":$PATH\nCOPY $REQUIREMENTS requirements.txt ./\nRUN ls\nRUN echo \"$REQUIREMENTS\"\nRUN pip3 install -U pip\nRUN pip3 install -r \"$REQUIREMENTS\"\n\n\n\nFROM alpine:3.13.0\nWORKDIR /home/spiderfoot\n\n# Place database and logs outside installation directory\nENV SPIDERFOOT_DATA /var/lib/spiderfoot\nENV SPIDERFOOT_LOGS /var/lib/spiderfoot/log\nENV SPIDERFOOT_CACHE /var/lib/spiderfoot/cache\n\n# Run everything as one command so that only one layer is created\nRUN apk --update --no-cache add python3 musl openssl libxslt tinyxml libxml2 jpeg zlib openjpeg \\\n && addgroup spiderfoot \\\n && adduser -G spiderfoot -h /home/spiderfoot -s /sbin/nologin \\\n -g \"SpiderFoot User\" -D spiderfoot \\\n && rm -rf /var/cache/apk/* \\\n && rm -rf /lib/apk/db \\\n && rm -rf /root/.cache \\\n && mkdir -p $SPIDERFOOT_DATA || true \\\n && mkdir -p $SPIDERFOOT_LOGS || true \\\n && mkdir -p $SPIDERFOOT_CACHE || true \\\n && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA \\\n && chown spiderfoot:spiderfoot $SPIDERFOOT_LOGS \\\n && chown spiderfoot:spiderfoot $SPIDERFOOT_CACHE\n\nCOPY . .\nCOPY --from=build /opt/venv /opt/venv\nENV PATH=\"/opt/venv/bin:$PATH\"\n\nUSER spiderfoot\n\nEXPOSE 5001\n\n# Run the application.\nENTRYPOINT [\"/opt/venv/bin/python\"]\nCMD [\"sf.py\", \"-l\", \"0.0.0.0:5001\"]\n" }, { "path": "Dockerfile.full", "content": "#\n# Spiderfoot Dockerfile (Full - includes all CLI tools, etc.)\n#\n# http://www.spiderfoot.net\n#\n# Written by: TheTechromancer\n#\n\nFROM python:3\n\n# Install tools/dependencies from apt\nRUN apt-get -y update && apt-get -y install nbtscan onesixtyone nmap\n\n# Compile other tools from source\nRUN mkdir /tools || true\nWORKDIR /tools\n\n# Install Golang tools\nRUN apt-get -y update && apt-get -y install golang\nENV GOPATH=\"/go\"\nENV PATH=\"$GOPATH/bin:$PATH\"\nRUN mkdir -p \"$GOPATH/src\" \"$GOPATH/bin\"\n\n# Install Ruby tools for WhatWeb\nRUN apt-get -y update && apt-get -y install ruby ruby-dev bundler\n# Install WhatWeb\nRUN git clone https://github.com/urbanadventurer/WhatWeb \\\n && gem install rchardet mongo json && cd /tools/WhatWeb \\\n && bundle install && cd /tools\n\nRUN groupadd spiderfoot \\\n && useradd -m -g spiderfoot -d /home/spiderfoot -s /sbin/nologin \\\n -c \"SpiderFoot User\" spiderfoot\n\n# Install RetireJS\nRUN apt remove -y cmdtest \\\n && apt remove -y yarn \\\n && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \\\n && echo 'deb https://dl.yarnpkg.com/debian/ stable main' |tee /etc/apt/sources.list.d/yarn.list \\\n && apt-get update \\\n && apt-get install yarn -y \\\n && yarn install \\\n && curl -fsSL https://deb.nodesource.com/setup_17.x | bash - \\\n && apt-get install -y nodejs \\\n && npm install -g retire\n\n# Install Google Chrome the New Way (Not via apt-key)\nRUN wget -qO - https://dl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/googlechrome-linux-keyring.gpg \\\n && echo \"deb [arch=amd64 signed-by=/usr/share/keyrings/googlechrome-linux-keyring.gpg] http://dl.google.com/linux/chrome/deb/ stable main\" | tee /etc/apt/sources.list.d/google-chrome.list \\\n && apt -y update && apt install --allow-unauthenticated -y google-chrome-stable\n\n# Install Wappalyzer\nRUN git clone https://github.com/AliasIO/wappalyzer.git \\\n && cd wappalyzer \\\n && yarn install && yarn run link\n\n# Install Nuclei\nRUN wget https://github.com/projectdiscovery/nuclei/releases/download/v2.6.5/nuclei_2.6.5_linux_amd64.zip \\\n && unzip nuclei_2.6.5_linux_amd64.zip \\\n && git clone https://github.com/projectdiscovery/nuclei-templates.git\n\n# Install testssl.sh\nRUN apt-get install -y bsdmainutils dnsutils coreutils\nRUN git clone https://github.com/drwetter/testssl.sh.git\n\n# Install Snallygaster and TruffleHog\nRUN pip3 install snallygaster trufflehog\n\n# Place database and logs outside installation directory\nENV SPIDERFOOT_DATA /var/lib/spiderfoot\nENV SPIDERFOOT_LOGS /var/lib/spiderfoot/log\nENV SPIDERFOOT_CACHE /var/lib/spiderfoot/cache\n\nRUN mkdir -p $SPIDERFOOT_DATA || true \\\n && mkdir -p $SPIDERFOOT_LOGS || true \\\n && mkdir -p $SPIDERFOOT_CACHE || true \\\n && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA \\\n && chown spiderfoot:spiderfoot $SPIDERFOOT_LOGS \\\n && chown spiderfoot:spiderfoot $SPIDERFOOT_CACHE\n\nWORKDIR /home/spiderfoot\nCOPY . .\n\nENV VIRTUAL_ENV=/opt/venv\nRUN mkdir -p \"$VIRTUAL_ENV\" || true\nENV PATH=\"$VIRTUAL_ENV/bin:$PATH\"\nRUN python -m venv \"$VIRTUAL_ENV\"\n\nARG REQUIREMENTS=requirements.txt\nCOPY \"$REQUIREMENTS\" requirements.txt\n\nRUN chown -R spiderfoot:spiderfoot /tools\nRUN chown -R spiderfoot:spiderfoot \"$VIRTUAL_ENV\"\nRUN chown -R spiderfoot:spiderfoot \"/home/spiderfoot\"\n\nUSER spiderfoot\n\nRUN pip install -U pip\nRUN pip install -r \"$REQUIREMENTS\"\n\n# Install Python tools\nRUN pip install dnstwist\n# CMSeeK\nWORKDIR /tools\nRUN git clone https://github.com/Tuhinshubhra/CMSeeK && cd CMSeeK \\\n && pip install -r requirements.txt && mkdir Results\n\n# Install wafw00f\nRUN git clone https://github.com/EnableSecurity/wafw00f \\\n && cd wafw00f \\\n && python3 setup.py install\nWORKDIR /home/spiderfoot\n\nEXPOSE 5001\n\n# Run the application\nCMD python -c 'from spiderfoot import SpiderFootDb; \\\ndb = SpiderFootDb({\"__database\": \"/var/lib/spiderfoot/spiderfoot.db\"}); \\\ndb.configSet({ \\\n \"sfp_tool_dnstwist:dnstwistpath\": \"/opt/venv/bin/dnstwist\", \\\n \"sfp_tool_cmseek:cmseekpath\": \"/tools/CMSeeK/cmseek.py\", \\\n \"sfp_tool_whatweb:whatweb_path\": \"/tools/WhatWeb/whatweb\", \\\n \"sfp_tool_wafw00f:wafw00f_path\": \"/opt/venv/bin/wafw00f\", \\\n \"sfp_tool_onesixtyone:onesixtyone_path\": \"/usr/bin/onesixtyone\", \\\n \"sfp_tool_retirejs:retirejs_path\": \"/usr/bin/retire\", \\\n \"sfp_tool_testsslsh:testsslsh_path\": \"/tools/testssl.sh/testssl.sh\", \\\n \"sfp_tool_snallygaster:snallygaster_path\": \"/usr/local/bin/snallygaster\", \\\n \"sfp_tool_trufflehog:trufflehog_path\": \"/usr/local/bin/trufflehog\", \\\n \"sfp_tool_nuclei:nuclei_path\": \"/tools/nuclei\", \\\n \"sfp_tool_nuclei:template_path\": \"/tools/nuclei-templates\", \\\n \"sfp_tool_wappalyzer:wappalyzer_path\": \"/tools/wappalyzer/src/drivers/npm/cli.js\", \\\n \"sfp_tool_nbtscan:nbtscan_path\": \"/usr/bin/nbtscan\", \\\n \"sfp_tool_nmap:nmappath\": \"DISABLED_BECAUSE_NMAP_REQUIRES_ROOT_TO_WORK\" \\\n})' || true && ./sf.py -l 0.0.0.0:5001\n" }, { "path": "LICENSE", "content": "Copyright 2022 Steve Micallef \n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, { "path": "README.md", "content": "\n\n\n[![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/smicallef/spiderfoot/master/LICENSE)\n[![Python Version](https://img.shields.io/badge/python-3.7+-green)](https://www.python.org)\n[![Stable Release](https://img.shields.io/badge/version-4.0-blue.svg)](https://github.com/smicallef/spiderfoot/releases/tag/v4.0)\n[![CI status](https://github.com/smicallef/spiderfoot/workflows/Tests/badge.svg)](https://github.com/smicallef/spiderfoot/actions?query=workflow%3A\"Tests\")\n[![Last Commit](https://img.shields.io/github/last-commit/smicallef/spiderfoot)](https://github.com/smicallef/spiderfoot/commits/master)\n[![Codecov](https://codecov.io/github/smicallef/spiderfoot/coverage.svg)](https://codecov.io/github/smicallef/spiderfoot)\n[![Twitter Follow](https://img.shields.io/twitter/follow/spiderfoot?label=follow&style=social)](https://twitter.com/spiderfoot)\n[![Discord](https://img.shields.io/discord/770524432464216074)](https://discord.gg/vyvztrG)\n\n**SpiderFoot** is an open source intelligence (OSINT) automation tool. It integrates with just about every data source available and utilises a range of methods for data analysis, making that data easy to navigate. \n\nSpiderFoot has an embedded web-server for providing a clean and intuitive web-based interface but can also be used completely via the command-line. It's written in **Python 3** and **MIT-licensed**.\n\n\n\n### FEATURES\n\n- Web based UI or CLI\n- Over 200 modules (see below)\n- Python 3.7+\n- YAML-configurable [correlation engine](/correlations/README.md) with [37 pre-defined rules](/correlations)\n- CSV/JSON/GEXF export\n- API key export/import\n- SQLite back-end for custom querying\n- Highly configurable\n- Fully documented\n- Visualisations\n- TOR integration for dark web searching\n- Dockerfile for Docker-based deployments\n- Can call other tools like DNSTwist, Whatweb, Nmap and CMSeeK\n- [Actively developed since 2012!](https://medium.com/@micallst/lessons-learned-from-my-10-year-open-source-project-4a4c8c2b4f64)\n\n### WANT MORE?\n\nNeed more from SpiderFoot? Check out [SpiderFoot HX](https://www.spiderfoot.net/hx) for:\n- 100% Cloud-based and managed for you\n- Attack Surface Monitoring with change notifications by email, REST and Slack\n- Multiple targets per scan\n- Multi-user collaboration\n- Authenticated and 2FA\n- Investigations\n- Customer support\n- Third party tools pre-installed & configured\n- Drive it with a fully RESTful API\n- TOR integration built-in\n- Screenshotting\n- Bring your own Python SpiderFoot modules\n- Feed scan data to Splunk, ElasticSearch and REST endpoints\n\nSee the full set of differences between SpiderFoot HX and the open source version [here](https://www.spiderfoot.net/open-source-vs-hx/).\n\n### USES\n\nSpiderFoot can be used offensively (e.g. in a red team exercise or penetration test) for reconnaissance of your target or defensively to gather information about what you or your organisation might have exposed over the Internet.\n\nYou can target the following entities in a SpiderFoot scan:\n\n - IP address\n - Domain/sub-domain name\n - Hostname\n - Network subnet (CIDR)\n - ASN\n - E-mail address\n - Phone number\n - Username\n - Person's name\n - Bitcoin address\n \nSpiderFoot's 200+ modules feed each other in a publisher/subscriber model to ensure maximum data extraction to do things like:\n\n- [Host/sub-domain/TLD enumeration/extraction](https://asciinema.org/a/295912)\n- [Email address, phone number and human name extraction](https://asciinema.org/a/295947)\n- [Bitcoin and Ethereum address extraction](https://asciinema.org/a/295957)\n- [Check for susceptibility to sub-domain hijacking](https://asciinema.org/a/344377)\n- DNS zone transfers\n- [Threat intelligence and Blacklist queries](https://asciinema.org/a/295949)\n- API integration with [SHODAN](https://asciinema.org/a/127601), [HaveIBeenPwned](https://asciinema.org/a/128731), [GreyNoise](https://asciinema.org/a/295943), AlienVault, SecurityTrails, etc.\n- [Social media account enumeration](https://asciinema.org/a/295923)\n- [S3/Azure/Digitalocean bucket enumeration/scraping](https://asciinema.org/a/295941)\n- IP geo-location\n- Web scraping, web content analysis\n- [Image, document and binary file meta data analysis](https://asciinema.org/a/296274)\n- Dark web searches\n- [Port scanning and banner grabbing](https://asciinema.org/a/295939)\n- [Data breach searches](https://asciinema.org/a/296145)\n- So much more...\n\n### INSTALLING & RUNNING\n\nTo install and run SpiderFoot, you need at least Python 3.7 and a number of Python libraries which you can install with `pip`. We recommend you install a packaged release since master will often have bleeding edge features and modules that aren't fully tested.\n\n#### Stable build (packaged release):\n\n```\n wget https://github.com/smicallef/spiderfoot/archive/v4.0.tar.gz\n tar zxvf v4.0.tar.gz\n cd spiderfoot-4.0\n pip3 install -r requirements.txt\n python3 ./sf.py -l 127.0.0.1:5001\n```\n\n#### Development build (cloning git master branch):\n\n```\n git clone https://github.com/smicallef/spiderfoot.git\n cd spiderfoot\n pip3 install -r requirements.txt\n python3 ./sf.py -l 127.0.0.1:5001\n```\n\nCheck out the [documentation](https://www.spiderfoot.net/documentation) and our [asciinema videos](https://asciinema.org/~spiderfoot) for more tutorials.\n\n### COMMUNITY\n\nWhether you're a contributor, user or just curious about SpiderFoot and OSINT in general, we'd love to have you join our community! SpiderFoot now has a [Discord server](https://discord.gg/vyvztrG) for seeking help from the community, requesting features or just general OSINT chit-chat.\n\n### WRITING CORRELATION RULES\n\nWe have a comprehensive write-up and reference of the correlation rule-set introduced in SpiderFoot 4.0 [here](/correlations/README.md).\n\nAlso take a look at the [template.yaml](/correlations/template.yaml) file for a walk through. The existing [37 rules](/correlations) are also quite readable and good as starting points for additional rules.\n\n### MODULES / INTEGRATIONS\n\nSpiderFoot has over 200 modules, most of which *don't require API keys*, and many of those that do require API keys *have a free tier*.\n\n| Name | Description | Type |\n|:---------| :-----------|:-------|\n[AbstractAPI](https://app.abstractapi.com/)|Look up domain, phone and IP address information from AbstractAPI.|Tiered API\n[abuse.ch](https://www.abuse.ch)|Check if a host/domain, IP address or netblock is malicious according to Abuse.ch.|Free API\n[AbuseIPDB](https://www.abuseipdb.com)|Check if an IP address is malicious according to AbuseIPDB.com blacklist.|Tiered API\n[Abusix Mail Intelligence](https://abusix.org/)|Check if a netblock or IP address is in the Abusix Mail Intelligence blacklist.|Tiered API\nAccount Finder|Look for possible associated accounts on over 500 social and other websites such as Instagram, Reddit, etc.|Internal\n[AdBlock Check](https://adblockplus.org/)|Check if linked pages would be blocked by AdBlock Plus.|Tiered API\n[AdGuard DNS](https://adguard.com/)|Check if a host would be blocked by AdGuard DNS.|Free API\n[Ahmia](https://ahmia.fi/)|Search Tor 'Ahmia' search engine for mentions of the target.|Free API\n[AlienVault IP Reputation](https://cybersecurity.att.com/)|Check if an IP or netblock is malicious according to the AlienVault IP Reputation database.|Free API\n[AlienVault OTX](https://otx.alienvault.com/)|Obtain information from AlienVault Open Threat Exchange (OTX)|Tiered API\n[Amazon S3 Bucket Finder](https://aws.amazon.com/s3/)|Search for potential Amazon S3 buckets associated with the target and attempt to list their contents.|Free API\n[Apple iTunes](https://itunes.apple.com/)|Search Apple iTunes for mobile apps.|Free API\n[Archive.org](https://archive.org/)|Identifies historic versions of interesting files/pages from the Wayback Machine.|Free API\n[ARIN](https://www.arin.net/)|Queries ARIN registry for contact information.|Free API\n[Azure Blob Finder](https://azure.microsoft.com/en-in/services/storage/blobs/)|Search for potential Azure blobs associated with the target and attempt to list their contents.|Free API\nBase64 Decoder|Identify Base64-encoded strings in URLs, often revealing interesting hidden information.|Internal\n[BGPView](https://bgpview.io/)|Obtain network information from BGPView API.|Free API\nBinary String Extractor|Attempt to identify strings in binary content.|Internal\n[BinaryEdge](https://www.binaryedge.io/)|Obtain information from BinaryEdge.io Internet scanning systems, including breaches, vulnerabilities, torrents and passive DNS.|Tiered API\n[Bing (Shared IPs)](https://www.bing.com/)|Search Bing for hosts sharing the same IP.|Tiered API\n[Bing](https://www.bing.com/)|Obtain information from bing to identify sub-domains and links.|Tiered API\nBitcoin Finder|Identify bitcoin addresses in scraped webpages.|Internal\n[Bitcoin Who's Who](https://bitcoinwhoswho.com/)|Check for Bitcoin addresses against the Bitcoin Who's Who database of suspect/malicious addresses.|Tiered API\n[BitcoinAbuse](https://www.bitcoinabuse.com/)|Check Bitcoin addresses against the bitcoinabuse.com database of suspect/malicious addresses.|Free API\n[Blockchain](https://www.blockchain.com/)|Queries blockchain.info to find the balance of identified bitcoin wallet addresses.|Free API\n[blocklist.de](http://www.blocklist.de/en/index.html)|Check if a netblock or IP is malicious according to blocklist.de.|Free API\n[BotScout](https://botscout.com/)|Searches BotScout.com's database of spam-bot IP addresses and e-mail addresses.|Tiered API\n[botvrij.eu](https://botvrij.eu/)|Check if a domain is malicious according to botvrij.eu.|Free API\n[BuiltWith](https://builtwith.com/)|Query BuiltWith.com's Domain API for information about your target's web technology stack, e-mail addresses and more.|Tiered API\n[C99](https://api.c99.nl/)|Queries the C99 API which offers various data (geo location, proxy detection, phone lookup, etc).|Commercial API\n[CallerName](http://callername.com/)|Lookup US phone number location and reputation information.|Free API\n[Censys](https://censys.io/)|Obtain host information from Censys.io.|Tiered API\n[Certificate Transparency](https://crt.sh/)|Gather hostnames from historical certificates in crt.sh.|Free API\n[CertSpotter](https://sslmate.com/certspotter/)|Gather information about SSL certificates from SSLMate CertSpotter API.|Tiered API\n[CINS Army List](https://cinsscore.com/)|Check if a netblock or IP address is malicious according to Collective Intelligence Network Security (CINS) Army list.|Free API\n[CIRCL.LU](https://www.circl.lu/)|Obtain information from CIRCL.LU's Passive DNS and Passive SSL databases.|Free API\n[CleanBrowsing.org](https://cleanbrowsing.org/)|Check if a host would be blocked by CleanBrowsing.org DNS content filters.|Free API\n[CleanTalk Spam List](https://cleantalk.org)|Check if a netblock or IP address is on CleanTalk.org's spam IP list.|Free API\n[Clearbit](https://clearbit.com/)|Check for names, addresses, domains and more based on lookups of e-mail addresses on clearbit.com.|Tiered API\n[CloudFlare DNS](https://www.cloudflare.com/)|Check if a host would be blocked by CloudFlare DNS.|Free API\n[CoinBlocker Lists](https://zerodot1.gitlab.io/CoinBlockerListsWeb/)|Check if a domain appears on CoinBlocker lists.|Free API\n[CommonCrawl](http://commoncrawl.org/)|Searches for URLs found through CommonCrawl.org.|Free API\n[Comodo Secure DNS](https://www.comodo.com/secure-dns/)|Check if a host would be blocked by Comodo Secure DNS.|Tiered API\nCompany Name Extractor|Identify company names in any obtained data.|Internal\nCookie Extractor|Extract Cookies from HTTP headers.|Internal\nCountry Name Extractor|Identify country names in any obtained data.|Internal\nCredit Card Number Extractor|Identify Credit Card Numbers in any data|Internal\n[Crobat API](https://sonar.omnisint.io/)|Search Crobat API for subdomains.|Free API\nCross-Referencer|Identify whether other domains are associated ('Affiliates') of the target by looking for links back to the target site(s).|Internal\n[CRXcavator](https://crxcavator.io/)|Search CRXcavator for Chrome extensions.|Free API\nCustom Threat Feed|Check if a host/domain, netblock, ASN or IP is malicious according to your custom feed.|Internal\n[CyberCrime-Tracker.net](https://cybercrime-tracker.net/)|Check if a host/domain or IP address is malicious according to CyberCrime-Tracker.net.|Free API\n[Debounce](https://debounce.io/)|Check whether an email is disposable|Free API\n[Dehashed](https://www.dehashed.com/)|Gather breach data from Dehashed API.|Commercial API\n[Digital Ocean Space Finder](https://www.digitalocean.com/products/spaces/)|Search for potential Digital Ocean Spaces associated with the target and attempt to list their contents.|Free API\nDNS Brute-forcer|Attempts to identify hostnames through brute-forcing common names and iterations.|Internal\nDNS Common SRV|Attempts to identify hostnames through brute-forcing common DNS SRV records.|Internal\n[DNS for Family](https://dnsforfamily.com/)|Check if a host would be blocked by DNS for Family.|Free API\nDNS Look-aside|Attempt to reverse-resolve the IP addresses next to your target to see if they are related.|Internal\nDNS Raw Records|Retrieves raw DNS records such as MX, TXT and others.|Internal\nDNS Resolver|Resolves hosts and IP addresses identified, also extracted from raw content.|Internal\nDNS Zone Transfer|Attempts to perform a full DNS zone transfer.|Internal\n[DNSDB](https://www.farsightsecurity.com)|Query FarSight's DNSDB for historical and passive DNS data.|Tiered API\n[DNSDumpster](https://dnsdumpster.com/)|Passive subdomain enumeration using HackerTarget's DNSDumpster|Free API\n[DNSGrep](https://opendata.rapid7.com/)|Obtain Passive DNS information from Rapid7 Sonar Project using DNSGrep API.|Free API\n[DroneBL](https://dronebl.org/)|Query the DroneBL database for open relays, open proxies, vulnerable servers, etc.|Free API\n[DuckDuckGo](https://duckduckgo.com/)|Query DuckDuckGo's API for descriptive information about your target.|Free API\nE-Mail Address Extractor|Identify e-mail addresses in any obtained data.|Internal\n[EmailCrawlr](https://emailcrawlr.com/)|Search EmailCrawlr for email addresses and phone numbers associated with a domain.|Tiered API\n[EmailFormat](https://www.email-format.com/)|Look up e-mail addresses on email-format.com.|Free API\n[EmailRep](https://emailrep.io/)|Search EmailRep.io for email address reputation.|Tiered API\n[Emerging Threats](https://rules.emergingthreats.net/)|Check if a netblock or IP address is malicious according to EmergingThreats.net.|Free API\nError String Extractor|Identify common error messages in content like SQL errors, etc.|Internal\nEthereum Address Extractor|Identify ethereum addresses in scraped webpages.|Internal\n[Etherscan](https://etherscan.io)|Queries etherscan.io to find the balance of identified ethereum wallet addresses.|Free API\nFile Metadata Extractor|Extracts meta data from documents and images.|Internal\n[Flickr](https://www.flickr.com/)|Search Flickr for domains, URLs and emails related to the specified domain.|Free API\n[Focsec](https://focsec.com/)|Look up IP address information from Focsec.|Tiered API\n[FortiGuard Antispam](https://www.fortiguard.com/)|Check if an IP address is malicious according to FortiGuard Antispam.|Free API\n[Fraudguard](https://fraudguard.io/)|Obtain threat information from Fraudguard.io|Tiered API\n[F-Secure Riddler.io](https://riddler.io/)|Obtain network information from F-Secure Riddler.io API.|Commercial API\n[FullContact](https://www.fullcontact.com)|Gather domain and e-mail information from FullContact.com API.|Tiered API\n[FullHunt](https://fullhunt.io/)|Identify domain attack surface using FullHunt API.|Tiered API\n[Github](https://github.com/)|Identify associated public code repositories on Github.|Free API\n[GLEIF](https://search.gleif.org/)|Look up company information from Global Legal Entity Identifier Foundation (GLEIF).|Tiered API\n[Google Maps](https://cloud.google.com/maps-platform/)|Identifies potential physical addresses and latitude/longitude coordinates.|Tiered API\n[Google Object Storage Finder](https://cloud.google.com/storage)|Search for potential Google Object Storage buckets associated with the target and attempt to list their contents.|Free API\n[Google SafeBrowsing](https://developers.google.com/safe-browsing/v4/lookup-api)|Check if the URL is included on any of the Safe Browsing lists.|Free API\n[Google](https://developers.google.com/custom-search)|Obtain information from the Google Custom Search API to identify sub-domains and links.|Tiered API\n[Gravatar](https://secure.gravatar.com/)|Retrieve user information from Gravatar API.|Free API\n[Grayhat Warfare](https://buckets.grayhatwarfare.com/)|Find bucket names matching the keyword extracted from a domain from Grayhat API.|Tiered API\n[Greensnow](https://greensnow.co/)|Check if a netblock or IP address is malicious according to greensnow.co.|Free API\n[grep.app](https://grep.app/)|Search grep.app API for links and emails related to the specified domain.|Free API\n[GreyNoise Community](https://greynoise.io/)|Obtain IP enrichment data from GreyNoise Community API|Tiered API\n[GreyNoise](https://greynoise.io/)|Obtain IP enrichment data from GreyNoise|Tiered API\n[HackerOne (Unofficial)](http://www.nobbd.de/)|Check external vulnerability scanning/reporting service h1.nobbd.de to see if the target is listed.|Free API\n[HackerTarget](https://hackertarget.com/)|Search HackerTarget.com for hosts sharing the same IP.|Free API\nHash Extractor|Identify MD5 and SHA hashes in web content, files and more.|Internal\n[HaveIBeenPwned](https://haveibeenpwned.com/)|Check HaveIBeenPwned.com for hacked e-mail addresses identified in breaches.|Commercial API\nHosting Provider Identifier|Find out if any IP addresses identified fall within known 3rd party hosting ranges, e.g. Amazon, Azure, etc.|Internal\n[Host.io](https://host.io)|Obtain information about domain names from host.io.|Tiered API\nHuman Name Extractor|Attempt to identify human names in fetched content.|Internal\n[Hunter.io](https://hunter.io/)|Check for e-mail addresses and names on hunter.io.|Tiered API\n[Hybrid Analysis](https://www.hybrid-analysis.com)|Search Hybrid Analysis for domains and URLs related to the target.|Free API\nIBAN Number Extractor|Identify International Bank Account Numbers (IBANs) in any data.|Internal\n[Iknowwhatyoudownload.com](https://iknowwhatyoudownload.com/en/peer/)|Check iknowwhatyoudownload.com for IP addresses that have been using torrents.|Tiered API\n[IntelligenceX](https://intelx.io/)|Obtain information from IntelligenceX about identified IP addresses, domains, e-mail addresses and phone numbers.|Tiered API\nInteresting File Finder|Identifies potential files of interest, e.g. office documents, zip files.|Internal\n[Internet Storm Center](https://isc.sans.edu)|Check if an IP address is malicious according to SANS ISC.|Free API\n[ipapi.co](https://ipapi.co/)|Queries ipapi.co to identify geolocation of IP Addresses using ipapi.co API|Tiered API\n[ipapi.com](https://ipapi.com/)|Queries ipapi.com to identify geolocation of IP Addresses using ipapi.com API|Tiered API\n[IPInfo.io](https://ipinfo.io)|Identifies the physical location of IP addresses identified using ipinfo.io.|Tiered API\n[IPQualityScore](https://www.ipqualityscore.com/)|Determine if target is malicious using IPQualityScore API|Tiered API\n[ipregistry](https://ipregistry.co/)|Query the ipregistry.co database for reputation and geo-location.|Tiered API\n[ipstack](https://ipstack.com/)|Identifies the physical location of IP addresses identified using ipstack.com.|Tiered API\n[JsonWHOIS.com](https://jsonwhois.com)|Search JsonWHOIS.com for WHOIS records associated with a domain.|Tiered API\nJunk File Finder|Looks for old/temporary and other similar files.|Internal\n[Keybase](https://keybase.io/)|Obtain additional information about domain names and identified usernames.|Free API\n[Koodous](https://koodous.com/apks/)|Search Koodous for mobile apps.|Tiered API\n[LeakIX](https://leakix.net/)|Search LeakIX for host data leaks, open ports, software and geoip.|Free API\n[Leak-Lookup](https://leak-lookup.com/)|Searches Leak-Lookup.com's database of breaches.|Free API\n[Maltiverse](https://maltiverse.com)|Obtain information about any malicious activities involving IP addresses|Free API\n[MalwarePatrol](https://www.malwarepatrol.net/)|Searches malwarepatrol.net's database of malicious URLs/IPs.|Tiered API\n[MetaDefender](https://metadefender.opswat.com/)|Search MetaDefender API for IP address and domain IP reputation.|Tiered API\n[Mnemonic PassiveDNS](https://www.mnemonic.no)|Obtain Passive DNS information from PassiveDNS.mnemonic.no.|Free API\n[multiproxy.org Open Proxies](https://multiproxy.org/)|Check if an IP address is an open proxy according to multiproxy.org open proxy list.|Free API\n[MySpace](https://myspace.com/)|Gather username and location from MySpace.com profiles.|Free API\n[NameAPI](https://www.nameapi.org/)|Check whether an email is disposable|Tiered API\n[NetworksDB](https://networksdb.io/)|Search NetworksDB.io API for IP address and domain information.|Tiered API\n[NeutrinoAPI](https://www.neutrinoapi.com/)|Search NeutrinoAPI for phone location information, IP address information, and host reputation.|Tiered API\n[numverify](http://numverify.com/)|Lookup phone number location and carrier information from numverify.com.|Tiered API\n[Onion.link](https://onion.link/)|Search Tor 'Onion City' search engine for mentions of the target domain using Google Custom Search.|Free API\n[Onionsearchengine.com](https://as.onionsearchengine.com)|Search Tor onionsearchengine.com for mentions of the target domain.|Free API\n[Onyphe](https://www.onyphe.io)|Check Onyphe data (threat list, geo-location, pastries, vulnerabilities) about a given IP.|Tiered API\n[Open Bug Bounty](https://www.openbugbounty.org/)|Check external vulnerability scanning/reporting service openbugbounty.org to see if the target is listed.|Free API\n[OpenCorporates](https://opencorporates.com)|Look up company information from OpenCorporates.|Tiered API\n[OpenDNS](https://www.opendns.com/)|Check if a host would be blocked by OpenDNS.|Free API\n[OpenNIC DNS](https://www.opennic.org/)|Resolves host names in the OpenNIC alternative DNS system.|Free API\n[OpenPhish](https://openphish.com/)|Check if a host/domain is malicious according to OpenPhish.com.|Free API\n[OpenStreetMap](https://www.openstreetmap.org/)|Retrieves latitude/longitude coordinates for physical addresses from OpenStreetMap API.|Free API\nPage Information|Obtain information about web pages (do they take passwords, do they contain forms, etc.)|Internal\n[PasteBin](https://pastebin.com/)|PasteBin search (via Google Search API) to identify related content.|Tiered API\nPGP Key Servers|Look up domains and e-mail addresses in PGP public key servers.|Internal\n[PhishStats](https://phishstats.info/)|Check if a netblock or IP address is malicious according to PhishStats.|Free API\n[PhishTank](https://phishtank.com/)|Check if a host/domain is malicious according to PhishTank.|Free API\nPhone Number Extractor|Identify phone numbers in scraped webpages.|Internal\nPort Scanner - TCP|Scans for commonly open TCP ports on Internet-facing systems.|Internal\n[Project Honey Pot](https://www.projecthoneypot.org/)|Query the Project Honey Pot database for IP addresses.|Free API\n[ProjectDiscovery Chaos](https://chaos.projectdiscovery.io)|Search for hosts/subdomains using chaos.projectdiscovery.io|Commercial API\n[Psbdmp](https://psbdmp.cc/)|Check psbdmp.cc (PasteBin Dump) for potentially hacked e-mails and domains.|Free API\n[Pulsedive](https://pulsedive.com/)|Obtain information from Pulsedive's API.|Tiered API\n[PunkSpider](https://punkspider.io/)|Check the QOMPLX punkspider.io service to see if the target is listed as vulnerable.|Free API\n[Quad9](https://quad9.net/)|Check if a host would be blocked by Quad9 DNS.|Free API\n[ReverseWhois](https://www.reversewhois.io/)|Reverse Whois lookups using reversewhois.io.|Free API\n[RIPE](https://www.ripe.net/)|Queries the RIPE registry (includes ARIN data) to identify netblocks and other info.|Free API\n[RiskIQ](https://community.riskiq.com/)|Obtain information from RiskIQ's (formerly PassiveTotal) Passive DNS and Passive SSL databases.|Tiered API\n[Robtex](https://www.robtex.com/)|Search Robtex.com for hosts sharing the same IP.|Free API\n[searchcode](https://searchcode.com/)|Search searchcode for code repositories mentioning the target domain.|Free API\n[SecurityTrails](https://securitytrails.com/)|Obtain Passive DNS and other information from SecurityTrails|Tiered API\n[Seon](https://seon.io/)|Queries seon.io to gather intelligence about IP Addresses, email addresses, and phone numbers|Commercial API\n[SHODAN](https://www.shodan.io/)|Obtain information from SHODAN about identified IP addresses.|Tiered API\nSimilar Domain Finder|Search various sources to identify similar looking domain names, for instance squatted domains.|Internal\n[Skymem](http://www.skymem.info/)|Look up e-mail addresses on Skymem.|Free API\n[SlideShare](https://www.slideshare.net)|Gather name and location from SlideShare profiles.|Free API\n[Snov](https://snov.io/)|Gather available email IDs from identified domains|Tiered API\n[Social Links](https://sociallinks.io/)|Queries SocialLinks.io to gather intelligence from social media platforms and dark web.|Commercial API\n[Social Media Profile Finder](https://developers.google.com/custom-search)|Tries to discover the social media profiles for human names identified.|Tiered API\nSocial Network Identifier|Identify presence on social media networks such as LinkedIn, Twitter and others.|Internal\n[SORBS](http://www.sorbs.net/)|Query the SORBS database for open relays, open proxies, vulnerable servers, etc.|Free API\n[SpamCop](https://www.spamcop.net/)|Check if a netblock or IP address is in the SpamCop database.|Free API\n[Spamhaus Zen](https://www.spamhaus.org/)|Check if a netblock or IP address is in the Spamhaus Zen database.|Free API\n[spur.us](https://spur.us/)|Obtain information about any malicious activities involving IP addresses found|Commercial API\n[SpyOnWeb](http://spyonweb.com/)|Search SpyOnWeb for hosts sharing the same IP address, Google Analytics code, or Google Adsense code.|Tiered API\nSSL Certificate Analyzer|Gather information about SSL certificates used by the target's HTTPS sites.|Internal\n[StackOverflow](https://www.stackexchange.com)|Search StackOverflow for any mentions of a target domain. Returns potentially related information.|Tiered API\n[Steven Black Hosts](https://github.com/StevenBlack/hosts)|Check if a domain is malicious (malware or adware) according to Steven Black Hosts list.|Free API\nStrange Header Identifier|Obtain non-standard HTTP headers returned by web servers.|Internal\nSubdomain Takeover Checker|Check if affiliated subdomains are vulnerable to takeover.|Internal\n[Sublist3r PassiveDNS](https://api.sublist3r.com)|Passive subdomain enumeration using Sublist3r's API|Free API\n[SURBL](http://www.surbl.org/)|Check if a netblock, IP address or domain is in the SURBL blacklist.|Free API\n[Talos Intelligence](https://talosintelligence.com/)|Check if a netblock or IP address is malicious according to TalosIntelligence.|Free API\n[TextMagic](https://www.textmagic.com/)|Obtain phone number type from TextMagic API|Tiered API\n[Threat Jammer](https://threatjammer.com)|Check if an IP address is malicious according to ThreatJammer.com|Tiered API\n[ThreatCrowd](https://www.threatcrowd.org)|Obtain information from ThreatCrowd about identified IP addresses, domains and e-mail addresses.|Free API\n[ThreatFox](https://threatfox.abuse.ch)|Check if an IP address is malicious according to ThreatFox.|Free API\n[ThreatMiner](https://www.threatminer.org/)|Obtain information from ThreatMiner's database for passive DNS and threat intelligence.|Free API\nTLD Searcher|Search all Internet TLDs for domains with the same name as the target (this can be very slow.)|Internal\n[Tool - CMSeeK]([https://github.com/Tuhinshubhra/CMSeeK](https://github.com/Tuhinshubhra/CMSeeK))|Identify what Content Management System (CMS) might be used.|Tool\n[Tool - DNSTwist]([https://github.com/elceef/dnstwist](https://github.com/elceef/dnstwist))|Identify bit-squatting, typo and other similar domains to the target using a local DNSTwist installation.|Tool\n[Tool - nbtscan]([http://www.unixwiz.net/tools/nbtscan.html](http://www.unixwiz.net/tools/nbtscan.html))|Scans for open NETBIOS nameservers on your target's network.|Tool\n[Tool - Nmap]([https://nmap.org/](https://nmap.org/))|Identify what Operating System might be used.|Tool\n[Tool - Nuclei]([https://nuclei.projectdiscovery.io/](https://nuclei.projectdiscovery.io/))|Fast and customisable vulnerability scanner.|Tool\n[Tool - onesixtyone]([https://github.com/trailofbits/onesixtyone](https://github.com/trailofbits/onesixtyone))|Fast scanner to find publicly exposed SNMP services.|Tool\n[Tool - Retire.js]([http://retirejs.github.io/retire.js/](http://retirejs.github.io/retire.js/))|Scanner detecting the use of JavaScript libraries with known vulnerabilities|Tool\n[Tool - snallygaster]([https://github.com/hannob/snallygaster](https://github.com/hannob/snallygaster))|Finds file leaks and other security problems on HTTP servers.|Tool\n[Tool - testssl.sh]([https://testssl.sh](https://testssl.sh))|Identify various TLS/SSL weaknesses, including Heartbleed, CRIME and ROBOT.|Tool\n[Tool - TruffleHog]([https://github.com/trufflesecurity/truffleHog](https://github.com/trufflesecurity/truffleHog))|Searches through git repositories for high entropy strings and secrets, digging deep into commit history.|Tool\n[Tool - WAFW00F]([https://github.com/EnableSecurity/wafw00f](https://github.com/EnableSecurity/wafw00f))|Identify what web application firewall (WAF) is in use on the specified website.|Tool\n[Tool - Wappalyzer]([https://www.wappalyzer.com/](https://www.wappalyzer.com/))|Wappalyzer indentifies technologies on websites.|Tool\n[Tool - WhatWeb]([https://github.com/urbanadventurer/whatweb](https://github.com/urbanadventurer/whatweb))|Identify what software is in use on the specified website.|Tool\n[TOR Exit Nodes](https://metrics.torproject.org/)|Check if an IP adddress or netblock appears on the Tor Metrics exit node list.|Free API\n[TORCH](https://torchsearch.wordpress.com/)|Search Tor 'TORCH' search engine for mentions of the target domain.|Free API\n[Trashpanda](https://got-hacked.wtf)|Queries Trashpanda to gather intelligence about mentions of target in pastesites|Tiered API\n[Trumail](https://trumail.io/)|Check whether an email is disposable|Free API\n[Twilio](https://www.twilio.com/)|Obtain information from Twilio about phone numbers. Ensure you have the Caller Name add-on installed in Twilio.|Tiered API\n[Twitter](https://twitter.com/)|Gather name and location from Twitter profiles.|Free API\n[UCEPROTECT](http://www.uceprotect.net/)|Check if a netblock or IP address is in the UCEPROTECT database.|Free API\n[URLScan.io](https://urlscan.io/)|Search URLScan.io cache for domain information.|Free API\n[Venmo](https://venmo.com/)|Gather user information from Venmo API.|Free API\n[ViewDNS.info](https://viewdns.info/)|Identify co-hosted websites and perform reverse Whois lookups using ViewDNS.info.|Tiered API\n[VirusTotal](https://www.virustotal.com/)|Obtain information from VirusTotal about identified IP addresses.|Tiered API\n[VoIP Blacklist (VoIPBL)](https://voipbl.org/)|Check if an IP address or netblock is malicious according to VoIP Blacklist (VoIPBL).|Free API\n[VXVault.net](http://vxvault.net/)|Check if a domain or IP address is malicious according to VXVault.net.|Free API\nWeb Analytics Extractor|Identify web analytics IDs in scraped webpages and DNS TXT records.|Internal\nWeb Framework Identifier|Identify the usage of popular web frameworks like jQuery, YUI and others.|Internal\nWeb Server Identifier|Obtain web server banners to identify versions of web servers being used.|Internal\nWeb Spider|Spidering of web-pages to extract content for searching.|Internal\n[WhatCMS](https://whatcms.org/)|Check web technology using WhatCMS.org API.|Tiered API\n[Whoisology](https://whoisology.com/)|Reverse Whois lookups using Whoisology.com.|Commercial API\nWhois|Perform a WHOIS look-up on domain names and owned netblocks.|Internal\n[Whoxy](https://www.whoxy.com/)|Reverse Whois lookups using Whoxy.com.|Commercial API\n[WiGLE](https://wigle.net/)|Query WiGLE to identify nearby WiFi access points.|Free API\n[Wikileaks](https://wikileaks.org/)|Search Wikileaks for mentions of domain names and e-mail addresses.|Free API\n[Wikipedia Edits](https://www.wikipedia.org/)|Identify edits to Wikipedia articles made from a given IP address or username.|Free API\n[XForce Exchange](https://exchange.xforce.ibmcloud.com/)|Obtain IP reputation and passive DNS information from IBM X-Force Exchange.|Tiered API\n[Yandex DNS](https://yandex.com/)|Check if a host would be blocked by Yandex DNS.|Free API\n[Zetalytics](https://zetalytics.com/)|Query the Zetalytics database for hosts on your target domain(s).|Tiered API\n[ZoneFile.io](https://zonefiles.io)|Search ZoneFiles.io Domain query API for domain information.|Tiered API\n[Zone-H Defacement Check](https://zone-h.org/)|Check if a hostname/domain appears on the zone-h.org 'special defacements' RSS feed.|Free API\n\n### DOCUMENTATION\n\nRead more at the [project website](https://www.spiderfoot.net/r.php?u=aHR0cHM6Ly93d3cuc3BpZGVyZm9vdC5uZXQv&s=os_gh), including more complete documentation, blog posts with tutorials/guides, plus information about [SpiderFoot HX](https://www.spiderfoot.net/r.php?u=aHR0cHM6Ly93d3cuc3BpZGVyZm9vdC5uZXQvaHgvCg==&s=os_gh).\n\nLatest updates announced on [Twitter](https://twitter.com/spiderfoot).\n" }, { "path": "THANKYOU", "content": "These people are owed some credit for their contributions to SpiderFoot, be it\nbug fixes or new functionality. Thank you!\n\n- Brendan Coles (https://github.com/bcoles)\n - Second highest contributor to the project\n - Tons of modules, fixes and established the whole testing framework\n - Too many contributions to list here, see https://github.com/smicallef/spiderfoot/pulls?q=is%3Apr+author%3Abcoles+is%3Aclosed\n\n- Krishnasis Mandal (https://github.com/krishnasism)\n - Many modules, see https://github.com/smicallef/spiderfoot/pulls?q=is%3Apr+author%3Akrishnasism+is%3Aclosed\n\n- Steve Bate (https://github.com/steve-bate)\n - Multiple bug fixes, performance and quality improvements\n\n- fallingcubes (https://github.com/fallingcubes)\n - Use of multiprocessing instead of threading for running multiple scans simultaneously\n\n- Counter Intrusive Technologies (https://github.com/counterintrusive)\n - sfp_citadel updates\n\n- Dhiraj Mishra (https://github.com/RootUp)\n - Created the sfp_h1nobbdde module\n\n- Chris Weber (http://www.casaba.com/blog/author/chris/)\n - Many recommendations for improvement\n\n- Henri Salo (https://github.com/fgeek)\n - Identified XSS issues (issue #29)\n\n- MarioVilas (https://github.com/MarioVilas)\n - Contributed a fix for CSV exports (issue #35)\n\n- johnnykv (https://github.com/johnnykv)\n - Contributed improved pip instructions\n\n- Russ McRee (@holisticinfosec)\n - Identified XSS/CSRF issues\n\n- Andrea De Pasquale (https://github.com/adepasquale)\n - Cache directory fix\n\n- Viyat Bhalodia (https://github.com/delta24)\n - Prettied up README.md\n\n- Gormogon (https://github.com/Gormogon)\n - Huge code clean up for PEP 8 compliance\n\n- Lin Zhemin (https://github.com/miaoski)\n - Added the ability to use a configurable document root\n\n- Micah Hoffman (https://github.com/WebBreacher)\n - Comprehensive list of URLs for identifying account usage in modules/sfp_accounts.py\n\n- Michael Pellon (https://github.com/mpellon)\n - Providing a Dockerfile\n\n- HackershubNL (https://github.com/HackershubNL)\n - Contributions towards authentication functionality\n\n- Mathew Woodyard (https://github.com/woodrad)\n - Bad Packets module improvements\n" }, { "path": "VERSION", "content": "SpiderFoot 4.0.0\n" }, { "path": "correlations/README.md", "content": "## Background\n\nSpiderFoot’s goal is to automate OSINT collection and analysis to the greatest extent possible. Since its inception, SpiderFoot has heavily focused on automating OSINT collection and entity extraction, but the automation of common analysis tasks -- beyond some reporting and visualisations -- has been left entirely to the user. The meant that the strength of SpiderFoot's data collection capabilities has sometimes been its weakness since with so much data collected, users have often needed to export it and use other tools to weed out data of interest.\n\n## Introducing Correlations\n\nWe started tackling this analysis gap with the launch of SpiderFoot HX in 2019 through the introduction of the \"Correlations\" feature. This feature was represented by some 30 \"correlation rules\" that ran with each scan, analyzing data and presenting results reflecting SpiderFoot's opinionated view on what may be important or interesting. Here are a few of those rules as examples:\n* Hosts/IPs reported as malicious by multiple data sources\n* Outlier web servers (can be an indication of shadow IT)\n* Databases exposed on the Internet\n* Open ports revealing software versions\n* and many more.\n\nWith the release of SpiderFoot 4.0 we wanted to bring this capability from SpiderFoot HX to the community, but also re-imagine it at the same time so that the community might not simply run rules we provide, but also write their own correlation rules and contribute them back. We also hope that just as with modules, we see a long list of contributions made in the years ahead so that all may benefit.\n\nWith that said, let's get into what these rules look like and how to write one.\n\n## Key concepts\n\n### YAML\nThe rules themselves are written in YAML. Why YAML? It’s easy to read, write, allows for comments and is increasingly commonplace in many modern tools.\n\n### Rule structure\nThe simplest way to think of a SpiderFoot correlation rule is like a simple database query that consists of a few sections:\n1. Defining the rule itself (`id`, `version` and `meta` sections).\n2. Stating what you'd like to extract from the scan results (`collections` section).\n3. Grouping that data in some way (`aggregation` section; optional).\n3. Performing some analysis over that data in some way (`analysis` section; optional).\n4. Presenting the results (`headline` section).\n\n### Example rule\nHere's an example rule that looks at SpiderFoot scan results for data revealing open TCP ports where the banner (the data returned upon connecting to the port) reports a software version. It does so by applying some regular expressions to the content of `TCP_PORT_OPEN_BANNER` data elements, filtering out some false positives and then grouping the results by the banner itself so that one correlation result is created per banner revealing a version:\n\n```yaml\nid: open_port_version\nversion: 1\nmeta:\n name: Open TCP port reveals version\n description: >\n A possible software version has been revealed on an open port. Such\n information may reveal the use of old/unpatched software used by\n the target.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: type\n value: TCP_PORT_OPEN_BANNER\n - method: regex\n field: data\n value: .*[0-9]\\.[0-9].*\n - method: regex\n field: data\n value: not .*Mime-Version.*\n - method: regex\n field: data\n value: not .*HTTP/1.*\naggregation:\n field: data\nheadline: \"Software version revealed on open port: {data}\"\n```\n\n### The outcome\nTo show this in practice, we can run a simple scan against a target, in this case focusing on performing a port scan:\n\n```\n-> # python3.9 ./sf.py -s www.binarypool.com -m sfp_dnsresolve,sfp_portscan_tcp \n2022-04-06 08:14:58,476 [INFO] sflib : Scan [94EB5F0B] for 'www.binarypool.com' initiated.\n...\nsfp_portscan_tcp Open TCP Port Banner SSH-2.0-OpenSSH_7.2p2 Ubuntu-4ubuntu2.10\n...\n2022-04-06 08:15:23,110 [INFO] correlation : New correlation [open_port_version]: Software version revealed on open port: SSH-2.0-OpenSSH_7.2p2 Ubuntu-4ubuntu2.10\n2022-04-06 08:15:23,244 [INFO] sflib : Scan [94EB5F0B] completed.\n```\nWe can see above that a port was found to be open by the `sfp_portscan_tcp` module, and it happens to include a version. The correlation rule `open_port_version` picked this up and reported it. This is also visible in the web interface:\n\n\n\n**NOTE:** Rules will only succeed if relevant data exists in your scan results in the first place. In other words, correlation rules analyze scan data, they don't collect data from targets.\n\n### How it works\nIn short, SpiderFoot translates the YAML rules into a combination queries against the backend database of scan results and Python logic to filter and group the results, creating \"correlation results\" in the SpiderFoot database. These results can be viewed in the SpiderFot web interface or from the SpiderFoot CLI. You can also query them directly out of the SQLite database if you like (they are in the `tbl_scan_correlation_results` table, and the `tbl_scan_correlation_results_events` table maps the events (data elements) to the correlation result).\n\n### The rules\n\nEach rule exists as a YAML file within the `/correlations` folder in the SpiderFoot installation path. Here you can see a list of rules in 4.0, which we hope to grow over time:\n\n```sh\ncert_expired.yaml host_only_from_certificatetransparency.yaml outlier_ipaddress.yaml\ncloud_bucket_open.yaml http_errors.yaml outlier_registrar.yaml\ncloud_bucket_open_related.yaml human_name_in_whois.yaml outlier_webserver.yaml\ndata_from_base64.yaml internal_host.yaml remote_desktop_exposed.yaml\ndata_from_docmeta.yaml multiple_malicious.yaml root_path_needs_auth.yaml\ndatabase_exposed.yaml multiple_malicious_affiliate.yaml stale_host.yaml\ndev_or_test_system.yaml multiple_malicious_cohost.yaml strong_affiliate_certs.yaml\ndns_zone_transfer_possible.yaml name_only_from_pasteleak_site.yaml strong_similardomain_crossref.yaml\negress_ip_from_wikipedia.yaml open_port_version.yaml template.yaml\nemail_in_multiple_breaches.yaml outlier_cloud.yaml vulnerability_critical.yaml\nemail_in_whois.yaml outlier_country.yaml vulnerability_high.yaml\nemail_only_from_pasteleak_site.yaml outlier_email.yaml vulnerability_mediumlow.yaml\nhost_only_from_bruteforce.yaml outlier_hostname.yaml\n```\n### Rule components\n\nThe rules themselves are broken down into the following components:\n\n**Meta**: Describes the rule itself so that humans understand what the rule does and the risk level of any results. This information is used mostly in the web interface and CLI.\n\n**Collections**: A collection represents a set of data pulled from scan results, to be used in later aggregation and analysis stages. Each rule can have multiple collections.\n\n**Aggregations**: An aggregation buckets the collected data into groups for analysis in distinct groups of data elements.\n\n**Analysis**: Analysis performs (you guessed it) analysis on the data to whittle down the data elements to what ultimately gets reported. For example, the analysis stage may look only for cases where the data field is repeated in the data set, indicating it was found multiple times and therefore discarding any only appearing once.\n\n**Headline**: The headline represents the actual correlation title that summarizes what was found. You can think of this as equivalent to a meal name (beef stew), and all the data elements as being the ingredients (beef, tomatoes, onions, etc.).\n\n### Creating a rule\n\nTo create your own rule, simply copy the `template.yaml` file in the `correlations` folder to a meaningful name that matches the ID you intend to provide it, e.g. `aws_cloud_usage.yaml` and edit the rule to fit your needs. Save it and re-start SpiderFoot for the rule to be loaded. If there are any syntax errors, SpiderFoot will abort at startup and (hopefully) give you enough information to know where the error is.\n\nThe `template.yaml` file is also a good next point of reference to better understand the structure of the rules and how to use them. We also recommend taking a look through the actual rules themselves to see the concepts in practice.\n\n## Rule Reference\n\n**id**: The internal ID for this rule, which needs to match the filename.\n\n**version**: The rule syntax version. This must be `1` for now.\n\n**meta**: This section contains a few important fields used to describe the rule.\n\n * **name**: A short, human readable name for the rule.\n * **description**: A longer (can be multi-paragraph) description of the rule.\n * **risk**: The risk level represented by this rule's findings. Can be `INFO`, `LOW`, `MEDIUM`, `HIGH`.\n\n**collection**: A correlation rule contains one or more `collect` blocks. Each `collect` block contains one or more `method` blocks telling SpiderFoot what criteria to use for extracting data from the database and how to filter it down. \n * **collect**: Technically, the first `method` block in each `collect` block is what actually pulls data from the database, and each subsequent `method` refines that dataset down to what you’re seeking. You may have multiple `collect` blocks overall but the rule remains that within each `collect`, the first `method` pulls data from the database and subsequent `method` blocks within the `collect` refine that data.\n\n * **method**: Each `method` block tells SpiderFoot how to collect and refine data. Each `collect` must contain at least one `method` block. Valid methods are `exact` for performing an exact match of the chosen `field` to the supplied `value`, or `regex` to perform regular expression matching.\n\n * **field**: Each `method` block has a `field` upon which the matching should be performed. Valid fields are `type` (e.g. `INTERNET_NAME`), `module` (e.g. `sfp_whois`) and `data`, which would be the value of the data element (e.g. in case of an `INTERNET_NAME`, the `data` would be the hostname). After the first `method` block, you can also prefix the field with `source.`, `child.` or `entity.` to refer to the fields of the source, children or relevant entities of the collected data, respectively (see `multiple_malicious.yaml` and `data_from_docmeta.yaml` as examples of this approach).\n\n * **value**: Here you supply the value or values you wish to match against the field you supplied in `field`. If your `method` was `regex`, this would be a regular expression.\n\n**aggregation**: With all the data elements in their collections, you can now aggregate them into buckets for further analysis or immediately generate results from the rule. While the collection phase is about obtaining the data from the database and filtering down to data of interest, the aggregation phase is about grouping that data in different ways in order to support analysis and/or grouping reported results.\n\nAggregation simply iterates through the data elements in each collection and places them into groups based on the `field` specified. For instance if you pick the `type` field, you’ll end up with data elements with the same `type` field grouped together. The purpose of this grouping is two-fold: to support the analysis stage, or if you don’t have the analysis stage, it’s how your correlation results will be grouped for the user.\n * **field**: The `field` defines how you'd like your data elements grouped together. Just like the `field` option in `method` blocks above, you may prefix the field with `source.`, `child.` or `entity.` to apply the aggregation on those fields of the data element instead. For example, if you intended to look for multiple occurrences of a hostname, you would specify `data` here as that field, since you want to count the number of times the value of the `data` field appears.\n\n**analysis**\nThe analysis section applies (you guessed it) some analysis to the aggregated results or collections directly if you didn’t perform any aggregation, and drops candidate results if they fail this stage. Various analysis `method` types exist, and each takes different options, described below.\n\n * **method**:\n * **threshold**: Drop any collection/aggregation of data elements that do not meet the defined thresholds. You would use this analysis rule when wanting to generate a result only when a data element has appeared more or less than a limit specified, for instance reporting when an email address is reported just once, or more than 100 times.\n * **field**: The field you want to apply the threshold too. As per above, you can use `child.`, `source.` and `entity.` field prefixes here too.\n * **count_unique_only**: By default the threshold is applied to the `field` specified on all data elements, but by setting `count_unique_only` to `true`, you can limit the threshold to only unique values in the `field` specified, so as not to also count duplicates.\n * **minimum**: The minimum number of data elements that must appear within the collection or aggregation.\n * **maximum**: The maximum number of data elements that must appear within the collection or aggregation.\n * **outlier**: Only keep outliers within any collection/aggregation of data elements.\n * **maximum_percent**: The maximum percentage of the overall results that an aggregation can represent. This method requires that you have performed an aggregation on a certain field in order to function. For example, if you aggregate on the `data` field of your collections, and one of those buckets contains less than 10% of the overall volume, it will be reported as an outlier.\n * **noisy_percent**: By default this is `10`, meaning that if the average percentage every bucket is below 10%, don't report outliers since the dataset is anomalous.\n * **first_collection_only**: Only keep data elements that appeared in the first collection but not any others. For example, this is handy for finding cases where data was found from one or several data sources but not others.\n * **field**: The field you want to use for looking up between collections.\n * **match_all_to_first_collection**: Only keep data elements that have matched in some way to the first collection. This requires an aggregation to have been performed, as the field used for aggregation is what will be used for checking for a match.\n * **match_method**: How to match between all collections and the first collection. Options are `contains` (simple wildcard match), `exact` and `subnet` which reports a match if the expected field may contain an IP address that is within the first collection field containing a subnet.\n\n**headline**\nAfter all data elements have been collected, filtered down, aggregated and analyzed, if data elements are remaining, these are what we call \"correlation results\" -> the results of your correlation rule. These need a \"headline\" to summarize the findings, which you can define here. To place any value from your data into the headline, you must enclose the field in `{}`, e.g. `{entity.data}`. There are two ways to write a `headline` rule. The typical way is to simply have `headline: titletexthere`, or have it as a block, in which case you can be more granular about how the correlation results are published:\n\n* **text**: The headline text, as described above.\n* **publish_collections**: The collection you wish to have associated with the correlation result. This is not often needed, but more in combination with the `match_all_to_first_collection` analysis rule in case your first collection is only used as a reference point and not actually contain any data elements you wish to publish with this correlation result. Take a look at the `egress_ip_from_wikipedia.yaml` rule for an example of this used in practice.\n\n### A note about `child.`, `source.` and `entity.` field prefixes\n\nEvery data element pulled in the first `match` rule in a collection will also have any children (data resulting from that data element), the source (the data element that this data element was generated from) and entity (the source, or source of source, etc. that was an entity like IP address, domain, etc.). This enables you to prefix subsequent (and only subsequent!) match block field names with `child.`, `source.` and `entity.` if you wish to match based on those fields. These prefixes, as shown above, can also be used in the `aggregation`, `analysis` and `headline` sections too.\n\nIt is vital to note that these prefixes **always** are in reference to the first `match` block within each `collect` block, since every subsequent `match` block is always a refinement of the first `match` block.\n\nThis can be complicated, so let's use an example to illustrate. Let's say your scan has found a hostname (a data element type of `INTERNET_NAME`) of `foo`, and it found that within some webpage content (a data element type of `TARGET_WEB_CONTENT`) of \"This is some web content: foo\", which was from a URL (data element type of `LINKED_URL_INTERNAL`) of \"https://bar/page.html\", which was from another host named `bar`. Here's the data discovery path:\n\n`bar` [`INTERNET_NAME`] -> `https://bar/page.html` [`LINKED_URL_INTERNAL`] -> `This is some web content: foo` [`TARGET_WEB_CONTENT`] -> `foo` [`INTERNET_NAME`]\n\nIf we were to look at `This is some web content: foo` in our rule, here are the `data` and `type` fields you would expect to exist (`module` would also exist but has been left out of this example for brevity):\n* `data`: `This is some web content: foo`\n* `type`: `TARGET_WEB_CONTENT`\n* `source.data`: `https://bar/page.html`\n* `source.type`: `LINKED_URL_INTERNAL`\n* `child.data`: `foo`\n* `child.type`: `INTERNET_NAME`\n* `entity.type`: `INTERNET_NAME`\n* `entity.data`: `bar`\n\nNotice how the `entity.type` and `entity.data` fields for \"This is some web content: foo\" is **not** the `LINKED_URL_INTERNAL` data element, but actually the `bar` `INTERNET_NAME` data element. This is because an `INTERNET_NAME` is an entity, but a `LINKED_URL_INTERNAL` is not.\n\nYou can look in `spiderfoot/db.py` to see which data types are entities and which are not.\n" }, { "path": "correlations/cert_expired.yaml", "content": "id: cert_expired\nversion: 1\nmeta:\n name: Expired SSL certificate found\n description: >\n A host was found with an expired SSL certificate. This may\n pose a risk to the security of the service exposed and/or\n cause connecting services to fail due to being unable to\n verify the certificate.\n risk: MEDIUM\ncollections:\n - collect:\n - method: exact\n field: type\n value: SSL_CERTIFICATE_EXPIRED\naggregation:\n field: source.data\nheadline: \"Expired SSL certificate found: {source.data}\"\n" }, { "path": "correlations/cloud_bucket_open.yaml", "content": "id: cloud_bucket_open\nversion: 1\nmeta:\n name: Cloud storage bucket open to the Internet\n description: >\n A cloud storage bucket (e.g. S3) referenced from the target\n website has been found to be open to the Internet. Such\n buckets should be restricted so that contents cannot be\n listed, even if needing to be publicly accessible.\n risk: HIGH\ncollections:\n - collect:\n - method: exact\n field: type\n value: CLOUD_STORAGE_BUCKET\n - method: exact\n field: source.type\n value: LINKED_URL_EXTERNAL\n - method: exact\n field: child.type\n value: CLOUD_STORAGE_BUCKET_OPEN\naggregation:\n field: data\nheadline: \"Cloud storage bucket found open: {data}\"\n" }, { "path": "correlations/cloud_bucket_open_related.yaml", "content": "id: cloud_bucket_open_related\nversion: 1\nmeta:\n name: Possibly related cloud storage bucket open to the Internet\n description: >\n A cloud storage bucket (e.g. S3) potentially related to\n the target has been found to be open to the Internet.\n\n As the buckets in this case are based on name-matching, verification\n for actual association with the target is necessary.\n risk: LOW\ncollections:\n - collect:\n - method: exact\n field: type\n value: CLOUD_STORAGE_BUCKET\n - method: exact\n field: source.type\n value: not LINKED_URL_EXTERNAL\n - method: exact\n field: child.type\n value: CLOUD_STORAGE_BUCKET_OPEN\naggregation:\n field: data\nheadline: \"Potentially relevant cloud storage bucket found open: {data}\"\n" }, { "path": "correlations/data_from_base64.yaml", "content": "id: data_from_base64\nversion: 1\nmeta:\n name: Data was found within base64-encoded data\n description: >\n Possibly interesting data was found within base64-encoded data,\n such as software versions, names, email addresses and hostnames.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: type\n value: BASE64_DATA\n - method: regex\n field: child.data\n value: .*\n - method: exact\n field: child.type\n value: not HASH\naggregation:\n field: child.data\nheadline: \"Interesting data was found within base64-encoded data: '{child.data}'\"\n" }, { "path": "correlations/data_from_docmeta.yaml", "content": "id: data_from_docmeta\nversion: 1\nmeta:\n name: Data was found within document/image meta data\n description: >\n Possibly interesting data was found within document/image meta data,\n such as software versions, names, email addresses and hostnames.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: type\n value: RAW_FILE_META_DATA\n - method: regex\n field: child.data\n value: .*\n - method: exact\n field: child.type\n value: not HASH\naggregation:\n field: child.data\nheadline: \"Interesting data was found within document meta data: '{child.data}'\"\n" }, { "path": "correlations/database_exposed.yaml", "content": "id: database_exposed\nversion: 1\nmeta:\n name: Database server exposed to the Internet\n description: >\n A database technology (MySQL, Oracle, Postgres, Redis,\n Hadoop, MongoDB, Spark) was found to be accessible over\n the Internet.\n\n Even if authentication is required such systems should not\n be exposed over the Internet due to the risk of misconfiguration\n or unpatched vulnerabilities.\n risk: HIGH\ncollections:\n - collect:\n - method: exact\n field: type\n value: TCP_PORT_OPEN\n - method: regex\n field: data\n value:\n # MySQL\n - .*:3306$\n # Oracle\n - .*:1521$\n # PostgreSQL\n - .*:5432$\n # Redis\n - .*:6379$\n - .*:6380$\n # Hadoop\n - .*:50070$\n - .*:50470$\n - .*:50090$\n - .*:500[12]0$\n - .*:50475$\n - .*:50075$\n - .*:8020$\n - .*:9000$\n # Spark\n - .*:7077$\n # MongoDB\n - .*:2701[789]$\naggregation:\n field: data\nheadline: \"Database server exposed to the Internet: {data}\"\n" }, { "path": "correlations/dev_or_test_system.yaml", "content": "id: dev_or_test_system\nversion: 1\nmeta:\n name: A host appearing to be a test or development system was found\n description: >\n A host that has dev, test, staging, internal or uat in its\n name was found. This may indicate a system that is exposed\n over the Internet unintentionally, and/or may be less secure\n than other systems.\n risk: MEDIUM\ncollections:\n - collect:\n - method: exact\n field: type\n value: INTERNET_NAME\n - method: regex\n field: data\n value:\n - .*dev.*\n - .*test.*\n - .*uat.*\n - .*internal.*\n - .*staging.*\naggregation:\n field: data\nheadline: \"Development or internal system found: {data}\"\n" }, { "path": "correlations/dns_zone_transfer_possible.yaml", "content": "id: dns_zone_transfer_possible\nversion: 1\nmeta:\n name: DNS Zone Transfer is possible\n description: >\n It was possible to perform a DNS Zone Transfer with the\n target's name server. Zone transfers can expose a significant\n amount of information about the target network and have no\n real need to be enabled in modern networks.\n risk: HIGH\ncollections:\n - collect:\n - method: exact\n field: module\n value: sfp_dnszonexfer\naggregation:\n field: source.data\nheadline: \"DNS Zone Transfer possible with {source.data}\"\n" }, { "path": "correlations/egress_ip_from_wikipedia.yaml", "content": "id: egress_ip_from_wikipedia\nversion: 1\nmeta:\n name: Wikipedia page edit from target-owned network\n description: >\n A Wikipedia page edit was found to come from an IP address\n within a network owned by the target. That IP address is\n highly likely to be that of a VPN server or office egress\n proxy/gateway.\n risk: MEDIUM\ncollections:\n - collect:\n - method: exact\n field: type\n value: NETBLOCK_OWNER\n - collect:\n - method: exact\n field: type\n value: IP_ADDRESS\n - method: exact\n field: child.type\n value: WIKIPEDIA_PAGE_EDIT\naggregation:\n field: data\nanalysis:\n - method: match_all_to_first_collection\n field: data\n # Could also be 'contains' or 'exact'\n match_method: subnet\nheadline:\n text: \"Wikipedia edit from IP within target-owned network: {data}\"\n publish_collections:\n - 1\n" }, { "path": "correlations/email_in_multiple_breaches.yaml", "content": "id: email_in_multiple_breaches\nversion: 1\nmeta:\n name: >\n An email address was reported to be in multiple breaches\n description: >\n An email address was reported to be in multiple breaches.\n\n The presence in multiple breaches may indicate that the password\n of the account is particularly weak, or that it was re-used across\n the sites involved in the breaches. Note that some breaches simply\n aggregate other breaches, or may be very old.\n risk: HIGH\ncollections:\n - collect:\n - method: exact\n field: type\n value: EMAILADDR_COMPROMISED\naggregation:\n field: source.data\nanalysis:\n - method: threshold\n field: source.data\n minimum: 2\nheadline: \"Email address reported in multiple breaches: {source.data}\"\n" }, { "path": "correlations/email_in_whois.yaml", "content": "id: email_in_whois\nversion: 1\nmeta:\n name: A target-relevant email address was found in Whois data\n description: >\n An email address was found in Whois data that is directly\n related to the target (e.g. same domain). The email found may\n be that of a highly privileged person responsible for\n maintaining the infrastructure of the target.\n\n Email addresses are increasingly rarely found in Whois data due\n to GDPR.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: type\n value: EMAILADDR\n - method: regex\n field: source.type\n value: .*WHOIS.*\naggregation:\n field: data\nheadline: \"Email address found in Whois record: {data}\"\n" }, { "path": "correlations/email_only_from_pasteleak_site.yaml", "content": "id: email_only_from_pasteleak_site\nversion: 1\nmeta:\n name: Email address only from paste/leak site\n description: >\n An email address was found mentioned in a paste/leak site but\n nowhere else.\n\n Since the email address was not found anywhere else, this may indicate\n that the address is in some way special, perhaps not intended to\n be publicly exposed/used or targeted in an attack.\n risk: MEDIUM\ncollections:\n - collect:\n - method: exact\n field: type\n value: EMAILADDR\n - method: exact\n field: source.type\n value:\n - LEAKSITE_CONTENT\n - collect:\n - method: exact\n field: type\n value: EMAILADDR\n - method: exact\n field: source.type\n value:\n - not LEAKSITE_CONTENT\naggregation:\n field: data\nanalysis:\n - method: first_collection_only\n field: data\nheadline: \"Email address found only in paste/leak site: {data}\"\n" }, { "path": "correlations/host_only_from_bruteforce.yaml", "content": "id: host_only_from_bruteforce\nversion: 1\nmeta:\n name: Host only from bruteforcing\n description: >\n A hostname was found only by brute-forcing but nowhere else.\n\n Since the host was not found anywhere else, this may indicate\n that the host is in some way special, perhaps not intended to\n be publicly exposed/used.\n risk: LOW\ncollections:\n - collect:\n - method: exact\n field: type\n value: INTERNET_NAME\n - method: exact\n field: module\n value: sfp_dnsbrute\n - collect:\n - method: exact\n field: type\n value: INTERNET_NAME\n - method: exact\n field: module\n value: not sfp_dnsbrute\naggregation:\n field: data\nanalysis:\n - method: first_collection_only\n field: data\nheadline: \"Host found only through bruteforcing: {data}\"\n" }, { "path": "correlations/host_only_from_certificatetransparency.yaml", "content": "id: host_only_from_certificatetransparency\nversion: 1\nmeta:\n name: Hostname only from certificate transparency\n description: >\n A hostname was found from certificate transparency but\n nowhere else.\n\n Since the host was not found anywhere else, this may indicate\n that the host is in some way special, perhaps not intended to\n be publicly exposed/used.\n risk: LOW\ncollections:\n - collect:\n - method: exact\n field: type\n value: INTERNET_NAME\n - method: exact\n field: module\n value:\n - sfp_crt\n - sfp_certspotter\n - collect:\n - method: exact\n field: type\n value: INTERNET_NAME\n - method: exact\n field: module\n value:\n - not sfp_crt\n - not sfp_certspotter\n - not sfp_dnsresolve\naggregation:\n field: data\nanalysis:\n - method: first_collection_only\n field: data\nheadline: \"Host found only in certificate transparency: {data}\"\n" }, { "path": "correlations/http_errors.yaml", "content": "id: http_errors\nversion: 1\nmeta:\n name: Multiple HTTP errors found\n description: >\n Non-successful HTTP error codes were encountered. 401 and 403\n are not included as these refer to authentication/authorization\n failures.\n risk: LOW\ncollections:\n - collect:\n - method: exact\n field: type\n value: HTTP_CODE\n - method: regex\n field: data\n value: ^[4-9].*$\n # Leave out authentication/authorization failures\n - method: regex\n field: data\n value: not 40[13]\naggregation:\n field: entity.data\nanalysis:\n - method: threshold\n minimum: 2\n field: data\nheadline: \"Multiple failure HTTP codes found at {entity.data}\"\n" }, { "path": "correlations/human_name_in_whois.yaml", "content": "id: human_name_in_whois\nversion: 1\nmeta:\n name: A human name was found in Whois data\n description: >\n A human name was found in Whois data. The name found may\n be that of a highly privileged person responsible for\n maintaining the infrastructure of the target.\n\n Names are increasingly rarely found in Whois data due to\n GDPR.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: type\n value: HUMAN_NAME\n - method: regex\n field: source.type\n value: .*WHOIS.*\naggregation:\n field: data\nheadline: \"Human name found in Whois record: {data}\"\n" }, { "path": "correlations/internal_host.yaml", "content": "id: internal_host\nversion: 1\nmeta:\n name: A host resolving to unroutable IPs was found\n description: >\n A host that resolves to an IP address on ranges reserved for\n internal use (10.0.0.0/8, etc.) was found to be publicly\n resolvable. This might reveal information about the internal\n infrastructure and workings of the target.\n risk: MEDIUM\ncollections:\n - collect:\n - method: exact\n field: type\n value: INTERNAL_IP_ADDRESS\n - collect:\n - method: exact\n field: type\n value: IP_ADDRESS\n - method: regex\n field: data\n value:\n - ^192\\\\.168\\\\..*\n - ^10\\\\..*\n - method: exact\n field: module\n value: sfp_dnsresolve\naggregation:\n field: data\nheadline: \"An internal host was found: {data}\"\n" }, { "path": "correlations/multiple_malicious.yaml", "content": "id: multiple_malicious\nversion: 1\nmeta:\n name: >\n An IP, host, subnet or email address was considered malicious by\n multiple sources\n description: >\n An IP, host, subnet or email address was considered malicious by\n multiple sources.\n\n Such cases have a high likelihood of being genuinely malicious and\n should be urgently investigated. Even if the entity in question is\n not compromised, it's likely to be blocked across parts of the\n Internet due to its presence in these lists.\n risk: HIGH\ncollections:\n - collect:\n - method: regex\n field: type\n value:\n - MALICIOUS_*\n - BLACKLIST_*\n # Filter out all subnets\n - method: regex\n field: source.data\n value: not .*/.*\n # Filter out affiliates and Co-hosts\n - method: regex\n field: type\n value:\n - not .*COHOST.*\n - not .*AFFILIATE.*\naggregation:\n field: source.data\nanalysis:\n - method: threshold\n field: source.data\n minimum: 2\nheadline: \"Entity considered malicious by multiple sources: {source.data}\"\n" }, { "path": "correlations/multiple_malicious_affiliate.yaml", "content": "id: multiple_malicious_affiliate\nversion: 1\nmeta:\n name: >\n An affiliated IP or host was considered malicious by multiple sources\n description: >\n An affiliated IP or host was considered malicious by multiple sources.\n\n Such cases have a high likelihood of being genuinely malicious and\n should be investigated depending on the nature of the relationship\n between the target and the affiliate. Even if the entity in question is\n not compromised, it's likely to be blocked across parts of the\n Internet due to its presence in these lists and may therefore have\n an impact on the target.\n risk: LOW\ncollections:\n - collect:\n - method: regex\n field: type\n value:\n - MALICIOUS_*\n - BLACKLIST_*\n # Filter to only affiliated entities\n - method: regex\n field: type\n value: .*AFFILIATE.*\naggregation:\n field: source.data\nanalysis:\n - method: threshold\n field: source.data\n minimum: 2\nheadline: \"Affiliated entity considered malicious by multiple sources: {source.data}\"\n" }, { "path": "correlations/multiple_malicious_cohost.yaml", "content": "id: multiple_malicious_cohost\nversion: 1\nmeta:\n name: >\n A co-hosted site was considered malicious by multiple sources\n description: >\n A co-hosted site was considered malicious by multiple sources.\n\n Such cases have a high likelihood of being genuinely malicious and\n should be investigated depending on the nature of the relationship\n between the target and the co-host. Even if the entity in question is\n not compromised, it's likely to be blocked across parts of the\n Internet due to its presence in these lists and may therefore have\n an impact on the target.\n risk: LOW\ncollections:\n - collect:\n - method: regex\n field: type\n value:\n - MALICIOUS_*\n - BLACKLIST_*\n # Filter to only co-hosted sites\n - method: regex\n field: type\n value: .*COHOST.*\naggregation:\n field: source.data\nanalysis:\n - method: threshold\n field: source.data\n minimum: 2\nheadline: \"Co-hosted site considered malicious by multiple sources: {source.data}\"\n" }, { "path": "correlations/name_only_from_pasteleak_site.yaml", "content": "id: name_only_from_pasteleak_site\nversion: 1\nmeta:\n name: Human name only from paste/leak site\n description: >\n A human name was found mentioned in a paste/leak site but\n nowhere else.\n\n Since the name was not found anywhere else, this may indicate\n that the name is in some way special, perhaps not intended to\n be publicly exposed/used or targeted in an attack.\n risk: MEDIUM\ncollections:\n - collect:\n - method: exact\n field: type\n value: INTERNET_NAME\n - method: exact\n field: source.type\n value:\n - LEAKSITE_CONTENT\n - collect:\n - method: exact\n field: type\n value: INTERNET_NAME\n - method: exact\n field: source.type\n value:\n - not LEAKSITE_CONTENT\naggregation:\n field: data\nanalysis:\n - method: first_collection_only\n field: data\nheadline: \"Human name found only in paste/leak site: {data}\"\n" }, { "path": "correlations/open_port_version.yaml", "content": "id: open_port_version\nversion: 1\nmeta:\n name: Open TCP port reveals version\n description: >\n A possible software version has been revealed on an open port. Such\n information may reveal the use of old/unpatched software used by\n the target.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: type\n value: TCP_PORT_OPEN_BANNER\n - method: regex\n field: data\n value: .*[0-9]\\.[0-9].*\n - method: regex\n field: data\n value: not .*Mime-Version.*\n - method: regex\n field: data\n value: not .*HTTP/1.*\naggregation:\n field: data\nheadline: \"Software version revealed on open port: {data}\"\n" }, { "path": "correlations/outlier_cloud.yaml", "content": "id: outlier_cloud\nversion: 1\nmeta:\n name: Outlier Cloud provider\n description: >\n A cloud provider that appeared in 10% or less of the\n total providers found. Outliers can often reveal\n entities that are rare and therefore interesting.\n\n Particularly in the case of cloud providers, an outlier\n may be an indicator of Shadow IT or unmtaintained\n infrastructure.\n risk: MEDIUM\ncollections:\n - collect:\n - method: exact\n field: type\n value: PROVIDER_HOSTING\naggregation:\n field: data\nanalysis:\n - method: outlier\n maximum_percent: 10\nheadline: \"Outlier cloud provider found: {data}\"\n" }, { "path": "correlations/outlier_country.yaml", "content": "id: outlier_country\nversion: 1\nmeta:\n name: Outlier country\n description: >\n A country that appeared in 10% or less of the total countries\n found. Outliers can often reveal entities that are rare and\n therefore interesting.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: type\n value: COUNTRY_NAME\naggregation:\n field: data\nanalysis:\n - method: outlier\n maximum_percent: 10\nheadline: \"Outlier country found: {data}\"\n" }, { "path": "correlations/outlier_email.yaml", "content": "id: outlier_email\nversion: 1\nmeta:\n name: Outlier email address found\n description: >\n An email address that appeared in 10% or less of the\n total email addresses found. Outliers can often reveal\n entities that are rare and therefore interesting.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: type\n value: EMAILADDR\naggregation:\n field: data\nanalysis:\n - method: outlier\n maximum_percent: 10\nheadline: \"Outlier email address found: {data}\"\n" }, { "path": "correlations/outlier_hostname.yaml", "content": "id: outlier_hostname\nversion: 1\nmeta:\n name: Outlier hostname\n description: >\n A hostname that appeared in 10% or less of the\n total hostnames found. Outliers can often reveal\n entities that are rare and therefore interesting.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: type\n value: INTERNET_NAME\naggregation:\n field: data\nanalysis:\n - method: outlier\n maximum_percent: 10\nheadline: \"Outlier hostname found: {data}\"\n" }, { "path": "correlations/outlier_ipaddress.yaml", "content": "id: outlier_ipaddress\nversion: 1\nmeta:\n name: Outlier IP address\n description: >\n An IP address that appeared in 10% or less of the\n total IP addresses found. Outliers can often reveal\n entities that are rare and therefore interesting.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: type\n value:\n - IP_ADDRESS\n - IPV6_ADDRESS\naggregation:\n field: data\nanalysis:\n - method: outlier\n maximum_percent: 10\nheadline: \"Outlier IP address found: {data}\"\n" }, { "path": "correlations/outlier_registrar.yaml", "content": "id: outlier_registrar\nversion: 1\nmeta:\n name: Outlier registrar\n description: >\n A registrar that appeared in 10% or less of the total registrars\n found. Outliers can often reveal entities that are rare and\n therefore interesting.\n\n Particularly in the case of registrars, an outlier may be an\n indicator of Shadow IT or unmtaintained infrastructure.\n risk: MEDIUM\ncollections:\n - collect:\n - method: exact\n field: type\n value: DOMAIN_REGISTRAR\naggregation:\n field: data\nanalysis:\n - method: outlier\n maximum_percent: 10\nheadline: \"Outlier registrar found: {data}\"\n" }, { "path": "correlations/outlier_webserver.yaml", "content": "id: outlier_webserver\nversion: 1\nmeta:\n name: Outlier web server\n description: >\n A web server that appeared in 10% or less of the\n total web servers found. Outliers can often reveal\n entities that are rare and therefore interesting.\n\n Particularly in the case of web servers, an outlier\n may be an indicator of Shadow IT or unmtaintained\n infrastructure.\n risk: MEDIUM\ncollections:\n - collect:\n - method: exact\n field: type\n value: WEBSERVER_BANNER\naggregation:\n field: data\nanalysis:\n - method: outlier\n maximum_percent: 10\nheadline: \"Outlier web server found: {data}\"\n" }, { "path": "correlations/remote_desktop_exposed.yaml", "content": "id: remote_desktop_exposed\nversion: 1\nmeta:\n name: Remote desktop technology found exposed to the Internet\n description: >\n A remote desktop technology (RDP, VNC, NoMachine)\n was found to be accessible over the Internet.\n\n Even if authentication is required such systems should not\n be exposed over the Internet due to the risk of misconfiguration\n or unpatched vulnerabilities.\n risk: HIGH\ncollections:\n - collect:\n - method: exact\n field: type\n value: TCP_PORT_OPEN\n - method: regex\n field: data\n value:\n - .*:5900$\n - .*:3389$\n - .*:4000$\naggregation:\n field: source.data\nheadline: \"Remote desktop exposed to the Internet: {source.data}\"\n" }, { "path": "correlations/root_path_needs_auth.yaml", "content": "id: root_path_needs_auth\nversion: 1\nmeta:\n name: Website root path needs authentication\n description: >\n The base path of a server needs authentication, indicating the\n server is likely sensitive or important.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: type\n value: HTTP_CODE\n - method: regex\n field: data\n value: 40[31]\n # The source might be a host or URL\n - method: regex\n field: source.data\n value:\n - ^https?://[a-zA-Z0-9\\.\\-]+/?#?$\n - ^[a-zA-Z0-9\\.\\-]+$\naggregation:\n field: source.data\nheadline: \"Base URL requires authentication: {source.data}\"\n" }, { "path": "correlations/stale_host.yaml", "content": "id: stale_host\nversion: 1\nmeta:\n name: Stale host found\n description: >\n A host appears to be stale based upon various indicators such\n as unsuccessful HTTP codes, expired SSL certificates, error\n messages, vulnerabilities and junk files found. Such hosts\n may be unmaintained, exposing the target to security risks.\n risk: HIGH\ncollections:\n - collect:\n - method: exact\n field: type\n value: SSL_CERTIFICATE_EXPIRED\n - collect:\n - method: exact\n field: type\n value: HTTP_CODE\n - method: regex\n field: data\n value: not ^[2-3].*$\n - method: regex\n field: data\n value: not 40[13]\n - collect:\n - method: exact\n field: type\n value: ERROR_MESSAGE\n - collect:\n - method: exact\n field: type\n value: JUNK_FILE\n - collect:\n - method: regex\n field: type\n value: VULNERABILITY_.*\naggregation:\n field: entity.data\nanalysis:\n - method: threshold\n field: type\n # Avoid multiple of the same type triggering. This means\n # a minimum of 3 of different types must appear in one\n # bucket.\n count_unique_only: true\n minimum: 3\nheadline: \"Potentially stale/unmaintained host: {entity.data}\"\n" }, { "path": "correlations/strong_affiliate_certs.yaml", "content": "id: strong_affiliate_certs\nversion: 1\nmeta:\n name: Affiliated host/domain with strong relationship to the target\n description: >\n A host or domain was found to be referenced multiple times from\n SSL certificates covering multiple target hosts.\n\n Strong affiliates indicate potential targets that should be\n included in the scope of scanning to ensure more complete coverage.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: module\n value:\n - sfp_sslcert\n - sfp_crt\n - sfp_certspotter\n - method: exact\n field: source.type\n value:\n - INTERNET_NAME\n - DOMAIN_NAME\n - method: exact\n field: type\n value:\n - AFFILIATE_INTERNET_NAME\n - AFFILIATE_DOMAIN_NAME\n - CO_HOSTED_SITE\n - CO_HOSTED_SITE_DOMAIN\naggregation:\n field: data\nanalysis:\n - method: threshold\n field: source.data\n count_unique_only: true\n minimum: 2\nheadline: \"Affiliate with strong target relationship: {data}\"\n" }, { "path": "correlations/strong_similardomain_crossref.yaml", "content": "id: strong_similardomain_crossref\nversion: 1\nmeta:\n name: Similar domain with strong relationship to the target\n description: >\n A similar domain was found to be referenced by the target, and\n references back, indicating a strong relationship.\n\n Strong affiliates indicate potential targets that should be\n included in the scope of scanning to ensure more complete coverage.\n risk: INFO\ncollections:\n - collect:\n - method: exact\n field: module\n value: sfp_crossref\n - method: exact\n field: type\n value: AFFILIATE_INTERNET_NAME\n - method: exact\n field: source.type\n value: SIMILARDOMAIN\naggregation:\n field: data\nheadline: \"Similar domain with strong target relationship: {data}\"\n" }, { "path": "correlations/template.yaml", "content": "# Set the ID for the rule. Ensure it matches the filename (without the .yaml\n# extension) and has no spaces or other special characters.\nid: set_a_meaningful_id_here\n\n# Use version 1 for now.\nversion: 1\n\n# The meta section sets meta attributes about the rule.\nmeta:\n\n # The human-readable name for the rule\n name: This is a briefly descriptive name.\n\n # A longer description for the rule\n description: >\n This is a more detailed description about the rule and ideally\n includes some rationale explaining the risk posed, or other\n relevant information that helps the user better understand the\n value of the rule results.\n\n You can have multiple paragraphs here too.\n\n For this example rule, our goal is to find all hosts that\n match \"foo\" or \"bar\", creating a correlation result when such a\n host is found more than once.\n\n # The risk the results of this rule posed. They can be one of:\n # - HIGH (reserved for genuine cases requiring immediate action)\n # - MEDIUM (potential high risk, needs deeper examination)\n # - LOW (potential risk, but probably low risk or false positive)\n # - INFO (no risk posed but potential interesting information)\n risk: MEDIUM\n\n# Define the collections to perform here. Refer to the README.md\n# in the correlations folder for more details.\ncollections:\n\n # Each 'collect' block represents a collection of data from\n # the SpiderFoot database. You need at least one 'collect'\n # block for the rule to actually do something.\n - collect:\n # Each 'collect' block may contain multiple blocks\n # for collecting and analyzing data. The first pulls\n # data from the SpiderFoot database, and subsequent\n # blocks further refine that data down to what you are\n # interested in. You need at least one for the 'collect'\n # block to have data.\n #\n # For example, this rule will extract all data elements\n # that exactly match the INTERNET_NAME type.\n - method: exact\n field: type\n value: INTERNET_NAME\n # This next rule will filter out all the INTERNET_NAME\n # results where data doesn't match any of the regular\n # expressions provided.\n - method: regex\n field: data\n value:\n - .*foo.*\n - .*bar.*\n # What we are now left with in this collection are all\n # INTERNET_NAME elements with data matching either .*foo.*\n # or .*bar.*\n\n# With the data from your one or more collections, you can\n# now use 'aggregation' to define how they should be grouped\n# (if at all) for analysis at the next stage (also optional).\naggregation:\n # Each data element will be placed into a bucket according\n # to its data field, which is the hostname since we have\n # INTERNET_NAME data elements.\n field: data\n\n# Analysis is an optional step to perform analysis on the\n# (optionally aggegated) data gathered to apply some criteria\n# determining whether a correlation creates results or not.\nanalysis:\n # Here we want to only generate a correlation result when\n # the value of the data element's data field (the hostname\n # in this case) appears at least two times.\n - method: threshold\n minimum: 2\n field: data\n\n# With the collection, aggegation and analysis performed,\n# we can now generate the correlation result with a title\n# that represents something meaningful to the user.\nheadline: \"A foo or bar host was found more than once: {data}\"\n" }, { "path": "correlations/vulnerability_critical.yaml", "content": "id: vulnerability_critical\nversion: 1\nmeta:\n name: At least one CRITICAL-rated vulnerability was found\n description: >\n A vulnerability rated as CRITICAL was found on a host.\n\n Such vulnerabilities are typically easily exploitable and should\n be urgently patched.\n risk: HIGH\ncollections:\n - collect:\n - method: exact\n field: type\n value: VULNERABILITY_CVE_CRITICAL\naggregation:\n field: entity.data\nheadline: \"Critical-rated vulnerability found on {entity.data}\"\n" }, { "path": "correlations/vulnerability_high.yaml", "content": "id: vulnerability_high\nversion: 1\nmeta:\n name: At least one HIGH-rated vulnerability was found\n description: >\n A vulnerability rated as HIGH was found on a host.\n\n Such vulnerabilities are typically easily exploitable and should\n be urgently patched.\n risk: HIGH\ncollections:\n - collect:\n - method: exact\n field: type\n value: VULNERABILITY_CVE_HIGH\naggregation:\n field: entity.data\nheadline: \"High-rated vulnerability found on {entity.data}\"\n" }, { "path": "correlations/vulnerability_mediumlow.yaml", "content": "id: vulnerability_mediumlow\nversion: 1\nmeta:\n name: Multiple MEDIUM or LOW-rated vulnerabilities were found\n description: >\n Multiple vulnerabilities rated as MEDIUM or LOW were found on a host.\n\n Such vulnerabilities pose a lower risk to the target, however in\n aggregate may be an indication that the host is not well maintained.\n risk: MEDIUM\ncollections:\n - collect:\n - method: exact\n field: type\n value:\n - VULNERABILITY_CVE_MEDIUM\n - VULNERABILITY_CVE_LOW\n - VULNERABILITY_GENERAL\naggregation:\n field: entity.data\nanalysis:\n - method: threshold\n field: entity.data\n minimum: 2\nheadline: \"Multiple medium/low vulnerabilities found on {entity.data}\"\n" }, { "path": "docker-compose-dev.yml", "content": "version: \"3\"\n\nservices:\n spiderfoot:\n volumes:\n - .:/home/spiderfoot\n" }, { "path": "docker-compose-full.yml", "content": "version: \"3\"\n\nservices:\n spiderfoot:\n build:\n context: ./\n dockerfile: ./Dockerfile.full\n" }, { "path": "docker-compose.yml", "content": "version: \"3\"\n\n# Basic usage:\n# $ docker-compose up\n#\n# Dev environment (code directory mapped into container):\n# $ docker-compose -f docker-compose.yml -f docker-compose-dev.yml up\n#\n# Full image (all CLI tools installed):\n# $ docker-compose -f docker-compose.yml -f docker-compose-full.yml up\n#\n# Spiderfoot data resides in a Docker volume\n#\n# $ ls -lh /var/lib/docker/volumes/spiderfoot_spiderfoot-data/_data\n# total 104K\n# drwxr-xr-x 2 user user 4.0K Sep 22 09:51 cache\n# -rw-r--r-- 1 user user 100K Sep 22 15:19 spiderfoot.db\n\nservices:\n spiderfoot:\n build:\n context: ./\n volumes:\n - spiderfoot-data:/var/lib/spiderfoot\n image: spiderfoot\n container_name: spiderfoot\n ports:\n - \"5001:5001\"\n restart: unless-stopped\n\nvolumes:\n spiderfoot-data:\n" }, { "path": "docs/Makefile", "content": "# Minimal makefile for Sphinx documentation\n#\n\n# You can set these variables from the command line, and also\n# from the environment for the first two.\nSPHINXOPTS ?=\nSPHINXBUILD ?= sphinx-build\nSOURCEDIR = .\nBUILDDIR = _build\n\n# Put it first so that \"make\" without argument is like \"make help\".\nhelp:\n\t@$(SPHINXBUILD) -M help \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n\n.PHONY: help Makefile\n\n# Catch-all target: route all unknown targets to Sphinx using the new\n# \"make mode\" option. $(O) is meant as a shortcut for $(SPHINXOPTS).\n%: Makefile\n\t@$(SPHINXBUILD) -M $@ \"$(SOURCEDIR)\" \"$(BUILDDIR)\" $(SPHINXOPTS) $(O)\n" }, { "path": "docs/conf.py", "content": "# Configuration file for the Sphinx documentation builder.\n#\n# This file only contains a selection of the most common options. For a full\n# list see the documentation:\n# https://www.sphinx-doc.org/en/master/usage/configuration.html\n\n# -- Path setup --------------------------------------------------------------\n\n# If extensions (or modules to document with autodoc) are in another directory,\n# add these directories to sys.path here. If the directory is relative to the\n# documentation root, use os.path.abspath to make it absolute, like shown here.\n#\n# import os\n# import sys\n# sys.path.insert(0, '/root/Desktop/spiderfoot/spiderfoot')\n\n\n# -- Project information -----------------------------------------------------\n\nproject = 'SpiderFoot'\ncopyright = '2012, Steve Micallef'\nauthor = 'Steve Micallef'\n\n\n# -- General configuration ---------------------------------------------------\n\n# Add any Sphinx extension module names here, as strings. They can be\n# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom\n# ones.\nextensions = [\n 'sphinx.ext.autodoc',\n 'sphinx.ext.napoleon',\n 'sphinx.ext.viewcode',\n 'sphinx.ext.todo',\n]\n\n# Add any paths that contain templates here, relative to this directory.\ntemplates_path = ['_templates']\n\n# The language for content autogenerated by Sphinx. Refer to documentation\n# for a list of supported languages.\n#\n# This is also used if you do content translation via gettext catalogs.\n# Usually you set \"language\" from the command line for these cases.\nlanguage = 'en'\n\n# List of patterns, relative to source directory, that match files and\n# directories to ignore when looking for source files.\n# This pattern also affects html_static_path and html_extra_path.\nexclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']\n\n\n# -- Options for HTML output -------------------------------------------------\n\n# The theme to use for HTML and HTML Help pages. See the documentation for\n# a list of builtin themes.\n#\nhtml_theme = 'alabaster'\n\n# Add any paths that contain custom static files (such as style sheets) here,\n# relative to this directory. They are copied after the builtin static files,\n# so a file named \"default.css\" will overwrite the builtin \"default.css\".\nhtml_static_path = ['_static']\n\n\n# -- Extension configuration -------------------------------------------------\n\n# -- Options for todo extension ----------------------------------------------\n\n# If true, `todo` and `todoList` produce output, else they produce nothing.\ntodo_include_todos = True\n" }, { "path": "docs/index.rst", "content": ".. spiderfoot documentation master file, created by\n sphinx-quickstart on Sat Jun 26 01:55:34 2021.\n You can adapt this file completely to your liking, but it should at least\n contain the root `toctree` directive.\n\nWelcome to spiderfoot's documentation!\n======================================\n\n.. toctree::\n :maxdepth: 4\n :caption: Contents:\n\n spiderfoot\n\n\nIndices and tables\n==================\n\n* :ref:`genindex`\n* :ref:`modindex`\n* :ref:`search`\n" }, { "path": "docs/make.bat", "content": "@ECHO OFF\r\n\r\npushd %~dp0\r\n\r\nREM Command file for Sphinx documentation\r\n\r\nif \"%SPHINXBUILD%\" == \"\" (\r\n\tset SPHINXBUILD=sphinx-build\r\n)\r\nset SOURCEDIR=.\r\nset BUILDDIR=_build\r\n\r\nif \"%1\" == \"\" goto help\r\n\r\n%SPHINXBUILD% >NUL 2>NUL\r\nif errorlevel 9009 (\r\n\techo.\r\n\techo.The 'sphinx-build' command was not found. Make sure you have Sphinx\r\n\techo.installed, then set the SPHINXBUILD environment variable to point\r\n\techo.to the full path of the 'sphinx-build' executable. Alternatively you\r\n\techo.may add the Sphinx directory to PATH.\r\n\techo.\r\n\techo.If you don't have Sphinx installed, grab it from\r\n\techo.http://sphinx-doc.org/\r\n\texit /b 1\r\n)\r\n\r\n%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%\r\ngoto end\r\n\r\n:help\r\n%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%\r\n\r\n:end\r\npopd\r\n" }, { "path": "docs/spiderfoot.rst", "content": "spiderfoot package\n==================\n\nSubmodules\n----------\n\nspiderfoot.db module\n--------------------\n\n.. automodule:: spiderfoot.db\n :members:\n :undoc-members:\n :show-inheritance:\n\nspiderfoot.event module\n-----------------------\n\n.. automodule:: spiderfoot.event\n :members:\n :undoc-members:\n :show-inheritance:\n\nspiderfoot.helpers module\n-------------------------\n\n.. automodule:: spiderfoot.helpers\n :members:\n :undoc-members:\n :show-inheritance:\n\nspiderfoot.plugin module\n------------------------\n\n.. automodule:: spiderfoot.plugin\n :members:\n :undoc-members:\n :show-inheritance:\n\nspiderfoot.target module\n------------------------\n\n.. automodule:: spiderfoot.target\n :members:\n :undoc-members:\n :show-inheritance:\n\nModule contents\n---------------\n\n.. automodule:: spiderfoot\n :members:\n :undoc-members:\n :show-inheritance:\n" }, { "path": "generate-certificate", "content": "#!/bin/sh\nif ! command -v openssl >/dev/null 2>&1 ; then\n echo \"Error: Could not find openssl in \\$PATH: $PATH\"\n exit 1\nfi\n\nif test -f spiderfoot.key; then\n echo \"Error: spiderfoot.key already exists\"\n exit 1\nfi\n\nif test -f spiderfoot.crt; then\n echo \"Error: spiderfoot.crt already exists\"\n exit 1\nfi\n\nopenssl req -new -newkey rsa:4096 -sha256 -x509 -days 365 -nodes -out spiderfoot.crt -keyout spiderfoot.key -subj \"/CN=localhost\"\n\nchmod 600 spiderfoot.crt\nchmod 600 spiderfoot.key\n" }, { "path": "modules/__init__.py", "content": "" }, { "path": "modules/sfp__stor_db.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_stor_db\n# Purpose: SpiderFoot plug-in for storing events to the local SpiderFoot\n# SQLite database.\n#\n# Author: Steve Micallef \n#\n# Created: 14/05/2012\n# Copyright: (c) Steve Micallef 2012\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootPlugin\n\n\nclass sfp__stor_db(SpiderFootPlugin):\n\n meta = {\n 'name': \"Storage\",\n 'summary': \"Stores scan results into the back-end SpiderFoot database. You will need this.\"\n }\n\n _priority = 0\n\n # Default options\n opts = {\n 'maxstorage': 1024, # max bytes for any piece of info stored (0 = unlimited)\n '_store': True\n }\n\n # Option descriptions\n optdescs = {\n 'maxstorage': \"Maximum bytes to store for any piece of information retrieved (0 = unlimited.)\"\n }\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n # Because this is a storage plugin, we are interested in everything so we\n # can store all events for later analysis.\n def watchedEvents(self):\n return [\"*\"]\n\n # Handle events sent to this module\n def handleEvent(self, sfEvent):\n if not self.opts['_store']:\n return\n\n if self.opts['maxstorage'] != 0 and len(sfEvent.data) > self.opts['maxstorage']:\n self.debug(\"Storing an event: \" + sfEvent.eventType)\n self.__sfdb__.scanEventStore(self.getScanId(), sfEvent, self.opts['maxstorage'])\n return\n\n self.debug(\"Storing an event: \" + sfEvent.eventType)\n self.__sfdb__.scanEventStore(self.getScanId(), sfEvent)\n\n# End of sfp__stor_db class\n" }, { "path": "modules/sfp__stor_stdout.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_stor_stdout\n# Purpose: SpiderFoot plug-in for dumping events to standard output.\n#\n# Author: Steve Micallef \n#\n# Created: 22/10/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootPlugin\n\n\nclass sfp__stor_stdout(SpiderFootPlugin):\n\n meta = {\n 'name': \"Command-line output\",\n 'summary': \"Dumps output to standard out. Used for when a SpiderFoot scan is run via the command-line.\"\n }\n\n _priority = 0\n firstEvent = True\n\n # Default options\n opts = {\n \"_format\": \"tab\", # tab, csv, json\n \"_requested\": [],\n \"_showonlyrequested\": False,\n \"_stripnewline\": False,\n \"_showsource\": False,\n \"_csvdelim\": \",\",\n \"_maxlength\": 0,\n \"_eventtypes\": dict()\n }\n\n # Option descriptions\n optdescs = {\n }\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n # Because this is a storage plugin, we are interested in everything so we\n # can store all events for later analysis.\n def watchedEvents(self):\n return [\"*\"]\n\n def output(self, event):\n d = self.opts['_csvdelim']\n if type(event.data) in [list, dict]:\n data = str(event.data)\n else:\n data = event.data\n\n if type(data) != str:\n data = str(event.data)\n\n if type(event.sourceEvent.data) in [list, dict]:\n srcdata = str(event.sourceEvent.data)\n else:\n srcdata = event.sourceEvent.data\n\n if type(srcdata) != str:\n srcdata = str(event.sourceEvent.data)\n\n if self.opts['_stripnewline']:\n data = data.replace(\"\\n\", \" \").replace(\"\\r\", \"\")\n srcdata = srcdata.replace(\"\\n\", \" \").replace(\"\\r\", \"\")\n\n if \"\" in data:\n data = data.replace(\"\", \"\").replace(\"\", \"\")\n if \"\" in srcdata:\n srcdata = srcdata.replace(\"\", \"\").replace(\"\", \"\")\n\n if self.opts['_maxlength'] > 0:\n data = data[0:self.opts['_maxlength']]\n srcdata = srcdata[0:self.opts['_maxlength']]\n\n if self.opts['_format'] == \"tab\":\n event_type = self.opts['_eventtypes'][event.eventType]\n if self.opts['_showsource']:\n print(f\"{event.module.ljust(30)}\\t{event_type.ljust(45)}\\t{srcdata}\\t{data}\")\n else:\n print(f\"{event.module.ljust(30)}\\t{event_type.ljust(45)}\\t{data}\")\n\n if self.opts['_format'] == \"csv\":\n print((event.module + d + self.opts['_eventtypes'][event.eventType] + d + srcdata + d + data))\n\n if self.opts['_format'] == \"json\":\n d = event.asDict()\n d['type'] = self.opts['_eventtypes'][event.eventType]\n if self.firstEvent:\n self.firstEvent = False\n else:\n print(\",\")\n print(json.dumps(d), end='')\n\n # Handle events sent to this module\n def handleEvent(self, sfEvent):\n if sfEvent.eventType == \"ROOT\":\n return\n\n if self.opts['_showonlyrequested']:\n if sfEvent.eventType in self.opts['_requested']:\n self.output(sfEvent)\n else:\n self.output(sfEvent)\n\n# End of sfp__stor_stdout class\n" }, { "path": "modules/sfp_abstractapi.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_abstractapi\n# Purpose: Search AbstractAPI for domain, phone and IP address information.\n#\n# Author: Krishnasis Mandal \n#\n# Created: 29/07/2021\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_abstractapi(SpiderFootPlugin):\n\n meta = {\n 'name': \"AbstractAPI\",\n 'summary': \"Look up domain, phone and IP address information from AbstractAPI.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Passive\", \"Footprint\", \"Investigate\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://app.abstractapi.com/\",\n 'model': \"FREE_NOAUTH_LIMITED\",\n 'references': [\n \"https://app.abstractapi.com/\",\n ],\n 'apiKeyInstructions': [\n \"Visit https://app.abstractapi.com/users/signup\",\n \"Register a free account\",\n \"Visit https://app.abstractapi.com/api/\",\n \"Visit each API page and click on 'Try it out'\",\n \"Your API Key will be listed under 'This is your private API key, specific to this API.'\",\n ],\n 'favIcon': \"https://app.abstractapi.com/favicon.ico\",\n 'logo': \"https://app.abstractapi.com/logo192.png\",\n 'description': \"Abstract provides powerful APIs to help you enrich any user experience or automate any workflow.\"\n }\n }\n\n opts = {\n \"companyenrichment_api_key\": \"\",\n \"phonevalidation_api_key\": \"\",\n \"ipgeolocation_api_key\": \"\",\n }\n\n optdescs = {\n \"companyenrichment_api_key\": \"AbstractAPI Company Enrichment API key.\",\n \"phonevalidation_api_key\": \"AbstractAPI Phone Validation API key.\",\n \"ipgeolocation_api_key\": \"AbstractAPI IP Geolocation API key.\",\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.errorState = False\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"DOMAIN_NAME\", \"PHONE_NUMBER\", \"IP_ADDRESS\", \"IPV6_ADDRESS\"]\n\n def producedEvents(self):\n return [\"COMPANY_NAME\", \"SOCIAL_MEDIA\", \"GEOINFO\", \"PHYSICAL_COORDINATES\", \"PROVIDER_TELCO\", \"RAW_RIR_DATA\"]\n\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from Abstract API.\")\n return None\n\n # Rate limited to one request per second\n if res['code'] == '429':\n self.error(\"You are being rate-limited by AbstractAPI.\")\n return None\n\n if res['code'] == '401':\n self.error(\"Unauthorized. Invalid AbstractAPI API key.\")\n self.errorState = True\n return None\n\n if res['code'] == '422':\n self.error(\"Usage quota reached. Insufficient API credit.\")\n self.errorState = True\n return None\n\n if res['code'] == '500' or res['code'] == '502' or res['code'] == '503':\n self.error(\"Abstract API service is unavailable\")\n self.errorState = True\n return None\n\n if res['code'] == '204':\n self.debug(\"No response data for target\")\n return None\n\n if res['code'] != '200':\n self.error(f\"Unexpected reply from AbstractAPI: {res['code']}\")\n return None\n\n if res['content'] is None:\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n def queryCompanyEnrichment(self, qry):\n \"\"\"Enrich domain with company information.\n\n Args:\n qry (str): domain name\n\n Returns:\n dict: company information\n \"\"\"\n\n api_key = self.opts['companyenrichment_api_key']\n if not api_key:\n return None\n\n params = urllib.parse.urlencode({\n 'api_key': api_key,\n 'domain': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n })\n\n res = self.sf.fetchUrl(\n f\"https://companyenrichment.abstractapi.com/v1/?{params}\",\n useragent=self.opts['_useragent']\n )\n\n time.sleep(1)\n\n if not res:\n self.debug(\"No response from AbstractAPI Company Enrichment API endpoint\")\n return None\n\n return self.parseApiResponse(res)\n\n def queryPhoneValidation(self, qry):\n \"\"\"Verify phone number and enrich with carrier and location information.\n\n Args:\n qry (str): phone number\n\n Returns:\n dict: phone number information\n \"\"\"\n\n api_key = self.opts['phonevalidation_api_key']\n if not api_key:\n return None\n\n params = urllib.parse.urlencode({\n 'api_key': api_key,\n 'phone': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n })\n\n res = self.sf.fetchUrl(\n f\"https://phonevalidation.abstractapi.com/v1/?{params}\",\n useragent=self.opts['_useragent']\n )\n\n time.sleep(1)\n\n if not res:\n self.debug(\"No response from AbstractAPI Phone Validation API endpoint\")\n return None\n\n return self.parseApiResponse(res)\n\n def queryIpGeolocation(self, qry):\n \"\"\"Enrich IP address with geolocation information.\n\n Args:\n qry (str): IPv4 address\n\n Returns:\n dict: location information\n \"\"\"\n\n api_key = self.opts['ipgeolocation_api_key']\n if not api_key:\n return None\n\n params = urllib.parse.urlencode({\n 'api_key': api_key,\n 'ip_address': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n })\n\n res = self.sf.fetchUrl(\n f\"https://ipgeolocation.abstractapi.com/v1/?{params}\",\n useragent=self.opts['_useragent']\n )\n\n time.sleep(1)\n\n if not res:\n self.debug(\"No response from AbstractAPI Phone Validation API endpoint\")\n return None\n\n return self.parseApiResponse(res)\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if self.opts[\"companyenrichment_api_key\"] == \"\" and self.opts[\"phonevalidation_api_key\"] == \"\" and self.opts[\"ipgeolocation_api_key\"] == \"\":\n self.error(\n f\"You enabled {self.__class__.__name__} but did not set any API keys!\"\n )\n self.errorState = True\n return\n\n if eventName not in self.watchedEvents():\n return\n\n if eventName == \"DOMAIN_NAME\":\n if self.opts[\"companyenrichment_api_key\"] == \"\":\n self.info(\n f\"No API key set for Company Enrichment API endpoint. Ignoring {eventData}\"\n )\n return\n\n data = self.queryCompanyEnrichment(eventData)\n\n if not data:\n return\n\n name = data.get('name')\n if not name:\n return\n\n if name == 'To Be Confirmed':\n return\n\n e = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(e)\n\n e = SpiderFootEvent(\"COMPANY_NAME\", name, self.__name__, event)\n self.notifyListeners(e)\n\n linkedin_url = data.get('linkedin_url')\n if linkedin_url:\n if linkedin_url.startswith('linkedin.com'):\n linkedin_url = f\"https://{linkedin_url}\"\n e = SpiderFootEvent(\"SOCIAL_MEDIA\", f\"LinkedIn (Company): {linkedin_url}\", self.__name__, event)\n self.notifyListeners(e)\n\n locality = data.get('locality')\n country = data.get('country')\n geoinfo = ', '.join(\n filter(None, [locality, country])\n )\n\n if geoinfo:\n e = SpiderFootEvent(\"GEOINFO\", geoinfo, self.__name__, event)\n self.notifyListeners(e)\n\n elif eventName == \"PHONE_NUMBER\":\n if self.opts[\"phonevalidation_api_key\"] == \"\":\n self.info(\n f\"No API key set for Phone Validation API endpoint. Ignoring {eventData}\"\n )\n return\n\n data = self.queryPhoneValidation(eventData)\n\n if not data:\n return\n\n valid = data.get('valid')\n if not valid:\n return\n\n e = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(e)\n\n carrier = data.get('carrier')\n if carrier:\n e = SpiderFootEvent(\"PROVIDER_TELCO\", carrier, self.__name__, event)\n self.notifyListeners(e)\n\n location = data.get('location')\n country = data.get('country')\n country_name = None\n if country:\n country_name = country.get('name')\n\n geoinfo = ', '.join(\n filter(None, [location, country_name])\n )\n\n if geoinfo:\n e = SpiderFootEvent(\"GEOINFO\", geoinfo, self.__name__, event)\n self.notifyListeners(e)\n\n elif eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']:\n if self.opts[\"ipgeolocation_api_key\"] == \"\":\n self.info(\n f\"No API key set for IP Geolocation API endpoint. Ignoring {eventData}\"\n )\n return\n\n data = self.queryIpGeolocation(eventData)\n\n if not data:\n return\n\n e = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(e)\n\n geoinfo = ', '.join(\n [\n _f for _f in [\n data.get('city'),\n data.get('region'),\n data.get('postal_code'),\n data.get('country'),\n data.get('continent'),\n ] if _f\n ]\n )\n\n if geoinfo:\n e = SpiderFootEvent(\"GEOINFO\", geoinfo, self.__name__, event)\n self.notifyListeners(e)\n\n latitude = data.get('latitude')\n longitude = data.get('longitude')\n if latitude and longitude:\n e = SpiderFootEvent(\"PHYSICAL_COORDINATES\", f\"{latitude}, {longitude}\", self.__name__, event)\n self.notifyListeners(e)\n\n# End of sfp_abstractapi class\n" }, { "path": "modules/sfp_abusech.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_abusech\n# Purpose: Check if a host/domain, IP address or netblock is malicious according\n# to Abuse.ch.\n#\n# Author: steve@binarypool.com\n#\n# Created: 14/12/2013\n# Copyright: (c) Steve Micallef, 2013\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom netaddr import IPAddress, IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_abusech(SpiderFootPlugin):\n\n meta = {\n 'name': \"abuse.ch\",\n 'summary': \"Check if a host/domain, IP address or netblock is malicious according to Abuse.ch.\",\n 'flags': [],\n 'useCases': [\"Passive\", \"Investigate\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://www.abuse.ch\",\n 'model': \"FREE_AUTH_UNLIMITED\",\n 'references': [\n \"https://feodotracker.abuse.ch/\",\n \"https://sslbl.abuse.ch/\",\n \"https://urlhaus.abuse.ch/\",\n ],\n 'apiKeyInstructions': [\n \"Visit https://bazaar.abuse.ch/api#api_key\",\n \"Login using a Twitter Account\",\n \"Navigate to 'Account Settings'\",\n \"The API key is listed under 'Your API Key'\",\n \"Visit https://urlhaus.abuse.ch/api/\",\n \"Login using a Twitter Account at https://urlhaus.abuse.ch/login/\",\n \"Navigate to https://urlhaus.abuse.ch/api/#account\",\n \"The API Key is listed under 'API-Key'\"\n ],\n 'favIcon': \"https://abuse.ch/favicon.ico\",\n 'logo': \"https://abuse.ch/images/abusech.svg\",\n 'description': \"abuse.ch is operated by a random swiss guy fighting malware for non-profit, \"\n \"running a couple of projects helping internet service providers and \"\n \"network operators protecting their infrastructure from malware.\\n\"\n \"IT-Security researchers, vendors and law enforcement agencies rely on data from abuse.ch,\"\n \"trying to make the internet a safer place.\",\n }\n }\n\n # Default options\n opts = {\n 'abusefeodoip': True,\n 'abusesslblip': True,\n 'abuseurlhaus': True,\n 'checkaffiliates': True,\n 'checkcohosts': True,\n 'cacheperiod': 18,\n 'checknetblocks': True,\n 'checksubnets': True\n }\n\n # Option descriptions\n optdescs = {\n 'abusefeodoip': \"Enable abuse.ch Feodo IP check?\",\n 'abusesslblip': \"Enable abuse.ch SSL Backlist IP check?\",\n 'abuseurlhaus': \"Enable abuse.ch URLhaus check?\",\n 'checkaffiliates': \"Apply checks to affiliates?\",\n 'checkcohosts': \"Apply checks to sites found to be co-hosted on the target's IP?\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\",\n 'checknetblocks': \"Report if any malicious IPs are found within owned netblocks?\",\n 'checksubnets': \"Check if any malicious IPs are found within the same subnet of the target?\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"IP_ADDRESS\",\n \"NETBLOCK_MEMBER\",\n \"AFFILIATE_INTERNET_NAME\",\n \"AFFILIATE_IPADDR\",\n \"CO_HOSTED_SITE\",\n \"NETBLOCK_OWNER\"\n ]\n\n # What events this module produces\n def producedEvents(self):\n return [\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_SUBNET\",\n \"MALICIOUS_COHOST\",\n \"MALICIOUS_NETBLOCK\"\n ]\n\n def queryFeodoTrackerBlacklist(self, target, targetType):\n blacklist = self.retrieveFeodoTrackerBlacklist()\n\n if not blacklist:\n return False\n\n if targetType == \"ip\":\n if target in blacklist:\n self.debug(f\"IP address {target} found in Abuse.ch Feodo Tracker.\")\n return True\n elif targetType == \"netblock\":\n netblock = IPNetwork(target)\n for ip in blacklist:\n if IPAddress(ip) in netblock:\n self.debug(f\"IP address {ip} found within netblock/subnet {target} in Abuse.ch Feodo Tracker.\")\n return True\n\n return False\n\n def retrieveFeodoTrackerBlacklist(self):\n blacklist = self.sf.cacheGet('abusech_feodo', 24)\n\n if blacklist is not None:\n return self.parseFeodoTrackerBlacklist(blacklist)\n\n res = self.sf.fetchUrl(\n \"https://feodotracker.abuse.ch/downloads/ipblocklist.txt\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from Abuse.ch Abuse.ch Feodo Tracker.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from Abuse.ch Feodo Tracker\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"abusech_feodo\", res['content'])\n\n return self.parseFeodoTrackerBlacklist(res['content'])\n\n def parseFeodoTrackerBlacklist(self, blacklist):\n \"\"\"Parse plaintext blacklist\n\n Args:\n blacklist (str): plaintext blacklist from Abuse.ch Feodo Tracker\n\n Returns:\n list: list of blacklisted IP addresses\n \"\"\"\n ips = list()\n\n if not blacklist:\n return ips\n\n for ip in blacklist.split('\\n'):\n ip = ip.strip()\n if not ip:\n continue\n if ip.startswith('#'):\n continue\n if not self.sf.validIP(ip):\n continue\n ips.append(ip)\n\n return ips\n\n def querySslBlacklist(self, target, targetType):\n blacklist = self.retrieveSslBlacklist()\n\n if not blacklist:\n return False\n\n if targetType == \"ip\":\n if target in blacklist:\n self.debug(f\"IP address {target} found in Abuse.ch SSL Blacklist.\")\n return True\n elif targetType == \"netblock\":\n netblock = IPNetwork(target)\n for ip in blacklist:\n if IPAddress(ip) in netblock:\n self.debug(f\"IP address {ip} found within netblock/subnet {target} in Abuse.ch SSL Blacklist.\")\n return True\n\n return False\n\n def retrieveSslBlacklist(self):\n blacklist = self.sf.cacheGet('abusech_ssl', 24)\n\n if blacklist is not None:\n return self.parseSslBlacklist(blacklist)\n\n res = self.sf.fetchUrl(\n \"https://sslbl.abuse.ch/blacklist/sslipblacklist.csv\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from Abuse.ch Abuse.ch Feodo Tracker.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from Abuse.ch Feodo Tracker\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"abusech_ssl\", res['content'])\n\n return self.parseSslBlacklist(res['content'])\n\n def parseSslBlacklist(self, blacklist):\n \"\"\"Parse plaintext blacklist\n\n Args:\n blacklist (str): CSV blacklist from Abuse.ch SSL Blacklist\n\n Returns:\n list: list of blacklisted IP addresses\n \"\"\"\n ips = list()\n\n if not blacklist:\n return ips\n\n for line in blacklist.split('\\n'):\n line = line.strip()\n if not line:\n continue\n if line.startswith('#'):\n continue\n csv = line.split(',')\n if len(csv) < 2:\n continue\n ip = csv[1]\n if not self.sf.validIP(ip):\n continue\n ips.append(ip)\n\n return ips\n\n def queryUrlHausBlacklist(self, target, targetType):\n blacklist = self.retrieveUrlHausBlacklist()\n\n if not blacklist:\n return False\n\n if targetType == \"ip\":\n if target in blacklist:\n self.debug(f\"IP address {target} found in Abuse.ch URL Haus Blacklist.\")\n return True\n elif targetType == \"netblock\":\n netblock = IPNetwork(target)\n for ip in blacklist:\n if IPAddress(ip) in netblock:\n self.debug(f\"IP address {ip} found within netblock/subnet {target} in Abuse.ch URL Haus Blacklist.\")\n return True\n elif targetType == \"domain\":\n if target.lower() in blacklist:\n self.debug(f\"Host name {target} found in Abuse.ch URL Haus Blacklist.\")\n return True\n\n return False\n\n def retrieveUrlHausBlacklist(self):\n blacklist = self.sf.cacheGet('abusech_urlhaus', 24)\n\n if blacklist is not None:\n return self.parseUrlHausBlacklist(blacklist)\n\n res = self.sf.fetchUrl(\n \"https://urlhaus.abuse.ch/downloads/csv_recent/\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from Abuse.ch URL Haus.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from Abuse.ch URL Haus\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"abusech_urlhaus\", res['content'])\n\n return self.parseUrlHausBlacklist(res['content'])\n\n def parseUrlHausBlacklist(self, blacklist):\n \"\"\"Parse plaintext blacklist\n\n Args:\n blacklist (str): plaintext blacklist from Abuse.ch URL Haus\n\n Returns:\n list: list of blacklisted hosts\n \"\"\"\n hosts = list()\n\n if not blacklist:\n return hosts\n\n for line in blacklist.split('\\n'):\n if not line:\n continue\n if line.startswith('#'):\n continue\n\n # Note: URL parsing and validation with sf.validHost() is too slow to use here\n url = line.strip().lower()\n if len(url.split(\"/\")) < 3:\n continue\n host = url.split(\"/\")[2].split(':')[0]\n if not host:\n continue\n if \".\" not in host:\n continue\n hosts.append(host)\n\n return hosts\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName == 'IP_ADDRESS':\n targetType = 'ip'\n evtType = 'MALICIOUS_IPADDR'\n elif eventName == 'AFFILIATE_IPADDR':\n if not self.opts.get('checkaffiliates', False):\n return\n targetType = 'ip'\n evtType = 'MALICIOUS_AFFILIATE_IPADDR'\n elif eventName == 'NETBLOCK_OWNER':\n if not self.opts.get('checknetblocks', False):\n return\n targetType = 'netblock'\n evtType = 'MALICIOUS_NETBLOCK'\n elif eventName == 'NETBLOCK_MEMBER':\n if not self.opts.get('checksubnets', False):\n return\n targetType = 'netblock'\n evtType = 'MALICIOUS_SUBNET'\n elif eventName == \"INTERNET_NAME\":\n targetType = 'domain'\n evtType = \"MALICIOUS_INTERNET_NAME\"\n elif eventName == 'AFFILIATE_INTERNET_NAME':\n if not self.opts.get('checkaffiliates', False):\n return\n targetType = 'domain'\n evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'\n elif eventName == 'CO_HOSTED_SITE':\n if not self.opts.get('checkcohosts', False):\n return\n targetType = 'domain'\n evtType = 'MALICIOUS_COHOST'\n else:\n return\n\n if targetType in ['ip', 'netblock']:\n self.debug(f\"Checking maliciousness of {eventData} ({eventName}) with Abuse.ch Feodo Tracker\")\n if self.queryFeodoTrackerBlacklist(eventData, targetType):\n url = \"https://feodotracker.abuse.ch/downloads/ipblocklist.txt\"\n text = f\"Abuse.ch Feodo Tracker [{eventData}]\\n{url}\"\n evt = SpiderFootEvent(evtType, text, self.__name__, event)\n self.notifyListeners(evt)\n\n self.debug(f\"Checking maliciousness of {eventData} ({eventName}) with Abuse.ch SSL Blacklist\")\n if self.querySslBlacklist(eventData, targetType):\n url = \"https://sslbl.abuse.ch/blacklist/sslipblacklist.csv\"\n text = f\"Abuse.ch SSL Blacklist [{eventData}]\\n{url}\"\n evt = SpiderFootEvent(evtType, text, self.__name__, event)\n self.notifyListeners(evt)\n\n if targetType in ['ip', 'domain']:\n self.debug(f\"Checking maliciousness of {eventData} ({eventName}) with Abuse.ch URL Haus\")\n if self.queryUrlHausBlacklist(eventData, targetType):\n url = \"https://urlhaus.abuse.ch/downloads/csv_recent/\"\n text = f\"Abuse.ch URL Haus Blacklist [{eventData}]\\n{url}\"\n evt = SpiderFootEvent(evtType, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_abusech class\n" }, { "path": "modules/sfp_abuseipdb.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_abuseipdb\n# Purpose: Check if an IP address is malicious according to AbuseIPDB.com.\n#\n# Author: steve@binarypool.com\n#\n# Created: 06/09/2018\n# Copyright: (c) Steve Micallef, 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_abuseipdb(SpiderFootPlugin):\n\n meta = {\n 'name': \"AbuseIPDB\",\n 'summary': \"Check if an IP address is malicious according to AbuseIPDB.com blacklist.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Passive\", \"Investigate\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://www.abuseipdb.com\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://docs.abuseipdb.com/#introduction\",\n \"https://www.abuseipdb.com/fail2ban.html\",\n \"https://www.abuseipdb.com/csf\",\n \"https://www.abuseipdb.com/suricata\",\n \"https://www.abuseipdb.com/splunk\",\n \"https://www.abuseipdb.com/categories\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://www.abuseipdb.com/pricing\",\n \"Select the plan required\",\n \"Register a new account with an email\",\n \"Navigate to https://www.abuseipdb.com/account/api\",\n \"The API Key is listed under 'Keys'\"\n ],\n 'favIcon': \"https://www.abuseipdb.com/favicon.ico\",\n 'logo': \"https://www.abuseipdb.com/img/abuseipdb.png.pagespeed.ce.CI8T6WsXU7.png\",\n 'description': \"AbuseIPDB is a project dedicated to helping combat the spread of hackers,\"\n \"spammers, and abusive activity on the internet.\\n\"\n \"Our mission is to help make Web safer by providing a central blacklist for\"\n \"webmasters, system administrators, and other interested parties to\"\n \"report and find IP addresses that have been associated with malicious activity online.\"\n }\n }\n\n opts = {\n 'api_key': '',\n 'confidenceminimum': 90,\n 'checkaffiliates': True,\n 'limit': 10000\n }\n\n optdescs = {\n 'api_key': \"AbuseIPDB.com API key.\",\n 'confidenceminimum': \"The minimium AbuseIPDB confidence level to require.\",\n 'checkaffiliates': \"Apply checks to affiliates?\",\n 'limit': 'Maximum number of results to retrieve.',\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"AFFILIATE_IPV6_ADDRESS\",\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n ]\n\n def queryBlacklist(self):\n blacklist = self.sf.cacheGet('abuseipdb', 24)\n\n if blacklist is not None:\n return self.parseBlacklist(blacklist)\n\n headers = {\n 'Key': self.opts['api_key'],\n 'Accept': \"text/plain\"\n }\n\n params = urllib.parse.urlencode({\n 'confidenceMinimum': self.opts['confidenceminimum'],\n 'limit': self.opts['limit'],\n 'plaintext': '1'\n })\n\n res = self.sf.fetchUrl(\n f\"https://api.abuseipdb.com/api/v2/blacklist?{params}\",\n timeout=60, # retrieving 10,000 results (default) or more can sometimes take a while\n useragent=self.opts['_useragent'],\n headers=headers\n )\n\n time.sleep(1)\n\n if res['code'] == '429':\n self.error(\"You are being rate-limited by AbuseIPDB\")\n self.errorState = True\n return None\n\n if res['code'] != \"200\":\n self.error(f\"Error retrieving search results, code {res['code']}\")\n self.errorState = True\n return None\n\n if res['code'] != \"200\":\n self.error(\"Error retrieving search results from AbuseIPDB\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from AbuseIPDB\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"abuseipdb\", res['content'])\n\n return self.parseBlacklist(res['content'])\n\n def parseBlacklist(self, blacklist):\n \"\"\"Parse plaintext blacklist\n\n Args:\n blacklist (str): plaintext blacklist from AbuseIPDB\n\n Returns:\n list: list of blacklisted IP addresses\n \"\"\"\n ips = list()\n\n if not blacklist:\n return ips\n\n for ip in blacklist.split('\\n'):\n ip = ip.strip()\n if ip.startswith('#'):\n continue\n if not self.sf.validIP(ip) and not self.sf.validIP6(ip):\n continue\n ips.append(ip)\n\n return ips\n\n def queryIpAddress(self, ip):\n \"\"\"Query API for an IPv4 or IPv6 address.\n\n Note: Currently unused.\n\n Args:\n ip (str): IP address\n\n Returns:\n str: API response as JSON\n \"\"\"\n\n headers = {\n 'Key': self.opts['api_key'],\n 'Accept': 'application/json',\n }\n\n params = urllib.parse.urlencode({\n 'ipAddress': ip,\n 'maxAgeInDays': 30,\n })\n\n res = self.sf.fetchUrl(\n f\"https://api.abuseipdb.com/api/v2/check?{params}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n headers=headers\n )\n\n time.sleep(1)\n\n if res['code'] == '429':\n self.error(\"You are being rate-limited by AbuseIPDB\")\n self.errorState = True\n return None\n\n if res['code'] != \"200\":\n self.error(\"Error retrieving search results from AbuseIPDB\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from AbuseIPDB\")\n self.errorState = True\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n return None\n\n def queryNetblock(self, ip):\n \"\"\"Query API for a netblock.\n\n Note: Currently unused.\n\n Args:\n ip (str): CIDR range\n\n Returns:\n str: API response as JSON\n \"\"\"\n\n headers = {\n 'Key': self.opts['api_key'],\n 'Accept': 'application/json',\n }\n\n params = urllib.parse.urlencode({\n 'ipAddress': ip,\n 'maxAgeInDays': 30,\n })\n\n res = self.sf.fetchUrl(\n f\"https://api.abuseipdb.com/api/v2/check-block?{params}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n headers=headers\n )\n\n time.sleep(1)\n\n if res['code'] == '429':\n self.error(\"You are being rate-limited by AbuseIPDB\")\n self.errorState = True\n return None\n\n if res['code'] != \"200\":\n self.error(\"Error retrieving search results from AbuseIPDB\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from AbuseIPDB\")\n self.errorState = True\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\n f\"You enabled {self.__class__.__name__} but did not set an API key!\"\n )\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName.startswith(\"AFFILIATE\") and not self.opts['checkaffiliates']:\n return\n\n if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']:\n blacklist_type = \"BLACKLISTED_IPADDR\"\n malicious_type = 'MALICIOUS_IPADDR'\n elif eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS']:\n blacklist_type = \"BLACKLISTED_AFFILIATE_IPADDR\"\n malicious_type = 'MALICIOUS_AFFILIATE_IPADDR'\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n self.debug(f\"Checking maliciousness of IP address {eventData} with AbuseIPDB\")\n\n blacklist = self.queryBlacklist()\n\n if not blacklist:\n return\n\n if eventData not in blacklist:\n return\n\n self.info(f\"Malicious IP address {eventData} found in AbuseIPDB blacklist\")\n\n url = f\"https://www.abuseipdb.com/check/{eventData}\"\n\n evt = SpiderFootEvent(\n malicious_type,\n f\"AbuseIPDB [{eventData}]\\n{url}\",\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(\n blacklist_type,\n f\"AbuseIPDB [{eventData}]\\n{url}\",\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n\n# End of sfp_abuseipdb class\n" }, { "path": "modules/sfp_abusix.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_abusix\n# Purpose: SpiderFoot plug-in for looking up whether IPs/Netblocks/Domains\n# appear in the Abusix Mail Intelligence blacklist.\n#\n# Author: \n#\n# Created: 2021-10-17\n# Copyright: (c) bcoles 2021\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport ipaddress\n\nfrom netaddr import IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_abusix(SpiderFootPlugin):\n\n meta = {\n 'name': \"Abusix Mail Intelligence\",\n 'summary': \"Check if a netblock or IP address is in the Abusix Mail Intelligence blacklist.\",\n 'flags': ['apikey'],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://abusix.org/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://abusix.com/products/abusix-mail-intelligence/\",\n \"https://docs.abusix.com/105726-setup-abusix-mail-intelligence/ami%2Fsetup%2Fexample-queries\",\n \"https://docs.abusix.com/105725-detailed-list-information/ami%2Freturn-codes\",\n ],\n 'apiKeyInstructions': [\n \"Visit https://app.abusix.com/signup\",\n \"Register a free account\",\n \"Browse to 'Account Settings' page\",\n \"The API key is listed on the 'Email protection' page.\"\n ],\n 'logo': \"https://abusix.com/wp-content/uploads/2020/10/Footer_logo.png\",\n 'description': \"Abusix Mail Intelligence is an innovative set of blocklists (RBL/DNSBL) \"\n \"that adds real-time threat data to your existing email protection. \"\n \"Considered as the first line of defense, blocklists help to prevent email-borne threats \"\n \"such as spam and malware from entering your network.\"\n }\n }\n\n opts = {\n 'api_key': \"\",\n 'checkaffiliates': True,\n 'checkcohosts': True,\n 'netblocklookup': True,\n 'maxnetblock': 24,\n 'maxv6netblock': 120,\n 'subnetlookup': True,\n 'maxsubnet': 24,\n 'maxv6subnet': 120,\n }\n\n optdescs = {\n 'api_key': \"Abusix Mail Intelligence API key.\",\n 'checkaffiliates': \"Apply checks to affiliates?\",\n 'checkcohosts': \"Apply checks to sites found to be co-hosted on the target's IP?\",\n 'netblocklookup': \"Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?\",\n 'maxnetblock': \"If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'maxv6netblock': \"If looking up owned netblocks, the maximum IPv6 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'subnetlookup': \"Look up all IPs on subnets which your target is a part of for blacklisting?\",\n 'maxsubnet': \"If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'maxv6subnet': \"If looking up subnets, the maximum IPv6 subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n }\n\n results = None\n errorState = False\n\n checks = {\n \"127.0.0.2\": \"black\",\n \"127.0.0.3\": \"black (composite/heuristic)\",\n \"127.0.0.4\": \"exploit / authbl\",\n \"127.0.0.5\": \"forged\",\n \"127.0.0.6\": \"backscatter\",\n \"127.0.0.11\": \"policy (generic rDNS)\",\n \"127.0.0.12\": \"policy (missing rDNS)\",\n \"127.0.0.100\": \"noip\",\n \"127.0.1.1\": \"dblack\",\n \"127.0.1.2\": \"dblack (Newly Observed Domain)\",\n \"127.0.1.3\": \"dblack (Unshortened)\",\n \"127.0.2.1\": \"white\",\n \"127.0.3.1\": \"shorthash\",\n \"127.0.3.2\": \"diskhash\",\n \"127.0.4.1\": \"btc-wallets\",\n \"127.0.5.1\": \"attachhash\",\n }\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.errorState = False\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n 'IP_ADDRESS',\n 'IPV6_ADDRESS',\n 'AFFILIATE_IPADDR',\n 'AFFILIATE_IPV6_ADDRESS',\n \"NETBLOCK_MEMBER\",\n \"NETBLOCKV6_MEMBER\",\n \"NETBLOCK_OWNER\",\n \"NETBLOCKV6_OWNER\",\n 'INTERNET_NAME',\n 'AFFILIATE_INTERNET_NAME',\n 'CO_HOSTED_SITE',\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"BLACKLISTED_SUBNET\",\n \"BLACKLISTED_NETBLOCK\",\n \"BLACKLISTED_INTERNET_NAME\",\n \"BLACKLISTED_AFFILIATE_INTERNET_NAME\",\n \"BLACKLISTED_COHOST\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_NETBLOCK\",\n \"MALICIOUS_SUBNET\",\n \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_COHOST\",\n ]\n\n def reverseIpAddress(self, ipaddr):\n if not self.sf.validIP(ipaddr):\n self.debug(f\"Invalid IPv4 address {ipaddr}\")\n return None\n return ipaddress.ip_address(ipaddr).reverse_pointer.replace('.in-addr.arpa', '')\n\n def reverseIp6Address(self, ipaddr):\n if not self.sf.validIP6(ipaddr):\n self.debug(f\"Invalid IPv6 address {ipaddr}\")\n return None\n return ipaddress.ip_address(ipaddr).reverse_pointer.replace('.ip6.arpa', '')\n\n def query(self, qaddr):\n \"\"\"Query Abusix Mail Intelligence DNS.\n\n Args:\n qaddr (str): Host name or IPv4 address.\n\n Returns:\n list: Abusix DNS entries\n \"\"\"\n if self.sf.validIP(qaddr):\n lookup = f\"{self.reverseIpAddress(qaddr)}.{self.opts['api_key']}.combined.mail.abusix.zone\"\n elif self.sf.validIP6(qaddr):\n lookup = f\"{self.reverseIp6Address(qaddr)}.{self.opts['api_key']}.combined.mail.abusix.zone\"\n else:\n lookup = f\"{qaddr}.{self.opts['api_key']}.combined.mail.abusix.zone\"\n\n self.debug(f\"Checking Abusix Mail Intelligence blacklist: {lookup}\")\n\n try:\n return self.sf.resolveHost(lookup)\n except Exception as e:\n self.debug(f\"Abusix Mail Intelligence did not resolve {qaddr} / {lookup}: {e}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if not self.opts['api_key']:\n self.error(f\"You enabled {self.__class__.__name__} but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS']:\n if not self.opts.get('checkaffiliates', False):\n return\n malicious_type = \"MALICIOUS_AFFILIATE_IPADDR\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_IPADDR\"\n elif eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']:\n malicious_type = \"MALICIOUS_IPADDR\"\n blacklist_type = \"BLACKLISTED_IPADDR\"\n elif eventName in ['NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']:\n if not self.opts['subnetlookup']:\n return\n\n if eventName == 'NETBLOCKV6_MEMBER':\n max_subnet = self.opts['maxv6subnet']\n else:\n max_subnet = self.opts['maxsubnet']\n\n if IPNetwork(eventData).prefixlen < max_subnet:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}\")\n return\n\n malicious_type = \"MALICIOUS_SUBNET\"\n blacklist_type = \"BLACKLISTED_SUBNET\"\n elif eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']:\n if not self.opts['netblocklookup']:\n return\n\n if eventName == 'NETBLOCKV6_OWNER':\n max_netblock = self.opts['maxv6netblock']\n else:\n max_netblock = self.opts['maxnetblock']\n\n if IPNetwork(eventData).prefixlen < max_netblock:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}\")\n return\n\n malicious_type = \"MALICIOUS_NETBLOCK\"\n blacklist_type = \"BLACKLISTED_NETBLOCK\"\n elif eventName == \"INTERNET_NAME\":\n malicious_type = \"MALICIOUS_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n if not self.opts.get('checkaffiliates', False):\n return\n malicious_type = \"MALICIOUS_AFFILIATE_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n if not self.opts.get('checkcohosts', False):\n return\n malicious_type = \"MALICIOUS_COHOST\"\n blacklist_type = \"BLACKLISTED_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n addrs = list()\n if eventName.startswith(\"NETBLOCK\"):\n for addr in IPNetwork(eventData):\n addrs.append(str(addr))\n else:\n addrs.append(eventData)\n\n for addr in addrs:\n if self.checkForStop():\n return\n\n if self.errorState:\n return\n\n res = self.query(addr)\n\n self.results[addr] = True\n\n if not res:\n continue\n\n self.debug(f\"{addr} found in Abusix Mail Intelligence DNS\")\n\n for result in res:\n k = str(result)\n\n if k not in self.checks:\n if 'mail.abusix.zone' not in result:\n # This is an error. The \"checks\" dict may need to be updated.\n self.error(f\"Abusix Mail Intelligence resolved address {addr} to unknown IP address {result} not found in Abusix Mail Intelligence list.\")\n continue\n\n text = f\"Abusix Mail Intelligence - {self.checks[k]} [{addr}]\\nhttps://lookup.abusix.com/search?q={addr}\"\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_abusix class\n" }, { "path": "modules/sfp_accounts.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_accounts\n# Purpose: Identify the existence of a given acount on various sites thanks\n# to Micah Hoffman's (https://github.com/WebBreacher) list.\n#\n# Author: Steve Micallef \n#\n# Created: 18/02/2015\n# Copyright: (c) Steve Micallef 2015\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport random\nimport threading\nimport time\nfrom queue import Empty as QueueEmpty\nfrom queue import Queue\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_accounts(SpiderFootPlugin):\n\n meta = {\n 'name': \"Account Finder\",\n 'summary': \"Look for possible associated accounts on over 500 social and other websites such as Instagram, Reddit, etc.\",\n 'useCases': [\"Footprint\", \"Passive\"],\n 'categories': [\"Social Media\"]\n }\n\n # Default options\n opts = {\n \"ignorenamedict\": True,\n \"ignoreworddict\": True,\n \"musthavename\": True,\n \"userfromemail\": True,\n \"permutate\": False,\n \"usernamesize\": 4,\n \"_maxthreads\": 20\n }\n\n # Option descriptions\n optdescs = {\n \"ignorenamedict\": \"Don't bother looking up names that are just stand-alone first names (too many false positives).\",\n \"ignoreworddict\": \"Don't bother looking up names that appear in the dictionary.\",\n \"musthavename\": \"The username must be mentioned on the social media page to consider it valid (helps avoid false positives).\",\n \"userfromemail\": \"Extract usernames from e-mail addresses at all? If disabled this can reduce false positives for common usernames but for highly unique usernames it would result in missed accounts.\",\n \"permutate\": \"Look for the existence of account name permutations. Useful to identify fraudulent social media accounts or account squatting.\",\n \"usernamesize\": \"The minimum length of a username to query across social media sites. Helps avoid false positives for very common short usernames.\",\n \"_maxthreads\": \"Maximum threads\"\n }\n\n results = None\n reportedUsers = list()\n siteResults = dict()\n sites = list()\n errorState = False\n distrustedChecked = False\n lock = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.commonNames = list()\n self.reportedUsers = list()\n self.errorState = False\n self.distrustedChecked = False\n self.__dataSource__ = \"Social Media\"\n self.lock = threading.Lock()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n self.commonNames = SpiderFootHelpers.humanNamesFromWordlists()\n self.words = SpiderFootHelpers.dictionaryWordsFromWordlists()\n\n content = self.sf.cacheGet(\"sfaccountsv2\", 48)\n if content is None:\n url = \"https://raw.githubusercontent.com/WebBreacher/WhatsMyName/main/wmn-data.json\"\n data = self.sf.fetchUrl(url, useragent=\"SpiderFoot\")\n\n if data['content'] is None:\n self.error(f\"Unable to fetch {url}\")\n self.errorState = True\n return\n\n content = data['content']\n self.sf.cachePut(\"sfaccountsv2\", content)\n\n try:\n self.sites = [site for site in json.loads(content)['sites'] if not site.get('valid', True) is False]\n except Exception as e:\n self.error(f\"Unable to parse social media accounts list: {e}\")\n self.errorState = True\n return\n\n def watchedEvents(self):\n return [\"EMAILADDR\", \"DOMAIN_NAME\", \"HUMAN_NAME\", \"USERNAME\"]\n\n def producedEvents(self):\n return [\"USERNAME\", \"ACCOUNT_EXTERNAL_OWNED\",\n \"SIMILAR_ACCOUNT_EXTERNAL\"]\n\n def checkSite(self, name, site):\n if 'uri_check' not in site:\n return\n\n url = site['uri_check'].format(account=name)\n if 'uri_pretty' in site:\n ret_url = site['uri_pretty'].format(account=name)\n else:\n ret_url = url\n retname = f\"{site['name']} (Category: {site['cat']})\\n{ret_url}\"\n\n post = None\n if site.get('post_body'):\n post = site['post_body']\n\n res = self.sf.fetchUrl(\n url,\n postData=post,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n noLog=True,\n verify=False\n )\n\n if not res['content']:\n with self.lock:\n self.siteResults[retname] = False\n return\n\n if site.get('e_code') != site.get('m_code'):\n if res['code'] != str(site.get('e_code')):\n with self.lock:\n self.siteResults[retname] = False\n return\n\n if site.get('e_string') not in res['content'] or (site.get('m_string') and site.get('m_string') in res['content']):\n with self.lock:\n self.siteResults[retname] = False\n return\n\n if self.opts['musthavename']:\n if name.lower() not in res['content'].lower():\n self.debug(f\"Skipping {site['name']} as username not mentioned.\")\n with self.lock:\n self.siteResults[retname] = False\n return\n\n # Some sites can't handle periods so treat bob.abc and bob as the same\n # TODO: fix this once WhatsMyName has support for usernames with '.'\n if \".\" in name:\n firstname = name.split(\".\")[0]\n if firstname + \"<\" in res['content'] or firstname + '\"' in res['content']:\n with self.lock:\n self.siteResults[retname] = False\n return\n\n with self.lock:\n self.siteResults[retname] = True\n\n def checkSites(self, username, sites=None):\n def processSiteQueue(username, queue):\n try:\n while True:\n site = queue.get(timeout=0.1)\n try:\n self.checkSite(username, site)\n except Exception as e:\n self.debug(f'Thread {threading.current_thread().name} exception: {e}')\n except QueueEmpty:\n return\n\n startTime = time.monotonic()\n\n # results will be collected in siteResults\n self.siteResults = {}\n\n sites = self.sites if sites is None else sites\n\n # load the queue\n queue = Queue()\n for site in sites:\n queue.put(site)\n\n # start the scan threads\n threads = []\n for i in range(min(len(sites), self.opts['_maxthreads'])):\n thread = threading.Thread(\n name=f'sfp_accounts_scan_{i}',\n target=processSiteQueue,\n args=(username, queue))\n thread.start()\n threads.append(thread)\n\n # wait for all scan threads to finish\n while threads:\n threads.pop(0).join()\n\n duration = time.monotonic() - startTime\n scanRate = len(sites) / duration\n self.debug(f'Scan statistics: name={username}, count={len(self.siteResults)}, duration={duration:.2f}, rate={scanRate:.0f}')\n\n return [site for site, found in self.siteResults.items() if found]\n\n def generatePermutations(self, username):\n permutations = list()\n prefixsuffix = ['_', '-']\n replacements = {\n 'a': ['4', 's'],\n 'b': ['v', 'n'],\n 'c': ['x', 'v'],\n 'd': ['s', 'f'],\n 'e': ['w', 'r'],\n 'f': ['d', 'g'],\n 'g': ['f', 'h'],\n 'h': ['g', 'j', 'n'],\n 'i': ['o', 'u', '1'],\n 'j': ['k', 'h', 'i'],\n 'k': ['l', 'j'],\n 'l': ['i', '1', 'k'],\n 'm': ['n'],\n 'n': ['m'],\n 'o': ['p', 'i', '0'],\n 'p': ['o', 'q'],\n 'r': ['t', 'e'],\n 's': ['a', 'd', '5'],\n 't': ['7', 'y', 'z', 'r'],\n 'u': ['v', 'i', 'y', 'z'],\n 'v': ['u', 'c', 'b'],\n 'w': ['v', 'vv', 'q', 'e'],\n 'x': ['z', 'y', 'c'],\n 'y': ['z', 'x'],\n 'z': ['y', 'x'],\n '0': ['o'],\n '1': ['l'],\n '2': ['5'],\n '3': ['e'],\n '4': ['a'],\n '5': ['s'],\n '6': ['b'],\n '7': ['t'],\n '8': ['b'],\n '9': []\n }\n pairs = {\n 'oo': ['00'],\n 'll': ['l1l', 'l1l', '111', '11'],\n '11': ['ll', 'lll', 'l1l', '1l1']\n }\n\n # Generate a set with replacements, then\n # add suffixes and prefixes.\n pos = 0\n for c in username:\n if c not in replacements:\n continue\n if len(replacements[c]) == 0:\n continue\n npos = pos + 1\n for xc in replacements[c]:\n newuser = username[0:pos] + xc + username[npos:len(username)]\n permutations.append(newuser)\n\n pos += 1\n\n # Search for common double-letter replacements\n for p in pairs:\n if p in username:\n for r in pairs[p]:\n permutations.append(username.replace(p, r))\n\n # Search for prefixed and suffixed usernames\n for c in prefixsuffix:\n permutations.append(username + c)\n permutations.append(c + username)\n\n # Search for double character usernames\n pos = 0\n for c in username:\n permutations.append(username[0:pos] + c + c + username[(pos + 1):len(username)])\n pos += 1\n\n return list(set(permutations))\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n users = list()\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n # Skip events coming from me unless they are USERNAME events\n if eventName != \"USERNAME\" and srcModuleName == \"sfp_accounts\":\n self.debug(f\"Ignoring {eventName}, from self.\")\n return\n\n if eventData in list(self.results.keys()):\n return\n\n self.results[eventData] = True\n\n # If being called for the first time, let's see how trusted the\n # sites are by attempting to fetch a garbage user.\n if not self.distrustedChecked:\n # Check if a state cache exists first, to not have to do this all the time\n content = self.sf.cacheGet(\"sfaccounts_state_v3\", 72)\n if content:\n if content != \"None\": # \"None\" is written to the cached file when no sites are distrusted\n delsites = list()\n for line in content.split(\"\\n\"):\n if line == '':\n continue\n delsites.append(line)\n self.sites = [d for d in self.sites if d['name'] not in delsites]\n else:\n randpool = 'abcdefghijklmnopqrstuvwxyz1234567890'\n randuser = ''.join([random.SystemRandom().choice(randpool) for x in range(10)])\n res = self.checkSites(randuser)\n if res:\n delsites = list()\n for site in res:\n sitename = site.split(\" (Category:\")[0]\n self.debug(f\"Distrusting {sitename}\")\n delsites.append(sitename)\n self.sites = [d for d in self.sites if d['name'] not in delsites]\n else:\n # The caching code needs *some* content\n delsites = \"None\"\n self.sf.cachePut(\"sfaccounts_state_v3\", delsites)\n\n self.distrustedChecked = True\n\n if eventName == \"HUMAN_NAME\":\n names = [eventData.lower().replace(\" \", \"\"), eventData.lower().replace(\" \", \".\")]\n for name in names:\n users.append(name)\n\n if eventName == \"DOMAIN_NAME\":\n kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])\n if not kw:\n return\n\n users.append(kw)\n\n if eventName == \"EMAILADDR\" and self.opts['userfromemail']:\n name = eventData.split(\"@\")[0].lower()\n users.append(name)\n\n if eventName == \"USERNAME\":\n users.append(eventData)\n\n for user in set(users):\n if user in self.opts['_genericusers'].split(\",\"):\n self.debug(f\"{user} is a generic account name, skipping.\")\n continue\n\n if self.opts['ignorenamedict'] and user in self.commonNames:\n self.debug(f\"{user} is found in our name dictionary, skipping.\")\n continue\n\n if self.opts['ignoreworddict'] and user in self.words:\n self.debug(f\"{user} is found in our word dictionary, skipping.\")\n continue\n\n if user not in self.reportedUsers and eventData != user:\n if len(user) < self.opts['usernamesize']:\n self.debug(f\"{user} is too short, skipping.\")\n continue\n\n evt = SpiderFootEvent(\"USERNAME\", user, self.__name__, event)\n self.notifyListeners(evt)\n self.reportedUsers.append(user)\n\n # Only look up accounts when we've received a USERNAME event (possibly from\n # ourselves), since we want them to have gone through some verification by\n # this module, and we don't want duplicates (one based on EMAILADDR and another\n # based on USERNAME).\n if eventName == \"USERNAME\":\n res = self.checkSites(user)\n for site in res:\n evt = SpiderFootEvent(\n \"ACCOUNT_EXTERNAL_OWNED\",\n site,\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n\n if self.opts['permutate']:\n permutations = self.generatePermutations(user)\n for puser in permutations:\n res = self.checkSites(puser)\n for site in res:\n evt = SpiderFootEvent(\n \"SIMILAR_ACCOUNT_EXTERNAL\",\n site,\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n# End of sfp_accounts class\n" }, { "path": "modules/sfp_adblock.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_adblock\n# Purpose: SpiderFoot plug-in to test if external/internally linked pages\n# would be blocked by AdBlock Plus.\n#\n# Author: Steve Micallef \n#\n# Created: 22/09/2014\n# Copyright: (c) Steve Micallef 2014\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport adblockparser\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_adblock(SpiderFootPlugin):\n\n meta = {\n 'name': \"AdBlock Check\",\n 'summary': \"Check if linked pages would be blocked by AdBlock Plus.\",\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://adblockplus.org/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://help.eyeo.com/en/adblockplus/\",\n \"https://adblockplus.org/en/download\",\n \"https://adblockplus.org/en/filters#options\",\n \"https://chrome.google.com/webstore/detail/adblock-plus-free-ad-bloc/cfhdojbkjhnklbpkdaibdccddilifddb\"\n ],\n 'favIcon': \"https://www.google.com/s2/favicons?domain=https://adblockplus.org/en/\",\n 'logo': \"https://adblockplus.org/img/navbar-logo.svg\",\n 'description': \"Adblock Plus is a free extension that allows you to customize your web experience.\"\n \"You can block annoying ads, disable tracking and lots more.\"\n \"It’s available for all major desktop browsers and for your mobile devices.\\n\"\n \"Block ads that interrupt your browsing experience.\"\n \"Say goodbye to video ads, pop-ups, flashing banners and more.\"\n \"Blocking these annoyances means pages load faster.\\n\"\n \"With Adblock Plus avoiding tracking and malware is easy.\"\n \"Blocking intrusive ads reduces the risk of \\\"malvertising\\\" infections.\"\n \"Blocking tracking stops companies following your online activity.\"\n }\n }\n\n # Default options\n opts = {\n \"blocklist\": \"https://easylist-downloads.adblockplus.org/easylist.txt\",\n 'cacheperiod': 24,\n }\n\n optdescs = {\n \"blocklist\": \"AdBlockPlus block list.\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\",\n }\n\n results = None\n rules = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.rules = None\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"LINKED_URL_INTERNAL\", \"LINKED_URL_EXTERNAL\", \"PROVIDER_JAVASCRIPT\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"URL_ADBLOCKED_INTERNAL\", \"URL_ADBLOCKED_EXTERNAL\"]\n\n def retrieveBlocklist(self, blocklist_url):\n if not blocklist_url:\n return None\n\n blocklist = self.sf.cacheGet(f\"adblock_{blocklist_url}\", 24)\n\n if blocklist is not None:\n return self.setBlocklistRules(blocklist)\n\n res = self.sf.fetchUrl(blocklist_url, timeout=30)\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} for {blocklist_url}\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(f\"Unable to download AdBlock Plus blocklist: {blocklist_url}\")\n self.errorState = True\n return None\n\n self.sf.cachePut(f\"adblock_{blocklist_url}\", res['content'])\n\n return self.setBlocklistRules(res['content'])\n\n def setBlocklistRules(self, blocklist):\n \"\"\"Parse AdBlock Plus blocklist and set blocklist rules\n\n Args:\n blocklist (str): plaintext AdBlock Plus blocklist\n \"\"\"\n if not blocklist:\n return\n\n lines = blocklist.split('\\n')\n self.debug(f\"Retrieved {len(lines)} AdBlock blocklist rules\")\n try:\n self.rules = adblockparser.AdblockRules(lines)\n except adblockparser.AdblockParsingError as e:\n self.errorState = True\n self.error(f\"Parsing error handling AdBlock list: {e}\")\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(\"Already checked this URL for AdBlock matching, skipping.\")\n return\n\n self.results[eventData] = True\n\n if self.errorState:\n return\n\n if not self.opts[\"blocklist\"]:\n self.error(\n f\"You enabled {self.__class__.__name__} but did not set a blocklist URL!\"\n )\n self.errorState = True\n return\n\n if not self.rules:\n self.retrieveBlocklist(self.opts['blocklist'])\n\n if not self.rules:\n self.error(\"No AdBlock Plus rules loaded\")\n self.errorState = True\n return\n\n try:\n if eventName == 'PROVIDER_JAVASCRIPT':\n if self.rules and self.rules.should_block(eventData, {'third-party': True, 'script': True}):\n evt = SpiderFootEvent(\"URL_ADBLOCKED_EXTERNAL\", eventData, self.__name__, event)\n self.notifyListeners(evt)\n\n if eventName == 'LINKED_URL_EXTERNAL':\n if self.rules and self.rules.should_block(eventData, {'third-party': True}):\n evt = SpiderFootEvent(\"URL_ADBLOCKED_EXTERNAL\", eventData, self.__name__, event)\n self.notifyListeners(evt)\n\n if eventName == 'LINKED_URL_INTERNAL':\n if self.rules and self.rules.should_block(eventData):\n evt = SpiderFootEvent(\"URL_ADBLOCKED_INTERNAL\", eventData, self.__name__, event)\n self.notifyListeners(evt)\n\n except ValueError as e:\n self.error(f\"Parsing error handling AdBlock list: {e}\")\n self.errorState = True\n\n# End of sfp_adblock class\n" }, { "path": "modules/sfp_adguard_dns.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_adguard_dns\n# Purpose: SpiderFoot plug-in for looking up whether hosts are blocked by\n# AdGuard DNS servers.\n#\n# Author: \n#\n# Created: 2021-10-11\n# Copyright: (c) bcoles 2021\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport dns.resolver\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_adguard_dns(SpiderFootPlugin):\n\n meta = {\n 'name': \"AdGuard DNS\",\n 'summary': \"Check if a host would be blocked by AdGuard DNS.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://adguard.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://adguard.com/en/adguard-dns/overview.html\",\n ],\n 'favIcon': \"https://adguard.com/img/favicons/favicon.ico\",\n 'logo': \"https://adguard.com/img/favicons/apple-touch-icon.png\",\n 'description': \"AdGuard DNS is a foolproof way to block Internet ads that does not require installing any applications. \"\n \"It is easy to use, absolutely free, easily set up on any device, and provides you with minimal necessary functions \"\n \"to block ads, counters, malicious websites, and adult content.\"\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"AFFILIATE_INTERNET_NAME\",\n \"CO_HOSTED_SITE\"\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_INTERNET_NAME\",\n \"BLACKLISTED_AFFILIATE_INTERNET_NAME\",\n \"BLACKLISTED_COHOST\",\n ]\n\n def queryDefaultDNS(self, qaddr):\n res = dns.resolver.Resolver()\n res.nameservers = [\"94.140.14.14\", \"94.140.15.15\"]\n\n try:\n return res.resolve(qaddr)\n except Exception:\n self.debug(f\"Unable to resolve {qaddr}\")\n\n return None\n\n def queryFamilyDNS(self, qaddr):\n res = dns.resolver.Resolver()\n res.nameservers = [\"94.140.14.15\", \"94.140.15.16\"]\n\n try:\n return res.resolve(qaddr)\n except Exception:\n self.debug(f\"Unable to resolve {qaddr}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if eventName == \"INTERNET_NAME\":\n blacklist_type = \"BLACKLISTED_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n blacklist_type = \"BLACKLISTED_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n blacklist_type = \"BLACKLISTED_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n family = self.sf.normalizeDNS(self.queryFamilyDNS(eventData))\n default = self.sf.normalizeDNS(self.queryDefaultDNS(eventData))\n\n if not family or not default:\n return\n\n if '94.140.14.35' in family:\n self.debug(f\"{eventData} blocked by AdGuard Family DNS\")\n evt = SpiderFootEvent(blacklist_type, f\"AdGuard - Family Filter [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n if '94.140.14.35' in default:\n self.debug(f\"{eventData} blocked by AdGuard Default DNS\")\n evt = SpiderFootEvent(blacklist_type, f\"AdGuard - Default Filter [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_adguard_dns class\n" }, { "path": "modules/sfp_ahmia.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_ahmia\n# Purpose: Searches the Tor search engine 'Ahmia' for content related to the\n# target.\n#\n# Author: Steve Micallef \n#\n# Created: 20/06/2017\n# Copyright: (c) Steve Micallef 2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_ahmia(SpiderFootPlugin):\n\n meta = {\n 'name': \"Ahmia\",\n 'flags': [\"tor\"],\n 'summary': \"Search Tor 'Ahmia' search engine for mentions of the target.\",\n 'useCases': [\"Footprint\", \"Investigate\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://ahmia.fi/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://ahmia.fi/documentation/\",\n \"https://github.com/ahmia/\",\n \"http://msydqstlz2kzerdg.onion/\",\n \"https://ahmia.fi/stats\"\n ],\n 'favIcon': \"https://ahmia.fi/static/images/favicon.ico\",\n 'logo': \"https://ahmia.fi/static/images/ahmiafi_black.png\",\n 'description': \"Ahmia searches hidden services on the Tor network. To access these hidden services,\"\n \"you need the Tor browser bundle. Abuse material is not allowed on Ahmia. \"\n \"See our service blacklist and report abuse material if you find it in the index. \"\n \"It will be removed as soon as possible.\\n\"\n \"Contributors to Ahmia believe that the Tor network is an important and \"\n \"resilient distributed platform for anonymity and privacy worldwide. \"\n \"By providing a search engine for what many call the \\\"deep web\\\" or \\\"dark net\\\", \"\n \"Ahmia makes hidden services accessible to a wide range of people, not just Tor network early adopters.\"\n }\n }\n\n # Default options\n opts = {\n 'fetchlinks': True,\n 'fullnames': True\n }\n\n # Option descriptions\n optdescs = {\n 'fetchlinks': \"Fetch the darknet pages (via TOR, if enabled) to verify they mention your target.\",\n 'fullnames': \"Search for human names?\"\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\", \"HUMAN_NAME\", \"EMAILADDR\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"DARKNET_MENTION_URL\", \"DARKNET_MENTION_CONTENT\"]\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if not self.opts['fullnames'] and eventName == 'HUMAN_NAME':\n self.debug(f\"Skipping HUMAN_NAME: {eventData}\")\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n params = urllib.parse.urlencode({\n 'q': eventData\n })\n\n data = self.sf.fetchUrl(\n f\"https://ahmia.fi/search/?{params}\",\n useragent=self.opts['_useragent'],\n timeout=15\n )\n\n if not data:\n self.info(f\"No results for {eventData} returned from Ahmia.fi.\")\n return\n\n content = data.get('content')\n\n if not content:\n self.info(f\"No results for {eventData} returned from Ahmia.fi.\")\n return\n\n # We don't bother with pagination as Ahmia seems fairly limited in coverage\n # and displays hundreds of results per page\n links = re.findall(\"redirect_url=(.[^\\\"]+)\\\"\", content, re.IGNORECASE | re.DOTALL)\n\n if not links:\n self.info(f\"No results for {eventData} returned from Ahmia.fi.\")\n return\n\n for link in links:\n if self.checkForStop():\n return\n\n if link in self.results:\n continue\n\n self.results[link] = True\n\n self.debug(f\"Found a darknet mention: {link}\")\n\n if not self.sf.urlFQDN(link).endswith(\".onion\"):\n continue\n\n if not self.opts['fetchlinks']:\n evt = SpiderFootEvent(\"DARKNET_MENTION_URL\", link, self.__name__, event)\n self.notifyListeners(evt)\n continue\n\n res = self.sf.fetchUrl(\n link,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n verify=False\n )\n\n if res['content'] is None:\n self.debug(f\"Ignoring {link} as no data returned\")\n continue\n\n if eventData not in res['content']:\n self.debug(f\"Ignoring {link} as no mention of {eventData}\")\n continue\n\n evt = SpiderFootEvent(\"DARKNET_MENTION_URL\", link, self.__name__, event)\n self.notifyListeners(evt)\n\n try:\n startIndex = res['content'].index(eventData) - 120\n endIndex = startIndex + len(eventData) + 240\n except Exception:\n self.debug(f\"String '{eventData}' not found in content.\")\n continue\n\n wdata = res['content'][startIndex:endIndex]\n evt = SpiderFootEvent(\"DARKNET_MENTION_CONTENT\", f\"...{wdata}...\", self.__name__, evt)\n self.notifyListeners(evt)\n\n# End of sfp_ahmia class\n" }, { "path": "modules/sfp_alienvault.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_alienvault\n# Purpose: Query AlienVault OTX\n#\n# Author: Steve Micallef\n#\n# Created: 26/03/2017\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\nfrom datetime import datetime\n\nfrom netaddr import IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_alienvault(SpiderFootPlugin):\n\n meta = {\n 'name': \"AlienVault OTX\",\n 'summary': \"Obtain information from AlienVault Open Threat Exchange (OTX)\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://otx.alienvault.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://otx.alienvault.com/faq\",\n \"https://otx.alienvault.com/api\",\n \"https://otx.alienvault.com/submissions/list\",\n \"https://otx.alienvault.com/pulse/create\",\n \"https://otx.alienvault.com/endpoint-security/welcome\",\n \"https://otx.alienvault.com/browse/\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://otx.alienvault.com/\",\n \"Sign up for a free account\",\n \"Navigate to https://otx.alienvault.com/settings\",\n \"The API key is listed under 'OTX Key'\"\n ],\n 'favIcon': \"https://www.google.com/s2/favicons?domain=https://otx.alienvault.com/\",\n 'logo': \"https://otx.alienvault.com/assets/images/otx-logo.svg\",\n 'description': \"The World’s First Truly Open Threat Intelligence Community\\n\"\n \"Open Threat Exchange is the neighborhood watch of the global intelligence community. \"\n \"It enables private companies, independent security researchers, and government agencies to \"\n \"openly collaborate and share the latest information about emerging threats, attack methods, \"\n \"and malicious actors, promoting greater security across the entire community.\\n\"\n \"OTX changed the way the intelligence community creates and consumes threat data. \"\n \"In OTX, anyone in the security community can contribute, discuss, research, validate, \"\n \"and share threat data. You can integrate community-generated OTX threat data directly \"\n \"into your AlienVault and third-party security products, so that your threat detection defenses \"\n \"are always up to date with the latest threat intelligence. \"\n \"Today, 100,000 participants in 140 countries contribute over 19 million threat indicators daily.\"\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\",\n \"verify\": True,\n \"reputation_age_limit_days\": 30,\n \"cohost_age_limit_days\": 30,\n \"threat_score_min\": 2,\n 'netblocklookup': True,\n 'maxnetblock': 24,\n 'maxv6netblock': 120,\n 'subnetlookup': True,\n 'maxsubnet': 24,\n 'maxv6subnet': 120,\n 'max_pages': 50,\n 'maxcohost': 100,\n 'checkaffiliates': True\n }\n\n # Option descriptions\n optdescs = {\n \"api_key\": \"AlienVault OTX API Key.\",\n \"verify\": \"Verify co-hosts are valid by checking if they still resolve to the shared IP.\",\n \"reputation_age_limit_days\": \"Ignore any reputation records older than this many days. 0 = unlimited.\",\n \"cohost_age_limit_days\": \"Ignore any co-hosts older than this many days. 0 = unlimited.\",\n \"threat_score_min\": \"Minimum AlienVault threat score.\",\n 'netblocklookup': \"Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?\",\n 'maxnetblock': \"If looking up owned netblocks, the maximum IPv4 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'maxv6netblock': \"If looking up owned netblocks, the maximum IPv6 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'subnetlookup': \"Look up all IPs on subnets which your target is a part of for blacklisting?\",\n 'maxsubnet': \"If looking up subnets, the maximum IPv4 subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'maxv6subnet': \"If looking up subnets, the maximum IPv6 subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'max_pages': \"Maximum number of pages of URL results to fetch.\",\n 'maxcohost': \"Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.\",\n 'checkaffiliates': \"Apply checks to affiliates?\"\n }\n\n results = None\n errorState = False\n cohostcount = 0\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.cohostcount = 0\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"AFFILIATE_IPV6_ADDRESS\",\n \"NETBLOCK_OWNER\",\n \"NETBLOCKV6_OWNER\",\n \"NETBLOCK_MEMBER\",\n \"NETBLOCKV6_MEMBER\",\n \"NETBLOCK_OWNER\",\n \"NETBLOCK_MEMBER\",\n ]\n\n # What events this module produces\n def producedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"AFFILIATE_IPV6_ADDRESS\",\n \"CO_HOSTED_SITE\",\n \"INTERNET_NAME\",\n \"INTERNET_NAME_UNRESOLVED\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_NETBLOCK\",\n \"LINKED_URL_INTERNAL\"\n ]\n\n # Parse API response\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from AlienVault OTX.\")\n return None\n\n # Future proofing - AlienVault OTX does not implement rate limiting\n if res['code'] == '429':\n self.error(\"You are being rate-limited by AienVault OTX\")\n self.errorState = True\n return None\n\n if res['code'] == \"403\":\n self.error(\"AlienVault OTX API key seems to have been rejected or you have exceeded usage limits for the month.\")\n self.errorState = True\n return None\n\n if res['content'] is None or res['code'] == \"404\":\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from AlienVault OTX: {e}\")\n\n return None\n\n def queryReputation(self, qry):\n if \":\" in qry:\n target_type = \"IPv6\"\n elif self.sf.validIP(qry):\n target_type = \"IPv4\"\n else:\n self.info(f\"Could not determine target type for {qry}\")\n return None\n\n headers = {\n 'Accept': 'application/json',\n 'X-OTX-API-KEY': self.opts['api_key']\n }\n\n res = self.sf.fetchUrl(\n f\"https://otx.alienvault.com/api/v1/indicators/{target_type}/{qry}/reputation\",\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\",\n headers=headers)\n\n return self.parseApiResponse(res)\n\n def queryPassiveDns(self, qry):\n if \":\" in qry:\n target_type = \"IPv6\"\n elif self.sf.validIP(qry):\n target_type = \"IPv4\"\n else:\n self.info(f\"Could not determine target type for {qry}\")\n return None\n\n headers = {\n 'Accept': 'application/json',\n 'X-OTX-API-KEY': self.opts['api_key']\n }\n\n res = self.sf.fetchUrl(\n f\"https://otx.alienvault.com/api/v1/indicators/{target_type}/{qry}/passive_dns\",\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\",\n headers=headers)\n\n return self.parseApiResponse(res)\n\n def queryDomainUrlList(self, qry, page=1, per_page=50):\n params = urllib.parse.urlencode({\n 'page': page,\n 'limit': per_page\n })\n\n headers = {\n 'Accept': 'application/json',\n 'X-OTX-API-KEY': self.opts['api_key']\n }\n res = self.sf.fetchUrl(\n f\"https://otx.alienvault.com/api/v1/indicators/domain/{qry}/url_list?{params}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\",\n headers=headers)\n\n return self.parseApiResponse(res)\n\n def queryHostnameUrlList(self, qry, page=1, per_page=50):\n params = urllib.parse.urlencode({\n 'page': page,\n 'limit': per_page\n })\n\n headers = {\n 'Accept': 'application/json',\n 'X-OTX-API-KEY': self.opts['api_key']\n }\n res = self.sf.fetchUrl(\n f\"https://otx.alienvault.com/api/v1/indicators/hostname/{qry}/url_list?{params}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\",\n headers=headers)\n\n return self.parseApiResponse(res)\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts['api_key'] == \"\":\n self.error(f\"You enabled {self.__class__.__name__} but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName.startswith(\"AFFILIATE\") and not self.opts['checkaffiliates']:\n return\n\n if eventName == 'INTERNET_NAME':\n urls = list()\n page = 1\n while page <= self.opts['max_pages']:\n if self.checkForStop():\n break\n if self.errorState:\n break\n\n data = self.queryHostnameUrlList(eventData, page=page)\n page += 1\n\n url_list = data.get('url_list')\n if not url_list:\n break\n\n for url in url_list:\n u = url.get('url')\n if not u:\n continue\n urls.append(u)\n\n if not data.get('has_next'):\n break\n\n if self.sf.isDomain(eventData, self.opts['_internettlds']):\n page = 1\n while page <= self.opts['max_pages']:\n if self.checkForStop():\n break\n if self.errorState:\n break\n\n data = self.queryDomainUrlList(eventData, page=page)\n page += 1\n\n url_list = data.get('url_list')\n if not url_list:\n break\n\n for url in url_list:\n u = url.get('url')\n if not u:\n continue\n urls.append(u)\n\n if not data.get('has_next'):\n break\n\n for url in set(urls):\n if not url:\n continue\n\n host = self.sf.urlFQDN(url.lower())\n\n if not self.getTarget().matches(host, includeChildren=True, includeParents=True):\n continue\n\n if url not in self.results:\n self.results[url] = True\n evt = SpiderFootEvent('LINKED_URL_INTERNAL', url, self.__name__, event)\n self.notifyListeners(evt)\n\n return\n\n if eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']:\n if not self.opts['netblocklookup']:\n return\n\n if eventName == 'NETBLOCKV6_OWNER':\n max_netblock = self.opts['maxv6netblock']\n else:\n max_netblock = self.opts['maxnetblock']\n\n if IPNetwork(eventData).prefixlen < max_netblock:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}\")\n return\n\n if eventName in ['NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']:\n if not self.opts['subnetlookup']:\n return\n\n if eventName == 'NETBLOCKV6_MEMBER':\n max_subnet = self.opts['maxv6subnet']\n else:\n max_subnet = self.opts['maxsubnet']\n\n if IPNetwork(eventData).prefixlen < max_subnet:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}\")\n return\n\n qrylist = list()\n if eventName.startswith(\"NETBLOCK\"):\n for ipaddr in IPNetwork(eventData):\n qrylist.append(str(ipaddr))\n self.results[str(ipaddr)] = True\n else:\n qrylist.append(eventData)\n\n # For IP addresses, do the additional passive DNS lookup\n if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']:\n ret = self.queryPassiveDns(eventData)\n\n if ret is None:\n self.info(f\"No Passive DNS info for {eventData}\")\n else:\n passive_dns = ret.get('passive_dns')\n if passive_dns:\n self.debug(f\"Found passive DNS results for {eventData} in AlienVault OTX\")\n for rec in passive_dns:\n host = rec.get('hostname')\n\n if not host:\n continue\n\n if self.getTarget().matches(host, includeParents=True):\n evtType = \"INTERNET_NAME\"\n if not self.sf.resolveHost(host) and not self.sf.resolveHost6(host):\n evtType = \"INTERNET_NAME_UNRESOVLED\"\n evt = SpiderFootEvent(evtType, host, self.__name__, event)\n self.notifyListeners(evt)\n continue\n\n if self.opts['cohost_age_limit_days'] > 0:\n try:\n last = rec.get(\"last\", \"\")\n last_dt = datetime.strptime(last, '%Y-%m-%dT%H:%M:%S')\n last_ts = int(time.mktime(last_dt.timetuple()))\n age_limit_ts = int(time.time()) - (86400 * self.opts['cohost_age_limit_days'])\n if last_ts < age_limit_ts:\n self.debug(f\"Passive DNS record {host} found for {eventData} is too old ({last_dt}), skipping.\")\n continue\n except Exception:\n self.info(\"Could not parse date from AlienVault data, so ignoring cohost_age_limit_days\")\n\n if self.opts[\"verify\"] and not self.sf.validateIP(host, eventData):\n self.debug(f\"Co-host {host} no longer resolves to {eventData}, skipping\")\n continue\n\n if self.cohostcount < self.opts['maxcohost']:\n e = SpiderFootEvent(\"CO_HOSTED_SITE\", host, self.__name__, event)\n self.notifyListeners(e)\n self.cohostcount += 1\n else:\n self.info(f\"Maximum co-host threshold exceeded ({self.opts['maxcohost']}), ignoring co-host {host}\")\n\n if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS', 'NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']:\n evtType = 'MALICIOUS_IPADDR'\n elif eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS', 'NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']:\n evtType = 'MALICIOUS_AFFILIATE_IPADDR'\n elif eventName == \"INTERNET_NAME\":\n evtType = 'MALICIOUS_INTERNET_NAME'\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n for addr in qrylist:\n if self.checkForStop():\n return\n if self.errorState:\n return\n\n rec = self.queryReputation(addr)\n\n if not rec:\n continue\n\n if rec.get(\"reputation\", None):\n self.debug(f\"Found reputation info for {addr} in AlienVault OTX\")\n rec_history = rec['reputation'].get(\"activities\", list())\n threat_score = rec['reputation']['threat_score']\n threat_score_min = self.opts['threat_score_min']\n\n if threat_score < threat_score_min:\n self.debug(f\"Threat score {threat_score} smaller than {threat_score_min}, skipping.\")\n continue\n\n descr = f\"AlienVault Threat Score: {threat_score}\"\n\n for result in rec_history:\n nm = result.get(\"name\", None)\n\n if nm is None or nm in descr:\n continue\n\n descr += \"\\n - \" + nm\n created = result.get(\"last_date\", \"\")\n if self.opts['reputation_age_limit_days'] > 0:\n try:\n # 2014-11-06T10:45:00.000\n created_dt = datetime.strptime(created, '%Y-%m-%dT%H:%M:%S')\n created_ts = int(time.mktime(created_dt.timetuple()))\n age_limit_ts = int(time.time()) - (86400 * self.opts['reputation_age_limit_days'])\n if created_ts < age_limit_ts:\n self.debug(f\"Reputation record found for {addr} is too old ({created_dt}), skipping.\")\n continue\n except Exception:\n self.info(\"Could not parse date from AlienVault data, so ignoring reputation_age_limit_days\")\n\n # For netblocks, we need to create the IP address event so that\n # the threat intel event is more meaningful.\n if eventName == 'NETBLOCK_OWNER':\n pevent = SpiderFootEvent(\"IP_ADDRESS\", addr, self.__name__, event)\n self.notifyListeners(pevent)\n if eventName == 'NETBLOCKV6_OWNER':\n pevent = SpiderFootEvent(\"IPV6_ADDRESS\", addr, self.__name__, event)\n self.notifyListeners(pevent)\n elif eventName == 'NETBLOCK_MEMBER':\n pevent = SpiderFootEvent(\"AFFILIATE_IPADDR\", addr, self.__name__, event)\n self.notifyListeners(pevent)\n elif eventName == 'NETBLOCKV6_MEMBER':\n pevent = SpiderFootEvent(\"AFFILIATE_IPV6_ADDRESS\", addr, self.__name__, event)\n self.notifyListeners(pevent)\n else:\n pevent = event\n\n e = SpiderFootEvent(evtType, descr, self.__name__, pevent)\n self.notifyListeners(e)\n\n# End of sfp_alienvault class\n" }, { "path": "modules/sfp_alienvaultiprep.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_alienvaultiprep\n# Purpose: Check if an IP or netblock is malicious according to the AlienVault\n# IP Reputation database.\n#\n# Author: steve@binarypool.com\n#\n# Created: 14/12/2013\n# Copyright: (c) Steve Micallef, 2013\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom netaddr import IPAddress, IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_alienvaultiprep(SpiderFootPlugin):\n\n meta = {\n 'name': \"AlienVault IP Reputation\",\n 'summary': \"Check if an IP or netblock is malicious according to the AlienVault IP Reputation database.\",\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://cybersecurity.att.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://cybersecurity.att.com/documentation/\",\n \"https://cybersecurity.att.com/resource-center\",\n \"https://success.alienvault.com/s/article/Can-I-use-the-OTX-IP-Reputation-List-as-a-blocklist\",\n ],\n 'favIcon': \"https://cdn-cybersecurity.att.com/images/uploads/logos/att-globe.svg\",\n 'logo': \"https://cdn-cybersecurity.att.com/images/uploads/logos/att-business-web.svg\",\n 'description': \"Looking at security through new eyes.\\n\"\n \"AT&T Business and AlienVault have joined forces to create AT&T Cybersecurity, \"\n \"with a vision to bring together the people, process, and technology \"\n \"that help businesses of any size stay ahead of threats.\",\n }\n }\n\n # Default options\n opts = {\n 'checkaffiliates': True,\n 'cacheperiod': 18,\n 'checknetblocks': True,\n 'checksubnets': True\n }\n\n # Option descriptions\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliates?\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\",\n 'checknetblocks': \"Report if any malicious IPs are found within owned netblocks?\",\n 'checksubnets': \"Check if any malicious IPs are found within the same subnet of the target?\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"NETBLOCK_MEMBER\",\n \"NETBLOCK_OWNER\"\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"BLACKLISTED_SUBNET\",\n \"BLACKLISTED_NETBLOCK\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_SUBNET\",\n \"MALICIOUS_NETBLOCK\",\n ]\n\n def queryBlacklist(self, target, targetType):\n blacklist = self.retrieveBlacklist()\n\n if not blacklist:\n return False\n\n if targetType == \"ip\":\n if target in blacklist:\n self.debug(f\"IP address {target} found in AlienVault IP Reputation Database blacklist.\")\n return True\n elif targetType == \"netblock\":\n netblock = IPNetwork(target)\n for ip in blacklist:\n if IPAddress(ip) in netblock:\n self.debug(f\"IP address {ip} found within netblock/subnet {target} in AlienVault IP Reputation Database blacklist.\")\n return True\n\n return False\n\n def retrieveBlacklist(self):\n blacklist = self.sf.cacheGet('alienvaultiprep', 24)\n\n if blacklist is not None:\n return self.parseBlacklist(blacklist)\n\n res = self.sf.fetchUrl(\n \"https://reputation.alienvault.com/reputation.generic\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from AlienVault IP Reputation Database.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from AlienVault IP Reputation Database\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"alienvaultiprep\", res['content'])\n\n return self.parseBlacklist(res['content'])\n\n def parseBlacklist(self, blacklist):\n \"\"\"Parse plaintext blacklist\n\n Args:\n blacklist (str): plaintext blacklist from AlienVault IP Reputation Database\n\n Returns:\n list: list of blacklisted IP addresses\n \"\"\"\n ips = list()\n\n if not blacklist:\n return ips\n\n for ip in blacklist.split('\\n'):\n ip = ip.strip().split(\" #\")[0]\n if ip.startswith('#'):\n continue\n if not self.sf.validIP(ip):\n continue\n ips.append(ip)\n\n return ips\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName == 'IP_ADDRESS':\n targetType = 'ip'\n malicious_type = \"MALICIOUS_IPADDR\"\n blacklist_type = \"BLACKLISTED_IPADDR\"\n elif eventName == 'AFFILIATE_IPADDR':\n if not self.opts.get('checkaffiliates', False):\n return\n targetType = 'ip'\n malicious_type = \"MALICIOUS_AFFILIATE_IPADDR\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_IPADDR\"\n elif eventName == 'NETBLOCK_OWNER':\n if not self.opts.get('checknetblocks', False):\n return\n targetType = 'netblock'\n malicious_type = \"MALICIOUS_NETBLOCK\"\n blacklist_type = \"BLACKLISTED_NETBLOCK\"\n elif eventName == 'NETBLOCK_MEMBER':\n if not self.opts.get('checksubnets', False):\n return\n targetType = 'netblock'\n malicious_type = \"MALICIOUS_SUBNET\"\n blacklist_type = \"BLACKLISTED_SUBNET\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n self.debug(f\"Checking maliciousness of {eventData} ({eventName}) with AlienVault IP Reputation Database\")\n\n if not self.queryBlacklist(eventData, targetType):\n return\n\n url = \"https://reputation.alienvault.com/reputation.generic\"\n text = f\"AlienVault IP Reputation Database [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_alienvaultiprep class\n" }, { "path": "modules/sfp_apple_itunes.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_apple_itunes\n# Purpose: Query Apple iTunes for mobile apps.\n#\n# Author: \n#\n# Created: 2020-09-19\n# Copyright: (c) bcoles 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_apple_itunes(SpiderFootPlugin):\n\n meta = {\n 'name': \"Apple iTunes\",\n 'summary': \"Search Apple iTunes for mobile apps.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Footprint\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://itunes.apple.com/\",\n 'model': \"FREE_AUTH_UNLIMITED\",\n 'favIcon': \"https://itunes.apple.com/favicon.ico\",\n 'logo': \"https://itunes.apple.com/favicon.ico\",\n 'description': \"The Apple iTunes store is a store for downloading \"\n \"and purchasing apps for Apple devices.\",\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n 'DOMAIN_NAME'\n ]\n\n def producedEvents(self):\n return [\n 'APPSTORE_ENTRY',\n 'INTERNET_NAME',\n 'LINKED_URL_INTERNAL',\n 'AFFILIATE_INTERNET_NAME',\n 'RAW_RIR_DATA'\n ]\n\n def query(self, qry, limit=100):\n params = urllib.parse.urlencode({\n 'media': 'software',\n 'entity': 'software,iPadSoftware,softwareDeveloper',\n 'limit': limit,\n 'term': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n })\n\n res = self.sf.fetchUrl(\n f\"https://itunes.apple.com/search?{params}\",\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout']\n )\n\n time.sleep(1)\n\n if res['content'] is None:\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from Apple iTunes: {e}\")\n return None\n\n results = data.get('results')\n\n if not results:\n self.debug(f\"No results found for {qry}\")\n return None\n\n return results\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if eventName not in self.watchedEvents():\n return\n\n self.results[eventData] = True\n\n domain_reversed = '.'.join(list(reversed(eventData.lower().split('.'))))\n data = self.query(domain_reversed)\n\n if not data:\n self.info(f\"No results found for {eventData}\")\n return\n\n urls = list()\n hosts = list()\n found = False\n\n for result in data:\n bundleId = result.get('bundleId')\n\n if not bundleId:\n continue\n\n trackName = result.get('trackName')\n\n if not trackName:\n continue\n\n version = result.get('version')\n\n if not version:\n continue\n\n app_full_name = f\"{trackName} {version} ({bundleId})\"\n\n if (\n domain_reversed != bundleId.lower()\n and not bundleId.lower().startswith(f\"{domain_reversed}.\")\n and not bundleId.lower().endswith(f\".{domain_reversed}\")\n and f\".{domain_reversed}.\" not in bundleId.lower()\n ):\n self.debug(f\"App {app_full_name} does not match {domain_reversed}, skipping\")\n continue\n\n trackViewUrl = result.get('trackViewUrl')\n\n if not trackViewUrl:\n continue\n\n app_data = f\"{app_full_name}\\n{trackViewUrl}\"\n\n evt = SpiderFootEvent('APPSTORE_ENTRY', app_data, self.__name__, event)\n self.notifyListeners(evt)\n found = True\n\n sellerUrl = result.get('sellerUrl')\n\n if not sellerUrl:\n continue\n\n urls.append(sellerUrl)\n\n for url in set(urls):\n host = self.sf.urlFQDN(url)\n\n if not host:\n continue\n\n if self.getTarget().matches(host, includeChildren=True, includeParents=True):\n evt = SpiderFootEvent('LINKED_URL_INTERNAL', url, self.__name__, event)\n self.notifyListeners(evt)\n found = True\n\n hosts.append(host)\n\n for host in set(hosts):\n if not host:\n continue\n\n if self.getTarget().matches(host, includeChildren=True, includeParents=True):\n evt = SpiderFootEvent('INTERNET_NAME', host, self.__name__, event)\n self.notifyListeners(evt)\n else:\n evt = SpiderFootEvent('AFFILIATE_INTERNET_NAME', host, self.__name__, event)\n self.notifyListeners(evt)\n found = True\n\n if found:\n evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(data), self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_apple_itunes class\n" }, { "path": "modules/sfp_archiveorg.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_archiveorg\n# Purpose: Queries archive.org (Wayback machine) for historic versions of\n# certain pages.\n#\n# Author: Steve Micallef \n#\n# Created: 20/07/2015\n# Copyright: (c) Steve Micallef 2015\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport datetime\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_archiveorg(SpiderFootPlugin):\n\n meta = {\n 'name': \"Archive.org\",\n 'summary': \"Identifies historic versions of interesting files/pages from the Wayback Machine.\",\n 'flags': [\"slow\"],\n 'useCases': [\"Footprint\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://archive.org/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://archive.org/projects/\",\n \"https://archive.org/services/docs/api/\"\n ],\n 'favIcon': \"https://archive.org/images/glogo.jpg\",\n 'logo': \"https://archive.org/images/glogo.jpg\",\n 'description': \"Internet Archive is a non-profit library of millions of free books, movies, software, music, websites, and more.\\n\"\n \"The Internet Archive, a 501(c)(3) non-profit, is building a digital library of Internet sites \"\n \"and other cultural artifacts in digital form. Like a paper library, we provide free access to \"\n \"researchers, historians, scholars, the print disabled, and the general public. \"\n \"Our mission is to provide Universal Access to All Knowledge.\\n\"\n \"We began in 1996 by archiving the Internet itself, a medium that was just beginning to grow in use. \"\n \"Like newspapers, the content published on the web was ephemeral - but unlike newspapers, no one was saving it. \"\n \"Today we have 20+ years of web history accessible through the Wayback Machine and we work with 625+ library and \"\n \"other partners through our Archive-It program to identify important web pages.\",\n }\n\n }\n\n # Default options\n opts = {\n 'farback': \"30,60,90\",\n 'intfiles': True,\n 'passwordpages': True,\n 'formpages': False,\n 'flashpages': False,\n 'javapages': False,\n 'staticpages': False,\n 'uploadpages': False,\n 'webframeworkpages': False,\n 'javascriptpages': False\n }\n\n # Option descriptions\n optdescs = {\n 'farback': \"Number of days back to look for older versions of files/pages in the Wayback Machine snapshots. Comma-separate the values, so for example 30,60,90 means to look for snapshots 30 days, 60 days and 90 days back.\",\n 'intfiles': \"Query the Wayback Machine for historic versions of Interesting Files.\",\n 'passwordpages': \"Query the Wayback Machine for historic versions of URLs with passwords.\",\n 'formpages': \"Query the Wayback Machine for historic versions of URLs with forms.\",\n 'uploadpages': \"Query the Wayback Machine for historic versions of URLs accepting uploads.\",\n 'flashpages': \"Query the Wayback Machine for historic versions of URLs containing Flash.\",\n 'javapages': \"Query the Wayback Machine for historic versions of URLs using Java Applets.\",\n 'staticpages': \"Query the Wayback Machine for historic versions of purely static URLs.\",\n \"webframeworkpages\": \"Query the Wayback Machine for historic versions of URLs using Javascript frameworks.\",\n \"javascriptpages\": \"Query the Wayback Machine for historic versions of URLs using Javascript.\"\n }\n\n results = None\n foundDates = list()\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.foundDates = list()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"INTERESTING_FILE\", \"URL_PASSWORD\", \"URL_FORM\", \"URL_FLASH\",\n \"URL_STATIC\", \"URL_JAVA_APPLET\", \"URL_UPLOAD\", \"URL_JAVASCRIPT\",\n \"URL_WEB_FRAMEWORK\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"INTERESTING_FILE_HISTORIC\", \"URL_PASSWORD_HISTORIC\",\n \"URL_FORM_HISTORIC\", \"URL_FLASH_HISTORIC\",\n \"URL_STATIC_HISTORIC\", \"URL_JAVA_APPLET_HISTORIC\",\n \"URL_UPLOAD_HISTORIC\", \"URL_WEB_FRAMEWORK_HISTORIC\",\n \"URL_JAVASCRIPT_HISTORIC\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventName == \"INTERESTING_FILE\" and not self.opts['intfiles']:\n return\n if eventName == \"URL_PASSWORD\" and not self.opts['passwordpages']:\n return\n if eventName == \"URL_STATIC\" and not self.opts['staticpages']:\n return\n if eventName == \"URL_FORM\" and not self.opts['formpages']:\n return\n if eventName == \"URL_UPLOAD\" and not self.opts['uploadpages']:\n return\n if eventName == \"URL_JAVA_APPLET\" and not self.opts['javapages']:\n return\n if eventName == \"URL_FLASH\" and not self.opts['flashpages']:\n return\n if eventName == \"URL_JAVASCRIPT\" and not self.opts['javascriptpages']:\n return\n if eventName == \"URL_WEB_FRAMEWORK\" and not self.opts['webframeworkpages']:\n return\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n for daysback in self.opts['farback'].split(\",\"):\n try:\n newDate = datetime.datetime.now() - datetime.timedelta(days=int(daysback))\n except Exception:\n self.error(\"Unable to parse option for number of days back to search.\")\n self.errorState = True\n return\n\n maxDate = newDate.strftime(\"%Y%m%d\")\n\n url = \"https://archive.org/wayback/available?url=\" + eventData + \\\n \"×tamp=\" + maxDate\n res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'])\n\n if res['content'] is None:\n self.error(f\"Unable to fetch {url}\")\n continue\n\n try:\n ret = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from Archive.org: {e}\")\n ret = None\n\n if not ret:\n self.debug(f\"Empty response from archive.org for {eventData}\")\n continue\n\n if len(ret['archived_snapshots']) < 1:\n self.debug(\"No archived snapshots for \" + eventData)\n continue\n\n wbmlink = ret['archived_snapshots']['closest']['url']\n if wbmlink in self.foundDates:\n self.debug(\"Snapshot already fetched.\")\n continue\n\n self.foundDates.append(wbmlink)\n name = eventName + \"_HISTORIC\"\n\n self.info(\"Found a historic file: \" + wbmlink)\n evt = SpiderFootEvent(name, wbmlink, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_archiveorg class\n" }, { "path": "modules/sfp_arin.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_arin\n# Purpose: Queries the ARIN internet registry to get netblocks and other\n# bits of info.\n#\n# Author: Steve Micallef \n#\n# Created: 23/02/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_arin(SpiderFootPlugin):\n\n meta = {\n 'name': \"ARIN\",\n 'summary': \"Queries ARIN registry for contact information.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Public Registries\"],\n 'dataSource': {\n 'website': \"https://www.arin.net/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://www.arin.net/resources/\",\n \"https://www.arin.net/reference/\",\n \"https://www.arin.net/participate/\",\n \"https://www.arin.net/resources/guide/request/\",\n \"https://www.arin.net/resources/registry/transfers/\",\n \"https://www.arin.net/resources/guide/ipv6/\"\n ],\n 'favIcon': \"https://www.arin.net/img/favicon.ico\",\n 'logo': \"https://www.arin.net/img/logo-stnd.svg\",\n 'description': \"ARIN is a nonprofit, member-based organization that administers IP addresses & \"\n \"ASNs in support of the operation and growth of the Internet.\\n\"\n \"Established in December 1997 as a Regional Internet Registry, \"\n \"the American Registry for Internet Numbers (ARIN) is responsible for the management \"\n \"and distribution of Internet number resources such as Internet Protocol (IP) addresses \"\n \"and Autonomous System Numbers (ASNs). ARIN manages these resources within its service region, \"\n \"which is comprised of Canada, the United States, and many Caribbean and North Atlantic islands.\",\n }\n }\n\n # Default options\n opts = {}\n optdescs = {}\n\n results = None\n currentEventSrc = None\n keywords = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.currentEventSrc = None\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['DOMAIN_NAME', 'HUMAN_NAME']\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"RAW_RIR_DATA\", \"HUMAN_NAME\"]\n\n # Fetch content and notify of the raw data\n def fetchRir(self, url):\n head = {\"Accept\": \"application/json\"}\n res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'], headers=head)\n if res['content'] is not None and res['code'] != \"404\":\n return res\n return None\n\n # Owner information about an AS\n def query(self, qtype, value):\n url = \"https://whois.arin.net/rest/\"\n\n if qtype == \"domain\":\n url += \"pocs;domain=@\" + value\n\n try:\n if qtype == \"name\":\n fname, lname = value.split(\" \", 1)\n if fname.endswith(\",\"):\n t = fname\n fname = lname\n lname = t\n url += \"pocs;first=\" + fname + \";last=\" + lname\n except Exception as e:\n self.debug(\"Couldn't process name: \" + value + \" (\" + str(e) + \")\")\n return None\n\n if qtype == \"contact\":\n url = value\n\n res = self.fetchRir(url)\n if not res:\n self.debug(\"No info found/available for \" + value + \" at ARIN.\")\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, self.currentEventSrc)\n self.notifyListeners(evt)\n return data\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n self.currentEventSrc = event\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n # Don't look up stuff twice\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName == \"DOMAIN_NAME\":\n ret = self.query(\"domain\", eventData)\n if not ret:\n return\n if \"pocs\" in ret:\n if \"pocRef\" in ret['pocs']:\n ref = list()\n # Might be a list or a dictionary\n if type(ret['pocs']['pocRef']) == dict:\n ref = [ret['pocs']['pocRef']]\n else:\n ref = ret['pocs']['pocRef']\n for p in ref:\n name = p['@name']\n if \",\" in name:\n sname = name.split(\", \", 1)\n name = sname[1] + \" \" + sname[0]\n\n evt = SpiderFootEvent(\"HUMAN_NAME\", name,\n self.__name__, self.currentEventSrc)\n self.notifyListeners(evt)\n\n # We just want the raw data so we can get potential\n # e-mail addresses.\n self.query(\"contact\", p['$'])\n\n if eventName == \"HUMAN_NAME\":\n ret = self.query(\"name\", eventData)\n if not ret:\n return\n if \"pocs\" in ret:\n if \"pocRef\" in ret['pocs']:\n ref = list()\n # Might be a list or a dictionary\n if type(ret['pocs']['pocRef']) == dict:\n ref = [ret['pocs']['pocRef']]\n else:\n ref = ret['pocs']['pocRef']\n for p in ref:\n # We just want the raw data so we can get potential\n # e-mail addresses.\n self.query(\"contact\", p['$'])\n\n# End of sfp_arin class\n" }, { "path": "modules/sfp_azureblobstorage.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_azureblobstorage\n# Purpose: SpiderFoot plug-in for identifying potential Azure blobs related\n# to the target.\n#\n# Author: Steve Micallef \n#\n# Created: 14/07/2019\n# Copyright: (c) Steve Micallef 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport random\nimport threading\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_azureblobstorage(SpiderFootPlugin):\n\n meta = {\n 'name': \"Azure Blob Finder\",\n 'summary': \"Search for potential Azure blobs associated with the target and attempt to list their contents.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Passive\"],\n 'categories': [\"Crawling and Scanning\"],\n 'dataSource': {\n 'website': \"https://azure.microsoft.com/en-in/services/storage/blobs/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'favIcon': 'https://azurecomcdn.azureedge.net/cvt-4fd6fa9ffb60246fd6387e4b34f89dc454cdf3df85d2b5d3215846066fceb0b6/images/icon/favicon.ico',\n 'logo': 'https://azurecomcdn.azureedge.net/cvt-4fd6fa9ffb60246fd6387e4b34f89dc454cdf3df85d2b5d3215846066fceb0b6/images/icon/favicon.ico',\n 'description': \"Massively scalable and secure object storage for cloud-native workloads,\"\n \"archives, data lakes, high-performance computing and machine learning.\"\n }\n }\n\n # Default options\n opts = {\n \"suffixes\": \"test,dev,web,beta,bucket,space,files,content,data,prod,staging,production,stage,app,media,development,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data,-prod,-staging,-production,-stage,-app,-media,-development\",\n \"_maxthreads\": 20\n }\n\n # Option descriptions\n optdescs = {\n \"suffixes\": \"List of suffixes to append to domains tried as blob storage names\",\n \"_maxthreads\": \"Maximum threads\"\n }\n\n results = None\n s3results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.s3results = self.tempStorage()\n self.results = self.tempStorage()\n self.lock = threading.Lock()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\", \"LINKED_URL_EXTERNAL\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"CLOUD_STORAGE_BUCKET\"]\n\n def checkSite(self, url):\n res = self.sf.fetchUrl(url, timeout=10, useragent=\"SpiderFoot\", noLog=True)\n\n if res['code']:\n with self.lock:\n self.s3results[url] = True\n\n def threadSites(self, siteList):\n self.s3results = dict()\n running = True\n i = 0\n t = []\n\n for site in siteList:\n if self.checkForStop():\n return None\n\n self.info(\"Spawning thread to check bucket: \" + site)\n tname = str(random.SystemRandom().randint(0, 999999999))\n t.append(threading.Thread(name='thread_sfp_azureblobstorages_' + tname,\n target=self.checkSite, args=(site,)))\n t[i].start()\n i += 1\n\n # Block until all threads are finished\n while running:\n found = False\n for rt in threading.enumerate():\n if rt.name.startswith(\"thread_sfp_azureblobstorages_\"):\n found = True\n\n if not found:\n running = False\n\n time.sleep(0.25)\n\n # Return once the scanning has completed\n return self.s3results\n\n def batchSites(self, sites):\n i = 0\n res = list()\n siteList = list()\n\n for site in sites:\n if i >= self.opts['_maxthreads']:\n data = self.threadSites(siteList)\n if data is None:\n return res\n\n for ret in list(data.keys()):\n if data[ret]:\n res.append(ret)\n i = 0\n siteList = list()\n\n siteList.append(site)\n i += 1\n\n return res\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventName == \"LINKED_URL_EXTERNAL\":\n if \".blob.core.windows.net\" in eventData:\n b = self.sf.urlFQDN(eventData)\n evt = SpiderFootEvent(\"CLOUD_STORAGE_BUCKET\", b, self.__name__, event)\n self.notifyListeners(evt)\n return\n\n targets = [eventData.replace('.', '')]\n kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])\n if kw:\n targets.append(kw)\n\n urls = list()\n for t in targets:\n suffixes = [''] + self.opts['suffixes'].split(',')\n for s in suffixes:\n if self.checkForStop():\n return\n\n b = t + s + \".blob.core.windows.net\"\n url = \"https://\" + b\n urls.append(url)\n\n # Batch the scans\n ret = self.batchSites(urls)\n for b in ret:\n evt = SpiderFootEvent(\"CLOUD_STORAGE_BUCKET\", b, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_azureblobstorage class\n" }, { "path": "modules/sfp_base64.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_base64\n# Purpose: Identifies (best-effort) Base64-encoded strings in URLs.\n#\n# Author: Steve Micallef \n#\n# Created: 31/01/2017\n# Copyright: (c) Steve Micallef 2017\n# -------------------------------------------------------------------------------\n\nimport base64\nimport re\nimport urllib.parse\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_base64(SpiderFootPlugin):\n\n meta = {\n 'name': \"Base64 Decoder\",\n 'summary': \"Identify Base64-encoded strings in URLs, often revealing interesting hidden information.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Content Analysis\"]\n }\n\n # Default options\n opts = {\n 'minlength': 10\n }\n\n # Option descriptions\n optdescs = {\n 'minlength': \"The minimum length a string that looks like a base64-encoded string needs to be.\"\n }\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.__dataSource__ = \"Target Website\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"LINKED_URL_INTERNAL\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"BASE64_DATA\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n decoded_data = urllib.parse.unquote(eventData)\n\n # Note: this will miss base64 encoded strings with no padding\n # (strings which do not end with '=' or '==')\n pat = re.compile(r\"([A-Za-z0-9+\\/]+={1,2})\")\n m = re.findall(pat, decoded_data)\n for match in m:\n if self.checkForStop():\n return\n\n minlen = int(self.opts['minlength'])\n if len(match) < minlen:\n continue\n\n # Base64-encoded strings don't look like normal strings\n caps = sum(1 for c in match if c.isupper())\n if caps < (minlen / 4):\n continue\n\n if isinstance(match, str):\n string = match\n else:\n string = str(match)\n\n self.info(f\"Found Base64 string: {match}\")\n\n try:\n string += f\" ({base64.b64decode(match).decode('utf-8')})\"\n except Exception as e:\n self.debug(f\"Unable to base64-decode string: {e}\")\n continue\n\n evt = SpiderFootEvent(\"BASE64_DATA\", string, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_base64 class\n" }, { "path": "modules/sfp_bgpview.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_bgpview\n# Purpose: Query BGPView API - https://bgpview.docs.apiary.io/\n#\n# Author: \n#\n# Created: 2019-09-03\n# Copyright: (c) bcoles 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_bgpview(SpiderFootPlugin):\n\n meta = {\n 'name': \"BGPView\",\n 'summary': \"Obtain network information from BGPView API.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Footprint\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://bgpview.io/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://bgpview.docs.apiary.io/#\",\n \"https://bgpview.docs.apiary.io/api-description-document\"\n ],\n 'favIcon': \"https://bgpview.io/favicon-32x32.png\",\n 'logo': \"https://bgpview.io/assets/logo.png\",\n 'description': \"BGPView is a simple API allowing consumers to view all sort of analytics data about the current state and structure of the internet.\",\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n 'IP_ADDRESS',\n 'IPV6_ADDRESS',\n 'BGP_AS_MEMBER',\n 'NETBLOCK_MEMBER',\n 'NETBLOCKV6_MEMBER'\n ]\n\n def producedEvents(self):\n return [\n 'BGP_AS_MEMBER',\n 'NETBLOCK_MEMBER',\n 'NETBLOCKV6_MEMBER',\n 'PHYSICAL_ADDRESS',\n 'RAW_RIR_DATA'\n ]\n\n def queryAsn(self, qry):\n res = self.sf.fetchUrl(\"https://api.bgpview.io/asn/\" + qry.replace('AS', ''),\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout'])\n\n time.sleep(1)\n\n if res['content'] is None:\n return None\n\n try:\n json_data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from BGPView: {e}\")\n return None\n\n if json_data.get('status') != 'ok':\n self.debug(\"No results found for ASN \" + qry)\n return None\n\n data = json_data.get('data')\n\n if not data:\n self.debug(\"No results found for ASN \" + qry)\n return None\n\n return data\n\n def queryIp(self, qry):\n res = self.sf.fetchUrl(\"https://api.bgpview.io/ip/\" + qry,\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout'])\n\n time.sleep(1)\n\n if res['content'] is None:\n return None\n\n try:\n json_data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from BGPView: {e}\")\n return None\n\n if json_data.get('status') != 'ok':\n self.debug(\"No results found for IP address \" + qry)\n return None\n\n data = json_data.get('data')\n\n if not data:\n self.debug(\"No results found for IP address \" + qry)\n return None\n\n return data\n\n def queryNetblock(self, qry):\n res = self.sf.fetchUrl(\"https://api.bgpview.io/prefix/\" + qry,\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout'])\n\n time.sleep(1)\n\n if res['content'] is None:\n return None\n\n try:\n json_data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from BGPView: {e}\")\n return None\n\n if json_data.get('status') != 'ok':\n self.debug(\"No results found for netblock \" + qry)\n return None\n\n data = json_data.get('data')\n\n if not data:\n self.debug(\"No results found for netblock \" + qry)\n return None\n\n return data\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName == 'BGP_AS_MEMBER':\n data = self.queryAsn(eventData)\n\n if not data:\n self.info(\"No results found for ASN \" + eventData)\n return\n\n e = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(e)\n\n address = data.get('owner_address')\n\n if not address:\n return\n\n evt = SpiderFootEvent('PHYSICAL_ADDRESS', ', '.join([_f for _f in address if _f]), self.__name__, event)\n self.notifyListeners(evt)\n\n if eventName in ['NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']:\n data = self.queryNetblock(eventData)\n\n if not data:\n self.info(\"No results found for netblock \" + eventData)\n return\n\n e = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(e)\n\n address = data.get('owner_address')\n\n if not address:\n return\n\n evt = SpiderFootEvent('PHYSICAL_ADDRESS', ', '.join([_f for _f in address if _f]), self.__name__, event)\n self.notifyListeners(evt)\n\n if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']:\n data = self.queryIp(eventData)\n\n if not data:\n self.info(\"No results found for IP address \" + eventData)\n return\n\n e = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(e)\n\n prefixes = data.get('prefixes')\n\n if not prefixes:\n self.info(\"No prefixes found for IP address \" + eventData)\n return\n\n for prefix in prefixes:\n p = prefix.get('prefix')\n if not p:\n continue\n\n if not prefix.get('asn'):\n continue\n\n asn = prefix.get('asn').get('asn')\n if not asn:\n continue\n\n self.info(f\"Netblock found: {p} ({asn})\")\n evt = SpiderFootEvent(\"BGP_AS_MEMBER\", str(asn), self.__name__, event)\n self.notifyListeners(evt)\n\n if self.sf.validIpNetwork(p):\n if \":\" in p:\n evt = SpiderFootEvent(\"NETBLOCKV6_MEMBER\", p, self.__name__, event)\n else:\n evt = SpiderFootEvent(\"NETBLOCK_MEMBER\", p, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_bgpview class\n" }, { "path": "modules/sfp_binaryedge.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_binaryedge\n# Purpose: Query binaryedge.io using their API\n#\n# Author: Steve Micallef\n#\n# Created: 02/04/2019\n# Copyright: (c) Steve Micallef 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\n\nfrom netaddr import IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_binaryedge(SpiderFootPlugin):\n\n meta = {\n 'name': \"BinaryEdge\",\n 'summary': \"Obtain information from BinaryEdge.io Internet scanning systems, including breaches, vulnerabilities, torrents and passive DNS.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://www.binaryedge.io/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://docs.binaryedge.io/\",\n \"https://www.binaryedge.io/data.html\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://www.binaryedge.io/pricing.html\",\n \"Select a plan\",\n \"Sign up with new account\",\n \"Go to Account\",\n \"The API key is listed under 'API Access'\"\n ],\n 'favIcon': \"https://www.binaryedge.io/img/favicon/favicon-32x32.png\",\n 'logo': \"https://www.binaryedge.io/img/logo.png\",\n 'description': \"We scan the entire public internet, create real-time threat intelligence streams, \"\n \"and reports that show the exposure of what is connected to the Internet.\\n\"\n \"We have built a distributed platform of scanners and honeypots, to acquire, classify and correlate different types of data.\\n\"\n \"We use all of these datapoints to match those digital assets to an organization, \"\n \"allowing us to provide a global, up-to-date, view of organizations known and unknown assets.\",\n }\n }\n\n opts = {\n 'binaryedge_api_key': \"\",\n 'torrent_age_limit_days': 30,\n 'cve_age_limit_days': 30,\n 'port_age_limit_days': 90,\n 'maxpages': 10,\n 'verify': True,\n 'netblocklookup': False,\n 'maxnetblock': 24,\n 'subnetlookup': False,\n 'maxsubnet': 24,\n 'maxcohost': 100\n }\n\n optdescs = {\n 'binaryedge_api_key': \"BinaryEdge.io API Key.\",\n 'torrent_age_limit_days': \"Ignore any torrent records older than this many days. 0 = unlimited.\",\n 'cve_age_limit_days': \"Ignore any vulnerability records older than this many days. 0 = unlimited.\",\n 'port_age_limit_days': \"Ignore any discovered open ports/banners older than this many days. 0 = unlimited.\",\n 'verify': 'Verify that any hostnames found on the target domain still resolve?',\n 'maxpages': \"Maximum number of pages to iterate through, to avoid exceeding BinaryEdge API usage limits. APIv2 has a maximum of 500 pages (10,000 results).\",\n 'netblocklookup': \"Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?\",\n 'maxnetblock': \"If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'subnetlookup': \"Look up all IPs on subnets which your target is a part of?\",\n 'maxsubnet': \"If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'maxcohost': \"Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.\"\n }\n\n results = None\n errorState = False\n cohostcount = 0\n reportedhosts = None\n checkedips = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.reportedhosts = self.tempStorage()\n self.checkedips = self.tempStorage()\n self.cohostcount = 0\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"DOMAIN_NAME\",\n \"EMAILADDR\",\n \"NETBLOCK_OWNER\",\n \"NETBLOCK_MEMBER\"\n ]\n\n def producedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"DOMAIN_NAME\",\n \"VULNERABILITY_CVE_CRITICAL\",\n \"VULNERABILITY_CVE_HIGH\",\n \"VULNERABILITY_CVE_MEDIUM\",\n \"VULNERABILITY_CVE_LOW\",\n \"VULNERABILITY_GENERAL\",\n \"TCP_PORT_OPEN\",\n \"TCP_PORT_OPEN_BANNER\",\n \"EMAILADDR_COMPROMISED\",\n \"UDP_PORT_OPEN\",\n \"UDP_PORT_OPEN_INFO\",\n \"CO_HOSTED_SITE\",\n \"MALICIOUS_IPADDR\"\n ]\n\n def query(self, qry, querytype, page=1):\n retarr = list()\n\n if self.errorState:\n return None\n\n if querytype == \"email\":\n queryurl = \"dataleaks/email\"\n elif querytype == \"ip\":\n queryurl = \"ip\"\n elif querytype == \"torrent\":\n queryurl = \"torrent/historical\"\n elif querytype == \"vuln\":\n queryurl = \"cve/ip\"\n elif querytype == \"subs\":\n queryurl = \"domains/subdomain\"\n elif querytype == \"passive\":\n queryurl = \"domains/ip\"\n else:\n self.error(f\"Invalid query type: {querytype}\")\n return None\n\n headers = {\n 'X-Key': self.opts['binaryedge_api_key']\n }\n\n res = self.sf.fetchUrl(\n f\"https://api.binaryedge.io/v2/query/{queryurl}/{qry}?page={page}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\",\n headers=headers\n )\n\n if res['code'] in [\"429\", \"500\"]:\n self.error(\"BinaryEdge.io API key seems to have been rejected or you have exceeded usage limits for the month.\")\n self.errorState = True\n return None\n\n if not res['content']:\n self.info(f\"No BinaryEdge.io info found for {qry}\")\n return None\n\n try:\n info = json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from BinaryEdge.io: {e}\")\n return None\n\n if info.get('page') and info['total'] > info.get('pagesize', 100) * info.get('page', 0):\n page = info['page'] + 1\n if page > self.opts['maxpages']:\n self.error(\"Maximum number of pages reached.\")\n return [info]\n retarr.append(info)\n e = self.query(qry, querytype, page)\n if e:\n retarr.extend(e)\n else:\n retarr.append(info)\n\n return retarr\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"binaryedge_api_key\"] == \"\":\n self.error(\n f\"You enabled {self.__class__.__name__} but did not set an API key!\"\n )\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName == \"EMAILADDR\":\n ret = self.query(eventData, \"email\")\n if ret is None:\n self.info(f\"No leak info for {eventData}\")\n return\n\n for rec in ret:\n events = rec.get('events')\n if not events:\n continue\n\n self.debug(\"Found compromised account results in BinaryEdge.io\")\n\n for leak in events:\n e = SpiderFootEvent('EMAILADDR_COMPROMISED', f\"{eventData} [{leak}]\", self.__name__, event)\n self.notifyListeners(e)\n\n # No further API endpoints available for email addresses. we can bail out here\n return\n\n if eventName == 'NETBLOCK_OWNER':\n if not self.opts['netblocklookup']:\n return\n\n net_size = IPNetwork(eventData).prefixlen\n max_netblock = self.opts['maxnetblock']\n if net_size < max_netblock:\n self.debug(f\"Network size bigger than permitted: {net_size} > {max_netblock}\")\n return\n\n if eventName == 'NETBLOCK_MEMBER':\n if not self.opts['subnetlookup']:\n return\n\n net_size = IPNetwork(eventData).prefixlen\n max_subnet = self.opts['maxsubnet']\n if net_size < max_subnet:\n self.debug(f\"Network size bigger than permitted: {net_size} > {max_subnet}\")\n return\n\n # For IP Addresses, do the additional passive DNS lookup\n if eventName == \"IP_ADDRESS\":\n evtType = \"CO_HOSTED_SITE\"\n ret = self.query(eventData, \"passive\")\n if ret is None:\n self.info(f\"No Passive DNS info for {eventData}\")\n return\n\n for rec in ret:\n events = rec.get('events')\n if not events:\n continue\n\n self.debug(\"Found passive DNS results in BinaryEdge.io\")\n for rec in events:\n host = rec['domain']\n if host == eventData:\n continue\n\n if self.getTarget().matches(host, includeParents=True):\n if self.opts['verify'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host):\n continue\n\n evt = SpiderFootEvent(\"INTERNET_NAME\", host, self.__name__, event)\n self.notifyListeners(evt)\n if self.sf.isDomain(host, self.opts['_internettlds']):\n evt = SpiderFootEvent(\"DOMAIN_NAME\", host, self.__name__, event)\n self.notifyListeners(evt)\n\n self.reportedhosts[host] = True\n continue\n\n if self.cohostcount < self.opts['maxcohost']:\n e = SpiderFootEvent(evtType, host, self.__name__, event)\n self.notifyListeners(e)\n self.cohostcount += 1\n\n if eventName == \"DOMAIN_NAME\":\n ret = self.query(eventData, \"subs\")\n if ret is None:\n self.info(f\"No hosts found for {eventData}\")\n return\n\n for rec in ret:\n events = rec.get('events')\n if not events:\n continue\n\n self.debug(\"Found host results in BinaryEdge.io\")\n for rec in events:\n if rec in self.reportedhosts:\n continue\n\n self.reportedhosts[rec] = True\n\n if self.opts['verify'] and not self.sf.resolveHost(rec) and not self.sf.resolveHost6(rec):\n self.debug(f\"Couldn't resolve {rec}, so skipping.\")\n continue\n\n e = SpiderFootEvent('INTERNET_NAME', rec, self.__name__, event)\n self.notifyListeners(e)\n\n # Loop through all IP addresses / host names\n qrylist = list()\n if eventName.startswith(\"NETBLOCK_\"):\n for ipaddr in IPNetwork(eventData):\n qrylist.append(str(ipaddr))\n self.results[str(ipaddr)] = True\n else:\n qrylist.append(eventData)\n\n for addr in qrylist:\n if self.checkForStop():\n return\n\n if self.errorState:\n return\n\n if addr in self.checkedips:\n continue\n\n ret = self.query(addr, \"torrent\")\n if ret is None:\n self.info(f\"No torrent info for {addr}\")\n continue\n\n for rec in ret:\n events = rec.get('events')\n if not events:\n continue\n\n self.debug(f\"Found torrent results for {addr} in BinaryEdge.io\")\n\n for rec in events:\n created_ts = rec['origin'].get('ts') / 1000\n age_limit_ts = int(time.time()) - (86400 * self.opts['torrent_age_limit_days'])\n\n if self.opts['torrent_age_limit_days'] > 0 and created_ts < age_limit_ts:\n self.debug(\"Record found but too old, skipping.\")\n continue\n\n dat = \"Torrent: \" + rec.get(\"torrent\", \"???\").get(\"name\") + \" @ \" + rec.get('torrent').get(\"source\", \"???\")\n e = SpiderFootEvent('MALICIOUS_IPADDR', dat, self.__name__, event)\n self.notifyListeners(e)\n\n for addr in qrylist:\n if self.checkForStop():\n return\n\n if self.errorState:\n return\n\n if addr in self.checkedips:\n continue\n\n ret = self.query(addr, \"vuln\")\n if ret is None:\n self.info(f\"No vulnerability info for {addr}\")\n continue\n\n for rec in ret:\n events = rec.get('events')\n if not events:\n continue\n\n results = events.get('results')\n if not results:\n continue\n\n self.debug(\"Found vulnerability results in BinaryEdge.io\")\n for rec in results:\n created_ts = rec.get('ts') / 1000\n age_limit_ts = int(time.time()) - (86400 * self.opts['cve_age_limit_days'])\n\n if self.opts['cve_age_limit_days'] > 0 and created_ts < age_limit_ts:\n self.debug(\"Record found but too old, skipping.\")\n continue\n\n cves = rec.get('cves')\n if cves:\n for c in cves:\n etype, cvetext = self.sf.cveInfo(c['cve'])\n e = SpiderFootEvent(etype, cvetext, self.__name__, event)\n self.notifyListeners(e)\n\n for addr in qrylist:\n if self.checkForStop():\n return\n\n if self.errorState:\n return\n\n if addr in self.checkedips:\n continue\n\n ret = self.query(addr, \"ip\")\n if ret is None:\n self.info(f\"No port/banner info for {addr}\")\n return\n\n for rec in ret:\n events = rec.get('events')\n if not events:\n continue\n\n self.debug(\"Found port/banner results in BinaryEdge.io\")\n\n ports = list()\n for res in events:\n for prec in res['results']:\n created_ts = prec['origin'].get('ts') / 1000\n age_limit_ts = int(time.time()) - (86400 * self.opts['port_age_limit_days'])\n\n if self.opts['port_age_limit_days'] > 0 and created_ts < age_limit_ts:\n self.debug(\"Record found but too old, skipping.\")\n continue\n\n port = str(prec['target']['port'])\n entity = prec['target']['ip'] + \":\" + port\n evttype = \"TCP_PORT_OPEN\"\n evtbtype = \"TCP_PORT_OPEN_BANNER\"\n\n if prec['target']['protocol'] == \"udp\":\n evttype = \"UDP_PORT_OPEN\"\n evtbtype = \"UDP_PORT_OPEN_INFO\"\n\n if f\"{evttype}:{port}\" not in ports:\n ev = SpiderFootEvent(evttype, entity, self.__name__, event)\n self.notifyListeners(ev)\n ports.append(f\"{evttype}:{port}\")\n\n try:\n banner = prec['result']['data']['service']['banner']\n if '\\\\r\\\\n\\\\r\\\\n' in banner and \"HTTP/\" in banner:\n # We don't want the content after HTTP banners\n banner = banner.split('\\\\r\\\\n\\\\r\\\\n')[0]\n banner = banner.replace(\"\\\\r\\\\n\", \"\\n\")\n except Exception:\n self.debug(\"No banner information found.\")\n continue\n\n e = SpiderFootEvent(evtbtype, banner, self.__name__, ev)\n self.notifyListeners(e)\n\n for addr in qrylist:\n self.checkedips[addr] = True\n\n# End of sfp_binaryedge class\n" }, { "path": "modules/sfp_bingsearch.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_bingsearch\n# Purpose: Searches Bing for content related to the domain in question.\n#\n# Author: Steve Micallef \n#\n# Created: 06/10/2013\n# Copyright: (c) Steve Micallef 2013\n# Licence: MIT\n# -------------------------------------------------------------------------------\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_bingsearch(SpiderFootPlugin):\n\n meta = {\n 'name': \"Bing\",\n 'summary': \"Obtain information from bing to identify sub-domains and links.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://www.bing.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://docs.microsoft.com/en-us/azure/cognitive-services/bing-web-search/\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://azure.microsoft.com/en-in/services/cognitive-services/bing-web-search-api/\",\n \"Register a free account\",\n \"Select on Bing Custom Search\",\n \"The API keys are listed under 'Key1' and 'Key2' (both should work)\"\n ],\n 'favIcon': \"https://www.bing.com/sa/simg/bing_p_rr_teal_min.ico\",\n 'logo': \"https://www.bing.com/sa/simg/bing_p_rr_teal_min.ico\",\n 'description': \"The Bing Search APIs let you build web-connected apps and services that \"\n \"find webpages, images, news, locations, and more without advertisements.\",\n }\n }\n\n # Default options\n opts = {\n \"pages\": 20,\n \"api_key\": \"\"\n }\n\n # Option descriptions\n optdescs = {\n \"pages\": \"Number of max bing results to request from the API.\",\n \"api_key\": \"Bing API Key for Bing search.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"INTERNET_NAME\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"LINKED_URL_INTERNAL\", \"RAW_RIR_DATA\"]\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_bingsearch but did not set a Bing API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(\"Already did a search for \" + eventData + \", skipping.\")\n return\n\n self.results[eventData] = True\n\n # Sites hosted on the domain\n\n res = self.sf.bingIterate(\n searchString=\"site:\" + eventData,\n opts={\n \"timeout\": self.opts[\"_fetchtimeout\"],\n \"useragent\": self.opts[\"_useragent\"],\n \"count\": self.opts[\"pages\"],\n \"api_key\": self.opts[\"api_key\"],\n },\n )\n if res is None:\n # Failed to talk to the bing API or no results returned\n return\n\n urls = res[\"urls\"]\n new_links = list(set(urls) - set(self.results.keys()))\n\n # Add new links to results\n for link in new_links:\n self.results[link] = True\n\n internal_links = [\n link for link in new_links if self.sf.urlFQDN(link).endswith(eventData)\n ]\n for link in internal_links:\n self.debug(\"Found a link: \" + link)\n\n evt = SpiderFootEvent(\"LINKED_URL_INTERNAL\", link, self.__name__, event)\n self.notifyListeners(evt)\n\n if internal_links:\n evt = SpiderFootEvent(\n \"RAW_RIR_DATA\", str(res), self.__name__, event\n )\n self.notifyListeners(evt)\n\n# End of sfp_bingsearch class\n" }, { "path": "modules/sfp_bingsharedip.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_bingsharedip\n# Purpose: Searches Bing for hosts sharing the same IP.\n#\n# Author: Steve Micallef \n#\n# Created: 12/04/2014\n# Copyright: (c) Steve Micallef 2014\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom netaddr import IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_bingsharedip(SpiderFootPlugin):\n\n meta = {\n 'name': \"Bing (Shared IPs)\",\n 'summary': \"Search Bing for hosts sharing the same IP.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://www.bing.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://docs.microsoft.com/en-us/azure/cognitive-services/bing-web-search/\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://azure.microsoft.com/en-in/services/cognitive-services/bing-web-search-api/\",\n \"Register a free account\",\n \"Select on Bing Custom Search\",\n \"The API keys are listed under 'Key1' and 'Key2' (both should work)\"\n ],\n 'favIcon': \"https://www.bing.com/sa/simg/bing_p_rr_teal_min.ico\",\n 'logo': \"https://www.bing.com/sa/simg/bing_p_rr_teal_min.ico\",\n 'description': \"The Bing Search APIs let you build web-connected apps and services that \"\n \"find webpages, images, news, locations, and more without advertisements.\",\n }\n }\n\n # Default options\n opts = {\n \"cohostsamedomain\": False,\n \"pages\": 20,\n \"verify\": True,\n \"maxcohost\": 100,\n \"api_key\": \"\"\n }\n\n # Option descriptions\n optdescs = {\n \"cohostsamedomain\": \"Treat co-hosted sites on the same target domain as co-hosting?\",\n \"pages\": \"Number of max bing results to request from API.\",\n \"verify\": \"Verify co-hosts are valid by checking if they still resolve to the shared IP.\",\n \"maxcohost\": \"Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.\",\n \"api_key\": \"Bing API Key for shared IP search.\"\n }\n\n results = None\n cohostcount = 0\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.cohostcount = 0\n self.__dataSource__ = \"Bing\"\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"IP_ADDRESS\", \"NETBLOCK_OWNER\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"CO_HOSTED_SITE\", \"IP_ADDRESS\", \"RAW_RIR_DATA\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n self.currentEventSrc = event\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts['api_key'] == \"\" and self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_bingsharedip but did not set a Bing API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n # Ignore IP addresses from myself as they are just for creating\n # a link from the netblock to the co-host.\n if eventName == \"IP_ADDRESS\" and srcModuleName == \"sfp_bingsharedip\":\n self.debug(\"Ignoring \" + eventName + \", from self.\")\n return\n\n if self.cohostcount > self.opts[\"maxcohost\"]:\n return\n\n qrylist = list()\n if eventName.startswith(\"NETBLOCK_\"):\n for ipaddr in IPNetwork(eventData):\n if str(ipaddr) not in self.results:\n qrylist.append(str(ipaddr))\n self.results[str(ipaddr)] = True\n else:\n qrylist.append(eventData)\n self.results[eventData] = True\n\n myres = list()\n\n for ip in qrylist:\n if self.checkForStop():\n return\n\n res = self.sf.bingIterate(\n searchString=\"ip:\" + ip,\n opts={\n \"timeout\": self.opts[\"_fetchtimeout\"],\n \"useragent\": self.opts[\"_useragent\"],\n \"count\": self.opts[\"pages\"],\n \"api_key\": self.opts[\"api_key\"],\n },\n )\n if res is None:\n # Failed to talk to bing api or no results returned\n return\n\n urls = res[\"urls\"]\n\n for url in urls:\n self.info(\"Found something on same IP: \" + url)\n site = self.sf.urlFQDN(url.lower())\n if site not in myres and site != ip:\n if not self.opts[\"cohostsamedomain\"]:\n if self.getTarget().matches(site, includeParents=True):\n self.debug(\n f\"Skipping {site} because it is on the same domain.\"\n )\n continue\n if self.opts[\"verify\"] and not self.sf.validateIP(site, ip):\n self.debug(f\"Host {site} no longer resolves to {ip}\")\n continue\n # Create an IP Address event stemming from the netblock as the\n # link to the co-host.\n if eventName == \"NETBLOCK_OWNER\":\n ipe = SpiderFootEvent(\"IP_ADDRESS\", ip, self.__name__, event)\n self.notifyListeners(ipe)\n evt = SpiderFootEvent(\n \"CO_HOSTED_SITE\", site, self.__name__, ipe\n )\n self.notifyListeners(evt)\n else:\n evt = SpiderFootEvent(\n \"CO_HOSTED_SITE\", site, self.__name__, event\n )\n self.notifyListeners(evt)\n self.cohostcount += 1\n myres.append(site)\n\n if urls:\n evt = SpiderFootEvent(\n \"RAW_RIR_DATA\", str(res), self.__name__, event\n )\n self.notifyListeners(evt)\n\n# End of sfp_bingsharedip class\n" }, { "path": "modules/sfp_binstring.py", "content": "# coding: utf-8\n# -------------------------------------------------------------------------------\n# Name: sfp_binstring\n# Purpose: Identify strings in binary content.\n#\n# Author: Steve Micallef \n#\n# Created: 03/12/2016\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport string\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_binstring(SpiderFootPlugin):\n\n meta = {\n 'name': \"Binary String Extractor\",\n 'summary': \"Attempt to identify strings in binary content.\",\n 'flags': [\"errorprone\"],\n 'useCases': [\"Footprint\"],\n 'categories': [\"Content Analysis\"]\n }\n\n # Default options\n opts = {\n 'minwordsize': 5,\n 'maxwords': 100,\n 'maxfilesize': 1000000,\n 'usedict': True,\n 'fileexts': ['png', 'gif', 'jpg', 'jpeg', 'tiff', 'tif',\n 'ico', 'flv', 'mp4', 'mp3', 'avi', 'mpg',\n 'mpeg', 'dat', 'mov', 'swf', 'exe', 'bin'],\n 'filterchars': '#}{|%^&*()=+,;[]~'\n }\n\n # Option descriptions\n optdescs = {\n 'minwordsize': \"Upon finding a string in a binary, ensure it is at least this length. Helps weed out false positives.\",\n 'usedict': \"Use the dictionary to further reduce false positives - any string found must contain a word from the dictionary (can be very slow, especially for larger files).\",\n 'fileexts': \"File types to fetch and analyse.\",\n 'maxfilesize': \"Maximum file size in bytes to download for analysis.\",\n 'maxwords': \"Stop reporting strings from a single binary after this many are found.\",\n 'filterchars': \"Ignore strings with these characters, as they may just be garbage ASCII.\"\n }\n\n results = list()\n d = None\n n = None\n fq = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = list()\n self.__dataSource__ = \"Target Website\"\n\n self.d = SpiderFootHelpers.dictionaryWordsFromWordlists()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def getStrings(self, content):\n words = list()\n result = \"\"\n\n if not content:\n return None\n\n for c in content:\n c = str(c)\n if len(words) >= self.opts['maxwords']:\n break\n if c in string.printable and c not in string.whitespace:\n result += c\n continue\n if len(result) >= self.opts['minwordsize']:\n if self.opts['usedict']:\n accept = False\n for w in self.d:\n if result.startswith(w) or result.endswith(w):\n accept = True\n break\n\n if self.opts['filterchars']:\n accept = True\n for x in self.opts['filterchars']:\n if x in result:\n accept = False\n break\n\n if not self.opts['filterchars'] and not self.opts['usedict']:\n accept = True\n\n if accept:\n words.append(result)\n\n result = \"\"\n\n if len(words) == 0:\n return None\n\n return words\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"LINKED_URL_INTERNAL\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"RAW_FILE_META_DATA\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n return\n\n self.results.append(eventData)\n\n for fileExt in self.opts['fileexts']:\n if eventData.lower().endswith(f\".{fileExt.lower()}\") or f\".{fileExt.lower()}?\" in eventData.lower():\n res = self.sf.fetchUrl(\n eventData,\n useragent=self.opts['_useragent'],\n disableContentEncoding=True,\n sizeLimit=self.opts['maxfilesize'],\n verify=False\n )\n\n if not res:\n continue\n\n self.debug(f\"Searching {eventData} for strings\")\n words = self.getStrings(res['content'])\n\n if words:\n wordstr = '\\n'.join(words[0:self.opts['maxwords']])\n evt = SpiderFootEvent(\"RAW_FILE_META_DATA\", wordstr, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_binstring class\n" }, { "path": "modules/sfp_bitcoin.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_bitcoin\n# Purpose: SpiderFoot plug-in for scanning retrieved content by other\n# modules (such as sfp_spider) and identifying bitcoin numbers.\n#\n# Author: Steve Micallef \n#\n# Created: 27/05/2017\n# Copyright: (c) Steve Micallef 2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport codecs\nimport re\nfrom hashlib import sha256\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_bitcoin(SpiderFootPlugin):\n\n meta = {\n 'name': \"Bitcoin Finder\",\n 'summary': \"Identify bitcoin addresses in scraped webpages.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Content Analysis\"]\n }\n\n opts = {}\n optdescs = {}\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"TARGET_WEB_CONTENT\"]\n\n def producedEvents(self):\n return [\"BITCOIN_ADDRESS\"]\n\n def to_bytes(self, n, length):\n h = '%x' % n\n return codecs.decode(('0' * (len(h) % 2) + h).zfill(length * 2), \"hex\")\n\n def decode_base58(self, bc, length):\n digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'\n n = 0\n for char in bc:\n n = n * 58 + digits58.index(char)\n return self.to_bytes(n, length)\n\n def check_bc(self, bc):\n bcbytes = self.decode_base58(bc, 25)\n return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n sourceData = self.sf.hashstring(eventData)\n\n if sourceData in self.results:\n return\n\n self.results[sourceData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n addrs = list()\n\n # thanks to https://stackoverflow.com/questions/21683680/regex-to-match-bitcoin-addresses\n # Does not support keys or testnet addresses\n matches = re.findall(r\"[\\s:=\\>](bc(0([ac-hj-np-z02-9]{39}|[ac-hj-np-z02-9]{59})|1[ac-hj-np-z02-9]{8,87})|[13][a-km-zA-HJ-NP-Z1-9]{25,35})\", eventData)\n for m in matches:\n address = m[0]\n self.debug(f\"Potential Bitcoin address match: {address}\")\n\n if address.startswith('1') or address.startswith('3'):\n if self.check_bc(address):\n addrs.append(address)\n else:\n addrs.append(address)\n\n for address in set(addrs):\n evt = SpiderFootEvent(\"BITCOIN_ADDRESS\", address, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_bitcoin class\n" }, { "path": "modules/sfp_bitcoinabuse.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_bitcoinabuse\n# Purpose: Check bitcoin address agains bitcoinabuse.com database\n#\n# Author: Leo Trubach \n#\n# Created: 2020-09-01\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib.parse\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_bitcoinabuse(SpiderFootPlugin):\n meta = {\n \"name\": \"BitcoinAbuse\",\n \"summary\": \"Check Bitcoin addresses against the bitcoinabuse.com database of suspect/malicious addresses.\",\n 'flags': [\"apikey\"],\n \"useCases\": [\"Passive\", \"Investigate\"],\n \"categories\": [\"Reputation Systems\"],\n \"dataSource\": {\n \"website\": \"https://www.bitcoinabuse.com/\",\n \"model\": \"FREE_AUTH_UNLIMITED\",\n \"references\": [\"https://www.bitcoinabuse.com/api-docs\"],\n \"apiKeyInstructions\": [\n \"Visit https://www.bitcoinabuse.com/register\",\n \"Register a free account\",\n \"Click on the account icon and click on 'Your Settings'\",\n \"Click on 'API'\",\n \"Enter a token name and press 'Create'\",\n ],\n \"favIcon\": \"https://www.bitcoinabuse.com/favicon-32x32.png\",\n \"logo\": \"https://www.bitcoinabuse.com/img/logo-sm.png\",\n \"description\": \"BitcoinAbuse.com is a public database of bitcoin \"\n \"addresses used by scammers, hackers, and criminals.\"\n \"Bitcoin is anonymous if used perfectly. Luckily, no \"\n \"one is perfect. Even hackers make mistakes. It only \"\n \"takes one slip to link stolen bitcoin to a hacker's \"\n \"their real identity. It is our hope that by making a \"\n \"public database of bitcoin addresses used by criminals it \"\n \"will be harder for criminals to convert the digital currency\"\n \" back into fiat money.\",\n },\n }\n opts = {\n \"api_key\": \"\",\n }\n optdescs = {\n \"api_key\": \"BitcoinAbuse API Key.\",\n }\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=None):\n self.sf = sfc\n self.errorState = False\n self.results = self.tempStorage()\n\n if userOpts:\n self.opts.update(userOpts)\n\n def watchedEvents(self):\n return [\"BITCOIN_ADDRESS\"]\n\n def producedEvents(self):\n return [\n \"MALICIOUS_BITCOIN_ADDRESS\",\n \"RAW_RIR_DATA\",\n ]\n\n def queryAddress(self, address: str):\n params = {\n \"address\": address,\n \"api_token\": self.opts[\"api_key\"]\n }\n\n res = self.sf.fetchUrl(\n f\"https://www.bitcoinabuse.com/api/reports/check?{urllib.parse.urlencode(params)}\",\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=\"SpiderFoot\",\n )\n\n # All endpoints other than Report Address have a rate limit of\n # 30 requests per minute or one request every two seconds on average.\n time.sleep(2)\n\n return self.parseApiResponse(res)\n\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from BitcoinAbuse.\")\n return None\n\n if res['code'] == '404':\n self.debug(\"No results for query\")\n return None\n\n if res['code'] == \"401\":\n self.error(\"Invalid BitcoinAbuse API key.\")\n self.errorState = True\n return None\n\n if res['code'] == '429':\n self.error(\"You are being rate-limited by BitcoinAbuse\")\n self.errorState = True\n return None\n\n if res['code'] in ['500', '502', '503']:\n self.error(\"BitcoinAbuse service unavailable\")\n self.errorState = True\n return None\n\n # Catch all other non-200 status codes, and presume something went wrong\n if res['code'] != '200':\n self.error(\"Failed to retrieve content from BitcoinAbuse\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from BitcoinAbuse: {e}\")\n\n return None\n\n def handleEvent(self, event):\n if self.errorState:\n return\n\n eventName = event.eventType\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(f\"You enabled {self.__class__.__name__} but did not set an API key!\")\n self.errorState = True\n return\n\n eventData = event.data\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName == \"BITCOIN_ADDRESS\":\n rec = self.queryAddress(eventData)\n\n if not rec:\n return\n\n if not isinstance(rec, dict):\n return\n\n count = rec.get(\"count\")\n\n if not count:\n return\n\n if not isinstance(count, int):\n return\n\n address = rec.get('address')\n\n if not address:\n return\n\n url = f\"https://www.bitcoinabuse.com/reports/{address}\"\n evt = SpiderFootEvent(\n \"MALICIOUS_BITCOIN_ADDRESS\",\n f\"BitcoinAbuse [{address}]\\n{url}\",\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n\n rirevt = SpiderFootEvent(\n \"RAW_RIR_DATA\", json.dumps(rec), self.__name__, event\n )\n self.notifyListeners(rirevt)\n" }, { "path": "modules/sfp_bitcoinwhoswho.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_bitcoinwhoswho\n# Purpose: Bitcoin Who's Who database lookup module\n#\n# Author: Leo Trubach \n#\n# Created: 2020-09-09\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport urllib.parse\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_bitcoinwhoswho(SpiderFootPlugin):\n meta = {\n 'name': \"Bitcoin Who's Who\",\n 'summary': \"Check for Bitcoin addresses against the Bitcoin Who's Who database of suspect/malicious addresses.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Passive\", \"Investigate\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://bitcoinwhoswho.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://bitcoinwhoswho.com/api\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://bitcoinwhoswho.com/signup\",\n \"Register a free account\",\n \"Verify your email and sign into your account\",\n \"Visit https://bitcoinwhoswho.com/api/register and request an API Key\",\n \"Wait for a few days, you'll receive it to your email\"\n ],\n 'favIcon': \"https://bitcoinwhoswho.com/public/images/ico/favicon.ico\",\n 'logo': \"https://bitcoinwhoswho.com/public/images/logo2.png\",\n 'description': (\n \"Bitcoin Who's Who is dedicated to profiling the extraordinary members of the bitcoin \"\n \"ecosystem.Our goal is to help you verify a bitcoin address owner and avoid a bitcoin \"\n \"scam or fraud.\"\n ),\n }\n }\n\n opts = {\n 'api_key': '',\n }\n\n optdescs = {\n \"api_key\": \"Bitcoin Who's Who API Key.\"\n }\n\n results = None\n\n errorState = False\n\n def setup(self, sfc, userOpts=None):\n self.sf = sfc\n self.results = self.tempStorage()\n\n if userOpts:\n self.opts.update(userOpts)\n\n def watchedEvents(self):\n return [\"BITCOIN_ADDRESS\"]\n\n def producedEvents(self):\n return [\"MALICIOUS_BITCOIN_ADDRESS\", \"RAW_RIR_DATA\"]\n\n def query(self, qry):\n qs = urllib.parse.urlencode({\"address\": qry})\n res = self.sf.fetchUrl(\n f\"https://bitcoinwhoswho.com/api/scam/{self.opts['api_key']}?{qs}\",\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=\"SpiderFoot\",\n )\n\n if res[\"content\"] is None:\n self.info(f\"No {self.meta['name']} info found for {qry}\")\n return None\n\n try:\n return json.loads(res[\"content\"])\n except Exception as e:\n self.error(f\"Error processing JSON response from {self.meta['name']}: {e}\")\n\n return None\n\n def emit(self, etype, data, pevent, notify=True):\n evt = SpiderFootEvent(etype, data, self.__name__, pevent)\n if notify:\n self.notifyListeners(evt)\n return evt\n\n def generate_events(self, data, pevent):\n if not isinstance(data, dict):\n return False\n\n scams = data.get(\"scams\", [])\n if scams:\n self.emit(\"MALICIOUS_BITCOIN_ADDRESS\", f\"Bitcoin Who's Who [{pevent.data}][https://bitcoinwhoswho.com/address/{pevent.data}]\", pevent)\n return True\n\n return False\n\n def handleEvent(self, event):\n if self.errorState:\n return\n\n self.debug(f\"Received event, {event.eventType}, from {event.module}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(f\"You enabled {self.__class__.__name__} but did not set an API key!\")\n self.errorState = True\n return\n\n if event.data in self.results:\n self.debug(f\"Skipping {event.data}, already checked.\")\n return\n self.results[event.data] = True\n\n if event.eventType == \"BITCOIN_ADDRESS\":\n data = self.query(event.data)\n r = self.generate_events(data, event)\n\n if r:\n self.emit(\"RAW_RIR_DATA\", json.dumps(data), event)\n\n# End of sfp_bitcoinwhoswho class\n" }, { "path": "modules/sfp_blockchain.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_blockchain\n# Purpose: SpiderFoot plug-in to look up a bitcoin wallet's balance by\n# querying blockchain.info.\n#\n# Author: Steve Micallef \n#\n# Created: 18/06/2017\n# Copyright: (c) Steve Micallef 2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_blockchain(SpiderFootPlugin):\n\n meta = {\n 'name': \"Blockchain\",\n 'summary': \"Queries blockchain.info to find the balance of identified bitcoin wallet addresses.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Public Registries\"],\n 'dataSource': {\n 'website': \"https://www.blockchain.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://exchange.blockchain.com/api/#introduction\",\n \"https://exchange.blockchain.com/markets\",\n \"https://exchange.blockchain.com/fees\",\n \"https://exchange.blockchain.com/trade\"\n ],\n 'favIcon': \"https://www.blockchain.com/static/favicon.ico\",\n 'logo': \"https://exchange.blockchain.com/api/assets/images/logo.png\",\n 'description': \"Blockchain Exchange is the most secure place to buy, sell, and trade crypto.\\n\"\n \"Use the most popular block explorer to search and \"\n \"verify transactions on the Bitcoin, Ethereum, and Bitcoin Cash blockchains.\\n\"\n \"Stay on top of Bitcoin and other top cryptocurrency prices, news, and market information.\",\n }\n }\n\n # Default options\n opts = {}\n optdescs = {}\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['BITCOIN_ADDRESS']\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"BITCOIN_BALANCE\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n # Wallet balance\n res = self.sf.fetchUrl(\"https://blockchain.info/balance?active=\" + eventData,\n timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])\n if res['content'] is None:\n self.info(\"No Blockchain info found for \" + eventData)\n return\n try:\n data = json.loads(res['content'])\n balance = float(data[eventData]['final_balance']) / 100000000\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return\n\n evt = SpiderFootEvent(\"BITCOIN_BALANCE\", str(balance) + \" BTC\", self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_blockchain class\n" }, { "path": "modules/sfp_blocklistde.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_blocklistde\n# Purpose: Check if a netblock or IP is malicious according to blocklist.de.\n#\n# Author: steve@binarypool.com\n#\n# Created: 14/12/2013\n# Copyright: (c) Steve Micallef, 2013\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom netaddr import IPAddress, IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_blocklistde(SpiderFootPlugin):\n\n meta = {\n 'name': \"blocklist.de\",\n 'summary': \"Check if a netblock or IP is malicious according to blocklist.de.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"http://www.blocklist.de/en/index.html\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"http://www.blocklist.de/en/api.html\",\n \"http://www.blocklist.de/en/rbldns.html\",\n \"http://www.blocklist.de/en/httpreports.html\",\n \"http://www.blocklist.de/en/export.html\",\n \"http://www.blocklist.de/en/delist.html?ip=\"\n ],\n 'favIcon': \"http://www.blocklist.de/templates/css/logo_web-size.jpg\",\n 'logo': \"http://www.blocklist.de/templates/css/logo_web-size.jpg\",\n 'description': \"www.blocklist.de is a free and voluntary service provided by a Fraud/Abuse-specialist, \"\n \"whose servers are often attacked via SSH-, Mail-Login-, FTP-, Webserver- and other services.\\n\"\n \"The mission is to report any and all attacks to the respective abuse departments of the infected PCs/servers, \"\n \"to ensure that the responsible provider can inform their customer about the infection and disable the attacker.\"\n }\n }\n\n opts = {\n 'checkaffiliates': True,\n 'cacheperiod': 18,\n 'checknetblocks': True,\n 'checksubnets': True\n }\n\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliates?\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\",\n 'checknetblocks': \"Report if any malicious IPs are found within owned netblocks?\",\n 'checksubnets': \"Check if any malicious IPs are found within the same subnet of the target?\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"AFFILIATE_IPV6_ADDRESS\",\n \"NETBLOCK_MEMBER\",\n \"NETBLOCKV6_MEMBER\",\n \"NETBLOCK_OWNER\",\n \"NETBLOCKV6_OWNER\",\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"BLACKLISTED_SUBNET\",\n \"BLACKLISTED_NETBLOCK\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_NETBLOCK\",\n \"MALICIOUS_SUBNET\",\n ]\n\n def queryBlacklist(self, target, targetType):\n blacklist = self.retrieveBlacklist()\n\n if not blacklist:\n return False\n\n if targetType == \"ip\":\n if target in blacklist:\n self.debug(f\"IP address {target} found in blocklist.de blacklist.\")\n return True\n elif targetType == \"netblock\":\n netblock = IPNetwork(target)\n for ip in blacklist:\n if IPAddress(ip) in netblock:\n self.debug(f\"IP address {ip} found within netblock/subnet {target} in blocklist.de blacklist.\")\n return True\n\n return False\n\n def retrieveBlacklist(self):\n blacklist = self.sf.cacheGet('blocklistde', 24)\n\n if blacklist is not None:\n return self.parseBlacklist(blacklist)\n\n res = self.sf.fetchUrl(\n \"https://lists.blocklist.de/lists/all.txt\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from blocklist.de.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from blocklist.de\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"blocklistde\", res['content'])\n\n return self.parseBlacklist(res['content'])\n\n def parseBlacklist(self, blacklist):\n \"\"\"Parse plaintext blacklist\n\n Args:\n blacklist (str): plaintext blacklist from blocklist.de\n\n Returns:\n list: list of blacklisted IP addresses\n \"\"\"\n ips = list()\n\n if not blacklist:\n return ips\n\n for ip in blacklist.split('\\n'):\n ip = ip.strip()\n if ip.startswith('#'):\n continue\n if not self.sf.validIP(ip) and not self.sf.validIP6(ip):\n continue\n ips.append(ip)\n\n return ips\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']:\n targetType = 'ip'\n malicious_type = 'MALICIOUS_IPADDR'\n blacklist_type = 'BLACKLISTED_IPADDR'\n elif eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS']:\n if not self.opts.get('checkaffiliates', False):\n return\n targetType = 'ip'\n malicious_type = 'MALICIOUS_AFFILIATE_IPADDR'\n blacklist_type = 'BLACKLISTED_AFFILIATE_IPADDR'\n elif eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']:\n if not self.opts.get('checknetblocks', False):\n return\n targetType = 'netblock'\n malicious_type = 'MALICIOUS_NETBLOCK'\n blacklist_type = 'BLACKLISTED_NETBLOCK'\n elif eventName in ['NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']:\n if not self.opts.get('checksubnets', False):\n return\n targetType = 'netblock'\n malicious_type = 'MALICIOUS_SUBNET'\n blacklist_type = 'BLACKLISTED_SUBNET'\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n self.debug(f\"Checking maliciousness of {eventData} ({eventName}) with blocklist.de\")\n\n if self.queryBlacklist(eventData, targetType):\n # https://www.blocklist.de/en/search.html?ip=\n url = \"https://lists.blocklist.de/lists/all.txt\"\n text = f\"blocklist.de [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_blocklistde class\n" }, { "path": "modules/sfp_botscout.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_botscout\n# Purpose: SpiderFoot plug-in to search botsout.com using their API, for\n# potential malicious IPs and e-mail addresses.\n#\n# Author: Steve Micallef \n#\n# Created: 25/07/2016\n# Copyright: (c) Steve Micallef 2016\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_botscout(SpiderFootPlugin):\n\n meta = {\n 'name': \"BotScout\",\n 'summary': \"Searches BotScout.com's database of spam-bot IP addresses and e-mail addresses.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Passive\", \"Investigate\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://botscout.com/\",\n 'model': \"FREE_NOAUTH_LIMITED\",\n 'references': [\n \"http://botscout.com/api.htm\",\n \"http://botscout.com/api_queries.htm\",\n \"http://botscout.com/getkey.htm\",\n \"http://botscout.com/corp_users.htm\"\n ],\n 'apiKeyInstructions': [\n \"Visit http://botscout.com/getkey.htm\",\n \"Register a free account\",\n \"The API key will be emailed to your account\"\n ],\n 'favIcon': \"https://botscout.com/favicon.ico\",\n 'logo': \"http://botscout.com/image/bslogo.gif\",\n 'description': \"BotScout helps prevent automated web scripts, known as \\\"bots\\\", \"\n \"from registering on forums, polluting databases, spreading spam, \"\n \"and abusing forms on web sites. We do this by tracking the names, IPs, \"\n \"and email addresses that bots use and logging them as unique signatures for future reference. \"\n \"We also provide a simple yet powerful API that you can use to test forms \"\n \"when they're submitted on your site.\",\n }\n }\n\n opts = {\n \"api_key\": \"\"\n }\n optdescs = {\n \"api_key\": \"Botscout.com API key. Without this you will be limited to 100 look-ups per day.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return ['IP_ADDRESS', 'EMAILADDR']\n\n def producedEvents(self):\n return [\"MALICIOUS_IPADDR\", \"BLACKLISTED_IPADDR\", \"MALICIOUS_EMAILADDR\"]\n\n def queryIp(self, ip):\n if not self.sf.validIP(ip):\n return None\n\n params = urllib.parse.urlencode({\n 'ip': ip,\n 'key': self.opts['api_key'],\n })\n\n res = self.sf.fetchUrl(\n f\"https://botscout.com/test/?{params}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n )\n\n return self.parseApiResponse(res)\n\n def queryEmail(self, email):\n if not SpiderFootHelpers.validEmail(email):\n return None\n\n params = urllib.parse.urlencode({\n 'mail': email,\n 'key': self.opts['api_key'],\n })\n\n res = self.sf.fetchUrl(\n f\"https://botscout.com/test/?{params}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n )\n\n return self.parseApiResponse(res)\n\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from BotScout.\")\n return None\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from BotScout.\")\n self.errorState = True\n return None\n\n if not res['content']:\n self.error(\"No response from BotScout.\")\n return None\n\n if res['content'].startswith(\"! \"):\n self.error(f\"Received error from BotScout: {res['content']}\")\n self.errorState = True\n return None\n\n if not res['content'].startswith(\"Y|\") and not res['content'].startswith(\"N|\"):\n self.error(\"Error encountered processing response from BotScout.\")\n return None\n\n return res['content']\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if not self.opts['api_key']:\n self.info(\"You enabled sfp_botscout but did not set an API key! Queries will be limited to 100 per day.\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData} as already searched.\")\n return\n\n self.results[eventData] = True\n\n if eventName == \"IP_ADDRESS\":\n res = self.queryIp(eventData)\n\n if not res:\n return\n\n if not res.startswith(\"Y|\"):\n return\n\n self.info(f\"Found BotScout entry for {eventData}: {res}\")\n\n url = f\"https://botscout.com/ipcheck.htm?ip={eventData}\"\n text = f\"BotScout [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(\"MALICIOUS_IPADDR\", text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(\"BLACKLISTED_IPADDR\", text, self.__name__, event)\n self.notifyListeners(evt)\n elif eventName == \"EMAILADDR\":\n res = self.queryEmail(eventData)\n\n if not res:\n return\n\n if not res.startswith(\"Y|\"):\n return\n\n url = f\"https://botscout.com/search.htm?sterm={eventData}&stype=q\"\n text = f\"BotScout [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(\"MALICIOUS_EMAILADDR\", text, self.__name__, event)\n self.notifyListeners(evt)\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n\n# End of sfp_botscout class\n" }, { "path": "modules/sfp_botvrij.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_botvrij\n# Purpose: Check if a domain is malicious according to botvrij.eu.\n#\n# Author: steve@binarypool.com\n#\n# Created: 16/05/2020\n# Copyright: (c) Steve Micallef, 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_botvrij(SpiderFootPlugin):\n\n meta = {\n 'name': \"botvrij.eu\",\n 'summary': \"Check if a domain is malicious according to botvrij.eu.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://botvrij.eu/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'description': \"Botvrij.eu provides different sets \"\n \" of open source IOCs that you can use in your \"\n \" security devices to detect possible malicious activity.\\n\"\n \"The information contains network info (IPs), file hashes,\"\n \" file paths, domain names, URLs.\",\n }\n }\n\n opts = {\n 'checkaffiliates': True,\n 'checkcohosts': True,\n 'cacheperiod': 18\n }\n\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliates?\",\n 'checkcohosts': \"Apply checks to sites found to be co-hosted on the target's IP?\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"AFFILIATE_INTERNET_NAME\",\n \"CO_HOSTED_SITE\",\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_INTERNET_NAME\",\n \"BLACKLISTED_AFFILIATE_INTERNET_NAME\",\n \"BLACKLISTED_COHOST\",\n \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_COHOST\",\n ]\n\n def queryBlacklist(self, target):\n blacklist = self.retrieveBlacklist()\n\n if not blacklist:\n return False\n\n if target.lower() in blacklist:\n self.debug(f\"Host name {target} found in botvrij.eu blacklist.\")\n return True\n\n return False\n\n def retrieveBlacklist(self):\n blacklist = self.sf.cacheGet('botvrij', 24)\n\n if blacklist is not None:\n return self.parseBlacklist(blacklist)\n\n res = self.sf.fetchUrl(\n \"https://www.botvrij.eu/data/blocklist/blocklist_full.csv\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from botvrij.eu.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from botvrij.eu\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"botvrij\", res['content'])\n\n return self.parseBlacklist(res['content'])\n\n def parseBlacklist(self, blacklist):\n \"\"\"Parse plaintext blacklist\n\n Args:\n blacklist (str): plaintext blacklist from botvrij.eu\n\n Returns:\n list: list of blacklisted host names\n \"\"\"\n hosts = list()\n\n if not blacklist:\n return hosts\n\n for line in blacklist.split('\\n'):\n if not line:\n continue\n if line.startswith('#'):\n continue\n host = line.strip().split(\",\")[0].lower()\n # Note: Validation with sf.validHost() is too slow to use here\n # if not self.sf.validHost(host, self.opts['_internettlds']):\n # continue\n hosts.append(host)\n\n return hosts\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName == \"INTERNET_NAME\":\n malicious_type = \"MALICIOUS_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n if not self.opts.get('checkaffiliates', False):\n return\n malicious_type = \"MALICIOUS_AFFILIATE_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n if not self.opts.get('checkcohosts', False):\n return\n malicious_type = \"MALICIOUS_COHOST\"\n blacklist_type = \"BLACKLISTED_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n self.debug(f\"Checking maliciousness of {eventData} ({eventName}) with botvrij.eu\")\n\n if not self.queryBlacklist(eventData):\n return\n\n url = \"https://www.botvrij.eu/data/blocklist/blocklist_full.csv\"\n text = f\"botvrij.eu Domain Blocklist [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_botvrij class\n" }, { "path": "modules/sfp_builtwith.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_builtwith\n# Purpose: Query builtwith.com using their API.\n#\n# Author: Steve Micallef \n#\n# Created: 10/08/2017\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_builtwith(SpiderFootPlugin):\n\n meta = {\n 'name': \"BuiltWith\",\n 'summary': \"Query BuiltWith.com's Domain API for information about your target's web technology stack, e-mail addresses and more.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://builtwith.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://api.builtwith.com/\",\n \"https://kb.builtwith.com/\",\n \"https://builtwith.com/screencast\",\n \"https://builtwith.com/faq\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://api.builtwith.com/free-api\",\n \"Register a free account\",\n \"Navigate to https://api.builtwith.com/free-api\",\n \"The API key is listed under 'Your API Key'\"\n ],\n 'favIcon': \"https://d28rh9vvmrd65v.cloudfront.net/favicon.ico\",\n 'logo': \"https://d28rh9vvmrd65v.cloudfront.net/favicon.ico\",\n 'description': \"Build lists of websites from our database of 38,701+ web technologies and over a quarter of a billion websites showing \"\n \"which sites use shopping carts, analytics, hosting and many more. \"\n \"Filter by location, traffic, vertical and more.\\n\"\n \"Know your prospects platform before you talk to them. \"\n \"Improve your conversions with validated market adoption.\\n\"\n \"Get advanced technology market share information and country based analytics for all web technologies.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\",\n \"maxage\": 30\n }\n\n # Option descriptions\n optdescs = {\n \"api_key\": \"Builtwith.com Domain API key.\",\n \"maxage\": \"The maximum age of the data returned, in days, in order to be considered valid.\"\n }\n\n # Be sure to completely clear any class variables in setup()\n # or you run the risk of data persisting between scan runs.\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n # Clear / reset any other class member variables here\n # or you risk them persisting between threads.\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"INTERNET_NAME\", \"EMAILADDR\", \"EMAILADDR_GENERIC\", \"RAW_RIR_DATA\",\n \"WEBSERVER_TECHNOLOGY\", \"PHONE_NUMBER\", \"DOMAIN_NAME\",\n \"CO_HOSTED_SITE\", \"IP_ADDRESS\", \"WEB_ANALYTICS_ID\"]\n\n def queryRelationships(self, t):\n url = f\"https://api.builtwith.com/rv1/api.json?LOOKUP={t}&KEY={self.opts['api_key']}\"\n\n res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=\"SpiderFoot\")\n\n if res['code'] == \"404\":\n return None\n\n if not res['content']:\n return None\n\n try:\n return json.loads(res['content'])['Relationships']\n except Exception as e:\n self.error(f\"Error processing JSON response from builtwith.com: {e}\")\n\n return None\n\n def queryDomainInfo(self, t):\n url = f\"https://api.builtwith.com/rv1/api.json?LOOKUP={t}&KEY={self.opts['api_key']}\"\n\n res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=\"SpiderFoot\")\n\n if res['code'] == \"404\":\n return None\n\n if not res['content']:\n return None\n\n try:\n return json.loads(res['content'])['Results'][0]\n except Exception as e:\n self.error(f\"Error processing JSON response from builtwith.com: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_builtwith but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n data = self.queryDomainInfo(eventData)\n if data is not None:\n if \"Meta\" in data:\n if data['Meta'].get(\"Names\", []):\n for nb in data['Meta']['Names']:\n e = SpiderFootEvent(\"RAW_RIR_DATA\", \"Possible full name: \" + nb['Name'],\n self.__name__, event)\n self.notifyListeners(e)\n if nb.get('Email', None):\n if SpiderFootHelpers.validEmail(nb['Email']):\n if nb['Email'].split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evttype = \"EMAILADDR_GENERIC\"\n else:\n evttype = \"EMAILADDR\"\n e = SpiderFootEvent(evttype, nb['Email'],\n self.__name__, event)\n self.notifyListeners(e)\n\n if data['Meta'].get(\"Emails\", []):\n for email in data['Meta']['Emails']:\n if SpiderFootHelpers.validEmail(email):\n if email.split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evttype = \"EMAILADDR_GENERIC\"\n else:\n evttype = \"EMAILADDR\"\n\n e = SpiderFootEvent(evttype, email,\n self.__name__, event)\n self.notifyListeners(e)\n\n if data['Meta'].get(\"Telephones\", []):\n for phone in data['Meta']['Telephones']:\n phone = phone.replace(\"-\", \"\").replace(\"(\", \"\").replace(\")\", \"\").replace(\" \", \"\")\n e = SpiderFootEvent(\"PHONE_NUMBER\", phone, self.__name__, event)\n self.notifyListeners(e)\n\n if \"Paths\" in data.get(\"Result\", []):\n for p in data[\"Result\"]['Paths']:\n if p.get(\"SubDomain\", \"\"):\n h = p[\"SubDomain\"] + \".\" + eventData\n ev = SpiderFootEvent(\"INTERNET_NAME\", h, self.__name__, event)\n self.notifyListeners(ev)\n if self.sf.isDomain(h, self.opts['_internettlds']):\n ev = SpiderFootEvent(\"DOMAIN_NAME\", h, self.__name__, event)\n self.notifyListeners(ev)\n else:\n ev = None\n\n # If we have a subdomain, let's get its tech info\n # and associate it with the subdomain event.\n for t in p.get(\"Technologies\", []):\n if ev:\n src = ev\n else:\n src = event\n agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage'])\n if t.get(\"LastDetected\", 0) < agelimit:\n self.debug(\"Data found too old, skipping.\")\n continue\n e = SpiderFootEvent(\"WEBSERVER_TECHNOLOGY\", t[\"Name\"],\n self.__name__, src)\n self.notifyListeners(e)\n\n data = self.queryRelationships(eventData)\n if data is None:\n return\n\n agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage'])\n\n for r in data:\n if \"Domain\" not in r or \"Identifiers\" not in r:\n self.debug(\"Data returned not in the format requested.\")\n continue\n\n if r['Domain'] != eventData:\n self.debug(\"Data returned doesn't match data requested, skipping.\")\n continue\n\n for i in r['Identifiers']:\n if \"Last\" not in i or \"Type\" not in i or \"Value\" not in i:\n self.debug(\"Data returned not in the format requested.\")\n continue\n\n if i['Last'] < agelimit:\n self.debug(\"Data found too old, skipping.\")\n continue\n\n evttype = None\n # Related through shared IP\n if i['Type'] == \"ip\":\n if self.sf.validIP(i['Value']):\n val = i['Value']\n evttype = \"IP_ADDRESS\"\n else:\n val = i['Value'].strip(\".\")\n if self.getTarget.matches(val):\n evttype = \"INTERNET_NAME\"\n else:\n evttype = \"CO_HOSTED_SITE\"\n\n # Create the name/co-host\n e = SpiderFootEvent(evttype, val, self.__name__, event)\n self.notifyListeners(e)\n continue\n\n # Related through shared analytics ID\n txt = i['Type'] + \": \" + str(i['Value'])\n e = SpiderFootEvent(\"WEB_ANALYTICS_ID\", txt, self.__name__, event)\n self.notifyListeners(e)\n\n if i['Matches']:\n for m in i['Matches']:\n if \"Domain\" not in m:\n continue\n evt = SpiderFootEvent(\"AFFILIATE_INTERNET_NAME\", m['Domain'], self.__name__, e)\n self.notifyListeners(evt)\n\n if self.sf.isDomain(m['Domain'], self.opts['_internettlds']):\n evt = SpiderFootEvent(\"AFFILIATE_DOMAIN_NAME\", m['Domain'], self.__name__, e)\n self.notifyListeners(evt)\n\n# End of sfp_builtwith class\n" }, { "path": "modules/sfp_c99.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_c99\n# Purpose: SpiderFoot plug-in that queries c99 API\n#\n# Author: Filip Aleksić \n#\n# Created: 2020-08-27\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_c99(SpiderFootPlugin):\n meta = {\n \"name\": \"C99\",\n \"summary\": \"Queries the C99 API which offers various data (geo location, proxy detection, phone lookup, etc).\",\n 'flags': [\"apikey\"],\n \"useCases\": [\"Footprint\", \"Passive\", \"Investigate\"],\n \"categories\": [\"Search Engines\"],\n \"dataSource\": {\n \"website\": \"https://api.c99.nl/\",\n \"model\": \"COMMERCIAL_ONLY\",\n \"references\": [\"https://api.c99.nl/api_overview\", \"https://api.c99.nl/faq\"],\n \"apiKeyInstructions\": [\n \"Visit https://api.c99.nl\",\n \"Click shop in the top navigation or go to https://api.c99.nl/dashboard/shop\",\n \"Click purchase key on option 'C99.NL API KEY' (you can also purchase a 1 year key)\",\n \"You will receive your API key by email.\",\n ],\n \"favIcon\": \"https://api.c99.nl/favicon.ico\",\n \"logo\": \"https://api.c99.nl/assets/images/logo.png\",\n \"description\": \"C99 API service is versatile source of information. \"\n \"They offer over 57 different APIs of which 10 are integrated in this module. \"\n \"APIs that are integrated are subdomain finder, phone lookup, Skype resolver, \"\n \"IP to Skype, firewall technology WAF detector, domain history, \"\n \"IP to domains, IP geo location, proxy detector.\",\n },\n }\n\n opts = {\n \"api_key\": \"\",\n \"verify\": True,\n \"cohostsamedomain\": False,\n \"maxcohost\": 100,\n }\n\n optdescs = {\n \"api_key\": \"C99 API Key.\",\n \"verify\": \"Verify identified domains still resolve to the associated specified IP address.\",\n \"maxcohost\": \"Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.\",\n \"cohostsamedomain\": \"Treat co-hosted sites on the same target domain as co-hosting?\",\n }\n\n results = None\n errorState = False\n cohostcount = 0\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.cohostcount = 0\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"DOMAIN_NAME\",\n \"PHONE_NUMBER\",\n \"IP_ADDRESS\",\n \"USERNAME\",\n \"EMAILADDR\",\n ]\n\n def producedEvents(self):\n return [\n \"RAW_RIR_DATA\",\n \"GEOINFO\",\n \"INTERNET_NAME\",\n \"INTERNET_NAME_UNRESOLVED\",\n \"PROVIDER_TELCO\",\n \"PHYSICAL_ADDRESS\",\n \"PHYSICAL_COORDINATES\",\n \"PROVIDER_DNS\",\n \"IP_ADDRESS\",\n \"USERNAME\",\n \"ACCOUNT_EXTERNAL_OWNED\",\n \"WEBSERVER_TECHNOLOGY\",\n \"PROVIDER_HOSTING\",\n \"CO_HOSTED_SITE\"\n ]\n\n def query(self, path, queryParam, queryData):\n res = self.sf.fetchUrl(\n f\"https://api.c99.nl/{path}?key={self.opts['api_key']}&{queryParam}={queryData}&json\",\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=\"SpiderFoot\",\n )\n\n if res[\"code\"] == \"429\":\n self.error(\"Reaching rate limit on C99 API\")\n self.errorState = True\n return None\n\n if res[\"code\"] == 400:\n self.error(\"Invalid request or API key on C99 API\")\n self.errorState = True\n return None\n\n if res[\"content\"] is None:\n self.info(f\"No C99 info found for {queryData}\")\n return None\n\n try:\n info = json.loads(res[\"content\"])\n except Exception as e:\n self.errorState = True\n self.error(f\"Error processing response from C99: {e}\")\n return None\n\n if not info.get('success', False):\n return None\n\n return info\n\n def emitRawRirData(self, data, event):\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n def emitPhoneData(self, phoneData, event):\n provider = phoneData.get(\"provider\")\n carrier = phoneData.get(\"carrier\")\n city = phoneData.get(\"city\")\n countryName = phoneData.get(\"country_name\")\n region = phoneData.get(\"region\")\n found = False\n\n if provider or carrier:\n evt = SpiderFootEvent(\n \"PROVIDER_TELCO\",\n f\"Provider: {provider}, Carrier: {carrier}\",\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n found = True\n\n if city or countryName or region:\n evt = SpiderFootEvent(\n \"PHYSICAL_ADDRESS\",\n f\"Country: {countryName}, Region: {region}, City: {city}\",\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n found = True\n\n if found:\n self.emitRawRirData(phoneData, event)\n\n def emitSubDomainData(self, subDomainData, event):\n found = False\n\n for subDomainElem in subDomainData:\n if self.checkForStop():\n return\n\n subDomain = subDomainElem.get(\"subdomain\", \"\").strip()\n\n if subDomain:\n self.emitHostname(subDomain, event)\n found = True\n\n if found:\n self.emitRawRirData(subDomainData, event)\n\n def emitDomainHistoryData(self, domainHistoryData, event):\n found = False\n\n for domainHistoryElem in domainHistoryData:\n if self.checkForStop():\n return\n\n ip = domainHistoryElem.get(\"ip_address\")\n\n if self.sf.validIP(ip):\n evt = SpiderFootEvent(\n \"IP_ADDRESS\",\n ip,\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n found = True\n\n if found:\n self.emitRawRirData(domainHistoryData, event)\n\n def emitIpToSkypeData(self, data, event):\n skype = data.get(\"skype\")\n\n if skype:\n evt = SpiderFootEvent(\n \"ACCOUNT_EXTERNAL_OWNED\",\n f\"Skype [{skype}]\",\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(\n \"USERNAME\",\n skype,\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n\n self.emitRawRirData(data, event)\n\n def emitIpToDomainsData(self, data, event):\n domains = data.get(\"domains\")\n found = False\n\n if isinstance(domains, list):\n for domain in domains:\n if self.checkForStop():\n return\n\n domain = domain.strip()\n if domain:\n self.emitHostname(domain, event)\n found = True\n\n if found:\n self.emitRawRirData(data, event)\n\n def emitProxyDetectionData(self, data, event):\n isProxy = data.get(\"proxy\")\n\n if isProxy:\n evt = SpiderFootEvent(\n \"WEBSERVER_TECHNOLOGY\",\n f\"Server is proxy: {isProxy}\",\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n self.emitRawRirData(data, event)\n\n def emitGeoIPData(self, data, event):\n found = False\n\n hostName = data.get(\"hostname\", \"\").strip()\n if hostName:\n self.emitHostname(hostName, event)\n found = True\n\n record = data.get(\"records\")\n\n if record:\n country = record.get(\"country_name\")\n region = record[\"region\"].get(\"name\") if record.get(\"region\") else None\n city = record.get(\"city\")\n postalCode = record.get(\"postal_code\")\n latitude = record.get(\"latitude\")\n longitude = record.get(\"longitude\")\n provider = record.get(\"isp\")\n\n if provider:\n evt = SpiderFootEvent(\n \"PROVIDER_HOSTING\",\n provider,\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n found = True\n\n if latitude and longitude:\n evt = SpiderFootEvent(\n \"PHYSICAL_COORDINATES\",\n f\"{latitude}, {longitude}\",\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n found = True\n\n if region or country or city or postalCode:\n evt = SpiderFootEvent(\n \"GEOINFO\",\n f\"Country: {country}, Region: {region}, City: {city}, Postal code: {postalCode}\",\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n found = True\n\n if found:\n self.emitRawRirData(data, event)\n\n def emitSkypeResolverData(self, data, event):\n ip = data.get(\"ip\")\n ips = data.get(\"ips\")\n found = False\n\n if ip and ip not in ips:\n evt = SpiderFootEvent(\n \"IP_ADDRESS\",\n ip,\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n found = True\n\n if isinstance(ips, list):\n found = True\n for ipElem in ips:\n if self.checkForStop():\n return\n\n evt = SpiderFootEvent(\n \"IP_ADDRESS\",\n ipElem.strip(),\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n\n if found:\n self.emitRawRirData(data, event)\n\n def emitWafDetectorData(self, data, event):\n firewall = data.get(\"result\")\n\n if firewall:\n evt = SpiderFootEvent(\n \"WEBSERVER_TECHNOLOGY\",\n f\"Firewall detected: {firewall}\",\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n self.emitRawRirData(data, event)\n\n def emitHostname(self, data, event):\n if not self.sf.validHost(data, self.opts['_internettlds']):\n return\n\n if self.opts[\"verify\"] and not self.sf.resolveHost(data) and not self.sf.resolveHost6(data):\n self.debug(f\"Host {data} could not be resolved.\")\n if self.getTarget().matches(data):\n evt = SpiderFootEvent(\"INTERNET_NAME_UNRESOLVED\", data, self.__name__, event)\n self.notifyListeners(evt)\n return\n\n if self.getTarget().matches(data):\n evt = SpiderFootEvent('INTERNET_NAME', data, self.__name__, event)\n self.notifyListeners(evt)\n if self.sf.isDomain(data, self.opts['_internettlds']):\n evt = SpiderFootEvent('DOMAIN_NAME', data, self.__name__, event)\n self.notifyListeners(evt)\n return\n\n if self.cohostcount < self.opts['maxcohost']:\n if self.opts[\"verify\"] and not self.sf.validateIP(data, event.data):\n self.debug(\"Host no longer resolves to our IP.\")\n return\n\n if not self.opts[\"cohostsamedomain\"]:\n if self.getTarget().matches(data, includeParents=True):\n self.debug(\n f\"Skipping {data} because it is on the same domain.\"\n )\n return\n\n if self.cohostcount < self.opts[\"maxcohost\"]:\n evt = SpiderFootEvent(\"CO_HOSTED_SITE\", data, self.__name__, event)\n self.notifyListeners(evt)\n self.cohostcount += 1\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n # Once we are in this state, return immediately.\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\"You enabled sfp_c99, but did not set an API key!\")\n self.errorState = True\n return\n\n # Don't look up stuff twice\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName == \"PHONE_NUMBER\":\n phoneData = self.query(\"phonelookup\", \"number\", eventData)\n phoneData = phoneData.get(\"details\") if phoneData else None\n\n if phoneData:\n self.emitPhoneData(phoneData, event)\n\n if eventName == \"DOMAIN_NAME\":\n subDomainData = self.query(\"subdomainfinder\", \"domain\", eventData)\n subDomainData = (\n subDomainData.get(\"subdomains\") if subDomainData is not None else None\n )\n\n if isinstance(subDomainData, list):\n self.emitSubDomainData(subDomainData, event)\n\n domainHistoryData = self.query(\"domainhistory\", \"domain\", eventData)\n domainHistoryData = (\n domainHistoryData.get(\"result\") if domainHistoryData else None\n )\n\n if isinstance(domainHistoryData, list):\n self.emitDomainHistoryData(domainHistoryData, event)\n\n wafDetectorData = self.query(\"firewalldetector\", \"url\", eventData)\n\n if wafDetectorData:\n self.emitWafDetectorData(wafDetectorData, event)\n\n if eventName == \"IP_ADDRESS\":\n ipToSkypeData = self.query(\"ip2skype\", \"ip\", eventData)\n\n if ipToSkypeData:\n self.emitIpToSkypeData(ipToSkypeData, event)\n\n ipToDomainsData = self.query(\"ip2domains\", \"ip\", eventData)\n\n if ipToDomainsData:\n self.emitIpToDomainsData(ipToDomainsData, event)\n\n proxyDetectionData = self.query(\"proxydetector\", \"ip\", eventData)\n\n if proxyDetectionData:\n self.emitProxyDetectionData(proxyDetectionData, event)\n\n geoIPData = self.query(\"geoip\", \"host\", eventData)\n\n if geoIPData:\n self.emitGeoIPData(geoIPData, event)\n\n if eventName == \"USERNAME\":\n skypeResolverData = self.query(\"skyperesolver\", \"username\", eventData)\n\n if skypeResolverData:\n self.emitSkypeResolverData(skypeResolverData, event)\n\n\n# End of sfp_c99 class\n" }, { "path": "modules/sfp_callername.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_callername\n# Purpose: SpiderFoot plug-in to search CallerName.com for a phone number\n# (US only) and retrieve location and reputation information.\n#\n# Author: \n#\n# Created: 2019-05-28\n# Copyright: (c) bcoles 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_callername(SpiderFootPlugin):\n\n meta = {\n 'name': \"CallerName\",\n 'summary': \"Lookup US phone number location and reputation information.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Real World\"],\n 'dataSource': {\n 'website': \"http://callername.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://callername.com/faq\",\n \"https://callername.com/stats\"\n ],\n 'favIcon': \"http://static.callername.com/favicon.ico\",\n 'logo': \"http://static.callername.com/img/logo.min.png\",\n 'description': \"CallerName is a free, reverse phone lookup service for both cell and landline numbers. \"\n \"It relies on a database of white pages and business pages taken from public sources. \"\n \"The easy-to-use and streamlined interface allow users to look up the caller ID information of any number quickly. \"\n \"Just type the unknown number into the search bar to start. \"\n \"You need not pay nor register to use this 100% free service.\",\n }\n }\n\n # Default options\n opts = {\n }\n\n # Option descriptions\n optdescs = {\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['PHONE_NUMBER']\n\n # What events this module produces\n def producedEvents(self):\n return ['GEOINFO', 'MALICIOUS_PHONE_NUMBER']\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n # Only US numbers are supported (+1)\n if not eventData.startswith('+1'):\n self.debug('Unsupported phone number: ' + eventData)\n return\n\n # Strip country code (+1) and formatting\n number = eventData.lstrip('+1').strip('(').strip(')').strip('-').strip(' ')\n\n if not number.isdigit():\n self.debug('Invalid phone number: ' + number)\n return\n\n # Query CallerName.com for the specified phone number\n url = f\"https://callername.com/{number}\"\n res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])\n\n time.sleep(1)\n\n if res['content'] is None:\n self.debug('No response from CallerName.com')\n return\n\n if res['code'] != '200':\n self.debug('No phone information found for ' + eventData)\n return\n\n location_match = re.findall(r'

.*?

(.+?)

', str(res['content']), re.MULTILINE | re.DOTALL)\n\n if location_match:\n location = location_match[0]\n\n if len(location) < 5 or len(location) > 100:\n self.debug(\"Skipping likely invalid location.\")\n else:\n evt = SpiderFootEvent('GEOINFO', location, self.__name__, event)\n self.notifyListeners(evt)\n\n rep_good_match = re.findall(r'>SAFE.*?>(\\d+) votes?<', str(res['content']))\n rep_bad_match = re.findall(r'>UNSAFE.*?>(\\d+) votes?<', str(res['content']))\n\n if rep_good_match and rep_bad_match:\n good_votes = int(rep_good_match[0])\n bad_votes = int(rep_bad_match[0])\n\n if bad_votes > good_votes:\n text = f\"CallerName [{eventData}]\\n{url}\"\n evt = SpiderFootEvent('MALICIOUS_PHONE_NUMBER', text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_callername class\n" }, { "path": "modules/sfp_censys.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_censys\n# Purpose: Query Censys.io API\n#\n# Author: Steve Micallef\n#\n# Created: 01/02/2017\n# Copyright: (c) Steve Micallef 2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport base64\nimport json\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\nfrom datetime import datetime\n\nfrom netaddr import IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_censys(SpiderFootPlugin):\n\n meta = {\n 'name': \"Censys\",\n 'summary': \"Obtain host information from Censys.io.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://censys.io/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://search.censys.io/api\",\n \"https://search.censys.io/search/language\",\n \"https://github.com/censys/censys-postman/blob/main/Censys_Search.postman_collection.json\",\n ],\n 'apiKeyInstructions': [\n \"Visit https://censys.io/\",\n \"Register a free account\",\n \"Navigate to https://censys.io/account\",\n \"Click on 'API'\",\n \"The API key combination is listed under 'API ID' and 'Secret'\"\n ],\n 'favIcon': \"https://censys.io/assets/favicon.png\",\n 'logo': \"https://censys.io/assets/logo.png\",\n 'description': \"Discover exposures and other common entry points for attackers.\\n\"\n \"Censys scans the entire internet constantly, including obscure ports. \"\n \"We use a combination of banner grabs and deep protocol handshakes \"\n \"to provide industry-leading visibility and an accurate depiction of what is live on the internet.\",\n }\n }\n\n opts = {\n \"censys_api_key_uid\": \"\",\n \"censys_api_key_secret\": \"\",\n 'delay': 3,\n 'netblocklookup': True,\n 'maxnetblock': 24,\n 'maxv6netblock': 120,\n \"age_limit_days\": 90,\n }\n\n optdescs = {\n \"censys_api_key_uid\": \"Censys.io API UID.\",\n \"censys_api_key_secret\": \"Censys.io API Secret.\",\n 'delay': 'Delay between requests, in seconds.',\n 'netblocklookup': \"Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?\",\n 'maxnetblock': \"If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'maxv6netblock': \"If looking up owned netblocks, the maximum IPv6 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n \"age_limit_days\": \"Ignore any records older than this many days. 0 = unlimited.\",\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\",\n \"NETBLOCK_OWNER\",\n \"NETBLOCKV6_OWNER\",\n ]\n\n def producedEvents(self):\n return [\n \"BGP_AS_MEMBER\",\n \"UDP_PORT_OPEN\",\n \"TCP_PORT_OPEN\",\n \"TCP_PORT_OPEN_BANNER\",\n \"OPERATING_SYSTEM\",\n \"SOFTWARE_USED\",\n \"WEBSERVER_HTTPHEADERS\",\n \"NETBLOCK_MEMBER\",\n \"NETBLOCKV6_MEMBER\",\n \"GEOINFO\",\n \"RAW_RIR_DATA\"\n ]\n\n def queryHosts(self, qry):\n secret = self.opts['censys_api_key_uid'] + ':' + self.opts['censys_api_key_secret']\n auth = base64.b64encode(secret.encode('utf-8')).decode('utf-8')\n\n headers = {\n 'Authorization': f\"Basic {auth}\"\n }\n\n res = self.sf.fetchUrl(\n f\"https://search.censys.io/api/v2/hosts/{qry}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\",\n headers=headers\n )\n\n # API rate limit: 0.4 actions/second (120.0 per 5 minute interval)\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n def queryHostsSearch(self, qry):\n secret = self.opts['censys_api_key_uid'] + ':' + self.opts['censys_api_key_secret']\n auth = base64.b64encode(secret.encode('utf-8')).decode('utf-8')\n\n headers = {\n 'Authorization': f\"Basic {auth}\"\n }\n\n params = urllib.parse.urlencode({\n 'q': qry,\n })\n\n res = self.sf.fetchUrl(\n f\"https://search.censys.io/api/v2/hosts/search/?{params}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\",\n headers=headers\n )\n\n # API rate limit: 0.4 actions/second (120.0 per 5 minute interval)\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from Censys.io.\")\n return None\n\n if res['code'] == \"400\":\n self.error(\"Invalid request.\")\n return None\n\n if res['code'] == \"404\":\n self.info('Censys.io returned no results')\n return None\n\n if res['code'] == \"403\":\n self.error(\"Invalid API key.\")\n self.errorState = True\n return None\n\n if res['code'] == \"429\":\n self.error(\"Request rate limit exceeded.\")\n self.errorState = True\n return None\n\n # Catch all non-200 status codes, and presume something went wrong\n if res['code'] != '200':\n self.error(f\"Unexpected HTTP response code {res['code']} from Censys API.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.info('Censys.io returned no results')\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from Censys.io: {e}\")\n return None\n\n error_type = data.get('error_type')\n if error_type:\n self.error(f\"Censys returned an unexpected error: {error_type}\")\n return None\n\n return data\n\n def handleEvent(self, event):\n if self.errorState:\n return\n\n eventName = event.eventType\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if self.opts['censys_api_key_uid'] == \"\" or self.opts['censys_api_key_secret'] == \"\":\n self.error(f\"You enabled {self.__class__.__name__} but did not set an API uid/secret!\")\n self.errorState = True\n return\n\n eventData = event.data\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']:\n if not self.opts['netblocklookup']:\n return\n\n if eventName == 'NETBLOCKV6_OWNER':\n max_netblock = self.opts['maxv6netblock']\n else:\n max_netblock = self.opts['maxnetblock']\n\n if IPNetwork(eventData).prefixlen < max_netblock:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}\")\n return\n\n qrylist = list()\n if eventName.startswith(\"NETBLOCK\"):\n for ipaddr in IPNetwork(eventData):\n qrylist.append(str(ipaddr))\n self.results[str(ipaddr)] = True\n else:\n qrylist.append(eventData)\n\n for addr in qrylist:\n if self.checkForStop():\n return\n\n data = self.queryHosts(addr)\n\n if not data:\n continue\n\n rec = data.get(\"result\")\n\n if not rec:\n self.info(f\"Censys.io returned no results for {addr}\")\n continue\n\n self.debug(f\"Found results for {addr} in Censys.io\")\n\n # For netblocks, we need to create the associated IP address event first.\n if eventName == 'NETBLOCK_OWNER':\n pevent = SpiderFootEvent(\"IP_ADDRESS\", addr, self.__name__, event)\n self.notifyListeners(pevent)\n elif eventName == 'NETBLOCKV6_OWNER':\n pevent = SpiderFootEvent(\"IPV6_ADDRESS\", addr, self.__name__, event)\n self.notifyListeners(pevent)\n else:\n pevent = event\n\n e = SpiderFootEvent(\"RAW_RIR_DATA\", json.dumps(rec), self.__name__, pevent)\n self.notifyListeners(e)\n\n try:\n # Date format: 2021-09-22T16:46:47.623Z\n created_dt = datetime.strptime(rec.get('last_updated_at', \"1970-01-01T00:00:00.000Z\"), '%Y-%m-%dT%H:%M:%S.%fZ')\n created_ts = int(time.mktime(created_dt.timetuple()))\n age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])\n\n if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts:\n self.debug(f\"Record found but too old ({created_dt}), skipping.\")\n continue\n except Exception as e:\n self.error(f\"Error encountered processing last_updated_at record for {addr}: {e}\")\n\n try:\n location = rec.get('location')\n if location:\n geoinfo = ', '.join(\n [\n _f for _f in [\n location.get('city'),\n location.get('province'),\n location.get('postal_code'),\n location.get('country'),\n location.get('continent'),\n ] if _f\n ]\n )\n if geoinfo:\n e = SpiderFootEvent(\"GEOINFO\", geoinfo, self.__name__, pevent)\n self.notifyListeners(e)\n except Exception as e:\n self.error(f\"Error encountered processing location record for {addr}: {e}\")\n\n try:\n services = rec.get('services')\n if services:\n softwares = list()\n tcp_banners = list()\n for service in services:\n port = service.get('port')\n\n if port:\n transport_protocol = service.get('transport_protocol')\n banner = service.get('banner')\n\n if transport_protocol == \"UDP\":\n evt = SpiderFootEvent(\"UDP_PORT_OPEN\", f\"{addr}:{port}\", self.__name__, pevent)\n self.notifyListeners(evt)\n elif transport_protocol == \"TCP\":\n evt = SpiderFootEvent(\"TCP_PORT_OPEN\", f\"{addr}:{port}\", self.__name__, pevent)\n self.notifyListeners(evt)\n if banner:\n tcp_banners.append(banner)\n\n software = service.get('software', list())\n if software:\n for sw in software:\n s = ' '.join(\n filter(\n None,\n [\n sw.get('vendor'),\n sw.get('product'),\n sw.get('version')\n ]\n )\n )\n if s:\n softwares.append(s)\n\n http = service.get('http')\n if http:\n response = http.get('response')\n if response:\n headers = response.get('headers')\n if headers:\n e = SpiderFootEvent(\n \"WEBSERVER_HTTPHEADERS\",\n json.dumps(headers, ensure_ascii=False),\n self.__name__,\n pevent\n )\n e.actualSource = addr\n self.notifyListeners(e)\n\n for software in set(softwares):\n evt = SpiderFootEvent(\"SOFTWARE_USED\", software, self.__name__, pevent)\n self.notifyListeners(evt)\n\n for banner in set(tcp_banners):\n evt = SpiderFootEvent(\"TCP_PORT_OPEN_BANNER\", str(banner), self.__name__, pevent)\n self.notifyListeners(evt)\n except Exception as e:\n self.error(f\"Error encountered processing services record for {addr}: {e}\")\n\n try:\n autonomous_system = rec.get('autonomous_system')\n if autonomous_system:\n asn = autonomous_system.get('asn')\n if asn:\n e = SpiderFootEvent(\"BGP_AS_MEMBER\", str(asn), self.__name__, pevent)\n self.notifyListeners(e)\n\n bgp_prefix = autonomous_system.get('bgp_prefix')\n if bgp_prefix and self.sf.validIpNetwork(bgp_prefix):\n if ':' in bgp_prefix:\n e = SpiderFootEvent(\"NETBLOCKV6_MEMBER\", str(bgp_prefix), self.__name__, pevent)\n else:\n e = SpiderFootEvent(\"NETBLOCK_MEMBER\", str(bgp_prefix), self.__name__, pevent)\n self.notifyListeners(e)\n except Exception as e:\n self.error(f\"Error encountered processing autonomous_system record for {addr}: {e}\")\n\n try:\n operating_system = rec.get('operating_system')\n if operating_system:\n os = ' '.join(\n filter(\n None,\n [\n operating_system.get('vendor'),\n operating_system.get('product'),\n operating_system.get('version'),\n operating_system.get('edition')\n ]\n )\n )\n\n if os:\n e = SpiderFootEvent(\"OPERATING_SYSTEM\", os, self.__name__, pevent)\n self.notifyListeners(e)\n except Exception as e:\n self.error(f\"Error encountered processing operating_system record for {addr}: {e}\")\n\n# End of sfp_censys class\n" }, { "path": "modules/sfp_certspotter.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_certspotter\n# Purpose: Gather information about SSL certificates from SSLMate CertSpotter API.\n#\n# Author: \n#\n# Created: 2021-08-15\n# Copyright: (c) bcoles\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport base64\nimport json\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_certspotter(SpiderFootPlugin):\n meta = {\n 'name': \"CertSpotter\",\n 'summary': \"Gather information about SSL certificates from SSLMate CertSpotter API.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Crawling and Scanning\"],\n 'dataSource': {\n 'website': \"https://sslmate.com/certspotter/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://sslmate.com/help/reference/ct_search_api_v1\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://sslmate.com/signup?for=ct_search_api\",\n \"Register a new account with an email\",\n \"Navigate to https://sslmate.com/account/\",\n \"The API key is listed under 'API Credentials'\",\n ],\n \"favIcon\": \"https://sslmate.com/assets/@faafe50b54dfb91476c01374043f217c.png\",\n \"logo\": \"https://sslmate.com/assets/@995de4b3fc64525a0c960b570432bcaf.png\",\n \"description\": \"Cert Spotter monitors your domains for expiring, unauthorized, \"\n \"and invalid SSL certificates, so you can act before an incident, not after.\"\n }\n }\n\n # Default options\n opts = {\n 'api_key': '',\n 'verify': True,\n 'max_pages': 20,\n 'certexpiringdays': 30\n }\n\n # Option descriptions\n optdescs = {\n 'api_key': 'CertSpotter API key.',\n 'verify': \"Verify certificate subject alternative names resolve.\",\n 'max_pages': \"Maximum number of pages of results to fetch.\",\n 'certexpiringdays': 'Number of days in the future a certificate expires to consider it as expiring.'\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in userOpts.keys():\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['DOMAIN_NAME']\n\n # What events this module produces\n def producedEvents(self):\n return [\n 'INTERNET_NAME',\n 'INTERNET_NAME_UNRESOLVED',\n 'DOMAIN_NAME',\n 'CO_HOSTED_SITE',\n 'CO_HOSTED_SITE_DOMAIN',\n 'SSL_CERTIFICATE_ISSUED',\n 'SSL_CERTIFICATE_ISSUER',\n 'SSL_CERTIFICATE_MISMATCH',\n 'SSL_CERTIFICATE_EXPIRED',\n 'SSL_CERTIFICATE_EXPIRING',\n 'SSL_CERTIFICATE_RAW',\n 'RAW_RIR_DATA'\n ]\n\n # Query CertSpotter issuances API endpoint\n def queryIssuances(self, domain, after=None):\n params = {\n 'domain': domain.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n 'include_subdomains': 'true',\n 'match_wildcards': 'true',\n 'after': (after or '')\n }\n\n expand = '&expand='.join(['dns_names', 'issuer', 'cert'])\n\n headers = {\n 'Accept': 'application/json',\n 'Authorization': \"Basic \" + base64.b64encode(f\"{self.opts['api_key']}:\".encode('utf-8')).decode('utf-8')\n }\n\n res = self.sf.fetchUrl(\n f\"https://api.certspotter.com/v1/issuances?{urllib.parse.urlencode(params)}&expand={expand}\",\n headers=headers,\n timeout=15,\n useragent=self.opts['_useragent'],\n )\n\n # Free plan - 1,000 single-hostname queries / hour; 100 full-domain queries / hour\n time.sleep(1)\n\n if res['content'] is None:\n self.debug('No response from CertSpotter API')\n return None\n\n if res['code'] == '429':\n self.error(\"You are being rate-limited by CertSpotter\")\n self.errorState = True\n return None\n\n if res['code'] != '200':\n self.error(f\"Unexpected HTTP response code {res['code']} from CertSpotter\")\n self.errorState = True\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n return\n\n if self.errorState:\n return\n\n if self.opts['api_key'] == \"\":\n self.error(f\"You enabled {self.__class__.__name__} but did not set an API key!\")\n self.errorState = True\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n max_pages = int(self.opts['max_pages'])\n page = 1\n last_id = None\n hosts = list()\n while page <= max_pages:\n if self.checkForStop():\n break\n\n if self.errorState:\n break\n\n data = self.queryIssuances(eventData, last_id)\n\n if data is None or len(data) == 0:\n break\n\n page += 1\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n for result in data:\n cert_hosts = result.get('dns_names')\n\n if cert_hosts:\n for d in cert_hosts:\n if d != eventData:\n hosts.append(d.replace(\"*.\", \"\"))\n\n if result.get('cert') is None:\n self.debug('Response data contains no certificate data')\n continue\n\n try:\n rawcert = \"-----BEGIN CERTIFICATE-----\\n\"\n rawcert += result.get('cert').get('data')\n rawcert += \"\\n-----END CERTIFICATE-----\\n\"\n cert = self.sf.parseCert(rawcert, eventData, self.opts['certexpiringdays'])\n except Exception as e:\n self.info(f\"Error parsing certificate: {e}\")\n continue\n\n if not cert.get('text'):\n self.info(\"Failed to parse the SSL certificate\")\n continue\n\n evt = SpiderFootEvent('SSL_CERTIFICATE_RAW', cert['text'], self.__name__, event)\n self.notifyListeners(evt)\n\n if cert.get('issuer'):\n evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUER', cert['issuer'], self.__name__, event)\n self.notifyListeners(evt)\n\n if cert.get('issued'):\n evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUED', cert['issued'], self.__name__, event)\n self.notifyListeners(evt)\n\n for san in set(cert.get('altnames', list())):\n hosts.append(san.replace(\"*.\", \"\"))\n\n if cert.get('expired'):\n evt = SpiderFootEvent(\"SSL_CERTIFICATE_EXPIRED\", cert.get('expirystr', 'Unknown'), self.__name__, event)\n self.notifyListeners(evt)\n continue\n\n if cert.get('expiring'):\n evt = SpiderFootEvent(\"SSL_CERTIFICATE_EXPIRING\", cert.get('expirystr', 'Unknown'), self.__name__, event)\n self.notifyListeners(evt)\n continue\n\n # \"To retrieve additional issuances, take the id field of the last issuance and pass it to the issuances endpoint in the after parameter\"\n last_id = data[-1].get('id')\n\n if last_id is None:\n break\n\n if not hosts:\n return\n\n if self.opts['verify']:\n self.info(f\"Resolving {len(set(hosts))} hostnames ...\")\n\n for domain in set(hosts):\n if self.checkForStop():\n return\n\n if domain in self.results:\n continue\n\n if self.getTarget().matches(domain, includeChildren=True, includeParents=True):\n evt_type = 'INTERNET_NAME'\n if self.opts['verify'] and not self.sf.resolveHost(domain) and not self.sf.resolveHost6(domain):\n self.debug(f\"Host {domain} could not be resolved\")\n evt_type += '_UNRESOLVED'\n else:\n evt_type = 'CO_HOSTED_SITE'\n\n evt = SpiderFootEvent(evt_type, domain, self.__name__, event)\n self.notifyListeners(evt)\n\n if self.sf.isDomain(domain, self.opts['_internettlds']):\n if evt_type == 'CO_HOSTED_SITE':\n evt = SpiderFootEvent('CO_HOSTED_SITE_DOMAIN', domain, self.__name__, event)\n self.notifyListeners(evt)\n else:\n evt = SpiderFootEvent('DOMAIN_NAME', domain, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_certspotter class\n" }, { "path": "modules/sfp_cinsscore.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_cinsscore\n# Purpose: Checks if an IP address is malicious according to the CINS Army list.\n#\n# Author: steve@binarypool.com\n#\n# Created: 13/05/2018\n# Copyright: (c) Steve Micallef, 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom netaddr import IPAddress, IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_cinsscore(SpiderFootPlugin):\n\n meta = {\n 'name': \"CINS Army List\",\n 'summary': \"Check if a netblock or IP address is malicious according to Collective Intelligence Network Security (CINS) Army list.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://cinsscore.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'favIcon': 'https://cinsscore.com/media/images/fav-icon.png',\n 'logo': 'https://cinsscore.com/media/images/logo-small-grey-inset.png',\n 'description': \"Leveraging data from our network of Sentinel \"\n \"devices and other trusted InfoSec sources, CINS is a \"\n \"Threat Intelligence database that provides an accurate \"\n \"and timely score for any IP address in the world.\"\n \"The CINS Army list is a subset of the CINS Active Threat Intelligence ruleset.\",\n }\n }\n\n opts = {\n 'checkaffiliates': True,\n 'cacheperiod': 18,\n 'checknetblocks': True,\n 'checksubnets': True\n }\n\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliate IP addresses?\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\",\n 'checknetblocks': \"Report if any malicious IPs are found within owned netblocks?\",\n 'checksubnets': \"Check if any malicious IPs are found within the same subnet of the target?\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"NETBLOCK_MEMBER\",\n \"NETBLOCK_OWNER\",\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"BLACKLISTED_SUBNET\",\n \"BLACKLISTED_NETBLOCK\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_SUBNET\",\n \"MALICIOUS_NETBLOCK\",\n ]\n\n def query(self, qry, targetType):\n cid = \"_cinsscore\"\n url = \"https://cinsscore.com/list/ci-badguys.txt\"\n\n data = dict()\n data[\"content\"] = self.sf.cacheGet(\"sfmal_\" + cid, self.opts.get('cacheperiod', 0))\n\n if data[\"content\"] is None:\n data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])\n\n if data[\"code\"] != \"200\":\n self.error(f\"Unable to fetch {url}\")\n self.errorState = True\n return None\n\n if data[\"content\"] is None:\n self.error(f\"Unable to fetch {url}\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"sfmal_\" + cid, data['content'])\n\n for line in data[\"content\"].split('\\n'):\n ip = line.strip().lower()\n\n if targetType == \"netblock\":\n try:\n if IPAddress(ip) in IPNetwork(qry):\n self.debug(f\"{ip} found within netblock/subnet {qry} in cinsscore.com list.\")\n return url\n except Exception as e:\n self.debug(f\"Error encountered parsing: {e}\")\n continue\n\n if targetType == \"ip\":\n if qry.lower() == ip:\n self.debug(f\"{qry} found in cinsscore.com list.\")\n return url\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName == 'IP_ADDRESS':\n targetType = 'ip'\n malicious_type = \"MALICIOUS_IPADDR\"\n blacklist_type = \"BLACKLISTED_IPADDR\"\n elif eventName == 'AFFILIATE_IPADDR':\n if not self.opts.get('checkaffiliates', False):\n return\n targetType = 'ip'\n malicious_type = \"MALICIOUS_AFFILIATE_IPADDR\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_IPADDR\"\n elif eventName == 'NETBLOCK_OWNER':\n if not self.opts.get('checknetblocks', False):\n return\n targetType = 'netblock'\n malicious_type = \"MALICIOUS_NETBLOCK\"\n blacklist_type = \"BLACKLISTED_NETBLOCK\"\n elif eventName == 'NETBLOCK_MEMBER':\n if not self.opts.get('checksubnets', False):\n return\n targetType = 'netblock'\n malicious_type = \"MALICIOUS_SUBNET\"\n blacklist_type = \"BLACKLISTED_SUBNET\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n self.debug(f\"Checking maliciousness of {eventData} with cinsscore.com\")\n\n url = self.query(eventData, targetType)\n\n if not url:\n return\n\n text = f\"cinsscore.com [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_cinsscore class\n" }, { "path": "modules/sfp_circllu.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_circllu\n# Purpose: Query circl.lu using their API\n#\n# Author: Steve Micallef\n#\n# Created: 16/02/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport base64\nimport json\nimport re\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_circllu(SpiderFootPlugin):\n\n meta = {\n 'name': \"CIRCL.LU\",\n 'summary': \"Obtain information from CIRCL.LU's Passive DNS and Passive SSL databases.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://www.circl.lu/\",\n 'model': \"FREE_AUTH_UNLIMITED\",\n 'references': [\n \"https://www.circl.lu/services/passive-dns/\",\n \"https://www.circl.lu/services/passive-ssl/\",\n \"https://www.circl.lu/services/\",\n \"https://www.circl.lu/pub/\",\n \"https://www.circl.lu/projects\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://www.circl.lu/contact/\",\n \"Contact with email or phone to request access for Passive DNS and Passive SSL API services\",\n \"The API access will be provided once approved\"\n ],\n 'favIcon': \"https://www.google.com/s2/favicons?domain=https://www.circl.lu/\",\n 'logo': \"https://www.circl.lu/assets/images/circl-logo.png\",\n 'description': \"The Computer Incident Response Center Luxembourg (CIRCL) is a government-driven initiative \"\n \"designed to gather, review, report and respond to computer security threats and incidents.\\n\"\n \"CIRCL provides a reliable and trusted point of contact for any users, companies and organizations \"\n \"based in Luxembourg, for the handling of attacks and incidents. \"\n \"Its team of experts acts like a fire brigade, with the ability to react promptly and \"\n \"efficiently whenever threats are suspected, detected or incidents occur.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key_login\": \"\",\n \"api_key_password\": \"\",\n \"age_limit_days\": 0,\n \"verify\": True,\n \"cohostsamedomain\": False,\n \"maxcohost\": 100\n }\n\n # Option descriptions\n optdescs = {\n \"api_key_login\": \"CIRCL.LU login.\",\n \"api_key_password\": \"CIRCL.LU password.\",\n \"age_limit_days\": \"Ignore any Passive DNS records older than this many days. 0 = unlimited.\",\n \"verify\": \"Verify co-hosts are valid by checking if they still resolve to the shared IP.\",\n \"cohostsamedomain\": \"Treat co-hosted sites on the same target domain as co-hosting?\",\n 'maxcohost': \"Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.\"\n }\n\n # Be sure to completely clear any class variables in setup()\n # or you run the risk of data persisting between scan runs.\n\n results = None\n errorState = False\n cohostcount = 0\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.cohostcount = 0\n\n # Clear / reset any other class member variables here\n # or you risk them persisting between threads.\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"INTERNET_NAME\", \"NETBLOCK_OWNER\", \"IP_ADDRESS\", \"DOMAIN_NAME\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"IP_ADDRESS\", \"SSL_CERTIFICATE_ISSUED\", \"CO_HOSTED_SITE\"]\n\n def query(self, qry, qtype):\n if self.errorState:\n return None\n\n if qtype == \"PDNS\":\n url = \"https://www.circl.lu/pdns/query/\" + qry\n else:\n url = \"https://www.circl.lu/v2pssl/query/\" + qry\n\n secret = self.opts['api_key_login'] + ':' + self.opts['api_key_password']\n b64_val = base64.b64encode(secret.encode('utf-8'))\n headers = {\n 'Authorization': f\"Basic {b64_val.decode('utf-8')}\"\n }\n\n # Be more forgiving with the timeout as some queries for subnets can be slow\n res = self.sf.fetchUrl(url, timeout=30,\n useragent=\"SpiderFoot\", headers=headers)\n\n if res['code'] not in [\"200\", \"201\"]:\n self.error(\"CIRCL.LU access seems to have been rejected or you have exceeded usage limits.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.info(\"No CIRCL.LU info found for \" + qry)\n return None\n\n return res['content']\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n ret = None\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n # Ignore messages from myself\n if srcModuleName == \"sfp_circllu\":\n self.debug(\"Ignoring \" + eventName + \", from self.\")\n return\n\n if self.opts['api_key_login'] == \"\" or self.opts['api_key_password'] == \"\":\n self.error(\"You enabled sfp_circllu but did not set an credentials!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName in ['IP_ADDRESS', 'NETBLOCK_OWNER']:\n # CIRCL.LU limit the maximum subnet size to 23\n # http://circl.lu/services/passive-ssl/\n if \"/\" in eventData:\n addr, mask = eventData.split(\"/\")\n if int(mask) < 23:\n self.debug(\"Network size bigger than permitted by CIRCL.LU.\")\n else:\n ret = self.query(eventData, \"PSSL\")\n if not ret:\n self.info(\"No CIRCL.LU passive SSL data found for \" + eventData)\n else:\n ret = self.query(eventData, \"PSSL\")\n if not ret:\n self.info(\"No CIRCL.LU passive SSL data found for \" + eventData)\n\n if ret:\n try:\n # Generate an event for the IP first, and then link the cert\n # to that event.\n j = json.loads(ret)\n for ip in j:\n ipe = event\n if ip != eventData:\n ipe = SpiderFootEvent(\"IP_ADDRESS\", ip, self.__name__, event)\n self.notifyListeners(ipe)\n for crt in j[ip]['subjects']:\n r = re.findall(r\".*[\\\"\\'](.+CN=([a-zA-Z0-9\\-\\*\\.])+)[\\\"\\'].*\",\n str(j[ip]['subjects'][crt]), re.IGNORECASE)\n if r:\n e = SpiderFootEvent(\"SSL_CERTIFICATE_ISSUED\", r[0][0], self.__name__, ipe)\n self.notifyListeners(e)\n except Exception as e:\n self.error(\"Invalid response returned from CIRCL.LU: \" + str(e))\n\n if eventName in ['IP_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME']:\n ret = self.query(eventData, \"PDNS\")\n if not ret:\n self.info(\"No CIRCL.LU passive DNS data found for \" + eventData)\n return\n\n # CIRCL.LU doesn't return valid JSON - it's one JSON record per line\n for line in ret.split(\"\\n\"):\n if len(line) < 2:\n continue\n try:\n rec = json.loads(line)\n except Exception as e:\n self.error(\"Invalid response returned from CIRCL.LU: \" + str(e))\n continue\n\n age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])\n if self.opts['age_limit_days'] > 0 and rec['time_last'] < age_limit_ts:\n self.debug(\"Record found but too old, skipping.\")\n continue\n\n cohosts = list()\n if eventName == \"IP_ADDRESS\":\n # Record could be pointing to our IP, or from our IP\n if rec['rrtype'] == \"A\" and rec['rdata'] == eventData:\n if not self.getTarget().matches(rec['rrname']):\n # We found a co-host\n cohosts.append(rec['rrname'])\n\n if eventName in [\"INTERNET_NAME\", \"DOMAIN_NAME\"]:\n # Record could be an A/CNAME of this entity, or something pointing to it\n if rec['rdata'] == eventData:\n if not self.getTarget().matches(rec['rrname']):\n # We found a co-host\n cohosts.append(rec['rrname'])\n\n for co in cohosts:\n if eventName == \"IP_ADDRESS\" and (self.opts['verify'] and not self.sf.validateIP(co, eventData)):\n self.debug(\"Host no longer resolves to our IP.\")\n continue\n\n if not self.opts['cohostsamedomain']:\n if self.getTarget().matches(co, includeParents=True):\n self.debug(\"Skipping \" + co + \" because it is on the same domain.\")\n continue\n\n if self.cohostcount < self.opts['maxcohost']:\n e = SpiderFootEvent(\"CO_HOSTED_SITE\", co, self.__name__, event)\n self.notifyListeners(e)\n self.cohostcount += 1\n\n# End of sfp_circllu class\n" }, { "path": "modules/sfp_citadel.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_citadel\n# Purpose: SpiderFoot plug-in to search Leak-Lookup using their API,\n# for potential data breaches.\n#\n# Author: sn \n#\n# Created: 15/08/2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_citadel(SpiderFootPlugin):\n\n meta = {\n 'name': \"Leak-Lookup\",\n 'summary': \"Searches Leak-Lookup.com's database of breaches.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Leaks, Dumps and Breaches\"],\n 'dataSource': {\n 'website': \"https://leak-lookup.com/\",\n 'model': \"FREE_AUTH_UNLIMITED\",\n 'references': [\n \"https://leak-lookup.com/api\",\n \"https://leak-lookup.com/databases\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://leak-lookup.com\",\n \"Register an account\",\n \"Login to your account\",\n \"Click on 'Account'\",\n \"Click on 'API'\",\n \"The API key is listed under 'API Key'\"\n ],\n 'favIcon': \"https://leak-lookup.com/favicon.png\",\n 'logo': \"https://leak-lookup.com/favicon.png\",\n 'description': \"Leak-Lookup allows you to search across thousands of data breaches \"\n \"to stay on top of credentials that may have been compromised in the wild.\\n\"\n \"The creators came together when they realized they had a vast trove of data \"\n \"that could be of great value to pen-testers seeking weaknesses in client passwords \"\n \"and those concerned about which of their credentials have been leaked into the wild.\\n\"\n \"Always looking forward, Leak-Lookup invests all of its profits back into securing the \"\n \"latest data breaches and leaks / dumps as they become available, ensuring that \"\n \"as well as historical data, Leak-Lookup becomes a field leader in credential monitoring.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\",\n \"timeout\": 60\n }\n optdescs = {\n \"api_key\": \"Leak-Lookup API key. Without this you're limited to the public API.\",\n \"timeout\": \"Custom timeout due to heavy traffic at times.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n self.__dataSource__ = \"Leak-Lookup.com\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['EMAILADDR']\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"EMAILADDR_COMPROMISED\"]\n\n # Query email address\n # https://leak-lookup.com/api\n def queryEmail(self, email):\n apikey = self.opts['api_key']\n\n if not apikey:\n # Public API key\n apikey = \"3edfb5603418f101926c64ca5dd0e409\"\n\n params = {\n 'query': email.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n 'type': 'email_address',\n 'key': apikey\n }\n\n res = self.sf.fetchUrl(\"https://leak-lookup.com/api/search\",\n postData=urllib.parse.urlencode(params),\n timeout=self.opts['timeout'],\n useragent=self.opts['_useragent'])\n\n if res['code'] == \"429\":\n time.sleep(10)\n return self.queryEmail(email)\n\n if res['content'] is None:\n self.debug('No response from Leak-Lookup.com')\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.errorState:\n return\n\n # Don't look up stuff twice\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n data = self.queryEmail(eventData)\n\n if data is None:\n return\n\n error = data.get('error')\n message = data.get('message')\n\n if error == 'true':\n self.error(f\"Error encountered processing {eventData}: {message}\")\n if \"MISSING API\" in message:\n self.errorState = True\n return\n return\n\n if not message:\n return\n\n for site in message:\n self.info(f\"Found Leak-Lookup entry for {eventData}: {site}\")\n evt = SpiderFootEvent(\"EMAILADDR_COMPROMISED\", f\"{eventData} [{site}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_citadel class\n" }, { "path": "modules/sfp_cleanbrowsing.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_cleanbrowsing\n# Purpose: SpiderFoot plug-in for looking up whether hosts are blocked by\n# CleanBrowsing.org DNS content family filters\n# (185.228.168.168 and 185.228.168.169).\n#\n# Author: Steve Micallef \n#\n# Created: 30/05/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport dns.resolver\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_cleanbrowsing(SpiderFootPlugin):\n\n meta = {\n 'name': \"CleanBrowsing.org\",\n 'summary': \"Check if a host would be blocked by CleanBrowsing.org DNS content filters.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://cleanbrowsing.org/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://cleanbrowsing.org/guides/\",\n \"https://cleanbrowsing.org/filters/\",\n \"https://cleanbrowsing.org/how-it-works\",\n \"https://cleanbrowsing.org/web-filtering-for-shools-and-cipa-compliance\",\n \"https://cleanbrowsing.org/getting-started\"\n ],\n 'favIcon': \"https://cleanbrowsing.org/favicon-new.ico\",\n 'logo': \"https://cleanbrowsing.org/images/logos/CleanBrowsing-logo-large-2019-Orange-II.png\",\n 'description': \"You get to decide what type of content is allowed in your home or network via our \"\n \"DNS-based content filtering service. Parents can protect their kids from adult content, \"\n \"schools can be CIPA compliant and businesses can block malicious domains and \"\n \"gain visibility into their network.\\n\"\n \"CleanBrowsing is a DNS-based content filtering service that offers a safe way to browse the web without surprises. \"\n \"It intercepts domain requests and filter sites that should be blocked, based on your requirements. \"\n \"Our free family filter, for example, blocks adult content, while still allowing Google, \"\n \"Youtube, Bing, DuckDuckGo and the rest of the web to load safely.\",\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"AFFILIATE_INTERNET_NAME\",\n \"CO_HOSTED_SITE\"\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_INTERNET_NAME\",\n \"BLACKLISTED_AFFILIATE_INTERNET_NAME\",\n \"BLACKLISTED_COHOST\",\n \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_COHOST\",\n ]\n\n def queryFamilyDNS(self, qaddr):\n res = dns.resolver.Resolver()\n res.nameservers = [\"185.228.168.168\", \"185.228.168.169\"]\n\n try:\n addrs = res.resolve(qaddr)\n self.debug(f\"Addresses returned: {addrs}\")\n except Exception:\n self.debug(f\"Unable to resolve {qaddr}\")\n return False\n\n if addrs:\n return True\n return False\n\n def queryAdultDNS(self, qaddr):\n res = dns.resolver.Resolver()\n res.nameservers = [\"185.228.168.10\", \"185.228.169.11\"]\n\n try:\n addrs = res.resolve(qaddr)\n self.debug(f\"Addresses returned: {addrs}\")\n except Exception:\n self.debug(f\"Unable to resolve {qaddr}\")\n return False\n\n if addrs:\n return True\n return False\n\n def querySecurityDNS(self, qaddr):\n res = dns.resolver.Resolver()\n res.nameservers = [\"185.228.168.9\", \"185.228.169.9\"]\n\n try:\n addrs = res.resolve(qaddr)\n self.debug(f\"Addresses returned: {addrs}\")\n except Exception:\n self.debug(f\"Unable to resolve {qaddr}\")\n return False\n\n if addrs:\n return True\n return False\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if eventName == \"INTERNET_NAME\":\n malicious_type = \"MALICIOUS_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n malicious_type = \"MALICIOUS_AFFILIATE_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n malicious_type = \"MALICIOUS_COHOST\"\n blacklist_type = \"BLACKLISTED_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n # Check that it resolves first, as it becomes a valid\n # malicious host only if NOT resolved by CleanBrowsing DNS.\n if not self.sf.resolveHost(eventData) and not self.sf.resolveHost6(eventData):\n return\n\n family = self.queryFamilyDNS(eventData)\n adult = self.queryAdultDNS(eventData)\n security = self.querySecurityDNS(eventData)\n\n # Host was found, not blocked\n if family and adult and security:\n return\n\n self.debug(f\"{eventData} was blocked by CleanBrowsing DNS\")\n\n if not security:\n evt = SpiderFootEvent(\n blacklist_type,\n f\"CleanBrowsing DNS - Security [{eventData}]\",\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(\n malicious_type,\n f\"CleanBrowsing DNS - Security [{eventData}]\",\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n elif not adult:\n evt = SpiderFootEvent(\n blacklist_type,\n f\"CleanBrowsing DNS - Adult [{eventData}]\",\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n elif not family:\n evt = SpiderFootEvent(\n blacklist_type,\n f\"CleanBrowsing DNS - Family [{eventData}]\",\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n\n# End of sfp_cleanbrowsing class\n" }, { "path": "modules/sfp_cleantalk.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_cleantalk\n# Purpose: Checks if a netblock or IP address is on CleanTalk.org's spam IP list.\n#\n# Author: steve@binarypool.com\n#\n# Created: 05/08/2018\n# Copyright: (c) Steve Micallef, 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom netaddr import IPAddress, IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_cleantalk(SpiderFootPlugin):\n\n meta = {\n 'name': \"CleanTalk Spam List\",\n 'summary': \"Check if a netblock or IP address is on CleanTalk.org's spam IP list.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://cleantalk.org\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://cleantalk.org/help\",\n \"https://cleantalk.org/help/introduction\",\n \"https://cleantalk.org/help/api-spam-check\",\n \"https://cleantalk.org/wordpress-security-malware-firewall\",\n \"https://cleantalk.org/price-anti-spam\",\n \"https://cleantalk.org/ssl-certificates/cheap-positivessl-certificate\",\n \"https://cleantalk.org/email-checker\",\n \"https://cleantalk.org/blacklists\"\n ],\n 'favIcon': \"https://cleantalk.org/favicons/favicon-16x16.png\",\n 'logo': \"https://cleantalk.org/favicons/favicon-16x16.png\",\n 'description': \"CleanTalk is a Cloud-Based spam filtering service that allows you to protect your website from spam. \"\n \"CleanTalk provides spam protection that invisible to visitors \"\n \"without using captcha or other methods when visitors have to prove that they are real people.\\n\"\n \"CleanTalk provides cloud anti-spam solutions for CMS and we developed plugins for the most of popular \"\n \"CMS: WordPress anti-spam plugin, Joomla anti-spam plugin, Drupal and etc. \"\n \"With our simple cloud spam checker, you can be sure your website is protected from spam bots, spam comments, and users.\",\n }\n }\n\n opts = {\n 'checkaffiliates': True,\n 'cacheperiod': 18,\n 'checknetblocks': True,\n 'checksubnets': True\n }\n\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliate IP addresses?\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\",\n 'checknetblocks': \"Report if any malicious IPs are found within owned netblocks?\",\n 'checksubnets': \"Check if any malicious IPs are found within the same subnet of the target?\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n 'IP_ADDRESS',\n 'AFFILIATE_IPADDR',\n 'NETBLOCK_OWNER',\n 'NETBLOCK_MEMBER'\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"BLACKLISTED_SUBNET\",\n \"BLACKLISTED_NETBLOCK\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_NETBLOCK\",\n \"MALICIOUS_SUBNET\",\n ]\n\n def query(self, qry, targetType):\n cid = \"_cleantalk\"\n url = \"https://iplists.firehol.org/files/cleantalk_7d.ipset\"\n\n data = dict()\n data[\"content\"] = self.sf.cacheGet(\"sfmal_\" + cid, self.opts.get('cacheperiod', 0))\n\n if data[\"content\"] is None:\n data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])\n\n if data[\"code\"] != \"200\":\n self.error(f\"Unable to fetch {url}\")\n self.errorState = True\n return None\n\n if data[\"content\"] is None:\n self.error(f\"Unable to fetch {url}\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"sfmal_\" + cid, data['content'])\n\n for line in data[\"content\"].split('\\n'):\n ip = line.strip().lower()\n\n if ip.startswith('#'):\n continue\n\n if targetType == \"netblock\":\n try:\n if IPAddress(ip) in IPNetwork(qry):\n self.debug(f\"{ip} found within netblock/subnet {qry} in CleanTalk Spam List.\")\n return url\n except Exception as e:\n self.debug(f\"Error encountered parsing: {e}\")\n continue\n\n if targetType == \"ip\":\n if qry.lower() == ip:\n self.debug(f\"{qry} found in CleanTalk Spam List.\")\n return url\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName == 'IP_ADDRESS':\n targetType = 'ip'\n malicious_type = \"MALICIOUS_IPADDR\"\n blacklist_type = \"BLACKLISTED_IPADDR\"\n elif eventName == 'AFFILIATE_IPADDR':\n if not self.opts.get('checkaffiliates', False):\n return\n targetType = 'ip'\n malicious_type = \"MALICIOUS_AFFILIATE_IPADDR\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_IPADDR\"\n elif eventName == 'NETBLOCK_OWNER':\n if not self.opts.get('checknetblocks', False):\n return\n targetType = 'netblock'\n malicious_type = \"MALICIOUS_NETBLOCK\"\n blacklist_type = \"BLACKLISTED_NETBLOCK\"\n elif eventName == 'NETBLOCK_MEMBER':\n if not self.opts.get('checksubnets', False):\n return\n targetType = 'netblock'\n malicious_type = \"MALICIOUS_SUBNET\"\n blacklist_type = \"BLACKLISTED_SUBNET\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n self.debug(f\"Checking maliciousness of {eventData} with CleanTalk Spam List\")\n\n url = self.query(eventData, targetType)\n\n if not url:\n return\n\n self.debug(f\"{eventData} found in Cleantalk Spam List\")\n\n text = f\"CleanTalk Spam List [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_cleantalk class\n" }, { "path": "modules/sfp_clearbit.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_clearbit\n# Purpose: Query clearbit.com using their API.\n#\n# Author: Steve Micallef \n#\n# Created: 20/03/2017\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport base64\nimport urllib.error\nimport urllib.parse\nimport urllib.request\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_clearbit(SpiderFootPlugin):\n\n meta = {\n 'name': \"Clearbit\",\n 'summary': \"Check for names, addresses, domains and more based on lookups of e-mail addresses on clearbit.com.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://clearbit.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://clearbit.com/docs\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://clearbit.com\",\n \"Register account for a Free Trial\",\n \"Navigate to https://dashboard.clearbit.com/api\",\n \"The API key is listed under 'Your API Key'\"\n ],\n 'favIcon': \"https://clearbit.com/assets/site/logo.png\",\n 'logo': \"https://clearbit.com/assets/site/logo.png\",\n 'description': \"Clearbit is the marketing data engine for all of your customer interactions. \"\n \"Deeply understand your customers, identify future prospects, \"\n \"and personalize every single marketing and sales interaction.\\n\"\n \"Rely on fresh, accurate data with our proprietary real-time lookups. \"\n \"Then act on new information immediately, with sales alerting and job change notifications.\\n\"\n \"Get company attributes like employee count, technologies used, and industry classification—and \"\n \"get employee details like role, seniority, and even job change notifications, right at your fingertips.\\n\"\n \"With our dataset and machine learning algorithms, you’ll have all of \"\n \"the information you need to convert leads and grow your business.\",\n }\n }\n\n opts = {\n \"api_key\": \"\"\n }\n\n optdescs = {\n \"api_key\": \"Clearbit.com API key.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"EMAILADDR\"]\n\n def producedEvents(self):\n return [\n \"RAW_RIR_DATA\",\n \"PHONE_NUMBER\",\n \"PHYSICAL_ADDRESS\",\n \"AFFILIATE_INTERNET_NAME\",\n \"EMAILADDR\",\n \"EMAILADDR_GENERIC\",\n \"INTERNET_NAME\"\n ]\n\n def query(self, email: str):\n api_key = self.opts['api_key']\n\n if isinstance(api_key, str):\n api_key = api_key.encode('utf-8')\n\n token = base64.b64encode(api_key + ':'.encode('utf-8'))\n headers = {\n 'Accept': 'application/json',\n 'Authorization': \"Basic \" + token.decode('utf-8')\n }\n params = {\n 'email': email\n }\n\n res = self.sf.fetchUrl(\n f\"https://person.clearbit.com/v2/combined/find?{urllib.parse.urlencode(params)}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\",\n headers=headers\n )\n\n return self.parseApiResponse(res)\n\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from Clearbit.\")\n return None\n\n if res['code'] == '404':\n self.debug(\"No results from Clearbit.\")\n return None\n\n if res['code'] == \"401\":\n self.error(\"Invalid Clearbit API key.\")\n self.errorState = True\n return None\n\n if res['code'] == \"402\":\n self.error(\"You have exceeded your Clearbit API request quota.\")\n self.errorState = True\n return None\n\n # Rate limit is 600 requests per minute\n # https://dashboard.clearbit.com/docs#rate-limiting\n if res['code'] == '429':\n self.error(\"You are being rate-limited by Clearbit.\")\n return None\n\n if res['code'] == '500' or res['code'] == '502' or res['code'] == '503':\n self.error(\"Clearbit service is unavailable.\")\n self.errorState = True\n return None\n\n # Catch all non-200 status codes, and presume something went wrong\n if res['code'] != '200':\n self.error(f\"Unexpected reply from Clearbit: {res['code']}\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from Clearbit: {e}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n if self.errorState:\n return\n\n if self.opts['api_key'] == \"\":\n self.error(f\"You enabled {self.__class__.__name__} but did not set an API key!\")\n self.errorState = True\n return\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n data = self.query(eventData)\n if not data:\n return\n\n try:\n # Get the name associated with the e-mail\n person = data.get('person')\n if person:\n name = person.get('name')\n if name:\n fullName = name.get('fullName')\n if fullName:\n evt = SpiderFootEvent(\n \"RAW_RIR_DATA\",\n f\"Possible full name: {fullName}\",\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n except Exception:\n self.debug(\"Unable to extract person name from JSON.\")\n pass\n\n # Get the location of the person, also indicating\n # the location of the employer.\n try:\n geo = data.get('geo')\n if geo:\n location = ', '.join(\n filter(\n None,\n [\n geo.get('streetNumber'),\n geo.get('streetName'),\n geo.get('city'),\n geo.get('postalCode'),\n geo.get('state'),\n geo.get('country')\n ]\n )\n )\n\n if location:\n evt = SpiderFootEvent(\"PHYSICAL_ADDRESS\", location, self.__name__, event)\n self.notifyListeners(evt)\n except Exception:\n self.debug(\"Unable to extract location from JSON.\")\n pass\n\n try:\n company = data.get('company')\n if company:\n domainAliases = company.get('domainAliases')\n if domainAliases:\n for d in domainAliases:\n if self.getTarget().matches(d):\n t = \"INTERNET_NAME\"\n else:\n t = \"AFFILIATE_INTERNET_NAME\"\n evt = SpiderFootEvent(\n t,\n d,\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n\n site = company.get('site')\n if site:\n if 'phoneNumbers' in site:\n for p in site['phoneNumbers']:\n evt = SpiderFootEvent(\"PHONE_NUMBER\", p, self.__name__, event)\n self.notifyListeners(evt)\n\n if 'emailAddresses' in company['site']:\n for e in site['emailAddresses']:\n if e.split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evttype = \"EMAILADDR_GENERIC\"\n else:\n evttype = \"EMAILADDR\"\n evt = SpiderFootEvent(evttype, e, self.__name__, event)\n self.notifyListeners(evt)\n\n # Get the location of the person, also indicating\n # the location of the employer.\n company_geo = company.get('geo')\n if company_geo:\n location = ', '.join(\n filter(\n None,\n [\n company_geo.get('streetNumber'),\n company_geo.get('streetName'),\n company_geo.get('city'),\n company_geo.get('postalCode'),\n company_geo.get('state'),\n company_geo.get('country')\n ]\n )\n )\n\n if location:\n evt = SpiderFootEvent(\"PHYSICAL_ADDRESS\", location, self.__name__, event)\n self.notifyListeners(evt)\n except Exception:\n self.debug(\"Unable to extract company info from JSON.\")\n pass\n\n# End of sfp_clearbit class\n" }, { "path": "modules/sfp_cloudflaredns.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_cloudflaredns\n# Purpose: SpiderFoot plug-in for looking up whether hosts are blocked by\n# CloudFlare family and malware filtering DNS servers.\n#\n# Author: Steve Micallef \n#\n# Created: 11/05/2020\n# Copyright: (c) Steve Micallef 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport dns.resolver\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_cloudflaredns(SpiderFootPlugin):\n\n meta = {\n 'name': \"CloudFlare DNS\",\n 'summary': \"Check if a host would be blocked by CloudFlare DNS.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://www.cloudflare.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://developers.cloudflare.com/1.1.1.1/1.1.1.1-for-families/\"\n ],\n 'favIcon': \"https://www.cloudflare.com/img/favicon/favicon-32x32.png\",\n 'logo': \"https://www.cloudflare.com/img/logo-web-badges/cf-logo-on-white-bg.svg\",\n 'description': \"1.1.1.1 for Families is the easiest way to add a layer of protection to \"\n \"your home network and protect it from malware and adult content. \"\n \"1.1.1.1 for Families leverages Cloudflare’s global network to ensure \"\n \"that it is fast and secure around the world.\",\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"AFFILIATE_INTERNET_NAME\",\n \"CO_HOSTED_SITE\"\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_INTERNET_NAME\",\n \"BLACKLISTED_AFFILIATE_INTERNET_NAME\",\n \"BLACKLISTED_COHOST\",\n \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_COHOST\",\n ]\n\n def queryFamilyDNS(self, qaddr):\n res = dns.resolver.Resolver()\n res.nameservers = [\"1.1.1.3\", \"1.0.0.3\"]\n\n try:\n return res.resolve(qaddr)\n except Exception:\n self.debug(f\"Unable to resolve {qaddr}\")\n\n return None\n\n def queryMalwareDNS(self, qaddr):\n res = dns.resolver.Resolver()\n res.nameservers = [\"1.1.1.2\", \"1.0.0.2\"]\n\n try:\n return res.resolve(qaddr)\n except Exception:\n self.debug(f\"Unable to resolve {qaddr}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if eventName == \"INTERNET_NAME\":\n e = \"BLACKLISTED_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n e = \"BLACKLISTED_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n e = \"BLACKLISTED_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n family = self.sf.normalizeDNS(self.queryFamilyDNS(eventData))\n malware = self.sf.normalizeDNS(self.queryMalwareDNS(eventData))\n\n if not family or not malware:\n return\n\n if '0.0.0.0' not in family and '0.0.0.0' not in malware:\n return\n\n # Host is blocked only by family filters\n if '0.0.0.0' not in malware:\n self.debug(f\"{eventData} blocked by CloudFlare Family DNS\")\n evt = SpiderFootEvent(e, f\"CloudFlare - Family [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n return\n\n # Host is blocked only by malware filters\n self.debug(f\"{eventData} blocked by CloudFlare Malware DNS\")\n evt = SpiderFootEvent(e, f\"CloudFlare - Malware [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n if eventName == \"INTERNET_NAME\":\n e = \"MALICIOUS_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n e = \"MALICIOUS_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n e = \"MALICIOUS_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n\n evt = SpiderFootEvent(e, f\"CloudFlare - Malware [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_cloudflaredns class\n" }, { "path": "modules/sfp_coinblocker.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_coinblocker\n# Purpose: Checks if a hostname is listed on CoinBlockerLists.\n#\n# Author: steve@binarypool.com\n#\n# Created: 07/09/2018\n# Copyright: (c) Steve Micallef, 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_coinblocker(SpiderFootPlugin):\n\n meta = {\n 'name': \"CoinBlocker Lists\",\n 'summary': \"Check if a domain appears on CoinBlocker lists.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://zerodot1.gitlab.io/CoinBlockerListsWeb/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://zerodot1.gitlab.io/CoinBlockerListsWeb/downloads.html\",\n \"https://zerodot1.gitlab.io/CoinBlockerListsWeb/references.html\",\n \"https://zerodot1.gitlab.io/CoinBlockerListsWeb/aboutthisproject.html\"\n ],\n 'favIcon': \"https://zerodot1.gitlab.io/CoinBlockerListsWeb/assets/img/favicon.png\",\n 'logo': \"https://zerodot1.gitlab.io/CoinBlockerListsWeb/assets/img/favicon.png\",\n 'description': \"The CoinBlockerLists are a project to prevent illegal mining in \"\n \"browsers or other applications using IPlists and URLLists.\\n\"\n \"It's not just to block everything without any reason, but to protect \"\n \"Internet users from illegal mining.\",\n }\n }\n\n opts = {\n 'checkaffiliates': True,\n 'checkcohosts': True,\n 'cacheperiod': 18,\n }\n\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliates?\",\n 'checkcohosts': \"Apply checks to sites found to be co-hosted on the target's IP?\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\",\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.errorState = False\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"AFFILIATE_INTERNET_NAME\",\n \"CO_HOSTED_SITE\",\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_INTERNET_NAME\",\n \"BLACKLISTED_AFFILIATE_INTERNET_NAME\",\n \"BLACKLISTED_COHOST\",\n \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_COHOST\",\n ]\n\n def queryBlocklist(self, target):\n blocklist = self.retrieveBlocklist()\n\n if not blocklist:\n return False\n\n if target.lower() in blocklist:\n self.debug(f\"Host name {target} found in CoinBlocker list.\")\n return True\n\n return False\n\n def retrieveBlocklist(self):\n blocklist = self.sf.cacheGet('coinblocker', self.opts.get('cacheperiod', 24))\n\n if blocklist is not None:\n return self.parseBlocklist(blocklist)\n\n url = \"https://zerodot1.gitlab.io/CoinBlockerLists/list.txt\"\n res = self.sf.fetchUrl(\n url,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from {url}\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(f\"Received no content from {url}\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"coinblocker\", res['content'])\n\n return self.parseBlocklist(res['content'])\n\n def parseBlocklist(self, blocklist):\n \"\"\"Parse plaintext CoinBlocker list\n\n Args:\n blocklist (str): plaintext CoinBlocker list\n\n Returns:\n list: list of blocked host names\n \"\"\"\n hosts = list()\n\n if not blocklist:\n return hosts\n\n for line in blocklist.split('\\n'):\n if not line:\n continue\n if line.startswith('#'):\n continue\n host = line.strip()\n # Note: Validation with sf.validHost() is too slow to use here\n # if not self.sf.validHost(host, self.opts['_internettlds']):\n # continue\n if not host:\n continue\n hosts.append(host.lower())\n\n return hosts\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName == \"INTERNET_NAME\":\n malicious_type = \"MALICIOUS_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n if not self.opts.get('checkaffiliates', False):\n return\n malicious_type = \"MALICIOUS_AFFILIATE_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n if not self.opts.get('checkcohosts', False):\n return\n malicious_type = \"MALICIOUS_COHOST\"\n blacklist_type = \"BACKLISTED_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n self.debug(f\"Checking maliciousness of {eventData} ({eventName}) with CoinBlocker list\")\n\n if not self.queryBlocklist(eventData):\n return\n\n url = \"https://zerodot1.gitlab.io/CoinBlockerLists/list.txt\"\n text = f\"CoinBlocker [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_coinblocker class\n" }, { "path": "modules/sfp_commoncrawl.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_commoncrawl\n# Purpose: Searches the commoncrawl.org project's indexes for URLs related\n# to the target.\n#\n# Author: Steve Micallef \n#\n# Created: 05/09/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport re\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_commoncrawl(SpiderFootPlugin):\n\n meta = {\n 'name': \"CommonCrawl\",\n 'summary': \"Searches for URLs found through CommonCrawl.org.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"http://commoncrawl.org/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://commoncrawl.org/the-data/get-started/\",\n \"https://commoncrawl.org/the-data/examples/\",\n \"https://commoncrawl.org/the-data/tutorials/\"\n ],\n 'favIcon': \"https://commoncrawl.org/wp-content/themes/commoncrawl/img/favicon.png\",\n 'logo': \"https://commoncrawl.org/wp-content/themes/commoncrawl/img/favicon.png\",\n 'description': \"We build and maintain an open repository of web crawl data that can be accessed and analyzed by anyone.\\n\"\n \"Everyone should have the opportunity to indulge their curiosities, analyze the world and pursue brilliant ideas. \"\n \"Small startups or even individuals can now access high quality crawl data that was previously \"\n \"only available to large search engine corporations.\",\n }\n }\n\n # Default options\n opts = {\n \"indexes\": 6\n }\n\n # Option descriptions\n optdescs = {\n \"indexes\": \"Number of most recent indexes to attempt, because results tend to be occasionally patchy.\"\n }\n\n results = None\n indexBase = list()\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.indexBase = list()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def search(self, target):\n ret = list()\n for index in self.indexBase:\n url = f\"https://index.commoncrawl.org/{index}-index?url={target}/*&output=json\"\n res = self.sf.fetchUrl(url, timeout=60,\n useragent=\"SpiderFoot\")\n\n if res['code'] in [\"400\", \"401\", \"402\", \"403\", \"404\"]:\n self.error(\"CommonCrawl search doesn't seem to be available.\")\n self.errorState = True\n return None\n\n if not res['content']:\n self.error(\"CommonCrawl search doesn't seem to be available.\")\n self.errorState = True\n return None\n\n ret.append(res['content'])\n\n return ret\n\n def getLatestIndexes(self):\n url = \"https://index.commoncrawl.org/\"\n res = self.sf.fetchUrl(url, timeout=60,\n useragent=\"SpiderFoot\")\n\n if res['code'] in [\"400\", \"401\", \"402\", \"403\", \"404\"]:\n self.error(\"CommonCrawl index collection doesn't seem to be available.\")\n self.errorState = True\n return list()\n\n if not res['content']:\n self.error(\"CommonCrawl index collection doesn't seem to be available.\")\n self.errorState = True\n return list()\n\n indexes = re.findall(r\".*(CC-MAIN-\\d+-\\d+).*\", str(res['content']))\n indexlist = dict()\n for m in indexes:\n ms = m.replace(\"CC-MAIN-\", \"\").replace(\"-\", \"\")\n indexlist[ms] = True\n\n topindexes = sorted(list(indexlist.keys()), reverse=True)[0:self.opts['indexes']]\n\n if len(topindexes) < self.opts['indexes']:\n self.error(\"Not able to find latest CommonCrawl indexes.\")\n self.errorState = True\n return list()\n\n retindex = list()\n for i in topindexes:\n retindex.append(\"CC-MAIN-\" + str(i)[0:4] + \"-\" + str(i)[4:6])\n self.debug(\"CommonCrawl indexes: \" + str(retindex))\n return retindex\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"INTERNET_NAME\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"LINKED_URL_INTERNAL\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.errorState:\n return\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if len(self.indexBase) == 0:\n self.indexBase = self.getLatestIndexes()\n\n if not self.indexBase:\n self.error(\"Unable to fetch CommonCrawl index.\")\n return\n\n if len(self.indexBase) == 0:\n self.error(\"Unable to fetch CommonCrawl index.\")\n return\n\n data = self.search(eventData)\n if not data:\n self.error(\"Unable to obtain content from CommonCrawl.\")\n return\n\n sent = list()\n for content in data:\n try:\n for line in content.split(\"\\n\"):\n if self.checkForStop():\n return\n\n if len(line) < 2:\n continue\n link = json.loads(line)\n if 'url' not in link:\n continue\n\n # CommonCrawl sometimes returns hosts with a trailing . after the domain\n link['url'] = link['url'].replace(eventData + \".\", eventData)\n\n if link['url'] in sent:\n continue\n sent.append(link['url'])\n\n evt = SpiderFootEvent(\"LINKED_URL_INTERNAL\", link['url'],\n self.__name__, event)\n self.notifyListeners(evt)\n except Exception as e:\n self.error(\"Malformed JSON from CommonCrawl.org: \" + str(e))\n return\n\n# End of sfp_commoncrawl class\n" }, { "path": "modules/sfp_comodo.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_comodo\n# Purpose: SpiderFoot plug-in for looking up whether hosts are blocked by\n# Comodo Secure DNS.\n#\n# Author: Steve Micallef \n#\n# Created: 30/05/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport dns.resolver\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_comodo(SpiderFootPlugin):\n\n meta = {\n 'name': \"Comodo Secure DNS\",\n 'summary': \"Check if a host would be blocked by Comodo Secure DNS.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://www.comodo.com/secure-dns/\",\n 'model': \"FREE_NOAUTH_LIMITED\",\n 'references': [\n \"https://cdome.comodo.com/pdf/Datasheet-Dome-Shield.pdf\",\n \"http://securedns.dnsbycomodo.com/\",\n \"https://www.comodo.com/secure-dns/secure-dns-assets/dowloads/ccs-dome-shield-whitepaper-threat-intelligence.pdf\",\n \"https://www.comodo.com/secure-dns/secure-dns-assets/dowloads/domeshield-all-use-cases.pdf\",\n ],\n 'favIcon': \"https://www.comodo.com/favicon.ico\",\n 'logo': \"https://www.comodo.com/new-assets/images/logo.png\",\n 'description': \"Comodo Secure DNS is a domain name resolution service \"\n \"that resolves your DNS requests through our worldwide network of redundant DNS servers, \"\n \"bringing you the most reliable fully redundant DNS service anywhere, for a safer, \"\n \"smarter and faster Internet experience.\"\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"AFFILIATE_INTERNET_NAME\",\n \"CO_HOSTED_SITE\"\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_INTERNET_NAME\",\n \"BLACKLISTED_AFFILIATE_INTERNET_NAME\",\n \"BLACKLISTED_COHOST\",\n \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_COHOST\",\n ]\n\n def query(self, qaddr):\n res = dns.resolver.Resolver()\n res.nameservers = [\"8.26.56.26\", \"8.20.247.20\"]\n\n try:\n addrs = res.resolve(qaddr)\n self.debug(f\"Addresses returned: {addrs}\")\n except Exception:\n self.debug(f\"Unable to resolve {qaddr}\")\n return False\n\n if addrs:\n return True\n return False\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if eventName == \"INTERNET_NAME\":\n malicious_type = \"MALICIOUS_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n malicious_type = \"MALICIOUS_AFFILIATE_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n malicious_type = \"MALICIOUS_COHOST\"\n blacklist_type = \"BLACKLISTED_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n # Check that it resolves first, as it becomes a valid\n # malicious host only if NOT resolved by Comodo Secure DNS.\n if not self.sf.resolveHost(eventData) and not self.sf.resolveHost6(eventData):\n return\n\n found = self.query(eventData)\n\n # Host was found, not blocked\n if found:\n return\n\n evt = SpiderFootEvent(blacklist_type, f\"Comodo Secure DNS [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(malicious_type, f\"Comodo Secure DNS [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_comodo class\n" }, { "path": "modules/sfp_company.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_company\n# Purpose: SpiderFoot plug-in for scanning retrieved content by other\n# modules (such as sfp_spider) and identifying company names.\n#\n# Author: Steve Micallef \n#\n# Created: 09/09/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_company(SpiderFootPlugin):\n\n meta = {\n 'name': \"Company Name Extractor\",\n 'summary': \"Identify company names in any obtained data.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Content Analysis\"]\n }\n\n opts = {\n 'filterjscss': True\n }\n\n optdescs = {\n 'filterjscss': \"Filter out company names that originated from CSS/JS content. Enabling this avoids detection of popular Javascript and web framework author company names.\"\n }\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"TARGET_WEB_CONTENT\", \"SSL_CERTIFICATE_ISSUED\",\n \"DOMAIN_WHOIS\", \"NETBLOCK_WHOIS\",\n \"AFFILIATE_DOMAIN_WHOIS\", \"AFFILIATE_WEB_CONTENT\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"COMPANY_NAME\", \"AFFILIATE_COMPANY_NAME\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n # Various ways to identify companies in text\n # Support up to three word company names with each starting with\n # a capital letter, allowing for hyphens brackets and numbers within.\n pattern_prefix = r\"(?=[,;:\\'\\\">\\(= ]|^)\\s?([A-Z0-9\\(\\)][A-Za-z0-9\\-&,\\.][^ \\\"\\';:><]*)?\\s?([A-Z0-9\\(\\)][A-Za-z0-9\\-&,\\.]?[^ \\\"\\';:><]*|[Aa]nd)?\\s?([A-Z0-9\\(\\)][A-Za-z0-9\\-&,\\.]?[^ \\\"\\';:><]*)?\\s+\"\n pattern_match_re = [\n 'LLC', r'L\\.L\\.C\\.?', 'AG', r'A\\.G\\.?', 'GmbH', r'Pty\\.?\\s+Ltd\\.?',\n r'Ltd\\.?', r'Pte\\.?', r'Inc\\.?', r'INC\\.?', 'Incorporated', 'Foundation',\n r'Corp\\.?', 'Corporation', 'SA', r'S\\.A\\.?', 'SIA', 'BV', r'B\\.V\\.?',\n 'NV', r'N\\.V\\.?', 'PLC', 'Limited', r'Pvt\\.?\\s+Ltd\\.?', 'SARL']\n pattern_match = [\n 'LLC', 'L.L.C', 'AG', 'A.G', 'GmbH', 'Pty',\n 'Ltd', 'Pte', 'Inc', 'INC', 'Foundation',\n 'Corp', 'SA', 'S.A', 'SIA', 'BV', 'B.V',\n 'NV', 'N.V', 'PLC', 'Limited', 'Pvt.', 'SARL']\n\n pattern_suffix = r\"(?=[ \\.,:<\\)\\'\\\"]|[$\\n\\r])\"\n\n # Filter out anything from the company name which matches the below\n filterpatterns = [\n \"Copyright\",\n r\"\\d{4}\" # To catch years\n ]\n\n # Don't re-parse company names\n if eventName in [\"COMPANY_NAME\", \"AFFILIATE_COMPANY_NAME\"]:\n return\n\n if eventName == \"TARGET_WEB_CONTENT\":\n url = event.actualSource\n if self.opts['filterjscss'] and (\".js\" in url or \".css\" in url):\n self.debug(\"Ignoring web content from CSS/JS.\")\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName} ({len(eventData)} bytes)\")\n\n # Strip out everything before the O=\n try:\n if eventName == \"SSL_CERTIFICATE_ISSUED\":\n eventData = eventData.split(\"O=\")[1]\n except Exception:\n self.debug(\"Couldn't strip out 'O=' from certificate issuer, proceeding anyway...\")\n\n # Find chunks of text containing what might be a company name first.\n # This is to avoid running very expensive regexps on large chunks of\n # data.\n chunks = list()\n for pat in pattern_match:\n start = 0\n m = eventData.find(pat, start)\n while m > 0:\n start = m - 50\n if start < 0:\n start = 0\n end = m + 10\n if end >= len(eventData):\n end = len(eventData) - 1\n chunks.append(eventData[start:end])\n offset = m + len(pat)\n m = eventData.find(pat, offset)\n\n myres = list()\n for chunk in chunks:\n for pat in pattern_match_re:\n matches = re.findall(pattern_prefix + \"(\" + pat + \")\" + pattern_suffix, chunk, re.MULTILINE | re.DOTALL)\n for match in matches:\n matched = 0\n for m in match:\n if len(m) > 0:\n matched += 1\n if matched <= 1:\n continue\n\n fullcompany = \"\"\n for m in match:\n flt = False\n for f in filterpatterns:\n if re.match(f, m):\n flt = True\n if not flt:\n fullcompany += m + \" \"\n\n fullcompany = re.sub(r\"\\s+\", \" \", fullcompany.strip())\n\n self.info(\"Found company name: \" + fullcompany)\n\n if fullcompany in myres:\n self.debug(\"Already found from this source.\")\n continue\n\n myres.append(fullcompany)\n\n if \"AFFILIATE_\" in eventName:\n etype = \"AFFILIATE_COMPANY_NAME\"\n else:\n etype = \"COMPANY_NAME\"\n\n evt = SpiderFootEvent(etype, fullcompany, self.__name__, event)\n if event.moduleDataSource:\n evt.moduleDataSource = event.moduleDataSource\n else:\n evt.moduleDataSource = \"Unknown\"\n self.notifyListeners(evt)\n\n# End of sfp_company class\n" }, { "path": "modules/sfp_cookie.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_cookie\n# Purpose: SpiderFoot plug-in for extracting cookies from HTTP headers.\n#\n# Author: Steve Micallef \n#\n# Created: 06/04/2014\n# Copyright: (c) Steve Micallef 2014\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_cookie(SpiderFootPlugin):\n\n meta = {\n 'name': \"Cookie Extractor\",\n 'summary': \"Extract Cookies from HTTP headers.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Content Analysis\"]\n }\n\n opts = {}\n optdescs = {}\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.__dataSource__ = \"Target Website\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"WEBSERVER_HTTPHEADERS\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"TARGET_WEB_COOKIE\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n eventSource = event.actualSource\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventSource in self.results:\n return\n\n self.results[eventSource] = True\n\n fqdn = self.sf.urlFQDN(eventSource)\n if not self.getTarget().matches(fqdn):\n self.debug(f\"Not collecting cookies from external sites. Ignoring HTTP headers from {fqdn}\")\n return\n\n try:\n data = json.loads(eventData)\n except Exception:\n self.error(\"Received HTTP headers from another module in an unexpected format.\")\n return\n\n cookie = data.get('cookie')\n if cookie:\n evt = SpiderFootEvent(\"TARGET_WEB_COOKIE\", cookie, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_cookie class\n" }, { "path": "modules/sfp_countryname.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_countryname\n# Purpose: SpiderFoot plug-in for scanning retrieved content by other\n# modules (such as sfp_iban, sfp_phone, sfp_whois) and identifying country names\n#\n# Author: Krishnasis Mandal \n#\n# Created: 28/04/2020\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nimport phonenumbers\nfrom phonenumbers.phonenumberutil import region_code_for_country_code\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_countryname(SpiderFootPlugin):\n\n meta = {\n 'name': \"Country Name Extractor\",\n 'summary': \"Identify country names in any obtained data.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Content Analysis\"]\n }\n\n opts = {\n 'cohosted': True,\n 'affiliate': True,\n 'noncountrytld': True,\n 'similardomain': False,\n }\n\n optdescs = {\n 'cohosted': \"Obtain country name from co-hosted sites\",\n 'affiliate': \"Obtain country name from affiliate sites\",\n 'noncountrytld': \"Parse TLDs not associated with any country as default country domains\",\n 'similardomain': \"Obtain country name from similar domains\"\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n # Clear / reset any other class member variables here\n # or you risk them persisting between threads.\n\n for opt in userOpts.keys():\n self.opts[opt] = userOpts[opt]\n\n def detectCountryFromPhone(self, srcPhoneNumber: str) -> str:\n \"\"\"Lookup name of country from phone number region code.\n\n Args:\n srcPhoneNumber (str): phone number\n\n Returns:\n str: country name\n \"\"\"\n\n if not isinstance(srcPhoneNumber, str):\n return None\n\n try:\n phoneNumber = phonenumbers.parse(srcPhoneNumber)\n except Exception:\n self.debug(f\"Skipped invalid phone number: {srcPhoneNumber}\")\n return None\n\n try:\n countryCode = region_code_for_country_code(phoneNumber.country_code)\n except Exception:\n self.debug(f\"Lookup of region code failed for phone number: {srcPhoneNumber}\")\n return None\n\n if not countryCode:\n return None\n\n return SpiderFootHelpers.countryNameFromCountryCode(countryCode.upper())\n\n def detectCountryFromDomainName(self, srcDomain: str) -> str:\n \"\"\"Lookup name of country from TLD of domain name.\n\n Args:\n srcDomain (str): domain\n\n Returns:\n str: country name\n \"\"\"\n if not isinstance(srcDomain, str):\n return None\n\n # Split domain into parts by '.'\n # Country TLDs are reserved\n domainParts = srcDomain.split(\".\")\n\n # Search for country TLD in the domain parts - reversed\n for part in domainParts[::-1]:\n country_name = SpiderFootHelpers.countryNameFromTld(part)\n if country_name:\n return country_name\n\n return None\n\n def detectCountryFromIBAN(self, srcIBAN: str) -> str:\n \"\"\"Detect name of country from IBAN.\n\n Args:\n srcIBAN (str): IBAN\n\n Returns:\n str: country name\n \"\"\"\n if not isinstance(srcIBAN, str):\n return None\n\n return SpiderFootHelpers.countryNameFromCountryCode(srcIBAN[0:2])\n\n def detectCountryFromData(self, srcData: str) -> list:\n \"\"\"Detect name of country from event data (WHOIS lookup, Geo Info, Physical Address, etc)\n\n Args:\n srcData (str): event data\n\n Returns:\n list: list of countries\n \"\"\"\n countries = list()\n\n if not srcData:\n return countries\n\n # Get dictionary of country codes and country names\n abbvCountryCodes = SpiderFootHelpers.countryCodes()\n\n # Look for countrycodes and country in source data\n for countryName in abbvCountryCodes.values():\n if countryName.lower() not in srcData.lower():\n continue\n\n # Look for country name in source data\n # Spaces are not included since \"New Jersey\" and others\n # will get interpreted as \"Jersey\", etc.\n matchCountries = re.findall(r\"[,'\\\"\\:\\=\\[\\(\\[\\n\\t\\r\\.] ?\" + countryName + r\"[,'\\\"\\:\\=\\[\\(\\[\\n\\t\\r\\.]\", srcData, re.IGNORECASE)\n\n if matchCountries:\n countries.append(countryName)\n\n # Look for \"Country: \", usually found in Whois records\n matchCountries = re.findall(\"country: (.+?)\", srcData, re.IGNORECASE)\n if matchCountries:\n for m in matchCountries:\n m = m.strip()\n if m in abbvCountryCodes:\n countries.append(abbvCountryCodes[m])\n if m in abbvCountryCodes.values():\n countries.append(m)\n\n return list(set(countries))\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"IBAN_NUMBER\", \"PHONE_NUMBER\", \"AFFILIATE_DOMAIN_NAME\",\n \"CO_HOSTED_SITE_DOMAIN\", \"DOMAIN_NAME\", \"SIMILARDOMAIN\",\n \"AFFILIATE_DOMAIN_WHOIS\", \"CO_HOSTED_SITE_DOMAIN_WHOIS\",\n \"DOMAIN_WHOIS\", \"GEOINFO\", \"PHYSICAL_ADDRESS\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"COUNTRY_NAME\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if event.moduleDataSource:\n moduleDataSource = event.moduleDataSource\n else:\n moduleDataSource = \"Unknown\"\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n eventDataHash = self.sf.hashstring(eventData)\n\n if eventDataHash in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventDataHash] = True\n\n countryNames = list()\n\n # Process the event data based on incoming event type\n if eventName == \"PHONE_NUMBER\":\n countryNames.append(self.detectCountryFromPhone(eventData))\n elif eventName == \"DOMAIN_NAME\":\n countryNames.append(self.detectCountryFromDomainName(eventData))\n elif eventName == \"AFFILIATE_DOMAIN_NAME\" and self.opts[\"affiliate\"]:\n countryNames.append(self.detectCountryFromDomainName(eventData))\n elif eventName == \"CO_HOSTED_SITE_DOMAIN\" and self.opts[\"cohosted\"]:\n countryNames.append(self.detectCountryFromDomainName(eventData))\n elif eventName == \"SIMILARDOMAIN\" and self.opts[\"similardomain\"]:\n countryNames.append(self.detectCountryFromDomainName(eventData))\n elif eventName == \"IBAN_NUMBER\":\n countryNames.append(self.detectCountryFromIBAN(eventData))\n elif eventName in [\"DOMAIN_WHOIS\", \"GEOINFO\", \"PHYSICAL_ADDRESS\"]:\n countryNames.extend(self.detectCountryFromData(eventData))\n elif eventName == \"AFFILIATE_DOMAIN_WHOIS\" and self.opts[\"affiliate\"]:\n countryNames.extend(self.detectCountryFromData(eventData))\n elif eventName == \"CO_HOSTED_SITE_DOMAIN_WHOIS\" and self.opts[\"cohosted\"]:\n countryNames.extend(self.detectCountryFromData(eventData))\n\n if not countryNames:\n self.debug(f\"Found no country names associated with {eventName}: {eventData}\")\n return\n\n for countryName in set(countryNames):\n if not countryName:\n continue\n\n self.debug(f\"Found country name: {countryName}\")\n\n evt = SpiderFootEvent(\"COUNTRY_NAME\", countryName, self.__name__, event)\n evt.moduleDataSource = moduleDataSource\n self.notifyListeners(evt)\n\n# End of sfp_countryname class\n" }, { "path": "modules/sfp_creditcard.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_creditcard\n# Purpose: SpiderFoot plug-in for scanning retrieved content by other\n# modules (such as sfp_spider) and identifying credit card numbers.\n#\n# Author: Krishnasis Mandal \n#\n# Created: 21/04/2020\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_creditcard(SpiderFootPlugin):\n\n meta = {\n 'name': \"Credit Card Number Extractor\",\n 'summary': \"Identify Credit Card Numbers in any data\",\n 'flags': [\"errorprone\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Content Analysis\"]\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n # Override datasource for sfp_creditcard module\n self.__dataSource__ = \"Target Website\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DARKNET_MENTION_CONTENT\", \"LEAKSITE_CONTENT\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"CREDIT_CARD_NUMBER\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n creditCards = SpiderFootHelpers.extractCreditCardsFromText(eventData)\n\n for creditCard in set(creditCards):\n self.info(f\"Found credit card number: {creditCard}\")\n evt = SpiderFootEvent(\"CREDIT_CARD_NUMBER\", creditCard, self.__name__, event)\n if event.moduleDataSource:\n evt.moduleDataSource = event.moduleDataSource\n else:\n evt.moduleDataSource = \"Unknown\"\n self.notifyListeners(evt)\n\n# End of sfp_creditcard class\n" }, { "path": "modules/sfp_crobat_api.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_crobat_api\n# Purpose: Search Crobat API for subdomains.\n# https://www.onsecurity.co.uk/blog/how-i-made-rapid7s-project-sonar-searchable\n# https://github.com/cgboal/sonarsearch\n#\n# Authors: \n#\n# Created: 2020-08-29\n# Copyright: (c) bcoles 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_crobat_api(SpiderFootPlugin):\n\n meta = {\n 'name': \"Crobat API\",\n 'summary': \"Search Crobat API for subdomains.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Passive DNS\"],\n 'dataSource': {\n 'website': \"https://sonar.omnisint.io/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'logo': \"https://sonar.omnisint.io/img/crobat.png\",\n 'description': \"The entire Rapid7 Sonar DNS dataset indexed,\"\n \" available at your fingertips.\",\n }\n }\n\n opts = {\n \"verify\": True,\n \"max_pages\": 10,\n \"delay\": 1\n }\n\n optdescs = {\n \"verify\": \"DNS resolve each identified subdomain.\",\n \"max_pages\": \"Maximum number of pages of results to fetch.\",\n \"delay\": \"Delay between requests, in seconds.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in userOpts.keys():\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"DOMAIN_NAME\"]\n\n def producedEvents(self):\n return [\"RAW_RIR_DATA\", \"INTERNET_NAME\", \"INTERNET_NAME_UNRESOLVED\"]\n\n def queryDomain(self, qry, page=0):\n headers = {\n \"Accept\": \"application/json\"\n }\n params = urllib.parse.urlencode({\n 'page': page\n })\n domain = qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n res = self.sf.fetchUrl(\n f\"https://sonar.omnisint.io/subdomains/{domain}?{params}\",\n headers=headers,\n timeout=30,\n useragent=self.opts['_useragent']\n )\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from Crobat API.\")\n return None\n\n # Future proofing - Crobat API does not implement rate limiting\n if res['code'] == '429':\n self.error(\"You are being rate-limited by Crobat API\")\n self.errorState = True\n return None\n\n # Catch all non-200 status codes, and presume something went wrong\n if res['code'] != '200':\n self.error(\"Failed to retrieve content from Crobat API\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n return None\n\n # returns \"null\" when page has no data\n if res['content'] == \"null\":\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n if not isinstance(data, list):\n self.error(\"Failed to retrieve content from Crobat API\")\n return None\n\n return data\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventName != \"DOMAIN_NAME\":\n return\n\n page = 0\n while page < self.opts['max_pages']:\n if self.checkForStop():\n return\n\n if self.errorState:\n return\n\n data = self.queryDomain(eventData, page)\n\n if not data:\n self.debug(f\"No information found for domain {eventData} (page: {page})\")\n return\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n page += 1\n\n for domain in set(data):\n if domain in self.results:\n continue\n\n if not self.getTarget().matches(domain, includeChildren=True, includeParents=True):\n continue\n\n if self.opts['verify'] and not self.sf.resolveHost(domain) and not self.sf.resolveHost6(domain):\n self.debug(f\"Host {domain} could not be resolved\")\n evt = SpiderFootEvent(\"INTERNET_NAME_UNRESOLVED\", domain, self.__name__, event)\n self.notifyListeners(evt)\n else:\n evt = SpiderFootEvent(\"INTERNET_NAME\", domain, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_crobat_api class\n" }, { "path": "modules/sfp_crossref.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_crossref\n# Purpose: SpiderFoot plug-in for scanning links identified from the\n# spidering process, and for external links, fetching them to\n# see if those sites link back to the original site, indicating a\n# potential relationship between the external sites.\n#\n# Author: Steve Micallef \n#\n# Created: 06/04/2012\n# Copyright: (c) Steve Micallef 2012\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_crossref(SpiderFootPlugin):\n\n meta = {\n 'name': \"Cross-Referencer\",\n 'summary': \"Identify whether other domains are associated ('Affiliates') of the target by looking for links back to the target site(s).\",\n 'flags': [],\n 'useCases': [\"Footprint\"],\n 'categories': [\"Crawling and Scanning\"]\n }\n\n opts = {\n 'checkbase': True\n }\n\n optdescs = {\n \"checkbase\": \"Check the base URL of the potential affiliate if no direct affiliation found?\"\n }\n\n fetched = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.fetched = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n 'LINKED_URL_EXTERNAL',\n 'SIMILARDOMAIN',\n 'CO_HOSTED_SITE',\n 'DARKNET_MENTION_URL'\n ]\n\n def producedEvents(self):\n return [\n 'AFFILIATE_INTERNET_NAME',\n 'AFFILIATE_WEB_CONTENT'\n ]\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n # SIMILARDOMAIN and CO_HOSTED_SITE events are domains, not URLs.\n # Assume HTTP.\n if eventName in ['SIMILARDOMAIN', 'CO_HOSTED_SITE']:\n url = 'http://' + eventData.lower()\n elif 'URL' in eventName:\n url = eventData\n else:\n return\n\n fqdn = self.sf.urlFQDN(url)\n\n # We are only interested in external sites for the crossref\n if self.getTarget().matches(fqdn):\n self.debug(f\"Ignoring {url} as not external\")\n return\n\n if eventData in self.fetched:\n self.debug(f\"Ignoring {url} as already tested\")\n return\n\n if not self.sf.resolveHost(fqdn) and not self.sf.resolveHost6(fqdn):\n self.debug(f\"Ignoring {url} as {fqdn} does not resolve\")\n return\n\n self.fetched[url] = True\n\n self.debug(f\"Testing URL for affiliation: {url}\")\n\n res = self.sf.fetchUrl(\n url,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n sizeLimit=10000000,\n verify=False\n )\n\n if res['content'] is None:\n self.debug(f\"Ignoring {url} as no data returned\")\n return\n\n matched = False\n for name in self.getTarget().getNames():\n # Search for mentions of our host/domain in the external site's data\n pat = re.compile(\n r\"([\\.\\'\\/\\\"\\ ]\" + re.escape(name) + r\"[\\.\\'\\/\\\"\\ ])\",\n re.IGNORECASE\n )\n matches = re.findall(pat, str(res['content']))\n\n if len(matches) > 0:\n matched = True\n break\n\n if not matched:\n # If the name wasn't found in the affiliate, and checkbase is set,\n # fetch the base URL of the affiliate to check for a crossref.\n if eventName == \"LINKED_URL_EXTERNAL\" and self.opts['checkbase']:\n # Check the base url to see if there is an affiliation\n url = SpiderFootHelpers.urlBaseUrl(eventData)\n if url in self.fetched:\n return\n\n self.fetched[url] = True\n\n res = self.sf.fetchUrl(\n url,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n sizeLimit=10000000,\n verify=False\n )\n\n if res['content'] is not None:\n for name in self.getTarget().getNames():\n pat = re.compile(\n r\"([\\.\\'\\/\\\"\\ ]\" + re.escape(name) + r\"[\\'\\/\\\"\\ ])\",\n re.IGNORECASE\n )\n matches = re.findall(pat, str(res['content']))\n\n if len(matches) > 0:\n matched = True\n break\n\n if not matched:\n return\n\n if not event.moduleDataSource:\n event.moduleDataSource = \"Unknown\"\n\n self.info(f\"Found link to target from affiliate: {url}\")\n\n evt1 = SpiderFootEvent(\n \"AFFILIATE_INTERNET_NAME\",\n self.sf.urlFQDN(url),\n self.__name__,\n event\n )\n evt1.moduleDataSource = event.moduleDataSource\n self.notifyListeners(evt1)\n\n evt2 = SpiderFootEvent(\n \"AFFILIATE_WEB_CONTENT\",\n res['content'],\n self.__name__,\n evt1\n )\n evt2.moduleDataSource = event.moduleDataSource\n self.notifyListeners(evt2)\n\n# End of sfp_crossref class\n" }, { "path": "modules/sfp_crt.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_crt\n# Purpose: SpiderFoot plug-in to identify historical certificates for a domain\n# from crt.sh, and from this identify hostnames.\n#\n# Author: Steve Micallef \n#\n# Created: 17/03/2017\n# Copyright: (c) Steve Micallef 2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib.parse\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_crt(SpiderFootPlugin):\n\n meta = {\n 'name': \"Certificate Transparency\",\n 'summary': \"Gather hostnames from historical certificates in crt.sh.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://crt.sh/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://sectigo.com/\",\n \"https://github.com/crtsh\"\n ],\n 'favIcon': \"https://crt.sh/sectigo_s.png\",\n 'logo': \"https://crt.sh/sectigo_s.png\",\n 'description': \"Free CT Log Certificate Search Tool from Sectigo (formerly Comodo CA).\"\n }\n }\n\n opts = {\n 'verify': True,\n 'fetchcerts': True,\n }\n\n optdescs = {\n 'verify': 'Verify certificate subject alternative names resolve.',\n 'fetchcerts': 'Fetch each certificate found, for processing by other modules.',\n }\n\n results = None\n cert_ids = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.errorState = False\n self.results = self.tempStorage()\n self.cert_ids = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return ['DOMAIN_NAME', 'INTERNET_NAME']\n\n def producedEvents(self):\n return [\n \"SSL_CERTIFICATE_RAW\",\n \"RAW_RIR_DATA\",\n \"INTERNET_NAME\",\n \"INTERNET_NAME_UNRESOLVED\",\n \"DOMAIN_NAME\",\n \"CO_HOSTED_SITE\",\n \"CO_HOSTED_SITE_DOMAIN\"\n ]\n\n def queryDomain(self, qry: str):\n params = {\n 'q': '%.' + qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n 'output': 'json'\n }\n\n res = self.sf.fetchUrl(\n \"https://crt.sh/?\" + urllib.parse.urlencode(params),\n timeout=30,\n useragent=self.opts['_useragent']\n )\n\n time.sleep(0.5)\n\n return self.parseApiResponse(res)\n\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from crt.sh\")\n return None\n\n if res['code'] == '404':\n self.debug(\"No results for query\")\n return None\n\n # Future proofing - crt.sh does not implement rate limiting\n if res['code'] == '429':\n self.error(\"You are being rate-limited by crt.sh\")\n self.errorState = True\n return None\n\n if res['code'] == '500' or res['code'] == '502' or res['code'] == '503':\n self.error(\"crt.sh service is unavailable\")\n self.errorState = True\n return None\n\n # Catch all other non-200 status codes, and presume something went wrong\n if res['code'] != '200':\n self.error(\"Failed to retrieve content from crt.sh\")\n self.errorState = True\n return None\n\n if not res['content']:\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n def handleEvent(self, event):\n if self.errorState:\n return\n\n if event.data in self.results:\n return\n\n self.results[event.data] = True\n\n self.debug(f\"Received event, {event.eventType}, from {event.module}\")\n\n data = self.queryDomain(event.data)\n\n if not data:\n self.debug(f\"No certificate transparency results for domain {event.data}\")\n return\n\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n domains = list()\n certs = list()\n\n # Extract all domains and certificate IDs from each certificate\n for cert_info in data:\n cert_id = cert_info.get('id')\n\n if not cert_id:\n continue\n\n if cert_id in self.cert_ids:\n continue\n\n self.cert_ids[cert_id] = True\n\n if self.opts['fetchcerts']:\n certs.append(cert_id)\n\n domain = cert_info.get('name_value')\n\n if not domain:\n continue\n\n for d in domain.split(\"\\n\"):\n if d.lower() == event.data.lower():\n continue\n domains.append(d.lower().replace(\"*.\", \"\"))\n\n if self.opts['fetchcerts'] and len(certs) > 0:\n self.info(f\"Retrieving {len(set(certs))} certificates ...\")\n\n # Fetch and store each certificate\n # Report SAN domains for each certificate\n for cert_id in set(certs):\n if self.checkForStop():\n return\n\n if self.errorState:\n break\n\n params = {\n 'd': str(cert_id)\n }\n\n res = self.sf.fetchUrl(\n 'https://crt.sh/?' + urllib.parse.urlencode(params),\n timeout=30,\n useragent=self.opts['_useragent']\n )\n\n time.sleep(0.5)\n\n if not res or not res['content']:\n self.error(f\"Error retrieving certificate with ID {cert_id}. No response from crt.sh\")\n continue\n\n try:\n cert = self.sf.parseCert(str(res['content']))\n except Exception as e:\n self.info(f\"Error parsing certificate: {e}\")\n continue\n\n cert_text = cert.get('text')\n\n if cert_text:\n evt = SpiderFootEvent(\"SSL_CERTIFICATE_RAW\", str(cert_text), self.__name__, event)\n self.notifyListeners(evt)\n\n sans = cert.get('altnames', list())\n\n if not sans:\n continue\n\n for san in sans:\n if san.lower() == event.data.lower():\n continue\n domains.append(san.lower().replace(\"*.\", \"\"))\n\n if self.opts['verify'] and len(domains) > 0:\n self.info(f\"Resolving {len(set(domains))} domains ...\")\n\n for domain in set(domains):\n if domain in self.results:\n continue\n\n if not self.sf.validHost(domain, self.opts['_internettlds']):\n continue\n\n if self.getTarget().matches(domain, includeChildren=True, includeParents=True):\n evt_type = 'INTERNET_NAME'\n if self.opts['verify'] and not self.sf.resolveHost(domain) and not self.sf.resolveHost6(domain):\n self.debug(f\"Host {domain} could not be resolved\")\n evt_type += '_UNRESOLVED'\n else:\n evt_type = 'CO_HOSTED_SITE'\n\n evt = SpiderFootEvent(evt_type, domain, self.__name__, event)\n self.notifyListeners(evt)\n\n if self.sf.isDomain(domain, self.opts['_internettlds']):\n if evt_type == 'CO_HOSTED_SITE':\n evt = SpiderFootEvent('CO_HOSTED_SITE_DOMAIN', domain, self.__name__, event)\n self.notifyListeners(evt)\n else:\n evt = SpiderFootEvent('DOMAIN_NAME', domain, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_crt class\n" }, { "path": "modules/sfp_crxcavator.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_crxcavator\n# Purpose: Query CRXcavator for Chrome extensions.\n#\n# Author: \n#\n# Created: 2020-09-19\n# Copyright: (c) bcoles 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_crxcavator(SpiderFootPlugin):\n\n meta = {\n 'name': \"CRXcavator\",\n 'summary': \"Search CRXcavator for Chrome extensions.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Footprint\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://crxcavator.io/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'favIcon': \"https://crxcavator.io/favicon-32x32.png\",\n 'logo': \"https://crxcavator.io/apple-touch-icon.png\",\n 'description': \"CRXcavator automatically scans the entire Chrome Web \"\n \"Store every 3 hours and produces a quantified risk score for \"\n \"each Chrome Extension based on several factors.\",\n }\n }\n\n opts = {\n \"verify\": True,\n }\n\n optdescs = {\n \"verify\": \"Verify identified hostnames resolve.\",\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n 'DOMAIN_NAME'\n ]\n\n def producedEvents(self):\n return [\n 'APPSTORE_ENTRY',\n 'INTERNET_NAME',\n 'INTERNET_NAME_UNRESOLVED',\n 'LINKED_URL_INTERNAL',\n 'AFFILIATE_INTERNET_NAME',\n 'AFFILIATE_INTERNET_NAME_UNRESOLVED',\n 'PHYSICAL_ADDRESS',\n 'RAW_RIR_DATA'\n ]\n\n def query(self, qry):\n params = urllib.parse.urlencode({\n 'q': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n })\n\n res = self.sf.fetchUrl(\n f\"https://api.crxcavator.io/v1/search?{params}\",\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout']\n )\n\n time.sleep(1)\n\n if res['content'] is None:\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from CRXcavator: {e}\")\n return None\n\n if not data:\n self.debug(f\"No results found for {qry}\")\n return None\n\n return data\n\n def queryExtension(self, extension_id):\n res = self.sf.fetchUrl(\n f\"https://api.crxcavator.io/v1/report/{extension_id}\",\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout']\n )\n\n time.sleep(1)\n\n if res['content'] is None:\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from CRXcavator: {e}\")\n return None\n\n if not data:\n self.debug(f\"No results found for extension {extension_id}\")\n return None\n\n return data\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if eventName not in self.watchedEvents():\n return\n\n self.results[eventData] = True\n\n domain_keyword = self.sf.domainKeyword(eventData, self.opts['_internettlds'])\n results = self.query(domain_keyword)\n\n if not results:\n self.info(f\"No results found for {domain_keyword}\")\n return\n\n evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(results), self.__name__, event)\n self.notifyListeners(evt)\n\n urls = list()\n hosts = list()\n locations = list()\n\n for result in results:\n if not isinstance(result, dict):\n continue\n\n extension_id = result.get('extension_id')\n\n if not extension_id:\n continue\n\n if '@' in extension_id:\n continue\n\n extensions = self.queryExtension(extension_id)\n\n if not extensions:\n continue\n\n evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(extensions), self.__name__, event)\n self.notifyListeners(evt)\n\n for extension in extensions:\n data = extension.get('data')\n\n if not data:\n continue\n\n manifest = data.get('manifest')\n\n if not manifest:\n continue\n\n version = manifest.get('version')\n\n if not version:\n continue\n\n webstore = data.get('webstore')\n\n if not webstore:\n continue\n\n name = webstore.get('name')\n\n if not name:\n continue\n\n app_full_name = f\"{name} {version} ({extension_id})\"\n\n privacy_policy = webstore.get('privacy_policy')\n support_site = webstore.get('support_site')\n offered_by = webstore.get('offered_by')\n website = webstore.get('website')\n\n if not privacy_policy and not support_site and not offered_by and not website:\n continue\n\n if (\n not self.getTarget().matches(self.sf.urlFQDN(privacy_policy), includeChildren=True, includeParents=True)\n and not self.getTarget().matches(self.sf.urlFQDN(website), includeChildren=True, includeParents=True)\n and not self.getTarget().matches(self.sf.urlFQDN(offered_by), includeChildren=True, includeParents=True)\n and not self.getTarget().matches(self.sf.urlFQDN(support_site), includeChildren=True, includeParents=True)\n ):\n self.debug(f\"Extension {app_full_name} does not match {eventData}, skipping\")\n continue\n\n app_data = f\"{name} {version}\\nhttps://chrome.google.com/webstore/detail/{extension_id}\"\n\n evt = SpiderFootEvent('APPSTORE_ENTRY', app_data, self.__name__, event)\n self.notifyListeners(evt)\n\n if privacy_policy:\n urls.append(privacy_policy)\n\n if support_site:\n urls.append(support_site)\n\n if website:\n urls.append(website)\n\n if offered_by:\n urls.append(offered_by)\n\n address = webstore.get('address')\n\n if address and len(address) > 10:\n locations.append(address)\n\n for url in set(urls):\n if not url:\n continue\n\n host = self.sf.urlFQDN(url)\n\n if not host:\n continue\n\n if self.getTarget().matches(host, includeChildren=True, includeParents=True):\n evt = SpiderFootEvent('LINKED_URL_INTERNAL', url, self.__name__, event)\n self.notifyListeners(evt)\n\n hosts.append(host)\n\n for host in set(hosts):\n if not host:\n continue\n\n if self.getTarget().matches(host, includeChildren=True, includeParents=True):\n evt_type = 'INTERNET_NAME'\n else:\n evt_type = 'AFFILIATE_INTERNET_NAME'\n\n if self.opts['verify'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host):\n self.debug(f\"Host {host} could not be resolved\")\n evt_type += '_UNRESOLVED'\n\n evt = SpiderFootEvent(evt_type, host, self.__name__, event)\n self.notifyListeners(evt)\n\n for location in set(locations):\n evt = SpiderFootEvent(\"PHYSICAL_ADDRESS\", location, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_crxcavator class\n" }, { "path": "modules/sfp_customfeed.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_customfeed\n# Purpose: Checks if an ASN, IP, hostname or domain is listed as malicious\n# in a user-supplied data feed.\n#\n# Author: steve@binarypool.com\n#\n# Created: 11/11/2018\n# Copyright: (c) Steve Micallef, 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom netaddr import IPAddress, IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\nmalchecks = {\n 'Custom Threat Data': {\n 'id': '_customfeed',\n 'checks': ['ip', 'netblock', 'asn', 'domain'],\n 'regex': '^{0}$'\n }\n}\n\n\nclass sfp_customfeed(SpiderFootPlugin):\n\n meta = {\n 'name': \"Custom Threat Feed\",\n 'summary': \"Check if a host/domain, netblock, ASN or IP is malicious according to your custom feed.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"]\n }\n\n # Default options\n opts = {\n 'checkaffiliates': True,\n 'checkcohosts': True,\n 'url': \"\",\n 'cacheperiod': 0\n }\n\n # Option descriptions\n optdescs = {\n 'url': \"The URL where the feed can be found. Exact matching is performed so the format must be a single line per host, ASN, domain, IP or netblock.\",\n 'checkaffiliates': \"Apply checks to affiliates?\",\n 'checkcohosts': \"Apply checks to sites found to be co-hosted on the target's IP?\",\n 'cacheperiod': \"Maximum age of data in hours before re-downloading. 0 to always download.\"\n }\n\n # Be sure to completely clear any class variables in setup()\n # or you run the risk of data persisting between scan runs.\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n # Clear / reset any other class member variables here\n # or you risk them persisting between threads.\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n # * = be notified about all events.\n def watchedEvents(self):\n return [\"INTERNET_NAME\", \"IP_ADDRESS\", \"AFFILIATE_INTERNET_NAME\",\n \"AFFILIATE_IPADDR\", \"CO_HOSTED_SITE\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"MALICIOUS_IPADDR\", \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_IPADDR\", \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_COHOST\"]\n\n # Look up 'list' type resources\n def resourceList(self, replaceme_id, target, targetType):\n targetDom = ''\n # Get the base domain if we're supplied a domain\n if targetType == \"domain\":\n targetDom = self.sf.hostDomain(target, self.opts['_internettlds'])\n if not targetDom:\n return None\n\n for check in list(malchecks.keys()):\n cid = malchecks[check]['id']\n url = self.opts['url']\n if replaceme_id == cid:\n data = dict()\n data['content'] = self.sf.cacheGet(\"sfmal_\" + cid, self.opts.get('cacheperiod', 0))\n if data['content'] is None:\n data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])\n if data['content'] is None:\n self.error(\"Unable to fetch \" + url)\n return None\n self.sf.cachePut(\"sfmal_\" + cid, data['content'])\n\n # If we're looking at netblocks\n if targetType == \"netblock\":\n iplist = list()\n # Get the regex, replace {0} with an IP address matcher to\n # build a list of IP.\n # Cycle through each IP and check if it's in the netblock.\n if 'regex' in malchecks[check]:\n rx = malchecks[check]['regex'].replace(\"{0}\", r\"(\\d+\\.\\d+\\.\\d+\\.\\d+)\")\n pat = re.compile(rx, re.IGNORECASE)\n self.debug(\"New regex for \" + check + \": \" + rx)\n for line in data['content'].split('\\n'):\n grp = re.findall(pat, line)\n if len(grp) > 0:\n # self.debug(\"Adding \" + grp[0] + \" to list.\")\n iplist.append(grp[0])\n else:\n iplist = data['content'].split('\\n')\n\n for ip in iplist:\n if len(ip) < 8 or ip.startswith(\"#\"):\n continue\n ip = ip.strip()\n\n try:\n if IPAddress(ip) in IPNetwork(target):\n self.debug(f\"{ip} found within netblock/subnet {target} in {check}\")\n return url\n except Exception as e:\n self.debug(f\"Error encountered parsing: {e}\")\n continue\n\n return None\n\n # If we're looking at hostnames/domains/IPs\n if 'regex' not in malchecks[check]:\n for line in data['content'].split('\\n'):\n if line == target or (targetType == \"domain\" and line == targetDom):\n self.debug(target + \"/\" + targetDom + \" found in \" + check + \" list.\")\n return url\n else:\n # Check for the domain and the hostname\n try:\n rxDom = str(malchecks[check]['regex']).format(targetDom)\n rxTgt = str(malchecks[check]['regex']).format(target)\n for line in data['content'].split('\\n'):\n if (targetType == \"domain\" and re.match(rxDom, line, re.IGNORECASE)) or \\\n re.match(rxTgt, line, re.IGNORECASE):\n self.debug(target + \"/\" + targetDom + \" found in \" + check + \" list.\")\n return url\n except Exception as e:\n self.debug(\"Error encountered parsing 2: \" + str(e))\n continue\n\n return None\n\n def lookupItem(self, resourceId, itemType, target):\n for check in list(malchecks.keys()):\n cid = malchecks[check]['id']\n if cid == resourceId and itemType in malchecks[check]['checks']:\n self.debug(\"Checking maliciousness of \" + target + \" (\"\n + itemType + \") with: \" + cid)\n return self.resourceList(cid, target, itemType)\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.errorState:\n return\n\n if self.opts['url'] == \"\":\n self.error(\"You enabled sfp_customfeed but defined no custom feed URL!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):\n return\n if eventName == 'AFFILIATE_IPADDR' \\\n and not self.opts.get('checkaffiliates', False):\n return\n if eventName == 'NETBLOCK_OWNER' and not self.opts.get('checknetblocks', False):\n return\n if eventName == 'NETBLOCK_MEMBER' and not self.opts.get('checksubnets', False):\n return\n\n for check in list(malchecks.keys()):\n cid = malchecks[check]['id']\n\n if eventName in ['IP_ADDRESS', 'AFFILIATE_IPADDR']:\n typeId = 'ip'\n if eventName == 'IP_ADDRESS':\n evtType = 'MALICIOUS_IPADDR'\n else:\n evtType = 'MALICIOUS_AFFILIATE_IPADDR'\n\n if eventName in ['BGP_AS_OWNER', 'BGP_AS_MEMBER']:\n typeId = 'asn'\n evtType = 'MALICIOUS_ASN'\n\n if eventName in ['INTERNET_NAME', 'CO_HOSTED_SITE',\n 'AFFILIATE_INTERNET_NAME', ]:\n typeId = 'domain'\n if eventName == \"INTERNET_NAME\":\n evtType = \"MALICIOUS_INTERNET_NAME\"\n if eventName == 'AFFILIATE_INTERNET_NAME':\n evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'\n if eventName == 'CO_HOSTED_SITE':\n evtType = 'MALICIOUS_COHOST'\n\n if eventName == 'NETBLOCK_OWNER':\n typeId = 'netblock'\n evtType = 'MALICIOUS_NETBLOCK'\n if eventName == 'NETBLOCK_MEMBER':\n typeId = 'netblock'\n evtType = 'MALICIOUS_SUBNET'\n\n url = self.lookupItem(cid, typeId, eventData)\n\n if self.checkForStop():\n return\n\n # Notify other modules of what you've found\n if url is not None:\n text = f\"{check} [{eventData}]\\n{url}\"\n evt = SpiderFootEvent(evtType, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_customfeed class\n" }, { "path": "modules/sfp_cybercrimetracker.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_cybercrimetracker\n# Purpose: Check if a host/domain or IP address is malicious according to cybercrime-tracker.net.\n#\n# Author: steve@binarypool.com\n#\n# Created: 14/12/2013\n# Copyright: (c) Steve Micallef, 2013\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_cybercrimetracker(SpiderFootPlugin):\n\n meta = {\n 'name': \"CyberCrime-Tracker.net\",\n 'summary': \"Check if a host/domain or IP address is malicious according to CyberCrime-Tracker.net.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://cybercrime-tracker.net/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://cybercrime-tracker.net/tools.php\",\n \"https://cybercrime-tracker.net/about.php\"\n ],\n 'favIcon': \"https://cybercrime-tracker.net/favicon.ico\",\n 'logo': \"https://cybercrime-tracker.net/favicon.ico\",\n 'description': \"CyberCrime is a C&C panel tracker, in other words, \"\n \"it lists the administration interfaces of certain in-the-wild botnets.\",\n }\n }\n\n # Default options\n opts = {\n 'checkaffiliates': True,\n 'checkcohosts': True,\n 'cacheperiod': 18\n }\n\n # Option descriptions\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliates?\",\n 'checkcohosts': \"Apply checks to sites found to be co-hosted on the target's IP?\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"IP_ADDRESS\",\n \"AFFILIATE_INTERNET_NAME\",\n \"AFFILIATE_IPADDR\",\n \"CO_HOSTED_SITE\"\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_INTERNET_NAME\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"BLACKLISTED_AFFILIATE_INTERNET_NAME\",\n \"BLACKLISTED_COHOST\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_COHOST\"\n ]\n\n def queryBlacklist(self, target):\n blacklist = self.retrieveBlacklist()\n\n if not blacklist:\n return False\n\n if target.lower() in blacklist:\n self.debug(f\"Host name {target} found in CyberCrime-Tracker.net blacklist.\")\n return True\n\n return False\n\n def retrieveBlacklist(self):\n blacklist = self.sf.cacheGet('cybercrime-tracker', 24)\n\n if blacklist is not None:\n return self.parseBlacklist(blacklist)\n\n res = self.sf.fetchUrl(\n \"https://cybercrime-tracker.net/all.php\",\n timeout=10,\n useragent=self.opts['_useragent'],\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from CyberCrime-Tracker.net.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from CyberCrime-Tracker.net\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"cybercrime-tracker\", res['content'])\n\n return self.parseBlacklist(res['content'])\n\n def parseBlacklist(self, blacklist):\n \"\"\"Parse plaintext blacklist\n\n Args:\n blacklist (str): plaintext blacklist from CyberCrime-Tracker.net\n\n Returns:\n list: list of blacklisted IP addresses and host names\n \"\"\"\n hosts = list()\n\n if not blacklist:\n return hosts\n\n for line in blacklist.split('\\n'):\n if not line:\n continue\n if line.startswith('#'):\n continue\n\n # Note: URL parsing and validation with sf.validHost() is too slow to use here\n host = line.split(\"/\")[0]\n if not host:\n continue\n if \".\" not in host:\n continue\n hosts.append(host.split(':')[0])\n\n return hosts\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName == 'IP_ADDRESS':\n malicious_type = \"MALICIOUS_IPADDR\"\n blacklist_type = \"BLACKLISTED_IPADDR\"\n elif eventName == 'AFFILIATE_IPADDR':\n if not self.opts.get('checkaffiliates', False):\n return\n malicious_type = \"MALICIOUS_AFFILIATE_IPADDR\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_IPADDR\"\n elif eventName == \"INTERNET_NAME\":\n malicious_type = \"MALICIOUS_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n if not self.opts.get('checkaffiliates', False):\n return\n malicious_type = \"MALICIOUS_AFFILIATE_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n if not self.opts.get('checkcohosts', False):\n return\n malicious_type = \"MALICIOUS_COHOST\"\n blacklist_type = \"BLACKLISTED_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n self.debug(f\"Checking maliciousness of {eventData} ({eventName}) with CyberCrime-Tracker.net\")\n\n if not self.queryBlacklist(eventData):\n return\n\n url = f\"https://cybercrime-tracker.net/index.php?search={eventData}\"\n text = f\"CyberCrime-Tracker.net Malicious Submissions [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_cybercrimetracker class\n" }, { "path": "modules/sfp_debounce.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_debounce\n# Purpose: Spiderfoot plugin to check if an email is\n# disposable using Debounce API.\n#\n# Author: Krishnasis Mandal \n#\n# Created: 2020-10-01\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_debounce(SpiderFootPlugin):\n\n meta = {\n 'name': \"Debounce\",\n 'summary': \"Check whether an email is disposable\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://debounce.io/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://debounce.io/free-disposable-check-api/\"\n ],\n 'favIcon': \"https://debounce.io/wp-content/uploads/2018/01/favicon-2.png\",\n 'logo': \"https://debounce.io/wp-content/uploads/2018/01/debounce-logo-2.png\",\n 'description': \"DeBounce provides a free & powerful API endpoint for checking \"\n \"a domain or email address against a realtime up-to-date list of disposable domains.\"\n \"CORS is enabled for all originating domains, \"\n \"so you can call the API directly from your client-side code.\",\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"EMAILADDR\"\n ]\n\n def producedEvents(self):\n return [\n \"EMAILADDR_DISPOSABLE\",\n \"RAW_RIR_DATA\"\n ]\n\n def queryEmailAddr(self, qry):\n res = self.sf.fetchUrl(\n f\"https://disposable.debounce.io?email={qry}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\"\n )\n\n if res['content'] is None:\n self.info(f\"No Debounce info found for {qry}\")\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from Debounce: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n self.results[eventData] = True\n\n data = self.queryEmailAddr(eventData)\n\n if data is None:\n return\n\n isDisposable = data.get('disposable')\n\n if isDisposable == \"true\":\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(\"EMAILADDR_DISPOSABLE\", eventData, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_debounce class\n" }, { "path": "modules/sfp_dehashed.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_dehashed\n# Purpose: Gather breach data from Dehashed API.\n#\n# Author: \n#\n# Created: 16-01-2021\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport base64\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_dehashed(SpiderFootPlugin):\n\n meta = {\n 'name': \"Dehashed\",\n 'summary': \"Gather breach data from Dehashed API.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Leaks, Dumps and Breaches\"],\n 'dataSource': {\n 'website': \"https://www.dehashed.com/\",\n 'model': \"COMMERCIAL_ONLY\",\n 'references': [\n \"https://www.dehashed.com/docs\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://www.dehashed.com/register\"\n \"Register a free account\",\n \"Visit https://www.dehashed.com/profile\",\n \"Your API key is listed under 'API Key'\",\n ],\n 'favIcon': \"https://www.dehashed.com/assets/img/favicon.ico\",\n 'logo': \"https://www.dehashed.com/assets/img/logo.png\",\n 'description': \"Have you been compromised? \"\n \"DeHashed provides free deep-web scans and protection against credential leaks. \"\n \"A modern personal asset search engine created for \"\n \"security analysts, journalists, security companies, \"\n \"and everyday people to help secure accounts and provide insight on compromised assets. \"\n \"Free breach alerts & breach notifications.\",\n }\n }\n\n # Default options\n opts = {\n 'api_key_username': '',\n 'api_key': '',\n 'per_page': 10000,\n 'max_pages': 2,\n 'pause': 1\n }\n\n # Option descriptions\n optdescs = {\n 'api_key_username': 'Dehashed username.',\n 'api_key': 'Dehashed API key.',\n 'per_page': 'Maximum number of results per page.(Max: 10000)',\n 'max_pages': 'Maximum number of pages to fetch(Max: 10 pages)',\n 'pause': 'Number of seconds to wait between each API call.'\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\n \"DOMAIN_NAME\",\n \"EMAILADDR\"\n ]\n\n # What events this module produces\n def producedEvents(self):\n return [\n 'EMAILADDR',\n 'EMAILADDR_COMPROMISED',\n 'PASSWORD_COMPROMISED',\n 'HASH_COMPROMISED',\n 'RAW_RIR_DATA'\n ]\n\n # Query Dehashed\n def query(self, event, per_page, start):\n if event.eventType == \"EMAILADDR\":\n queryString = f\"https://api.dehashed.com/search?query=email:\\\"{event.data}\\\"&page={start}&size={self.opts['per_page']}\"\n if event.eventType == \"DOMAIN_NAME\":\n queryString = f\"https://api.dehashed.com/search?query=email:\\\"@{event.data}\\\"&page={start}&size={self.opts['per_page']}\"\n\n token = (base64.b64encode(self.opts['api_key_username'].encode('utf8') + \":\".encode('utf-8') + self.opts['api_key'].encode('utf-8'))).decode('utf-8')\n headers = {\n 'Accept': 'application/json',\n 'Authorization': f'Basic {token}'\n }\n\n res = self.sf.fetchUrl(queryString,\n headers=headers,\n timeout=15,\n useragent=self.opts['_useragent'],\n verify=True)\n\n time.sleep(self.opts['pause'])\n\n if res['code'] == \"400\":\n self.error(\"Too many requests were performed in a small amount of time. Please wait a bit before querying the API.\")\n time.sleep(5)\n res = self.sf.fetchUrl(queryString, headers=headers, timeout=15, useragent=self.opts['_useragent'], verify=True)\n\n if res['code'] == \"401\":\n self.error(\"Invalid API credentials\")\n self.errorState = True\n return None\n\n if res['code'] != \"200\":\n self.error(\"Unable to fetch data from Dehashed.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.debug('No response from Dehashed')\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if srcModuleName == self.__name__:\n return\n\n if eventData in self.results:\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts['api_key'] == \"\" or self.opts['api_key_username'] == \"\":\n self.error(\"You enabled sfp_dehashed but did not set an API key/API Key Username!\")\n self.errorState = True\n return\n\n currentPage = 1\n maxPages = self.opts['max_pages']\n perPage = self.opts['per_page']\n\n while currentPage <= maxPages:\n if self.checkForStop():\n return\n\n if self.errorState:\n break\n\n data = self.query(event, perPage, currentPage)\n\n if not data:\n return\n\n breachResults = set()\n emailResults = set()\n\n if not data.get('entries'):\n return\n\n for row in data.get('entries'):\n email = row.get('email')\n password = row.get('password')\n passwordHash = row.get('hashed_password')\n leakSource = row.get('database_name', 'Unknown')\n\n if f\"{email} [{leakSource}]\" in breachResults:\n continue\n\n breachResults.add(f\"{email} [{leakSource}]\")\n\n if eventName == \"EMAILADDR\":\n if email == eventData:\n evt = SpiderFootEvent('EMAILADDR_COMPROMISED', f\"{email} [{leakSource}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n if password:\n evt = SpiderFootEvent('PASSWORD_COMPROMISED', f\"{email}:{password} [{leakSource}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n if passwordHash:\n evt = SpiderFootEvent('HASH_COMPROMISED', f\"{email}:{passwordHash} [{leakSource}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(row), self.__name__, event)\n self.notifyListeners(evt)\n\n if eventName == \"DOMAIN_NAME\":\n pevent = SpiderFootEvent(\"EMAILADDR\", email, self.__name__, event)\n if email not in emailResults:\n self.notifyListeners(pevent)\n emailResults.add(email)\n\n evt = SpiderFootEvent('EMAILADDR_COMPROMISED', f\"{email} [{leakSource}]\", self.__name__, pevent)\n self.notifyListeners(evt)\n\n if password:\n evt = SpiderFootEvent('PASSWORD_COMPROMISED', f\"{email}:{password} [{leakSource}]\", self.__name__, pevent)\n self.notifyListeners(evt)\n\n if passwordHash:\n evt = SpiderFootEvent('HASH_COMPROMISED', f\"{email}:{passwordHash} [{leakSource}]\", self.__name__, pevent)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(row), self.__name__, pevent)\n self.notifyListeners(evt)\n\n currentPage += 1\n\n if data.get('total') < self.opts['per_page']:\n break\n\n# End of sfp_dehashed class\n" }, { "path": "modules/sfp_digitaloceanspace.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_digitaloceanspace\n# Purpose: SpiderFoot plug-in for identifying potential Digital Ocean spaces\n# related to the target.\n#\n# Author: Steve Micallef \n#\n# Created: 16/06/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport random\nimport threading\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_digitaloceanspace(SpiderFootPlugin):\n\n meta = {\n 'name': \"Digital Ocean Space Finder\",\n 'summary': \"Search for potential Digital Ocean Spaces associated with the target and attempt to list their contents.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Passive\"],\n 'categories': [\"Crawling and Scanning\"],\n 'dataSource': {\n 'website': \"https://www.digitalocean.com/products/spaces/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'favIcon': 'https://www.digitalocean.com/_next/static/media/favicon-32x32.b7ef9ede.png',\n 'logo': 'https://www.digitalocean.com/_next/static/media/logo.87a8f3b8.svg',\n 'description': \"Store and deliver vast amounts of content.\"\n \"S3-compatible object storage with a built-in CDN that makes scaling easy, reliable, and affordable.\"\n }\n }\n\n # Default options\n opts = {\n \"endpoints\": \"nyc3.digitaloceanspaces.com,sgp1.digitaloceanspaces.com,ams3.digitaloceanspaces.com\",\n \"suffixes\": \"test,dev,web,beta,bucket,space,files,content,data,prod,staging,production,stage,app,media,development,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data,-prod,-staging,-production,-stage,-app,-media,-development\",\n \"_maxthreads\": 20\n }\n\n # Option descriptions\n optdescs = {\n \"endpoints\": \"Different Digital Ocean locations to check where spaces may exist.\",\n \"suffixes\": \"List of suffixes to append to domains tried as space names\",\n \"_maxthreads\": \"Maximum threads\"\n }\n\n results = None\n s3results = dict()\n lock = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.s3results = dict()\n self.results = self.tempStorage()\n self.lock = threading.Lock()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\", \"LINKED_URL_EXTERNAL\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"CLOUD_STORAGE_BUCKET\", \"CLOUD_STORAGE_BUCKET_OPEN\"]\n\n def checkSite(self, url):\n res = self.sf.fetchUrl(url, timeout=10, useragent=\"SpiderFoot\", noLog=True)\n\n if not res['content']:\n return\n\n if \"NoSuchBucket\" in res['content']:\n self.debug(f\"Not a valid bucket: {url}\")\n return\n\n # Bucket found\n if res['code'] in [\"301\", \"302\", \"200\"]:\n # Bucket has files\n if \"ListBucketResult\" in res['content']:\n with self.lock:\n self.s3results[url] = res['content'].count(\"\")\n else:\n # Bucket has no files\n with self.lock:\n self.s3results[url] = 0\n\n def threadSites(self, siteList):\n self.s3results = dict()\n running = True\n i = 0\n t = []\n\n for site in siteList:\n if self.checkForStop():\n return None\n\n self.info(\"Spawning thread to check bucket: \" + site)\n tname = str(random.SystemRandom().randint(0, 999999999))\n t.append(threading.Thread(name='thread_sfp_digitaloceanspaces_' + tname,\n target=self.checkSite, args=(site,)))\n t[i].start()\n i += 1\n\n # Block until all threads are finished\n while running:\n found = False\n for rt in threading.enumerate():\n if rt.name.startswith(\"thread_sfp_digitaloceanspaces_\"):\n found = True\n\n if not found:\n running = False\n\n time.sleep(0.25)\n\n # Return once the scanning has completed\n return self.s3results\n\n def batchSites(self, sites):\n i = 0\n res = list()\n siteList = list()\n\n for site in sites:\n if i >= self.opts['_maxthreads']:\n data = self.threadSites(siteList)\n if data is None:\n return res\n\n for ret in list(data.keys()):\n if data[ret]:\n # bucket:filecount\n res.append(ret + \":\" + str(data[ret]))\n i = 0\n siteList = list()\n\n siteList.append(site)\n i += 1\n\n return res\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventName == \"LINKED_URL_EXTERNAL\":\n if \".digitaloceanspaces.com\" in eventData:\n b = self.sf.urlFQDN(eventData)\n evt = SpiderFootEvent(\"CLOUD_STORAGE_BUCKET\", b, self.__name__, event)\n self.notifyListeners(evt)\n return\n\n targets = [eventData.replace('.', '')]\n kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])\n if kw:\n targets.append(kw)\n\n urls = list()\n for t in targets:\n for e in self.opts['endpoints'].split(','):\n suffixes = [''] + self.opts['suffixes'].split(',')\n for s in suffixes:\n if self.checkForStop():\n return\n\n b = t + s + \".\" + e\n url = \"https://\" + b\n urls.append(url)\n\n # Batch the scans\n ret = self.batchSites(urls)\n for b in ret:\n bucket = b.split(\":\")\n evt = SpiderFootEvent(\"CLOUD_STORAGE_BUCKET\", bucket[0] + \":\" + bucket[1], self.__name__, event)\n self.notifyListeners(evt)\n if bucket[2] != \"0\":\n evt = SpiderFootEvent(\"CLOUD_STORAGE_BUCKET_OPEN\", bucket[0] + \":\" + bucket[1] + \": \" + bucket[2] + \" files found.\",\n self.__name__, evt)\n self.notifyListeners(evt)\n\n\n# End of sfp_digitaloceanspace class\n" }, { "path": "modules/sfp_dns_for_family.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_dns_for_family\n# Purpose: Check if a host would be blocked by DNS for Family.\n#\n# Author: \n#\n# Created: 2021-10-11\n# Copyright: (c) bcoles 2021\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport dns.resolver\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_dns_for_family(SpiderFootPlugin):\n\n meta = {\n 'name': \"DNS for Family\",\n 'summary': \"Check if a host would be blocked by DNS for Family.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://dnsforfamily.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://dnsforfamily.com/\",\n ],\n 'favIcon': \"https://dnsforfamily.com/Icons/favicon-32x32.png\",\n 'logo': \"https://dnsforfamily.com/Templates/assets/images/logo.svg\",\n 'description': \"DNS for Family aims to block websites which are considered as porn. \"\n \"So that you and your children can safely surf internet to their heart's desire, \"\n \"without worrying to be followed to harmful websites.\"\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"AFFILIATE_INTERNET_NAME\",\n \"CO_HOSTED_SITE\"\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_INTERNET_NAME\",\n \"BLACKLISTED_AFFILIATE_INTERNET_NAME\",\n \"BLACKLISTED_COHOST\",\n ]\n\n def queryAddr(self, qaddr):\n if not qaddr:\n return None\n\n res = dns.resolver.Resolver()\n res.nameservers = [\"94.130.180.225\", \"78.47.64.161\"]\n\n try:\n return res.resolve(qaddr)\n except Exception:\n self.debug(f\"Unable to resolve {qaddr}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if eventName == \"INTERNET_NAME\":\n blacklist_type = \"BLACKLISTED_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n blacklist_type = \"BLACKLISTED_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n blacklist_type = \"BLACKLISTED_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n res = self.queryAddr(eventData)\n\n if not res:\n return\n\n for result in res:\n k = str(result)\n if k != '159.69.10.249':\n continue\n\n self.debug(f\"{eventData} blocked by DNS for Family\")\n evt = SpiderFootEvent(blacklist_type, f\"DNS for Family [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_dns_for_family class\n" }, { "path": "modules/sfp_dnsbrute.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_dnsbrute\n# Purpose: SpiderFoot plug-in for attempting to resolve through brute-forcing\n# common hostnames.\n#\n# Author: Steve Micallef \n#\n# Created: 06/07/2017\n# Copyright: (c) Steve Micallef 2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport importlib\nimport random\nimport threading\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_dnsbrute(SpiderFootPlugin):\n\n meta = {\n 'name': \"DNS Brute-forcer\",\n 'summary': \"Attempts to identify hostnames through brute-forcing common names and iterations.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\"],\n 'categories': [\"DNS\"]\n }\n\n # Default options\n opts = {\n \"skipcommonwildcard\": True,\n \"domainonly\": True,\n \"commons\": True,\n \"top10000\": False,\n \"numbersuffix\": True,\n \"numbersuffixlimit\": True,\n \"_maxthreads\": 100\n }\n\n # Option descriptions\n optdescs = {\n 'skipcommonwildcard': \"If wildcard DNS is detected, don't bother brute-forcing.\",\n 'domainonly': \"Only attempt to brute-force names on domain names, not hostnames (some hostnames are also sub-domains).\",\n 'commons': \"Try a list of about 750 common hostnames/sub-domains.\",\n 'top10000': \"Try a further 10,000 common hostnames/sub-domains. Will make the scan much slower.\",\n 'numbersuffix': \"For any host found, try appending 1, 01, 001, -1, -01, -001, 2, 02, etc. (up to 10)\",\n 'numbersuffixlimit': \"Limit using the number suffixes for hosts that have already been resolved? If disabled this will significantly extend the duration of scans.\",\n \"_maxthreads\": \"Maximum threads\"\n }\n\n events = None\n sublist = None\n lock = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.sublist = self.tempStorage()\n self.events = self.tempStorage()\n self.__dataSource__ = \"DNS\"\n self.lock = threading.Lock()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n if self.opts['commons']:\n with importlib.resources.open_text('spiderfoot.dicts', 'subdomains.txt') as f:\n for s in f.readlines():\n s = s.strip()\n self.sublist[s] = True\n\n if self.opts['top10000']:\n with importlib.resources.open_text('spiderfoot.dicts', 'subdomains-10000.txt') as f:\n for s in f.readlines():\n s = s.strip()\n self.sublist[s] = True\n\n # What events is this module interested in for input\n def watchedEvents(self):\n ret = ['DOMAIN_NAME']\n if not self.opts['domainonly'] or self.opts['numbersuffix']:\n ret.append('INTERNET_NAME')\n return ret\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"INTERNET_NAME\"]\n\n def tryHost(self, name):\n try:\n if self.sf.resolveHost(name) or self.sf.resolveHost6(name):\n with self.lock:\n self.hostResults[name] = True\n except Exception:\n with self.lock:\n self.hostResults[name] = False\n\n def tryHostWrapper(self, hostList, sourceEvent):\n self.hostResults = dict()\n running = True\n i = 0\n t = []\n\n # Spawn threads for scanning\n self.info(\"Spawning threads to check hosts: \" + str(hostList))\n for name in hostList:\n tn = 'thread_sfp_dnsbrute_' + str(random.SystemRandom().randint(1, 999999999))\n t.append(threading.Thread(name=tn, target=self.tryHost, args=(name,)))\n t[i].start()\n i += 1\n\n # Block until all threads are finished\n while running:\n found = False\n for rt in threading.enumerate():\n if rt.name.startswith(\"thread_sfp_dnsbrute_\"):\n found = True\n\n if not found:\n running = False\n\n time.sleep(0.05)\n\n for res in self.hostResults:\n if self.hostResults.get(res, False):\n self.sendEvent(sourceEvent, res)\n\n # Store the result internally and notify listening modules\n def sendEvent(self, source, result):\n self.info(\"Found a brute-forced host: \" + result)\n # Report the host\n evt = SpiderFootEvent(\"INTERNET_NAME\", result, self.__name__, source)\n self.notifyListeners(evt)\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n eventDataHash = self.sf.hashstring(eventData)\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if srcModuleName == \"sfp_dnsbrute\":\n return\n\n if eventDataHash in self.events:\n return\n self.events[eventDataHash] = True\n\n if eventName == \"INTERNET_NAME\" and not self.getTarget().matches(eventData, includeChildren=False):\n if not self.opts['numbersuffix']:\n return\n\n if self.checkForStop():\n return\n\n h, dom = eventData.split(\".\", 1)\n\n # Try resolving common names\n wildcard = self.sf.checkDnsWildcard(dom)\n if self.opts['skipcommonwildcard'] and wildcard:\n self.debug(\"Wildcard DNS detected on \" + dom + \" so skipping host iteration.\")\n return\n\n dom = \".\" + dom\n nextsubs = dict()\n for i in range(10):\n nextsubs[h + str(i) + dom] = True\n nextsubs[h + \"0\" + str(i) + dom] = True\n nextsubs[h + \"00\" + str(i) + dom] = True\n nextsubs[h + \"-\" + str(i) + dom] = True\n nextsubs[h + \"-0\" + str(i) + dom] = True\n nextsubs[h + \"-00\" + str(i) + dom] = True\n\n self.tryHostWrapper(list(nextsubs.keys()), event)\n\n # The rest of the module is for handling targets only\n return\n\n # Only for the target, from this point forward...\n if not self.getTarget().matches(eventData, includeChildren=False):\n return\n\n # Try resolving common names\n self.debug(\"Iterating through possible sub-domains.\")\n wildcard = self.sf.checkDnsWildcard(eventData)\n if self.opts['skipcommonwildcard'] and wildcard:\n self.debug(\"Wildcard DNS detected.\")\n return\n\n targetList = list()\n for sub in self.sublist:\n if self.checkForStop():\n return\n\n name = f\"{sub}.{eventData}\"\n\n if len(targetList) <= self.opts['_maxthreads']:\n targetList.append(name)\n else:\n self.tryHostWrapper(targetList, event)\n targetList = list()\n\n # Scan whatever may be left over.\n if len(targetList) > 0:\n self.tryHostWrapper(targetList, event)\n\n if self.opts['numbersuffix'] and not self.opts['numbersuffixlimit']:\n nextsubs = dict()\n dom = \".\" + eventData\n for s in self.sublist:\n if self.checkForStop():\n return\n\n for i in range(10):\n nextsubs[s + str(i) + dom] = True\n nextsubs[s + \"0\" + str(i) + dom] = True\n nextsubs[s + \"00\" + str(i) + dom] = True\n nextsubs[s + \"-\" + str(i) + dom] = True\n nextsubs[s + \"-0\" + str(i) + dom] = True\n nextsubs[s + \"-00\" + str(i) + dom] = True\n\n if len(list(nextsubs.keys())) >= self.opts['_maxthreads']:\n self.tryHostWrapper(list(nextsubs.keys()), event)\n nextsubs = dict()\n\n # Scan whatever may be left over.\n if len(nextsubs) > 0:\n self.tryHostWrapper(list(nextsubs.keys()), event)\n\n\n# End of sfp_dnsbrute class\n" }, { "path": "modules/sfp_dnscommonsrv.py", "content": "# -*- coding: utf-8 -*-\n# -----------------------------------------------------------------------------\n# Name: sfp_dnscommonsrv\n# Purpose: SpiderFoot plug-in for attempting to resolve through\n# brute-forcing common DNS SRV records.\n#\n# Author: Michael Scherer \n#\n# Created: 22/08/2017\n# Copyright: (c) Michael Scherer 2017\n# Licence: MIT\n# -----------------------------------------------------------------------------\n\nimport dns.resolver\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_dnscommonsrv(SpiderFootPlugin):\n\n meta = {\n 'name': \"DNS Common SRV\",\n 'summary': \"Attempts to identify hostnames through brute-forcing common DNS SRV records.\",\n 'flags': [\"slow\"],\n 'useCases': [\"Footprint\", \"Investigate\"],\n 'categories': [\"DNS\"]\n }\n\n opts = {}\n\n optdescs = {}\n\n events = None\n\n commonsrv = [\n # LDAP/Kerberos, used for Active Directory\n # https://technet.microsoft.com/en-us/library/cc961719.aspx\n '_ldap._tcp',\n '_gc._msdcs',\n '_ldap._tcp.pdc._msdcs',\n '_ldap._tcp.gc._msdcs',\n '_kerberos._tcp.dc._msdcs',\n '_kerberos._tcp',\n '_kerberos._udp',\n '_kerberos-master._tcp',\n '_kerberos-master._udp',\n '_kpasswd._tcp',\n '_kpasswd._udp',\n '_ntp._udp',\n\n # SIP\n '_sip._tcp',\n '_sip._udp',\n '_sip._tls',\n '_sips._tcp',\n\n # STUN\n # https://tools.ietf.org/html/rfc5389\n '_stun._tcp',\n '_stun._udp',\n '_stuns._tcp',\n\n # TURN\n # https://tools.ietf.org/html/rfc5928\n '_turn._tcp',\n '_turn._udp',\n '_turns._tcp',\n\n # XMPP\n # http://xmpp.org/rfcs/rfc6120.html\n '_jabber._tcp',\n '_xmpp-client._tcp',\n '_xmpp-server._tcp'\n ]\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.events = self.tempStorage()\n self.__dataSource__ = \"DNS\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return ['INTERNET_NAME', 'DOMAIN_NAME']\n\n def producedEvents(self):\n return [\"INTERNET_NAME\", \"AFFILIATE_INTERNET_NAME\"]\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if srcModuleName == \"sfp_dnscommonsrv\":\n self.debug(f\"Ignoring {eventName}, from self.\")\n return\n\n eventDataHash = self.sf.hashstring(eventData)\n parentEvent = event\n\n if eventDataHash in self.events:\n return\n\n self.events[eventDataHash] = True\n\n res = dns.resolver.Resolver()\n if self.opts.get('_dnsserver', \"\") != \"\":\n res.nameservers = [self.opts['_dnsserver']]\n\n self.debug(\"Iterating through possible SRV records.\")\n\n # Try resolving common names\n for srv in self.commonsrv:\n if self.checkForStop():\n return\n\n name = srv + \".\" + eventData\n\n # Skip hosts we've processed already\n if self.sf.hashstring(name) in self.events:\n continue\n\n try:\n answers = res.query(name, 'SRV', timeout=10)\n except Exception:\n answers = []\n\n if not answers:\n continue\n\n evt = SpiderFootEvent(\n \"DNS_SRV\",\n name,\n self.__name__,\n parentEvent\n )\n self.notifyListeners(evt)\n\n for a in answers:\n # Strip off the trailing .\n tgt_clean = a.target.to_text().rstrip(\".\")\n\n if self.getTarget().matches(tgt_clean):\n evt_type = \"INTERNET_NAME\"\n else:\n evt_type = \"AFFILIATE_INTERNET_NAME\"\n\n evt = SpiderFootEvent(\n evt_type,\n tgt_clean,\n self.__name__,\n parentEvent\n )\n self.notifyListeners(evt)\n\n# End of sfp_dnscommonsrv class\n" }, { "path": "modules/sfp_dnsdb.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_dnsdb\n# Purpose: SpiderFoot plug-in that resolves and gets history of domains and IPs\n#\n# Author: Filip Aleksić \n#\n# Created: 2020-09-09\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport re\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_dnsdb(SpiderFootPlugin):\n meta = {\n \"name\": \"DNSDB\",\n \"summary\": \"Query FarSight's DNSDB for historical and passive DNS data.\",\n 'flags': [\"apikey\"],\n \"useCases\": [\"Passive\", \"Footprint\", \"Investigate\"],\n \"categories\": [\"Passive DNS\"],\n \"dataSource\": {\n \"website\": \"https://www.farsightsecurity.com\",\n \"model\": \"FREE_AUTH_LIMITED\",\n \"references\": [\n \"https://docs.dnsdb.info/dnsdb-apiv2/\",\n \"https://www.farsightsecurity.com/get-started/\"\n \"https://www.farsightsecurity.com/solutions/dnsdb/\",\n ],\n \"apiKeyInstructions\": [\n \"Visit https://www.farsightsecurity.com/get-started/\",\n \"Select the model that best fit your needs (free or premium)\",\n \"Fill in the form to get API key\",\n \"Check your email for your API Key \",\n ],\n \"favIcon\": \"https://www.farsightsecurity.com/favicon.ico\",\n \"logo\": \"https://www.farsightsecurity.com/assets/media/svg/farsight-logo.svg\",\n \"description\": \"Farsight Security’s DNSDB is the world’s largest \"\n \"database of DNS resolution and change data. Started in 2010 and \"\n \"updated in real-time, DNSDB provides the most comprehensive \"\n \"history of domains and IP addresses worldwide.\",\n },\n }\n\n opts = {\n \"api_key\": \"\",\n \"age_limit_days\": 0,\n \"verify\": True,\n \"cohostsamedomain\": False,\n \"maxcohost\": 100,\n }\n\n optdescs = {\n \"api_key\": \"DNSDB API Key.\",\n \"age_limit_days\": \"Ignore any DNSDB records older than this many days. 0 = unlimited.\",\n \"verify\": \"Verify co-hosts are valid by checking if they still resolve to the shared IP.\",\n \"cohostsamedomain\": \"Treat co-hosted sites on the same target domain as co-hosting?\",\n \"maxcohost\": \"Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.\",\n }\n\n results = None\n errorState = False\n cohostcount = 0\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.cohostcount = 0\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"IP_ADDRESS\", \"IPV6_ADDRESS\", \"DOMAIN_NAME\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\n \"RAW_RIR_DATA\",\n \"INTERNET_NAME\",\n \"INTERNET_NAME_UNRESOLVED\",\n \"PROVIDER_DNS\",\n \"DNS_TEXT\",\n \"PROVIDER_MAIL\",\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\",\n \"CO_HOSTED_SITE\",\n ]\n\n def query(self, endpoint, queryType, query):\n if endpoint not in (\"rrset\", \"rdata\"):\n self.error(f\"Endpoint MUST be rrset or rdata, you sent {endpoint}\")\n return None\n\n if queryType not in (\"name\", \"ip\"):\n self.error(f\"Query type MUST be name or ip, you sent {queryType}\")\n return None\n\n headers = {\"Accept\": \"application/x-ndjson\", \"X-API-Key\": self.opts[\"api_key\"]}\n\n res = self.sf.fetchUrl(\n f\"https://api.dnsdb.info/dnsdb/v2/lookup/{endpoint}/{queryType}/{query}\",\n timeout=30,\n useragent=\"SpiderFoot\",\n headers=headers,\n )\n\n if res[\"code\"] == \"429\":\n self.error(\"You are being rate-limited by DNSDB\")\n self.errorState = True\n return None\n\n if res[\"content\"] is None:\n self.info(f\"No DNSDB record found for {query}\")\n return None\n\n splittedContent = res[\"content\"].strip().split(\"\\n\")\n if len(splittedContent) == 2:\n self.info(f\"No DNSDB record found for {query}\")\n return None\n\n if len(splittedContent) < 2:\n self.info(f\"Unexpected DNSDB response {query}\")\n return None\n\n try:\n records = []\n for content in splittedContent:\n records.append(json.loads(content))\n except json.JSONDecodeError as e:\n self.error(f\"Error processing JSON response from DNSDB: {e}\")\n return None\n\n return records[1:-1]\n\n def isTooOld(self, lastSeen):\n ageLimitTs = int(time.time()) - (86400 * self.opts[\"age_limit_days\"])\n if self.opts[\"age_limit_days\"] > 0 and lastSeen < ageLimitTs:\n self.debug(\"Record found but too old, skipping.\")\n return True\n return False\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\"You enabled sfp_dnsdb but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n self.results[eventData] = True\n\n responseData = set()\n coHosts = set()\n\n if eventName == \"DOMAIN_NAME\":\n rrsetRecords = self.query(\"rrset\", \"name\", eventData)\n if rrsetRecords is None:\n return\n\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(rrsetRecords), self.__name__, event)\n self.notifyListeners(evt)\n\n for record in rrsetRecords:\n record = record.get(\"obj\")\n if self.checkForStop():\n return\n\n if self.isTooOld(record.get(\"time_last\", 0)):\n continue\n\n if record.get(\"rrtype\") not in (\n \"A\",\n \"AAAA\",\n \"MX\",\n \"NS\",\n \"TXT\",\n \"CNAME\",\n ):\n continue\n\n for data in record.get(\"rdata\"):\n data = data.rstrip(\".\")\n if data in responseData:\n continue\n responseData.add(data)\n\n if record.get(\"rrtype\") == \"A\":\n if not self.sf.validIP(data):\n self.debug(f\"Skipping invalid IP address {data}\")\n continue\n\n if self.opts[\"verify\"] and not self.sf.validateIP(\n eventData, data\n ):\n self.debug(\n f\"Host {eventData} no longer resolves to {data}\"\n )\n continue\n\n evt = SpiderFootEvent(\"IP_ADDRESS\", data, self.__name__, event)\n\n if record.get(\"rrtype\") == \"AAAA\":\n\n if not self.getTarget().matches(\n data, includeChildren=True, includeParents=True\n ):\n continue\n\n if not self.sf.validIP6(data):\n self.debug(\"Skipping invalid IPv6 address \" + data)\n continue\n\n if self.opts[\"verify\"] and not self.sf.validateIP(\n eventData, data\n ):\n self.debug(\n \"Host \" + eventData + \" no longer resolves to \" + data\n )\n continue\n\n evt = SpiderFootEvent(\"IPV6_ADDRESS\", data, self.__name__, event)\n elif record.get(\"rrtype\") == \"MX\":\n data = re.sub(r'.*\\s+(.*)', r'\\1', data)\n evt = SpiderFootEvent(\"PROVIDER_MAIL\", data, self.__name__, event)\n elif record.get(\"rrtype\") == \"NS\":\n evt = SpiderFootEvent(\"PROVIDER_DNS\", data, self.__name__, event)\n elif record.get(\"rrtype\") == \"TXT\":\n data = data.replace('\"', '')\n evt = SpiderFootEvent(\"DNS_TEXT\", data, self.__name__, event)\n elif record.get(\"rrtype\") == \"CNAME\":\n if not self.getTarget().matches(data):\n coHosts.add(data)\n\n self.notifyListeners(evt)\n\n rdataRecords = self.query(\"rdata\", \"name\", eventData)\n\n if rdataRecords is None:\n return\n\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(rdataRecords), self.__name__, event)\n self.notifyListeners(evt)\n\n for record in rdataRecords:\n record = record.get(\"obj\")\n if self.isTooOld(record.get(\"time_last\", 0)):\n continue\n\n if record.get(\"rrtype\") not in (\"NS\", \"CNAME\"):\n continue\n data = record.get(\"rrname\").rstrip(\".\")\n\n if data in responseData:\n continue\n responseData.add(data)\n if record.get(\"rrtype\") == \"NS\":\n evt = SpiderFootEvent(\"PROVIDER_DNS\", data, self.__name__, event)\n elif record.get(\"rrtype\") == \"CNAME\":\n if not self.getTarget().matches(data):\n coHosts.add(data)\n\n elif eventName in (\"IP_ADDRESS\", \"IPV6_ADDRESS\"):\n rdataRecords = self.query(\"rdata\", \"ip\", eventData)\n if rdataRecords is None:\n return\n\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(rdataRecords), self.__name__, event)\n self.notifyListeners(evt)\n\n for record in rdataRecords:\n record = record.get(\"obj\")\n if self.checkForStop():\n return\n\n if self.isTooOld(record.get(\"time_last\", 0)):\n continue\n\n if record.get(\"rrtype\") not in (\"A\", \"AAAA\"):\n continue\n\n data = record.get(\"rrname\").rstrip(\".\")\n\n if data in responseData:\n continue\n responseData.add(data)\n\n if not self.getTarget().matches(data):\n coHosts.add(data)\n continue\n\n if self.opts[\"verify\"] and not self.sf.resolveHost(data) and not self.sf.resolveHost6(data):\n self.debug(f\"Host {data} could not be resolved\")\n evt = SpiderFootEvent(\"INTERNET_NAME_UNRESOLVED\", data, self.__name__, event)\n else:\n evt = SpiderFootEvent(\"INTERNET_NAME\", data, self.__name__, event)\n self.notifyListeners(evt)\n\n for co in coHosts:\n if eventName == \"IP_ADDRESS\" and (\n self.opts[\"verify\"] and not self.sf.validateIP(co, eventData)\n ):\n self.debug(\"Host no longer resolves to our IP.\")\n continue\n\n if not self.opts[\"cohostsamedomain\"]:\n if self.getTarget().matches(co, includeParents=True):\n self.debug(\n \"Skipping \" + co + \" because it is on the same domain.\"\n )\n continue\n\n if self.cohostcount < self.opts[\"maxcohost\"]:\n evt = SpiderFootEvent(\"CO_HOSTED_SITE\", co, self.__name__, event)\n self.notifyListeners(evt)\n self.cohostcount += 1\n\n# End of sfp_dnsdb class\n" }, { "path": "modules/sfp_dnsdumpster.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_dnsdumpster\n# Purpose: SpiderFoot plug-in for subdomain enumeration using\n# dnsdumpster.com\n#\n# Author: TheTechromancer\n#\n# Created: 05/21/2021\n# Copyright: (c) Steve Micallef 2021\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom bs4 import BeautifulSoup\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_dnsdumpster(SpiderFootPlugin):\n\n meta = {\n \"name\": \"DNSDumpster\",\n \"summary\": \"Passive subdomain enumeration using HackerTarget's DNSDumpster\",\n \"useCases\": [\"Investigate\", \"Footprint\", \"Passive\"],\n \"categories\": [\"Passive DNS\"],\n \"dataSource\": {\n \"website\": \"https://dnsdumpster.com/\",\n \"model\": \"FREE_NOAUTH_UNLIMITED\",\n \"description\": \"DNSdumpster.com is a FREE domain research tool that can discover hosts related to a domain.\",\n }\n }\n\n # Default options\n opts = {}\n\n # Option descriptions\n optdescs = {}\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.debug(\"Setting up sfp_dnsdumpster\")\n self.results = self.tempStorage()\n self.opts.update(userOpts)\n\n def watchedEvents(self):\n return [\"DOMAIN_NAME\", \"INTERNET_NAME\"]\n\n def producedEvents(self):\n return [\"INTERNET_NAME\", \"INTERNET_NAME_UNRESOLVED\"]\n\n def query(self, domain):\n ret = []\n # first, get the CSRF tokens\n url = \"https://dnsdumpster.com\"\n res1 = self.sf.fetchUrl(\n url,\n useragent=self.opts.get(\"_useragent\", \"Spiderfoot\")\n )\n if res1[\"code\"] not in [\"200\"]:\n self.error(f\"Bad response code \\\"{res1['code']}\\\" from DNSDumpster\")\n else:\n self.debug(f\"Valid response code \\\"{res1['code']}\\\" from DNSDumpster\")\n html = BeautifulSoup(str(res1[\"content\"]), features=\"lxml\")\n csrftoken = None\n csrfmiddlewaretoken = None\n try:\n for cookie in res1[\"headers\"].get(\"set-cookie\", \"\").split(\";\"):\n k, v = cookie.split('=', 1)\n if k == \"csrftoken\":\n csrftoken = str(v)\n csrfmiddlewaretoken = html.find(\"input\", {\"name\": \"csrfmiddlewaretoken\"}).attrs.get(\"value\", None)\n except Exception:\n pass\n\n # Abort if we didn't get the tokens\n if not csrftoken or not csrfmiddlewaretoken:\n self.error(\"Error obtaining CSRF tokens\")\n self.errorState = True\n return ret\n\n self.debug(\"Successfully obtained CSRF tokens\")\n\n # Otherwise, do the needful\n url = \"https://dnsdumpster.com/\"\n subdomains = set()\n res2 = self.sf.fetchUrl(\n url,\n cookies={\n \"csrftoken\": csrftoken\n },\n postData={\n \"csrfmiddlewaretoken\": csrfmiddlewaretoken,\n \"targetip\": str(domain).lower(),\n \"user\": \"free\"\n },\n headers={\n \"origin\": \"https://dnsdumpster.com\",\n \"referer\": \"https://dnsdumpster.com/\"\n },\n useragent=self.opts.get(\"_useragent\", \"Spiderfoot\")\n )\n if res2[\"code\"] not in [\"200\"]:\n self.error(f\"Bad response code \\\"{res2['code']}\\\" from DNSDumpster\")\n return ret\n\n html = BeautifulSoup(str(res2[\"content\"]), features=\"lxml\")\n escaped_domain = re.escape(domain)\n match_pattern = re.compile(r\"^[\\w\\.-]+\\.\" + escaped_domain + r\"$\")\n for subdomain in html.findAll(text=match_pattern):\n subdomains.add(str(subdomain).strip().lower())\n\n return list(subdomains)\n\n def sendEvent(self, source, host):\n if self.sf.resolveHost(host) or self.sf.resolveHost6(host):\n e = SpiderFootEvent(\"INTERNET_NAME\", host, self.__name__, source)\n else:\n e = SpiderFootEvent(\"INTERNET_NAME_UNRESOLVED\", host, self.__name__, source)\n self.notifyListeners(e)\n\n def handleEvent(self, event):\n query = str(event.data).lower()\n\n self.debug(f\"Received event, {event.eventType}, from {event.module}\")\n\n # skip if we've already processed this event (or its parent domain/subdomain)\n target = self.getTarget()\n eventDataHash = self.sf.hashstring(query)\n if eventDataHash in self.results or \\\n (target.matches(query, includeParents=True) and not\n target.matches(query, includeChildren=False)):\n self.debug(f\"Skipping already-processed event, {event.eventType}, from {event.module}\")\n return\n self.results[eventDataHash] = True\n\n for hostname in self.query(query):\n if target.matches(hostname, includeParents=True) and not \\\n target.matches(hostname, includeChildren=False):\n self.sendEvent(event, hostname)\n else:\n self.debug(f\"Invalid subdomain: {hostname}\")\n" }, { "path": "modules/sfp_dnsgrep.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_dnsgrep\n# Purpose: SpiderFoot plug-in for retrieving domain names\n# from Rapid7 Sonar Project data sets using DNSGrep API.\n# - https://opendata.rapid7.com/about/\n# - https://blog.erbbysam.com/index.php/2019/02/09/dnsgrep/\n# - https://github.com/erbbysam/DNSGrep\n#\n# Author: \n#\n# Created: 2020-03-14\n# Copyright: (c) bcoles 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_dnsgrep(SpiderFootPlugin):\n\n meta = {\n 'name': \"DNSGrep\",\n 'summary': \"Obtain Passive DNS information from Rapid7 Sonar Project using DNSGrep API.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Passive DNS\"],\n 'dataSource': {\n 'website': \"https://opendata.rapid7.com/\",\n 'model': \"FREE_AUTH_UNLIMITED\",\n 'references': [\n \"https://opendata.rapid7.com/apihelp/\",\n \"https://www.rapid7.com/about/research\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://opendata.rapid7.com/apihelp/\",\n \"Submit form requesting for access\",\n \"After getting access, navigate to https://insight.rapid7.com/platform#/apiKeyManagement\",\n \"Create an User Key\",\n \"The API key will be listed after creation\"\n ],\n 'favIcon': \"https://www.rapid7.com/includes/img/favicon.ico\",\n 'logo': \"https://www.rapid7.com/includes/img/Rapid7_logo.svg\",\n 'description': \"Offering researchers and community members open access to data from Project Sonar, \"\n \"which conducts internet-wide surveys to gain insights into global exposure \"\n \"to common vulnerabilities.\",\n }\n }\n\n # Default options\n opts = {\n 'timeout': 30,\n 'dns_resolve': True\n }\n\n # Option descriptions\n optdescs = {\n 'timeout': \"Query timeout, in seconds.\",\n 'dns_resolve': \"DNS resolve each identified domain.\"\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in userOpts.keys():\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"INTERNET_NAME\", \"INTERNET_NAME_UNRESOLVED\", \"RAW_RIR_DATA\"]\n\n # Query the DNSGrep REST API\n def query(self, qry):\n params = {\n 'q': '.' + qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n }\n\n res = self.sf.fetchUrl('https://dns.bufferover.run/dns?' + urllib.parse.urlencode(params),\n timeout=self.opts['timeout'],\n useragent=self.opts['_useragent'])\n\n if res['content'] is None:\n self.info(\"No results found for \" + qry)\n return None\n\n if res['code'] != '200':\n self.debug(\"Error retrieving search results for \" + qry)\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from DNSGrep: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n return\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n data = self.query(eventData)\n\n if data is None:\n self.info(\"No DNS records found for \" + eventData)\n return\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n domains = list()\n\n # Forward DNS A records\n fdns = data.get(\"FDNS_A\")\n if fdns:\n for r in fdns:\n try:\n ip, domain = r.split(',')\n except Exception:\n continue\n\n domains.append(domain)\n\n # Reverse DNS records\n rdns = data.get(\"RDNS\")\n if rdns:\n for r in rdns:\n try:\n ip, domain = r.split(',')\n except Exception:\n continue\n\n domains.append(domain)\n\n for domain in domains:\n if domain in self.results:\n continue\n\n if not self.getTarget().matches(domain, includeParents=True):\n continue\n\n evt_type = \"INTERNET_NAME\"\n\n if self.opts[\"dns_resolve\"] and not self.sf.resolveHost(domain) and not self.sf.resolveHost6(domain):\n self.debug(f\"Host {domain} could not be resolved\")\n evt_type += \"_UNRESOLVED\"\n\n evt = SpiderFootEvent(evt_type, domain, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_dnsgrep class\n" }, { "path": "modules/sfp_dnsneighbor.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_dnsneighbor\n# Purpose: SpiderFoot plug-in for gathering IP addresses from sub-domains\n# and hostnames identified, and optionally affiliates.\n#\n# Author: Steve Micallef \n#\n# Created: 07/07/2017\n# Copyright: (c) Steve Micallef 2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport ipaddress\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_dnsneighbor(SpiderFootPlugin):\n\n meta = {\n 'name': \"DNS Look-aside\",\n 'summary': \"Attempt to reverse-resolve the IP addresses next to your target to see if they are related.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\"],\n 'categories': [\"DNS\"]\n }\n\n # Default options\n opts = {\n 'lookasidebits': 4,\n 'validatereverse': True\n }\n\n # Option descriptions\n optdescs = {\n 'validatereverse': \"Validate that reverse-resolved hostnames still resolve back to that IP before considering them as aliases of your target.\",\n 'lookasidebits': \"If look-aside is enabled, the netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts).\"\n }\n\n events = None\n domresults = None\n hostresults = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.events = self.tempStorage()\n self.domresults = self.tempStorage()\n self.hostresults = self.tempStorage()\n self.__dataSource__ = \"DNS\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['IP_ADDRESS']\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"AFFILIATE_IPADDR\", \"IP_ADDRESS\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n eventDataHash = self.sf.hashstring(eventData)\n addrs = None\n parentEvent = event\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventDataHash in self.events:\n return\n\n self.events[eventDataHash] = True\n\n try:\n address = ipaddress.ip_address(eventData)\n netmask = address.max_prefixlen - min(address.max_prefixlen, max(1, int(self.opts.get(\"lookasidebits\"))))\n network = ipaddress.ip_network(f\"{eventData}/{netmask}\", strict=False)\n except ValueError:\n self.error(f\"Invalid IP address received: {eventData}\")\n return\n\n self.debug(f\"Lookaside max: {network.network_address}, min: {network.broadcast_address}\")\n\n for ip in network:\n sip = str(ip)\n self.debug(\"Attempting look-aside lookup of: \" + sip)\n if self.checkForStop():\n return\n\n if sip in self.hostresults or sip == eventData:\n continue\n\n addrs = self.sf.resolveIP(sip)\n if not addrs:\n self.debug(\"Look-aside resolve for \" + sip + \" failed.\")\n continue\n\n # Report addresses that resolve to hostnames on the same\n # domain or sub-domain as the target.\n if self.getTarget().matches(sip):\n affil = False\n else:\n affil = True\n for a in addrs:\n if self.getTarget().matches(a):\n affil = False\n\n # Generate the event for the look-aside IP, but don't let it re-trigger\n # this module by adding it to self.events first.\n self.events[sip] = True\n ev = self.processHost(sip, parentEvent, affil)\n\n if not ev:\n continue\n\n for addr in addrs:\n if self.checkForStop():\n return\n\n if addr == sip:\n continue\n\n if self.sf.validIP(addr) or self.sf.validIP6(addr):\n parent = parentEvent\n else:\n # Hostnames from the IP need to be linked to the IP\n parent = ev\n\n if self.getTarget().matches(addr):\n # Generate an event for the IP, then\n # let the handling by this module take\n # care of follow-up processing.\n self.processHost(addr, parent, False)\n else:\n self.processHost(addr, parent, True)\n\n def processHost(self, host, parentEvent, affiliate=None):\n parentHash = self.sf.hashstring(parentEvent.data)\n if host not in self.hostresults:\n self.hostresults[host] = [parentHash]\n else:\n if parentHash in self.hostresults[host] or parentEvent.data == host:\n self.debug(\"Skipping host, \" + host + \", already processed.\")\n return None\n self.hostresults[host] = self.hostresults[host] + [parentHash]\n\n self.debug(\"Found host: \" + host)\n # If the returned hostname is aliased to our\n # target in some way, flag it as an affiliate\n if affiliate is None:\n affil = True\n if self.getTarget().matches(host):\n affil = False\n else:\n # If the IP the host resolves to is in our\n # list of aliases,\n if not self.sf.validIP(host) and not self.sf.validIP6(host):\n hostips = self.sf.resolveHost(host)\n if hostips:\n for hostip in hostips:\n if self.getTarget().matches(hostip):\n affil = False\n break\n hostips6 = self.sf.resolveHost6(host)\n if hostips6:\n for hostip in hostips6:\n if self.getTarget().matches(hostip):\n affil = False\n break\n else:\n affil = affiliate\n\n if not self.sf.validIP(host):\n return None\n\n if affil:\n htype = \"AFFILIATE_IPADDR\"\n else:\n htype = \"IP_ADDRESS\"\n\n # If names were found, leave them to sfp_dnsresolve to resolve\n if not htype:\n return None\n\n # Report the host\n evt = SpiderFootEvent(htype, host, self.__name__, parentEvent)\n self.notifyListeners(evt)\n\n return evt\n\n# End of sfp_dnsneighbor class\n" }, { "path": "modules/sfp_dnsraw.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_dnsraw\n# Purpose: SpiderFoot plug-in for collecting raw DNS records.\n# Also extracts hostnames from SPF records.\n#\n# Author: Steve Micallef \n#\n# Created: 07/07/2017\n# Copyright: (c) Steve Micallef 2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nimport dns.query\nimport dns.rdatatype\nimport dns.resolver\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_dnsraw(SpiderFootPlugin):\n\n meta = {\n 'name': \"DNS Raw Records\",\n 'summary': \"Retrieves raw DNS records such as MX, TXT and others.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"DNS\"]\n }\n\n # Default options\n opts = {\n 'verify': True,\n }\n\n # Option descriptions\n optdescs = {\n 'verify': \"Verify identified hostnames resolve.\"\n }\n\n events = None\n checked = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.events = self.tempStorage()\n self.checked = self.tempStorage()\n self.__dataSource__ = \"DNS\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['INTERNET_NAME', 'DOMAIN_NAME', 'DOMAIN_NAME_PARENT']\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"PROVIDER_MAIL\", \"PROVIDER_DNS\", \"RAW_DNS_RECORDS\", \"DNS_TEXT\", \"DNS_SPF\",\n 'INTERNET_NAME', 'INTERNET_NAME_UNRESOLVED',\n 'AFFILIATE_INTERNET_NAME', 'AFFILIATE_INTERNET_NAME_UNRESOLVED']\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n eventDataHash = self.sf.hashstring(eventData)\n parentEvent = event\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventDataHash in self.events:\n self.debug(\"Skipping duplicate event for \" + eventData)\n return\n\n self.events[eventDataHash] = True\n\n self.debug(\"Gathering DNS records for \" + eventData)\n\n domains = list()\n\n # Process the raw data alone\n recs = {\n 'CNAME': r'\\S+\\s+(?:\\d+)?\\s+IN\\s+CNAME\\s+(\\S+)\\.',\n 'MX': r'\\S+\\s+(?:\\d+)?\\s+IN\\s+MX\\s+\\d+\\s+(\\S+)\\.',\n 'NS': r'\\S+\\s+(?:\\d+)?\\s+IN\\s+NS\\s+(\\S+)\\.',\n 'TXT': r'\\S+\\s+TXT\\s+\\\"(.[^\\\"]*)\"'\n }\n\n for rec in list(recs.keys()):\n if self.checkForStop():\n return\n\n try:\n req = dns.message.make_query(eventData, dns.rdatatype.from_text(rec))\n\n if self.opts.get('_dnsserver', \"\") != \"\":\n n = self.opts['_dnsserver']\n else:\n ns = dns.resolver.get_default_resolver()\n n = ns.nameservers[0]\n\n res = dns.query.udp(req, n, timeout=30)\n\n if not len(res.answer):\n continue\n except Exception as e:\n self.error(f\"Failed to obtain DNS response for {eventData} ({e})\")\n continue\n\n # Iterate through DNS answers\n for x in res.answer:\n if str(x) in self.checked:\n continue\n\n self.checked[str(x)] = True\n\n evt = SpiderFootEvent(\"RAW_DNS_RECORDS\", str(x), self.__name__, parentEvent)\n self.notifyListeners(evt)\n\n for rx in list(recs.keys()):\n self.debug(\"Checking \" + str(x) + \" + against \" + recs[rx])\n pat = re.compile(recs[rx], re.IGNORECASE | re.DOTALL)\n grps = re.findall(pat, str(x))\n\n if len(grps) == 0:\n continue\n\n for m in grps:\n self.debug(\"Matched: \" + m)\n strdata = str(m)\n\n if rx == \"CNAME\":\n domains.append(strdata.lower())\n\n if rx == \"MX\":\n evt = SpiderFootEvent(\"PROVIDER_MAIL\", strdata.lower(), self.__name__, parentEvent)\n self.notifyListeners(evt)\n domains.append(strdata.lower())\n\n if rx == \"NS\":\n evt = SpiderFootEvent(\"PROVIDER_DNS\", strdata.lower(), self.__name__, parentEvent)\n self.notifyListeners(evt)\n domains.append(strdata.lower())\n\n if rx == \"TXT\":\n evt = SpiderFootEvent(\"DNS_TEXT\", strdata, self.__name__, parentEvent)\n self.notifyListeners(evt)\n\n if \"v=spf\" in strdata or \"spf2.0/\" in strdata:\n evt = SpiderFootEvent(\"DNS_SPF\", strdata, self.__name__, parentEvent)\n self.notifyListeners(evt)\n\n matches = re.findall(r'include:(.+?) ', strdata, re.IGNORECASE | re.DOTALL)\n if matches:\n for domain in matches:\n if '_' in domain:\n continue\n domains.append(domain.lower())\n\n for domain in set(domains):\n if self.getTarget().matches(domain, includeChildren=True, includeParents=True):\n evt_type = 'INTERNET_NAME'\n else:\n evt_type = 'AFFILIATE_INTERNET_NAME'\n\n if self.opts['verify'] and not self.sf.resolveHost(domain) and not self.sf.resolveHost6(domain):\n self.debug(f\"Host {domain} could not be resolved\")\n evt_type += '_UNRESOLVED'\n\n evt = SpiderFootEvent(evt_type, domain, self.__name__, parentEvent)\n self.notifyListeners(evt)\n\n# End of sfp_dnsraw class\n" }, { "path": "modules/sfp_dnsresolve.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_dnsresolve\n# Purpose: SpiderFoot plug-in for extracting hostnames from identified data\n# and resolving them.\n#\n# Author: Steve Micallef \n#\n# Created: 07/07/2017\n# Copyright: (c) Steve Micallef 2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\nimport urllib\n\nfrom netaddr import IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_dnsresolve(SpiderFootPlugin):\n\n meta = {\n 'name': \"DNS Resolver\",\n 'summary': \"Resolves hosts and IP addresses identified, also extracted from raw content.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"DNS\"]\n }\n\n # Default options\n opts = {\n 'validatereverse': True,\n 'skipcommononwildcard': True,\n 'netblocklookup': True,\n 'maxnetblock': 24,\n 'maxv6netblock': 120,\n }\n\n # Option descriptions\n optdescs = {\n 'skipcommononwildcard': \"If wildcard DNS is detected, only attempt to look up the first common sub-domain from the common sub-domain list.\",\n 'validatereverse': \"Validate that reverse-resolved hostnames still resolve back to that IP before considering them as aliases of your target.\",\n 'netblocklookup': \"Look up all IPs on netblocks deemed to be owned by your target for possible hosts on the same target subdomain/domain?\",\n 'maxnetblock': \"Maximum owned IPv4 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'maxv6netblock': \"Maximum owned IPv6 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\"\n }\n\n events = None\n domresults = None\n hostresults = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.events = self.tempStorage()\n self.domresults = self.tempStorage()\n self.hostresults = self.tempStorage()\n self.__dataSource__ = \"DNS\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def enrichTarget(self, target):\n ret = list()\n # If it's an IP, get the hostname it reverse resolves to\n self.info(\"Identifying aliases for specified target(s)\")\n ret = self.resolveTargets(target, self.opts['validatereverse'])\n if not ret:\n return target\n\n for host in ret:\n self.debug(\"Found an alias: \" + host)\n if self.sf.validIP(host):\n target.setAlias(host, \"IP_ADDRESS\")\n elif self.sf.validIP6(host):\n target.setAlias(host, \"IPV6_ADDRESS\")\n else:\n target.setAlias(host, \"INTERNET_NAME\")\n idnahost = host.encode(\"idna\")\n if idnahost != host:\n target.setAlias(idnahost.decode('ascii', errors='replace'), \"INTERNET_NAME\")\n\n # If the target was a hostname/sub-domain, we can\n # add the domain as an alias for the target. But\n # not if the target was an IP or subnet.\n # if target.targetType == \"INTERNET_NAME\":\n # dom = self.sf.hostDomain(host, self.opts['_internettlds'])\n # target.setAlias(dom, \"INTERNET_NAME\")\n\n self.info(f\"Target aliases identified: {target.targetAliases}\")\n\n return target\n\n def resolveTargets(self, target, validateReverse: bool) -> list:\n \"\"\"Resolve alternative names for a given target.\n\n Args:\n target (SpiderFootTarget): target object\n validateReverse (bool): validate domain names resolve\n\n Returns:\n list: list of domain names and IP addresses\n \"\"\"\n ret = list()\n\n if not target:\n return ret\n\n t = target.targetType\n v = target.targetValue\n\n if t in [\"IP_ADDRESS\", \"IPV6_ADDRESS\"]:\n r = self.sf.resolveIP(v)\n if r:\n ret.extend(r)\n if t == \"INTERNET_NAME\":\n r = self.sf.resolveHost(v)\n if r:\n ret.extend(r)\n r = self.sf.resolveHost6(v)\n if r:\n ret.extend(r)\n if t == \"NETBLOCK_OWNER\":\n max_netblock = self.opts['maxnetblock']\n if IPNetwork(v).prefixlen < max_netblock:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(v).prefixlen} > {max_netblock}\")\n return list(set(ret))\n\n for addr in IPNetwork(v):\n if self.checkForStop():\n return list(set(ret))\n\n ipaddr = str(addr)\n if ipaddr.split(\".\")[3] in ['255', '0']:\n continue\n\n if '255' in ipaddr.split(\".\"):\n continue\n\n ret.append(ipaddr)\n\n # Add the reverse-resolved hostnames as aliases too\n names = self.sf.resolveIP(ipaddr)\n if not names:\n continue\n\n if not validateReverse:\n ret.extend(names)\n continue\n\n for host in names:\n chk = self.sf.resolveHost(host)\n if chk and ipaddr in chk:\n ret.append(host)\n if t == \"NETBLOCKV6_OWNER\":\n max_netblock = self.opts['maxv6netblock']\n if IPNetwork(v).prefixlen < max_netblock:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(v).prefixlen} > {max_netblock}\")\n return list(set(ret))\n\n for addr in IPNetwork(v):\n if self.checkForStop():\n return list(set(ret))\n\n ipaddr = str(addr)\n\n ret.append(ipaddr)\n\n # Add the reverse-resolved hostnames as aliases too\n names = self.sf.resolveIP(ipaddr)\n if not names:\n continue\n\n if not validateReverse:\n ret.extend(names)\n continue\n\n for host in names:\n chk = self.sf.resolveHost6(host)\n if chk and ipaddr in chk:\n ret.append(host)\n\n return list(set(ret))\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\n # Events that need some kind of DNS treatment\n \"CO_HOSTED_SITE\", \"AFFILIATE_INTERNET_NAME\", \"NETBLOCK_OWNER\", \"NETBLOCKV6_OWNER\",\n \"IP_ADDRESS\", \"IPV6_ADDRESS\", \"INTERNET_NAME\", \"AFFILIATE_IPADDR\", \"AFFILIATE_IPV6_ADDRESS\",\n # Events that may contain hostnames in their content\n \"TARGET_WEB_CONTENT\", \"BASE64_DATA\", \"AFFILIATE_DOMAIN_WHOIS\",\n \"CO_HOSTED_SITE_DOMAIN_WHOIS\", \"DOMAIN_WHOIS\", \"NETBLOCK_WHOIS\",\n \"LEAKSITE_CONTENT\", \"RAW_DNS_RECORDS\", \"RAW_FILE_META_DATA\",\n \"RAW_RIR_DATA\", \"SIMILARDOMAIN_WHOIS\",\n \"SSL_CERTIFICATE_RAW\", \"SSL_CERTIFICATE_ISSUED\", \"TCP_PORT_OPEN_BANNER\",\n \"WEBSERVER_BANNER\", \"WEBSERVER_HTTPHEADERS\"\n ]\n\n # What events this module produces\n def producedEvents(self):\n return [\"IP_ADDRESS\", \"INTERNET_NAME\", \"AFFILIATE_INTERNET_NAME\",\n \"AFFILIATE_IPADDR\", \"AFFILIATE_IPV6_ADDRESS\", \"DOMAIN_NAME\", \"IPV6_ADDRESS\", \"INTERNAL_IP_ADDRESS\",\n \"DOMAIN_NAME_PARENT\", \"CO_HOSTED_SITE_DOMAIN\", \"AFFILIATE_DOMAIN_NAME\",\n \"INTERNET_NAME_UNRESOLVED\"]\n\n # Handle events sent to this module\n def handleEvent(self, event) -> None:\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n eventDataHash = self.sf.hashstring(eventData)\n addrs = None\n parentEvent = event\n\n # Don't be recursive for names\n if srcModuleName in [\"sfp_dnsresolve\"] and \"_NAME\" in eventName:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventDataHash in self.events:\n self.debug(\"Skipping duplicate event.\")\n return\n\n self.events[eventDataHash] = True\n\n # Parse Microsoft workaround \"ipv6-literal.net\" fake domain for IPv6 UNC paths\n # For internal use on Windows systems (should not resolve in DNS)\n if eventData.endswith(\".ipv6-literal.net\") and eventName == \"AFFILIATE_INTERNET_NAME\":\n ipv6 = eventData.split(\".ipv6-literal.net\")[0].replace('-', ':').replace('s', '%').split('%')[0]\n if self.sf.validIP6(ipv6):\n if self.getTarget().matches(ipv6):\n evt = SpiderFootEvent(\"IPV6_ADDRESS\", ipv6, self.__name__, parentEvent)\n else:\n evt = SpiderFootEvent(\"AFFILIATE_IPV6_ADDRESS\", ipv6, self.__name__, parentEvent)\n self.notifyListeners(evt)\n return\n\n # Convert ARPA in-addr.arpa address to IPv4 address\n if eventData.endswith(\".in-addr.arpa\") and eventName == \"AFFILIATE_INTERNET_NAME\":\n ipv4 = '.'.join(reversed(eventData.split('.in-addr.arpa')[0].split('.')))\n if self.sf.validIP(ipv4):\n if self.getTarget().matches(ipv4):\n evt = SpiderFootEvent(\"IP_ADDRESS\", ipv4, self.__name__, parentEvent)\n else:\n evt = SpiderFootEvent(\"AFFILIATE_IPADDR\", ipv4, self.__name__, parentEvent)\n self.notifyListeners(evt)\n\n # Simply translates these to their domains\n if eventName in [\"CO_HOSTED_SITE\", \"AFFILIATE_INTERNET_NAME\"]:\n # If the co-host or affiliate is a domain name, generate\n # a domain event.\n if eventName == \"AFFILIATE_INTERNET_NAME\":\n ev = \"AFFILIATE_DOMAIN_NAME\"\n else:\n ev = \"CO_HOSTED_SITE_DOMAIN\"\n\n # What we've been provided might be a domain, so report it\n if self.sf.isDomain(eventData, self.opts['_internettlds']):\n evt = SpiderFootEvent(ev, eventData, self.__name__, parentEvent)\n self.notifyListeners(evt)\n\n # In case the domain of the provided host is different, report that too\n dom = self.sf.hostDomain(eventData, self.opts['_internettlds'])\n if dom and dom != eventData:\n evt = SpiderFootEvent(ev, dom, self.__name__, parentEvent)\n self.notifyListeners(evt)\n\n # Resolve host names\n if eventName in [\"INTERNET_NAME\", \"AFFILIATE_INTERNET_NAME\"]:\n addrs = list()\n addrs.extend(self.sf.resolveHost(eventData))\n addrs.extend(self.sf.resolveHost6(eventData))\n\n if not addrs:\n return\n\n addrs.append(eventData)\n\n # We now have a set of hosts/IPs to do something with.\n for addr in set(addrs):\n if self.checkForStop():\n return\n\n # IP addresses resolved from hosts are assumed\n # to be part of the target (non-affiliates),\n # unless the source event was an AFFILIATE event.\n affiliate = False\n if self.getTarget().matches(addr):\n affiliate = False\n elif eventName.startswith(\"AFFILIATE_\"):\n affiliate = True\n self.processHost(addr, parentEvent, affiliate)\n\n # Reverse resolve IP addresses\n elif eventName in [\"IP_ADDRESS\", \"IPV6_ADDRESS\", \"AFFILIATE_IPADDR\", \"AFFILIATE_IPV6_ADDRESS\"]:\n addrs = self.sf.resolveIP(eventData)\n\n if not addrs:\n return\n\n addrs.append(eventData)\n\n # We now have a set of hosts/IPs to do something with.\n for addr in set(addrs):\n if self.checkForStop():\n return\n\n # IP addresses resolved from hosts are assumed\n # to be part of the target (non-affiliates),\n # unless the source event was an AFFILIATE event.\n affiliate = False\n if self.getTarget().matches(addr):\n affiliate = False\n elif eventName.startswith(\"AFFILIATE_\"):\n affiliate = True\n self.processHost(addr, parentEvent, affiliate)\n\n elif eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']:\n if not self.opts['netblocklookup']:\n return\n\n if eventName == 'NETBLOCKV6_OWNER':\n max_netblock = self.opts['maxv6netblock']\n else:\n max_netblock = self.opts['maxnetblock']\n\n if IPNetwork(eventData).prefixlen < max_netblock:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}\")\n return\n\n self.debug(f\"Looking up IPs in owned netblock: {eventData}\")\n for ip in IPNetwork(eventData):\n if self.checkForStop():\n return\n\n ipaddr = str(ip)\n\n # Skip 0 and 255 for IPv4 addresses\n if self.sf.validIP(ipaddr):\n if ipaddr.split(\".\")[3] in ['255', '0']:\n continue\n if '255' in ipaddr.split(\".\"):\n continue\n\n addrs = self.sf.resolveIP(ipaddr)\n if not addrs:\n continue\n\n self.debug(f\"Found {len(addrs)} reversed hostnames from {ipaddr} ({addrs})\")\n for addr in addrs:\n if self.checkForStop():\n return\n\n # Generate an event for the IP, then\n # let the handling by this module take\n # care of follow-up processing.\n self.processHost(addr, parentEvent, False)\n\n # For everything else (ie, raw data), search for IPs/hosts\n else:\n # ignore co-hosted sites\n if eventName == \"CO_HOSTED_SITE\":\n return\n\n data = urllib.parse.unquote(eventData).lower()\n # We get literal \\n from RAW_RIR_DATA in cases where JSON responses\n # have been str()'d, breaking interpretation of hostnames.\n if eventName == 'RAW_RIR_DATA':\n data = re.sub(r'(\\\\x[0-f]{2}|\\\\n|\\\\r)', '\\n', data)\n\n for name in self.getTarget().getNames():\n if self.checkForStop():\n return\n\n offset = data.find(name)\n if offset < 0:\n continue\n\n pat = re.compile(r\"[^a-z0-9\\-\\.]([a-z0-9\\-\\.]*\\.\" + name + \")\", re.DOTALL | re.MULTILINE)\n while offset >= 0:\n # If the target was found at the beginning of the content, skip past it\n if offset == 0:\n offset += len(name)\n continue\n\n if offset <= 100:\n # Start from the beginning of the text\n start = 0\n else:\n # Start looking for a host 100 chars before the target name\n start = offset - 100\n\n # Get up to 100 bytes before the name to try and get hostnames\n chunkhost = data[start:(offset + start + len(name) + 1)]\n matches = None\n try:\n matches = re.findall(pat, chunkhost)\n except Exception as e:\n self.error(f\"Error applying regex to data ({e})\")\n\n if matches:\n for match in matches:\n # Wildcard certs will come in as .blah.blah\n if match.startswith(\".\"):\n m = match[1:]\n else:\n m = match\n self.processHost(m, parentEvent, False)\n\n offset = data.find(name, start + len(chunkhost))\n\n # Process a host/IP, parentEvent is the event that represents this entity\n def processHost(self, host, parentEvent, affiliate=None) -> None:\n parentHash = self.sf.hashstring(parentEvent.data)\n if host in self.hostresults:\n if parentHash in self.hostresults[host] or parentEvent.data == host:\n self.debug(f\"Skipping host, {host}, already processed.\")\n return\n self.hostresults[host] = self.hostresults[host] + [parentHash]\n else:\n self.hostresults[host] = [parentHash]\n\n self.debug(f\"Found host: {host}\")\n\n # If the returned hostname is aliased to our\n # target in some way, flag it as an affiliate\n if affiliate is None:\n affil = True\n if self.getTarget().matches(host):\n affil = False\n # If the IP the host resolves to is in our\n # list of aliases,\n if not self.sf.validIP(host):\n hostips = self.sf.resolveHost(host)\n if hostips:\n for hostip in hostips:\n if self.getTarget().matches(hostip):\n affil = False\n break\n hostips6 = self.sf.resolveHost6(host)\n if hostips6:\n for hostip6 in hostips6:\n if self.getTarget().matches(hostip6):\n affil = False\n break\n else:\n affil = affiliate\n\n if affil:\n if self.sf.isValidLocalOrLoopbackIp(host):\n htype = \"INTERNAL_IP_ADDRESS\"\n elif self.sf.validIP(host):\n htype = \"AFFILIATE_IPADDR\"\n elif self.sf.validIP6(host):\n htype = \"AFFILIATE_IPV6_ADDRESS\"\n else:\n htype = \"AFFILIATE_INTERNET_NAME\"\n else:\n if self.sf.isValidLocalOrLoopbackIp(host):\n htype = \"INTERNAL_IP_ADDRESS\"\n elif self.sf.validIP(host):\n htype = \"IP_ADDRESS\"\n elif self.sf.validIP6(host):\n htype = \"IPV6_ADDRESS\"\n else:\n htype = \"INTERNET_NAME\"\n\n if htype in [\"INTERNET_NAME\", \"AFFILIATE_INTERNET_NAME\"]:\n if not self.sf.resolveHost(host) and not self.sf.resolveHost6(host):\n evt = SpiderFootEvent(f\"{htype}_UNRESOLVED\", host, self.__name__, parentEvent)\n self.notifyListeners(evt)\n return\n\n # Report the host\n if host != parentEvent.data:\n evt = SpiderFootEvent(htype, host, self.__name__, parentEvent)\n self.notifyListeners(evt)\n else:\n evt = parentEvent\n\n if htype == \"INTERNET_NAME\":\n dom = self.sf.hostDomain(host, self.opts['_internettlds'])\n if dom:\n self.processDomain(dom, evt, False, host)\n\n # Try obtain the IPv6 address\n ip6s = self.sf.resolveHost6(host)\n if not ip6s:\n return\n for ip6 in ip6s:\n parentHash = self.sf.hashstring(evt.data)\n if ip6 not in self.hostresults:\n self.hostresults[ip6] = [parentHash]\n else:\n if parentHash in self.hostresults[ip6] or evt.data == ip6:\n self.debug(f\"Skipping host, {ip6}, already processed.\")\n continue\n self.hostresults[ip6] = self.hostresults[ip6] + [parentHash]\n\n evt6 = SpiderFootEvent(\"IPV6_ADDRESS\", ip6, self.__name__, evt)\n self.notifyListeners(evt6)\n\n if htype == \"AFFILIATE_INTERNET_NAME\":\n dom = self.sf.hostDomain(host, self.opts['_internettlds'])\n if not dom:\n return\n if dom == host and not self.sf.isDomain(dom, self.opts['_internettlds']):\n return\n self.processDomain(dom, evt, True, host)\n\n def processDomain(self, domainName, parentEvent, affil=False, host=None) -> None:\n if domainName in self.domresults:\n self.debug(f\"Skipping domain, {domainName}, already processed.\")\n return\n\n self.domresults[domainName] = True\n\n if affil:\n domevt = SpiderFootEvent(\"AFFILIATE_DOMAIN_NAME\", domainName,\n self.__name__, parentEvent)\n self.notifyListeners(domevt)\n return\n\n if self.getTarget().matches(domainName):\n domevt = SpiderFootEvent(\"DOMAIN_NAME\", domainName,\n self.__name__, parentEvent)\n self.notifyListeners(domevt)\n else:\n # Only makes sense to link this event with a source event\n # that sits on the parent domain.\n if not host:\n return\n if parentEvent.data.endswith(\".\" + domainName):\n domevt = SpiderFootEvent(\"DOMAIN_NAME_PARENT\", domainName,\n self.__name__, parentEvent)\n self.notifyListeners(domevt)\n\n# End of sfp_dnsresolve class\n" }, { "path": "modules/sfp_dnszonexfer.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_dnszonexfer\n# Purpose: SpiderFoot plug-in for attempting a DNS zone transfer.\n#\n# Author: Steve Micallef \n#\n# Created: 22/08/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nimport dns.query\nimport dns.zone\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_dnszonexfer(SpiderFootPlugin):\n\n meta = {\n 'name': \"DNS Zone Transfer\",\n 'summary': \"Attempts to perform a full DNS zone transfer.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\"],\n 'categories': [\"DNS\"]\n }\n\n opts = {\n \"timeout\": 30\n }\n\n optdescs = {\n \"timeout\": \"Timeout in seconds\"\n }\n\n events = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.events = self.tempStorage()\n self.__dataSource__ = \"DNS\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return ['PROVIDER_DNS']\n\n def producedEvents(self):\n return [\"RAW_DNS_RECORDS\", \"INTERNET_NAME\"]\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n eventDataHash = self.sf.hashstring(eventData)\n parentEvent = event\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if srcModuleName == \"sfp_dnszonexfer\":\n self.debug(f\"Ignoring {eventName}, from self.\")\n return\n\n if eventDataHash in self.events:\n self.debug(\"Skipping duplicate event for \" + eventData)\n return\n\n self.events[eventDataHash] = True\n\n res = dns.resolver.Resolver()\n if self.opts.get('_dnsserver', \"\") != \"\":\n res.nameservers = [self.opts['_dnsserver']]\n\n # Get the name server's IP. This is to avoid DNS leaks\n # when attempting to resolve the name server during\n # the zone transfer.\n if not self.sf.validIP(eventData) and not self.sf.validIP6(eventData):\n nsips = self.sf.resolveHost(eventData)\n if not nsips:\n return\n\n if not nsips:\n self.error(\"Couldn't resolve the name server, so not attempting zone transfer.\")\n return\n\n for n in nsips:\n if self.sf.validIP(n):\n nsip = n\n break\n else:\n nsip = eventData\n\n for name in self.getTarget().getNames():\n self.debug(\"Trying for name: \" + name)\n try:\n ret = list()\n z = dns.zone.from_xfr(dns.query.xfr(nsip, name, timeout=int(self.opts[\"timeout\"])))\n names = list(z.nodes.keys())\n for n in names:\n ret.append(z[n].to_text(n))\n\n evt = SpiderFootEvent(\"RAW_DNS_RECORDS\", \"\\n\".join(ret), self.__name__, parentEvent)\n self.notifyListeners(evt)\n\n # Try and pull out individual records\n for row in ret:\n pat = re.compile(r\"^(\\S+)\\.?\\s+\\d+\\s+IN\\s+[AC].*\", re.IGNORECASE | re.DOTALL)\n grps = re.findall(pat, row)\n if len(grps) > 0:\n for strdata in grps:\n self.debug(\"Matched: \" + strdata)\n if strdata.endswith(\".\"):\n strdata = strdata[:-1]\n else:\n strdata = strdata + \".\" + name\n\n evt = SpiderFootEvent(\"INTERNET_NAME\", strdata, self.__name__, parentEvent)\n self.notifyListeners(evt)\n\n except Exception as e:\n self.info(f\"Unable to perform DNS zone transfer for {eventData} ({name}): {e}\")\n\n# End of sfp_dnszonexfer class\n" }, { "path": "modules/sfp_dronebl.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_dronebl\n# Purpose: SpiderFoot plug-in for looking up whether IPs/Netblocks/Domains\n# appear in the DroneBL blocklist, indicating potential open-relays,\n# open proxies, malicious servers, vulnerable servers, etc.\n#\n# Author: Steve Micallef \n#\n# Created: 07/01/2014\n# Copyright: (c) Steve Micallef 2014\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom netaddr import IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_dronebl(SpiderFootPlugin):\n\n meta = {\n 'name': \"DroneBL\",\n 'summary': \"Query the DroneBL database for open relays, open proxies, vulnerable servers, etc.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://dronebl.org/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://dronebl.org/docs/howtouse\",\n \"https://dronebl.org/rpckey_signup\",\n \"https://dronebl.org/docs/rpc2\"\n ],\n 'favIcon': \"https://dronebl.org/images/favicon.ico\",\n 'logo': \"https://dronebl.org/images/dronebl-logo.svg\",\n 'description': \"DroneBL is a realtime monitor of abusable IPs, which has \"\n \"the goal of stopping abuse of infected machines.\\n\"\n \"A real-time tracker of abusable IPs.\",\n }\n }\n\n # Default options\n opts = {\n 'netblocklookup': True,\n 'maxnetblock': 24,\n 'subnetlookup': True,\n 'maxsubnet': 24\n }\n\n # Option descriptions\n optdescs = {\n 'netblocklookup': \"Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?\",\n 'maxnetblock': \"If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'subnetlookup': \"Look up all IPs on subnets which your target is a part of for blacklisting?\",\n 'maxsubnet': \"If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\"\n }\n\n results = None\n\n checks = {\n \"127.0.0.3\": \"dronebl.org - IRC Drone\",\n \"127.0.0.5\": \"dronebl.org - Bottler\",\n \"127.0.0.6\": \"dronebl.org - Unknown spambot or drone\",\n \"127.0.0.7\": \"dronebl.org - DDOS Drone\",\n \"127.0.0.8\": \"dronebl.org - SOCKS Proxy\",\n \"127.0.0.9\": \"dronebl.org - HTTP Proxy\",\n \"127.0.0.10\": \"dronebl.org - ProxyChain\",\n \"127.0.0.11\": \"dronebl.org - Web Page Proxy\",\n \"127.0.0.12\": \"dronebl.org - Open DNS Resolver\",\n \"127.0.0.13\": \"dronebl.org - Brute force attackers\",\n \"127.0.0.14\": \"dronebl.org - Open Wingate Proxy\",\n \"127.0.0.15\": \"dronebl.org - Compromised router / gateway\",\n \"127.0.0.16\": \"dronebl.org - Autorooting worms\",\n \"127.0.0.17\": \"dronebl.org - Automatically determined botnet IPs (experimental)\",\n \"127.0.0.18\": \"dronebl.org - Possibly compromised DNS/MX\",\n \"127.0.0.19\": \"dronebl.org - Abused VPN Service\",\n \"127.0.0.255\": \"dronebl.org - Unknown\"\n }\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n 'IP_ADDRESS',\n 'AFFILIATE_IPADDR',\n 'NETBLOCK_OWNER',\n 'NETBLOCK_MEMBER'\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"BLACKLISTED_SUBNET\",\n \"BLACKLISTED_NETBLOCK\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_NETBLOCK\",\n \"MALICIOUS_SUBNET\",\n \"VPN_HOST\",\n \"PROXY_HOST\"\n ]\n\n # Swap 1.2.3.4 to 4.3.2.1\n def reverseAddr(self, ipaddr):\n if not self.sf.validIP(ipaddr):\n self.debug(f\"Invalid IPv4 address {ipaddr}\")\n return None\n return '.'.join(reversed(ipaddr.split('.')))\n\n def queryAddr(self, qaddr):\n \"\"\"Query DroneBL DNS for an IPv4 address.\n\n Args:\n qaddr (str): IPv4 address.\n\n Returns:\n list: DroneBL DNS entries\n \"\"\"\n if not self.sf.validIP(qaddr):\n self.debug(f\"Invalid IPv4 address {qaddr}\")\n return None\n\n try:\n lookup = self.reverseAddr(qaddr) + '.dnsbl.dronebl.org'\n self.debug(f\"Checking DroneBL blacklist: {lookup}\")\n return self.sf.resolveHost(lookup)\n except Exception as e:\n self.debug(f\"DroneBL did not resolve {qaddr} / {lookup}: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if eventName == 'NETBLOCK_OWNER':\n if not self.opts['netblocklookup']:\n return\n\n max_netblock = self.opts['maxnetblock']\n if IPNetwork(eventData).prefixlen < max_netblock:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}\")\n return\n\n if eventName == 'NETBLOCK_MEMBER':\n if not self.opts['subnetlookup']:\n return\n\n max_subnet = self.opts['maxsubnet']\n if IPNetwork(eventData).prefixlen < max_subnet:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}\")\n return\n\n if eventName == \"AFFILIATE_IPADDR\":\n malicious_type = \"MALICIOUS_AFFILIATE_IPADDR\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_IPADDR\"\n elif eventName == \"IP_ADDRESS\":\n malicious_type = \"MALICIOUS_IPADDR\"\n blacklist_type = \"BLACKLISTED_IPADDR\"\n elif eventName == \"NETBLOCK_OWNER\":\n malicious_type = \"MALICIOUS_NETBLOCK\"\n blacklist_type = \"BLACKLISTED_NETBLOCK\"\n elif eventName == \"NETBLOCK_MEMBER\":\n malicious_type = \"MALICIOUS_SUBNET\"\n blacklist_type = \"BLACKLISTED_SUBNET\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n addrs = list()\n if eventName.startswith(\"NETBLOCK_\"):\n for addr in IPNetwork(eventData):\n addrs.append(str(addr))\n else:\n addrs.append(eventData)\n\n for addr in addrs:\n if self.checkForStop():\n return\n\n res = self.queryAddr(addr)\n\n self.results[addr] = True\n\n if not res:\n continue\n\n self.debug(f\"{addr} found in DroneBL DNS: {res}\")\n\n for result in res:\n k = str(result)\n if k not in self.checks:\n if not result.endswith('.dnsbl.dronebl.org'):\n # This is an error. The \"checks\" dict may need to be updated.\n self.error(f\"DroneBL resolved address {addr} to unknown IP address {result} not found in DroneBL list.\")\n continue\n\n evt = SpiderFootEvent(blacklist_type, f\"{self.checks[k]} [{addr}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n if k in [\n \"127.0.0.3\",\n \"127.0.0.5\",\n \"127.0.0.6\",\n \"127.0.0.7\",\n \"127.0.0.13\",\n \"127.0.0.15\",\n \"127.0.0.16\",\n \"127.0.0.17\",\n \"127.0.0.18\",\n \"127.0.0.19\",\n ]:\n evt = SpiderFootEvent(malicious_type, f\"{self.checks[k]} [{addr}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n if k in [\"127.0.0.8\", \"127.0.0.9\", \"127.0.0.10\", \"127.0.0.11\", \"127.0.0.14\"]:\n evt = SpiderFootEvent(\"PROXY_HOST\", addr, self.__name__, event)\n self.notifyListeners(evt)\n\n if k == \"127.0.0.19\":\n evt = SpiderFootEvent(\"VPN_HOST\", addr, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_dronebl class\n" }, { "path": "modules/sfp_duckduckgo.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_duckduckgo\n# Purpose: Queries DuckDuckGo's API for information abotut the target.\n#\n# Author: Steve Micallef \n#\n# Created: 21/07/2015\n# Copyright: (c) Steve Micallef 2015\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_duckduckgo(SpiderFootPlugin):\n\n meta = {\n 'name': \"DuckDuckGo\",\n 'summary': \"Query DuckDuckGo's API for descriptive information about your target.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://duckduckgo.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://api.duckduckgo.com/api\",\n \"https://help.duckduckgo.com/company/partnerships/\",\n \"https://help.duckduckgo.com/duckduckgo-help-pages/\"\n ],\n 'favIcon': \"https://duckduckgo.com/favicon.ico\",\n 'logo': \"https://duckduckgo.com/assets/icons/meta/DDG-icon_256x256.png\",\n 'description': \"Our Instant Answer API gives you free access to many of our instant answers like: \"\n \"topic summaries , categories, disambiguation, and !bang redirects.\",\n }\n }\n\n # Default options\n opts = {\n \"affiliatedomains\": True\n }\n\n # Option descriptions\n optdescs = {\n \"affiliatedomains\": \"For affiliates, look up the domain name, not the hostname. This will usually return more meaningful information about the affiliate.\"\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\", \"DOMAIN_NAME_PARENT\",\n \"INTERNET_NAME\", \"AFFILIATE_INTERNET_NAME\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"DESCRIPTION_CATEGORY\", \"DESCRIPTION_ABSTRACT\",\n \"AFFILIATE_DESCRIPTION_CATEGORY\",\n \"AFFILIATE_DESCRIPTION_ABSTRACT\"]\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n if self.opts['affiliatedomains'] and \"AFFILIATE_\" in eventName:\n eventData = self.sf.hostDomain(eventData, self.opts['_internettlds'])\n if not eventData:\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n url = \"https://api.duckduckgo.com/?q=\" + eventData + \"&format=json&pretty=1\"\n res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\")\n\n if res['content'] is None:\n self.error(f\"Unable to fetch {url}\")\n return\n\n try:\n ret = json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from DuckDuckGo: {e}\")\n return\n\n if not ret['Heading']:\n self.debug(f\"No DuckDuckGo information for {eventData}\")\n return\n\n abstract_text = ret.get('AbstractText')\n if abstract_text:\n event_type = \"DESCRIPTION_ABSTRACT\"\n\n if \"AFFILIATE\" in eventName:\n event_type = \"AFFILIATE_\" + event_type\n\n evt = SpiderFootEvent(event_type, str(abstract_text), self.__name__, event)\n self.notifyListeners(evt)\n\n related_topics = ret.get('RelatedTopics')\n if related_topics:\n event_type = \"DESCRIPTION_CATEGORY\"\n\n if \"AFFILIATE\" in eventName:\n event_type = \"AFFILIATE_\" + event_type\n\n for topic in related_topics:\n if not isinstance(topic, dict):\n self.debug(\"No category text found from DuckDuckGo.\")\n continue\n\n category = topic.get('Text')\n\n if not category:\n self.debug(\"No category text found from DuckDuckGo.\")\n continue\n\n evt = SpiderFootEvent(event_type, category, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_duckduckgo class\n" }, { "path": "modules/sfp_email.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_email\n# Purpose: SpiderFoot plug-in for scanning retrieved content by other\n# modules (such as sfp_spider) and identifying e-mail addresses\n#\n# Author: Steve Micallef \n#\n# Created: 06/04/2012\n# Copyright: (c) Steve Micallef 2012\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_email(SpiderFootPlugin):\n\n meta = {\n 'name': \"E-Mail Address Extractor\",\n 'summary': \"Identify e-mail addresses in any obtained data.\",\n 'useCases': [\"Passive\", \"Investigate\", \"Footprint\"],\n 'categories': [\"Content Analysis\"]\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n\n for opt in userOpts.keys():\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"TARGET_WEB_CONTENT\", \"BASE64_DATA\", \"AFFILIATE_DOMAIN_WHOIS\",\n \"CO_HOSTED_SITE_DOMAIN_WHOIS\", \"DOMAIN_WHOIS\", \"NETBLOCK_WHOIS\",\n \"LEAKSITE_CONTENT\", \"RAW_DNS_RECORDS\", \"RAW_FILE_META_DATA\",\n 'RAW_RIR_DATA', \"SIMILARDOMAIN_WHOIS\",\n \"SSL_CERTIFICATE_RAW\", \"SSL_CERTIFICATE_ISSUED\", \"TCP_PORT_OPEN_BANNER\",\n \"WEBSERVER_BANNER\", \"WEBSERVER_HTTPHEADERS\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"EMAILADDR\", \"EMAILADDR_GENERIC\", \"AFFILIATE_EMAILADDR\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n emails = SpiderFootHelpers.extractEmailsFromText(eventData)\n for email in set(emails):\n evttype = \"EMAILADDR\"\n email = email.lower()\n\n # Get the domain and strip potential ending .\n mailDom = email.split('@')[1].strip('.')\n if not self.sf.validHost(mailDom, self.opts['_internettlds']):\n self.debug(f\"Skipping {email} as not a valid e-mail.\")\n continue\n\n if not self.getTarget().matches(mailDom, includeChildren=True, includeParents=True) and not self.getTarget().matches(email):\n self.debug(\"External domain, so possible affiliate e-mail\")\n evttype = \"AFFILIATE_EMAILADDR\"\n\n if eventName.startswith(\"AFFILIATE_\"):\n evttype = \"AFFILIATE_EMAILADDR\"\n\n if not evttype.startswith(\"AFFILIATE_\") and email.split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evttype = \"EMAILADDR_GENERIC\"\n\n self.info(f\"Found e-mail address: {email}\")\n mail = email.strip('.')\n\n evt = SpiderFootEvent(evttype, mail, self.__name__, event)\n if event.moduleDataSource:\n evt.moduleDataSource = event.moduleDataSource\n else:\n evt.moduleDataSource = \"Unknown\"\n self.notifyListeners(evt)\n\n# End of sfp_email class\n" }, { "path": "modules/sfp_emailcrawlr.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_emailcrawlr\n# Purpose: Search EmailCrawlr for email addresses and phone numbers\n# associated with a domain.\n#\n# Authors: \n#\n# Created: 2020-06-19\n# Copyright: (c) bcoles 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_emailcrawlr(SpiderFootPlugin):\n\n meta = {\n 'name': \"EmailCrawlr\",\n 'summary': \"Search EmailCrawlr for email addresses and phone numbers associated with a domain.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://emailcrawlr.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://emailcrawlr.com/docs\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://emailcrawlr.com\",\n \"Sign up for free account\",\n \"Navigate to https://emailcrawlr.com/dashboard\",\n \"The API key is listed under 'API Key'\"\n ],\n 'favIcon': \"https://emailcrawlr.com/assets/fav-165eaa698b0dc774f0b250fadb2b41266e4c70dfbd7fb5fd4413e4bdecfd229d.png\",\n 'logo': \"https://emailcrawlr.com/assets/logo_black-d136617b2fc5d52df6eea245a4db78477d8d99f873e08c24a9c3c7defe1c1379.png\",\n 'description': \"By using the EmailCrawlr JSON API you can: \"\n \"Get key information about company websites.\\n\"\n \"Find all email addresses associated with a domain.\\n\"\n \"Get social accounts associated with an email.\\n\"\n \"Verify email address deliverability.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\",\n \"delay\": 1,\n }\n\n # Option descriptions\n optdescs = {\n \"api_key\": \"EmailCrawlr API key.\",\n \"delay\": \"Delay between requests, in seconds.\",\n }\n\n results = None\n errorState = False\n\n # Initialize module and module options\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in userOpts.keys():\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"RAW_RIR_DATA\", \"EMAILADDR\", \"EMAILADDR_GENERIC\",\n \"PHONE_NUMBER\", \"GEOINFO\"]\n\n # Query domain\n # https://emailcrawlr.com/docs\n def queryDomain(self, qry):\n params = {\n 'domain': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n }\n headers = {\n 'Accept': \"application/json\",\n 'x-api-key': self.opts['api_key']\n }\n\n res = self.sf.fetchUrl(\n f\"https://api.emailcrawlr.com/v2/domain?{urllib.parse.urlencode(params)}\",\n headers=headers,\n timeout=15,\n useragent=self.opts['_useragent']\n )\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n # Parse API response\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from EmailCrawlr.\")\n return None\n\n if res['code'] == '404':\n self.debug(\"No results for query\")\n return None\n\n if res['code'] == \"401\":\n self.error(\"Invalid EmailCrawlr API key.\")\n self.errorState = True\n return None\n\n if res['code'] == '429':\n self.error(\"You are being rate-limited by EmailCrawlr\")\n self.errorState = True\n return None\n\n if res['code'] == '503':\n self.error(\"EmailCrawlr service unavailable\")\n self.errorState = True\n return None\n\n # Catch all other non-200 status codes, and presume something went wrong\n if res['code'] != '200':\n self.error(\"Failed to retrieve content from EmailCrawlr\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if eventData in self.results:\n return\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_emailcrawlr but did not set an API key!\")\n self.errorState = True\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventName in [\"DOMAIN_NAME\"]:\n data = self.queryDomain(eventData)\n\n if data is None:\n self.debug(f\"No information found for domain {eventData}\")\n return\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n emails = data.get(\"emails\")\n\n if not emails:\n self.info(f\"No emails found for domain {eventData}\")\n return\n\n for res in emails:\n email = res.get('email')\n if email:\n mail_domain = email.lower().split('@')[1]\n if self.getTarget().matches(mail_domain, includeChildren=True):\n if email.split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evttype = \"EMAILADDR_GENERIC\"\n else:\n evttype = \"EMAILADDR\"\n\n evt = SpiderFootEvent(evttype, email, self.__name__, event)\n self.notifyListeners(evt)\n\n name = res.get('name')\n if name:\n full_name = name.get('name')\n if full_name and len(full_name) > 3:\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", f\"Possible full name: {full_name}\",\n self.__name__, event)\n self.notifyListeners(evt)\n\n phone_numbers = res.get('numbers')\n if phone_numbers:\n for number in phone_numbers:\n if number:\n evt = SpiderFootEvent(\"PHONE_NUMBER\", number, self.__name__, event)\n self.notifyListeners(evt)\n\n location = res.get('location')\n if location:\n loc = ', '.join([_f for _f in [location.get('city'), location.get('country')] if _f])\n if loc:\n evt = SpiderFootEvent(\"GEOINFO\", loc, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_emailcrawlr class\n" }, { "path": "modules/sfp_emailformat.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_emailformat\n# Purpose: SpiderFoot plug-in for retrieving e-mail addresses\n# belonging to your target from email-format.com.\n#\n# Author: \n#\n# Created: 29/09/2018\n# Copyright: (c) bcoles 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom bs4 import BeautifulSoup\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_emailformat(SpiderFootPlugin):\n\n meta = {\n 'name': \"EmailFormat\",\n 'summary': \"Look up e-mail addresses on email-format.com.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://www.email-format.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://www.email-format.com/i/api_access/\",\n \"https://www.email-format.com/i/api_v2/\",\n \"https://www.email-format.com/i/api_v1/\"\n ],\n 'favIcon': \"https://www.google.com/s2/favicons?domain=https://www.email-format.com/\",\n 'logo': \"https://www.google.com/s2/favicons?domain=https://www.email-format.com/\",\n 'description': \"Save time and energy - find the email address formats in use at thousands of companies.\",\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return ['INTERNET_NAME', \"DOMAIN_NAME\"]\n\n def producedEvents(self):\n return [\"EMAILADDR\", \"EMAILADDR_GENERIC\"]\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n # Get e-mail addresses on this domain\n res = self.sf.fetchUrl(f\"https://www.email-format.com/d/{eventData}/\", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])\n\n if res['content'] is None:\n return\n\n html = BeautifulSoup(res[\"content\"], features=\"lxml\")\n if not html:\n return\n\n tbody = html.find('tbody')\n if tbody:\n data = str(tbody.contents)\n else:\n # fall back to raw page contents\n data = res[\"content\"]\n\n emails = SpiderFootHelpers.extractEmailsFromText(data)\n for email in emails:\n # Skip unrelated emails\n mailDom = email.lower().split('@')[1]\n if not self.getTarget().matches(mailDom):\n self.debug(f\"Skipped address: {email}\")\n continue\n\n # Skip masked emails\n if re.match(r\"^[0-9a-f]{8}\\.[0-9]{7}@\", email):\n self.debug(f\"Skipped address: {email}\")\n continue\n\n self.info(f\"Found e-mail address: {email}\")\n if email.split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evttype = \"EMAILADDR_GENERIC\"\n else:\n evttype = \"EMAILADDR\"\n\n evt = SpiderFootEvent(evttype, email, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_emailformat class\n" }, { "path": "modules/sfp_emailrep.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_emailrep\n# Purpose: Searches EmailRep.io for email address reputation.\n#\n# Author: \n#\n# Created: 2019-08-07\n# Copyright: (c) bcoles 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_emailrep(SpiderFootPlugin):\n\n meta = {\n 'name': \"EmailRep\",\n 'summary': \"Search EmailRep.io for email address reputation.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://emailrep.io/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://docs.emailrep.io/\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://emailrep.io/free\",\n \"Request a free API Key\",\n \"The API key will be sent to registered email account on approval\"\n ],\n 'favIcon': \"https://emailrep.io/assets/img/favicon.png\",\n 'logo': \"https://emailrep.io/assets/img/logo-light.png\",\n 'description': \"Illuminate the \\\"reputation\\\" behind an email address.\\n\"\n \"EmailRep uses hundreds of factors like domain age, traffic rankings, \"\n \"presence on social media sites, professional networking sites, personal connections, \"\n \"public records, deliverability, data breaches, dark web credential leaks, \"\n \"phishing emails, threat actor emails, and more to answer these types of questions.\",\n }\n }\n\n opts = {\n 'api_key': '',\n }\n\n optdescs = {\n 'api_key': 'EmailRep API key.',\n }\n\n results = None\n errorState = False\n errorWarned = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return ['EMAILADDR']\n\n def producedEvents(self):\n return ['RAW_RIR_DATA', 'EMAILADDR_COMPROMISED', 'MALICIOUS_EMAILADDR']\n\n # https://emailrep.io/docs/\n def query(self, qry):\n headers = {\n 'Accept': \"application/json\"\n }\n\n if self.opts['api_key'] != '':\n headers['Key'] = self.opts['api_key']\n\n res = self.sf.fetchUrl(\n 'https://emailrep.io/' + qry,\n headers=headers,\n useragent='SpiderFoot',\n timeout=self.opts['_fetchtimeout']\n )\n\n # Documentation does not indicate rate limit threshold (50 queries/day)\n time.sleep(1)\n\n if res['content'] is None:\n return None\n\n if res['code'] == '400':\n self.error('API error: Bad request')\n self.errorState = True\n return None\n\n if res['code'] == '401':\n self.error('API error: Invalid API key')\n self.errorState = True\n return None\n\n if res['code'] == '429':\n self.error('API error: Too Many Requests')\n self.errorState = True\n return None\n\n if res['code'] != '200':\n self.error('Unexpected reply from EmailRep.io: ' + res['code'])\n self.errorState = True\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts['api_key'] == '' and not self.errorWarned:\n self.error(\"Warning: You enabled sfp_emailrep but did not set an API key! Queries will be rate limited.\")\n self.errorWarned = True\n\n res = self.query(eventData)\n\n if res is None:\n return\n\n details = res.get('details')\n\n if not details:\n return\n\n credentials_leaked = details.get('credentials_leaked')\n if credentials_leaked:\n evt = SpiderFootEvent('EMAILADDR_COMPROMISED', eventData + \" [Unknown]\", self.__name__, event)\n self.notifyListeners(evt)\n\n malicious_activity = details.get('malicious_activity')\n if malicious_activity:\n evt = SpiderFootEvent('MALICIOUS_EMAILADDR', 'EmailRep [' + eventData + ']', self.__name__, event)\n self.notifyListeners(evt)\n\n if malicious_activity or credentials_leaked:\n evt = SpiderFootEvent('RAW_RIR_DATA', str(res), self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_emailrep class\n" }, { "path": "modules/sfp_emergingthreats.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_emergingthreats\n# Purpose: Checks if an IP address or netblock is malicious according to\n# EmergingThreats.net.\n#\n# Author: steve@binarypool.com\n#\n# Created: 16/05/2020\n# Copyright: (c) Steve Micallef, 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom netaddr import IPAddress, IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_emergingthreats(SpiderFootPlugin):\n\n meta = {\n 'name': \"Emerging Threats\",\n 'summary': \"Check if a netblock or IP address is malicious according to EmergingThreats.net.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://rules.emergingthreats.net/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://doc.emergingthreats.net/\"\n ],\n 'favIcon': \"https://doc.emergingthreats.net/pub/Main/WebPreferences/favicon.ico\",\n 'logo': \"https://doc.emergingthreats.net/logo.png\",\n 'description': \"Emerging Threats delivers the most timely and accurate threat intelligence.\\n\"\n \"Emerging Threat (ET) intelligence helps prevent attacks and reduce risk by \"\n \"helping you understand the historical context of where these threats originated, \"\n \"who is behind them, when have they attacked, what methods they used, and what they're after. \"\n \"Get on-demand access to current and historical metadata on IPs, domains, \"\n \"and other related threat intelligence to help research threats and investigate incidents.\",\n }\n }\n\n opts = {\n 'checkaffiliates': True,\n 'cacheperiod': 18,\n 'checknetblocks': True,\n 'checksubnets': True\n }\n\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliate IP addresses?\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\",\n 'checknetblocks': \"Report if any malicious IPs are found within owned netblocks?\",\n 'checksubnets': \"Check if any malicious IPs are found within the same subnet of the target?\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"NETBLOCK_MEMBER\",\n \"NETBLOCK_OWNER\",\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"BLACKLISTED_SUBNET\",\n \"BLACKLISTED_NETBLOCK\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_SUBNET\",\n \"MALICIOUS_NETBLOCK\",\n ]\n\n def query(self, qry, targetType):\n cid = \"_emergingthreats\"\n url = \"https://rules.emergingthreats.net/blockrules/compromised-ips.txt\"\n\n data = dict()\n data[\"content\"] = self.sf.cacheGet(\"sfmal_\" + cid, self.opts.get('cacheperiod', 0))\n\n if data[\"content\"] is None:\n data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])\n\n if data[\"code\"] != \"200\":\n self.error(f\"Unable to fetch {url}\")\n self.errorState = True\n return None\n\n if data[\"content\"] is None:\n self.error(f\"Unable to fetch {url}\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"sfmal_\" + cid, data['content'])\n\n for line in data[\"content\"].split('\\n'):\n ip = line.strip().lower()\n\n if targetType == \"netblock\":\n try:\n if IPAddress(ip) in IPNetwork(qry):\n self.debug(f\"{ip} found within netblock/subnet {qry} in EmergingThreats.net list.\")\n return url\n except Exception as e:\n self.debug(f\"Error encountered parsing: {e}\")\n continue\n\n if targetType == \"ip\":\n if qry.lower() == ip:\n self.debug(f\"{qry} found in EmergingThreats.net list.\")\n return url\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName == 'IP_ADDRESS':\n targetType = 'ip'\n malicious_type = \"MALICIOUS_IPADDR\"\n blacklist_type = \"BLACKLISTED_IPADDR\"\n elif eventName == 'AFFILIATE_IPADDR':\n if not self.opts.get('checkaffiliates', False):\n return\n targetType = 'ip'\n malicious_type = \"MALICIOUS_AFFILIATE_IPADDR\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_IPADDR\"\n elif eventName == 'NETBLOCK_OWNER':\n if not self.opts.get('checknetblocks', False):\n return\n targetType = 'netblock'\n malicious_type = \"MALICIOUS_NETBLOCK\"\n blacklist_type = \"BLACKLISTED_NETBLOCK\"\n elif eventName == 'NETBLOCK_MEMBER':\n if not self.opts.get('checksubnets', False):\n return\n targetType = 'netblock'\n malicious_type = \"MALICIOUS_SUBNET\"\n blacklist_type = \"BLACKLISTED_SUBNET\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n self.debug(f\"Checking maliciousness of {eventData} with EmergingThreats.net\")\n\n url = self.query(eventData, targetType)\n\n if not url:\n return\n\n text = f\"EmergingThreats.net [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_emergingthreats class\n" }, { "path": "modules/sfp_errors.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_errors\n# Purpose: Identify common error messages in content like SQL errors, etc.\n#\n# Author: Steve Micallef \n#\n# Created: 18/01/2015\n# Copyright: (c) Steve Micallef 2015\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n# Taken from Google Dorks on exploit-db.com\nregexps = dict({\n \"PHP Error\": [\"PHP pase error\", \"PHP warning\", \"PHP error\",\n \"unexpected T_VARIABLE\", \"warning: failed opening\", \"include_path=\"],\n \"Generic Error\": [\"Internal Server Error\", \"Incorrect syntax\"],\n \"Oracle Error\": [r\"ORA-\\d+\", \"TNS:.?no listen\"],\n \"ASP Error\": [\"NET_SessionId\"],\n \"MySQL Error\": [r\"mysql_query\\(\", r\"mysql_connect\\(\"],\n \"ODBC Error\": [r\"\\[ODBC SQL\"]\n\n})\n\n\nclass sfp_errors(SpiderFootPlugin):\n\n meta = {\n 'name': \"Error String Extractor\",\n 'summary': \"Identify common error messages in content like SQL errors, etc.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Passive\"],\n 'categories': [\"Content Analysis\"]\n }\n\n # Default options\n opts = {}\n\n # Option descriptions\n optdescs = {\n # For each option in opts you should have a key/value pair here\n # describing it. It will end up in the UI to explain the option\n # to the end-user.\n }\n\n # Target\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.__dataSource__ = \"Target Website\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n # * = be notified about all events.\n def watchedEvents(self):\n return [\"TARGET_WEB_CONTENT\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"ERROR_MESSAGE\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n # We only want web content from the target\n if srcModuleName != \"sfp_spider\":\n return\n\n eventSource = event.actualSource\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventSource not in list(self.results.keys()):\n self.results[eventSource] = list()\n\n # We only want web content for pages on the target site\n if not self.getTarget().matches(self.sf.urlFQDN(eventSource)):\n self.debug(\"Not collecting web content information for external sites.\")\n return\n\n for regexpGrp in list(regexps.keys()):\n if regexpGrp in self.results[eventSource]:\n continue\n\n for regex in regexps[regexpGrp]:\n pat = re.compile(regex, re.IGNORECASE)\n matches = re.findall(pat, eventData)\n if len(matches) > 0 and regexpGrp not in self.results[eventSource]:\n self.info(\"Matched \" + regexpGrp + \" in content from \" + eventSource)\n self.results[eventSource] = self.results[eventSource] + [regexpGrp]\n evt = SpiderFootEvent(\"ERROR_MESSAGE\", regexpGrp,\n self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_errors class\n" }, { "path": "modules/sfp_ethereum.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_ethereum\n# Purpose: SpiderFoot plug-in for scanning retrieved content by other\n# modules (such as sfp_spider) and identifying ethereum addresses.\n#\n# Author: Steve Micallef \n#\n# Created: 03/09/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_ethereum(SpiderFootPlugin):\n\n meta = {\n 'name': \"Ethereum Address Extractor\",\n 'summary': \"Identify ethereum addresses in scraped webpages.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Content Analysis\"]\n }\n\n # Default options\n opts = {}\n optdescs = {}\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"TARGET_WEB_CONTENT\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"ETHEREUM_ADDRESS\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n sourceData = self.sf.hashstring(eventData)\n\n if sourceData in self.results:\n return\n\n self.results[sourceData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n # thanks to https://stackoverflow.com/questions/21683680/regex-to-match-ethereum-addresses\n matches = re.findall(r\"[\\s:=\\>](0x[a-fA-F0-9]{40})\", eventData)\n for m in matches:\n self.debug(\"Ethereum address match: \" + m)\n evt = SpiderFootEvent(\"ETHEREUM_ADDRESS\", m, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_ethereum class\n" }, { "path": "modules/sfp_etherscan.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_etherscan\n# Purpose: SpiderFoot plug-in to look up a ethereum wallet's balance by\n# querying etherscan.io.\n#\n# Author: Krishnasis Mandal \n#\n# Created: 26/01/2021\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_etherscan(SpiderFootPlugin):\n\n meta = {\n 'name': \"Etherscan\",\n 'summary': \"Queries etherscan.io to find the balance of identified ethereum wallet addresses.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Public Registries\"],\n 'dataSource': {\n 'website': \"https://etherscan.io\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://etherscan.io/apis\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://etherscan.io\",\n \"Register a free account\",\n \"Browse to https://etherscan.io/myapikey\",\n \"Click on Add beside API Key\",\n \"Your API Key will be listed under API Key Token\",\n ],\n 'favIcon': \"https://etherscan.io/images/favicon3.ico\",\n 'logo': \"https://etherscan.io/images/brandassets/etherscan-logo-circle.png\",\n 'description': \"Etherscan allows you to explore and search the Ethereum blockchain \"\n \"for transactions, addresses, tokens, prices and other activities taking place on Ethereum (ETH)\",\n }\n }\n\n # Default options\n opts = {\n 'api_key': '',\n 'pause': 1\n }\n\n # Option descriptions\n optdescs = {\n 'api_key': \"API Key for etherscan.io\",\n 'pause': \"Number of seconds to wait between each API call.\"\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\n \"ETHEREUM_ADDRESS\"\n ]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\n \"ETHEREUM_BALANCE\",\n \"RAW_RIR_DATA\"\n ]\n\n def query(self, qry):\n queryString = f\"https://api.etherscan.io/api?module=account&action=balance&address={qry}&tag=latest&apikey={self.opts['api_key']}\"\n # Wallet balance\n res = self.sf.fetchUrl(queryString,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'])\n\n time.sleep(self.opts['pause'])\n\n if res['content'] is None:\n self.info(f\"No Etherscan data found for {qry}\")\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.errorState:\n return\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_etherscan but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n data = self.query(eventData)\n if data is None:\n self.info(f\"No Etherscan data found for {eventData}\")\n return\n\n # Value returned by etherscan was too large in comparison to actual wallet balance\n balance = float(data.get('result')) / 1000000000000000000\n\n evt = SpiderFootEvent(\"ETHEREUM_BALANCE\", f\"{str(balance)} ETH\", self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_etherscan class\n" }, { "path": "modules/sfp_filemeta.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_filemeta\n# Purpose: From Spidering and from searching search engines, extracts file\n# meta data from files matching certain file extensions.\n#\n# Author: Steve Micallef \n#\n# Created: 25/04/2014\n# Copyright: (c) Steve Micallef 2014\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport io\nimport mimetypes\n\nimport PyPDF2\n\nimport docx\n\nimport exifread\n\nimport pptx\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_filemeta(SpiderFootPlugin):\n\n meta = {\n 'name': \"File Metadata Extractor\",\n 'summary': \"Extracts meta data from documents and images.\",\n 'flags': [],\n 'useCases': [\"Footprint\"],\n 'categories': [\"Content Analysis\"]\n }\n\n # Default options\n opts = {\n 'fileexts': [\"docx\", \"pptx\", 'pdf', 'jpg', 'jpeg', 'tiff', 'tif'],\n 'timeout': 300\n }\n\n # Option descriptions\n optdescs = {\n 'fileexts': \"File extensions of files you want to analyze the meta data of (only PDF, DOCX, XLSX and PPTX are supported.)\",\n 'timeout': \"Download timeout for files, in seconds.\"\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.__dataSource__ = \"Target Website\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"LINKED_URL_INTERNAL\", \"INTERESTING_FILE\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"RAW_FILE_META_DATA\", \"SOFTWARE_USED\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n for fileExt in self.opts['fileexts']:\n if self.checkForStop():\n return\n\n if \".\" + fileExt.lower() in eventData.lower():\n # Fetch the file, allow much more time given that these files are\n # typically large.\n ret = self.sf.fetchUrl(eventData, timeout=self.opts['timeout'],\n useragent=self.opts['_useragent'], disableContentEncoding=True,\n sizeLimit=10000000,\n verify=False)\n if ret['content'] is None:\n self.error(f\"Unable to fetch file for meta analysis: {eventData}\")\n return\n\n if len(ret['content']) < 512:\n self.error(f\"Strange content encountered, size of {len(ret['content'])}\")\n return\n\n meta = None\n data = None\n # Based on the file extension, handle it\n if fileExt.lower() == \"pdf\":\n try:\n raw = io.BytesIO(ret['content'])\n # data = metapdf.MetaPdfReader().read_metadata(raw)\n pdf = PyPDF2.PdfFileReader(raw, strict=False)\n data = pdf.getDocumentInfo()\n meta = str(data)\n self.debug(\"Obtained meta data from \" + eventData)\n except Exception as e:\n self.error(f\"Unable to parse meta data from: {eventData} ({e})\")\n return\n\n if fileExt.lower() in [\"docx\"]:\n try:\n c = io.BytesIO(ret['content'])\n doc = docx.Document(c)\n mtype = mimetypes.guess_type(eventData)[0]\n self.debug(\"Office type: \" + str(mtype))\n a = doc.core_properties.author\n c = doc.core_properties.comments\n data = [_f for _f in [a, c] if _f]\n meta = \", \".join(data)\n except Exception as e:\n self.error(f\"Unable to process file: {eventData} ({e})\")\n return\n\n if fileExt.lower() in [\"pptx\"]:\n try:\n c = io.BytesIO(ret['content'])\n doc = pptx.Presentation(c)\n mtype = mimetypes.guess_type(eventData)[0]\n self.debug(\"Office type: \" + str(mtype))\n a = doc.core_properties.author\n c = doc.core_properties.comments\n data = [_f for _f in [a, c] if _f]\n meta = \", \".join(data)\n except Exception as e:\n self.error(f\"Unable to process file: {eventData} ({e})\")\n return\n\n if fileExt.lower() in [\"jpg\", \"jpeg\", \"tiff\"]:\n try:\n raw = io.BytesIO(ret['content'])\n data = exifread.process_file(raw)\n if data is None or len(data) == 0:\n continue\n meta = str(data)\n except Exception as e:\n self.error(f\"Unable to parse meta data from: {eventData} ({e})\")\n return\n\n if meta is not None and data is not None:\n rawevt = SpiderFootEvent(\"RAW_FILE_META_DATA\", meta,\n self.__name__, event)\n self.notifyListeners(rawevt)\n\n val = list()\n try:\n if \"/Producer\" in data:\n val.append(str(data['/Producer']))\n\n if \"/Creator\" in data:\n val.append(str(data['/Creator']))\n\n if \"Application\" in data:\n val.append(str(data['Application']))\n\n if \"Image Software\" in data:\n val.append(str(data['Image Software']))\n except Exception as e:\n self.error(\"Failed to parse PDF, \" + eventData + \": \" + str(e))\n return\n\n for v in val:\n if v and not isinstance(v, PyPDF2.generic.NullObject):\n self.debug(\"VAL: \" + str(val))\n # Strip non-ASCII\n v = ''.join([i if ord(i) < 128 else ' ' for i in v])\n evt = SpiderFootEvent(\"SOFTWARE_USED\", v, self.__name__, rawevt)\n self.notifyListeners(evt)\n" }, { "path": "modules/sfp_flickr.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_flickr\n# Purpose: Search Flickr API for domains, URLs and emails related to the\n# specified domain.\n#\n# Author: \n#\n# Created: 2018-10-08\n# Copyright: (c) bcoles 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport re\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_flickr(SpiderFootPlugin):\n\n meta = {\n 'name': \"Flickr\",\n 'summary': \"Search Flickr for domains, URLs and emails related to the specified domain.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Social Media\"],\n 'dataSource': {\n 'website': \"https://www.flickr.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://www.flickr.com/services/api/\",\n \"https://www.flickr.com/services/developer/api/\",\n \"https://code.flickr.net/\"\n ],\n 'favIcon': \"https://combo.staticflickr.com/pw/favicon.ico\",\n 'logo': \"https://combo.staticflickr.com/pw/favicon.ico\",\n 'description': \"Flickr is almost certainly the best online photo management and sharing application in the world.\\n \"\n \"On Flickr, members upload photos, share them securely, supplement their photos with \"\n \"metadata like license information, geo-location, people, tags, etc., \"\n \"and interact with their family, friends, contacts or anyone in the community. \"\n \"Practically all the features on Flickr's various platforms -- web, mobile and desktop -- \"\n \"are accompanied by a longstanding API program. \"\n \"Since 2005, developers have collaborated on top of Flickr's APIs to build fun, creative, \"\n \"and gorgeous experiences around photos that extend beyond Flickr.\",\n }\n }\n\n # Default options\n opts = {\n 'pause': 1,\n 'per_page': 100,\n 'maxpages': 20,\n 'dns_resolve': True,\n }\n\n # Option descriptions\n optdescs = {\n 'pause': \"Number of seconds to pause between fetches.\",\n 'per_page': \"Maximum number of results per page.\",\n 'maxpages': \"Maximum number of pages of results to fetch.\",\n 'dns_resolve': \"DNS resolve each identified domain.\",\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"EMAILADDR\", \"EMAILADDR_GENERIC\", \"INTERNET_NAME\",\n \"DOMAIN_NAME\", \"LINKED_URL_INTERNAL\"]\n\n # Retrieve API key\n def retrieveApiKey(self):\n res = self.sf.fetchUrl(\"https://www.flickr.com/\", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])\n\n if res['content'] is None:\n return None\n\n keys = re.findall(r'YUI_config.flickr.api.site_key = \"([a-zA-Z0-9]+)\"', str(res['content']))\n\n if not keys:\n return None\n\n return keys[0]\n\n # Query the REST API\n def query(self, qry, api_key, page=1, per_page=200):\n params = {\n \"sort\": \"relevance\",\n \"parse_tags\": \"1\",\n \"content_type\": \"7\",\n \"extras\": \"description,owner_name,path_alias,realname\",\n \"hermes\": \"1\",\n \"hermesClient\": \"1\",\n \"reqId\": \"\",\n \"nojsoncallback\": \"1\",\n \"viewerNSID\": \"\",\n \"method\": \"flickr.photos.search\",\n \"csrf\": \"\",\n \"lang\": \"en-US\",\n \"per_page\": str(per_page),\n \"page\": str(page),\n \"text\": qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n \"api_key\": api_key,\n \"format\": \"json\"\n }\n\n res = self.sf.fetchUrl(\"https://api.flickr.com/services/rest?\" + urllib.parse.urlencode(params),\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout'])\n\n time.sleep(self.opts['pause'])\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked\")\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if srcModuleName == 'sfp_flickr':\n self.debug(f\"Ignoring {eventData}, from self.\")\n return\n\n # Retrieve API key\n api_key = self.retrieveApiKey()\n\n if not api_key:\n self.error(\"Failed to obtain API key\")\n return\n\n self.debug(f\"Retrieved API key: {api_key}\")\n\n # Query API for event data\n hosts = list()\n page = 1\n pages = self.opts['maxpages']\n per_page = self.opts['per_page']\n while page <= pages:\n if self.checkForStop():\n return\n\n if self.errorState:\n return\n\n data = self.query(eventData, api_key, page=page, per_page=per_page)\n\n if data is None:\n return\n\n # Check the response is ok\n if data.get('stat') != \"ok\":\n self.debug(\"Error retrieving search results.\")\n return\n\n photos = data.get('photos')\n\n if not photos:\n self.debug(\"No search results.\")\n return\n\n # Calculate number of pages to retrieve\n result_pages = int(photos.get('pages', 0))\n\n if result_pages < pages:\n pages = result_pages\n\n if 'max_allowed_pages' in photos:\n allowed_pages = int(photos.get('max_allowed_pages', 0))\n if pages > allowed_pages:\n pages = allowed_pages\n\n self.info(f\"Parsing page {page} of {pages}\")\n\n # Extract data\n for photo in photos.get('photo', list()):\n emails = SpiderFootHelpers.extractEmailsFromText(str(photo))\n for email in emails:\n if email in self.results:\n continue\n\n mail_domain = email.lower().split('@')[1]\n\n if not self.getTarget().matches(mail_domain, includeChildren=True, includeParents=True):\n self.debug(f\"Skipped unrelated address: {email}\")\n continue\n\n self.info(\"Found e-mail address: \" + email)\n if email.split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evttype = \"EMAILADDR_GENERIC\"\n else:\n evttype = \"EMAILADDR\"\n\n evt = SpiderFootEvent(evttype, email, self.__name__, event)\n self.notifyListeners(evt)\n self.results[email] = True\n\n links = SpiderFootHelpers.extractUrlsFromText(str(photo))\n for link in links:\n if link in self.results:\n continue\n\n host = self.sf.urlFQDN(link)\n\n if not self.getTarget().matches(host, includeChildren=True, includeParents=True):\n self.debug(f\"Skipped unrelated URL: {link}\")\n continue\n\n hosts.append(host)\n\n self.debug(f\"Found a URL: {link}\")\n evt = SpiderFootEvent('LINKED_URL_INTERNAL', link, self.__name__, event)\n self.notifyListeners(evt)\n self.results[link] = True\n\n page += 1\n\n for host in set(hosts):\n if self.checkForStop():\n return\n\n if self.errorState:\n return\n\n if self.opts['dns_resolve'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host):\n self.debug(f\"Host {host} could not be resolved\")\n evt = SpiderFootEvent(\"INTERNET_NAME_UNRESOLVED\", host, self.__name__, event)\n self.notifyListeners(evt)\n continue\n\n evt = SpiderFootEvent(\"INTERNET_NAME\", host, self.__name__, event)\n self.notifyListeners(evt)\n if self.sf.isDomain(host, self.opts[\"_internettlds\"]):\n evt = SpiderFootEvent(\"DOMAIN_NAME\", host, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_flickr class\n" }, { "path": "modules/sfp_focsec.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_focsec\n# Purpose: Look up IP address information from Focsec.\n#\n# Author: \n#\n# Created: 2021-10-09\n# Copyright: (c) bcoles 2021\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport urllib\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_focsec(SpiderFootPlugin):\n\n meta = {\n 'name': \"Focsec\",\n 'summary': \"Look up IP address information from Focsec.\",\n 'flags': ['apikey'],\n 'useCases': [\"Passive\", \"Footprint\", \"Investigate\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://focsec.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://docs.focsec.com/#ip\",\n ],\n \"apiKeyInstructions\": [\n \"Visit https://focsec.com/signup\",\n \"Register an account\",\n \"Visit https://focsec.com/account/dashboard and use the API key provided\",\n ],\n 'favIcon': \"https://focsec.com/static/favicon.png\",\n 'logo': \"https://focsec.com/static/web/images/logo.png\",\n 'description': \"Our API lets you know if a user's IP address is associated with a VPN, Proxy, TOR or malicious bots.\"\n \"Take your applications security to the next level by detecting suspicious activity early on.\"\n }\n }\n\n opts = {\n \"api_key\": \"\",\n }\n\n optdescs = {\n \"api_key\": \"Focsec API Key.\",\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.errorState = False\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\"\n ]\n\n def producedEvents(self):\n return [\n \"RAW_RIR_DATA\",\n \"GEOINFO\",\n \"MALICIOUS_IPADDR\",\n \"PROXY_HOST\",\n \"VPN_HOST\",\n \"TOR_EXIT_NODE\",\n ]\n\n def query(self, qry):\n \"\"\"Retrieve IP address information from Focsec.\n\n Args:\n qry (str): IPv4/IPv6 address\n\n Returns:\n dict: JSON formatted results\n \"\"\"\n\n params = urllib.parse.urlencode({\n 'api_key': self.opts[\"api_key\"],\n })\n\n res = self.sf.fetchUrl(\n f\"https://api.focsec.com/v1/ip/{qry}?{params}\",\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=self.opts['_useragent']\n )\n\n if not res:\n self.error(\"No response from Focsec.\")\n return None\n\n if res['code'] == \"400\":\n self.error(\"Bad request.\")\n self.errorState = True\n return None\n\n if res['code'] == \"401\":\n self.error(\"Unauthorized - Invalid API key.\")\n self.errorState = True\n return None\n\n if res['code'] == \"402\":\n self.error(\"Unauthorized - Payment Required. Subscription or trial period expired.\")\n self.errorState = True\n return None\n\n if res['code'] == \"404\":\n self.debug(f\"No results for {qry}\")\n return None\n\n # Future proofing - Focsec does not implement rate limiting\n if res['code'] == \"429\":\n self.error(\"You are being rate-limited by Focsec.\")\n return None\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from Focsec.\")\n return None\n\n if not res['content']:\n self.debug(\"No results from Focsec.\")\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\n f\"You enabled {self.__class__.__name__} but did not set an API key!\"\n )\n self.errorState = True\n return\n\n data = self.query(eventData)\n\n if not data:\n self.debug(f\"Found no results for {eventData}\")\n return\n\n e = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(e)\n\n is_bot = data.get('is_bot')\n if is_bot:\n e = SpiderFootEvent(\"MALICIOUS_IPADDR\", f\"Focsec [{eventData}]\", self.__name__, event)\n self.notifyListeners(e)\n\n is_tor = data.get('is_tor')\n if is_tor:\n e = SpiderFootEvent(\"TOR_EXIT_NODE\", eventData, self.__name__, event)\n self.notifyListeners(e)\n\n is_vpn = data.get('is_vpn')\n if is_vpn:\n e = SpiderFootEvent(\"VPN_HOST\", eventData, self.__name__, event)\n self.notifyListeners(e)\n\n is_proxy = data.get('is_proxy')\n if is_proxy:\n e = SpiderFootEvent(\"PROXY_HOST\", eventData, self.__name__, event)\n self.notifyListeners(e)\n\n location = ', '.join(\n filter(\n None,\n [\n data.get('city'),\n data.get('country'),\n ]\n )\n )\n\n if location:\n e = SpiderFootEvent(\"GEOINFO\", location, self.__name__, event)\n self.notifyListeners(e)\n\n# End of sfp_focsec class\n" }, { "path": "modules/sfp_fortinet.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_fortinet\n# Purpose: Check if an IP address is malicious according to FortiGuard Antispam.\n#\n# Author: steve@binarypool.com\n#\n# Created: 14/12/2013\n# Copyright: (c) Steve Micallef, 2013\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_fortinet(SpiderFootPlugin):\n\n meta = {\n 'name': \"FortiGuard Antispam\",\n 'summary': \"Check if an IP address is malicious according to FortiGuard Antispam.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://www.fortiguard.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://www.fortiguard.com/learnmore#as\",\n ],\n 'favIcon': \"https://www.fortiguard.com/static/images/favicon.ico\",\n 'logo': \"https://www.fortiguard.com/static/images/Fortinet-logo%20white.png?v=880\",\n 'description': \"FortiGuard Antispam provides a comprehensive and multi-layered approach to detect and filter spam processed by organizations.\"\n }\n }\n\n opts = {\n 'checkaffiliates': True\n }\n\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliates?\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"AFFILIATE_IPV6_ADDRESS\",\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n ]\n\n def query(self, ip):\n if not ip:\n return None\n\n res = self.sf.fetchUrl(\n f\"https://www.fortiguard.com/search?q={ip}&engine=8\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from FortiGuard Antispam.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from FortiGuard Antispam\")\n self.errorState = True\n return None\n\n return res['content']\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']:\n malicious_type = 'MALICIOUS_IPADDR'\n blacklist_type = 'BLACKLISTED_IPADDR'\n elif eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS']:\n if not self.opts.get('checkaffiliates', False):\n return\n malicious_type = 'MALICIOUS_AFFILIATE_IPADDR'\n blacklist_type = 'BLACKLISTED_AFFILIATE_IPADDR'\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n data = self.query(eventData)\n\n if not data:\n return\n\n if \"Your signature is on the blocklist\" not in data:\n return\n\n url = f\"https://www.fortiguard.com/search?q={eventData}&engine=8\"\n text = f\"FortiGuard Antispam [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_fortinet class\n" }, { "path": "modules/sfp_fraudguard.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_fraudguard\n# Purpose: Query fraudguard.io using their API\n#\n# Author: Steve Micallef\n#\n# Created: 18/06/2017\n# Copyright: (c) Steve Micallef 2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport base64\nimport json\nimport time\nfrom datetime import datetime\n\nfrom netaddr import IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_fraudguard(SpiderFootPlugin):\n\n meta = {\n 'name': \"Fraudguard\",\n 'summary': \"Obtain threat information from Fraudguard.io\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://fraudguard.io/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://docs.fraudguard.io/\",\n \"https://faq.fraudguard.io/\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://app.fraudguard.io/register\",\n \"Register a free account\",\n \"Navigate to https://app.fraudguard.io/keys\",\n \"The API key combination is listed under Username and Password\"\n ],\n 'favIcon': \"https://fraudguard.io/img/favicon.ico\",\n 'logo': \"https://s3.amazonaws.com/fraudguard.io/img/header.png\",\n 'description': \"FraudGuard is a service designed to provide an easy way to validate usage \"\n \"by continuously collecting and analyzing real-time internet traffic. \"\n \"Utilizing just a few simple API endpoints we make integration as simple as possible \"\n \"and return data such as: Risk Level, Threat Type, Geo Location, etc. Super fast, super simple.\\n\"\n \"Lookup any IP address by querying our threat engine.\",\n }\n }\n\n # Default options\n opts = {\n \"fraudguard_api_key_account\": \"\",\n \"fraudguard_api_key_password\": \"\",\n \"age_limit_days\": 90,\n 'netblocklookup': True,\n 'maxnetblock': 24,\n 'maxv6netblock': 120,\n 'subnetlookup': True,\n 'maxsubnet': 24,\n 'maxv6subnet': 120,\n 'checkaffiliates': True\n }\n\n # Option descriptions\n optdescs = {\n \"fraudguard_api_key_account\": \"Fraudguard.io API username.\",\n \"fraudguard_api_key_password\": \"Fraudguard.io API password.\",\n \"age_limit_days\": \"Ignore any records older than this many days. 0 = unlimited.\",\n 'netblocklookup': \"Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?\",\n 'maxnetblock': \"If looking up owned netblocks, the maximum IPv4 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'maxv6netblock': \"If looking up owned netblocks, the maximum IPv6 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'subnetlookup': \"Look up all IPs on subnets which your target is a part of for blacklisting?\",\n 'maxsubnet': \"If looking up subnets, the maximum IPv4 subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'maxv6subnet': \"If looking up subnets, the maximum IPv6 subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'checkaffiliates': \"Apply checks to affiliates?\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.errorState = False\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"AFFILIATE_IPV6_ADDRESS\",\n \"NETBLOCK_MEMBER\",\n \"NETBLOCKV6_MEMBER\",\n \"NETBLOCK_OWNER\",\n \"NETBLOCKV6_OWNER\",\n ]\n\n def producedEvents(self):\n return [\n \"GEOINFO\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_SUBNET\",\n \"MALICIOUS_NETBLOCK\"\n ]\n\n def query(self, qry):\n \"\"\"Query IP address\n\n Args:\n qry (str): IPv4/IPv6 address\n\n Returns:\n dict: JSON formatted results\n \"\"\"\n\n fraudguard_url = \"https://api.fraudguard.io/ip/\" + qry\n api_key_account = self.opts['fraudguard_api_key_account']\n if type(api_key_account) == str:\n api_key_account = api_key_account.encode('utf-8')\n api_key_password = self.opts['fraudguard_api_key_password']\n if type(api_key_password) == str:\n api_key_password = api_key_password.encode('utf-8')\n token = base64.b64encode(api_key_account + ':'.encode('utf-8') + api_key_password)\n headers = {\n 'Authorization': \"Basic \" + token.decode('utf-8')\n }\n\n res = self.sf.fetchUrl(\n fraudguard_url,\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\",\n headers=headers\n )\n\n if res['code'] in [\"400\", \"429\", \"500\", \"403\"]:\n self.error(\"Fraudguard.io API key seems to have been rejected or you have exceeded usage limits for the month.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.info(f\"No Fraudguard.io info found for {qry}\")\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from Fraudguard.io: {e}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts['fraudguard_api_key_account'] == \"\" or self.opts['fraudguard_api_key_password'] == \"\":\n self.error(\"You enabled sfp_fraudguard but did not set an API username/password!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if eventName.startswith(\"AFFILIATE\") and not self.opts['checkaffiliates']:\n return\n\n if eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']:\n if not self.opts['netblocklookup']:\n return\n\n if eventName == 'NETBLOCKV6_OWNER':\n max_netblock = self.opts['maxv6netblock']\n else:\n max_netblock = self.opts['maxnetblock']\n\n if IPNetwork(eventData).prefixlen < max_netblock:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}\")\n return\n\n if eventName in ['NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']:\n if not self.opts['subnetlookup']:\n return\n\n if eventName == 'NETBLOCKV6_MEMBER':\n max_subnet = self.opts['maxv6subnet']\n else:\n max_subnet = self.opts['maxsubnet']\n\n if IPNetwork(eventData).prefixlen < max_subnet:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}\")\n return\n\n if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS', 'NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']:\n evtType = 'MALICIOUS_IPADDR'\n elif eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS', 'NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']:\n evtType = 'MALICIOUS_AFFILIATE_IPADDR'\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n qrylist = list()\n if eventName.startswith(\"NETBLOCK\"):\n for ipaddr in IPNetwork(eventData):\n qrylist.append(str(ipaddr))\n self.results[str(ipaddr)] = True\n else:\n qrylist.append(eventData)\n self.results[eventData] = True\n\n for addr in qrylist:\n if self.checkForStop():\n return\n\n data = self.query(addr)\n\n if not data:\n continue\n\n self.debug(f\"Found results for {addr} in Fraudguard.io\")\n\n # Format: 2016-12-24T07:25:35+00:00'\n created_dt = datetime.strptime(data.get('discover_date'), '%Y-%m-%d %H:%M:%S')\n created_ts = int(time.mktime(created_dt.timetuple()))\n age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])\n if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts:\n self.debug(f\"Record found but too old ({created_dt}), skipping.\")\n continue\n\n # For netblocks, we need to create the IP address event so that\n # the threat intel event is more meaningful.\n if eventName == 'NETBLOCK_OWNER':\n pevent = SpiderFootEvent(\"IP_ADDRESS\", addr, self.__name__, event)\n self.notifyListeners(pevent)\n elif eventName == 'NETBLOCKV6_OWNER':\n pevent = SpiderFootEvent(\"IPV6_ADDRESS\", addr, self.__name__, event)\n self.notifyListeners(pevent)\n elif eventName == 'NETBLOCK_MEMBER':\n pevent = SpiderFootEvent(\"AFFILIATE_IPADDR\", addr, self.__name__, event)\n self.notifyListeners(pevent)\n elif eventName == 'NETBLOCKV6_MEMBER':\n pevent = SpiderFootEvent(\"AFFILIATE_IPV6_ADDRESS\", addr, self.__name__, event)\n self.notifyListeners(pevent)\n else:\n pevent = event\n\n geoinfo = [\n _f for _f in [\n data.get('state'),\n data.get('city'),\n data.get('postal_code'),\n data.get('country')\n ] if _f and _f != \"unknown\"\n ]\n if geoinfo:\n location = ', '.join(filter(None, geoinfo))\n e = SpiderFootEvent(\"GEOINFO\", location, self.__name__, pevent)\n self.notifyListeners(e)\n\n threat = data.get('threat')\n if threat and threat != \"unknown\":\n risk_level = data.get('risk_level')\n e = SpiderFootEvent(evtType, f\"{threat} (risk level: {risk_level}) [{addr}]\", self.__name__, pevent)\n self.notifyListeners(e)\n\n# End of sfp_fraudguard class\n" }, { "path": "modules/sfp_fsecure_riddler.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_fsecure_riddler\n# Purpose: Query F-Secure Riddler.io API.\n#\n# Author: \n#\n# Created: 2019-09-16\n# Copyright: (c) bcoles 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_fsecure_riddler(SpiderFootPlugin):\n\n meta = {\n 'name': \"F-Secure Riddler.io\",\n 'summary': \"Obtain network information from F-Secure Riddler.io API.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Investigate\", \"Footprint\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://riddler.io/\",\n 'model': \"PRIVATE_ONLY\",\n 'references': [\n \"https://riddler.io/help/api\",\n \"https://riddler.io/help/search\",\n \"https://riddler.io/static/riddler_white_paper.pdf\",\n \"https://www.f-secure.com/en/business/products/vulnerability-management/radar\"\n ],\n 'apiKeyInstructions': [\n \"Registration is disabled for new accounts\"\n ],\n 'favIcon': \"https://riddler.io/static/images/favicon.png\",\n 'logo': \"https://riddler.io/static/images/logo.png\",\n 'description': \"Riddler.io allows you to search in a high quality dataset with more than 396,831,739 hostnames. \"\n \"Unlike others, we do not rely on simple port scanning techniques - we crawl the web, \"\n \"ensuring an in-depth quality data set you will not find anywhere else.\\n\"\n \"Use Riddler to enumerate possible attack vectors during your pen-test or use the very same data \"\n \"to monitor potential threats before it is too late.\",\n }\n }\n\n opts = {\n 'verify': True,\n 'username': '',\n 'password': ''\n }\n\n optdescs = {\n 'verify': 'Verify host names resolve',\n 'username': 'F-Secure Riddler.io username',\n 'password': 'F-Secure Riddler.io password'\n }\n\n results = None\n token = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return ['DOMAIN_NAME', 'INTERNET_NAME',\n 'INTERNET_NAME_UNRESOLVED', 'IP_ADDRESS']\n\n def producedEvents(self):\n return ['INTERNET_NAME', 'AFFILIATE_INTERNET_NAME',\n 'INTERNET_NAME_UNRESOLVED', 'AFFILIATE_INTERNET_NAME_UNRESOLVED',\n 'DOMAIN_NAME', 'AFFILIATE_DOMAIN_NAME',\n 'IP_ADDRESS',\n 'PHYSICAL_COORDINATES', 'RAW_RIR_DATA']\n\n # https://riddler.io/help/api\n def login(self):\n params = {\n 'email': self.opts['username'].encode('raw_unicode_escape').decode(\"ascii\"),\n 'password': self.opts['password'].encode('raw_unicode_escape').decode(\"ascii\")\n }\n headers = {\n 'Content-Type': 'application/json',\n }\n\n res = self.sf.fetchUrl('https://riddler.io/auth/login',\n postData=json.dumps(params),\n headers=headers,\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout'])\n\n if res['content'] is None:\n return\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from F-Secure Riddler: {e}\")\n return\n\n try:\n token = data.get('response').get('user').get('authentication_token')\n except Exception:\n self.error('Login failed')\n self.errorState = True\n return\n\n if not token:\n self.error('Login failed')\n self.errorState = True\n return\n\n self.token = token\n\n # https://riddler.io/help/search\n def query(self, qry):\n params = {\n 'query': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n }\n headers = {\n 'Authentication-Token': self.token,\n 'Content-Type': 'application/json',\n }\n\n res = self.sf.fetchUrl('https://riddler.io/api/search',\n postData=json.dumps(params),\n headers=headers,\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout'])\n\n time.sleep(1)\n\n if res['code'] in [\"400\", \"401\", \"402\", \"403\", \"500\"]:\n self.error(f\"Unexpected HTTP response code {res['code']} from F-Secure Riddler\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from F-Secure Riddler: {e}\")\n return None\n\n if not data:\n self.debug(f\"No results found for {qry}\")\n return None\n\n return data\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if srcModuleName == 'sfp_fsecure_riddler':\n self.debug(f\"Ignoring {eventName}, from self.\")\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.opts['username'] == '' or self.opts['password'] == '':\n self.error('You enabled sfp_fsecure_riddler but did not set an API username/password!')\n self.errorState = True\n return\n\n if not self.token:\n self.login()\n\n if not self.token:\n self.error('Could not login to F-Secure Riddler')\n self.errorState = True\n return\n\n self.results[eventData] = True\n\n data = None\n\n if eventName in ['INTERNET_NAME', 'DOMAIN_NAME']:\n data = self.query(\"pld:\" + eventData)\n elif eventName == 'IP_ADDRESS':\n data = self.query(\"ip:\" + eventData)\n\n if not data:\n self.info(f\"No results found for {eventData}\")\n return\n\n e = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(e)\n\n hosts = list()\n addrs = list()\n coords = list()\n\n for result in data:\n host = result.get('host')\n\n if not host:\n continue\n\n if not self.getTarget().matches(host, includeChildren=True, includeParents=True):\n continue\n\n hosts.append(host)\n\n addr = result.get('addr')\n\n if addr:\n addrs.append(addr)\n\n coord = result.get('cordinates')\n\n if coord and len(coord) == 2:\n coords.append(str(coord[0]) + ', ' + str(coord[1]))\n\n if self.opts['verify'] and len(hosts) > 0:\n self.info(\"Resolving \" + str(len(set(hosts))) + \" domains ...\")\n\n for host in set(hosts):\n if self.getTarget().matches(host, includeChildren=True, includeParents=True):\n evt_type = 'INTERNET_NAME'\n else:\n evt_type = 'AFFILIATE_INTERNET_NAME'\n\n if self.opts['verify'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host):\n self.debug(f\"Host {host} could not be resolved\")\n evt_type += '_UNRESOLVED'\n\n evt = SpiderFootEvent(evt_type, host, self.__name__, event)\n self.notifyListeners(evt)\n\n if self.sf.isDomain(host, self.opts['_internettlds']):\n if evt_type.startswith('AFFILIATE'):\n evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', host, self.__name__, event)\n self.notifyListeners(evt)\n else:\n evt = SpiderFootEvent('DOMAIN_NAME', host, self.__name__, event)\n self.notifyListeners(evt)\n\n for addr in set(addrs):\n if self.sf.validIP(addr):\n evt = SpiderFootEvent('IP_ADDRESS', addr, self.__name__, event)\n self.notifyListeners(evt)\n\n for coord in set(coords):\n evt = SpiderFootEvent('PHYSICAL_COORDINATES', coord, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_fsecure_riddler class\n" }, { "path": "modules/sfp_fullcontact.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_fullcontact\n# Purpose: Gather domain and e-mail information from FullContact.com API.\n#\n# Author: Steve Micallef \n#\n# Created: 06/02/2018\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nfrom datetime import datetime\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_fullcontact(SpiderFootPlugin):\n\n meta = {\n 'name': \"FullContact\",\n 'summary': \"Gather domain and e-mail information from FullContact.com API.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://www.fullcontact.com\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://dashboard.fullcontact.com/api-ref\",\n \"https://www.fullcontact.com/developer-portal/\",\n \"https://www.fullcontact.com/insights-bundles/\",\n \"https://dashboard.fullcontact.com/docs\",\n \"https://www.fullcontact.com/faq/\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://fullcontact.com\",\n \"Register a free account\",\n \"Navigate to https://dashboard.fullcontact.com\",\n \"Click on 'Get an API Key'\",\n \"Verify your account using your contact number\",\n \"The API Key will be listed under 'Your API Keys'\"\n ],\n 'favIcon': \"https://1a3asl4eps7u26kl661u3bi9-wpengine.netdna-ssl.com/wp-content/uploads/2019/11/cropped-full-contact-isologo-32x32.png\",\n 'logo': \"https://1a3asl4eps7u26kl661u3bi9-wpengine.netdna-ssl.com/wp-content/themes/fc-theme/assets/images/common/full-contact-logo.svg?1574450351\",\n 'description': \"Connecting data. Consolidating identities. Applying insights. Amplifying media reach. \"\n \"We provide person-centered identity resolution to improve your customer interactions with a simple, \"\n \"real-time API integration.\\n\"\n \"FullContact is a privacy-safe Identity Resolution company building trust between people and brands. \"\n \"We deliver the capabilities needed to create tailored customer experiences by unifying data and \"\n \"applying insights in the moments that matter.\",\n }\n }\n\n opts = {\n \"api_key\": \"\",\n \"max_age_days\": \"365\"\n }\n\n optdescs = {\n \"api_key\": \"FullContact.com API key.\",\n \"max_age_days\": \"Maximum number of age in days for a record before it's considered invalid and not reported.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"DOMAIN_NAME\", \"EMAILADDR\"]\n\n def producedEvents(self):\n return [\n \"EMAILADDR\",\n \"EMAILADDR_GENERIC\",\n \"RAW_RIR_DATA\",\n \"PHONE_NUMBER\",\n \"GEOINFO\",\n \"PHYSICAL_ADDRESS\"\n ]\n\n def query(self, url, data, failcount=0):\n headers = {\n 'Authorization': f\"Bearer {self.opts['api_key']}\"\n }\n\n res = self.sf.fetchUrl(\n url,\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\",\n postData=json.dumps(data),\n headers=headers\n )\n\n if res['code'] in [\"401\", \"400\"]:\n self.error(\"API key rejected by FullContact\")\n self.errorState = True\n return None\n\n if res['code'] == \"403\":\n if failcount == 3:\n self.error(\"Throttled or other blocking by FullContact\")\n return None\n\n time.sleep(2)\n failcount += 1\n return self.query(url, data, failcount)\n\n if not res['content']:\n self.error(\"No content returned from FullContact\")\n return None\n\n try:\n ret = json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from FullContact: {e}\")\n return None\n\n if \"updated\" in ret and int(self.opts['max_age_days']) > 0:\n last_dt = datetime.strptime(ret['updated'], '%Y-%m-%d')\n last_ts = int(time.mktime(last_dt.timetuple()))\n age_limit_ts = int(time.time()) - (86400 * int(self.opts['max_age_days']))\n\n if last_ts < age_limit_ts:\n self.debug(\"FullContact record found but too old.\")\n return None\n\n return ret\n\n def queryCompany(self, domain):\n url = \"https://api.fullcontact.com/v3/company.enrich\"\n\n if not domain:\n return None\n\n return self.query(url, {\"domain\": domain})\n\n def queryPersonByEmail(self, email):\n url = \"https://api.fullcontact.com/v3/person.enrich\"\n\n if not email:\n return None\n\n return self.query(url, {'email': email})\n\n def queryPersonByName(self, name):\n url = \"https://api.fullcontact.com/v3/person.enrich\"\n\n if not name:\n return None\n\n return self.query(url, {'fullName': name})\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\n f\"You enabled {self.__class__.__name__} but did not set an API key!\"\n )\n self.errorState = True\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName == \"EMAILADDR\":\n data = self.queryPersonByEmail(eventData)\n\n if not data:\n return\n\n full_name = data.get('fullName')\n\n if full_name:\n e = SpiderFootEvent(\"RAW_RIR_DATA\", f\"Possible full name: {full_name}\", self.__name__, event)\n self.notifyListeners(e)\n\n return\n\n if eventName == \"DOMAIN_NAME\":\n data = self.queryCompany(eventData)\n\n if not data:\n return\n\n if data.get(\"details\"):\n data = data['details']\n\n if data.get(\"emails\"):\n for r in data['emails']:\n email = r.get('value')\n\n if not email:\n continue\n\n if email.split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evttype = \"EMAILADDR_GENERIC\"\n else:\n evttype = \"EMAILADDR\"\n\n e = SpiderFootEvent(evttype, email, self.__name__, event)\n self.notifyListeners(e)\n\n if data.get(\"phones\"):\n for r in data['phones']:\n phone = r.get('value')\n\n if not phone:\n continue\n\n e = SpiderFootEvent(\"PHONE_NUMBER\", phone, self.__name__, event)\n self.notifyListeners(e)\n\n if data.get(\"locations\"):\n for r in data['locations']:\n location = ', '.join([_f for _f in [r.get('city'), r.get('country')] if _f])\n if location:\n e = SpiderFootEvent(\n \"GEOINFO\",\n location,\n self.__name__,\n event\n )\n self.notifyListeners(e)\n\n if r.get(\"formatted\"):\n # Seems to contain some junk sometimes\n if len(r['formatted']) > 10:\n e = SpiderFootEvent(\n \"PHYSICAL_ADDRESS\",\n r['formatted'],\n self.__name__,\n event\n )\n self.notifyListeners(e)\n\n if data.get(\"keyPeople\"):\n for r in data['keyPeople']:\n full_name = r.get('fullName')\n if full_name:\n e = SpiderFootEvent(\n \"RAW_RIR_DATA\",\n f\"Possible full name: {full_name}\",\n self.__name__,\n event\n )\n self.notifyListeners(e)\n\n# End of sfp_fullcontact class\n" }, { "path": "modules/sfp_fullhunt.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_fullhunt\n# Purpose: Identify domain attack surface using FullHunt API.\n#\n# Author: \n#\n# Created: 2021-10-26\n# Copyright: (c) bcoles 2021\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_fullhunt(SpiderFootPlugin):\n\n meta = {\n 'name': \"FullHunt\",\n 'summary': \"Identify domain attack surface using FullHunt API.\",\n 'flags': ['apikey'],\n 'useCases': [\"Passive\", \"Footprint\", \"Investigate\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://fullhunt.io/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://api-docs.fullhunt.io/\",\n ],\n 'apiKeyInstructions': [\n \"Visit https://fullhunt.io/\",\n \"Register a free account\",\n \"Navigate to https://fullhunt.io/user/settings/\",\n \"Your API key is listed under 'API Access'\"\n ],\n 'favIcon': \"https://fullhunt.io/static/theme/images/logo/favicon.ico\",\n 'logo': \"https://fullhunt.io/static/theme/images/logo/Icon.png\",\n 'description': \"Discover, monitor, and secure your attack surface. \"\n \"FullHunt delivers the best platform in the market for attack surface security.\"\n }\n }\n\n opts = {\n \"api_key\": \"\",\n }\n\n optdescs = {\n \"api_key\": \"FullHunt API key.\",\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.errorState = False\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"DOMAIN_NAME\",\n ]\n\n def producedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"INTERNET_NAME_UNRESOLVED\",\n \"AFFILIATE_INTERNET_NAME\",\n \"AFFILIATE_INTERNET_NAME_UNRESOLVED\",\n \"TCP_PORT_OPEN\",\n \"PROVIDER_DNS\",\n \"PROVIDER_MAIL\",\n \"RAW_RIR_DATA\"\n ]\n\n def queryDomainDetails(self, qry):\n \"\"\"Search for hosts on a domain.\n\n Args:\n qry (str): domain name\n\n Returns:\n dict: search results\n \"\"\"\n headers = {\n 'X-API-KEY': self.opts['api_key']\n }\n\n res = self.sf.fetchUrl(\n f\"https://fullhunt.io/api/v1/domain/{qry}/details\",\n timeout=30,\n headers=headers,\n useragent=self.opts['_useragent']\n )\n\n return self.parseApiResponse(res)\n\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from FullHunt.\")\n return None\n\n if res['code'] == \"400\":\n self.error(\"Bad Request -- Your request is invalid.\")\n return None\n\n if res['code'] == \"401\":\n self.errorState = True\n self.error(\"Unauthorized -- Your API key is wrong.\")\n return None\n\n if res['code'] == \"403\":\n self.errorState = True\n self.error(\"Forbidden -- The requested resource is forbidden.\")\n return None\n\n if res['code'] == \"404\":\n self.error(\"Not Found -- The requested resource could not be found.\")\n return None\n\n if res['code'] == \"429\":\n self.errorState = True\n self.error(\"Too Many Requests -- You are sending too many requests.\")\n return None\n\n try:\n results = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n return results.get('hosts')\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\n f\"You enabled {self.__class__.__name__} but did not set an API key!\"\n )\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n res = self.queryDomainDetails(eventData)\n\n if not res:\n self.debug(f\"Found no results for {eventData}\")\n return\n\n e = SpiderFootEvent(\"RAW_RIR_DATA\", str(res), self.__name__, event)\n self.notifyListeners(e)\n\n hosts = list()\n name_servers = list()\n mail_servers = list()\n\n for record in res:\n host = record.get('host')\n\n if not host:\n continue\n\n hosts.append(host)\n\n dns = record.get('dns')\n if dns:\n mx = dns.get('mx')\n if mx:\n for mail_server in mx:\n mail_servers.append(mail_server.rstrip(\".\"))\n\n ns = dns.get('ns')\n if ns:\n for name_server in ns:\n name_servers.append(name_server.rstrip(\".\"))\n\n cname = dns.get('cname')\n if cname:\n for c in cname:\n hosts.append(c.rstrip(\".\"))\n\n network_ports = record.get('network_ports')\n if network_ports:\n for port in network_ports:\n e = SpiderFootEvent(\"TCP_PORT_OPEN\", f\"{host}:{port}\", self.__name__, event)\n self.notifyListeners(e)\n\n for host in set(mail_servers):\n if not host:\n continue\n\n hosts.append(host)\n\n e = SpiderFootEvent(\"PROVIDER_MAIL\", host, self.__name__, event)\n self.notifyListeners(e)\n\n for host in set(name_servers):\n if not host:\n continue\n\n hosts.append(host)\n\n e = SpiderFootEvent(\"PROVIDER_DNS\", host, self.__name__, event)\n self.notifyListeners(e)\n\n for host in set(hosts):\n if not host:\n continue\n\n if host in self.results:\n continue\n\n self.results[host] = True\n\n if self.getTarget().matches(host, includeChildren=True):\n evt_type = \"INTERNET_NAME\"\n else:\n evt_type = \"AFFILIATE_INTERNET_NAME\"\n\n if not self.sf.resolveHost(host) and not self.sf.resolveHost6(host):\n self.debug(f\"Host {host} could not be resolved\")\n evt_type += \"_UNRESOLVED\"\n\n evt = SpiderFootEvent(evt_type, host, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_fullhunt class\n" }, { "path": "modules/sfp_github.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_github\n# Purpose: Identifies public code repositories in Github associated with\n# your target.\n#\n# Author: Steve Micallef \n#\n# Created: 21/07/2015\n# Copyright: (c) Steve Micallef 2015\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_github(SpiderFootPlugin):\n\n meta = {\n 'name': \"Github\",\n 'summary': \"Identify associated public code repositories on Github.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Passive\"],\n 'categories': [\"Social Media\"],\n 'dataSource': {\n 'website': \"https://github.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://developer.github.com/\"\n ],\n 'favIcon': \"https://github.githubassets.com/favicons/favicon.png\",\n 'logo': \"https://github.githubassets.com/favicons/favicon.png\",\n 'description': \"GitHub brings together the world's largest community of \"\n \"developers to discover, share, and build better software.\",\n }\n }\n\n # Default options\n opts = {\n 'namesonly': True\n }\n\n # Option descriptions\n optdescs = {\n 'namesonly': \"Match repositories by name only, not by their descriptions. Helps reduce false positives.\"\n }\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\", \"USERNAME\", \"SOCIAL_MEDIA\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"RAW_RIR_DATA\", \"GEOINFO\", \"PUBLIC_CODE_REPO\"]\n\n # Build up repo info for use as an event\n def buildRepoInfo(self, item):\n # Get repos matching the name\n name = item.get('name')\n if name is None:\n self.debug(\"Incomplete Github information found (name).\")\n return None\n\n html_url = item.get('html_url')\n if html_url is None:\n self.debug(\"Incomplete Github information found (url).\")\n return None\n\n description = item.get('description')\n if description is None:\n self.debug(\"Incomplete Github information found (description).\")\n return None\n\n return \"\\n\".join([f\"Name: {name}\", f\"URL: {html_url}\", f\"Description: {description}\"])\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n srcModuleName = event.module\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Already did a search for {eventData}, skipping.\")\n return\n\n self.results[eventData] = True\n\n # Extract name and location from profile\n if eventName == \"SOCIAL_MEDIA\":\n try:\n network = eventData.split(\": \")[0]\n url = eventData.split(\": \")[1].replace(\"\", \"\").replace(\"\", \"\")\n except Exception as e:\n self.debug(f\"Unable to parse SOCIAL_MEDIA: {eventData} ({e})\")\n return\n\n if network != \"Github\":\n self.debug(f\"Skipping social network profile, {url}, as not a GitHub profile\")\n return\n\n try:\n urlParts = url.split(\"/\")\n username = urlParts[len(urlParts) - 1]\n except Exception:\n self.debug(f\"Couldn't get a username out of {url}\")\n return\n\n res = self.sf.fetchUrl(\n f\"https://api.github.com/users/{username}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent']\n )\n\n if res['content'] is None:\n return\n\n try:\n json_data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return\n\n if not json_data.get('login'):\n self.debug(f\"{username} is not a valid GitHub profile\")\n return\n\n full_name = json_data.get('name')\n\n if not full_name:\n self.debug(f\"{username} is not a valid GitHub profile\")\n return\n\n e = SpiderFootEvent(\"RAW_RIR_DATA\", f\"Possible full name: {full_name}\", self.__name__, event)\n self.notifyListeners(e)\n\n location = json_data.get('location')\n\n if location is None:\n return\n\n if len(location) < 3 or len(location) > 100:\n self.debug(f\"Skipping likely invalid location: {location}\")\n return\n\n e = SpiderFootEvent(\"GEOINFO\", location, self.__name__, event)\n self.notifyListeners(e)\n\n return\n\n if eventName == \"DOMAIN_NAME\":\n username = self.sf.domainKeyword(eventData, self.opts['_internettlds'])\n if not username:\n return\n\n if eventName == \"USERNAME\":\n username = eventData\n\n self.debug(f\"Looking at {username}\")\n failed = False\n\n # Get all the repositories based on direct matches with the\n # name identified\n url = f\"https://api.github.com/search/repositories?q={username}\"\n res = self.sf.fetchUrl(\n url,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent']\n )\n\n if res['content'] is None:\n self.error(f\"Unable to fetch {url}\")\n failed = True\n\n if not failed:\n try:\n ret = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from GitHub: {e}\")\n ret = None\n\n if ret is None:\n self.error(f\"Unable to process empty response from Github for: {username}\")\n failed = True\n\n if not failed:\n if ret.get('total_count', \"0\") == \"0\" or len(ret['items']) == 0:\n self.debug(f\"No Github information for {username}\")\n failed = True\n\n if not failed:\n for item in ret['items']:\n repo_info = self.buildRepoInfo(item)\n if repo_info is not None:\n if self.opts['namesonly'] and username != item['name']:\n continue\n\n evt = SpiderFootEvent(\"PUBLIC_CODE_REPO\", repo_info, self.__name__, event)\n self.notifyListeners(evt)\n\n # Now look for users matching the name found\n failed = False\n url = f\"https://api.github.com/search/users?q={username}\"\n res = self.sf.fetchUrl(\n url,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent']\n )\n\n if res['content'] is None:\n self.error(f\"Unable to fetch {url}\")\n failed = True\n\n if not failed:\n try:\n ret = json.loads(res['content'])\n if ret is None:\n self.error(f\"Unable to process empty response from Github for: {username}\")\n failed = True\n except Exception:\n self.error(f\"Unable to process invalid response from Github for: {username}\")\n failed = True\n\n if not failed:\n if ret.get('total_count', \"0\") == \"0\" or len(ret['items']) == 0:\n self.debug(\"No Github information for \" + username)\n failed = True\n\n if not failed:\n # For each user matching the username, get their repos\n for item in ret['items']:\n if item.get('repos_url') is None:\n self.debug(\"Incomplete Github information found (repos_url).\")\n continue\n\n url = item['repos_url']\n res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'])\n\n if res['content'] is None:\n self.error(f\"Unable to fetch {url}\")\n continue\n\n try:\n repret = json.loads(res['content'])\n except Exception as e:\n self.error(f\"Invalid JSON returned from Github: {e}\")\n continue\n\n if repret is None:\n self.error(f\"Unable to process empty response from Github for: {username}\")\n continue\n\n for item in repret:\n if type(item) != dict:\n self.debug(\"Encountered an unexpected or empty response from Github.\")\n continue\n\n repo_info = self.buildRepoInfo(item)\n if repo_info is not None:\n if self.opts['namesonly'] and item['name'] != username:\n continue\n if eventName == \"USERNAME\" and \"/\" + username + \"/\" not in item.get('html_url', ''):\n continue\n\n evt = SpiderFootEvent(\"PUBLIC_CODE_REPO\", repo_info,\n self.__name__, event)\n self.notifyListeners(evt)\n\n\n# End of sfp_github class\n" }, { "path": "modules/sfp_gleif.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_gleif\n# Purpose: Search the Global LEI Index for company information using\n# Global Legal Entity Identifier Foundation (GLEIF) search API.\n#\n# Author: \n#\n# Created: 2021-06-21\n# Copyright: (c) bcoles 2021\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport urllib\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_gleif(SpiderFootPlugin):\n\n meta = {\n 'name': \"GLEIF\",\n 'summary': \"Look up company information from Global Legal Entity Identifier Foundation (GLEIF).\",\n 'flags': [],\n 'useCases': [\"Passive\", \"Footprint\", \"Investigate\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://search.gleif.org/\",\n 'model': \"FREE_NOAUTH_LIMITED\",\n 'references': [\n \"https://www.gleif.org/en/lei-data/gleif-api\",\n \"https://api.gleif.org/docs\",\n ],\n 'favIcon': \"https://www.gleif.org/favicon.ico\",\n 'logo': \"https://search.gleif.org/static/img/gleif-logo.svg\",\n 'description': \"The Global Legal Entity Identifier Foundation (GLEIF) Global LEI Index contains \"\n \"historical and current LEI records including related reference data in one authoritative, central repository.\"\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"COMPANY_NAME\", \"LEI\"]\n\n def producedEvents(self):\n return [\"COMPANY_NAME\", \"LEI\", \"PHYSICAL_ADDRESS\", \"RAW_RIR_DATA\"]\n\n def searchLegalName(self, qry):\n \"\"\"Fuzzy search for legal entity by name\n\n Args:\n qry (str): legal entity name\n\n Returns:\n dict: search results\n \"\"\"\n\n params = urllib.parse.urlencode({\n 'q': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n 'field': \"entity.legalName\"\n })\n\n headers = {\n 'Accept': 'application/vnd.api+json'\n }\n\n res = self.sf.fetchUrl(\n f\"https://api.gleif.org/api/v1/fuzzycompletions?{params}\",\n timeout=30,\n headers=headers,\n useragent=self.opts['_useragent']\n )\n\n if res['code'] == \"429\":\n self.error(\"You are being rate-limited by GLEIF.\")\n return None\n\n try:\n results = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n data = results.get('data')\n if not data:\n return None\n if not len(data):\n return None\n\n return data\n\n def searchAutocompletions(self, qry):\n \"\"\"Search for legal entity name autocompletions\n\n Args:\n qry (str): legal entity name\n\n Returns:\n dict: search results\n \"\"\"\n\n params = urllib.parse.urlencode({\n 'q': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n 'field': \"fulltext\"\n })\n\n headers = {\n 'Accept': 'application/vnd.api+json'\n }\n\n res = self.sf.fetchUrl(\n f\"https://api.gleif.org/api/v1/autocompletions?{params}\",\n timeout=30,\n headers=headers,\n useragent=self.opts['_useragent']\n )\n\n if res['code'] == \"429\":\n self.error(\"You are being rate-limited by GLEIF.\")\n return None\n\n try:\n results = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n data = results.get('data')\n if not data:\n return None\n if not len(data):\n return None\n\n return data\n\n def retrieveRecord(self, lei):\n headers = {\n 'Accept': 'application/vnd.api+json'\n }\n\n res = self.sf.fetchUrl(\n f\"https://api.gleif.org/api/v1/lei-records/{lei}\",\n timeout=self.opts['_fetchtimeout'],\n headers=headers,\n useragent=self.opts['_useragent']\n )\n\n if res['code'] == \"404\":\n self.error(f\"No record for LEI: {lei}\")\n return None\n\n if res['code'] == \"429\":\n self.error(\"You are being rate-limited by GLEIF.\")\n return None\n\n try:\n results = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n data = results.get('data')\n\n if not len(data):\n return None\n if not data:\n return None\n\n return data\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n leis = list()\n\n if eventName == \"LEI\":\n leis.append(eventData)\n elif eventName == \"COMPANY_NAME\":\n self.results[eventData] = True\n res = self.searchAutocompletions(eventData)\n\n if res is None:\n self.debug(f\"Found no results for {eventData}\")\n return\n\n e = SpiderFootEvent(\"RAW_RIR_DATA\", str(res), self.__name__, event)\n self.notifyListeners(e)\n\n for record in res:\n relationships = record.get('relationships')\n if not relationships:\n continue\n\n lei_records = relationships.get('lei-records')\n if not lei_records:\n continue\n\n data = lei_records.get('data')\n if not data:\n continue\n\n lei = data.get('id')\n if not SpiderFootHelpers.validLEI(lei):\n continue\n\n leis.append(lei)\n\n self.info(f\"Found {len(leis)} LEIs matching {eventData}\")\n\n for lei in set(leis):\n if lei in self.results:\n continue\n\n if not SpiderFootHelpers.validLEI(lei):\n continue\n\n self.results[lei] = True\n\n e = SpiderFootEvent(\"LEI\", lei, self.__name__, event)\n self.notifyListeners(e)\n\n self.results[lei] = True\n\n res = self.retrieveRecord(lei)\n if not res:\n self.debug(f\"Found no results for {eventData}\")\n continue\n\n attributes = res.get('attributes')\n if not attributes:\n continue\n\n entity = attributes.get('entity')\n if not entity:\n continue\n\n legal_name = entity.get('legalName')\n if legal_name:\n entity_name = legal_name.get('value')\n if entity_name:\n e = SpiderFootEvent(\"COMPANY_NAME\", entity_name, self.__name__, event)\n self.notifyListeners(e)\n\n addresses = list()\n\n address = entity.get('legalAddress')\n if address.get('addressLines'):\n address_lines = ', '.join(filter(None, address.get('addressLines')))\n location = ', '.join(\n filter(\n None,\n [\n address_lines,\n address.get('city'),\n address.get('region'),\n address.get('country'),\n address.get('postalCode')\n ]\n )\n )\n\n if location:\n addresses.append(location)\n\n address = entity.get('headquartersAddress')\n if address.get('addressLines'):\n address_lines = ', '.join(filter(None, address.get('addressLines')))\n location = ', '.join(\n filter(\n None,\n [\n address_lines,\n address.get('city'),\n address.get('region'),\n address.get('country'),\n address.get('postalCode')\n ]\n )\n )\n\n if location:\n addresses.append(location)\n\n for address in set(addresses):\n e = SpiderFootEvent(\"PHYSICAL_ADDRESS\", address, self.__name__, event)\n self.notifyListeners(e)\n\n# End of sfp_gleif class\n" }, { "path": "modules/sfp_google_tag_manager.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_google_tag_manager\n# Purpose: Search Google Tag Manager (GTM) for hosts sharing the same GTM code.\n#\n# Author: \n#\n# Created: 2023-10-29\n# Copyright: (c) bcoles 2023\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_google_tag_manager(SpiderFootPlugin):\n\n meta = {\n 'name': \"Google Tag Manager\",\n 'summary': \"Search Google Tag Manager (GTM) for hosts sharing the same GTM code.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Passive DNS\"],\n 'dataSource': {\n 'website': \"https://tagmanager.google.com\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://marketingplatform.google.com/about/tag-manager/\",\n \"https://developers.google.com/tag-manager/quickstart\",\n \"https://developers.google.com/tag-manager/devguide\"\n ],\n 'favIcon': \"https://google.com/favicon.ico\",\n 'logo': \"https://google.com/favicon.ico\",\n 'description': \"Manage all your website tags without editing code. Google Tag Manager \"\n \"delivers simple, reliable, easily integrated tag management solutions for free.\"\n }\n }\n\n opts = {\n \"verify\": True\n }\n\n optdescs = {\n \"verify\": \"Verify identified hostnames resolve to an IP address.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return ['WEB_ANALYTICS_ID']\n\n def producedEvents(self):\n return [\n 'DOMAIN_NAME',\n 'INTERNET_NAME',\n 'AFFILIATE_DOMAIN_NAME',\n 'AFFILIATE_INTERNET_NAME',\n ]\n\n # from: https://stackoverflow.com/a/43211062\n def is_valid_hostname(self, hostname: str = None) -> bool:\n if not hostname:\n return False\n if len(hostname) > 255:\n return False\n\n hostname = hostname.rstrip(\".\")\n allowed = re.compile(\"(?!-)[A-Z0-9-_]{1,63}(? set:\n if not tag_id:\n return None\n\n params = urllib.parse.urlencode({\n 'id': tag_id,\n })\n\n res = self.sf.fetchUrl(\n f\"https://googletagmanager.com/gtm.js?{params}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent']\n )\n\n if res['code'] != \"200\":\n self.debug(f\"Invalid GTM tag id: {tag_id}\")\n return None\n\n data = res['content']\n\n if not data:\n self.debug(f\"Invalid GTM tag id: {tag_id}\")\n return None\n\n hosts = list()\n\n for host in re.findall(r'\"map\",\"key\",\"(.+?)\"', data):\n if '.' not in host:\n continue\n if self.is_valid_hostname(host):\n hosts.append(host)\n\n for host in re.findall(r',\"arg1\":\"(.+?)\"', data):\n if '.' not in host:\n continue\n if self.is_valid_hostname(host):\n hosts.append(host)\n\n for url in SpiderFootHelpers.extractUrlsFromText(str(data).replace(\"\\\\/\", \"/\")):\n host = self.sf.urlFQDN(url)\n if not host:\n continue\n if '.' not in host:\n continue\n hosts.append(host)\n\n return set(hosts)\n\n def handleEvent(self, event):\n self.debug(f\"Received event, {event.eventType}, from {event.module}\")\n\n if self.errorState:\n return\n\n if event.data in self.results:\n return\n\n self.results[event.data] = True\n\n try:\n network = event.data.split(\": \")[0]\n tag_id = event.data.split(\": \")[1]\n except Exception as e:\n self.error(f\"Unable to parse WEB_ANALYTICS_ID: {event.data} ({e})\")\n return\n\n if network != 'Google Tag Manager':\n return\n\n hosts = self.queryGoogleTagId(tag_id)\n\n if not hosts:\n self.info(f\"No hosts found for {tag_id}\")\n return\n\n self.info(f\"Retrieved {len(hosts)} results\")\n\n for host in hosts:\n # we ignore unresolved hosts due to large number of false positives\n if self.opts['verify'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host):\n self.debug(f\"Potential host name '{host}' could not be resolved\")\n continue\n\n if self.getTarget().matches(host, includeChildren=True, includeParents=True):\n evt_type = 'INTERNET_NAME'\n else:\n evt_type = 'AFFILIATE_INTERNET_NAME'\n\n evt = SpiderFootEvent(evt_type, host, self.__name__, event)\n self.notifyListeners(evt)\n\n if self.sf.isDomain(host, self.opts['_internettlds']):\n if evt_type.startswith('AFFILIATE'):\n evt_type = 'AFFILIATE_DOMAIN_NAME'\n else:\n evt_type = 'DOMAIN_NAME'\n\n evt = SpiderFootEvent(evt_type, host, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_google_tag_manager class\n" }, { "path": "modules/sfp_googlemaps.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_googlemaps\n# Purpose: SpiderFoot plug-in to identify historical certificates for a domain\n# from googlemaps.sh, and from this identify hostnames.\n#\n# Author: Steve Micallef \n#\n# Created: 18/03/2017\n# Copyright: (c) Steve Micallef 2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport urllib\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_googlemaps(SpiderFootPlugin):\n\n meta = {\n 'name': \"Google Maps\",\n 'summary': \"Identifies potential physical addresses and latitude/longitude coordinates.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Real World\"],\n 'dataSource': {\n 'website': \"https://cloud.google.com/maps-platform/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://developers.google.com/maps/documentation/?_ga=2.135220017.1220421370.1587340370-900596925.1587340370\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://cloud.google.com/maps-platform/\",\n \"Register a free Google account\",\n \"Click on 'Get Started'\",\n \"Click on 'API'\",\n \"Select the type of API\",\n \"Navigate to https://console.cloud.google.com/apis/credentials\",\n \"Click on 'Credentials'\",\n \"The API Key will be listed under 'API Keys'\"\n ],\n 'favIcon': \"https://www.gstatic.com/devrel-devsite/prod/v2210deb8920cd4a55bd580441aa58e7853afc04b39a9d9ac4198e1cd7fbe04ef/cloud/images/favicons/onecloud/favicon.ico\",\n 'logo': \"https://www.gstatic.com/devrel-devsite/prod/v2210deb8920cd4a55bd580441aa58e7853afc04b39a9d9ac4198e1cd7fbe04ef/cloud/images/cloud-logo.svg\",\n 'description': \"Explore where real-world insights and immersive location experiences can take your business.\\n\"\n \"Build with reliable, comprehensive data for over 200 countries and territories.\\n\"\n \"has been done here. If line breaks are needed for breaking up\\n\"\n \"Scale confidently, backed by our infrastructure.\",\n }\n }\n\n opts = {\n \"api_key\": \"\"\n }\n optdescs = {\n \"api_key\": \"Google Geocoding API Key.\"\n }\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return ['DOMAIN_NAME', 'PHYSICAL_ADDRESS']\n\n def producedEvents(self):\n return [\"PHYSICAL_ADDRESS\", \"PHYSICAL_COORDINATES\", \"RAW_RIR_DATA\"]\n\n def query(self, address):\n params = urllib.parse.urlencode({\n 'key': self.opts['api_key'],\n 'address': address.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n })\n\n res = self.sf.fetchUrl(\n f\"https://maps.googleapis.com/maps/api/geocode/json?{params}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent']\n )\n\n if res['content'] is None:\n self.info(f\"No location info found for {address}\")\n return None\n\n return res\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\n f\"You enabled {self.__class__.__name__} but did not set an API key!\"\n )\n self.errorState = True\n return\n\n res = self.query(eventData)\n\n if not res:\n self.debug(f\"No information found for {eventData}\")\n return\n\n evt = SpiderFootEvent(\n \"RAW_RIR_DATA\",\n res['content'],\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n\n try:\n data = json.loads(res['content'])['results'][0]\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return\n\n if srcModuleName == \"sfp_googlemaps\":\n return\n\n geometry = data.get('geometry')\n if geometry:\n location = data.get('location')\n if location:\n lat = location.get('lat')\n lng = location.get('lng')\n if lat and lng:\n evt = SpiderFootEvent(\n \"PHYSICAL_COORDINATES\",\n f\"{lat},{lng}\",\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n\n formatted_address = data.get('formatted_address')\n if formatted_address:\n evt = SpiderFootEvent(\n \"PHYSICAL_ADDRESS\",\n data['formatted_address'],\n self.__name__,\n event\n )\n self.notifyListeners(evt)\n\n# End of sfp_googlemaps class\n" }, { "path": "modules/sfp_googleobjectstorage.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_googleobjectstorage\n# Purpose: SpiderFoot plug-in for identifying potential Google Object Storage\n# buckets related to the target.\n#\n# Author: Steve Micallef \n#\n# Created: 24/01/2020\n# Copyright: (c) Steve Micallef 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport random\nimport threading\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_googleobjectstorage(SpiderFootPlugin):\n\n meta = {\n 'name': \"Google Object Storage Finder\",\n 'summary': \"Search for potential Google Object Storage buckets associated with the target and attempt to list their contents.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Passive\"],\n 'categories': [\"Crawling and Scanning\"],\n 'dataSource': {\n 'website': \"https://cloud.google.com/storage\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'favIcon': 'https://www.gstatic.com/devrel-devsite/prod/v4c1e9ea53f4457a5de1027b4eeb4608c1000a427e20261ba1771dd3fc26d5df8/cloud/images/favicons/onecloud/favicon.ico',\n 'logo': 'https://www.gstatic.com/devrel-devsite/prod/v4c1e9ea53f4457a5de1027b4eeb4608c1000a427e20261ba1771dd3fc26d5df8/cloud/images/cloud-logo.svg',\n 'description': \"Object storage for companies of all sizes.\"\n \"Secure, durable, and with low latency. Store any amount of data.\"\n \"Retrieve it as often as you'd like.\"\n }\n }\n\n # Default options\n opts = {\n \"suffixes\": \"test,dev,web,beta,bucket,space,files,content,data,prod,staging,production,stage,app,media,development,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data,-prod,-staging,-production,-stage,-app,-media,-development\",\n \"_maxthreads\": 20\n }\n\n # Option descriptions\n optdescs = {\n \"suffixes\": \"List of suffixes to append to domains tried as bucket names\",\n \"_maxthreads\": \"Maximum threads\"\n }\n\n results = None\n gosresults = dict()\n lock = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.gosresults = dict()\n self.results = self.tempStorage()\n self.lock = threading.Lock()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\", \"LINKED_URL_EXTERNAL\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"CLOUD_STORAGE_BUCKET\", \"CLOUD_STORAGE_BUCKET_OPEN\"]\n\n def checkSite(self, url):\n res = self.sf.fetchUrl(url, timeout=10, useragent=\"SpiderFoot\", noLog=True)\n\n if not res['content']:\n return\n\n if \"NoSuchBucket\" in res['content']:\n self.debug(f\"Not a valid bucket: {url}\")\n return\n\n # Bucket found\n if res['code'] in [\"301\", \"302\", \"200\"]:\n # Bucket has files\n if \"ListBucketResult\" in res['content']:\n with self.lock:\n self.gosresults[url] = res['content'].count(\"\")\n else:\n # Bucket has no files\n with self.lock:\n self.gosresults[url] = 0\n\n def threadSites(self, siteList):\n self.gosresults = dict()\n running = True\n i = 0\n t = []\n\n for site in siteList:\n if self.checkForStop():\n return None\n\n self.info(\"Spawning thread to check bucket: \" + site)\n tname = str(random.SystemRandom().randint(0, 999999999))\n t.append(threading.Thread(name='thread_sfp_googleobjectstorage_' + tname,\n target=self.checkSite, args=(site,)))\n t[i].start()\n i += 1\n\n # Block until all threads are finished\n while running:\n found = False\n for rt in threading.enumerate():\n if rt.name.startswith(\"thread_sfp_googleobjectstorage_\"):\n found = True\n\n if not found:\n running = False\n\n time.sleep(0.25)\n\n # Return once the scanning has completed\n return self.gosresults\n\n def batchSites(self, sites):\n i = 0\n res = list()\n siteList = list()\n\n for site in sites:\n if i >= self.opts['_maxthreads']:\n data = self.threadSites(siteList)\n if data is None:\n return res\n\n for ret in list(data.keys()):\n if data[ret]:\n # bucket:filecount\n res.append(ret + \":\" + str(data[ret]))\n i = 0\n siteList = list()\n\n siteList.append(site)\n i += 1\n\n return res\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventName == \"LINKED_URL_EXTERNAL\":\n if \".storage.googleapis.com\" in eventData:\n b = self.sf.urlFQDN(eventData)\n evt = SpiderFootEvent(\"CLOUD_STORAGE_BUCKET\", b, self.__name__, event)\n self.notifyListeners(evt)\n return\n\n targets = [eventData.replace('.', '')]\n kw = self.sf.domainKeyword(eventData, self.opts['_internettlds'])\n if kw:\n targets.append(kw)\n\n urls = list()\n for t in targets:\n suffixes = [''] + self.opts['suffixes'].split(',')\n for s in suffixes:\n if self.checkForStop():\n return\n\n b = t + s + \".storage.googleapis.com\"\n url = \"https://\" + b\n urls.append(url)\n\n # Batch the scans\n ret = self.batchSites(urls)\n for b in ret:\n bucket = b.split(\":\")\n evt = SpiderFootEvent(\"CLOUD_STORAGE_BUCKET\", bucket[0] + \":\" + bucket[1], self.__name__, event)\n self.notifyListeners(evt)\n if bucket[2] != \"0\":\n bucketname = bucket[1].replace(\"//\", \"\")\n evt = SpiderFootEvent(\"CLOUD_STORAGE_BUCKET_OPEN\", bucketname + \": \" + bucket[2] + \" files found.\",\n self.__name__, evt)\n self.notifyListeners(evt)\n\n\n# End of sfp_googleobjectstorage class\n" }, { "path": "modules/sfp_googlesafebrowsing.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_googlesafebrowsing\n# Purpose: SpiderFoot plug-in to check if the URL is included on any of the\n# Google Safe Browsing lists\n#\n# Author: Filip Aleksić \n#\n# Created: 2020-08-18\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_googlesafebrowsing(SpiderFootPlugin):\n\n meta = {\n \"name\": \"Google SafeBrowsing\",\n \"summary\": \"Check if the URL is included on any of the Safe Browsing lists.\",\n 'flags': [\"slow\", \"apikey\"],\n \"useCases\": [\"Passive\", \"Investigate\"],\n \"categories\": [\"Reputation Systems\"],\n \"dataSource\": {\n \"website\": \"https://developers.google.com/safe-browsing/v4/lookup-api\",\n \"model\": \"FREE_AUTH_UNLIMITED\",\n \"references\": [\n \"https://developers.google.com/safe-browsing/v4/reference/rest\"\n ],\n \"apiKeyInstructions\": [\n \"Visit https://console.developers.google.com/\",\n \"Register a free Google account or sign in\",\n \"Create or select existing Google Developer Console project\",\n \"Go to the Cloud Console API Library https://console.cloud.google.com/apis/library\",\n \"From the projects list, select the project you want to use\",\n \"In the API Library select 'Safe Browsing APIs'\",\n \"On the API page, click ENABLE\",\n \"Navigate to the APIs & Services→Credentials panel in Cloud Console\",\n \"Select Create credentials, then select API key from the dropdown menu\",\n \"The API key created dialog box displays your newly created key\",\n ],\n \"favIcon\": \"https://www.gstatic.com/devrel-devsite/prod/v1241c04ebcb2127897d6c18221acbd64e7ed5c46e5217fd83dd808e592c47bf6/developers/images/favicon.png\",\n \"logo\": \"https://developers.google.com/safe-browsing/images/SafeBrowsing_Icon.png\",\n \"description\": \"The Safe Browsing APIs (v4) let your client applications check URLs \"\n \"against Google's constantly updated lists of unsafe web resources. \"\n \"Any URL found on a Safe Browsing list is considered unsafe.\",\n },\n }\n\n opts = {\"api_key\": \"\"}\n\n optdescs = {\n \"api_key\": \"Google Safe Browsing API key.\",\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"IP_ADDRESS\",\n \"AFFILIATE_INTERNET_NAME\",\n \"AFFILIATE_IPADDR\",\n \"CO_HOSTED_SITE\",\n ]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_COHOST\",\n \"RAW_RIR_DATA\",\n ]\n\n def query(self, qry):\n\n headers = {\"Content-Type\": \"application/json\"}\n url = (\n \"https://safebrowsing.googleapis.com/v4/threatMatches\"\n f\":find?key={self.opts['api_key']}\"\n )\n payload = {\n \"client\": {\"clientId\": \"SpiderFoot\", \"clientVersion\": \"3.2\"},\n \"threatInfo\": {\n \"threatTypes\": [\n \"THREAT_TYPE_UNSPECIFIED\",\n \"MALWARE\",\n \"SOCIAL_ENGINEERING\",\n \"UNWANTED_SOFTWARE\",\n \"POTENTIALLY_HARMFUL_APPLICATION\",\n ],\n \"platformTypes\": [\"PLATFORM_TYPE_UNSPECIFIED\", \"ANY_PLATFORM\"],\n \"threatEntryTypes\": [\n \"THREAT_ENTRY_TYPE_UNSPECIFIED\",\n \"URL\",\n \"EXECUTABLE\",\n ],\n \"threatEntries\": [\n {\n \"url\": qry.encode(\"raw_unicode_escape\").decode(\n \"ascii\", errors=\"replace\"\n )\n }\n ],\n },\n }\n res = self.sf.fetchUrl(\n url,\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=self.opts[\"_useragent\"],\n headers=headers,\n postData=json.dumps(payload),\n )\n\n if res[\"code\"] == \"400\":\n self.error(\"Invalid request payload on Google Safe Browsing API\")\n self.errorState = True\n return None\n\n if res[\"code\"] == \"429\":\n self.error(\"Reaching rate limit on Google Safe Browsing API\")\n self.errorState = True\n return None\n\n if res[\"code\"] == \"403\":\n self.error(\n \"Permission denied, invalid API key on Google Safe Browsing API\"\n )\n self.errorState = True\n return None\n\n if res[\"code\"] in [\"500\", \"503\", \"504\"]:\n self.error(\n \"Google Safe Browsing API is having some troubles or unavailable.\"\n )\n self.errorState = True\n return None\n\n try:\n info = json.loads(res[\"content\"])\n if info == {}:\n self.info(\"No Google Safe Browsing matches found for \" + qry)\n return None\n\n except Exception as e:\n self.error(f\"Error processing JSON response from SHODAN: {e}\")\n return None\n\n return info\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\n \"You enabled sfp_googlesafebrowsing but did not set an API key!\"\n )\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n evtType = \"\"\n if eventName in [\"IP_ADDRESS\", \"AFFILIATE_IPADDR\"]:\n if eventName == \"IP_ADDRESS\":\n evtType = \"MALICIOUS_IPADDR\"\n else:\n evtType = \"MALICIOUS_AFFILIATE_IPADDR\"\n\n if eventName in [\"INTERNET_NAME\", \"CO_HOSTED_SITE\", \"AFFILIATE_INTERNET_NAME\"]:\n if eventName == \"INTERNET_NAME\":\n evtType = \"MALICIOUS_INTERNET_NAME\"\n if eventName == \"AFFILIATE_INTERNET_NAME\":\n evtType = \"MALICIOUS_AFFILIATE_INTERNET_NAME\"\n if eventName == \"CO_HOSTED_SITE\":\n evtType = \"MALICIOUS_COHOST\"\n\n rec = self.query(eventData)\n\n if rec is None:\n return\n\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(rec), self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(\n evtType, \"Google SafeBrowsing [\" + eventData + \"]\", self.__name__, event\n )\n self.notifyListeners(evt)\n\n\n# End of sfp_googlesafebrowsing class\n" }, { "path": "modules/sfp_googlesearch.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_googlesearch\n# Purpose: Searches Google for content related to the domain in question.\n#\n# Author: Steve Micallef \n#\n# Created: 07/05/2012\n# Copyright: (c) Steve Micallef 2012\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_googlesearch(SpiderFootPlugin):\n\n meta = {\n 'name': \"Google\",\n 'summary': \"Obtain information from the Google Custom Search API to identify sub-domains and links.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://developers.google.com/custom-search\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://developers.google.com/custom-search/v1\",\n \"https://developers.google.com/custom-search/docs/overview\",\n \"https://cse.google.com/cse\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://developers.google.com/custom-search/v1/introduction\",\n \"Register a free Google account\",\n \"Click on 'Get A Key'\",\n \"Connect a Project\",\n \"The API Key will be listed under 'YOUR API KEY'\"\n ],\n 'favIcon': \"https://www.gstatic.com/devrel-devsite/prod/v2210deb8920cd4a55bd580441aa58e7853afc04b39a9d9ac4198e1cd7fbe04ef/developers/images/favicon.png\",\n 'logo': \"https://www.gstatic.com/devrel-devsite/prod/v2210deb8920cd4a55bd580441aa58e7853afc04b39a9d9ac4198e1cd7fbe04ef/developers/images/favicon.png\",\n 'description': \"Google Custom Search enables you to create a search engine for your website, your blog, or a collection of websites. \"\n \"You can configure your engine to search both web pages and images. \"\n \"You can fine-tune the ranking, add your own promotions and customize the look and feel of the search results. \"\n \"You can monetize the search by connecting your engine to your Google AdSense account.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\",\n \"cse_id\": \"013611106330597893267:tfgl3wxdtbp\"\n }\n\n # Option descriptions\n optdescs = {\n \"api_key\": \"Google API Key for Google search.\",\n \"cse_id\": \"Google Custom Search Engine ID.\"\n }\n\n # Target\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"INTERNET_NAME\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"LINKED_URL_INTERNAL\", \"RAW_RIR_DATA\"]\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_googlesearch but did not set a Google API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(\"Already did a search for \" + eventData + \", skipping.\")\n return\n\n self.results[eventData] = True\n\n # Sites hosted on the domain\n res = self.sf.googleIterate(\n searchString=\"site:\" + eventData,\n opts={\n \"timeout\": self.opts[\"_fetchtimeout\"],\n \"useragent\": self.opts[\"_useragent\"],\n \"api_key\": self.opts[\"api_key\"],\n \"cse_id\": self.opts[\"cse_id\"],\n },\n )\n if res is None:\n # Failed to talk to the Google API or no results returned\n return\n\n urls = res[\"urls\"]\n new_links = list(set(urls) - set(self.results.keys()))\n\n # Add new links to results\n for link in new_links:\n self.results[link] = True\n\n internal_links = [\n link for link in new_links if self.sf.urlFQDN(link).endswith(eventData)\n ]\n for link in internal_links:\n self.debug(\"Found a link: \" + link)\n\n evt = SpiderFootEvent(\"LINKED_URL_INTERNAL\", link, self.__name__, event)\n self.notifyListeners(evt)\n\n if internal_links:\n evt = SpiderFootEvent(\n \"RAW_RIR_DATA\", str(res), self.__name__, event\n )\n self.notifyListeners(evt)\n\n# End of sfp_googlesearch class\n" }, { "path": "modules/sfp_gravatar.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_gravatar\n# Purpose: SpiderFoot plug-in to search Gravatar API for an email address\n# and retrieve user information, including username, name, phone\n# numbers, additional email addresses, and social media usernames.\n#\n# Author: \n#\n# Created: 2019-05-26\n# Copyright: (c) bcoles 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport hashlib\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_gravatar(SpiderFootPlugin):\n\n meta = {\n 'name': \"Gravatar\",\n 'summary': \"Retrieve user information from Gravatar API.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Social Media\"],\n 'dataSource': {\n 'website': \"https://secure.gravatar.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://secure.gravatar.com/site/implement/\"\n ],\n 'favIcon': \"https://secure.gravatar.com/favicon.ico\",\n 'logo': \"https://secure.gravatar.com/favicon.ico\",\n 'description': \"Your Gravatar is an image that follows you from site to site \"\n \"appearing beside your name when you do things like comment or post on a blog.\\n\"\n \"A Gravatar is a Globally Recognized Avatar. You upload it and create your profile just once, \"\n \"and then when you participate in any Gravatar-enabled site, your Gravatar image will automatically follow you there.\",\n }\n }\n\n # Default options\n opts = {\n }\n\n # Option descriptions\n optdescs = {\n }\n\n results = None\n reportedUsers = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.reportedUsers = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['EMAILADDR']\n\n # What events this module produces\n def producedEvents(self):\n return ['RAW_RIR_DATA', 'USERNAME',\n 'EMAILADDR', 'EMAILADDR_GENERIC', 'PHONE_NUMBER', 'GEOINFO',\n 'ACCOUNT_EXTERNAL_OWNED', 'SOCIAL_MEDIA']\n\n # Query Gravatar API for the specified email address\n # https://secure.gravatar.com/site/implement/\n # https://secure.gravatar.com/site/implement/profiles/\n def query(self, qry):\n email_hash = hashlib.md5(qry.encode('utf-8', errors='replace').lower()).hexdigest() # noqa: DUO130\n output = 'json'\n\n res = self.sf.fetchUrl(\"https://secure.gravatar.com/\" + email_hash + '.' + output,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'])\n\n time.sleep(1)\n\n if res['content'] is None:\n self.debug('No response from gravatar.com')\n return None\n\n if res['code'] != '200':\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n if data.get('entry') is None or len(data.get('entry')) == 0:\n return None\n\n return data.get('entry')[0]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n data = self.query(eventData)\n\n if data is None:\n self.debug(\"No user information found for \" + eventData)\n return\n\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n if data.get('preferredUsername') is not None:\n un = data.get('preferredUsername')\n evt = SpiderFootEvent(\"USERNAME\", un, self.__name__, event)\n self.notifyListeners(evt)\n self.reportedUsers[un] = True\n\n names = list()\n if data.get('name') is not None:\n if type(data.get('name')) != list:\n names.append(data.get('name'))\n else:\n names = data.get('name')\n\n for name in names:\n full_name = name.get('formatted')\n if full_name:\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", f\"Possible full name: {full_name}\", self.__name__, event)\n self.notifyListeners(evt)\n\n # TODO: re-enable once location validation is implemented\n # location can not be trusted\n # if data.get('currentLocation') is not None:\n # location = data.get('currentLocation')\n # if len(location) < 3 or len(location) > 100:\n # self.debug(\"Skipping likely invalid location.\")\n # else:\n # evt = SpiderFootEvent(\"GEOINFO\", location, self.__name__, event)\n # self.notifyListeners(evt)\n\n if data.get('phoneNumbers') is not None:\n for number in data.get('phoneNumbers'):\n if number.get('value') is not None:\n evt = SpiderFootEvent(\"PHONE_NUMBER\", number.get('value'), self.__name__, event)\n self.notifyListeners(evt)\n\n if data.get('emails') is not None:\n for email in data.get('emails'):\n em = email.get('value')\n if not em:\n continue\n if SpiderFootHelpers.validEmail(em) and em != eventData:\n if em.split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evttype = \"EMAILADDR_GENERIC\"\n else:\n evttype = \"EMAILADDR\"\n\n evt = SpiderFootEvent(evttype, em, self.__name__, event)\n self.notifyListeners(evt)\n\n if data.get('ims') is not None:\n for im in data.get('ims'):\n v = im.get('value')\n if v is None:\n continue\n t = im.get('type').capitalize() + \" (Instant Messenger)\\n\" + v\n evt = SpiderFootEvent(\"ACCOUNT_EXTERNAL_OWNED\", t, self.__name__, event)\n self.notifyListeners(evt)\n if v not in self.reportedUsers:\n evt = SpiderFootEvent(\"USERNAME\", v, self.__name__, event)\n self.notifyListeners(evt)\n self.reportedUsers[v] = True\n\n if data.get('accounts') is not None:\n for account in data.get('accounts'):\n url = account.get('url')\n platform = account.get('shortname')\n if platform is not None and url is not None:\n t = platform.capitalize() + \": \" + url + \"\"\n evt = SpiderFootEvent(\"SOCIAL_MEDIA\", t, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_gravatar class\n" }, { "path": "modules/sfp_grayhatwarfare.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_grayhatwarfare\n# Purpose: Find bucket names matching the keyword extracted from a domain from Grayhat API.\n#\n# Author: \n#\n# Created: 24-01-2021\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_grayhatwarfare(SpiderFootPlugin):\n\n meta = {\n 'name': \"Grayhat Warfare\",\n 'summary': \"Find bucket names matching the keyword extracted from a domain from Grayhat API.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://buckets.grayhatwarfare.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://buckets.grayhatwarfare.com/docs/api/v1\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://grayhatwarfare.com/register\",\n \"Register an account\",\n \"Visit https://grayhatwarfare.com/account/settings\",\n \"Your API key is listed under 'Api Key'\",\n ],\n 'favIcon': \"https://buckets.grayhatwarfare.com/assets/template/images/favicon.png\",\n 'logo': \"https://buckets.grayhatwarfare.com/assets/images/logo/logo-sm.png\",\n 'description': \"It is a searchable database of open buckets.\"\n \"Has up to million results of each bucket.\"\n \"Full text search with binary logic (can search for keywords and also stopwords)\",\n }\n }\n\n # Default options\n opts = {\n 'api_key': '',\n 'per_page': 1000,\n 'max_pages': 2,\n 'pause': 1\n }\n\n # Option descriptions\n optdescs = {\n 'api_key': 'Grayhat Warfare API key.',\n 'per_page': 'Maximum number of results per page (Max: 1000).',\n 'max_pages': 'Maximum number of pages to fetch.',\n 'pause': 'Number of seconds to wait between each API call.'\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\n \"DOMAIN_NAME\",\n ]\n\n # What events this module produces\n def producedEvents(self):\n return [\n 'CLOUD_STORAGE_BUCKET',\n 'CLOUD_STORAGE_BUCKET_OPEN',\n 'RAW_RIR_DATA'\n ]\n\n # Query Grayhat Warfare\n def query(self, keyword, start):\n params = urllib.parse.urlencode({\n 'keywords': keyword.encode('raw_unicode_escape'),\n 'access_token': self.opts['api_key']\n })\n\n headers = {\n 'Accept': 'application/json',\n }\n\n res = self.sf.fetchUrl(\n f\"https://buckets.grayhatwarfare.com/api/v1/buckets/{start}/{self.opts['per_page']}?{params}\",\n headers=headers,\n timeout=15,\n useragent=self.opts['_useragent'],\n verify=True\n )\n\n time.sleep(self.opts['pause'])\n\n if res['code'] != \"200\":\n self.error(\"Unable to fetch data from Grayhat Warfare API.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.debug('No response from Grayhat Warfare API.')\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_grayhatwarfare but did not set an API key!\")\n self.errorState = True\n return\n\n currentIndex = 0\n currentPage = 0\n maxPages = self.opts['max_pages']\n perPage = self.opts['per_page']\n\n keyword = self.sf.domainKeyword(eventData, self.opts['_internettlds'])\n\n while currentPage < maxPages:\n currentIndex = currentPage * perPage\n if self.checkForStop():\n return\n\n if self.errorState:\n break\n\n data = self.query(keyword=keyword, start=currentIndex)\n\n if not data:\n return\n\n for row in data.get('buckets'):\n bucketName = row.get('bucket')\n bucketKeyword = bucketName.split('.')[0]\n self.debug(bucketKeyword)\n if bucketKeyword.startswith(keyword) or bucketKeyword.endswith(keyword):\n evt = SpiderFootEvent('CLOUD_STORAGE_BUCKET', bucketName, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent('CLOUD_STORAGE_BUCKET_OPEN', f\"{bucketName}: {row.get('fileCount')} files found.\", self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(row), self.__name__, event)\n self.notifyListeners(evt)\n\n currentPage += 1\n if data.get('buckets_count') < perPage:\n break\n\n# End of sfp_grayhatwarfare class\n" }, { "path": "modules/sfp_greensnow.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_greensnow\n# Purpose: Checks if an IP address or netblock is malicious according to greensnow.co.\n#\n# Author: steve@binarypool.com\n#\n# Created: 16/05/2020\n# Copyright: (c) Steve Micallef, 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom netaddr import IPAddress, IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_greensnow(SpiderFootPlugin):\n\n meta = {\n 'name': \"Greensnow\",\n 'summary': \"Check if a netblock or IP address is malicious according to greensnow.co.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://greensnow.co/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://blocklist.greensnow.co/greensnow.txt\",\n \"https://greensnow.co/faq\"\n ],\n 'favIcon': \"https://greensnow.co/favicon.ico\",\n 'logo': \"https://greensnow.co/img/logo.png\",\n 'description': \"GreenSnow is a team consisting of the best specialists in computer security, \"\n \"we harvest a large number of IPs from different computers located around the world. \"\n \"GreenSnow is comparable with SpamHaus.org for attacks of any kind except for spam. \"\n \"Our list is updated automatically and you can withdraw at any time your IP address if it has been listed.\",\n }\n }\n\n opts = {\n 'checkaffiliates': True,\n 'cacheperiod': 18,\n 'checknetblocks': True,\n 'checksubnets': True\n }\n\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliate IP addresses?\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\",\n 'checknetblocks': \"Report if any malicious IPs are found within owned netblocks?\",\n 'checksubnets': \"Check if any malicious IPs are found within the same subnet of the target?\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n 'IP_ADDRESS',\n 'AFFILIATE_IPADDR',\n 'NETBLOCK_OWNER',\n 'NETBLOCK_MEMBER',\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"BLACKLISTED_SUBNET\",\n \"BLACKLISTED_NETBLOCK\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_NETBLOCK\",\n \"MALICIOUS_SUBNET\",\n ]\n\n def query(self, qry, targetType):\n cid = \"_greensnow\"\n url = \"https://blocklist.greensnow.co/greensnow.txt\"\n\n data = dict()\n data[\"content\"] = self.sf.cacheGet(\"sfmal_\" + cid, self.opts.get('cacheperiod', 0))\n\n if data[\"content\"] is None:\n data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])\n\n if data[\"code\"] != \"200\":\n self.error(f\"Unable to fetch {url}\")\n self.errorState = True\n return None\n\n if data[\"content\"] is None:\n self.error(f\"Unable to fetch {url}\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"sfmal_\" + cid, data['content'])\n\n for line in data[\"content\"].split('\\n'):\n ip = line.strip().lower()\n\n if targetType == \"netblock\":\n try:\n if IPAddress(ip) in IPNetwork(qry):\n self.debug(f\"{ip} found within netblock/subnet {qry} in greensnow.co list.\")\n return f\"https://greensnow.co/view/{ip}\"\n except Exception as e:\n self.debug(f\"Error encountered parsing: {e}\")\n continue\n\n if targetType == \"ip\":\n if qry.lower() == ip:\n self.debug(f\"{qry} found in greensnow.co list.\")\n return f\"https://greensnow.co/view/{ip}\"\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName == 'IP_ADDRESS':\n targetType = 'ip'\n malicious_type = \"MALICIOUS_IPADDR\"\n blacklist_type = \"BLACKLISTED_IPADDR\"\n elif eventName == 'AFFILIATE_IPADDR':\n if not self.opts.get('checkaffiliates', False):\n return\n targetType = 'ip'\n malicious_type = \"MALICIOUS_AFFILIATE_IPADDR\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_IPADDR\"\n elif eventName == 'NETBLOCK_OWNER':\n if not self.opts.get('checknetblocks', False):\n return\n targetType = 'netblock'\n malicious_type = \"MALICIOUS_NETBLOCK\"\n blacklist_type = \"BLACKLISTED_NETBLOCK\"\n elif eventName == 'NETBLOCK_MEMBER':\n if not self.opts.get('checksubnets', False):\n return\n targetType = 'netblock'\n malicious_type = \"MALICIOUS_SUBNET\"\n blacklist_type = \"BLACKLISTED_SUBNET\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n self.debug(f\"Checking maliciousness of {eventData} ({eventName}) with greensnow.co\")\n\n url = self.query(eventData, targetType)\n\n if not url:\n return\n\n text = f\"greensnow.co [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_greensnow class\n" }, { "path": "modules/sfp_grep_app.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_grep_app\n# Purpose: Searches grep.app API for domains, URLs and emails related to the\n# specified domain.\n#\n# Author: \n#\n# Created: 2020-04-12\n# Copyright: (c) bcoles 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport math\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_grep_app(SpiderFootPlugin):\n\n meta = {\n 'name': \"grep.app\",\n 'summary': \"Search grep.app API for links and emails related to the specified domain.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://grep.app/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [],\n 'favIcon': \"https://grep.app/favicon-16x16.png\",\n 'logo': \"https://grep.app/apple-touch-icon.png\",\n 'description': \"grep.app searches code from over a half million public repositories on GitHub.\\n\"\n \"It searches for the exact string you enter, including any punctuation or other characters.\\n\"\n \"You can also search by regular expression, using the RE2 syntax.\",\n }\n }\n\n # Default options\n opts = {\n 'max_pages': 20,\n 'dns_resolve': True,\n }\n\n # Option descriptions\n optdescs = {\n 'max_pages': \"Maximum number of pages of results to fetch.\",\n 'dns_resolve': \"DNS resolve each identified domain.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"DOMAIN_NAME\"]\n\n def producedEvents(self):\n return [\"EMAILADDR\", \"EMAILADDR_GENERIC\", \"DOMAIN_NAME\",\n \"INTERNET_NAME\", \"RAW_RIR_DATA\",\n \"INTERNET_NAME_UNRESOLVED\", \"LINKED_URL_INTERNAL\"]\n\n def query(self, qry, page):\n params = {\n 'q': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n 'page': str(page)\n }\n\n res = self.sf.fetchUrl(\"https://grep.app/api/search?\" + urllib.parse.urlencode(params),\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout'])\n\n time.sleep(1)\n\n if res['content'] is None:\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if srcModuleName == 'sfp_grep_app':\n self.debug(\"Ignoring \" + eventData + \", from self.\")\n return\n\n hosts = list()\n page = 1\n per_page = 10\n pages = self.opts['max_pages']\n while page <= pages:\n if self.checkForStop():\n return\n\n if self.errorState:\n return\n\n res = self.query(eventData, page)\n\n if res is None:\n return\n\n facets = res.get('facets')\n\n if facets is None:\n return\n\n count = facets.get('count')\n\n if count is None:\n return\n\n last_page = math.ceil(count / per_page)\n\n if last_page is None:\n pages = 0\n\n if last_page < pages:\n pages = last_page\n\n self.info(\"Parsing page \" + str(page) + \" of \" + str(pages))\n page += 1\n\n hits = res.get('hits')\n\n if hits is None:\n return\n\n data = hits.get('hits')\n\n if data is None:\n return\n\n for result in data:\n if result is None:\n continue\n\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(result), self.__name__, event)\n self.notifyListeners(evt)\n\n content = result.get('content')\n\n if content is None:\n continue\n\n snippet = content.get('snippet')\n\n if snippet is None:\n continue\n\n links = self.sf.extractUrlsFromText(snippet.replace('', '').replace('', ''))\n if links:\n for link in links:\n if link in self.results:\n continue\n\n host = self.sf.urlFQDN(link)\n\n if not self.getTarget().matches(host, includeChildren=True, includeParents=True):\n continue\n\n hosts.append(host)\n\n if not self.getTarget().matches(self.sf.urlFQDN(link), includeChildren=True, includeParents=True):\n self.debug(\"Skipped unrelated link: \" + link)\n continue\n\n self.debug('Found a link: ' + link)\n evt = SpiderFootEvent('LINKED_URL_INTERNAL', link, self.__name__, event)\n self.notifyListeners(evt)\n self.results[link] = True\n\n emails = SpiderFootHelpers.extractEmailsFromText(snippet.replace('', '').replace('', ''))\n if emails:\n for email in emails:\n if email in self.results:\n continue\n\n mail_domain = email.lower().split('@')[1]\n if not self.getTarget().matches(mail_domain, includeChildren=True, includeParents=True):\n self.debug(\"Skipped unrelated email address: \" + email)\n continue\n\n self.info(\"Found e-mail address: \" + email)\n if email.split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evttype = \"EMAILADDR_GENERIC\"\n else:\n evttype = \"EMAILADDR\"\n\n evt = SpiderFootEvent(evttype, email, self.__name__, event)\n self.notifyListeners(evt)\n self.results[email] = True\n\n for host in set(hosts):\n if self.checkForStop():\n return\n\n if self.errorState:\n return\n\n if self.opts['dns_resolve'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host):\n self.debug(f\"Host {host} could not be resolved\")\n evt = SpiderFootEvent(\"INTERNET_NAME_UNRESOLVED\", host, self.__name__, event)\n self.notifyListeners(evt)\n continue\n\n evt = SpiderFootEvent(\"INTERNET_NAME\", host, self.__name__, event)\n self.notifyListeners(evt)\n if self.sf.isDomain(host, self.opts[\"_internettlds\"]):\n evt = SpiderFootEvent(\"DOMAIN_NAME\", host, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_grep_app class\n" }, { "path": "modules/sfp_greynoise.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_greynoise\n# Purpose: Query GreyNoise's API\n#\n# Author: Steve Micallef\n# Updated By: Brad Chiappetta, GreyNoise\n#\n# Created: 20/11/2018\n# Updated: 31-Aug-2022\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nfrom datetime import datetime\nfrom netaddr import IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_greynoise(SpiderFootPlugin):\n\n meta = {\n \"name\": \"GreyNoise\",\n \"summary\": \"Obtain IP enrichment data from GreyNoise\",\n \"flags\": [\"apikey\"],\n \"useCases\": [\"Investigate\", \"Passive\"],\n \"categories\": [\"Reputation Systems\"],\n \"dataSource\": {\n \"website\": \"https://greynoise.io/\",\n \"model\": \"FREE_AUTH_LIMITED\",\n \"references\": [\"https://docs.greynoise.io/\", \"https://viz.greynoise.io/signup\"],\n \"apiKeyInstructions\": [\n \"Visit https://viz.greynoise.io/signup\",\n \"Sign up for a free account\",\n \"Navigate to https://viz.greynoise.io/account\",\n \"The API key is listed under 'API Key'\",\n ],\n \"favIcon\": \"https://viz.greynoise.io/favicon.ico\",\n \"logo\": \"https://viz.greynoise.io/_nuxt/img/greynoise-logo.dccd59d.png\",\n \"description\": \"At GreyNoise, we collect and analyze untargeted, widespread, \"\n \"and opportunistic scan and attack activity that reaches every server directly connected to the Internet. \"\n \"Mass scanners (such as Shodan and Censys), search engines, bots, worms, \"\n \"and crawlers generate logs and events omnidirectionally on every IP address in the IPv4 space. \"\n \"GreyNoise gives you the ability to filter this useless noise out.\",\n },\n }\n\n # Default options\n opts = {\n \"api_key\": \"\",\n \"age_limit_days\": 30,\n \"netblocklookup\": True,\n \"maxnetblock\": 24,\n \"subnetlookup\": True,\n \"maxsubnet\": 24\n # 'asnlookup': True\n }\n\n # Option descriptions\n optdescs = {\n \"api_key\": \"GreyNoise API Key.\",\n \"age_limit_days\": \"Ignore any records older than this many days. 0 = unlimited.\",\n \"netblocklookup\": \"Look up netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?\",\n \"maxnetblock\": \"If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n \"subnetlookup\": \"Look up subnets which your target is a part of for blacklisting?\",\n \"maxsubnet\": \"If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\"\n # 'asnlookup': \"Look up ASNs that your target is a member of?\"\n }\n\n # Be sure to completely clear any class variables in setup()\n # or you run the risk of data persisting between scan runs.\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n # Clear / reset any other class member variables here\n # or you risk them persisting between threads.\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"IP_ADDRESS\", \"AFFILIATE_IPADDR\", \"NETBLOCK_MEMBER\", \"NETBLOCK_OWNER\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_ASN\",\n \"MALICIOUS_SUBNET\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_NETBLOCK\",\n \"COMPANY_NAME\",\n \"GEOINFO\",\n \"BGP_AS_MEMBER\",\n \"OPERATING_SYSTEM\",\n \"RAW_RIR_DATA\",\n ]\n\n def queryIP(self, qry, qry_type):\n gn_context_url = \"https://api.greynoise.io/v2/noise/context/\"\n gn_gnql_url = \"https://api.greynoise.io/v2/experimental/gnql?query=\"\n\n headers = {\"key\": self.opts[\"api_key\"]}\n res = {}\n if qry_type == \"ip\":\n self.debug(f\"Querying GreyNoise for IP: {qry}\")\n res = {}\n ip_response = self.sf.fetchUrl(\n gn_context_url + qry,\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=\"greynoise-spiderfoot-v1.2.0\",\n headers=headers,\n )\n if ip_response[\"code\"] == \"200\":\n res = json.loads(ip_response[\"content\"])\n else:\n self.debug(f\"Querying GreyNoise for Netblock: {qry}\")\n query_response = self.sf.fetchUrl(\n gn_gnql_url + qry,\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=\"greynoise-spiderfoot-v1.1.0\",\n headers=headers,\n )\n if query_response[\"code\"] == \"200\":\n res = json.loads(query_response[\"content\"])\n\n if not res:\n self.error(\"Greynoise API key seems to have been rejected or you have exceeded usage limits.\")\n self.errorState = True\n return None\n\n return res\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\"You enabled sfp_greynoise but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName == \"NETBLOCK_OWNER\":\n if not self.opts[\"netblocklookup\"]:\n return\n\n max_netblock = self.opts['maxnetblock']\n if IPNetwork(eventData).prefixlen < max_netblock:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}\")\n return\n\n if eventName == \"NETBLOCK_MEMBER\":\n if not self.opts[\"subnetlookup\"]:\n return\n\n max_subnet = self.opts['maxsubnet']\n if IPNetwork(eventData).prefixlen < max_subnet:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}\")\n return\n\n if eventName == \"IP_ADDRESS\":\n evtType = \"MALICIOUS_IPADDR\"\n qryType = \"ip\"\n if eventName.startswith(\"NETBLOCK_\"):\n evtType = \"MALICIOUS_IPADDR\"\n qryType = \"netblock\"\n if eventName == \"AFFILIATE_IPADDR\":\n evtType = \"MALICIOUS_AFFILIATE_IPADDR\"\n qryType = \"ip\"\n\n ret = self.queryIP(eventData, qryType)\n\n if not ret:\n return\n\n if \"data\" not in ret and \"seen\" not in ret:\n return\n\n if \"data\" in ret and len(ret[\"data\"]) > 0:\n for rec in ret[\"data\"]:\n if rec.get(\"seen\", None):\n self.debug(f\"Found threat info in Greynoise: {rec['ip']}\")\n lastseen = rec.get(\"last_seen\", \"1970-01-01\")\n lastseen_dt = datetime.strptime(lastseen, \"%Y-%m-%d\")\n lastseen_ts = int(time.mktime(lastseen_dt.timetuple()))\n age_limit_ts = int(time.time()) - (86400 * self.opts[\"age_limit_days\"])\n if self.opts[\"age_limit_days\"] > 0 and lastseen_ts < age_limit_ts:\n self.debug(f\"Record [{rec['ip']}] found but too old, skipping.\")\n return\n\n # Only report meta data about the target, not affiliates\n if rec.get(\"metadata\") and eventName == \"IP_ADDRESS\":\n met = rec.get(\"metadata\")\n if met.get(\"country\", \"unknown\") != \"unknown\":\n loc = \"\"\n if met.get(\"city\"):\n loc = met.get(\"city\") + \", \"\n loc += met.get(\"country\")\n e = SpiderFootEvent(\"GEOINFO\", loc, self.__name__, event)\n self.notifyListeners(e)\n if met.get(\"asn\", \"unknown\") != \"unknown\":\n asn = met.get(\"asn\").replace(\"AS\", \"\")\n e = SpiderFootEvent(\"BGP_AS_MEMBER\", asn, self.__name__, event)\n self.notifyListeners(e)\n if met.get(\"organization\", \"unknown\") != \"unknown\":\n e = SpiderFootEvent(\"COMPANY_NAME\", met.get(\"organization\"), self.__name__, event)\n self.notifyListeners(e)\n if met.get(\"os\", \"unknown\") != \"unknown\":\n e = SpiderFootEvent(\"OPERATING_SYSTEM\", met.get(\"os\"), self.__name__, event)\n self.notifyListeners(e)\n e = SpiderFootEvent(\"RAW_RIR_DATA\", str(rec), self.__name__, event)\n self.notifyListeners(e)\n\n if rec.get(\"classification\"):\n descr = (\n \"GreyNoise - Mass-Scanning IP Detected [\"\n + rec.get(\"ip\")\n + \"]\\n - Classification: \"\n + rec.get(\"classification\")\n )\n if rec.get(\"tags\"):\n descr += \"\\n - \" + \"Scans For Tags: \" + \", \".join(rec.get(\"tags\"))\n if rec.get(\"cve\"):\n descr += \"\\n - \" + \"Scans For CVEs: \" + \", \".join(rec.get(\"cve\"))\n if rec.get(\"raw_data\") and not (rec.get(\"tags\") or ret.get(\"cve\")):\n descr += \"\\n - \" + \"Raw data: \" + str(rec.get(\"raw_data\"))\n descr += \"\\nhttps://viz.greynoise.io/ip/\" + rec.get(\"ip\") + \"\"\n e = SpiderFootEvent(evtType, descr, self.__name__, event)\n self.notifyListeners(e)\n\n if \"seen\" in ret:\n if ret.get(\"seen\", None):\n lastseen = ret.get(\"last_seen\", \"1970-01-01\")\n lastseen_dt = datetime.strptime(lastseen, \"%Y-%m-%d\")\n lastseen_ts = int(time.mktime(lastseen_dt.timetuple()))\n age_limit_ts = int(time.time()) - (86400 * self.opts[\"age_limit_days\"])\n if self.opts[\"age_limit_days\"] > 0 and lastseen_ts < age_limit_ts:\n self.debug(\"Record found but too old, skipping.\")\n return\n\n # Only report meta data about the target, not affiliates\n if ret.get(\"metadata\") and eventName == \"IP_ADDRESS\":\n met = ret.get(\"metadata\")\n if met.get(\"country\", \"unknown\") != \"unknown\":\n loc = \"\"\n if met.get(\"city\"):\n loc = met.get(\"city\") + \", \"\n loc += met.get(\"country\")\n e = SpiderFootEvent(\"GEOINFO\", loc, self.__name__, event)\n self.notifyListeners(e)\n if met.get(\"asn\", \"unknown\") != \"unknown\":\n asn = met.get(\"asn\").replace(\"AS\", \"\")\n e = SpiderFootEvent(\"BGP_AS_MEMBER\", asn, self.__name__, event)\n self.notifyListeners(e)\n if met.get(\"organization\", \"unknown\") != \"unknown\":\n e = SpiderFootEvent(\"COMPANY_NAME\", met.get(\"organization\"), self.__name__, event)\n self.notifyListeners(e)\n if met.get(\"os\", \"unknown\") != \"unknown\":\n e = SpiderFootEvent(\"OPERATING_SYSTEM\", met.get(\"os\"), self.__name__, event)\n self.notifyListeners(e)\n e = SpiderFootEvent(\"RAW_RIR_DATA\", str(ret), self.__name__, event)\n self.notifyListeners(e)\n\n if ret.get(\"classification\"):\n descr = (\n \"GreyNoise - Mass-Scanning IP Detected [\"\n + eventData\n + \"]\\n - Classification: \"\n + ret.get(\"classification\")\n )\n if ret.get(\"tags\"):\n descr += \"\\n - \" + \"Scans For Tags: \" + \", \".join(ret.get(\"tags\"))\n if ret.get(\"cve\"):\n descr += \"\\n - \" + \"Scans For CVEs: \" + \", \".join(ret.get(\"cve\"))\n if ret.get(\"raw_data\") and not (ret.get(\"tags\") or ret.get(\"cve\")):\n descr += \"\\n - \" + \"Raw data: \" + str(ret.get(\"raw_data\"))\n descr += \"\\nhttps://viz.greynoise.io/ip/\" + ret.get(\"ip\") + \"\"\n e = SpiderFootEvent(evtType, descr, self.__name__, event)\n self.notifyListeners(e)\n\n# End of sfp_greynoise class\n" }, { "path": "modules/sfp_greynoise_community.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_greynoise_community\n# Purpose: Query GreyNoise's Community API\n#\n# Author: Brad Chiappetta, GreyNoise\n# Updated By: Brad Chiappetta, GreyNoise\n#\n# Created: 31-Aug-2022\n# Updated: 31-Aug-2022\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nfrom datetime import datetime\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_greynoise_community(SpiderFootPlugin):\n\n meta = {\n \"name\": \"GreyNoise Community\",\n \"summary\": \"Obtain IP enrichment data from GreyNoise Community API\",\n \"flags\": [\"apikey\"],\n \"useCases\": [\"Investigate\", \"Passive\"],\n \"categories\": [\"Reputation Systems\"],\n \"dataSource\": {\n \"website\": \"https://greynoise.io/\",\n \"model\": \"FREE_AUTH_LIMITED\",\n \"references\": [\"https://docs.greynoise.io/reference/get_v3-community-ip\", \"https://viz.greynoise.io/signup\"],\n \"apiKeyInstructions\": [\n \"Visit https://viz.greynoise.io/signup\",\n \"Sign up for a free account\",\n \"Navigate to https://viz.greynoise.io/account/\",\n \"The API key is listed under 'API Key'\",\n ],\n \"favIcon\": \"https://viz.greynoise.io/favicon.ico\",\n \"logo\": \"https://viz.greynoise.io/_nuxt/img/greynoise-logo.dccd59d.png\",\n \"description\": \"At GreyNoise, we collect and analyze untargeted, widespread, \"\n \"and opportunistic scan and attack activity that reaches every server directly connected to the Internet. \"\n \"Mass scanners (such as Shodan and Censys), search engines, bots, worms, \"\n \"and crawlers generate logs and events omnidirectionally on every IP address in the IPv4 space. \"\n \"GreyNoise gives you the ability to filter this useless noise out.\",\n },\n }\n\n # Default options\n opts = {\n \"api_key\": \"\",\n \"age_limit_days\": 30,\n }\n\n # Option descriptions\n optdescs = {\n \"api_key\": \"GreyNoise Community API Key.\",\n \"age_limit_days\": \"Ignore any records older than this many days. 0 = unlimited.\",\n }\n\n # Be sure to completely clear any class variables in setup()\n # or you run the risk of data persisting between scan runs.\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n # Clear / reset any other class member variables here\n # or you risk them persisting between threads.\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"IP_ADDRESS\", \"AFFILIATE_IPADDR\", \"NETBLOCK_MEMBER\", \"NETBLOCK_OWNER\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\n \"MALICIOUS_IPADDR\",\n \"COMPANY_NAME\",\n \"RAW_RIR_DATA\",\n ]\n\n def queryIP(self, qry, qry_type):\n gn_community_url = \"https://api.greynoise.io/v3/community/\"\n\n headers = {\"key\": self.opts[\"api_key\"]}\n res = {}\n if qry_type == \"ip\":\n self.debug(f\"Querying GreyNoise Community API for IP: {qry}\")\n ip_res = {}\n ip_response = self.sf.fetchUrl(\n gn_community_url + qry,\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=\"greynoise-spiderfoot-community-v1.2.0\",\n headers=headers,\n )\n if ip_response[\"code\"] == \"200\":\n ip_res = json.loads(ip_response[\"content\"])\n res = ip_res\n\n if not res:\n self.error(\"Greynoise API key seems to have been rejected or you have exceeded usage limits.\")\n self.errorState = True\n return None\n\n return res\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\"You enabled sfp_greynoise_community but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName == \"IP_ADDRESS\":\n evtType = \"MALICIOUS_IPADDR\"\n qryType = \"ip\"\n if eventName == \"AFFILIATE_IPADDR\":\n evtType = \"MALICIOUS_AFFILIATE_IPADDR\"\n qryType = \"ip\"\n\n ret = self.queryIP(eventData, qryType)\n\n if not ret:\n return\n\n if \"data\" not in ret and \"noise\" not in ret:\n return\n\n if \"noise\" in ret:\n if ret.get(\"noise\", None):\n lastseen = ret.get(\"last_seen\", \"1970-01-01\")\n lastseen_dt = datetime.strptime(lastseen, \"%Y-%m-%d\")\n lastseen_ts = int(time.mktime(lastseen_dt.timetuple()))\n age_limit_ts = int(time.time()) - (86400 * self.opts[\"age_limit_days\"])\n if self.opts[\"age_limit_days\"] > 0 and lastseen_ts < age_limit_ts:\n self.debug(\"Record found but too old, skipping.\")\n return\n e = SpiderFootEvent(\"RAW_RIR_DATA\", str(ret), self.__name__, event)\n self.notifyListeners(e)\n\n # Only report meta data about the target, not affiliates\n if ret.get(\"name\", \"unknown\") != \"unknown\":\n e = SpiderFootEvent(\"COMPANY_NAME\", ret.get(\"name\"), self.__name__, event)\n self.notifyListeners(e)\n\n if ret.get(\"classification\"):\n descr = (\n \"GreyNoise - Mass-Scanning IP Detected [\"\n + eventData\n + \"]\\n - Classification: \"\n + ret.get(\"classification\")\n )\n descr += \"\\nhttps://viz.greynoise.io/ip/\" + ret.get(\"ip\") + \"\"\n e = SpiderFootEvent(evtType, descr, self.__name__, event)\n self.notifyListeners(e)\n\n# End of sfp_greynoise_community class\n" }, { "path": "modules/sfp_h1nobbdde.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_h1.nobbd.de\n# Purpose: Query the the unofficial HackerOne disclosure timeline database\n# to see if our target appears.\n#\n# Author: Dhiraj Mishra \n# Created: 28/10/2018\n# Copyright: (c) Dhiraj Mishra\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_h1nobbdde(SpiderFootPlugin):\n\n meta = {\n 'name': \"HackerOne (Unofficial)\",\n 'summary': \"Check external vulnerability scanning/reporting service h1.nobbd.de to see if the target is listed.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Leaks, Dumps and Breaches\"],\n 'dataSource': {\n 'website': \"http://www.nobbd.de/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"http://www.nobbd.de/index.php#projekte\",\n \"https://twitter.com/disclosedh1\"\n ],\n 'favIcon': \"http://www.nobbd.de/favicon.ico\",\n 'logo': \"http://www.nobbd.de/favicon.ico\",\n 'description': \"Unofficial Bug Monitoring platform for HackerOne.\",\n }\n }\n\n # Default options\n opts = {\n }\n\n # Option descriptions\n optdescs = {\n }\n\n # Be sure to completely clear any class variables in setup()\n # or you run the risk of data persisting between scan runs.\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n # Clear / reset any other class member variables here\n # or you risk them persisting between threads.\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"DOMAIN_NAME\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"VULNERABILITY_DISCLOSURE\"]\n\n # Query h1.nobbd.de\n def queryOBB(self, qry):\n ret = list()\n url = \"http://h1.nobbd.de/search.php?q=\" + qry\n res = self.sf.fetchUrl(url, timeout=30, useragent=self.opts['_useragent'])\n\n if res['content'] is None:\n self.debug(\"No content returned from h1.nobbd.de\")\n return None\n\n try:\n rx = re.compile(\"\" + m[0] + \"\")\n except Exception as e:\n self.error(f\"Error processing response from h1.nobbd.de: {e}\")\n return None\n\n return ret\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n data = list()\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n obb = self.queryOBB(eventData)\n if obb:\n data.extend(obb)\n\n for n in data:\n e = SpiderFootEvent(\"VULNERABILITY_DISCLOSURE\", n, self.__name__, event)\n self.notifyListeners(e)\n\n# End of sfp_h1nobbdde class\n" }, { "path": "modules/sfp_hackertarget.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_hackertarget\n# Purpose: SpiderFoot plug-in to search HackerTarget.com for hosts sharing\n# the same IP. Optionally, also perform a basic TCP/UDP port scan\n# for commonly open ports using HackerTarget.com port scan tools.\n#\n# Author: Steve Micallef \n#\n# Created: 12/04/2014\n# Copyright: (c) Steve Micallef 2014\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport re\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom netaddr import IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_hackertarget(SpiderFootPlugin):\n\n meta = {\n 'name': \"HackerTarget\",\n 'summary': \"Search HackerTarget.com for hosts sharing the same IP.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\"],\n 'categories': [\"Passive DNS\"],\n 'dataSource': {\n 'website': \"https://hackertarget.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://hackertarget.com/research/\",\n \"https://hackertarget.com/category/tools/\"\n ],\n 'favIcon': \"https://www.google.com/s2/favicons?domain=https://hackertarget.com/\",\n 'logo': \"https://hackertarget.com/wp-content/uploads/2018/03/online-security.png\",\n 'description': \"Simplify the security assessment process with hosted vulnerability scanners. \"\n \"From attack surface discovery to vulnerability identification, \"\n \"actionable network intelligence for IT & security operations. \"\n \"Proactively hunt for security weakness. \"\n \"Pivot from attack surface discovery to vulnerability identification.\",\n }\n }\n\n opts = {\n 'cohostsamedomain': False,\n 'verify': True,\n 'netblocklookup': True,\n 'maxnetblock': 24,\n 'maxcohost': 100,\n 'http_headers': False,\n }\n\n optdescs = {\n 'cohostsamedomain': \"Treat co-hosted sites on the same target domain as co-hosting?\",\n 'verify': \"Verify co-hosts are valid by checking if they still resolve to the shared IP.\",\n 'netblocklookup': \"Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?\",\n 'maxnetblock': \"If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'maxcohost': \"Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.\",\n 'http_headers': \"Retrieve IP HTTP headers using HackerTarget.com\",\n }\n\n results = None\n errorState = False\n cohostcount = 0\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.cohostcount = 0\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"NETBLOCK_OWNER\",\n 'DOMAIN_NAME_PARENT'\n ]\n\n def producedEvents(self):\n return [\n \"CO_HOSTED_SITE\",\n \"IP_ADDRESS\",\n 'WEBSERVER_HTTPHEADERS',\n 'RAW_DNS_RECORDS',\n 'INTERNET_NAME',\n 'INTERNET_NAME_UNRESOLVED',\n 'DOMAIN_NAME',\n 'AFFILIATE_DOMAIN_NAME',\n 'AFFILIATE_INTERNET_NAME',\n 'AFFILIATE_INTERNET_NAME_UNRESOLVED'\n ]\n\n def httpHeaders(self, ip):\n \"\"\"Retrieve HTTP headers for IP address\n\n Args:\n ip (str): IPv4 address\n\n Returns:\n dict: HTTP headers\n \"\"\"\n params = urllib.parse.urlencode({\n 'q': ip\n })\n\n res = self.sf.fetchUrl(\n f\"https://api.hackertarget.com/httpheaders/?{params}\",\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout']\n )\n\n if res['content'] is None:\n self.error(f\"Unable to fetch HTTP headers for {ip} from HackerTarget.com.\")\n return None\n\n if res['code'] == '429':\n self.error(\"You are being rate-limited by HackerTarget\")\n self.errorState = True\n return None\n\n if not res['content'].startswith('HTTP/'):\n self.debug(f\"Found no HTTP headers for {ip}\")\n return None\n\n headers = dict()\n\n for header in res['content'].splitlines():\n if ': ' not in header:\n continue\n k = header.split(': ')[0].lower()\n v = ': '.join(header.split(': ')[1:])\n headers[k] = v\n\n return headers\n\n def zoneTransfer(self, ip):\n \"\"\"Retrieve DNS zone transfer\n\n Args:\n ip (str): IPv4 address\n\n Returns:\n list: DNS zone\n \"\"\"\n params = urllib.parse.urlencode({\n 'q': ip\n })\n\n res = self.sf.fetchUrl(\n f\"https://api.hackertarget.com/zonetransfer/?{params}\",\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout']\n )\n\n if res['content'] is None:\n self.error(f\"Unable to fetch DNS zone for {ip} from HackerTarget.com.\")\n return None\n\n if res['code'] == '429':\n self.error(\"You are being rate-limited by HackerTarget\")\n self.errorState = True\n return None\n\n records = list()\n\n for record in res['content'].splitlines():\n if record.strip().startswith(';'):\n continue\n if record.strip() == '':\n continue\n records.append(record.strip())\n\n return records\n\n def reverseIpLookup(self, ip):\n \"\"\"Reverse lookup hosts on the same IP address\n\n Args:\n ip (str): IPv4 address\n\n Returns:\n list: (co)hosts on provided IP addresses\n \"\"\"\n params = urllib.parse.urlencode({\n 'q': ip\n })\n\n res = self.sf.fetchUrl(\n f\"https://api.hackertarget.com/reverseiplookup/?{params}\",\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout']\n )\n\n if res['content'] is None:\n self.error(\"Unable to fetch hackertarget.com content.\")\n return None\n\n if res['code'] == '429':\n self.error(\"You are being rate-limited by HackerTarget\")\n self.errorState = True\n return None\n\n if \"No records\" in res['content']:\n return None\n\n hosts = res['content'].split('\\n')\n\n self.debug(f\"Found {len(hosts)} on {ip}\")\n\n return hosts\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n self.currentEventSrc = event\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if srcModuleName == \"sfp_hackertarget\" and eventName == \"IP_ADDRESS\":\n self.debug(f\"Ignoring {eventName}, from self.\")\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if eventName == 'NETBLOCK_OWNER':\n if not self.opts['netblocklookup']:\n return\n\n max_netblock = self.opts['maxnetblock']\n net_size = IPNetwork(eventData).prefixlen\n if net_size < max_netblock:\n self.debug(f\"Network size bigger than permitted: {net_size} > {max_netblock}\")\n return\n\n if eventName == 'DOMAIN_NAME_PARENT':\n records = self.zoneTransfer(eventData)\n\n if not records:\n return\n\n evt = SpiderFootEvent('RAW_DNS_RECORDS', \"\\n\".join(records), self.__name__, event)\n self.notifyListeners(evt)\n\n # Try and pull out individual records\n for row in records:\n pat = re.compile(r\"^(\\S+)\\.?\\s+\\d+\\s+IN\\s+[AC].*\", re.IGNORECASE | re.DOTALL)\n grps = re.findall(pat, row)\n\n if len(grps) == 0:\n continue\n\n hosts = list()\n\n for strdata in grps:\n self.debug(\"Matched: \" + strdata)\n if strdata.endswith(\".\"):\n hosts.append(strdata[:-1])\n else:\n hosts.append(strdata)\n\n for host in set(hosts):\n if self.getTarget().matches(host, includeChildren=True, includeParents=True):\n evt_type = 'INTERNET_NAME'\n else:\n evt_type = 'AFFILIATE_INTERNET_NAME'\n\n if self.opts['verify'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host):\n self.debug(f\"Host {host} could not be resolved\")\n evt_type += '_UNRESOLVED'\n\n evt = SpiderFootEvent(evt_type, host, self.__name__, event)\n self.notifyListeners(evt)\n\n if self.sf.isDomain(host, self.opts['_internettlds']):\n if evt_type.startswith('AFFILIATE'):\n evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', host, self.__name__, event)\n self.notifyListeners(evt)\n else:\n evt = SpiderFootEvent('DOMAIN_NAME', host, self.__name__, event)\n self.notifyListeners(evt)\n\n return\n\n qrylist = list()\n if eventName.startswith(\"NETBLOCK_\"):\n for ipaddr in IPNetwork(eventData):\n if str(ipaddr) not in self.results:\n qrylist.append(str(ipaddr))\n self.results[str(ipaddr)] = True\n else:\n qrylist.append(eventData)\n self.results[eventData] = True\n\n myres = list()\n\n for ip in qrylist:\n if self.checkForStop():\n return\n\n hosts = self.reverseIpLookup(ip)\n if not hosts:\n continue\n\n for h in hosts:\n if \" \" in h:\n continue\n\n self.info(f\"Found something on same IP: {h}\")\n\n if not self.opts['cohostsamedomain']:\n if self.getTarget().matches(h, includeParents=True):\n self.debug(f\"Skipping {h} because it is on the same domain.\")\n continue\n\n if h not in myres and h != ip:\n if self.opts['verify'] and not self.sf.validateIP(h, ip):\n self.debug(f\"Host {h} no longer resolves to {ip}\")\n continue\n\n if self.cohostcount < self.opts['maxcohost']:\n # Create an IP Address event stemming from the netblock as the\n # link to the co-host.\n if eventName == \"NETBLOCK_OWNER\":\n ipe = SpiderFootEvent(\"IP_ADDRESS\", ip, self.__name__, event)\n self.notifyListeners(ipe)\n evt = SpiderFootEvent(\"CO_HOSTED_SITE\", h.lower(), self.__name__, ipe)\n self.notifyListeners(evt)\n else:\n evt = SpiderFootEvent(\"CO_HOSTED_SITE\", h.lower(), self.__name__, event)\n self.notifyListeners(evt)\n\n myres.append(h.lower())\n self.cohostcount += 1\n\n # For netblocks, we need to create the IP address event so that\n # the threat intel event is more meaningful.\n if eventName == 'NETBLOCK_OWNER':\n pevent = SpiderFootEvent(\"IP_ADDRESS\", ip, self.__name__, event)\n self.notifyListeners(pevent)\n else:\n pevent = event\n\n if self.opts.get('http_headers', True):\n http_headers = self.httpHeaders(ip)\n if http_headers is not None:\n e = SpiderFootEvent('WEBSERVER_HTTPHEADERS', json.dumps(http_headers), self.__name__, pevent)\n e.actualSource = ip\n self.notifyListeners(e)\n\n# End of sfp_hackertarget class\n" }, { "path": "modules/sfp_hashes.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_hashes\n# Purpose: SpiderFoot plug-in for scanning retrieved content by other\n# modules (such as sfp_spider) and identifying hashes\n#\n# Author: Steve Micallef \n#\n# Created: 24/01/2020\n# Copyright: (c) Steve Micallef 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_hashes(SpiderFootPlugin):\n\n meta = {\n 'name': \"Hash Extractor\",\n 'summary': \"Identify MD5 and SHA hashes in web content, files and more.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Content Analysis\"]\n }\n\n # Default options\n opts = {\n # options specific to this module\n }\n\n # Option descriptions\n optdescs = {\n }\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n\n for opt in userOpts.keys():\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"TARGET_WEB_CONTENT\", \"BASE64_DATA\",\n \"LEAKSITE_CONTENT\", \"RAW_DNS_RECORDS\",\n \"RAW_FILE_META_DATA\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"HASH\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n hashes = SpiderFootHelpers.extractHashesFromText(eventData)\n for hashtup in hashes:\n hashalgo, hashval = hashtup\n\n evt = SpiderFootEvent(\"HASH\", f\"[{hashalgo}] {hashval}\", self.__name__, event)\n if event.moduleDataSource:\n evt.moduleDataSource = event.moduleDataSource\n else:\n evt.moduleDataSource = \"Unknown\"\n self.notifyListeners(evt)\n\n# End of sfp_hashes class\n" }, { "path": "modules/sfp_haveibeenpwned.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_haveibeenpwned\n# Purpose: Query haveibeenpwned.com to see if an e-mail account has been hacked.\n#\n# Author: Steve Micallef \n#\n# Created: 19/02/2015\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport re\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_haveibeenpwned(SpiderFootPlugin):\n\n meta = {\n 'name': \"HaveIBeenPwned\",\n 'summary': \"Check HaveIBeenPwned.com for hacked e-mail addresses identified in breaches.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Leaks, Dumps and Breaches\"],\n 'dataSource': {\n 'website': \"https://haveibeenpwned.com/\",\n 'model': \"COMMERCIAL_ONLY\",\n 'references': [\n \"https://haveibeenpwned.com/API/v3\",\n \"https://haveibeenpwned.com/FAQs\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://haveibeenpwned.com/API/Key\",\n \"Register an account\",\n \"Visit https://haveibeenpwned.com/API/Key\",\n ],\n 'favIcon': \"https://haveibeenpwned.com/favicon.ico\",\n 'logo': \"https://haveibeenpwned.com/favicon.ico\",\n 'description': \"Check if you have an account that has been compromised in a data breach.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\"\n }\n\n # Option descriptions\n optdescs = {\n \"api_key\": \"HaveIBeenPwned.com API key.\"\n }\n\n # Be sure to completely clear any class variables in setup()\n # or you run the risk of data persisting between scan runs.\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n # Clear / reset any other class member variables here\n # or you risk them persisting between threads.\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"EMAILADDR\", \"PHONE_NUMBER\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"EMAILADDR_COMPROMISED\", \"PHONE_NUMBER_COMPROMISED\", \"LEAKSITE_CONTENT\", \"LEAKSITE_URL\"]\n\n def query(self, qry):\n if self.opts['api_key']:\n version = \"3\"\n else:\n version = \"2\"\n\n url = f\"https://haveibeenpwned.com/api/v{version}/breachedaccount/{qry}\"\n hdrs = {\"Accept\": f\"application/vnd.haveibeenpwned.v{version}+json\"}\n retry = 0\n\n if self.opts['api_key']:\n hdrs['hibp-api-key'] = self.opts['api_key']\n\n while retry < 2:\n # https://haveibeenpwned.com/API/v2#RateLimiting\n time.sleep(1.5)\n res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\", headers=hdrs)\n\n if res['code'] == \"200\":\n break\n\n if res['code'] == \"404\":\n return None\n\n if res['code'] == \"429\":\n # Back off a little further\n time.sleep(2)\n retry += 1\n\n if res['code'] == \"401\":\n self.error(\"Failed to authenticate key with HaveIBeenPwned.com.\")\n self.errorState = True\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from HaveIBeenPwned?: {e}\")\n\n return None\n\n def queryPaste(self, qry):\n url = f\"https://haveibeenpwned.com/api/v3/pasteaccount/{qry}\"\n headers = {\n 'Accept': \"application/json\",\n 'hibp-api-key': self.opts['api_key']\n }\n\n retry = 0\n\n while retry < 2:\n # https://haveibeenpwned.com/API/v2#RateLimiting\n time.sleep(1.5)\n res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\", headers=headers)\n\n if res['code'] == \"200\":\n break\n\n if res['code'] == \"404\":\n return None\n\n if res['code'] == \"429\":\n # Back off a little further\n time.sleep(2)\n retry += 1\n\n if res['code'] == \"401\":\n self.error(\"Failed to authenticate key with HaveIBeenPwned.com.\")\n self.errorState = True\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from HaveIBeenPwned?: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_haveibeenpwned but did not set an API key!\")\n self.errorState = True\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n data = self.query(eventData)\n if data is not None:\n for n in data:\n try:\n site = n[\"Name\"]\n except Exception as e:\n self.debug(f\"Unable to parse result from HaveIBeenPwned?: {e}\")\n continue\n\n # Notify other modules of what you've found\n if eventName == 'EMAILADDR':\n e = SpiderFootEvent(\"EMAILADDR_COMPROMISED\", eventData + \" [\" + site + \"]\",\n self.__name__, event)\n else:\n e = SpiderFootEvent(\"PHONE_NUMBER_COMPROMISED\", eventData + \" [\" + site + \"]\",\n self.__name__, event)\n self.notifyListeners(e)\n\n # This API endpoint doesn't support phone numbers\n if eventName == \"PHONE_NUMBER\":\n return\n pasteData = self.queryPaste(eventData)\n if pasteData is None:\n return\n\n sites = {\n \"Pastebin\": \"https://pastebin.com/\",\n \"Pastie\": \"http://pastie.org/p/\",\n \"Slexy\": \"https://slexy.org/view/\",\n \"Ghostbin\": \"https://ghostbin.com/paste/\",\n \"JustPaste\": \"https://justpaste.it/\",\n }\n links = set()\n for n in pasteData:\n try:\n source = n.get(\"Source\")\n site = source\n if source in sites:\n site = f\"{sites[n.get('Source')]}{n.get('Id')}\"\n links.add(site)\n\n except Exception as e:\n self.debug(f\"Unable to parse result from HaveIBeenPwned?: {e}\")\n continue\n\n for link in links:\n try:\n self.debug(\"Found a link: \" + link)\n\n if self.checkForStop():\n return\n\n res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])\n\n if res['content'] is None:\n self.debug(f\"Ignoring {link} as no data returned\")\n continue\n\n if re.search(r\"[^a-zA-Z\\-\\_0-9]\" + re.escape(eventData) + r\"[^a-zA-Z\\-\\_0-9]\", res['content'], re.IGNORECASE) is None:\n continue\n\n evt1 = SpiderFootEvent(\"LEAKSITE_URL\", link, self.__name__, event)\n self.notifyListeners(evt1)\n\n evt2 = SpiderFootEvent(\"LEAKSITE_CONTENT\", res['content'], self.__name__, evt1)\n self.notifyListeners(evt2)\n\n except Exception as e:\n self.debug(f\"Unable to parse result from HaveIBeenPwned?: {e}\")\n continue\n\n# End of sfp_haveibeenpwned class\n" }, { "path": "modules/sfp_honeypot.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_honeypot\n# Purpose: SpiderFoot plug-in for looking up whether IPs appear in the\n# ProjectHoneyPot.org database.\n#\n# Author: Steve Micallef \n#\n# Created: 16/04/2014\n# Copyright: (c) Steve Micallef 2014\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom netaddr import IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_honeypot(SpiderFootPlugin):\n\n meta = {\n 'name': \"Project Honey Pot\",\n 'summary': \"Query the Project Honey Pot database for IP addresses.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://www.projecthoneypot.org/\",\n 'model': \"FREE_AUTH_UNLIMITED\",\n 'references': [\n \"https://www.projecthoneypot.org/httpbl_api.php\",\n \"https://www.projecthoneypot.org/services_overview.php\",\n \"https://www.projecthoneypot.org/faq.php\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://www.projecthoneypot.org\",\n \"Sign up for a free account\",\n \"Navigate to https://www.projecthoneypot.org/httpbl_configure.php'\",\n \"Request for an API key\",\n \"The API key is listed under 'Your http:BL Access Key'\"\n ],\n 'favIcon': \"https://www.projecthoneypot.org/favicon.ico\",\n 'logo': \"https://www.projecthoneypot.org/images/php_logo.gif\",\n 'description': \"Project Honey Pot is the first and only distributed system for identifying spammers \"\n \"and the spambots they use to scrape addresses from your website. \"\n \"Using the Project Honey Pot system you can install addresses \"\n \"that are custom-tagged to the time and IP address of a visitor to your site. \"\n \"If one of these addresses begins receiving email we not only can tell that the messages are spam, \"\n \"but also the exact moment when the address was harvested and the IP address that gathered it.\",\n }\n }\n\n opts = {\n 'api_key': \"\",\n 'searchengine': False,\n 'threatscore': 0,\n 'timelimit': 30,\n 'netblocklookup': True,\n 'maxnetblock': 24,\n 'subnetlookup': True,\n 'maxsubnet': 24\n }\n\n optdescs = {\n 'api_key': \"ProjectHoneyPot.org API key.\",\n 'searchengine': \"Include entries considered search engines?\",\n 'threatscore': \"Threat score minimum, 0 being everything and 255 being only the most serious.\",\n 'timelimit': \"Maximum days old an entry can be. 255 is the maximum, 0 means you'll get nothing.\",\n 'netblocklookup': \"Look up all IPs on netblocks deemed to be owned by your target for possible hosts on the same target subdomain/domain?\",\n 'maxnetblock': \"If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'subnetlookup': \"Look up all IPs on subnets which your target is a part of?\",\n 'maxsubnet': \"If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\"\n }\n\n results = None\n errorState = False\n\n # Status codes according to:\n # http://www.projecthoneypot.org/httpbl_api.php\n statuses = {\n \"0\": \"Search Engine\",\n \"1\": \"Suspicious\",\n \"2\": \"Harvester\",\n \"3\": \"Suspicious & Harvester\",\n \"4\": \"Comment Spammer\",\n \"5\": \"Suspicious & Comment Spammer\",\n \"6\": \"Harvester & Comment Spammer\",\n \"7\": \"Suspicious & Harvester & Comment Spammer\",\n \"8\": \"Unknown (8)\",\n \"9\": \"Unknown (9)\",\n \"10\": \"Unknown (10)\"\n }\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"NETBLOCK_OWNER\",\n \"NETBLOCK_MEMBER\",\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"BLACKLISTED_NETBLOCK\",\n \"BLACKLISTED_SUBNET\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_NETBLOCK\",\n \"MALICIOUS_SUBNET\",\n ]\n\n # Swap 1.2.3.4 to 4.3.2.1\n def reverseAddr(self, ipaddr):\n return '.'.join(reversed(ipaddr.split('.')))\n\n # Returns text about the IP status returned from DNS\n def parseDNS(self, addr):\n bits = addr.split(\".\")\n if int(bits[1]) > self.opts['timelimit']:\n return None\n\n if int(bits[2]) < self.opts['threatscore']:\n return None\n\n if int(bits[3]) == 0 and self.opts['searchengine']:\n return None\n\n return f\"{self.statuses[bits[3]]}\\nLast Activity: {bits[1]} days ago\\nThreat Level: {bits[2]}\"\n\n def queryAddr(self, qaddr, parentEvent):\n eventName = parentEvent.eventType\n\n text = None\n try:\n lookup = f\"{self.opts['api_key']}.{self.reverseAddr(qaddr)}.dnsbl.httpbl.org\"\n\n self.debug(f\"Checking ProjectHoneyPot: {lookup}\")\n addrs = self.sf.resolveHost(lookup)\n if not addrs:\n return\n\n self.debug(f\"Addresses returned: {addrs}\")\n\n for addr in addrs:\n text = self.parseDNS(addr)\n if text is not None:\n break\n except Exception as e:\n self.debug(f\"ProjectHoneyPot did not resolve {qaddr} / {lookup}: {e}\")\n\n if not text:\n return\n\n if eventName == \"AFFILIATE_IPADDR\":\n malicious_type = \"MALICIOUS_AFFILIATE_IPADDR\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_IPADDR\"\n elif eventName == \"IP_ADDRESS\":\n malicious_type = \"MALICIOUS_IPADDR\"\n blacklist_type = \"BLACKLISTED_IPADDR\"\n elif eventName == \"NETBLOCK_OWNER\":\n malicious_type = \"MALICIOUS_NETBLOCK\"\n blacklist_type = \"BLACKLISTED_NETBLOCK\"\n elif eventName == \"NETBLOCK_MEMBER\":\n malicious_type = \"MALICIOUS_SUBNET\"\n blacklist_type = \"BLACKLISTED_SUBNET\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n url = f\"https://www.projecthoneypot.org/ip_{qaddr}\"\n\n evt = SpiderFootEvent(malicious_type, f\"ProjectHoneyPot ({qaddr}): {text}\\n{url}\", self.__name__, parentEvent)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, f\"ProjectHoneyPot ({qaddr}): {text}\\n{url}\", self.__name__, parentEvent)\n self.notifyListeners(evt)\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if not self.opts['api_key']:\n self.error(f\"You enabled {self.__class__.__name__} but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if eventName == 'NETBLOCK_OWNER':\n if not self.opts['netblocklookup']:\n return\n\n max_netblock = self.opts['maxnetblock']\n if IPNetwork(eventData).prefixlen < max_netblock:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}\")\n return\n\n if eventName == 'NETBLOCK_MEMBER':\n if not self.opts['subnetlookup']:\n return\n\n max_subnet = self.opts['maxsubnet']\n if IPNetwork(eventData).prefixlen < max_subnet:\n self.debug(f\"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}\")\n return\n\n if eventName.startswith(\"NETBLOCK_\"):\n for addr in IPNetwork(eventData):\n if self.checkForStop():\n return\n self.queryAddr(str(addr), event)\n else:\n self.queryAddr(eventData, event)\n\n# End of sfp_honeypot class\n" }, { "path": "modules/sfp_hosting.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_hosting\n# Purpose: SpiderFoot plug-in for looking up whether IPs/Netblocks/Domains\n# appear in an IP categorization table of hosting providers.\n#\n# Author: Steve Micallef \n#\n# Created: 16/08/2015\n# Copyright: (c) Steve Micallef 2015\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom netaddr import IPAddress\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_hosting(SpiderFootPlugin):\n\n meta = {\n 'name': \"Hosting Provider Identifier\",\n 'summary': \"Find out if any IP addresses identified fall within known 3rd party hosting ranges, e.g. Amazon, Azure, etc.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Content Analysis\"]\n }\n\n # Default options\n opts = {\n }\n\n # Option descriptions\n optdescs = {\n }\n\n # Target\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.__dataSource__ = \"DNS\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['IP_ADDRESS']\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"PROVIDER_HOSTING\"]\n\n def queryAddr(self, qaddr):\n data = dict()\n url = \"https://raw.githubusercontent.com/client9/ipcat/master/datacenters.csv\"\n\n data['content'] = self.sf.cacheGet(\"sfipcat\", 48)\n if data['content'] is None:\n data = self.sf.fetchUrl(url, useragent=self.opts['_useragent'])\n\n if data['content'] is None:\n self.error(\"Unable to fetch \" + url)\n return None\n\n self.sf.cachePut(\"sfipcat\", data['content'])\n\n for line in data['content'].split('\\n'):\n if \",\" not in line:\n continue\n try:\n [start, end, title, url] = line.split(\",\")\n except Exception:\n continue\n\n try:\n if IPAddress(qaddr) > IPAddress(start) and IPAddress(qaddr) < IPAddress(end):\n return [title, url]\n except Exception as e:\n self.debug(\"Encountered an issue processing an IP: \" + str(e))\n continue\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n ret = self.queryAddr(eventData)\n if ret:\n evt = SpiderFootEvent(\"PROVIDER_HOSTING\", ret[0] + \": \" + ret[1],\n self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_hosting class\n" }, { "path": "modules/sfp_hostio.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_hostio\n# Purpose: Host.io database query module\n#\n# Author: Lev Trubach \n#\n# Created: 2020-08-21\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_hostio(SpiderFootPlugin):\n\n meta = {\n \"name\": \"Host.io\",\n \"summary\": \"Obtain information about domain names from host.io.\",\n 'flags': [\"apikey\"],\n \"useCases\": [\"Passive\"],\n \"categories\": [\"Passive DNS\"],\n \"dataSource\": {\n \"website\": \"https://host.io\",\n \"model\": \"FREE_AUTH_LIMITED\",\n \"references\": [\"https://host.io/docs\"],\n \"apiKeyInstructions\": [\n \"Visit https://host.io/signup\",\n \"Register a free account\",\n \"Visit https://host.io/dashboard and use the authentication token provided\",\n ],\n \"favIcon\": \"https://host.io/static/images/hostio/favicon.png?v2\",\n \"logo\": \"https://host.io/static/images/hostio/favicon.png?v2\", # Seems like they embed it as SVG\n \"description\": \"We collect data on every known domain name, from every TLD, and update it every month. \"\n \"Our data includes DNS records and website data for each of the domains.\"\n \"We process terabytes of data and summarize it to produce our final results. \"\n \"Browse through our site to see backlinks, redirects, server details or IP address \"\n \"and hosting provider details courtesy of IPinfo.io.\",\n },\n }\n\n opts = {\n \"api_key\": \"\",\n }\n\n optdescs = {\n \"api_key\": \"Host.io API Key.\",\n }\n\n errorState = False\n\n def setup(self, sfc, userOpts=None):\n if userOpts is None:\n userOpts = {}\n self.sf = sfc\n self.results = self.tempStorage()\n self.opts.update(userOpts)\n\n def watchedEvents(self):\n return [\n \"DOMAIN_NAME\",\n ]\n\n # What events this module produces\n def producedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"RAW_RIR_DATA\",\n \"EMAILADDR\",\n \"WEB_ANALYTICS_ID\",\n \"WEBSERVER_TECHNOLOGY\",\n \"PHYSICAL_COORDINATES\",\n \"DESCRIPTION_ABSTRACT\",\n \"GEOINFO\",\n ]\n\n # When querying third parties, it's best to have a dedicated function\n # to do so and avoid putting it in handleEvent()\n def handle_error_response(self, qry, res):\n try:\n error_info = json.loads(res[\"content\"])\n except Exception:\n error_info = None\n if error_info:\n error_message = error_info.get(\"error\")\n else:\n error_message = None\n if error_message:\n error_str = f\", message {error_message}\"\n else:\n error_str = \"\"\n self.info(f\"Failed to get results for {qry}, code {res['code']}{error_str}\")\n\n def query(self, qry):\n res = self.sf.fetchUrl(\n f\"https://host.io/api/full/{qry}\",\n headers={\"Authorization\": f\"Bearer {self.opts['api_key']}\"},\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=\"SpiderFoot\",\n )\n if res[\"code\"] != \"200\":\n self.handle_error_response(qry, res)\n return None\n\n if res[\"content\"] is None:\n self.info(f\"No Host.io info found for {qry}\")\n return None\n\n try:\n return json.loads(res[\"content\"])\n except Exception as e:\n self.error(f\"Error processing JSON response from Host.io: {e}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\n f\"You enabled {self.__class__.__name__} but did not set an API key!\"\n )\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData} as already mapped.\")\n return\n\n self.results[eventData] = True\n\n data = self.query(event.data)\n if not data:\n self.error(f\"No data received for {event.data}\")\n return\n\n found = False\n ipinfo = data.get(\"ipinfo\")\n if ipinfo and isinstance(ipinfo, dict):\n for address, ip_data in data[\"ipinfo\"].items():\n # Not supporting co-hosted sites yet\n if not self.sf.validIP(address):\n continue\n evt = SpiderFootEvent(\"IP_ADDRESS\", address, self.__name__, event)\n self.notifyListeners(evt)\n found = True\n\n loc = ip_data.get(\"loc\")\n if loc and isinstance(loc, str):\n loc_evt = SpiderFootEvent(\n \"PHYSICAL_COORDINATES\", loc, self.__name__, evt\n )\n self.notifyListeners(loc_evt)\n found = True\n\n geo_info = ', '.join(filter(None, (ip_data.get(k) for k in (\"city\", \"region\", \"country\"))))\n if geo_info:\n geo_info_evt = SpiderFootEvent(\n \"GEOINFO\", geo_info, self.__name__, evt\n )\n self.notifyListeners(geo_info_evt)\n found = True\n\n related = data.get(\"related\")\n if related and isinstance(related, dict):\n email_section = related.get(\"email\")\n if email_section and isinstance(email_section, list):\n for email_data in email_section:\n if isinstance(email_data, dict):\n value = email_data[\"value\"]\n if value and isinstance(value, str):\n for email in value.split(','):\n email = email.strip('.')\n evt = SpiderFootEvent(\n \"EMAILADDR\", email, self.__name__, event\n )\n self.notifyListeners(evt)\n found = True\n\n web = data.get(\"web\")\n if web and isinstance(web, dict):\n server = web.get(\"server\")\n if server and isinstance(server, str):\n evt = SpiderFootEvent(\n \"WEBSERVER_TECHNOLOGY\", server, self.__name__, event\n )\n self.notifyListeners(evt)\n found = True\n\n google_analytics = web.get(\"googleanalytics\")\n if google_analytics and isinstance(google_analytics, str):\n evt = SpiderFootEvent(\n \"WEB_ANALYTICS_ID\", google_analytics, self.__name__, event\n )\n self.notifyListeners(evt)\n found = True\n\n title = web.get(\"title\")\n if title and isinstance(title, str):\n evt = SpiderFootEvent(\n \"DESCRIPTION_ABSTRACT\", title, self.__name__, event\n )\n self.notifyListeners(evt)\n found = True\n\n if found:\n evt = SpiderFootEvent(\n \"RAW_RIR_DATA\", json.dumps(data), self.__name__, event\n )\n self.notifyListeners(evt)\n" }, { "path": "modules/sfp_hunter.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_hunter\n# Purpose: Query hunter.io using their API.\n#\n# Author: Steve Micallef \n#\n# Created: 22/02/2017\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_hunter(SpiderFootPlugin):\n\n meta = {\n 'name': \"Hunter.io\",\n 'summary': \"Check for e-mail addresses and names on hunter.io.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://hunter.io/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://hunter.io/api\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://hunter.io/\",\n \"Sign up for a free account\",\n \"Click on 'Account Settings'\",\n \"Click on 'API'\",\n \"The API key is listed under 'Your API Key'\"\n ],\n 'favIcon': \"https://hunter.io/assets/head/favicon-d5796c45076e78aa5cf22dd53c5a4a54155062224bac758a412f3a849f38690b.ico\",\n 'logo': \"https://hunter.io/assets/head/touch-icon-iphone-fd9330e31552eeaa12b177489943de997551bfd991c4c44e8c3d572e78aea5f3.png\",\n 'description': \"Hunter lets you find email addresses in seconds and connect with the people that matter for your business.\\n\"\n \"The Domain Search lists all the people working in a company with their name \"\n \"and email address found on the web. With 100+ million email addresses indexed, \"\n \"effective search filters and scoring, it's the most powerful email-finding tool ever created.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\"\n }\n\n # Option descriptions\n optdescs = {\n \"api_key\": \"Hunter.io API key.\"\n }\n\n # Be sure to completely clear any class variables in setup()\n # or you run the risk of data persisting between scan runs.\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n # Clear / reset any other class member variables here\n # or you risk them persisting between threads.\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\", \"INTERNET_NAME\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"EMAILADDR\", \"EMAILADDR_GENERIC\", \"RAW_RIR_DATA\"]\n\n def query(self, qry, offset=0, limit=10):\n params = {\n \"domain\": qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n \"api_key\": self.opts['api_key'],\n \"offset\": str(offset),\n \"limit\": str(limit)\n }\n\n url = f\"https://api.hunter.io/v2/domain-search?{urllib.parse.urlencode(params)}\"\n\n res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=\"SpiderFoot\")\n\n if res['code'] == \"404\":\n return None\n\n if not res['content']:\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from hunter.io: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_hunter but did not set an API key!\")\n self.errorState = True\n return\n\n data = self.query(eventData, 0, 10)\n if not data:\n return\n\n if \"data\" not in data:\n return\n\n # Check if we have more results on further pages\n if \"meta\" in data:\n maxgoal = data['meta'].get('results', 10)\n else:\n maxgoal = 10\n\n rescount = len(data['data'].get('emails', list()))\n\n while rescount <= maxgoal:\n for email in data['data'].get('emails', list()):\n # Notify other modules of what you've found\n em = email.get('value')\n if not em:\n continue\n if em.split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evttype = \"EMAILADDR_GENERIC\"\n else:\n evttype = \"EMAILADDR\"\n\n e = SpiderFootEvent(evttype, em, self.__name__, event)\n self.notifyListeners(e)\n\n if 'first_name' in email and 'last_name' in email:\n if email['first_name'] is not None and email['last_name'] is not None:\n n = email['first_name'] + \" \" + email['last_name']\n e = SpiderFootEvent(\"RAW_RIR_DATA\", \"Possible full name: \" + n,\n self.__name__, event)\n self.notifyListeners(e)\n\n if rescount >= maxgoal:\n return\n\n data = self.query(eventData, rescount, 10)\n if data is None:\n return\n if \"data\" not in data:\n return\n\n rescount += len(data['data'].get('emails', list()))\n\n# End of sfp_hunter class\n" }, { "path": "modules/sfp_hybrid_analysis.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_hybrid_analysis\n# Purpose: Search Hybrid Analysis for domains and URLs related to the target.\n#\n# Authors: \n#\n# Created: 2020-08-09\n# Copyright: (c) bcoles 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_hybrid_analysis(SpiderFootPlugin):\n\n meta = {\n 'name': \"Hybrid Analysis\",\n 'summary': \"Search Hybrid Analysis for domains and URLs related to the target.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://www.hybrid-analysis.com\",\n 'model': \"FREE_AUTH_UNLIMITED\",\n 'references': [\n \"https://www.hybrid-analysis.com/knowledge-base\",\n \"https://www.hybrid-analysis.com/docs/api/v2\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://www.hybrid-analysis.com/signup\",\n \"Register a free account\",\n \"Navigate to https://www.hybrid-analysis.com/my-account?tab=%23api-key-tab\",\n \"Create an API Key\",\n \"The API key is listed under 'API Key'\"\n ],\n 'favIcon': \"https://www.hybrid-analysis.com/favicon.ico\",\n 'logo': \"https://www.hybrid-analysis.com/img/logo.svg\",\n 'description': \"A free malware analysis service for the community. \"\n \"Using this service you can submit files for in-depth static and dynamic analysis.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\",\n \"verify\": True,\n \"delay\": 1\n }\n\n # Option descriptions\n optdescs = {\n \"api_key\": \"Hybrid Analysis API key.\",\n \"verify\": \"Verify identified domains still resolve to the associated specified IP address.\",\n \"delay\": \"Delay between requests, in seconds.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in userOpts.keys():\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"IP_ADDRESS\", \"DOMAIN_NAME\"]\n\n def producedEvents(self):\n return [\"RAW_RIR_DATA\", \"INTERNET_NAME\", \"DOMAIN_NAME\", \"LINKED_URL_INTERNAL\"]\n\n def queryDomain(self, qry):\n \"\"\"Query a domain\n\n Args:\n qry (str): domain\n\n Returns:\n str: API response as JSON\n \"\"\"\n\n params = {\n \"domain\": qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n }\n headers = {\n \"Accept\": \"application/json\",\n 'api-key': self.opts['api_key']\n }\n res = self.sf.fetchUrl(\n 'https://www.hybrid-analysis.com/api/v2/search/terms',\n headers=headers,\n timeout=15,\n useragent=\"Falcon Sandbox\",\n postData=params\n )\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n def queryHost(self, qry):\n \"\"\"Query a host\n\n Args:\n qry (str): host\n\n Returns:\n str: API response as JSON\n \"\"\"\n\n params = {\n \"host\": qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n }\n headers = {\n \"Accept\": \"application/json\",\n 'api-key': self.opts['api_key']\n }\n res = self.sf.fetchUrl(\n 'https://www.hybrid-analysis.com/api/v2/search/terms',\n headers=headers,\n timeout=15,\n useragent=\"Falcon Sandbox\",\n postData=params\n )\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n def queryHash(self, qry):\n \"\"\"Query a hash\n\n Args:\n qry (str): hash\n\n Returns:\n str: API response as JSON\n \"\"\"\n\n params = {\n \"hash\": qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n }\n headers = {\n \"Accept\": \"application/json\",\n 'api-key': self.opts['api_key']\n }\n res = self.sf.fetchUrl(\n 'https://www.hybrid-analysis.com/api/v2/search/hash',\n headers=headers,\n timeout=15,\n useragent=\"Falcon Sandbox\",\n postData=params\n )\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n def parseApiResponse(self, res: dict):\n \"\"\"Parse HTTP response from API\n\n Args:\n res (dict): HTTP response from SpiderFoot.fetchUrl()\n\n Returns:\n str: API response as JSON\n \"\"\"\n if not res:\n self.error(\"No response from Hybrid Analysis.\")\n return None\n\n if res['code'] == '400':\n self.error(\"Failed to retrieve content from Hybrid Analysis: Invalid request\")\n self.debug(f\"API response: {res['content']}\")\n return None\n\n # Future proofing - Hybrid Analysis does not implement rate limiting\n if res['code'] == '429':\n self.error(\"Failed to retrieve content from Hybrid Analysis: rate limit exceeded\")\n self.errorState = True\n return None\n\n # Catch all non-200 status codes, and presume something went wrong\n if res['code'] != '200':\n self.error(f\"Failed to retrieve content from Hybrid Analysis: Unexpected response status {res['code']}\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventName not in [\"IP_ADDRESS\", \"DOMAIN_NAME\"]:\n return\n\n if eventName == \"IP_ADDRESS\":\n data = self.queryHost(eventData)\n elif eventName == \"DOMAIN_NAME\":\n data = self.queryDomain(eventData)\n else:\n return\n\n if data is None:\n self.debug(f\"No information found for{eventData}\")\n return\n\n results = data.get(\"result\")\n\n if not results:\n return\n\n hashes = []\n\n for result in results:\n file_hash = result.get('sha256')\n if file_hash:\n hashes.append(file_hash)\n\n if not hashes:\n return\n\n self.info(f\"Found {len(hashes)} results for {eventData}\")\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n urls = []\n domains = []\n\n for file_hash in hashes:\n results = self.queryHash(file_hash)\n\n if not results:\n self.debug(f\"No information found for hash {file_hash}\")\n continue\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(results), self.__name__, event)\n self.notifyListeners(evt)\n\n for result in results:\n if not result:\n continue\n\n result_domains = result.get('domains')\n if result_domains:\n for r in result_domains:\n domains.append(r)\n\n submissions = result.get('submissions')\n if submissions:\n for submission in submissions:\n url = submission.get('url')\n if url:\n urls.append(url)\n\n for url in set(urls):\n host = self.sf.urlFQDN(url.lower())\n\n if not self.getTarget().matches(host, includeChildren=True, includeParents=True):\n continue\n\n domains.append(host)\n\n evt = SpiderFootEvent('LINKED_URL_INTERNAL', url, self.__name__, event)\n self.notifyListeners(evt)\n\n for domain in set(domains):\n if self.checkForStop():\n return\n\n if domain in self.results:\n continue\n\n if not self.getTarget().matches(domain, includeChildren=True, includeParents=True):\n continue\n\n if self.opts['verify'] and not self.sf.resolveHost(domain) and not self.sf.resolveHost6(domain):\n self.debug(f\"Host {domain} could not be resolved\")\n evt = SpiderFootEvent(\"INTERNET_NAME_UNRESOLVED\", domain, self.__name__, event)\n self.notifyListeners(evt)\n else:\n evt = SpiderFootEvent(\"INTERNET_NAME\", domain, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_hybrid_analysis class\n" }, { "path": "modules/sfp_iban.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_iban\n# Purpose: SpiderFoot plug-in for scanning retrieved content by other\n# modules (such as sfp_spider) and identifying IBANs.\n#\n# Author: Krishnasis Mandal \n#\n# Created: 26/04/2020\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_iban(SpiderFootPlugin):\n\n meta = {\n 'name': \"IBAN Number Extractor\",\n 'summary': \"Identify International Bank Account Numbers (IBANs) in any data.\",\n 'flags': [\"errorprone\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Content Analysis\"]\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n # Override datasource for sfp_iban module\n self.__dataSource__ = \"Target Website\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"TARGET_WEB_CONTENT\", \"DARKNET_MENTION_CONTENT\",\n \"LEAKSITE_CONTENT\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"IBAN_NUMBER\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n ibans = SpiderFootHelpers.extractIbansFromText(eventData)\n for ibanNumber in set(ibans):\n self.info(f\"Found IBAN number: {ibanNumber}\")\n evt = SpiderFootEvent(\"IBAN_NUMBER\", ibanNumber, self.__name__, event)\n if event.moduleDataSource:\n evt.moduleDataSource = event.moduleDataSource\n else:\n evt.moduleDataSource = \"Unknown\"\n self.notifyListeners(evt)\n\n# End of sfp_iban class\n" }, { "path": "modules/sfp_iknowwhatyoudownload.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_iknowwhatyoudownload\n# Purpose: Query iknowwhatyoudownload.com for IP addresses using torrents.\n#\n# Author: Steve Micallef \n#\n# Created: 03/09/2018\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_iknowwhatyoudownload(SpiderFootPlugin):\n\n meta = {\n 'name': \"Iknowwhatyoudownload.com\",\n 'summary': \"Check iknowwhatyoudownload.com for IP addresses that have been using torrents.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Secondary Networks\"],\n 'dataSource': {\n 'website': \"https://iknowwhatyoudownload.com/en/peer/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://iknowwhatyoudownload.com/en/api/\",\n \"https://iknowwhatyoudownload.com/en/link/\",\n \"https://iknowwhatyoudownload.com/en/peer/\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://iknowwhatyoudownload.com/en/api/\",\n \"Request Demo Key with email id\",\n \"The API key will be sent to your email\"\n ],\n 'favIcon': \"https://iknowwhatyoudownload.com/assets/img/utorrent2.png\",\n 'logo': \"https://iknowwhatyoudownload.com/assets/img/logo.png\",\n 'description': \"Our system collects torrent files in two ways: parsing torrent sites, and listening DHT network. \"\n \"The system contains more than 7 million torrents (as of Oct 2021) which were classified and which are using now \"\n \"for collecting peer sharing facts (up to 200.000.000 daily).\",\n }\n }\n\n opts = {\n \"daysback\": 30,\n \"api_key\": \"\"\n }\n\n optdescs = {\n \"daysback\": \"How far back (in days) to look for activity.\",\n \"api_key\": \"Iknowwhatyoudownload.com API key.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"IP_ADDRESS\", \"IPV6_ADDRESS\"]\n\n def producedEvents(self):\n return [\"MALICIOUS_IPADDR\"]\n\n def query(self, qry):\n \"\"\"Search iknowwhatyoudownload.com for an IPv4/IPv6 address.\n\n Args:\n qry: IPv4/IPv6 address\n\n Returns:\n dict: JSON response containing dowloaded content\n \"\"\"\n params = urllib.parse.urlencode({\n 'ip': qry,\n 'days': self.opts['daysback'],\n 'key': self.opts['api_key'],\n })\n\n res = self.sf.fetchUrl(\n f\"https://api.antitor.com/history/peer/?{params}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\"\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from iknowwhatyoudownload.com.\")\n return None\n\n if res['content'] is None:\n self.info(f\"No results for {qry} from iknowwhatyoudownload.com\")\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from iknowwhatyoudownload.com: {e}\")\n return None\n\n error = data.get('error')\n if error and error == \"INVALID_DAYS\":\n self.errorState = True\n self.error(f\"The number of days you have configured ({self.opts['daysback']}) was not accepted. If you have the demo key, try 30 days or less.\")\n return None\n\n contents = data.get('contents')\n\n if not contents:\n return None\n\n return contents\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.errorState:\n return\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_iknowwhatyoudownload but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n data = self.query(eventData)\n\n if not data:\n return\n\n retdata = [f\"https://iknowwhatyoudownload.com/en/peer/?ip={eventData}\"]\n\n for d in data:\n torrent = d.get('torrent')\n\n if not torrent:\n continue\n\n download_name = torrent.get('name')\n download_date = d.get(\"endDate\", \"Date unknown\")\n retdata.append(f\"{download_name} ({download_date})\")\n\n e = SpiderFootEvent(\"MALICIOUS_IPADDR\", \"\\n\".join(retdata), self.__name__, event)\n self.notifyListeners(e)\n\n# End of sfp_iknowwhatyoudownload class\n" }, { "path": "modules/sfp_intelx.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_intelx\n# Purpose: Query IntelligenceX (intelx.io) for identified IP addresses,\n# domains, e-mail addresses and phone numbers.\n#\n# Author: Steve Micallef \n#\n# Created: 28/04/2019\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport datetime\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_intelx(SpiderFootPlugin):\n\n meta = {\n 'name': \"IntelligenceX\",\n 'summary': \"Obtain information from IntelligenceX about identified IP addresses, domains, e-mail addresses and phone numbers.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://intelx.io/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://ginseg.com/wp-content/uploads/sites/2/2019/07/Manual-Intelligence-X-API.pdf\",\n \"https://blog.intelx.io/2019/01/25/new-developer-tab/\",\n \"https://github.com/IntelligenceX/SDK\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://intelx.io/\",\n \"Register a free account\",\n \"Navigate to https://intelx.io/account?tab=developer\",\n \"The API key is listed under 'Your API details'\"\n ],\n 'favIcon': \"https://intelx.io/favicon/favicon-32x32.png\",\n 'logo': \"https://intelx.io/assets/img/IntelligenceX.svg\",\n 'description': \"Intelligence X is an independent European technology company founded in 2018 by Peter Kleissner. \"\n \"Its mission is to develop and maintain the search engine and data archive.\\n\"\n \"The search works with selectors, i.e. specific search terms such as \"\n \"email addresses, domains, URLs, IPs, CIDRs, Bitcoin addresses, IPFS hashes, etc.\\n\"\n \"It searches in places such as the darknet, document sharing platforms, whois data, public data leaks and others.\\n\"\n \"It keeps a historical data archive of results, \"\n \"similar to how the Wayback Machine from archive.org stores historical copies of websites.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\",\n \"base_url\": \"2.intelx.io\",\n \"checkcohosts\": False,\n \"checkaffiliates\": False,\n 'netblocklookup': False,\n 'maxnetblock': 24,\n 'subnetlookup': False,\n 'maxsubnet': 24,\n 'maxage': 90\n }\n\n # Option descriptions\n optdescs = {\n \"api_key\": \"IntelligenceX API key.\",\n \"base_url\": \"API URL, as provided in your IntelligenceX account settings.\",\n \"checkcohosts\": \"Check co-hosted sites?\",\n \"checkaffiliates\": \"Check affiliates?\",\n 'netblocklookup': \"Look up all IPs on netblocks deemed to be owned by your target for possible hosts on the same target subdomain/domain?\",\n 'maxnetblock': \"If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'subnetlookup': \"Look up all IPs on subnets which your target is a part of?\",\n 'maxsubnet': \"If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'maxage': \"Maximum age (in days) of results to be considered valid. 0 = unlimited.\"\n }\n\n # Be sure to completely clear any class variables in setup()\n # or you run the risk of data persisting between scan runs.\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n # Clear / reset any other class member variables here\n # or you risk them persisting between threads.\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"IP_ADDRESS\", \"AFFILIATE_IPADDR\", \"INTERNET_NAME\", \"EMAILADDR\",\n \"CO_HOSTED_SITE\", \"PHONE_NUMBER\", \"BITCOIN_ADDRESS\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"LEAKSITE_URL\", \"DARKNET_MENTION_URL\",\n \"INTERNET_NAME\", \"DOMAIN_NAME\",\n \"EMAILADDR\", \"EMAILADDR_GENERIC\"]\n\n def query(self, qry, qtype):\n retdata = list()\n\n headers = {\n \"User-Agent\": \"SpiderFoot\",\n \"x-key\": self.opts['api_key'],\n }\n\n payload = {\n \"term\": qry,\n \"buckets\": [],\n \"lookuplevel\": 0,\n \"maxresults\": 100,\n \"timeout\": 0,\n \"datefrom\": \"\",\n \"dateto\": \"\",\n \"sort\": 4,\n \"media\": 0,\n \"terminate\": []\n }\n\n url = 'https://' + self.opts['base_url'] + '/' + qtype + '/search'\n res = self.sf.fetchUrl(url, postData=json.dumps(payload),\n headers=headers, timeout=self.opts['_fetchtimeout'])\n\n if res['content'] is None:\n self.info(\"No IntelligenceX info found for \" + qry)\n return None\n\n if res['code'] == \"402\":\n self.info(\"IntelligenceX credits expired.\")\n self.errorState = True\n return None\n\n try:\n ret = json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from IntelligenceX: {e}\")\n self.errorState = True\n return None\n\n if ret.get('status', -1) == 0:\n # Craft API URL with the id to return results\n resulturl = f\"{url}/result?id={ret['id']}\"\n limit = 30\n count = 0\n status = 3 # status 3 = No results yet, keep trying. 0 = Success with results\n while status in [3, 0] and count < limit:\n if self.checkForStop():\n return None\n\n res = self.sf.fetchUrl(resulturl, headers=headers)\n if res['content'] is None:\n self.info(\"No IntelligenceX info found for results from \" + qry)\n return None\n\n if res['code'] == \"402\":\n self.info(\"IntelligenceX credits expired.\")\n self.errorState = True\n return None\n\n try:\n ret = json.loads(res['content'])\n except Exception as e:\n self.error(\"Error processing JSON response from IntelligenceX: \" + str(e))\n return None\n\n status = ret['status']\n count += 1\n\n retdata.append(ret)\n # No more results left\n if status == 1:\n # print data in json format to manipulate as desired\n break\n\n time.sleep(1)\n\n return retdata\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if self.opts['api_key'] == \"\" or self.opts['base_url'] == \"\":\n self.error(\"You enabled sfp_intelx but did not set an API key and/or base URL!\")\n self.errorState = True\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName.startswith(\"AFFILIATE\") and not self.opts['checkaffiliates']:\n return\n\n if eventName == 'CO_HOSTED_SITE' and not self.opts['checkcohosts']:\n return\n\n data = self.query(eventData, \"intelligent\")\n if data is None:\n return\n\n self.info(\"Found IntelligenceX leak data for \" + eventData)\n agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage'])\n for info in data:\n for rec in info.get(\"records\", dict()):\n try:\n last_seen = int(datetime.datetime.strptime(rec['added'].split(\".\")[0], '%Y-%m-%dT%H:%M:%S').strftime('%s')) * 1000\n if self.opts['maxage'] > 0 and last_seen < agelimit:\n self.debug(\"Record found but too old, skipping.\")\n continue\n\n val = None\n evt = None\n if \"pastes\" in rec['bucket']:\n evt = \"LEAKSITE_URL\"\n val = rec['keyvalues'][0]['value']\n if rec['bucket'].startswith(\"darknet.\"):\n evt = \"DARKNET_MENTION_URL\"\n val = rec['name']\n\n if not val or not evt:\n # Try generically extracting it\n if \"systemid\" not in rec:\n continue\n evt = \"LEAKSITE_URL\"\n val = \"https://intelx.io/?did=\" + rec['systemid']\n except Exception as e:\n self.error(f\"Error processing content from IntelX: {e}\")\n continue\n\n # Notify other modules of what you've found\n e = SpiderFootEvent(evt, val, self.__name__, event)\n self.notifyListeners(e)\n\n if \"public.intelx.io\" in self.opts['base_url'] or eventName != \"INTERNET_NAME\":\n return\n\n data = self.query(eventData, \"phonebook\")\n if data is None:\n return\n\n self.info(f\"Found IntelligenceX host and email data for {eventData}\")\n for info in data:\n for rec in info.get(\"selectors\", dict()):\n try:\n val = rec['selectorvalueh']\n evt = None\n if rec['selectortype'] == 1: # Email\n evt = \"EMAILADDR\"\n if val.split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evt = \"EMAILADDR_GENERIC\"\n if rec['selectortype'] == 2: # Domain\n evt = \"INTERNET_NAME\"\n if val == eventData:\n continue\n if rec['selectortype'] == 3: # URL\n evt = \"LINKED_URL_INTERNAL\"\n\n if not val or not evt:\n self.debug(\"Unexpected record, skipping.\")\n continue\n except Exception as e:\n self.error(f\"Error processing content from IntelX: {e}\")\n continue\n\n # Notify other modules of what you've found\n e = SpiderFootEvent(evt, val, self.__name__, event)\n self.notifyListeners(e)\n\n if evt == \"INTERNET_NAME\" and self.sf.isDomain(val, self.opts['_internettlds']):\n e = SpiderFootEvent(\"DOMAIN_NAME\", val, self.__name__, event)\n self.notifyListeners(e)\n\n# End of sfp_intelx class\n" }, { "path": "modules/sfp_intfiles.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_intfiles\n# Purpose: From Spidered pages found, identifies files of potential interest.\n#\n# Author: Steve Micallef \n#\n# Created: 06/04/2014\n# Copyright: (c) Steve Micallef 2014\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_intfiles(SpiderFootPlugin):\n\n meta = {\n 'name': \"Interesting File Finder\",\n 'summary': \"Identifies potential files of interest, e.g. office documents, zip files.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Passive\"],\n 'categories': [\"Crawling and Scanning\"]\n }\n\n # Default options\n opts = {\n 'fileexts': [\"doc\", \"docx\", \"ppt\", \"pptx\", \"pdf\", 'xls', 'xlsx', 'zip']\n }\n\n # Option descriptions\n optdescs = {\n 'fileexts': \"File extensions of files you consider interesting.\"\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.__dataSource__ = \"Target Website\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"LINKED_URL_INTERNAL\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"INTERESTING_FILE\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n for fileExt in self.opts['fileexts']:\n if \".\" + fileExt.lower() in eventData.lower():\n evt = SpiderFootEvent(\"INTERESTING_FILE\", eventData,\n self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_intfiles class\n" }, { "path": "modules/sfp_ipapico.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_ipapico\n# Purpose: SpiderFoot plug-in to identify the Geo-location of IP addresses\n# identified by other modules using ipapi.co\n#\n# Author: Krishnasis Mandal \n#\n# Created: 02/02/2021\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_ipapico(SpiderFootPlugin):\n\n meta = {\n 'name': \"ipapi.co\",\n 'summary': \"Queries ipapi.co to identify geolocation of IP Addresses using ipapi.co API\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Real World\"],\n 'dataSource': {\n 'website': \"https://ipapi.co/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://ipapi.co/api/\"\n ],\n 'favIcon': \"https://ipapi.co/static/images/favicon.b64f1de785e1.ico\",\n 'logo': \"https://ipapi.co/static/images/favicon.34f0ec468301.png\",\n 'description': \"Powerful & Simple REST API for IP Address Geolocation.\"\n \"ipapi.co provides a REST API to find the location of an IP address.\",\n }\n }\n\n # Default options\n opts = {\n }\n\n # Option descriptions\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\"\n ]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\n \"GEOINFO\",\n \"RAW_RIR_DATA\"\n ]\n\n def query(self, qry):\n queryString = f\"https://ipapi.co/{qry}/json/\"\n\n res = self.sf.fetchUrl(queryString,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'])\n time.sleep(1.5)\n\n if res['content'] is None:\n self.info(f\"No ipapi.co data found for {qry}\")\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n # Don't look up stuff twice\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n data = self.query(eventData)\n\n if data is None:\n self.info(\"No results returned from ipapi.co\")\n return\n\n if data.get('country'):\n location = ', '.join(filter(None, [data.get('city'), data.get('region'), data.get('region_code'), data.get('country_name'), data.get('country')]))\n evt = SpiderFootEvent('GEOINFO', location, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n\n# End of sfp_ipapico class\n" }, { "path": "modules/sfp_ipapicom.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_ipapicom\n# Purpose: SpiderFoot plug-in to identify the Geo-location of IP addresses\n# identified by other modules using ipapi.com\n#\n# Author: Krishnasis Mandal \n#\n# Created: 29/01/2021\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_ipapicom(SpiderFootPlugin):\n\n meta = {\n 'name': \"ipapi.com\",\n 'summary': \"Queries ipapi.com to identify geolocation of IP Addresses using ipapi.com API\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Real World\"],\n 'dataSource': {\n 'website': \"https://ipapi.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://ipapi.com/documentation\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://ipapi.com/\",\n \"Register a free account\",\n \"Browse to https://ipapi.com/dashboard\",\n \"Your API Key will be listed under Your API Access Key\",\n ],\n 'favIcon': \"https://ipapi.com/site_images/ipapi_shortcut_icon.ico\",\n 'logo': \"https://ipapi.com/site_images/ipapi_icon.png\",\n 'description': \"ipapi provides an easy-to-use API interface allowing customers \"\n \"to look various pieces of information IPv4 and IPv6 addresses are associated with. \"\n \"For each IP address processed, the API returns more than 45 unique data points, \"\n \"such as location data, connection data, ISP information, time zone, currency and security assessment data.\",\n }\n }\n\n # Default options\n opts = {\n 'api_key': '',\n }\n\n # Option descriptions\n optdescs = {\n 'api_key': \"ipapi.com API Key.\",\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\"\n ]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\n \"GEOINFO\",\n \"RAW_RIR_DATA\"\n ]\n\n def query(self, qry):\n queryString = f\"http://api.ipapi.com/api/{qry}?access_key={self.opts['api_key']}\"\n\n res = self.sf.fetchUrl(queryString,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'])\n time.sleep(1.5)\n\n if res['code'] == \"429\":\n self.error(\"You are being rate-limited by IP-API.com.\")\n self.errorState = True\n return None\n if res['content'] is None:\n self.info(f\"No ipapi.com data found for {qry}\")\n return None\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.errorState:\n return\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_ipapicom but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n data = self.query(eventData)\n if not data:\n return\n\n if data.get('country_name'):\n location = ', '.join(filter(None, [data.get('city'), data.get('region_name'), data.get('region_code'), data.get('country_name'), data.get('country_code')]))\n evt = SpiderFootEvent('GEOINFO', location, self.__name__, event)\n self.notifyListeners(evt)\n\n if data.get('latitude') and data.get('longitude'):\n evt = SpiderFootEvent(\"PHYSICAL_COORDINATES\", f\"{data.get('latitude')}, {data.get('longitude')}\", self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n\n# End of sfp_ipapicom class\n" }, { "path": "modules/sfp_ipinfo.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_ipinfo\n# Purpose: SpiderFoot plug-in to identify the Geo-location of IP addresses\n# identified by other modules using ipinfo.io.\n#\n# Author: Steve Micallef \n#\n# Created: 17/06/2017\n# Copyright: (c) Steve Micallef 2017\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_ipinfo(SpiderFootPlugin):\n\n meta = {\n 'name': \"IPInfo.io\",\n 'summary': \"Identifies the physical location of IP addresses identified using ipinfo.io.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Real World\"],\n 'dataSource': {\n 'website': \"https://ipinfo.io\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://ipinfo.io/developers\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://ipinfo.io/\",\n \"Sign up for a free account\",\n \"Navigate to https://ipinfo.io/account\",\n \"The API key is listed above 'is your access token'\"\n ],\n 'favIcon': \"https://ipinfo.io/static/favicon-96x96.png?v3\",\n 'logo': \"https://ipinfo.io/static/deviceicons/android-icon-96x96.png\",\n 'description': \"The Trusted Source for IP Address Data.\\n\"\n \"With IPinfo, you can pinpoint your users’ locations, customize their experiences, \"\n \"prevent fraud, ensure compliance, and so much more.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\"\n }\n optdescs = {\n \"api_key\": \"Ipinfo.io access token.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['IP_ADDRESS', 'IPV6_ADDRESS']\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"GEOINFO\"]\n\n # https://ipinfo.io/developers\n def queryIP(self, ip):\n headers = {\n 'Authorization': \"Bearer \" + self.opts['api_key']\n }\n res = self.sf.fetchUrl(\"https://ipinfo.io/\" + ip + \"/json\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n headers=headers)\n\n if res['code'] == \"429\":\n self.error(\"You are being rate-limited by ipinfo.io.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.info(\"No GeoIP info found for \" + ip)\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_ipinfo but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n data = self.queryIP(eventData)\n\n if data is None:\n return\n\n if 'country' not in data:\n return\n\n location = ', '.join([_f for _f in [data.get('city'), data.get('region'), data.get('country')] if _f])\n self.info(\"Found GeoIP for \" + eventData + \": \" + location)\n\n evt = SpiderFootEvent(\"GEOINFO\", location, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_ipinfo class\n" }, { "path": "modules/sfp_ipqualityscore.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_ipqualityscore\n# Purpose: Spiderfoot module to check whether a target is malicious\n# using IPQualityScore API\n#\n# Author: Krishnasis Mandal \n#\n# Created: 2020-10-07\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_ipqualityscore(SpiderFootPlugin):\n\n meta = {\n \"name\": \"IPQualityScore\",\n \"summary\": \"Determine if target is malicious using IPQualityScore API\",\n 'flags': [\"apikey\"],\n \"useCases\": [\"Investigate\", \"Passive\"],\n \"categories\": [\"Reputation Systems\"],\n \"dataSource\": {\n \"website\": \"https://www.ipqualityscore.com/\",\n \"model\": \"FREE_AUTH_LIMITED\",\n \"references\": [\n \"https://www.ipqualityscore.com/documentation/overview\"\n ],\n \"apiKeyInstructions\": [\n \"Visit https://www.ipqualityscore.com/\",\n \"Click on 'Plans'\",\n \"Register a free account\",\n \"Visit https://www.ipqualityscore.com/user/settings\",\n \"Your API key will be listed under 'API Key'\"\n ],\n \"favIcon\": \"https://www.ipqualityscore.com/templates/img/icons/fav/favicon-32x32.png\",\n \"logo\": \"https://www.ipqualityscore.com/templates/img/logo.png\",\n \"description\": \"IPQualityScore's suite of fraud prevention tools automate quality control \"\n \"to prevent bots, fake accounts, fraudsters, suspicious transactions, \"\n \"& malicious users without interrupting the user experience.\",\n },\n }\n\n opts = {\n \"api_key\": \"\",\n \"abuse_score_threshold\": 85,\n \"strictness\": 0\n }\n\n optdescs = {\n \"api_key\": \"IPQualityScore API Key\",\n \"abuse_score_threshold\": \"Minimum abuse score for target to be considered malicious (0 - 100)\",\n \"strictness\": \"Depth of the reputation checks to be performed on the target (0 - 2)\"\n }\n\n errorState = False\n\n def setup(self, sfc, userOpts=None):\n if userOpts is None:\n userOpts = {}\n self.sf = sfc\n self.results = self.tempStorage()\n self.opts.update(userOpts)\n\n def watchedEvents(self):\n return [\n \"DOMAIN_NAME\",\n \"EMAILADDR\",\n \"IP_ADDRESS\",\n \"PHONE_NUMBER\",\n ]\n\n def producedEvents(self):\n return [\n \"EMAILADDR_DISPOSABLE\",\n \"EMAILADDR_COMPROMISED\",\n \"GEOINFO\",\n \"MALICIOUS_PHONE_NUMBER\",\n \"MALICIOUS_EMAILADDR\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_INTERNET_NAME\",\n \"PHONE_NUMBER_TYPE\",\n \"RAW_RIR_DATA\"\n ]\n\n def handle_error_response(self, qry, res):\n try:\n error_info = json.loads(res[\"content\"])\n except Exception:\n error_info = None\n if error_info:\n error_message = error_info.get(\"message\")\n else:\n error_message = None\n if error_message:\n error_str = f\", message {error_message}\"\n else:\n error_str = \"\"\n self.error(f\"Failed to get results for {qry}, code {res['code']}{error_str}\")\n\n def query(self, qry, eventName):\n queryString = \"\"\n if eventName == \"PHONE_NUMBER\":\n queryString = f\"https://ipqualityscore.com/api/json/phone/{self.opts['api_key']}/{qry}?strictness={self.opts['strictness']}\"\n elif eventName == \"EMAILADDR\":\n queryString = f\"https://ipqualityscore.com/api/json/email/{self.opts['api_key']}/{qry}?strictness={self.opts['strictness']}\"\n elif eventName in ['IP_ADDRESS', 'DOMAIN_NAME']:\n queryString = f\"https://ipqualityscore.com/api/json/ip/{self.opts['api_key']}/{qry}?strictness={self.opts['strictness']}\"\n\n res = self.sf.fetchUrl(\n queryString,\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=\"SpiderFoot\",\n )\n\n if not res['content']:\n self.info(f\"No IPQualityScore info found for {qry}\")\n return None\n\n try:\n r = json.loads(res['content'])\n if res[\"code\"] != \"200\" or not r.get(\"success\"):\n self.handle_error_response(qry, res)\n return None\n return r\n except Exception as e:\n self.error(f\"Error processing JSON response from IPQualityScore: {e}\")\n\n return None\n\n def getGeoInfo(self, data):\n geoInfo = \"\"\n\n city = data.get('city')\n country = data.get('country')\n if not country:\n country = data.get('country_code')\n zipcode = data.get('zip_code')\n region = data.get('region')\n\n if city:\n geoInfo += city + \", \"\n if region:\n geoInfo += region + \", \"\n if country:\n geoInfo += country + \" \"\n if zipcode:\n geoInfo += zipcode\n\n return geoInfo\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\n f\"You enabled {self.__class__.__name__} but did not set an API Key!\"\n )\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData} as already mapped.\")\n return\n self.results[eventData] = True\n\n data = self.query(eventData, eventName)\n if not data:\n return\n\n fraudScore = data.get('fraud_score')\n recentAbuse = data.get('recent_abuse')\n botStatus = data.get('bot_status')\n malicious = False\n maliciousDesc = \"\"\n\n if fraudScore >= self.opts['abuse_score_threshold'] or recentAbuse or botStatus:\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(evt)\n malicious = True\n maliciousDesc = f\"IPQualityScore [{eventData}]\\n\"\n\n if eventName == \"PHONE_NUMBER\":\n if malicious:\n maliciousDesc += f\" - FRAUD SCORE: {fraudScore}\\n - ACTIVE: {data.get('active')}\\n - RISKY: {data.get('risky')}\\n - RECENT ABUSE: {recentAbuse}\"\n evt = SpiderFootEvent(\"MALICIOUS_PHONE_NUMBER\", maliciousDesc, self.__name__, event)\n self.notifyListeners(evt)\n\n phoneNumberType = data.get('line_type')\n if phoneNumberType:\n evt = SpiderFootEvent(\"PHONE_NUMBER_TYPE\", phoneNumberType, self.__name__, event)\n self.notifyListeners(evt)\n\n geoInfo = self.getGeoInfo(data)\n if geoInfo:\n evt = SpiderFootEvent(\"GEOINFO\", geoInfo, self.__name__, event)\n self.notifyListeners(evt)\n\n elif eventName == \"EMAILADDR\":\n if malicious:\n maliciousDesc += f\" - FRAUD SCORE: {fraudScore}\\n - HONEYPOT: {data.get('honeypot')}\\n - SPAM TRAP SCORE: {data.get('spam_trap_score')}\\n - RECENT ABUSE: {recentAbuse}\"\n evt = SpiderFootEvent(\"MALICIOUS_EMAILADDR\", maliciousDesc, self.__name__, event)\n self.notifyListeners(evt)\n\n if data.get('disposable'):\n evt = SpiderFootEvent(\"EMAILADDR_DISPOSABLE\", eventData, self.__name__, event)\n self.notifyListeners(evt)\n\n if data.get('leaked'):\n evt = SpiderFootEvent(\"EMAILADDR_COMPROMISED\", f\"{eventData} [Unknown]\", self.__name__, event)\n self.notifyListeners(evt)\n\n elif eventName in ['IP_ADDRESS', 'DOMAIN_NAME']:\n if malicious:\n maliciousDesc += f\" - FRAUD SCORE: {fraudScore}\\n - BOT STATUS: {botStatus}\\n - RECENT ABUSE: {recentAbuse}\\n - ABUSE VELOCITY: {data.get('abuse_velocity')}\\n - VPN: {data.get('vpn')}\\n - ACTIVE VPN: {data.get('active_vpn')}\\n - TOR: {data.get('tor')}\\n - ACTIVE TOR: {data.get('active_tor')}\"\n\n if eventName == \"IP_ADDRESS\":\n evt = SpiderFootEvent(\"MALICIOUS_IPADDR\", maliciousDesc, self.__name__, event)\n elif eventName == \"DOMAIN_NAME\":\n evt = SpiderFootEvent(\"MALICIOUS_INTERNET_NAME\", maliciousDesc, self.__name__, event)\n self.notifyListeners(evt)\n\n geoInfo = self.getGeoInfo(data)\n if geoInfo:\n evt = SpiderFootEvent(\"GEOINFO\", geoInfo, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_ipqualityscore class\n" }, { "path": "modules/sfp_ipregistry.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_ipregistry\n# Purpose: ipregistry database query module.\n#\n# Author: Leo Trubach \n#\n# Created: 2020-09-08\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport urllib.parse\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_ipregistry(SpiderFootPlugin):\n meta = {\n \"name\": \"ipregistry\",\n \"summary\": \"Query the ipregistry.co database for reputation and geo-location.\",\n 'flags': [\"apikey\"],\n \"useCases\": [\"Passive\", \"Footprint\", \"Investigate\"],\n \"categories\": [\"Reputation Systems\"],\n \"dataSource\": {\n \"website\": \"https://ipregistry.co/\",\n \"model\": \"FREE_AUTH_LIMITED\",\n \"references\": [\"https://ipregistry.co/docs\"],\n \"apiKeyInstructions\": [\n \"Visit https://dashboard.ipregistry.co/signup\",\n \"Register a free account\",\n \"Click on 'API Keys' in left navbar\",\n \"Click on 'Click to reveal API key' for existing Default key\",\n ],\n \"favIcon\": \"https://cdn.ipregistry.co/icons/favicon-32x32.png\",\n \"logo\": \"https://ipregistry.co/assets/ipregistry.svg\",\n \"description\": \"Ipregistry is a trusted and in-depth IP \"\n \"Geolocation and Threat detections source of information that can\"\n \"benefit publishers, ad networks, retailers, financial services, \"\n \"e-commerce stores and more.\",\n },\n }\n\n opts = {\n \"api_key\": \"\",\n }\n\n optdescs = {\n \"api_key\": \"Ipregistry API Key.\",\n }\n\n results = None\n\n errorState = False\n\n def setup(self, sfc, userOpts=None):\n self.sf = sfc\n self.results = self.tempStorage()\n\n if userOpts:\n self.opts.update(userOpts)\n\n def watchedEvents(self):\n return [\"IP_ADDRESS\", \"IPV6_ADDRESS\"]\n\n def producedEvents(self):\n return [\"GEOINFO\", \"MALICIOUS_IPADDR\", \"PHYSICAL_COORDINATES\", \"RAW_RIR_DATA\"]\n\n def query(self, qry):\n qs = urllib.parse.urlencode({\"key\": self.opts[\"api_key\"]})\n res = self.sf.fetchUrl(\n f\"https://api.ipregistry.co/{qry}?{qs}\",\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=\"SpiderFoot\",\n )\n\n if res[\"content\"] is None:\n self.info(f\"No {self.meta['name']} info found for {qry}\")\n return None\n\n try:\n return json.loads(res[\"content\"])\n except Exception as e:\n self.error(\n f\"Error processing JSON response from {self.meta['name']}: {e}\"\n )\n\n return None\n\n def emit(self, etype, data, pevent):\n evt = SpiderFootEvent(etype, data, self.__name__, pevent)\n self.notifyListeners(evt)\n return evt\n\n def generate_location_events(self, location, pevent):\n if not isinstance(location, dict):\n return\n physical_location = None\n country = location.get(\"country\")\n if isinstance(country, dict):\n country_name = country.get(\"name\")\n else:\n country_name = None\n region = location.get(\"region\")\n if isinstance(region, dict):\n region_name = region.get(\"name\")\n else:\n region_name = None\n latitude = location.get(\"latitude\")\n longitude = location.get(\"longitude\")\n if latitude and longitude:\n physical_location = f\"{latitude}, {longitude}\"\n geo_info = \", \".join(\n [\n _f\n for _f in [\n location.get(\"city\"),\n region_name,\n location.get(\"postal\"),\n country_name,\n ]\n if _f\n ]\n )\n if geo_info:\n self.emit(\"GEOINFO\", geo_info, pevent)\n if physical_location:\n self.emit(\"PHYSICAL_COORDINATES\", physical_location, pevent)\n\n def generate_security_events(self, security, pevent):\n if not isinstance(security, dict):\n return\n malicious = any(\n security.get(k) for k in (\"is_abuser\", \"is_attacker\", \"is_threat\")\n )\n if malicious:\n self.emit(\"MALICIOUS_IPADDR\", f\"ipregistry [{pevent.data}]\", pevent)\n\n def generate_events(self, data, pevent):\n if not isinstance(data, dict):\n return\n self.generate_location_events(data.get(\"location\"), pevent)\n self.generate_security_events(data.get(\"security\"), pevent)\n\n def handleEvent(self, event):\n if self.errorState:\n return\n\n self.debug(f\"Received event, {event.eventType}, from {event.module}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(f\"You enabled {self.__class__.__name__} but did not set an API key!\")\n self.errorState = True\n return\n\n if event.data in self.results:\n self.debug(f\"Skipping {event.data}, already checked.\")\n return\n self.results[event.data] = True\n\n if event.eventType in (\"IP_ADDRESS\", \"IPV6_ADDRESS\"):\n data = self.query(event.data)\n self.generate_events(data, event)\n\n self.emit(\"RAW_RIR_DATA\", json.dumps(data), event)\n" }, { "path": "modules/sfp_ipstack.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_ipstack\n# Purpose: SpiderFoot plug-in to identify the Geo-location of IP addresses\n# identified by other modules using ipstack.com.\n#\n# Author: Steve Micallef \n#\n# Created: 20/08/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_ipstack(SpiderFootPlugin):\n\n meta = {\n 'name': \"ipstack\",\n 'summary': \"Identifies the physical location of IP addresses identified using ipstack.com.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Real World\"],\n 'dataSource': {\n 'website': \"https://ipstack.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://ipstack.com/documentation\",\n \"https://ipstack.com/faq\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://ipstack.com/product\",\n \"Click on 'Get Free API Key'\",\n \"Click on 'Dashboard'\",\n \"The API key is listed under 'Your API Access Key'\"\n ],\n 'favIcon': \"https://ipstack.com/ipstack_images/ipstack_logo.svg\",\n 'logo': \"https://ipstack.com/ipstack_images/ipstack_logo.svg\",\n 'description': \"Locate and identify website visitors by IP address.\\n\"\n \"ipstack offers one of the leading IP to geolocation APIS \"\n \"and global IP database services worldwide.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\"\n }\n optdescs = {\n \"api_key\": \"Ipstack.com API key.\"\n }\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['IP_ADDRESS']\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"GEOINFO\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.errorState:\n return\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_ipstack but did not set an API key!\")\n self.errorState = True\n return\n\n # Don't look up stuff twice\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n res = self.sf.fetchUrl(\"http://api.ipstack.com/\" + eventData + \"?access_key=\" + self.opts['api_key'],\n timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])\n if res['content'] is None:\n self.info(\"No GeoIP info found for \" + eventData)\n\n try:\n hostip = json.loads(res['content'])\n if 'success' in hostip and hostip['success'] is False:\n self.error(\"Invalid ipstack.com API key or usage limits exceeded.\")\n self.errorState = True\n return\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return\n\n geoinfo = hostip.get('country_name')\n if geoinfo:\n self.info(f\"Found GeoIP for {eventData}: {geoinfo}\")\n evt = SpiderFootEvent(\"GEOINFO\", geoinfo, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_ipstack class\n" }, { "path": "modules/sfp_isc.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_isc\n# Purpose: Check if an IP address is malicious according to SANS ISC.\n#\n# Author: steve@binarypool.com\n#\n# Created: 14/12/2013\n# Copyright: (c) Steve Micallef, 2013\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_isc(SpiderFootPlugin):\n\n meta = {\n 'name': \"Internet Storm Center\",\n 'summary': \"Check if an IP address is malicious according to SANS ISC.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://isc.sans.edu\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://isc.sans.edu/api/\",\n \"https://isc.sans.edu/howto.html\",\n \"https://isc.sans.edu/honeypot.html\",\n \"https://isc.sans.edu/glossary.html\",\n \"https://isc.sans.edu/fightback.html\"\n ],\n 'favIcon': \"https://isc.sans.edu/iscfavicon.ico\",\n 'logo': \"https://isc.sans.edu/images/logos/isc/large.png\",\n 'description': \"The ISC provides a free analysis and warning service to thousands of Internet users \"\n \"and organizations, and is actively working with Internet Service Providers to \"\n \"fight back against the most malicious attackers.\\n\"\n \"Thousands of sensors that work with most firewalls, intrusion detection systems, \"\n \"home broadband devices, and nearly all operating systems are constantly collecting information about \"\n \"unwanted traffic arriving from the Internet. \"\n \"These devices feed the DShield database where human volunteers as well as machines pour through \"\n \"the data looking for abnormal trends and behavior. \"\n \"The resulting analysis is posted to the ISC's main web page where it can be automatically retrieved \"\n \"by simple scripts or can be viewed in near real time by any Internet user.\",\n }\n }\n\n opts = {\n 'checkaffiliates': True\n }\n\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliates?\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"AFFILIATE_IPV6_ADDRESS\",\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n ]\n\n def query(self, ip):\n if not ip:\n return None\n\n res = self.sf.fetchUrl(\n f\"https://isc.sans.edu/api/ip/{ip}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from ISC.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from ISC\")\n self.errorState = True\n return None\n\n return res['content']\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']:\n malicious_type = 'MALICIOUS_IPADDR'\n blacklist_type = 'BLACKLISTED_IPADDR'\n elif eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS']:\n if not self.opts.get('checkaffiliates', False):\n return\n malicious_type = 'MALICIOUS_AFFILIATE_IPADDR'\n blacklist_type = 'BLACKLISTED_AFFILIATE_IPADDR'\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n data = self.query(eventData)\n\n if not data:\n return\n\n attacks = re.findall(r\"([0-9]+)\", data)\n\n if not attacks:\n return\n\n url = f\"https://isc.sans.edu/api/ip/{eventData}\"\n text = f\"Internet Storm Center [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_isc class\n" }, { "path": "modules/sfp_jsonwhoiscom.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_jsonwhoiscom\n# Purpose: Search JsonWHOIS.com for WHOIS records associated with a domain.\n#\n# Authors: \n#\n# Created: 2020-06-20\n# Copyright: (c) bcoles 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_jsonwhoiscom(SpiderFootPlugin):\n\n meta = {\n 'name': \"JsonWHOIS.com\",\n 'summary': \"Search JsonWHOIS.com for WHOIS records associated with a domain.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://jsonwhois.com\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://jsonwhois.com/docs\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://jsonwhois.com\",\n \"Sign up for a free account\",\n \"Navigate to https://jsonwhois.com/dashboard\",\n \"The API key is listed under 'Api Key'\"\n ],\n 'favIcon': \"https://jsonwhois.com/assets/fav.ico\",\n 'logo': \"https://jsonwhois.com/assets/fav.ico\",\n 'description': \"Get access to accurate Whois records for generic and country TLDs. \"\n \"Around 1000 gTLDs include .com, .org, .net, .us, .biz, .info, .mobi, .pro, .asia and many other new ones.\\n\"\n \"Raw and parsed Whois data are both accessible for downloads in the form of \"\n \"MYSQL or MYSQL database dumps and Comma Separated Values (.CSV) files.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\",\n \"delay\": 1,\n }\n\n # Option descriptions\n optdescs = {\n \"api_key\": \"JsonWHOIS.com API key.\",\n \"delay\": \"Delay between requests, in seconds.\",\n }\n\n results = None\n errorState = False\n\n # Initialize module and module options\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in userOpts.keys():\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\", \"AFFILIATE_DOMAIN_NAME\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"RAW_RIR_DATA\", \"DOMAIN_REGISTRAR\", \"DOMAIN_WHOIS\", \"PROVIDER_DNS\",\n \"EMAILADDR\", \"EMAILADDR_GENERIC\", \"PHONE_NUMBER\", \"PHYSICAL_ADDRESS\",\n \"AFFILIATE_DOMAIN_UNREGISTERED\"]\n\n # Query domain\n # https://jsonwhois.com/docs\n def queryDomain(self, qry):\n params = {\n 'domain': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n }\n headers = {\n \"Accept\": \"application/json\",\n \"Authorization\": \"Token token=\" + self.opts[\"api_key\"]\n }\n\n res = self.sf.fetchUrl(\n f\"https://jsonwhois.com/api/v1/whois?{urllib.parse.urlencode(params)}\",\n headers=headers,\n timeout=15,\n useragent=self.opts['_useragent']\n )\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n # Parse API response\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from JsonWHOIS.com.\")\n return None\n\n if res['code'] == '404':\n self.debug(\"No results for query\")\n return None\n\n # Sometimes JsonWHOIS.com returns HTTP 500 errors rather than 404\n if res['code'] == '500' and res['content'] == '{\"error\":\"Call failed\"}':\n self.debug(\"No results for query\")\n return None\n\n if res['code'] == \"401\":\n self.error(\"Invalid JsonWHOIS.com API key.\")\n self.errorState = True\n return None\n\n if res['code'] == '429':\n self.error(\"You are being rate-limited by JsonWHOIS.com\")\n self.errorState = True\n return None\n\n if res['code'] == '503':\n self.error(\"JsonWHOIS.com service unavailable\")\n self.errorState = True\n return None\n\n # Catch all other non-200 status codes, and presume something went wrong\n if res['code'] != '200':\n self.error(\"Failed to retrieve content from JsonWHOIS.com\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if eventData in self.results:\n return\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_jsonwhoiscom but did not set an API key!\")\n self.errorState = True\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n res = self.queryDomain(eventData)\n\n if res is None:\n self.debug(f\"No information found for domain {eventData}\")\n return\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(res), self.__name__, event)\n self.notifyListeners(evt)\n\n dns_providers = list()\n\n nameservers = res.get('nameservers')\n if nameservers:\n for nameserver in nameservers:\n if nameserver:\n nameserver_name = nameserver.get('name')\n if nameserver_name:\n dns_providers.append(nameserver_name)\n\n contacts = list()\n\n registrant_contacts = res.get('registrant_contacts')\n if registrant_contacts:\n for contact in registrant_contacts:\n contacts.append(contact)\n\n admin_contacts = res.get('admin_contacts')\n if admin_contacts:\n for contact in admin_contacts:\n contacts.append(contact)\n\n technical_contacts = res.get('technical_contacts')\n if technical_contacts:\n for contact in technical_contacts:\n contacts.append(contact)\n\n emails = list()\n names = list()\n phones = list()\n locations = list()\n\n for contact in contacts:\n email = contact.get('email')\n if email:\n if SpiderFootHelpers.validEmail(email):\n emails.append(email)\n\n name = contact.get(\"name\")\n if name:\n names.append(name)\n\n phone = contact.get('phone')\n if phone:\n phone = phone.replace(\" \", \"\").replace(\"-\", \"\").replace(\"(\", \"\").replace(\")\", \"\").replace(\".\", \"\")\n phones.append(phone)\n\n country = SpiderFootHelpers.countryNameFromCountryCode(contact.get('country_code'))\n location = ', '.join([_f for _f in [contact.get('address'), contact.get('city'), contact.get('state'), contact.get('zip'), country] if _f])\n if location:\n locations.append(location)\n\n for email in set(emails):\n mail_domain = email.lower().split('@')[1]\n if self.getTarget().matches(mail_domain, includeChildren=True):\n if email.split(\"@\")[0] in self.opts['_genericusers'].split(\",\"):\n evttype = \"EMAILADDR_GENERIC\"\n else:\n evttype = \"EMAILADDR\"\n evt = SpiderFootEvent(evttype, email, self.__name__, event)\n self.notifyListeners(evt)\n else:\n evt = SpiderFootEvent(\"AFFILIATE_EMAILADDR\", email, self.__name__, event)\n self.notifyListeners(evt)\n\n if eventName in [\"DOMAIN_NAME\"]:\n raw = res.get('raw')\n if raw:\n evt = SpiderFootEvent(\"DOMAIN_WHOIS\", raw, self.__name__, event)\n self.notifyListeners(evt)\n\n registrar = res.get(\"registrar\")\n if registrar:\n registrar_name = registrar.get(\"name\")\n if registrar_name:\n evt = SpiderFootEvent(\"DOMAIN_REGISTRAR\", registrar_name, self.__name__, event)\n self.notifyListeners(evt)\n\n for dns_provider in set(dns_providers):\n evt = SpiderFootEvent(\"PROVIDER_DNS\", dns_provider, self.__name__, event)\n self.notifyListeners(evt)\n\n for name in set(names):\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", f\"Possible full name {name}\", self.__name__, event)\n self.notifyListeners(evt)\n\n for phone in set(phones):\n evt = SpiderFootEvent(\"PHONE_NUMBER\", phone, self.__name__, event)\n self.notifyListeners(evt)\n\n for location in set(locations):\n evt = SpiderFootEvent(\"PHYSICAL_ADDRESS\", location, self.__name__, event)\n self.notifyListeners(evt)\n\n if eventName in [\"AFFILIATE_DOMAIN_NAME\"]:\n raw = res.get('raw')\n if raw:\n evt = SpiderFootEvent(\"AFFILIATE_DOMAIN_WHOIS\", raw, self.__name__, event)\n self.notifyListeners(evt)\n\n available = res.get('available?')\n if available:\n evt = SpiderFootEvent(\"AFFILIATE_DOMAIN_UNREGISTERED\", eventData, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_jsonwhoiscom class\n" }, { "path": "modules/sfp_junkfiles.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_junkfiles\n# Purpose: From Spidering, identifies backup and temporary files.\n#\n# Author: Steve Micallef \n#\n# Created: 23/08/2014\n# Copyright: (c) Steve Micallef 2014\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport random\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_junkfiles(SpiderFootPlugin):\n\n meta = {\n 'name': \"Junk File Finder\",\n 'summary': \"Looks for old/temporary and other similar files.\",\n 'flags': [\"slow\", \"errorprone\", \"invasive\"],\n 'useCases': [\"Footprint\"],\n 'categories': [\"Crawling and Scanning\"]\n }\n\n # Default options\n opts = {\n 'fileexts': ['tmp', 'bak', 'old'],\n 'urlextstry': ['asp', 'php', 'jsp', ],\n 'files': [\"old\", \"passwd\", \".htaccess\", \".htpasswd\",\n \"Thumbs.db\", \"backup\"],\n 'dirs': ['zip', 'tar.gz', 'tgz', 'tar']\n }\n\n # Option descriptions\n optdescs = {\n 'fileexts': \"File extensions to try.\",\n 'urlextstry': \"Try those extensions against URLs with these extensions.\",\n 'files': \"Try to fetch each of these files from the directory of the URL.\",\n 'dirs': \"Try to fetch the containing folder with these extensions.\"\n }\n\n results = None\n hosts = None\n skiphosts = None\n bases = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.hosts = self.tempStorage()\n self.skiphosts = self.tempStorage()\n self.bases = self.tempStorage()\n self.__dataSource__ = \"Target Website\"\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"LINKED_URL_INTERNAL\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"JUNK_FILE\"]\n\n # Test how trustworthy a result is\n def checkValidity(self, junkUrl):\n # Try and fetch an obviously missing version of the junk file\n fetch = junkUrl + str(random.SystemRandom().randint(0, 99999999))\n res = self.sf.fetchUrl(fetch, headOnly=True,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n verify=False)\n if res['code'] != \"404\":\n host = SpiderFootHelpers.urlBaseUrl(junkUrl)\n self.skiphosts[host] = True\n return False\n return True\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n host = SpiderFootHelpers.urlBaseUrl(eventData)\n\n if host in self.skiphosts:\n self.debug(\"Skipping \" + host + \" because it doesn't return 404s.\")\n return\n\n # http://www/blah/abc.php -> try http://www/blah/abc.php.[fileexts]\n for ext in self.opts['urlextstry']:\n if host in self.skiphosts:\n self.debug(\"Skipping \" + host + \" because it doesn't return 404s.\")\n return\n\n if \".\" + ext + \"?\" in eventData or \".\" + ext + \"#\" in eventData or \\\n eventData.endswith(\".\" + ext):\n bits = eventData.split(\"?\")\n for x in self.opts['fileexts']:\n if self.checkForStop():\n return\n\n self.debug(\"Trying \" + x + \" against \" + eventData)\n fetch = bits[0] + \".\" + x\n if fetch in self.results:\n self.debug(\"Skipping, already fetched.\")\n continue\n\n self.results[fetch] = True\n\n res = self.sf.fetchUrl(fetch, headOnly=True,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n sizeLimit=10000000,\n verify=False)\n if res['realurl'] != fetch:\n self.debug(\"Skipping because \" + res['realurl'] + \" isn't the fetched URL of \" + fetch)\n continue\n if res['code'] == \"200\":\n if not self.checkValidity(fetch):\n continue\n\n evt = SpiderFootEvent(\"JUNK_FILE\", fetch, self.__name__, event)\n self.notifyListeners(evt)\n\n base = SpiderFootHelpers.urlBaseDir(eventData)\n if not base or base in self.bases:\n return\n\n self.bases[base] = True\n\n # http://www/blah/abc.html -> try http://www/blah/[files]\n for f in self.opts['files']:\n if self.checkForStop():\n return\n\n if host in self.skiphosts:\n self.debug(\"Skipping \" + host + \" because it doesn't return 404s.\")\n return\n\n self.debug(\"Trying \" + f + \" against \" + eventData)\n fetch = base + f\n if fetch in self.results:\n self.debug(\"Skipping, already fetched.\")\n continue\n\n self.results[fetch] = True\n\n res = self.sf.fetchUrl(fetch, headOnly=True,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n verify=False)\n if res['realurl'] != fetch:\n self.debug(\"Skipping because \" + res['realurl'] + \" isn't the fetched URL of \" + fetch)\n continue\n if res['code'] == \"200\":\n if not self.checkValidity(fetch):\n continue\n\n evt = SpiderFootEvent(\"JUNK_FILE\", fetch, self.__name__, event)\n self.notifyListeners(evt)\n\n # don't do anything with the root directory of a site\n self.debug(f\"Base: {base}, event: {eventData}\")\n if base in [eventData, eventData + \"/\"]:\n return\n\n # http://www/blah/abc.html -> try http://www/blah.[dirs]\n for dirfile in self.opts['dirs']:\n if self.checkForStop():\n return\n\n if host in self.skiphosts:\n self.debug(\"Skipping \" + host + \" because it doesn't return 404s.\")\n return\n\n if base.count('/') == 3:\n self.debug(\"Skipping base url.\")\n continue\n\n self.debug(\"Trying \" + dirfile + \" against \" + eventData)\n fetch = base[0:len(base) - 1] + \".\" + dirfile\n if fetch in self.results:\n self.debug(\"Skipping, already fetched.\")\n continue\n\n self.results[fetch] = True\n\n res = self.sf.fetchUrl(fetch, headOnly=True,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n verify=False)\n if res['realurl'] != fetch:\n self.debug(\"Skipping because \" + res['realurl'] + \" isn't the fetched URL of \" + fetch)\n continue\n if res['code'] == \"200\":\n if not self.checkValidity(fetch):\n continue\n\n evt = SpiderFootEvent(\"JUNK_FILE\", fetch, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_junkfiles class\n" }, { "path": "modules/sfp_keybase.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_keybase\n# Purpose: Spiderfoot plugin to query KeyBase API\n# to gather additional information about domain names and identified\n# usernames.\n#\n# Author: Krishnasis Mandal \n#\n# Created: 22/05/2020\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport re\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_keybase(SpiderFootPlugin):\n\n meta = {\n 'name': \"Keybase\",\n 'summary': \"Obtain additional information about domain names and identified usernames.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Public Registries\"],\n 'dataSource': {\n 'website': \"https://keybase.io/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://keybase.io/docs/api/1.0/call/user/lookup\",\n ],\n 'favIcon': \"https://keybase.io/images/icons/icon-keybase-logo-48.png\",\n 'logo': \"https://keybase.io/images/icons/icon-keybase-logo-48.png\",\n 'description': \"Keybase is a key directory that maps social media identities to encryption keys \"\n \"in a publicly auditable manner.\",\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"USERNAME\", \"LINKED_URL_EXTERNAL\", \"DOMAIN_NAME\"]\n\n def producedEvents(self):\n return [\n \"RAW_RIR_DATA\", \"SOCIAL_MEDIA\", \"USERNAME\",\n \"GEOINFO\", \"BITCOIN_ADDRESS\", \"PGP_KEY\"\n ]\n\n def query(self, selector: str, qry: str) -> str:\n \"\"\"Search Keybase for a domain name or username.\n\n Args:\n selector (str): query type (\"usernames\" | \"domain\")\n qry (str): username\n\n Returns:\n str: Search results as JSON string\n \"\"\"\n if not selector:\n return None\n\n if not qry:\n return None\n\n params = {\n selector: qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n }\n\n headers = {\n 'Accept': \"application/json\"\n }\n\n res = self.sf.fetchUrl(\n 'https://keybase.io/_/api/1.0/user/lookup.json?' + urllib.parse.urlencode(params),\n headers=headers,\n timeout=15,\n useragent=self.opts['_useragent']\n )\n\n # In this case, it will always be 200 if keybase is queried\n # The actual response codes are stored in status tag of the response\n if res['code'] != '200':\n self.error(f\"Unexpected reply from Keybase: {res['code']}\")\n return None\n\n try:\n content = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n status = content.get('status')\n if not status:\n return None\n\n code = status.get('code')\n\n if code != 0:\n self.error(f\"Unexpected JSON response code reply from Keybase: {code}\")\n return None\n\n them = content.get('them')\n if not isinstance(them, list):\n return None\n\n return them\n\n def handleEvent(self, event) -> None:\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n # Extract username if a Keybase link is received\n if eventName == \"LINKED_URL_EXTERNAL\":\n linkRegex = r\"^https?://keybase.io\\/[A-Za-z0-9\\-_\\.]+\"\n links = re.findall(linkRegex, eventData)\n\n if len(links) == 0:\n self.debug(f\"Skipping URL {eventData}, as not a keybase link\")\n return\n\n userName = links[0].split(\"/\")[3]\n\n data = self.query('usernames', userName)\n elif eventName == \"USERNAME\":\n data = self.query('usernames', eventData)\n elif eventName == \"DOMAIN_NAME\":\n data = self.query('domain', eventData)\n else:\n return\n\n if not data:\n self.debug(f\"No data found for {eventName}: {eventData}\")\n return\n\n for user in data:\n if not user:\n continue\n\n # Basic information about the username\n basics = user.get('basics')\n if not basics:\n continue\n\n username = basics.get('username')\n if not username:\n continue\n\n # Failsafe to prevent reporting any wrongly received data\n if eventName == \"USERNAME\":\n if eventData.lower() != username.lower():\n self.error(\"Username does not match received response, skipping\")\n continue\n\n # For newly discovereed usernames, create a username event to be used as a source event\n if eventName in ['LINKED_URL_EXTERNAL', 'DOMAIN_NAME']:\n if username in self.results:\n self.debug(f\"Skipping {userName}, already checked.\")\n continue\n\n source_event = SpiderFootEvent(\"USERNAME\", username, self.__name__, event)\n self.notifyListeners(source_event)\n self.results[username] = True\n else:\n source_event = event\n\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(user), self.__name__, source_event)\n self.notifyListeners(evt)\n\n # Profile information about the username\n profile = user.get('profile')\n if profile:\n # Get and report full name of user\n fullName = profile.get('full_name')\n if fullName:\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", f\"Possible full name: {fullName}\", self.__name__, source_event)\n self.notifyListeners(evt)\n\n # Get and report location of user\n location = profile.get('location')\n if location:\n evt = SpiderFootEvent(\"GEOINFO\", location, self.__name__, source_event)\n self.notifyListeners(evt)\n\n # Extract social media information\n proofsSummary = user.get('proofs_summary')\n\n if proofsSummary:\n socialMediaData = proofsSummary.get('all')\n\n if socialMediaData:\n for socialMedia in socialMediaData:\n socialMediaSite = socialMedia.get('proof_type')\n socialMediaURL = socialMedia.get('service_url')\n\n if socialMediaSite and socialMediaURL:\n socialMedia = socialMediaSite + \": \" + \"\" + socialMediaURL + \"\"\n evt = SpiderFootEvent(\"SOCIAL_MEDIA\", socialMedia, self.__name__, source_event)\n self.notifyListeners(evt)\n\n # Get cryptocurrency addresses\n cryptoAddresses = user.get('cryptocurrency_addresses')\n\n # Extract and report bitcoin addresses if any\n if cryptoAddresses:\n bitcoinAddresses = cryptoAddresses.get('bitcoin')\n\n if bitcoinAddresses:\n for bitcoinAddress in bitcoinAddresses:\n btcAddress = bitcoinAddress.get('address')\n\n if not btcAddress:\n continue\n\n evt = SpiderFootEvent(\"BITCOIN_ADDRESS\", btcAddress, self.__name__, source_event)\n self.notifyListeners(evt)\n\n # Extract PGP Keys\n pgpRegex = r\"-----BEGIN\\s*PGP\\s*(?:PUBLIC?)\\s*KEY\\s*BLOCK-----(.*?)-----END\\s*PGP\\s*(?:PUBLIC?)\\s*KEY\\s*BLOCK-----\"\n\n pgpKeys = re.findall(pgpRegex, str(user))\n\n for pgpKey in pgpKeys:\n if len(pgpKey) < 300:\n self.debug(f\"PGP key size ({len(pgpKey)} bytes) is likely invalid (smaller than 300 bytes), skipping.\")\n continue\n\n # Remove unescaped \\n literals\n pgpKey = pgpKey.replace(\"\\\\n\", \"\\n\")\n\n # Avoid reporting of duplicate keys\n pgpKeyHash = self.sf.hashstring(pgpKey)\n\n if pgpKeyHash in self.results:\n continue\n\n self.results[pgpKeyHash] = True\n\n evt = SpiderFootEvent(\"PGP_KEY\", pgpKey, self.__name__, source_event)\n self.notifyListeners(evt)\n\n# End of sfp_keybase class\n" }, { "path": "modules/sfp_koodous.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_koodous\n# Purpose: Query Koodous for mobile apps.\n#\n# Author: \n#\n# Created: 2020-09-19\n# Copyright: (c) bcoles 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport re\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_koodous(SpiderFootPlugin):\n\n meta = {\n 'name': \"Koodous\",\n 'summary': \"Search Koodous for mobile apps.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Investigate\", \"Footprint\", \"Passive\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'model': \"FREE_AUTH_LIMITED\",\n \"apiKeyInstructions\": [\n \"Visit https://koodous.com/apks\",\n \"Register a free account\",\n \"Visit https://koodous.com/settings/developers and use the authentication token provided\",\n ],\n 'references': [\n \"https://docs.koodous.com/api/apks.html\",\n \"https://docs.koodous.com/apks.html#apks-search-system\"\n ],\n 'website': \"https://koodous.com/apks/\",\n 'favIcon': \"https://koodous.com/favicon.ico\",\n 'logo': \"https://koodous.com/assets/img/koodous-logo.png\",\n \"description\": \"The Collaborative Platform for Android Malware Analysts.\"\n }\n }\n\n opts = {\n \"api_key\": \"\",\n 'max_pages': 10,\n }\n\n optdescs = {\n \"api_key\": \"Koodous API key.\",\n 'max_pages': \"Maximum number of pages of results to fetch.\",\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.errorState = False\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n 'DOMAIN_NAME'\n ]\n\n def producedEvents(self):\n return [\n 'APPSTORE_ENTRY',\n 'RAW_RIR_DATA'\n ]\n\n def queryPackageName(self, qry, cursor=''):\n package_name = qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace')\n\n params = urllib.parse.urlencode({\n 'cursor': cursor,\n 'search': f\"package:{package_name}.*\"\n })\n\n res = self.sf.fetchUrl(\n f\"https://developer.koodous.com/apks/?{params}\",\n headers={\"Authorization\": f\"Token {self.opts['api_key']}\"},\n useragent=self.opts['_useragent'],\n timeout=self.opts['_fetchtimeout']\n )\n\n # 100 requests per minute\n time.sleep(1)\n\n return self.parseApiResponse(res)\n\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from Koodous.\")\n return None\n\n if res['code'] == '404':\n self.debug(\"No results from Koodous.\")\n return None\n\n if res['code'] == \"401\":\n self.error(\"Invalid Koodous API key.\")\n self.errorState = True\n return None\n\n if res['code'] == '429':\n self.error(\"You are being rate-limited by Koodous.\")\n self.errorState = True\n return None\n\n if res['code'] == '500' or res['code'] == '502' or res['code'] == '503':\n self.error(\"Koodous service is unavailable\")\n self.errorState = True\n return None\n\n # Catch all non-200 status codes, and presume something went wrong\n if res['code'] != '200':\n self.error(f\"Unexpected reply from Koodous: {res['code']}\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from Koodous: {e}\")\n\n return None\n\n def handleEvent(self, event):\n if self.errorState:\n return\n\n self.debug(f\"Received event, {event.eventType}, from {event.module}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\n f\"You enabled {self.__class__.__name__} but did not set an API key!\"\n )\n self.errorState = True\n return\n\n eventData = event.data\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n # Reverse domain name to create potential package name\n domain_reversed = '.'.join(list(reversed(eventData.lower().split('.'))))\n\n max_pages = int(self.opts['max_pages'])\n page = 1\n cursor = ''\n while page <= max_pages:\n found = False\n\n if self.checkForStop():\n return\n\n data = self.queryPackageName(domain_reversed, cursor)\n\n page += 1\n\n if not data:\n self.errorState = True\n return\n\n results = data.get('results')\n\n for result in results:\n package_name = result.get('package_name')\n\n if not package_name:\n continue\n\n app = result.get('app')\n\n # results can have a null app name, but it is probably a duplicate\n if not app:\n continue\n\n # TODO: compare company name with target\n # company = result.get('company')\n\n version = result.get('version')\n\n if version:\n app_full_name = f\"{app} {version} ({package_name})\"\n else:\n app_full_name = f\"{app} ({package_name})\"\n\n if (\n domain_reversed != package_name.lower()\n and not package_name.lower().startswith(f\"{domain_reversed}.\")\n and not package_name.lower().endswith(f\".{domain_reversed}\")\n and f\".{domain_reversed}.\" not in package_name.lower()\n ):\n self.debug(f\"App {app_full_name} does not match {domain_reversed}, skipping\")\n continue\n\n sha256 = result.get('sha256')\n\n if not sha256:\n continue\n\n app_data = f\"{app_full_name}\\nhttps://koodous.com/apks/{sha256}\"\n\n evt = SpiderFootEvent('APPSTORE_ENTRY', app_data, self.__name__, event)\n self.notifyListeners(evt)\n found = True\n\n if found:\n evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(data), self.__name__, event)\n self.notifyListeners(evt)\n\n if not data.get('next'):\n break\n\n next_cursor = re.findall('cursor=(.+?)&', data.get('next'))\n if not next_cursor:\n break\n\n cursor = urllib.parse.unquote(next_cursor[0])\n\n# End of sfp_koodous class\n" }, { "path": "modules/sfp_leakix.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_leakix\n# Purpose: Search LeakIX for host data leaks, open ports, software and geoip.\n#\n# Authors: \n#\n# Created: 2020-06-16\n# Copyright: (c) bcoles 2020\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_leakix(SpiderFootPlugin):\n\n meta = {\n 'name': \"LeakIX\",\n 'summary': \"Search LeakIX for host data leaks, open ports, software and geoip.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Leaks, Dumps and Breaches\"],\n 'dataSource': {\n 'website': \"https://leakix.net/\",\n 'model': \"FREE_AUTH_UNLIMITED\",\n 'references': [\n \"https://leakix.net/api-documentation\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://leakix.net/\",\n \"Register a free account\",\n \"Go to your 'Settings'\",\n \"Click on 'API key'\",\n \"Click on 'Reset key' to generate a new key\"\n ],\n 'favIcon': \"https://leakix.net/public/img/favicon.png\",\n 'logo': \"https://leakix.net/public/img/logoleakix-v1.png\",\n 'description': \"LeakIX provides insights into devices and servers that are compromised \"\n \"and compromised database schemas online.\\n\"\n \"In this scope we inspect found services for weak credentials.\",\n }\n }\n\n # Default options\n opts = {\n 'api_key': \"\",\n 'delay': 1,\n \"verify\": True,\n }\n\n # Option descriptions\n optdescs = {\n 'api_key': \"LeakIX API key\",\n 'delay': 'Delay between requests, in seconds.',\n \"verify\": \"Verify discovered hostnames are valid by checking if they still resolve.\",\n }\n\n results = None\n errorState = False\n\n # Initialize module and module options\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in userOpts.keys():\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"IP_ADDRESS\", \"DOMAIN_NAME\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"RAW_RIR_DATA\", \"GEOINFO\", \"TCP_PORT_OPEN\",\n \"OPERATING_SYSTEM\", \"SOFTWARE_USED\", \"WEBSERVER_BANNER\",\n \"LEAKSITE_CONTENT\", \"INTERNET_NAME\"]\n\n # Query host\n # https://leakix.net/api-documentation\n def queryApi(self, qryType, qry):\n headers = {\n \"Accept\": \"application/json\",\n \"api-key\": self.opts[\"api_key\"]\n }\n res = self.sf.fetchUrl(\n 'https://leakix.net/' + qryType + '/' + qry,\n headers=headers,\n timeout=15,\n useragent=self.opts['_useragent']\n )\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n # Parse API response\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from LeakIX.\")\n return None\n\n if res['code'] == '404':\n self.debug(\"Host not found\")\n return None\n\n # Future proofing - LeakIX does not implement rate limiting\n if res['code'] == '429':\n self.error(\"You are being rate-limited by LeakIX\")\n self.errorState = True\n return None\n\n # Catch all non-200 status codes, and presume something went wrong\n if res['code'] != '200':\n self.error(\"Failed to retrieve content from LeakIX\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n ports = list()\n hosts = list()\n oses = list()\n softwares = list()\n ips = list()\n banners = list()\n locs = list()\n\n if self.errorState:\n return\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if self.opts['api_key'] == \"\":\n self.debug(\"You enabled sfp_leakix but did not set an API key, results are limited\")\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventName in [\"IP_ADDRESS\", \"DOMAIN_NAME\"]:\n if eventName == \"IP_ADDRESS\":\n data = self.queryApi(\"host\", eventData)\n if eventName == \"DOMAIN_NAME\":\n data = self.queryApi(\"domain\", eventData)\n\n if data is None:\n self.debug(\"No information found for host \" + eventData)\n return\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n services = data.get(\"Services\")\n\n if services:\n for service in services:\n src = event\n ipevt = None\n hostname = service.get('host')\n if hostname and eventName == \"DOMAIN_NAME\" and self.getTarget().matches(hostname) and hostname not in hosts:\n if self.opts[\"verify\"] and not self.sf.resolveHost(hostname) and not self.sf.resolveHost6(hostname):\n self.debug(f\"Host {hostname} could not be resolved\")\n evt = SpiderFootEvent(\"INTERNET_NAME_UNRESOLVED\", hostname, self.__name__, event)\n else:\n evt = SpiderFootEvent(\"INTERNET_NAME\", hostname, self.__name__, event)\n self.notifyListeners(evt)\n src = evt\n hosts.append(hostname)\n ip = service.get('ip')\n if ip and eventName != \"IP_ADDRESS\" and self.sf.validIP(ip) and ip not in ips:\n evt = SpiderFootEvent(\"IP_ADDRESS\", ip, self.__name__, src)\n self.notifyListeners(evt)\n ips.append(ip)\n ipevt = evt\n port = service.get('port')\n if port and ip + \":\" + port not in ports:\n evt = SpiderFootEvent(\"TCP_PORT_OPEN\", ip + ':' + port, self.__name__, src)\n self.notifyListeners(evt)\n ports.append(ip + \":\" + port)\n headers = service.get('headers')\n if headers:\n servers = headers.get('Server')\n if servers:\n for server in servers:\n if server and server not in banners:\n evt = SpiderFootEvent('WEBSERVER_BANNER', server, self.__name__, src)\n self.notifyListeners(evt)\n banners.append(server)\n\n geoip = service.get('geoip')\n if geoip:\n location = ', '.join([_f for _f in [geoip.get('city_name'), geoip.get('region_name'), geoip.get('country_name')] if _f])\n if location:\n if ip and self.sf.validIP(ip) and ipevt:\n # GEOINFO should be linked to an IP_ADDRESS\n evt = SpiderFootEvent(\"GEOINFO\", location, self.__name__, ipevt)\n self.notifyListeners(evt)\n locs.append(location)\n\n software = service.get('software')\n if software:\n software_version = ' '.join([_f for _f in [software.get('name'), software.get('version')] if _f])\n if software_version and software_version not in softwares:\n evt = SpiderFootEvent(\"SOFTWARE_USED\", software_version, self.__name__, src)\n self.notifyListeners(evt)\n softwares.append(software_version)\n\n os = software.get('os')\n if os and os not in oses:\n evt = SpiderFootEvent('OPERATING_SYSTEM', os, self.__name__, src)\n self.notifyListeners(evt)\n oses.append(os)\n\n leaks = data.get(\"Leaks\")\n\n if leaks:\n for leak in leaks:\n leak_protocol = leak.get('type')\n hostname = leak.get('host')\n # If protocol is web, our hostname not empty and is not an IP ,\n # and doesn't belong to our target, discard ( happens when sharing Hosting/CDN IPs )\n if leak_protocol == \"web\" and hostname and not self.sf.validIP(hostname) and not self.getTarget().matches(hostname):\n continue\n leak_data = leak.get('data')\n if leak_data:\n evt = SpiderFootEvent(\"LEAKSITE_CONTENT\", leak_data, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_leakix class\n" }, { "path": "modules/sfp_maltiverse.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_maltiverse\n# Purpose: Spiderfoot plugin to search Maltiverse API\n# for any malicious activity by the target\n#\n# Author: Krishnasis Mandal \n#\n# Created: 20/05/2020\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nfrom datetime import datetime\n\nfrom netaddr import IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_maltiverse(SpiderFootPlugin):\n\n meta = {\n 'name': \"Maltiverse\",\n 'summary': \"Obtain information about any malicious activities involving IP addresses\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://maltiverse.com\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://maltiverse.com/faq\",\n \"https://app.swaggerhub.com/apis-docs/maltiverse/api/1.0.0-oas3\"\n ],\n 'favIcon': \"https://maltiverse.com/favicon.ico\",\n 'logo': \"https://maltiverse.com/assets/images/logo/logo.png\",\n 'description': \"The Open IOC Search Engine.\\n\"\n \"Enhance your SIEM or Firewall and crosscheck your event data with \"\n \"top quality Threat Intelligence information to highlight what requires action.\",\n }\n }\n\n opts = {\n 'checkaffiliates': True,\n 'subnetlookup': False,\n 'netblocklookup': True,\n 'maxnetblock': 24,\n 'maxsubnet': 24,\n \"age_limit_days\": 30\n }\n\n # Option descriptions. Delete any options not applicable to this module.\n optdescs = {\n 'checkaffiliates': \"Check affiliates?\",\n 'subnetlookup': \"Look up all IPs on subnets which your target is a part of?\",\n 'netblocklookup': \"Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?\",\n 'maxnetblock': \"If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n 'maxsubnet': \"If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)\",\n \"age_limit_days\": \"Ignore any records older than this many days. 0 = unlimited.\",\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n # For a list of all events, check sfdb.py.\n def watchedEvents(self):\n return [\"IP_ADDRESS\", \"NETBLOCK_OWNER\", \"NETBLOCK_MEMBER\",\n \"AFFILIATE_IPADDR\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"IP_ADDRESS\", \"MALICIOUS_IPADDR\", \"RAW_RIR_DATA\",\n \"MALICIOUS_AFFILIATE_IPADDR\"]\n\n # Check whether the IP Address is malicious using Maltiverse API\n # https://app.swaggerhub.com/apis-docs/maltiverse/api/1.0.0-oas3#/IPv4/getIP\n def queryIPAddress(self, qry):\n\n headers = {\n 'Accept': \"application/json\",\n }\n\n res = self.sf.fetchUrl(\n 'https://api.maltiverse.com/ip/' + str(qry),\n headers=headers,\n timeout=15,\n useragent=self.opts['_useragent']\n )\n\n if res['code'] == \"400\":\n self.error(\"Bad request. \" + qry + \" is not a valid IP Address\")\n return None\n\n if res['code'] == \"404\":\n self.error(\"API endpoint not found\")\n return None\n\n if res['code'] != \"200\":\n self.debug(\"No information found from Maltiverse for IP Address\")\n return None\n\n try:\n # Maltiverse returns \\\\n instead of \\n in the response\n data = str(res['content']).replace(\"\\\\n\", \" \")\n return json.loads(data)\n except Exception:\n self.error(\"Incorrectly formatted data received as JSON response\")\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n # Don't look up stuff twice\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n if eventName == 'NETBLOCK_OWNER':\n if not self.opts['netblocklookup']:\n return\n\n if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:\n self.debug(\"Network size bigger than permitted: \"\n + str(IPNetwork(eventData).prefixlen) + \" > \"\n + str(self.opts['maxnetblock']))\n return\n\n if eventName == 'NETBLOCK_MEMBER':\n if not self.opts['subnetlookup']:\n return\n\n if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:\n self.debug(\"Network size bigger than permitted: \"\n + str(IPNetwork(eventData).prefixlen) + \" > \"\n + str(self.opts['maxsubnet']))\n return\n\n qrylist = list()\n if eventName.startswith(\"NETBLOCK_\"):\n for ipaddr in IPNetwork(eventData):\n qrylist.append(str(ipaddr))\n self.results[str(ipaddr)] = True\n else:\n # If user has enabled affiliate checking\n if eventName == \"AFFILIATE_IPADDR\" and not self.opts['checkaffiliates']:\n return\n qrylist.append(eventData)\n\n for addr in qrylist:\n\n if self.checkForStop():\n return\n\n data = self.queryIPAddress(addr)\n\n if data is None:\n break\n\n maliciousIP = data.get('ip_addr')\n\n if maliciousIP is None:\n continue\n\n if addr != maliciousIP:\n self.error(\"Reported address doesn't match requested, skipping\")\n continue\n\n blacklistedRecords = data.get('blacklist')\n\n if blacklistedRecords is None or len(blacklistedRecords) == 0:\n self.debug(\"No blacklist information found for IP\")\n continue\n\n # Data is reported about the IP Address\n if eventName.startswith(\"NETBLOCK_\"):\n ipEvt = SpiderFootEvent(\"IP_ADDRESS\", addr, self.__name__, event)\n self.notifyListeners(ipEvt)\n\n if eventName.startswith(\"NETBLOCK_\"):\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, ipEvt)\n self.notifyListeners(evt)\n else:\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n maliciousIPDesc = f\"Maltiverse [{maliciousIP}]\\n\"\n\n for blacklistedRecord in blacklistedRecords:\n lastSeen = blacklistedRecord.get('last_seen')\n if lastSeen is None:\n continue\n\n try:\n lastSeenDate = datetime.strptime(str(lastSeen), \"%Y-%m-%d %H:%M:%S\")\n except Exception:\n self.error(\"Invalid date in JSON response, skipping\")\n continue\n\n today = datetime.now()\n\n difference = (today - lastSeenDate).days\n\n if difference > int(self.opts[\"age_limit_days\"]):\n self.debug(\"Record found is older than age limit, skipping\")\n continue\n\n maliciousIPDesc += \" - DESCRIPTION : \" + str(blacklistedRecord.get(\"description\")) + \"\\n\"\n\n maliciousIPDescHash = self.sf.hashstring(maliciousIPDesc)\n\n if maliciousIPDescHash in self.results:\n continue\n\n self.results[maliciousIPDescHash] = True\n\n if eventName.startswith(\"NETBLOCK_\"):\n evt = SpiderFootEvent(\"MALICIOUS_IPADDR\", maliciousIPDesc, self.__name__, ipEvt)\n elif eventName.startswith(\"AFFILIATE_\"):\n evt = SpiderFootEvent(\"MALICIOUS_AFFILIATE_IPADDR\", maliciousIPDesc, self.__name__, event)\n else:\n evt = SpiderFootEvent(\"MALICIOUS_IPADDR\", maliciousIPDesc, self.__name__, event)\n\n self.notifyListeners(evt)\n\n# End of sfp_maltiverse class\n" }, { "path": "modules/sfp_malwarepatrol.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_malwarepatrol\n# Purpose: SpiderFoot plug-in to search MalwarePatrol's daatabase for\n# potential malicious IPs/hostnames.\n#\n# Author: Steve Micallef \n#\n# Created: 25/07/2016\n# Copyright: (c) Steve Micallef 2016\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_malwarepatrol(SpiderFootPlugin):\n\n meta = {\n 'name': \"MalwarePatrol\",\n 'summary': \"Searches malwarepatrol.net's database of malicious URLs/IPs.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://www.malwarepatrol.net/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://www.malwarepatrol.net/tech-support/\",\n \"https://www.malwarepatrol.net/integrations-formats-tip-siem-soar/\",\n \"https://www.malwarepatrol.net/community-contribution-suspicious-emails/\",\n \"https://www.malwarepatrol.net/non-commercial/#lists\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://www.malwarepatrol.net/free-guard-block-list/\",\n \"Register a free account\",\n \"The Password/Receipt number will be mailed to your email\"\n ],\n 'favIcon': \"https://www.malwarepatrol.net/wp-content/uploads/2020/02/rsz_12logo-150x150.png\",\n 'logo': \"https://www.malwarepatrol.net/wp-content/uploads/2016/06/rsz_mp_logo_clear_-_small.png\",\n 'description': \"Based in the USA and Brazil, our history is one of community spirit and dedication to \"\n \"Internet security that began in 2005 when a group started sharing malicious links using a simple mailing list.\\n\"\n \"Collecting, analyzing, and sharing data for over a decade has allowed us \"\n \"to develop an extensive network of sensors, sharing agreements, and community contributors. \"\n \"The result is our vast database of unique and historically rich – “intelligent” – threat data.\",\n }\n }\n\n opts = {\n \"api_key\": \"\",\n 'checkaffiliates': True,\n 'checkcohosts': True,\n }\n\n optdescs = {\n \"api_key\": \"Malwarepatrol.com 'receipt' ID, provided once signing up for their open-source feed. Without this you cannot obtain their feed.\",\n 'checkaffiliates': \"Apply checks to affiliates?\",\n 'checkcohosts': \"Apply checks to sites found to be co-hosted on the target's IP?\",\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"AFFILIATE_INTERNET_NAME\",\n \"IP_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"CO_HOSTED_SITE\",\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"BLACKLISTED_INTERNET_NAME\",\n \"BLACKLISTED_AFFILIATE_INTERNET_NAME\",\n \"BLACKLISTED_COHOST\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_COHOST\",\n ]\n\n def queryAddr(self, qaddr):\n data = dict()\n url = \"http://lists.malwarepatrol.net/cgi/getfile?receipt=\" + \\\n self.opts['api_key'] + \"&product=8&list=smoothwall\"\n\n data['content'] = self.sf.cacheGet(\"sfmalwarepatrol\", 72)\n if data['content'] is None:\n data = self.sf.fetchUrl(url, useragent=self.opts['_useragent'])\n if data['content'] is None:\n self.error(\"Unable to fetch \" + url)\n return None\n self.sf.cachePut(\"sfmalwarepatrol\", data['content'])\n\n for line in data['content'].split('\\n'):\n if len(line) < 2 or line.startswith('#'):\n continue\n\n if line.startswith(qaddr):\n return True\n\n return False\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if not self.opts['api_key']:\n self.error(\"You enabled sfp_malwarepatrol but did not provide a receipt ID!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData} as already searched.\")\n return\n\n self.results[eventData] = True\n\n if eventName == 'IP_ADDRESS':\n malicious_type = \"MALICIOUS_IPADDR\"\n blacklist_type = \"BLACKLISTED_IPADDR\"\n elif eventName == 'AFFILIATE_IPADDR':\n if not self.opts.get('checkaffiliates', False):\n return\n malicious_type = \"MALICIOUS_AFFILIATE_IPADDR\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_IPADDR\"\n elif eventName == \"INTERNET_NAME\":\n malicious_type = \"MALICIOUS_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n if not self.opts.get('checkaffiliates', False):\n return\n malicious_type = \"MALICIOUS_AFFILIATE_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n if not self.opts.get('checkcohosts', False):\n return\n malicious_type = \"MALICIOUS_COHOST\"\n blacklist_type = \"BLACKLISTED_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n if not self.queryAddr(eventData):\n return\n\n text = f\"MalwarePatrol [{eventData}]\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_malwarepatrol class\n" }, { "path": "modules/sfp_metadefender.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_metadefender\n# Purpose: Search MetaDefender API for IP address and domain reputation.\n#\n# Author: \n#\n# Created: 2019-09-21\n# Copyright: (c) bcoles 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_metadefender(SpiderFootPlugin):\n\n meta = {\n 'name': \"MetaDefender\",\n 'summary': \"Search MetaDefender API for IP address and domain IP reputation.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://metadefender.opswat.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://onlinehelp.opswat.com/mdcloud/\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://metadefender.opswat.com/\",\n \"Register a free account\",\n \"Navigate to https://metadefender.opswat.com/account\",\n \"The API key is listed under 'API key'\"\n ],\n 'favIcon': \"https://mcl-cdn.opswat.com/1.40.3-729f31db/city/icons/icon-48x48.png?v=61be50566cce944a710aaa90ba6bbb8d\",\n 'logo': \"https://mcl-cdn.opswat.com/1.40.3-729f31db/city/icons/icon-48x48.png?v=61be50566cce944a710aaa90ba6bbb8d\",\n 'description': \"File Analysis - Analyzing binaries with 30+ anti-malware engines.\\n\"\n \"Heuristic analysis to detect more unknown and targeted attacks.\\n\"\n \"Binary vulnerability data assessment, IP/Domain reputation, Threat Intelligence Feeds\",\n }\n }\n\n # Default options\n opts = {\n 'api_key': '',\n # 10 requests / minute for free users\n # 100 requests / minute for paid users\n 'delay': 6\n }\n\n # Option descriptions\n optdescs = {\n 'api_key': 'MetaDefender API key.',\n 'delay': 'Delay between requests, in seconds.'\n }\n\n results = None\n errorState = False\n\n # Initialize module and module options\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['IP_ADDRESS', 'INTERNET_NAME']\n\n # What events this module produces\n def producedEvents(self):\n return [\n 'MALICIOUS_IPADDR',\n 'MALICIOUS_INTERNET_NAME',\n 'BLACKLISTED_IPADDR',\n 'BLACKLISTED_INTERNET_NAME',\n 'GEOINFO'\n ]\n\n # Query domain REST API\n # https://onlinehelp.opswat.com/mdcloud/4.5_Domain_Reputation.html\n def queryDomain(self, qry):\n headers = {\n 'Accept': 'application/json',\n 'apikey': self.opts['api_key']\n }\n res = self.sf.fetchUrl('https://api.metadefender.com/v4/domain/' + qry,\n headers=headers,\n timeout=15,\n useragent=self.opts['_useragent'])\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n # Query ip REST API\n # https://onlinehelp.opswat.com/mdcloud/4.1_IP_Reputation.html\n def queryIp(self, qry):\n headers = {\n 'Accept': 'application/json',\n 'apikey': self.opts['api_key']\n }\n res = self.sf.fetchUrl('https://api.metadefender.com/v4/ip/' + qry,\n headers=headers,\n timeout=15,\n useragent=self.opts['_useragent'])\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n # Parse API response\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from MetaDefender.\")\n return None\n\n if res['code'] == \"401\":\n self.error(\"Invalid MetaDefender API key\")\n self.errorState = True\n return None\n\n # https://onlinehelp.opswat.com/mdcloud/3._Rate_Limiting.html\n # https://onlinehelp.opswat.com/mdcloud/4._Throttling.html\n if res['code'] == \"429\":\n self.error(\"You are being rate-limited by MetaDefender\")\n self.errorState = True\n return None\n\n if res['code'] == \"404\":\n return None\n\n if res['content'] is None:\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n if self.errorState:\n return\n\n if eventData in self.results:\n return\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_metadefender but did not set an API key!\")\n self.errorState = True\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventName == 'IP_ADDRESS':\n data = self.queryIp(eventData)\n\n if data is None:\n self.debug(\"No matches found for \" + eventData)\n return\n\n geo_info = data.get('geo_info')\n\n if geo_info:\n location = ', '.join([_f for _f in [geo_info.get('city').get('name'), geo_info.get('country').get('name')] if _f])\n evt = SpiderFootEvent('GEOINFO', location, self.__name__, event)\n self.notifyListeners(evt)\n\n res = data.get('lookup_results')\n\n if not res:\n self.debug(\"No matches found for \" + eventData)\n return\n\n sources = res.get('sources')\n\n if not sources:\n self.debug(\"No matches found for \" + eventData)\n return\n\n for m in sources:\n if not m.get('assessment'):\n continue\n if m['assessment'] != \"trustworthy\":\n continue\n provider = m.get('provider')\n evt = SpiderFootEvent('MALICIOUS_IPADDR', provider + ' [' + eventData + ']', self.__name__, event)\n self.notifyListeners(evt)\n evt = SpiderFootEvent('BLACKLISTED_IPADDR', provider + ' [' + eventData + ']', self.__name__, event)\n self.notifyListeners(evt)\n\n if eventName == 'INTERNET_NAME':\n data = self.queryDomain(eventData)\n\n if data is None:\n self.debug(\"No matches found for \" + eventData)\n return\n\n res = data.get('lookup_results')\n\n if not res:\n self.debug(\"No matches found for \" + eventData)\n return\n\n sources = res.get('sources')\n\n if not sources:\n self.debug(\"No matches found for \" + eventData)\n return\n\n for m in sources:\n if not m.get('assessment'):\n continue\n if m['assessment'] == \"trustworthy\":\n continue\n provider = m.get('provider')\n evt = SpiderFootEvent('MALICIOUS_INTERNET_NAME', provider + ' [' + eventData + ']', self.__name__, event)\n self.notifyListeners(evt)\n evt = SpiderFootEvent('BLACKLISTED_INTERNET_NAME', provider + ' [' + eventData + ']', self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_metadefender class\n" }, { "path": "modules/sfp_mnemonic.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_mnemonic\n# Purpose: SpiderFoot plug-in for retrieving passive DNS information\n# from Mnemonic PassiveDNS API.\n#\n# Author: \n#\n# Created: 2018-10-12\n# Copyright: (c) bcoles 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_mnemonic(SpiderFootPlugin):\n\n meta = {\n 'name': \"Mnemonic PassiveDNS\",\n 'summary': \"Obtain Passive DNS information from PassiveDNS.mnemonic.no.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Passive DNS\"],\n 'dataSource': {\n 'website': \"https://www.mnemonic.no\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://www.mnemonic.no/resources/whitepapers/\",\n \"https://www.mnemonic.no/research-and-development/\",\n \"https://docs.mnemonic.no/display/public/API/PassiveDNS+Integration+Guide\"\n ],\n 'favIcon': \"https://www.mnemonic.no/favicon-96x96.png\",\n 'logo': \"https://www.mnemonic.no/UI/logo.svg\",\n 'description': \"mnemonic helps businesses manage their security risks, \"\n \"protect their data and defend against cyber threats.\\n\"\n \"Our expert team of security consultants, product specialists, \"\n \"threat researchers, incident responders and ethical hackers, combined \"\n \"with our Argus security platform ensures we stay ahead of \"\n \"advanced cyberattacks and protect our customers from evolving threats.\",\n }\n }\n\n opts = {\n 'per_page': 500,\n 'max_pages': 2,\n 'timeout': 30,\n 'maxage': 180, # 6 months\n 'verify': True,\n 'cohostsamedomain': False,\n 'maxcohost': 100\n }\n\n optdescs = {\n 'per_page': \"Maximum number of results per page.\",\n 'max_pages': \"Maximum number of pages of results to fetch.\",\n 'timeout': \"Query timeout, in seconds.\",\n 'maxage': \"The maximum age of the data returned, in days, in order to be considered valid.\",\n 'verify': \"Verify identified domains still resolve to the associated specified IP address.\",\n 'cohostsamedomain': \"Treat co-hosted sites on the same target domain as co-hosting?\",\n 'maxcohost': \"Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.\",\n }\n\n cohostcount = 0\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.cohostcount = 0\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n 'IP_ADDRESS',\n 'IPV6_ADDRESS',\n 'INTERNET_NAME',\n 'DOMAIN_NAME'\n ]\n\n def producedEvents(self):\n return [\n 'IP_ADDRESS',\n 'IPV6_ADDRESS',\n 'INTERNAL_IP_ADDRESS',\n 'CO_HOSTED_SITE',\n 'INTERNET_NAME',\n 'DOMAIN_NAME'\n ]\n\n def query(self, qry, limit=500, offset=0):\n \"\"\"Query the Mnemonic PassiveDNS v3 API.\n\n Args:\n qry (str): domain name or IP address\n limit (int): Limit the number of returned values.\n offset (int): Skip the initial number of values in the resultset.\n\n Returns:\n dict: results as JSON\n \"\"\"\n\n params = urllib.parse.urlencode({\n 'limit': limit,\n 'offset': offset\n })\n\n res = self.sf.fetchUrl(\n f\"https://api.mnemonic.no/pdns/v3/{qry}?{params}\",\n timeout=self.opts['timeout'],\n useragent=self.opts['_useragent']\n )\n\n # Unauthenticated users are limited to 100 requests per minute, and 1000 requests per day.\n time.sleep(0.75)\n\n if res['content'] is None:\n self.info(\"No results found for \" + qry)\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response from Mnemonic: {e}\")\n return None\n\n response_code = data.get('responseCode')\n\n if not response_code:\n self.debug(\"Error retrieving search results.\")\n return None\n\n if response_code == 402:\n self.debug(\"Error retrieving search results: Resource limit exceeded\")\n self.errorState = True\n return None\n\n if response_code != 200:\n self.debug(f\"Error retrieving search results: {response_code}\")\n return None\n\n if 'data' not in data:\n self.info(f\"No results found for {qry}\")\n return None\n\n size = data.get('size')\n count = data.get('count')\n\n if not count or not size:\n self.info(f\"No results found for {qry}\")\n return None\n\n self.info(f\"Retrieved {size} of {count} results\")\n\n return data['data']\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n position = 0\n max_pages = int(self.opts['max_pages'])\n per_page = int(self.opts['per_page'])\n agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage'])\n self.cohostcount = 0\n cohosts = list()\n\n while position < (per_page * max_pages):\n if self.checkForStop():\n break\n\n if self.errorState:\n break\n\n data = self.query(eventData, limit=per_page, offset=position)\n\n if data is None:\n self.info(f\"No passive DNS data found for {eventData}\")\n break\n\n position += per_page\n\n for r in data:\n if \"*\" in r['query'] or \"%\" in r['query']:\n continue\n\n if r['lastSeenTimestamp'] < agelimit:\n self.debug(f\"Record {r['answer']} found for {r['query']} is too old, skipping.\")\n continue\n\n if eventName in ['IP_ADDRESS']:\n if r['rrtype'] == 'a':\n if self.sf.validIP(r['query']):\n cohosts.append(r['query'])\n continue\n\n if eventName in ['INTERNET_NAME', 'DOMAIN_NAME']:\n # Ignore PTR records\n if r['rrtype'] == 'ptr':\n continue\n\n if r['rrtype'] == 'cname':\n if not self.getTarget().matches(r['query'], includeParents=True):\n continue\n\n cohosts.append(r['query'])\n\n if self.opts['verify']:\n continue\n\n answer = r.get('answer')\n\n if r['rrtype'] == 'a':\n if not self.sf.validIP(answer):\n continue\n\n if self.sf.isValidLocalOrLoopbackIp(answer):\n evt = SpiderFootEvent(\"INTERNAL_IP_ADDRESS\", answer, self.__name__, event)\n else:\n evt = SpiderFootEvent(\"IP_ADDRESS\", answer, self.__name__, event)\n self.notifyListeners(evt)\n\n if r['rrtype'] == 'aaaa':\n if not self.sf.validIP6(r['answer']):\n continue\n\n if self.sf.isValidLocalOrLoopbackIp(answer):\n evt = SpiderFootEvent(\"INTERNAL_IP_ADDRESS\", answer, self.__name__, event)\n else:\n evt = SpiderFootEvent(\"IPV6_ADDRESS\", answer, self.__name__, event)\n self.notifyListeners(evt)\n\n for co in set(cohosts):\n if self.checkForStop():\n return\n\n if co in self.results:\n continue\n\n if eventName in [\"IP_ADDRESS\", \"IPV6_ADDRESS\"]:\n if self.opts['verify'] and not self.sf.validateIP(co, eventData):\n self.debug(f\"Host {co} no longer resolves to {eventData}\")\n continue\n\n if self.opts['cohostsamedomain']:\n if self.cohostcount < self.opts['maxcohost']:\n evt = SpiderFootEvent(\"CO_HOSTED_SITE\", co, self.__name__, event)\n self.notifyListeners(evt)\n self.cohostcount += 1\n continue\n\n if self.getTarget().matches(co, includeParents=True):\n if self.opts['verify'] and not self.sf.resolveHost(co) and not self.sf.resolveHost6(co):\n self.debug(f\"Host {co} could not be resolved\")\n evt = SpiderFootEvent(\"INTERNET_NAME_UNRESOLVED\", co, self.__name__, event)\n self.notifyListeners(evt)\n continue\n\n evt = SpiderFootEvent(\"INTERNET_NAME\", co, self.__name__, event)\n self.notifyListeners(evt)\n\n if self.sf.isDomain(co, self.opts['_internettlds']):\n evt = SpiderFootEvent(\"DOMAIN_NAME\", co, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_mnemonic class\n" }, { "path": "modules/sfp_multiproxy.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_multiproxy\n# Purpose: Check if an IP arress is an open proxy according to multiproxy.org\n# open proxy list.\n#\n# Author: steve@binarypool.com\n#\n# Created: 14/12/2013\n# Copyright: (c) Steve Micallef, 2013\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom netaddr import IPAddress, IPNetwork\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_multiproxy(SpiderFootPlugin):\n\n meta = {\n 'name': \"multiproxy.org Open Proxies\",\n 'summary': \"Check if an IP address is an open proxy according to multiproxy.org open proxy list.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Secondary Networks\"],\n 'dataSource': {\n 'website': \"https://multiproxy.org/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://multiproxy.org/faq.htm\",\n \"https://multiproxy.org/env_check.htm\",\n \"https://multiproxy.org/anon_proxy.htm\",\n \"https://multiproxy.org/help.htm\"\n ],\n 'favIcon': \"https://www.google.com/s2/favicons?domain=https://multiproxy.org/\",\n 'logo': \"https://multiproxy.org/images/mproxy_title.png\",\n 'description': \"MultiProxy is a multifunctional personal proxy server that protects your privacy \"\n \"while on the Internet as well as speeds up your downloads, \"\n \"especially if you are trying to get several files form overseas or from otherwise rather slow server. \"\n \"It can also completely hide your IP address by dynamically connecting to \"\n \"non-transparent anonymizing public proxy servers. \"\n \"You can also test a list of proxy servers and sort them by connection speed and level of anonimity.\",\n }\n }\n\n opts = {\n 'checkaffiliates': True,\n 'cacheperiod': 18\n }\n\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliates?\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n 'IP_ADDRESS',\n 'AFFILIATE_IPADDR',\n 'NETBLOCK_OWNER',\n 'NETBLOCK_MEMBER',\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_IPADDR\",\n \"BLACKLISTED_AFFILIATE_IPADDR\",\n \"BLACKLISTED_SUBNET\",\n \"BLACKLISTED_NETBLOCK\",\n \"MALICIOUS_IPADDR\",\n \"MALICIOUS_AFFILIATE_IPADDR\",\n \"MALICIOUS_NETBLOCK\",\n \"MALICIOUS_SUBNET\",\n ]\n\n def queryProxyList(self, target, targetType):\n proxy_list = self.retrieveProxyList()\n\n if not proxy_list:\n self.errorState = True\n return False\n\n if targetType == \"ip\":\n if target in proxy_list:\n self.debug(f\"IP address {target} found in multiproxy.org open proxy list.\")\n return True\n elif targetType == \"netblock\":\n netblock = IPNetwork(target)\n for ip in proxy_list:\n if IPAddress(ip) in netblock:\n self.debug(f\"IP address {ip} found within netblock/subnet {target} in multiproxy.org open proxy list.\")\n return True\n\n return False\n\n def retrieveProxyList(self):\n proxy_list = self.sf.cacheGet('multiproxyopenproxies', 24)\n\n if proxy_list is not None:\n return self.parseProxyList(proxy_list)\n\n res = self.sf.fetchUrl(\n \"http://multiproxy.org/txt_all/proxy.txt\",\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from multiproxy.org.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from multiproxy.org\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"multiproxyopenproxies\", res['content'])\n\n return self.parseProxyList(res['content'])\n\n def parseProxyList(self, proxy_list):\n \"\"\"Parse plaintext open proxy list\n\n Args:\n proxy_list (str): plaintext open proxy list from multiproxy.org\n\n Returns:\n list: list of open proxy IP addresses\n \"\"\"\n ips = list()\n\n if not proxy_list:\n return ips\n\n for ip in proxy_list.split('\\n'):\n ip = ip.strip().split(\":\")[0]\n if ip.startswith('#'):\n continue\n if not self.sf.validIP(ip):\n continue\n ips.append(ip)\n\n return ips\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName == 'IP_ADDRESS':\n targetType = 'ip'\n malicious_type = \"MALICIOUS_IPADDR\"\n blacklist_type = \"BLACKLISTED_IPADDR\"\n elif eventName == 'AFFILIATE_IPADDR':\n if not self.opts.get('checkaffiliates', False):\n return\n targetType = 'ip'\n malicious_type = \"MALICIOUS_AFFILIATE_IPADDR\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_IPADDR\"\n elif eventName == 'NETBLOCK_OWNER':\n if not self.opts.get('checknetblocks', False):\n return\n targetType = 'netblock'\n malicious_type = \"MALICIOUS_NETBLOCK\"\n blacklist_type = \"BLACKLISTED_NETBLOCK\"\n elif eventName == 'NETBLOCK_MEMBER':\n if not self.opts.get('checksubnets', False):\n return\n targetType = 'netblock'\n malicious_type = \"MALICIOUS_SUBNET\"\n blacklist_type = \"BLACKLISTED_SUBNET\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n self.debug(f\"Checking maliciousness of {eventData} ({eventName}) with multiproxy.org open proxy list\")\n\n if not self.queryProxyList(eventData, targetType):\n return\n\n url = \"http://multiproxy.org/txt_all/proxy.txt\"\n text = f\"multiproxy.org Open Proxies [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_multiproxy class\n" }, { "path": "modules/sfp_myspace.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_myspace\n# Purpose: Query MySpace for username and location information.\n#\n# Author: \n#\n# Created: 2018-10-07\n# Copyright: (c) bcoles 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_myspace(SpiderFootPlugin):\n\n meta = {\n 'name': \"MySpace\",\n 'summary': \"Gather username and location from MySpace.com profiles.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Social Media\"],\n 'dataSource': {\n 'website': \"https://myspace.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://www.programmableweb.com/api/myspace\"\n ],\n 'favIcon': \"https://x.myspacecdn.com/new/common/images/favicons/favicon.ico\",\n 'logo': \"https://x.myspacecdn.com/new/common/images/favicons/114-Retina-iPhone.png\",\n 'description': \"Myspace is a place where people come to connect, discover, and share.\\n\"\n \"Through an open design, compelling editorial features, \"\n \"and analytics-based recommendations, Myspace creates a creative community \"\n \"of people who connect around mutual affinity and inspiration for the purpose \"\n \"of shaping, sharing, and discovering what's next.\",\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.__dataSource__ = \"MySpace.com\"\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"EMAILADDR\", \"SOCIAL_MEDIA\"]\n\n def producedEvents(self):\n return [\"SOCIAL_MEDIA\", \"GEOINFO\"]\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n # Search by email address\n if eventName == \"EMAILADDR\":\n email = eventData\n res = self.sf.fetchUrl(\"https://myspace.com/search/people?q=\" + email,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'])\n\n if res['content'] is None:\n self.error(f\"Could not fetch MySpace content for {email}\")\n return\n\n # Extract HTML containing potential profile matches\n profiles = re.findall(r'[^<]+', str(res['content']))\n\n if not profiles:\n self.debug(f\"No profiles found for e-mail: {email}\")\n return\n\n # The first result is the closest match, but whether it's an exact match is unknown.\n profile = profiles[0]\n\n # Check for email address as name, at the risk of missed results.\n try:\n matches = re.findall(r'\\.\\?\\!]+' + email + r'[\\&; :\\\"\\#\\*\\)\\\"\\'\\;\\,\\<\\.\\?\\!]+', profile, re.IGNORECASE)\n except Exception:\n self.debug(\"Malformed e-mail address, skipping.\")\n return\n\n if not matches:\n self.debug(\"No concrete match for that e-mail.\")\n return\n\n name = matches[0]\n e = SpiderFootEvent(\n \"SOCIAL_MEDIA\",\n f\"MySpace: https://myspace.com/{name}\",\n self.__name__,\n event\n )\n self.notifyListeners(e)\n\n # Retrieve location from MySpace profile\n if eventName == \"SOCIAL_MEDIA\":\n try:\n network = eventData.split(\": \")[0]\n url = eventData.split(\": \")[1].replace(\"\", \"\").replace(\"\", \"\")\n except Exception as e:\n self.debug(f\"Unable to parse SOCIAL_MEDIA: {eventData} ({e})\")\n return\n\n if network != \"MySpace\":\n self.debug(f\"Skipping social network profile, {url}, as not a MySpace profile\")\n return\n\n res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'])\n\n if res['content'] is None:\n return\n\n data = re.findall(r'
100:\n self.debug(\"Skipping likely invalid location.\")\n return\n\n e = SpiderFootEvent(\"GEOINFO\", location, self.__name__, event)\n self.notifyListeners(e)\n\n# End of sfp_myspace class\n" }, { "path": "modules/sfp_nameapi.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_nameapi\n# Purpose: Spiderfoot plugin to check if an email is\n# disposable using nameapi.org API.\n#\n# Author: Krishnasis Mandal \n#\n# Created: 2020-10-02\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_nameapi(SpiderFootPlugin):\n\n meta = {\n 'name': \"NameAPI\",\n 'summary': \"Check whether an email is disposable\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://www.nameapi.org/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://www.nameapi.org/en/developer/manuals/rest-web-services/53/web-services/disposable-email-address-detector/\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://nameapi.org\",\n \"Click on 'Get API Key'\",\n \"Register a free account\",\n \"The API key will be sent to your email\"\n ],\n 'favIcon': \"https://www.nameapi.org/fileadmin/favicon.ico\",\n 'logo': \"https://www.nameapi.org/fileadmin/templates/nameprofiler/images/name-api-logo.png\",\n 'description': \"The NameAPI DEA-Detector checks email addresses \"\n \"against a list of known trash domains such as mailinator.com.\\n\"\n \"It classifies those as disposable which operate as a time-limited, \"\n \"web based way of receiving emails, for example, sign up confirmations.\",\n }\n }\n\n opts = {\n 'api_key': ''\n }\n\n optdescs = {\n 'api_key': \"API Key for NameAPI\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"EMAILADDR\"\n ]\n\n def producedEvents(self):\n return [\n \"EMAILADDR_DISPOSABLE\",\n \"RAW_RIR_DATA\"\n ]\n\n def queryEmailAddr(self, qry):\n res = self.sf.fetchUrl(\n f\"http://api.nameapi.org/rest/v5.3/email/disposableemailaddressdetector?apiKey={self.opts['api_key']}&emailAddress={qry}\",\n timeout=self.opts['_fetchtimeout'],\n useragent=\"SpiderFoot\"\n )\n\n if res['content'] is None:\n self.info(f\"No NameAPI info found for {qry}\")\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from NameAPI: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\n f\"You enabled {self.__class__.__name__} but did not set an API key!\"\n )\n self.errorState = True\n return\n\n self.results[eventData] = True\n\n data = self.queryEmailAddr(eventData)\n\n if data is None:\n return\n\n isDisposable = data.get('disposable')\n\n if isDisposable == \"YES\":\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(\"EMAILADDR_DISPOSABLE\", eventData, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_nameapi class\n" }, { "path": "modules/sfp_names.py", "content": "# coding: utf-8\n# -------------------------------------------------------------------------------\n# Name: sfp_names\n# Purpose: Identify human names in content fetched.\n#\n# Author: Steve Micallef \n#\n# Created: 24/03/2014\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_names(SpiderFootPlugin):\n\n meta = {\n 'name': \"Human Name Extractor\",\n 'summary': \"Attempt to identify human names in fetched content.\",\n 'flags': [\"errorprone\"],\n 'useCases': [\"Footprint\", \"Passive\"],\n 'categories': [\"Content Analysis\"]\n }\n\n # Default options\n opts = {\n 'algolimit': 75,\n 'emailtoname': True,\n 'filterjscss': True\n }\n\n # Option descriptions\n optdescs = {\n 'algolimit': \"A value between 0-100 to tune the sensitivity of the name finder. Less than 40 will give you a lot of junk, over 50 and you'll probably miss things but will have less false positives.\",\n 'emailtoname': \"Convert e-mail addresses in the form of firstname.surname@target to names?\",\n 'filterjscss': \"Filter out names that originated from CSS/JS content. Enabling this avoids detection of popular Javascript and web framework author names.\"\n }\n\n results = None\n d = None\n n = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.d = SpiderFootHelpers.dictionaryWordsFromWordlists()\n self.n = SpiderFootHelpers.humanNamesFromWordlists()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"TARGET_WEB_CONTENT\", \"EMAILADDR\",\n \"DOMAIN_WHOIS\", \"NETBLOCK_WHOIS\",\n \"RAW_RIR_DATA\", \"RAW_FILE_META_DATA\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"HUMAN_NAME\"]\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n # If the source event is web content, check if the source URL was javascript\n # or CSS, in which case optionally ignore it.\n if eventName == \"TARGET_WEB_CONTENT\":\n url = event.actualSource\n if url is not None:\n if self.opts['filterjscss'] and (\".js\" in url or \".css\" in url):\n self.debug(\"Ignoring web content from CSS/JS.\")\n return\n\n # Find names in email addresses in \".@\" format\n if eventName == \"EMAILADDR\" and self.opts['emailtoname']:\n potential_name = eventData.split(\"@\")[0]\n\n if \".\" not in potential_name:\n return\n\n name = \" \".join(map(str.capitalize, potential_name.split(\".\")))\n\n # Names usually do not contain numbers\n if re.search(\"[0-9]\", name):\n return\n\n evt = SpiderFootEvent(\"HUMAN_NAME\", name, self.__name__, event)\n if event.moduleDataSource:\n evt.moduleDataSource = event.moduleDataSource\n else:\n evt.moduleDataSource = \"Unknown\"\n self.notifyListeners(evt)\n return\n\n # For RAW_RIR_DATA, there are only specific modules we\n # expect to see RELEVANT names within.\n if eventName == \"RAW_RIR_DATA\":\n if srcModuleName not in [\"sfp_builtwith\", \"sfp_clearbit\", \"sfp_emailcrawlr\",\n \"sfp_fullcontact\", \"sfp_github\", \"sfp_hunter\",\n \"sfp_opencorporates\", \"sfp_slideshare\", \"sfp_jsonwhoiscom\",\n \"sfp_twitter\", \"sfp_gravatar\", \"sfp_keybase\"]:\n self.debug(\"Ignoring RAW_RIR_DATA from untrusted module.\")\n return\n\n # Stage 1: Find things that look (very vaguely) like names\n rx = re.compile(r\"([A-Z][a-z�������������]+)\\s+.?.?\\s?([A-Z][�������������a-zA-Z\\'\\-]+)\")\n m = re.findall(rx, eventData)\n for r in m:\n # Start off each match as 0 points.\n p = 0\n notindict = False\n\n # Shouldn't encounter \"Firstname's Secondname\"\n first = r[0].lower()\n if first[len(first) - 2] == \"'\" or first[len(first) - 1] == \"'\":\n continue\n\n # Strip off trailing ' or 's\n secondOrig = r[1].replace(\"'s\", \"\")\n secondOrig = secondOrig.rstrip(\"'\")\n second = r[1].lower().replace(\"'s\", \"\")\n second = second.rstrip(\"'\")\n\n # If both words are not in the dictionary, add 75 points.\n if first not in self.d and second not in self.d:\n self.debug(f\"Both first and second names are not in the dictionary, so high chance of name: ({first}:{second}).\")\n p += 75\n notindict = True\n else:\n self.debug(first + \" was found or \" + second + \" was found in dictionary.\")\n\n # If the first word is a known popular first name, award 50 points.\n if first in self.n:\n p += 50\n\n # If either word is 2 characters, subtract 50 points.\n if len(first) == 2 or len(second) == 2:\n p -= 50\n\n # If the first word is in the dictionary but the second isn't,\n # subtract 40 points.\n if not notindict:\n if first in self.d and second not in self.d:\n p -= 20\n\n # If the second word is in the dictionary but the first isn't,\n # reduce 20 points.\n if first not in self.d and second in self.d:\n p -= 40\n\n name = r[0] + \" \" + secondOrig\n\n self.debug(\"Name of \" + name + \" has score: \" + str(p))\n if p >= self.opts['algolimit']:\n # Notify other modules of what you've found\n evt = SpiderFootEvent(\"HUMAN_NAME\", name, self.__name__, event)\n if event.moduleDataSource:\n evt.moduleDataSource = event.moduleDataSource\n else:\n evt.moduleDataSource = \"Unknown\"\n self.notifyListeners(evt)\n\n\n# End of sfp_names class\n" }, { "path": "modules/sfp_networksdb.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_networksdb\n# Purpose: SpiderFoot plug-in to search NetworksDB.io API for IP address and\n# domain information.\n#\n# Author: \n#\n# Created: 2019-09-16\n# Copyright: (c) bcoles 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_networksdb(SpiderFootPlugin):\n\n meta = {\n 'name': \"NetworksDB\",\n 'summary': \"Search NetworksDB.io API for IP address and domain information.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Passive DNS\"],\n 'dataSource': {\n 'website': \"https://networksdb.io/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://networksdb.io/api/docs\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://networksdb.io/api/order/free\",\n \"Register a free account\",\n \"Click on 'Generate a Free API Key'\",\n \"The API key is listed under 'API Key: Free plan'\"\n ],\n 'favIcon': \"https://networksdb.io/img/favicon/favicon-96x96.png\",\n 'logo': \"https://networksdb.io/img/logo.png\",\n 'description': \"Our database contains information about the public IPv4 and IPv6 addresses, \"\n \"networks and domains owned by companies and organisations across the world \"\n \"along with city-level IP geolocation data and autonomous system information.\",\n }\n }\n # Default options\n opts = {\n 'api_key': '',\n 'delay': 1,\n 'verify': True,\n 'cohostsamedomain': False,\n 'maxcohost': 100\n }\n\n # Option descriptions\n optdescs = {\n 'api_key': 'NetworksDB API key.',\n 'delay': 'Delay between requests, in seconds.',\n 'verify': \"Verify co-hosts are valid by checking if they still resolve to the shared IP.\",\n 'cohostsamedomain': \"Treat co-hosted sites on the same target domain as co-hosting?\",\n 'maxcohost': \"Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.\"\n }\n\n cohostcount = 0\n results = None\n errorState = False\n\n # Initialize module and module options\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.cohostcount = 0\n self.errorState = False\n\n for opt in userOpts.keys():\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"IP_ADDRESS\", \"IPV6_ADDRESS\", \"INTERNET_NAME\", \"DOMAIN_NAME\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"INTERNET_NAME\", \"IP_ADDRESS\", \"IPV6_ADDRESS\", \"NETBLOCK_MEMBER\",\n \"CO_HOSTED_SITE\", \"GEOINFO\", \"RAW_RIR_DATA\"]\n\n # Query IP Address Info\n # https://networksdb.io/api/docs#ipinfo\n def queryIpInfo(self, qry):\n params = {\n 'ip': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n }\n headers = {\n 'Accept': 'application/json',\n 'X-Api-Key': self.opts['api_key'],\n 'Content-Type': 'application/x-www-form-urlencoded',\n }\n res = self.sf.fetchUrl('https://networksdb.io/api/ip-info',\n headers=headers,\n postData=urllib.parse.urlencode(params),\n timeout=15,\n useragent=self.opts['_useragent'])\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n # Query IP Geolocation\n # https://networksdb.io/api/docs#geoip\n def queryIpGeo(self, qry):\n params = {\n 'ip': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n }\n headers = {\n 'Accept': 'application/json',\n 'X-Api-Key': self.opts['api_key'],\n 'Content-Type': 'application/x-www-form-urlencoded',\n }\n res = self.sf.fetchUrl('https://networksdb.io/api/ip-geo',\n headers=headers,\n postData=urllib.parse.urlencode(params),\n timeout=15,\n useragent=self.opts['_useragent'])\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n # Query Domains on IP (Reverse DNS)\n # https://networksdb.io/api/docs#revdns\n def queryReverseDns(self, qry):\n params = {\n 'ip': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n }\n headers = {\n 'Accept': 'application/json',\n 'X-Api-Key': self.opts['api_key'],\n 'Content-Type': 'application/x-www-form-urlencoded',\n }\n res = self.sf.fetchUrl('https://networksdb.io/api/reverse-dns',\n headers=headers,\n postData=urllib.parse.urlencode(params),\n timeout=15,\n useragent=self.opts['_useragent'])\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n # Query IPs for Domain (Forward DNS)\n # https://networksdb.io/api/docs#fwddns\n def queryForwardDns(self, qry):\n params = {\n 'domain': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n }\n headers = {\n 'Accept': 'application/json',\n 'X-Api-Key': self.opts['api_key'],\n 'Content-Type': 'application/x-www-form-urlencoded',\n }\n res = self.sf.fetchUrl('https://networksdb.io/api/dns',\n headers=headers,\n postData=urllib.parse.urlencode(params),\n timeout=15,\n useragent=self.opts['_useragent'])\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n # Query Autonomous System Info\n # https://networksdb.io/api/docs#asinfo\n # Note: currently unused\n def queryAsnInfo(self, qry):\n params = {\n 'asn': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n }\n headers = {\n 'Accept': 'application/json',\n 'X-Api-Key': self.opts['api_key'],\n 'Content-Type': 'application/x-www-form-urlencoded',\n }\n res = self.sf.fetchUrl('https://networksdb.io/api/asn',\n headers=headers,\n postData=urllib.parse.urlencode(params),\n timeout=15,\n useragent=self.opts['_useragent'])\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n # Query Autonomous System Networks\n # https://networksdb.io/api/docs#asnets\n # Note: currently unused\n def queryAsnNetworks(self, qry):\n params = {\n 'asn': qry,\n }\n headers = {\n 'Accept': 'application/json',\n 'X-Api-Key': self.opts['api_key'],\n 'Content-Type': 'application/x-www-form-urlencoded',\n }\n res = self.sf.fetchUrl('https://networksdb.io/api/asn-networks',\n headers=headers,\n postData=urllib.parse.urlencode(params),\n timeout=15,\n useragent=self.opts['_useragent'])\n\n time.sleep(self.opts['delay'])\n\n return self.parseApiResponse(res)\n\n # Parse API response\n # https://networksdb.io/api/plans\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from NetworksDB.\")\n return None\n\n # Future proofing - NetworksDB does not implement rate limiting\n if res['code'] == '429':\n self.error(\"You are being rate-limited by NetworksDB\")\n self.errorState = True\n return None\n\n if res['code'] == '403':\n self.error(\"Authentication failed\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.error(f\"Error processing JSON response from NetworksDB: {e}\")\n return None\n\n if data.get('warning'):\n self.debug(\"Received warning from NetworksDB: \" + data.get('warning'))\n\n if data.get('error'):\n self.error(\"Received error from NetworksDB: \" + data.get('error'))\n\n return data\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if eventData in self.results:\n return\n\n if self.opts['api_key'] == '':\n self.error(\"You enabled sfp_networksdb but did not set an API key!\")\n self.errorState = True\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventName in [\"IP_ADDRESS\", \"IPV6_ADDRESS\"]:\n data = self.queryIpInfo(eventData)\n\n if data is None:\n self.debug(\"No IP address information found for \" + eventData)\n else:\n evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n network = data.get('network')\n if network:\n cidr = network.get('cidr')\n if cidr and cidr != 'N/A' and self.sf.validIpNetwork(cidr):\n if \":\" in cidr:\n evt = SpiderFootEvent('NETBLOCKV6_MEMBER', cidr, self.__name__, event)\n else:\n evt = SpiderFootEvent('NETBLOCK_MEMBER', cidr, self.__name__, event)\n self.notifyListeners(evt)\n\n data = self.queryIpGeo(eventData)\n\n if data is None:\n self.debug(\"No IP geolocation information found for \" + eventData)\n else:\n evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n if data.get('country'):\n location = ', '.join(filter(None, [data.get('city'), data.get('state'), data.get('country')]))\n evt = SpiderFootEvent('GEOINFO', location, self.__name__, event)\n self.notifyListeners(evt)\n\n data = self.queryReverseDns(eventData)\n\n cohosts = list()\n\n if data is None:\n self.debug(\"No reverse DNS results for \" + eventData)\n else:\n evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n results = data.get('results')\n if results:\n for domain in results:\n cohosts.append(domain)\n\n for co in set(cohosts):\n if self.checkForStop():\n return\n\n if co in self.results:\n continue\n\n if self.opts['verify'] and not self.sf.validateIP(co, eventData):\n self.debug(\"Host \" + co + \" no longer resolves to \" + eventData)\n continue\n\n if not self.opts['cohostsamedomain']:\n if self.getTarget().matches(co, includeParents=True):\n evt = SpiderFootEvent('INTERNET_NAME', co, self.__name__, event)\n self.notifyListeners(evt)\n if self.sf.isDomain(co, self.opts['_internettlds']):\n evt = SpiderFootEvent('DOMAIN_NAME', co, self.__name__, event)\n self.notifyListeners(evt)\n continue\n\n if self.cohostcount < self.opts['maxcohost']:\n evt = SpiderFootEvent('CO_HOSTED_SITE', co, self.__name__, event)\n self.notifyListeners(evt)\n self.cohostcount += 1\n\n if eventName in [\"INTERNET_NAME\", \"DOMAIN_NAME\"]:\n data = self.queryForwardDns(eventData)\n\n if data is None:\n self.debug(\"No forward DNS results for \" + eventData)\n return\n\n res = data.get('results')\n\n if not res:\n self.debug(\"No forward DNS results for \" + eventData)\n return\n\n evt = SpiderFootEvent('RAW_RIR_DATA', str(res), self.__name__, event)\n self.notifyListeners(evt)\n\n for ip in res:\n if self.sf.validIP(ip):\n evt = SpiderFootEvent('IP_ADDRESS', ip, self.__name__, event)\n self.notifyListeners(evt)\n elif self.sf.validIP6(ip):\n evt = SpiderFootEvent('IPV6_ADDRESS', ip, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_networksdb class\n" }, { "path": "modules/sfp_neutrinoapi.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_neutrinoapi\n# Purpose: SpiderFoot plug-in to search NeutrinoAPI for IP address info,\n# check IP address reputation, and search for phone location.\n#\n# Author: \n#\n# Created: 2018-11-30\n# Copyright: (c) bcoles 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_neutrinoapi(SpiderFootPlugin):\n\n meta = {\n 'name': \"NeutrinoAPI\",\n 'summary': \"Search NeutrinoAPI for phone location information, IP address information, and host reputation.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://www.neutrinoapi.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://www.neutrinoapi.com/api/api-basics/\",\n \"https://www.neutrinoapi.com/api/phone-validate/\",\n \"https://www.neutrinoapi.com/api/ip-info/\",\n \"https://www.neutrinoapi.com/api/ip-blocklist/\",\n \"https://www.neutrinoapi.com/api/host-reputation/\",\n ],\n 'apiKeyInstructions': [\n \"Visit https://www.neutrinoapi.com/\",\n \"Sign up for a free account\",\n \"Click on 'My Account'\",\n \"The API key is listed under 'Master Key'\"\n ],\n 'favIcon': \"https://www.neutrinoapi.com/favicon.png\",\n 'logo': \"https://www.neutrinoapi.com/favicon.png\",\n 'description': \"Neutrino API - The general-purpose API - Build smarter applications.\"\n }\n }\n\n # Default options\n opts = {\n 'user_id': '',\n 'api_key': '',\n 'timeout': 30\n }\n\n # Option descriptions\n optdescs = {\n 'user_id': \"NeutrinoAPI user ID.\",\n 'api_key': \"NeutrinoAPI API key.\",\n 'timeout': \"Query timeout, in seconds.\"\n }\n\n results = None\n errorState = False\n\n # Initialize module and module options\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.__dataSource__ = \"NeutrinoAPI\"\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['IP_ADDRESS', 'IPV6_ADDRESS', 'PHONE_NUMBER']\n\n # What events this module produces\n def producedEvents(self):\n return [\n 'RAW_RIR_DATA',\n 'BLACKLISTED_IPADDR',\n 'MALICIOUS_IPADDR',\n 'PROXY_HOST',\n 'VPN_HOST',\n 'TOR_EXIT_NODE',\n 'GEOINFO',\n ]\n\n # Query the phone-validate REST API\n # https://www.neutrinoapi.com/api/phone-validate/\n def queryPhoneValidate(self, qry):\n res = self.sf.fetchUrl(\n 'https://neutrinoapi.com/phone-validate',\n postData={\"output-format\": \"json\", \"number\": qry, \"user-id\": self.opts['user_id'], \"api-key\": self.opts['api_key']},\n timeout=self.opts['timeout'],\n useragent=self.opts['_useragent']\n )\n\n return self.parseApiResponse(res)\n\n # Query the ip-info REST API\n # https://www.neutrinoapi.com/api/ip-info/\n def queryIpInfo(self, qry):\n res = self.sf.fetchUrl(\n \"https://neutrinoapi.com/ip-info\",\n postData={\"output-format\": \"json\", \"ip\": qry, \"user-id\": self.opts['user_id'], \"api-key\": self.opts['api_key']},\n timeout=self.opts['timeout'],\n useragent=self.opts['_useragent']\n )\n\n return self.parseApiResponse(res)\n\n # Query the ip-blocklist REST API\n # https://www.neutrinoapi.com/api/ip-blocklist/\n def queryIpBlocklist(self, qry):\n res = self.sf.fetchUrl(\n \"https://neutrinoapi.com/ip-blocklist\",\n postData={\"output-format\": \"json\", \"ip\": qry, \"vpn-lookup\": True, \"user-id\": self.opts['user_id'], \"api-key\": self.opts['api_key']},\n timeout=self.opts['timeout'],\n useragent=self.opts['_useragent']\n )\n\n return self.parseApiResponse(res)\n\n # Query the host-reputation REST API\n # https://www.neutrinoapi.com/api/host-reputation/\n def queryHostReputation(self, qry):\n res = self.sf.fetchUrl(\n \"https://neutrinoapi.com/host-reputation\",\n postData={\"output-format\": \"json\", \"host\": qry, \"user-id\": self.opts['user_id'], \"api-key\": self.opts['api_key']},\n timeout=self.opts['timeout'],\n useragent=self.opts['_useragent']\n )\n\n return self.parseApiResponse(res)\n\n # Parse API response\n def parseApiResponse(self, res: dict):\n if not res:\n self.error(\"No response from NeutrinoAPI.\")\n return None\n\n if res['code'] == \"403\":\n self.error(\"Authentication failed\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n if res['code'] == \"400\":\n if data.get('api-error-msg'):\n self.error(\"Error: \" + data.get('api-error-msg'))\n if \"EXCEED\" in data.get('api-error-msg'):\n self.errorState = True\n return None\n else:\n self.error(\"Error: HTTP 400\")\n return None\n\n return data\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_neutrinoapi but did not set an API key!\")\n self.errorState = True\n return\n\n if self.opts['user_id'] == \"\":\n self.error(\"You enabled sfp_neutrinoapi but did not set a user ID!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if eventName == 'PHONE_NUMBER':\n data = self.queryPhoneValidate(eventData)\n\n if data is None:\n self.debug(\"No phone info results found for \" + eventData)\n else:\n if data.get('location') is not None and data.get('country') is not None:\n if data.get('location') == data.get('country'):\n location = data.get('location')\n else:\n location = data.get('location') + ', ' + data.get('country')\n\n evt = SpiderFootEvent(\"GEOINFO\", location, self.__name__, event)\n self.notifyListeners(evt)\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']:\n data = self.queryIpInfo(eventData)\n\n if data is None:\n self.debug(\"No IP info results found for \" + eventData)\n else:\n if data.get('city') is not None and data.get('region') is not None and data.get('country-code') is not None:\n location = data.get('city') + ', ' + data.get('region') + ', ' + data.get('country-code')\n evt = SpiderFootEvent(\"GEOINFO\", location, self.__name__, event)\n self.notifyListeners(evt)\n\n data = self.queryIpBlocklist(eventData)\n\n if data is None:\n self.debug(\"No IP blocklist results found for \" + eventData)\n else:\n if data.get('is-listed'):\n evt = SpiderFootEvent(\"MALICIOUS_IPADDR\", f\"NeutrinoAPI - IP Blocklist [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n evt = SpiderFootEvent(\"BLACKLISTED_IPADDR\", f\"NeutrinoAPI - IP Blocklist [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n if data.get('is-proxy'):\n evt = SpiderFootEvent(\"PROXY_HOST\", eventData, self.__name__, event)\n self.notifyListeners(evt)\n\n if data.get('is-vpn'):\n evt = SpiderFootEvent(\"VPN_HOST\", eventData, self.__name__, event)\n self.notifyListeners(evt)\n\n if data.get('is-tor'):\n evt = SpiderFootEvent(\"TOR_EXIT_NODE\", eventData, self.__name__, event)\n self.notifyListeners(evt)\n\n data = self.queryHostReputation(eventData)\n\n if data is None:\n self.debug(\"No host reputation results found for \" + eventData)\n else:\n if data.get('is-listed'):\n evt = SpiderFootEvent(\"MALICIOUS_IPADDR\", f\"NeutrinoAPI - Host Reputation [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n evt = SpiderFootEvent(\"BLACKLISTED_IPADDR\", f\"NeutrinoAPI - Host Reputation [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_neutrinoapi class\n" }, { "path": "modules/sfp_numverify.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_numverify\n# Purpose: SpiderFoot plug-in to search numverify.com API for a phone number\n# and retrieve location and carrier information.\n#\n# Author: \n#\n# Created: 2019-05-25\n# Copyright: (c) bcoles 2019\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin\n\n\nclass sfp_numverify(SpiderFootPlugin):\n\n meta = {\n 'name': \"numverify\",\n 'summary': \"Lookup phone number location and carrier information from numverify.com.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Real World\"],\n 'dataSource': {\n 'website': \"http://numverify.com/\",\n 'model': \"FREE_AUTH_LIMITED\",\n 'references': [\n \"https://numverify.com/documentation\",\n \"https://numverify.com/faq\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://numverify.com\",\n \"Sign up for a free account\",\n \"Navigate to https://numverify.com/dashboard\",\n \"The API key is listed under 'Your API Access Key'\"\n ],\n 'favIcon': \"https://numverify.com/images/icons/numverify_shortcut_icon.ico\",\n 'logo': \"https://numverify.com/images/logos/numverify_header.png\",\n 'description': \"Global Phone Number Validation & Lookup JSON API.\\n\"\n \"NumVerify offers a full-featured yet simple RESTful JSON API for \"\n \"national and international phone number validation and information lookup for a total of 232 countries around the world.\\n\"\n \"Requested numbers are processed in real-time, cross-checked with the latest international numbering plan databases \"\n \"and returned in handy JSON format enriched with useful carrier, geographical location and line type data.\",\n }\n }\n\n # Default options\n opts = {\n 'api_key': ''\n }\n\n # Option descriptions\n optdescs = {\n 'api_key': 'numverify API key.'\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['PHONE_NUMBER']\n\n # What events this module produces\n def producedEvents(self):\n return ['RAW_RIR_DATA', 'GEOINFO', 'PROVIDER_TELCO']\n\n # Query numverify API for the specified phone number\n # https://numverify.com/documentation\n def query(self, qry):\n number = qry.strip('+').strip('(').strip(')')\n\n params = {\n 'number': number.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n 'country_code': '',\n 'format': '0', # set to \"1\" for prettified debug output\n 'access_key': self.opts['api_key']\n }\n\n # Free API does not support HTTPS for no adequately explained reason\n res = self.sf.fetchUrl(\"http://apilayer.net/api/validate?\" + urllib.parse.urlencode(params),\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'])\n\n time.sleep(1)\n\n if res['content'] is None:\n self.debug('No response from apilayer.net')\n return None\n\n if res['code'] == '101':\n self.error('API error: invalid API key')\n self.errorState = True\n return None\n\n if res['code'] == '102':\n self.error('API error: user account deactivated')\n self.errorState = True\n return None\n\n if res['code'] == '104':\n self.error('API error: usage limit exceeded')\n self.errorState = True\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n if data.get('error') is not None:\n self.error('API error: ' + str(data.get('error')))\n return None\n\n return data\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_numverify but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n data = self.query(eventData)\n\n if data is None:\n self.debug(\"No phone information found for \" + eventData)\n return\n\n evt = SpiderFootEvent(\"RAW_RIR_DATA\", str(data), self.__name__, event)\n self.notifyListeners(evt)\n\n if data.get('country_code'):\n country = SpiderFootHelpers.countryNameFromCountryCode(data.get('country_code'))\n location = ', '.join([_f for _f in [data.get('location'), country] if _f])\n evt = SpiderFootEvent(\"GEOINFO\", location, self.__name__, event)\n self.notifyListeners(evt)\n else:\n self.debug(\"No location information found for \" + eventData)\n\n if data.get('carrier'):\n evt = SpiderFootEvent(\"PROVIDER_TELCO\", data.get('carrier'), self.__name__, event)\n self.notifyListeners(evt)\n else:\n self.debug(\"No carrier information found for \" + eventData)\n\n# End of sfp_numverify class\n" }, { "path": "modules/sfp_onioncity.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_onioncity\n# Purpose: Searches the Tor search engine 'Onion City' using Google Custom\n# Search for content related to the domain in question.\n#\n# Author: Steve Micallef \n#\n# Created: 15/07/2015\n# Copyright: (c) Steve Micallef 2015\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_onioncity(SpiderFootPlugin):\n\n meta = {\n 'name': \"Onion.link\",\n 'summary': \"Search Tor 'Onion City' search engine for mentions of the target domain using Google Custom Search.\",\n 'flags': [\"apikey\", \"tor\"],\n 'useCases': [\"Footprint\", \"Investigate\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://onion.link/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://developers.google.com/custom-search/v1\",\n \"https://developers.google.com/custom-search/docs/overview\",\n \"https://cse.google.com/cse\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://developers.google.com/custom-search/v1/introduction\",\n \"Register a free Google account\",\n \"Click on 'Get A Key'\",\n \"Connect a Project\",\n \"The API Key will be listed under 'YOUR API KEY'\"\n ],\n 'favIcon': \"https://www.google.com/s2/favicons?domain=https://onion.link\",\n 'logo': \"https://onion.link/images/OC.png\",\n 'description': \"Enabling search and global access to Tor's onionsites.\",\n }\n }\n\n # Default options\n opts = {\n \"api_key\": \"\",\n \"cse_id\": \"013611106330597893267:tfgl3wxdtbp\",\n 'fetchlinks': True,\n 'fullnames': True\n }\n\n # Option descriptions\n optdescs = {\n \"api_key\": \"Google API Key for Onion.link search.\",\n \"cse_id\": \"Google Custom Search Engine ID.\",\n 'fetchlinks': \"Fetch the darknet pages (via TOR, if enabled) to verify they mention your target.\",\n 'fullnames': \"Search for human names?\"\n }\n\n # Target\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"INTERNET_NAME\", \"DOMAIN_NAME\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"DARKNET_MENTION_URL\", \"DARKNET_MENTION_CONTENT\",\n \"RAW_RIR_DATA\"]\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if self.errorState:\n return\n\n if not self.opts['fullnames'] and eventName == 'HUMAN_NAME':\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts['api_key'] == \"\":\n self.error(\"You enabled sfp_onioncity but did not set a Google API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(f\"Already did a search for {eventData}, skipping.\")\n return\n\n self.results[eventData] = True\n\n # Sites hosted on the domain\n res = self.sf.googleIterate(\n searchString=\"+site:onion.link \" + eventData,\n opts={\n \"timeout\": self.opts[\"_fetchtimeout\"],\n \"useragent\": self.opts[\"_useragent\"],\n \"api_key\": self.opts[\"api_key\"],\n \"cse_id\": self.opts[\"cse_id\"],\n },\n )\n if res is None:\n # Failed to talk to the bing API or no results returned\n return\n\n urls = res[\"urls\"]\n new_links = list(set(urls) - set(self.results.keys()))\n\n # Add new links to results\n for link in new_links:\n self.results[link] = True\n\n # Submit the Google results for analysis\n googlesearch_url = res[\"webSearchUrl\"]\n response = self.sf.fetchUrl(\n googlesearch_url,\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=self.opts[\"_useragent\"],\n )\n if response['code'] in [\"200\", \"201\", \"202\"]:\n evt = SpiderFootEvent(\n \"RAW_RIR_DATA\", response[\"content\"], self.__name__, event\n )\n self.notifyListeners(evt)\n else:\n self.error(\"Failed to fetch Google web search URL\")\n\n # Check if we've been asked to stop\n if self.checkForStop():\n return\n\n darknet_links = [\n link for link in new_links if self.sf.urlFQDN(link).endswith(\".onion.link\")\n ]\n\n for link in darknet_links:\n self.debug(\"Found a darknet mention: \" + link)\n torlink = link.replace(\".onion.link\", \".onion\")\n if self.opts['fetchlinks']:\n res = self.sf.fetchUrl(torlink, timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n verify=False)\n\n if res['content'] is None:\n self.debug(\"Ignoring \" + link + \" as no data returned\")\n continue\n\n # Sometimes onion city search results false positives\n if re.search(r\"[^a-zA-Z\\-\\_0-9]\" + re.escape(eventData)\n + r\"[^a-zA-Z\\-\\_0-9]\", res['content'], re.IGNORECASE) is None:\n self.debug(\"Ignoring \" + link + \" as no mention of \" + eventData)\n continue\n\n evt = SpiderFootEvent(\"DARKNET_MENTION_URL\", torlink, self.__name__, event)\n self.notifyListeners(evt)\n\n try:\n startIndex = res['content'].index(eventData) - 120\n endIndex = startIndex + len(eventData) + 240\n except Exception:\n self.debug(\"String not found in content.\")\n continue\n\n data = res['content'][startIndex:endIndex]\n evt = SpiderFootEvent(\"DARKNET_MENTION_CONTENT\", \"...\" + data + \"...\",\n self.__name__, evt)\n self.notifyListeners(evt)\n else:\n evt = SpiderFootEvent(\"DARKNET_MENTION_URL\", torlink, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_onioncity class\n" }, { "path": "modules/sfp_onionsearchengine.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_onionsearchengine\n# Purpose: Searches the Tor search engine onionsearchengine.com for content\n# related to the domain in question.\n#\n# Author: Steve Micallef \n#\n# Created: 27/10/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_onionsearchengine(SpiderFootPlugin):\n\n meta = {\n 'name': \"Onionsearchengine.com\",\n 'summary': \"Search Tor onionsearchengine.com for mentions of the target domain.\",\n 'flags': [\"tor\"],\n 'useCases': [\"Footprint\", \"Investigate\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://as.onionsearchengine.com\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://helpdesk.onionsearchengine.com/?v=knowledgebase\",\n \"https://onionsearchengine.com/add_url.php\"\n ],\n 'favIcon': \"https://as.onionsearchengine.com/images/onionsearchengine.jpg\",\n 'logo': \"https://as.onionsearchengine.com/images/onionsearchengine.jpg\",\n 'description': \"No cookies, no javascript, no trace. We protect your privacy.\\n\"\n \"Onion search engine is search engine with ability to find content on tor network / deepweb / darkweb.\",\n }\n }\n\n # Default options\n opts = {\n 'timeout': 10,\n 'max_pages': 20,\n 'fetchlinks': True,\n 'blacklist': ['.*://relate.*'],\n 'fullnames': True\n }\n\n # Option descriptions\n optdescs = {\n 'timeout': \"Query timeout, in seconds.\",\n 'max_pages': \"Maximum number of pages of results to fetch.\",\n 'fetchlinks': \"Fetch the darknet pages (via TOR, if enabled) to verify they mention your target.\",\n 'blacklist': \"Exclude results from sites matching these patterns.\",\n 'fullnames': \"Search for human names?\"\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"DOMAIN_NAME\", \"HUMAN_NAME\", \"EMAILADDR\"]\n\n # What events this module produces\n # This is to support the end user in selecting modules based on events\n # produced.\n def producedEvents(self):\n return [\"DARKNET_MENTION_URL\", \"DARKNET_MENTION_CONTENT\"]\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n if not self.opts['fullnames'] and eventName == 'HUMAN_NAME':\n return\n\n if eventData in self.results:\n self.debug(\"Already did a search for \" + eventData + \", skipping.\")\n return\n\n self.results[eventData] = True\n\n keepGoing = True\n page = 1\n while keepGoing and page <= int(self.opts['max_pages']):\n # Check if we've been asked to stop\n if self.checkForStop():\n return\n\n params = {\n 'search': '\"' + eventData.encode('raw_unicode_escape').decode(\"ascii\", errors='replace') + '\"',\n 'submit': 'Search',\n 'page': str(page)\n }\n\n # Sites hosted on the domain\n data = self.sf.fetchUrl('https://onionsearchengine.com/search.php?' + urllib.parse.urlencode(params),\n useragent=self.opts['_useragent'],\n timeout=self.opts['timeout'])\n\n if data is None or not data.get('content'):\n self.info(\"No results returned from onionsearchengine.com.\")\n return\n\n page += 1\n\n if \"url.php?u=\" not in data['content']:\n # Work around some kind of bug in the site\n if \"you didn't submit a keyword\" in data['content']:\n continue\n return\n\n if \"forward >\" not in data['content']:\n keepGoing = False\n\n links = re.findall(r\"url\\.php\\?u=(.[^\\\"\\']+)[\\\"\\']\",\n data['content'], re.IGNORECASE | re.DOTALL)\n\n for link in links:\n if self.checkForStop():\n return\n\n if link in self.results:\n continue\n\n self.results[link] = True\n\n blacklist = False\n for r in self.opts['blacklist']:\n if re.match(r, link, re.IGNORECASE):\n self.debug(\"Skipping \" + link + \" as it matches blacklist \" + r)\n blacklist = True\n if blacklist:\n continue\n\n self.debug(\"Found a darknet mention: \" + link)\n\n if not self.sf.urlFQDN(link).endswith(\".onion\"):\n continue\n\n if not self.opts['fetchlinks']:\n evt = SpiderFootEvent(\"DARKNET_MENTION_URL\", link, self.__name__, event)\n self.notifyListeners(evt)\n continue\n\n res = self.sf.fetchUrl(link,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent'],\n verify=False)\n\n if res['content'] is None:\n self.debug(\"Ignoring \" + link + \" as no data returned\")\n continue\n\n if eventData not in res['content']:\n self.debug(\"Ignoring \" + link + \" as no mention of \" + eventData)\n continue\n\n evt = SpiderFootEvent(\"DARKNET_MENTION_URL\", link, self.__name__, event)\n self.notifyListeners(evt)\n\n try:\n startIndex = res['content'].index(eventData) - 120\n endIndex = startIndex + len(eventData) + 240\n except Exception:\n self.debug('String \"' + eventData + '\" not found in content.')\n continue\n\n data = res['content'][startIndex:endIndex]\n evt = SpiderFootEvent(\"DARKNET_MENTION_CONTENT\", \"...\" + data + \"...\",\n self.__name__, evt)\n self.notifyListeners(evt)\n\n# End of sfp_onionsearchengine class\n" }, { "path": "modules/sfp_onyphe.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_onyphe\n# Purpose: SpiderFoot plug-in to check if the IP is included on Onyphe\n# data (threat list, geo-location, pastries, vulnerabilities)\n#\n# Author: Filip Aleksić \n#\n# Created: 2020-08-21\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport time\nfrom datetime import datetime\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_onyphe(SpiderFootPlugin):\n\n meta = {\n \"name\": \"Onyphe\",\n \"summary\": \"Check Onyphe data (threat list, geo-location, pastries, vulnerabilities) about a given IP.\",\n 'flags': [\"apikey\"],\n \"useCases\": [\"Footprint\", \"Passive\", \"Investigate\"],\n \"categories\": [\"Reputation Systems\"],\n \"dataSource\": {\n \"website\": \"https://www.onyphe.io\",\n \"model\": \"FREE_AUTH_LIMITED\",\n \"references\": [\"https://www.onyphe.io/documentation/api\"],\n \"apiKeyInstructions\": [\n \"Visit https://www.onyphe.io/login/#register\",\n \"Register a free account\",\n \"You should receive your API key on your email, or you can get it by yourself following next steps\",\n \"Go to your account settings https://www.onyphe.io/auth/account\",\n \"The API key is listed under 'API Key'\",\n ],\n \"favIcon\": \"https://www.onyphe.io/favicon.ico\",\n \"logo\": \"https://www.onyphe.io/img/logo-solo.png\",\n \"description\": \"ONYPHE is a search engine for open-source \"\n \"and cyber threat intelligence data collected by crawling \"\n \"various sources available on the Internet or by listening \"\n \"to Internet background noise. They make this data available \"\n \"through API that we use. We check their data to see following \"\n \"information about the IP: geo-location, does it have some \"\n \"vulnerabilities, is it on some pastries (PasteBin) and \"\n \"is it on their threat list\",\n },\n }\n\n opts = {\n \"api_key\": \"\",\n \"paid_plan\": False,\n \"max_page\": 10,\n \"verify\": True,\n \"age_limit_days\": 30,\n \"cohostsamedomain\": False,\n \"maxcohost\": 100,\n }\n optdescs = {\n \"api_key\": \"Onyphe access token.\",\n \"paid_plan\": \"Are you using paid plan? Paid plan has pagination enabled\",\n \"max_page\": \"Maximum number of pages to iterate through. Onyphe has a maximum of 1000 pages (10,000 results). Only matters for paid plans\",\n \"verify\": \"Verify identified domains still resolve to the associated specified IP address.\",\n \"age_limit_days\": \"Ignore any records older than this many days. 0 = unlimited.\",\n \"maxcohost\": \"Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.\",\n \"cohostsamedomain\": \"Treat co-hosted sites on the same target domain as co-hosting?\",\n }\n\n results = None\n errorState = False\n cohostcount = 0\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"IP_ADDRESS\", \"IPV6_ADDRESS\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\n \"GEOINFO\",\n \"MALICIOUS_IPADDR\",\n \"LEAKSITE_CONTENT\",\n \"VULNERABILITY_CVE_CRITICAL\",\n \"VULNERABILITY_CVE_HIGH\",\n \"VULNERABILITY_CVE_MEDIUM\",\n \"VULNERABILITY_CVE_LOW\",\n \"VULNERABILITY_GENERAL\",\n \"RAW_RIR_DATA\",\n \"INTERNET_NAME\",\n \"INTERNET_NAME_UNRESOLVED\",\n \"PHYSICAL_COORDINATES\",\n ]\n\n def query(self, endpoint, ip, page=1):\n retarr = list()\n\n headers = {\n \"Content-Type\": \"application/json\",\n \"Authorization\": f\"apikey {self.opts['api_key']}\",\n }\n res = self.sf.fetchUrl(\n f\"https://www.onyphe.io/api/v2/simple/{endpoint}/{ip}?page={page}\",\n timeout=self.opts[\"_fetchtimeout\"],\n useragent=self.opts[\"_useragent\"],\n headers=headers,\n )\n\n if res[\"code\"] == \"429\":\n self.error(\"Reaching rate limit on Onyphe API\")\n self.errorState = True\n return None\n\n if res[\"code\"] == 400:\n self.error(\"Invalid request or API key on Onyphe\")\n self.errorState = True\n return None\n\n try:\n info = json.loads(res[\"content\"])\n if \"status\" in info and info[\"status\"] == \"nok\":\n self.error(\n f\"Unexpected error happened while requesting data from Onyphe. Error message: {info.get('text', '')}\"\n )\n self.errorState = True\n return None\n\n if \"results\" not in info or info[\"results\"] == []:\n self.info(f\"No Onyphe {endpoint} data found for {ip}\")\n return None\n except Exception as e:\n self.debug(f\"{e.__class__} {res['code']} {res['content']}\")\n self.error(\"Error processing JSON response from Onyphe.\")\n return None\n\n # Go through other pages if user has paid plan\n try:\n current_page = int(info[\"page\"])\n if (\n self.opts[\"paid_plan\"]\n and info.get(\"page\")\n and int(info.get(\"max_page\")) > current_page\n ):\n page = current_page + 1\n\n if page > self.opts[\"max_page\"]:\n self.error(\n \"Maximum number of pages from options for Onyphe reached.\"\n )\n return [info]\n retarr.append(info)\n response = self.query(endpoint, ip, page)\n if response:\n retarr.extend(response)\n else:\n retarr.append(info)\n\n except ValueError:\n self.error(\n f\"Unexpected value for page in response from Onyphe, url: https://www.onyphe.io/api/v2/simple/{endpoint}/{ip}?page={page}\"\n )\n self.errorState = True\n return None\n\n return retarr\n\n def emitLocationEvent(self, location, eventData, event):\n if location is None:\n return\n self.info(f\"Found location for {eventData}: {location}\")\n\n evt = SpiderFootEvent(\"PHYSICAL_COORDINATES\", location, self.__name__, event)\n self.notifyListeners(evt)\n\n def emitDomainData(self, response, eventData, event):\n domains = set()\n if response.get(\"domain\") is not None and isinstance(\n response['domain'], list\n ):\n for dom in response['domain']:\n domains.add(dom)\n\n if response.get(\"subdomains\") is not None and isinstance(\n response[\"subdomains\"], list\n ):\n for subDomain in response[\"subdomains\"]:\n domains.add(subDomain)\n\n for domain in domains:\n if self.getTarget().matches(domain):\n if self.opts['verify']:\n if self.sf.resolveHost(domain) or self.sf.resolveHost6(domain):\n evt = SpiderFootEvent('INTERNET_NAME', domain, self.__name__, event)\n else:\n evt = SpiderFootEvent('INTERNET_NAME_UNRESOLVED', domain, self.__name__, event)\n self.notifyListeners(evt)\n\n if self.sf.isDomain(domain, self.opts['_internettlds']):\n evt = SpiderFootEvent('DOMAIN_NAME', domain, self.__name__, event)\n self.notifyListeners(evt)\n continue\n\n if self.cohostcount < self.opts['maxcohost']:\n if self.opts[\"verify\"] and not self.sf.validateIP(domain, eventData):\n self.debug(\"Host no longer resolves to our IP.\")\n continue\n\n if not self.opts[\"cohostsamedomain\"] and self.getTarget().matches(domain, includeParents=True):\n self.debug(f\"Skipping {domain} because it is on the same domain.\")\n continue\n\n evt = SpiderFootEvent(\"CO_HOSTED_SITE\", domain, self.__name__, event)\n self.notifyListeners(evt)\n self.cohostcount += 1\n\n def isFreshEnough(self, result):\n limit = self.opts[\"age_limit_days\"]\n if limit <= 0:\n return True\n\n timestamp = result.get(\"@timestamp\")\n if timestamp is None:\n self.debug(\"Record doesn't have timestamp defined\")\n return False\n\n last_dt = datetime.strptime(timestamp, \"%Y-%m-%dT%H:%M:%S.%fZ\")\n last_ts = int(time.mktime(last_dt.timetuple()))\n age_limit_ts = int(time.time()) - (86400 * limit)\n\n if last_ts < age_limit_ts:\n self.debug(\"Record found but too old, skipping.\")\n return False\n\n return True\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n sentData = set()\n\n if self.errorState:\n return\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts[\"api_key\"] == \"\":\n self.error(\"You enabled sfp_onyphe, but did not set an API key!\")\n self.errorState = True\n return\n\n if eventData in self.results:\n self.debug(\"Skipping \" + eventData + \" as already mapped.\")\n return\n\n self.results[eventData] = True\n\n geoLocDataArr = self.query(\"geoloc\", eventData)\n\n if geoLocDataArr is not None:\n evt = SpiderFootEvent(\n \"RAW_RIR_DATA\", str(geoLocDataArr), self.__name__, event\n )\n self.notifyListeners(evt)\n\n for geoLocData in geoLocDataArr:\n if self.checkForStop():\n return\n\n for result in geoLocData[\"results\"]:\n if not self.isFreshEnough(result):\n continue\n\n location = \", \".join(\n [\n _f\n for _f in [\n result.get(\"city\"),\n result.get(\"country\"),\n ]\n if _f\n ]\n )\n self.info(\"Found GeoIP for \" + eventData + \": \" + location)\n\n if location in sentData:\n self.debug(f\"Skipping {location}, already sent\")\n continue\n\n sentData.add(location)\n\n evt = SpiderFootEvent(\"GEOINFO\", location, self.__name__, event)\n self.notifyListeners(evt)\n\n coordinates = result.get(\"location\")\n if coordinates is None:\n continue\n\n if coordinates in sentData:\n self.debug(f\"Skipping {coordinates}, already sent\")\n continue\n sentData.add(coordinates)\n\n self.emitLocationEvent(coordinates, eventData, event)\n\n self.emitDomainData(result, eventData, event)\n\n pastriesDataArr = self.query(\"pastries\", eventData)\n\n if pastriesDataArr is not None:\n evt = SpiderFootEvent(\n \"RAW_RIR_DATA\", str(pastriesDataArr), self.__name__, event\n )\n self.notifyListeners(evt)\n\n for pastriesData in pastriesDataArr:\n if self.checkForStop():\n return\n\n for result in pastriesData[\"results\"]:\n pastry = result.get(\"content\")\n if pastry is None:\n continue\n\n if pastry in sentData:\n self.debug(f\"Skipping {pastry}, already sent\")\n continue\n sentData.add(pastry)\n\n if not self.isFreshEnough(result):\n continue\n\n evt = SpiderFootEvent(\n \"LEAKSITE_CONTENT\", pastry, self.__name__, event\n )\n self.notifyListeners(evt)\n\n threatListDataArr = self.query(\"threatlist\", eventData)\n\n if threatListDataArr is not None:\n evt = SpiderFootEvent(\n \"RAW_RIR_DATA\", str(threatListDataArr), self.__name__, event\n )\n self.notifyListeners(evt)\n\n for threatListData in threatListDataArr:\n if self.checkForStop():\n return\n\n for result in threatListData[\"results\"]:\n threatList = result.get(\"threatlist\")\n\n if threatList is None:\n continue\n\n if threatList in sentData:\n self.debug(f\"Skipping {threatList}, already sent\")\n continue\n sentData.add(threatList)\n\n if not self.isFreshEnough(result):\n continue\n\n evt = SpiderFootEvent(\n \"MALICIOUS_IPADDR\",\n result.get(\"threatlist\"),\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n\n vulnerabilityDataArr = self.query(\"vulnscan\", eventData)\n\n if vulnerabilityDataArr is not None:\n evt = SpiderFootEvent(\n \"RAW_RIR_DATA\", str(vulnerabilityDataArr), self.__name__, event\n )\n self.notifyListeners(evt)\n\n for vulnerabilityData in vulnerabilityDataArr:\n if self.checkForStop():\n return\n\n for result in vulnerabilityData[\"results\"]:\n if not self.isFreshEnough(result):\n continue\n\n cves = result.get(\"cve\")\n\n if cves is None:\n continue\n\n for cve in cves:\n if not cve:\n continue\n\n if cve in sentData:\n continue\n sentData.add(cve)\n\n etype, cvetext = self.sf.cveInfo(cve)\n evt = SpiderFootEvent(\n etype,\n cvetext,\n self.__name__,\n event,\n )\n self.notifyListeners(evt)\n\n# End of sfp_onyphe class\n" }, { "path": "modules/sfp_openbugbounty.py", "content": "# -------------------------------------------------------------------------------\n# Name: sfp_openbugbounty\n# Purpose: Query the Open Bug Bounty database to see if our target appears.\n#\n# Author: Steve Micallef \n#\n# Created: 04/10/2015\n# Copyright: (c) Steve Micallef\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_openbugbounty(SpiderFootPlugin):\n\n meta = {\n 'name': \"Open Bug Bounty\",\n 'summary': \"Check external vulnerability scanning/reporting service openbugbounty.org to see if the target is listed.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Leaks, Dumps and Breaches\"],\n 'dataSource': {\n 'website': \"https://www.openbugbounty.org/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://www.openbugbounty.org/cert/\"\n ],\n 'favIcon': \"https://www.openbugbounty.org/favicon.ico\",\n 'logo': \"https://www.openbugbounty.org/images/design/logo-obbnew.svg\",\n 'description': \"Open Bug Bounty is an open, disintermediated, cost-free, and community-driven bug bounty platform \"\n \"for coordinated, responsible and ISO 29147 compatible vulnerability disclosure.\\n\"\n \"The role of Open Bug Bounty is limited to independent verification of the \"\n \"submitted vulnerabilities and proper notification of website owners by all available means. \"\n \"Once notified, the website owner and the researcher are in direct contact to \"\n \"remediate the vulnerability and coordinate its disclosure. \"\n \"At this and at any later stages, we never act as an intermediary between \"\n \"website owners and security researchers.\",\n }\n }\n # Default options\n opts = {\n }\n\n # Option descriptions\n optdescs = {\n }\n\n # Be sure to completely clear any class variables in setup()\n # or you run the risk of data persisting between scan runs.\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n # Clear / reset any other class member variables here\n # or you risk them persisting between threads.\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return [\"INTERNET_NAME\"]\n\n # What events this module produces\n def producedEvents(self):\n return [\"VULNERABILITY_DISCLOSURE\"]\n\n # Query XSSposed.org\n def queryOBB(self, qry):\n ret = list()\n base = \"https://www.openbugbounty.org\"\n url = \"https://www.openbugbounty.org/search/?search=\" + qry\n res = self.sf.fetchUrl(url, timeout=30, useragent=self.opts['_useragent'])\n\n if res['content'] is None:\n self.debug(\"No content returned from openbugbounty.org\")\n return None\n\n try:\n rx = re.compile(\".*
(.*\" + qry + \").*?
.*\", re.IGNORECASE)\n for m in rx.findall(str(res['content'])):\n # Report it\n if m[1] == qry or m[1].endswith(\".\" + qry):\n ret.append(\"From openbugbounty.org: \" + base + m[0] + \"\")\n except Exception as e:\n self.error(\"Error processing response from openbugbounty.org: \" + str(e))\n return None\n return ret\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n data = list()\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n obb = self.queryOBB(eventData)\n if obb:\n data.extend(obb)\n\n for n in data:\n # Notify other modules of what you've found\n e = SpiderFootEvent(\"VULNERABILITY_DISCLOSURE\", n, self.__name__, event)\n self.notifyListeners(e)\n\n# End of sfp_openbugbounty class\n" }, { "path": "modules/sfp_opencorporates.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_opencorporates\n# Purpose: SpiderFoot plug-in for retrieving company information from\n# OpenCorporates.\n#\n# Author: \n#\n# Created: 2018-10-21\n# Copyright: (c) bcoles 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport urllib\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_opencorporates(SpiderFootPlugin):\n\n meta = {\n 'name': \"OpenCorporates\",\n 'summary': \"Look up company information from OpenCorporates.\",\n 'flags': [\"apikey\"],\n 'useCases': [\"Passive\", \"Footprint\", \"Investigate\"],\n 'categories': [\"Search Engines\"],\n 'dataSource': {\n 'website': \"https://opencorporates.com\",\n 'model': \"FREE_NOAUTH_LIMITED\",\n 'references': [\n \"https://api.opencorporates.com/documentation/API-Reference\"\n ],\n 'apiKeyInstructions': [\n \"Visit https://opencorporates.com/api_accounts/new\"\n \"Register a new account with an email\",\n \"Navigate to https://opencorporates.com/users/account and select 'Get Account'\",\n \"Select the plan required\",\n \"Navigate to https://opencorporates.com/users/account\",\n \"The API key is listed under 'API Account'\",\n ],\n 'favIcon': \"https://opencorporates.com/assets/favicons/favicon.png\",\n 'logo': \"https://opencorporates.com/contents/ui/theme/img/oc-logo.svg\",\n 'description': \"The largest open database of companies in the world.\\n\"\n \"As the largest, open database of companies in the world, \"\n \"our business is making high-quality, official company data openly available. \"\n \"Data that can be trusted, accessed, analysed and interrogated when and how it’s needed.\",\n }\n }\n\n opts = {\n 'confidence': 100,\n 'api_key': ''\n }\n\n optdescs = {\n 'confidence': \"Confidence that the search result objects are correct (numeric value between 0 and 100).\",\n 'api_key': 'OpenCorporates.com API key. Without this you will be limited to 50 look-ups per day.'\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\"COMPANY_NAME\"]\n\n def producedEvents(self):\n return [\"COMPANY_NAME\", \"PHYSICAL_ADDRESS\", \"RAW_RIR_DATA\"]\n\n def searchCompany(self, qry):\n \"\"\"Search for company name\n\n Args:\n qry (str): company name\n\n Returns:\n str\n \"\"\"\n\n version = '0.4'\n\n apiparam = \"\"\n if self.opts['api_key']:\n apiparam = \"&api_token=\" + self.opts['api_key']\n\n params = urllib.parse.urlencode({\n 'q': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n 'format': 'json',\n 'order': 'score',\n 'confidence': self.opts['confidence']\n })\n\n res = self.sf.fetchUrl(\n f\"https://api.opencorporates.com/v{version}/companies/search?{params}{apiparam}\",\n timeout=60, # High timeouts as they can sometimes take a while\n useragent=self.opts['_useragent']\n )\n\n if res['code'] == \"401\":\n self.error(\"Invalid OpenCorporates API key.\")\n return None\n\n if res['code'] == \"403\":\n self.error(\"You are being rate-limited by OpenCorporates.\")\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n if 'results' not in data:\n return None\n\n return data['results']\n\n def retrieveCompanyDetails(self, jurisdiction_code, company_number):\n url = f\"https://api.opencorporates.com/companies/{jurisdiction_code}/{company_number}\"\n\n if self.opts['api_key']:\n url += \"?api_token=\" + self.opts['api_key']\n\n res = self.sf.fetchUrl(\n url,\n timeout=self.opts['_fetchtimeout'],\n useragent=self.opts['_useragent']\n )\n\n if res['code'] == \"401\":\n self.error(\"Invalid OpenCorporates API key.\")\n return None\n\n if res['code'] == \"403\":\n self.error(\"You are being rate-limited by OpenCorporates.\")\n return None\n\n try:\n data = json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n return None\n\n if 'results' not in data:\n return None\n\n return data['results']\n\n # Extract company address, previous names, and officer names\n def extractCompanyDetails(self, company, sevt):\n\n # Extract registered address\n location = company.get('registered_address_in_full')\n\n if location:\n if len(location) < 3 or len(location) > 100:\n self.debug(\"Skipping likely invalid location.\")\n else:\n if company.get('registered_address'):\n country = company.get('registered_address').get('country')\n if country:\n if not location.endswith(country):\n location += \", \" + country\n\n location = location.replace(\"\\n\", ',')\n self.info(\"Found company address: \" + location)\n e = SpiderFootEvent(\"PHYSICAL_ADDRESS\", location, self.__name__, sevt)\n self.notifyListeners(e)\n\n # Extract previous company names\n previous_names = company.get('previous_names')\n\n if previous_names:\n for previous_name in previous_names:\n p = previous_name.get('company_name')\n if p:\n self.info(\"Found previous company name: \" + p)\n e = SpiderFootEvent(\"COMPANY_NAME\", p, self.__name__, sevt)\n self.notifyListeners(e)\n\n # Extract officer names\n officers = company.get('officers')\n\n if officers:\n for officer in officers:\n n = officer.get('name')\n if n:\n self.info(\"Found company officer: \" + n)\n e = SpiderFootEvent(\"RAW_RIR_DATA\", \"Possible full name: \" + n, self.__name__, sevt)\n self.notifyListeners(e)\n\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if self.opts['api_key'] == '':\n self.error(f\"Warning: You enabled {self.__class__.__name__} but did not set an API key! Queries will be limited to 50 per day and 200 per month.\")\n\n res = self.searchCompany(f\"{eventData}*\")\n\n if res is None:\n self.debug(\"Found no results for \" + eventData)\n return\n\n companies = res.get('companies')\n\n if not companies:\n self.debug(\"Found no results for \" + eventData)\n return\n\n for c in companies:\n company = c.get('company')\n\n if not company:\n continue\n\n # Check for match\n if eventData.lower() != company.get('name').lower():\n continue\n\n # Extract company details from search results\n self.extractCompanyDetails(company, event)\n\n # Retrieve further details\n jurisdiction_code = company.get('jurisdiction_code')\n company_number = company.get('company_number')\n\n if not company_number or not jurisdiction_code:\n continue\n\n res = self.retrieveCompanyDetails(jurisdiction_code, company_number)\n\n if not res:\n continue\n\n c = res.get('company')\n\n if not c:\n continue\n\n self.extractCompanyDetails(c, event)\n\n# End of sfp_opencorporates class\n" }, { "path": "modules/sfp_opendns.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_opendns\n# Purpose: SpiderFoot plug-in for looking up whether hosts are blocked by\n# OpenDNS.\n#\n# Author: Steve Micallef \n#\n# Created: 30/05/2018\n# Copyright: (c) Steve Micallef 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport dns.resolver\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_opendns(SpiderFootPlugin):\n\n meta = {\n 'name': \"OpenDNS\",\n 'summary': \"Check if a host would be blocked by OpenDNS.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://www.opendns.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://www.opendns.com/setupguide/?url=familyshield\",\n \"https://support.opendns.com/hc/en-us/categories/204012807-OpenDNS-Knowledge-Base\",\n \"https://support.opendns.com/hc/en-us/categories/204012907-OpenDNS-Device-Configuration\"\n ],\n 'favIcon': \"https://www.google.com/s2/favicons?domain=https://www.opendns.com/\",\n 'logo': \"https://d15ni2z53ptwz9.cloudfront.net/opendns-www/img/logo-opendns.png\",\n 'description': \"Cisco Umbrella provides protection against threats on the internet such as \"\n \"malware, phishing, and ransomware.\\n\"\n \"OpenDNS is a suite of consumer products aimed at making your internet faster, safer, and more reliable. \"\n \"FamilyShield is the single easiest way to protect your kids online, block adult websites, \"\n \"and protect your family from phishing and malware.\",\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n checks = {\n \"146.112.61.105\": \"OpenDNS - Botnet\",\n \"146.112.61.106\": \"OpenDNS - Adult\",\n \"146.112.61.107\": \"OpenDNS - Malware\",\n \"146.112.61.108\": \"OpenDNS - Phishing\",\n \"146.112.61.109\": \"OpenDNS - Blocked\",\n \"146.112.61.110\": \"OpenDNS - Malware\",\n }\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"AFFILIATE_INTERNET_NAME\",\n \"CO_HOSTED_SITE\"\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_INTERNET_NAME\",\n \"BLACKLISTED_AFFILIATE_INTERNET_NAME\",\n \"BLACKLISTED_COHOST\",\n \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_COHOST\",\n ]\n\n def queryAddr(self, qaddr):\n if not qaddr:\n return None\n\n res = dns.resolver.Resolver()\n res.nameservers = [\"208.67.222.123\", \"208.67.220.123\"]\n\n try:\n return res.resolve(qaddr)\n except Exception:\n self.debug(f\"Unable to resolve {qaddr}\")\n\n return None\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if eventName == \"INTERNET_NAME\":\n malicious_type = \"MALICIOUS_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n malicious_type = \"MALICIOUS_AFFILIATE_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n malicious_type = \"MALICIOUS_COHOST\"\n blacklist_type = \"BLACKLISTED_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n res = self.queryAddr(eventData)\n\n if not res:\n return\n\n self.debug(f\"{eventData} found in OpenDNS Blocklist: {res}\")\n\n for result in res:\n k = str(result)\n if k not in self.checks:\n continue\n\n evt = SpiderFootEvent(blacklist_type, f\"{self.checks[k]} [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n if k in ['146.112.61.105', '146.112.61.107', '146.112.61.108', '146.112.61.110']:\n evt = SpiderFootEvent(malicious_type, f\"{self.checks[k]} [{eventData}]\", self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_opendns class\n" }, { "path": "modules/sfp_opennic.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_opnenic\n# Purpose: SpiderFoot plug-in for resolving host names on the OpenNIC\n# alternative DNS system.\n#\n# Author: \n#\n# Created: 2021-10-16\n# Copyright: (c) bcoles 2021\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport dns.resolver\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_opennic(SpiderFootPlugin):\n\n meta = {\n 'name': \"OpenNIC DNS\",\n 'summary': \"Resolves host names in the OpenNIC alternative DNS system.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Footprint\", \"Passive\"],\n 'categories': [\"DNS\"],\n 'dataSource': {\n 'website': \"https://www.opennic.org/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://wiki.opennic.org/\",\n \"https://servers.opennic.org\",\n ],\n 'description': \"An organization of hobbyists who run an alternative DNS network, \"\n \"also provides access to domains not administered by ICANN.\"\n }\n }\n\n opts = {\n 'checkaffiliates': True,\n }\n\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliates?\",\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"INTERNET_NAME_UNRESOLVED\",\n \"AFFILIATE_INTERNET_NAME\",\n \"AFFILIATE_INTERNET_NAME_UNRESOLVED\",\n ]\n\n def producedEvents(self):\n return [\n \"IP_ADDRESS\",\n \"IPV6_ADDRESS\",\n \"AFFILIATE_IPADDR\",\n \"AFFILIATE_IPV6_ADDRESS\",\n ]\n\n def queryOpenNIC(self, qaddr):\n res = dns.resolver.Resolver()\n # https://servers.opennicproject.org/\n res.nameservers = [\n \"192.3.165.37\",\n \"35.211.96.150\",\n \"51.89.88.77\",\n \"94.247.43.254\",\n \"138.197.140.189\"\n ]\n\n try:\n return res.resolve(qaddr)\n except Exception:\n self.debug(f\"Unable to resolve {qaddr}\")\n\n return None\n\n def tlds(self):\n \"\"\"Valid OpenNIC top-level domains.\n\n Returns:\n list: OpenNIC TLDs (and peer TLDs).\n \"\"\"\n return [\n 'bbs',\n 'chan',\n 'cyb',\n 'dyn',\n 'epic',\n 'free',\n 'geek',\n 'glue',\n 'gopher',\n 'indy',\n 'libre',\n 'neo',\n 'null',\n 'o',\n 'oss',\n 'oz',\n 'parody',\n 'pirate',\n\n # Peers\n 'bazar',\n 'bit',\n 'coin',\n 'emc',\n 'fur',\n 'ku',\n 'lib',\n 'te',\n 'ti',\n 'uu',\n ]\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n return\n\n self.results[eventData] = True\n\n if eventData.split('.')[-1] not in self.tlds():\n return\n\n affiliate = False\n\n if \"AFFILIATE\" in eventName:\n if not self.opts.get('checkaffiliates', False):\n return\n affiliate = True\n\n addrs = self.sf.normalizeDNS(self.queryOpenNIC(eventData))\n\n if not addrs:\n return\n\n self.debug(f\"OpenNIC resolved {eventData} to addresses: {addrs}\")\n\n for addr in set(addrs):\n if self.sf.validIP(addr):\n if affiliate and not self.getTarget().matches(addr, includeParents=True):\n evt = SpiderFootEvent(\"AFFILIATE_IPADDR\", addr, self.__name__, event)\n else:\n evt = SpiderFootEvent(\"IP_ADDRESS\", addr, self.__name__, event)\n\n self.notifyListeners(evt)\n elif self.sf.validIP6(addr):\n if affiliate and not self.getTarget().matches(addr, includeParents=True):\n evt = SpiderFootEvent(\"AFFILIATE_IPV6_ADDRESS\", addr, self.__name__, event)\n else:\n evt = SpiderFootEvent(\"IPV6_ADDRESS\", addr, self.__name__, event)\n\n self.notifyListeners(evt)\n else:\n continue\n\n# End of sfp_opennic class\n" }, { "path": "modules/sfp_openphish.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_openphish\n# Purpose: Check if a host/domain is malicious according to OpenPhish.com.\n#\n# Author: steve@binarypool.com\n#\n# Created: 28/06/2018\n# Copyright: (c) Steve Micallef, 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_openphish(SpiderFootPlugin):\n\n meta = {\n 'name': \"OpenPhish\",\n 'summary': \"Check if a host/domain is malicious according to OpenPhish.com.\",\n 'flags': [],\n 'useCases': [\"Investigate\", \"Passive\"],\n 'categories': [\"Reputation Systems\"],\n 'dataSource': {\n 'website': \"https://openphish.com/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://openphish.com/faq.html\",\n \"https://openphish.com/feed.txt\"\n ],\n 'favIcon': \"\",\n 'logo': \"https://openphish.com/static/openphish_logo2.png\",\n 'description': \"Timely. Accurate. Relevant Threat Intelligence.\\n\"\n \"OpenPhish is a fully automated self-contained platform for phishing intelligence. \"\n \"It identifies phishing sites and performs intelligence analysis \"\"in real time \"\n \"without human intervention and without using any external resources, such as blacklists.\",\n }\n }\n\n opts = {\n 'checkaffiliates': True,\n 'checkcohosts': True,\n 'cacheperiod': 18\n }\n\n optdescs = {\n 'checkaffiliates': \"Apply checks to affiliates?\",\n 'checkcohosts': \"Apply checks to sites found to be co-hosted on the target's IP?\",\n 'cacheperiod': \"Hours to cache list data before re-fetching.\"\n }\n\n results = None\n errorState = False\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n self.errorState = False\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n def watchedEvents(self):\n return [\n \"INTERNET_NAME\",\n \"AFFILIATE_INTERNET_NAME\",\n \"CO_HOSTED_SITE\",\n ]\n\n def producedEvents(self):\n return [\n \"BLACKLISTED_INTERNET_NAME\",\n \"BLACKLISTED_AFFILIATE_INTERNET_NAME\",\n \"BLACKLISTED_COHOST\",\n \"MALICIOUS_INTERNET_NAME\",\n \"MALICIOUS_AFFILIATE_INTERNET_NAME\",\n \"MALICIOUS_COHOST\",\n ]\n\n def queryBlacklist(self, target):\n blacklist = self.retrieveBlacklist()\n\n if not blacklist:\n return False\n\n if target.lower() in blacklist:\n self.debug(f\"Host name {target} found in OpenPhish blacklist.\")\n return True\n\n return False\n\n def retrieveBlacklist(self):\n blacklist = self.sf.cacheGet('openphish', 24)\n\n if blacklist is not None:\n return self.parseBlacklist(blacklist)\n\n res = self.sf.fetchUrl(\n \"https://www.openphish.com/feed.txt\",\n timeout=10,\n useragent=self.opts['_useragent'],\n )\n\n if res['code'] != \"200\":\n self.error(f\"Unexpected HTTP response code {res['code']} from OpenPhish.\")\n self.errorState = True\n return None\n\n if res['content'] is None:\n self.error(\"Received no content from OpenPhish\")\n self.errorState = True\n return None\n\n self.sf.cachePut(\"openphish\", res['content'])\n\n return self.parseBlacklist(res['content'])\n\n def parseBlacklist(self, blacklist):\n \"\"\"Parse plaintext blacklist\n\n Args:\n blacklist (str): plaintext blacklist from OpenPhish\n\n Returns:\n list: list of blacklisted host names\n \"\"\"\n hosts = list()\n\n if not blacklist:\n return hosts\n\n for line in blacklist.split('\\n'):\n if not line:\n continue\n if not line.startswith('http'):\n continue\n\n # Note: URL parsing and validation with sf.validHost() is too slow to use here\n url = line.strip().lower()\n if len(url.split(\"/\")) < 3:\n continue\n host = url.split(\"/\")[2]\n if not host:\n continue\n if \".\" not in host:\n continue\n hosts.append(host)\n\n return hosts\n\n def handleEvent(self, event):\n eventName = event.eventType\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {event.module}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n if self.errorState:\n return\n\n self.results[eventData] = True\n\n if eventName == \"INTERNET_NAME\":\n malicious_type = \"MALICIOUS_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_INTERNET_NAME\"\n elif eventName == \"AFFILIATE_INTERNET_NAME\":\n if not self.opts.get('checkaffiliates', False):\n return\n malicious_type = \"MALICIOUS_AFFILIATE_INTERNET_NAME\"\n blacklist_type = \"BLACKLISTED_AFFILIATE_INTERNET_NAME\"\n elif eventName == \"CO_HOSTED_SITE\":\n if not self.opts.get('checkcohosts', False):\n return\n malicious_type = \"MALICIOUS_COHOST\"\n blacklist_type = \"BLACKLISTED_COHOST\"\n else:\n self.debug(f\"Unexpected event type {eventName}, skipping\")\n return\n\n self.debug(f\"Checking maliciousness of {eventData} ({eventName}) with OpenPhish\")\n\n if not self.queryBlacklist(eventData):\n return\n\n url = \"https://www.openphish.com/feed.txt\"\n text = f\"OpenPhish [{eventData}]\\n{url}\"\n\n evt = SpiderFootEvent(malicious_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n evt = SpiderFootEvent(blacklist_type, text, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_openphish class\n" }, { "path": "modules/sfp_openstreetmap.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_openstreetmap\n# Purpose: SpiderFoot plug-in to retrieve latitude/longitude coordinates\n# for physical addresses from OpenStreetMap API.\n#\n# Author: \n#\n# Created: 2018-10-27\n# Copyright: (c) bcoles 2018\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport json\nimport re\nimport time\nimport urllib.error\nimport urllib.parse\nimport urllib.request\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n\nclass sfp_openstreetmap(SpiderFootPlugin):\n\n meta = {\n 'name': \"OpenStreetMap\",\n 'summary': \"Retrieves latitude/longitude coordinates for physical addresses from OpenStreetMap API.\",\n 'flags': [],\n 'useCases': [\"Footprint\", \"Investigate\", \"Passive\"],\n 'categories': [\"Real World\"],\n 'dataSource': {\n 'website': \"https://www.openstreetmap.org/\",\n 'model': \"FREE_NOAUTH_UNLIMITED\",\n 'references': [\n \"https://wiki.openstreetmap.org/wiki/API\",\n \"https://wiki.openstreetmap.org/wiki/API_v0.6\"\n ],\n 'favIcon': \"https://www.openstreetmap.org/assets/osm_logo-b7061f13a03615f787a7e0e56a0db5252eb2a217ab063183e78526a8cc10989b.svg\",\n 'logo': \"https://www.openstreetmap.org/assets/osm_logo-b7061f13a03615f787a7e0e56a0db5252eb2a217ab063183e78526a8cc10989b.svg\",\n 'description': \"OpenStreetMap powers map data on thousands of web sites, mobile apps, and hardware devices.\",\n }\n }\n\n opts = {\n }\n\n optdescs = {\n }\n\n results = None\n\n def setup(self, sfc, userOpts=dict()):\n self.sf = sfc\n self.results = self.tempStorage()\n\n for opt in list(userOpts.keys()):\n self.opts[opt] = userOpts[opt]\n\n # What events is this module interested in for input\n def watchedEvents(self):\n return ['PHYSICAL_ADDRESS']\n\n # What events this module produces\n def producedEvents(self):\n return ['PHYSICAL_COORDINATES']\n\n # Search for address\n # https://operations.osmfoundation.org/policies/nominatim/\n def query(self, qry):\n params = {\n 'q': qry.encode('raw_unicode_escape').decode(\"ascii\", errors='replace'),\n 'format': 'json',\n 'polygon': '0',\n 'addressdetails': '0'\n }\n\n res = self.sf.fetchUrl(\"https://nominatim.openstreetmap.org/search?\" + urllib.parse.urlencode(params),\n timeout=self.opts['_fetchtimeout'], useragent='SpiderFoot')\n\n if res['content'] is None:\n self.info(\"No location info found for \" + qry)\n return None\n\n try:\n return json.loads(res['content'])\n except Exception as e:\n self.debug(f\"Error processing JSON response: {e}\")\n\n return None\n\n # Handle events sent to this module\n def handleEvent(self, event):\n eventName = event.eventType\n srcModuleName = event.module\n eventData = event.data\n\n self.debug(f\"Received event, {eventName}, from {srcModuleName}\")\n\n if eventData in self.results:\n self.debug(f\"Skipping {eventData}, already checked.\")\n return\n\n self.results[eventData] = True\n\n address = eventData\n\n # Skip post office boxes\n if address.lower().startswith('po box'):\n self.debug(\"Skipping PO BOX address\")\n return\n\n rx1 = re.compile(r'^(c/o|care of|attn:|attention:)\\s+[0-9a-z\\s\\.]', flags=re.IGNORECASE)\n # Remove address prefixes for delivery instructions\n address = re.sub(rx1, r'', address)\n\n rx2 = re.compile(r'^(Level|Floor|Suite|Room)\\s+[0-9a-z]+,', flags=re.IGNORECASE)\n # Remove address prefixes known to return no results (floor, level, suite, etc).\n address = re.sub(rx2, r'', address)\n\n # Search for address\n data = self.query(eventData)\n\n # Usage Policy mandates no more than 1 request per second\n time.sleep(1)\n\n if data is None:\n self.debug(\"Found no results for \" + eventData)\n return\n\n self.info(\"Found \" + str(len(data)) + \" matches for \" + eventData)\n\n for location in data:\n try:\n lat = location.get('lat')\n lon = location.get('lon')\n except Exception as e:\n self.debug(\"Failed to get lat/lon: \" + str(e))\n continue\n\n if not lat or not lon:\n continue\n\n coords = str(lat) + \",\" + str(lon)\n self.debug(\"Found coordinates: \" + coords)\n\n evt = SpiderFootEvent(\"PHYSICAL_COORDINATES\", coords, self.__name__, event)\n self.notifyListeners(evt)\n\n# End of sfp_openstreetmap class\n" }, { "path": "modules/sfp_pageinfo.py", "content": "# -*- coding: utf-8 -*-\n# -------------------------------------------------------------------------------\n# Name: sfp_pageinfo\n# Purpose: SpiderFoot plug-in for scanning retrieved content by other\n# modules (such as sfp_spider) and building up information about\n# the page, such as whether it uses Javascript, has forms, and more.\n#\n# Author: Steve Micallef \n#\n# Created: 02/05/2012\n# Copyright: (c) Steve Micallef 2012\n# Licence: MIT\n# -------------------------------------------------------------------------------\n\nimport re\n\nfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin\n\n# Indentify pages that use Javascript libs, handle passwords, have forms,\n# permit file uploads and more to come.\nregexps = dict({\n 'URL_JAVASCRIPT': list(['text/javascript', '\n \n\n \n \n \n \n\n \n \n\n \n\n\n\n
\n
\n
\n\n
\n

SpiderFoot ${version}

\n
\n\n
\n

SpiderFoot is an open source, MIT-licensed footprinting tool, created by Steve Micallef. It is designed to be easy to use, fast and extensible.

\n

If you have any bugs to report, or requests for enhancements, please contact the support mailing list at support@spiderfoot.net.\n

You are running version ${version}.

\n

Visit the project website at https://www.spiderfoot.net/.

\n

View the documentation at https://www.spiderfoot.net/documentation/.

\n

Check out the Github repo at https://github.com/smicallef/spiderfoot.

\n

Latest updates @ Twitter https://twitter.com/spiderfoot.

\n

Join the community in our Discord Server.

\n

Want something cloud-based with more capabilities? Check out SpiderFoot HX.

\n
\n\n
\n \n
\n\n
\n
\n
\n\n \n \n \n \n \n \n \n \n \n \n\n
\n\n" }, { "path": "spiderfoot/templates/error.tmpl", "content": "<%include file=\"HEADER.tmpl\"/>\n
\n

${message}

\n

\nPlease go back and try again. If you're pretty sure this is a bug, please contact the support mailing list at support@spiderfoot.net.\n
\n<%include file=\"FOOTER.tmpl\"/>\n" }, { "path": "spiderfoot/templates/newscan.tmpl", "content": "<%include file=\"HEADER.tmpl\"/>\n\n

New Scan

\n
\n
\n
\n \n
\n \n
\n \n
\n \n
\n
\n
\n
\n
\n
\n
\n   Your scan target may be one of the following. SpiderFoot will automatically detect the target type based on the format of your input:\n
\n
\n
\n
\n Domain Name: e.g. example.com
IPv4 Address: e.g. 1.2.3.4
IPv6 Address: e.g. 2606:4700:4700::1111
Hostname/Sub-domain: e.g. abc.example.com
Subnet: e.g. 1.2.3.0/24
Bitcoin Address: e.g. 1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R\n
\n
E-mail address: e.g. bob@example.com
Phone Number: e.g. +12345678901 (E.164 format)
Human Name: e.g. "John Smith" (must be in quotes)
Username: e.g. "jsmith2000" (must be in quotes)
Network ASN: e.g. 1234\n
\n
\n
\n
\n
\n\n
\n \n \n \n \n \n \n
AllGet anything and everything about the target.

All SpiderFoot modules will be enabled (slow) but every possible piece of information about the target will be obtained and analysed.

FootprintUnderstand what information this target exposes to the Internet.

Gain an understanding about the target's network perimeter, associated identities and other information that is obtained through a lot of web crawling and search engine use.\n

InvestigateBest for when you suspect the target to be malicious but need more information.

Some basic footprinting will be performed in addition to querying of blacklists and other sources that may have information about your target's maliciousness.\n

PassiveWhen you don't want the target to even suspect they are being investigated.

As much information will be gathered without touching the target or their affiliates, therefore only modules that do not touch the target will be enabled.\n\n
\n\n \n <%\n modlist = dict()\n for item in modules:\n modlist[modules[item]['name']] = item\n %>\n % for it in sorted(modlist, key=lambda v: v.upper()):\n <% item = modlist[it] %>\n <% keylist = dict((k, v) for k, v in modules[item]['opts'].items() if not k.startswith('_')) %>\n <% keyicon = \"\" %>\n % if len(keylist) > 0:\n <% apikeylist = dict((k, v) for k, v in modules[item]['opts'].items() if k.find(\"api_key\") >= 0) %>\n <%\n if len(apikeylist) > 0:\n keyicon = \"  \"\n %>\n % endif\n\n % if item != \"sfp__stor_db\" and item != \"sfp__stor_stdout\":\n \n % endif\n % endfor\n
${modules[item]['name']}${keyicon}${modules[item]['descr']}
\n \n <% count = 0 %>\n % for item in sorted(types):\n % if count % 2 == 0:\n \n % endif\n \n % if not count % 2 == 0:\n \n % endif\n <% count = count + 1 %>\n % endfor\n
${item[0]}
\n
\n
\n
\n \n \n \n
\n
\n
\n
\n\n\n<%include file=\"FOOTER.tmpl\"/>\n" }, { "path": "spiderfoot/templates/opts.tmpl", "content": "<%include file=\"HEADER.tmpl\"/>\n \n

Settings

\n% if updated:\n
\n \n

Success!

\n Settings updated. These will take effect the next time you run a scan.\n
\n% endif\n
\n
\n \n \n \n \n \n \n \n

\n
\n\n
\n
    \n
  • Global
    • \n% for mod in sorted(opts['__modules__'].keys()):\n <% keylist = dict((k, v) for k, v in opts['__modules__'][mod]['opts'].items() if not k.startswith('_')) %>\n % if len(keylist) > 0:\n <% keyicon = \"\" %>\n <% apikeylist = dict((k, v) for k, v in opts['__modules__'][mod]['opts'].items() if k.find(\"api_key\") >= 0) %>\n <% \n if len(apikeylist) > 0: \n keyicon = \"  \"\n %>\n
  • ${opts['__modules__'][mod]['name']}${keyicon}
    • \n % endif\n% endfor\n
\n
\n\n
\n\n
\n

Global Settings

\n\n\n% for opt in sorted(opts.keys()):\n % if not opt.startswith(\"__\"):\n \n % endif\n% endfor\n\n
Option Value
${opts['__globaloptdescs__'].get(opt) or \"No description available\"}\n % if type(opts[opt]) is int:\n
\n \n
\n % endif\n % if type(opts[opt]) is str:\n
\n \n
\n % endif\n % if type(opts[opt]) is bool:\n <%\n if opts[opt] == True:\n seltrue = \"selected\"\n selfalse = \"\"\n else:\n selfalse = \"selected\" \n seltrue = \"\"\n %>\n \n % endif\n % if type(opts[opt]) is list and type(opts[opt][0]) is str:\n <% expandedopt = ','.join(opts[opt]) %>\n
\n \n
\n % endif\n % if type(opts[opt]) is list and type(opts[opt][0]) is int:\n <% expandedopt = ','.join(str(x) for x in opts[opt]) %>\n
\n \n
\n % endif\n
\n
\n% for mod in sorted(opts['__modules__'].keys()):\n <%\n summary = opts['__modules__'][mod].get('descr') or 'No summary available.'\n categories = opts['__modules__'][mod].get('cats') or []\n labels = opts['__modules__'][mod].get('labels') or []\n meta = opts['__modules__'][mod].get('meta')\n modopts = opts['__modules__'][mod].get('opts')\n optdescs = opts['__modules__'][mod].get('optdescs')\n data_source = meta.get('dataSource') or {}\n website = data_source.get('website') or ''\n description = data_source.get('description') or ''\n apiKeyInstructions = data_source.get('apiKeyInstructions') or ''\n %>\n
\n

${opts['__modules__'][mod]['name']} (${mod})

\n\n \n \n \n\n % if categories:\n \n % endif\n\n % if labels:\n \n % endif\n\n % if website:\n \n % endif\n\n \n
Summary${summary}

${description}
Categories:${', '.join(categories)}
Tags:${', '.join(labels)}
Website:${website}
\n\n

Settings

\n \n \n \n\n % for opt in sorted(modopts.keys()):\n % if not opt.startswith(\"_\"):\n \n \n \n % endif\n % endfor\n \n
Option Value
${optdescs.get(opt) or \"No description available.\"}\n % if \"api_key\" in opt:\n % if apiKeyInstructions:\n <%\n import re\n instructions = \"\"\n for step in apiKeyInstructions:\n if \"https://\" in step or \"http://\" in step:\n step = re.sub(r'(https?://.[^ ]+)', '\\\\1', step)\n instructions += f\"
  • {step}
    • \"\n %>\n   ${instructions}\">\n % endif\n % endif\n     		\n % if type(modopts[opt]) is int:\n
    \n \n
    \n % endif\n % if type(modopts[opt]) is str:\n
    \n \n
    \n % endif\n % if type(modopts[opt]) is bool:\n <%\n if modopts[opt] == True:\n seltrue = \"selected\"\n selfalse = \"\"\n else:\n selfalse = \"selected\"\n seltrue = \"\"\n %>\n \n % endif\n % if type(modopts[opt]) is list and type(modopts[opt][0]) is str:\n <% expandedopt = ','.join(modopts[opt]) %>\n
    \n \n
    \n % endif\n % if type(modopts[opt]) is list and type(modopts[opt][0]) is int:\n <% expandedopt = ','.join(str(x) for x in modopts[opt]) %>\n
    \n \n
    \n % endif\n\n
    \n
    \n% endfor\n
    \n
    \n\n

    \n<%include file=\"FOOTER.tmpl\"/>\n" }, { "path": "spiderfoot/templates/scaninfo.tmpl", "content": "## -*- coding: utf-8 -*-\n<%include file=\"HEADER.tmpl\"/>\n<%\n if status == \"FINISHED\":\n statusy = \"alert-success\"\n elif status.startswith(\"ABORT\"):\n statusy = \"alert-warning\"\n elif status == \"CREATED\" or status == \"RUNNING\" or status == \"STARTED\" or status == \"STARTING\" or status == \"INITIALIZING\":\n statusy = \"alert-info\"\n elif status == \"ERROR-FAILED\":\n statusy = \"alert-danger\"\n else:\n statusy = \"alert-info\"\n end\n%>\n

    ${name} ${status} 

    \n
    \n
    \n  Summary\n  Correlations\n  Browse\n  Graph\n  Scan Settings\n  Log\n
    \n
    \n \n
    \n
    \n \n \n \n \n
    \n \n
    \n R\n F\n \n
    \n \n
    \n \n \n \n \n \n \n\n<%include file=\"FOOTER.tmpl\"/>\n" }, { "path": "spiderfoot/templates/scanlist.tmpl", "content": "<%include file=\"HEADER.tmpl\"/>\n

    Scans  

    \n
    \n% if newscan:\n
    \n \n

    Success!

    \n Scan '${newscan}' has successfully been initiated. Reload this page for up-to-date status on the scan.\n
    \n% endif\n% if rerunscans:\n
    \n \n

    Success!

    \n Scans have successfully been re-initiated as separate instances. Reload this page for up-to-date status on the scans.\n
    \n% endif\n% if stoppedscan and len(errors) == 0:\n
    \n \n

    Scan aborted.

    \n Please allow a minute or two for the scan to cleanly shut itself down.\n
    \n% endif\n\n% if stoppedscan and len(errors) > 0:\n
    \n \n

    Error

    \n Some or all of the scans could not be aborted:
    \n
      \n % for err in errors:\n
    • ${err}
      • \n % endfor\n
    \n
    \n\n% endif\n\n\n
    \n\n\n\n<%include file=\"FOOTER.tmpl\"/>\n" }, { "path": "spiderfoot/threadpool.py", "content": "import queue\nimport logging\nimport threading\nfrom time import sleep\nfrom contextlib import suppress\n\n\nclass SpiderFootThreadPool:\n \"\"\"\n Each thread in the pool is spawned only once, and reused for best performance.\n\n Example 1: using map()\n with SpiderFootThreadPool(self.opts[\"_maxthreads\"]) as pool:\n # callback(\"a\", \"arg1\"), callback(\"b\", \"arg1\"), ...\n for result in pool.map(\n callback,\n [\"a\", \"b\", \"c\", \"d\"],\n \"arg1\",\n taskName=\"sfp_testmodule\"\n saveResult=True\n ):\n yield result\n\n Example 2: using submit()\n with SpiderFootThreadPool(self.opts[\"_maxthreads\"]) as pool:\n pool.start()\n # callback(\"arg1\"), callback(\"arg2\")\n pool.submit(callback, \"arg1\", taskName=\"sfp_testmodule\", saveResult=True)\n pool.submit(callback, \"arg2\", taskName=\"sfp_testmodule\", saveResult=True)\n for result in pool.shutdown()[\"sfp_testmodule\"]:\n yield result\n \"\"\"\n\n def __init__(self, threads: int = 100, qsize: int = 10, name: str = '') -> None:\n \"\"\"Initialize the SpiderFootThreadPool class.\n\n Args:\n threads (int): Max number of threads\n qsize (int): Queue size\n name (str): Name\n \"\"\"\n self.log = logging.getLogger(f\"spiderfoot.{__name__}\")\n self.threads = int(threads)\n self.qsize = int(qsize)\n self.pool = [None] * self.threads\n self.name = str(name)\n self.inputThread = None\n self.inputQueues = dict()\n self.outputQueues = dict()\n self._stop = False\n self._lock = threading.Lock()\n\n def start(self) -> None:\n self.log.debug(f'Starting thread pool \"{self.name}\" with {self.threads:,} threads')\n for i in range(self.threads):\n t = ThreadPoolWorker(pool=self, name=f\"{self.name}_worker_{i + 1}\")\n t.start()\n self.pool[i] = t\n\n @property\n def stop(self) -> bool:\n return self._stop\n\n @stop.setter\n def stop(self, val: bool):\n assert val in (True, False), \"stop must be either True or False\"\n for t in self.pool:\n with suppress(Exception):\n t.stop = val\n self._stop = val\n\n def shutdown(self, wait: bool = True) -> dict:\n \"\"\"Shut down the pool.\n\n Args:\n wait (bool): Whether to wait for the pool to finish executing\n\n Returns:\n results (dict): (unordered) results in the format: {\"taskName\": [returnvalue1, returnvalue2, ...]}\n \"\"\"\n results = dict()\n self.log.debug(f'Shutting down thread pool \"{self.name}\" with wait={wait}')\n if wait:\n while not self.finished and not self.stop:\n with self._lock:\n outputQueues = list(self.outputQueues)\n for taskName in outputQueues:\n moduleResults = list(self.results(taskName))\n try:\n results[taskName] += moduleResults\n except KeyError:\n results[taskName] = moduleResults\n sleep(.1)\n self.stop = True\n # make sure input queues are empty\n with self._lock:\n inputQueues = list(self.inputQueues.values())\n for q in inputQueues:\n with suppress(Exception):\n while 1:\n q.get_nowait()\n with suppress(Exception):\n q.close()\n # make sure output queues are empty\n with self._lock:\n outputQueues = list(self.outputQueues.items())\n for taskName, q in outputQueues:\n moduleResults = list(self.results(taskName))\n try:\n results[taskName] += moduleResults\n except KeyError:\n results[taskName] = moduleResults\n with suppress(Exception):\n q.close()\n return results\n\n def submit(self, callback, *args, **kwargs) -> None:\n \"\"\"Submit a function call to the pool.\n The \"taskName\" and \"maxThreads\" arguments are optional.\n\n Args:\n callback (function): callback function\n *args: Passed through to callback\n **kwargs: Passed through to callback, except for taskName and maxThreads\n \"\"\"\n taskName = kwargs.get('taskName', 'default')\n maxThreads = kwargs.pop('maxThreads', 100)\n # block if this module's thread limit has been reached\n while self.countQueuedTasks(taskName) >= maxThreads:\n sleep(.01)\n continue\n self.log.debug(f\"Submitting function \\\"{callback.__name__}\\\" from module \\\"{taskName}\\\" to thread pool \\\"{self.name}\\\"\")\n self.inputQueue(taskName).put((callback, args, kwargs))\n\n def countQueuedTasks(self, taskName: str) -> int:\n \"\"\"For the specified task, returns the number of queued function calls\n plus the number of functions which are currently executing\n\n Args:\n taskName (str): Name of task\n\n Returns:\n int: the number of queued function calls plus the number of functions which are currently executing\n \"\"\"\n queuedTasks = 0\n with suppress(Exception):\n queuedTasks += self.inputQueues[taskName].qsize()\n runningTasks = 0\n for t in self.pool:\n with suppress(Exception):\n if t.taskName == taskName:\n runningTasks += 1\n return queuedTasks + runningTasks\n\n def inputQueue(self, taskName: str = \"default\") -> str:\n try:\n return self.inputQueues[taskName]\n except KeyError:\n self.inputQueues[taskName] = queue.Queue(self.qsize)\n return self.inputQueues[taskName]\n\n def outputQueue(self, taskName: str = \"default\") -> str:\n try:\n return self.outputQueues[taskName]\n except KeyError:\n self.outputQueues[taskName] = queue.Queue(self.qsize)\n return self.outputQueues[taskName]\n\n def map(self, callback, iterable, *args, **kwargs) -> None: # noqa: A003\n \"\"\"map.\n\n Args:\n callback: the function to thread\n iterable: each entry will be passed as the first argument to the function\n args: additional arguments to pass to callback function\n kwargs: keyword arguments to pass to callback function\n\n Yields:\n return values from completed callback function\n \"\"\"\n taskName = kwargs.get(\"taskName\", \"default\")\n self.inputThread = threading.Thread(target=self.feedQueue, args=(callback, iterable, args, kwargs))\n self.inputThread.start()\n self.start()\n sleep(.1)\n yield from self.results(taskName, wait=True)\n\n def results(self, taskName: str = \"default\", wait: bool = False) -> None:\n while 1:\n result = False\n with suppress(Exception):\n while 1:\n yield self.outputQueue(taskName).get_nowait()\n result = True\n if self.countQueuedTasks(taskName) == 0 or not wait:\n break\n if not result:\n # sleep briefly to save CPU\n sleep(.1)\n\n def feedQueue(self, callback, iterable, args, kwargs) -> None:\n for i in iterable:\n if self.stop:\n break\n self.submit(callback, i, *args, **kwargs)\n\n @property\n def finished(self):\n if self.stop:\n return True\n\n finishedThreads = [not t.busy for t in self.pool if t is not None]\n try:\n inputThreadAlive = self.inputThread.is_alive()\n except AttributeError:\n inputThreadAlive = False\n\n inputQueuesEmpty = [q.empty() for q in self.inputQueues.values()]\n return not inputThreadAlive and all(inputQueuesEmpty) and all(finishedThreads)\n\n def __enter__(self):\n return self\n\n def __exit__(self, exception_type, exception_value, traceback):\n self.shutdown()\n\n\nclass ThreadPoolWorker(threading.Thread):\n\n def __init__(self, pool, name: str = None) -> None:\n\n self.log = logging.getLogger(f\"spiderfoot.{__name__}\")\n self.pool = pool\n self.taskName = \"\" # which module submitted the callback\n self.busy = False\n self.stop = False\n\n super().__init__(name=name)\n\n def run(self) -> None:\n # Round-robin through each module's input queue\n while not self.stop:\n ran = False\n with self.pool._lock:\n inputQueues = list(self.pool.inputQueues.values())\n for q in inputQueues:\n if self.stop:\n break\n try:\n self.busy = True\n callback, args, kwargs = q.get_nowait()\n self.taskName = kwargs.pop(\"taskName\", \"default\")\n saveResult = kwargs.pop(\"saveResult\", False)\n try:\n result = callback(*args, **kwargs)\n ran = True\n except Exception: # noqa: B902\n import traceback\n self.log.error(f'Error in thread worker {self.name}: {traceback.format_exc()}')\n break\n if saveResult:\n self.pool.outputQueue(self.taskName).put(result)\n except queue.Empty:\n self.busy = False\n finally:\n self.busy = False\n self.taskName = \"\"\n # sleep briefly to save CPU\n if not ran:\n sleep(.05)\n" }, { "path": "test/README.md", "content": "# Tests\n\nSpiderFoot includes various test suites.\n\n\n## Unit and Integration Tests\n\nUnit and integration tests require test dependencies to be installed:\n\n```\npip3 install -r test/requirements.txt\n```\n\nTo run the tests locally, run `./test/run` from the SpiderFoot root directory.\n\nThese tests are run on all pull requests automatically.\n\nModule integration tests are excluded.\n\nTo run all unit and integration tests, including module integration tests, run:\n\n```\npython3 -m pytest -n auto --flake8 --dist loadfile --durations=5 --cov-report html --cov=. .\n```\n\n\n## Module Integration Tests\n\nThe module integration tests check module integration with remote third-party data sources.\n\nTo run the tests:\n\n```\npython3 -m pytest -n auto --flake8 --dist loadfile --durations=5 --cov-report html --cov=. test/integration/modules/\n```\n\n\n## Acceptance Tests\n\nThe acceptance tests check that the web intereface is working as\nintended and that SpiderFooot is operating correctly as a whole.\n\nThese tests use a headless browser (Firefox by default), and\nmust be run with `./test/acceptance` as current working directory.\n\nRequires SpiderFoot web server to be running on default port (`5001`).\n\nRequires test dependencies to be installed:\n\n```\npip3 install -r test/acceptance/requirements.txt\n```\n\nTo run the tests, start the SpiderFoot web interface on the default port:\n\n```\npython3 ./sf.py -l 127.0.0.1:5001\n```\n\nThen run robot (override the `BROWSER` variable if necessary):\n\n```\ncd test/acceptance\nrobot --variable BROWSER:Firefox --outputdir results scan.robot\n```\n" }, { "path": "test/__init__.py", "content": "" }, { "path": "test/acceptance/requirements.txt", "content": "# To run the tests, start sf web interface:\n# python3 ./sf.py -l 127.0.0.1:5001\n# then:\n# robot --variable BROWSER:Firefox --outputdir results example.robot\n-r ../requirements.txt\nrobotframework\nrobotframework-seleniumlibrary\n\n" }, { "path": "test/acceptance/run", "content": "#!/bin/bash\n# Run acceptance tests\n#\n# Must be run with ./test/acceptance as current working directory.\n#\n# Requires SpiderFoot web server to be running on default port.\n#\n# Requires robotframework:\n# cd test/acceptance; pip3 install -r requirements.txt\n\ntime robot --outputdir results scan.robot\n" }, { "path": "test/acceptance/scan.robot", "content": "# To run the tests, start sf web interface:\n# python3 ./sf.py -l 127.0.0.1:5001\n# then run robot (override the BROWSER variable if necessary):\n# robot --variable BROWSER:Firefox --outputdir results scan.robot\n\n*** Settings ***\nLibrary SeleniumLibrary\nTest Teardown Close All Browsers\n\n*** Variables ***\n${BROWSER} Firefox\n${HOST} 127.0.0.1\n${PORT} 5001\n${URL} http://${HOST}:${PORT}\n\n*** Keywords ***\nCreate a module scan\n [Arguments] ${scan_name} ${scan_target} ${module_name}\n Open browser ${URL}/newscan ${BROWSER}\n Press Keys name:scanname ${scan_name}\n Press Keys name:scantarget ${scan_target}\n Click Element id:moduletab\n Click Element id:btn-deselect-all\n Scroll To Element id:module_${module_name}\n Set Focus To Element id:module_${module_name}\n Click Element id:module_${module_name}\n Scroll To Element id:btn-run-scan\n Click Element id:btn-run-scan\n\nCreate a use case scan\n [Arguments] ${scan_name} ${scan_target} ${use_case}\n Open browser ${URL}/newscan ${BROWSER}\n Press Keys name:scanname ${scan_name}\n Press Keys name:scantarget ${scan_target}\n Click Element id:usecase_${use_case}\n Scroll To Element id:btn-run-scan\n Click Element id:btn-run-scan\n \nScan info page should render tabs\n Element Should Be Visible id:btn-status\n Element Should Be Visible id:btn-browse\n Element Should Be Visible id:btn-correlations\n Element Should Be Visible id:btn-graph\n Element Should Be Visible id:btn-info\n Element Should Be Visible id:btn-log\n\nScan info Summary tab should render\n Scan info page should render tabs\n Element Should Be Visible id:vbarsummary\n\nScan info Browse tab should render\n Scan info page should render tabs\n Element Should Be Visible id:btn-refresh\n Element Should Be Visible id:btn-export\n Element Should Be Visible id:searchvalue\n Element Should Be Visible id:searchbutton\n\nScan info Correlations tab should render\n Scan info page should render tabs\n Element Should Be Visible id:scansummary-content\n\nScan info Graph tab should render\n Scan info page should render tabs\n Element Should Be Visible id:graph-container\n\nScan info Settings tab should render\n Scan info page should render tabs\n Page Should Contain Meta Information\n Page Should Contain Global Settings\n\nScan info Log tab should render\n Scan info page should render tabs\n Element Should Be Visible id:btn-refresh\n Element Should Be Visible id:btn-download-logs\n\nScan list page should render\n Element Should Be Visible id:scanlist\n Element Should Be Visible id:btn-rerun\n Element Should Be Visible id:btn-stop\n Element Should Be Visible id:btn-refresh\n Element Should Be Visible id:btn-export\n Element Should Be Visible id:btn-delete\n\nSettings page should render\n Element Should Be Visible id:savesettingsform\n Element Should Be Visible id:btn-save-changes\n Element Should Be Visible id:btn-import-config\n Element Should Be Visible id:btn-opt-export\n Element Should Be Visible id:btn-reset-settings\n\nNew scan page should render\n Element Should Be Visible id:scanname\n Element Should Be Visible id:scantarget\n Element Should Be Visible id:usetab\n Element Should Be Visible id:typetab\n Element Should Be Visible id:moduletab\n\nScroll To Element\n [Arguments] ${locator}\n ${x}= Get Horizontal Position ${locator}\n ${y}= Get Vertical Position ${locator}\n Execute Javascript window.scrollTo(${x} - 100, ${y} - 100)\n Wait Until Element is visible ${locator} timeout=5s\n\n*** Test Cases ***\nMain navigation pages should render correctly\n Open browser ${URL} ${BROWSER}\n Click Element id:nav-link-newscan\n Wait Until Page Contains New Scan timeout=5s\n New scan page should render\n Click Element id:nav-link-scans\n Wait Until Page Contains Started timeout=5s\n Scan list page should render\n Click Element id:nav-link-settings\n Wait Until Page Contains Settings timeout=5s\n Settings page should render\n\nScan info page should render correctly\n Create a module scan test scan info spiderfoot.net sfp_countryname\n Wait Until Page Contains Browse timeout=5s\n Wait Until Element Contains scanstatusbadge FINISHED timeout=10s\n Click Element id:btn-status\n Scan info Summary tab should render\n Click Element id:btn-browse\n Scan info Browse tab should render\n Click Element id:btn-graph\n Scan info Graph tab should render\n Click Element id:btn-info\n Scan info Settings tab should render\n Click Element id:btn-log\n Scan info Log tab should render\n\nScan list page should list scans\n Create a module scan test scan list spiderfoot.net sfp_countryname\n Click Element id:nav-link-scans\n Wait Until Page Contains test scan list timeout=5s\n\nA sfp_dnsresolve scan should resolve INTERNET_NAME to IP_ADDRESS\n Create a module scan dns resolve spiderfoot.net sfp_dnsresolve\n Wait Until Page Contains Browse timeout=5s\n Wait Until Element Contains scanstatusbadge FINISHED timeout=10s\n Click Element id:btn-browse\n Scan info Browse tab should render\n Page Should Contain Domain Name\n Page Should Contain Internet Name\n Page Should Contain IP Address\n\nA sfp_dnsresolve scan should reverse resolve IP_ADDRESS to INTERNET_NAME\n Create a module scan reverse resolve 1.1.1.1 sfp_dnsresolve\n Wait Until Page Contains Browse timeout=5s\n Wait Until Element Contains scanstatusbadge FINISHED timeout=10s\n Click Element id:btn-browse\n Scan info Browse tab should render\n Page Should Contain Domain Name\n Page Should Contain Internet Name\n Page Should Contain IP Address\n\n#A passive scan with unresolvable target internet name should fail\n# Create a use case scan shouldnotresolve.doesnotexist.local passive\n# Wait Until Page Contains Browse timeout=5s\n# Wait Until Element Contains scanstatusbadge RUNNING timeout=10s\n# Click Element id:btn-browse\n# Page Should Contain Domain Name\n# Page Should Contain Internet Name\n# Page Should Contain IP Address\n\n" }, { "path": "test/bandit", "content": "#!/bin/sh\n# Run bandit static analysis checks\n#\n# Must be run from SpiderFoot root directory; ie:\n# ./test/bandit\n\npython3 -m bandit *.py spiderfoot/* modules/*.py\n" }, { "path": "test/conftest.py", "content": "import pytest\nfrom spiderfoot import SpiderFootHelpers\n\n\n@pytest.fixture(autouse=True)\ndef default_options(request):\n request.cls.default_options = {\n '_debug': False,\n '__logging': True, # Logging in general\n '__outputfilter': None, # Event types to filter from modules' output\n '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests\n '_dnsserver': '', # Override the default resolver\n '_fetchtimeout': 5, # number of seconds before giving up on a fetch\n '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',\n '_internettlds_cache': 72,\n '_genericusers': \",\".join(SpiderFootHelpers.usernamesFromWordlists(['generic-usernames'])),\n '__database': f\"{SpiderFootHelpers.dataPath()}/spiderfoot.test.db\", # note: test database file\n '__modules__': None, # List of modules. Will be set after start-up.\n '__correlationrules__': None, # List of correlation rules. Will be set after start-up.\n '_socks1type': '',\n '_socks2addr': '',\n '_socks3port': '',\n '_socks4user': '',\n '_socks5pwd': '',\n '__logstdout': False\n }\n\n request.cls.web_default_options = {\n 'root': '/'\n }\n\n request.cls.cli_default_options = {\n \"cli.debug\": False,\n \"cli.silent\": False,\n \"cli.color\": True,\n \"cli.output\": \"pretty\",\n \"cli.history\": True,\n \"cli.history_file\": \"\",\n \"cli.spool\": False,\n \"cli.spool_file\": \"\",\n \"cli.ssl_verify\": True,\n \"cli.username\": \"\",\n \"cli.password\": \"\",\n \"cli.server_baseurl\": \"http://127.0.0.1:5001\"\n }\n" }, { "path": "test/integration/__init__.py", "content": "" }, { "path": "test/integration/modules/__init__.py", "content": "" }, { "path": "test/integration/modules/test_sfp__stor_db.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp__stor_db import sfp__stor_db\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegration_stor_db(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp__stor_db()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp__stor_stdout.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp__stor_stdout import sfp__stor_stdout\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegration_stor_stdout(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp__stor_stdout()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_abstractapi.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_abstractapi import sfp_abstractapi\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationAbstractapi(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_abstractapi()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_abusech.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_abusech import sfp_abusech\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationAbusech(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_abusech()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_abuseipdb.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_abuseipdb import sfp_abuseipdb\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationabuseipdb(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_abuseipdb()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_abusix.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_abusix import sfp_abusix\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationAbusix(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_abusix()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_accounts.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_accounts import sfp_accounts\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationaccounts(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_accounts()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_adblock.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_adblock import sfp_adblock\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationAdblock(unittest.TestCase):\n\n def test_handleEvent_event_data_provider_javascript_url_matching_ad_filter_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_adblock()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'URL_ADBLOCKED_EXTERNAL'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'https://example.local/lib/ad.js'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_adblock)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'PROVIDER_JAVASCRIPT'\n event_data = 'https://example.local/lib/ad.js'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_external_url_matching_ad_filter_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_adblock()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'URL_ADBLOCKED_EXTERNAL'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'https://example.local/lib/ad.js'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_adblock)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'LINKED_URL_EXTERNAL'\n event_data = 'https://example.local/lib/ad.js'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_external_url_not_matching_ad_filter_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_adblock()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_adblock)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'LINKED_URL_EXTERNAL'\n event_data = 'https://example.local/lib/example.js'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_adguard_dns.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_adguard_dns import sfp_adguard_dns\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationAdGuardDns(unittest.TestCase):\n\n def test_handleEvent_event_data_adult_internet_name_blocked_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_adguard_dns()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'BLACKLISTED_INTERNET_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'AdGuard - Family Filter [pornhub.com]'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_adguard_dns)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'pornhub.com'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_adguard_dns()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_adguard_dns)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'spiderfoot.net'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_ahmia.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ahmia import sfp_ahmia\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationAhmia(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ahmia()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_alienvault.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_alienvault import sfp_alienvault\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationalienvault(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_alienvault()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_alienvaultiprep.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_alienvaultiprep import sfp_alienvaultiprep\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationAlienvaultiprep(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_alienvaultiprep()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_apple_itunes.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_apple_itunes import sfp_apple_itunes\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationAppleItunes(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_apple_itunes()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_archiveorg.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_archiveorg import sfp_archiveorg\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationarchiveorg(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_archiveorg()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_arin.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_arin import sfp_arin\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationarin(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_arin()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_azureblobstorage.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_azureblobstorage import sfp_azureblobstorage\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationazureblobstorage(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_azureblobstorage()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_bgpview.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_bgpview import sfp_bgpview\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationBgpview(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bgpview()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_binaryedge.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_binaryedge import sfp_binaryedge\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationBinaryedge(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_binaryedge()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_bingsearch.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_bingsearch import sfp_bingsearch\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationbingsearch(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bingsearch()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_bingsharedip.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_bingsharedip import sfp_bingsharedip\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationbingsharedip(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bingsharedip()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_bitcoinabuse.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_bitcoinabuse import sfp_bitcoinabuse\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationBitcoinAbuse(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bitcoinabuse()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_bitcoinwhoswho.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_bitcoinwhoswho import sfp_bitcoinwhoswho\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationBitcoinwhoswho(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bitcoinwhoswho()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_blockchain.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_blockchain import sfp_blockchain\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationBlockchain(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_blockchain()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_blocklistde.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_blocklistde import sfp_blocklistde\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationBlocklistde(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_blocklistde()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_botscout.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_botscout import sfp_botscout\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationBotscout(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_botscout()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_botvrij.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_botvrij import sfp_botvrij\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationBotvrij(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_botvrij()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_builtwith.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_builtwith import sfp_builtwith\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationBuiltwith(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_builtwith()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_c99.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_c99 import sfp_c99\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationC99(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_c99()\n module.setup(sf, dict())\n\n target_value = \"example target value\"\n target_type = \"IP_ADDRESS\"\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = \"ROOT\"\n event_data = \"example data\"\n event_module = \"\"\n source_event = \"\"\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_callername.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_callername import sfp_callername\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationcallername(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_callername()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_censys.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_censys import sfp_censys\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCensys(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_censys()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_certspotter.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_certspotter import sfp_certspotter\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCertspotter(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_certspotter()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_cinsscore.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_cinsscore import sfp_cinsscore\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCinsscore(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_cinsscore()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_circllu.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_circllu import sfp_circllu\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCircllu(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_circllu()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_citadel.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_citadel import sfp_citadel\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationcitadel(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_citadel()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_cleanbrowsing.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_cleanbrowsing import sfp_cleanbrowsing\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationcleanbrowsing(unittest.TestCase):\n\n def test_handleEvent_event_data_adult_internet_name_blocked_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_cleanbrowsing()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'BLACKLISTED_INTERNET_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'CleanBrowsing DNS - Adult [pornhub.com]'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_cleanbrowsing)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'pornhub.com'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_cleanbrowsing()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_cleanbrowsing)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'spiderfoot.net'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_cleantalk.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_cleantalk import sfp_cleantalk\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationcleantalk(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_cleantalk()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n module.opts['_fetchtimeout'] = 15\n module.optdescs['_fetchtimeout'] = ''\n module.opts['_useragent'] = ''\n module.optdescs['_useragent'] = ''\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_cleantalk)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'IP_ADDRESS'\n event_data = '1.0.0.1'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_clearbit.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_clearbit import sfp_clearbit\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationclearbit(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_clearbit()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_cloudflaredns.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_cloudflaredns import sfp_cloudflaredns\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCloudflaredns(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_cloudflaredns()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_cloudflaredns)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'cloudflare.com'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n\n def test_handleEvent_event_data_adult_internet_name_blocked_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_cloudflaredns()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'BLACKLISTED_INTERNET_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'CloudFlare - Family [pornhub.com]'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_cloudflaredns)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'pornhub.com'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n" }, { "path": "test/integration/modules/test_sfp_coinblocker.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_coinblocker import sfp_coinblocker\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCoinblocker(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_coinblocker()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_commoncrawl.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_commoncrawl import sfp_commoncrawl\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationcommoncrawl(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_commoncrawl()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_comodo.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_comodo import sfp_comodo\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationcomodo(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_comodo()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_comodo)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'comodo.com'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_crobat_api.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_crobat_api import sfp_crobat_api\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCrobat_api(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_crobat_api()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_crossref.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_crossref import sfp_crossref\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCrossref(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_crossref()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_crt.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_crt import sfp_crt\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCrt(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_crt()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_crxcavator.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_crxcavator import sfp_crxcavator\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCrxcavator(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_crxcavator()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_customfeed.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_customfeed import sfp_customfeed\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCustomfeed(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_customfeed()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_cybercrimetracker.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_cybercrimetracker import sfp_cybercrimetracker\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCybercrimetracker(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_cybercrimetracker()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_debounce.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_debounce import sfp_debounce\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDebounce(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_debounce()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_dehashed.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dehashed import sfp_dehashed\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDehashed(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dehashed()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_digitaloceanspace.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_digitaloceanspace import sfp_digitaloceanspace\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDigitaloceanspace(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_digitaloceanspace()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_dns_for_family.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dns_for_family import sfp_dns_for_family\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDnsForFamily(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dns_for_family()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_dns_for_family)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'dns_for_family.com'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n\n def test_handleEvent_event_data_adult_internet_name_blocked_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dns_for_family()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'BLACKLISTED_INTERNET_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'DNS for Family [pornhub.com]'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_dns_for_family)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'pornhub.com'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n" }, { "path": "test/integration/modules/test_sfp_dnsbrute.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsbrute import sfp_dnsbrute\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDnsBrute(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsbrute()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_dnscommonsrv.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnscommonsrv import sfp_dnscommonsrv\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDnsCommonsrv(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnscommonsrv()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_dnsdb.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsdb import sfp_dnsdb\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationdnsdb(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsdb()\n module.setup(sf, dict())\n\n target_value = \"example target value\"\n target_type = \"IP_ADDRESS\"\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = \"ROOT\"\n event_data = \"example data\"\n event_module = \"\"\n source_event = \"\"\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_dnsdumpster.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsdumpster import sfp_dnsdumpster\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDnsDumpster(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsdumpster()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_dnsgrep.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsgrep import sfp_dnsgrep\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDnsgrep(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsgrep()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_dnsneighbor.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsneighbor import sfp_dnsneighbor\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDnsneighbor(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsneighbor()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_dnsraw.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsraw import sfp_dnsraw\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationdnsraw(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsraw()\n module.setup(sf, dict())\n\n target_value = '1.1.1.1'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = '1.1.1.1'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_dnsresolve.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsresolve import sfp_dnsresolve\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDnsResolve(unittest.TestCase):\n\n def test_enrichTarget_should_return_SpiderFootTarget(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsresolve()\n module.setup(sf, dict())\n\n target_value = '127.0.0.1'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n\n result = module.enrichTarget(target)\n self.assertIsInstance(result, SpiderFootTarget)\n self.assertEqual(result.targetType, target_type)\n self.assertEqual(result.targetValue, target_value)\n\n def test_resolveTargets_should_return_list(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsresolve()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n invalid_types = [None, \"\", list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n resolve_targets = module.resolveTargets(invalid_type, False)\n self.assertIsInstance(resolve_targets, list)\n\n target = SpiderFootTarget(\"spiderfoot.net\", \"INTERNET_NAME\")\n resolve_targets = module.resolveTargets(target, False)\n self.assertIsInstance(resolve_targets, list)\n self.assertIn('spiderfoot.net', resolve_targets)\n\n target = SpiderFootTarget(\"127.0.0.1\", \"IP_ADDRESS\")\n resolve_targets = module.resolveTargets(target, False)\n self.assertIsInstance(resolve_targets, list)\n self.assertIn('127.0.0.1', resolve_targets)\n\n target = SpiderFootTarget(\"::1\", \"IPV6_ADDRESS\")\n resolve_targets = module.resolveTargets(target, False)\n self.assertIsInstance(resolve_targets, list)\n self.assertIn('::1', resolve_targets)\n\n target = SpiderFootTarget(\"127.0.0.1/32\", \"NETBLOCK_OWNER\")\n resolve_targets = module.resolveTargets(target, False)\n self.assertIsInstance(resolve_targets, list)\n self.assertIn('127.0.0.1', resolve_targets)\n\n # note: test fails on MacOSX on CI\n def test_handleEvent_event_data_ip_address_should_return_internet_name_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsresolve()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'INTERNET_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"one.one.one.one\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_dnsresolve)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'IP_ADDRESS'\n event_data = '1.1.1.1'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n # note: test fails on MacOSX on CI\n def test_handleEvent_event_data_ipv6_address_should_return_internet_name_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsresolve()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'INTERNET_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"one.one.one.one\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_dnsresolve)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'IPV6_ADDRESS'\n event_data = '2606:4700:4700::1111'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n # note: test fails on MacOSX on CI\n def test_handleEvent_event_data_affiliate_ip_address_should_return_affiliate_internet_name_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsresolve()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'AFFILIATE_INTERNET_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"one.one.one.one\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_dnsresolve)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'AFFILIATE_IPADDR'\n event_data = '1.1.1.1'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_raw_rir_data_containing_subdomain_should_return_internet_name_event(self):\n \"\"\"\n Test handleEvent(self, event)\n \"\"\"\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsresolve()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'INTERNET_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"www.spiderfoot.net\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_dnsresolve)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'RAW_RIR_DATA'\n event_data = 'example data www.spiderfoot.net example data'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n" }, { "path": "test/integration/modules/test_sfp_dnszonexfer.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnszonexfer import sfp_dnszonexfer\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDnsZoneXfer(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnszonexfer()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_dronebl.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dronebl import sfp_dronebl\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDronebl(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dronebl()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_dronebl)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'IP_ADDRESS'\n event_data = '1.0.0.1'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_duckduckgo.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_duckduckgo import sfp_duckduckgo\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationDuckduckgo(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_duckduckgo()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_email.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_email import sfp_email\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationEmail(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_email()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_emailcrawlr.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_emailcrawlr import sfp_emailcrawlr\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationemailcrawlr(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_emailcrawlr()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_emailformat.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_emailformat import sfp_emailformat\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationEmailFormat(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_emailformat()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_emailrep.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_emailrep import sfp_emailrep\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationEmailrep(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_emailrep()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_emergingthreats.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_emergingthreats import sfp_emergingthreats\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationEmergingthreats(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_emergingthreats()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_etherscan.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_etherscan import sfp_etherscan\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationEtherscan(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_etherscan()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_filemeta.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_filemeta import sfp_filemeta\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationFilemeta(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_filemeta()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_flickr.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_flickr import sfp_flickr\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationFlickr(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_flickr()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_focsec.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_focsec import sfp_focsec\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationFocsec(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_focsec()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_fortinet.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_fortinet import sfp_fortinet\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationFortinet(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_fortinet()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_fraudguard.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_fraudguard import sfp_fraudguard\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationFraudguard(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_fraudguard()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_fsecure_riddler.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_fsecure_riddler import sfp_fsecure_riddler\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationFsecureRiddler(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_fsecure_riddler()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_fullcontact.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_fullcontact import sfp_fullcontact\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationFullContact(unittest.TestCase):\n\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_fullcontact()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_fullhunt.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_fullhunt import sfp_fullhunt\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationFullhunt(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_fullhunt()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_github.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_github import sfp_github\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationGithub(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_github()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_gleif.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_gleif import sfp_gleif\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationGleif(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_gleif()\n module.setup(sf, dict())\n\n target_value = '7ZW8QJWVPR4P1J1KQY45'\n target_type = 'LEI'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_google_tag_manager.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_google_tag_manager import sfp_google_tag_manager\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationGoogleTagManager(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_google_tag_manager()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_googlemaps.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_googlemaps import sfp_googlemaps\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationGoogleMaps(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_googlemaps()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_googleobjectstorage.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_googleobjectstorage import sfp_googleobjectstorage\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationGoogleObjectStorage(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_googleobjectstorage()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_googlesafebrowsing.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_googlesafebrowsing import sfp_googlesafebrowsing\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationGoogleSafebrowsing(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_googlesafebrowsing()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_googlesearch.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_googlesearch import sfp_googlesearch\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationGoogleSearch(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_googlesearch()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_gravatar.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_gravatar import sfp_gravatar\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationGravatar(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_gravatar()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_grayhatwarfare.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_grayhatwarfare import sfp_grayhatwarfare\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationGrayhatWarfare(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_grayhatwarfare()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_greensnow.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_greensnow import sfp_greensnow\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationgreensnow(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_greensnow()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_grep_app.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_grep_app import sfp_grep_app\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationGrepApp(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_grep_app()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_greynoise.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_greynoise import sfp_greynoise\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationGreynoise(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_greynoise()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_h1nobbdde.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_h1nobbdde import sfp_h1nobbdde\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationH1nobbdde(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_h1nobbdde()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_hackertarget.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_hackertarget import sfp_hackertarget\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationHackertarget(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_hackertarget()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_haveibeenpwned.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_haveibeenpwned import sfp_haveibeenpwned\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationHaveibeenpwned(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_haveibeenpwned()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_honeypot.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_honeypot import sfp_honeypot\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationHoneypot(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_honeypot()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_hosting.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_hosting import sfp_hosting\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationHosting(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent_event_data_ip_address_hosted_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_hosting()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'PROVIDER_HOSTING'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"Amazon AWS: http://www.amazon.com/aws/\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_hosting)\n\n event_type = 'ROOT'\n event_data = '3.1.1.1'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n @unittest.skip(\"todo\")\n def test_handleEvent_event_data_ip_address_not_hosted_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_hosting()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_hosting)\n\n event_type = 'ROOT'\n event_data = '127.0.0.1'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_hostio.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_hostio import sfp_hostio\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationHostio(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_hostio()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_hunter.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_hunter import sfp_hunter\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationHunter(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_hunter()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_hybrid_analysis.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_hybrid_analysis import sfp_hybrid_analysis\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationHybridAnalysis(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_hybrid_analysis()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_iknowwhatyoudownload.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_iknowwhatyoudownload import sfp_iknowwhatyoudownload\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationIknowwhatyoudownload(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_iknowwhatyoudownload()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_intelx.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_intelx import sfp_intelx\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationIntelx(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_intelx()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_ipapico.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ipapico import sfp_ipapico\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationIpapico(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipapico()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_ipapicom.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ipapicom import sfp_ipapicom\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationIpapicom(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipapicom()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_ipinfo.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ipinfo import sfp_ipinfo\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationIpinfo(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipinfo()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_ipqualityscore.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ipqualityscore import sfp_ipqualityscore\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationIpqualityscore(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipqualityscore()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'PHONE_NUMBER'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_ipregistry.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ipregistry import sfp_ipregistry\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationIpregistry(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipregistry()\n module.setup(sf, dict())\n\n target_value = \"example target value\"\n target_type = \"IP_ADDRESS\"\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = \"ROOT\"\n event_data = \"example data\"\n event_module = \"\"\n source_event = \"\"\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_ipstack.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ipstack import sfp_ipstack\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationIpstack(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipstack()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_isc.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_isc import sfp_isc\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationIsc(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_isc()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_jsonwhoiscom.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_jsonwhoiscom import sfp_jsonwhoiscom\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationJsonwhoiscom(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_jsonwhoiscom()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_junkfiles.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_junkfiles import sfp_junkfiles\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationJunkfiles(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_junkfiles()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_keybase.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_keybase import sfp_keybase\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationKeybase(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_keybase()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_koodous.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_koodous import sfp_koodous\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationKoodous(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_koodous()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_leakix.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_leakix import sfp_leakix\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationleakix(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_leakix()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_maltiverse.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_maltiverse import sfp_maltiverse\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationMaltiverse(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_maltiverse()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_malwarepatrol.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_malwarepatrol import sfp_malwarepatrol\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationMalwarepatrol(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_malwarepatrol()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_metadefender.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_metadefender import sfp_metadefender\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationMetadefender(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_metadefender()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_mnemonic.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_mnemonic import sfp_mnemonic\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationMnemonic(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_mnemonic()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_multiproxy.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_multiproxy import sfp_multiproxy\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationMultiproxy(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_multiproxy()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_myspace.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_myspace import sfp_myspace\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationMyspace(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_myspace()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_nameapi.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_nameapi import sfp_nameapi\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationNameapi(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_nameapi()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_networksdb.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_networksdb import sfp_networksdb\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationNetworksdb(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_networksdb()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_neutrinoapi.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_neutrinoapi import sfp_neutrinoapi\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationNeutrinoapi(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_neutrinoapi()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_numverify.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_numverify import sfp_numverify\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationNumverify(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_numverify()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_onioncity.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_onioncity import sfp_onioncity\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationOnioncity(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_onioncity()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_onionsearchengine.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_onionsearchengine import sfp_onionsearchengine\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationOnionsearchengine(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_onionsearchengine()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_onyphe.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_onyphe import sfp_onyphe\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationOnyphe(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_onyphe()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_openbugbounty.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_openbugbounty import sfp_openbugbounty\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationOpenbugbounty(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_openbugbounty()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_opencorporates.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_opencorporates import sfp_opencorporates\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationOpencorporates(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_opencorporates()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_opendns.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_opendns import sfp_opendns\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationOpendns(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_opendns()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_opendns)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'opendns.com'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n\n def test_handleEvent_event_data_adult_internet_name_blocked_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_opendns()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'BLACKLISTED_INTERNET_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'OpenDNS - Adult [pornhub.com]'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_opendns)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'pornhub.com'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n" }, { "path": "test/integration/modules/test_sfp_opennic.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_opennic import sfp_opennic\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationOpenNic(unittest.TestCase):\n\n def test_handleEvent_event_data_internet_name_with_opennic_tld_should_return_ip_address_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_opennic()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'IP_ADDRESS'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_opennic)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'opennic.glue'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n" }, { "path": "test/integration/modules/test_sfp_openphish.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_openphish import sfp_openphish\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationOpenphish(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_openphish()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_openstreetmap.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_openstreetmap import sfp_openstreetmap\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationOpenstreetmap(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_openstreetmap()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_pageinfo.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_pageinfo import sfp_pageinfo\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationPageInfo(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_pageinfo()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_pastebin.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_pastebin import sfp_pastebin\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationPastebin(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_pastebin()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_pgp.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_pgp import sfp_pgp\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationPgp(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_pgp()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_phishstats.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_phishstats import sfp_phishstats\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationPhishstats(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_phishstats()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_phishtank.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_phishtank import sfp_phishtank\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationPhishtank(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_phishtank()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_portscan_tcp.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_portscan_tcp import sfp_portscan_tcp\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationPortscanTcp(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_portscan_tcp()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_projectdiscovery.py", "content": "import pytest\nimport unittest\nfrom modules.sfp_projectdiscovery import sfp_projectdiscovery\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationProjectdiscovery(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_projectdiscovery()\n module.setup(sf, dict())\n\n target_value = \"example target value\"\n target_type = \"IP_ADDRESS\"\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = \"ROOT\"\n event_data = \"example data\"\n event_module = \"\"\n source_event = \"\"\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_psbdmp.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_psbdmp import sfp_psbdmp\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationPsbdmp(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_psbdmp()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_pulsedive.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_pulsedive import sfp_pulsedive\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationPulsedive(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_pulsedive()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_punkspider.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_punkspider import sfp_punkspider\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationPunkspider(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_punkspider()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_quad9.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_quad9 import sfp_quad9\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationQuad9(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_quad9()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_quad9)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'quad9.net'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_reversewhois.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_reversewhois import sfp_reversewhois\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationReversewhois(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_reversewhois()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'DOMAIN_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_ripe.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ripe import sfp_ripe\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationRipe(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ripe()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_riskiq.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_riskiq import sfp_riskiq\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationRiskiq(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_riskiq()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_robtex.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_robtex import sfp_robtex\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationRobtex(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_robtex()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_s3bucket.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_s3bucket import sfp_s3bucket\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationS3bucket(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_s3bucket()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_searchcode.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_searchcode import sfp_searchcode\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationCodesearch(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_searchcode()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'DOMAIN_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_securitytrails.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_securitytrails import sfp_securitytrails\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSecuritytrails(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_securitytrails()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_seon.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_seon import sfp_seon\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationseon(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_seon()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_shodan.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_shodan import sfp_shodan\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationShodan(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_shodan()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_similar.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_similar import sfp_similar\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSimilar(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_similar()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_skymem.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_skymem import sfp_skymem\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSkymem(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_skymem()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_slideshare.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_slideshare import sfp_slideshare\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSlideshare(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_slideshare()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_snov.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_snov import sfp_snov\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSnov(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_snov()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_sociallinks.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_sociallinks import sfp_sociallinks\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationsociallinks(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_sociallinks()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_socialprofiles.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_socialprofiles import sfp_socialprofiles\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSocialprofiles(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_socialprofiles()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'HUMAN_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_sorbs.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_sorbs import sfp_sorbs\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSorbs(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_sorbs()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_sorbs)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'IP_ADDRESS'\n event_data = '1.0.0.1'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_spamcop.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_spamcop import sfp_spamcop\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSpamcop(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_spamcop()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_spamcop)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'IP_ADDRESS'\n event_data = '1.0.0.1'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_spamhaus.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_spamhaus import sfp_spamhaus\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSpamhaus(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_spamhaus()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_spamhaus)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'IP_ADDRESS'\n event_data = '1.0.0.1'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_spider.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_spider import sfp_spider\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSpider(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_spider()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_spur.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_spur import sfp_spur\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSpur(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_spur()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_spyonweb.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_spyonweb import sfp_spyonweb\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSpyonweb(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_spyonweb()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_sslcert.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_sslcert import sfp_sslcert\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSslCert(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_sslcert()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_stackoverflow.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_stackoverflow import sfp_stackoverflow\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationStackoverflow(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_stackoverflow()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_stevenblack_hosts.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_stevenblack_hosts import sfp_stevenblack_hosts\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationStevenblackHosts(unittest.TestCase):\n\n def test_handleEvent_event_data_affiliate_internet_name_matching_ad_server_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_stevenblack_hosts()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n module.opts['_fetchtimeout'] = 15\n module.optdescs['_fetchtimeout'] = ''\n module.opts['_useragent'] = ''\n module.optdescs['_useragent'] = ''\n\n def new_notifyListeners(self, event):\n expected = 'MALICIOUS_AFFILIATE_INTERNET_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'Steven Black Hosts Blocklist [ads.google.com]\\nhttps://raw.githubusercontent.com/StevenBlack/hosts/master/hosts'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_stevenblack_hosts)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'AFFILIATE_INTERNET_NAME'\n event_data = 'ads.google.com'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_affiliate_internet_name_not_matching_ad_server_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_stevenblack_hosts()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n module.opts['_fetchtimeout'] = 15\n module.optdescs['_fetchtimeout'] = ''\n module.opts['_useragent'] = ''\n module.optdescs['_useragent'] = ''\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_stevenblack_hosts)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'AFFILIATE_INTERNET_NAME'\n event_data = 'no.ads.safe.local'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_subdomain_takeover.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_subdomain_takeover import sfp_subdomain_takeover\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSubdomainTakeover(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_subdomain_takeover()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_sublist3r.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_sublist3r import sfp_sublist3r\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSublist3r(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_sublist3r()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_surbl.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_surbl import sfp_surbl\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationSurbl(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_surbl()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_surbl)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'IP_ADDRESS'\n event_data = '1.0.0.1'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_talosintel.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_talosintel import sfp_talosintel\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationTalosintel(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_talosintel()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_textmagic.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_textmagic import sfp_textmagic\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationTextmagic(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_textmagic()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'PHONE_NUMBER'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_threatcrowd.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_threatcrowd import sfp_threatcrowd\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationThreatcrowd(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_threatcrowd()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_threatfox.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_threatfox import sfp_threatfox\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationThreatFox(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_threatfox()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_threatminer.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_threatminer import sfp_threatminer\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationThreatminer(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_threatminer()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_tldsearch.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tldsearch import sfp_tldsearch\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationTldsearch(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tldsearch()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_torch.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_torch import sfp_torch\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationTorch(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_torch()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_torexits.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_torexits import sfp_torexits\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationTorExits(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_torexits()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_trashpanda.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_trashpanda import sfp_trashpanda\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationTrashpanda(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_trashpanda()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_trumail.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_trumail import sfp_trumail\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationTrumail(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_trumail()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_twilio.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_twilio import sfp_twilio\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationTwilio(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_twilio()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_twitter.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_twitter import sfp_twitter\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationTwitter(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_twitter()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_uceprotect.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_uceprotect import sfp_uceprotect\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationUceprotect(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_uceprotect()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_uceprotect)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'IP_ADDRESS'\n event_data = '1.0.0.1'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_urlscan.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_urlscan import sfp_urlscan\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationUrlscan(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_urlscan()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_venmo.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_venmo import sfp_venmo\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationVenmo(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_venmo()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_viewdns.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_viewdns import sfp_viewdns\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationViewdns(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_viewdns()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_virustotal.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_virustotal import sfp_virustotal\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationVirustotal(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_virustotal()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_voipbl.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_voipbl import sfp_voipbl\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationVoipbl(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_voipbl()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_vxvault.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_vxvault import sfp_vxvault\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationVxvault(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_vxvault()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_webserver.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_webserver import sfp_webserver\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationWebserver(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_webserver()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_whatcms.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_whatcms import sfp_whatcms\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationWhatCMS(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_whatcms()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_whois.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_whois import sfp_whois\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationwhois(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_whois()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_whoisology.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_whoisology import sfp_whoisology\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationWhoisology(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_whoisology()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_whoxy.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_whoxy import sfp_whoxy\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationWhoxy(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_whoxy()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_wigle.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_wigle import sfp_wigle\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationWigle(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_wigle()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_wikileaks.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_wikileaks import sfp_wikileaks\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationWikileaks(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_wikileaks()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_wikipediaedits.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_wikipediaedits import sfp_wikipediaedits\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationWikipediaedits(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_wikipediaedits()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_xforce.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_xforce import sfp_xforce\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationXforce(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_xforce()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/integration/modules/test_sfp_yandexdns.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_yandexdns import sfp_yandexdns\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationYandexDns(unittest.TestCase):\n\n def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_yandexdns()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_yandexdns)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'yandex.com'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n\n def test_handleEvent_event_data_adult_internet_name_blocked_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_yandexdns()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'BLACKLISTED_INTERNET_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'Yandex - Adult [pornhub.com]'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_yandexdns)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'INTERNET_NAME'\n event_data = 'pornhub.com'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n" }, { "path": "test/integration/modules/test_sfp_zetalytics.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_zetalytics import sfp_zetalytics\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationZetalytics(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_zetalytics()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/modules/test_sfp_zoneh.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_zoneh import sfp_zoneh\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntegrationZoneh(unittest.TestCase):\n\n @unittest.skip(\"todo\")\n def test_handleEvent(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_zoneh()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/integration/test_sf.py", "content": "# test_sf.py\nimport subprocess\nimport sys\nimport unittest\n\n\nclass TestSf(unittest.TestCase):\n \"\"\"\n Test TestSf\n \"\"\"\n\n default_types = [\n \"\"\n ]\n\n default_modules = [\n \"sfp_base64\",\n \"sfp_bitcoin\",\n \"sfp_company\",\n \"sfp_cookie\",\n \"sfp_countryname\",\n \"sfp_creditcard\",\n \"sfp_email\",\n \"sfp_errors\",\n \"sfp_ethereum\",\n \"sfp_filemeta\",\n \"sfp_hashes\",\n \"sfp_iban\",\n \"sfp_names\",\n \"sfp_pageinfo\",\n \"sfp_phone\",\n \"sfp_strangeheaders\",\n \"sfp_webframework\",\n \"sfp_webserver\",\n \"sfp_webanalytics\",\n ]\n\n def execute(self, command):\n proc = subprocess.Popen(\n command,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n )\n out, err = proc.communicate()\n return out, err, proc.returncode\n\n def test_no_args_should_print_arg_l_required(self):\n out, err, code = self.execute([sys.executable, \"sf.py\"])\n self.assertIn(b\"SpiderFoot requires -l : to start the web server. Try --help for guidance.\", out)\n self.assertEqual(b\"\", err)\n self.assertEqual(255, code)\n\n def test_help_arg_should_print_help_and_exit(self):\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-h\"])\n self.assertIn(b\"show this help message and exit\", out)\n self.assertEqual(b\"\", err)\n self.assertEqual(0, code)\n\n def test_modules_arg_should_print_modules_and_exit(self):\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-M\"])\n self.assertIn(b\"Modules available:\", err)\n self.assertEqual(0, code)\n\n def test_types_arg_should_print_types_and_exit(self):\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-T\"])\n self.assertIn(b\"Types available:\", err)\n self.assertEqual(0, code)\n\n @unittest.skip(\"todo\")\n def test_l_arg_should_start_web_server(self):\n listen = \"127.0.0.1:5001\"\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-l\", listen])\n self.assertIn(bytes(f\"Starting web server at {listen}\", 'utf-8'), err)\n self.assertEqual(0, code)\n\n def test_debug_arg_should_enable_and_print_debug_output(self):\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-d\", \"-m\", \"example module\", \"-s\", \"spiderfoot.net\"])\n self.assertIn(b\"[DEBUG]\", err)\n self.assertIn(b\"sfp__stor_db : Storing an event: ROOT\", err)\n self.assertEqual(0, code)\n\n def test_quiet_arg_should_hide_debug_output(self):\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-q\", \"-m\", \"example module\", \"-s\", \"spiderfoot.net\"])\n self.assertNotIn(b\"[INFO]\", err)\n self.assertEqual(0, code)\n\n def test_run_scan_invalid_target_should_exit(self):\n invalid_target = '.'\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-s\", invalid_target])\n self.assertIn(bytes(f\"Could not determine target type. Invalid target: {invalid_target}\", 'utf-8'), err)\n self.assertEqual(255, code)\n\n def test_run_scan_with_modules_no_target_should_exit(self):\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-m\", \",\".join(self.default_modules)])\n self.assertIn(b\"You must specify a target when running in scan mode\", err)\n self.assertEqual(255, code)\n\n def test_run_scan_with_types_no_target_should_exit(self):\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-t\", \",\".join(self.default_types)])\n self.assertIn(b\"You must specify a target when running in scan mode\", err)\n self.assertEqual(255, code)\n\n def test_run_scan_with_invalid_module_should_run_scan_and_exit(self):\n module = \"invalid module\"\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-m\", module, \"-s\", \"spiderfoot.net\"])\n self.assertIn(bytes(f\"Failed to load module: {module}\", 'utf-8'), err)\n self.assertEqual(0, code)\n\n def test_run_scan_with_invalid_type_should_exit(self):\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-t\", \"invalid type\", \"-s\", \"spiderfoot.net\"])\n self.assertIn(b\"Based on your criteria, no modules were enabled\", err)\n self.assertEqual(255, code)\n\n def test_run_scan_should_run_scan_and_exit(self):\n target = \"spiderfoot.net\"\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-m\", \",\".join(self.default_modules), \"-s\", target])\n self.assertIn(b\"Scan completed with status FINISHED\", err)\n self.assertEqual(0, code)\n\n for module in self.default_modules:\n with self.subTest(module=module):\n self.assertIn(module.encode(), err)\n\n @unittest.skip(\"output buffering sometimes causes this test to fail\")\n def test_run_scan_should_print_scan_result_and_exit(self):\n target = \"spiderfoot.net\"\n out, err, code = self.execute([sys.executable, \"sf.py\", \"-m\", \",\".join(self.default_modules), \"-s\", target, \"-o\", \"csv\"])\n self.assertIn(b\"Scan completed with status FINISHED\", err)\n self.assertEqual(0, code)\n\n for module in self.default_modules:\n with self.subTest(module=module):\n self.assertIn(module.encode(), err)\n\n expected_output = [\n \"Source,Type,Data\",\n \"SpiderFoot UI,Internet Name,spiderfoot.net,spiderfoot.net\\n\",\n \"SpiderFoot UI,Domain Name,spiderfoot.net,spiderfoot.net\\n\",\n \"sfp_countryname,Country Name,spiderfoot.net,United States\\n\",\n ]\n for output in expected_output:\n self.assertIn(bytes(output, 'utf-8'), out)\n" }, { "path": "test/integration/test_sfcli.py", "content": "# test_sfcli.py\nimport subprocess\nimport sys\nimport unittest\n\n\nclass TestSfcli(unittest.TestCase):\n \"\"\"\n Test TestSfcli\n \"\"\"\n\n def execute(self, command):\n proc = subprocess.Popen(\n command,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n )\n out, err = proc.communicate()\n return out, err, proc.returncode\n\n def test_help_arg_should_print_help_and_exit(self):\n out, err, code = self.execute([sys.executable, \"sfcli.py\", \"-h\"])\n self.assertIn(b\"show this help message and exit\", out)\n self.assertEqual(b\"\", err)\n self.assertEqual(0, code)\n" }, { "path": "test/integration/test_sfwebui.py", "content": "# test_sfwebui.py\nimport os\nimport unittest\n\nimport cherrypy\nfrom cherrypy.test import helper\n\nfrom spiderfoot import SpiderFootHelpers\nfrom sfwebui import SpiderFootWebUi\n\n\nclass TestSpiderFootWebUiRoutes(helper.CPWebCase):\n @staticmethod\n def setup_server():\n default_config = {\n '_debug': False, # Debug\n '__logging': True, # Logging in general\n '__outputfilter': None, # Event types to filter from modules' output\n '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests\n '_dnsserver': '', # Override the default resolver\n '_fetchtimeout': 5, # number of seconds before giving up on a fetch\n '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',\n '_internettlds_cache': 72,\n '_genericusers': \",\".join(SpiderFootHelpers.usernamesFromWordlists(['generic-usernames'])),\n '__database': f\"{SpiderFootHelpers.dataPath()}/spiderfoot.test.db\", # note: test database file\n '__modules__': None, # List of modules. Will be set after start-up.\n '__correlationrules__': None, # List of correlation rules. Will be set after start-up.\n '_socks1type': '',\n '_socks2addr': '',\n '_socks3port': '',\n '_socks4user': '',\n '_socks5pwd': '',\n '__logstdout': False\n }\n\n default_web_config = {\n 'root': '/'\n }\n\n mod_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../modules/'\n default_config['__modules__'] = SpiderFootHelpers.loadModulesAsDict(mod_dir, ['sfp_template.py'])\n\n conf = {\n '/query': {\n 'tools.encode.text_only': False,\n 'tools.encode.add_charset': True,\n },\n '/static': {\n 'tools.staticdir.on': True,\n 'tools.staticdir.dir': 'static',\n 'tools.staticdir.root': f\"{os.path.dirname(os.path.abspath(__file__))}/../../spiderfoot\",\n }\n }\n\n cherrypy.tree.mount(SpiderFootWebUi(default_web_config, default_config), script_name=default_web_config.get('root'), config=conf)\n\n def test_invalid_page_returns_404(self):\n self.getPage(\"/doesnotexist\")\n self.assertStatus('404 Not Found')\n\n def test_static_returns_200(self):\n self.getPage(\"/static/img/spiderfoot-header.png\")\n self.assertStatus('200 OK')\n\n def test_scaneventresultexport_invalid_scan_id_returns_200(self):\n self.getPage(\"/scaneventresultexport?id=doesnotexist&type=doesnotexist\")\n self.assertStatus('200 OK')\n\n def test_scaneventresultexportmulti(self):\n self.getPage(\"/scaneventresultexportmulti?ids=doesnotexist\")\n self.assertStatus('200 OK')\n\n def test_scansearchresultexport(self):\n self.getPage(\"/scansearchresultexport?id=doesnotexist\")\n self.assertStatus('200 OK')\n\n def test_scanexportjsonmulti(self):\n self.getPage(\"/scanexportjsonmulti?ids=doesnotexist\")\n self.assertStatus('200 OK')\n\n def test_scanviz(self):\n self.getPage(\"/scanviz?id=doesnotexist\")\n self.assertStatus('200 OK')\n\n def test_scanvizmulti(self):\n self.getPage(\"/scanvizmulti?ids=doesnotexist\")\n self.assertStatus('200 OK')\n\n def test_scanopts_invalid_scan_returns_200(self):\n self.getPage(\"/scanopts?id=doesnotexist\")\n self.assertStatus('200 OK')\n\n def test_rerunscan(self):\n self.getPage(\"/rerunscan?id=doesnotexist\")\n self.assertStatus('200 OK')\n self.assertInBody(\"Invalid scan ID.\")\n\n def test_rerunscanmulti_invalid_scan_id_returns_200(self):\n self.getPage(\"/rerunscanmulti?ids=doesnotexist\")\n self.assertStatus('200 OK')\n self.assertInBody(\"Invalid scan ID.\")\n\n def test_newscan_returns_200(self):\n self.getPage(\"/newscan\")\n self.assertStatus('200 OK')\n self.assertInBody(\"Scan Name\")\n self.assertInBody(\"Scan Target\")\n\n def test_clonescan(self):\n self.getPage(\"/clonescan?id=doesnotexist\")\n self.assertStatus('200 OK')\n self.assertInBody(\"Invalid scan ID.\")\n\n def test_index_returns_200(self):\n self.getPage(\"/\")\n self.assertStatus('200 OK')\n\n def test_scaninfo_invalid_scan_returns_200(self):\n self.getPage(\"/scaninfo?id=doesnotexist\")\n self.assertStatus('200 OK')\n self.assertInBody(\"Scan ID not found.\")\n\n @unittest.skip(\"todo\")\n def test_opts_returns_200(self):\n self.getPage(\"/opts\")\n self.assertStatus('200 OK')\n\n def test_optsexport(self):\n self.getPage(\"/optsexport\")\n self.assertStatus('200 OK')\n self.getPage(\"/optsexport?pattern=api_key\")\n self.assertStatus('200 OK')\n self.assertHeader(\"Content-Disposition\", \"attachment; filename=\\\"SpiderFoot.cfg\\\"\")\n self.assertInBody(\":api_key=\")\n\n def test_optsraw(self):\n self.getPage(\"/optsraw\")\n self.assertStatus('200 OK')\n\n def test_scandelete_invalid_scan_id_returns_404(self):\n self.getPage(\"/scandelete?id=doesnotexist\")\n self.assertStatus('404 Not Found')\n self.assertInBody('Scan doesnotexist does not exist')\n\n @unittest.skip(\"todo\")\n def test_savesettings(self):\n self.getPage(\"/savesettings\")\n self.assertStatus('200 OK')\n\n @unittest.skip(\"todo\")\n def test_savesettingsraw(self):\n self.getPage(\"/savesettingsraw\")\n self.assertStatus('200 OK')\n\n def test_resultsetfp(self):\n self.getPage(\"/resultsetfp?id=doesnotexist&resultids=doesnotexist&fp=1\")\n self.assertStatus('200 OK')\n self.assertInBody(\"No IDs supplied.\")\n\n def test_eventtypes(self):\n self.getPage(\"/eventtypes\")\n self.assertStatus('200 OK')\n self.assertInBody('\"DOMAIN_NAME\"')\n\n def test_modules(self):\n self.getPage(\"/modules\")\n self.assertStatus('200 OK')\n self.assertInBody('\"name\":')\n\n def test_ping_returns_200(self):\n self.getPage(\"/ping\")\n self.assertStatus('200 OK')\n self.assertInBody('\"SUCCESS\"')\n\n def test_query_returns_200(self):\n self.getPage(\"/query?query=SELECT+1\")\n self.assertStatus('200 OK')\n self.assertInBody('[{\"1\": 1}]')\n\n def test_startscan_invalid_scan_name_returns_error(self):\n self.getPage(\"/startscan?scanname=&scantarget=&modulelist=&typelist=&usecase=\")\n self.assertStatus('200 OK')\n self.assertInBody('Invalid request: scan name was not specified.')\n\n def test_startscan_invalid_scan_target_returns_error(self):\n self.getPage(\"/startscan?scanname=example-scan&scantarget=&modulelist=&typelist=&usecase=\")\n self.assertStatus('200 OK')\n self.assertInBody('Invalid request: scan target was not specified.')\n\n def test_startscan_unrecognized_scan_target_returns_error(self):\n self.getPage(\"/startscan?scanname=example-scan&scantarget=invalid-target&modulelist=doesnotexist&typelist=doesnotexist&usecase=doesnotexist\")\n self.assertStatus('200 OK')\n self.assertInBody('Invalid target type. Could not recognize it as a target SpiderFoot supports.')\n\n def test_startscan_invalid_modules_returns_error(self):\n self.getPage(\"/startscan?scanname=example-scan&scantarget=spiderfoot.net&modulelist=&typelist=&usecase=\")\n self.assertStatus('200 OK')\n self.assertInBody('Invalid request: no modules specified for scan.')\n\n def test_startscan_invalid_typelist_returns_error(self):\n self.getPage(\"/startscan?scanname=example-scan&scantarget=spiderfoot.net&modulelist=&typelist=doesnotexist&usecase=\")\n self.assertStatus('200 OK')\n self.assertInBody('Invalid request: no modules specified for scan.')\n\n def test_startscan_should_start_a_scan(self):\n self.getPage(\"/startscan?scanname=spiderfoot.net&scantarget=spiderfoot.net&modulelist=doesnotexist&typelist=doesnotexist&usecase=doesnotexist\")\n self.assertStatus('303 See Other')\n\n def test_stopscan_invalid_scan_id_returns_404(self):\n self.getPage(\"/stopscan?id=doesnotexist\")\n self.assertStatus('404 Not Found')\n self.assertInBody('Scan doesnotexist does not exist')\n\n def test_scanlog_invalid_scan_returns_200(self):\n self.getPage(\"/scanlog?id=doesnotexist\")\n self.assertStatus('200 OK')\n\n def test_scanerrors_invalid_scan_returns_200(self):\n self.getPage(\"/scanerrors?id=doesnotexist\")\n self.assertStatus('200 OK')\n\n def test_scanlist_returns_200(self):\n self.getPage(\"/scanlist\")\n self.assertStatus('200 OK')\n\n def test_scanstatus_invalid_scan_returns_200(self):\n self.getPage(\"/scanstatus?id=doesnotexist\")\n self.assertStatus('200 OK')\n\n def test_scansummary_invalid_scan_returns_200(self):\n self.getPage(\"/scansummary?id=doesnotexist&by=anything\")\n self.assertStatus('200 OK')\n\n def test_scaneventresults_invalid_scan_returns_200(self):\n self.getPage(\"/scaneventresults?id=doesnotexist&eventType=anything\")\n self.assertStatus('200 OK')\n\n def test_scaneventresultsunique_invalid_scan_returns_200(self):\n self.getPage(\"/scaneventresultsunique?id=doesnotexist&eventType=anything\")\n self.assertStatus('200 OK')\n\n def test_search_returns_200(self):\n self.getPage(\"/search?id=doesnotexist&eventType=doesnotexist&value=doesnotexist\")\n self.assertStatus('200 OK')\n\n def test_scanhistory_invalid_scan_returns_200(self):\n self.getPage(\"/scanhistory?id=doesnotexist\")\n self.assertStatus('200 OK')\n\n def test_scanelementtypediscovery_invalid_scan_id_returns_200(self):\n self.getPage(\"/scanelementtypediscovery?id=doesnotexist&eventType=anything\")\n self.assertStatus('200 OK')\n" }, { "path": "test/requirements.txt", "content": "-r ../requirements.txt\ndarglint==1.8.1\ndlint==0.14.0\nflake8==5.0.4\nflake8-annotations==2.9.1\nflake8-blind-except==0.2.1\nflake8-bugbear==23.2.13\nflake8-builtins==2.1.0\nflake8-quotes==3.3.2\nflake8-return==1.2.0\nflake8-sfs==0.0.4\nflake8-simplify==0.19.3\npytest==7.2.1\npytest-cov==4.0.0\npytest-mock==3.10.0\npytest-xdist==3.2.0\npycodestyle==2.9.0\nresponses==0.22.0\n" }, { "path": "test/run", "content": "#!/bin/bash\n# Run unit and integration tests (excluding module integration tests).\n# These same tests are run on all pull requests automatically.\n#\n# Must be run from SpiderFoot root directory; ie:\n# ./test/run\n\necho Running flake8 ...\ntime python3 -m flake8 . --count --show-source --statistics || exit\n\necho Running pytest ...\ntime python3 -m pytest -n auto --dist loadfile --ignore=test/integration/modules/ --durations=5 --cov-report html --cov=. .\n" }, { "path": "test/unit/__init__.py", "content": "" }, { "path": "test/unit/modules/__init__.py", "content": "" }, { "path": "test/unit/modules/test_sfp__stor_db.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp__stor_db import sfp__stor_db\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleStor_db(unittest.TestCase):\n\n @unittest.skip(\"This module contains an extra private option\")\n def test_opts(self):\n module = sfp__stor_db()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp__stor_db()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp__stor_db()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp__stor_db()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp__stor_stdout.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp__stor_stdout import sfp__stor_stdout\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleStor_stdout(unittest.TestCase):\n\n @unittest.skip(\"This module contains an extra private option\")\n def test_opts(self):\n module = sfp__stor_stdout()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp__stor_stdout()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp__stor_stdout()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp__stor_stdout()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_abstractapi.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_abstractapi import sfp_abstractapi\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleAbstractapi(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_abstractapi()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_abstractapi()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_abstractapi()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_abstractapi()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"204\", \"429\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_abstractapi()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"422\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_abstractapi()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_abstractapi()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_abusech.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_abusech import sfp_abusech\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleAbusech(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_abusech()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_abusech()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_abusech()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_abusech()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_abuseipdb.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_abuseipdb import sfp_abuseipdb\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleAbuseipdb(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_abuseipdb()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_abuseipdb()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_abuseipdb()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_abuseipdb()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_abuseipdb()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_abusix.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_abusix import sfp_abusix\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleAbusix(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_abusix()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_abusix()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_abusix()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_abusix()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_abusix()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_accounts.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_accounts import sfp_accounts\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleAccounts(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_accounts()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_accounts()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_accounts()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_accounts()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_adblock.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_adblock import sfp_adblock\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleAdblock(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_adblock()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_adblock()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_adblock()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_adblock()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_adguard_dns.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_adguard_dns import sfp_adguard_dns\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleAdGuardDns(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_adguard_dns()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_adguard_dns()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_adguard_dns()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_adguard_dns()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_ahmia.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ahmia import sfp_ahmia\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleAhmia(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_ahmia()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ahmia()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_ahmia()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_ahmia()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_alienvault.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_alienvault import sfp_alienvault\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleAlienvault(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_alienvault()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_alienvault()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_alienvault()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_alienvault()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"404\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_alienvault()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n # TODO: http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n http_codes = [\"403\", \"429\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_alienvault()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_alienvault()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_alienvaultiprep.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_alienvaultiprep import sfp_alienvaultiprep\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleAlienvaultiprep(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_alienvaultiprep()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_alienvaultiprep()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_alienvaultiprep()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_alienvaultiprep()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_apple_itunes.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_apple_itunes import sfp_apple_itunes\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleAppleItunes(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_apple_itunes()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_apple_itunes()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_apple_itunes()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_apple_itunes()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_archiveorg.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_archiveorg import sfp_archiveorg\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleArchiveorg(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_archiveorg()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_archiveorg()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_archiveorg()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_archiveorg()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_arin.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_arin import sfp_arin\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleArin(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_arin()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_arin()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_arin()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_arin()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_azureblobstorage.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_azureblobstorage import sfp_azureblobstorage\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleAzureblobstorage(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_azureblobstorage()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_azureblobstorage()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_azureblobstorage()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_azureblobstorage()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_base64.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_base64 import sfp_base64\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleBase64(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_base64()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_base64()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_base64()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_base64()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_url_containing_base64_string_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_base64()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'BASE64_DATA'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"U3BpZGVyRm9vdA== (SpiderFoot)\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_base64)\n\n event_type = 'ROOT'\n event_data = 'https://spiderfoot.net/path?param=example%20data%20U3BpZGVyRm9vdA%3d%3d%20example%20data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_not_containing_base64_string_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_base64()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_base64)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_bgpview.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_bgpview import sfp_bgpview\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleBgpview(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_bgpview()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_bgpview()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_bgpview()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_bgpview()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_binaryedge.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_binaryedge import sfp_binaryedge\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleBinaryedge(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_binaryedge()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_binaryedge()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_binaryedge()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_binaryedge()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_binaryedge()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_bingsearch.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_bingsearch import sfp_bingsearch\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleBingsearch(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_bingsearch()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_bingsearch()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_bingsearch()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_bingsearch()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bingsearch()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_bingsharedip.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_bingsharedip import sfp_bingsharedip\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleBingsharedip(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_bingsharedip()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_bingsharedip()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_bingsharedip()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_bingsharedip()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bingsharedip()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_binstring.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_binstring import sfp_binstring\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleBinstring(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_binstring()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_binstring()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_binstring()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_binstring()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_bitcoin.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_bitcoin import sfp_bitcoin\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleBitcoin(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_bitcoin()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_bitcoin()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_bitcoin()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_bitcoin()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_containing_bitcoin_string_in_legacy_base58_format_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bitcoin()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'BITCOIN_ADDRESS'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = '1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_bitcoin)\n\n event_type = 'ROOT'\n event_data = 'example data 1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_containing_bitcoin_string_in_bech32_format_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bitcoin()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'BITCOIN_ADDRESS'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'bc1q4r8h8vqk02gnvlus758qmpk8jmajpy2ld23xtr73a39ps0r9z82qq0qqye'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_bitcoin)\n\n event_type = 'ROOT'\n event_data = 'example data bc1q4r8h8vqk02gnvlus758qmpk8jmajpy2ld23xtr73a39ps0r9z82qq0qqye example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_not_containing_bitcoin_string_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bitcoin()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_bitcoin)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_bitcoinabuse.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_bitcoinabuse import sfp_bitcoinabuse\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleBitcoinAbuse(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_bitcoinabuse()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_bitcoinabuse()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_bitcoinabuse()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_bitcoinabuse()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"404\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_bitcoinabuse()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_bitcoinabuse()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bitcoinabuse()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_bitcoinwhoswho.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_bitcoinwhoswho import sfp_bitcoinwhoswho\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleBitcoinwhoswho(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_bitcoinwhoswho()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bitcoinwhoswho()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_bitcoinwhoswho()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_bitcoinwhoswho()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_bitcoinwhoswho()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_blockchain.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_blockchain import sfp_blockchain\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleBlockchain(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_blockchain()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_blockchain()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_blockchain()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_blockchain()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_blocklistde.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_blocklistde import sfp_blocklistde\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleBlocklistde(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_blocklistde()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_blocklistde()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_blocklistde()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_blocklistde()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_botscout.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_botscout import sfp_botscout\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleBotscout(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_botscout()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_botscout()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_botscout()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_botscout()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_botscout()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_botscout()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_botvrij.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_botvrij import sfp_botvrij\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModulebotvrij(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_botvrij()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_botvrij()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_botvrij()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_botvrij()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_builtwith.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_builtwith import sfp_builtwith\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModulebuiltwith(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_builtwith()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_builtwith()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_builtwith()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_builtwith()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_builtwith()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_c99.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_c99 import sfp_c99\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleC99(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_c99()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_c99()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_c99()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_c99()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_c99()\n module.setup(sf, dict())\n\n target_value = \"example target value\"\n target_type = \"IP_ADDRESS\"\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = \"ROOT\"\n event_data = \"example data\"\n event_module = \"\"\n source_event = \"\"\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_callername.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_callername import sfp_callername\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCallername(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_callername()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_callername()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_callername()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_callername()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_censys.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_censys import sfp_censys\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleCensys(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_censys()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_censys()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_censys()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_censys()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"400\", \"404\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_censys()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_censys()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_censys()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_certspotter.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_certspotter import sfp_certspotter\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleiCertspotter(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_certspotter()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_certspotter()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_certspotter()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_certspotter()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_certspotter()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_cinsscore.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_cinsscore import sfp_cinsscore\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCinsscore(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_cinsscore()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_cinsscore()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_cinsscore()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_cinsscore()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_circllu.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_circllu import sfp_circllu\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleCircllu(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_circllu()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_circllu()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_circllu()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_circllu()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_circllu()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_citadel.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_citadel import sfp_citadel\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModulecitadel(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_citadel()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_citadel()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_citadel()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_citadel()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_cleanbrowsing.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_cleanbrowsing import sfp_cleanbrowsing\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCleanbrowsing(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_cleanbrowsing()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_cleanbrowsing()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_cleanbrowsing()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_cleanbrowsing()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_cleantalk.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_cleantalk import sfp_cleantalk\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCleantalk(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_cleantalk()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_cleantalk()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_cleantalk()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_cleantalk()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_clearbit.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_clearbit import sfp_clearbit\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleClearbit(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_clearbit()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_clearbit()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_clearbit()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_clearbit()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"404\", \"429\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_clearbit()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_clearbit()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_clearbit()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_cloudflaredns.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_cloudflaredns import sfp_cloudflaredns\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCloudflaredns(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_cloudflaredns()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_cloudflaredns()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_cloudflaredns()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_cloudflaredns()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_coinblocker.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_coinblocker import sfp_coinblocker\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCoinblocker(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_coinblocker()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_coinblocker()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_coinblocker()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_coinblocker()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_commoncrawl.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_commoncrawl import sfp_commoncrawl\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCommoncrawl(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_commoncrawl()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_commoncrawl()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_commoncrawl()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_commoncrawl()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_comodo.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_comodo import sfp_comodo\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleComodo(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_comodo()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_comodo()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_comodo()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_comodo()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_company.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_company import sfp_company\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleCompany(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_company()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_company()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_company()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_company()\n self.assertIsInstance(module.producedEvents(), list)\n\n @unittest.skip(\"todo\")\n def test_handleEvent_event_data_ssl_certificate_issued_containing_company_name_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_company()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'COMPANY_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"SpiderFoot Corporation\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_company)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'SSL_CERTIFICATE_ISSUED'\n event_data = 'O=SpiderFoot Corporation'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n @unittest.skip(\"todo\")\n def test_handleEvent_event_data_domain_whois_containing_company_name_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_company()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'COMPANY_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"SpiderFoot Corporation\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_company)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'DOMAIN_WHOIS'\n event_data = 'Registrant Organization: SpiderFoot Corporation'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n @unittest.skip(\"todo\")\n def test_handleEvent_event_data_target_web_content_containing_company_name_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_company()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'COMPANY_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"SpiderFoot Corporation\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_company)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'TARGET_WEB_CONTENT'\n event_data = '

    Copyright SpiderFoot Corporation 2022.

    '\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n" }, { "path": "test/unit/modules/test_sfp_cookie.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_cookie import sfp_cookie\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleCookie(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_cookie()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_cookie()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_cookie()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_cookie()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_containing_cookie_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_cookie()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'TARGET_WEB_COOKIE'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'example cookie'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_cookie)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'WEBSERVER_HTTPHEADERS'\n event_data = '{\"cookie\": \"example cookie\"}'\n event_module = 'sfp_spider'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n evt.actualSource = \"https://spiderfoot.net/example\"\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_not_containing_cookie_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_cookie()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_cookie)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'WEBSERVER_HTTPHEADERS'\n event_data = '{\"not cookie\": \"example cookie\"}'\n event_module = 'sfp_spider'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n evt.actualSource = \"https://spiderfoot.net/example\"\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_countryname.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_countryname import sfp_countryname\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleCountryName(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_countryname()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_countryname()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_countryname()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_countryname()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_phone_number_event_data_containing_countrycode_should_create_country_name_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_countryname()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'COUNTRY_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"United States\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_countryname)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'PHONE_NUMBER'\n event_data = '+12345678901'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_domain_whois_event_data_containing_countryname_string_should_create_country_name_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_countryname()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'COUNTRY_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"United States\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_countryname)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'DOMAIN_WHOIS'\n event_data = 'example data 123 Fake St, Fakeville, \"United States\" example data'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_domain_whois_event_data_not_containing_countryname_string_should_not_create_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_countryname()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_countryname)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'DOMAIN_WHOIS'\n event_data = 'example data'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_creditcard.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_creditcard import sfp_creditcard\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleCreditCard(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_creditcard()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_creditcard()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_creditcard()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_creditcard()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_containing_creditcard_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_creditcard()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'CREDIT_CARD_NUMBER'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = '4111111111111111'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_creditcard)\n\n event_type = 'ROOT'\n event_data = 'example data 4111 1111 1111 1111 example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_not_containing_creditcard_string_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_creditcard()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_creditcard)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_crobat_api.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_crobat_api import sfp_crobat_api\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCrobatApi(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_crobat_api()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_crobat_api()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_crobat_api()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_crobat_api()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_crobat_api()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_crobat_api()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_crossref.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_crossref import sfp_crossref\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCrossref(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_crossref()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_crossref()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_crossref()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_crossref()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_crt.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_crt import sfp_crt\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCrt(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_crt()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_crt()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_crt()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_crt()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"404\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_crt()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_crt()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_crxcavator.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_crxcavator import sfp_crxcavator\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCrxcavator(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_crxcavator()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_crxcavator()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_crxcavator()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_crxcavator()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_customfeed.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_customfeed import sfp_customfeed\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleCustomfeed(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_customfeed()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_customfeed()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_customfeed()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_customfeed()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_feed_url_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_customfeed()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_cybercrimetracker.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_cybercrimetracker import sfp_cybercrimetracker\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCybercrimetracker(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_cybercrimetracker()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_cybercrimetracker()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_cybercrimetracker()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_cybercrimetracker()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_debounce.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_debounce import sfp_debounce\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDebounce(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_debounce()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_debounce()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_debounce()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_debounce()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_dehashed.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dehashed import sfp_dehashed\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleDehashed(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_dehashed()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dehashed()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_dehashed()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_dehashed()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dehashed()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_digitaloceanspace.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_digitaloceanspace import sfp_digitaloceanspace\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDigitaloceanspace(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_digitaloceanspace()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_digitaloceanspace()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_digitaloceanspace()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_digitaloceanspace()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_dns_for_family.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dns_for_family import sfp_dns_for_family\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDnsForFamily(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_dns_for_family()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_dns_for_family()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_dns_for_family()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_dns_for_family()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_dnsbrute.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsbrute import sfp_dnsbrute\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDnsBrute(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_dnsbrute()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_dnsbrute()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_dnsbrute()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_dnsbrute()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_dnscommonsrv.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnscommonsrv import sfp_dnscommonsrv\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDnsCommonsrv(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_dnscommonsrv()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_dnscommonsrv()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_dnscommonsrv()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_dnscommonsrv()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_dnsdb.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsdb import sfp_dnsdb\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleDnsDb(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_dnsdb()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_dnsdb()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_dnsdb()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_dnsdb()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_dnsdb()\n module.setup(sf, dict())\n\n target_value = \"example target value\"\n target_type = \"IP_ADDRESS\"\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = \"ROOT\"\n event_data = \"example data\"\n event_module = \"\"\n source_event = \"\"\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_dnsdumpster.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsdumpster import sfp_dnsdumpster\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDnsDumpster(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_dnsdumpster()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_dnsdumpster()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_dnsdumpster()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_dnsdumpster()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_dnsgrep.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsgrep import sfp_dnsgrep\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDnsGrep(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_dnsgrep()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_dnsgrep()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_dnsgrep()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_dnsgrep()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_dnsneighbor.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsneighbor import sfp_dnsneighbor\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDnsNeighbor(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_dnsneighbor()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_dnsneighbor()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_dnsneighbor()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_dnsneighbor()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_dnsraw.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsraw import sfp_dnsraw\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDnsRaw(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_dnsraw()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_dnsraw()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_dnsraw()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_dnsraw()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_dnsresolve.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnsresolve import sfp_dnsresolve\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDnsResolve(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_dnsresolve()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_dnsresolve()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_dnsresolve()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_dnsresolve()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_dnszonexfer.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dnszonexfer import sfp_dnszonexfer\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDnsZonexfer(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_dnszonexfer()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_dnszonexfer()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_dnszonexfer()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_dnszonexfer()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_dronebl.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_dronebl import sfp_dronebl\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDronebl(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_dronebl()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_dronebl()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_dronebl()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_dronebl()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_duckduckgo.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_duckduckgo import sfp_duckduckgo\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleDuckduckgo(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_duckduckgo()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_duckduckgo()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_duckduckgo()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_duckduckgo()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_email.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_email import sfp_email\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleEmail(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_email()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_email()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_email()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_email()\n self.assertIsInstance(module.producedEvents(), list)\n\n @unittest.skip(\"todo\")\n def test_handleEvent_event_data_target_web_content_containing_email_address_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_email()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'EMAILADDR'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'firstname.lastname@spiderfoot.net'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_email)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'TARGET_WEB_CONTENT'\n event_data = '

    sample data firstname.lastname@spiderfoot.net sample data.

    '\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n" }, { "path": "test/unit/modules/test_sfp_emailcrawlr.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_emailcrawlr import sfp_emailcrawlr\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleEmailcrawlr(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_emailcrawlr()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_emailcrawlr()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_emailcrawlr()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_emailcrawlr()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"404\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_emailcrawlr()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_emailcrawlr()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_emailcrawlr()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_emailformat.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_emailformat import sfp_emailformat\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleEmailformat(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_emailformat()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_emailformat()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_emailformat()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_emailformat()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_emailrep.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_emailrep import sfp_emailrep\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleEmailrep(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_emailrep()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_emailrep()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_emailrep()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_emailrep()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_emergingthreats.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_emergingthreats import sfp_emergingthreats\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleEmergingthreats(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_emergingthreats()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_emergingthreats()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_emergingthreats()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_emergingthreats()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_errors.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_errors import sfp_errors\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleErrors(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_errors()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_errors()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_errors()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_errors()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_should_only_handle_events_from_sfp_spider(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_errors()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_errors)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'TARGET_WEB_CONTENT'\n event_data = 'example data Internal Server Error example data'\n event_module = 'something else entirely'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n evt.actualSource = 'https://spiderfoot.net/'\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n\n def test_handleEvent_should_only_handle_events_within_target_scope(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_errors()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_errors)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'TARGET_WEB_CONTENT'\n event_data = 'example data Internal Server Error example data'\n event_module = 'sfp_spider'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n evt.actualSource = 'https://something.else.entirely/'\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n\n def test_handleEvent_event_data_containing_error_string_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_errors()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'ERROR_MESSAGE'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'Generic Error'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_errors)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n self.assertIsNone(result)\n\n event_type = 'TARGET_WEB_CONTENT'\n event_data = 'example data Internal Server Error example data'\n event_module = 'sfp_spider'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n evt.actualSource = 'https://spiderfoot.net/'\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_not_containing_error_string_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_errors()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_errors)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n event_type = 'TARGET_WEB_CONTENT'\n event_data = 'example data'\n event_module = 'sfp_spider'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n evt.actualSource = 'https://spiderfoot.net/'\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_ethereum.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ethereum import sfp_ethereum\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleEthereum(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_ethereum()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_ethereum()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_ethereum()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_ethereum()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_containing_ethereum_string_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ethereum()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'ETHEREUM_ADDRESS'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = '0x89205A3A3b2A69De6Dbf7f01ED13B2108B2c43e7'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_ethereum)\n\n event_type = 'ROOT'\n event_data = 'example data 0x89205A3A3b2A69De6Dbf7f01ED13B2108B2c43e7 example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_not_containing_ethereum_string_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ethereum()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_ethereum)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_etherscan.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_etherscan import sfp_etherscan\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleEtherscan(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_etherscan()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_etherscan()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_etherscan()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_etherscan()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_etherscan()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_filemeta.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_filemeta import sfp_filemeta\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleFilemeta(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_filemeta()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_filemeta()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_filemeta()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_filemeta()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_flickr.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_flickr import sfp_flickr\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleFlickr(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_flickr()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_flickr()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_flickr()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_flickr()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_focsec.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_focsec import sfp_focsec\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleFocsec(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_focsec()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_focsec()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_focsec()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_focsec()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_focsec()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_fortinet.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_fortinet import sfp_fortinet\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleFortinet(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_fortinet()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_fortinet()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_fortinet()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_fortinet()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_fraudguard.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_fraudguard import sfp_fraudguard\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleFraudguard(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_fraudguard()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_fraudguard()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_fraudguard()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_fraudguard()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_fraudguard()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_fsecure_riddler.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_fsecure_riddler import sfp_fsecure_riddler\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleFsecureRiddler(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_fsecure_riddler()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_fsecure_riddler()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_fsecure_riddler()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_fsecure_riddler()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_fsecure_riddler()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_fullcontact.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_fullcontact import sfp_fullcontact\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleFullcontact(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_fullcontact()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_fullcontact()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_fullcontact()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_fullcontact()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_fullcontact()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_fullhunt.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_fullhunt import sfp_fullhunt\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleFullhunt(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_fullhunt()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_fullhunt()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_fullhunt()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_fullhunt()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"400\", \"404\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_fullhunt()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"403\", \"429\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_fullhunt()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_fullhunt()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_github.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_github import sfp_github\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleGithub(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_github()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_github()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_github()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_github()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_social_media_not_github_profile_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_github()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_github)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'SOCIAL_MEDIA'\n event_data = 'Not GitHub: example_username'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_gleif.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_gleif import sfp_gleif\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleGleif(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_gleif()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_gleif()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_gleif()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_gleif()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_invalid_lei_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_gleif()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_gleif)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'LEI'\n event_data = 'invalid LEI'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_google_tag_manager.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_google_tag_manager import sfp_google_tag_manager\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModulesGoogleTagManager(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_google_tag_manager()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_google_tag_manager()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_google_tag_manager()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_google_tag_manager()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_googlemaps.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_googlemaps import sfp_googlemaps\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleGoogleMaps(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_googlemaps()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_googlemaps()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_googlemaps()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_googlemaps()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_googlemaps()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_googleobjectstorage.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_googleobjectstorage import sfp_googleobjectstorage\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleGoogleobjectstorage(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_googleobjectstorage()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_googleobjectstorage()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_googleobjectstorage()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_googleobjectstorage()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_googlesafebrowsing.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_googlesafebrowsing import sfp_googlesafebrowsing\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleGoogleSafebrowsing(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_googlesafebrowsing()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_googlesafebrowsing()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_googlesafebrowsing()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_googlesafebrowsing()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_googlesafebrowsing()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_googlesearch.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_googlesearch import sfp_googlesearch\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleGoogleSearch(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_googlesearch()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_googlesearch()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_googlesearch()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_googlesearch()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_googlesearch()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_gravatar.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_gravatar import sfp_gravatar\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleGravatar(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_gravatar()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_gravatar()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_gravatar()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_gravatar()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_grayhatwarfare.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_grayhatwarfare import sfp_grayhatwarfare\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleGrayhatWarfare(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_grayhatwarfare()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_grayhatwarfare()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_grayhatwarfare()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_grayhatwarfare()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_grayhatwarfare()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_greensnow.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_greensnow import sfp_greensnow\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleGreensnow(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_greensnow()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_greensnow()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_greensnow()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_greensnow()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_grep_app.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_grep_app import sfp_grep_app\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleGrepApp(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_grep_app()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_grep_app()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_grep_app()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_grep_app()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_greynoise.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_greynoise import sfp_greynoise\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleGreynoise(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_greynoise()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_greynoise()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_greynoise()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_greynoise()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_greynoise()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_h1nobbdde.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_h1nobbdde import sfp_h1nobbdde\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleH1nobbdde(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_h1nobbdde()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_h1nobbdde()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_h1nobbdde()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_h1nobbdde()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_hackertarget.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_hackertarget import sfp_hackertarget\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleHackertarget(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_hackertarget()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_hackertarget()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_hackertarget()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_hackertarget()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_hashes.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_hashes import sfp_hashes\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleHashes(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_hashes()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_hashes()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_hashes()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_hashes()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_containing_hashes_string_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_hashes()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'HASH'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"[MD5] e17cff4eb3e8fbe6ca3b83fb47532dba\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_hashes)\n\n event_type = 'ROOT'\n event_data = 'example data e17cff4eb3e8fbe6ca3b83fb47532dba example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_not_containing_hashes_string_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_hashes()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_hashes)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_haveibeenpwned.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_haveibeenpwned import sfp_haveibeenpwned\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleHaveibeenpwned(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_haveibeenpwned()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_haveibeenpwned()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_haveibeenpwned()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_haveibeenpwned()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_haveibeenpwned()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_honeypot.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_honeypot import sfp_honeypot\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleHoneypot(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_honeypot()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_honeypot()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_honeypot()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_honeypot()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_honeypot()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_hosting.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_hosting import sfp_hosting\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleHosting(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_hosting()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_hosting()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_hosting()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_hosting()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_hostio.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_hostio import sfp_hostio\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleHostio(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_hostio()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_hostio()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_hostio()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_hostio()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_hostio()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_hunter.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_hunter import sfp_hunter\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleHunter(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_hunter()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_hunter()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_hunter()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_hunter()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_hunter()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_hybrid_analysis.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_hybrid_analysis import sfp_hybrid_analysis\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleHybridAnalysis(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_hybrid_analysis()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_hybrid_analysis()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_hybrid_analysis()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_hybrid_analysis()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"400\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_hybrid_analysis()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_hybrid_analysis()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_iban.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_iban import sfp_iban\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIban(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_iban()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_iban()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_iban()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_iban()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_containing_iban_string_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_iban()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'IBAN_NUMBER'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'BE71096123456769'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_iban)\n\n event_type = 'ROOT'\n event_data = 'BE71096123456769'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_not_containing_iban_string_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_iban()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_iban)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_iknowwhatyoudownload.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_iknowwhatyoudownload import sfp_iknowwhatyoudownload\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIknowwhatyoudownload(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_iknowwhatyoudownload()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_iknowwhatyoudownload()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_iknowwhatyoudownload()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_iknowwhatyoudownload()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_iknowwhatyoudownload()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_intelx.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_intelx import sfp_intelx\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntelx(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_intelx()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_intelx()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_intelx()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_intelx()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_intelx()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_intfiles.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_intfiles import sfp_intfiles\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIntfiles(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_intfiles()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_intfiles()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_intfiles()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_intfiles()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_internal_url_with_interesting_file_extension_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_intfiles()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'INTERESTING_FILE'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'https://spiderfoot.net/example.zip'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_intfiles)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'LINKED_URL_INTERNAL'\n event_data = 'https://spiderfoot.net/example.zip'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_internal_url_without_interesting_file_extension_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_intfiles()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_intfiles)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'LINKED_URL_INTERNAL'\n event_data = 'https://spiderfoot.net/example'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_ipapico.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ipapico import sfp_ipapico\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleipApico(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_ipapico()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipapico()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_ipapico()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_ipapico()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_ipapicom.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ipapicom import sfp_ipapicom\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleipapicom(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_ipapicom()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipapicom()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_ipapicom()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_ipapicom()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipapicom()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_ipinfo.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ipinfo import sfp_ipinfo\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIpinfo(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_ipinfo()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_ipinfo()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_ipinfo()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_ipinfo()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipinfo()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_ipqualityscore.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ipqualityscore import sfp_ipqualityscore\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIpqualityscore(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_ipqualityscore()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_ipqualityscore()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_ipqualityscore()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_ipqualityscore()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipqualityscore()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'PHONE_NUMBER'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_ipregistry.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ipregistry import sfp_ipregistry\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIpRegistry(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_ipregistry()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipregistry()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_ipregistry()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_ipregistry()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipregistry()\n module.setup(sf, dict())\n\n target_value = \"example target value\"\n target_type = \"IP_ADDRESS\"\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = \"ROOT\"\n event_data = \"example data\"\n event_module = \"\"\n source_event = \"\"\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_ipstack.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ipstack import sfp_ipstack\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleIpstack(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_ipstack()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_ipstack()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_ipstack()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_ipstack()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_ipstack()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_isc.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_isc import sfp_isc\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleisc(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_isc()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_isc()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_isc()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_isc()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_jsonwhoiscom.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_jsonwhoiscom import sfp_jsonwhoiscom\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleJsonwhoiscom(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_jsonwhoiscom()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_jsonwhoiscom()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_jsonwhoiscom()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_jsonwhoiscom()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"404\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_jsonwhoiscom()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_jsonwhoiscom()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_jsonwhoiscom()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_junkfiles.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_junkfiles import sfp_junkfiles\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleJunkfiles(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_junkfiles()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_junkfiles()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_junkfiles()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_junkfiles()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_keybase.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_keybase import sfp_keybase\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleKeybase(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_keybase()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_keybase()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_keybase()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_keybase()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_koodous.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_koodous import sfp_koodous\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleKoodous(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_koodous()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_koodous()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_koodous()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_koodous()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"404\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_koodous()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_koodous()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_koodous()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_leakix.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_leakix import sfp_leakix\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleLeakix(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_leakix()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_leakix()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_leakix()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_leakix()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"404\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_leakix()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_leakix()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_maltiverse.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_maltiverse import sfp_maltiverse\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleMaltiverse(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_maltiverse()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_maltiverse()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_maltiverse()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_maltiverse()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_malwarepatrol.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_malwarepatrol import sfp_malwarepatrol\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModulemalwarepatrol(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_malwarepatrol()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_malwarepatrol()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_malwarepatrol()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_malwarepatrol()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_malwarepatrol()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_metadefender.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_metadefender import sfp_metadefender\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleMetadefender(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_metadefender()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_metadefender()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_metadefender()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_metadefender()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"404\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_metadefender()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n # TODO http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n http_codes = [\"401\", \"429\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_metadefender()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_metadefender()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_mnemonic.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_mnemonic import sfp_mnemonic\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleMnemonic(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_mnemonic()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_mnemonic()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_mnemonic()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_mnemonic()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_multiproxy.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_multiproxy import sfp_multiproxy\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleMultiproxy(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_multiproxy()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_multiproxy()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_multiproxy()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_multiproxy()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_myspace.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_myspace import sfp_myspace\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleMyspace(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_myspace()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_myspace()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_myspace()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_myspace()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_social_media_not_myspace_profile_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_myspace()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_myspace)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'SOCIAL_MEDIA'\n event_data = 'Not MySpace: example_username'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_nameapi.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_nameapi import sfp_nameapi\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleNameapi(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_nameapi()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_nameapi()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_nameapi()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_nameapi()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_nameapi()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_names.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_names import sfp_names\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleNames(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_names()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_names()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_names()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_names()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_email_address_containing_human_names_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_names()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'HUMAN_NAME'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"Firstname Lastname\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_names)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'EMAILADDR'\n event_data = 'firstname.lastname@spiderfoot.net'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_email_address_containing_human_names_containing_numbers_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_names()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_names)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'EMAILADDR'\n event_data = 'firstname.lastname1@spiderfoot.net'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n\n def test_handleEvent_event_data_email_address_not_containing_names_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_names()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_names)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'EMAILADDR'\n event_data = 'lastname@spiderfoot.net'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_networksdb.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_networksdb import sfp_networksdb\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleNetworksdb(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_networksdb()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_networksdb()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_networksdb()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_networksdb()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"400\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_networksdb()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n # TODO: http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n http_codes = [\"403\", \"429\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_networksdb()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_networksdb()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_neutrinoapi.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_neutrinoapi import sfp_neutrinoapi\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleNeutrinoapi(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_neutrinoapi()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_neutrinoapi()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_neutrinoapi()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_neutrinoapi()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"400\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_neutrinoapi()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n # TODO: http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n http_codes = [\"403\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_neutrinoapi()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_neutrinoapi()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_numverify.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_numverify import sfp_numverify\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleNumverify(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_numverify()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_numverify()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_numverify()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_numverify()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_numverify()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_onioncity.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_onioncity import sfp_onioncity\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleonioncity(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_onioncity()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_onioncity()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_onioncity()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_onioncity()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_onioncity()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_onionsearchengine.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_onionsearchengine import sfp_onionsearchengine\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleOnionsearchengine(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_onionsearchengine()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_onionsearchengine()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_onionsearchengine()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_onionsearchengine()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_onyphe.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_onyphe import sfp_onyphe\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleOnyphe(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_onyphe()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_onyphe()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_onyphe()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_onyphe()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_onyphe()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_openbugbounty.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_openbugbounty import sfp_openbugbounty\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleOpenbugbounty(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_openbugbounty()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_openbugbounty()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_openbugbounty()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_openbugbounty()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_opencorporates.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_opencorporates import sfp_opencorporates\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleOpencorporates(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_opencorporates()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_opencorporates()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_opencorporates()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_opencorporates()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_opendns.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_opendns import sfp_opendns\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleOpendns(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_opendns()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_opendns()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_opendns()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_opendns()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_opennic.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_opennic import sfp_opennic\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleOpenNic(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_opennic()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_opennic()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_opennic()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_opennic()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_openphish.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_openphish import sfp_openphish\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleOpenphish(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_openphish()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_openphish()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_openphish()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_openphish()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_openstreetmap.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_openstreetmap import sfp_openstreetmap\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleopenstreetmap(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_openstreetmap()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_openstreetmap()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_openstreetmap()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_openstreetmap()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_pageinfo.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_pageinfo import sfp_pageinfo\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModulePageInfo(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_pageinfo()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_pageinfo()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_pageinfo()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_pageinfo()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_pastebin.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_pastebin import sfp_pastebin\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModulePastebin(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_pastebin()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_pastebin()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_pastebin()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_pastebin()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_pastebin()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_pgp.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_pgp import sfp_pgp\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModulePgp(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_pgp()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_pgp()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_pgp()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_pgp()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_keyserver_urls_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_pgp()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n module.opts['keyserver_search1'] = ''\n module.opts['keyserver_search2'] = ''\n module.opts['keyserver_fetch1'] = ''\n module.opts['keyserver_fetch2'] = ''\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_phishstats.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_phishstats import sfp_phishstats\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModulePhishstats(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_phishstats()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_phishstats()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_phishstats()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_phishstats()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_phishtank.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_phishtank import sfp_phishtank\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModulePhishtank(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_phishtank()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_phishtank()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_phishtank()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_phishtank()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_phone.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_phone import sfp_phone\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModulePhone(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_phone()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_phone()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_phone()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_phone()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_domain_whois_event_data_containing_phone_string_should_create_phone_number_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_phone()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'PHONE_NUMBER'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"+12025550111\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_phone)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'DOMAIN_WHOIS'\n event_data = 'example data +1 202 555 0111 example data'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_domain_whois_event_data_not_containing_phone_string_should_not_create_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_phone()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_phone)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'DOMAIN_WHOIS'\n event_data = 'example data'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n\n def test_handleEvent_phone_number_event_data_containing_phone_string_should_return_provider_telco_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_phone()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'PROVIDER_TELCO'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"Swisscom\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_phone)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'PHONE_NUMBER'\n event_data = '+41798765432'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n" }, { "path": "test/unit/modules/test_sfp_portscan_tcp.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_portscan_tcp import sfp_portscan_tcp\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModulePortscanTcp(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_portscan_tcp()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_portscan_tcp()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_portscan_tcp()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_portscan_tcp()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_projectdiscovery.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_projectdiscovery import sfp_projectdiscovery\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleProjectdiscovery(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_projectdiscovery()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_projectdiscovery()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_projectdiscovery()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_projectdiscovery()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_projectdiscovery()\n module.setup(sf, dict())\n\n target_value = \"example target value\"\n target_type = \"IP_ADDRESS\"\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = \"ROOT\"\n event_data = \"example data\"\n event_module = \"\"\n source_event = \"\"\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_psbdmp.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_psbdmp import sfp_psbdmp\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModulePsbdmp(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_psbdmp()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_psbdmp()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_psbdmp()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_psbdmp()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_pulsedive.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_pulsedive import sfp_pulsedive\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModulePulsedive(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_pulsedive()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_pulsedive()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_pulsedive()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_pulsedive()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_pulsedive()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_punkspider.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_punkspider import sfp_punkspider\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModulePunkspider(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_punkspider()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_punkspider()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_punkspider()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_punkspider()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"404\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_punkspider()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_punkspider()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_quad9.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_quad9 import sfp_quad9\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleQuad9(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_quad9()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_quad9()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_quad9()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_quad9()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_reversewhois.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_reversewhois import sfp_reversewhois\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleReversewhois(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_reversewhois()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_reversewhois()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_reversewhois()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_reversewhois()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_ripe.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_ripe import sfp_ripe\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleRipe(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_ripe()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_ripe()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_ripe()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_ripe()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_riskiq.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_riskiq import sfp_riskiq\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleRiskiq(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_riskiq()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_riskiq()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_riskiq()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_riskiq()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_riskiq()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_robtex.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_robtex import sfp_robtex\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleRobtex(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_robtex()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_robtex()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_robtex()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_robtex()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_s3bucket.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_s3bucket import sfp_s3bucket\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleS3bucket(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_s3bucket()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_s3bucket()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_s3bucket()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_s3bucket()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_searchcode.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_searchcode import sfp_searchcode\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleCodesearch(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_searchcode()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_searchcode()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_searchcode()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_searchcode()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_securitytrails.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_securitytrails import sfp_securitytrails\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleSecuritytrails(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_securitytrails()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_securitytrails()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_securitytrails()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_securitytrails()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_securitytrails()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_seon.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_seon import sfp_seon\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleSeon(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_seon()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_seon()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_seon()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_seon()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_seon()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_shodan.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_shodan import sfp_shodan\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleShodan(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_shodan()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_shodan()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_shodan()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_shodan()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_shodan()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_similar.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_similar import sfp_similar\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleSimilar(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_similar()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_similar()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_similar()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_similar()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_skymem.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_skymem import sfp_skymem\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleSkymem(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_skymem()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_skymem()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_skymem()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_skymem()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_slideshare.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_slideshare import sfp_slideshare\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleSlideshare(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_slideshare()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_slideshare()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_slideshare()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_slideshare()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_social_media_not_slideshare_profile_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_slideshare()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_slideshare)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'SOCIAL_MEDIA'\n event_data = 'Not SlideShare: example_username'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_snov.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_snov import sfp_snov\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleSnov(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_snov()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_snov()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_snov()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_snov()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_snov()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_social.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_social import sfp_social\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleSocial(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_social()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_social()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_social()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_social()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_url_containing_social_media_profile_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_social()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'SOCIAL_MEDIA'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"LinkedIn (Individual): https://linkedin.com/in/spiderfoot\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_social)\n\n event_type = 'ROOT'\n event_data = 'https://linkedin.com/in/spiderfoot'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_url_not_containing_social_media_profile_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_social()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_social)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_sociallinks.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_sociallinks import sfp_sociallinks\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleSociallinks(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_sociallinks()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_sociallinks()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_sociallinks()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_sociallinks()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_sociallinks()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_socialprofiles.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_socialprofiles import sfp_socialprofiles\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleSocialprofiles(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_socialprofiles()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_socialprofiles()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_socialprofiles()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_socialprofiles()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_socialprofiles()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'HUMAN_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_sorbs.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_sorbs import sfp_sorbs\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleSorbs(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_sorbs()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_sorbs()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_sorbs()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_sorbs()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_spamcop.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_spamcop import sfp_spamcop\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleSpamcop(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_spamcop()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_spamcop()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_spamcop()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_spamcop()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_spamhaus.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_spamhaus import sfp_spamhaus\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleSpamhaus(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_spamhaus()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_spamhaus()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_spamhaus()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_spamhaus()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_spider.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_spider import sfp_spider\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleSpider(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_spider()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_spider()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_spider()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_spider()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_spur.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_spur import sfp_spur\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleSpur(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_spur()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_spur()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_spur()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_spur()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_spur()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_spyonweb.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_spyonweb import sfp_spyonweb\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleSpyonweb(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_spyonweb()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_spyonweb()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_spyonweb()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_spyonweb()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_spyonweb()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_sslcert.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_sslcert import sfp_sslcert\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleSslCert(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_sslcert()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_sslcert()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_sslcert()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_sslcert()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_stackoverflow.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_stackoverflow import sfp_stackoverflow\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleStackoverflow(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_stackoverflow()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_stackoverflow()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_stackoverflow()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_stackoverflow()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_stevenblack_hosts.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_stevenblack_hosts import sfp_stevenblack_hosts\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleStevenblackHosts(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_stevenblack_hosts()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_stevenblack_hosts()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_stevenblack_hosts()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_stevenblack_hosts()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_strangeheaders.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_strangeheaders import sfp_strangeheaders\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleStrangeHeaders(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_strangeheaders()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_strangeheaders()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_strangeheaders()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_strangeheaders()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_containing_unusual_header_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_strangeheaders()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'WEBSERVER_STRANGEHEADER'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = 'unusual header: example header value'\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_strangeheaders)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'WEBSERVER_HTTPHEADERS'\n event_data = '{\"unusual header\": \"example header value\"}'\n event_module = 'sfp_spider'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n evt.actualSource = \"https://spiderfoot.net/example\"\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_not_containing_unusual_header_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_strangeheaders()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_strangeheaders)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'WEBSERVER_HTTPHEADERS'\n event_data = '{\"server\": \"example server\"}'\n event_module = 'sfp_spider'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n evt.actualSource = \"https://spiderfoot.net/example\"\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_subdomain_takeover.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_subdomain_takeover import sfp_subdomain_takeover\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleSubdomain_takeover(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_subdomain_takeover()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_subdomain_takeover()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_subdomain_takeover()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_subdomain_takeover()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_sublist3r.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_sublist3r import sfp_sublist3r\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleSublist3r(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_sublist3r()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_sublist3r()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_sublist3r()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_sublist3r()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_surbl.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_surbl import sfp_surbl\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleSurbl(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_surbl()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_surbl()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_surbl()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_surbl()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_talosintel.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_talosintel import sfp_talosintel\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleTalosintel(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_talosintel()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_talosintel()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_talosintel()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_talosintel()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_template.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_template import sfp_template\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleTemplate(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_template()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_template()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_template()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_template()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_textmagic.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_textmagic import sfp_textmagic\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleTextmagic(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_textmagic()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_textmagic()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_textmagic()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_textmagic()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_textmagic()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'PHONE_NUMBER'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_threatcrowd.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_threatcrowd import sfp_threatcrowd\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleThreatcrowd(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_threatcrowd()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_threatcrowd()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_threatcrowd()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_threatcrowd()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_threatfox.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_threatfox import sfp_threatfox\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleThreatFox(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_threatfox()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_threatfox()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_threatfox()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_threatfox()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_threatminer.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_threatminer import sfp_threatminer\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleThreatminer(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_threatminer()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_threatminer()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_threatminer()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_threatminer()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_tldsearch.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tldsearch import sfp_tldsearch\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleTldsearch(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tldsearch()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tldsearch()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tldsearch()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tldsearch()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_tool_cmseek.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_cmseek import sfp_tool_cmseek\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleToolCmseek(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_cmseek()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_cmseek()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_cmseek()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_cmseek()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_tool_path_configured_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_cmseek()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_tool_dnstwist.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_dnstwist import sfp_tool_dnstwist\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleToolDnstwist(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_dnstwist()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_tool_dnstwist()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_dnstwist()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_dnstwist()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_tool_nbtscan.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_nbtscan import sfp_tool_nbtscan\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleToolNbtscan(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_nbtscan()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_tool_nbtscan()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_nbtscan()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_nbtscan()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_tool_path_configured_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_nbtscan()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_tool_nmap.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_nmap import sfp_tool_nmap\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleToolNmap(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_nmap()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_tool_nmap()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_nmap()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_nmap()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_tool_path_configured_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_nmap()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_tool_nuclei.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_nuclei import sfp_tool_nuclei\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleToolNuclei(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_nuclei()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_tool_nuclei()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_nuclei()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_nuclei()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_tool_path_configured_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_nuclei()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_tool_onesixtyone.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_onesixtyone import sfp_tool_onesixtyone\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleToolOnesixtyone(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_onesixtyone()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_tool_onesixtyone()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_onesixtyone()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_onesixtyone()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_tool_path_configured_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_onesixtyone()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_tool_retirejs.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_retirejs import sfp_tool_retirejs\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleToolRetirejs(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_retirejs()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_tool_retirejs()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_retirejs()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_retirejs()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_tool_path_configured_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_retirejs()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_tool_snallygaster.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_snallygaster import sfp_tool_snallygaster\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleToolSnallygaster(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_snallygaster()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_tool_snallygaster()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_snallygaster()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_snallygaster()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_tool_path_configured_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_snallygaster()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_tool_testsslsh.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_testsslsh import sfp_tool_testsslsh\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleToolTestsslsh(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_testsslsh()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_tool_testsslsh()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_testsslsh()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_testsslsh()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_tool_path_configured_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_testsslsh()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_tool_trufflehog.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_trufflehog import sfp_tool_trufflehog\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleToolTrufflehog(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_trufflehog()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_tool_trufflehog()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_trufflehog()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_trufflehog()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_tool_path_configured_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_trufflehog()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example value'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_tool_wafw00f.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_wafw00f import sfp_tool_wafw00f\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleToolWafw00f(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_wafw00f()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_tool_wafw00f()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_wafw00f()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_wafw00f()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_tool_path_configured_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_wafw00f()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_tool_wappalyzer.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_wappalyzer import sfp_tool_wappalyzer\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleWappalyzer(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_wappalyzer()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_tool_wappalyzer()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_wappalyzer()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_wappalyzer()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_tool_path_configured_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_wappalyzer()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_tool_whatweb.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_tool_whatweb import sfp_tool_whatweb\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleToolWhatweb(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_tool_whatweb()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_tool_whatweb()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_tool_whatweb()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_tool_whatweb()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_tool_path_configured_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_tool_whatweb()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_torch.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_torch import sfp_torch\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleTorch(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_torch()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_torch()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_torch()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_torch()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_torexits.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_torexits import sfp_torexits\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleTorexits(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_torexits()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_torexits()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_torexits()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_torexits()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_trashpanda.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_trashpanda import sfp_trashpanda\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleTrashpanda(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_trashpanda()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_trashpanda()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_trashpanda()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_trashpanda()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_trashpanda()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'EMAILADDR'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_trumail.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_trumail import sfp_trumail\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleTrumail(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_trumail()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_trumail()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_trumail()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_trumail()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_twilio.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_twilio import sfp_twilio\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuletwilio(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_twilio()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_twilio()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_twilio()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_twilio()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_twilio()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_twitter.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_twitter import sfp_twitter\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleTwitter(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_twitter()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_twitter()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_twitter()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_twitter()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_social_media_not_twitter_profile_should_not_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_twitter()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_twitter)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'SOCIAL_MEDIA'\n event_data = 'Not Twitter: example_username'\n event_module = 'example module'\n source_event = evt\n\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_uceprotect.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_uceprotect import sfp_uceprotect\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleUceprotect(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_uceprotect()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_uceprotect()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_uceprotect()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_uceprotect()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_urlscan.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_urlscan import sfp_urlscan\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleUrlscan(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_urlscan()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_urlscan()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_urlscan()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_urlscan()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_venmo.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_venmo import sfp_venmo\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleVenmo(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_venmo()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_venmo()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_venmo()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_venmo()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_viewdns.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_viewdns import sfp_viewdns\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModulViewdns(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_viewdns()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_viewdns()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_viewdns()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_viewdns()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_viewdns()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_virustotal.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_virustotal import sfp_virustotal\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleVirustotal(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_virustotal()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_virustotal()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_virustotal()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_virustotal()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_virustotal()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_voipbl.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_voipbl import sfp_voipbl\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleVoipbl(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_voipbl()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_voipbl()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_voipbl()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_voipbl()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_vxvault.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_vxvault import sfp_vxvault\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleVxvault(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_vxvault()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_vxvault()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_vxvault()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_vxvault()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_webanalytics.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_webanalytics import sfp_webanalytics\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleWebAnalytics(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_webanalytics()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_webanalytics()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_webanalytics()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_webanalytics()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_target_web_content_containing_web_analytics_string_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_webanalytics()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'WEB_ANALYTICS_ID'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"Google Analytics: ua-1111111111-123\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_webanalytics)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'TARGET_WEB_CONTENT'\n event_data = '

    example data ua-1111111111-123 example data

    '\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_target_web_content_not_containing_web_analytics_string_should_not_create_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_webanalytics()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_webanalytics)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'TARGET_WEB_CONTENT'\n event_data = 'example data'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n\n def test_handleEvent_event_dns_text_containing_web_analytics_string_should_return_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_webanalytics()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'WEB_ANALYTICS_ID'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"Google Site Verification: abcdefghijklmnopqrstuvwxyz1234567890abc_def\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_webanalytics)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'DNS_TEXT'\n event_data = 'google-site-verification=abcdefghijklmnopqrstuvwxyz1234567890abc_def'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_dns_text_not_containing_web_analytics_string_should_not_create_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_webanalytics()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_webanalytics)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'DNS_TEXT'\n event_data = 'example data'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_webframework.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_webframework import sfp_webframework\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleWebFramework(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_webframework()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_webframework()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_webframework()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_webframework()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_event_data_web_content_containing_webframework_string_should_create_url_web_framework_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_webframework()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n expected = 'URL_WEB_FRAMEWORK'\n if str(event.eventType) != expected:\n raise Exception(f\"{event.eventType} != {expected}\")\n\n expected = \"Wordpress\"\n if str(event.data) != expected:\n raise Exception(f\"{event.data} != {expected}\")\n\n raise Exception(\"OK\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_webframework)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'TARGET_WEB_CONTENT'\n event_data = 'example data /wp-includes/ example data'\n event_module = 'sfp_spider'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n evt.actualSource = \"https://spiderfoot.net/\"\n\n with self.assertRaises(Exception) as cm:\n module.handleEvent(evt)\n\n self.assertEqual(\"OK\", str(cm.exception))\n\n def test_handleEvent_event_data_web_content_not_containing_webframework_string_should_not_create_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_webframework()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_webframework)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'TARGET_WEB_CONTENT'\n event_data = 'example data'\n event_module = 'example module'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n evt.actualSource = \"https://spiderfoot.net/\"\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n\n def test_handleEvent_event_data_web_content_from_external_url_containing_webframework_string_should_not_create_event(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_webframework()\n module.setup(sf, dict())\n\n target_value = 'spiderfoot.net'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n def new_notifyListeners(self, event):\n raise Exception(f\"Raised event {event.eventType}: {event.data}\")\n\n module.notifyListeners = new_notifyListeners.__get__(module, sfp_webframework)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n event_type = 'TARGET_WEB_CONTENT'\n event_data = 'example data /wp-includes/ example data'\n event_module = 'sfp_spider'\n source_event = evt\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n evt.actualSource = \"https://externalhost.local/\"\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_webserver.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_webserver import sfp_webserver\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleWebserver(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_webserver()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_webserver()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_webserver()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_webserver()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_whatcms.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_whatcms import sfp_whatcms\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleWhatCMS(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_whatcms()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_whatcms()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_whatcms()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_whatcms()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_whatcms()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_whatcms()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_whatcms()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_whois.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_whois import sfp_whois\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleWhois(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_whois()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_whois()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_whois()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_whois()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_whoisology.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_whoisology import sfp_whoisology\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleWhoisology(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_whoisology()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_whoisology()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_whoisology()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_whoisology()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_whoisology()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_whoxy.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_whoxy import sfp_whoxy\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleWhoxy(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_whoxy()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_whoxy()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_whoxy()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_whoxy()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_whoxy()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_wigle.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_wigle import sfp_wigle\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleWigle(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_wigle()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_wigle()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_wigle()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_wigle()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_wigle()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n" }, { "path": "test/unit/modules/test_sfp_wikileaks.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_wikileaks import sfp_wikileaks\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModulewikileaks(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_wikileaks()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_wikileaks()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_wikileaks()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_wikileaks()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_wikipediaedits.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_wikipediaedits import sfp_wikipediaedits\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModulewikipediaedits(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_wikipediaedits()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_wikipediaedits()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_wikipediaedits()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_wikipediaedits()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_xforce.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_xforce import sfp_xforce\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleXforce(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_xforce()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_xforce()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_xforce()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_xforce()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\", \"400\", \"404\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_xforce()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"402\", \"403\", \"429\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_xforce()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_xforce()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'IP_ADDRESS'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_yandexdns.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_yandexdns import sfp_yandexdns\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleYandexdns(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_yandexdns()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_yandexdns()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_yandexdns()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_yandexdns()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/modules/test_sfp_zetalytics.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_zetalytics import sfp_zetalytics\nfrom sflib import SpiderFoot\nfrom spiderfoot import SpiderFootEvent, SpiderFootTarget\n\n\n@pytest.mark.usefixtures\nclass TestModuleZetalytics(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_zetalytics()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_zetalytics()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_zetalytics()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_zetalytics()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_handleEvent_no_api_key_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n module = sfp_zetalytics()\n module.setup(sf, dict())\n\n target_value = 'example target value'\n target_type = 'INTERNET_NAME'\n target = SpiderFootTarget(target_value, target_type)\n module.setTarget(target)\n\n event_type = 'ROOT'\n event_data = 'example data'\n event_module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, event_module, source_event)\n\n result = module.handleEvent(evt)\n\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_zonefiles.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_zonefiles import sfp_zonefiles\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleZoneFiles(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_zonefiles()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_zonefiles()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_zonefiles()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_zonefiles()\n self.assertIsInstance(module.producedEvents(), list)\n\n def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"200\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_zonefiles()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertFalse(module.errorState)\n\n def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self):\n sf = SpiderFoot(self.default_options)\n\n http_codes = [\"401\", \"403\", \"404\", \"429\", \"500\", \"502\", \"503\"]\n for code in http_codes:\n with self.subTest(code=code):\n module = sfp_zonefiles()\n module.setup(sf, dict())\n result = module.parseApiResponse({\"code\": code, \"content\": None})\n self.assertIsNone(result)\n self.assertTrue(module.errorState)\n" }, { "path": "test/unit/modules/test_sfp_zoneh.py", "content": "import pytest\nimport unittest\n\nfrom modules.sfp_zoneh import sfp_zoneh\nfrom sflib import SpiderFoot\n\n\n@pytest.mark.usefixtures\nclass TestModuleZoneh(unittest.TestCase):\n\n def test_opts(self):\n module = sfp_zoneh()\n self.assertEqual(len(module.opts), len(module.optdescs))\n\n def test_setup(self):\n sf = SpiderFoot(self.default_options)\n module = sfp_zoneh()\n module.setup(sf, dict())\n\n def test_watchedEvents_should_return_list(self):\n module = sfp_zoneh()\n self.assertIsInstance(module.watchedEvents(), list)\n\n def test_producedEvents_should_return_list(self):\n module = sfp_zoneh()\n self.assertIsInstance(module.producedEvents(), list)\n" }, { "path": "test/unit/spiderfoot/test_spiderfootcorrelator.py", "content": "# test_spiderfootcorrelator.py\nimport unittest\n\nfrom spiderfoot import SpiderFootCorrelator, SpiderFootDb\n\n\nclass TestSpiderFootCorrelator(unittest.TestCase):\n \"\"\"\n Test SpiderFootCorrelator\n \"\"\"\n\n def test_init_argument_dbh_invalid_type_should_raise_TypeError(self):\n invalid_types = [None, str(), list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n SpiderFootCorrelator(invalid_type, {})\n\n def test_init_argument_ruleset_invalid_type_should_raise_TypeError(self):\n invalid_types = [None, str(), list(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n SpiderFootCorrelator(None, invalid_type)\n\n def test_init_argument_ruleset_invalid_rule_should_raise_SyntaxError(self):\n sfdb = SpiderFootDb(self.default_options, False)\n\n ruleset = {\"sample rule\": \"invalid yaml\"}\n with self.assertRaises(SyntaxError):\n SpiderFootCorrelator(sfdb, ruleset)\n\n def test_run_correlations_invalid_scan_instance_should_raise_ValueError(self):\n sfdb = SpiderFootDb(self.default_options, False)\n\n correlator = SpiderFootCorrelator(sfdb, {}, 'example scan id')\n with self.assertRaises(ValueError):\n correlator.run_correlations()\n\n def test_build_db_criteria_argument_matchrule_invalid_type_should_raise_TypeError(self):\n sfdb = SpiderFootDb(self.default_options, False)\n correlator = SpiderFootCorrelator(sfdb, {})\n\n invalid_types = [None, str(), list(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n correlator.build_db_criteria(invalid_type)\n\n def test_enrich_event_sources_argument_rule_invalid_type_should_raise_TypeError(self):\n sfdb = SpiderFootDb(self.default_options, False)\n correlator = SpiderFootCorrelator(sfdb, {})\n\n invalid_types = [None, str(), list(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n correlator.enrich_event_sources(invalid_type)\n\n def test_enrich_event_children_argument_rule_invalid_type_should_raise_TypeError(self):\n sfdb = SpiderFootDb(self.default_options, False)\n correlator = SpiderFootCorrelator(sfdb, {})\n\n invalid_types = [None, str(), list(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n correlator.enrich_event_children(invalid_type)\n\n def test_enrich_event_entities_argument_rule_invalid_type_should_raise_TypeError(self):\n sfdb = SpiderFootDb(self.default_options, False)\n correlator = SpiderFootCorrelator(sfdb, {})\n\n invalid_types = [None, str(), list(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n correlator.enrich_event_entities(invalid_type)\n\n def test_process_rule_argument_rule_invalid_type_should_raise_TypeError(self):\n sfdb = SpiderFootDb(self.default_options, False)\n correlator = SpiderFootCorrelator(sfdb, {})\n\n invalid_types = [None, str(), list(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n correlator.process_rule(invalid_type)\n\n def test_build_correlation_title_argument_rule_invalid_type_should_raise_TypeError(self):\n sfdb = SpiderFootDb(self.default_options, False)\n correlator = SpiderFootCorrelator(sfdb, {})\n\n invalid_types = [None, str(), list(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n correlator.build_correlation_title(invalid_type, [])\n\n def test_build_correlation_title_argument_data_invalid_type_should_raise_TypeError(self):\n sfdb = SpiderFootDb(self.default_options, False)\n correlator = SpiderFootCorrelator(sfdb, {})\n\n invalid_types = [None, str(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n correlator.build_correlation_title({}, invalid_type)\n\n def test_create_correlation_argument_rule_invalid_type_should_raise_TypeError(self):\n sfdb = SpiderFootDb(self.default_options, False)\n correlator = SpiderFootCorrelator(sfdb, {})\n\n invalid_types = [None, str(), list(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n correlator.create_correlation(invalid_type, [], readonly=True)\n\n def test_create_correlation_argument_data_invalid_type_should_raise_TypeError(self):\n sfdb = SpiderFootDb(self.default_options, False)\n correlator = SpiderFootCorrelator(sfdb, {})\n\n invalid_types = [None, str(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n correlator.create_correlation({}, invalid_type, readonly=True)\n\n def test_check_ruleset_validity_should_return_bool(self):\n sfdb = SpiderFootDb(self.default_options, False)\n correlator = SpiderFootCorrelator(sfdb, {})\n\n ruleset = [{\"sample\": \"sample\"}]\n self.assertIsInstance(correlator.check_ruleset_validity(ruleset), bool)\n\n invalid_types = [None, str(), list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n self.assertIsInstance(correlator.check_ruleset_validity(invalid_type), bool)\n\n def test_check_rule_validity_invalid_rule_should_return_false(self):\n sfdb = SpiderFootDb(self.default_options, False)\n correlator = SpiderFootCorrelator(sfdb, {})\n\n invalid_types = [None, str(), list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n self.assertFalse(correlator.check_rule_validity(invalid_type))\n\n def test_check_rule_validity_rule_missing_mandatory_field_should_return_false(self):\n sfdb = SpiderFootDb(self.default_options, False)\n correlator = SpiderFootCorrelator(sfdb, {})\n\n rule = {\n \"id\": \"sample\",\n \"collections\": [],\n \"headline\": \"sample\"\n }\n self.assertFalse(correlator.check_rule_validity(rule))\n\n rule = {\n \"id\": \"sample\",\n \"meta\": \"sample\",\n \"headline\": \"sample\"\n }\n self.assertFalse(correlator.check_rule_validity(rule))\n\n rule = {\n \"id\": \"sample\",\n \"meta\": \"sample\",\n \"collections\": []\n }\n self.assertFalse(correlator.check_rule_validity(rule))\n" }, { "path": "test/unit/spiderfoot/test_spiderfootdb.py", "content": "# test_spiderfootdb.py\nimport pytest\nimport unittest\n\nfrom spiderfoot import SpiderFootDb, SpiderFootEvent\n\n\n@pytest.mark.usefixtures\nclass TestSpiderFootDb(unittest.TestCase):\n \"\"\"\n Test SpiderFootDb\n \"\"\"\n\n def test_init_argument_opts_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test __init__(self, opts, init=False)\n \"\"\"\n invalid_types = [None, \"\", list(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n SpiderFootDb(invalid_type)\n\n def test_init_argument_opts_with_empty_value_should_raise_ValueError(self):\n \"\"\"\n Test __init__(self, opts, init=False)\n \"\"\"\n with self.assertRaises(ValueError):\n SpiderFootDb(dict())\n\n def test_init_argument_opts_with_empty_key___database_value_should_raise_ValueError(self):\n \"\"\"\n Test __init__(self, opts, init=False)\n \"\"\"\n with self.assertRaises(ValueError):\n opts = dict()\n opts['__database'] = None\n SpiderFootDb(opts)\n\n def test_init_should_create_SpiderFootDb_object(self):\n \"\"\"\n Test __init__(self, opts, init=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n self.assertIsInstance(sfdb, SpiderFootDb)\n\n @unittest.skip(\"todo\")\n def test_create_should_create_database_schema(self):\n \"\"\"\n Test create(self)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n sfdb.create()\n self.assertEqual('TBD', 'TBD')\n\n def test_close_should_close_database_connection(self):\n \"\"\"\n Test close(self)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n sfdb.close()\n\n def test_search_should_return_a_list(self):\n \"\"\"\n Test search(self, criteria, filterFp=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n criteria = {\n 'scan_id': \"example scan id\",\n 'type': \"example type\",\n 'value': \"example value\",\n 'regex': \"example regex\"\n }\n\n search_results = sfdb.search(criteria, False)\n self.assertIsInstance(search_results, list)\n self.assertFalse(search_results)\n\n def test_search_argument_criteria_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test search(self, criteria, filterFp=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n invalid_types = [None, \"\", list(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.search(invalid_type, False)\n\n def test_search_argument_criteria_key_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test search(self, criteria, filterFp=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n criteria = {\n 'type': \"example type\",\n 'value': \"example value\",\n 'regex': []\n }\n\n with self.assertRaises(TypeError):\n sfdb.search(criteria, False)\n\n def test_search_argument_criteria_no_valid_criteria_should_raise_ValueError(self):\n \"\"\"\n Test search(self, criteria, filterFp=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n criteria = {\n 'invalid_criteria': \"example invalid criteria\"\n }\n\n with self.assertRaises(ValueError):\n sfdb.search(criteria, False)\n\n def test_search_argument_criteria_one_criteria_should_raise_ValueError(self):\n \"\"\"\n Test search(self, criteria, filterFp=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n criteria = {\n 'type': \"example type\"\n }\n\n with self.assertRaises(ValueError):\n sfdb.search(criteria, False)\n\n def test_eventTypes_should_return_a_list(self):\n \"\"\"\n Test eventTypes(self)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n event_types = sfdb.eventTypes()\n self.assertIsInstance(event_types, list)\n\n def test_scanLogEvent_should_create_a_scan_log_event(self):\n \"\"\"\n Test scanLogEvent(self, instanceId, classification, message, component=None)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n sfdb.scanLogEvent(\"\", \"\", \"\", None)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_scanLogEvent_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanLogEvent(self, instanceId, classification, message, component=None)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanLogEvent(invalid_type, \"\", \"\")\n\n def test_scanLogEvent_argument_classification_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanLogEvent(self, instanceId, classification, message, component=None)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanLogEvent(instance_id, invalid_type, \"\")\n\n def test_scanLogEvent_argument_message_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanLogEvent(self, instanceId, classification, message, component=None)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n invalid_types = [None, list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanLogEvent(instance_id, \"\", invalid_type)\n\n @unittest.skip(\"todo\")\n def test_scanInstanceCreate_should_create_a_scan_instance(self):\n \"\"\"\n Test scanInstanceCreate(self, instanceId, scanName, scanTarget)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_name = \"example scan name\"\n scan_target = \"example scan target\"\n\n sfdb.scanInstanceCreate(instance_id, scan_name, scan_target)\n\n self.assertEqual('TBD', 'TBD')\n\n @unittest.skip(\"todo\")\n def test_scanInstanceCreate_argument_instanceId_already_exists_should_halt_and_catch_fire(self):\n \"\"\"\n Test scanInstanceCreate(self, instanceId, scanName, scanTarget)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_name = \"example scan name\"\n scan_target = \"example scan target\"\n\n sfdb.scanInstanceCreate(instance_id, scan_name, scan_target)\n\n instance_id = \"example instance id\"\n scan_name = \"example scan name\"\n scan_target = \"example scan target\"\n\n with self.assertRaises(IOError):\n sfdb.scanInstanceCreate(instance_id, scan_name, scan_target)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_scanInstanceCreate_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanInstanceCreate(self, instanceId, scanName, scanTarget)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n scan_name = \"\"\n scan_target = \"spiderfoot.net\"\n invalid_types = [None, list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanInstanceCreate(invalid_type, scan_name, scan_target)\n\n def test_scanInstanceCreate_argument_scanName_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanInstanceCreate(self, instanceId, scanName, scanTarget)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_target = \"spiderfoot.net\"\n invalid_types = [None, list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanInstanceCreate(instance_id, invalid_type, scan_target)\n\n def test_scanInstanceCreate_argument_scanTarget_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanInstanceCreate(self, instanceId, scanName, scanTarget)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_name = \"\"\n invalid_types = [None, list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanInstanceCreate(instance_id, scan_name, invalid_type)\n\n def test_scanInstanceSet(self):\n \"\"\"\n Test scanInstanceSet(self, instanceId, started=None, ended=None, status=None)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n scan_instance = 'example scan instance'\n sfdb.scanInstanceSet(scan_instance, None, None, None)\n self.assertEqual('TBD', 'TBD')\n\n def test_scanInstanceSet_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanInstanceSet(self, instanceId, started=None, ended=None, status=None)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n started = None\n ended = None\n status = None\n\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanInstanceSet(invalid_type, started, ended, status)\n\n def test_scanInstanceGet_should_return_scan_info(self):\n \"\"\"\n Test scanInstanceGet(self, instanceId)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_name = \"example scan name\"\n scan_target = \"example scan target\"\n\n sfdb.scanInstanceCreate(instance_id, scan_name, scan_target)\n\n scan_instance_get = sfdb.scanInstanceGet(instance_id)\n\n self.assertEqual(len(scan_instance_get), 6)\n\n self.assertIsInstance(scan_instance_get[0], str)\n self.assertEqual(scan_instance_get[0], scan_name)\n\n self.assertIsInstance(scan_instance_get[1], str)\n self.assertEqual(scan_instance_get[1], scan_target)\n\n self.assertIsInstance(scan_instance_get[2], float)\n\n self.assertIsInstance(scan_instance_get[3], float)\n\n self.assertIsInstance(scan_instance_get[4], float)\n\n self.assertIsInstance(scan_instance_get[5], str)\n self.assertEqual(scan_instance_get[5], 'CREATED')\n\n def test_scanInstanceGet_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanInstanceGet(self, instanceId)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanInstanceGet(invalid_type)\n\n def test_scanResultSummary_should_return_a_list(self):\n \"\"\"\n Test scanResultSummary(self, instanceId, by=\"type\")\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_results_summary = sfdb.scanResultSummary(instance_id, \"type\")\n self.assertIsInstance(scan_results_summary, list)\n\n def test_scanResultSummary_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanResultSummary(self, instanceId, by=\"type\")\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanResultSummary(invalid_type)\n\n def test_scanResultSummary_argument_by_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanResultSummary(self, instanceId, by=\"type\")\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanResultSummary(instance_id, invalid_type)\n\n with self.assertRaises(ValueError):\n sfdb.scanResultSummary(instance_id, \"invalid filter type\")\n\n def test_scanResultSummary_argument_by_invalid_value_should_raise_ValueError(self):\n \"\"\"\n Test scanResultSummary(self, instanceId, by=\"type\")\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n with self.assertRaises(ValueError):\n sfdb.scanResultSummary(instance_id, \"invalid filter type\")\n\n def test_scanResultEvent_should_return_a_list(self):\n \"\"\"\n Test scanResultEvent(self, instanceId, eventType='ALL', filterFp=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_result_event = sfdb.scanResultEvent(instance_id, \"\", False)\n self.assertIsInstance(scan_result_event, list)\n\n def test_scanResultEvent_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanResultEvent(self, instanceId, eventType='ALL', filterFp=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ALL'\n filter_fp = None\n\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanResultEvent(invalid_type, event_type, filter_fp)\n\n def test_scanResultEvent_argument_eventType_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanResultEvent(self, instanceId, eventType='ALL', filterFp=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n invalid_types = [None, dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanResultEvent(instance_id, invalid_type, None)\n\n def test_scanResultEventUnique_should_return_a_list(self):\n \"\"\"\n Test scanResultEventUnique(self, instanceId, eventType='ALL', filterFp=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_result_event = sfdb.scanResultEventUnique(instance_id, \"\", False)\n self.assertIsInstance(scan_result_event, list)\n\n def test_scanResultEventUnique_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanResultEventUnique(self, instanceId, eventType='ALL', filterFp=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ALL'\n filter_fp = None\n\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanResultEventUnique(invalid_type, event_type, filter_fp)\n\n def test_scanResultEventUnique_argument_eventType_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanResultEventUnique(self, instanceId, eventType='ALL', filterFp=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n invalid_types = [None, list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanResultEventUnique(instance_id, invalid_type, None)\n\n def test_scanLogs_should_return_a_list(self):\n \"\"\"\n Test scanLogs(self, instanceId, limit=None, fromRowId=None, reverse=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_logs = sfdb.scanLogs(instance_id, None, None, None)\n self.assertIsInstance(scan_logs, list)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_scanLogs_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanLogs(self, instanceId, limit=None, fromRowId=None, reverse=False)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n limit = None\n from_row_id = None\n reverse = None\n\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanLogs(invalid_type, limit, from_row_id, reverse)\n\n def test_scanErrors_should_return_a_list(self):\n \"\"\"\n Test scanErrors(self, instanceId, limit=None)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n instance_id = \"example instance id\"\n scan_instance = sfdb.scanErrors(instance_id)\n self.assertIsInstance(scan_instance, list)\n\n def test_scanErrors_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanErrors(self, instanceId, limit=None)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanErrors(invalid_type)\n\n def test_scanInstanceDelete(self):\n \"\"\"\n Test scanInstanceDelete(self, instanceId)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n instance_id = \"example instance id\"\n sfdb.scanInstanceDelete(instance_id)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_scanInstanceDelete_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanInstanceDelete(self, instanceId)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanInstanceDelete(invalid_type)\n\n @unittest.skip(\"todo\")\n def test_scanResultsUpdateFP(self):\n \"\"\"\n Test scanResultsUpdateFP(self, instanceId, resultHashes, fpFlag)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_name = \"example scan name\"\n scan_target = \"example scan target\"\n\n sfdb.scanInstanceCreate(instance_id, scan_name, scan_target)\n\n result_hashes = None\n fp_flag = None\n sfdb.scanResultsUpdateFP(instance_id, result_hashes, fp_flag)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_scanResultsUpdateFP_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanResultsUpdateFP(self, instanceId, resultHashes, fpFlag)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n result_hashes = []\n fp_flag = None\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanResultsUpdateFP(invalid_type, result_hashes, fp_flag)\n\n def test_scanResultsUpdateFP_argument_resultHashes_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanResultsUpdateFP(self, instanceId, resultHashes, fpFlag)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n fp_flag = None\n invalid_types = [None, \"\", dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanResultsUpdateFP(instance_id, invalid_type, fp_flag)\n\n def test_configSet_should_set_config_opts(self):\n \"\"\"\n Test configSet(self, optMap=dict())\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n opts = dict()\n opts['example'] = 'example non-default config opt'\n sfdb.configSet(opts)\n\n config = sfdb.configGet()\n self.assertIsInstance(config, dict)\n self.assertIn('example', config)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_configSet_argument_optmap_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test configSet(self, optMap=dict())\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n invalid_types = [None, \"\", list()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.configSet(invalid_type)\n\n def test_configGet_should_return_a_dict(self):\n \"\"\"\n Test configGet(self)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n config = sfdb.configGet()\n self.assertIsInstance(config, dict)\n\n def test_configClear_should_clear_config(self):\n \"\"\"\n Test configClear(self)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n opts = dict()\n opts['example'] = 'example non-default config opt'\n sfdb.configSet(opts)\n\n config = sfdb.configGet()\n self.assertIsInstance(config, dict)\n self.assertIn('example', config)\n\n sfdb.configClear()\n\n config = sfdb.configGet()\n self.assertIsInstance(config, dict)\n self.assertNotIn('example', config)\n\n def test_scanConfigSet_argument_optMap_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanConfigSet(self, id, optMap=dict())\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n invalid_types = [None, \"\"]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanConfigSet(instance_id, invalid_type)\n\n def test_scanConfigSet_argument_instanceId_with_empty_value_should_raise_ValueError(self):\n \"\"\"\n Test scanConfigSet(self, id, optMap=dict())\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n with self.assertRaises(ValueError):\n sfdb.scanConfigSet(\"\", dict())\n\n def test_scanConfigGet_should_return_a_dict(self):\n \"\"\"\n Test scanConfigGet(self, instanceId)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_config = sfdb.scanConfigGet(instance_id)\n self.assertIsInstance(scan_config, dict)\n\n def test_scanConfigGet_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanConfigGet(self, instanceId)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanConfigGet(invalid_type)\n\n def test_scanEventStore_should_store_a_scan_event(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n instance_id = \"example instance id\"\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event = \"\"\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanEventStore(invalid_type, event)\n\n def test_scanEventStore_argument_instanceId_with_empty_value_should_raise_ValueError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event = \"\"\n with self.assertRaises(ValueError):\n sfdb.scanEventStore(\"\", event)\n\n def test_scanEventStore_argument_sfEvent_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n invalid_types = [None, \"\", list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanEventStore(instance_id, invalid_type)\n\n def test_scanEventStore_argument_sfEvent_with_invalid_eventType_property_type_should_raise_TypeError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n invalid_types = [None, list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n event.eventType = invalid_type\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_sfEvent_with_empty_eventType_property_value_should_raise_ValueError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n\n with self.assertRaises(ValueError):\n event.eventType = ''\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_sfEvent_with_invalid_data_property_type_should_raise_TypeError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n invalid_types = [None, list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n event.data = invalid_type\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_sfEvent_with_empty_data_property_value_should_raise_ValueError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n\n with self.assertRaises(ValueError):\n event.data = ''\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_sfEvent_with_invalid_module_property_type_should_raise_TypeError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n invalid_types = [None, list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n event.module = invalid_type\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_sfEvent_with_empty_module_property_value_should_raise_ValueError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n with self.assertRaises(ValueError):\n event.module = ''\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_sfEvent_with_invalid_confidence_property_type_should_raise_TypeError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n invalid_types = [None, list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n event.confidence = invalid_type\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_sfEvent_with_empty_confidence_property_value_should_raise_ValueError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n invalid_values = [-1, 101]\n for invalid_value in invalid_values:\n with self.subTest(invalid_value=invalid_value):\n with self.assertRaises(ValueError):\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n event.confidence = invalid_value\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_sfEvent_with_invalid_visibility_property_type_should_raise_TypeError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n invalid_types = [None, list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n event.visibility = invalid_type\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_sfEvent_with_empty_visibility_property_value_should_raise_ValueError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n invalid_values = [-1, 101]\n for invalid_value in invalid_values:\n with self.subTest(invalid_value=invalid_value):\n with self.assertRaises(ValueError):\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n event.visibility = invalid_value\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_sfEvent_with_invalid_risk_property_type_should_raise_TypeError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n invalid_types = [None, list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n event.risk = invalid_type\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_sfEvent_with_empty_risk_property_value_should_raise_ValueError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n invalid_values = [-1, 101]\n for invalid_value in invalid_values:\n with self.subTest(invalid_value=invalid_value):\n with self.assertRaises(ValueError):\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n event.risk = invalid_value\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanEventStore_argument_sfEvent_with_invalid_sourceEvent_property_type_should_raise_TypeError(self):\n \"\"\"\n Test scanEventStore(self, instanceId, sfEvent, truncateSize=0)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n event_type = 'ROOT'\n event_data = 'example data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = 'example module'\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n instance_id = \"example instance id\"\n invalid_types = [None, \"\", list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n event = SpiderFootEvent(event_type, event_data, module, source_event)\n event.sourceEvent = invalid_type\n sfdb.scanEventStore(instance_id, event)\n\n def test_scanInstanceList_should_return_a_list(self):\n \"\"\"\n Test scanInstanceList(self)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n scan_instances = sfdb.scanInstanceList()\n self.assertIsInstance(scan_instances, list)\n\n def test_scanResultHistory_should_return_a_list(self):\n \"\"\"\n Test scanResultHistory(self, instanceId)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_result_history = sfdb.scanResultHistory(instance_id)\n self.assertIsInstance(scan_result_history, list)\n\n def test_scanResultHistory_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanResultHistory(self, instanceId)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n invalid_types = [None, list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanResultHistory(invalid_type)\n\n def test_scanElementSourcesDirect_should_return_a_list(self):\n \"\"\"\n Test scanElementSourcesDirect(self, instanceId, elementIdList)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n element_id_list = []\n scan_element_sources_direct = sfdb.scanElementSourcesDirect(instance_id, element_id_list)\n self.assertIsInstance(scan_element_sources_direct, list)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_scanElementSourcesDirect_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanElementSourcesDirect(self, instanceId, elementIdList)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n element_id_list = []\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanElementSourcesDirect(invalid_type, element_id_list)\n\n def test_scanElementSourcesDirect_argument_elementIdList_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanElementSourcesDirect(self, instanceId, elementIdList)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n invalid_types = [None, \"\", dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanElementSourcesDirect(instance_id, invalid_type)\n\n def test_scanElementChildrenDirect_should_return_a_list(self):\n \"\"\"\n Test scanElementChildrenDirect(self, instanceId, elementIdList)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_element_children_direct = sfdb.scanElementChildrenDirect(instance_id, list())\n self.assertIsInstance(scan_element_children_direct, list)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_scanElementChildrenDirect_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanElementChildrenDirect(self, instanceId, elementIdList)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n element_id_list = []\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanElementChildrenDirect(invalid_type, element_id_list)\n\n def test_scanElementChildrenDirect_argument_elementIdList_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanElementChildrenDirect(self, instanceId, elementIdList)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n invalid_types = [None, \"\", dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanElementChildrenDirect(instance_id, invalid_type)\n\n def test_scanElementSourcesAll_should_return_a_list(self):\n \"\"\"\n Test scanElementSourcesAll(self, instanceId, childData)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n child_data = [\"example child\", \"example child\"]\n scan_element_sources_all = sfdb.scanElementSourcesAll(instance_id, child_data)\n self.assertIsInstance(scan_element_sources_all, list)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_scanElementSourcesAll_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanElementSourcesAll(self, instanceId, childData)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n invalid_types = [None, list(), dict(), int()]\n child_data = []\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanElementSourcesAll(invalid_type, child_data)\n\n def test_scanElementSourcesAll_argument_childData_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanElementSourcesAll(self, instanceId, childData)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n invalid_types = [None, \"\", dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanElementSourcesAll(instance_id, invalid_type)\n\n def test_scanElementSourcesAll_argument_childData_with_empty_value_should_raise_ValueError(self):\n \"\"\"\n Test scanElementSourcesAll(self, instanceId, childData)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n child_data = []\n\n with self.assertRaises(ValueError):\n sfdb.scanElementSourcesAll(instance_id, child_data)\n\n def test_scanElementChildrenAll_should_return_a_list(self):\n \"\"\"\n Test scanElementChildrenAll(self, instanceId, parentIds)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n scan_element_children_all = sfdb.scanElementChildrenAll(instance_id, list())\n self.assertIsInstance(scan_element_children_all, list)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_scanElementChildrenAll_argument_instanceId_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanElementChildrenAll(self, instanceId, parentIds)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n invalid_types = [None, list(), dict(), int()]\n parent_ids = []\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanElementChildrenAll(invalid_type, parent_ids)\n\n def test_scanElementChildrenAll_argument_parentIds_of_invalid_type_should_raise_TypeError(self):\n \"\"\"\n Test scanElementChildrenAll(self, instanceId, parentIds)\n \"\"\"\n sfdb = SpiderFootDb(self.default_options, False)\n\n instance_id = \"example instance id\"\n invalid_types = [None, \"\", dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.scanElementChildrenAll(instance_id, invalid_type)\n\n def test_correlationResultCreate_arguments_of_invalid_type_should_raise_TypeError(self):\n sfdb = SpiderFootDb(self.default_options, False)\n\n invalid_types = [None, list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.correlationResultCreate(invalid_type, \"\", \"\", \"\", \"\", \"\", \"\", [])\n\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.correlationResultCreate(\"\", invalid_type, \"\", \"\", \"\", \"\", \"\", [])\n\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.correlationResultCreate(\"\", \"\", invalid_type, \"\", \"\", \"\", \"\", [])\n\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.correlationResultCreate(\"\", \"\", \"\", invalid_type, \"\", \"\", \"\", [])\n\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.correlationResultCreate(\"\", \"\", \"\", \"\", invalid_type, \"\", \"\", [])\n\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.correlationResultCreate(\"\", \"\", \"\", \"\", \"\", invalid_type, \"\", [])\n\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n sfdb.correlationResultCreate(\"\", \"\", \"\", \"\", \"\", \"\", invalid_type, [])\n" }, { "path": "test/unit/spiderfoot/test_spiderfootevent.py", "content": "# test_spiderfootevent.py\nimport unittest\n\nfrom spiderfoot import SpiderFootEvent\n\n\nclass TestSpiderFootEvent(unittest.TestCase):\n\n def test_init_root_event_should_create_event(self):\n event_data = 'example event data'\n module = 'example module'\n source_event = ''\n\n event_type = 'ROOT'\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n self.assertIsInstance(evt, SpiderFootEvent)\n\n def test_init_nonroot_event_with_root_sourceEvent_should_create_event(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example non-root event type'\n event_data = 'example event data'\n module = 'example module'\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n self.assertIsInstance(evt, SpiderFootEvent)\n\n def test_init_argument_eventType_of_invalid_type_should_raise_TypeError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n module = 'example module'\n\n invalid_types = [None, bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n SpiderFootEvent(invalid_type, event_data, module, source_event)\n\n def test_init_argument_eventType_with_empty_value_should_raise_ValueError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = ''\n module = 'example module'\n\n with self.assertRaises(ValueError):\n SpiderFootEvent(event_type, event_data, module, source_event)\n\n def test_init_argument_data_of_invalid_type_should_raise_TypeError(self):\n event_type = 'ROOT'\n module = ''\n source_event = ''\n\n invalid_types = [None, bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n SpiderFootEvent(event_type, invalid_type, module, source_event)\n\n def test_init_argument_data_with_empty_value_should_raise_ValueError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = ''\n module = 'example module'\n\n with self.assertRaises(ValueError):\n SpiderFootEvent(event_type, event_data, module, source_event)\n\n def test_init_argument_module_of_invalid_type_should_raise_TypeError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = SpiderFootEvent(event_type, event_data, module, \"ROOT\")\n\n event_type = 'example non-root event type'\n invalid_types = [None, bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n SpiderFootEvent(event_type, event_data, invalid_type, source_event)\n\n def test_init_argument_module_with_empty_value_should_raise_ValueError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example event type'\n event_data = 'example event data'\n module = ''\n\n with self.assertRaises(ValueError):\n SpiderFootEvent(event_type, event_data, module, source_event)\n\n def test_init_argument_sourceEvent_of_invalid_type_should_raise_TypeError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n\n event_type = 'example non-root event type'\n module = 'example module'\n invalid_types = [None, \"\", bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n SpiderFootEvent(event_type, event_data, module, invalid_type)\n\n def test_init_argument_confidence_of_invalid_type_should_raise_TypeError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n\n invalid_types = [None, \"\", bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt.confidence = invalid_type\n\n def test_init_argument_confidence_invalid_value_should_raise_ValueError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n\n invalid_values = [-1, 101]\n for invalid_value in invalid_values:\n with self.subTest(invalid_value=invalid_value):\n with self.assertRaises(ValueError):\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt.confidence = invalid_value\n\n def test_init_argument_visibility_of_invalid_type_should_raise_TypeError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n\n invalid_types = [None, \"\", bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt.visibility = invalid_type\n\n def test_init_argument_visibility_invalid_value_should_raise_ValueError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n\n invalid_values = [-1, 101]\n for invalid_value in invalid_values:\n with self.subTest(invalid_value=invalid_value):\n with self.assertRaises(ValueError):\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt.visibility = invalid_value\n\n def test_init_argument_risk_of_invalid_type_should_raise_TypeError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n\n invalid_types = [None, \"\", bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt.risk = invalid_type\n\n def test_init_argument_risk_invalid_value_should_raise_ValueError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n\n invalid_values = [-1, 101]\n for invalid_value in invalid_values:\n with self.subTest(invalid_value=invalid_value):\n with self.assertRaises(ValueError):\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt.risk = invalid_value\n\n def test_confidence_attribute_should_return_confidence_as_integer(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n confidence = 100\n\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt.confidence = confidence\n\n self.assertEqual(confidence, evt.confidence)\n\n def test_confidence_attribute_setter_invalid_type_should_raise_TypeError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n\n invalid_types = [None, \"\", bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n evt.confidence = invalid_type\n\n def test_visibility_attribute_should_return_visibility_as_integer(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n visibility = 100\n\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt.visibility = visibility\n\n self.assertEqual(visibility, evt.visibility)\n\n def test_visibility_attribute_setter_invalid_type_should_raise_TypeError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n\n invalid_types = [None, \"\", bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n evt.visibility = invalid_type\n\n def test_risk_attribute_should_return_risk_as_integer(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n risk = 100\n\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt.risk = risk\n\n self.assertEqual(risk, evt.risk)\n\n def test_risk_attribute_setter_invalid_type_should_raise_TypeError(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n\n invalid_types = [None, \"\", bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n evt.risk = invalid_type\n\n def test_actualSource_attribute_should_return_actual_source_as_string(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n\n actual_source = 'example actual source'\n evt.actualSource = actual_source\n\n self.assertEqual(actual_source, evt.actualSource)\n\n def test_sourceEventHash_attribute_should_return_source_event_hash_as_string(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n\n self.assertEqual(\"ROOT\", evt.sourceEventHash)\n\n def test_moduleDataSource_attribute_should_return_module_data_source_as_string(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n\n module_data_source = 'example module data source'\n evt.moduleDataSource = module_data_source\n\n self.assertEqual(module_data_source, evt.moduleDataSource)\n\n def test_asdict_root_event_should_return_event_as_a_dict(self):\n event_data = 'example event data'\n module = 'example module data'\n source_event = ''\n\n event_type = 'ROOT'\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt_dict = evt.asDict()\n\n self.assertIsInstance(evt_dict, dict)\n self.assertEqual(evt_dict['type'], event_type)\n self.assertEqual(evt_dict['data'], event_data)\n self.assertEqual(evt_dict['module'], module)\n\n def test_asdict_nonroot_event_should_return_event_as_a_dict(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n source_event = SpiderFootEvent(event_type, event_data, module, source_event)\n\n event_type = 'example non-root event type'\n event_data = 'example event data'\n module = 'example_module'\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt_dict = evt.asDict()\n\n self.assertIsInstance(evt_dict, dict)\n self.assertEqual(evt_dict['type'], event_type)\n self.assertEqual(evt_dict['data'], event_data)\n self.assertEqual(evt_dict['module'], module)\n\n def test_hash_attribute_root_event_should_return_root_as_a_string(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = ''\n source_event = ''\n\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt_hash = evt.hash\n\n self.assertEqual('ROOT', evt_hash)\n\n def test_hash_attribute_nonroot_event_should_return_a_string(self):\n event_type = 'ROOT'\n event_data = 'example event data'\n module = 'example module'\n source_event = SpiderFootEvent(event_type, event_data, module, \"ROOT\")\n\n event_type = 'not ROOT'\n evt = SpiderFootEvent(event_type, event_data, module, source_event)\n evt_hash = evt.hash\n\n self.assertIsInstance(evt_hash, str)\n" }, { "path": "test/unit/spiderfoot/test_spiderfoothelpers.py", "content": "# test_spiderfoot.py\nimport pytest\nimport unittest\n\nfrom spiderfoot import SpiderFootHelpers\n\n\n@pytest.mark.usefixtures\nclass TestSpiderFootHelpers(unittest.TestCase):\n\n def test_data_path_should_return_a_string(self):\n data_path = SpiderFootHelpers.dataPath()\n self.assertIsInstance(data_path, str)\n\n def test_cache_path_should_return_a_string(self):\n cache_path = SpiderFootHelpers.cachePath()\n self.assertIsInstance(cache_path, str)\n\n def test_log_path_should_return_a_string(self):\n log_path = SpiderFootHelpers.logPath()\n self.assertIsInstance(log_path, str)\n\n def test_target_type(self):\n target_type = SpiderFootHelpers.targetTypeFromString(\"0.0.0.0\")\n self.assertEqual('IP_ADDRESS', target_type)\n target_type = SpiderFootHelpers.targetTypeFromString(\"noreply@spiderfoot.net\")\n self.assertEqual('EMAILADDR', target_type)\n target_type = SpiderFootHelpers.targetTypeFromString(\"0.0.0.0/0\")\n self.assertEqual('NETBLOCK_OWNER', target_type)\n target_type = SpiderFootHelpers.targetTypeFromString(\"+1234567890\")\n self.assertEqual('PHONE_NUMBER', target_type)\n target_type = SpiderFootHelpers.targetTypeFromString('\"Human Name\"')\n self.assertEqual('HUMAN_NAME', target_type)\n target_type = SpiderFootHelpers.targetTypeFromString('\"abc123\"')\n self.assertEqual('USERNAME', target_type)\n target_type = SpiderFootHelpers.targetTypeFromString(\"1234567890\")\n self.assertEqual('BGP_AS_OWNER', target_type)\n target_type = SpiderFootHelpers.targetTypeFromString(\"::1\")\n self.assertEqual('IPV6_ADDRESS', target_type)\n target_type = SpiderFootHelpers.targetTypeFromString(\"spiderfoot.net\")\n self.assertEqual('INTERNET_NAME', target_type)\n target_type = SpiderFootHelpers.targetTypeFromString(\"1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R\")\n self.assertEqual('BITCOIN_ADDRESS', target_type)\n\n def test_target_type_invalid_seed_should_return_none(self):\n target_type = SpiderFootHelpers.targetTypeFromString(None)\n self.assertEqual(None, target_type)\n\n target_type = SpiderFootHelpers.targetTypeFromString(\"\")\n self.assertEqual(None, target_type)\n\n target_type = SpiderFootHelpers.targetTypeFromString('\"\"')\n self.assertEqual(None, target_type)\n\n def test_urlRelativeToAbsolute_argument_url_relative_path_should_return_an_absolute_path(self):\n relative_url = SpiderFootHelpers.urlRelativeToAbsolute('/somewhere/else/../../path?param=value#fragment')\n self.assertIsInstance(relative_url, str)\n self.assertEqual('/path?param=value#fragment', relative_url)\n\n def test_url_base_dir_should_return_a_string(self):\n base_dir = SpiderFootHelpers.urlBaseDir('http://localhost.local/path?param=value#fragment')\n self.assertIsInstance(base_dir, str)\n self.assertEqual('http://localhost.local/', base_dir)\n\n def test_url_base_url_should_return_a_string(self):\n base_url = SpiderFootHelpers.urlBaseUrl('http://localhost.local/path?param=value#fragment')\n self.assertIsInstance(base_url, str)\n self.assertEqual('http://localhost.local', base_url)\n\n def test_dictionaryWordsFromWordlists_should_return_a_set(self):\n words = SpiderFootHelpers.dictionaryWordsFromWordlists()\n self.assertIsInstance(words, set)\n self.assertTrue(len(words))\n\n def test_dictionaryWordsFromWordlists_argument_wordlists_missing_wordlist_should_raise_IOError(self):\n with self.assertRaises(IOError):\n SpiderFootHelpers.dictionaryWordsFromWordlists(['does not exist'])\n\n def test_humanNamesFromWordlists_should_return_a_set(self):\n names = SpiderFootHelpers.humanNamesFromWordlists()\n self.assertIsInstance(names, set)\n self.assertTrue(len(names))\n\n def test_humanNamesFromWordlists_argument_wordlists_missing_wordlist_should_raise_IOError(self):\n with self.assertRaises(IOError):\n SpiderFootHelpers.humanNamesFromWordlists(['does not exist'])\n\n def test_usernamesFromWordlists_should_return_a_set(self):\n users = SpiderFootHelpers.usernamesFromWordlists()\n self.assertIsInstance(users, set)\n self.assertTrue(len(users))\n\n def test_usernamesFromWordlists_argument_wordlists_missing_wordlist_should_raise_IOError(self):\n with self.assertRaises(IOError):\n SpiderFootHelpers.usernamesFromWordlists(['does not exist'])\n\n def test_buildGraphData_invalid_data_type_should_raise_TypeError(self):\n invalid_types = [None, \"\", bytes(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n SpiderFootHelpers.buildGraphData(invalid_type)\n\n def test_buildGraphData_empty_data_should_raise_ValueError(self):\n with self.assertRaises(ValueError):\n SpiderFootHelpers.buildGraphData([])\n\n def test_buildGraphData_data_row_with_invalid_number_of_columns_should_raise_ValueError(self):\n with self.assertRaises(ValueError):\n SpiderFootHelpers.buildGraphData(\n [\n ['only one column']\n ]\n )\n\n def test_buildGraphData_should_return_a_set(self):\n graph_data = SpiderFootHelpers.buildGraphData(\n [\n [\"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\"]\n ]\n )\n self.assertIsInstance(graph_data, set)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_buildGraphGexf_should_return_bytes(self):\n gexf = SpiderFootHelpers.buildGraphGexf('test root', 'test title', [[\"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"ENTITY\", \"test\", \"test\", \"test\"]])\n self.assertIsInstance(gexf, bytes)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_buildGraphJson_should_return_a_string(self):\n json = SpiderFootHelpers.buildGraphJson('test root', [[\"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"test\", \"ENTITY\", \"test\", \"test\", \"test\"]])\n self.assertIsInstance(json, str)\n\n self.assertEqual('TBD', 'TBD')\n\n def test_dataParentChildToTree_invalid_data_type_should_return_TypeError(self):\n invalid_types = [None, \"\", bytes(), list(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n SpiderFootHelpers.dataParentChildToTree(invalid_type)\n\n def test_dataParentChildToTree_empty_data_should_return_ValueError(self):\n with self.assertRaises(ValueError):\n SpiderFootHelpers.dataParentChildToTree(dict())\n\n def test_dataParentChildToTree_should_return_dict(self):\n tree = SpiderFootHelpers.dataParentChildToTree({\"test\": {\"123\": \"456\"}})\n self.assertIsInstance(tree, dict)\n\n def test_genScanInstanceId_should_return_a_string(self):\n scan_instance_id = SpiderFootHelpers.genScanInstanceId()\n self.assertIsInstance(scan_instance_id, str)\n\n def test_extractLinksFromHtml_argument_data_not_containing_links_should_return_an_empty_dict(self):\n parse_links = SpiderFootHelpers.extractLinksFromHtml('url', 'example html content', 'domains')\n self.assertIsInstance(parse_links, dict)\n self.assertFalse(parse_links)\n\n def test_extractLinksFromHtml_argument_data_containing_malformed_html_with_links_should_return_a_dict_of_urls(self):\n url = 'http://spiderfoot.net/'\n parse_links = SpiderFootHelpers.extractLinksFromHtml(\n url,\n 'example html content',\n 'domains'\n )\n self.assertIsInstance(parse_links, dict)\n\n self.assertEqual(\n parse_links,\n {\n 'http://spiderfoot.net/xmlrpc.php': {'source': url, 'original': 'http://spiderfoot.net/xmlrpc.php'},\n 'http://spiderfoot.net/path': {'source': url, 'original': 'http://spiderfoot.net/path'},\n 'http://spiderfoot.net/relative-path': {'source': url, 'original': '/relative-path'},\n }\n )\n\n def test_extractLinksFromHtml_invalid_url_should_raise_TypeError(self):\n invalid_types = [None, bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n SpiderFootHelpers.extractLinksFromHtml(invalid_type, 'example html content', 'domains')\n\n def test_extractLinksFromHtml_invalid_data_should_raise_TypeError(self):\n invalid_types = [None, bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n with self.assertRaises(TypeError):\n SpiderFootHelpers.extractLinksFromHtml(\"\", invalid_type, 'domains')\n\n def test_extractLinksFromHtml_invalid_domains_should_return_a_dict(self):\n invalid_types = [None, \"\", bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n parse_links = SpiderFootHelpers.extractLinksFromHtml('url', 'example html content', invalid_type)\n self.assertIsInstance(parse_links, dict)\n\n def test_extractHashesFromText_should_return_a_list(self):\n invalid_types = [None, list(), bytes(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n hashes = SpiderFootHelpers.extractHashesFromText(invalid_type)\n self.assertIsInstance(hashes, list)\n\n def test_extractHashesFromText_argument_data_should_return_hahes(self):\n md5_hash = \"e17cff4eb3e8fbe6ca3b83fb47532dba\"\n sha1_hash = \"f81efbe70f8116fcf3dc4e9b37725dcb949719f5\"\n sha256_hash = \"7cd444af3d8de9e195b1f1cb55e7b7d9409dcd4648247c853a2f64b7578dc9b7\"\n sha512_hash = \"a55a2fe120d7d7d6e2ba930e6c56faa30b9d24a3178a0aff1d89312a89d61d8a9d5b7743e3af6b1a318d99974a1145ed76f85aa8c6574074dfb347613ccd3249\"\n\n hashes = SpiderFootHelpers.extractHashesFromText(f\"spiderfoot{md5_hash}spiderfoot{sha1_hash}spiderfoot{sha256_hash}spiderfoot{sha512_hash}spiderfoot\")\n\n self.assertIsInstance(hashes, list)\n self.assertIn((\"MD5\", md5_hash), hashes)\n self.assertIn((\"SHA1\", sha1_hash), hashes)\n self.assertIn((\"SHA256\", sha256_hash), hashes)\n self.assertIn((\"SHA512\", sha512_hash), hashes)\n\n def test_valid_email_should_return_a_boolean(self):\n invalid_types = [None, \"\", bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n valid_email = SpiderFootHelpers.validEmail(invalid_type)\n self.assertIsInstance(valid_email, bool)\n self.assertFalse(valid_email)\n\n valid_email = SpiderFootHelpers.validEmail('%@localhost.local')\n self.assertIsInstance(valid_email, bool)\n self.assertFalse(valid_email)\n\n valid_email = SpiderFootHelpers.validEmail('...@localhost.local')\n self.assertIsInstance(valid_email, bool)\n self.assertFalse(valid_email)\n\n valid_email = SpiderFootHelpers.validEmail('root@localhost.local\\n.com')\n self.assertIsInstance(valid_email, bool)\n self.assertFalse(valid_email)\n\n valid_email = SpiderFootHelpers.validEmail('root@localhost')\n self.assertIsInstance(valid_email, bool)\n self.assertFalse(valid_email)\n\n valid_email = SpiderFootHelpers.validEmail('root@localhost.local')\n self.assertIsInstance(valid_email, bool)\n self.assertTrue(valid_email)\n\n def test_validPhoneNumber_should_return_a_boolean(self):\n invalid_types = [None, \"\", bytes(), list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n valid_phone = SpiderFootHelpers.validPhoneNumber(invalid_type)\n self.assertIsInstance(valid_phone, bool)\n self.assertFalse(valid_phone)\n\n valid_phone = SpiderFootHelpers.validPhoneNumber('+1234567890')\n self.assertIsInstance(valid_phone, bool)\n self.assertFalse(valid_phone)\n\n valid_phone = SpiderFootHelpers.validPhoneNumber('+12345678901234567890')\n self.assertIsInstance(valid_phone, bool)\n self.assertFalse(valid_phone)\n\n valid_phone = SpiderFootHelpers.validPhoneNumber('+12345678901')\n self.assertIsInstance(valid_phone, bool)\n self.assertTrue(valid_phone)\n\n def test_validLEI_should_return_a_boolean(self):\n invalid_types = [None, \"\", bytes(), list(), dict(), int()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n valid_phone = SpiderFootHelpers.validLEI(invalid_type)\n self.assertIsInstance(valid_phone, bool)\n self.assertFalse(valid_phone)\n\n valid_lei = SpiderFootHelpers.validLEI('7ZW8QJWVPR4P1J1KQYZZ')\n self.assertIsInstance(valid_lei, bool)\n self.assertFalse(valid_lei)\n\n valid_lei = SpiderFootHelpers.validLEI('7ZW8QJWVPR4P1J1KQY45')\n self.assertIsInstance(valid_lei, bool)\n self.assertTrue(valid_lei)\n\n def test_countryCodes_should_return_a_dict(self):\n country_code_dict = SpiderFootHelpers.countryCodes()\n self.assertIsInstance(country_code_dict, dict)\n\n def test_countryNameFromCountryCode_argument_countryCode_should_return_country_as_a_string(self):\n country_name = SpiderFootHelpers.countryNameFromCountryCode('US')\n self.assertIsInstance(country_name, str)\n self.assertEqual(country_name, \"United States\")\n\n def test_countryNameFromTld_argument_tld_should_return_country_as_a_string(self):\n tlds = ['com', 'net', 'org', 'gov', 'mil']\n for tld in tlds:\n with self.subTest(tld=tld):\n country_name = SpiderFootHelpers.countryNameFromTld(tld)\n self.assertIsInstance(country_name, str)\n self.assertEqual(country_name, \"United States\")\n\n def test_extractEmailsFromText_should_return_list_of_emails_from_string(self):\n emails = SpiderFootHelpers.extractEmailsFromText(\"

    From:noreply@spiderfoot.net

    Subject:Hello user@spiderfoot.net, here's some text

    \")\n self.assertIsInstance(emails, list)\n self.assertIn('noreply@spiderfoot.net', emails)\n self.assertIn('user@spiderfoot.net', emails)\n\n def test_extractEmailsFromText_invalid_data_should_return_list(self):\n invalid_types = [None, \"\", bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n emails = SpiderFootHelpers.extractEmailsFromText(invalid_type)\n self.assertIsInstance(emails, list)\n\n def test_extractPgpKeysFromText_should_return_list_of_pgp_keys_from_string(self):\n key1 = \"-----BEGIN PGP PUBLIC KEY BLOCK-----Version: software v1.2.3\\nComment: sample comment\\n\\nmQINBFRUAGoBEACuk6ze2V2pZtScf1Ul25N2CX19AeL7sVYwnyrTYuWdG2FmJx4x\\nDLTLVUazp2AEm/JhskulL/7VCZPyg7ynf+o20Tu9/6zUD7p0rnQA2k3Dz+7dKHHh\\neEsIl5EZyFy1XodhUnEIjel2nGe6f1OO7Dr3UIEQw5JnkZyqMcbLCu9sM2twFyfa\\na8JNghfjltLJs3/UjJ8ZnGGByMmWxrWQUItMpQjGr99nZf4L+IPxy2i8O8WQewB5\\nfvfidBGruUYC+mTw7CusaCOQbBuZBiYduFgH8hRW97KLmHn0xzB1FV++KI7syo8q\\nXGo8Un24WP40IT78XjKO\\n=nUop\\n-----END PGP PUBLIC KEY BLOCK-----\"\n key2 = \"-----BEGIN PGP PRIVATE KEY BLOCK-----Version: software v1.2.3\\nComment: sample comment\\n\\nmQINBFRUAGoBEACuk6ze2V2pZtScf1Ul25N2CX19AeL7sVYwnyrTYuWdG2FmJx4x\\nDLTLVUazp2AEm/JhskulL/7VCZPyg7ynf+o20Tu9/6zUD7p0rnQA2k3Dz+7dKHHh\\neEsIl5EZyFy1XodhUnEIjel2nGe6f1OO7Dr3UIEQw5JnkZyqMcbLCu9sM2twFyfa\\na8JNghfjltLJs3/UjJ8ZnGGByMmWxrWQUItMpQjGr99nZf4L+IPxy2i8O8WQewB5\\nfvfidBGruUYC+mTw7CusaCOQbBuZBiYduFgH8hRW97KLmHn0xzB1FV++KI7syo8q\\nXGo8Un24WP40IT78XjKO\\n=nUop\\n-----END PGP PRIVATE KEY BLOCK-----\"\n keys = SpiderFootHelpers.extractPgpKeysFromText(f\"

    sample{key1}sample

    sample{key2}sample

    \")\n self.assertIsInstance(keys, list)\n self.assertIn(key1, keys)\n self.assertIn(key2, keys)\n self.assertEqual(len(keys), 2)\n\n def test_extractIbansFromText_should_return_a_list(self):\n invalid_types = [None, \"\", bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n ibans = SpiderFootHelpers.extractIbansFromText(invalid_type)\n self.assertIsInstance(ibans, list)\n\n # Example IBANS from https://www.iban.com/structure\n ibans = [\n \"AL35202111090000000001234567\",\n \"AD1400080001001234567890\",\n \"AT483200000012345864\",\n \"AZ96AZEJ00000000001234567890\",\n \"BH02CITI00001077181611\",\n \"BY86AKBB10100000002966000000\",\n \"BE71096123456769\",\n \"BA393385804800211234\",\n \"BR1500000000000010932840814P2\",\n \"BG18RZBB91550123456789\",\n \"CR23015108410026012345\",\n \"HR1723600001101234565\",\n \"CY21002001950000357001234567\",\n \"CZ5508000000001234567899\",\n \"DK9520000123456789\",\n \"DO22ACAU00000000000123456789\",\n \"EG800002000156789012345180002\",\n \"SV43ACAT00000000000000123123\",\n \"EE471000001020145685\",\n \"FO9264600123456789\",\n \"FI1410093000123458\",\n \"FR7630006000011234567890189\",\n \"GE60NB0000000123456789\",\n \"DE75512108001245126199\",\n \"GI04BARC000001234567890\",\n \"GR9608100010000001234567890\",\n \"GL8964710123456789\",\n \"GT20AGRO00000000001234567890\",\n \"VA59001123000012345678\",\n \"HU93116000060000000012345676\",\n \"IS750001121234563108962099\",\n \"IQ20CBIQ861800101010500\",\n \"IE64IRCE92050112345678\",\n \"IL170108000000012612345\",\n \"IT60X0542811101000000123456\",\n \"JO71CBJO0000000000001234567890\",\n \"KZ563190000012344567\",\n \"XK051212012345678906\",\n \"KW81CBKU0000000000001234560101\",\n \"LV97HABA0012345678910\",\n \"LB92000700000000123123456123\",\n \"LI7408806123456789012\",\n \"LT601010012345678901\",\n \"LU120010001234567891\",\n \"MT31MALT01100000000000000000123\",\n \"MR1300020001010000123456753\",\n \"MU43BOMM0101123456789101000MUR\",\n \"MD21EX000000000001234567\",\n \"MC5810096180790123456789085\",\n \"ME25505000012345678951\",\n \"NL02ABNA0123456789\",\n \"MK07200002785123453\",\n \"NO8330001234567\",\n \"PK36SCBL0000001123456702\",\n \"PS92PALS000000000400123456702\",\n \"PL10105000997603123456789123\",\n \"PT50002700000001234567833\",\n \"QA54QNBA000000000000693123456\",\n \"RO09BCYP0000001234567890\",\n \"LC14BOSL123456789012345678901234\",\n \"SM76P0854009812123456789123\",\n \"ST23000200000289355710148\",\n \"SA4420000001234567891234\",\n \"RS35105008123123123173\",\n \"SC52BAHL01031234567890123456USD\",\n \"SK8975000000000012345671\",\n \"SI56192001234567892\",\n \"ES7921000813610123456789\",\n \"SE7280000810340009783242\",\n \"CH5604835012345678009\",\n \"TL380010012345678910106\",\n \"TN5904018104004942712345\",\n \"TR320010009999901234567890\",\n \"UA903052992990004149123456789\",\n \"AE460090000000123456789\",\n \"GB33BUKB20201555555555\",\n \"VG21PACG0000000123456789\"\n ]\n for iban in ibans:\n with self.subTest(iban=iban):\n extract_ibans = SpiderFootHelpers.extractIbansFromText(iban)\n self.assertIsInstance(extract_ibans, list)\n self.assertIn(iban, extract_ibans)\n\n # Invalid IBANs\n ibans = [\n # Invalid country code\n \"ZZ21PACG0000000123456789\",\n # Invalid length for country code\n \"VG123456789012345\",\n # Invalid mod 97 remainder\n \"VG21PACG0000000123456111\"\n ]\n for iban in ibans:\n with self.subTest(iban=iban):\n extract_ibans = SpiderFootHelpers.extractIbansFromText(iban)\n self.assertIsInstance(extract_ibans, list)\n self.assertNotIn(iban, extract_ibans)\n\n def test_extractCreditCardsFromText_should_return_a_list(self):\n invalid_types = [None, \"\", bytes(), list(), dict()]\n for invalid_type in invalid_types:\n with self.subTest(invalid_type=invalid_type):\n cards = SpiderFootHelpers.extractCreditCardsFromText(invalid_type)\n self.assertIsInstance(cards, list)\n\n cards = SpiderFootHelpers.extractCreditCardsFromText(\"spiderfoot4111 1111 1111 1111spiderfoot\")\n self.assertIsInstance(cards, list)\n self.assertEqual([\"4111111111111111\"], cards)\n\n def test_sslDerToPem_should_return_a_certificate_as_a_string(self):\n pem = SpiderFootHelpers.sslDerToPem(\n b\"0\\x82\\x07J0\\x82\\x062\\xa0\\x03\\x02\\x01\\x02\\x02\\x10\\x0c\\x1f\\xcb\\x18E\\x18\\xc7\\xe3\\x86gA#mks\\xf10\\r\\x06\\t*\\x86H\\x86\\xf7\\r\\x01\\x01\\x0b\\x05\\x000O1\\x0b0\\t\\x06\\x03U\\x04\\x06\\x13\\x02US1\\x150\\x13\\x06\\x03U\\x04\\n\\x13\\x0cDigiCert Inc1)0\\'\\x06\\x03U\\x04\\x03\\x13 DigiCert TLS RSA SHA256 2020 CA10\\x1e\\x17\\r230113000000Z\\x17\\r240213235959Z0\\x81\\x961\\x0b0\\t\\x06\\x03U\\x04\\x06\\x13\\x02US1\\x130\\x11\\x06\\x03U\\x04\\x08\\x13\\nCalifornia1\\x140\\x12\\x06\\x03U\\x04\\x07\\x13\\x0bLos Angeles1B0@\\x06\\x03U\\x04\\n\\x0c9Internet\\xc2\\xa0Corporation\\xc2\\xa0for\\xc2\\xa0Assigned\\xc2\\xa0Names\\xc2\\xa0and\\xc2\\xa0Numbers1\\x180\\x16\\x06\\x03U\\x04\\x03\\x13\\x0fwww.example.org0\\x82\\x01\\x220\\r\\x06\\t*\\x86H\\x86\\xf7\\r\\x01\\x01\\x01\\x05\\x00\\x03\\x82\\x01\\x0f\\x000\\x82\\x01\\n\\x02\\x82\\x01\\x01\\x00\\xc2\\x80w\\x89Y\\xb8Eo\\xbaJ\\xd9\\x11\\xfa{\\xad\\xc7W\\xd0z\\xfb\\xb6\\xfa\\xdd\\x05\\xbb\\xa2\\x81q\\xbb\\xe1\\x7f!\\xd2_.\\xf0\\xd2rNu4\\xf8\\x8db\\xe3J\\xdaQ\\x90\\xd4\\x01=\\x9c\\x0c\\xc0q\\xf7\\xe6/\\xb6\\xd6\\x07g&\\xd0\\xde\\xff\\x17\\xce\\xf0\\x85\\xfd1\\xc1f\\xca\\x87e\\x05G*_\\xc0\\xab\\xb8\\x8c\\xc3\\xbf\\xd0\\x17\\x7fc\\xa3\\\\\\xf0F\\xfb\\x86\\xaa\\xfbM\\xd7*^\\x7f\\x9a\\xe0\\x13\\x97}\\xbe\\xfb}5W\\r]^\\x81\\x985\\xea\\x16B\\xa2\\xd3\\xb0t\\xf7Y-\\xed8\\xe7\\xfez\\x1b\\xb36\\xe6~\\xae?\\x9e\\xa6\\x16\\x83\\xdeS\\x01N\\x81\\x00\\xae\\xbbB\\xf5\\x1fu)4\\xcd\\xe9\\x84\\x808\\xae<7\\x14\\xc0\\xf0\\'\\xce0R\\xb9\\x8a\\xdc_\\x22\\xa0y\\xf8ONI\\x04\\xe2u|\\xaa/*\\x1e\\x03\\xecqL\\xa3*a\\xfco\\xca\\x91\\x1e\\x93Z.x\\x08X\\xf6\\xee\\xbb4 ]\\x9a\\xe6\\xaf\\xc6\\xd7\\xf2\\xbf\\n{\\xfa\\x8e\\x92w\\xe3l{\\x0c@\\x86dJ\\x15\\xecp\\xd7r\\x8ec0\\xe1\\x0b\\xefZ0\\x97.%\\x02\\x03\\x01\\x00\\x01\\xa3\\x82\\x03\\xd80\\x82\\x03\\xd40\\x1f\\x06\\x03U\\x1d#\\x04\\x180\\x16\\x80\\x14\\xb7k\\xa2\\xea\\xa8\\xaa\\x84\\x8cy\\xea\\xb4\\xda\\x0f\\x98\\xb2\\xc5\\x95v\\xb9\\xf40\\x1d\\x06\\x03U\\x1d\\x0e\\x04\\x16\\x04\\x14\\xb0\\x93?\\xe8\\x17\\x82\\xfdl\\xb2\\xb6\\x17\\x87\\xcb\\xe3\\x80\\xfe\\x82\\x9b\\x01\\x9e0\\x81\\x81\\x06\\x03U\\x1d\\x11\\x04z0x\\x82\\x0fwww.example.org\\x82\\x0bexample.net\\x82\\x0bexample.edu\\x82\\x0bexample.com\\x82\\x0bexample.org\\x82\\x0fwww.example.com\\x82\\x0fwww.example.edu\\x82\\x0fwww.example.net0\\x0e\\x06\\x03U\\x1d\\x0f\\x01\\x01\\xff\\x04\\x04\\x03\\x02\\x05\\xa00\\x1d\\x06\\x03U\\x1d%\\x04\\x160\\x14\\x06\\x08+\\x06\\x01\\x05\\x05\\x07\\x03\\x01\\x06\\x08+\\x06\\x01\\x05\\x05\\x07\\x03\\x020\\x81\\x8f\\x06\\x03U\\x1d\\x1f\\x04\\x81\\x870\\x81\\x840@\\xa0>\\xa0<\\x86:http://crl3.digicert.com/DigiCertTLSRSASHA2562020CA1-4.crl0@\\xa0>\\xa0<\\x86:http://crl4.digicert.com/DigiCertTLSRSASHA2562020CA1-4.crl0>\\x06\\x03U\\x1d \\x0470503\\x06\\x06g\\x81\\x0c\\x01\\x02\\x020)0\\'\\x06\\x08+\\x06\\x01\\x05\\x05\\x07\\x02\\x01\\x16\\x1bhttp://www.digicert.com/CPS0\\x7f\\x06\\x08+\\x06\\x01\\x05\\x05\\x07\\x01\\x01\\x04s0q0$\\x06\\x08+\\x06\\x01\\x05\\x05\\x070\\x01\\x86\\x18http://ocsp.digicert.com0I\\x06\\x08+\\x06\\x01\\x05\\x05\\x070\\x02\\x86=http://cacerts.digicert.com/DigiCertTLSRSASHA2562020CA1-1.crt0\\t\\x06\\x03U\\x1d\\x13\\x04\\x020\\x000\\x82\\x01\\x7f\\x06\\n+\\x06\\x01\\x04\\x01\\xd6y\\x02\\x04\\x02\\x04\\x82\\x01o\\x04\\x82\\x01k\\x01i\\x00v\\x00\\xee\\xcd\\xd0d\\xd5\\xdb\\x1a\\xce\\xc5\\\\\\xb7\\x9d\\xb4\\xcd\\x13\\xa22\\x87F|\\xbc\\xec\\xde\\xc3QHYFq\\x1f\\xb5\\x9b\\x00\\x00\\x01\\x85\\xabH\\x05#\\x00\\x00\\x04\\x03\\x00G0E\\x02!\\x00\\xaa\\xdf\\x9f+\\xa8\\xc5t`:\\xb6\\xfd\\x04Z\\xdfkk\\x1d\\x16`\\x15x\\xad\\xefc\\x81\\x98*\\xd38\\xb8\\xd9\\x05\\x02 @a\\xd7\\x22\\xa9>\\xf8\\x17\\xd4\\x1a\\xde\\x13L\\x01Rj\\xe29U!%.\\xfb*\\x01u\\xf7w\\xd3\\xdb\\xce\\xfb\\x00w\\x00s\\xd9\\x9e\\x89\\x1bL\\x96x\\xa0 }G\\x9d\\xe6\\xb2\\xc6\\x1c\\xd0Q^q\\x19*\\x8ck\\x80\\x10z\\xc1wr\\xb5\\x00\\x00\\x01\\x85\\xabH\\x05\\x9f\\x00\\x00\\x04\\x03\\x00H0F\\x02!\\x00\\xd7d\\x94\\x14\\xaek\\x80\\xba\\x91\\xce\\xf8\\x1c\\xaf\\xb6sW\\x89\\xe5\\xf9\\x9b}\\x96Z\\x00\\xcd\\x12\\xdf=\\xce\\xefH\\xf0\\x02!\\x00\\x97=\\xbc\\x12s\\x1dk\\x13\\xe0c\\x15\\xac\\x19\\x95X\\xcb\\x8f\\xfdO\\xb0\\xcd\\nA\\x07,|p\\xd9%D\\xcb\\xc0\\x00v\\x00H\\xb0\\xe3k\\xda\\xa6G4\\x0f\\xe5j\\x02\\xfa\\x9d0\\xeb\\x1cR\\x01\\xcbV\\xdd,\\x81\\xd9\\xbb\\xbf\\xab9\\xd8\\x84s\\x00\\x00\\x01\\x85\\xabH\\x05^\\x00\\x00\\x04\\x03\\x00G0E\\x02!\\x00\\xde[\\x84{a\\xa3%\\x8c\\'p\\x90\\x07\\xfdb`Q!2\\x05\\x15\\x90XG\\x0c\\xcf\\xe7\\x94OS\\x84,!\\x02 \\x0f\\xbc\\xf2W\\xca\\x9e\\xda\\xdaL\\xf0%}\\xcf\\xed\\xfa\\x87\\xe5y(\\xde\\xb3\\xe1\\x0b4h]\\x87z[\\xe4$\\n0\\r\\x06\\t*\\x86H\\x86\\xf7\\r\\x01\\x01\\x0b\\x05\\x00\\x03\\x82\\x01\\x01\\x00Y\\xe4J\\xd8\\xa9\\x82\\xba\\x9aJ\\xf1c\\x0cmv&u\\xb3some HTML with \\\"some quotes\\\" & some JavaScript\\n

    \",\n \"Some more input. This function accepts a list\"\n ])\n self.assertIsInstance(clean_input, list)\n self.assertEqual(clean_input, [\n '<p>some HTML with \"some quotes\" & some JavaScript\\n<script>alert('JavaScript')</script></p>',\n \"Some more input. This function accepts a list\"\n ])\n\n def test_search_base_should_return_a_list(self):\n \"\"\"\n Test searchBase(self, id=None, eventType=None, value=None)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n search_results = sfwebui.searchBase(None, None, None)\n self.assertIsInstance(search_results, list)\n\n search_results = sfwebui.searchBase(None, None, \"//\")\n self.assertIsInstance(search_results, list)\n\n @unittest.skip(\"todo\")\n def test_scan_correlations_export(self):\n opts = self.default_options\n opts['__modules__'] = dict()\n scan_id = \"\"\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n search_results = sfwebui.scancorrelationsexport(scan_id, \"csv\", \"excel\")\n self.assertIsInstance(search_results, bytes)\n search_results = sfwebui.scancorrelationsexport(scan_id, \"xlxs\", \"excel\")\n self.assertIsInstance(search_results, bytes)\n\n @unittest.skip(\"todo\")\n def test_scan_event_result_export_should_return_bytes(self):\n \"\"\"\n Test scaneventresultexport(self, id, type, filetype=\"csv\", dialect=\"excel\")\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n search_results = sfwebui.scaneventresultexport(\"\", \"\")\n self.assertIsInstance(search_results, bytes)\n search_results = sfwebui.scaneventresultexport(\"\", \"\", \"excel\")\n self.assertIsInstance(search_results, bytes)\n\n @unittest.skip(\"todo\")\n def test_scan_event_result_export_multi(self):\n \"\"\"\n Test scaneventresultexportmulti(self, ids, filetype=\"csv\", dialect=\"excel\")\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n search_results = sfwebui.scaneventresultexportmulti(\"\", \"\")\n self.assertIsInstance(search_results, bytes)\n search_results = sfwebui.scaneventresultexportmulti(\"\", \"excel\")\n self.assertIsInstance(search_results, bytes)\n\n @unittest.skip(\"todo\")\n def test_scan_search_result_export(self):\n \"\"\"\n Test scansearchresultexport(self, id, eventType=None, value=None, filetype=\"csv\", dialect=\"excel\")\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n search_results = sfwebui.scansearchresultexport(\"\")\n self.assertIsInstance(search_results, bytes)\n search_results = sfwebui.scansearchresultexport(\"\", None, None, \"excel\")\n self.assertIsInstance(search_results, bytes)\n\n def test_scan_export_logs_invalid_scan_id_should_return_string(self):\n \"\"\"\n Test scanexportlogs(self: 'SpiderFootWebUi', id: str, dialect: str = \"excel\") -> str\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n logs = sfwebui.scanexportlogs(None, \"excel\")\n self.assertIsInstance(logs, str)\n self.assertIn(\"Scan ID not found.\", logs)\n\n @unittest.skip(\"todo\")\n def test_scan_export_logs_should_return_bytes(self):\n \"\"\"\n Test scanexportlogs(self: 'SpiderFootWebUi', id: str, dialect: str = \"excel\") -> str\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n logs = sfwebui.scanexportlogs(\"scan id\", \"excel\")\n self.assertIsInstance(logs, bytes)\n\n @unittest.skip(\"todo\")\n def test_scan_export_json_multi(self):\n \"\"\"\n Test scanexportjsonmulti(self, ids)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n search_results = sfwebui.scanexportjsonmulti(None)\n self.assertIsInstance(search_results, bytes)\n\n @unittest.skip(\"todo\")\n def test_scan_viz_should_return_a_string(self):\n \"\"\"\n Test scanviz(self, id, gexf=\"0\")\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_viz = sfwebui.scanviz(None, None)\n self.assertIsInstance(scan_viz, str)\n\n @unittest.skip(\"todo\")\n def test_scan_viz_multi_should_return_a_string(self):\n \"\"\"\n Test scanvizmulti(self, ids, gexf=\"1\")\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_viz_multi = sfwebui.scanvizmulti(None, None)\n self.assertIsInstance(scan_viz_multi, str)\n\n def test_scanopts_should_return_dict(self):\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_opts = sfwebui.scanopts(\"example scan instance\")\n self.assertIsInstance(scan_opts, dict)\n self.assertEqual(scan_opts, dict())\n\n def test_rerunscan_invalid_scan_id_should_return_error(self):\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n rerunscan = sfwebui.rerunscan(\"example scan instance\")\n self.assertIsInstance(rerunscan, str)\n self.assertIn(\"Invalid scan ID\", rerunscan)\n\n @unittest.skip(\"todo\")\n def test_rerunscanmulti(self):\n \"\"\"\n Test rerunscanmulti(self, ids)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n rerunscanmulti = sfwebui.rerunscanmulti(\"example scan instance\")\n self.assertIsInstance(rerunscanmulti, str)\n\n @unittest.skip(\"todo\")\n def test_newscan(self):\n \"\"\"\n Test newscan(self)\n \"\"\"\n self.assertEqual('TBD', 'TBD')\n\n def test_clonescan(self):\n \"\"\"\n Test clonescan(self, id)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n clone_scan = sfwebui.clonescan(\"example scan instance\")\n self.assertIsInstance(clone_scan, str)\n\n def test_index(self):\n \"\"\"\n Test index(self)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n index = sfwebui.index()\n self.assertIsInstance(index, str)\n\n def test_scaninfo(self):\n \"\"\"\n Test scaninfo(self, id)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_info = sfwebui.scaninfo(\"example scan instance\")\n self.assertIsInstance(scan_info, str)\n\n def test_opts(self):\n opts = self.default_options\n opts['__modules__'] = dict()\n opts['__globaloptdescs__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n opts_page = sfwebui.opts()\n self.assertIsInstance(opts_page, str)\n self.assertIn('Settings', opts_page)\n\n def test_optsexport_should_return_a_string(self):\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n opts_export = sfwebui.optsexport(None)\n self.assertIsInstance(opts_export, str)\n\n def test_optsraw_should_return_a_list(self):\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n opts_raw = sfwebui.optsraw()\n self.assertIsInstance(opts_raw, list)\n self.assertEqual(opts_raw[0], 'SUCCESS')\n\n def test_error(self):\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n\n message = \"example message\"\n scan_error = sfwebui.error(message)\n self.assertIsInstance(scan_error, str)\n self.assertIn(\"example message\", scan_error)\n\n def test_scandelete_invalid_scanid_should_return_an_error(self):\n \"\"\"\n Test scandelete(self, id)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_delete = sfwebui.scandelete(\"example scan id\")\n self.assertIsInstance(scan_delete, dict)\n self.assertEqual(\"Scan example scan id does not exist\", scan_delete.get('error').get('message'))\n\n def test_savesettings_invalid_csrf_token_should_return_an_error(self):\n \"\"\"\n Test savesettings(self, allopts, token, configFile=None)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n save_settings = sfwebui.savesettings(None, \"invalid token\", None)\n self.assertIsInstance(save_settings, str)\n self.assertIn(\"Invalid token\", save_settings)\n\n @unittest.skip(\"todo\")\n def test_savesettings(self):\n self.assertEqual('TBD', 'TBD')\n\n def test_savesettingsraw_invalid_csrf_token_should_return_an_error(self):\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n save_settings_raw = sfwebui.savesettingsraw(None, \"invalid token\")\n self.assertIsInstance(save_settings_raw, bytes)\n self.assertIn(b\"Invalid token\", save_settings_raw)\n\n @unittest.skip(\"todo\")\n def test_savesettingsraw(self):\n self.assertEqual('TBD', 'TBD')\n\n def test_reset_settings_should_return_true(self):\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n reset_settings = sfwebui.reset_settings()\n self.assertIsInstance(reset_settings, bool)\n self.assertTrue(reset_settings)\n\n @unittest.skip(\"todo\")\n def test_result_set_fp(self):\n \"\"\"\n Test resultsetfp(self, id, resultids, fp)\n \"\"\"\n self.assertEqual('TBD', 'TBD')\n\n def test_eventtypes_should_return_list(self):\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n event_types = sfwebui.eventtypes()\n self.assertIsInstance(event_types, list)\n\n def test_modules_should_return_list(self):\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n modules = sfwebui.modules()\n self.assertIsInstance(modules, list)\n\n def test_correlationrules_should_return_list(self):\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n correlationrules = sfwebui.correlationrules()\n self.assertIsInstance(correlationrules, list)\n\n def test_ping_should_return_list(self):\n \"\"\"\n Test ping(self)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n ping = sfwebui.ping()\n self.assertIsInstance(ping, list)\n self.assertEqual(ping[0], 'SUCCESS')\n\n def test_query_should_perform_sql_query(self):\n \"\"\"\n Test query(self, query)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n\n select = \"12345\"\n query = sfwebui.query(f\"SELECT {select}\")\n self.assertIsInstance(query, list)\n self.assertEqual(len(query), 1)\n self.assertEqual(str(query[0].get(select)), str(select))\n\n def test_query_invalid_query_should_return_error(self):\n \"\"\"\n Test query(self, query)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n\n query = sfwebui.query(None)\n self.assertIsInstance(query, dict)\n self.assertEqual(\"Invalid query.\", query.get('error').get('message'))\n\n query = sfwebui.query(\"UPDATE 1\")\n self.assertIsInstance(query, dict)\n self.assertEqual(\"Non-SELECTs are unpredictable and not recommended.\", query.get('error').get('message'))\n\n @unittest.skip(\"todo\")\n def test_start_scan_should_start_a_scan(self):\n \"\"\"\n Test startscan(self, scanname, scantarget, modulelist, typelist, usecase)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n start_scan = sfwebui.startscan('example scan name', 'spiderfoot.net', 'example module list', None, None)\n self.assertEqual(start_scan, start_scan)\n self.assertEqual('TBD', 'TBD')\n\n def test_start_scan_invalid_scanname_should_return_error(self):\n \"\"\"\n Test startscan(self, scanname, scantarget, modulelist, typelist, usecase)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n start_scan = sfwebui.startscan(None, 'example scan target', None, None, None)\n self.assertIn('Invalid request: scan name was not specified.', start_scan)\n start_scan = sfwebui.startscan('', 'example scan target', None, None, None)\n self.assertIn('Invalid request: scan name was not specified.', start_scan)\n\n def test_start_scan_invalid_scantarget_should_return_error(self):\n \"\"\"\n Test startscan(self, scanname, scantarget, modulelist, typelist, usecase)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n start_scan = sfwebui.startscan('example scan name', None, None, None, None)\n self.assertIn('Invalid request: scan target was not specified.', start_scan)\n start_scan = sfwebui.startscan('example scan name', '', None, None, None)\n self.assertIn('Invalid request: scan target was not specified.', start_scan)\n\n def test_start_scan_unrecognized_scantarget_type_should_return_error(self):\n \"\"\"\n Test startscan(self, scanname, scantarget, modulelist, typelist, usecase)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n start_scan = sfwebui.startscan('example scan name', 'example scan target', 'example module list', None, None)\n self.assertIn('Invalid target type. Could not recognize it as a target SpiderFoot supports.', start_scan)\n\n def test_start_scan_invalid_modules_should_return_error(self):\n \"\"\"\n Test startscan(self, scanname, scantarget, modulelist, typelist, usecase)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n start_scan = sfwebui.startscan('example scan name', 'spiderfoot.net', None, None, None)\n self.assertIn('Invalid request: no modules specified for scan.', start_scan)\n start_scan = sfwebui.startscan('example scan name', 'spiderfoot.net', '', '', '')\n self.assertIn('Invalid request: no modules specified for scan.', start_scan)\n\n def test_start_scan_invalid_typelist_should_return_error(self):\n \"\"\"\n Test startscan(self, scanname, scantarget, modulelist, typelist, usecase)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n start_scan = sfwebui.startscan('example scan name', 'spiderfoot.net', None, 'invalid type list', None)\n self.assertIn('Invalid request: no modules specified for scan.', start_scan)\n start_scan = sfwebui.startscan('example scan name', 'spiderfoot.net', '', 'invalid type list', '')\n self.assertIn('Invalid request: no modules specified for scan.', start_scan)\n\n def test_stopscan_invalid_scanid_should_return_an_error(self):\n \"\"\"\n Test stopscan(self, id)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n stop_scan = sfwebui.stopscan(\"example scan id\")\n self.assertIsInstance(stop_scan, dict)\n self.assertEqual(\"Scan example scan id does not exist\", stop_scan.get('error').get('message'))\n\n def test_scanlog_should_return_a_list(self):\n \"\"\"\n Test scanlog(self, id, limit=None, rowId=None, reverse=None)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_log = sfwebui.scanlog(None, None, None, None)\n self.assertIsInstance(scan_log, list)\n scan_log = sfwebui.scanlog('', '', '', '')\n self.assertIsInstance(scan_log, list)\n\n def test_scanerrors_should_return_a_list(self):\n \"\"\"\n Test scanerrors(self, id, limit=None)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_errors = sfwebui.scanerrors(None, None)\n self.assertIsInstance(scan_errors, list)\n scan_errors = sfwebui.scanerrors('', '')\n self.assertIsInstance(scan_errors, list)\n\n def test_scanlist_should_return_a_list(self):\n \"\"\"\n Test scanlist(self)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_list = sfwebui.scanlist()\n self.assertIsInstance(scan_list, list)\n\n def test_scanstatus_should_return_a_list(self):\n \"\"\"\n Test scanstatus(self, id)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_status = sfwebui.scanstatus(\"example scan instance\")\n self.assertIsInstance(scan_status, list)\n\n def test_scansummary_should_return_a_list(self):\n \"\"\"\n Test scansummary(self, id, by)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_summary = sfwebui.scansummary(None, None)\n self.assertIsInstance(scan_summary, list)\n scan_summary = sfwebui.scansummary('', '')\n self.assertIsInstance(scan_summary, list)\n\n def test_scaneventresults_should_return_a_list(self):\n \"\"\"\n Test scaneventresults(self, id, eventType, filterfp=False)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_results = sfwebui.scaneventresults(None, None, None)\n self.assertIsInstance(scan_results, list)\n scan_results = sfwebui.scaneventresults('', '', '')\n self.assertIsInstance(scan_results, list)\n\n def test_scaneventresultsunique_should_return_a_list(self):\n \"\"\"\n Test scaneventresultsunique(self, id, eventType, filterfp=False)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_results = sfwebui.scaneventresultsunique(None, None, None)\n self.assertIsInstance(scan_results, list)\n scan_results = sfwebui.scaneventresultsunique('', '', '')\n self.assertIsInstance(scan_results, list)\n\n def test_search_should_return_a_list(self):\n \"\"\"\n Test search(self, id=None, eventType=None, value=None)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n search_results = sfwebui.search(None, None, None)\n self.assertIsInstance(search_results, list)\n search_results = sfwebui.search('', '', '')\n self.assertIsInstance(search_results, list)\n\n def test_scan_history_missing_scanid_should_return_error(self):\n \"\"\"\n Test scanhistory(self, id)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n\n scan_history = sfwebui.scanhistory(None)\n self.assertIsInstance(scan_history, dict)\n self.assertEqual(\"No scan specified\", scan_history.get('error').get('message'))\n scan_history = sfwebui.scanhistory(\"example scan id\")\n self.assertIsInstance(scan_history, list)\n\n def test_scan_history_should_return_a_list(self):\n \"\"\"\n Test scanhistory(self, id)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_history = sfwebui.scanhistory(\"example scan id\")\n self.assertIsInstance(scan_history, list)\n\n def test_scan_element_type_discovery_should_return_a_dict(self):\n \"\"\"\n Test scanelementtypediscovery(self, id, eventType)\n \"\"\"\n opts = self.default_options\n opts['__modules__'] = dict()\n sfwebui = SpiderFootWebUi(self.web_default_options, opts)\n scan_element_type_discovery = sfwebui.scanelementtypediscovery(None, None)\n self.assertIsInstance(scan_element_type_discovery, dict)\n scan_element_type_discovery = sfwebui.scanelementtypediscovery('', '')\n self.assertIsInstance(scan_element_type_discovery, dict)\n" }, { "path": "test/update-requirements", "content": "#!/bin/bash\n# Update and lock all pip packages to latest version.\n#\n# Must be run from SpiderFoot root directory; ie:\n# ./test/update-requirements\n#\n# Requires lock-requirements:\n# pip3 install lock-requirements\n\nlock requirements.txt\ngit diff requirements.txt\n\nlock test/requirements.txt\ngit diff test/requirements.txt\n\n# python3 -m safety check -r test/requirements.txt\n" } ]