[
{
"path": ".dockerignore",
"content": ".github/*\n.gitattributes\n.git-blame-ignore-revs\n.idea/\n.pytest_cache\n.mypy_cache\ntests/*\nREADME/\nbin/\ntheHarvester-logo.png\ntheHarvester-logo.webp\nCHANGELOG.md\n"
},
{
"path": ".git-blame-ignore-revs",
"content": "# #1492 run `black .` and `isort .`\nc13843ec0d513ac7f9c35b7bd0501fa46e356415"
},
{
"path": ".gitattributes",
"content": "# Set the default behavior, which is to have git automatically determine\n# whether a file is a text or binary, unless otherwise specified.\n\n* text=auto\n\n# Basic .gitattributes for a python repo.\n\n# Source files\n# ============\n*.pxd text diff=python\n*.py text diff=python\n*.py3 text diff=python\n*.pyw text diff=python\n*.pyx text diff=python\n\n# Binary files\n# ============\n*.db binary\n*.p binary\n*.pkl binary\n*.pyc binary\n*.pyd binary\n*.pyo binary\n\n# Note: .db, .p, and .pkl files are associated with the python modules\n# ``pickle``, ``dbm.*``, # ``shelve``, ``marshal``, ``anydbm``, & ``bsddb``\n# (among others).\n"
},
{
"path": ".github/FUNDING.yml",
"content": "# These are supported funding model platforms\n\ngithub: [L1ghtn1ng, NotoriousRebel]\nopen_collective: # Replace with a single Open Collective username\nko_fi: #\ntidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel\ncommunity_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry\nliberapay: # Replace with a single Liberapay username\nissuehunt: # Replace with a single IssueHunt username\notechie: # Replace with a single Otechie username\ncustom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']\n"
},
{
"path": ".github/ISSUE_TEMPLATE/issue-template.md",
"content": "---\nname: Issue Template\nabout: A template for new issues.\ntitle: \"[Bug|Feature Request|Other] Short Description of Issue\"\nlabels: ''\n\n---\n\n## Note we do not support installing theHarvester on android\n\n**Feature Request or Bug or Another**\nFeature Request | Bug | Other\n\n**Describe the feature request or bug or other**\nA clear and concise description of what the bug, feature request,\nor other request is.\n\n**To Reproduce**\nSteps to reproduce the behaviour:\n1. Run tool like this: '...'\n2. See error\n\n**Expected behaviour**\nA clear and concise description of what you expected to happen.\n\n**Screenshots**\nIf possible please add screenshots to help explain your problem.\n\n**System Information (System that tool is running on):**\n - OS: [e.g. Windows10]\n - Version [e.g. 2.7]\n\n**Additional context**\nAdd any other context about the problem here.\n"
},
{
"path": ".github/dependabot.yml",
"content": "version: 2\nupdates:\n- package-ecosystem: github-actions\n directory: \"/\"\n schedule:\n interval: daily\n timezone: Europe/London\n- package-ecosystem: uv\n directory: \"/\"\n schedule:\n interval: daily\n timezone: Europe/London\n open-pull-requests-limit: 10\n target-branch: master\n allow:\n - dependency-type: direct\n - dependency-type: indirect\n"
},
{
"path": ".github/workflows/codeql-analysis.yml",
"content": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# You may wish to alter this file to override the set of languages analyzed,\n# or to provide custom queries or build logic.\n#\n# ******** NOTE ********\n# We have attempted to detect the languages in your repository. Please check\n# the `language` matrix defined below to confirm you have the correct set of\n# supported CodeQL languages.\n#\nname: \"CodeQL\"\n\non:\n push:\n branches: [ master, dev ]\n pull_request:\n # The branches below must be a subset of the branches above\n branches: [ master, dev ]\n schedule:\n - cron: '19 11 * * 4'\n\njobs:\n analyze:\n name: Analyze\n runs-on: ubuntu-latest\n\n strategy:\n fail-fast: false\n matrix:\n language: [ 'python' ]\n # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]\n # Learn more:\n # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v6\n\n # Initializes the CodeQL tools for scanning.\n - name: Initialize CodeQL\n uses: github/codeql-action/init@v4\n with:\n languages: ${{ matrix.language }}\n # If you wish to specify custom queries, you can do so here or in a config file.\n # By default, queries listed here will override any specified in a config file.\n # Prefix the list here with \"+\" to use these queries and those in the config file.\n # queries: ./path/to/local/query, your-org/your-repo/queries@main\n\n # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).\n # If this step fails, then you should remove it and run the build manually (see below)\n - name: Autobuild\n uses: github/codeql-action/autobuild@v4\n\n # âšī¸ Command-line programs to run using the OS shell.\n # đ https://git.io/JvXDl\n\n # âī¸ If the Autobuild fails above, remove it and uncomment the following three lines\n # and modify them (or add more) to build your code if your project\n # uses a compiled language\n\n #- run: |\n # make bootstrap\n # make release\n\n - name: Perform CodeQL Analysis\n uses: github/codeql-action/analyze@v4\n"
},
{
"path": ".github/workflows/docker-build-push.yml",
"content": "name: Build and Push Docker Image\n\non:\n push:\n branches:\n - master\n\npermissions:\n contents: read\n packages: write\n\njobs:\n build-and-push:\n runs-on: ubuntu-latest\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v6\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v4\n\n - name: Log in to GitHub Container Registry\n uses: docker/login-action@v4\n with:\n registry: ghcr.io\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Extract metadata for Docker\n id: meta\n uses: docker/metadata-action@v6\n with:\n images: ghcr.io/${{ github.repository_owner }}/theharvester\n tags: |\n latest\n type=ref,event=branch\n type=sha\n\n - name: Build and push Docker image\n uses: docker/build-push-action@v7\n with:\n context: .\n file: Dockerfile\n push: true\n platforms: linux/amd64,linux/arm64\n tags: ${{ steps.meta.outputs.tags }}\n labels: ${{ steps.meta.outputs.labels }}\n"
},
{
"path": ".github/workflows/dockerci.yml",
"content": "name: TheHarvester Docker Image CI\n\non: [push, pull_request]\n\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n - name: Build the Docker image\n run: docker build --tag theharvester .\n - name: Smoke test\n run: docker run --rm theharvester --help | grep restfulHarvest\n"
},
{
"path": ".github/workflows/theHarvester.yml",
"content": "name: TheHarvester Python CI\n\non:\n push:\n branches:\n - '*'\n\n pull_request:\n branches:\n - '*'\n\njobs:\n Python:\n runs-on: ${{ matrix.os }}\n strategy:\n max-parallel: 10\n matrix:\n os: [ ubuntu-latest ]\n python-version: [ '3.12', '3.13', '3.14' ]\n\n steps:\n - uses: actions/checkout@v6\n - name: Install uv\n uses: astral-sh/setup-uv@v7\n with:\n python-version: ${{ matrix.python-version }}\n enable-cache: true\n cache-dependency-glob: \"uv.lock\"\n\n - name: Install dependencies\n run: |\n sudo mkdir -p /usr/local/etc/theHarvester\n sudo cp theHarvester/data/*.yaml /usr/local/etc/theHarvester/\n sudo chown -R runner:runner /usr/local/etc/theHarvester/\n uv sync --all-groups --frozen\n echo \"$GITHUB_WORKSPACE/.venv/bin\" >> $GITHUB_PATH\n\n - name: Lint with ruff\n uses: astral-sh/ruff-action@v3\n with:\n args: check --fix\n\n - name: Format with ruff\n uses: astral-sh/ruff-action@v3\n with:\n args: format\n\n - name: Commit changes for ruff formating and linting\n if: github.event_name == 'push'\n run: |\n git config user.name github-actions\n git config user.email github-actions@github.com\n git add .\n git commit -m \"Apply ruff fixes and formatting\" || true # Use || true to prevent failure if no changes\n git push origin HEAD:${{ github.ref_name }}\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Test with pytest\n run: |\n pytest tests/**\n\n - name: Run theHarvester module Baidu\n run: |\n theHarvester -d yale.edu -b baidu\n\n - name: Run theHarvester module CertSpotter\n run: |\n theHarvester -d yale.edu -b certspotter\n\n - name: Run theHarvester module Crtsh\n run: |\n theHarvester -d hcl.com -b crtsh\n\n - name: Run theHarvester module DuckDuckGo\n run: |\n theHarvester -d yale.edu -b duckduckgo\n\n - name: Run theHarvester module HackerTarget\n run: |\n theHarvester -d yale.edu -b hackertarget\n\n - name: Run theHarvester module Otx\n run: |\n theHarvester -d yale.edu -b otx\n\n - name: Run theHarvester module RapidDns\n run: |\n theHarvester -d yale.edu -b rapiddns\n\n - name: Run theHarvester module Urlscan\n run: |\n theHarvester -d yale.edu -b urlscan\n\n - name: Run theHarvester module Yahoo\n run: |\n theHarvester -d yale.edu -b yahoo\n\n - name: Run theHarvester module DNS brute force\n run: |\n theHarvester -d yale.edu -c\n"
},
{
"path": ".gitignore",
"content": "*.idea\n*.pyc\n*.sqlite\n*.html\n*.htm\n*.vscode\n*.xml\n*.json\ndebug_results.txt\nvenv\n.mypy_cache\n.pytest_cache\nbuild/\ndist/\ntheHarvester.egg-info\napi-keys.yaml\n.DS_Store\n.venv\n.venv/**\n.pyre\n.junie"
},
{
"path": "CHANGELOG.md",
"content": "# Changelog\n\nAll notable changes to this project will be documented in this file.\n\nThe format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),\nand this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n\n## [Unreleased]\n\n## [4.10.1] - 2026-02-22\n\n### Changed\n- Updated Censys integration to align with current API documentation ([67419190](https://github.com/laramies/theHarvester/commit/67419190)).\n- Updated RocketReach integration to align with latest API documentation and tests ([ffc7420d](https://github.com/laramies/theHarvester/commit/ffc7420d)).\n- Refactored async file handling in CLI paths: replace blocking path calls with awaited operations and improve path sanitization ([e98bf5bb](https://github.com/laramies/theHarvester/commit/e98bf5bb), [607016a1](https://github.com/laramies/theHarvester/commit/607016a1)).\n- Migrated packaging/build configuration to `flit-core` and updated entrypoint/version wiring ([d2cae0be](https://github.com/laramies/theHarvester/commit/d2cae0be)).\n- Refactored and standardized output utilities, with new regression tests for output formatting and dedup helpers ([fa2dedd3](https://github.com/laramies/theHarvester/commit/fa2dedd3)).\n- Updated dependencies: bump `fastapi`, `playwright`, `ruff`, `ty`, and `uvicorn` ([1dfa6e98](https://github.com/laramies/theHarvester/commit/1dfa6e98), [46865337](https://github.com/laramies/theHarvester/commit/46865337), [c1ac137d](https://github.com/laramies/theHarvester/commit/c1ac137d), [7eaec4da](https://github.com/laramies/theHarvester/commit/7eaec4da)).\n- Updated packaging dependency `wheel` to `0.46.3` ([46865337](https://github.com/laramies/theHarvester/commit/46865337)).\n\n### Fixed\n- Fixed CriminalIP integration for current API behavior, including safer scan/report handling and hostname normalization (issue #2229) ([06c2fbd9](https://github.com/laramies/theHarvester/commit/06c2fbd9)).\n- Fixed Shodan engine processing to return hostnames consistently and avoid worker processing errors (issue #2227) ([419291a3](https://github.com/laramies/theHarvester/commit/419291a3)).\n- Fixed Bitbucket search flow so discovery runs successfully ([a1968f71](https://github.com/laramies/theHarvester/commit/a1968f71)).\n- Improved module API key error messages for clearer diagnostics ([e1b775e3](https://github.com/laramies/theHarvester/commit/e1b775e3)).\n- Improved BuiltWith URL handling logic in CLI processing ([15872350](https://github.com/laramies/theHarvester/commit/15872350)).\n\n## [4.10.0] - 2026-01-18\n\n### Added\n- LeakIX API key support and improved request header configuration ([31861c19](https://github.com/laramies/theHarvester/commit/31861c19)).\n- Bitbucket API key entry in `theHarvester/data/api-keys.yaml` ([6be673fa](https://github.com/laramies/theHarvester/commit/6be673fa)).\n- Fix issue #469 Add socks proxy support ([e38bb8fb](https://github.com/laramies/theHarvester/commit/e38bb8fb)).\n\n### Changed\n- CI: switch GitHub workflow to `ruff-action` for linting and formatting ([8ddcd1a8](https://github.com/laramies/theHarvester/commit/8ddcd1a8)).\n- Dockerfile: add `apt-get update/upgrade` and clean up apt cache layers ([3a5d504b](https://github.com/laramies/theHarvester/commit/3a5d504b)).\n- Dependencies updated: bump `aiodns`, `ruff`, `ty`, `filelock`, and `librt` ([40759146](https://github.com/laramies/theHarvester/commit/40759146)).\n- Codebase formatting and lint fixes applied (Ruff) ([7c6dec53](https://github.com/laramies/theHarvester/commit/7c6dec53)).\n- Tests: expand proxy parameter default structure to include both `http` and `socks5` fields ([bc2fce07](https://github.com/laramies/theHarvester/commit/bc2fce07)).\n- `api-keys.yaml` synchronized with `Core` API key references; add consistency test coverage ([ffe1f3a8](https://github.com/laramies/theHarvester/commit/ffe1f3a8)).\n\n### Removed\n- `Core.bing_key()` removed ([814c7811](https://github.com/laramies/theHarvester/commit/814c7811)).\n\n### Fixed\n- Fix mypy type-checking errors ([0991356b](https://github.com/laramies/theHarvester/commit/0991356b)).\n\n### Security\n- Improve input sanitization and add security-focused tests ([3d7489c9](https://github.com/laramies/theHarvester/commit/3d7489c9)).\n\n[Unreleased]: https://github.com/laramies/theHarvester/compare/06520b40...master\n[4.10.1]: https://github.com/laramies/theHarvester/compare/4.10.0...06520b40\n[4.10.0]: https://github.com/laramies/theHarvester/compare/4.9.2...4.10.0\n"
},
{
"path": "Dockerfile",
"content": "FROM python:3.14-slim-trixie\n\nLABEL maintainer=\"@jay_townsend1 & @NotoriousRebel1\"\n\nRUN useradd -m -u 1000 -s /bin/bash theharvester\n\nRUN apt-get update && apt-get upgrade -yqq && apt-get clean && \\\n rm -rf /var/lib/apt/lists/*\n# Set workdir and copy project files\nWORKDIR /app\nCOPY . /app\n\n# Create and sync environment using uv\n# Compile bytecode for faster startup and install to system site-packages\nRUN --mount=from=ghcr.io/astral-sh/uv,source=/uv,target=/bin/uv \\\n UV_PROJECT_ENVIRONMENT=/usr/local uv sync --locked --no-dev --no-cache --compile-bytecode\n\n# Use non-root user\nUSER theharvester\n\n# Expose port if the service listens on 80\nEXPOSE 80\n\n# Run the application as theharvester user\nENTRYPOINT [\"restfulHarvest\", \"-H\", \"0.0.0.0\", \"-p\", \"80\"]\n"
},
{
"path": "README/CONTRIBUTING.md",
"content": "# Contributing to theHarvester Project\nWelcome to theHarvester project, so you would like to contribute.\nThe following below must be met to get accepted.\n\n# CI\nMake sure all CI passes and you do not introduce any alerts from ruff\n\n# Unit Tests\nFor new modules a unit test for that module is required and we use pytest.\n\n# Coding Standards\n* No single letter variables and variable names must represent the action that it is performing\n* Have static typing on functions etc\n* Make sure no errors are reported from mypy\n* No issues reported with ruff\n \n# Submitting Bugs\nIf you find a bug in a module that you want to submit an issue for and know how to write python code.\nPlease create a unit test for that bug(If possible) and submit a fix for it as it would be a big help to the project. \n"
},
{
"path": "README/COPYING",
"content": " GNU GENERAL PUBLIC LICENSE\n Version 2, June 1991\n\n Copyright (C) 1989, 1991 Free Software Foundation, Inc.\n 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA\n Everyone is permitted to copy and distribute verbatim copies\n of this license document, but changing it is not allowed.\n\n Preamble\n\n The licenses for most software are designed to take away your\nfreedom to share and change it. By contrast, the GNU General Public\nLicense is intended to guarantee your freedom to share and change free\nsoftware--to make sure the software is free for all its users. This\nGeneral Public License applies to most of the Free Software\nFoundation's software and to any other program whose authors commit to\nusing it. (Some other Free Software Foundation software is covered by\nthe GNU Library General Public License instead.) You can apply it to\nyour programs, too.\n\n When we speak of free software, we are referring to freedom, not\nprice. Our General Public Licenses are designed to make sure that you\nhave the freedom to distribute copies of free software (and charge for\nthis service if you wish), that you receive source code or can get it\nif you want it, that you can change the software or use pieces of it\nin new free programs; and that you know you can do these things.\n\n To protect your rights, we need to make restrictions that forbid\nanyone to deny you these rights or to ask you to surrender the rights.\nThese restrictions translate to certain responsibilities for you if you\ndistribute copies of the software, or if you modify it.\n\n For example, if you distribute copies of such a program, whether\ngratis or for a fee, you must give the recipients all the rights that\nyou have. You must make sure that they, too, receive or can get the\nsource code. And you must show them these terms so they know their\nrights.\n\n We protect your rights with two steps: (1) copyright the software, and\n(2) offer you this license which gives you legal permission to copy,\ndistribute and/or modify the software.\n\n Also, for each author's protection and ours, we want to make certain\nthat everyone understands that there is no warranty for this free\nsoftware. If the software is modified by someone else and passed on, we\nwant its recipients to know that what they have is not the original, so\nthat any problems introduced by others will not reflect on the original\nauthors' reputations.\n\n Finally, any free program is threatened constantly by software\npatents. We wish to avoid the danger that redistributors of a free\nprogram will individually obtain patent licenses, in effect making the\nprogram proprietary. To prevent this, we have made it clear that any\npatent must be licensed for everyone's free use or not licensed at all.\n\n The precise terms and conditions for copying, distribution and\nmodification follow.\n\n GNU GENERAL PUBLIC LICENSE\n TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION\n\n 0. This License applies to any program or other work which contains\na notice placed by the copyright holder saying it may be distributed\nunder the terms of this General Public License. The \"Program\", below,\nrefers to any such program or work, and a \"work based on the Program\"\nmeans either the Program or any derivative work under copyright law:\nthat is to say, a work containing the Program or a portion of it,\neither verbatim or with modifications and/or translated into another\nlanguage. (Hereinafter, translation is included without limitation in\nthe term \"modification\".) Each licensee is addressed as \"you\".\n\nActivities other than copying, distribution and modification are not\ncovered by this License; they are outside its scope. The act of\nrunning the Program is not restricted, and the output from the Program\nis covered only if its contents constitute a work based on the\nProgram (independent of having been made by running the Program).\nWhether that is true depends on what the Program does.\n\n 1. You may copy and distribute verbatim copies of the Program's\nsource code as you receive it, in any medium, provided that you\nconspicuously and appropriately publish on each copy an appropriate\ncopyright notice and disclaimer of warranty; keep intact all the\nnotices that refer to this License and to the absence of any warranty;\nand give any other recipients of the Program a copy of this License\nalong with the Program.\n\nYou may charge a fee for the physical act of transferring a copy, and\nyou may at your option offer warranty protection in exchange for a fee.\n\n 2. You may modify your copy or copies of the Program or any portion\nof it, thus forming a work based on the Program, and copy and\ndistribute such modifications or work under the terms of Section 1\nabove, provided that you also meet all of these conditions:\n\n a) You must cause the modified files to carry prominent notices\n stating that you changed the files and the date of any change.\n\n b) You must cause any work that you distribute or publish, that in\n whole or in part contains or is derived from the Program or any\n part thereof, to be licensed as a whole at no charge to all third\n parties under the terms of this License.\n\n c) If the modified program normally reads commands interactively\n when run, you must cause it, when started running for such\n interactive use in the most ordinary way, to print or display an\n announcement including an appropriate copyright notice and a\n notice that there is no warranty (or else, saying that you provide\n a warranty) and that users may redistribute the program under\n these conditions, and telling the user how to view a copy of this\n License. (Exception: if the Program itself is interactive but\n does not normally print such an announcement, your work based on\n the Program is not required to print an announcement.)\n\nThese requirements apply to the modified work as a whole. If\nidentifiable sections of that work are not derived from the Program,\nand can be reasonably considered independent and separate works in\nthemselves, then this License, and its terms, do not apply to those\nsections when you distribute them as separate works. But when you\ndistribute the same sections as part of a whole which is a work based\non the Program, the distribution of the whole must be on the terms of\nthis License, whose permissions for other licensees extend to the\nentire whole, and thus to each and every part regardless of who wrote it.\n\nThus, it is not the intent of this section to claim rights or contest\nyour rights to work written entirely by you; rather, the intent is to\nexercise the right to control the distribution of derivative or\ncollective works based on the Program.\n\nIn addition, mere aggregation of another work not based on the Program\nwith the Program (or with a work based on the Program) on a volume of\na storage or distribution medium does not bring the other work under\nthe scope of this License.\n\n 3. You may copy and distribute the Program (or a work based on it,\nunder Section 2) in object code or executable form under the terms of\nSections 1 and 2 above provided that you also do one of the following:\n\n a) Accompany it with the complete corresponding machine-readable\n source code, which must be distributed under the terms of Sections\n 1 and 2 above on a medium customarily used for software interchange; or,\n\n b) Accompany it with a written offer, valid for at least three\n years, to give any third party, for a charge no more than your\n cost of physically performing source distribution, a complete\n machine-readable copy of the corresponding source code, to be\n distributed under the terms of Sections 1 and 2 above on a medium\n customarily used for software interchange; or,\n\n c) Accompany it with the information you received as to the offer\n to distribute corresponding source code. (This alternative is\n allowed only for noncommercial distribution and only if you\n received the program in object code or executable form with such\n an offer, in accord with Subsection b above.)\n\nThe source code for a work means the preferred form of the work for\nmaking modifications to it. For an executable work, complete source\ncode means all the source code for all modules it contains, plus any\nassociated interface definition files, plus the scripts used to\ncontrol compilation and installation of the executable. However, as a\nspecial exception, the source code distributed need not include\nanything that is normally distributed (in either source or binary\nform) with the major components (compiler, kernel, and so on) of the\noperating system on which the executable runs, unless that component\nitself accompanies the executable.\n\nIf distribution of executable or object code is made by offering\naccess to copy from a designated place, then offering equivalent\naccess to copy the source code from the same place counts as\ndistribution of the source code, even though third parties are not\ncompelled to copy the source along with the object code.\n\n 4. You may not copy, modify, sublicense, or distribute the Program\nexcept as expressly provided under this License. Any attempt\notherwise to copy, modify, sublicense or distribute the Program is\nvoid, and will automatically terminate your rights under this License.\nHowever, parties who have received copies, or rights, from you under\nthis License will not have their licenses terminated so long as such\nparties remain in full compliance.\n\n 5. You are not required to accept this License, since you have not\nsigned it. However, nothing else grants you permission to modify or\ndistribute the Program or its derivative works. These actions are\nprohibited by law if you do not accept this License. Therefore, by\nmodifying or distributing the Program (or any work based on the\nProgram), you indicate your acceptance of this License to do so, and\nall its terms and conditions for copying, distributing or modifying\nthe Program or works based on it.\n\n 6. Each time you redistribute the Program (or any work based on the\nProgram), the recipient automatically receives a license from the\noriginal licensor to copy, distribute or modify the Program subject to\nthese terms and conditions. You may not impose any further\nrestrictions on the recipients' exercise of the rights granted herein.\nYou are not responsible for enforcing compliance by third parties to\nthis License.\n\n 7. If, as a consequence of a court judgment or allegation of patent\ninfringement or for any other reason (not limited to patent issues),\nconditions are imposed on you (whether by court order, agreement or\notherwise) that contradict the conditions of this License, they do not\nexcuse you from the conditions of this License. If you cannot\ndistribute so as to satisfy simultaneously your obligations under this\nLicense and any other pertinent obligations, then as a consequence you\nmay not distribute the Program at all. For example, if a patent\nlicense would not permit royalty-free redistribution of the Program by\nall those who receive copies directly or indirectly through you, then\nthe only way you could satisfy both it and this License would be to\nrefrain entirely from distribution of the Program.\n\nIf any portion of this section is held invalid or unenforceable under\nany particular circumstance, the balance of the section is intended to\napply and the section as a whole is intended to apply in other\ncircumstances.\n\nIt is not the purpose of this section to induce you to infringe any\npatents or other property right claims or to contest validity of any\nsuch claims; this section has the sole purpose of protecting the\nintegrity of the free software distribution system, which is\nimplemented by public license practices. Many people have made\ngenerous contributions to the wide range of software distributed\nthrough that system in reliance on consistent application of that\nsystem; it is up to the author/donor to decide if he or she is willing\nto distribute software through any other system and a licensee cannot\nimpose that choice.\n\nThis section is intended to make thoroughly clear what is believed to\nbe a consequence of the rest of this License.\n\n 8. If the distribution and/or use of the Program is restricted in\ncertain countries either by patents or by copyrighted interfaces, the\noriginal copyright holder who places the Program under this License\nmay add an explicit geographical distribution limitation excluding\nthose countries, so that distribution is permitted only in or among\ncountries not thus excluded. In such case, this License incorporates\nthe limitation as if written in the body of this License.\n\n 9. The Free Software Foundation may publish revised and/or new versions\nof the General Public License from time to time. Such new versions will\nbe similar in spirit to the present version, but may differ in detail to\naddress new problems or concerns.\n\nEach version is given a distinguishing version number. If the Program\nspecifies a version number of this License which applies to it and \"any\nlater version\", you have the option of following the terms and conditions\neither of that version or of any later version published by the Free\nSoftware Foundation. If the Program does not specify a version number of\nthis License, you may choose any version ever published by the Free Software\nFoundation.\n\n 10. If you wish to incorporate parts of the Program into other free\nprograms whose distribution conditions are different, write to the author\nto ask for permission. For software which is copyrighted by the Free\nSoftware Foundation, write to the Free Software Foundation; we sometimes\nmake exceptions for this. Our decision will be guided by the two goals\nof preserving the free status of all derivatives of our free software and\nof promoting the sharing and reuse of software generally.\n\n NO WARRANTY\n\n 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY\nFOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN\nOTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES\nPROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED\nOR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\nMERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS\nTO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE\nPROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,\nREPAIR OR CORRECTION.\n\n 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\nWILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR\nREDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\nINCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\nOUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED\nTO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY\nYOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER\nPROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE\nPOSSIBILITY OF SUCH DAMAGES.\n\n END OF TERMS AND CONDITIONS\n"
},
{
"path": "README/LICENSES",
"content": "Released under the GPL v 2.0.\n\nIf you did not receive a copy of the GPL, try http://www.gnu.org/.\n\nCopyright 2011 Christian Martorella \n\ntheHarvester is free software; you can redistribute it and/or modify\nit under the terms of the GNU General Public License as published by\nthe Free Software Foundation version 2 of the License.\n\ntheHarvester is distributed in the hope that it will be useful,\nbut WITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\nGNU General Public License for more details.\nFoundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA\n"
},
{
"path": "README.md",
"content": "\n\n \n[](https://inventory.raw.pm/)\n\n[](https://repology.org/project/theharvester/versions)\n\nAbout\n-----\ntheHarvester is a simple to use, yet powerful tool designed to be used during the reconnaissance stage of a red\nteam assessment or penetration test. It performs open source intelligence (OSINT) gathering to help determine\na domain's external threat landscape. The tool gathers names, emails, IPs, subdomains, and URLs by using\nmultiple public resources that include:\n\nInstall and dependencies\n------------------------\n* Python 3.12 or higher.\n* https://github.com/laramies/theHarvester/wiki/Installation\n\nInstall uv:\n ```bash\n curl -LsSf https://astral.sh/uv/install.sh | sh\n ```\n\nClone the repository:\n ```bash\n git clone https://github.com/laramies/theHarvester\n cd theHarvester\n ```\n\nInstall dependencies and create a virtual environment:\n ```bash\n uv sync\n ```\n\nRun theHarvester:\n ```bash\n uv run theHarvester\n ```\n\n## Development\n\nTo install development dependencies:\n```bash\nuv sync --all-groups\n```\n\nTo run tests:\n```bash\nuv run pytest\n```\n\nTo run linting and formatting:\n```bash\nuv run ruff check\n```\n```bash\nuv run ruff format\n```\n\nPassive modules\n---------------\n\n* baidu: Baidu search engine (https://www.baidu.com)\n\n* bevigil: CloudSEK BeVigil scans mobile application for OSINT assets (https://bevigil.com/osint-api)\n\n* brave: Brave search engine - now uses official Brave Search API (https://api-dashboard.search.brave.com)\n\n* bufferoverun: Fast domain name lookups for TLS certificates in IPv4 space (https://tls.bufferover.run)\n\n* builtwith: Find out what websites are built with (https://builtwith.com)\n\n* censys: Uses certificates searches to enumerate subdomains and gather emails (https://censys.io)\n\n* certspotter: Cert Spotter monitors Certificate Transparency logs (https://sslmate.com/certspotter)\n\n* criminalip: Specialized Cyber Threat Intelligence (CTI) search engine (https://www.criminalip.io)\n\n* crtsh: Comodo Certificate search (https://crt.sh)\n\n* dehashed: Take your data security to the next level is (https://dehashed.com)\n\n* dnsdumpster: Domain research tool that can discover hosts related to a domain (https://dnsdumpster.com)\n\n* duckduckgo: DuckDuckGo search engine (https://duckduckgo.com)\n\n* fofa: FOFA search eingine (https://en.fofa.info)\n\n* fullhunt: Next-generation attack surface security platform (https://fullhunt.io)\n\n* github-code: GitHub code search engine (https://www.github.com)\n\n* hackertarget: Online vulnerability scanners and network intelligence to help organizations (https://hackertarget.com)\n\n* haveibeenpwned: Check if your email address is in a data breach (https://haveibeenpwned.com)\n\n* hunter: Hunter search engine (https://hunter.io)\n\n* hunterhow: Internet search engines for security researchers (https://hunter.how)\n\n* intelx: Intelx search engine (https://intelx.io)\n\n* leakix: LeakIX search engine (https://leakix.net)\n\n* leaklookup: Data breach search engine (https://leak-lookup.com)\n\n* mojeek: Mojeek search engine (https://www.mojeek.com)\n\n* netlas: A Shodan or Censys competitor (https://app.netlas.io)\n\n* onyphe: Cyber defense search engine (https://www.onyphe.io)\n\n* otx: AlienVault open threat exchange (https://otx.alienvault.com)\n\n* pentesttools: Cloud-based toolkit for offensive security testing, focused on web applications and network penetration testing (https://pentest-tools.com)\n\n* projecdiscovery: Actively collects and maintains internet-wide assets data, to enhance research and analyse changes around DNS for better insights (https://chaos.projectdiscovery.io)\n\n* rapiddns: DNS query tool which make querying subdomains or sites of a same IP easy (https://rapiddns.io)\n\n* rocketreach: Access real-time verified personal/professional emails, phone numbers, and social media links (https://rocketreach.co)\n\n* securityscorecard: helps TPRM and SOC teams detect, prioritize, and remediate vendor risk across their entire supplier ecosystem at scale (https://securityscorecard.com)\n\n* securityTrails: Security Trails search engine, the world's largest repository of historical DNS data (https://securitytrails.com)\n\n* -s, --shodan: Shodan search engine will search for ports and banners from discovered hosts (https://shodan.io)\n\n* subdomaincenter: A subdomain finder tool used to find subdomains of a given domain (https://www.subdomain.center)\n\n* subdomainfinderc99: A subdomain finder is a tool used to find the subdomains of a given domain (https://subdomainfinder.c99.nl)\n\n* thc: Free subdomain enumeration service with no API key required (https://ip.thc.org)\n\n* threatminer: Data mining for threat intelligence (https://www.threatminer.org)\n\n* tomba: Tomba search engine (https://tomba.io)\n\n* urlscan: A sandbox for the web that is a URL and website scanner (https://urlscan.io)\n\n* venacus: Venacus search engine (https://venacus.com)\n\n* virustotal: Domain search (https://www.virustotal.com)\n\n* whoisxml: Subdomain search (https://subdomains.whoisxmlapi.com/api/pricing)\n\n* yahoo: Yahoo search engine (https://www.yahoo.com)\n\n* windvane: Windvane search engine (https://windvane.lichoin.com)\n\n* zoomeye: China's version of Shodan (https://www.zoomeye.org)\n\nActive modules\n--------------\n* DNS brute force: dictionary brute force enumeration\n* Screenshots: Take screenshots of subdomains that were found\n\nModules that require an API key\n-------------------------------\nDocumentation to setup API keys can be found at - https://github.com/laramies/theHarvester/wiki/Installation#api-keys\n\n* bevigil - 50 free queries/month. 1k queries/month $50\n* brave - free plan available. Pro plans for higher limits\n* bufferoverun - 100 free queries/month. 10k/month $25\n* builtwith - 50 free queries ever. $2950/yr\n* censys - 500 credits $100\n* criminalip - 100 free queries/month. 700k/month $59\n* dehashed - 500 credts $15, 5k credits $150\n* dnsdumpster - 50 free querries/day, $49\n* fofa - query credits 10,000/month. 100k results/month $25\n* fullhunt - 50 free queries. 200 queries $29/month, 500 queries $59 \n* github-code\n* haveibeenpwned - 10 email searches/min $4.50, 50 email searches/min $22\n* hunter - 50 free credits/month. 12k credits/yr $34\n* hunterhow - 10k free API results per 30 days. 50k API results per 30 days $10\n* intelx - free account is very limited. Business acount $2900\n* leakix - free 25 results pages, 3000 API requests/month. Bounty Hunter $29\n* leaklookup - 20 credits $10, 50 credits $20, 140 credits $50, 300 credits $100\n* mojeek - 5000 free credits $6.50, $1.30 CPM (Personal), $2.60 CPM (Startup), $3.90 CPM (Business)\n* netlas - 50 free requests/day. 1k requests $49, 10k requests $249\n* onyphe - 10M results/month $587\n* pentesttools - 5 assets netsec $95/month, 5 assets webnetsec $140/month\n* projecdiscovery - requires work email. Free monthly discovery and vulnerability scans on sign-up email domain, enterprise $\n* rocketreach - 100 email lookups/month $48, 250 email lookups/month $108\n* securityscorecard - requires a work email\n* securityTrails - 50 free queries/month. 20k queries/month $500\n* shodan - Freelancer $69 month, Small Business $359 month\n* tomba - 25 free searches/month. 1k searches/month $39, 5k searches/month $89\n* venacus - 1 free search/day. 10 searches/day $12, 30 searches/day $36\n* virustotal - 500 free lookups/day, 15.5k lookups/month. Busines accounts requires a work email\n* whoisxml - 2k queries $50, 5k queries $105\n* windvane - 100 free queries\n* zoomeye - 5 free results/day. 30/results/day $190/yr\n\n## Package versions\n[](https://repology.org/project/theharvester/versions)\n\nComments, bugs, and requests\n----------------------------\n* [](https://twitter.com/laramies) Christian Martorella @laramies\n cmartorella@edge-security.com\n* [](https://twitter.com/NotoriousRebel1) Matthew Brown @NotoriousRebel1\n* [](https://twitter.com/jay_townsend1) Jay \"L1ghtn1ng\" Townsend @jay_townsend1\n\nMain contributors\n-----------------\n* [](https://twitter.com/NotoriousRebel1) Matthew Brown @NotoriousRebel1\n* [](https://twitter.com/jay_townsend1) Jay \"L1ghtn1ng\" Townsend @jay_townsend1\n* [](https://twitter.com/discoverscripts) Lee Baird @discoverscripts\n\n\nThanks\n------\n* John Matherly - Shodan project\n* Ahmed Aboul Ela - subdomain names dictionaries (big and small)\n"
},
{
"path": "bin/restfulHarvest",
"content": "#!/usr/bin/env python3\nfrom theHarvester.restfulHarvest import main\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "bin/theHarvester",
"content": "#!/usr/bin/env python3\n# Note: This script runs theHarvester\nimport sys\n\nfrom theHarvester.theHarvester import main\n\nif sys.version_info.major < 3 or sys.version_info.minor < 10:\n print('[!] Make sure you have Python 3.10+ installed, quitting.\\n\\n')\n sys.exit(1)\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "docker-compose.yml",
"content": "services:\n theharvester.svc.local:\n container_name: theHarvester\n volumes:\n - ./theHarvester/data/api-keys.yaml:/root/.theHarvester/api-keys.yaml\n - ./theHarvester/data/api-keys.yaml:/etc/theHarvester/api-keys.yaml\n - ./theHarvester/data/proxies.yaml:/etc/theHarvester/proxies.yaml\n - ./theHarvester/data/proxies.yaml:/root/.theHarvester/proxies.yaml\n build: .\n ports:\n - \"5000:80\"\n\nnetworks:\n default:\n name: app_theHarvester_network\n"
},
{
"path": "pyproject.toml",
"content": "[project]\nname = \"theHarvester\"\ndescription = \"theHarvester is a very simple, yet effective tool designed to be used in the early stages of a penetration test\"\nreadme = \"README.md\"\nlicense = \"GPL-2.0-only\"\nauthors = [\n { name = \"Christian Martorella\", email = \"cmartorella@edge-security.com\" },\n { name = \"Jay Townsend\", email = \"jay@cybermon.uk\" },\n { name = \"Matthew Brown\", email = \"36310667+NotoriousRebel@users.noreply.github.com\" },\n]\nrequires-python = \">=3.12\"\nurls.Homepage = \"https://github.com/laramies/theHarvester\"\nclassifiers = [\n \"Programming Language :: Python :: 3\",\n \"Programming Language :: Python :: 3.12\",\n \"Programming Language :: Python :: 3.13\",\n \"Programming Language :: Python :: 3.14\",\n \"Operating System :: OS Independent\",\n]\ndynamic = [\"version\"]\ndependencies = [\n \"aiodns==4.0.0\",\n \"aiofiles==25.1.0\",\n \"aiohttp==3.13.3\",\n \"aiohttp-socks==0.11.0\",\n \"aiomultiprocess==0.9.1\",\n \"aiosqlite==0.22.1\",\n \"beautifulsoup4==4.14.3\",\n \"censys==2.2.19\",\n \"certifi==2026.2.25\",\n \"dnspython==2.8.0\",\n \"fastapi==0.135.1\",\n \"lxml==6.0.2\",\n \"netaddr==1.3.0\",\n \"playwright==1.58.0\",\n \"PyYAML==6.0.3\",\n \"python-dateutil==2.9.0.post0\",\n \"httpx==0.28.1\",\n \"retrying==1.4.2\",\n \"shodan==1.31.0\",\n \"slowapi==0.1.9\",\n \"ujson==5.12.0\",\n \"uvicorn==0.41.0\",\n \"uvloop==0.22.1; platform_system != 'Windows'\",\n \"winloop==0.4.0; platform_system == 'Windows'\",\n]\n\n[dependency-groups]\ndev = [\n \"mypy==1.19.1\",\n \"mypy-extensions==1.1.0\",\n \"pytest==9.0.2\",\n \"pytest-asyncio==1.3.0\",\n \"types-certifi==2021.10.8.3\",\n \"types-chardet==5.0.4.6\",\n \"types-python-dateutil==2.9.0.20260305\",\n \"types-PyYAML==6.0.12.20250915\",\n \"ruff==0.15.5\",\n \"types-ujson==5.10.0.20250822\",\n \"wheel==0.46.3\",\n \"ty==0.0.21\",\n]\n\n[project.scripts]\ntheHarvester = \"theHarvester.theHarvester:main\"\nrestfulHarvest = \"theHarvester.restfulHarvest:main\"\n\n[tool.pytest.ini_options]\nminversion = \"8.3.3\"\nasyncio_mode = \"auto\"\nasyncio_default_fixture_loop_scope = \"function\"\naddopts = \"--no-header\"\ntestpaths = [\"tests\"]\n\n[build-system]\nrequires = [\"flit_core >=3.11,<4\"]\nbuild-backend = \"flit_core.buildapi\"\n\n[tool.mypy]\npython_version = \"3.13\"\nwarn_unused_configs = true\nignore_missing_imports = true\nshow_traceback = true\nshow_error_codes = true\nnamespace_packages = true\ncheck_untyped_defs = true\n\n[tool.uv]\npython-preference = \"managed\"\n\n[tool.uv.pip]\npython-version = \"3.13\"\n\n[tool.ty.src]\nrespect-ignore-files = false\nexclude = [\n \".venv/**\",\n \"tests/**\",\n \".github/*\"\n]\n\n[tool.ruff]\n# Exclude a variety of commonly ignored directories.\nexclude = [\n \"tests\",\n \".eggs\",\n \".git\",\n \".git-rewrite\",\n \".mypy_cache\",\n \".pyenv\",\n \".pytest_cache\",\n \".pytype\",\n \".ruff_cache\",\n \".github\",\n \".venv\",\n \".vscode\",\n \".idea\",\n \"__pypackages__\",\n \"build\",\n \"dist\",\n \"site-packages\",\n \"venv\",\n]\n\nline-length = 130\ntarget-version = \"py313\"\nshow-fixes = true\n\n[tool.ruff.lint]\nselect = [\"E\",\n \"F\",\n \"N\",\n \"I\",\n \"UP\",\n \"TCH\",\n \"FA\",\n \"RUF\",\n \"PT\",\n \"TC\",\n \"ASYNC\"\n ]\nignore = [\n \"E501\",\n \"ASYNC230\",\n \"N999\",\n \"PLR0915\"\n ]\n\n# Allow fix for all enabled rules (when `--fix`) is provided.\nfixable = [\"ALL\"]\nunfixable = []\n\n# Allow unused variables when underscore-prefixed.\ndummy-variable-rgx = \"^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$\"\n\n[tool.ruff.format]\n# Like Black, use double quotes for strings.\nquote-style = \"single\"\nindent-style = \"space\"\n\n# Like Black, respect magic trailing commas.\nskip-magic-trailing-comma = false\n\n# Like Black, automatically detect the appropriate line ending.\nline-ending = \"auto\"\n"
},
{
"path": "tests/__init__.py",
"content": ""
},
{
"path": "tests/discovery/__init__.py",
"content": ""
},
{
"path": "tests/discovery/test_baidusearch.py",
"content": "import pytest\n\nfrom theHarvester.discovery import baidusearch\n\n\nclass TestBaiduSearch:\n @pytest.mark.asyncio\n async def test_process_and_parsing(self, monkeypatch):\n called = {}\n\n async def fake_fetch_all(urls, headers=None, proxy=False):\n called[\"urls\"] = urls\n called[\"headers\"] = headers\n called[\"proxy\"] = proxy\n return [\n \"Contact foo@example.com on a.example.com \\n\",\n \" bar@sub.example.com is here and www.example.com appears \\n\",\n \" Visit sub.a.example.com. baz@example.com \\n\",\n ]\n\n # Patch the AsyncFetcher.fetch_all to avoid network I/O\n import theHarvester.lib.core as core_module\n\n monkeypatch.setattr(core_module.AsyncFetcher, \"fetch_all\", fake_fetch_all)\n # Make user agent deterministic (not strictly necessary, but stable)\n monkeypatch.setattr(core_module.Core, \"get_user_agent\", staticmethod(lambda: \"UA\"), raising=True)\n\n search = baidusearch.SearchBaidu(word=\"example.com\", limit=21)\n await search.process(proxy=True)\n\n expected_urls = [\n \"https://www.baidu.com/s?wd=%40example.com&pn=0&oq=example.com\",\n \"https://www.baidu.com/s?wd=%40example.com&pn=10&oq=example.com\",\n \"https://www.baidu.com/s?wd=%40example.com&pn=20&oq=example.com\",\n ]\n assert called[\"urls\"] == expected_urls\n assert called[\"proxy\"] is True\n\n emails = await search.get_emails()\n hosts = await search.get_hostnames()\n\n # Ensure our expected values are present\n assert \"foo@example.com\" in emails\n assert \"bar@sub.example.com\" in emails\n assert \"baz@example.com\" in emails\n\n assert {\"a.example.com\", \"www.example.com\", \"sub.a.example.com\"} <= set(hosts)\n\n @pytest.mark.asyncio\n async def test_pagination_limit_exclusive(self, monkeypatch):\n captured = {}\n\n async def fake_fetch_all(urls, headers=None, proxy=False):\n captured[\"urls\"] = urls\n return [\"\"] * len(urls)\n\n import theHarvester.lib.core as core_module\n\n monkeypatch.setattr(core_module.AsyncFetcher, \"fetch_all\", fake_fetch_all)\n monkeypatch.setattr(core_module.Core, \"get_user_agent\", staticmethod(lambda: \"UA\"), raising=True)\n\n search = baidusearch.SearchBaidu(word=\"example.com\", limit=20)\n await search.process()\n\n # For limit=20, range(0, 20, 10) yields 0 and 10 only (20 is excluded)\n assert captured[\"urls\"] == [\n \"https://www.baidu.com/s?wd=%40example.com&pn=0&oq=example.com\",\n \"https://www.baidu.com/s?wd=%40example.com&pn=10&oq=example.com\",\n ]\n"
},
{
"path": "tests/discovery/test_censys.py",
"content": "import sys\nimport types\n\nimport pytest\n\nif 'aiohttp_socks' not in sys.modules:\n aiohttp_socks_stub = types.ModuleType('aiohttp_socks')\n\n class _ProxyConnector:\n @staticmethod\n def from_url(*_args, **_kwargs):\n return None\n\n setattr(aiohttp_socks_stub, 'ProxyConnector', _ProxyConnector)\n sys.modules['aiohttp_socks'] = aiohttp_socks_stub\n\nfrom theHarvester.discovery import censysearch\nfrom theHarvester.discovery.constants import MissingKey\n\n\nclass _FakeQuery:\n def __init__(self, pages):\n self.pages = pages\n\n def __iter__(self):\n return iter(self.pages)\n\n\n@pytest.mark.asyncio\nasync def test_missing_key_raises(monkeypatch) -> None:\n monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: (None, None))\n\n with pytest.raises(MissingKey):\n censysearch.SearchCensys('example.com')\n\n\n@pytest.mark.asyncio\nasync def test_search_uses_documented_pagination_and_fields(monkeypatch) -> None:\n monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: ('id', 'secret'))\n\n calls = {}\n\n class _FakeCensysCerts:\n def __init__(self, api_id, api_secret, user_agent):\n calls['init'] = {'api_id': api_id, 'api_secret': api_secret, 'user_agent': user_agent}\n\n def search(self, **kwargs):\n calls['search'] = kwargs\n return _FakeQuery(\n [\n [\n {'names': ['a.example.com'], 'parsed': {'subject': {'email_address': 'admin@example.com'}}},\n {'names': ['b.example.com'], 'parsed': {'subject': {'email_address': ['ops@example.com']}}},\n ],\n [\n {'names': ['c.example.com'], 'parsed': {'subject': {'email_address': None}}},\n ],\n ]\n )\n\n monkeypatch.setattr(censysearch, 'CensysCerts', _FakeCensysCerts)\n\n search = censysearch.SearchCensys('example.com', limit=250)\n await search.process()\n\n assert calls['init']['api_id'] == 'id'\n assert calls['init']['api_secret'] == 'secret'\n assert calls['search']['query'] == 'names: example.com'\n assert calls['search']['per_page'] == 100\n assert calls['search']['pages'] == 3\n assert calls['search']['fields'] == ['names', 'parsed.subject.email_address']\n assert await search.get_hostnames() == {'a.example.com', 'b.example.com', 'c.example.com'}\n assert await search.get_emails() == {'admin@example.com', 'ops@example.com'}\n\n\n@pytest.mark.asyncio\nasync def test_search_respects_limit_across_page_data(monkeypatch) -> None:\n monkeypatch.setattr(censysearch.Core, 'censys_key', lambda: ('id', 'secret'))\n\n class _FakeCensysCerts:\n def __init__(self, api_id, api_secret, user_agent):\n del api_id, api_secret, user_agent\n\n def search(self, **kwargs):\n del kwargs\n return _FakeQuery(\n [\n [\n {'names': ['1.example.com']},\n {'names': ['2.example.com']},\n {'names': ['3.example.com']},\n {'names': ['4.example.com']},\n {'names': ['5.example.com']},\n ]\n ]\n )\n\n monkeypatch.setattr(censysearch, 'CensysCerts', _FakeCensysCerts)\n\n search = censysearch.SearchCensys('example.com', limit=3)\n await search.process()\n\n assert await search.get_hostnames() == {'1.example.com', '2.example.com', '3.example.com'}\n"
},
{
"path": "tests/discovery/test_certspotter.py",
"content": "#!/usr/bin/env python3\n# coding=utf-8\nimport os\nfrom typing import Optional\n\nimport pytest\nimport httpx\n\nfrom theHarvester.discovery import certspottersearch\nfrom theHarvester.lib.core import *\n\ngithub_ci: Optional[str] = os.getenv(\n \"GITHUB_ACTIONS\"\n) # Github set this to be the following: true instead of True\n\n\nclass TestCertspotter(object):\n @staticmethod\n def domain() -> str:\n return \"metasploit.com\"\n\n\n@pytest.mark.skipif(github_ci == 'true', reason=\"Skipping this test for now\")\nclass TestCertspotterSearch(object):\n @pytest.mark.asyncio\n async def test_api(self) -> None:\n base_url = f\"https://api.certspotter.com/v1/issuances?domain={TestCertspotter.domain()}&expand=dns_names\"\n headers = {\"User-Agent\": Core.get_user_agent()}\n request = httpx.get(base_url, headers=headers)\n assert request.status_code == 200\n\n @pytest.mark.asyncio\n async def test_search(self) -> None:\n search = certspottersearch.SearchCertspoter(TestCertspotter.domain())\n await search.process()\n assert isinstance(await search.get_hostnames(), set)\n\n\nif __name__ == \"__main__\":\n pytest.main()\n"
},
{
"path": "tests/discovery/test_criminalip.py",
"content": "#!/usr/bin/env python3\n# coding=utf-8\nimport pytest\n\nfrom theHarvester.discovery import criminalip\n\n\n@pytest.mark.asyncio\nasync def test_parser_handles_missing_legacy_fields(monkeypatch) -> None:\n monkeypatch.setattr(criminalip.Core, 'criminalip_key', lambda: 'test-key')\n\n search = criminalip.SearchCriminalIP('example.com')\n payload = {\n 'data': {\n 'certificates': [{'subject': 'www.example.com'}],\n 'connected_domain_subdomain': [{'main_domain': {'domain': 'example.com'}, 'subdomains': [{'domain': 'api.example.com'}]}],\n 'connected_ip': [{'ip': '93.184.216.34'}],\n 'connected_ip_info': [\n {\n 'asn': '15133',\n 'ip': '93.184.216.34',\n 'domain_list': [{'domain': 'mail.example.com'}],\n }\n ],\n 'cookies': [{'domain': '.portal.example.com'}],\n 'dns_record': {\n 'dns_record_type_a': {'ipv4': [{'ip': '93.184.216.34'}], 'ipv6': []},\n 'dns_record_type_ns': ['ns1.example.com.'],\n },\n 'html_page_link_domains': [{'domain': 'www.iana.org', 'mapped_ips': [{'ip': '192.0.33.8'}]}],\n 'links': [{'url': 'https://docs.example.com/guide'}],\n 'mapped_ip': [{'ip': '203.0.113.10'}],\n 'network_logs': {\n 'data': [{'url': 'https://cdn.example.com/script.js', 'as_number': '64500', 'ip_port': '198.51.100.10:443'}]\n },\n 'page_redirections': [[{'url': 'https://login.example.com'}]],\n 'subdomains': [{'subdomain_name': 'blog.example.com'}],\n }\n }\n\n await search.parser(payload)\n\n hostnames = await search.get_hostnames()\n ips = await search.get_ips()\n asns = await search.get_asns()\n\n assert {'api.example.com', 'blog.example.com', 'cdn.example.com', 'docs.example.com', 'login.example.com'}.issubset(hostnames)\n assert {'93.184.216.34', '198.51.100.10', '203.0.113.10'}.issubset(ips)\n assert {'15133', '64500'}.issubset(asns)\n\n\n@pytest.mark.asyncio\nasync def test_do_search_uses_v2_report_endpoint(monkeypatch) -> None:\n monkeypatch.setattr(criminalip.Core, 'criminalip_key', lambda: 'test-key')\n monkeypatch.setattr(criminalip.Core, 'get_user_agent', lambda: 'test-agent')\n\n called_urls = []\n\n async def fake_post_fetch(url, **kwargs):\n assert url == 'https://api.criminalip.io/v1/domain/scan'\n return {'status': 200, 'data': {'scan_id': 12345}}\n\n async def fake_fetch_all(urls, **kwargs):\n called_urls.append(urls[0])\n if '/v1/domain/status/' in urls[0]:\n return [{'status': 200, 'data': {'scan_percentage': 100}}]\n if '/v2/domain/report/' in urls[0]:\n return [\n {\n 'status': 200,\n 'data': {\n 'certificates': [],\n 'connected_domain_subdomain': [],\n 'connected_ip': [],\n 'connected_ip_info': [],\n 'cookies': [],\n 'dns_record': {},\n 'html_page_link_domains': [],\n 'links': [],\n 'mapped_ip': [],\n 'network_logs': {'data': []},\n 'page_redirections': [],\n 'subdomains': [],\n },\n }\n ]\n return [{'status': 500}]\n\n monkeypatch.setattr(criminalip.AsyncFetcher, 'post_fetch', fake_post_fetch)\n monkeypatch.setattr(criminalip.AsyncFetcher, 'fetch_all', fake_fetch_all)\n\n search = criminalip.SearchCriminalIP('example.com')\n await search.process()\n\n assert any('/v2/domain/report/12345' in url for url in called_urls)\n assert all('/v1/domain/report/' not in url for url in called_urls)\n"
},
{
"path": "tests/discovery/test_githubcode.py",
"content": "from unittest.mock import MagicMock\nimport pytest\nfrom httpx import Response\nfrom theHarvester.discovery import githubcode\nfrom theHarvester.discovery.constants import MissingKey\nfrom theHarvester.lib.core import Core\n\n\nclass TestSearchGithubCode:\n class OkResponse:\n response = Response(status_code=200)\n\n # Mocking the json method properly\n def __init__(self):\n self.response = Response(status_code=200)\n object.__setattr__(\n self.response,\n \"json\",\n MagicMock(\n return_value={\n \"items\": [\n {\"text_matches\": [{\"fragment\": \"test1\"}]},\n {\"text_matches\": [{\"fragment\": \"test2\"}]},\n ]\n }\n ),\n )\n\n class FailureResponse:\n def __init__(self):\n self.response = Response(status_code=401)\n object.__setattr__(self.response, \"json\", MagicMock(return_value={}))\n\n class RetryResponse:\n def __init__(self):\n self.response = Response(status_code=403)\n object.__setattr__(self.response, \"json\", MagicMock(return_value={}))\n\n class MalformedResponse:\n def __init__(self):\n self.response = Response(status_code=200)\n object.__setattr__(\n self.response,\n \"json\",\n MagicMock(\n return_value={\n \"items\": [\n {\"fail\": True},\n {\"text_matches\": []},\n {\"text_matches\": [{\"weird\": \"result\"}]},\n ]\n }\n ),\n )\n\n @pytest.mark.asyncio\n async def test_missing_key(self):\n with pytest.raises(MissingKey):\n Core.github_key = MagicMock(return_value=None) # type: ignore[method-assign]\n githubcode.SearchGithubCode(word=\"test\", limit=500)\n\n @pytest.mark.asyncio\n async def test_fragments_from_response(self):\n Core.github_key = MagicMock(return_value=\"test_key\") # type: ignore[method-assign]\n test_class_instance = githubcode.SearchGithubCode(word=\"test\", limit=500)\n test_result = await test_class_instance.fragments_from_response(\n self.OkResponse().response.json()\n )\n print(\"test_result: \", test_result)\n assert test_result == [\"test1\", \"test2\"]\n\n @pytest.mark.asyncio\n async def test_invalid_fragments_from_response(self):\n Core.github_key = MagicMock(return_value=\"test_key\") # type: ignore[method-assign]\n test_class_instance = githubcode.SearchGithubCode(word=\"test\", limit=500)\n test_result = await test_class_instance.fragments_from_response(\n self.MalformedResponse().response.json()\n )\n assert test_result == []\n\n @pytest.mark.asyncio\n async def test_next_page(self):\n Core.github_key = MagicMock(return_value=\"test_key\") # type: ignore[method-assign]\n test_class_instance = githubcode.SearchGithubCode(word=\"test\", limit=500)\n test_result = githubcode.SuccessResult(list(), next_page=2, last_page=4)\n assert 2 == await test_class_instance.next_page_or_end(test_result)\n\n @pytest.mark.asyncio\n async def test_last_page(self):\n Core.github_key = MagicMock(return_value=\"test_key\") # type: ignore[method-assign]\n test_class_instance = githubcode.SearchGithubCode(word=\"test\", limit=500)\n test_result = githubcode.SuccessResult(list(), 0, 0)\n assert await test_class_instance.next_page_or_end(test_result) == 0\n\n @pytest.mark.asyncio\n async def test_infinite_loop_fix_page_zero(self):\n \"\"\"Test that the loop condition properly exits when page becomes 0\"\"\"\n Core.github_key = MagicMock(return_value=\"test_key\") # type: ignore[method-assign]\n test_class_instance = githubcode.SearchGithubCode(word=\"test\", limit=500)\n\n # Test the fixed condition: page != 0\n page = 0\n counter = 0\n limit = 10\n\n # The condition should be False when page is 0, preventing infinite loop\n condition_result = counter <= limit and page != 0\n assert condition_result is False, \"Loop should exit when page is 0\"\n\n @pytest.mark.asyncio\n async def test_infinite_loop_fix_page_nonzero(self):\n \"\"\"Test that the loop condition continues when page is non-zero\"\"\"\n Core.github_key = MagicMock(return_value=\"test_key\") # type: ignore[method-assign]\n test_class_instance = githubcode.SearchGithubCode(word=\"test\", limit=500)\n\n # Test with non-zero page values\n for page in [1, 2, 3, 10]:\n counter = 0\n limit = 10\n\n # The condition should be True when page is non-zero\n condition_result = counter <= limit and page != 0\n assert condition_result is True, f\"Loop should continue when page is {page}\"\n\n @pytest.mark.asyncio\n async def test_infinite_loop_fix_old_vs_new_condition(self):\n \"\"\"Test that demonstrates the difference between old and new conditions\"\"\"\n Core.github_key = MagicMock(return_value=\"test_key\") # type: ignore[method-assign]\n test_class_instance = githubcode.SearchGithubCode(word=\"test\", limit=500)\n\n page = 0\n counter = 0\n limit = 10\n\n # Old problematic condition (would cause infinite loop)\n old_condition = counter <= limit and page is not None\n\n # New fixed condition (properly exits)\n new_condition = counter <= limit and page != 0\n\n # Old condition would be True (causing infinite loop)\n assert old_condition is True, \"Old condition would cause infinite loop when page=0\"\n\n # New condition is False (properly exits)\n assert new_condition is False, \"New condition properly exits when page=0\"\n\n\nif __name__ == \"__main__\":\n pytest.main()\n"
},
{
"path": "tests/discovery/test_githubcode_additions.py",
"content": "from unittest.mock import MagicMock, AsyncMock\nimport asyncio\nimport pytest\nfrom theHarvester.discovery import githubcode\nfrom theHarvester.lib.core import Core\n\n\nclass TestSearchGithubCodeProcess:\n @pytest.mark.asyncio\n async def test_process_stops_after_max_retries(self, monkeypatch):\n Core.github_key = MagicMock(return_value=\"test_key\") # type: ignore[method-assign]\n inst = githubcode.SearchGithubCode(word=\"test\", limit=10)\n\n # Speed up by avoiding actual sleeps\n monkeypatch.setattr(githubcode, \"get_delay\", lambda: 0, raising=False)\n monkeypatch.setattr(asyncio, \"sleep\", AsyncMock(return_value=None))\n\n # Force RetryResult every time\n monkeypatch.setattr(\n inst,\n \"handle_response\",\n AsyncMock(return_value=githubcode.RetryResult(0)),\n )\n monkeypatch.setattr(\n inst,\n \"do_search\",\n AsyncMock(return_value=(\"\", {}, 403, {})),\n )\n\n inst.max_retries = 2\n await inst.process()\n assert inst.page == 0, \"Process should stop after exceeding max retries\"\n assert inst.retry_count == 3, \"Retry count should exceed max_retries before stopping\"\n\n @pytest.mark.asyncio\n async def test_process_stops_on_error_result(self, monkeypatch):\n Core.github_key = MagicMock(return_value=\"test_key\") # type: ignore[method-assign]\n inst = githubcode.SearchGithubCode(word=\"test\", limit=10)\n\n monkeypatch.setattr(githubcode, \"get_delay\", lambda: 0, raising=False)\n monkeypatch.setattr(asyncio, \"sleep\", AsyncMock(return_value=None))\n\n # Force ErrorResult\n monkeypatch.setattr(\n inst,\n \"handle_response\",\n AsyncMock(return_value=githubcode.ErrorResult(500, \"err\")),\n )\n monkeypatch.setattr(\n inst,\n \"do_search\",\n AsyncMock(return_value=(\"\", {}, 500, {})),\n )\n\n await inst.process()\n assert inst.page == 0, \"Process should stop on error result to avoid infinite loop\"\n\n @pytest.mark.asyncio\n async def test_process_breaks_on_same_page_pagination(self, monkeypatch):\n Core.github_key = MagicMock(return_value=\"test_key\") # type: ignore[method-assign]\n inst = githubcode.SearchGithubCode(word=\"test\", limit=10)\n\n monkeypatch.setattr(githubcode, \"get_delay\", lambda: 0, raising=False)\n monkeypatch.setattr(asyncio, \"sleep\", AsyncMock(return_value=None))\n\n # Force SuccessResult that does not advance the page\n monkeypatch.setattr(\n inst,\n \"handle_response\",\n AsyncMock(return_value=githubcode.SuccessResult([], next_page=1, last_page=0)),\n )\n monkeypatch.setattr(\n inst,\n \"do_search\",\n AsyncMock(return_value=(\"\", {\"items\": []}, 200, {})),\n )\n\n await inst.process()\n assert inst.page == 0, \"Process should stop when pagination does not advance\"\n"
},
{
"path": "tests/discovery/test_otx.py",
"content": "#!/usr/bin/env python3\n# coding=utf-8\nimport os\nfrom typing import Optional\nimport httpx\nimport pytest\n\nfrom theHarvester.discovery import otxsearch\nfrom theHarvester.lib.core import *\n\ngithub_ci: Optional[str] = os.getenv(\n \"GITHUB_ACTIONS\"\n) # Github set this to be the following: true instead of True\n\n\nclass TestOtx(object):\n @staticmethod\n def domain() -> str:\n return \"apple.com\"\n\n @pytest.mark.asyncio\n async def test_search(self) -> None:\n search = otxsearch.SearchOtx(TestOtx.domain())\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip(\"Skipping OTX search due to network error\")\n assert isinstance(await search.get_hostnames(), set)\n assert isinstance(await search.get_ips(), set)\n\n\nif __name__ == \"__main__\":\n pytest.main()\n"
},
{
"path": "tests/discovery/test_rocketreach.py",
"content": "import sys\nimport types\n\nimport pytest\n\nif 'aiohttp_socks' not in sys.modules:\n aiohttp_socks_stub = types.ModuleType('aiohttp_socks')\n\n class _ProxyConnector:\n @staticmethod\n def from_url(*_args, **_kwargs):\n return None\n\n setattr(aiohttp_socks_stub, 'ProxyConnector', _ProxyConnector)\n sys.modules['aiohttp_socks'] = aiohttp_socks_stub\n\nfrom theHarvester.discovery import rocketreach\nfrom theHarvester.discovery.constants import MissingKey\n\n\n@pytest.mark.asyncio\nasync def test_missing_key_raises(monkeypatch) -> None:\n monkeypatch.setattr(rocketreach.Core, 'rocketreach_key', lambda: None)\n with pytest.raises(MissingKey):\n rocketreach.SearchRocketReach('example.com', 10)\n\n\n@pytest.mark.asyncio\nasync def test_do_search_uses_people_data_endpoint_and_start_pagination(monkeypatch) -> None:\n monkeypatch.setattr(rocketreach.Core, 'rocketreach_key', lambda: 'test-key')\n monkeypatch.setattr(rocketreach.Core, 'get_user_agent', lambda: 'test-agent')\n monkeypatch.setattr(rocketreach, 'get_delay', lambda: 0)\n\n async def fake_sleep(_seconds):\n return None\n\n monkeypatch.setattr(rocketreach.asyncio, 'sleep', fake_sleep)\n\n calls = []\n\n async def fake_post_fetch(url, headers=None, data=None, json=False, **kwargs):\n calls.append((url, headers, data, json, kwargs))\n if len(calls) == 1:\n first_page_profiles = []\n for index in range(100):\n first_page_profiles.append(\n {\n 'linkedin_url': f'https://linkedin.com/in/user{index}',\n 'emails': [{'email': f'user{index}@example.com'}],\n }\n )\n return {\n 'profiles': first_page_profiles,\n 'pagination': {'page': 1, 'total': 150},\n }\n\n second_page_profiles = []\n for index in range(100, 150):\n second_page_profiles.append(\n {\n 'linkedin_url': f'https://linkedin.com/in/user{index}',\n 'emails': [{'email': f'user{index}@example.com'}],\n }\n )\n return {\n 'profiles': second_page_profiles,\n 'pagination': {'page': 2, 'total': 150},\n }\n\n monkeypatch.setattr(rocketreach.AsyncFetcher, 'post_fetch', fake_post_fetch)\n\n search = rocketreach.SearchRocketReach('example.com', 150)\n await search.process()\n\n assert len(calls) == 2\n first_url, first_headers, first_data, first_json, _ = calls[0]\n second_url, _, second_data, _, _ = calls[1]\n\n assert first_url == 'https://api.rocketreach.co/api/v2/person/search'\n assert second_url == 'https://api.rocketreach.co/api/v2/person/search'\n assert first_headers['Api-Key'] == 'test-key'\n assert first_headers['User-Agent'] == 'test-agent'\n assert first_json is True\n assert first_data == {'query': {'current_employer_domain': ['example.com']}, 'start': 0, 'page_size': 100}\n assert second_data == {'query': {'current_employer_domain': ['example.com']}, 'start': 100, 'page_size': 50}\n\n links = await search.get_links()\n emails = await search.get_emails()\n assert len(links) == 150\n assert len(emails) == 150\n assert 'https://linkedin.com/in/user0' in links\n assert 'https://linkedin.com/in/user149' in links\n assert 'user0@example.com' in emails\n assert 'user149@example.com' in emails\n\n\n@pytest.mark.asyncio\nasync def test_do_search_stops_on_throttling_message(monkeypatch) -> None:\n monkeypatch.setattr(rocketreach.Core, 'rocketreach_key', lambda: 'test-key')\n monkeypatch.setattr(rocketreach.Core, 'get_user_agent', lambda: 'test-agent')\n monkeypatch.setattr(rocketreach, 'get_delay', lambda: 0)\n\n async def fake_sleep(_seconds):\n return None\n\n monkeypatch.setattr(rocketreach.asyncio, 'sleep', fake_sleep)\n\n calls = []\n\n async def fake_post_fetch(url, headers=None, data=None, json=False, **kwargs):\n calls.append((url, data))\n return {'detail': 'Request was throttled. Credits will become available in 10 seconds.'}\n\n monkeypatch.setattr(rocketreach.AsyncFetcher, 'post_fetch', fake_post_fetch)\n\n search = rocketreach.SearchRocketReach('example.com', 10)\n await search.process()\n\n assert len(calls) == 1\n"
},
{
"path": "tests/discovery/test_shodan_engine.py",
"content": "import socket\nimport sys\nfrom collections import OrderedDict\n\nimport pytest\n\n\nclass TestShodanEngine:\n @pytest.mark.asyncio\n async def test_shodan_engine_processes_without_work_item_error_and_yields_hostnames(self, monkeypatch, capsys):\n # Import inside the test so monkeypatching affects the already-imported module namespace.\n import theHarvester.__main__ as main_module\n\n # Make DNS resolution deterministic and offline.\n monkeypatch.setattr(socket, \"gethostbyname\", lambda _domain: \"1.2.3.4\", raising=True)\n\n # Avoid filesystem/sqlite side effects.\n class DummyStashManager:\n async def do_init(self) -> None:\n return None\n\n async def store_all(self, domain, all, res_type, source) -> None: # noqa: A002\n return None\n\n monkeypatch.setattr(main_module.stash, \"StashManager\", DummyStashManager, raising=True)\n\n # Stub Shodan search to avoid network and API key requirements.\n class DummySearchShodan:\n async def search_ip(self, ip):\n return OrderedDict({ip: {\"hostnames\": [\"a.example.com\", \"b.example.com\"]}})\n\n monkeypatch.setattr(main_module.shodansearch, \"SearchShodan\", DummySearchShodan, raising=True)\n\n # Run the CLI path that uses the engine queue/worker (`-b shodan`).\n monkeypatch.setattr(sys, \"argv\", [\"theHarvester\", \"-d\", \"example.com\", \"-b\", \"shodan\"], raising=True)\n\n with pytest.raises(SystemExit) as excinfo:\n await main_module.start()\n assert excinfo.value.code == 0\n\n out = capsys.readouterr().out\n assert 'A error occurred while processing a \"work item\"' not in out\n assert \"a.example.com\" in out\n assert \"b.example.com\" in out\n"
},
{
"path": "tests/discovery/test_thc.py",
"content": "#!/usr/bin/env python3\n# coding=utf-8\n\"\"\"\nTests for THC (ip.thc.org) discovery module.\n\nTHC provides multiple endpoints:\n- Subdomain enumeration\n- CNAME lookup\n- Reverse DNS lookup\n\nAPI Documentation: https://ip.thc.org/docs/\n\"\"\"\nimport os\nfrom typing import Optional\n\nimport httpx\nimport pytest\n\nfrom theHarvester.discovery import thc\nfrom theHarvester.lib.core import Core\n\ngithub_ci: Optional[str] = os.getenv('GITHUB_ACTIONS')\n\n\n# =============================================================================\n# 1. Direct API Tests (Endpoint Validation)\n# =============================================================================\nclass TestThcApi:\n \"\"\"Tests to validate that the THC API responds correctly.\"\"\"\n\n @pytest.mark.asyncio\n async def test_api_subdomains_download_endpoint_responds(self) -> None:\n \"\"\"Verify that the subdomain download endpoint responds.\"\"\"\n url = 'https://ip.thc.org/api/v1/subdomains/download?domain=google.com&limit=10&hide_header=true'\n headers = {'User-Agent': Core.get_user_agent()}\n try:\n response = httpx.get(url, headers=headers, timeout=30)\n assert response.status_code == 200\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n\n @pytest.mark.asyncio\n async def test_api_subdomains_returns_text_format(self) -> None:\n \"\"\"Verify that the response is plain text.\"\"\"\n url = 'https://ip.thc.org/api/v1/subdomains/download?domain=google.com&limit=5&hide_header=true'\n headers = {'User-Agent': Core.get_user_agent()}\n try:\n response = httpx.get(url, headers=headers, timeout=30)\n content_type = response.headers.get('content-type', '')\n assert 'text' in content_type or 'octet-stream' in content_type or response.status_code == 200\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n\n @pytest.mark.asyncio\n async def test_api_cli_subdomain_endpoint(self) -> None:\n \"\"\"Verify CLI endpoint /sb/{domain}.\"\"\"\n url = 'https://ip.thc.org/sb/google.com?l=5&noheader'\n headers = {'User-Agent': Core.get_user_agent()}\n try:\n response = httpx.get(url, headers=headers, timeout=30)\n assert response.status_code == 200\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n\n @pytest.mark.asyncio\n async def test_api_returns_rate_limit_headers(self) -> None:\n \"\"\"Verify that the API returns rate limit headers.\"\"\"\n url = 'https://ip.thc.org/api/v1/subdomains/download?domain=example.com&limit=1&hide_header=true'\n headers = {'User-Agent': Core.get_user_agent()}\n try:\n response = httpx.get(url, headers=headers, timeout=30)\n assert 'x-ratelimit-limit' in response.headers\n assert 'x-ratelimit-remaining' in response.headers\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n\n\n# =============================================================================\n# 2. Subdomain Search Tests (Main Functionality)\n# =============================================================================\nclass TestThcSubdomainSearch:\n \"\"\"Tests for subdomain search functionality.\"\"\"\n\n @staticmethod\n def domain() -> str:\n return 'tesla.com'\n\n @staticmethod\n def small_domain() -> str:\n return 'thc.org'\n\n @pytest.mark.asyncio\n async def test_search_returns_set(self) -> None:\n \"\"\"Verify that get_hostnames() returns a set.\"\"\"\n search = thc.SearchThc(self.domain())\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n result = await search.get_hostnames()\n assert isinstance(result, set)\n\n @pytest.mark.asyncio\n async def test_search_finds_subdomains(self) -> None:\n \"\"\"Verify that it finds subdomains for a known domain.\"\"\"\n search = thc.SearchThc(self.domain())\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n result = await search.get_hostnames()\n assert len(result) > 0, 'Should find at least one subdomain for tesla.com'\n\n @pytest.mark.asyncio\n async def test_search_results_contain_target_domain(self) -> None:\n \"\"\"Verify that all results contain the target domain.\"\"\"\n search = thc.SearchThc(self.small_domain())\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n result = await search.get_hostnames()\n for hostname in result:\n assert self.small_domain() in hostname, f'{hostname} should contain {self.small_domain()}'\n\n @pytest.mark.asyncio\n async def test_search_no_duplicates(self) -> None:\n \"\"\"Verify that there are no duplicates in the results.\"\"\"\n search = thc.SearchThc(self.domain())\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n result = await search.get_hostnames()\n result_list = list(result)\n assert len(result_list) == len(set(result_list))\n\n\n# =============================================================================\n# 3. Edge Case Tests\n# =============================================================================\nclass TestThcEdgeCases:\n \"\"\"Tests for edge cases and error handling.\"\"\"\n\n @pytest.mark.asyncio\n async def test_search_nonexistent_domain(self) -> None:\n \"\"\"Verify behavior with non-existent domain.\"\"\"\n search = thc.SearchThc('this-domain-definitely-does-not-exist-12345.com')\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n except Exception:\n pass\n result = await search.get_hostnames()\n assert isinstance(result, set)\n\n @pytest.mark.asyncio\n async def test_search_empty_domain(self) -> None:\n \"\"\"Verify behavior with empty domain.\"\"\"\n search = thc.SearchThc('')\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n except Exception:\n pass\n result = await search.get_hostnames()\n assert isinstance(result, set)\n\n @pytest.mark.asyncio\n async def test_search_special_characters_domain(self) -> None:\n \"\"\"Verify behavior with special characters.\"\"\"\n search = thc.SearchThc('example.com; DROP TABLE domains;--')\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n except Exception:\n pass\n result = await search.get_hostnames()\n assert isinstance(result, set)\n\n @pytest.mark.asyncio\n async def test_search_unicode_domain(self) -> None:\n \"\"\"Verify behavior with IDN/unicode domain.\"\"\"\n search = thc.SearchThc('xn--mnchen-3ya.de')\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n except Exception:\n pass\n result = await search.get_hostnames()\n assert isinstance(result, set)\n\n @pytest.mark.asyncio\n async def test_search_subdomain_as_input(self) -> None:\n \"\"\"Verify behavior when a subdomain is passed as input.\"\"\"\n search = thc.SearchThc('www.google.com')\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n result = await search.get_hostnames()\n assert isinstance(result, set)\n\n\n# =============================================================================\n# 4. Proxy Tests\n# =============================================================================\nclass TestThcProxy:\n \"\"\"Tests for proxy functionality.\"\"\"\n\n @staticmethod\n def domain() -> str:\n return 'example.com'\n\n @pytest.mark.asyncio\n async def test_process_accepts_proxy_parameter(self) -> None:\n \"\"\"Verify that process() accepts proxy parameter.\"\"\"\n search = thc.SearchThc(self.domain())\n try:\n await search.process(proxy=False)\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n result = await search.get_hostnames()\n assert isinstance(result, set)\n\n @pytest.mark.asyncio\n async def test_proxy_attribute_is_set(self) -> None:\n \"\"\"Verify that the proxy attribute is set correctly.\"\"\"\n search = thc.SearchThc(self.domain())\n assert search.proxy is False\n\n\n# =============================================================================\n# 5. Initialization and Attributes Tests\n# =============================================================================\nclass TestThcInitialization:\n \"\"\"Tests for class initialization and structure.\"\"\"\n\n def test_init_sets_word(self) -> None:\n \"\"\"Verify that __init__ sets the domain.\"\"\"\n domain = 'test.com'\n search = thc.SearchThc(domain)\n assert search.word == domain\n\n def test_init_creates_empty_results(self) -> None:\n \"\"\"Verify that results is initialized empty.\"\"\"\n search = thc.SearchThc('test.com')\n assert hasattr(search, 'results')\n assert len(search.results) == 0\n\n def test_init_proxy_default_false(self) -> None:\n \"\"\"Verify that proxy is False by default.\"\"\"\n search = thc.SearchThc('test.com')\n assert search.proxy is False\n\n def test_init_has_rate_limit_settings(self) -> None:\n \"\"\"Verify that rate limit settings are initialized.\"\"\"\n search = thc.SearchThc('test.com')\n assert hasattr(search, 'max_retries')\n assert hasattr(search, 'base_delay')\n assert search.max_retries == 3\n assert search.base_delay == 2\n\n def test_class_has_required_methods(self) -> None:\n \"\"\"Verify that the class has the required methods.\"\"\"\n search = thc.SearchThc('test.com')\n assert hasattr(search, 'do_search')\n assert hasattr(search, 'get_hostnames')\n assert hasattr(search, 'process')\n assert callable(search.do_search)\n assert callable(search.get_hostnames)\n assert callable(search.process)\n\n\n# =============================================================================\n# 6. Response Format Tests\n# =============================================================================\nclass TestThcResponseFormat:\n \"\"\"Tests to verify response format.\"\"\"\n\n @staticmethod\n def domain() -> str:\n return 'github.com'\n\n @pytest.mark.asyncio\n async def test_hostnames_are_strings(self) -> None:\n \"\"\"Verify that all hostnames are strings.\"\"\"\n search = thc.SearchThc(self.domain())\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n result = await search.get_hostnames()\n for hostname in result:\n assert isinstance(hostname, str)\n\n @pytest.mark.asyncio\n async def test_hostnames_are_valid_format(self) -> None:\n \"\"\"Verify that hostnames have valid format.\"\"\"\n search = thc.SearchThc(self.domain())\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n result = await search.get_hostnames()\n for hostname in result:\n assert ' ' not in hostname\n assert '\\n' not in hostname\n assert '\\t' not in hostname\n\n @pytest.mark.asyncio\n async def test_hostnames_are_lowercase(self) -> None:\n \"\"\"Verify that hostnames are lowercase.\"\"\"\n search = thc.SearchThc(self.domain())\n try:\n await search.process()\n except (httpx.TimeoutException, httpx.RequestError):\n pytest.skip('Skipping due to network error')\n result = await search.get_hostnames()\n for hostname in result:\n assert hostname == hostname.lower()\n\n\n# =============================================================================\n# 7. Integration Tests with theHarvester\n# =============================================================================\n@pytest.mark.skipif(github_ci == 'true', reason='Skip integration tests in CI')\nclass TestThcIntegration:\n \"\"\"Integration tests with theHarvester framework.\"\"\"\n\n @pytest.mark.asyncio\n async def test_module_can_be_imported(self) -> None:\n \"\"\"Verify that the module can be imported.\"\"\"\n from theHarvester.discovery import thc as thc_module\n assert thc_module is not None\n\n @pytest.mark.asyncio\n async def test_search_class_exists(self) -> None:\n \"\"\"Verify that SearchThc class exists.\"\"\"\n from theHarvester.discovery import thc as thc_module\n assert hasattr(thc_module, 'SearchThc')\n\n @pytest.mark.asyncio\n async def test_compatible_with_store_function(self) -> None:\n \"\"\"Verify compatibility with store function from __main__.py.\"\"\"\n search = thc.SearchThc('example.com')\n assert hasattr(search, 'process')\n assert hasattr(search, 'get_hostnames')\n\n\nif __name__ == '__main__':\n pytest.main()\n"
},
{
"path": "tests/lib/test_core.py",
"content": "from __future__ import annotations\n\nfrom pathlib import Path\nfrom typing import Any\nfrom unittest import mock\n\nimport pytest\nimport yaml\n\nimport theHarvester.lib.core as core_module\nfrom theHarvester.lib.core import CONFIG_DIRS, DATA_DIR, AsyncFetcher, Core\n\n\n@pytest.fixture(autouse=True)\ndef mock_environ(monkeypatch, tmp_path: Path):\n monkeypatch.setenv(\"HOME\", str(tmp_path))\n\n\ndef mock_read_text(mocked: dict[Path, str | Exception]):\n read_text = Path.read_text\n\n def _read_text(self: Path, *args, **kwargs):\n if result := mocked.get(self):\n if isinstance(result, Exception):\n raise result\n return result\n return read_text(self, *args, **kwargs)\n\n return _read_text\n\n\n@pytest.mark.parametrize(\n (\"name\", \"contents\", \"expected\"),\n [\n (\"api-keys\", \"apikeys: {}\", {}),\n (\"proxies\", \"http: [localhost:8080]\", {\"http\": [\"http://localhost:8080\"], \"socks5\": []}),\n ],\n)\n@pytest.mark.parametrize(\"dir\", CONFIG_DIRS)\ndef test_read_config_searches_config_dirs(\n name: str, contents: str, expected: Any, dir: Path, capsys\n):\n file = dir.expanduser() / f\"{name}.yaml\"\n config_files = [d.expanduser() / file.name for d in CONFIG_DIRS]\n side_effect = mock_read_text(\n {f: contents if f == file else FileNotFoundError() for f in config_files}\n )\n\n with mock.patch(\"pathlib.Path.read_text\", autospec=True, side_effect=side_effect):\n got = Core.api_keys() if name == \"api-keys\" else Core.proxy_list()\n\n assert got == expected\n assert f\"Read {file.name} from {file}\" in capsys.readouterr().out\n\n\n@pytest.mark.parametrize(\"name\", (\"api-keys\", \"proxies\"))\ndef test_read_config_copies_default_to_home(name: str, capsys):\n file = Path(f\"~/.theHarvester/{name}.yaml\").expanduser()\n config_files = [d.expanduser() / file.name for d in CONFIG_DIRS]\n side_effect = mock_read_text({f: FileNotFoundError() for f in config_files})\n\n with mock.patch(\"pathlib.Path.read_text\", autospec=True, side_effect=side_effect):\n got = Core.api_keys() if name == \"api-keys\" else Core.proxy_list()\n\n default = yaml.safe_load((DATA_DIR / file.name).read_text())\n expected = (\n default[\"apikeys\"]\n if name == \"api-keys\"\n else {\n \"http\": [f\"http://{h}\" for h in default[\"http\"]] if default.get(\"http\") else [],\n \"socks5\": [f\"socks5://{h}\" for h in default[\"socks5\"]] if default.get(\"socks5\") else [],\n }\n )\n assert got == expected\n assert f\"Created default {file.name} at {file}\" in capsys.readouterr().out\n assert file.exists()\n\n\nclass DummyResponse:\n def __init__(self, text_value: str = 'response-text', json_value: Any = None):\n self.text_value = text_value\n self.json_value = {'ok': True} if json_value is None else json_value\n\n async def __aenter__(self):\n return self\n\n async def __aexit__(self, exc_type, exc, tb):\n return False\n\n async def text(self):\n return self.text_value\n\n async def json(self):\n return self.json_value\n\n\nclass DummySession:\n instances: list['DummySession'] = []\n\n def __init__(self, *, headers=None, timeout=None, connector=None):\n self.headers = headers\n self.timeout = timeout\n self.connector = connector\n self.closed = False\n self.requests: list[tuple[str, str, dict[str, Any]]] = []\n DummySession.instances.append(self)\n\n async def __aenter__(self):\n return self\n\n async def __aexit__(self, exc_type, exc, tb):\n await self.close()\n return False\n\n def request(self, method: str, url: str, **kwargs):\n self.requests.append((method, url, kwargs))\n return DummyResponse()\n\n def get(self, url: str, **kwargs):\n self.requests.append(('GET', url, kwargs))\n return DummyResponse()\n\n def post(self, url: str, **kwargs):\n self.requests.append(('POST', url, kwargs))\n return DummyResponse(json_value={'posted': True})\n\n async def close(self):\n self.closed = True\n\n\ndef reset_dummy_sessions() -> None:\n DummySession.instances.clear()\n\n\nasync def fake_sleep(_seconds: float) -> None:\n return None\n\n\ndef test_api_keys_yaml_is_in_sync_with_core_accessors():\n required = core_module.Core._API_KEY_FIELDS\n assert required, \"No API-key references were detected in `Core`\"\n\n config = yaml.safe_load((DATA_DIR / \"api-keys.yaml\").read_text(encoding=\"utf-8\"))\n apikeys = config[\"apikeys\"]\n\n missing_providers = sorted(set(required) - set(apikeys))\n assert not missing_providers, f\"Missing providers in api-keys.yaml: {missing_providers}\"\n\n missing_fields: dict[str, list[str]] = {}\n for provider, fields in required.items():\n for field in sorted(fields):\n if field not in apikeys[provider]:\n missing_fields.setdefault(provider, []).append(field)\n\n assert not missing_fields, f\"Missing fields in api-keys.yaml: {missing_fields}\"\n\n\n@pytest.mark.parametrize(\n (\"accessor_name\", \"expected\"),\n [\n (\"bevigil_key\", \"bevigil-key\"),\n (\"censys_key\", (\"censys-id\", \"censys-secret\")),\n (\"fofa_key\", (\"fofa-key\", \"fofa-email\")),\n (\"tomba_key\", (\"tomba-key\", \"tomba-secret\")),\n ],\n)\ndef test_api_key_accessors_delegate_to_shared_mapping(monkeypatch, accessor_name: str, expected: Any):\n monkeypatch.setattr(\n Core,\n 'api_keys',\n staticmethod(\n lambda: {\n 'bevigil': {'key': 'bevigil-key'},\n 'censys': {'id': 'censys-id', 'secret': 'censys-secret'},\n 'fofa': {'key': 'fofa-key', 'email': 'fofa-email'},\n 'tomba': {'key': 'tomba-key', 'secret': 'tomba-secret'},\n }\n ),\n )\n\n accessor = getattr(Core, accessor_name)\n assert accessor() == expected\n\n\n@pytest.mark.asyncio\nasync def test_fetch_creates_session_with_default_headers(monkeypatch) -> None:\n reset_dummy_sessions()\n monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession)\n monkeypatch.setattr(core_module.ssl, 'create_default_context', lambda cafile=None: 'ssl-context')\n monkeypatch.setattr(core_module.certifi, 'where', lambda: '/tmp/cacert.pem')\n monkeypatch.setattr(core_module.asyncio, 'sleep', fake_sleep)\n monkeypatch.setattr(Core, 'get_user_agent', staticmethod(lambda: 'test-agent'))\n\n result = await AsyncFetcher.fetch(url='https://example.com', follow_redirects=False)\n\n assert result == 'response-text'\n assert len(DummySession.instances) == 1\n session = DummySession.instances[0]\n assert session.headers == {'User-Agent': 'test-agent'}\n assert session.closed is True\n assert session.requests == [\n ('GET', 'https://example.com', {'ssl': 'ssl-context', 'allow_redirects': False})\n ]\n\n\n@pytest.mark.asyncio\nasync def test_fetch_uses_http_proxy_when_enabled(monkeypatch) -> None:\n reset_dummy_sessions()\n monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession)\n monkeypatch.setattr(core_module.ssl, 'create_default_context', lambda cafile=None: 'ssl-context')\n monkeypatch.setattr(core_module.certifi, 'where', lambda: '/tmp/cacert.pem')\n monkeypatch.setattr(core_module.asyncio, 'sleep', fake_sleep)\n monkeypatch.setattr(AsyncFetcher, '_get_random_proxy', staticmethod(lambda proxy_dict: ('http://proxy.local:8080', 'http')))\n\n async def fake_create_connector(proxy_url, proxy_type, ssl_context=None):\n return 'connector'\n\n monkeypatch.setattr(AsyncFetcher, '_create_connector', fake_create_connector)\n\n result = await AsyncFetcher.fetch(url='https://example.com', proxy=True)\n\n assert result == 'response-text'\n session = DummySession.instances[0]\n assert session.connector == 'connector'\n assert session.requests == [\n ('GET', 'https://example.com', {'ssl': 'ssl-context', 'proxy': 'http://proxy.local:8080'})\n ]\n\n\n@pytest.mark.asyncio\nasync def test_post_fetch_decodes_string_payload_and_posts_params(monkeypatch) -> None:\n reset_dummy_sessions()\n monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession)\n monkeypatch.setattr(core_module.asyncio, 'sleep', fake_sleep)\n monkeypatch.setattr(core_module.ssl, 'create_default_context', lambda cafile=None: 'ssl-context')\n monkeypatch.setattr(core_module.certifi, 'where', lambda: '/tmp/cacert.pem')\n monkeypatch.setattr(Core, 'get_user_agent', staticmethod(lambda: 'test-agent'))\n\n result = await AsyncFetcher.post_fetch(\n 'https://example.com/api',\n data='{\"query\": \"example\"}',\n params={'page': 2},\n json=True,\n )\n\n assert result == {'ok': True}\n session = DummySession.instances[0]\n assert session.headers == {'User-Agent': 'test-agent'}\n assert session.requests == [\n ('POST', 'https://example.com/api', {'data': {'query': 'example'}, 'ssl': 'ssl-context', 'params': {'page': 2}})\n ]\n\n\n@pytest.mark.asyncio\nasync def test_post_fetch_proxy_branch_uses_get_with_http_proxy(monkeypatch) -> None:\n reset_dummy_sessions()\n created_connectors = []\n monkeypatch.setattr(core_module.aiohttp, 'ClientSession', DummySession)\n monkeypatch.setattr(core_module.asyncio, 'sleep', fake_sleep)\n monkeypatch.setattr(core_module.ssl, 'create_default_context', lambda cafile=None: 'ssl-context')\n monkeypatch.setattr(core_module.certifi, 'where', lambda: '/tmp/cacert.pem')\n monkeypatch.setattr(AsyncFetcher, '_get_random_proxy', staticmethod(lambda proxy_dict: ('http://proxy.local:8080', 'http')))\n\n async def fake_create_connector(proxy_url, proxy_type, ssl_context=None):\n created_connectors.append((proxy_url, proxy_type, ssl_context))\n return 'connector'\n\n monkeypatch.setattr(AsyncFetcher, '_create_connector', fake_create_connector)\n\n result = await AsyncFetcher.post_fetch('https://example.com/resource', proxy=True)\n\n assert result == 'response-text'\n assert created_connectors == [('http://proxy.local:8080', 'http', 'ssl-context')]\n session = DummySession.instances[0]\n assert session.connector == 'connector'\n assert session.requests == [\n ('GET', 'https://example.com/resource', {'proxy': 'http://proxy.local:8080'})\n ]\n"
},
{
"path": "tests/lib/test_output.py",
"content": "from __future__ import annotations\n\n\nfrom theHarvester.lib.output import print_linkedin_sections, sorted_unique\n\n\ndef test_sorted_unique_sorts_and_deduplicates() -> None:\n assert sorted_unique([\"b\", \"a\", \"b\"]) == [\"a\", \"b\"]\n\n\ndef test_print_linkedin_sections_prints_links_when_present(capsys) -> None:\n # Regression coverage: the CLI previously never printed LinkedIn links when the list was non-empty.\n print_linkedin_sections(\n engines=[\"linkedin\"],\n people=[],\n links=[\"https://b.example\", \"https://a.example\", \"https://a.example\"],\n )\n\n out = capsys.readouterr().out\n assert \"No LinkedIn users found\" in out\n assert \"LinkedIn Links found: 3\" in out\n assert \"https://a.example\" in out\n assert \"https://b.example\" in out\n\n\ndef test_print_linkedin_sections_prints_people_and_links(capsys) -> None:\n print_linkedin_sections(\n engines=[\"rocketreach\"],\n people=[\"bob\", \"alice\", \"bob\"],\n links=[\"https://z.example\", \"https://z.example\"],\n )\n\n out = capsys.readouterr().out\n assert \"LinkedIn Users found: 3\" in out\n assert \"alice\" in out\n assert \"bob\" in out\n assert \"LinkedIn Links found: 2\" in out\n assert \"https://z.example\" in out\n"
},
{
"path": "tests/test_hackertarget_apikey.py",
"content": "īģŋimport pytest\nfrom theHarvester.discovery import hackertarget as ht_mod\nfrom theHarvester.lib.core import Core\n\n\nclass TestHackerTargetApiKey:\n\n @pytest.mark.asyncio\n async def test_do_search_with_apikey(self, monkeypatch):\n # make Core.hackertarget_key return a known key\n monkeypatch.setattr(Core, \"hackertarget_key\", lambda: \"TESTKEY\")\n\n # monkeypatch AsyncFetcher.fetch_all to capture requested URLs\n async def fake_fetch_all(urls, headers=None, proxy=False):\n # ensure apikey present in each URL\n assert all(\"apikey=TESTKEY\" in u for u in urls)\n return [\"1.2.3.4,host.example.com\\n\", \"No PTR records found\\n\"]\n\n monkeypatch.setattr(ht_mod.AsyncFetcher, \"fetch_all\", fake_fetch_all)\n\n s = ht_mod.SearchHackerTarget(\"example.com\")\n await s.do_search()\n\n # after do_search, total_results should include our fake response (commas replaced by colons)\n assert \"1.2.3.4:host.example.com\" in s.total_results\n\n @pytest.mark.asyncio\n async def test_do_search_without_apikey(self, monkeypatch):\n monkeypatch.setattr(Core, \"hackertarget_key\", lambda: None)\n\n async def fake_fetch_all(urls, headers=None, proxy=False):\n assert all(\"apikey=\" not in u for u in urls)\n return [\"1.2.3.4,host.example.com\\n\"]\n\n monkeypatch.setattr(ht_mod.AsyncFetcher, \"fetch_all\", fake_fetch_all)\n\n s = ht_mod.SearchHackerTarget(\"example.com\")\n await s.do_search()\n assert \"1.2.3.4:host.example.com\" in s.total_results\n"
},
{
"path": "tests/test_mojeek.py",
"content": "import pytest\nfrom theHarvester.discovery import mojeek\n\nclass TestMojeekSearch:\n\n @pytest.mark.asyncio\n async def test_process_and_parsing(self, monkeypatch):\n called = {}\n\n async def fake_fetch_all(urls, headers=None, proxy=False):\n called[\"urls\"] = urls\n called[\"headers\"] = headers\n called[\"proxy\"] = proxy\n return [\n \"Contact admin@exemple.com sur www.exemple.com \\n\",\n \" dev@exemple.com est prÊsent sur api.exemple.com \\n\"\n ]\n\n import theHarvester.lib.core as core_module\n monkeypatch.setattr(core_module.AsyncFetcher, \"fetch_all\", fake_fetch_all)\n monkeypatch.setattr(core_module.Core, \"get_user_agent\", staticmethod(lambda: \"UA\"), raising=True)\n\n search = mojeek.SearchMojeek(word=\"exemple.com\", limit=20)\n await search.process(proxy=True)\n\n expected_urls = [\n \"https://www.mojeek.com/search?q=%40exemple.com&s=0\",\n \"https://www.mojeek.com/search?q=%40exemple.com&s=10\"\n ]\n \n assert any(\"mojeek.com\" in url for url in called[\"urls\"])\n \n emails = await search.get_emails()\n hosts = await search.get_hostnames()\n\n assert \"admin@exemple.com\" in emails\n assert \"dev@exemple.com\" in emails\n assert \"www.exemple.com\" in hosts\n assert \"api.exemple.com\" in hosts\n\n @pytest.mark.asyncio\n async def test_pagination_limit(self, monkeypatch):\n captured = {}\n\n async def fake_fetch_all(urls, headers=None, proxy=False):\n captured[\"urls\"] = urls\n return [\"\"] * len(urls)\n\n import theHarvester.lib.core as core_module\n monkeypatch.setattr(core_module.AsyncFetcher, \"fetch_all\", fake_fetch_all)\n monkeypatch.setattr(core_module.Core, \"get_user_agent\", staticmethod(lambda: \"UA\"), raising=True)\n\n search = mojeek.SearchMojeek(word=\"exemple.com\", limit=10)\n await search.process()\n \n assert len(captured[\"urls\"]) == 1\n"
},
{
"path": "tests/test_myparser.py",
"content": "#!/usr/bin/env python3\n# coding=utf-8\n\nimport pytest\n\nfrom theHarvester.parsers import myparser\n\n\nclass TestMyParser(object):\n @pytest.mark.asyncio\n async def test_emails(self) -> None:\n word = \"domain.com\"\n results = \"@domain.com***a@domain***banotherdomain.com***c@domain.com***d@sub.domain.com***\"\n parse = myparser.Parser(results, word)\n emails = sorted(await parse.emails())\n assert emails, [\"c@domain.com\", \"d@sub.domain.com\"]\n\n\nif __name__ == \"__main__\":\n pytest.main()\n"
},
{
"path": "tests/test_security.py",
"content": "import os\nimport re\nimport tempfile\nfrom pathlib import Path\n\nimport pytest\nfrom fastapi.testclient import TestClient\n\nfrom theHarvester.__main__ import sanitize_filename, sanitize_for_xml\n\n\nclass TestCORSConfiguration:\n \"\"\"Test CORS security configuration.\"\"\"\n\n def test_cors_does_not_allow_credentials_with_wildcard_origins(self):\n \"\"\"\n Security Test: CORS should not allow credentials with wildcard origins.\n\n This prevents credential theft attacks where any origin can make\n authenticated requests to the API.\n \"\"\"\n from theHarvester.lib.api.api import app\n\n # Find CORS middleware in the app\n cors_middleware = None\n for middleware in app.user_middleware:\n if 'CORSMiddleware' in str(middleware.cls):\n cors_middleware = middleware\n break\n\n assert cors_middleware is not None, 'CORS middleware should be configured'\n\n # Check that if allow_origins contains '*', allow_credentials must be False\n # Access kwargs from the middleware\n options = cors_middleware.kwargs\n allow_origins = options.get('allow_origins', [])\n allow_credentials = options.get('allow_credentials', False)\n\n if isinstance(allow_origins, (list, tuple, set)) and '*' in allow_origins:\n assert (\n allow_credentials is False\n ), 'CRITICAL: CORS must not allow credentials with wildcard origins (CVE risk)'\n\n def test_cors_restricts_http_methods(self):\n \"\"\"\n Security Test: CORS should restrict HTTP methods to only what's needed.\n\n Reduces attack surface by limiting available methods.\n \"\"\"\n from theHarvester.lib.api.api import app\n\n cors_middleware = None\n for middleware in app.user_middleware:\n if 'CORSMiddleware' in str(middleware.cls):\n cors_middleware = middleware\n break\n\n assert cors_middleware is not None\n\n options = cors_middleware.kwargs\n allow_methods = options.get('allow_methods', [])\n\n # Should not allow all methods\n assert allow_methods != ['*'], 'CORS should restrict HTTP methods, not allow all (*)'\n\n # Should only allow necessary methods (GET, POST for this API)\n if isinstance(allow_methods, list):\n dangerous_methods = {'DELETE', 'PUT', 'PATCH', 'TRACE', 'CONNECT'}\n allowed_set = {m.upper() for m in allow_methods}\n assert not (\n allowed_set & dangerous_methods\n ), f'Unnecessary HTTP methods detected: {allowed_set & dangerous_methods}'\n\n\nclass TestXMLInjectionPrevention:\n \"\"\"Test XML injection prevention.\"\"\"\n\n def test_sanitize_for_xml_escapes_special_characters(self):\n \"\"\"\n Security Test: Verify XML special characters are properly escaped.\n\n Prevents XML injection attacks.\n \"\"\"\n # Test all XML special characters\n test_cases = [\n ('&', '&'),\n ('<', '<'),\n ('>', '>'),\n ('\"', '"'),\n (\"'\", '''),\n ('', '<script>alert("XSS")</script>'),\n ('user@example.com & ', 'user@example.com & <test>'),\n ('Normal text', 'Normal text'),\n ]\n\n for input_text, expected_output in test_cases:\n result = sanitize_for_xml(input_text)\n assert result == expected_output, f'Failed to properly escape: {input_text}'\n\n def test_sanitize_for_xml_prevents_xml_entity_injection(self):\n \"\"\"\n Security Test: Prevent XML entity injection attempts.\n \"\"\"\n malicious_inputs = [\n ']>',\n '',\n '',\n '<script>',\n ]\n\n for malicious_input in malicious_inputs:\n result = sanitize_for_xml(malicious_input)\n # Ensure dangerous characters are escaped\n assert '<' in result or '&' in result, f'Failed to sanitize: {malicious_input}'\n assert '<' not in result or result == malicious_input.replace('<', '<'), f'XML tags not escaped: {malicious_input}'\n\n def test_command_line_args_are_sanitized_in_xml_output(self):\n \"\"\"\n Security Test: Command line arguments must be sanitized before XML output.\n\n This test is a conceptual check - in real usage, ensure the XML writing\n code uses sanitize_for_xml() on all user-controlled data.\n \"\"\"\n # Simulate dangerous command line arguments\n dangerous_args = [\n '--domain=test.com',\n \"--source=''\",\n '--output=\"; rm -rf /',\n '--domain=example.com¶m=',\n ]\n\n for arg in dangerous_args:\n sanitized = sanitize_for_xml(arg)\n # Verify no unescaped XML special characters remain\n assert '