[ { "path": ".gitattributes", "content": "* text=auto\n\n*.py text eol=lf\n*.conf text eol=lf\n*.txt text eol=lf\n*.json text eol=lf\n*.md text eol=lf\n*.sh text eol=lf\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug Report\nabout: Create a report to help us improve\ntitle: \"\"\nlabels: bug\nassignees: \"\"\n---\n\n**Describe the bug**\nWhat happened vs what was expected?\n\n**BBOT Command**\nExample: `bbot -m httpx -t evilcorp.com`\n\n**OS, BBOT Installation Method + Version**\nExample: `OS: Arch Linux, Installation method: pip, BBOT version: 1.0.3.545`\nNote: You can get the BBOT version with `bbot --version`\nNote: BBOT is designed from the ground up to run on Linux. Windows and MacOS are not officially supported. If you are using one of these platforms, it's recommended to use Docker.\n\n**BBOT Config**\nAttach your full BBOT preset (to show it, add `--current-preset` to your BBOT command).\n\n**Logs/Screenshots**\nIf possible, produce the bug while `--debug` is enabled, and attach the relevant parts of the output.\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.md", "content": "---\nname: Feature Request\nabout: Request a new feature\ntitle: \"\"\nlabels: enhancement\nassignees: \"\"\n---\n\n**Description**\nWhich feature would you like to see added to BBOT? What are its use cases?\n" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: \"pip\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n target-branch: \"dev\"\n open-pull-requests-limit: 10\n - package-ecosystem: github-actions\n directory: /\n groups:\n github-actions:\n patterns:\n - \"*\" # Group all Actions updates into a single larger pull request\n schedule:\n interval: weekly\n target-branch: \"dev\" \n" }, { "path": ".github/workflows/benchmark.yml", "content": "name: Performance Benchmarks\n\non:\n pull_request:\n paths:\n - 'bbot/**/*.py'\n - 'pyproject.toml'\n - '.github/workflows/benchmark.yml'\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}\n cancel-in-progress: true\n\npermissions:\n contents: read\n pull-requests: write\n\njobs:\n benchmark:\n runs-on: ubuntu-latest\n \n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: 0 # Need full history for branch comparison\n\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: \"3.11\"\n\n - name: Install dependencies\n run: |\n pip install poetry\n poetry install --with dev\n\n - name: Install system dependencies\n run: |\n sudo apt-get update\n sudo apt-get install -y libmagic1\n\n # Generate benchmark comparison report using our branch-based script\n - name: Generate benchmark comparison report\n run: |\n poetry run python bbot/scripts/benchmark_report.py \\\n --base ${{ github.base_ref }} \\\n --current ${{ github.head_ref }} \\\n --output benchmark_report.md \\\n --keep-results\n continue-on-error: true\n\n # Upload benchmark results as artifacts\n - name: Upload benchmark results\n uses: actions/upload-artifact@v6\n with:\n name: benchmark-results\n path: |\n benchmark_report.md\n base_benchmark_results.json\n current_benchmark_results.json\n retention-days: 30\n\n # Comment on PR with benchmark results\n - name: Comment benchmark results on PR\n uses: actions/github-script@v8\n with:\n script: |\n const fs = require('fs');\n \n try {\n const report = fs.readFileSync('benchmark_report.md', 'utf8');\n \n // Find existing benchmark comment (with pagination)\n const comments = await github.rest.issues.listComments({\n owner: context.repo.owner,\n repo: context.repo.repo,\n issue_number: context.issue.number,\n per_page: 100, // Get more comments per page\n });\n \n // Debug: log all comments to see what we're working with\n console.log(`Found ${comments.data.length} comments on this PR`);\n comments.data.forEach((comment, index) => {\n console.log(`Comment ${index}: user=${comment.user.login}, body preview=\"${comment.body.substring(0, 100)}...\"`);\n });\n \n const existingComments = comments.data.filter(comment => \n comment.body.toLowerCase().includes('performance benchmark') &&\n comment.user.login === 'github-actions[bot]'\n );\n \n console.log(`Found ${existingComments.length} existing benchmark comments`);\n \n if (existingComments.length > 0) {\n // Sort comments by creation date to find the most recent\n const sortedComments = existingComments.sort((a, b) => \n new Date(b.created_at) - new Date(a.created_at)\n );\n \n const mostRecentComment = sortedComments[0];\n console.log(`Updating most recent benchmark comment: ${mostRecentComment.id} (created: ${mostRecentComment.created_at})`);\n \n await github.rest.issues.updateComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n comment_id: mostRecentComment.id,\n body: report\n });\n console.log('Updated existing benchmark comment');\n \n // Delete any older duplicate comments\n if (existingComments.length > 1) {\n console.log(`Deleting ${existingComments.length - 1} older duplicate comments`);\n for (let i = 1; i < sortedComments.length; i++) {\n const commentToDelete = sortedComments[i];\n console.log(`Attempting to delete comment ${commentToDelete.id} (created: ${commentToDelete.created_at})`);\n \n try {\n await github.rest.issues.deleteComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n comment_id: commentToDelete.id\n });\n console.log(`Successfully deleted duplicate comment: ${commentToDelete.id}`);\n } catch (error) {\n console.error(`Failed to delete comment ${commentToDelete.id}: ${error.message}`);\n console.error(`Error details:`, error);\n }\n }\n }\n } else {\n // Create new comment\n await github.rest.issues.createComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n issue_number: context.issue.number,\n body: report\n });\n console.log('Created new benchmark comment');\n }\n } catch (error) {\n console.error('Failed to post benchmark results:', error);\n \n // Post a fallback comment\n const fallbackMessage = [\n '## Performance Benchmark Report',\n '',\n '> ⚠ī¸ **Failed to generate detailed benchmark comparison**',\n '> ',\n '> The benchmark comparison failed to run. This might be because:',\n '> - Benchmark tests don\\'t exist on the base branch yet',\n '> - Dependencies are missing', \n '> - Test execution failed',\n '> ',\n '> Please check the [workflow logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.',\n '> ',\n '> 📁 Benchmark artifacts may be available for download from the workflow run.'\n ].join('\\\\n');\n \n await github.rest.issues.createComment({\n owner: context.repo.owner,\n repo: context.repo.repo,\n issue_number: context.issue.number,\n body: fallbackMessage\n });\n } " }, { "path": ".github/workflows/codeql.yml", "content": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# You may wish to alter this file to override the set of languages analyzed,\n# or to provide custom queries or build logic.\n#\n# ******** NOTE ********\n# We have attempted to detect the languages in your repository. Please check\n# the `language` matrix defined below to confirm you have the correct set of\n# supported CodeQL languages.\n#\nname: \"CodeQL Advanced\"\n\non:\n push:\n branches: [ \"stable\", \"dev\" ]\n pull_request:\n branches: [ \"stable\", \"dev\" ]\n schedule:\n - cron: '32 3 * * 5'\n\njobs:\n analyze:\n name: Analyze (${{ matrix.language }})\n # Runner size impacts CodeQL analysis time. To learn more, please see:\n # - https://gh.io/recommended-hardware-resources-for-running-codeql\n # - https://gh.io/supported-runners-and-hardware-resources\n # - https://gh.io/using-larger-runners (GitHub.com only)\n # Consider using larger runners or machines with greater resources for possible analysis time improvements.\n runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}\n permissions:\n # required for all workflows\n security-events: write\n\n # required to fetch internal or private CodeQL packs\n packages: read\n\n # only required for workflows in private repositories\n actions: read\n contents: read\n\n strategy:\n fail-fast: false\n matrix:\n include:\n - language: python\n build-mode: none\n # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'\n # Use `c-cpp` to analyze code written in C, C++ or both\n # Use 'java-kotlin' to analyze code written in Java, Kotlin or both\n # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both\n # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,\n # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.\n # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how\n # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages\n steps:\n - name: Checkout repository\n uses: actions/checkout@v6\n\n # Add any setup steps before running the `github/codeql-action/init` action.\n # This includes steps like installing compilers or runtimes (`actions/setup-node`\n # or others). This is typically only required for manual builds.\n # - name: Setup runtime (example)\n # uses: actions/setup-example@v1\n\n # Initializes the CodeQL tools for scanning.\n - name: Initialize CodeQL\n uses: github/codeql-action/init@v4\n with:\n languages: ${{ matrix.language }}\n build-mode: ${{ matrix.build-mode }}\n # If you wish to specify custom queries, you can do so here or in a config file.\n # By default, queries listed here will override any specified in a config file.\n # Prefix the list here with \"+\" to use these queries and those in the config file.\n\n # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs\n # queries: security-extended,security-and-quality\n\n # If the analyze step fails for one of the languages you are analyzing with\n # \"We were unable to automatically build your code\", modify the matrix above\n # to set the build mode to \"manual\" for that language. Then modify this step\n # to build your code.\n # ℹī¸ Command-line programs to run using the OS shell.\n # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun\n - if: matrix.build-mode == 'manual'\n shell: bash\n run: |\n echo 'If you are using a \"manual\" build mode for one or more of the' \\\n 'languages you are analyzing, replace this with the commands to build' \\\n 'your code, for example:'\n echo ' make bootstrap'\n echo ' make release'\n exit 1\n\n - name: Perform CodeQL Analysis\n uses: github/codeql-action/analyze@v4\n with:\n category: \"/language:${{matrix.language}}\"\n" }, { "path": ".github/workflows/distro_tests.yml", "content": "name: Tests (Linux Distros)\non:\n pull_request:\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}\n cancel-in-progress: true\n\njobs:\n test-distros:\n runs-on: ubuntu-latest\n container:\n image: ${{ matrix.os }}\n strategy:\n fail-fast: false\n matrix:\n os: [\"ubuntu:22.04\", \"ubuntu:24.04\", \"debian\", \"archlinux\", \"fedora\", \"kalilinux/kali-rolling\", \"parrotsec/security\"]\n steps:\n - uses: actions/checkout@v6\n - name: Install Python and Poetry\n run: |\n if [ -f /etc/os-release ]; then\n . /etc/os-release\n if [ \"$ID\" = \"ubuntu\" ] || [ \"$ID\" = \"debian\" ] || [ \"$ID\" = \"kali\" ] || [ \"$ID\" = \"parrotsec\" ]; then\n export DEBIAN_FRONTEND=noninteractive\n apt-get update\n apt-get -y install curl git bash build-essential docker.io libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev\n elif [ \"$ID\" = \"alpine\" ]; then\n apk add --no-cache bash gcc g++ musl-dev libffi-dev docker curl git make openssl-dev bzip2-dev zlib-dev xz-dev sqlite-dev\n elif [ \"$ID\" = \"arch\" ]; then\n pacman -Syu --noconfirm curl docker git bash base-devel\n elif [ \"$ID\" = \"fedora\" ]; then\n dnf install -y curl docker git bash gcc make patch p7zip p7zip-plugins openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel python3-libdnf5\n elif [ \"$ID\" = \"gentoo\" ]; then\n echo \"media-libs/libglvnd X\" >> /etc/portage/package.use/libglvnd\n emerge-webrsync\n emerge --update --newuse dev-vcs/git media-libs/mesa curl docker bash\n fi\n fi\n\n # Re-run the script with bash\n exec bash -c \"\n curl https://pyenv.run | bash\n export PATH=\\\"$HOME/.pyenv/bin:\\$PATH\\\"\n export PATH=\\\"$HOME/.local/bin:\\$PATH\\\"\n eval \\\"\\$(pyenv init --path)\\\"\n eval \\\"\\$(pyenv init -)\\\"\n eval \\\"\\$(pyenv virtualenv-init -)\\\"\n pyenv install 3.11\n pyenv global 3.11\n pyenv rehash\n python3.11 -m pip install --user pipx\n python3.11 -m pipx ensurepath\n pipx install poetry\n \"\n - name: Set OS Environment Variable\n run: echo \"OS_NAME=${{ matrix.os }}\" | sed 's|[:/]|_|g' >> $GITHUB_ENV\n - name: Run tests\n run: |\n export PATH=\"$HOME/.local/bin:$PATH\"\n export PATH=\"$HOME/.pyenv/bin:$PATH\"\n export PATH=\"$HOME/.pyenv/shims:$PATH\"\n export BBOT_DISTRO_TESTS=true\n poetry env use python3.11\n poetry install\n poetry run pytest --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .\n - name: Upload Debug Logs\n if: always()\n uses: actions/upload-artifact@v6\n with:\n name: pytest-debug-logs-${{ env.OS_NAME }}\n path: pytest_debug.log\n" }, { "path": ".github/workflows/docs_updater.yml", "content": "name: Daily Docs Update\n\non:\n schedule:\n - cron: '30 2 * * *' # Runs daily at 2:30 AM UTC, a less congested time\n workflow_dispatch: # Allows manual triggering of the workflow\n\njobs:\n update_docs:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n with:\n token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}\n ref: dev # Checkout the dev branch\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: \"3.x\"\n - name: Install dependencies\n run: |\n pip install poetry\n poetry install\n - name: Generate docs\n run: |\n poetry run bbot/scripts/docs.py\n - name: Create or Update Pull Request\n uses: peter-evans/create-pull-request@v8\n with:\n token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}\n branch: update-docs\n base: dev\n title: \"Automated Docs Update\"\n body: \"This is an automated pull request to update the documentation.\"\n" }, { "path": ".github/workflows/tests.yml", "content": "name: Tests\non:\n push:\n branches:\n - stable\n - dev\n pull_request:\n\nconcurrency:\n group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}\n cancel-in-progress: true\n\njobs:\n test:\n runs-on: ubuntu-latest\n strategy:\n # if one python version fails, let the others finish\n fail-fast: false\n matrix:\n python-version: [\"3.9\", \"3.10\", \"3.11\", \"3.12\", \"3.13\"]\n steps:\n - uses: actions/checkout@v6\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: ${{ matrix.python-version }}\n - name: Set Python Version Environment Variable\n run: echo \"PYTHON_VERSION=${{ matrix.python-version }}\" | sed 's|[:/]|_|g' >> $GITHUB_ENV\n - name: Install dependencies\n run: |\n pip install poetry\n poetry install\n - name: Lint\n run: |\n poetry run ruff check\n poetry run ruff format --check\n - name: Run tests\n run: |\n poetry run pytest -vv --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .\n - name: Upload Debug Logs\n if: always()\n uses: actions/upload-artifact@v6\n with:\n name: pytest-debug-logs-${{ env.PYTHON_VERSION }}\n path: pytest_debug.log\n - name: Upload Code Coverage\n uses: codecov/codecov-action@v5\n with:\n token: ${{ secrets.CODECOV_TOKEN }}\n files: ./cov.xml\n verbose: true\n publish_code:\n needs: test\n runs-on: ubuntu-latest\n if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable')\n continue-on-error: true\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: 0\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n - name: Login to Docker Hub\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: \"3.x\"\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n pip install poetry build\n poetry self add \"poetry-dynamic-versioning[plugin]\"\n - name: Build Pypi package\n if: github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev'\n run: python -m build\n - name: Publish Pypi package\n if: github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev'\n uses: pypa/gh-action-pypi-publish@release/v1.13\n with:\n password: ${{ secrets.PYPI_API_TOKEN }}\n - name: Get BBOT version\n id: version\n run: |\n FULL_VERSION=$(poetry version | cut -d' ' -f2)\n echo \"BBOT_VERSION=$FULL_VERSION\" >> $GITHUB_OUTPUT\n # Extract major.minor (e.g., 2.7 from 2.7.1)\n MAJOR_MINOR=$(echo \"$FULL_VERSION\" | cut -d'.' -f1-2)\n echo \"BBOT_VERSION_MAJOR_MINOR=$MAJOR_MINOR\" >> $GITHUB_OUTPUT\n # Extract major (e.g., 2 from 2.7.1)\n MAJOR=$(echo \"$FULL_VERSION\" | cut -d'.' -f1)\n echo \"BBOT_VERSION_MAJOR=$MAJOR\" >> $GITHUB_OUTPUT\n - name: Publish to Docker Hub (dev)\n if: github.event_name == 'push' && github.ref == 'refs/heads/dev'\n uses: docker/build-push-action@v6\n with:\n push: true\n context: .\n tags: |\n blacklanternsecurity/bbot:latest\n blacklanternsecurity/bbot:dev\n blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }}\n blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }}\n blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }}\n - name: Publish to Docker Hub (stable)\n if: github.event_name == 'push' && github.ref == 'refs/heads/stable'\n uses: docker/build-push-action@v6\n with:\n push: true\n context: .\n tags: |\n blacklanternsecurity/bbot:stable\n blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }}\n blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }}\n blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }}\n - name: Publish Full Docker Image to Docker Hub (dev)\n if: github.event_name == 'push' && github.ref == 'refs/heads/dev'\n uses: docker/build-push-action@v6\n with:\n push: true\n file: Dockerfile.full\n context: .\n tags: |\n blacklanternsecurity/bbot:latest-full\n blacklanternsecurity/bbot:dev-full\n blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }}-full\n blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }}-full\n blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }}-full\n - name: Publish Full Docker Image to Docker Hub (stable)\n if: github.event_name == 'push' && github.ref == 'refs/heads/stable'\n uses: docker/build-push-action@v6\n with:\n push: true\n file: Dockerfile.full\n context: .\n tags: |\n blacklanternsecurity/bbot:stable-full\n blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION }}-full\n blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR_MINOR }}-full\n blacklanternsecurity/bbot:${{ steps.version.outputs.BBOT_VERSION_MAJOR }}-full\n - name: Docker Hub Description\n if: github.event_name == 'push' && github.ref == 'refs/heads/dev'\n uses: peter-evans/dockerhub-description@v5\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n repository: blacklanternsecurity/bbot\n - name: Clean up old Docker Hub tags (up to 50 most recent tags plus 'latest')\n if: github.event_name == 'push' && github.ref == 'refs/heads/dev'\n run: |\n # Install jq for JSON processing\n sudo apt-get update && sudo apt-get install -y jq\n\n IMAGE=\"blacklanternsecurity/bbot\"\n\n # Clean up dev tags (keep 50 most recent)\n for tag_pattern in \"rc$\" \"rc-full$\"; do\n echo \"Cleaning up tags ending with $tag_pattern...\"\n\n tags_response=$(curl -s -H \"Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}\" \\\n \"https://hub.docker.com/v2/repositories/$IMAGE/tags/?page_size=100\")\n\n tags_to_delete=$(echo \"$tags_response\" | jq -r --arg pattern \"$tag_pattern\" \\\n '.results[] | select(.name | test($pattern)) | [.last_updated, .name] | @tsv' | \\\n sort -r | tail -n +51 | cut -f2)\n\n for tag in $tags_to_delete; do\n echo \"Deleting $IMAGE tag: $tag\"\n curl -X DELETE -H \"Authorization: Bearer ${{ secrets.DOCKER_TOKEN }}\" \\\n \"https://hub.docker.com/v2/repositories/$IMAGE/tags/$tag/\"\n done\n\n echo \"Cleanup completed for tags ending with $tag_pattern. Kept 50 most recent.\"\n done\n outputs:\n BBOT_VERSION: ${{ steps.version.outputs.BBOT_VERSION }}\n publish_docs:\n runs-on: ubuntu-latest\n if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev')\n steps:\n - uses: actions/checkout@v6\n with:\n token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}\n - uses: actions/setup-python@v6\n with:\n python-version: \"3.11\"\n - run: echo \"cache_id=$(date --utc '+%V')\" >> $GITHUB_ENV\n - uses: actions/cache@v5\n with:\n key: mkdocs-material-${{ env.cache_id }}\n path: .cache\n restore-keys: |\n mkdocs-material-\n - name: Install dependencies\n run: |\n pip install poetry\n poetry install --only=docs\n - name: Configure Git\n run: |\n git config user.name github-actions\n git config user.email github-actions@github.com\n git fetch origin gh-pages:refs/remotes/origin/gh-pages\n if git show-ref --verify --quiet refs/heads/gh-pages; then\n git branch -f gh-pages origin/gh-pages\n else\n git branch --track gh-pages origin/gh-pages\n fi\n - name: Generate docs (stable branch)\n if: github.ref == 'refs/heads/stable'\n run: |\n poetry run mike deploy Stable\n - name: Generate docs (dev branch)\n if: github.ref == 'refs/heads/dev'\n run: |\n poetry run mike deploy Dev\n - name: Publish docs\n run: |\n git switch gh-pages\n git push\n # tag_commit:\n # needs: publish_code\n # runs-on: ubuntu-latest\n # if: github.event_name == 'push' && github.ref == 'refs/heads/stable'\n # steps:\n # - uses: actions/checkout@v6\n # with:\n # ref: ${{ github.head_ref }}\n # fetch-depth: 0 # Fetch all history for all tags and branches\n # - name: Configure git\n # run: |\n # git config --local user.email \"info@blacklanternsecurity.com\"\n # git config --local user.name \"GitHub Actions\"\n # - name: Tag commit\n # run: |\n # VERSION=\"${{ needs.publish_code.outputs.BBOT_VERSION }}\"\n # if [[ \"${{ github.ref }}\" == \"refs/heads/dev\" ]]; then\n # TAG_MESSAGE=\"Dev Release $VERSION\"\n # elif [[ \"${{ github.ref }}\" == \"refs/heads/stable\" ]]; then\n # TAG_MESSAGE=\"Stable Release $VERSION\"\n # fi\n # git tag -a $VERSION -m \"$TAG_MESSAGE\"\n # git push origin --tags\n" }, { "path": ".github/workflows/version_updater.yml", "content": "name: Version Updater\non:\n schedule:\n # Runs at 00:00 every day\n - cron: '0 0 * * *'\n workflow_dispatch: # Adds the ability to manually trigger the workflow\n\njobs:\n update-nuclei-version:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n with:\n ref: dev\n fetch-depth: 0\n token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: '3.x'\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n pip install requests\n - name: Get latest version\n id: get-latest-version\n run: |\n import os, requests\n response = requests.get('https://api.github.com/repos/projectdiscovery/nuclei/releases/latest')\n version = response.json()['tag_name'].lstrip('v')\n release_notes = response.json()['body']\n with open(os.getenv('GITHUB_ENV'), 'a') as env_file:\n env_file.write(f\"latest_version={version}\\n\")\n env_file.write(f\"release_notes<> $GITHUB_ENV\n - name: Update version\n id: update-version\n if: env.latest_version != env.current_version\n run: \"sed -i '0,/\\\"version\\\": \\\".*\\\",/ s/\\\"version\\\": \\\".*\\\",/\\\"version\\\": \\\"${{ env.latest_version }}\\\",/g' bbot/modules/nuclei.py\"\n - name: Create pull request to update the version\n if: steps.update-version.outcome == 'success'\n uses: peter-evans/create-pull-request@v8\n with:\n token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}\n commit-message: \"Update nuclei\"\n title: \"Update nuclei to ${{ env.latest_version }}\"\n body: |\n This PR uses https://api.github.com/repos/projectdiscovery/nuclei/releases/latest to obtain the latest version of nuclei and update the version in bbot/modules/nuclei.py.\"\n\n # Release notes:\n ${{ env.release_notes }}\n branch: \"update-nuclei\"\n committer: blsaccess \n author: blsaccess \n assignees: \"TheTechromancer\"\n update-trufflehog-version:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n with:\n ref: dev\n fetch-depth: 0\n token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}\n - name: Set up Python\n uses: actions/setup-python@v6\n with:\n python-version: '3.x'\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n pip install requests\n - name: Get latest version\n id: get-latest-version\n run: |\n import os, requests\n response = requests.get('https://api.github.com/repos/trufflesecurity/trufflehog/releases/latest')\n version = response.json()['tag_name'].lstrip('v')\n release_notes = response.json()['body']\n with open(os.getenv('GITHUB_ENV'), 'a') as env_file:\n env_file.write(f\"latest_version={version}\\n\")\n env_file.write(f\"release_notes<> $GITHUB_ENV\n - name: Update version\n id: update-version\n if: env.latest_version != env.current_version\n run: \"sed -i '0,/\\\"version\\\": \\\".*\\\",/ s/\\\"version\\\": \\\".*\\\",/\\\"version\\\": \\\"${{ env.latest_version }}\\\",/g' bbot/modules/trufflehog.py\"\n - name: Create pull request to update the version\n if: steps.update-version.outcome == 'success'\n uses: peter-evans/create-pull-request@v8\n with:\n token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}\n commit-message: \"Update trufflehog\"\n title: \"Update trufflehog to ${{ env.latest_version }}\"\n body: |\n This PR uses https://api.github.com/repos/trufflesecurity/trufflehog/releases/latest to obtain the latest version of trufflehog and update the version in bbot/modules/trufflehog.py.\n\n # Release notes:\n ${{ env.release_notes }}\n branch: \"update-trufflehog\"\n committer: blsaccess \n author: blsaccess \n assignees: \"TheTechromancer\"\n" }, { "path": ".gitignore", "content": "__pycache__/\n.coverage*\n/data/\n/neo4j/\n" }, { "path": ".gitmodules", "content": "[submodule \"bbot/modules/playground\"]\n path = bbot/modules/playground\n url = https://github.com/blacklanternsecurity/bbot-module-playground\n branch = main\n" }, { "path": ".pre-commit-config.yaml", "content": "# Learn more about this config here: https://pre-commit.com/\n\n# To enable these pre-commit hooks run:\n# `pipx install pre-commit` or `brew install pre-commit`\n# Then in the project root directory run `pre-commit install`\n\nrepos:\n - repo: https://github.com/pre-commit/pre-commit-hooks\n rev: v5.0.0\n hooks:\n - id: check-added-large-files\n - id: check-ast\n - id: check-builtin-literals\n - id: check-byte-order-marker\n - id: check-case-conflict\n # - id: check-docstring-first\n # - id: check-executables-have-shebangs\n - id: check-json\n - id: check-merge-conflict\n # - id: check-shebang-scripts-are-executable\n - id: check-symlinks\n - id: check-toml\n - id: check-vcs-permalinks\n - id: check-xml\n # - id: check-yaml\n - id: debug-statements\n - id: destroyed-symlinks\n # - id: detect-private-key\n - id: end-of-file-fixer\n - id: file-contents-sorter\n - id: fix-byte-order-marker\n - id: forbid-new-submodules\n - id: forbid-submodules\n - id: mixed-line-ending\n - id: requirements-txt-fixer\n - id: sort-simple-yaml\n - id: trailing-whitespace\n\n - repo: https://github.com/astral-sh/ruff-pre-commit\n rev: v0.8.0\n hooks:\n - id: ruff\n - id: ruff-format\n\n - repo: https://github.com/abravalheri/validate-pyproject\n rev: v0.23\n hooks:\n - id: validate-pyproject\n" }, { "path": "Dockerfile", "content": "FROM python:3.11-slim\n\nENV LANG=C.UTF-8\nENV LC_ALL=C.UTF-8\nENV PIP_NO_CACHE_DIR=off\n\nWORKDIR /usr/src/bbot\n\nRUN apt-get update && apt-get install -y openssl gcc git make unzip curl wget vim nano sudo\n\nCOPY . .\n\nRUN pip install .\n\nWORKDIR /root\n\nENTRYPOINT [ \"bbot\" ]\n" }, { "path": "Dockerfile.full", "content": "FROM python:3.11-slim\n\nENV LANG=C.UTF-8\nENV LC_ALL=C.UTF-8\nENV PIP_NO_CACHE_DIR=off\n\nWORKDIR /usr/src/bbot\n\nRUN apt-get update && apt-get install -y openssl gcc git make unzip curl wget vim nano sudo\n\nCOPY . .\n\nRUN pip install .\n\nRUN bbot --install-all-deps\n\nWORKDIR /root\n\nENTRYPOINT [ \"bbot\" ]\n" }, { "path": "LICENSE", "content": " GNU AFFERO GENERAL PUBLIC LICENSE\n Version 3, 19 November 2007\n\n Copyright (C) 2007 Free Software Foundation, Inc. \n Everyone is permitted to copy and distribute verbatim copies\n of this license document, but changing it is not allowed.\n\n Preamble\n\n The GNU Affero General Public License is a free, copyleft license for\nsoftware and other kinds of works, specifically designed to ensure\ncooperation with the community in the case of network server software.\n\n The licenses for most software and other practical works are designed\nto take away your freedom to share and change the works. By contrast,\nour General Public Licenses are intended to guarantee your freedom to\nshare and change all versions of a program--to make sure it remains free\nsoftware for all its users.\n\n When we speak of free software, we are referring to freedom, not\nprice. Our General Public Licenses are designed to make sure that you\nhave the freedom to distribute copies of free software (and charge for\nthem if you wish), that you receive source code or can get it if you\nwant it, that you can change the software or use pieces of it in new\nfree programs, and that you know you can do these things.\n\n Developers that use our General Public Licenses protect your rights\nwith two steps: (1) assert copyright on the software, and (2) offer\nyou this License which gives you legal permission to copy, distribute\nand/or modify the software.\n\n A secondary benefit of defending all users' freedom is that\nimprovements made in alternate versions of the program, if they\nreceive widespread use, become available for other developers to\nincorporate. Many developers of free software are heartened and\nencouraged by the resulting cooperation. However, in the case of\nsoftware used on network servers, this result may fail to come about.\nThe GNU General Public License permits making a modified version and\nletting the public access it on a server without ever releasing its\nsource code to the public.\n\n The GNU Affero General Public License is designed specifically to\nensure that, in such cases, the modified source code becomes available\nto the community. It requires the operator of a network server to\nprovide the source code of the modified version running there to the\nusers of that server. Therefore, public use of a modified version, on\na publicly accessible server, gives the public access to the source\ncode of the modified version.\n\n An older license, called the Affero General Public License and\npublished by Affero, was designed to accomplish similar goals. This is\na different license, not a version of the Affero GPL, but Affero has\nreleased a new version of the Affero GPL which permits relicensing under\nthis license.\n\n The precise terms and conditions for copying, distribution and\nmodification follow.\n\n TERMS AND CONDITIONS\n\n 0. Definitions.\n\n \"This License\" refers to version 3 of the GNU Affero General Public License.\n\n \"Copyright\" also means copyright-like laws that apply to other kinds of\nworks, such as semiconductor masks.\n\n \"The Program\" refers to any copyrightable work licensed under this\nLicense. Each licensee is addressed as \"you\". \"Licensees\" and\n\"recipients\" may be individuals or organizations.\n\n To \"modify\" a work means to copy from or adapt all or part of the work\nin a fashion requiring copyright permission, other than the making of an\nexact copy. The resulting work is called a \"modified version\" of the\nearlier work or a work \"based on\" the earlier work.\n\n A \"covered work\" means either the unmodified Program or a work based\non the Program.\n\n To \"propagate\" a work means to do anything with it that, without\npermission, would make you directly or secondarily liable for\ninfringement under applicable copyright law, except executing it on a\ncomputer or modifying a private copy. Propagation includes copying,\ndistribution (with or without modification), making available to the\npublic, and in some countries other activities as well.\n\n To \"convey\" a work means any kind of propagation that enables other\nparties to make or receive copies. Mere interaction with a user through\na computer network, with no transfer of a copy, is not conveying.\n\n An interactive user interface displays \"Appropriate Legal Notices\"\nto the extent that it includes a convenient and prominently visible\nfeature that (1) displays an appropriate copyright notice, and (2)\ntells the user that there is no warranty for the work (except to the\nextent that warranties are provided), that licensees may convey the\nwork under this License, and how to view a copy of this License. If\nthe interface presents a list of user commands or options, such as a\nmenu, a prominent item in the list meets this criterion.\n\n 1. Source Code.\n\n The \"source code\" for a work means the preferred form of the work\nfor making modifications to it. \"Object code\" means any non-source\nform of a work.\n\n A \"Standard Interface\" means an interface that either is an official\nstandard defined by a recognized standards body, or, in the case of\ninterfaces specified for a particular programming language, one that\nis widely used among developers working in that language.\n\n The \"System Libraries\" of an executable work include anything, other\nthan the work as a whole, that (a) is included in the normal form of\npackaging a Major Component, but which is not part of that Major\nComponent, and (b) serves only to enable use of the work with that\nMajor Component, or to implement a Standard Interface for which an\nimplementation is available to the public in source code form. A\n\"Major Component\", in this context, means a major essential component\n(kernel, window system, and so on) of the specific operating system\n(if any) on which the executable work runs, or a compiler used to\nproduce the work, or an object code interpreter used to run it.\n\n The \"Corresponding Source\" for a work in object code form means all\nthe source code needed to generate, install, and (for an executable\nwork) run the object code and to modify the work, including scripts to\ncontrol those activities. However, it does not include the work's\nSystem Libraries, or general-purpose tools or generally available free\nprograms which are used unmodified in performing those activities but\nwhich are not part of the work. For example, Corresponding Source\nincludes interface definition files associated with source files for\nthe work, and the source code for shared libraries and dynamically\nlinked subprograms that the work is specifically designed to require,\nsuch as by intimate data communication or control flow between those\nsubprograms and other parts of the work.\n\n The Corresponding Source need not include anything that users\ncan regenerate automatically from other parts of the Corresponding\nSource.\n\n The Corresponding Source for a work in source code form is that\nsame work.\n\n 2. Basic Permissions.\n\n All rights granted under this License are granted for the term of\ncopyright on the Program, and are irrevocable provided the stated\nconditions are met. This License explicitly affirms your unlimited\npermission to run the unmodified Program. The output from running a\ncovered work is covered by this License only if the output, given its\ncontent, constitutes a covered work. This License acknowledges your\nrights of fair use or other equivalent, as provided by copyright law.\n\n You may make, run and propagate covered works that you do not\nconvey, without conditions so long as your license otherwise remains\nin force. You may convey covered works to others for the sole purpose\nof having them make modifications exclusively for you, or provide you\nwith facilities for running those works, provided that you comply with\nthe terms of this License in conveying all material for which you do\nnot control copyright. Those thus making or running the covered works\nfor you must do so exclusively on your behalf, under your direction\nand control, on terms that prohibit them from making any copies of\nyour copyrighted material outside their relationship with you.\n\n Conveying under any other circumstances is permitted solely under\nthe conditions stated below. Sublicensing is not allowed; section 10\nmakes it unnecessary.\n\n 3. Protecting Users' Legal Rights From Anti-Circumvention Law.\n\n No covered work shall be deemed part of an effective technological\nmeasure under any applicable law fulfilling obligations under article\n11 of the WIPO copyright treaty adopted on 20 December 1996, or\nsimilar laws prohibiting or restricting circumvention of such\nmeasures.\n\n When you convey a covered work, you waive any legal power to forbid\ncircumvention of technological measures to the extent such circumvention\nis effected by exercising rights under this License with respect to\nthe covered work, and you disclaim any intention to limit operation or\nmodification of the work as a means of enforcing, against the work's\nusers, your or third parties' legal rights to forbid circumvention of\ntechnological measures.\n\n 4. Conveying Verbatim Copies.\n\n You may convey verbatim copies of the Program's source code as you\nreceive it, in any medium, provided that you conspicuously and\nappropriately publish on each copy an appropriate copyright notice;\nkeep intact all notices stating that this License and any\nnon-permissive terms added in accord with section 7 apply to the code;\nkeep intact all notices of the absence of any warranty; and give all\nrecipients a copy of this License along with the Program.\n\n You may charge any price or no price for each copy that you convey,\nand you may offer support or warranty protection for a fee.\n\n 5. Conveying Modified Source Versions.\n\n You may convey a work based on the Program, or the modifications to\nproduce it from the Program, in the form of source code under the\nterms of section 4, provided that you also meet all of these conditions:\n\n a) The work must carry prominent notices stating that you modified\n it, and giving a relevant date.\n\n b) The work must carry prominent notices stating that it is\n released under this License and any conditions added under section\n 7. This requirement modifies the requirement in section 4 to\n \"keep intact all notices\".\n\n c) You must license the entire work, as a whole, under this\n License to anyone who comes into possession of a copy. This\n License will therefore apply, along with any applicable section 7\n additional terms, to the whole of the work, and all its parts,\n regardless of how they are packaged. This License gives no\n permission to license the work in any other way, but it does not\n invalidate such permission if you have separately received it.\n\n d) If the work has interactive user interfaces, each must display\n Appropriate Legal Notices; however, if the Program has interactive\n interfaces that do not display Appropriate Legal Notices, your\n work need not make them do so.\n\n A compilation of a covered work with other separate and independent\nworks, which are not by their nature extensions of the covered work,\nand which are not combined with it such as to form a larger program,\nin or on a volume of a storage or distribution medium, is called an\n\"aggregate\" if the compilation and its resulting copyright are not\nused to limit the access or legal rights of the compilation's users\nbeyond what the individual works permit. Inclusion of a covered work\nin an aggregate does not cause this License to apply to the other\nparts of the aggregate.\n\n 6. Conveying Non-Source Forms.\n\n You may convey a covered work in object code form under the terms\nof sections 4 and 5, provided that you also convey the\nmachine-readable Corresponding Source under the terms of this License,\nin one of these ways:\n\n a) Convey the object code in, or embodied in, a physical product\n (including a physical distribution medium), accompanied by the\n Corresponding Source fixed on a durable physical medium\n customarily used for software interchange.\n\n b) Convey the object code in, or embodied in, a physical product\n (including a physical distribution medium), accompanied by a\n written offer, valid for at least three years and valid for as\n long as you offer spare parts or customer support for that product\n model, to give anyone who possesses the object code either (1) a\n copy of the Corresponding Source for all the software in the\n product that is covered by this License, on a durable physical\n medium customarily used for software interchange, for a price no\n more than your reasonable cost of physically performing this\n conveying of source, or (2) access to copy the\n Corresponding Source from a network server at no charge.\n\n c) Convey individual copies of the object code with a copy of the\n written offer to provide the Corresponding Source. This\n alternative is allowed only occasionally and noncommercially, and\n only if you received the object code with such an offer, in accord\n with subsection 6b.\n\n d) Convey the object code by offering access from a designated\n place (gratis or for a charge), and offer equivalent access to the\n Corresponding Source in the same way through the same place at no\n further charge. You need not require recipients to copy the\n Corresponding Source along with the object code. If the place to\n copy the object code is a network server, the Corresponding Source\n may be on a different server (operated by you or a third party)\n that supports equivalent copying facilities, provided you maintain\n clear directions next to the object code saying where to find the\n Corresponding Source. Regardless of what server hosts the\n Corresponding Source, you remain obligated to ensure that it is\n available for as long as needed to satisfy these requirements.\n\n e) Convey the object code using peer-to-peer transmission, provided\n you inform other peers where the object code and Corresponding\n Source of the work are being offered to the general public at no\n charge under subsection 6d.\n\n A separable portion of the object code, whose source code is excluded\nfrom the Corresponding Source as a System Library, need not be\nincluded in conveying the object code work.\n\n A \"User Product\" is either (1) a \"consumer product\", which means any\ntangible personal property which is normally used for personal, family,\nor household purposes, or (2) anything designed or sold for incorporation\ninto a dwelling. In determining whether a product is a consumer product,\ndoubtful cases shall be resolved in favor of coverage. For a particular\nproduct received by a particular user, \"normally used\" refers to a\ntypical or common use of that class of product, regardless of the status\nof the particular user or of the way in which the particular user\nactually uses, or expects or is expected to use, the product. A product\nis a consumer product regardless of whether the product has substantial\ncommercial, industrial or non-consumer uses, unless such uses represent\nthe only significant mode of use of the product.\n\n \"Installation Information\" for a User Product means any methods,\nprocedures, authorization keys, or other information required to install\nand execute modified versions of a covered work in that User Product from\na modified version of its Corresponding Source. The information must\nsuffice to ensure that the continued functioning of the modified object\ncode is in no case prevented or interfered with solely because\nmodification has been made.\n\n If you convey an object code work under this section in, or with, or\nspecifically for use in, a User Product, and the conveying occurs as\npart of a transaction in which the right of possession and use of the\nUser Product is transferred to the recipient in perpetuity or for a\nfixed term (regardless of how the transaction is characterized), the\nCorresponding Source conveyed under this section must be accompanied\nby the Installation Information. But this requirement does not apply\nif neither you nor any third party retains the ability to install\nmodified object code on the User Product (for example, the work has\nbeen installed in ROM).\n\n The requirement to provide Installation Information does not include a\nrequirement to continue to provide support service, warranty, or updates\nfor a work that has been modified or installed by the recipient, or for\nthe User Product in which it has been modified or installed. Access to a\nnetwork may be denied when the modification itself materially and\nadversely affects the operation of the network or violates the rules and\nprotocols for communication across the network.\n\n Corresponding Source conveyed, and Installation Information provided,\nin accord with this section must be in a format that is publicly\ndocumented (and with an implementation available to the public in\nsource code form), and must require no special password or key for\nunpacking, reading or copying.\n\n 7. Additional Terms.\n\n \"Additional permissions\" are terms that supplement the terms of this\nLicense by making exceptions from one or more of its conditions.\nAdditional permissions that are applicable to the entire Program shall\nbe treated as though they were included in this License, to the extent\nthat they are valid under applicable law. If additional permissions\napply only to part of the Program, that part may be used separately\nunder those permissions, but the entire Program remains governed by\nthis License without regard to the additional permissions.\n\n When you convey a copy of a covered work, you may at your option\nremove any additional permissions from that copy, or from any part of\nit. (Additional permissions may be written to require their own\nremoval in certain cases when you modify the work.) You may place\nadditional permissions on material, added by you to a covered work,\nfor which you have or can give appropriate copyright permission.\n\n Notwithstanding any other provision of this License, for material you\nadd to a covered work, you may (if authorized by the copyright holders of\nthat material) supplement the terms of this License with terms:\n\n a) Disclaiming warranty or limiting liability differently from the\n terms of sections 15 and 16 of this License; or\n\n b) Requiring preservation of specified reasonable legal notices or\n author attributions in that material or in the Appropriate Legal\n Notices displayed by works containing it; or\n\n c) Prohibiting misrepresentation of the origin of that material, or\n requiring that modified versions of such material be marked in\n reasonable ways as different from the original version; or\n\n d) Limiting the use for publicity purposes of names of licensors or\n authors of the material; or\n\n e) Declining to grant rights under trademark law for use of some\n trade names, trademarks, or service marks; or\n\n f) Requiring indemnification of licensors and authors of that\n material by anyone who conveys the material (or modified versions of\n it) with contractual assumptions of liability to the recipient, for\n any liability that these contractual assumptions directly impose on\n those licensors and authors.\n\n All other non-permissive additional terms are considered \"further\nrestrictions\" within the meaning of section 10. If the Program as you\nreceived it, or any part of it, contains a notice stating that it is\ngoverned by this License along with a term that is a further\nrestriction, you may remove that term. If a license document contains\na further restriction but permits relicensing or conveying under this\nLicense, you may add to a covered work material governed by the terms\nof that license document, provided that the further restriction does\nnot survive such relicensing or conveying.\n\n If you add terms to a covered work in accord with this section, you\nmust place, in the relevant source files, a statement of the\nadditional terms that apply to those files, or a notice indicating\nwhere to find the applicable terms.\n\n Additional terms, permissive or non-permissive, may be stated in the\nform of a separately written license, or stated as exceptions;\nthe above requirements apply either way.\n\n 8. Termination.\n\n You may not propagate or modify a covered work except as expressly\nprovided under this License. Any attempt otherwise to propagate or\nmodify it is void, and will automatically terminate your rights under\nthis License (including any patent licenses granted under the third\nparagraph of section 11).\n\n However, if you cease all violation of this License, then your\nlicense from a particular copyright holder is reinstated (a)\nprovisionally, unless and until the copyright holder explicitly and\nfinally terminates your license, and (b) permanently, if the copyright\nholder fails to notify you of the violation by some reasonable means\nprior to 60 days after the cessation.\n\n Moreover, your license from a particular copyright holder is\nreinstated permanently if the copyright holder notifies you of the\nviolation by some reasonable means, this is the first time you have\nreceived notice of violation of this License (for any work) from that\ncopyright holder, and you cure the violation prior to 30 days after\nyour receipt of the notice.\n\n Termination of your rights under this section does not terminate the\nlicenses of parties who have received copies or rights from you under\nthis License. If your rights have been terminated and not permanently\nreinstated, you do not qualify to receive new licenses for the same\nmaterial under section 10.\n\n 9. Acceptance Not Required for Having Copies.\n\n You are not required to accept this License in order to receive or\nrun a copy of the Program. Ancillary propagation of a covered work\noccurring solely as a consequence of using peer-to-peer transmission\nto receive a copy likewise does not require acceptance. However,\nnothing other than this License grants you permission to propagate or\nmodify any covered work. These actions infringe copyright if you do\nnot accept this License. Therefore, by modifying or propagating a\ncovered work, you indicate your acceptance of this License to do so.\n\n 10. Automatic Licensing of Downstream Recipients.\n\n Each time you convey a covered work, the recipient automatically\nreceives a license from the original licensors, to run, modify and\npropagate that work, subject to this License. You are not responsible\nfor enforcing compliance by third parties with this License.\n\n An \"entity transaction\" is a transaction transferring control of an\norganization, or substantially all assets of one, or subdividing an\norganization, or merging organizations. If propagation of a covered\nwork results from an entity transaction, each party to that\ntransaction who receives a copy of the work also receives whatever\nlicenses to the work the party's predecessor in interest had or could\ngive under the previous paragraph, plus a right to possession of the\nCorresponding Source of the work from the predecessor in interest, if\nthe predecessor has it or can get it with reasonable efforts.\n\n You may not impose any further restrictions on the exercise of the\nrights granted or affirmed under this License. For example, you may\nnot impose a license fee, royalty, or other charge for exercise of\nrights granted under this License, and you may not initiate litigation\n(including a cross-claim or counterclaim in a lawsuit) alleging that\nany patent claim is infringed by making, using, selling, offering for\nsale, or importing the Program or any portion of it.\n\n 11. Patents.\n\n A \"contributor\" is a copyright holder who authorizes use under this\nLicense of the Program or a work on which the Program is based. The\nwork thus licensed is called the contributor's \"contributor version\".\n\n A contributor's \"essential patent claims\" are all patent claims\nowned or controlled by the contributor, whether already acquired or\nhereafter acquired, that would be infringed by some manner, permitted\nby this License, of making, using, or selling its contributor version,\nbut do not include claims that would be infringed only as a\nconsequence of further modification of the contributor version. For\npurposes of this definition, \"control\" includes the right to grant\npatent sublicenses in a manner consistent with the requirements of\nthis License.\n\n Each contributor grants you a non-exclusive, worldwide, royalty-free\npatent license under the contributor's essential patent claims, to\nmake, use, sell, offer for sale, import and otherwise run, modify and\npropagate the contents of its contributor version.\n\n In the following three paragraphs, a \"patent license\" is any express\nagreement or commitment, however denominated, not to enforce a patent\n(such as an express permission to practice a patent or covenant not to\nsue for patent infringement). To \"grant\" such a patent license to a\nparty means to make such an agreement or commitment not to enforce a\npatent against the party.\n\n If you convey a covered work, knowingly relying on a patent license,\nand the Corresponding Source of the work is not available for anyone\nto copy, free of charge and under the terms of this License, through a\npublicly available network server or other readily accessible means,\nthen you must either (1) cause the Corresponding Source to be so\navailable, or (2) arrange to deprive yourself of the benefit of the\npatent license for this particular work, or (3) arrange, in a manner\nconsistent with the requirements of this License, to extend the patent\nlicense to downstream recipients. \"Knowingly relying\" means you have\nactual knowledge that, but for the patent license, your conveying the\ncovered work in a country, or your recipient's use of the covered work\nin a country, would infringe one or more identifiable patents in that\ncountry that you have reason to believe are valid.\n\n If, pursuant to or in connection with a single transaction or\narrangement, you convey, or propagate by procuring conveyance of, a\ncovered work, and grant a patent license to some of the parties\nreceiving the covered work authorizing them to use, propagate, modify\nor convey a specific copy of the covered work, then the patent license\nyou grant is automatically extended to all recipients of the covered\nwork and works based on it.\n\n A patent license is \"discriminatory\" if it does not include within\nthe scope of its coverage, prohibits the exercise of, or is\nconditioned on the non-exercise of one or more of the rights that are\nspecifically granted under this License. You may not convey a covered\nwork if you are a party to an arrangement with a third party that is\nin the business of distributing software, under which you make payment\nto the third party based on the extent of your activity of conveying\nthe work, and under which the third party grants, to any of the\nparties who would receive the covered work from you, a discriminatory\npatent license (a) in connection with copies of the covered work\nconveyed by you (or copies made from those copies), or (b) primarily\nfor and in connection with specific products or compilations that\ncontain the covered work, unless you entered into that arrangement,\nor that patent license was granted, prior to 28 March 2007.\n\n Nothing in this License shall be construed as excluding or limiting\nany implied license or other defenses to infringement that may\notherwise be available to you under applicable patent law.\n\n 12. No Surrender of Others' Freedom.\n\n If conditions are imposed on you (whether by court order, agreement or\notherwise) that contradict the conditions of this License, they do not\nexcuse you from the conditions of this License. If you cannot convey a\ncovered work so as to satisfy simultaneously your obligations under this\nLicense and any other pertinent obligations, then as a consequence you may\nnot convey it at all. For example, if you agree to terms that obligate you\nto collect a royalty for further conveying from those to whom you convey\nthe Program, the only way you could satisfy both those terms and this\nLicense would be to refrain entirely from conveying the Program.\n\n 13. Remote Network Interaction; Use with the GNU General Public License.\n\n Notwithstanding any other provision of this License, if you modify the\nProgram, your modified version must prominently offer all users\ninteracting with it remotely through a computer network (if your version\nsupports such interaction) an opportunity to receive the Corresponding\nSource of your version by providing access to the Corresponding Source\nfrom a network server at no charge, through some standard or customary\nmeans of facilitating copying of software. This Corresponding Source\nshall include the Corresponding Source for any work covered by version 3\nof the GNU General Public License that is incorporated pursuant to the\nfollowing paragraph.\n\n Notwithstanding any other provision of this License, you have\npermission to link or combine any covered work with a work licensed\nunder version 3 of the GNU General Public License into a single\ncombined work, and to convey the resulting work. The terms of this\nLicense will continue to apply to the part which is the covered work,\nbut the work with which it is combined will remain governed by version\n3 of the GNU General Public License.\n\n 14. Revised Versions of this License.\n\n The Free Software Foundation may publish revised and/or new versions of\nthe GNU Affero General Public License from time to time. Such new versions\nwill be similar in spirit to the present version, but may differ in detail to\naddress new problems or concerns.\n\n Each version is given a distinguishing version number. If the\nProgram specifies that a certain numbered version of the GNU Affero General\nPublic License \"or any later version\" applies to it, you have the\noption of following the terms and conditions either of that numbered\nversion or of any later version published by the Free Software\nFoundation. If the Program does not specify a version number of the\nGNU Affero General Public License, you may choose any version ever published\nby the Free Software Foundation.\n\n If the Program specifies that a proxy can decide which future\nversions of the GNU Affero General Public License can be used, that proxy's\npublic statement of acceptance of a version permanently authorizes you\nto choose that version for the Program.\n\n Later license versions may give you additional or different\npermissions. However, no additional obligations are imposed on any\nauthor or copyright holder as a result of your choosing to follow a\nlater version.\n\n 15. Disclaimer of Warranty.\n\n THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY\nAPPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT\nHOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY\nOF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,\nTHE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\nPURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM\nIS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF\nALL NECESSARY SERVICING, REPAIR OR CORRECTION.\n\n 16. Limitation of Liability.\n\n IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\nWILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS\nTHE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY\nGENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE\nUSE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF\nDATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD\nPARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),\nEVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF\nSUCH DAMAGES.\n\n 17. Interpretation of Sections 15 and 16.\n\n If the disclaimer of warranty and limitation of liability provided\nabove cannot be given local legal effect according to their terms,\nreviewing courts shall apply local law that most closely approximates\nan absolute waiver of all civil liability in connection with the\nProgram, unless a warranty or assumption of liability accompanies a\ncopy of the Program in return for a fee.\n\n END OF TERMS AND CONDITIONS\n\n How to Apply These Terms to Your New Programs\n\n If you develop a new program, and you want it to be of the greatest\npossible use to the public, the best way to achieve this is to make it\nfree software which everyone can redistribute and change under these terms.\n\n To do so, attach the following notices to the program. It is safest\nto attach them to the start of each source file to most effectively\nstate the exclusion of warranty; and each file should have at least\nthe \"copyright\" line and a pointer to where the full notice is found.\n\n \n Copyright (C) \n\n This program is free software: you can redistribute it and/or modify\n it under the terms of the GNU Affero General Public License as published by\n the Free Software Foundation, either version 3 of the License, or\n (at your option) any later version.\n\n This program is distributed in the hope that it will be useful,\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n GNU Affero General Public License for more details.\n\n You should have received a copy of the GNU Affero General Public License\n along with this program. If not, see .\n\nAlso add information on how to contact you by electronic and paper mail.\n\n If your software can interact with users remotely through a computer\nnetwork, you should also make sure that it provides a way for users to\nget its source. For example, if your program is a web application, its\ninterface could display a \"Source\" link that leads users to an archive\nof the code. There are many ways you could offer source, and different\nsolutions will be better for different programs; see section 13 for the\nspecific requirements.\n\n You should also get your employer (if you work as a programmer) or school,\nif any, to sign a \"copyright disclaimer\" for the program, if necessary.\nFor more information on this, and how to apply and follow the GNU AGPL, see\n.\n" }, { "path": "README.md", "content": "[![bbot_banner](https://github.com/user-attachments/assets/f02804ce-9478-4f1e-ac4d-9cf5620a3214)](https://github.com/blacklanternsecurity/bbot)\n\n[![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![License](https://img.shields.io/badge/license-AGPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Recon Village 2024](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://www.reconvillage.org/talks) [![PyPi Downloads](https://static.pepy.tech/personalized-badge/bbot?right_color=orange&left_color=grey)](https://pepy.tech/project/bbot) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A\"tests\") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA)\n\n### **BEE¡bot** is a multipurpose scanner inspired by [Spiderfoot](https://github.com/smicallef/spiderfoot), built to automate your **Recon**, **Bug Bounties**, and **ASM**!\n\nhttps://github.com/blacklanternsecurity/bbot/assets/20261699/e539e89b-92ea-46fa-b893-9cde94eebf81\n\n_A BBOT scan in real-time - visualization with [VivaGraphJS](https://github.com/blacklanternsecurity/bbot-vivagraphjs)_\n\n## Installation\n\n```bash\n# stable version\npipx install bbot\n\n# bleeding edge (dev branch)\npipx install --pip-args '\\--pre' bbot\n```\n\n_For more installation methods, including [Docker](https://hub.docker.com/r/blacklanternsecurity/bbot), see [Getting Started](https://www.blacklanternsecurity.com/bbot/Stable/)_\n\n## Example Commands\n\n### 1) Subdomain Finder\n\nPassive API sources plus a recursive DNS brute-force with target-specific subdomain mutations.\n\n```bash\n# find subdomains of evilcorp.com\nbbot -t evilcorp.com -p subdomain-enum\n\n# passive sources only\nbbot -t evilcorp.com -p subdomain-enum -rf passive\n```\n\n\n\n
\nsubdomain-enum.yml\n\n```yaml\ndescription: Enumerate subdomains via APIs, brute-force\n\nflags:\n # enable every module with the subdomain-enum flag\n - subdomain-enum\n\noutput_modules:\n # output unique subdomains to TXT file\n - subdomains\n\nconfig:\n dns:\n threads: 25\n brute_threads: 1000\n # put your API keys here\n # modules:\n # github:\n # api_key: \"\"\n # chaos:\n # api_key: \"\"\n # securitytrails:\n # api_key: \"\"\n\n```\n\n
\n\n\n\nBBOT consistently finds 20-50% more subdomains than other tools. The bigger the domain, the bigger the difference. To learn how this is possible, see [How It Works](https://www.blacklanternsecurity.com/bbot/Dev/how_it_works/).\n\n![subdomain-stats-ebay](https://github.com/blacklanternsecurity/bbot/assets/20261699/de3e7f21-6f52-4ac4-8eab-367296cd385f)\n\n### 2) Web Spider\n\n```bash\n# crawl evilcorp.com, extracting emails and other goodies\nbbot -t evilcorp.com -p spider\n```\n\n\n\n
\nspider.yml\n\n```yaml\ndescription: Recursive web spider\n\nmodules:\n - httpx\n\nblacklist:\n # Prevent spider from invalidating sessions by logging out\n - \"RE:/.*(sign|log)[_-]?out\"\n\nconfig:\n web:\n # how many links to follow in a row\n spider_distance: 2\n # don't follow links whose directory depth is higher than 4\n spider_depth: 4\n # maximum number of links to follow per page\n spider_links_per_page: 25\n\n```\n\n
\n\n\n\n### 3) Email Gatherer\n\n```bash\n# quick email enum with free APIs + scraping\nbbot -t evilcorp.com -p email-enum\n\n# pair with subdomain enum + web spider for maximum yield\nbbot -t evilcorp.com -p email-enum subdomain-enum spider\n```\n\n\n\n
\nemail-enum.yml\n\n```yaml\ndescription: Enumerate email addresses from APIs, web crawling, etc.\n\nflags:\n - email-enum\n\noutput_modules:\n - emails\n\n```\n\n
\n\n\n\n### 4) Web Scanner\n\n```bash\n# run a light web scan against www.evilcorp.com\nbbot -t www.evilcorp.com -p web-basic\n\n# run a heavy web scan against www.evilcorp.com\nbbot -t www.evilcorp.com -p web-thorough\n```\n\n\n\n
\nweb-basic.yml\n\n```yaml\ndescription: Quick web scan\n\ninclude:\n - iis-shortnames\n\nflags:\n - web-basic\n\n```\n\n
\n\n\n\n\n\n
\nweb-thorough.yml\n\n```yaml\ndescription: Aggressive web scan\n\ninclude:\n # include the web-basic preset\n - web-basic\n\nflags:\n - web-thorough\n\n```\n\n
\n\n\n\n### 5) Everything Everywhere All at Once\n\n```bash\n# everything everywhere all at once\nbbot -t evilcorp.com -p kitchen-sink --allow-deadly\n\n# roughly equivalent to:\nbbot -t evilcorp.com -p subdomain-enum cloud-enum code-enum email-enum spider web-basic paramminer dirbust-light web-screenshots --allow-deadly\n```\n\n\n\n
\nkitchen-sink.yml\n\n```yaml\ndescription: Everything everywhere all at once\n\ninclude:\n - subdomain-enum\n - cloud-enum\n - code-enum\n - email-enum\n - spider\n - web-basic\n - paramminer\n - dirbust-light\n - web-screenshots\n - baddns-intense\n\nconfig:\n modules:\n baddns:\n enable_references: True\n\n```\n\n
\n\n\n\n## How it Works\n\nClick the graph below to explore the [inner workings](https://www.blacklanternsecurity.com/bbot/Stable/how_it_works/) of BBOT.\n\n[![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/e55ba6bd-6d97-48a6-96f0-e122acc23513)](https://www.blacklanternsecurity.com/bbot/Stable/how_it_works/)\n\n## Output Modules\n\n- [Neo4j](docs/scanning/output.md#neo4j)\n- [Teams](docs/scanning/output.md#teams)\n- [Discord](docs/scanning/output.md#discord)\n- [Slack](docs/scanning/output.md#slack)\n- [Postgres](docs/scanning/output.md#postgres)\n- [MySQL](docs/scanning/output.md#mysql)\n- [SQLite](docs/scanning/output.md#sqlite)\n- [Splunk](docs/scanning/output.md#splunk)\n- [Elasticsearch](docs/scanning/output.md#elasticsearch)\n- [CSV](docs/scanning/output.md#csv)\n- [JSON](docs/scanning/output.md#json)\n- [HTTP](docs/scanning/output.md#http)\n- [Websocket](docs/scanning/output.md#websocket)\n\n...and [more](docs/scanning/output.md)!\n\n## BBOT as a Python Library\n\n#### Synchronous\n```python\nfrom bbot.scanner import Scanner\n\nif __name__ == \"__main__\":\n scan = Scanner(\"evilcorp.com\", presets=[\"subdomain-enum\"])\n for event in scan.start():\n print(event)\n```\n\n#### Asynchronous\n```python\nfrom bbot.scanner import Scanner\n\nasync def main():\n scan = Scanner(\"evilcorp.com\", presets=[\"subdomain-enum\"])\n async for event in scan.async_start():\n print(event.json())\n\nif __name__ == \"__main__\":\n import asyncio\n asyncio.run(main())\n```\n\n
\nSEE: This Nefarious Discord Bot\n\nA [BBOT Discord Bot](https://www.blacklanternsecurity.com/bbot/Stable/dev/#discord-bot-example) that responds to the `/scan` command. Scan the internet from the comfort of your discord server!\n\n![bbot-discord](https://github.com/blacklanternsecurity/bbot/assets/20261699/22b268a2-0dfd-4c2a-b7c5-548c0f2cc6f9)\n\n
\n\n## Feature Overview\n\n- Support for Multiple Targets\n- Web Screenshots\n- Suite of Offensive Web Modules\n- NLP-powered Subdomain Mutations\n- Native Output to Neo4j (and more)\n- Automatic dependency install with Ansible\n- Search entire attack surface with custom YARA rules\n- Python API + Developer Documentation\n\n## Targets\n\nBBOT accepts an unlimited number of targets via `-t`. You can specify targets either directly on the command line or in files (or both!):\n\n```bash\nbbot -t evilcorp.com evilcorp.org 1.2.3.0/24 -p subdomain-enum\n```\n\nTargets can be any of the following:\n\n- DNS Name (`evilcorp.com`)\n- IP Address (`1.2.3.4`)\n- IP Range (`1.2.3.0/24`)\n- Open TCP Port (`192.168.0.1:80`)\n- URL (`https://www.evilcorp.com`)\n- Email Address (`bob@evilcorp.com`)\n- Organization (`ORG:evilcorp`)\n- Username (`USER:bobsmith`)\n- Filesystem (`FILESYSTEM:/tmp/asdf`)\n- Mobile App (`MOBILE_APP:https://play.google.com/store/apps/details?id=com.evilcorp.app`)\n\nFor more information, see [Targets](https://www.blacklanternsecurity.com/bbot/Stable/scanning/#targets-t). To learn how BBOT handles scope, see [Scope](https://www.blacklanternsecurity.com/bbot/Stable/scanning/#scope).\n\n## API Keys\n\nSimilar to Amass or Subfinder, BBOT supports API keys for various third-party services such as SecurityTrails, etc.\n\nThe standard way to do this is to enter your API keys in **`~/.config/bbot/bbot.yml`**. Note that multiple API keys are allowed:\n```yaml\nmodules:\n shodan_dns:\n api_key: 4f41243847da693a4f356c0486114bc6\n c99:\n # multiple API keys\n api_key:\n - 21a270d5f59c9b05813a72bb41707266\n - ea8f243d9885cf8ce9876a580224fd3c\n - 5bc6ed268ab6488270e496d3183a1a27\n virustotal:\n api_key: dd5f0eee2e4a99b71a939bded450b246\n securitytrails:\n api_key: d9a05c3fd9a514497713c54b4455d0b0\n```\n\nIf you like, you can also specify them on the command line:\n```bash\nbbot -c modules.virustotal.api_key=dd5f0eee2e4a99b71a939bded450b246\n```\n\nFor details, see [Configuration](https://www.blacklanternsecurity.com/bbot/Stable/scanning/configuration/).\n\n## Complete Lists of Modules, Flags, etc.\n\n- Complete list of [Modules](https://www.blacklanternsecurity.com/bbot/Stable/modules/list_of_modules/).\n- Complete list of [Flags](https://www.blacklanternsecurity.com/bbot/Stable/scanning/#list-of-flags).\n- Complete list of [Presets](https://www.blacklanternsecurity.com/bbot/Stable/scanning/presets_list/).\n - Complete list of [Global Config Options](https://www.blacklanternsecurity.com/bbot/Stable/scanning/configuration/#global-config-options).\n - Complete list of [Module Config Options](https://www.blacklanternsecurity.com/bbot/Stable/scanning/configuration/#module-config-options).\n\n## Documentation\n\n\n- **User Manual**\n - **Basics**\n - [Getting Started](https://www.blacklanternsecurity.com/bbot/Stable/)\n - [How it Works](https://www.blacklanternsecurity.com/bbot/Stable/how_it_works)\n - [Comparison to Other Tools](https://www.blacklanternsecurity.com/bbot/Stable/comparison)\n - **Scanning**\n - [Scanning Overview](https://www.blacklanternsecurity.com/bbot/Stable/scanning/)\n - **Presets**\n - [Overview](https://www.blacklanternsecurity.com/bbot/Stable/scanning/presets)\n - [List of Presets](https://www.blacklanternsecurity.com/bbot/Stable/scanning/presets_list)\n - [Events](https://www.blacklanternsecurity.com/bbot/Stable/scanning/events)\n - [Output](https://www.blacklanternsecurity.com/bbot/Stable/scanning/output)\n - [Tips and Tricks](https://www.blacklanternsecurity.com/bbot/Stable/scanning/tips_and_tricks)\n - [Advanced Usage](https://www.blacklanternsecurity.com/bbot/Stable/scanning/advanced)\n - [Configuration](https://www.blacklanternsecurity.com/bbot/Stable/scanning/configuration)\n - **Modules**\n - [List of Modules](https://www.blacklanternsecurity.com/bbot/Stable/modules/list_of_modules)\n - [Nuclei](https://www.blacklanternsecurity.com/bbot/Stable/modules/nuclei)\n - [Custom YARA Rules](https://www.blacklanternsecurity.com/bbot/Stable/modules/custom_yara_rules)\n - [Lightfuzz](https://www.blacklanternsecurity.com/bbot/Stable/modules/lightfuzz)\n - **Misc**\n - [Contribution](https://www.blacklanternsecurity.com/bbot/Stable/contribution)\n - [Release History](https://www.blacklanternsecurity.com/bbot/Stable/release_history)\n - [Troubleshooting](https://www.blacklanternsecurity.com/bbot/Stable/troubleshooting)\n- **Developer Manual**\n - [Development Overview](https://www.blacklanternsecurity.com/bbot/Stable/dev/)\n - [Setting Up a Dev Environment](https://www.blacklanternsecurity.com/bbot/Stable/dev/dev_environment)\n - [BBOT Internal Architecture](https://www.blacklanternsecurity.com/bbot/Stable/dev/architecture)\n - [How to Write a BBOT Module](https://www.blacklanternsecurity.com/bbot/Stable/dev/module_howto)\n - [Unit Tests](https://www.blacklanternsecurity.com/bbot/Stable/dev/tests)\n - [Discord Bot Example](https://www.blacklanternsecurity.com/bbot/Stable/dev/discord_bot)\n - **Code Reference**\n - [Scanner](https://www.blacklanternsecurity.com/bbot/Stable/dev/scanner)\n - [Presets](https://www.blacklanternsecurity.com/bbot/Stable/dev/presets)\n - [Event](https://www.blacklanternsecurity.com/bbot/Stable/dev/event)\n - [Target](https://www.blacklanternsecurity.com/bbot/Stable/dev/target)\n - [BaseModule](https://www.blacklanternsecurity.com/bbot/Stable/dev/basemodule)\n - [BBOTCore](https://www.blacklanternsecurity.com/bbot/Stable/dev/core)\n - [Engine](https://www.blacklanternsecurity.com/bbot/Stable/dev/engine)\n - **Helpers**\n - [Overview](https://www.blacklanternsecurity.com/bbot/Stable/dev/helpers/)\n - [Command](https://www.blacklanternsecurity.com/bbot/Stable/dev/helpers/command)\n - [DNS](https://www.blacklanternsecurity.com/bbot/Stable/dev/helpers/dns)\n - [Interactsh](https://www.blacklanternsecurity.com/bbot/Stable/dev/helpers/interactsh)\n - [Miscellaneous](https://www.blacklanternsecurity.com/bbot/Stable/dev/helpers/misc)\n - [Web](https://www.blacklanternsecurity.com/bbot/Stable/dev/helpers/web)\n - [Word Cloud](https://www.blacklanternsecurity.com/bbot/Stable/dev/helpers/wordcloud)\n\n\n## Contribution\n\nSome of the best BBOT modules were written by the community. BBOT is being constantly improved; every day it grows more powerful!\n\nWe welcome contributions. Not just code, but ideas too! If you have an idea for a new feature, please let us know in [Discussions](https://github.com/blacklanternsecurity/bbot/discussions). If you want to get your hands dirty, see [Contribution](https://www.blacklanternsecurity.com/bbot/Stable/contribution/). There you can find setup instructions and a simple tutorial on how to write a BBOT module. We also have extensive [Developer Documentation](https://www.blacklanternsecurity.com/bbot/Stable/dev/).\n\nThanks to these amazing people for contributing to BBOT! :heart:\n\n

\n\n \n\n

\n\nSpecial thanks to:\n\n- @TheTechromancer for creating BBOT\n- @liquidsec for his extensive work on BBOT's web hacking features, including [badsecrets](https://github.com/blacklanternsecurity/badsecrets) and [baddns](https://github.com/blacklanternsecurity/baddns)\n- Steve Micallef (@smicallef) for creating Spiderfoot\n- @kerrymilan for his Neo4j and Ansible expertise\n- @domwhewell-sage for his family of badass code-looting modules\n- @aconite33 and @amiremami for their ruthless testing\n- Aleksei Kornev (@alekseiko) for granting us ownership of the bbot Pypi repository <3\n" }, { "path": "bbot/__init__.py", "content": "# version placeholder (replaced by poetry-dynamic-versioning)\n__version__ = \"v0.0.0\"\n\nfrom .scanner import Scanner, Preset\n\n__all__ = [\"Scanner\", \"Preset\"]\n" }, { "path": "bbot/cli.py", "content": "#!/usr/bin/env python3\n\nimport io\nimport sys\nimport logging\nimport multiprocessing\nfrom bbot.errors import *\nfrom bbot import __version__\nfrom bbot.logger import log_to_stderr\nfrom bbot.core.helpers.misc import chain_lists, rm_rf\n\n\nif multiprocessing.current_process().name == \"MainProcess\":\n silent = \"-s\" in sys.argv or \"--silent\" in sys.argv\n\n if not silent:\n ascii_art = rf\"\"\" \u001b[1;38;5;208m ______ \u001b[0m _____ ____ _______\n \u001b[1;38;5;208m| ___ \\\u001b[0m| __ \\ / __ \\__ __|\n \u001b[1;38;5;208m| |___) \u001b[0m| |__) | | | | | |\n \u001b[1;38;5;208m| ___ <\u001b[0m| __ <| | | | | |\n \u001b[1;38;5;208m| |___) \u001b[0m| |__) | |__| | | |\n \u001b[1;38;5;208m|______/\u001b[0m|_____/ \\____/ |_|\n \u001b[1;38;5;208mBIGHUGE\u001b[0m BLS OSINT TOOL {__version__}\n\nwww.blacklanternsecurity.com/bbot\n\"\"\"\n print(ascii_art, file=sys.stderr)\n\nscan_name = \"\"\n\n\nasync def _main():\n import asyncio\n import traceback\n from contextlib import suppress\n\n # fix tee buffering\n sys.stdout.reconfigure(line_buffering=True)\n\n log = logging.getLogger(\"bbot.cli\")\n\n from bbot.scanner import Scanner\n from bbot.scanner.preset import Preset\n\n global scan_name\n\n try:\n # start by creating a default scan preset\n preset = Preset(_log=True, name=\"bbot_cli_main\")\n # parse command line arguments and merge into preset\n try:\n preset.parse_args()\n except BBOTArgumentError as e:\n log_to_stderr(str(e), level=\"WARNING\")\n log.trace(traceback.format_exc())\n return\n # ensure arguments (-c config options etc.) are valid\n options = preset.args.parsed\n\n # print help if no arguments\n if len(sys.argv) == 1:\n print(preset.args.parser.format_help())\n sys.exit(1)\n return\n\n # --version\n if options.version:\n print(__version__)\n sys.exit(0)\n return\n\n # --list-presets\n if options.list_presets:\n print(\"\")\n print(\"### PRESETS ###\")\n print(\"\")\n for row in preset.presets_table().splitlines():\n print(row)\n return\n\n # if we're listing modules or their options\n if options.list_modules or options.list_output_modules or options.list_module_options or options.module_help:\n # if no modules or flags are specified, enable everything\n if not (options.modules or options.output_modules or options.flags):\n for module, preloaded in preset.module_loader.preloaded().items():\n module_type = preloaded.get(\"type\", \"scan\")\n preset.add_module(module, module_type=module_type)\n\n if options.modules or options.output_modules or options.flags:\n preset._default_output_modules = options.output_modules\n preset._default_internal_modules = []\n\n preset.bake()\n\n # --list-modules\n if options.list_modules:\n print(\"\")\n print(\"### MODULES ###\")\n print(\"\")\n modules = sorted(set(preset.scan_modules + preset.internal_modules))\n for row in preset.module_loader.modules_table(modules).splitlines():\n print(row)\n return\n\n # --list-output-modules\n if options.list_output_modules:\n print(\"\")\n print(\"### OUTPUT MODULES ###\")\n print(\"\")\n for row in preset.module_loader.modules_table(preset.output_modules).splitlines():\n print(row)\n return\n\n # --list-module-options\n if options.list_module_options:\n print(\"\")\n print(\"### MODULE OPTIONS ###\")\n print(\"\")\n for row in preset.module_loader.modules_options_table(preset.modules).splitlines():\n print(row)\n return\n\n # --module-help\n if options.module_help:\n module_name = options.module_help\n all_modules = list(preset.module_loader.preloaded())\n if module_name not in all_modules:\n log.hugewarning(f'Module \"{module_name}\" not found')\n return\n\n # Load the module class\n loaded_modules = preset.module_loader.load_modules([module_name])\n module_name, module_class = next(iter(loaded_modules.items()))\n print(module_class.help_text())\n return\n\n # --list-flags\n if options.list_flags:\n flags = preset.flags if preset.flags else None\n print(\"\")\n print(\"### FLAGS ###\")\n print(\"\")\n for row in preset.module_loader.flags_table(flags=flags).splitlines():\n print(row)\n return\n\n try:\n scan = Scanner(preset=preset)\n except (PresetAbortError, ValidationError) as e:\n log.warning(str(e))\n return\n\n deadly_modules = [\n m for m in scan.preset.scan_modules if \"deadly\" in preset.preloaded_module(m).get(\"flags\", [])\n ]\n if deadly_modules and not options.allow_deadly:\n log.hugewarning(f\"You enabled the following deadly modules: {','.join(deadly_modules)}\")\n log.hugewarning(\"Deadly modules are highly intrusive\")\n log.hugewarning(\"Please specify --allow-deadly to continue\")\n return False\n\n # --current-preset\n if options.current_preset:\n print(scan.preset.to_yaml())\n sys.exit(0)\n return\n\n # --current-preset-full\n if options.current_preset_full:\n print(scan.preset.to_yaml(full_config=True))\n sys.exit(0)\n return\n\n # --install-all-deps\n if options.install_all_deps:\n preloaded_modules = preset.module_loader.preloaded()\n scan_modules = [k for k, v in preloaded_modules.items() if str(v.get(\"type\", \"\")) == \"scan\"]\n output_modules = [k for k, v in preloaded_modules.items() if str(v.get(\"type\", \"\")) == \"output\"]\n log.verbose(\"Creating dummy scan with all modules + output modules for deps installation\")\n dummy_scan = Scanner(preset=preset, modules=scan_modules, output_modules=output_modules)\n dummy_scan.helpers.depsinstaller.force_deps = True\n log.info(\"Installing module dependencies\")\n await dummy_scan.load_modules()\n log.verbose(\"Running module setups\")\n succeeded, hard_failed, soft_failed = await dummy_scan.setup_modules(deps_only=True)\n # remove any leftovers from the dummy scan\n rm_rf(dummy_scan.home, ignore_errors=True)\n rm_rf(dummy_scan.temp_dir, ignore_errors=True)\n if succeeded:\n log.success(\n f\"Successfully installed dependencies for {len(succeeded):,} modules: {','.join(succeeded)}\"\n )\n if soft_failed or hard_failed:\n failed = soft_failed + hard_failed\n log.warning(f\"Failed to install dependencies for {len(failed):,} modules: {', '.join(failed)}\")\n return False\n return True\n\n scan_name = str(scan.name)\n\n log.verbose(\"\")\n log.verbose(\"### MODULES ENABLED ###\")\n log.verbose(\"\")\n for row in scan.preset.module_loader.modules_table(scan.preset.modules).splitlines():\n log.verbose(row)\n\n scan.helpers.word_cloud.load()\n await scan._prep()\n\n if not options.dry_run:\n log.trace(f\"Command: {' '.join(sys.argv)}\")\n\n if sys.stdin.isatty():\n # warn if any targets belong directly to a cloud provider\n if not scan.preset.strict_scope:\n for event in scan.target.seeds.event_seeds:\n if event.type == \"DNS_NAME\":\n cloudcheck_result = await scan.helpers.cloudcheck.lookup(event.host)\n if cloudcheck_result:\n scan.hugewarning(\n f'YOUR TARGET CONTAINS A CLOUD DOMAIN: \"{event.host}\". You\\'re in for a wild ride!'\n )\n\n if not options.yes:\n log.hugesuccess(f\"Scan ready. Press enter to execute {scan.name}\")\n input()\n\n import os\n import re\n import fcntl\n from bbot.core.helpers.misc import smart_decode\n\n def handle_keyboard_input(keyboard_input):\n kill_regex = re.compile(r\"kill (?P[a-z0-9_ ,]+)\")\n if keyboard_input:\n log.verbose(f'Got keyboard input: \"{keyboard_input}\"')\n kill_match = kill_regex.match(keyboard_input)\n if kill_match:\n modules = kill_match.group(\"modules\")\n if modules:\n modules = chain_lists(modules)\n for module in modules:\n if module in scan.modules:\n log.hugewarning(f'Killing module: \"{module}\"')\n scan.kill_module(module, message=\"killed by user\")\n else:\n log.warning(f'Invalid module: \"{module}\"')\n else:\n scan.preset.core.logger.toggle_log_level(logger=log)\n scan.modules_status(_log=True)\n\n reader = asyncio.StreamReader()\n protocol = asyncio.StreamReaderProtocol(reader)\n await asyncio.get_running_loop().connect_read_pipe(lambda: protocol, sys.stdin)\n\n # set stdout and stderr to blocking mode\n # this is needed to prevent BlockingIOErrors in logging etc.\n fds = []\n for stream in [sys.stdout, sys.stderr]:\n try:\n fds.append(stream.fileno())\n except io.UnsupportedOperation:\n log.debug(f\"Can't get fileno for {stream}\")\n for fd in fds:\n flags = fcntl.fcntl(fd, fcntl.F_GETFL)\n fcntl.fcntl(fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK)\n\n async def akeyboard_listen():\n try:\n allowed_errors = 10\n while 1:\n keyboard_input = None\n try:\n keyboard_input = smart_decode((await reader.readline()).strip())\n allowed_errors = 10\n except Exception as e:\n log_to_stderr(f\"Error in keyboard listen loop: {e}\", level=\"TRACE\")\n log_to_stderr(traceback.format_exc(), level=\"TRACE\")\n allowed_errors -= 1\n if keyboard_input is not None:\n handle_keyboard_input(keyboard_input)\n if allowed_errors <= 0:\n break\n except Exception as e:\n log_to_stderr(f\"Error in keyboard listen task: {e}\", level=\"ERROR\")\n log_to_stderr(traceback.format_exc(), level=\"TRACE\")\n\n keyboard_listen_task = asyncio.create_task(akeyboard_listen()) # noqa F841\n\n await scan.async_start_without_generator()\n\n return True\n\n except BBOTError as e:\n log.error(str(e))\n log.trace(traceback.format_exc())\n\n finally:\n # save word cloud\n with suppress(BaseException):\n scan.helpers.word_cloud.save()\n # remove output directory if empty\n with suppress(BaseException):\n scan.home.rmdir()\n\n\ndef main():\n import asyncio\n import traceback\n from bbot.core import CORE\n\n global scan_name\n try:\n asyncio.run(_main())\n except asyncio.CancelledError:\n if CORE.logger.log_level <= logging.DEBUG:\n log_to_stderr(traceback.format_exc(), level=\"DEBUG\")\n except KeyboardInterrupt:\n msg = \"Interrupted\"\n if scan_name:\n msg = f\"You killed {scan_name}\"\n log_to_stderr(msg, level=\"WARNING\")\n if CORE.logger.log_level <= logging.DEBUG:\n log_to_stderr(traceback.format_exc(), level=\"DEBUG\")\n exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "bbot/core/__init__.py", "content": "from .core import BBOTCore\n\nCORE = BBOTCore()\n" }, { "path": "bbot/core/config/__init__.py", "content": "import sys\nimport multiprocessing as mp\n\ntry:\n mp.set_start_method(\"spawn\")\nexcept Exception:\n start_method = mp.get_start_method()\n if start_method != \"spawn\":\n print(\n f\"[WARN] Multiprocessing spawn method is set to {start_method}. This may negatively affect performance.\",\n file=sys.stderr,\n )\n" }, { "path": "bbot/core/config/files.py", "content": "import sys\nfrom pathlib import Path\nfrom omegaconf import OmegaConf\n\nfrom ...logger import log_to_stderr\nfrom ...errors import ConfigLoadError\n\n\nbbot_code_dir = Path(__file__).parent.parent.parent\n\n\nclass BBOTConfigFiles:\n config_dir = (Path.home() / \".config\" / \"bbot\").resolve()\n defaults_filename = (bbot_code_dir / \"defaults.yml\").resolve()\n config_filename = (config_dir / \"bbot.yml\").resolve()\n secrets_filename = (config_dir / \"secrets.yml\").resolve()\n\n def __init__(self, core):\n self.core = core\n\n def _get_config(self, filename, name=\"config\"):\n filename = Path(filename).resolve()\n try:\n conf = OmegaConf.load(str(filename))\n cli_silent = any(x in sys.argv for x in (\"-s\", \"--silent\"))\n if __name__ == \"__main__\" and not cli_silent:\n log_to_stderr(f\"Loaded {name} from {filename}\")\n return conf\n except Exception as e:\n if filename.exists():\n raise ConfigLoadError(f\"Error parsing config at {filename}:\\n\\n{e}\")\n return OmegaConf.create()\n\n def get_custom_config(self):\n return OmegaConf.merge(\n self._get_config(self.config_filename, name=\"config\"),\n self._get_config(self.secrets_filename, name=\"secrets\"),\n )\n\n def get_default_config(self):\n return self._get_config(self.defaults_filename, name=\"defaults\")\n" }, { "path": "bbot/core/config/logger.py", "content": "import os\nimport sys\nimport atexit\nimport logging\nfrom copy import copy\nimport multiprocessing\nimport logging.handlers\nfrom pathlib import Path\nfrom contextlib import suppress\n\nfrom ..helpers.misc import mkdir, error_and_exit\nfrom ..multiprocess import SHARED_INTERPRETER_STATE\nfrom ...logger import colorize, loglevel_mapping, GzipRotatingFileHandler\n\ndebug_format = logging.Formatter(\"%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s\")\n\n\nclass ColoredFormatter(logging.Formatter):\n \"\"\"\n Pretty colors for terminal\n \"\"\"\n\n formatter = logging.Formatter(\"%(levelname)s %(message)s\")\n module_formatter = logging.Formatter(\"%(levelname)s %(name)s: %(message)s\")\n\n def format(self, record):\n colored_record = copy(record)\n levelname = colored_record.levelname\n levelshort = loglevel_mapping.get(levelname, \"INFO\")\n colored_record.levelname = colorize(f\"[{levelshort}]\", level=levelname)\n if levelname == \"CRITICAL\" or levelname.startswith(\"HUGE\"):\n colored_record.msg = colorize(colored_record.msg, level=levelname)\n # remove name\n if colored_record.name.startswith(\"bbot.modules.\"):\n colored_record.name = colored_record.name.split(\"bbot.modules.\")[-1]\n return self.module_formatter.format(colored_record)\n return self.formatter.format(colored_record)\n\n\nclass BBOTLogger:\n \"\"\"\n The main BBOT logger.\n\n The job of this class is to manage the different log handlers in BBOT,\n allow adding new log handlers, and easily switching log levels on the fly.\n \"\"\"\n\n def __init__(self, core):\n # custom logging levels\n if getattr(logging, \"HUGEWARNING\", None) is None:\n self.addLoggingLevel(\"TRACE\", 49)\n self.addLoggingLevel(\"HUGEWARNING\", 31)\n self.addLoggingLevel(\"HUGESUCCESS\", 26)\n self.addLoggingLevel(\"SUCCESS\", 25)\n self.addLoggingLevel(\"HUGEINFO\", 21)\n self.addLoggingLevel(\"HUGEVERBOSE\", 16)\n self.addLoggingLevel(\"VERBOSE\", 15)\n self.verbosity_levels_toggle = [logging.INFO, logging.VERBOSE, logging.DEBUG]\n\n self._loggers = None\n self._log_handlers = None\n self._log_level = None\n self.core_logger = logging.getLogger(\"bbot\")\n self.core = core\n\n self.listener = None\n\n # if we haven't set up logging yet, do it now\n if \"_BBOT_LOGGING_SETUP\" not in os.environ:\n os.environ[\"_BBOT_LOGGING_SETUP\"] = \"1\"\n self.queue = multiprocessing.Queue()\n self.setup_queue_handler()\n # Start the QueueListener\n self.listener = logging.handlers.QueueListener(self.queue, *self.log_handlers.values())\n self.listener.start()\n atexit.register(self.cleanup_logging)\n\n self.log_level = logging.INFO\n\n def cleanup_logging(self):\n # Close the queue handler\n with suppress(Exception):\n self.queue_handler.close()\n\n # Clean all other loggers\n for logger in logging.Logger.manager.loggerDict.values():\n if hasattr(logger, \"handlers\"): # Logger, not PlaceHolder\n for handler in list(logger.handlers):\n with suppress(Exception):\n logger.removeHandler(handler)\n with suppress(Exception):\n handler.close()\n\n # Stop queue listener\n with suppress(Exception):\n self.listener.stop()\n\n def setup_queue_handler(self, logging_queue=None, log_level=logging.DEBUG):\n if logging_queue is None:\n logging_queue = self.queue\n else:\n self.queue = logging_queue\n self.queue_handler = logging.handlers.QueueHandler(logging_queue)\n\n self.core_logger.addHandler(self.queue_handler)\n self.core_logger.setLevel(log_level)\n # disable asyncio logging for child processes\n if not SHARED_INTERPRETER_STATE.is_main_process:\n logging.getLogger(\"asyncio\").setLevel(logging.ERROR)\n\n def addLoggingLevel(self, levelName, levelNum, methodName=None):\n \"\"\"\n Comprehensively adds a new logging level to the `logging` module and the\n currently configured logging class.\n\n `levelName` becomes an attribute of the `logging` module with the value\n `levelNum`. `methodName` becomes a convenience method for both `logging`\n itself and the class returned by `logging.getLoggerClass()` (usually just\n `logging.Logger`). If `methodName` is not specified, `levelName.lower()` is\n used.\n\n To avoid accidental clobberings of existing attributes, this method will\n raise an `AttributeError` if the level name is already an attribute of the\n `logging` module or if the method name is already present\n\n Example\n -------\n >>> addLoggingLevel('TRACE', logging.DEBUG - 5)\n >>> logging.getLogger(__name__).setLevel('TRACE')\n >>> logging.getLogger(__name__).trace('that worked')\n >>> logging.trace('so did this')\n >>> logging.TRACE\n 5\n\n \"\"\"\n if not methodName:\n methodName = levelName.lower()\n\n if hasattr(logging, levelName):\n raise AttributeError(f\"{levelName} already defined in logging module\")\n if hasattr(logging, methodName):\n raise AttributeError(f\"{methodName} already defined in logging module\")\n if hasattr(logging.getLoggerClass(), methodName):\n raise AttributeError(f\"{methodName} already defined in logger class\")\n\n # This method was inspired by the answers to Stack Overflow post\n # http://stackoverflow.com/q/2183233/2988730, especially\n # http://stackoverflow.com/a/13638084/2988730\n def logForLevel(self, message, *args, **kwargs):\n if self.isEnabledFor(levelNum):\n self._log(levelNum, message, args, **kwargs)\n\n def logToRoot(message, *args, **kwargs):\n logging.log(levelNum, message, *args, **kwargs)\n\n logging.addLevelName(levelNum, levelName)\n setattr(logging, levelName, levelNum)\n setattr(logging.getLoggerClass(), methodName, logForLevel)\n setattr(logging, methodName, logToRoot)\n\n @property\n def loggers(self):\n if self._loggers is None:\n self._loggers = [\n logging.getLogger(\"bbot\"),\n logging.getLogger(\"asyncio\"),\n ]\n return self._loggers\n\n def add_log_handler(self, handler, formatter=None):\n if self.listener is None:\n return\n if handler.formatter is None:\n handler.setFormatter(debug_format)\n if handler not in self.listener.handlers:\n self.listener.handlers = self.listener.handlers + (handler,)\n\n def remove_log_handler(self, handler):\n if self.listener is None:\n return\n if handler in self.listener.handlers:\n new_handlers = list(self.listener.handlers)\n new_handlers.remove(handler)\n self.listener.handlers = tuple(new_handlers)\n\n def include_logger(self, logger):\n if logger not in self.loggers:\n self.loggers.append(logger)\n if self.log_level is not None:\n logger.setLevel(self.log_level)\n for handler in self.log_handlers.values():\n self.add_log_handler(handler)\n\n def stderr_filter(self, record):\n if record.levelno == logging.TRACE and self.log_level > logging.DEBUG:\n return False\n if record.levelno < self.log_level:\n return False\n return True\n\n @property\n def log_handlers(self):\n if self._log_handlers is None:\n log_dir = Path(self.core.home) / \"logs\"\n if not mkdir(log_dir, raise_error=False):\n error_and_exit(f\"Failure creating or error writing to BBOT logs directory ({log_dir})\")\n\n # Main log file\n main_handler = GzipRotatingFileHandler(f\"{log_dir}/bbot.log\", maxBytes=1024 * 1024 * 100, backupCount=100)\n\n # Separate log file for debugging\n debug_handler = GzipRotatingFileHandler(\n f\"{log_dir}/bbot.debug.log\", maxBytes=1024 * 1024 * 100, backupCount=100\n )\n\n # Log to stderr\n stderr_handler = logging.StreamHandler(sys.stderr)\n stderr_handler.addFilter(self.stderr_filter)\n # log to files\n debug_handler.addFilter(lambda x: x.levelno == logging.TRACE or (x.levelno < logging.VERBOSE))\n main_handler.addFilter(lambda x: x.levelno != logging.TRACE and x.levelno >= logging.VERBOSE)\n\n # Set log format\n debug_handler.setFormatter(debug_format)\n main_handler.setFormatter(debug_format)\n stderr_handler.setFormatter(ColoredFormatter(\"%(levelname)s %(name)s: %(message)s\"))\n\n self._log_handlers = {\n \"stderr\": stderr_handler,\n \"file_debug\": debug_handler,\n \"file_main\": main_handler,\n }\n return self._log_handlers\n\n @property\n def log_level(self):\n if self._log_level is None:\n return logging.INFO\n return self._log_level\n\n @log_level.setter\n def log_level(self, level):\n self.set_log_level(level)\n\n def set_log_level(self, level, logger=None):\n if isinstance(level, str):\n level = logging.getLevelName(level)\n if logger is not None:\n logger.hugeinfo(f\"Setting log level to {logging.getLevelName(level)}\")\n self._log_level = level\n for logger in self.loggers:\n logger.setLevel(level)\n\n def toggle_log_level(self, logger=None):\n if self.log_level in self.verbosity_levels_toggle:\n for i, level in enumerate(self.verbosity_levels_toggle):\n if self.log_level == level:\n self.set_log_level(\n self.verbosity_levels_toggle[(i + 1) % len(self.verbosity_levels_toggle)], logger=logger\n )\n break\n else:\n self.set_log_level(self.verbosity_levels_toggle[0], logger=logger)\n" }, { "path": "bbot/core/core.py", "content": "import os\nimport logging\nfrom copy import copy\nfrom pathlib import Path\nfrom contextlib import suppress\nfrom omegaconf import OmegaConf\n\nfrom bbot.errors import BBOTError\nfrom .multiprocess import SHARED_INTERPRETER_STATE\n\n\nDEFAULT_CONFIG = None\n\n\nclass BBOTCore:\n \"\"\"\n This is the first thing that loads when you import BBOT.\n\n Unlike a Preset, BBOTCore holds only the config, not scan-specific stuff like targets, flags, modules, etc.\n\n Its main jobs are:\n\n - set up logging\n - keep separation between the `default` and `custom` config (this allows presets to only display the config options that have changed)\n - allow for easy merging of configs\n - load quickly\n \"\"\"\n\n # used for filtering out sensitive config values\n secrets_strings = [\"api_key\", \"username\", \"password\", \"token\", \"secret\", \"_id\"]\n # don't filter/remove entries under this key\n secrets_exclude_keys = [\"modules\"]\n\n def __init__(self):\n self._logger = None\n self._files_config = None\n\n self._config = None\n self._custom_config = None\n\n # bare minimum == logging\n self.logger\n self.log = logging.getLogger(\"bbot.core\")\n\n self._prep_multiprocessing()\n\n def _prep_multiprocessing(self):\n import multiprocessing\n from .helpers.process import BBOTProcess\n\n if SHARED_INTERPRETER_STATE.is_main_process:\n # if this is the main bbot process, set the logger and queue for the first time\n from functools import partialmethod\n\n BBOTProcess.__init__ = partialmethod(\n BBOTProcess.__init__, log_level=self.logger.log_level, log_queue=self.logger.queue\n )\n\n # this makes our process class the default for process pools, etc.\n mp_context = multiprocessing.get_context(\"spawn\")\n mp_context.Process = BBOTProcess\n\n @property\n def home(self):\n return Path(self.config[\"home\"]).expanduser().resolve()\n\n @property\n def cache_dir(self):\n return self.home / \"cache\"\n\n @property\n def tools_dir(self):\n return self.home / \"tools\"\n\n @property\n def temp_dir(self):\n return self.home / \"temp\"\n\n @property\n def lib_dir(self):\n return self.home / \"lib\"\n\n @property\n def scans_dir(self):\n return self.home / \"scans\"\n\n @property\n def config(self):\n \"\"\"\n .config is just .default_config + .custom_config merged together\n\n any new values should be added to custom_config.\n \"\"\"\n if self._config is None:\n self._config = OmegaConf.merge(self.default_config, self.custom_config)\n # set read-only flag (change .custom_config instead)\n OmegaConf.set_readonly(self._config, True)\n return self._config\n\n @property\n def default_config(self):\n \"\"\"\n The default BBOT config (from `defaults.yml`). Read-only.\n \"\"\"\n global DEFAULT_CONFIG\n if DEFAULT_CONFIG is None:\n self.default_config = self.files_config.get_default_config()\n # ensure bbot home dir\n if \"home\" not in self.default_config:\n self.default_config[\"home\"] = \"~/.bbot\"\n return DEFAULT_CONFIG\n\n @default_config.setter\n def default_config(self, value):\n # we temporarily clear out the config so it can be refreshed if/when default_config changes\n global DEFAULT_CONFIG\n self._config = None\n DEFAULT_CONFIG = value\n # set read-only flag (change .custom_config instead)\n OmegaConf.set_readonly(DEFAULT_CONFIG, True)\n\n @property\n def custom_config(self):\n \"\"\"\n Custom BBOT config (from `~/.config/bbot/bbot.yml`)\n \"\"\"\n # we temporarily clear out the config so it can be refreshed if/when custom_config changes\n self._config = None\n if self._custom_config is None:\n self.custom_config = self.files_config.get_custom_config()\n return self._custom_config\n\n @custom_config.setter\n def custom_config(self, value):\n # we temporarily clear out the config so it can be refreshed if/when custom_config changes\n self._config = None\n # ensure the modules key is always a dictionary\n modules_entry = value.get(\"modules\", None)\n if modules_entry is not None and not OmegaConf.is_dict(modules_entry):\n value[\"modules\"] = {}\n self._custom_config = value\n\n def no_secrets_config(self, config):\n from .helpers.misc import clean_dict\n\n with suppress(ValueError):\n config = OmegaConf.to_object(config)\n\n return clean_dict(\n config,\n *self.secrets_strings,\n fuzzy=True,\n exclude_keys=self.secrets_exclude_keys,\n )\n\n def secrets_only_config(self, config):\n from .helpers.misc import filter_dict\n\n with suppress(ValueError):\n config = OmegaConf.to_object(config)\n\n return filter_dict(\n config,\n *self.secrets_strings,\n fuzzy=True,\n exclude_keys=self.secrets_exclude_keys,\n )\n\n def merge_custom(self, config):\n \"\"\"\n Merge a config into the custom config.\n \"\"\"\n self.custom_config = OmegaConf.merge(self.custom_config, OmegaConf.create(config))\n\n def merge_default(self, config):\n \"\"\"\n Merge a config into the default config.\n \"\"\"\n self.default_config = OmegaConf.merge(self.default_config, OmegaConf.create(config))\n\n def copy(self):\n \"\"\"\n Return a semi-shallow copy of self. (`custom_config` is copied, but `default_config` stays the same)\n \"\"\"\n core_copy = copy(self)\n core_copy._custom_config = self._custom_config.copy()\n return core_copy\n\n @property\n def files_config(self):\n \"\"\"\n Get the configs from `bbot.yml` and `defaults.yml`\n \"\"\"\n if self._files_config is None:\n from .config import files\n\n self.files = files\n self._files_config = files.BBOTConfigFiles(self)\n return self._files_config\n\n def create_process(self, *args, **kwargs):\n if os.environ.get(\"BBOT_TESTING\", \"\") == \"True\":\n process = self.create_thread(*args, **kwargs)\n else:\n if SHARED_INTERPRETER_STATE.is_scan_process:\n from .helpers.process import BBOTProcess\n\n process = BBOTProcess(*args, **kwargs)\n else:\n import multiprocessing\n\n raise BBOTError(f\"Tried to start server from process {multiprocessing.current_process().name}\")\n process.daemon = True\n return process\n\n def create_thread(self, *args, **kwargs):\n from .helpers.process import BBOTThread\n\n return BBOTThread(*args, **kwargs)\n\n @property\n def logger(self):\n self.config\n if self._logger is None:\n from .config.logger import BBOTLogger\n\n self._logger = BBOTLogger(self)\n return self._logger\n" }, { "path": "bbot/core/engine.py", "content": "import os\nimport sys\nimport zmq\nimport pickle\nimport asyncio\nimport inspect\nimport logging\nimport tempfile\nimport traceback\nimport contextlib\nimport contextvars\nimport zmq.asyncio\nimport multiprocessing\nfrom pathlib import Path\nfrom concurrent.futures import CancelledError\nfrom contextlib import asynccontextmanager, suppress\n\nfrom bbot.core import CORE\nfrom bbot.errors import BBOTEngineError\nfrom bbot.core.helpers.async_helpers import get_event_loop\nfrom bbot.core.multiprocess import SHARED_INTERPRETER_STATE\nfrom bbot.core.helpers.misc import rand_string, in_exception_chain\n\n\nerror_sentinel = object()\n\n\nclass EngineBase:\n \"\"\"\n Base Engine class for Server and Client.\n\n An Engine is a simple and lightweight RPC implementation that allows offloading async tasks\n to a separate process. It leverages ZeroMQ in a ROUTER-DEALER configuration.\n\n BBOT makes use of this by spawning a dedicated engine for DNS and HTTP tasks.\n This offloads I/O and helps free up the main event loop for other tasks.\n\n To use Engine, you must subclass both EngineClient and EngineServer.\n\n See the respective EngineClient and EngineServer classes for usage examples.\n \"\"\"\n\n ERROR_CLASS = BBOTEngineError\n\n def __init__(self, debug=False):\n self._shutdown_status = False\n self.log = logging.getLogger(f\"bbot.core.{self.__class__.__name__.lower()}\")\n self._engine_debug = debug\n\n def pickle(self, obj):\n try:\n return pickle.dumps(obj)\n except Exception as e:\n self.log.error(f\"Error serializing object: {obj}: {e}\")\n self.log.trace(traceback.format_exc())\n return error_sentinel\n\n def unpickle(self, binary):\n try:\n return pickle.loads(binary)\n except Exception as e:\n self.log.error(f\"Error deserializing binary: {e}\")\n self.log.trace(f\"Offending binary: {binary}\")\n self.log.trace(traceback.format_exc())\n return error_sentinel\n\n async def _infinite_retry(self, callback, *args, **kwargs):\n interval = kwargs.pop(\"_interval\", 300)\n context = kwargs.pop(\"_context\", \"\")\n # default overall timeout of 10 minutes (300 second interval * 2 iterations)\n max_retries = kwargs.pop(\"_max_retries\", 1)\n if not context:\n context = f\"{callback.__name__}({args}, {kwargs})\"\n retries = 0\n while not self._shutdown_status:\n try:\n return await asyncio.wait_for(callback(*args, **kwargs), timeout=interval)\n except (TimeoutError, asyncio.exceptions.TimeoutError):\n self.log.debug(f\"{self.name}: Timeout after {interval:,} seconds {context}, retrying...\")\n retries += 1\n if max_retries is not None and retries > max_retries:\n raise TimeoutError(f\"Timed out after {(max_retries + 1) * interval:,} seconds {context}\")\n\n def engine_debug(self, *args, **kwargs):\n if self._engine_debug:\n self.log.trace(*args, **kwargs)\n\n\nclass EngineClient(EngineBase):\n \"\"\"\n The client portion of BBOT's RPC Engine.\n\n To create an engine, you must create a subclass of this class and also\n define methods for each of your desired functions.\n\n Note that this only supports async functions. If you need to offload a synchronous function to another CPU, use BBOT's multiprocessing pool instead.\n\n Any CPU or I/O intense logic should be implemented in the EngineServer.\n\n These functions are typically stubs whose only job is to forward the arguments to the server.\n\n Functions with the same names should be defined on the EngineServer.\n\n The EngineClient must specify its associated server class via the `SERVER_CLASS` variable.\n\n Depending on whether your function is a generator, you will use either `run_and_return()`, or `run_and_yield`.\n\n Examples:\n >>> from bbot.core.engine import EngineClient\n >>>\n >>> class MyClient(EngineClient):\n >>> SERVER_CLASS = MyServer\n >>>\n >>> async def my_function(self, **kwargs)\n >>> return await self.run_and_return(\"my_function\", **kwargs)\n >>>\n >>> async def my_generator(self, **kwargs):\n >>> async for _ in self.run_and_yield(\"my_generator\", **kwargs):\n >>> yield _\n \"\"\"\n\n SERVER_CLASS = None\n\n def __init__(self, debug=False, **kwargs):\n self.name = f\"EngineClient {self.__class__.__name__}\"\n super().__init__(debug=debug)\n self.process = None\n if self.SERVER_CLASS is None:\n raise ValueError(f\"Must set EngineClient SERVER_CLASS, {self.SERVER_CLASS}\")\n self.CMDS = dict(self.SERVER_CLASS.CMDS)\n for k, v in list(self.CMDS.items()):\n self.CMDS[v] = k\n self.socket_address = f\"zmq_{rand_string(8)}.sock\"\n self.socket_path = Path(tempfile.gettempdir()) / self.socket_address\n self.server_kwargs = kwargs.pop(\"server_kwargs\", {})\n self._server_process = None\n self.context = zmq.asyncio.Context()\n self.context.setsockopt(zmq.LINGER, 0)\n self.sockets = set()\n\n def check_error(self, message):\n if isinstance(message, dict) and len(message) == 1 and \"_e\" in message:\n self.engine_debug(f\"{self.name}: got error message: {message}\")\n error, trace = message[\"_e\"]\n error = self.ERROR_CLASS(error)\n error.engine_traceback = trace\n self.engine_debug(f\"{self.name}: raising {error.__class__.__name__}\")\n raise error\n return False\n\n async def run_and_return(self, command, *args, **kwargs):\n fn_str = f\"{command}({args}, {kwargs})\"\n self.engine_debug(f\"{self.name}: executing run-and-return {fn_str}\")\n if self._shutdown_status and not command == \"_shutdown\":\n self.log.verbose(f\"{self.name} has been shut down and is not accepting new tasks\")\n return\n async with self.new_socket() as socket:\n try:\n message = self.make_message(command, args=args, kwargs=kwargs)\n if message is error_sentinel:\n return\n await socket.send(message)\n binary = await self._infinite_retry(socket.recv, _context=f\"waiting for return value from {fn_str}\")\n except BaseException:\n try:\n await self.send_cancel_message(socket, fn_str)\n except Exception:\n self.log.debug(f\"{self.name}: {fn_str} failed to send cancel message after exception\")\n self.log.trace(traceback.format_exc())\n raise\n # self.log.debug(f\"{self.name}.{command}({kwargs}) got binary: {binary}\")\n message = self.unpickle(binary)\n self.engine_debug(f\"{self.name}: {fn_str} got return value: {message}\")\n # error handling\n if self.check_error(message):\n return\n return message\n\n async def run_and_yield(self, command, *args, **kwargs):\n fn_str = f\"{command}({args}, {kwargs})\"\n self.engine_debug(f\"{self.name}: executing run-and-yield {fn_str}\")\n if self._shutdown_status:\n self.log.verbose(\"Engine has been shut down and is not accepting new tasks\")\n return\n message = self.make_message(command, args=args, kwargs=kwargs)\n if message is error_sentinel:\n return\n async with self.new_socket() as socket:\n # TODO: synchronize server-side generator by limiting qsize\n # socket.setsockopt(zmq.RCVHWM, 1)\n # socket.setsockopt(zmq.SNDHWM, 1)\n await socket.send(message)\n while 1:\n try:\n binary = await self._infinite_retry(\n socket.recv, _context=f\"waiting for new iteration from {fn_str}\"\n )\n # self.log.debug(f\"{self.name}.{command}({kwargs}) got binary: {binary}\")\n message = self.unpickle(binary)\n self.engine_debug(f\"{self.name}: {fn_str} got iteration: {message}\")\n # error handling\n if self.check_error(message) or self.check_stop(message):\n break\n yield message\n except (StopAsyncIteration, GeneratorExit) as e:\n exc_name = e.__class__.__name__\n self.engine_debug(f\"{self.name}.{command} got {exc_name}\")\n try:\n await self.send_cancel_message(socket, fn_str)\n except Exception:\n self.engine_debug(f\"{self.name}.{command} failed to send cancel message after {exc_name}\")\n self.log.trace(traceback.format_exc())\n break\n\n async def send_cancel_message(self, socket, context):\n \"\"\"\n Send a cancel message and wait for confirmation from the server\n \"\"\"\n # -1 == special \"cancel\" signal\n message = pickle.dumps({\"c\": -1})\n await self._infinite_retry(socket.send, message)\n while 1:\n response = await self._infinite_retry(\n socket.recv, _context=f\"waiting for CANCEL_OK from {context}\", _max_retries=4\n )\n response = pickle.loads(response)\n if isinstance(response, dict):\n response = response.get(\"m\", \"\")\n if response == \"CANCEL_OK\":\n break\n\n async def send_shutdown_message(self):\n async with self.new_socket() as socket:\n # -99 == special shutdown message\n message = pickle.dumps({\"c\": -99})\n with suppress(TimeoutError, asyncio.exceptions.TimeoutError):\n await asyncio.wait_for(socket.send(message), 0.5)\n with suppress(TimeoutError, asyncio.exceptions.TimeoutError):\n while 1:\n response = await asyncio.wait_for(socket.recv(), 0.5)\n response = pickle.loads(response)\n if isinstance(response, dict):\n response = response.get(\"m\", \"\")\n if response == \"SHUTDOWN_OK\":\n break\n\n def check_stop(self, message):\n if isinstance(message, dict) and len(message) == 1 and \"_s\" in message:\n return True\n return False\n\n def make_message(self, command, args=None, kwargs=None):\n try:\n cmd_id = self.CMDS[command]\n except KeyError:\n raise KeyError(f'Command \"{command}\" not found. Available commands: {\",\".join(self.available_commands)}')\n message = {\"c\": cmd_id}\n if args:\n message[\"a\"] = args\n if kwargs:\n message[\"k\"] = kwargs\n return pickle.dumps(message)\n\n @property\n def available_commands(self):\n return [s for s in self.CMDS if isinstance(s, str)]\n\n def start_server(self):\n process_name = multiprocessing.current_process().name\n if SHARED_INTERPRETER_STATE.is_scan_process:\n kwargs = dict(self.server_kwargs)\n # if we're in tests, we use a single event loop to avoid weird race conditions\n # this allows us to more easily mock http, etc.\n if os.environ.get(\"BBOT_TESTING\", \"\") == \"True\":\n kwargs[\"_loop\"] = get_event_loop()\n kwargs[\"debug\"] = self._engine_debug\n self.process = CORE.create_process(\n target=self.server_process,\n args=(\n self.SERVER_CLASS,\n self.socket_path,\n ),\n kwargs=kwargs,\n custom_name=f\"BBOT {self.__class__.__name__}\",\n )\n self.process.start()\n return self.process\n else:\n raise BBOTEngineError(\n f\"Tried to start server from process {process_name}. Did you forget \\\"if __name__ == '__main__'?\\\"\"\n )\n\n @staticmethod\n def server_process(server_class, socket_path, **kwargs):\n try:\n loop = kwargs.pop(\"_loop\", None)\n engine_server = server_class(socket_path, **kwargs)\n if loop is not None:\n future = asyncio.run_coroutine_threadsafe(engine_server.worker(), loop)\n future.result()\n else:\n asyncio.run(engine_server.worker())\n except (asyncio.CancelledError, KeyboardInterrupt, CancelledError):\n return\n except Exception:\n import traceback\n\n log = logging.getLogger(\"bbot.core.engine.server\")\n log.critical(f\"Unhandled error in {server_class.__name__} server process: {traceback.format_exc()}\")\n\n @asynccontextmanager\n async def new_socket(self):\n if self._server_process is None:\n self._server_process = self.start_server()\n while not self.socket_path.exists():\n self.engine_debug(f\"{self.name}: waiting for server process to start...\")\n await asyncio.sleep(0.1)\n socket = self.context.socket(zmq.DEALER)\n socket.setsockopt(zmq.LINGER, 0) # Discard pending messages immediately disconnect() or close()\n socket.setsockopt(zmq.SNDHWM, 0) # Unlimited send buffer\n socket.setsockopt(zmq.RCVHWM, 0) # Unlimited receive buffer\n socket.connect(f\"ipc://{self.socket_path}\")\n self.sockets.add(socket)\n try:\n yield socket\n finally:\n self.sockets.remove(socket)\n with suppress(Exception):\n socket.close()\n\n async def shutdown(self):\n if not self._shutdown_status:\n self._shutdown_status = True\n self.log.verbose(f\"{self.name}: shutting down...\")\n # send shutdown signal\n await self.send_shutdown_message()\n # then terminate context\n try:\n self.context.destroy(linger=0)\n except Exception:\n print(traceback.format_exc(), file=sys.stderr)\n try:\n self.context.term()\n except Exception:\n print(traceback.format_exc(), file=sys.stderr)\n # delete socket file on exit\n self.socket_path.unlink(missing_ok=True)\n\n\nclass EngineServer(EngineBase):\n \"\"\"\n The server portion of BBOT's RPC Engine.\n\n Methods defined here must match the methods in your EngineClient.\n\n To use the functions, you must create mappings for them in the CMDS attribute, as shown below.\n\n Examples:\n >>> from bbot.core.engine import EngineServer\n >>>\n >>> class MyServer(EngineServer):\n >>> CMDS = {\n >>> 0: \"my_function\",\n >>> 1: \"my_generator\",\n >>> }\n >>>\n >>> def my_function(self, arg1=None):\n >>> await asyncio.sleep(1)\n >>> return str(arg1)\n >>>\n >>> def my_generator(self):\n >>> for i in range(10):\n >>> await asyncio.sleep(1)\n >>> yield i\n \"\"\"\n\n CMDS = {}\n\n def __init__(self, socket_path, debug=False):\n self.name = f\"EngineServer {self.__class__.__name__}\"\n super().__init__(debug=debug)\n self.engine_debug(f\"{self.name}: finished setup 1 (_debug={self._engine_debug})\")\n self.socket_path = socket_path\n self.client_id_var = contextvars.ContextVar(\"client_id\", default=None)\n # task <--> client id mapping\n self.tasks = {}\n # child tasks spawned by main tasks\n self.child_tasks = {}\n self.engine_debug(f\"{self.name}: finished setup 2 (_debug={self._engine_debug})\")\n if self.socket_path is not None:\n # create ZeroMQ context\n self.context = zmq.asyncio.Context()\n # ROUTER socket can handle multiple concurrent requests\n self.socket = self.context.socket(zmq.ROUTER)\n self.socket.setsockopt(zmq.LINGER, 0) # Discard pending messages immediately disconnect() or close()\n self.socket.setsockopt(zmq.SNDHWM, 0) # Unlimited send buffer\n self.socket.setsockopt(zmq.RCVHWM, 0) # Unlimited receive buffer\n # create socket file\n self.socket.bind(f\"ipc://{self.socket_path}\")\n self.engine_debug(f\"{self.name}: finished setup 3 (_debug={self._engine_debug})\")\n\n @contextlib.contextmanager\n def client_id_context(self, value):\n token = self.client_id_var.set(value)\n try:\n yield\n finally:\n self.client_id_var.reset(token)\n\n async def run_and_return(self, client_id, command_fn, *args, **kwargs):\n fn_str = f\"{command_fn.__name__}({args}, {kwargs})\"\n self.engine_debug(fn_str)\n with self.client_id_context(client_id):\n try:\n self.engine_debug(f\"{self.name}: starting run-and-return {fn_str}\")\n try:\n result = await command_fn(*args, **kwargs)\n except BaseException as e:\n if in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):\n log_fn = self.log.debug\n else:\n log_fn = self.log.error\n error = f\"{self.name}: error in {fn_str}: {e}\"\n trace = traceback.format_exc()\n log_fn(error)\n self.log.trace(trace)\n result = {\"_e\": (error, trace)}\n finally:\n self.tasks.pop(client_id, None)\n self.engine_debug(f\"{self.name}: sending response to {fn_str}: {result}\")\n await self.send_socket_multipart(client_id, result)\n except BaseException as e:\n self.log.critical(\n f\"Unhandled exception in {self.name}.run_and_return({client_id}, {command_fn}, {args}, {kwargs}): {e}\"\n )\n self.log.critical(traceback.format_exc())\n finally:\n self.engine_debug(f\"{self.name} finished run-and-return {fn_str}\")\n\n async def run_and_yield(self, client_id, command_fn, *args, **kwargs):\n fn_str = f\"{command_fn.__name__}({args}, {kwargs})\"\n with self.client_id_context(client_id):\n try:\n self.engine_debug(f\"{self.name}: starting run-and-yield {fn_str}\")\n try:\n async for _ in command_fn(*args, **kwargs):\n self.engine_debug(f\"{self.name}: sending iteration for {fn_str}: {_}\")\n await self.send_socket_multipart(client_id, _)\n except BaseException as e:\n if in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):\n log_fn = self.log.debug\n else:\n log_fn = self.log.error\n error = f\"{self.name}: error in {fn_str}: {e}\"\n trace = traceback.format_exc()\n log_fn(error)\n self.log.trace(trace)\n result = {\"_e\": (error, trace)}\n await self.send_socket_multipart(client_id, result)\n finally:\n self.engine_debug(f\"{self.name}: reached end of run-and-yield iteration for {fn_str}\")\n # _s == special signal that means StopIteration\n await self.send_socket_multipart(client_id, {\"_s\": None})\n self.tasks.pop(client_id, None)\n except BaseException as e:\n self.log.critical(\n f\"Unhandled exception in {self.name}.run_and_yield({client_id}, {command_fn}, {args}, {kwargs}): {e}\"\n )\n self.log.critical(traceback.format_exc())\n finally:\n self.engine_debug(f\"{self.name}: finished run-and-yield {fn_str}\")\n\n async def send_socket_multipart(self, client_id, message):\n try:\n message = pickle.dumps(message)\n await self._infinite_retry(self.socket.send_multipart, [client_id, message])\n except Exception as e:\n self.log.verbose(f\"{self.name}: error sending ZMQ message: {e}\")\n self.log.trace(traceback.format_exc())\n\n def check_error(self, message):\n if message is error_sentinel:\n return True\n\n async def worker(self):\n self.engine_debug(f\"{self.name}: starting worker\")\n try:\n while 1:\n client_id, binary = await self.socket.recv_multipart()\n message = self.unpickle(binary)\n self.engine_debug(f\"{self.name} got message: {message}\")\n if self.check_error(message):\n continue\n\n cmd = message.get(\"c\", None)\n if not isinstance(cmd, int):\n self.log.warning(f\"{self.name}: no command sent in message: {message}\")\n continue\n\n # -1 == cancel task\n if cmd == -1:\n self.engine_debug(f\"{self.name} got cancel signal\")\n await self.send_socket_multipart(client_id, {\"m\": \"CANCEL_OK\"})\n await self.cancel_task(client_id)\n continue\n\n # -99 == shutdown task\n if cmd == -99:\n self.log.verbose(f\"{self.name} got shutdown signal\")\n await self.send_socket_multipart(client_id, {\"m\": \"SHUTDOWN_OK\"})\n await self._shutdown()\n return\n\n args = message.get(\"a\", ())\n if not isinstance(args, tuple):\n self.log.warning(f\"{self.name}: received invalid args of type {type(args)}, should be tuple\")\n continue\n kwargs = message.get(\"k\", {})\n if not isinstance(kwargs, dict):\n self.log.warning(f\"{self.name}: received invalid kwargs of type {type(kwargs)}, should be dict\")\n continue\n\n command_name = self.CMDS[cmd]\n command_fn = getattr(self, command_name, None)\n\n if command_fn is None:\n self.log.warning(f'{self.name} has no function named \"{command_fn}\"')\n continue\n\n if inspect.isasyncgenfunction(command_fn):\n self.engine_debug(f\"{self.name}: creating run-and-yield coroutine for {command_name}()\")\n coroutine = self.run_and_yield(client_id, command_fn, *args, **kwargs)\n else:\n self.engine_debug(f\"{self.name}: creating run-and-return coroutine for {command_name}()\")\n coroutine = self.run_and_return(client_id, command_fn, *args, **kwargs)\n\n self.engine_debug(f\"{self.name}: creating task for {command_name}() coroutine\")\n task = asyncio.create_task(coroutine)\n self.tasks[client_id] = task, command_fn, args, kwargs\n self.engine_debug(f\"{self.name}: finished creating task for {command_name}() coroutine\")\n except BaseException as e:\n await self._shutdown()\n if not in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):\n self.log.error(f\"{self.name}: error in EngineServer worker: {e}\")\n self.log.trace(traceback.format_exc())\n finally:\n self.engine_debug(f\"{self.name}: finished worker()\")\n\n async def _shutdown(self):\n if not self._shutdown_status:\n self.log.verbose(f\"{self.name}: shutting down...\")\n self._shutdown_status = True\n await self.cancel_all_tasks()\n context = getattr(self, \"context\", None)\n if context is not None:\n try:\n context.destroy(linger=0)\n except Exception:\n self.log.trace(traceback.format_exc())\n try:\n context.term()\n except Exception:\n self.log.trace(traceback.format_exc())\n self.log.verbose(f\"{self.name}: finished shutting down\")\n\n async def task_pool(self, fn, args_kwargs, threads=10, timeout=300, global_kwargs=None):\n if global_kwargs is None:\n global_kwargs = {}\n\n tasks = {}\n args_kwargs = list(args_kwargs)\n\n def new_task():\n if args_kwargs:\n kwargs = {}\n tracker = None\n args = args_kwargs.pop(0)\n if isinstance(args, (list, tuple)):\n # you can specify a custom tracker value if you want\n # this helps with correlating results\n with suppress(ValueError):\n args, kwargs, tracker = args\n # or you can just specify args/kwargs\n with suppress(ValueError):\n args, kwargs = args\n\n if not isinstance(kwargs, dict):\n raise ValueError(f\"kwargs must be dict (got: {kwargs})\")\n if not isinstance(args, (list, tuple)):\n args = [args]\n\n task = self.new_child_task(fn(*args, **kwargs, **global_kwargs))\n tasks[task] = (args, kwargs, tracker)\n\n for _ in range(threads): # Start initial batch of tasks\n new_task()\n\n while tasks: # While there are tasks pending\n # Wait for the first task to complete\n finished = await self.finished_tasks(tasks, timeout=timeout)\n for task in finished:\n result = task.result()\n (args, kwargs, tracker) = tasks.pop(task)\n yield (args, kwargs, tracker), result\n new_task()\n\n def new_child_task(self, coro):\n \"\"\"\n Create a new asyncio task, making sure to track it based on the client id.\n\n This allows the task to be automatically cancelled if its parent is cancelled.\n \"\"\"\n client_id = self.client_id_var.get()\n task = asyncio.create_task(coro)\n\n if client_id:\n\n def remove_task(t):\n tasks = self.child_tasks.get(client_id, set())\n tasks.discard(t)\n if not tasks:\n self.child_tasks.pop(client_id, None)\n\n task.add_done_callback(remove_task)\n\n try:\n self.child_tasks[client_id].add(task)\n except KeyError:\n self.child_tasks[client_id] = {task}\n\n return task\n\n async def finished_tasks(self, tasks, timeout=None):\n \"\"\"\n Given a list of asyncio tasks, return the ones that are finished with an optional timeout\n \"\"\"\n if tasks:\n try:\n done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED, timeout=timeout)\n return done\n except BaseException as e:\n if isinstance(e, (TimeoutError, asyncio.exceptions.TimeoutError)):\n self.log.warning(f\"{self.name}: Timeout after {timeout:,} seconds in finished_tasks({tasks})\")\n for task in list(tasks):\n task.cancel()\n self._await_cancelled_task(task)\n else:\n if not in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):\n self.log.error(f\"{self.name}: Unhandled exception in finished_tasks({tasks}): {e}\")\n self.log.trace(traceback.format_exc())\n raise\n return set()\n\n async def cancel_task(self, client_id):\n parent_task = self.tasks.pop(client_id, None)\n if parent_task is None:\n return\n parent_task, _cmd, _args, _kwargs = parent_task\n self.engine_debug(f\"{self.name}: Cancelling client id {client_id} (task: {parent_task})\")\n parent_task.cancel()\n child_tasks = self.child_tasks.pop(client_id, set())\n if child_tasks:\n self.engine_debug(f\"{self.name}: Cancelling {len(child_tasks):,} child tasks for client id {client_id}\")\n for child_task in child_tasks:\n child_task.cancel()\n\n for task in [parent_task] + list(child_tasks):\n await self._await_cancelled_task(task)\n\n async def _await_cancelled_task(self, task):\n try:\n await asyncio.wait_for(task, timeout=10)\n except (TimeoutError, asyncio.exceptions.TimeoutError):\n self.log.trace(f\"{self.name}: Timeout cancelling task: {task}\")\n return\n except (KeyboardInterrupt, asyncio.CancelledError):\n return\n except BaseException as e:\n self.log.error(f\"Unhandled error in {task.get_coro().__name__}(): {e}\")\n self.log.trace(traceback.format_exc())\n\n async def cancel_all_tasks(self):\n for client_id in list(self.tasks):\n await self.cancel_task(client_id)\n for client_id, tasks in self.child_tasks.items():\n for task in list(tasks):\n await self._await_cancelled_task(task)\n" }, { "path": "bbot/core/event/__init__.py", "content": "from .base import make_event, update_event, is_event, event_from_json\n\n__all__ = [\"make_event\", \"update_event\", \"is_event\", \"event_from_json\"]\n" }, { "path": "bbot/core/event/base.py", "content": "import io\nimport re\nimport uuid\nimport json\nimport base64\nimport logging\nimport tarfile\nimport datetime\nimport ipaddress\nimport traceback\n\nfrom pathlib import Path\nfrom typing import Optional\nfrom copy import copy, deepcopy\nfrom contextlib import suppress\nfrom radixtarget import RadixTarget\nfrom pydantic import BaseModel, field_validator\nfrom urllib.parse import urlparse, urljoin, parse_qs\n\n\nfrom bbot.errors import *\nfrom .helpers import EventSeed\nfrom bbot.core.helpers import (\n extract_words,\n is_domain,\n is_subdomain,\n is_ip,\n is_ip_type,\n is_ptr,\n is_uri,\n url_depth,\n domain_stem,\n make_netloc,\n make_ip_type,\n recursive_decode,\n sha1,\n smart_decode,\n split_host_port,\n tagify,\n validators,\n get_file_extension,\n)\nfrom bbot.core.helpers.web.envelopes import BaseEnvelope\n\n\nlog = logging.getLogger(\"bbot.core.event\")\n\n\nclass BaseEvent:\n \"\"\"\n Represents a piece of data discovered during a BBOT scan.\n\n An Event contains various attributes that provide metadata about the discovered data.\n The attributes assist in understanding the context of the Event and facilitate further\n filtering and querying. Events are integral in the construction of visual graphs and\n are the cornerstone of data exchange between BBOT modules.\n\n You can inherit from this class when creating a new event type. However, it's not always\n necessary. You only need to subclass if you want to layer additional functionality on\n top of the base class.\n\n Attributes:\n type (str): Specifies the type of the event, e.g., `IP_ADDRESS`, `DNS_NAME`.\n id (str): An identifier for the event (event type + sha1 hash of data). NOT universally unique.\n uuid (UUID): A universally unique identifier for the event.\n data (str or dict): The main data for the event, e.g., a URL or IP address.\n data_graph (str): Representation of `self.data` for graph nodes (e.g. Neo4j).\n data_human (str): Representation of `self.data` for human output.\n data_id (str): Representation of `self.data` used to calculate the event's ID (and ultimately its hash, which is used for deduplication)\n data_json (str): Representation of `self.data` to be used in JSON serialization.\n host (str, IPvXAddress, or IPvXNetwork): The associated IP address or hostname for the event\n host_stem (str): An abbreviated representation of hostname that removes the TLD, e.g. \"www.evilcorp\". Used by the word cloud.\n port (int or None): The port associated with the event, if applicable, else None.\n words (set): A list of relevant keywords extracted from the event. Used by the word cloud.\n scope_distance (int): Indicates how many hops the event is from the main scope; 0 means in-scope.\n web_spider_distance (int): The spider distance from the web root, specific to web crawling.\n scan (Scanner): The scan object that generated the event.\n timestamp (datetime.datetime): The time at which the data was discovered.\n resolved_hosts (list of str): List of hosts to which the event data resolves, applicable for URLs and DNS names.\n parent (BaseEvent): The parent event that led to the discovery of this event.\n parent_id (str): The `id` attribute of the parent event.\n parent_uuid (str): The `uuid` attribute of the parent event.\n tags (set of str): Descriptive tags for the event, e.g., `mx-record`, `in-scope`.\n module (BaseModule): The module that discovered the event.\n module_sequence (str): The sequence of modules that participated in the discovery.\n\n Examples:\n ```json\n {\n \"type\": \"URL\",\n \"id\": \"URL:017ec8e5dc158c0fd46f07169f8577fb4b45e89a\",\n \"data\": \"http://www.blacklanternsecurity.com/\",\n \"web_spider_distance\": 0,\n \"scope_distance\": 0,\n \"scan\": \"SCAN:4d786912dbc97be199da13074699c318e2067a7f\",\n \"timestamp\": 1688526222.723366,\n \"resolved_hosts\": [\"185.199.108.153\"],\n \"parent\": \"OPEN_TCP_PORT:cf7e6a937b161217eaed99f0c566eae045d094c7\",\n \"tags\": [\"in-scope\", \"distance-0\", \"dir\", \"ip-185-199-108-153\", \"status-301\", \"http-title-301-moved-permanently\"],\n \"module\": \"httpx\",\n \"module_sequence\": \"httpx\"\n }\n ```\n \"\"\"\n\n # Always emit this event type even if it's not in scope\n _always_emit = False\n # Always emit events with these tags even if they're not in scope\n _always_emit_tags = [\"affiliate\", \"target\"]\n # Bypass scope checking and dns resolution, distribute immediately to modules\n # This is useful for \"end-of-line\" events like FINDING and VULNERABILITY\n _quick_emit = False\n # Data validation, if data is a dictionary\n _data_validator = None\n # Whether to increment scope distance if the child and parent hosts are the same\n # Normally we don't want this, since scope distance only increases if the host changes\n # But for some events like SOCIAL media profiles, this is required to prevent spidering all of facebook.com\n _scope_distance_increment_same_host = False\n # Don't allow duplicates to occur within a parent chain\n # In other words, don't emit the event if the same one already exists in its discovery context\n _suppress_chain_dupes = False\n\n # using __slots__ dramatically reduces memory usage in large scans\n __slots__ = [\n # Core identification attributes\n \"_uuid\",\n \"_id\",\n \"_hash\",\n \"_data\",\n \"_data_hash\",\n # Host-related attributes\n \"__host\",\n \"_host_original\",\n \"_port\",\n # Parent-related attributes\n \"_parent\",\n \"_parent_id\",\n \"_parent_uuid\",\n # Event metadata\n \"_type\",\n \"_tags\",\n \"_omit\",\n \"__words\",\n \"_priority\",\n \"_scope_distance\",\n \"_module_priority\",\n \"_graph_important\",\n \"_resolved_hosts\",\n \"_discovery_context\",\n \"_discovery_context_regex\",\n \"_stats_recorded\",\n \"_internal\",\n \"_confidence\",\n \"_dummy\",\n \"_module\",\n # DNS-related attributes\n \"dns_children\",\n \"raw_dns_records\",\n \"dns_resolve_distance\",\n # Web-related attributes\n \"web_spider_distance\",\n \"parsed_url\",\n \"url_extension\",\n \"num_redirects\",\n # File-related attributes\n \"_data_path\",\n # Public attributes\n \"module\",\n \"scan\",\n \"timestamp\",\n ]\n\n def __init__(\n self,\n data,\n event_type,\n parent=None,\n context=None,\n module=None,\n scan=None,\n tags=None,\n confidence=100,\n timestamp=None,\n _dummy=False,\n _internal=None,\n ):\n \"\"\"\n Initializes an Event object with the given parameters.\n\n In most cases, you should use `make_event()` instead of instantiating this class directly.\n `make_event()` is much friendlier, and can auto-detect the event type for you.\n\n Attributes:\n data (str, dict): The primary data for the event.\n event_type (str, optional): Type of the event, e.g., 'IP_ADDRESS'.\n parent (BaseEvent, optional): Parent event that led to this event's discovery. Defaults to None.\n module (str, optional): Module that discovered the event. Defaults to None.\n scan (Scan, optional): BBOT Scan object. Required unless _dummy is True. Defaults to None.\n tags (list of str, optional): Descriptive tags for the event. Defaults to None.\n confidence (int, optional): Confidence level for the event, on a scale of 1-100. Defaults to 100.\n timestamp (datetime, optional): Time of event discovery. Defaults to current UTC time.\n _dummy (bool, optional): If True, disables certain data validations. Defaults to False.\n _internal (Any, optional): If specified, makes the event internal. Defaults to None.\n\n Raises:\n ValidationError: If either `scan` or `parent` are not specified and `_dummy` is False.\n \"\"\"\n self._uuid = uuid.uuid4()\n self._id = None\n self._hash = None\n self._data = None\n self.__host = None\n self._tags = set()\n self._port = None\n self._omit = False\n self.__words = None\n self._parent = None\n self._priority = None\n self._parent_id = None\n self._parent_uuid = None\n self._host_original = None\n self._scope_distance = None\n self._module_priority = None\n self._graph_important = False\n self._resolved_hosts = set()\n self.dns_children = {}\n self.raw_dns_records = {}\n self._discovery_context = \"\"\n self._discovery_context_regex = re.compile(r\"\\{(?:event|module)[^}]*\\}\")\n self.web_spider_distance = 0\n\n # for creating one-off events without enforcing parent requirement\n self._dummy = _dummy\n self.module = module\n self._type = event_type\n\n # keep track of whether this event has been recorded by the scan\n self._stats_recorded = False\n\n if timestamp is not None:\n self.timestamp = timestamp\n else:\n try:\n self.timestamp = datetime.datetime.now(datetime.UTC)\n except AttributeError:\n self.timestamp = datetime.datetime.utcnow()\n\n self.confidence = int(confidence)\n self._internal = False\n\n # self.scan holds the instantiated scan object (for helpers, etc.)\n self.scan = scan\n if (not self.scan) and (not self._dummy):\n raise ValidationError(\"Must specify scan\")\n\n try:\n self.data = self._sanitize_data(data)\n except Exception as e:\n log.trace(traceback.format_exc())\n raise ValidationError(f'Error sanitizing event data \"{data}\" for type \"{self.type}\": {e}')\n\n if not self.data:\n raise ValidationError(f'Invalid event data \"{data}\" for type \"{self.type}\"')\n\n self.parent = parent\n if (not self.parent) and (not self._dummy):\n raise ValidationError(\"Must specify event parent\")\n\n if tags is not None:\n for tag in tags:\n self.add_tag(tag)\n\n # internal events are not ingested by output modules\n if not self._dummy:\n # removed this second part because it was making certain sslcert events internal\n if _internal: # or parent._internal:\n self.internal = True\n\n if not context:\n context = getattr(self.module, \"default_discovery_context\", \"\")\n if context:\n self.discovery_context = context\n\n @property\n def data(self):\n return self._data\n\n @property\n def confidence(self):\n return self._confidence\n\n @confidence.setter\n def confidence(self, confidence):\n self._confidence = min(100, max(1, int(confidence)))\n\n @property\n def cumulative_confidence(self):\n \"\"\"\n Considers the confidence of parent events. This is useful for filtering out speculative/unreliable events.\n\n E.g. an event with a confidence of 50 whose parent is also 50 would have a cumulative confidence of 25.\n\n A confidence of 100 will reset the cumulative confidence to 100.\n \"\"\"\n if self._confidence == 100 or self.parent is None or self.parent is self:\n return self._confidence\n return int(self._confidence * self.parent.cumulative_confidence / 100)\n\n @property\n def resolved_hosts(self):\n if is_ip(self.host):\n return {\n self.host,\n }\n return self._resolved_hosts\n\n @data.setter\n def data(self, data):\n self._hash = None\n self._data_hash = None\n self._id = None\n self.__host = None\n self._port = None\n self._data = data\n\n @property\n def internal(self):\n return self._internal\n\n @internal.setter\n def internal(self, value):\n \"\"\"\n Marks the event as internal, excluding it from output but allowing normal exchange between scan modules.\n\n Internal events are typically speculative and may not be interesting by themselves but can lead to\n the discovery of interesting events. This method sets the `_internal` attribute to True and adds the\n \"internal\" tag.\n\n Examples of internal events include `OPEN_TCP_PORT`s from the `speculate` module,\n `IP_ADDRESS`es from the `ipneighbor` module, or out-of-scope `DNS_NAME`s that originate\n from DNS resolutions.\n\n The purpose of internal events is to enable speculative/explorative discovery without cluttering\n the console with irrelevant or uninteresting events.\n \"\"\"\n if value not in (True, False):\n raise ValueError(f'\"internal\" must be boolean, not {type(value)}')\n if value is True:\n self.add_tag(\"internal\")\n else:\n self.remove_tag(\"internal\")\n self._internal = value\n\n @property\n def host(self):\n \"\"\"\n An abbreviated representation of the data that allows comparison with other events.\n For host types, this is a hostname.\n This allows comparison of an email or a URL with a domain, and vice versa\n bob@evilcorp.com --> evilcorp.com\n https://evilcorp.com --> evilcorp.com\n evilcorp.com:80 --> evilcorp.com\n\n For IP_* types, this is an instantiated object representing the event's data\n E.g. for IP_ADDRESS, it could be an ipaddress.IPv4Address() or IPv6Address() object\n \"\"\"\n if self.__host is None:\n self.host = self._host()\n return self.__host\n\n @host.setter\n def host(self, host):\n if self._host_original is None:\n self._host_original = host\n self.__host = host\n\n @property\n def host_original(self):\n \"\"\"\n Original host data, in case it was changed due to a wildcard DNS, etc.\n \"\"\"\n if self._host_original is None:\n return self.host\n return self._host_original\n\n @property\n def host_filterable(self):\n \"\"\"\n A string version of the event that's used for regex-based blacklisting.\n\n For example, the user can specify \"REGEX:.*.evilcorp.com\" in their blacklist, and this regex\n will be applied against this property.\n \"\"\"\n parsed_url = getattr(self, \"parsed_url\", None)\n if parsed_url is not None:\n return parsed_url.geturl()\n if self.host is not None:\n return str(self.host)\n return \"\"\n\n @property\n def port(self):\n self.host\n if getattr(self, \"parsed_url\", None):\n if self.parsed_url.port is not None:\n return self.parsed_url.port\n elif self.parsed_url.scheme == \"https\":\n return 443\n elif self.parsed_url.scheme == \"http\":\n return 80\n return self._port\n\n @property\n def netloc(self):\n if self.host and is_ip_type(self.host, network=False):\n return make_netloc(self.host, self.port)\n return None\n\n @property\n def host_stem(self):\n \"\"\"\n An abbreviated representation of hostname that removes the TLD\n E.g. www.evilcorp.com --> www.evilcorp\n \"\"\"\n if self.host and type(self.host) == str:\n return domain_stem(self.host)\n else:\n return f\"{self.host}\"\n\n @property\n def discovery_context(self):\n return self._discovery_context\n\n @discovery_context.setter\n def discovery_context(self, context):\n def replace(match):\n s = match.group()\n return s.format(module=self.module, event=self)\n\n try:\n self._discovery_context = self._discovery_context_regex.sub(replace, context)\n except Exception as e:\n log.trace(f\"Error formatting discovery context for {self}: {e} (context: '{context}')\")\n self._discovery_context = context\n\n @property\n def discovery_path(self):\n \"\"\"\n This event's full discovery context, including those of all its parents\n \"\"\"\n discovery_path = []\n if self.parent is not None and self.parent is not self:\n discovery_path = self.parent.discovery_path\n return discovery_path + [self.discovery_context]\n\n @property\n def parent_chain(self):\n \"\"\"\n This event's full discovery context, including those of all its parents\n \"\"\"\n parent_chain = []\n if self.parent is not None and self.parent is not self:\n parent_chain = self.parent.parent_chain\n return parent_chain + [str(self.uuid)]\n\n @property\n def words(self):\n if self.__words is None:\n self.__words = set(self._words())\n return self.__words\n\n def _words(self):\n return set()\n\n @property\n def tags(self):\n return self._tags\n\n @tags.setter\n def tags(self, tags):\n self._tags = set()\n if isinstance(tags, str):\n tags = (tags,)\n for tag in tags:\n self.add_tag(tag)\n\n def add_tag(self, tag):\n self._tags.add(tagify(tag))\n\n def add_tags(self, tags):\n for tag in set(tags):\n self.add_tag(tag)\n\n def remove_tag(self, tag):\n with suppress(KeyError):\n self._tags.remove(tagify(tag))\n\n @property\n def always_emit(self):\n \"\"\"\n If this returns True, the event will always be distributed to output modules regardless of scope distance\n \"\"\"\n always_emit_tags = any(t in self.tags for t in self._always_emit_tags)\n no_host_information = not bool(self.host)\n return self._always_emit or always_emit_tags or no_host_information\n\n @property\n def id(self):\n \"\"\"\n A uniquely identifiable hash of the event from the event type + a SHA1 of its data\n \"\"\"\n if self._id is None:\n self._id = f\"{self.type}:{self.data_hash.hex()}\"\n return self._id\n\n @property\n def uuid(self):\n \"\"\"\n A universally unique identifier for the event\n \"\"\"\n return f\"{self.type}:{self._uuid}\"\n\n @property\n def data_hash(self):\n \"\"\"\n A raw byte hash of the event's data\n \"\"\"\n if self._data_hash is None:\n self._data_hash = sha1(self.data_id).digest()\n return self._data_hash\n\n @property\n def scope_distance(self):\n return self._scope_distance\n\n @scope_distance.setter\n def scope_distance(self, scope_distance):\n \"\"\"\n Setter for the scope_distance attribute, ensuring it only decreases.\n\n The scope_distance attribute is designed to never increase; it can only be set to smaller values than\n the current one. If a larger value is provided, it is ignored. The setter also updates the event's\n tags to reflect the new scope distance.\n\n Parameters:\n scope_distance (int): The new scope distance to set, must be a non-negative integer.\n\n Note:\n The method will automatically update the relevant 'distance-' tags associated with the event.\n \"\"\"\n if scope_distance < 0:\n raise ValueError(f\"Invalid scope distance: {scope_distance}\")\n # ensure scope distance does not increase (only allow setting to smaller values)\n if self.scope_distance is None:\n new_scope_distance = scope_distance\n else:\n new_scope_distance = min(self.scope_distance, scope_distance)\n if self._scope_distance != new_scope_distance:\n # remove old scope distance tags\n self._scope_distance = new_scope_distance\n self.refresh_scope_tags()\n # apply recursively to parent events\n parent_scope_distance = getattr(self.parent, \"scope_distance\", None)\n if parent_scope_distance is not None and self.parent is not self:\n self.parent.scope_distance = new_scope_distance + 1\n\n def refresh_scope_tags(self):\n for t in list(self.tags):\n if t.startswith(\"distance-\"):\n self.remove_tag(t)\n if self.host:\n if self.scope_distance == 0:\n self.add_tag(\"in-scope\")\n self.remove_tag(\"affiliate\")\n else:\n self.remove_tag(\"in-scope\")\n self.add_tag(f\"distance-{self.scope_distance}\")\n\n @property\n def scope_description(self):\n \"\"\"\n Returns a single word describing the scope of the event.\n\n \"in-scope\" if the event is in scope, \"affiliate\" if it's an affiliate, otherwise \"distance-{scope_distance}\"\n \"\"\"\n if self.scope_distance == 0:\n return \"in-scope\"\n elif \"affiliate\" in self.tags:\n return \"affiliate\"\n return f\"distance-{self.scope_distance}\"\n\n @property\n def parent(self):\n return self._parent\n\n @parent.setter\n def parent(self, parent):\n \"\"\"\n Setter for the parent attribute, ensuring it's a valid event and updating scope distance.\n\n Sets the parent of the event and automatically adjusts the scope distance based on the parent event's\n scope distance. The scope distance is incremented by 1 if the host of the parent event is different\n from the current event's host.\n\n Parameters:\n parent (BaseEvent): The new parent event to set. Must be a valid event object.\n\n Note:\n If an invalid parent is provided and the event is not a dummy, a warning will be logged.\n \"\"\"\n if is_event(parent):\n self._parent = parent\n hosts_are_same = (self.host and parent.host) and (self.host == parent.host)\n new_scope_distance = int(parent.scope_distance)\n if self.host and parent.scope_distance is not None:\n # only increment the scope distance if the host changes\n if self._scope_distance_increment_same_host or not hosts_are_same:\n new_scope_distance += 1\n self.scope_distance = new_scope_distance\n # inherit certain tags\n if hosts_are_same:\n # inherit web spider distance from parent\n self.web_spider_distance = getattr(parent, \"web_spider_distance\", 0)\n event_has_url = getattr(self, \"parsed_url\", None) is not None\n for t in parent.tags:\n if t in (\"affiliate\",):\n self.add_tag(t)\n elif t.startswith(\"mutation-\"):\n self.add_tag(t)\n # only add these tags if the event has a URL\n if event_has_url:\n if t in (\"spider-danger\", \"spider-max\"):\n self.add_tag(t)\n elif not self._dummy:\n log.warning(f\"Tried to set invalid parent on {self}: (got: {repr(parent)} ({type(parent)}))\")\n\n @property\n def children(self):\n return []\n\n @property\n def parent_id(self):\n parent_id = getattr(self.get_parent(), \"id\", None)\n if parent_id is not None:\n return parent_id\n return self._parent_id\n\n @property\n def parent_uuid(self):\n parent_uuid = getattr(self.get_parent(), \"uuid\", None)\n if parent_uuid is not None:\n return parent_uuid\n return self._parent_uuid\n\n @property\n def validators(self):\n \"\"\"\n Depending on whether the scan attribute is accessible, return either a config-aware or non-config-aware validator\n\n This exists to prevent a chicken-and-egg scenario during the creation of certain events such as URLs,\n whose sanitization behavior is different depending on the config.\n\n However, thanks to this property, validation can still work in the absence of a config.\n \"\"\"\n if self.scan is not None:\n return self.scan.helpers.config_aware_validators\n return validators\n\n def get_parent(self):\n \"\"\"\n Takes into account events with the _omit flag\n \"\"\"\n if getattr(self.parent, \"_omit\", False):\n return self.parent.get_parent()\n return self.parent\n\n def get_parents(self, omit=False, include_self=False):\n parents = []\n e = self\n if include_self:\n parents.append(self)\n while 1:\n if omit:\n parent = e.get_parent()\n else:\n parent = e.parent\n if parent is None:\n break\n if e == parent:\n break\n parents.append(parent)\n e = parent\n return parents\n\n def clone(self):\n # Create a shallow copy of the event first\n cloned_event = copy(self)\n # Re-assign a new UUID\n cloned_event._uuid = uuid.uuid4()\n return cloned_event\n\n def _host(self):\n return \"\"\n\n def _sanitize_data(self, data):\n \"\"\"\n Validates and sanitizes the event's data during instantiation.\n\n By default, uses the '_data_load' method to pre-process the data and then applies the '_data_validator'\n to validate and create a sanitized dictionary. Raises a ValidationError if any of the validations fail.\n Subclasses can override this method to provide custom validation logic.\n\n Returns:\n Any: The sanitized data.\n\n Raises:\n ValidationError: If the data fails to validate.\n \"\"\"\n data = self._data_load(data)\n if self._data_validator is not None:\n if not isinstance(data, dict):\n raise ValidationError(f\"data is not of type dict: {data}\")\n data = self._data_validator(**data).model_dump(exclude_none=True)\n return self.sanitize_data(data)\n\n def sanitize_data(self, data):\n return data\n\n @property\n def data_human(self):\n \"\"\"\n Human representation of event.data\n \"\"\"\n return self._data_human()\n\n def _data_human(self):\n if isinstance(self.data, (dict, list)):\n with suppress(Exception):\n return json.dumps(self.data, sort_keys=True)\n return smart_decode(self.data)\n\n def _data_load(self, data):\n \"\"\"\n How to load the event data (JSON-decode it, etc.)\n \"\"\"\n return data\n\n @property\n def data_id(self):\n \"\"\"\n Representation of the event.data used to calculate the event's ID\n \"\"\"\n return self._data_id()\n\n def _data_id(self):\n return self.data\n\n @property\n def pretty_string(self):\n \"\"\"\n A human-friendly representation of the event's data. Used for graph representation.\n\n If the event's data is a dictionary, the function will try to return a JSON-formatted string.\n Otherwise, it will use smart_decode to convert the data into a string representation.\n\n Override if necessary.\n\n Returns:\n str: The graphical representation of the event's data.\n \"\"\"\n return self._pretty_string()\n\n def _pretty_string(self):\n return self._data_human()\n\n @property\n def data_graph(self):\n \"\"\"\n Representation of event.data for neo4j graph nodes\n \"\"\"\n return self.pretty_string\n\n @property\n def data_json(self):\n \"\"\"\n JSON representation of event.data\n \"\"\"\n return self.data\n\n def __contains__(self, other):\n \"\"\"\n Membership checks for Events.\n\n Supports:\n - some_event in other_event (event vs event)\n - \"host:port\" in other_event (string coerced to an event)\n \"\"\"\n # Fast path: already an Event\n if is_event(other):\n other_event = other\n else:\n try:\n other_event = make_event(other, dummy=True)\n except ValidationError:\n return False\n\n # if hashes match\n if other_event == self:\n return True\n # if hosts match (including subnet / domain containment)\n if self.host and other_event.host:\n if self.host == other_event.host:\n return True\n # hostnames and IPs\n radixtarget = RadixTarget()\n radixtarget.insert(self.host)\n return bool(radixtarget.search(other_event.host))\n return False\n\n def json(self, mode=\"json\", siem_friendly=False):\n \"\"\"\n Serializes the event object to a JSON-compatible dictionary.\n\n By default, it includes attributes such as 'type', 'id', 'data', 'scope_distance', and others that are present.\n Additional specific attributes can be serialized based on the mode specified.\n\n Parameters:\n mode (str): Specifies the data serialization mode. Default is \"json\". Other options include \"graph\", \"human\", and \"id\".\n siem_friendly (bool): Whether to format the JSON in a way that's friendly to SIEM ingestion by Elastic, Splunk, etc. This ensures the value of \"data\" is always the same type (a dictionary).\n\n Returns:\n dict: JSON-serializable dictionary representation of the event object.\n \"\"\"\n j = {}\n # type, ID, scope description\n for i in (\"type\", \"id\", \"uuid\", \"scope_description\", \"netloc\"):\n v = getattr(self, i, \"\")\n if v:\n j.update({i: str(v)})\n # event data\n data_attr = getattr(self, f\"data_{mode}\", None)\n if data_attr is not None:\n data = data_attr\n else:\n data = smart_decode(self.data)\n if siem_friendly:\n j[\"data\"] = {self.type: data}\n else:\n j[\"data\"] = data\n # host, dns children\n if self.host:\n j[\"host\"] = str(self.host)\n j[\"resolved_hosts\"] = sorted(str(h) for h in self.resolved_hosts)\n j[\"dns_children\"] = {k: list(v) for k, v in self.dns_children.items()}\n if isinstance(self.port, int):\n j[\"port\"] = self.port\n # web spider distance\n web_spider_distance = getattr(self, \"web_spider_distance\", None)\n if web_spider_distance is not None:\n j[\"web_spider_distance\"] = web_spider_distance\n # scope distance\n j[\"scope_distance\"] = self.scope_distance\n # scan\n if self.scan:\n j[\"scan\"] = self.scan.id\n # timestamp\n j[\"timestamp\"] = self.timestamp.isoformat()\n # parent event\n parent_id = self.parent_id\n if parent_id:\n j[\"parent\"] = parent_id\n parent_uuid = self.parent_uuid\n if parent_uuid:\n j[\"parent_uuid\"] = parent_uuid\n # tags\n if self.tags:\n j.update({\"tags\": list(self.tags)})\n # parent module\n if self.module:\n j.update({\"module\": str(self.module)})\n # sequence of modules that led to discovery\n if self.module_sequence:\n j.update({\"module_sequence\": str(self.module_sequence)})\n # discovery context\n j[\"discovery_context\"] = self.discovery_context\n j[\"discovery_path\"] = self.discovery_path\n j[\"parent_chain\"] = self.parent_chain\n\n # parameter envelopes\n parameter_envelopes = getattr(self, \"envelopes\", None)\n if parameter_envelopes is not None:\n j[\"envelopes\"] = parameter_envelopes.to_dict()\n\n # normalize non-primitive python objects\n\n for k, v in list(j.items()):\n if k == \"data\":\n continue\n if type(v) not in (str, int, float, bool, list, dict, type(None)):\n try:\n j[k] = json.dumps(v, sort_keys=True)\n except Exception:\n j[k] = smart_decode(v)\n return j\n\n @staticmethod\n def from_json(j):\n \"\"\"\n Convenience shortcut to create an Event object from a JSON-compatible dictionary.\n\n Calls the `event_from_json()` function to deserialize the event.\n\n Parameters:\n j (dict): The JSON-compatible dictionary containing event data.\n\n Returns:\n Event: The deserialized Event object.\n \"\"\"\n return event_from_json(j)\n\n @property\n def module_sequence(self):\n \"\"\"\n Get a human-friendly string that represents the sequence of modules responsible for generating this event.\n\n Includes the names of omitted parent events to provide a complete view of the module sequence leading to this event.\n\n Returns:\n str: The module sequence in human-friendly format.\n \"\"\"\n module_name = getattr(self.module, \"name\", \"\")\n if getattr(self.parent, \"_omit\", False):\n module_name = f\"{self.parent.module_sequence}->{module_name}\"\n return module_name\n\n @property\n def module_priority(self):\n if self._module_priority is None:\n module = getattr(self, \"module\", None)\n self._module_priority = int(max(1, min(5, getattr(module, \"priority\", 3))))\n return self._module_priority\n\n @module_priority.setter\n def module_priority(self, priority):\n self._module_priority = int(max(1, min(5, priority)))\n\n @property\n def priority(self):\n if self._priority is None:\n timestamp = self.timestamp.timestamp()\n if self.parent.timestamp == self.timestamp:\n self._priority = (timestamp,)\n else:\n self._priority = getattr(self.parent, \"priority\", ()) + (timestamp,)\n\n return self._priority\n\n @property\n def type(self):\n return self._type\n\n @type.setter\n def type(self, val):\n self._type = val\n self._hash = None\n self._id = None\n\n @property\n def _host_size(self):\n \"\"\"\n Used for sorting events by their host size, so that parent ones (e.g. IP subnets) come first\n \"\"\"\n if self.host:\n if isinstance(self.host, str):\n # smaller domains should come first\n return len(self.host)\n else:\n try:\n # bigger IP subnets should come first\n return -self.host.num_addresses\n except AttributeError:\n # IP addresses default to 1\n return 1\n return 0\n\n def __iter__(self):\n \"\"\"\n For dict(event)\n \"\"\"\n yield from self.json().items()\n\n def __lt__(self, other):\n \"\"\"\n For queue sorting\n \"\"\"\n return self.priority < getattr(other, \"priority\", (0,))\n\n def __gt__(self, other):\n \"\"\"\n For queue sorting\n \"\"\"\n return self.priority > getattr(other, \"priority\", (0,))\n\n def __eq__(self, other):\n \"\"\"\n Event equality is **only** defined between Event instances.\n\n Equality is based on the event hash (derived from its id). Comparisons to\n non-Event types raise a ValueError to make incorrect comparisons explicit.\n \"\"\"\n if not is_event(other):\n raise ValueError(\"Event equality is only defined between Event instances\")\n return hash(self) == hash(other)\n\n def __hash__(self):\n if self._hash is None:\n self._hash = hash(self.id)\n return self._hash\n\n def __str__(self):\n max_event_len = 80\n d = str(self.data).replace(\"\\n\", \"\\\\n\")\n return f'{self.type}(\"{d[:max_event_len]}{(\"...\" if len(d) > max_event_len else \"\")}\", module={self.module}, tags={self.tags})'\n\n def __repr__(self):\n return str(self)\n\n\nclass SCAN(BaseEvent):\n def _data_human(self):\n return f\"{self.data['name']} ({self.data['id']})\"\n\n @property\n def discovery_path(self):\n return []\n\n @property\n def parent_chain(self):\n return []\n\n\nclass FINISHED(BaseEvent):\n \"\"\"\n Special signal event to indicate end of scan\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self._priority = (999999999999999,)\n\n\nclass DefaultEvent(BaseEvent):\n def sanitize_data(self, data):\n return data\n\n\nclass DictEvent(BaseEvent):\n def sanitize_data(self, data):\n url = data.get(\"url\", \"\")\n if url:\n self.parsed_url = self.validators.validate_url_parsed(url)\n return data\n\n def _data_load(self, data):\n if isinstance(data, str):\n return json.loads(data)\n return data\n\n\nclass DictHostEvent(DictEvent):\n def _host(self):\n if isinstance(self.data, dict) and \"host\" in self.data:\n return make_ip_type(self.data[\"host\"])\n else:\n parsed = getattr(self, \"parsed_url\", None)\n if parsed is not None:\n return make_ip_type(parsed.hostname)\n\n\nclass ClosestHostEvent(DictHostEvent):\n # if a host/path/url isn't specified, this event type grabs it from the closest parent\n # inherited by FINDING and VULNERABILITY\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n if not self.host:\n for parent in self.get_parents(include_self=True):\n # inherit closest URL\n if \"url\" not in self.data:\n parent_url = getattr(parent, \"parsed_url\", None)\n if parent_url is not None:\n self.data[\"url\"] = parent_url.geturl()\n # inherit closest path\n if \"path\" not in self.data and isinstance(parent.data, dict) and not parent.type == \"HTTP_RESPONSE\":\n parent_path = parent.data.get(\"path\", None)\n if parent_path is not None:\n self.data[\"path\"] = parent_path\n # inherit closest host\n if parent.host:\n self.data[\"host\"] = str(parent.host)\n # we do this to refresh the hash\n self.data = self.data\n break\n # die if we still haven't found a host\n if not self.host and not self.data.get(\"path\", \"\"):\n raise ValueError(f\"No host was found in event parents: {self.get_parents()}. Host must be specified!\")\n\n\nclass DictPathEvent(DictEvent):\n def sanitize_data(self, data):\n new_data = dict(data)\n new_data[\"path\"] = str(new_data[\"path\"])\n file_blobs = getattr(self.scan, \"_file_blobs\", False)\n folder_blobs = getattr(self.scan, \"_folder_blobs\", False)\n blob = None\n try:\n self._data_path = Path(data[\"path\"])\n # prepend the scan's home dir if the path is relative\n if not self._data_path.is_absolute():\n self._data_path = self.scan.home / self._data_path\n if self._data_path.is_file():\n self.add_tag(\"file\")\n if file_blobs:\n with open(self._data_path, \"rb\") as file:\n blob = file.read()\n elif self._data_path.is_dir():\n self.add_tag(\"folder\")\n if folder_blobs:\n blob = self._tar_directory(self._data_path)\n except KeyError:\n pass\n if blob:\n new_data[\"blob\"] = base64.b64encode(blob).decode(\"utf-8\")\n\n return new_data\n\n def _tar_directory(self, dir_path):\n tar_buffer = io.BytesIO()\n with tarfile.open(fileobj=tar_buffer, mode=\"w:gz\") as tar:\n # Add the entire directory to the tar archive\n tar.add(dir_path, arcname=dir_path.name)\n return tar_buffer.getvalue()\n\n\nclass ASN(DictEvent):\n _always_emit = True\n _quick_emit = True\n\n\nclass CODE_REPOSITORY(DictHostEvent):\n _always_emit = True\n\n class _data_validator(BaseModel):\n url: str\n _validate_url = field_validator(\"url\")(validators.validate_url)\n\n def _pretty_string(self):\n return self.data[\"url\"]\n\n\nclass IP_ADDRESS(BaseEvent):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n ip = ipaddress.ip_address(self.data)\n self.add_tag(f\"ipv{ip.version}\")\n if ip.is_private:\n self.add_tag(\"private-ip\")\n self.dns_resolve_distance = getattr(self.parent, \"dns_resolve_distance\", 0)\n\n def sanitize_data(self, data):\n return validators.validate_host(data)\n\n def _host(self):\n return ipaddress.ip_address(self.data)\n\n\nclass DnsEvent(BaseEvent):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n # prevent runaway DNS entries\n self.dns_resolve_distance = 0\n parent = getattr(self, \"parent\", None)\n module = getattr(self, \"module\", None)\n module_type = getattr(module, \"_type\", \"\")\n parent_module = getattr(parent, \"module\", None)\n parent_module_type = getattr(parent_module, \"_type\", \"\")\n if module_type == \"DNS\":\n self.dns_resolve_distance = getattr(parent, \"dns_resolve_distance\", 0)\n if parent_module_type == \"DNS\":\n self.dns_resolve_distance += 1\n # self.add_tag(f\"resolve-distance-{self.dns_resolve_distance}\")\n # tag subdomain / domain\n if is_subdomain(self.host):\n self.add_tag(\"subdomain\")\n elif is_domain(self.host):\n self.add_tag(\"domain\")\n # tag private IP\n try:\n if self.host.is_private:\n self.add_tag(\"private-ip\")\n except AttributeError:\n pass\n\n\nclass IP_RANGE(DnsEvent):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self.add_tag(f\"ipv{self.host.version}\")\n\n def sanitize_data(self, data):\n return str(ipaddress.ip_network(str(data), strict=False))\n\n def _host(self):\n return ipaddress.ip_network(self.data)\n\n\nclass DNS_NAME(DnsEvent):\n def sanitize_data(self, data):\n return validators.validate_host(data)\n\n def _host(self):\n return self.data\n\n def _words(self):\n stem = self.host_stem\n if not is_ptr(stem):\n split_stem = stem.split(\".\")\n if split_stem:\n leftmost_segment = split_stem[0]\n if leftmost_segment == \"_wildcard\":\n stem = \".\".join(split_stem[1:])\n if stem:\n return extract_words(stem)\n return set()\n\n\nclass OPEN_TCP_PORT(BaseEvent):\n def sanitize_data(self, data):\n return validators.validate_open_port(data)\n\n def _host(self):\n host, self._port = split_host_port(self.data)\n return host\n\n def _words(self):\n if not is_ip(self.host) and not is_ptr(self.host):\n return extract_words(self.host_stem)\n return set()\n\n\nclass OPEN_UDP_PORT(OPEN_TCP_PORT):\n pass\n\n\nclass URL_UNVERIFIED(BaseEvent):\n _status_code_regex = re.compile(r\"^status-(\\d{1,3})$\")\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self.num_redirects = getattr(self.parent, \"num_redirects\", 0)\n\n def _data_id(self):\n data = super()._data_id()\n\n # remove the querystring for URL/URL_UNVERIFIED events, because we will conditionally add it back in (based on settings)\n if self.__class__.__name__.startswith(\"URL\") and self.scan is not None:\n prefix = data.split(\"?\")[0]\n\n # consider spider-danger tag when deduping\n if \"spider-danger\" in self.tags:\n prefix += \"spider-danger\"\n\n if not self.scan.config.get(\"url_querystring_remove\", True) and self.parsed_url.query:\n query_dict = parse_qs(self.parsed_url.query)\n if self.scan.config.get(\"url_querystring_collapse\", True):\n # Only consider parameter names in dedup (collapse values)\n cleaned_query = \"|\".join(sorted(query_dict.keys()))\n else:\n # Consider parameter names and values in dedup\n cleaned_query = \"&\".join(\n f\"{key}={','.join(sorted(values))}\" for key, values in sorted(query_dict.items())\n )\n data = f\"{prefix}:{self.parsed_url.scheme}:{self.parsed_url.netloc}:{self.parsed_url.path}:{cleaned_query}\"\n return data\n\n def sanitize_data(self, data):\n self.parsed_url = self.validators.validate_url_parsed(data)\n\n # special handling of URL extensions\n if self.parsed_url is not None:\n url_path = self.parsed_url.path\n if url_path:\n parsed_path_lower = str(url_path).lower()\n extension = get_file_extension(parsed_path_lower)\n if extension:\n self.url_extension = extension\n self.add_tag(f\"extension-{extension}\")\n\n # tag as dir or endpoint\n if str(self.parsed_url.path).endswith(\"/\"):\n self.add_tag(\"dir\")\n else:\n self.add_tag(\"endpoint\")\n\n data = self.parsed_url.geturl()\n return data\n\n def add_tag(self, tag):\n self_url = getattr(self, \"parsed_url\", \"\")\n self_host = getattr(self, \"host\", \"\")\n # autoincrement web spider distance if the \"spider-danger\" tag is added\n if tag == \"spider-danger\" and \"spider-danger\" not in self.tags and self_url and self_host:\n parent_hosts_and_urls = set()\n for p in self.get_parents():\n # URL_UNVERIFIED events don't count because they haven't been visited yet\n if p.type == \"URL_UNVERIFIED\":\n continue\n url = getattr(p, \"parsed_url\", \"\")\n parent_hosts_and_urls.add((p.host, url))\n # if there's a URL anywhere in our parent chain that's different from ours but shares our host, we're in dAnGeR\n dangerous_parent = any(\n p_host == self.host and p_url != self_url for p_host, p_url in parent_hosts_and_urls\n )\n if dangerous_parent:\n # increment the web spider distance\n if self.type == \"URL_UNVERIFIED\":\n self.web_spider_distance += 1\n if self.is_spider_max:\n self.add_tag(\"spider-max\")\n super().add_tag(tag)\n\n @property\n def is_spider_max(self):\n if self.scan:\n depth = url_depth(self.parsed_url)\n if (self.web_spider_distance > self.scan.web_spider_distance) or (depth > self.scan.web_spider_depth):\n return True\n return False\n\n def with_port(self):\n netloc_with_port = make_netloc(self.host, self.port)\n return self.parsed_url._replace(netloc=netloc_with_port)\n\n def _words(self):\n first_elem = self.parsed_url.path.lstrip(\"/\").split(\"/\")[0]\n if \".\" not in first_elem:\n return extract_words(first_elem)\n return set()\n\n def _host(self):\n return make_ip_type(self.parsed_url.hostname)\n\n @property\n def http_status(self):\n for t in self.tags:\n match = self._status_code_regex.match(t)\n if match:\n return int(match.groups()[0])\n return 0\n\n\nclass URL(URL_UNVERIFIED):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n\n if not self._dummy and not any(t.startswith(\"status-\") for t in self.tags):\n raise ValidationError(\n 'Must specify HTTP status tag for URL event, e.g. \"status-200\". Use URL_UNVERIFIED if the URL is unvisited.'\n )\n\n @property\n def resolved_hosts(self):\n # TODO: remove this when we rip out httpx\n return {\".\".join(i.split(\"-\")[1:]) for i in self.tags if i.startswith(\"ip-\")}\n\n @property\n def pretty_string(self):\n return self.data\n\n\nclass STORAGE_BUCKET(DictEvent, URL_UNVERIFIED):\n _always_emit = True\n _suppress_chain_dupes = True\n\n class _data_validator(BaseModel):\n name: str\n url: str\n _validate_url = field_validator(\"url\")(validators.validate_url)\n\n def sanitize_data(self, data):\n data = super().sanitize_data(data)\n data[\"name\"] = data[\"name\"].lower()\n return data\n\n def _words(self):\n return self.data[\"name\"]\n\n\nclass URL_HINT(URL_UNVERIFIED):\n pass\n\n\nclass WEB_PARAMETER(DictHostEvent):\n @property\n def children(self):\n # if we have any subparams, raise a new WEB_PARAMETER for each one\n children = []\n envelopes = getattr(self, \"envelopes\", None)\n if envelopes is not None:\n subparams = sorted(list(self.envelopes.get_subparams()))\n\n if envelopes.selected_subparam is None:\n current_subparam = subparams[0]\n envelopes.selected_subparam = current_subparam[0]\n if len(subparams) > 1:\n for subparam, _ in subparams[1:]:\n clone = self.clone()\n clone.envelopes = deepcopy(envelopes)\n clone.envelopes.selected_subparam = subparam\n clone.parent = self\n children.append(clone)\n return children\n\n def sanitize_data(self, data):\n original_value = data.get(\"original_value\", None)\n if original_value is not None:\n try:\n envelopes = BaseEnvelope.detect(original_value)\n setattr(self, \"envelopes\", envelopes)\n except ValueError as e:\n log.verbose(f\"Error detecting envelopes for {self}: {e}\")\n return data\n\n def _data_id(self):\n # dedupe by url:name:param_type\n url = self.data.get(\"url\", \"\")\n name = self.data.get(\"name\", \"\")\n param_type = self.data.get(\"type\", \"\")\n envelopes = getattr(self, \"envelopes\", \"\")\n subparam = getattr(envelopes, \"selected_subparam\", \"\")\n\n return f\"{url}:{name}:{param_type}:{subparam}\"\n\n def _outgoing_dedup_hash(self, event):\n return hash(\n (\n str(event.host),\n event.data[\"url\"],\n event.data.get(\"name\", \"\"),\n event.data.get(\"type\", \"\"),\n event.data.get(\"envelopes\", \"\"),\n )\n )\n\n def _url(self):\n return self.data[\"url\"]\n\n def __str__(self):\n max_event_len = 200\n d = str(self.data)\n return f'{self.type}(\"{d[:max_event_len]}{(\"...\" if len(d) > max_event_len else \"\")}\", module={self.module}, tags={self.tags})'\n\n\nclass EMAIL_ADDRESS(BaseEvent):\n def sanitize_data(self, data):\n return validators.validate_email(data)\n\n def _host(self):\n data = str(self.data).rsplit(\"@\", 1)[-1]\n host, self._port = split_host_port(data)\n return host\n\n def _words(self):\n return extract_words(self.host_stem)\n\n\nclass HTTP_RESPONSE(URL_UNVERIFIED, DictEvent):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n # count number of consecutive redirects\n self.num_redirects = getattr(self.parent, \"num_redirects\", 0)\n if str(self.http_status).startswith(\"3\"):\n self.num_redirects += 1\n\n def _data_id(self):\n return self.data[\"method\"] + \"|\" + self.data[\"url\"]\n\n def sanitize_data(self, data):\n url = data.get(\"url\", \"\")\n self.parsed_url = self.validators.validate_url_parsed(url)\n data[\"url\"] = self.parsed_url.geturl()\n\n if not \"raw_header\" in data:\n raise ValueError(\"raw_header is required for HTTP_RESPONSE events\")\n\n if \"header-dict\" not in data:\n header_dict = {}\n for i in data.get(\"raw_header\", \"\").splitlines():\n if len(i) > 0 and \":\" in i:\n k, v = i.split(\":\", 1)\n k = k.strip().lower()\n v = v.lstrip()\n if k in header_dict:\n header_dict[k].append(v)\n else:\n header_dict[k] = [v]\n data[\"header-dict\"] = header_dict\n\n # move URL to the front of the dictionary for visibility\n data = dict(data)\n new_data = {\"url\": data.pop(\"url\")}\n new_data.update(data)\n\n return new_data\n\n def _words(self):\n return set()\n\n def _pretty_string(self):\n return f\"{self.data['hash']['header_mmh3']}:{self.data['hash']['body_mmh3']}\"\n\n @property\n def raw_response(self):\n \"\"\"\n Formats the status code, headers, and body into a single string formatted as an HTTP/1.1 response.\n \"\"\"\n raw_header = self.data.get(\"raw_header\", \"\")\n body = self.data.get(\"body\", \"\")\n return f\"{raw_header}{body}\"\n\n @property\n def http_status(self):\n try:\n return int(self.data.get(\"status_code\", 0))\n except (ValueError, TypeError):\n return 0\n\n @property\n def http_title(self):\n http_title = self.data.get(\"title\", \"\")\n try:\n return recursive_decode(http_title)\n except Exception:\n return http_title\n\n @property\n def redirect_location(self):\n location = self.data.get(\"location\", \"\")\n # if it's a redirect\n if location:\n # get the url scheme\n scheme = is_uri(location, return_scheme=True)\n # if there's no scheme (i.e. it's a relative redirect)\n if not scheme:\n # then join the location with the current url\n location = urljoin(self.parsed_url.geturl(), location)\n return location\n\n\nclass VULNERABILITY(ClosestHostEvent):\n _always_emit = True\n _quick_emit = True\n severity_colors = {\n \"CRITICAL\": \"đŸŸĒ\",\n \"HIGH\": \"đŸŸĨ\",\n \"MEDIUM\": \"🟧\",\n \"LOW\": \"🟨\",\n \"UNKNOWN\": \"âŦœ\",\n }\n\n def sanitize_data(self, data):\n self.add_tag(data[\"severity\"].lower())\n return data\n\n class _data_validator(BaseModel):\n host: Optional[str] = None\n severity: str\n description: str\n url: Optional[str] = None\n path: Optional[str] = None\n _validate_url = field_validator(\"url\")(validators.validate_url)\n _validate_host = field_validator(\"host\")(validators.validate_host)\n _validate_severity = field_validator(\"severity\")(validators.validate_severity)\n\n def _pretty_string(self):\n return f\"[{self.data['severity']}] {self.data['description']}\"\n\n\nclass FINDING(ClosestHostEvent):\n _always_emit = True\n _quick_emit = True\n\n class _data_validator(BaseModel):\n host: Optional[str] = None\n description: str\n url: Optional[str] = None\n path: Optional[str] = None\n _validate_url = field_validator(\"url\")(validators.validate_url)\n _validate_host = field_validator(\"host\")(validators.validate_host)\n\n def _pretty_string(self):\n return self.data[\"description\"]\n\n\nclass TECHNOLOGY(DictHostEvent):\n class _data_validator(BaseModel):\n host: str\n technology: str\n url: Optional[str] = None\n _validate_url = field_validator(\"url\")(validators.validate_url)\n _validate_host = field_validator(\"host\")(validators.validate_host)\n\n def _data_id(self):\n # dedupe by host+port+tech\n tech = self.data.get(\"technology\", \"\")\n return f\"{self.host}:{self.port}:{tech}\"\n\n def _pretty_string(self):\n return self.data[\"technology\"]\n\n\nclass VHOST(DictHostEvent):\n class _data_validator(BaseModel):\n host: str\n vhost: str\n url: Optional[str] = None\n _validate_url = field_validator(\"url\")(validators.validate_url)\n _validate_host = field_validator(\"host\")(validators.validate_host)\n\n def _pretty_string(self):\n return self.data[\"vhost\"]\n\n\nclass PROTOCOL(DictHostEvent):\n class _data_validator(BaseModel):\n host: str\n protocol: str\n port: Optional[int] = None\n banner: Optional[str] = None\n _validate_host = field_validator(\"host\")(validators.validate_host)\n _validate_port = field_validator(\"port\")(validators.validate_port)\n\n def sanitize_data(self, data):\n new_data = dict(data)\n new_data[\"protocol\"] = data.get(\"protocol\", \"\").upper()\n return new_data\n\n @property\n def port(self):\n return self.data.get(\"port\", None)\n\n def _pretty_string(self):\n return self.data[\"protocol\"]\n\n\nclass GEOLOCATION(BaseEvent):\n _always_emit = True\n _quick_emit = True\n\n\nclass PASSWORD(BaseEvent):\n _always_emit = True\n _quick_emit = True\n\n\nclass HASHED_PASSWORD(BaseEvent):\n _always_emit = True\n _quick_emit = True\n\n\nclass USERNAME(BaseEvent):\n _always_emit = True\n _quick_emit = True\n\n\nclass SOCIAL(DictHostEvent):\n _always_emit = True\n _quick_emit = True\n _scope_distance_increment_same_host = True\n\n\nclass WEBSCREENSHOT(DictPathEvent, DictHostEvent):\n _always_emit = True\n _quick_emit = True\n\n\nclass AZURE_TENANT(DictEvent):\n _always_emit = True\n _quick_emit = True\n\n\nclass WAF(DictHostEvent):\n _always_emit = True\n _quick_emit = True\n\n class _data_validator(BaseModel):\n url: str\n host: str\n waf: str\n info: Optional[str] = None\n _validate_url = field_validator(\"url\")(validators.validate_url)\n _validate_host = field_validator(\"host\")(validators.validate_host)\n\n def _pretty_string(self):\n return self.data[\"waf\"]\n\n\nclass FILESYSTEM(DictPathEvent):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n if self._data_path.is_file():\n # detect type of file content using magic\n from bbot.core.helpers.libmagic import get_magic_info, get_compression\n\n try:\n extension, mime_type, description, confidence = get_magic_info(self.data[\"path\"])\n self.data[\"magic_extension\"] = extension\n self.data[\"magic_mime_type\"] = mime_type\n self.data[\"magic_description\"] = description\n self.data[\"magic_confidence\"] = confidence\n # detection compression\n compression = get_compression(mime_type)\n if compression:\n self.add_tag(\"compressed\")\n self.add_tag(f\"{compression}-archive\")\n self.data[\"compression\"] = compression\n # refresh hash\n self.data = self.data\n except Exception as e:\n log.debug(f\"Error detecting file type: {type(e).__name__}: {e}\")\n\n\nclass RAW_DNS_RECORD(DictHostEvent, DnsEvent):\n # don't emit raw DNS records for affiliates\n _always_emit_tags = [\"target\"]\n\n\nclass MOBILE_APP(DictEvent):\n _always_emit = True\n\n def _sanitize_data(self, data):\n if isinstance(data, str):\n data = {\"url\": data}\n if \"url\" not in data:\n raise ValidationError(\"url is required for MOBILE_APP events\")\n url = data[\"url\"]\n # parse URL\n try:\n self.parsed_url = urlparse(url)\n except Exception as e:\n raise ValidationError(f\"Error parsing URL {url}: {e}\")\n if not \"id\" in data:\n # extract \"id\" getparam\n params = parse_qs(self.parsed_url.query)\n try:\n _id = params[\"id\"][0]\n except Exception:\n raise ValidationError(\"id is required for MOBILE_APP events\")\n data[\"id\"] = _id\n return data\n\n def _pretty_string(self):\n return self.data[\"url\"]\n\n\ndef update_event(\n event,\n parent=None,\n context=None,\n module=None,\n scan=None,\n tags=None,\n internal=None,\n):\n \"\"\"\n Updates an existing event object with additional metadata.\n\n Parameters:\n event (BaseEvent): The event object to update.\n parent (BaseEvent, optional): New parent event.\n context (str, optional): Discovery context to set.\n module (str or BaseModule, optional): Module that discovered the event.\n scan (Scan, optional): BBOT Scan object associated with the event.\n tags (Union[str, List[str]], optional): Tags to merge into the event.\n internal (Any, optional): Marks the event as internal if True.\n\n Returns:\n BaseEvent: The updated event object.\n \"\"\"\n if not is_event(event):\n raise ValidationError(f\"update_event() expects an Event, got {type(event)}\")\n\n # allow tags to be either a string or an array\n if not tags:\n tags = []\n elif isinstance(tags, str):\n tags = [tags]\n tags = set(tags)\n\n if scan is not None and not event.scan:\n event.scan = scan\n if module is not None:\n event.module = module\n if parent is not None:\n event.parent = parent\n if context is not None:\n event.discovery_context = context\n if internal is True:\n event.internal = True\n if tags:\n event.tags = tags.union(event.tags)\n return event\n\n\ndef make_event(\n data,\n event_type=None,\n parent=None,\n context=None,\n module=None,\n scan=None,\n tags=None,\n confidence=100,\n dummy=False,\n internal=None,\n):\n \"\"\"\n Creates and returns a new event object.\n\n This function serves as a factory for creating new event objects from raw data.\n If you need to modify an existing event, use ``update_event()`` instead.\n\n Parameters:\n data (Union[str, dict]): The primary data for the event.\n event_type (str, optional): Type of the event, e.g., 'IP_ADDRESS'. Auto-detected if not provided.\n parent (BaseEvent, optional): Parent event leading to this event's discovery.\n context (str, optional): Description of circumstances leading to event's discovery.\n module (str, optional): Module that discovered the event.\n scan (Scan, optional): BBOT Scan object associated with the event.\n scans (List[Scan], optional): Multiple BBOT Scan objects, primarily used for unserialization.\n tags (Union[str, List[str]], optional): Descriptive tags for the event, as a list or a single string.\n confidence (int, optional): Confidence level for the event, on a scale of 1-100. Defaults to 100.\n dummy (bool, optional): Disables data validations if set to True. Defaults to False.\n internal (Any, optional): Makes the event internal if set to True. Defaults to None.\n\n Returns:\n BaseEvent: A new event object.\n\n Raises:\n ValidationError: Raised when there's an error in event data or type sanitization.\n \"\"\"\n if not data:\n raise ValidationError(\"No data provided\")\n\n # do not allow passing an existing event here – use update_event() instead\n if is_event(data):\n raise ValidationError(\n \"make_event() does not accept an existing event object. Use update_event(event, ...) to modify an event.\"\n )\n\n # allow tags to be either a string or an array\n if not tags:\n tags = []\n elif isinstance(tags, str):\n tags = [tags]\n tags = set(tags)\n\n # if event_type is not provided, autodetect it\n if event_type is None:\n event_seed = EventSeed(data)\n event_type = event_seed.type\n data = event_seed.data\n if not dummy:\n log.debug(f'Autodetected event type \"{event_type}\" based on data: \"{data}\"')\n\n event_type = str(event_type).strip().upper()\n\n # Catch these common whoopsies\n if event_type in (\"DNS_NAME\", \"IP_ADDRESS\"):\n # DNS_NAME <--> EMAIL_ADDRESS confusion\n if validators.soft_validate(data, \"email\"):\n event_type = \"EMAIL_ADDRESS\"\n else:\n # DNS_NAME <--> IP_ADDRESS confusion\n try:\n data = validators.validate_host(data)\n except Exception as e:\n log.trace(traceback.format_exc())\n raise ValidationError(f'Error sanitizing event data \"{data}\" for type \"{event_type}\": {e}')\n data_is_ip = is_ip(data)\n if event_type == \"DNS_NAME\" and data_is_ip:\n event_type = \"IP_ADDRESS\"\n elif event_type == \"IP_ADDRESS\" and not data_is_ip:\n event_type = \"DNS_NAME\"\n # USERNAME <--> EMAIL_ADDRESS confusion\n if event_type == \"USERNAME\" and validators.soft_validate(data, \"email\"):\n event_type = \"EMAIL_ADDRESS\"\n tags.add(\"affiliate\")\n # Convert single-host IP_RANGE to IP_ADDRESS\n if event_type == \"IP_RANGE\":\n with suppress(Exception):\n net = ipaddress.ip_network(data, strict=False)\n if net.prefixlen == net.max_prefixlen:\n event_type = \"IP_ADDRESS\"\n data = net.network_address\n\n event_class = globals().get(event_type, DefaultEvent)\n return event_class(\n data,\n event_type=event_type,\n parent=parent,\n context=context,\n module=module,\n scan=scan,\n tags=tags,\n confidence=confidence,\n _dummy=dummy,\n _internal=internal,\n )\n\n\ndef event_from_json(j, siem_friendly=False):\n \"\"\"\n Creates an event object from a JSON dictionary.\n\n This function deserializes a JSON dictionary to create a new event object, using the `make_event` function\n for the actual object creation. It sets additional attributes such as the timestamp and scope distance\n based on the input JSON.\n\n Parameters:\n j (Dict): JSON dictionary containing the event attributes.\n Must include keys \"data\" and \"type\".\n\n Returns:\n BaseEvent: A new event object initialized with attributes from the JSON dictionary.\n\n Raises:\n ValidationError: Raised when the JSON dictionary is missing required fields.\n\n Note:\n The function assumes that the input JSON dictionary is valid and may raise exceptions\n if required keys are missing. Make sure to validate the JSON input beforehand.\n \"\"\"\n try:\n event_type = j[\"type\"]\n kwargs = {\n \"event_type\": event_type,\n \"tags\": j.get(\"tags\", []),\n \"confidence\": j.get(\"confidence\", 100),\n \"context\": j.get(\"discovery_context\", None),\n \"dummy\": True,\n }\n if siem_friendly:\n data = j[\"data\"][event_type]\n else:\n data = j[\"data\"]\n kwargs[\"data\"] = data\n event = make_event(**kwargs)\n event_uuid = j.get(\"uuid\", None)\n if event_uuid is not None:\n event._uuid = uuid.UUID(event_uuid.split(\":\")[-1])\n\n resolved_hosts = j.get(\"resolved_hosts\", [])\n event._resolved_hosts = set(resolved_hosts)\n event.timestamp = datetime.datetime.fromisoformat(j[\"timestamp\"])\n event.scope_distance = j[\"scope_distance\"]\n parent_id = j.get(\"parent\", None)\n if parent_id is not None:\n event._parent_id = parent_id\n parent_uuid = j.get(\"parent_uuid\", None)\n if parent_uuid is not None:\n parent_type, parent_uuid = parent_uuid.split(\":\", 1)\n event._parent_uuid = parent_type + \":\" + str(uuid.UUID(parent_uuid))\n return event\n except KeyError as e:\n raise ValidationError(f\"Event missing required field: {e}\")\n\n\ndef is_event(e):\n return BaseEvent in e.__class__.__mro__\n" }, { "path": "bbot/core/event/helpers.py", "content": "import ipaddress\nimport regex as re\nfrom functools import cached_property\nfrom bbot.errors import ValidationError\nfrom bbot.core.helpers import validators\nfrom bbot.core.helpers.misc import split_host_port, make_ip_type\nfrom bbot.core.helpers import regexes, smart_decode, smart_encode_punycode\n\nbbot_event_seeds = {}\n\n\n\"\"\"\nAn \"Event Seed\" is a lightweight event containing only the minimum logic required to:\n - parse input to determine the event type + data\n - validate+sanitize the data\n - extract the host for scope purposes\n\nIt's useful for quickly parsing target lists without the cpu+memory overhead of creating full-fledged BBOT events\n\nNot every type of BBOT event needs to be represented here. Only ones that are meant to be targets.\n\"\"\"\n\n\nclass EventSeedRegistry(type):\n \"\"\"\n Metaclass for EventSeed that registers all subclasses in a registry.\n \"\"\"\n\n def __new__(mcs, name, bases, attrs):\n global bbot_event_seeds\n cls = super().__new__(mcs, name, bases, attrs)\n # Don't register the base EventSeed class\n if name != \"BaseEventSeed\":\n bbot_event_seeds[cls.__name__] = cls\n return cls\n\n\ndef EventSeed(input):\n input = smart_encode_punycode(smart_decode(input).strip())\n for _, event_class in bbot_event_seeds.items():\n if hasattr(event_class, \"precheck\"):\n if event_class.precheck(input):\n return event_class(input)\n else:\n for regex in event_class.regexes:\n match = regex.match(input)\n if match:\n data = event_class.handle_match(match)\n return event_class(data)\n raise ValidationError(f'Unable to autodetect data type from \"{input}\"')\n\n\nclass BaseEventSeed(metaclass=EventSeedRegistry):\n regexes = []\n _target_type = \"TARGET\"\n\n __slots__ = [\"data\", \"host\", \"port\", \"input\"]\n\n def __init__(self, data):\n self.data, self.host, self.port = self._sanitize_and_extract_host(data)\n self.input = self._override_input(data)\n\n @staticmethod\n def handle_match(match):\n \"\"\"\n Given a regex match, returns the event data\n \"\"\"\n return match.group(0)\n\n def _sanitize_and_extract_host(self, data):\n \"\"\"\n Given the event data, returns the host\n\n Returns:\n tuple: (data, host, port)\n \"\"\"\n return data, None, None\n\n def _override_input(self, input):\n return self.data\n\n @property\n def type(self):\n return self.__class__.__name__\n\n @cached_property\n def _hash(self):\n return hash(self.input)\n\n def __hash__(self):\n return self._hash\n\n def __eq__(self, other):\n return hash(self) == hash(other)\n\n def __str__(self):\n return f\"EventSeed({self.input})\"\n\n def __repr__(self):\n return str(self)\n\n\nclass IP_ADDRESS(BaseEventSeed):\n regexes = regexes.event_type_regexes[\"IP_ADDRESS\"]\n\n @staticmethod\n def precheck(data):\n try:\n return ipaddress.ip_address(data)\n except ValueError:\n return False\n\n @staticmethod\n def _sanitize_and_extract_host(data):\n validated = ipaddress.ip_address(data)\n return str(validated), validated, None\n\n\nclass DNS_NAME(BaseEventSeed):\n regexes = regexes.event_type_regexes[\"DNS_NAME\"]\n\n @staticmethod\n def _sanitize_and_extract_host(data):\n validated = validators.validate_host(data)\n return validated, validated, None\n\n\nclass IP_RANGE(BaseEventSeed):\n regexes = regexes.event_type_regexes[\"IP_RANGE\"]\n\n @staticmethod\n def precheck(data):\n try:\n return ipaddress.ip_network(str(data), strict=False)\n except ValueError:\n return False\n\n @staticmethod\n def _sanitize_and_extract_host(data):\n validated = ipaddress.ip_network(str(data), strict=False)\n return str(validated), validated, None\n\n\nclass OPEN_TCP_PORT(BaseEventSeed):\n regexes = regexes.event_type_regexes[\"OPEN_TCP_PORT\"]\n\n @staticmethod\n def _sanitize_and_extract_host(data):\n validated = validators.validate_open_port(data)\n host, port = split_host_port(validated)\n host = make_ip_type(host)\n return str(validated), host, port\n\n\nclass URL_UNVERIFIED(BaseEventSeed):\n regexes = regexes.event_type_regexes[\"URL\"]\n\n _scheme_to_port = {\n \"https\": 443,\n \"http\": 80,\n }\n\n @staticmethod\n def _sanitize_and_extract_host(data):\n parsed_url = validators.clean_url(data, url_querystring_remove=False)\n scheme = parsed_url.scheme\n host = make_ip_type(validators.validate_host(parsed_url.hostname))\n port = parsed_url.port\n if port is None:\n port = URL_UNVERIFIED._scheme_to_port.get(scheme, None)\n return parsed_url.geturl(), host, port\n\n\nclass EMAIL_ADDRESS(BaseEventSeed):\n regexes = regexes.event_type_regexes[\"EMAIL_ADDRESS\"]\n\n @staticmethod\n def _sanitize_and_extract_host(data):\n validated = validators.validate_email(data)\n host = validated.rsplit(\"@\", 1)[-1]\n host, port = split_host_port(host)\n return validated, host, port\n\n\nclass ORG_STUB(BaseEventSeed):\n regexes = (re.compile(r\"^(?:ORG|ORG_STUB):(.*)\"),)\n\n def _override_input(self, input):\n return f\"ORG_STUB:{self.data}\"\n\n @staticmethod\n def handle_match(match):\n return match.group(1)\n\n\nclass USERNAME(BaseEventSeed):\n regexes = (re.compile(r\"^(?:USER|USERNAME):(.*)\"),)\n\n def _override_input(self, input):\n return f\"USERNAME:{self.data}\"\n\n @staticmethod\n def handle_match(match):\n return match.group(1)\n\n\nclass FILESYSTEM(BaseEventSeed):\n regexes = (re.compile(r\"^(?:FILESYSTEM|FILE|FOLDER|DIR|PATH):(.*)\"),)\n\n def _override_input(self, input):\n return f\"FILESYSTEM:{self.data['path']}\"\n\n @staticmethod\n def handle_match(match):\n return {\"path\": match.group(1)}\n\n\nclass MOBILE_APP(BaseEventSeed):\n regexes = (re.compile(r\"^(?:MOBILE_APP|APK|IPA|APP):(.*)\"),)\n\n def _override_input(self, input):\n return f\"MOBILE_APP:{self.data['url']}\"\n\n @staticmethod\n def handle_match(match):\n return {\"url\": match.group(1)}\n\n\nclass BLACKLIST_REGEX(BaseEventSeed):\n regexes = (re.compile(r\"^(?:RE|REGEX):(.*)\"),)\n _target_type = \"BLACKLIST\"\n\n def _override_input(self, input):\n return f\"REGEX:{self.data}\"\n\n @staticmethod\n def handle_match(match):\n return match.group(1)\n" }, { "path": "bbot/core/flags.py", "content": "flag_descriptions = {\n \"active\": \"Makes active connections to target systems\",\n \"affiliates\": \"Discovers affiliated hostnames/domains\",\n \"aggressive\": \"Generates a large amount of network traffic\",\n \"baddns\": \"Runs all modules from the DNS auditing tool BadDNS\",\n \"cloud-enum\": \"Enumerates cloud resources\",\n \"code-enum\": \"Find public code repositories and search them for secrets etc.\",\n \"deadly\": \"Highly aggressive\",\n \"download\": \"Modules that download files, apps, or repositories\",\n \"email-enum\": \"Enumerates email addresses\",\n \"iis-shortnames\": \"Scans for IIS Shortname vulnerability\",\n \"passive\": \"Never connects to target systems\",\n \"portscan\": \"Discovers open ports\",\n \"report\": \"Generates a report at the end of the scan\",\n \"safe\": \"Non-intrusive, safe to run\",\n \"service-enum\": \"Identifies protocols running on open ports\",\n \"slow\": \"May take a long time to complete\",\n \"social-enum\": \"Enumerates social media\",\n \"subdomain-enum\": \"Enumerates subdomains\",\n \"subdomain-hijack\": \"Detects hijackable subdomains\",\n \"web-basic\": \"Basic, non-intrusive web scan functionality\",\n \"web-paramminer\": \"Discovers HTTP parameters through brute-force\",\n \"web-screenshots\": \"Takes screenshots of web pages\",\n \"web-thorough\": \"More advanced web scanning functionality\",\n}\n" }, { "path": "bbot/core/helpers/__init__.py", "content": "from .url import *\nfrom .misc import *\nfrom . import regexes as regexes\nfrom . import validators as validators\n" }, { "path": "bbot/core/helpers/async_helpers.py", "content": "import time\nimport uuid\nimport random\nimport asyncio\nimport logging\nimport functools\nfrom contextlib import suppress\nfrom cachetools import keys, LRUCache\nfrom contextlib import asynccontextmanager\n\nlog = logging.getLogger(\"bbot.core.helpers.async_helpers\")\n\n\nclass ShuffleQueue(asyncio.Queue):\n def _put(self, item):\n random_index = random.randint(0, self.qsize())\n self._queue.insert(random_index, item)\n\n def _get(self):\n return self._queue.popleft()\n\n\nclass _Lock(asyncio.Lock):\n def __init__(self, name):\n self.name = name\n super().__init__()\n\n\nclass NamedLock:\n \"\"\"\n Returns a unique asyncio.Lock() based on a provided string\n\n Useful for preventing multiple operations from occurring on the same data in parallel\n E.g. simultaneous DNS lookups on the same hostname\n \"\"\"\n\n def __init__(self, max_size=10000):\n self._cache = LRUCache(maxsize=max_size)\n\n @asynccontextmanager\n async def lock(self, name):\n try:\n lock = self._cache[name]\n except KeyError:\n lock = _Lock(name)\n self._cache[name] = lock\n async with lock:\n yield\n\n\nclass TaskCounter:\n def __init__(self):\n self.tasks = {}\n self._lock = None\n\n @property\n def value(self):\n return sum([t.n for t in self.tasks.values()])\n\n @property\n def lock(self):\n if self._lock is None:\n self._lock = asyncio.Lock()\n return self._lock\n\n def count(self, task_name, n=1, asyncio_task=None, _log=True):\n if callable(task_name):\n task_name = f\"{task_name.__qualname__}()\"\n return self.Task(self, task_name, n=n, _log=_log, asyncio_task=asyncio_task)\n\n class Task:\n def __init__(self, manager, task_name, n=1, _log=True, asyncio_task=None):\n self.manager = manager\n self.task_name = task_name\n self.task_id = None\n self.start_time = None\n self.log = _log\n self.n = n\n self._asyncio_task = asyncio_task\n\n async def __aenter__(self):\n self.task_id = uuid.uuid4()\n # if self.log:\n # log.trace(f\"Starting task {self.task_name} ({self.task_id})\")\n async with self.manager.lock:\n self.start_time = time.time()\n self.manager.tasks[self.task_id] = self\n return self\n\n async def __aexit__(self, exc_type, exc_val, exc_tb):\n async with self.manager.lock:\n self.manager.tasks.pop(self.task_id, None)\n # if self.log:\n # log.trace(f\"Finished task {self.task_name} ({self.task_id})\")\n\n @property\n def asyncio_task(self):\n if self._asyncio_task is None:\n raise AttributeError(\"No asyncio task associated with this task\")\n return self._asyncio_task\n\n @property\n def function_name(self):\n with suppress(AttributeError):\n return self.asyncio_task.get_coro().__name__\n return \"\"\n\n async def cancel(self):\n self.asyncio_task.cancel()\n with suppress(asyncio.CancelledError):\n await self.asyncio_task\n\n @property\n def running_for(self):\n return time.time() - self.start_time\n\n def __str__(self):\n return f\"{self.task_name} running for {self.running_for:.1f}s\"\n\n\ndef get_event_loop():\n try:\n return asyncio.get_running_loop()\n except RuntimeError:\n log.verbose(\"Starting new event loop\")\n return asyncio.new_event_loop()\n\n\ndef async_to_sync_gen(async_gen):\n loop = get_event_loop()\n try:\n while True:\n yield loop.run_until_complete(async_gen.__anext__())\n except StopAsyncIteration:\n pass\n\n\ndef async_cachedmethod(cache, key=keys.hashkey):\n def decorator(method):\n async def wrapper(self, *args, **kwargs):\n method_cache = cache(self)\n k = key(*args, **kwargs)\n try:\n return method_cache[k]\n except KeyError:\n pass\n ret = await method(self, *args, **kwargs)\n try:\n method_cache[k] = ret\n except ValueError:\n pass\n return ret\n\n return functools.wraps(method)(wrapper)\n\n return decorator\n" }, { "path": "bbot/core/helpers/bloom.py", "content": "import os\nimport mmh3\nimport mmap\nimport xxhash\n\n\nclass BloomFilter:\n \"\"\"\n Simple bloom filter implementation capable of roughly 400K lookups/s.\n\n BBOT uses bloom filters in scenarios like DNS brute-forcing, where it's useful to keep track\n of which mutations have been tried so far.\n\n A 100-megabyte bloom filter (800M bits) can store 10M entries with a .01% false-positive rate.\n A python hash is 36 bytes. So if you wanted to store these in a set, this would take up\n 36 * 10M * 2 (key+value) == 720 megabytes. So we save roughly 7 times the space.\n \"\"\"\n\n def __init__(self, size=8000000):\n self.size = size # total bits\n self.byte_size = (size + 7) // 8 # calculate byte size needed for the given number of bits\n\n # Create an anonymous mmap region, compatible with both Windows and Unix\n if os.name == \"nt\": # Windows\n # -1 indicates an anonymous memory map in Windows\n self.mmap_file = mmap.mmap(-1, self.byte_size)\n else: # Unix/Linux\n # Use MAP_ANONYMOUS along with MAP_SHARED\n self.mmap_file = mmap.mmap(-1, self.byte_size, prot=mmap.PROT_WRITE, flags=mmap.MAP_ANON | mmap.MAP_SHARED)\n\n self.clear_all_bits()\n\n def add(self, item):\n for hash_value in self._hashes(item):\n index = hash_value // 8\n position = hash_value % 8\n current_byte = self.mmap_file[index]\n self.mmap_file[index] = current_byte | (1 << position)\n\n def check(self, item):\n for hash_value in self._hashes(item):\n index = hash_value // 8\n position = hash_value % 8\n current_byte = self.mmap_file[index]\n if not (current_byte & (1 << position)):\n return False\n return True\n\n def clear_all_bits(self):\n self.mmap_file.seek(0)\n # Write zeros across the entire mmap length\n self.mmap_file.write(b\"\\x00\" * self.byte_size)\n\n def _hashes(self, item):\n if not isinstance(item, bytes):\n if not isinstance(item, str):\n item = str(item)\n item = item.encode(\"utf-8\")\n\n return [\n abs(hash(item)) % self.size,\n abs(mmh3.hash(item)) % self.size,\n abs(xxhash.xxh32(item).intdigest()) % self.size,\n ]\n\n def close(self):\n \"\"\"Explicitly close the memory-mapped file.\"\"\"\n self.mmap_file.close()\n\n def __del__(self):\n try:\n self.close()\n except Exception:\n pass\n\n def __contains__(self, item):\n return self.check(item)\n" }, { "path": "bbot/core/helpers/cache.py", "content": "import os\nimport time\nimport logging\n\nfrom .misc import sha1\n\nlog = logging.getLogger(\"bbot.core.helpers.cache\")\n\n\ndef cache_get(self, key, text=True, cache_hrs=24 * 7):\n \"\"\"\n Get an item from the cache. Default expiration is 1 week.\n Returns None if item is not in cache\n \"\"\"\n filename = self.cache_filename(key)\n if filename.is_file():\n valid = self.is_cached(key, cache_hrs)\n if valid:\n open_kwargs = {}\n if text:\n open_kwargs.update({\"mode\": \"r\", \"encoding\": \"utf-8\", \"errors\": \"ignore\"})\n else:\n open_kwargs[\"mode\"] = \"rb\"\n log.debug(f'Using cached content for \"{key}\"')\n return open(filename, **open_kwargs).read()\n else:\n log.debug(f'Cached content for \"{key}\" is older than {cache_hrs:,} hours')\n\n\ndef cache_put(self, key, content):\n \"\"\"\n Put an item in the cache.\n \"\"\"\n filename = self.cache_filename(key)\n if type(content) == bytes:\n open_kwargs = {\"mode\": \"wb\"}\n else:\n open_kwargs = {\"mode\": \"w\", \"encoding\": \"utf-8\"}\n content = str(content)\n with open(filename, **open_kwargs) as f:\n f.write(content)\n\n\ndef is_cached(self, key, cache_hrs=24 * 7):\n filename = self.cache_filename(key)\n if filename.is_file():\n (m, i, d, n, u, g, sz, atime, mtime, ctime) = os.stat(filename)\n return mtime > time.time() - cache_hrs * 3600\n return False\n\n\ndef cache_filename(self, key):\n return self.cache_dir / sha1(key).hexdigest()\n" }, { "path": "bbot/core/helpers/command.py", "content": "import os\nimport asyncio\nimport logging\nimport traceback\nfrom signal import SIGINT\nfrom subprocess import CompletedProcess, CalledProcessError, SubprocessError\n\nfrom .misc import smart_decode, smart_encode, which\n\nlog = logging.getLogger(\"bbot.core.helpers.command\")\n\n\nasync def run(self, *command, check=False, text=True, idle_timeout=None, **kwargs):\n \"\"\"Runs a command asynchronously and gets its output as a string.\n\n This method is a simple helper for executing a command and capturing its output.\n If an error occurs during execution, it can optionally raise an error or just log the stderr.\n\n Args:\n *command (str): The command to run as separate arguments.\n check (bool, optional): If set to True, raises an error if the subprocess exits with a non-zero status.\n Defaults to False.\n text (bool, optional): If set to True, decodes the subprocess output to string. Defaults to True.\n idle_timeout (int, optional): Sets a limit on the number of seconds the process can run before throwing a TimeoutError\n **kwargs (dict): Additional keyword arguments for the subprocess.\n\n Returns:\n CompletedProcess: A completed process object with attributes for the command, return code, stdout, and stderr.\n\n Raises:\n CalledProcessError: If the subprocess exits with a non-zero status and `check=True`.\n\n Examples:\n >>> process = await run([\"ls\", \"/tmp\"])\n >>> process.stdout\n \"file1.txt\\nfile2.txt\"\n \"\"\"\n # proc_tracker optionally keeps track of which processes are running under which modules\n # this allows for graceful SIGINTing of a module's processes in the case when it's killed\n proc_tracker = kwargs.pop(\"_proc_tracker\", set())\n log_stderr = kwargs.pop(\"_log_stderr\", True)\n proc, _input, command = await self._spawn_proc(*command, **kwargs)\n if proc is not None:\n proc_tracker.add(proc)\n try:\n if _input is not None:\n if isinstance(_input, (list, tuple)):\n _input = b\"\\n\".join(smart_encode(i) for i in _input) + b\"\\n\"\n else:\n _input = smart_encode(_input)\n\n try:\n if idle_timeout is not None:\n stdout, stderr = await asyncio.wait_for(proc.communicate(_input), timeout=idle_timeout)\n else:\n stdout, stderr = await proc.communicate(_input)\n except asyncio.exceptions.TimeoutError:\n proc.send_signal(SIGINT)\n raise\n\n # surface stderr\n if text:\n if stderr is not None:\n stderr = smart_decode(stderr)\n if stdout is not None:\n stdout = smart_decode(stdout)\n if proc.returncode:\n if check:\n raise CalledProcessError(proc.returncode, command, output=stdout, stderr=stderr)\n if stderr and log_stderr:\n command_str = \" \".join(command)\n log.warning(f\"Stderr for run({command_str}):\\n\\t{stderr}\")\n\n return CompletedProcess(command, proc.returncode, stdout, stderr)\n finally:\n proc_tracker.remove(proc)\n\n\nasync def run_live(self, *command, check=False, text=True, idle_timeout=None, **kwargs):\n \"\"\"Runs a command asynchronously and iterates through its output line by line in realtime.\n\n This method is useful for executing a command and capturing its output on-the-fly, as it is generated.\n If an error occurs during execution, it can optionally raise an error or just log the stderr.\n\n Args:\n *command (str): The command to run as separate arguments.\n check (bool, optional): If set to True, raises an error if the subprocess exits with a non-zero status.\n Defaults to False.\n text (bool, optional): If set to True, decodes the subprocess output to string. Defaults to True.\n idle_timeout (int, optional): Sets a limit on the number of seconds the process can remain idle (no lines sent to stdout) before throwing a TimeoutError\n **kwargs (dict): Additional keyword arguments for the subprocess.\n\n Yields:\n str or bytes: The output lines of the command, either as a decoded string (if `text=True`)\n or as bytes (if `text=False`).\n\n Raises:\n CalledProcessError: If the subprocess exits with a non-zero status and `check=True`.\n\n Examples:\n >>> async for line in run_live([\"tail\", \"-f\", \"/var/log/auth.log\"]):\n ... log.info(line)\n \"\"\"\n # proc_tracker optionally keeps track of which processes are running under which modules\n # this allows for graceful SIGINTing of a module's processes in the case when it's killed\n proc_tracker = kwargs.pop(\"_proc_tracker\", set())\n log_stderr = kwargs.pop(\"_log_stderr\", True)\n proc, _input, command = await self._spawn_proc(*command, **kwargs)\n if proc is not None:\n proc_tracker.add(proc)\n try:\n input_task = None\n if _input is not None:\n input_task = asyncio.create_task(_write_stdin(proc, _input))\n\n while 1:\n try:\n if idle_timeout is not None:\n line = await asyncio.wait_for(proc.stdout.readline(), timeout=idle_timeout)\n else:\n line = await proc.stdout.readline()\n except asyncio.exceptions.TimeoutError:\n proc.send_signal(SIGINT)\n raise\n except ValueError as e:\n command_str = \" \".join(command)\n log.warning(f\"Error executing command {command_str}: {e}\")\n log.trace(traceback.format_exc())\n continue\n if not line:\n break\n if text:\n line = smart_decode(line).rstrip(\"\\r\\n\")\n else:\n line = line.rstrip(b\"\\r\\n\")\n yield line\n\n if input_task is not None:\n try:\n await input_task\n except ConnectionError:\n log.trace(f\"ConnectionError in command: {command}, kwargs={kwargs}\")\n log.trace(traceback.format_exc())\n await proc.wait()\n\n if proc.returncode:\n stdout, stderr = await proc.communicate()\n if text:\n if stderr is not None:\n stderr = smart_decode(stderr)\n if stdout is not None:\n stdout = smart_decode(stdout)\n if check:\n raise CalledProcessError(proc.returncode, command, output=stdout, stderr=stderr)\n # surface stderr\n if stderr and log_stderr:\n command_str = \" \".join(command)\n log.warning(f\"Stderr for run_live({command_str}):\\n\\t{stderr}\")\n finally:\n proc_tracker.remove(proc)\n\n\nasync def _spawn_proc(self, *command, **kwargs):\n \"\"\"Spawns an asynchronous subprocess.\n\n Prepares the command and associated keyword arguments. If the `input` argument is provided,\n it checks to ensure that the `stdin` argument is not also provided. Once prepared, it creates\n and returns the subprocess. If the command executable is not found, it logs a warning and traceback.\n\n Args:\n *command (str): The command to run as separate arguments.\n **kwargs (dict): Additional keyword arguments for the subprocess.\n\n Raises:\n ValueError: If both stdin and input arguments are provided.\n\n Returns:\n tuple: A tuple containing the created process (or None if creation failed), the input (or None if not provided),\n and the prepared command (or None if subprocess creation failed).\n\n Examples:\n >>> _spawn_proc(\"ls\", \"-l\", input=\"data\")\n (, \"data\", [\"ls\", \"-l\"])\n \"\"\"\n try:\n command, kwargs = self._prepare_command_kwargs(command, kwargs)\n except SubprocessError as e:\n command_str = \" \".join([str(s) for s in command])\n log.warning(f\"Error running command: '{command_str}': {e}\")\n log.trace(traceback.format_exc())\n return None, None, None\n _input = kwargs.pop(\"input\", None)\n if _input is not None:\n if kwargs.get(\"stdin\") is not None:\n raise ValueError(\"stdin and input arguments may not both be used.\")\n kwargs[\"stdin\"] = asyncio.subprocess.PIPE\n\n log.hugeverbose(f\"run: {' '.join(command)}\")\n try:\n proc = await asyncio.create_subprocess_exec(*command, **kwargs)\n return proc, _input, command\n except FileNotFoundError as e:\n log.warning(f\"{e} - missing executable?\")\n log.trace(traceback.format_exc())\n return None, None, None\n\n\nasync def _write_proc_line(proc, chunk):\n try:\n proc.stdin.write(smart_encode(chunk) + b\"\\n\")\n await proc.stdin.drain()\n return True\n except Exception as e:\n proc_args = [str(s) for s in getattr(proc, \"args\", [])]\n command = \" \".join(proc_args).strip()\n if command:\n log.warning(f\"Error writing line to stdin for command: {command}: {e}\")\n log.trace(traceback.format_exc())\n return False\n\n\nasync def _write_stdin(proc, _input):\n \"\"\"\n Asynchronously writes input to an active subprocess's stdin.\n\n This function takes an `_input` parameter, which can be of type str, bytes,\n list, tuple, or an asynchronous generator. The input is then written line by\n line to the stdin of the given `proc`.\n\n Args:\n proc (subprocess.Popen): An active subprocess object.\n _input (str, bytes, list, tuple, async generator): The data to write to stdin.\n \"\"\"\n if _input is not None:\n if isinstance(_input, (str, bytes)):\n _input = [_input]\n if isinstance(_input, (list, tuple)):\n for chunk in _input:\n write_result = await _write_proc_line(proc, chunk)\n if not write_result:\n break\n else:\n async for chunk in _input:\n write_result = await _write_proc_line(proc, chunk)\n if not write_result:\n break\n proc.stdin.close()\n\n\ndef _prepare_command_kwargs(self, command, kwargs):\n \"\"\"\n Prepare arguments for passing into `asyncio.create_subprocess_exec()`.\n\n This method modifies the `kwargs` dictionary in place to prepare it for\n use in the `asyncio.create_subprocess_exec()` method. It sets the default\n values for keys like 'limit', 'stdout', and 'stderr' if they are not\n already present. It also handles the case when 'sudo' needs to be run.\n\n Args:\n command (list): The command to be run in the subprocess.\n kwargs (dict): The keyword arguments to be passed to `asyncio.create_subprocess_exec()`.\n\n Returns:\n tuple: A tuple containing the modified `command` and `kwargs`.\n\n Examples:\n >>> _prepare_command_kwargs(['ls', '-l'], {})\n (['ls', '-l'], {'limit': 104857600, 'stdout': -1, 'stderr': -1})\n\n >>> _prepare_command_kwargs(['ls', '-l'], {'sudo': True})\n (['sudo', '-E', '-A', 'LD_LIBRARY_PATH=...', 'PATH=...', 'ls', '-l'], {'limit': 104857600, 'stdout': -1, 'stderr': -1, 'env': environ(...)})\n \"\"\"\n # limit = 100MB (this is needed for cases like httpx that are sending large JSON blobs over stdout)\n if \"limit\" not in kwargs:\n kwargs[\"limit\"] = 1024 * 1024 * 100\n if \"stdout\" not in kwargs:\n kwargs[\"stdout\"] = asyncio.subprocess.PIPE\n if \"stderr\" not in kwargs:\n kwargs[\"stderr\"] = asyncio.subprocess.PIPE\n sudo = kwargs.pop(\"sudo\", False)\n\n if len(command) == 1 and isinstance(command[0], (list, tuple)):\n command = command[0]\n\n command = [str(s) for s in command]\n\n if not command:\n raise SubprocessError(\"Must specify a command\")\n\n # use full path of binary, if not already specified\n binary = command[0]\n if \"/\" not in binary:\n binary_full_path = which(binary)\n if binary_full_path is None:\n raise SubprocessError(f'Command \"{binary}\" was not found')\n command[0] = binary_full_path\n\n env = kwargs.get(\"env\", os.environ)\n if sudo and os.geteuid() != 0:\n self.depsinstaller.ensure_root()\n env[\"SUDO_ASKPASS\"] = str((self.tools_dir / self.depsinstaller.askpass_filename).resolve())\n env[\"BBOT_SUDO_PASS\"] = self.depsinstaller.encrypted_sudo_pw\n kwargs[\"env\"] = env\n\n PATH = os.environ.get(\"PATH\", \"\")\n LD_LIBRARY_PATH = os.environ.get(\"LD_LIBRARY_PATH\", \"\")\n command = [\"sudo\", \"-E\", \"-A\", f\"LD_LIBRARY_PATH={LD_LIBRARY_PATH}\", f\"PATH={PATH}\"] + command\n return command, kwargs\n" }, { "path": "bbot/core/helpers/depsinstaller/__init__.py", "content": "from .installer import DepsInstaller\n\n__all__ = [\"DepsInstaller\"]\n" }, { "path": "bbot/core/helpers/depsinstaller/installer.py", "content": "import os\nimport sys\nimport stat\nimport json\nimport mmh3\nimport orjson\nimport shutil\nimport getpass\nimport logging\nfrom time import sleep\nfrom pathlib import Path\nfrom threading import Lock\nfrom itertools import chain\nfrom contextlib import suppress\nfrom secrets import token_bytes\nfrom ansible_runner.interface import run\nfrom subprocess import CalledProcessError\n\nfrom bbot import __version__\nfrom ..misc import can_sudo_without_password, os_platform, rm_at_exit, get_python_constraints\n\nlog = logging.getLogger(\"bbot.core.helpers.depsinstaller\")\n\n\nclass DepsInstaller:\n CORE_DEPS = {\n # core BBOT dependencies in the format of binary: package_name\n # each one will only be installed if the binary is not found\n \"unzip\": \"unzip\",\n \"zipinfo\": \"unzip\",\n \"curl\": \"curl\",\n \"git\": \"git\",\n \"make\": \"make\",\n \"gcc\": \"gcc\",\n \"bash\": \"bash\",\n \"which\": \"which\",\n \"tar\": \"tar\",\n \"xz\": [\n {\n \"name\": \"Install xz-utils (Debian)\",\n \"package\": {\"name\": [\"xz-utils\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Debian'\",\n },\n {\n \"name\": \"Install xz (Non-Debian)\",\n \"package\": {\"name\": [\"xz\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] != 'Debian'\",\n },\n ],\n # debian why are you like this\n \"7z\": [\n {\n \"name\": \"Install 7zip (Debian)\",\n \"package\": {\"name\": [\"p7zip-full\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Debian'\",\n },\n {\n \"name\": \"Install 7zip (Non-Debian)\",\n \"package\": {\"name\": [\"p7zip\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] != 'Debian'\",\n },\n {\n \"name\": \"Install p7zip-plugins (Fedora)\",\n \"package\": {\"name\": [\"p7zip-plugins\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['distribution'] == 'Fedora'\",\n },\n ],\n # to compile just about any tool, we need the openssl dev headers\n \"openssl_dev_headers\": [\n {\n \"name\": \"Install OpenSSL library and development headers (Debian/Ubuntu)\",\n \"package\": {\"name\": [\"libssl-dev\", \"openssl\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Debian'\",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Install OpenSSL library and development headers (RedHat/CentOS/Fedora)\",\n \"package\": {\"name\": [\"openssl\", \"openssl-devel\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'RedHat' or ansible_facts['os_family'] == 'Suse' \",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Install OpenSSL library and development headers (Arch)\",\n \"package\": {\"name\": [\"openssl\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Archlinux'\",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Install OpenSSL library and development headers (Alpine)\",\n \"package\": {\"name\": [\"openssl\", \"openssl-dev\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Alpine'\",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Install OpenSSL library and development headers (FreeBSD)\",\n \"package\": {\"name\": [\"openssl\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'FreeBSD'\",\n \"ignore_errors\": True,\n },\n ],\n }\n\n def __init__(self, parent_helper):\n self.parent_helper = parent_helper\n self.preset = self.parent_helper.preset\n self.core = self.preset.core\n\n self.os_platform = os_platform()\n\n # respect BBOT's http timeout\n self.web_config = self.parent_helper.config.get(\"web\", {})\n http_timeout = self.web_config.get(\"http_timeout\", 30)\n os.environ[\"ANSIBLE_TIMEOUT\"] = str(http_timeout)\n\n # cache encrypted sudo pass\n self.askpass_filename = \"sudo_askpass.py\"\n self._sudo_password = None\n self._sudo_cache_setup = False\n self._setup_sudo_cache()\n self._installed_sudo_askpass = False\n\n self.data_dir = self.parent_helper.cache_dir / \"depsinstaller\"\n self.parent_helper.mkdir(self.data_dir)\n self.setup_status_cache = self.data_dir / \"setup_status.json\"\n self.command_status = self.data_dir / \"command_status\"\n self.parent_helper.mkdir(self.command_status)\n self.setup_status = self.read_setup_status()\n\n # make sure we're using a minimal git config\n self.minimal_git_config = self.data_dir / \"minimal_git.config\"\n self.minimal_git_config.touch()\n os.environ[\"GIT_CONFIG_GLOBAL\"] = str(self.minimal_git_config)\n\n self.deps_config = self.parent_helper.config.get(\"deps\", {})\n self.deps_behavior = self.deps_config.get(\"behavior\", \"abort_on_failure\").lower()\n self.ansible_debug = self.core.logger.log_level <= logging.DEBUG\n self.venv = \"\"\n if sys.prefix != sys.base_prefix:\n self.venv = sys.prefix\n\n self.ensure_root_lock = Lock()\n\n async def install(self, *modules):\n await self.install_core_deps()\n succeeded = []\n failed = []\n try:\n notified = False\n for m in modules:\n # assume success if we're ignoring dependencies\n if self.deps_behavior == \"disable\":\n succeeded.append(m)\n continue\n # abort if module name is unknown\n if m not in self.all_modules_preloaded:\n log.verbose(f'Module \"{m}\" not found')\n failed.append(m)\n continue\n preloaded = self.all_modules_preloaded[m]\n log.debug(f\"Installing {m} - Preloaded Deps {preloaded['deps']}\")\n # make a hash of the dependencies and check if it's already been handled\n # take into consideration whether the venv or bbot home directory changes\n module_hash = self.parent_helper.sha1(\n json.dumps(preloaded[\"deps\"], sort_keys=True)\n + self.venv\n + str(self.parent_helper.bbot_home)\n + os.uname()[1]\n + str(__version__)\n ).hexdigest()\n success = self.setup_status.get(module_hash, None)\n dependencies = list(chain(*preloaded[\"deps\"].values()))\n if len(dependencies) <= 0:\n log.debug(f'No dependency work to do for module \"{m}\"')\n succeeded.append(m)\n continue\n else:\n if (\n success is None\n or (success is False and self.deps_behavior == \"retry_failed\")\n or self.deps_behavior == \"force_install\"\n ):\n if not notified:\n log.hugeinfo(\"Installing module dependencies. Please be patient, this may take a while.\")\n notified = True\n log.verbose(f'Installing dependencies for module \"{m}\"')\n # get sudo access if we need it\n if preloaded.get(\"sudo\", False) is True:\n self.ensure_root(f'Module \"{m}\" needs root privileges to install its dependencies.')\n success = await self.install_module(m)\n self.setup_status[module_hash] = success\n if success or self.deps_behavior == \"ignore_failed\":\n log.debug(f'Setup succeeded for module \"{m}\"')\n succeeded.append(m)\n else:\n log.warning(f'Setup failed for module \"{m}\"')\n failed.append(m)\n else:\n if success or self.deps_behavior == \"ignore_failed\":\n log.debug(\n f'Skipping dependency install for module \"{m}\" because it\\'s already done (--force-deps to re-run)'\n )\n succeeded.append(m)\n else:\n log.warning(\n f'Skipping dependency install for module \"{m}\" because it failed previously (--retry-deps to retry or --ignore-failed-deps to ignore)'\n )\n failed.append(m)\n\n finally:\n self.write_setup_status()\n\n succeeded.sort()\n failed.sort()\n return succeeded, failed\n\n async def install_module(self, module):\n success = True\n preloaded = self.all_modules_preloaded[module]\n\n # apt\n deps_apt = preloaded[\"deps\"][\"apt\"]\n if deps_apt:\n self.apt_install(deps_apt)\n\n # shell\n deps_shell = preloaded[\"deps\"][\"shell\"]\n if deps_shell:\n success &= self.shell(module, deps_shell)\n\n # pip\n deps_pip = preloaded[\"deps\"][\"pip\"]\n deps_pip_constraints = preloaded[\"deps\"][\"pip_constraints\"]\n if deps_pip:\n success &= await self.pip_install(deps_pip, constraints=deps_pip_constraints)\n\n # shared/common\n deps_common = preloaded[\"deps\"][\"common\"]\n if deps_common:\n for dep_common in deps_common:\n if self.setup_status.get(dep_common, False) is True and self.deps_behavior != \"force_install\":\n log.debug(\n f'Skipping installation of dependency \"{dep_common}\" for module \"{module}\" since it is already installed'\n )\n continue\n ansible_tasks = self.preset.module_loader._shared_deps[dep_common]\n result = self.tasks(module, ansible_tasks)\n self.setup_status[dep_common] = result\n success &= result\n\n # ansible tasks\n ansible_tasks = preloaded[\"deps\"][\"ansible\"]\n if ansible_tasks:\n success &= self.tasks(module, ansible_tasks)\n\n return success\n\n async def pip_install(self, packages, constraints=None):\n packages_str = \",\".join(packages)\n log.info(f\"Installing the following pip packages: {packages_str}\")\n\n command = [sys.executable, \"-m\", \"pip\", \"install\", \"--upgrade\"] + packages\n\n # if no custom constraints are provided, use the constraints of the currently installed version of bbot\n if constraints is not None:\n constraints = get_python_constraints()\n\n constraints_tempfile = self.parent_helper.tempfile(constraints, pipe=False)\n command.append(\"--constraint\")\n command.append(constraints_tempfile)\n\n process = None\n try:\n process = await self.parent_helper.run(command, check=True)\n message = f'Successfully installed pip packages \"{packages_str}\"'\n output = process.stdout\n if output is not None:\n message = output.splitlines()[-1]\n log.info(message)\n return True\n except CalledProcessError as err:\n log.warning(f\"Failed to install pip packages {packages_str} (return code {err.returncode}): {err.stderr}\")\n return False\n\n def apt_install(self, packages):\n \"\"\"\n Install packages with the OS's default package manager (apt, pacman, dnf, etc.)\n \"\"\"\n args, kwargs = self._make_apt_ansible_args(packages)\n success, err = self.ansible_run(module=\"package\", args=args, **kwargs)\n if success:\n log.info(f'Successfully installed OS packages \"{\",\".join(sorted(packages))}\"')\n else:\n log.warning(\n f\"Failed to install OS packages ({err}). Recommend installing the following packages manually:\"\n )\n for p in packages:\n log.warning(f\" - {p}\")\n return success\n\n def _make_apt_ansible_args(self, packages):\n packages_str = \",\".join(sorted(packages))\n log.info(f\"Installing the following OS packages: {packages_str}\")\n args = {\"name\": packages_str, \"state\": \"present\"} # , \"update_cache\": True, \"cache_valid_time\": 86400}\n kwargs = {}\n # don't sudo brew\n if self.os_platform != \"darwin\":\n kwargs = {\n \"ansible_args\": {\n \"ansible_become\": True,\n \"ansible_become_method\": \"sudo\",\n }\n }\n return args, kwargs\n\n def shell(self, module, commands):\n tasks = []\n for i, command in enumerate(commands):\n command_hash = self.parent_helper.sha1(f\"{module}_{i}_{command}\").hexdigest()\n command_status_file = self.command_status / command_hash\n if type(command) == str:\n command = {\"cmd\": command}\n command[\"cmd\"] += f\" && touch {command_status_file}\"\n tasks.append(\n {\n \"name\": f\"{module}.deps_shell step {i + 1}\",\n \"ansible.builtin.shell\": command,\n \"args\": {\"executable\": \"/bin/bash\", \"creates\": str(command_status_file)},\n }\n )\n success, err = self.ansible_run(tasks=tasks)\n if success:\n log.info(f\"Successfully ran {len(commands):,} shell commands\")\n else:\n log.warning(\"Failed to run shell dependencies\")\n return success\n\n def tasks(self, module, tasks):\n log.info(f\"Running {len(tasks):,} Ansible tasks for {module}\")\n success, err = self.ansible_run(tasks=tasks)\n if success:\n log.info(f\"Successfully ran {len(tasks):,} Ansible tasks for {module}\")\n else:\n log.warning(f\"Failed to run Ansible tasks for {module}\")\n return success\n\n def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):\n _ansible_args = {\"ansible_connection\": \"local\", \"ansible_python_interpreter\": sys.executable}\n if ansible_args is not None:\n _ansible_args.update(ansible_args)\n module_args = None\n if args:\n module_args = \" \".join([f'{k}=\"{v}\"' for k, v in args.items()])\n log.debug(f\"ansible_run(module={module}, args={args}, ansible_args={ansible_args})\")\n playbook = None\n if tasks:\n for task in tasks:\n if \"package\" in task:\n # special case for macos\n if self.os_platform == \"darwin\":\n # don't sudo brew\n task[\"become\"] = False\n # brew doesn't support update_cache\n task[\"package\"].pop(\"update_cache\", \"\")\n playbook = {\"hosts\": \"all\", \"tasks\": tasks}\n log.debug(json.dumps(playbook, indent=2))\n if self._sudo_password is not None:\n _ansible_args[\"ansible_become_password\"] = self._sudo_password\n playbook_hash = self.parent_helper.sha1(str(playbook)).hexdigest()\n data_dir = self.data_dir / (module if module else f\"playbook_{playbook_hash}\")\n shutil.rmtree(data_dir, ignore_errors=True)\n self.parent_helper.mkdir(data_dir)\n\n res = run(\n playbook=playbook,\n private_data_dir=str(data_dir),\n host_pattern=\"localhost\",\n inventory={\n \"all\": {\"hosts\": {\"localhost\": _ansible_args}},\n },\n module=module,\n module_args=module_args,\n quiet=True,\n verbosity=0,\n cancel_callback=lambda: None,\n )\n\n log.debug(f\"Ansible status: {res.status}\")\n log.debug(f\"Ansible return code: {res.rc}\")\n success = res.status == \"successful\"\n err = \"\"\n for e in res.events:\n if self.ansible_debug and not success:\n log.debug(json.dumps(e, indent=2))\n if e[\"event\"] == \"runner_on_failed\":\n err = e[\"event_data\"][\"res\"][\"msg\"]\n break\n return success, err\n\n def read_setup_status(self):\n setup_status = {}\n if self.setup_status_cache.is_file():\n with open(self.setup_status_cache) as f:\n with suppress(Exception):\n setup_status = json.load(f)\n return setup_status\n\n def write_setup_status(self):\n with open(self.setup_status_cache, \"w\") as f:\n json.dump(self.setup_status, f)\n\n def ensure_root(self, message=\"\"):\n self._install_sudo_askpass()\n # skip if we've already done this\n if self._sudo_password is not None:\n return\n with self.ensure_root_lock:\n # first check if the environment variable is set\n _sudo_password = os.environ.get(\"BBOT_SUDO_PASS\", None)\n if _sudo_password is not None or os.geteuid() == 0 or can_sudo_without_password():\n # if we're already root or we can sudo without a password, there's no need to prompt\n return\n\n if message:\n log.warning(message)\n while not self._sudo_password:\n # sleep for a split second to flush previous log messages\n sleep(0.1)\n _sudo_password = getpass.getpass(prompt=\"[USER] Please enter sudo password: \")\n if self.parent_helper.verify_sudo_password(_sudo_password):\n log.success(\"Authentication successful\")\n self._sudo_password = _sudo_password\n else:\n log.warning(\"Incorrect password\")\n\n async def install_core_deps(self):\n # skip if we've already successfully installed core deps for this definition\n core_deps_hash = str(mmh3.hash(orjson.dumps(self.CORE_DEPS, option=orjson.OPT_SORT_KEYS)))\n core_deps_cache_file = self.parent_helper.cache_dir / core_deps_hash\n if core_deps_cache_file.exists():\n log.debug(\"Skipping core dependency installation (cache hit)\")\n return\n\n to_install = set()\n to_install_friendly = set()\n playbook = []\n self._install_sudo_askpass()\n # ensure tldextract data is cached\n self.parent_helper.tldextract(\"evilcorp.co.uk\")\n # install any missing commands\n for command, package_name_or_playbook in self.CORE_DEPS.items():\n if not self.parent_helper.which(command):\n to_install_friendly.add(command)\n if isinstance(package_name_or_playbook, str):\n to_install.add(package_name_or_playbook)\n else:\n playbook.extend(package_name_or_playbook)\n # install ansible community.general collection\n overall_success = True\n if not self.setup_status.get(\"ansible:community.general\", False):\n log.info(\"Installing Ansible Community General Collection\")\n try:\n command = [\"ansible-galaxy\", \"collection\", \"install\", \"community.general\"]\n await self.parent_helper.run(command, check=True)\n self.setup_status[\"ansible:community.general\"] = True\n log.info(\"Successfully installed Ansible Community General Collection\")\n except CalledProcessError as err:\n log.warning(\n f\"Failed to install Ansible Community.General Collection (return code {err.returncode}): {err.stderr}\"\n )\n overall_success = False\n # construct ansible playbook\n if to_install:\n playbook.append(\n {\n \"name\": \"Install Core BBOT Dependencies\",\n \"package\": {\"name\": list(to_install), \"state\": \"present\"},\n \"become\": True,\n }\n )\n # run playbook\n if playbook:\n log.info(f\"Installing core BBOT dependencies: {','.join(sorted(to_install_friendly))}\")\n self.ensure_root()\n success, _ = self.ansible_run(tasks=playbook)\n overall_success &= success\n\n # mark cache only if everything succeeded (or nothing needed doing)\n if overall_success:\n with suppress(Exception):\n core_deps_cache_file.touch()\n\n def _setup_sudo_cache(self):\n if not self._sudo_cache_setup:\n self._sudo_cache_setup = True\n # write temporary encryption key, to be deleted upon scan completion\n self._sudo_temp_keyfile = self.parent_helper.temp_filename()\n # remove it at exit\n rm_at_exit(self._sudo_temp_keyfile)\n # generate random 32-byte key\n random_key = token_bytes(32)\n # write key to file and set secure permissions\n self._sudo_temp_keyfile.write_bytes(random_key)\n self._sudo_temp_keyfile.chmod(0o600)\n # export path to environment variable, for use in askpass script\n os.environ[\"BBOT_SUDO_KEYFILE\"] = str(self._sudo_temp_keyfile.resolve())\n\n @property\n def encrypted_sudo_pw(self):\n if self._sudo_password is None:\n return \"\"\n return self._encrypt_sudo_pw(self._sudo_password)\n\n def _encrypt_sudo_pw(self, pw):\n from Crypto.Cipher import AES\n from Crypto.Util.Padding import pad\n\n key = self._sudo_temp_keyfile.read_bytes()\n cipher = AES.new(key, AES.MODE_CBC)\n ct_bytes = cipher.encrypt(pad(pw.encode(), AES.block_size))\n iv = cipher.iv.hex()\n ct = ct_bytes.hex()\n return f\"{iv}:{ct}\"\n\n def _install_sudo_askpass(self):\n if not self._installed_sudo_askpass:\n self._installed_sudo_askpass = True\n # install custom askpass script\n askpass_src = Path(__file__).resolve().parent / self.askpass_filename\n askpass_dst = self.parent_helper.tools_dir / self.askpass_filename\n shutil.copy(askpass_src, askpass_dst)\n askpass_dst.chmod(askpass_dst.stat().st_mode | stat.S_IEXEC)\n\n @property\n def all_modules_preloaded(self):\n return self.preset.module_loader.preloaded()\n" }, { "path": "bbot/core/helpers/depsinstaller/sudo_askpass.py", "content": "#!/usr/bin/env python3\nimport os\nimport sys\nfrom pathlib import Path\nfrom Crypto.Cipher import AES\nfrom Crypto.Util.Padding import unpad\n\nENV_VAR_NAME = \"BBOT_SUDO_PASS\"\nKEY_ENV_VAR_PATH = \"BBOT_SUDO_KEYFILE\"\n\n\ndef decrypt_password(encrypted_data, key):\n iv, ciphertext = encrypted_data.split(\":\")\n iv = bytes.fromhex(iv)\n ct = bytes.fromhex(ciphertext)\n cipher = AES.new(key, AES.MODE_CBC, iv)\n pt = unpad(cipher.decrypt(ct), AES.block_size)\n return pt.decode(\"utf-8\")\n\n\ndef main():\n encrypted_password = os.environ.get(ENV_VAR_NAME, \"\")\n # remove variable from environment once we've got it\n os.environ.pop(ENV_VAR_NAME, None)\n encryption_keypath = Path(os.environ.get(KEY_ENV_VAR_PATH, \"\"))\n\n if not encrypted_password or not encryption_keypath.is_file():\n print(\"Error: Encrypted password or encryption key not found in environment variables.\", file=sys.stderr)\n sys.exit(1)\n\n try:\n key = encryption_keypath.read_bytes()\n decrypted_password = decrypt_password(encrypted_password, key)\n print(decrypted_password, end=\"\")\n except Exception as e:\n print(f'Error decrypting password \"{encrypted_password}\": {str(e)}', file=sys.stderr)\n sys.exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "bbot/core/helpers/diff.py", "content": "import logging\nimport xmltodict\nfrom deepdiff import DeepDiff\nfrom contextlib import suppress\nfrom xml.parsers.expat import ExpatError\nfrom bbot.errors import HttpCompareError\n\nlog = logging.getLogger(\"bbot.core.helpers.diff\")\n\n\nclass HttpCompare:\n def __init__(\n self,\n baseline_url,\n parent_helper,\n method=\"GET\",\n data=None,\n json=None,\n allow_redirects=False,\n include_cache_buster=True,\n headers=None,\n cookies=None,\n timeout=10,\n ):\n self.parent_helper = parent_helper\n self.baseline_url = baseline_url\n self.include_cache_buster = include_cache_buster\n self.method = method\n self.data = data\n self.json = json\n self.allow_redirects = allow_redirects\n self._baselined = False\n self.headers = headers\n self.cookies = cookies\n self.timeout = 10\n\n @staticmethod\n def merge_dictionaries(headers1, headers2):\n if headers2 is None:\n return headers1\n else:\n merged_headers = headers1.copy()\n merged_headers.update(headers2)\n return merged_headers\n\n async def _baseline(self):\n if not self._baselined:\n # vanilla URL\n if self.include_cache_buster:\n url_1 = self.parent_helper.add_get_params(self.baseline_url, self.gen_cache_buster()).geturl()\n else:\n url_1 = self.baseline_url\n baseline_1 = await self.parent_helper.request(\n url_1,\n follow_redirects=self.allow_redirects,\n method=self.method,\n data=self.data,\n json=self.json,\n headers=self.headers,\n cookies=self.cookies,\n retries=2,\n timeout=self.timeout,\n )\n await self.parent_helper.sleep(0.5)\n # put random parameters in URL, headers, and cookies\n get_params = {self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}\n\n if self.include_cache_buster:\n get_params.update(self.gen_cache_buster())\n url_2 = self.parent_helper.add_get_params(self.baseline_url, get_params).geturl()\n baseline_2 = await self.parent_helper.request(\n url_2,\n headers=self.merge_dictionaries(\n {self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}, self.headers\n ),\n cookies=self.merge_dictionaries(\n {self.parent_helper.rand_string(6): self.parent_helper.rand_string(6)}, self.cookies\n ),\n follow_redirects=self.allow_redirects,\n method=self.method,\n data=self.data,\n json=self.json,\n retries=2,\n timeout=self.timeout,\n )\n\n self.baseline = baseline_1\n if baseline_1 is None or baseline_2 is None:\n log.debug(\"HTTP error while establishing baseline, aborting\")\n raise HttpCompareError(\n f\"Can't get baseline from source URL: {url_1}:{baseline_1} / {url_2}:{baseline_2}\"\n )\n if baseline_1.status_code != baseline_2.status_code:\n log.debug(\"Status code not stable during baseline, aborting\")\n raise HttpCompareError(\"Can't get baseline from source URL\")\n\n try:\n baseline_1_json = xmltodict.parse(baseline_1.text)\n baseline_2_json = xmltodict.parse(baseline_2.text)\n except ExpatError:\n log.debug(f\"Can't HTML parse for {self.baseline_url}. Switching to text parsing as a backup\")\n baseline_1_json = baseline_1.text.split(\"\\n\")\n baseline_2_json = baseline_2.text.split(\"\\n\")\n\n ddiff = DeepDiff(\n baseline_1_json, baseline_2_json, ignore_order=True, view=\"tree\", threshold_to_diff_deeper=0\n )\n self.ddiff_filters = []\n\n for k in ddiff.keys():\n for x in list(ddiff[k]):\n self.ddiff_filters.append(x.path())\n\n self.baseline_json = baseline_1_json\n self.baseline_ignore_headers = [\n h.lower()\n for h in [\n \"date\",\n \"last-modified\",\n \"content-length\",\n \"ETag\",\n \"X-Pad\",\n \"X-Backside-Transport\",\n \"keep-alive\",\n ]\n ]\n dynamic_headers = self.compare_headers(baseline_1.headers, baseline_2.headers)\n\n self.baseline_ignore_headers += [x.lower() for x in dynamic_headers]\n self._baselined = True\n\n def gen_cache_buster(self):\n return {self.parent_helper.rand_string(6): \"1\"}\n\n def compare_headers(self, headers_1, headers_2):\n differing_headers = []\n\n for i, headers in enumerate((headers_1, headers_2)):\n for header, value in list(headers.items()):\n if header.lower() in self.baseline_ignore_headers:\n with suppress(KeyError):\n log.debug(f'found ignored header \"{header}\" in headers_{i + 1} and removed')\n del headers[header]\n\n ddiff = DeepDiff(headers_1, headers_2, ignore_order=True, view=\"tree\", threshold_to_diff_deeper=0)\n\n for k in ddiff.keys():\n for x in list(ddiff[k]):\n try:\n header_value = str(x).split(\"'\")[1]\n except KeyError:\n continue\n differing_headers.append(header_value)\n return differing_headers\n\n def compare_body(self, content_1, content_2):\n if content_1 == content_2:\n return True\n\n ddiff = DeepDiff(\n content_1,\n content_2,\n ignore_order=True,\n view=\"tree\",\n exclude_paths=self.ddiff_filters,\n threshold_to_diff_deeper=0,\n )\n\n if len(ddiff.keys()) == 0:\n return True\n else:\n return False\n\n async def compare(\n self,\n subject,\n headers=None,\n cookies=None,\n check_reflection=False,\n method=\"GET\",\n data=None,\n json=None,\n allow_redirects=False,\n timeout=None,\n ):\n \"\"\"\n Compares a URL with the baseline, with optional headers or cookies added\n\n Returns (match (bool), reason (str), reflection (bool),subject_response (requests response object))\n where \"match\" is whether the content matched against the baseline, and\n \"reason\" is the location of the change (\"code\", \"body\", \"header\", or None), and\n \"reflection\" is whether the value was reflected in the HTTP response\n \"\"\"\n\n await self._baseline()\n\n if timeout is None:\n timeout = self.timeout\n\n reflection = False\n if self.include_cache_buster:\n cache_key, cache_value = list(self.gen_cache_buster().items())[0]\n url = self.parent_helper.add_get_params(subject, {cache_key: cache_value}).geturl()\n else:\n url = subject\n subject_response = await self.parent_helper.request(\n url,\n headers=headers,\n cookies=cookies,\n follow_redirects=allow_redirects,\n method=method,\n data=data,\n json=json,\n timeout=timeout,\n )\n\n if subject_response is None:\n # this can be caused by a WAF not liking the header, so we really aren't interested in it\n return (True, \"403\", reflection, subject_response)\n\n if check_reflection:\n for arg in (headers, cookies):\n if arg is not None:\n for k, v in arg.items():\n if v in subject_response.text:\n reflection = True\n break\n\n subject_params = self.parent_helper.get_get_params(subject)\n for k, v in subject_params.items():\n if self.include_cache_buster and k != cache_key:\n for item in v:\n if item in subject_response.text:\n reflection = True\n break\n try:\n subject_json = xmltodict.parse(subject_response.text)\n\n except ExpatError:\n log.debug(f\"Can't HTML parse for {subject.split('?')[0]}. Switching to text parsing as a backup\")\n subject_json = subject_response.text.split(\"\\n\")\n\n diff_reasons = []\n\n if self.baseline.status_code != subject_response.status_code:\n log.debug(\n f\"status code was different [{str(self.baseline.status_code)}] -> [{str(subject_response.status_code)}], no match\"\n )\n diff_reasons.append(\"code\")\n\n different_headers = self.compare_headers(self.baseline.headers, subject_response.headers)\n if different_headers:\n log.debug(\"headers were different, no match\")\n diff_reasons.append(\"header\")\n\n if self.compare_body(self.baseline_json, subject_json) is False:\n log.debug(\"difference in HTML body, no match\")\n\n diff_reasons.append(\"body\")\n\n if not diff_reasons:\n return (True, [], reflection, subject_response)\n else:\n return (False, diff_reasons, reflection, subject_response)\n\n async def canary_check(self, url, mode, rounds=3):\n \"\"\"\n test detection using a canary to find hosts giving bad results\n \"\"\"\n await self._baseline()\n headers = None\n cookies = None\n for i in range(0, rounds):\n random_params = {self.parent_helper.rand_string(7): self.parent_helper.rand_string(7)}\n new_url = str(url)\n if mode == \"getparam\":\n new_url = self.parent_helper.add_get_params(url, random_params).geturl()\n elif mode == \"header\":\n headers = random_params\n elif mode == \"cookie\":\n cookies = random_params\n else:\n raise ValueError(f'Invalid mode: \"{mode}\", choose from: getparam, header, cookie')\n\n match, reasons, reflection, subject_response = await self.compare(\n new_url, headers=headers, cookies=cookies, check_reflection=True\n )\n\n # if a nonsense header \"caused\" a difference, we need to abort. We also need to abort if our canary was reflected\n if match is False or reflection is True:\n return False\n return True\n" }, { "path": "bbot/core/helpers/dns/__init__.py", "content": "from .dns import DNSHelper # noqa\n" }, { "path": "bbot/core/helpers/dns/brute.py", "content": "import json\nimport random\nimport asyncio\nimport logging\nimport subprocess\n\n\nclass DNSBrute:\n \"\"\"\n Helper for DNS brute-forcing.\n\n Examples:\n >>> domain = \"evilcorp.com\"\n >>> subdomains = [\"www\", \"mail\"]\n >>> results = await self.helpers.dns.brute(self, domain, subdomains)\n \"\"\"\n\n _nameservers_url = (\n \"https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt\"\n )\n\n def __init__(self, parent_helper):\n self.parent_helper = parent_helper\n self.log = logging.getLogger(\"bbot.helper.dns.brute\")\n self.dns_config = self.parent_helper.config.get(\"dns\", {})\n self.num_canaries = 100\n self.max_resolvers = self.dns_config.get(\"brute_threads\", 1000)\n self.nameservers_url = self.dns_config.get(\"brute_nameservers\", self._nameservers_url)\n self.devops_mutations = list(self.parent_helper.word_cloud.devops_mutations)\n self.digit_regex = self.parent_helper.re.compile(r\"\\d+\")\n self._resolver_file = None\n self._dnsbrute_lock = None\n\n async def __call__(self, *args, **kwargs):\n return await self.dnsbrute(*args, **kwargs)\n\n @property\n def dnsbrute_lock(self):\n if self._dnsbrute_lock is None:\n self._dnsbrute_lock = asyncio.Lock()\n return self._dnsbrute_lock\n\n async def dnsbrute(self, module, domain, subdomains, type=None):\n subdomains = list(subdomains)\n\n if type is None:\n type = \"A\"\n type = str(type).strip().upper()\n\n wildcard_domains = await self.parent_helper.dns.is_wildcard_domain(domain, (type, \"CNAME\"))\n wildcard_rdtypes = set()\n for domain, rdtypes in wildcard_domains.items():\n wildcard_rdtypes.update(rdtypes)\n if wildcard_domains:\n self.log.hugewarning(\n f\"Aborting massdns on {domain} because it's a wildcard domain ({','.join(sorted(wildcard_rdtypes))})\"\n )\n return []\n\n canaries = self.gen_random_subdomains(self.num_canaries)\n canaries_list = list(canaries)\n canaries_pre = canaries_list[: int(self.num_canaries / 2)]\n canaries_post = canaries_list[int(self.num_canaries / 2) :]\n # sandwich subdomains between canaries\n subdomains = canaries_pre + subdomains + canaries_post\n\n results = []\n canaries_triggered = []\n async for hostname, ip, rdtype in self._massdns(module, domain, subdomains, rdtype=type):\n sub = hostname.split(domain)[0]\n if sub in canaries:\n canaries_triggered.append(sub)\n else:\n results.append(hostname)\n\n if len(canaries_triggered) > 5:\n self.log.info(\n f\"Aborting massdns on {domain} due to false positive: ({len(canaries_triggered):,} canaries triggered - {','.join(canaries_triggered)})\"\n )\n return []\n\n # everything checks out\n return results\n\n async def _massdns(self, module, domain, subdomains, rdtype):\n \"\"\"\n {\n \"name\": \"www.blacklanternsecurity.com.\",\n \"type\": \"A\",\n \"class\": \"IN\",\n \"status\": \"NOERROR\",\n \"data\": {\n \"answers\": [\n {\n \"ttl\": 3600,\n \"type\": \"CNAME\",\n \"class\": \"IN\",\n \"name\": \"www.blacklanternsecurity.com.\",\n \"data\": \"blacklanternsecurity.github.io.\"\n },\n {\n \"ttl\": 3600,\n \"type\": \"A\",\n \"class\": \"IN\",\n \"name\": \"blacklanternsecurity.github.io.\",\n \"data\": \"185.199.108.153\"\n }\n ]\n },\n \"resolver\": \"168.215.165.186:53\"\n }\n \"\"\"\n resolver_file = await self.resolver_file()\n command = (\n \"massdns\",\n \"-r\",\n resolver_file,\n \"-s\",\n self.max_resolvers,\n \"-t\",\n rdtype,\n \"-o\",\n \"J\",\n \"-q\",\n )\n subdomains = self.gen_subdomains(subdomains, domain)\n hosts_yielded = set()\n async with self.dnsbrute_lock:\n async for line in module.run_process_live(*command, stderr=subprocess.DEVNULL, input=subdomains):\n try:\n j = json.loads(line)\n except json.decoder.JSONDecodeError:\n self.log.debug(f\"Failed to decode line: {line}\")\n continue\n answers = j.get(\"data\", {}).get(\"answers\", [])\n if type(answers) == list and len(answers) > 0:\n answer = answers[0]\n hostname = answer.get(\"name\", \"\").strip(\".\").lower()\n if hostname.endswith(f\".{domain}\"):\n data = answer.get(\"data\", \"\")\n rdtype = answer.get(\"type\", \"\").upper()\n if data and rdtype:\n hostname_hash = hash(hostname)\n if hostname_hash not in hosts_yielded:\n hosts_yielded.add(hostname_hash)\n yield hostname, data, rdtype\n\n async def gen_subdomains(self, prefixes, domain):\n for p in prefixes:\n if domain:\n p = f\"{p}.{domain}\"\n yield p\n\n async def resolver_file(self):\n if self._resolver_file is None:\n self._resolver_file_original = await self.parent_helper.wordlist(\n self.nameservers_url,\n cache_hrs=24 * 7,\n )\n nameservers = set(self.parent_helper.read_file(self._resolver_file_original))\n nameservers.difference_update(self.parent_helper.dns.system_resolvers)\n # exclude system nameservers from brute-force\n # this helps prevent rate-limiting which might cause BBOT's main dns queries to fail\n self._resolver_file = self.parent_helper.tempfile(nameservers, pipe=False)\n return self._resolver_file\n\n def gen_random_subdomains(self, n=50):\n delimiters = (\".\", \"-\")\n lengths = list(range(3, 8))\n for i in range(0, max(0, n - 5)):\n d = delimiters[i % len(delimiters)]\n l = lengths[i % len(lengths)]\n segments = [random.choice(self.devops_mutations) for _ in range(l)]\n segments.append(self.parent_helper.rand_string(length=8, digits=False))\n subdomain = d.join(segments)\n yield subdomain\n for _ in range(5):\n yield self.parent_helper.rand_string(length=8, digits=False)\n\n def has_excessive_digits(self, d):\n \"\"\"\n Identifies dns names with excessive numbers, e.g.:\n - w1-2-3.evilcorp.com\n - ptr1234.evilcorp.com\n \"\"\"\n is_ptr = self.parent_helper.is_ptr(d)\n digits = self.digit_regex.findall(d)\n excessive_digits = len(digits) > 2\n long_digits = any(len(d) > 3 for d in digits)\n return is_ptr or excessive_digits or long_digits\n" }, { "path": "bbot/core/helpers/dns/dns.py", "content": "import dns\nimport logging\nimport dns.exception\nimport dns.asyncresolver\nfrom cachetools import LFUCache\nfrom radixtarget import RadixTarget\n\nfrom bbot.errors import DNSError\nfrom bbot.core.engine import EngineClient\nfrom bbot.core.helpers.async_helpers import async_cachedmethod\nfrom ..misc import clean_dns_record, is_ip, is_domain, is_dns_name\n\nfrom .engine import DNSEngine\n\nlog = logging.getLogger(\"bbot.core.helpers.dns\")\n\n\nclass DNSHelper(EngineClient):\n SERVER_CLASS = DNSEngine\n ERROR_CLASS = DNSError\n\n \"\"\"Helper class for DNS-related operations within BBOT.\n\n This class provides mechanisms for host resolution, wildcard domain detection, event tagging, and more.\n It centralizes all DNS-related activities in BBOT, offering both synchronous and asynchronous methods\n for DNS resolution, as well as various utilities for batch resolution and DNS query filtering.\n\n Attributes:\n parent_helper: A reference to the instantiated `ConfigAwareHelper` (typically `scan.helpers`).\n resolver (BBOTAsyncResolver): An asynchronous DNS resolver tailored for BBOT with rate-limiting capabilities.\n timeout (int): The timeout value for DNS queries. Defaults to 5 seconds.\n retries (int): The number of retries for failed DNS queries. Defaults to 1.\n abort_threshold (int): The threshold for aborting after consecutive failed queries. Defaults to 50.\n runaway_limit (int): Maximum allowed distance for consecutive DNS resolutions. Defaults to 5.\n all_rdtypes (list): A list of DNS record types to be considered during operations.\n wildcard_ignore (tuple): Domains to be ignored during wildcard detection.\n wildcard_tests (int): Number of tests to be run for wildcard detection. Defaults to 5.\n _wildcard_cache (dict): Cache for wildcard detection results.\n _dns_cache (LRUCache): Cache for DNS resolution results, limited in size.\n resolver_file (Path): File containing system's current resolver nameservers.\n\n Args:\n parent_helper: The parent helper object with configuration details and utilities.\n\n Raises:\n DNSError: If an issue arises when creating the BBOTAsyncResolver instance.\n\n Examples:\n >>> dns_helper = DNSHelper(parent_config)\n >>> resolved_host = dns_helper.resolver.resolve(\"example.com\")\n \"\"\"\n\n def __init__(self, parent_helper):\n self.parent_helper = parent_helper\n self.config = self.parent_helper.config\n self.dns_config = self.config.get(\"dns\", {})\n engine_debug = self.config.get(\"engine\", {}).get(\"debug\", False)\n super().__init__(server_kwargs={\"config\": self.config}, debug=engine_debug)\n\n # resolver\n self.timeout = self.dns_config.get(\"timeout\", 5)\n self.resolver = dns.asyncresolver.Resolver()\n self.resolver.rotate = True\n self.resolver.timeout = self.timeout\n self.resolver.lifetime = self.timeout\n\n self.runaway_limit = self.dns_config.get(\"runaway_limit\", 5)\n\n # wildcard handling\n self.wildcard_disable = self.dns_config.get(\"wildcard_disable\", False)\n self.wildcard_ignore = RadixTarget()\n for d in self.dns_config.get(\"wildcard_ignore\", []):\n self.wildcard_ignore.insert(d)\n\n # copy the system's current resolvers to a text file for tool use\n self.system_resolvers = dns.resolver.Resolver().nameservers\n # TODO: DNS server speed test (start in background task)\n self.resolver_file = self.parent_helper.tempfile(self.system_resolvers, pipe=False)\n\n # brute force helper\n self._brute = None\n\n self._is_wildcard_cache = LFUCache(maxsize=1000)\n self._is_wildcard_domain_cache = LFUCache(maxsize=1000)\n\n async def resolve(self, query, **kwargs):\n return await self.run_and_return(\"resolve\", query=query, **kwargs)\n\n async def resolve_raw(self, query, **kwargs):\n return await self.run_and_return(\"resolve_raw\", query=query, **kwargs)\n\n async def resolve_batch(self, queries, **kwargs):\n agen = self.run_and_yield(\"resolve_batch\", queries=queries, **kwargs)\n while 1:\n try:\n yield await agen.__anext__()\n except (StopAsyncIteration, GeneratorExit):\n await agen.aclose()\n break\n\n async def resolve_raw_batch(self, queries):\n agen = self.run_and_yield(\"resolve_raw_batch\", queries=queries)\n while 1:\n try:\n yield await agen.__anext__()\n except (StopAsyncIteration, GeneratorExit):\n await agen.aclose()\n break\n\n @property\n def brute(self):\n if self._brute is None:\n from .brute import DNSBrute\n\n self._brute = DNSBrute(self.parent_helper)\n return self._brute\n\n @async_cachedmethod(\n lambda self: self._is_wildcard_cache,\n key=lambda query, rdtypes, raw_dns_records: (query, tuple(sorted(rdtypes)), bool(raw_dns_records)),\n )\n async def is_wildcard(self, query, rdtypes, raw_dns_records=None):\n \"\"\"\n Use this method to check whether a *host* is a wildcard entry\n\n This can reliably tell the difference between a valid DNS record and a wildcard within a wildcard domain.\n\n If you want to know whether a domain is using wildcard DNS, use `is_wildcard_domain()` instead.\n\n Args:\n query (str): The hostname to check for a wildcard entry.\n ips (list, optional): List of IPs to compare against, typically obtained from a previous DNS resolution of the query.\n rdtype (str, optional): The DNS record type (e.g., \"A\", \"AAAA\") to consider during the check.\n\n Returns:\n dict: A dictionary indicating if the query is a wildcard for each checked DNS record type.\n Keys are DNS record types like \"A\", \"AAAA\", etc.\n Values are tuples where the first element is a boolean indicating if the query is a wildcard,\n and the second element is the wildcard parent if it's a wildcard.\n\n Raises:\n ValueError: If only one of `ips` or `rdtype` is specified or if no valid IPs are specified.\n\n Examples:\n >>> is_wildcard(\"www.github.io\")\n {\"A\": (True, \"github.io\"), \"AAAA\": (True, \"github.io\")}\n\n >>> is_wildcard(\"www.evilcorp.com\", ips=[\"93.184.216.34\"], rdtype=\"A\")\n {\"A\": (False, \"evilcorp.com\")}\n\n Note:\n `is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive)\n \"\"\"\n query = self._wildcard_prevalidation(query)\n if not query:\n return {}\n\n # skip check if the query is a domain\n if is_domain(query):\n return {}\n\n return await self.run_and_return(\"is_wildcard\", query=query, rdtypes=rdtypes, raw_dns_records=raw_dns_records)\n\n @async_cachedmethod(\n lambda self: self._is_wildcard_domain_cache, key=lambda domain, rdtypes: (domain, tuple(sorted(rdtypes)))\n )\n async def is_wildcard_domain(self, domain, rdtypes):\n domain = self._wildcard_prevalidation(domain)\n if not domain:\n return {}\n\n return await self.run_and_return(\"is_wildcard_domain\", domain=domain, rdtypes=rdtypes)\n\n def _wildcard_prevalidation(self, host):\n if self.wildcard_disable:\n return False\n\n host = clean_dns_record(host)\n # skip check if it's an IP or a plain hostname\n if is_ip(host) or \".\" not in host:\n return False\n\n # skip if query isn't a dns name\n if not is_dns_name(host):\n return False\n\n # skip check if the query's parent domain is excluded in the config\n wildcard_ignore = self.wildcard_ignore.search(host)\n if wildcard_ignore:\n log.debug(f\"Skipping wildcard detection on {host} because {wildcard_ignore} is excluded in the config\")\n return False\n\n return host\n\n async def _mock_dns(self, mock_data, custom_lookup_fn=None):\n from .mock import MockResolver\n\n self.resolver = MockResolver(mock_data, custom_lookup_fn=custom_lookup_fn)\n await self.run_and_return(\"_mock_dns\", mock_data=mock_data, custom_lookup_fn=custom_lookup_fn)\n" }, { "path": "bbot/core/helpers/dns/engine.py", "content": "import os\nimport dns\nimport time\nimport asyncio\nimport logging\nimport traceback\nfrom cachetools import LRUCache\nfrom contextlib import suppress\n\nfrom bbot.core.engine import EngineServer\nfrom bbot.core.helpers.async_helpers import NamedLock\nfrom bbot.core.helpers.dns.helpers import extract_targets\nfrom bbot.core.helpers.misc import (\n is_ip,\n rand_string,\n parent_domain,\n domain_parents,\n)\n\n\nlog = logging.getLogger(\"bbot.core.helpers.dns.engine.server\")\n\nall_rdtypes = [\"A\", \"AAAA\", \"SRV\", \"MX\", \"NS\", \"SOA\", \"CNAME\", \"TXT\"]\n\n\nclass DNSEngine(EngineServer):\n CMDS = {\n 0: \"resolve\",\n 1: \"resolve_raw\",\n 2: \"resolve_batch\",\n 3: \"resolve_raw_batch\",\n 4: \"is_wildcard\",\n 5: \"is_wildcard_domain\",\n 99: \"_mock_dns\",\n }\n\n def __init__(self, socket_path, config={}, debug=False):\n super().__init__(socket_path, debug=debug)\n\n self.config = config\n self.dns_config = self.config.get(\"dns\", {})\n # config values\n self.timeout = self.dns_config.get(\"timeout\", 5)\n self.retries = self.dns_config.get(\"retries\", 1)\n self.abort_threshold = self.dns_config.get(\"abort_threshold\", 50)\n\n # resolver\n self.resolver = dns.asyncresolver.Resolver()\n self.resolver.rotate = True\n self.resolver.timeout = self.timeout\n self.resolver.lifetime = self.timeout\n\n # skip certain queries\n dns_omit_queries = self.dns_config.get(\"omit_queries\", None)\n if not dns_omit_queries:\n dns_omit_queries = []\n self.dns_omit_queries = {}\n for d in dns_omit_queries:\n d = d.split(\":\")\n if len(d) == 2:\n rdtype, query = d\n rdtype = rdtype.upper()\n query = query.lower()\n try:\n self.dns_omit_queries[rdtype].add(query)\n except KeyError:\n self.dns_omit_queries[rdtype] = {query}\n\n # wildcard handling\n self.wildcard_ignore = self.dns_config.get(\"wildcard_ignore\", None)\n if not self.wildcard_ignore:\n self.wildcard_ignore = []\n self.wildcard_ignore = tuple([str(d).strip().lower() for d in self.wildcard_ignore])\n self.wildcard_tests = self.dns_config.get(\"wildcard_tests\", 5)\n self._wildcard_cache = {}\n # since wildcard detection takes some time, This is to prevent multiple\n # modules from kicking off wildcard detection for the same domain at the same time\n self._wildcard_lock = NamedLock()\n\n self._dns_connectivity_lock = None\n self._last_dns_success = None\n self._last_connectivity_warning = time.time()\n # keeps track of warnings issued for wildcard detection to prevent duplicate warnings\n self._dns_warnings = set()\n self._errors = {}\n self._debug = self.dns_config.get(\"debug\", False)\n self._dns_cache = LRUCache(maxsize=10000)\n\n async def resolve(self, query, **kwargs):\n \"\"\"Resolve DNS names and IP addresses to their corresponding results.\n\n This is a high-level function that can translate a given domain name to its associated IP addresses\n or an IP address to its corresponding domain names. It's structured for ease of use within modules\n and will abstract away most of the complexity of DNS resolution, returning a simple set of results.\n\n Args:\n query (str): The domain name or IP address to resolve.\n **kwargs: Additional arguments to be passed to the resolution process.\n\n Returns:\n set: A set containing resolved domain names or IP addresses.\n\n Examples:\n >>> results = await resolve(\"1.2.3.4\")\n {\"evilcorp.com\"}\n\n >>> results = await resolve(\"evilcorp.com\")\n {\"1.2.3.4\", \"dead::beef\"}\n \"\"\"\n results = set()\n try:\n answers, errors = await self.resolve_raw(query, **kwargs)\n for answer in answers:\n for _, host in extract_targets(answer):\n results.add(host)\n except BaseException:\n self.log.trace(f\"Caught exception in resolve({query}, {kwargs}):\")\n self.log.trace(traceback.format_exc())\n raise\n\n self.debug(f\"Results for {query} with kwargs={kwargs}: {results}\")\n return results\n\n async def resolve_raw(self, query, **kwargs):\n \"\"\"Resolves the given query to its associated DNS records.\n\n This function is a foundational method for DNS resolution in this class. It understands both IP addresses and\n hostnames and returns their associated records in a raw format provided by the dnspython library.\n\n Args:\n query (str): The IP address or hostname to resolve.\n type (str or list[str], optional): Specifies the DNS record type(s) to fetch. Can be a single type like 'A'\n or a list like ['A', 'AAAA']. If set to 'any', 'all', or '*', it fetches all supported types. If not\n specified, the function defaults to fetching 'A' and 'AAAA' records.\n **kwargs: Additional arguments that might be passed to the resolver.\n\n Returns:\n tuple: A tuple containing two lists:\n - list: A list of tuples where each tuple consists of a record type string (like 'A') and the associated\n raw dnspython answer.\n - list: A list of tuples where each tuple consists of a record type string and the associated error if\n there was an issue fetching the record.\n\n Examples:\n >>> await resolve_raw(\"8.8.8.8\")\n ([('PTR', )], [])\n\n >>> await resolve_raw(\"dns.google\")\n (, [])\n \"\"\"\n # DNS over TCP is more reliable\n # But setting this breaks DNS resolution on Ubuntu because systemd-resolve doesn't support TCP\n # kwargs[\"tcp\"] = True\n try:\n query = str(query).strip()\n kwargs.pop(\"rdtype\", None)\n rdtype = kwargs.pop(\"type\", \"A\")\n if is_ip(query):\n return await self._resolve_ip(query, **kwargs)\n else:\n return await self._resolve_hostname(query, rdtype=rdtype, **kwargs)\n except BaseException:\n self.log.trace(f\"Caught exception in resolve_raw({query}, {kwargs}):\")\n self.log.trace(traceback.format_exc())\n raise\n\n async def _resolve_hostname(self, query, **kwargs):\n \"\"\"Translate a hostname into its corresponding IP addresses.\n\n This is the foundational function for converting a domain name into its associated IP addresses. It's designed\n for internal use within the class and handles retries, caching, and a variety of error/timeout scenarios.\n It also respects certain configurations that might ask to skip certain types of queries. Results are returned\n in the default dnspython answer object format.\n\n Args:\n query (str): The hostname to resolve.\n rdtype (str, optional): The type of DNS record to query (e.g., 'A', 'AAAA'). Defaults to 'A'.\n retries (int, optional): The number of times to retry on failure. Defaults to class-wide `retries`.\n use_cache (bool, optional): Whether to check the cache before trying a fresh resolution. Defaults to True.\n **kwargs: Additional arguments that might be passed to the resolver.\n\n Returns:\n tuple: A tuple containing:\n - list: A list of resolved IP addresses.\n - list: A list of errors encountered during the resolution process.\n\n Examples:\n >>> results, errors = await _resolve_hostname(\"google.com\")\n (, [])\n \"\"\"\n self.debug(f\"Resolving {query} with kwargs={kwargs}\")\n results = []\n errors = []\n rdtype = kwargs.get(\"rdtype\", \"A\")\n\n # skip certain queries if requested\n if rdtype in self.dns_omit_queries:\n if any(h == query or query.endswith(f\".{h}\") for h in self.dns_omit_queries[rdtype]):\n self.debug(f\"Skipping {rdtype}:{query} because it's omitted in the config\")\n return results, errors\n\n parent = parent_domain(query)\n retries = kwargs.pop(\"retries\", self.retries)\n use_cache = kwargs.pop(\"use_cache\", True)\n tries_left = int(retries) + 1\n parent_hash = hash((parent, rdtype))\n dns_cache_hash = hash((query, rdtype))\n while tries_left > 0:\n try:\n if use_cache:\n results = self._dns_cache.get(dns_cache_hash, [])\n if not results:\n error_count = self._errors.get(parent_hash, 0)\n if error_count >= self.abort_threshold:\n connectivity = await self._connectivity_check()\n if connectivity:\n self.log.verbose(\n f'Aborting query \"{query}\" because failed {rdtype} queries for \"{parent}\" ({error_count:,}) exceeded abort threshold ({self.abort_threshold:,})'\n )\n if parent_hash not in self._dns_warnings:\n self.log.verbose(\n f'Aborting future {rdtype} queries to \"{parent}\" because error count ({error_count:,}) exceeded abort threshold ({self.abort_threshold:,})'\n )\n self._dns_warnings.add(parent_hash)\n return results, errors\n results = await self._catch(self.resolver.resolve, query, **kwargs)\n if use_cache:\n self._dns_cache[dns_cache_hash] = results\n if parent_hash in self._errors:\n self._errors[parent_hash] = 0\n break\n except (\n dns.resolver.NoNameservers,\n dns.exception.Timeout,\n dns.resolver.LifetimeTimeout,\n TimeoutError,\n asyncio.exceptions.TimeoutError,\n ) as e:\n try:\n self._errors[parent_hash] += 1\n except KeyError:\n self._errors[parent_hash] = 1\n errors.append(e)\n # don't retry if we get a SERVFAIL\n if isinstance(e, dns.resolver.NoNameservers):\n break\n tries_left -= 1\n err_msg = (\n f'DNS error or timeout for {rdtype} query \"{query}\" ({self._errors[parent_hash]:,} so far): {e}'\n )\n if tries_left > 0:\n retry_num = (retries + 1) - tries_left\n self.debug(err_msg)\n self.debug(f\"Retry (#{retry_num}) resolving {query} with kwargs={kwargs}\")\n else:\n self.log.verbose(err_msg)\n\n if results:\n self._last_dns_success = time.time()\n self.debug(f\"Answers for {query} with kwargs={kwargs}: {list(results)}\")\n\n if errors:\n self.debug(f\"Errors for {query} with kwargs={kwargs}: {errors}\")\n\n return results, errors\n\n async def _resolve_ip(self, query, **kwargs):\n \"\"\"Translate an IP address into a corresponding DNS name.\n\n This is the most basic function that will convert an IP address into its associated domain name. It handles\n retries, caching, and multiple types of timeout/error scenarios internally. The function is intended for\n internal use and should not be directly called by modules without understanding its intricacies.\n\n Args:\n query (str): The IP address to be reverse-resolved.\n retries (int, optional): The number of times to retry on failure. Defaults to 0.\n use_cache (bool, optional): Whether to check the cache for the result before attempting resolution. Defaults to True.\n **kwargs: Additional arguments to be passed to the resolution process.\n\n Returns:\n tuple: A tuple containing:\n - list: A list of resolved domain names (in default dnspython answer format).\n - list: A list of errors encountered during resolution.\n\n Examples:\n >>> results, errors = await _resolve_ip(\"8.8.8.8\")\n (, [])\n \"\"\"\n self.debug(f\"Reverse-resolving {query} with kwargs={kwargs}\")\n retries = kwargs.pop(\"retries\", 0)\n use_cache = kwargs.pop(\"use_cache\", True)\n tries_left = int(retries) + 1\n results = []\n errors = []\n dns_cache_hash = hash((query, \"PTR\"))\n while tries_left > 0:\n try:\n if use_cache:\n results = self._dns_cache.get(dns_cache_hash, [])\n if not results:\n results = await self._catch(self.resolver.resolve_address, query, **kwargs)\n if use_cache:\n self._dns_cache[dns_cache_hash] = results\n break\n except (\n dns.resolver.NoNameservers,\n dns.exception.Timeout,\n dns.resolver.LifetimeTimeout,\n TimeoutError,\n asyncio.exceptions.TimeoutError,\n ) as e:\n errors.append(e)\n # don't retry if we get a SERVFAIL\n if isinstance(e, dns.resolver.NoNameservers):\n self.debug(f\"{e} (query={query}, kwargs={kwargs})\")\n break\n else:\n tries_left -= 1\n if tries_left > 0:\n retry_num = (retries + 2) - tries_left\n self.debug(f\"Retrying (#{retry_num}) {query} with kwargs={kwargs}\")\n\n if results:\n self._last_dns_success = time.time()\n\n return results, errors\n\n async def resolve_batch(self, queries, threads=10, **kwargs):\n \"\"\"\n A helper to execute a bunch of DNS requests.\n\n Args:\n queries (list): List of queries to resolve.\n **kwargs: Additional keyword arguments to pass to `resolve()`.\n\n Yields:\n tuple: A tuple containing the original query and its resolved value.\n\n Examples:\n >>> import asyncio\n >>> async def example_usage():\n ... async for result in resolve_batch(['www.evilcorp.com', 'evilcorp.com']):\n ... print(result)\n ('www.evilcorp.com', {'1.1.1.1'})\n ('evilcorp.com', {'2.2.2.2'})\n \"\"\"\n async for (args, _, _), responses in self.task_pool(\n self.resolve, args_kwargs=queries, threads=threads, global_kwargs=kwargs\n ):\n yield args[0], responses\n\n async def resolve_raw_batch(self, queries, threads=10, **kwargs):\n queries_kwargs = [[q[0], {\"type\": q[1]}] for q in queries]\n async for (args, kwargs, _), (answers, errors) in self.task_pool(\n self.resolve_raw, args_kwargs=queries_kwargs, threads=threads, global_kwargs=kwargs\n ):\n query = args[0]\n rdtype = kwargs[\"type\"]\n yield ((query, rdtype), (answers, errors))\n\n async def _catch(self, callback, *args, **kwargs):\n \"\"\"\n Asynchronously catches exceptions thrown during DNS resolution and logs them.\n\n This method wraps around a given asynchronous callback function to handle different\n types of DNS exceptions and general exceptions. It logs the exceptions for debugging\n and, in some cases, re-raises them.\n\n Args:\n callback (callable): The asynchronous function to be executed.\n *args: Positional arguments to pass to the callback.\n **kwargs: Keyword arguments to pass to the callback.\n\n Returns:\n Any: The return value of the callback function, or an empty list if an exception is caught.\n\n Raises:\n dns.resolver.NoNameservers: When no nameservers could be reached.\n \"\"\"\n try:\n return await callback(*args, **kwargs)\n except dns.resolver.NoNameservers:\n raise\n except (dns.exception.Timeout, dns.resolver.LifetimeTimeout, TimeoutError):\n self.log.debug(f\"DNS query with args={args}, kwargs={kwargs} timed out after {self.timeout} seconds\")\n raise\n except dns.exception.DNSException as e:\n self.debug(f\"{e} (args={args}, kwargs={kwargs})\")\n except Exception as e:\n self.log.warning(f\"Error in {callback.__qualname__}() with args={args}, kwargs={kwargs}: {e}\")\n self.log.trace(traceback.format_exc())\n return []\n\n async def is_wildcard(self, query, rdtypes, raw_dns_records=None):\n \"\"\"\n Use this method to check whether a *host* is a wildcard entry\n\n This can reliably tell the difference between a valid DNS record and a wildcard within a wildcard domain.\n\n It works by making a bunch of random DNS queries to the parent domain, compiling a list of wildcard IPs,\n then comparing those to the IPs of the host in question. If the host's IP matches the wildcard ones, it's a wildcard.\n\n If you want to know whether a domain is using wildcard DNS, use `is_wildcard_domain()` instead.\n\n Args:\n query (str): The hostname to check for a wildcard entry.\n rdtypes (list): The DNS record type (e.g., \"A\", \"AAAA\") to consider during the check.\n raw_dns_records (dict, optional): Dictionary of {rdtype: [answer1, answer2, ...], ...} containing raw dnspython answers for the query.\n\n Returns:\n dict: A dictionary indicating if the query is a wildcard for each checked DNS record type.\n Keys are DNS record types like \"A\", \"AAAA\", etc.\n Values are tuples where the first element is a boolean indicating if the query is a wildcard,\n and the second element is the wildcard parent if it's a wildcard.\n\n Examples:\n >>> is_wildcard(\"www.github.io\", rdtypes=[\"A\", \"AAAA\", \"MX\"])\n {\"A\": (True, \"github.io\"), \"AAAA\": (True, \"github.io\"), \"MX\": (False, \"github.io\")}\n\n >>> is_wildcard(\"www.evilcorp.com\", rdtypes=[\"A\"])\n {\"A\": (False, \"evilcorp.com\")}\n\n Note:\n `is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive)\n \"\"\"\n if isinstance(rdtypes, str):\n rdtypes = [rdtypes]\n\n result = {}\n\n # if the work of resolving hasn't been done yet, do it\n if raw_dns_records is None:\n raw_dns_records = {}\n queries = [(query, rdtype) for rdtype in rdtypes]\n async for (_, rdtype), (answers, errors) in self.resolve_raw_batch(queries):\n if answers:\n for answer in answers:\n try:\n raw_dns_records[rdtype].add(answer)\n except KeyError:\n raw_dns_records[rdtype] = {answer}\n else:\n if errors:\n self.debug(f\"Failed to resolve {query} ({rdtype}) during wildcard detection\")\n result[rdtype] = (\"ERROR\", query)\n\n # clean + process the raw records into a baseline\n baseline = {}\n baseline_raw = {}\n for rdtype, answers in raw_dns_records.items():\n for answer in answers:\n text_answer = answer.to_text()\n try:\n baseline_raw[rdtype].add(text_answer)\n except KeyError:\n baseline_raw[rdtype] = {text_answer}\n for _, host in extract_targets(answer):\n try:\n baseline[rdtype].add(host)\n except KeyError:\n baseline[rdtype] = {host}\n\n # if it's unresolved, it's a big nope\n if not raw_dns_records:\n return result\n\n # once we've resolved the base query and have IP addresses to work with\n # we can compare the IPs to the ones we have on file for wildcards\n\n # only bother to check the rdypes that actually resolve\n rdtypes_to_check = set(raw_dns_records)\n\n # for every parent domain, starting with the shortest\n parents = list(domain_parents(query))\n for parent in parents[::-1]:\n # check if the parent domain is set up with wildcards\n wildcard_results = await self.is_wildcard_domain(parent, rdtypes_to_check)\n\n # for every rdtype\n for rdtype in list(baseline_raw):\n # skip if we already found a wildcard for this rdtype\n if rdtype in result:\n continue\n\n # get our baseline IPs from above\n _baseline = baseline.get(rdtype, set())\n _baseline_raw = baseline_raw.get(rdtype, set())\n\n wildcard_rdtypes = wildcard_results.get(parent, {})\n wildcards = wildcard_rdtypes.get(rdtype, None)\n if wildcards is None:\n continue\n wildcards, wildcard_raw = wildcards\n\n if wildcard_raw:\n # skip this rdtype from now on\n rdtypes_to_check.remove(rdtype)\n\n # check if any of our baseline IPs are in the wildcard results\n is_wildcard = any(r in wildcards for r in _baseline)\n is_wildcard_raw = any(r in wildcard_raw for r in _baseline_raw)\n\n # if there are any matches, we have a wildcard\n if is_wildcard or is_wildcard_raw:\n result[rdtype] = (True, parent)\n else:\n # otherwise, it's still suspicious, because we had random stuff resolve at this level\n result[rdtype] = (\"POSSIBLE\", parent)\n\n # any rdtype that wasn't a wildcard, mark it as False\n for rdtype, answers in baseline_raw.items():\n if answers and rdtype not in result:\n result[rdtype] = (False, query)\n\n return result\n\n async def is_wildcard_domain(self, domain, rdtypes):\n \"\"\"\n Check whether a given host or its children make use of wildcard DNS entries. Wildcard DNS can have\n various implications, particularly in subdomain enumeration and subdomain takeovers.\n\n Args:\n domain (str): The domain to check for wildcard DNS entries.\n rdtypes (list): Which DNS record types to check.\n\n Returns:\n dict: A dictionary where the keys are the parent domains that have wildcard DNS entries,\n and the values are another dictionary of DNS record types (\"A\", \"AAAA\", etc.) mapped to\n sets of their resolved IP addresses.\n\n Examples:\n >>> is_wildcard_domain(\"github.io\")\n {\"github.io\": {\"A\": {\"1.2.3.4\"}, \"AAAA\": {\"dead::beef\"}}}\n\n >>> is_wildcard_domain(\"example.com\")\n {}\n \"\"\"\n if isinstance(rdtypes, str):\n rdtypes = [rdtypes]\n rdtypes = set(rdtypes)\n\n wildcard_results = {}\n # make a list of its parents\n parents = list(domain_parents(domain, include_self=True))\n # and check each of them, beginning with the highest parent (i.e. the root domain)\n for i, host in enumerate(parents[::-1]):\n host_results = {}\n queries = [((host, rdtype), {}) for rdtype in rdtypes]\n async for ((_, rdtype), _, _), (results, results_raw) in self.task_pool(\n self._is_wildcard_zone, args_kwargs=queries\n ):\n # if we hit a wildcard, we can skip this rdtype from now on\n if results_raw:\n rdtypes.remove(rdtype)\n host_results[rdtype] = results, results_raw\n\n if host_results:\n wildcard_results[host] = host_results\n\n return wildcard_results\n\n async def _is_wildcard_zone(self, host, rdtype):\n \"\"\"\n Check whether a specific DNS zone+rdtype has a wildcard configuration\n \"\"\"\n rdtype = rdtype.upper()\n\n # have we checked this host before?\n host_hash = hash((host, rdtype))\n async with self._wildcard_lock.lock(host_hash):\n # if we've seen this host before\n try:\n wildcard_results, wildcard_results_raw = self._wildcard_cache[host_hash]\n self.debug(f\"Got {host}:{rdtype} from cache\")\n except KeyError:\n wildcard_results = set()\n wildcard_results_raw = set()\n self.debug(f\"Checking if {host}:{rdtype} is a wildcard\")\n\n # determine if this is a wildcard domain\n # resolve a bunch of random subdomains of the same parent\n rand_queries = []\n for _ in range(self.wildcard_tests):\n rand_query = f\"{rand_string(digits=False, length=10)}.{host}\"\n rand_queries.append((rand_query, rdtype))\n\n async for (query, rdtype), (answers, errors) in self.resolve_raw_batch(rand_queries, use_cache=False):\n for answer in answers:\n # consider both the raw record\n wildcard_results_raw.add(answer.to_text())\n # and all the extracted hosts\n for _, t in extract_targets(answer):\n wildcard_results.add(t)\n\n if wildcard_results:\n self.log.info(f\"Encountered domain with wildcard DNS ({rdtype}): *.{host}\")\n else:\n self.debug(f\"Finished checking {host}:{rdtype}, it is not a wildcard\")\n self._wildcard_cache[host_hash] = wildcard_results, wildcard_results_raw\n\n return wildcard_results, wildcard_results_raw\n\n async def _is_wildcard(self, query, rdtypes, dns_children):\n if isinstance(rdtypes, str):\n rdtypes = [rdtypes]\n\n @property\n def dns_connectivity_lock(self):\n if self._dns_connectivity_lock is None:\n self._dns_connectivity_lock = asyncio.Lock()\n return self._dns_connectivity_lock\n\n async def _connectivity_check(self, interval=5):\n \"\"\"\n Periodically checks for an active internet connection by attempting DNS resolution.\n\n Args:\n interval (int, optional): The time interval, in seconds, at which to perform the check.\n Defaults to 5 seconds.\n\n Returns:\n bool: True if there is an active internet connection, False otherwise.\n\n Examples:\n >>> await _connectivity_check()\n True\n \"\"\"\n if self._last_dns_success is not None:\n if time.time() - self._last_dns_success < interval:\n return True\n dns_server_working = []\n async with self.dns_connectivity_lock:\n with suppress(Exception):\n dns_server_working = await self._catch(self.resolver.resolve, \"www.google.com\", rdtype=\"A\")\n if dns_server_working:\n self._last_dns_success = time.time()\n return True\n if time.time() - self._last_connectivity_warning > interval:\n self.log.warning(\"DNS queries are failing, please check your internet connection\")\n self._last_connectivity_warning = time.time()\n self._errors.clear()\n return False\n\n def debug(self, *args, **kwargs):\n if self._debug:\n self.log.trace(*args, **kwargs)\n\n @property\n def in_tests(self):\n return os.getenv(\"BBOT_TESTING\", \"\") == \"True\"\n\n async def _mock_dns(self, mock_data, custom_lookup_fn=None):\n from .mock import MockResolver\n\n def deserialize_function(func_source):\n assert self.in_tests, \"Can only mock when BBOT_TESTING=True\"\n if func_source is None:\n return None\n namespace = {}\n exec(func_source, {}, namespace)\n return namespace[\"custom_lookup\"]\n\n self.resolver = MockResolver(mock_data, custom_lookup_fn=deserialize_function(custom_lookup_fn))\n" }, { "path": "bbot/core/helpers/dns/helpers.py", "content": "import logging\n\nfrom bbot.core.helpers.regexes import dns_name_extraction_regex\nfrom bbot.core.helpers.misc import clean_dns_record, smart_decode\n\nlog = logging.getLogger(\"bbot.core.helpers.dns\")\n\n\n# the following are the result of a 1-day internet survey to find the top SRV records\n# the scan resulted in 36,282 SRV records. the count for each one is shown.\ncommon_srvs = [\n \"_sipfederationtls._tcp\", # 6909\n \"_sip._tls\", # 6853\n \"_autodiscover._tcp\", # 4268\n \"_xmpp-server._tcp\", # 1437\n \"_sip._tcp\", # 1193\n \"_sips._tcp\", # 1183\n \"_caldavs._tcp\", # 1179\n \"_carddavs._tcp\", # 1132\n \"_caldav._tcp\", # 1035\n \"_carddav._tcp\", # 1024\n \"_sip._udp\", # 1007\n \"_imaps._tcp\", # 1007\n \"_submission._tcp\", # 906\n \"_h323cs._tcp\", # 846\n \"_h323ls._udp\", # 782\n \"_xmpp-client._tcp\", # 689\n \"_pop3s._tcp\", # 394\n \"_jabber._tcp\", # 277\n \"_imap._tcp\", # 267\n \"_turn._udp\", # 256\n \"_pop3._tcp\", # 221\n \"_ldap._tcp\", # 213\n \"_smtps._tcp\", # 195\n \"_sipinternaltls._tcp\", # 192\n \"_vlmcs._tcp\", # 165\n \"_kerberos._udp\", # 163\n \"_kerberos._tcp\", # 148\n \"_kpasswd._udp\", # 128\n \"_kpasswd._tcp\", # 100\n \"_ntp._udp\", # 90\n \"_gc._tcp\", # 73\n \"_kerberos-master._udp\", # 66\n \"_ldap._tcp.dc._msdcs\", # 63\n \"_matrix._tcp\", # 62\n \"_smtp._tcp\", # 61\n \"_stun._udp\", # 57\n \"_kerberos._tcp.dc._msdcs\", # 54\n \"_ldap._tcp.gc._msdcs\", # 49\n \"_kerberos-adm._tcp\", # 44\n \"_ldap._tcp.pdc._msdcs\", # 43\n \"_kerberos-master._tcp\", # 43\n \"_http._tcp\", # 37\n \"_h323rs._tcp\", # 36\n \"_sipinternal._tcp\", # 35\n \"_turn._tcp\", # 33\n \"_stun._tcp\", # 33\n \"_h323ls._tcp\", # 33\n \"_x-puppet._tcp\", # 30\n \"_h323cs._udp\", # 27\n \"_stuns._tcp\", # 26\n \"_jabber-client._tcp\", # 25\n \"_x-puppet-ca._tcp\", # 22\n \"_ts3._udp\", # 22\n \"_minecraft._tcp\", # 22\n \"_turns._tcp\", # 21\n \"_ldaps._tcp\", # 21\n \"_xmpps-client._tcp\", # 20\n \"_https._tcp\", # 19\n \"_ftp._tcp\", # 19\n \"_xmpp-server._udp\", # 18\n \"_xmpp-client._udp\", # 17\n \"_jabber._udp\", # 17\n \"_jabber-client._udp\", # 17\n \"_xmpps-server._tcp\", # 15\n \"_finger._tcp\", # 14\n \"_stuns._udp\", # 12\n \"_hkp._tcp\", # 12\n \"_vlmcs._udp\", # 11\n \"_turns._udp\", # 11\n \"_tftp._udp\", # 11\n \"_ssh._tcp\", # 11\n \"_rtps._udp\", # 11\n \"_mysqlsrv._tcp\", # 11\n \"_hkps._tcp\", # 11\n \"_h323be._udp\", # 11\n \"_dns._tcp\", # 11\n \"_wss._tcp\", # 10\n \"_wpad._tcp\", # 10\n \"_whois._tcp\", # 10\n \"_webexconnect._tcp\", # 10\n \"_webexconnects._tcp\", # 10\n \"_vnc._tcp\", # 10\n \"_test._tcp\", # 10\n \"_telnet._tcp\", # 10\n \"_telnets._tcp\", # 10\n \"_teamspeak._tcp\", # 10\n \"_svns._tcp\", # 10\n \"_svcp._tcp\", # 10\n \"_smb._tcp\", # 10\n \"_sip-tls._tcp\", # 10\n \"_sftp._tcp\", # 10\n \"_secure-pop3._tcp\", # 10\n \"_secure-imap._tcp\", # 10\n \"_rtsp._tcp\", # 10\n \"_rtps._tcp\", # 10\n \"_rpc._tcp\", # 10\n \"_rfb._tcp\", # 10\n \"_raop._tcp\", # 10\n \"_pstn._tcp\", # 10\n \"_presence._tcp\", # 10\n \"_pkixrep._tcp\", # 10\n \"_pgprevokations._tcp\", # 10\n \"_pgpkeys._tcp\", # 10\n \"_ocsp._tcp\", # 10\n \"_nntp._tcp\", # 10\n \"_nfs._tcp\", # 10\n \"_netbios-ssn._tcp\", # 10\n \"_netbios-ns._tcp\", # 10\n \"_netbios-dgm._tcp\", # 10\n \"_mumble._tcp\", # 10\n \"_msrpc._tcp\", # 10\n \"_mqtts._tcp\", # 10\n \"_minecraft._udp\", # 10\n \"_iscsi._tcp\", # 10\n \"_ircs._tcp\", # 10\n \"_ipp._tcp\", # 10\n \"_ipps._tcp\", # 10\n \"_h323be._tcp\", # 10\n \"_gits._tcp\", # 10\n \"_ftps._tcp\", # 10\n \"_ftpes._tcp\", # 10\n \"_dnss._udp\", # 10\n \"_dnss._tcp\", # 10\n \"_diameter._tcp\", # 10\n \"_crl._tcp\", # 10\n \"_crls._tcp\", # 10\n \"_cmp._tcp\", # 10\n \"_certificates._tcp\", # 10\n \"_aix._tcp\", # 10\n \"_afpovertcp._tcp\", # 10\n \"_collab-edge._tls\", # 6\n \"_tcp\", # 5\n \"_client._smtp\", # 3\n \"_udp\", # 2\n \"_tls\", # 2\n \"_msdcs\", # 2\n \"_gc._msdcs\", # 2\n \"_ldaps._tcp.dc._msdcs\", # 1\n \"_kerberos._tcp.kdc._msdcs\", # 1\n \"_kerberos.tcp.dc._msdcs\", # 1\n \"_imap\", # 1\n \"_iax\", # 1\n]\n\n\ndef extract_targets(record):\n \"\"\"\n Extracts hostnames or IP addresses from a given DNS record.\n\n This method reads the DNS record's type and based on that, extracts the target\n hostnames or IP addresses it points to. The type of DNS record\n (e.g., \"A\", \"MX\", \"CNAME\", etc.) determines which fields are used for extraction.\n\n Args:\n record (dns.rdata.Rdata): The DNS record to extract information from.\n\n Returns:\n set: A set of tuples, each containing the DNS record type and the extracted value.\n\n Examples:\n >>> from dns.rrset import from_text\n >>> record = from_text('www.example.com', 3600, 'IN', 'A', '192.0.2.1')\n >>> extract_targets(record[0])\n {('A', '192.0.2.1')}\n\n >>> record = from_text('example.com', 3600, 'IN', 'MX', '10 mail.example.com.')\n >>> extract_targets(record[0])\n {('MX', 'mail.example.com')}\n\n \"\"\"\n results = set()\n\n def add_result(rdtype, _record):\n cleaned = clean_dns_record(_record)\n if cleaned:\n results.add((rdtype, cleaned))\n\n rdtype = str(record.rdtype.name).upper()\n if rdtype in (\"A\", \"AAAA\", \"NS\", \"CNAME\", \"PTR\"):\n add_result(rdtype, record)\n elif rdtype == \"SOA\":\n add_result(rdtype, record.mname)\n elif rdtype == \"MX\":\n add_result(rdtype, record.exchange)\n elif rdtype == \"SRV\":\n add_result(rdtype, record.target)\n elif rdtype == \"TXT\":\n for s in record.strings:\n s = smart_decode(s)\n for match in dns_name_extraction_regex.finditer(s):\n start, end = match.span()\n host = s[start:end]\n add_result(rdtype, host)\n elif rdtype == \"NSEC\":\n add_result(rdtype, record.next)\n else:\n log.warning(f'Unknown DNS record type \"{rdtype}\"')\n return results\n\n\ndef service_record(host, rdtype=None):\n \"\"\"\n Indicates that the provided host name and optional rdtype is an SRV or related service record.\n\n These types of records do/should not have A/AAAA/CNAME or similar records, and are simply used to advertise configuration information and/or policy information for different Internet facing services.\n\n This function exists to provide a consistent way in which to perform this test, rather than having duplicated code in multiple places in different modules.\n\n The response provides a way for modules to quickly test whether a host name is relevant and worth inspecting or using in context of what the module does.\n\n NOTE: While underscores are technically not supposed to exist in DNS names as per RFC's, they can be used, so we can't assume that a name that contains or starts with an underscore is a service record and so must check for specific strings.\n\n Args:\n host (string): A DNS host name\n\n Returns:\n bool: A boolean, True indicates that the host is an SRV or similar record, False indicates that it is not.\n\n Examples:\n >>> service_record('_xmpp._tcp.example.com')\n True\n\n >>> service_record('_custom._service.example.com', 'SRV')\n True\n\n >>> service_record('_dmarc.example.com')\n True\n\n >>> service_record('www.example.com')\n False\n \"\"\"\n\n # if we were providing an rdtype, check if it is SRV\n # NOTE: we don't care what the name is if rdtype == SRV\n if rdtype and str(rdtype).upper() == \"SRV\":\n return True\n\n # we did not receive rdtype, so we'll have to inspect host name parts\n parts = str(host).split(\".\")\n\n if not parts:\n return False\n\n # DMARC TXT records, e.g. _dmarc.example.com\n if parts[0] == \"_dmarc\":\n return True\n\n # MTA-STS TXT records, e.g. _mta-sts.example.com\n if parts[0] == \"_mta-sts\":\n return True\n\n if len(parts) < 2:\n return False\n\n # classic SRV record names, e.g. _ldap._tcp.example.com\n if parts[1] == \"_udp\" or parts[1] == \"_tcp\":\n return True\n\n # TLS indicating records, used by SMTP TLS-RPT etc, e.g. _smtp._tls.example.com\n if parts[1] == \"_tls\":\n return True\n\n # BIMI TXT records, e.g. selector._bimi.example.com\n if parts[1] == \"_bimi\":\n return True\n\n # DKIM TXT records, e.g. selector._domainkey.example.com\n if parts[1] == \"_domainkey\":\n return True\n\n return False\n" }, { "path": "bbot/core/helpers/dns/mock.py", "content": "import dns\nimport logging\n\nlog = logging.getLogger(\"bbot.core.helpers.dns.mock\")\n\n\nclass MockResolver:\n def __init__(self, mock_data=None, custom_lookup_fn=None):\n self.mock_data = mock_data if mock_data else {}\n self._custom_lookup_fn = custom_lookup_fn\n self.nameservers = [\"127.0.0.1\"]\n\n async def resolve_address(self, ipaddr, *args, **kwargs):\n modified_kwargs = {}\n modified_kwargs.update(kwargs)\n modified_kwargs[\"rdtype\"] = \"PTR\"\n return await self.resolve(str(dns.reversename.from_address(ipaddr)), *args, **modified_kwargs)\n\n def _lookup(self, query, rdtype):\n query = query.strip(\".\")\n ret = []\n if self._custom_lookup_fn is not None:\n answers = self._custom_lookup_fn(query, rdtype)\n if answers is not None:\n ret.extend(list(answers))\n answers = self.mock_data.get(query, {}).get(rdtype, [])\n if answers:\n ret.extend(list(answers))\n if not ret:\n raise dns.resolver.NXDOMAIN(f\"No answer found for {query} {rdtype}\")\n return ret\n\n def create_dns_response(self, query_name, answers, rdtype):\n query_name = query_name.strip(\".\")\n message_text = f\"\"\"id 1234\nopcode QUERY\nrcode NOERROR\nflags QR AA RD\n;QUESTION\n{query_name}. IN {rdtype}\n;ANSWER\"\"\"\n for answer in answers:\n if answer == \"\":\n answer = '\"\"'\n message_text += f\"\\n{query_name}. 1 IN {rdtype} {answer}\"\n\n message_text += \"\\n;AUTHORITY\\n;ADDITIONAL\\n\"\n message = dns.message.from_text(message_text)\n # log.verbose(message_text)\n return message\n\n async def resolve(self, query_name, rdtype=None):\n if rdtype is None:\n rdtype = \"A\"\n elif isinstance(rdtype, str):\n rdtype = rdtype.upper()\n else:\n rdtype = str(rdtype.name).upper()\n\n domain_name = dns.name.from_text(query_name)\n rdtype_obj = dns.rdatatype.from_text(rdtype)\n\n if \"_NXDOMAIN\" in self.mock_data and query_name in self.mock_data[\"_NXDOMAIN\"]:\n # Simulate the NXDOMAIN exception\n raise dns.resolver.NXDOMAIN\n\n try:\n answers = self._lookup(query_name, rdtype)\n log.verbose(f\"Answers for {query_name}:{rdtype}: {answers}\")\n response = self.create_dns_response(query_name, answers, rdtype)\n answer = dns.resolver.Answer(domain_name, rdtype_obj, dns.rdataclass.IN, response)\n return answer\n except dns.resolver.NXDOMAIN:\n return []\n" }, { "path": "bbot/core/helpers/files.py", "content": "import os\nimport logging\nimport traceback\nfrom contextlib import suppress\n\nfrom .misc import rm_at_exit\n\n\nlog = logging.getLogger(\"bbot.core.helpers.files\")\n\n\ndef tempfile(self, content, pipe=True, extension=None):\n \"\"\"\n Creates a temporary file or named pipe and populates it with content.\n\n Args:\n content (list, set, tuple, str): The content to populate the temporary file with.\n pipe (bool, optional): If True, a named pipe is used instead of a true file.\n This allows Python data to be piped directly into the process without taking up disk space.\n Defaults to True.\n\n Returns:\n str: The filepath of the created temporary file or named pipe.\n\n Examples:\n >>> tempfile([\"This\", \"is\", \"temp\", \"content\"])\n '/home/user/.bbot/temp/pgxml13bov87oqrvjz7a'\n\n >>> tempfile([\"Another\", \"temp\", \"file\"], pipe=False)\n '/home/user/.bbot/temp/someotherfile'\n \"\"\"\n filename = self.temp_filename(extension)\n rm_at_exit(filename)\n try:\n if type(content) not in (set, list, tuple):\n content = (content,)\n if pipe:\n os.mkfifo(filename)\n self.feed_pipe(filename, content, text=True)\n else:\n with open(filename, \"w\", errors=\"ignore\") as f:\n for c in content:\n line = f\"{self.smart_decode(c)}\\n\"\n f.write(line)\n except Exception as e:\n log.error(f\"Error creating temp file: {e}\")\n log.trace(traceback.format_exc())\n\n return filename\n\n\ndef _feed_pipe(self, pipe, content, text=True):\n \"\"\"\n Feeds content into a named pipe or file-like object.\n\n Args:\n pipe (str or file-like object): The named pipe or file-like object to feed the content into.\n content (iterable): The content to be written into the pipe or file.\n text (bool, optional): If True, the content is decoded using smart_decode function.\n If False, smart_encode function is used. Defaults to True.\n\n Notes:\n The method tries to determine if 'pipe' is a file-like object that has a 'write' method.\n If so, it writes directly to that object. Otherwise, it opens 'pipe' as a file for writing.\n \"\"\"\n try:\n if text:\n decode_fn = self.smart_decode\n newline = \"\\n\"\n else:\n decode_fn = self.smart_encode\n newline = b\"\\n\"\n try:\n if hasattr(pipe, \"write\"):\n try:\n for c in content:\n pipe.write(decode_fn(c) + newline)\n finally:\n with suppress(Exception):\n pipe.close()\n else:\n with open(pipe, \"w\") as p:\n for c in content:\n p.write(decode_fn(c) + newline)\n except BrokenPipeError:\n log.debug(\"Broken pipe in _feed_pipe()\")\n except ValueError:\n log.debug(f\"Error _feed_pipe(): {traceback.format_exc()}\")\n except KeyboardInterrupt:\n self.scan.stop()\n except Exception as e:\n log.error(f\"Error in _feed_pipe(): {e}\")\n log.trace(traceback.format_exc())\n\n\ndef feed_pipe(self, pipe, content, text=True):\n \"\"\"\n Starts a new thread to feed content into a named pipe or file-like object using _feed_pipe().\n\n Args:\n pipe (str or file-like object): The named pipe or file-like object to feed the content into.\n content (iterable): The content to be written into the pipe or file.\n text (bool, optional): If True, the content is decoded using smart_decode function.\n If False, smart_encode function is used. Defaults to True.\n \"\"\"\n t = self.preset.core.create_thread(\n target=self._feed_pipe,\n args=(pipe, content),\n kwargs={\"text\": text},\n daemon=True,\n custom_name=\"bbot feed_pipe()\",\n )\n t.start()\n\n\ndef tempfile_tail(self, callback):\n \"\"\"\n Create a named pipe and execute a callback function on each line that is written to the pipe.\n\n Useful for ingesting output from a program (e.g. nuclei) directly from a file in real-time as\n each line is written. The idea is you create the file with this function and then tell the CLI\n program to output to it as a normal output file. We are then able to scoop up the output line\n by line as it's written to our \"file\" (which is actually a named pipe, shhh! ;)\n\n Args:\n callback (Callable): A function that will be invoked with each line written to the pipe as its argument.\n\n Returns:\n str: The filename of the created named pipe.\n \"\"\"\n filename = self.temp_filename()\n rm_at_exit(filename)\n try:\n os.mkfifo(filename)\n t = self.preset.core.create_thread(\n target=tail, args=(filename, callback), daemon=True, custom_name=\"bbot tempfile_tail()\"\n )\n t.start()\n except Exception as e:\n log.error(f\"Error setting up tail for file {filename}: {e}\")\n log.trace(traceback.format_exc())\n return\n return filename\n\n\ndef tail(filename, callback):\n \"\"\"\n Continuously read lines from a file and execute a callback function on each line.\n\n Args:\n filename (str): The path of the file to tail.\n callback (Callable): A function to call on each line read from the file.\n\n Examples:\n >>> def print_callback(line):\n ... print(f\"Received: {line}\")\n >>> tail(\"/path/to/file\", print_callback)\n \"\"\"\n try:\n with open(filename, errors=\"ignore\") as f:\n for line in f:\n line = line.rstrip(\"\\r\\n\")\n callback(line)\n except Exception as e:\n log.error(f\"Error tailing file {filename}: {e}\")\n log.trace(traceback.format_exc())\n" }, { "path": "bbot/core/helpers/git.py", "content": "from pathlib import Path\n\n\ndef sanitize_git_repo(repo_folder: Path):\n # sanitizing the git config is infeasible since there are too many different ways to do evil things\n # instead, we move it out of .git and into the repo folder, so we don't miss any secrets etc. inside\n config_file = repo_folder / \".git\" / \"config\"\n if config_file.exists():\n config_file.rename(repo_folder / \"git_config_original\")\n # move the index file\n index_file = repo_folder / \".git\" / \"index\"\n if index_file.exists():\n index_file.rename(repo_folder / \"git_index_original\")\n # move the hooks folder\n hooks_folder = repo_folder / \".git\" / \"hooks\"\n if hooks_folder.exists():\n hooks_folder.rename(repo_folder / \"git_hooks_original\")\n" }, { "path": "bbot/core/helpers/helper.py", "content": "import os\nimport logging\nfrom pathlib import Path\nimport multiprocessing as mp\nfrom functools import partial\nfrom concurrent.futures import ProcessPoolExecutor\n\nfrom . import misc\nfrom .dns import DNSHelper\nfrom .web import WebHelper\nfrom .diff import HttpCompare\nfrom .regex import RegexHelper\nfrom .wordcloud import WordCloud\nfrom .interactsh import Interactsh\nfrom .yara_helper import YaraHelper\nfrom .depsinstaller import DepsInstaller\nfrom .async_helpers import get_event_loop\n\nfrom bbot.scanner.target import BaseTarget\n\nlog = logging.getLogger(\"bbot.core.helpers\")\n\n\nclass ConfigAwareHelper:\n \"\"\"\n Centralized helper class that provides unified access to various helper functions.\n\n This class serves as a convenient interface for accessing helper methods across different files.\n It is designed to be configuration-aware, allowing helper functions to utilize scan-specific\n configurations like rate-limits. The class leverages Python's `__getattribute__` magic method\n to provide seamless access to helper functions across various namespaces.\n\n Attributes:\n config (dict): Configuration settings for the BBOT scan instance.\n _scan (Scan): A BBOT scan instance.\n bbot_home (Path): Home directory for BBOT.\n cache_dir (Path): Directory for storing cache files.\n temp_dir (Path): Directory for storing temporary files.\n tools_dir (Path): Directory for storing tools, e.g. compiled binaries.\n lib_dir (Path): Directory for storing libraries.\n scans_dir (Path): Directory for storing scan results.\n wordlist_dir (Path): Directory for storing wordlists.\n current_dir (Path): The current working directory.\n keep_old_scans (int): The number of old scans to keep.\n\n Examples:\n >>> helper = ConfigAwareHelper(config)\n >>> ips = helper.dns.resolve(\"www.evilcorp.com\")\n \"\"\"\n\n from . import ntlm\n from . import regexes\n from . import validators\n from .files import tempfile, feed_pipe, _feed_pipe, tempfile_tail\n from .cache import cache_get, cache_put, cache_filename, is_cached\n from .command import run, run_live, _spawn_proc, _prepare_command_kwargs\n\n def __init__(self, preset):\n self.preset = preset\n self.bbot_home = self.preset.bbot_home\n self.cache_dir = self.bbot_home / \"cache\"\n self.temp_dir = self.bbot_home / \"temp\"\n self.tools_dir = self.bbot_home / \"tools\"\n self.lib_dir = self.bbot_home / \"lib\"\n self.scans_dir = self.bbot_home / \"scans\"\n self.wordlist_dir = Path(__file__).parent.parent.parent / \"wordlists\"\n self.current_dir = Path.cwd()\n self.keep_old_scans = self.config.get(\"keep_scans\", 20)\n self.mkdir(self.cache_dir)\n self.mkdir(self.temp_dir)\n self.mkdir(self.tools_dir)\n self.mkdir(self.lib_dir)\n\n self._loop = None\n\n # multiprocessing thread pool\n start_method = mp.get_start_method()\n if start_method != \"spawn\":\n self.warning(f\"Multiprocessing spawn method is set to {start_method}.\")\n\n # we spawn 1 fewer processes than cores\n # this helps to avoid locking up the system or competing with the main python process for cpu time\n num_processes = max(1, mp.cpu_count() - 1)\n self.process_pool = ProcessPoolExecutor(max_workers=num_processes)\n\n self._cloud = None\n\n self.re = RegexHelper(self)\n self.yara = YaraHelper(self)\n self._dns = None\n self._web = None\n self._cloudcheck = None\n self.config_aware_validators = self.validators.Validators(self)\n self.depsinstaller = DepsInstaller(self)\n self.word_cloud = WordCloud(self)\n self.dummy_modules = {}\n\n @property\n def dns(self):\n if self._dns is None:\n self._dns = DNSHelper(self)\n return self._dns\n\n @property\n def web(self):\n if self._web is None:\n self._web = WebHelper(self)\n return self._web\n\n @property\n def cloudcheck(self):\n if self._cloudcheck is None:\n from cloudcheck import CloudCheck\n\n self._cloudcheck = CloudCheck()\n return self._cloudcheck\n\n def bloom_filter(self, size):\n from .bloom import BloomFilter\n\n return BloomFilter(size)\n\n def interactsh(self, *args, **kwargs):\n return Interactsh(self, *args, **kwargs)\n\n def http_compare(\n self,\n url,\n allow_redirects=False,\n include_cache_buster=True,\n headers=None,\n cookies=None,\n method=\"GET\",\n data=None,\n json=None,\n timeout=10,\n ):\n return HttpCompare(\n url,\n self,\n allow_redirects=allow_redirects,\n include_cache_buster=include_cache_buster,\n headers=headers,\n cookies=cookies,\n timeout=timeout,\n method=method,\n data=data,\n json=json,\n )\n\n def temp_filename(self, extension=None):\n \"\"\"\n temp_filename() --> Path(\"/home/user/.bbot/temp/pgxml13bov87oqrvjz7a\")\n \"\"\"\n filename = self.rand_string(20)\n if extension is not None:\n filename = f\"{filename}.{extension}\"\n return self.temp_dir / filename\n\n def clean_old_scans(self):\n def _filter(x):\n return x.is_dir() and self.regexes.scan_name_regex.match(x.name)\n\n self.clean_old(self.scans_dir, keep=self.keep_old_scans, filter=_filter)\n\n def make_target(self, *targets, **kwargs):\n return BaseTarget(*targets, **kwargs)\n\n @property\n def config(self):\n return self.preset.config\n\n @property\n def web_config(self):\n return self.preset.web_config\n\n @property\n def scan(self):\n return self.preset.scan\n\n @property\n def loop(self):\n \"\"\"\n Get the current event loop\n \"\"\"\n if self._loop is None:\n self._loop = get_event_loop()\n return self._loop\n\n def run_in_executor(self, callback, *args, **kwargs):\n \"\"\"\n Run a synchronous task in the event loop's default thread pool executor\n\n Examples:\n Execute callback:\n >>> result = await self.helpers.run_in_executor(callback_fn, arg1, arg2)\n \"\"\"\n callback = partial(callback, **kwargs)\n return self.loop.run_in_executor(None, callback, *args)\n\n def run_in_executor_mp(self, callback, *args, **kwargs):\n \"\"\"\n Same as run_in_executor() except with a process pool executor\n Use only in cases where callback is CPU-bound\n\n Examples:\n Execute callback:\n >>> result = await self.helpers.run_in_executor_mp(callback_fn, arg1, arg2)\n \"\"\"\n callback = partial(callback, **kwargs)\n return self.loop.run_in_executor(self.process_pool, callback, *args)\n\n @property\n def in_tests(self):\n return os.environ.get(\"BBOT_TESTING\", \"\") == \"True\"\n\n def __getattribute__(self, attr):\n \"\"\"\n Do not be afraid, the angel said.\n\n Overrides Python's built-in __getattribute__ to provide convenient access to helper methods.\n\n This method first attempts to find an attribute within this class itself. If unsuccessful,\n it then looks in the 'misc', 'dns', and 'web' helper modules, in that order. If the attribute\n is still not found, an AttributeError is raised.\n\n Args:\n attr (str): The attribute name to look for.\n\n Returns:\n Any: The attribute value, if found.\n\n Raises:\n AttributeError: If the attribute is not found in any of the specified places.\n \"\"\"\n try:\n # first try self\n return super().__getattribute__(attr)\n except AttributeError:\n try:\n # then try misc\n return getattr(misc, attr)\n except AttributeError:\n try:\n # then try dns\n return getattr(self.dns, attr)\n except AttributeError:\n try:\n # then try web\n return getattr(self.web, attr)\n except AttributeError:\n try:\n # then try validators\n return getattr(self.validators, attr)\n except AttributeError:\n # then die\n raise AttributeError(f'Helper has no attribute \"{attr}\"')\n" }, { "path": "bbot/core/helpers/interactsh.py", "content": "# based on https://github.com/ElSicarius/interactsh-python/blob/main/sources/interactsh.py\nimport json\nimport base64\nimport random\nimport asyncio\nimport logging\nimport traceback\nfrom uuid import uuid4\n\nfrom Crypto.Hash import SHA256\nfrom Crypto.PublicKey import RSA\nfrom Crypto.Cipher import AES, PKCS1_OAEP\n\nfrom bbot.errors import InteractshError\n\nlog = logging.getLogger(\"bbot.core.helpers.interactsh\")\n\nserver_list = [\"oast.pro\", \"oast.live\", \"oast.site\", \"oast.online\", \"oast.fun\", \"oast.me\"]\n\n\nclass Interactsh:\n \"\"\"\n A pure python implementation of ProjectDiscovery's interact.sh.\n\n *\"Interactsh is an open-source tool for detecting out-of-band interactions. It is a tool designed to detect vulnerabilities that cause external interactions.\"*\n\n - https://app.interactsh.com\n - https://github.com/projectdiscovery/interactsh\n\n This class facilitates interactions with the interact.sh service for\n out-of-band data exfiltration and vulnerability confirmation. It allows\n for customization by accepting server and token parameters from the\n configuration provided by `parent_helper`.\n\n Attributes:\n parent_helper (ConfigAwareHelper): An instance of a helper class containing configuration data.\n server (str): The server to be used. If None (the default), a random server will be chosen from a predetermined list.\n correlation_id (str): An identifier to correlate requests and responses. Default is None.\n custom_server (str): Optional. A custom interact.sh server. Loaded from configuration.\n token (str): Optional. A token for interact.sh API. Loaded from configuration.\n _poll_task (AsyncTask): The task responsible for polling the interact.sh server.\n\n Examples:\n ```python\n # instantiate interact.sh client (no requests are sent yet)\n >>> interactsh_client = self.helpers.interactsh()\n # register with an interact.sh server\n >>> interactsh_domain = await interactsh_client.register()\n [INFO] Registering with interact.sh server: oast.me\n [INFO] Successfully registered to interactsh server oast.me with correlation_id rg99x2f860h5466ou3so [rg99x2f860h5466ou3so86i07n1m3013k.oast.me]\n # simulate an out-of-band interaction\n >>> await self.helpers.request(f\"https://{interactsh_domain}/test\")\n # wait for out-of-band interaction to be registered\n >>> await asyncio.sleep(10)\n >>> data_list = await interactsh_client.poll()\n >>> print(data_list)\n [\n {\n \"protocol\": \"dns\",\n \"unique-id\": \"rg99x2f860h5466ou3so86i07n1m3013k\",\n \"full-id\": \"rg99x2f860h5466ou3so86i07n1m3013k\",\n \"q-type\": \"A\",\n \"raw-request\": \"...\",\n \"remote-address\": \"1.2.3.4\",\n \"timestamp\": \"2023-09-15T21:09:23.187226851Z\"\n },\n {\n \"protocol\": \"http\",\n \"unique-id\": \"rg99x2f860h5466ou3so86i07n1m3013k\",\n \"full-id\": \"rg99x2f860h5466ou3so86i07n1m3013k\",\n \"raw-request\": \"GET /test HTTP/1.1 ...\",\n \"remote-address\": \"1.2.3.4\",\n \"timestamp\": \"2023-09-15T21:09:24.155677967Z\"\n }\n ]\n # finally, shut down the client\n >>> await interactsh_client.deregister()\n ```\n \"\"\"\n\n def __init__(self, parent_helper, poll_interval=10):\n self.parent_helper = parent_helper\n self.server = None\n self.correlation_id = None\n self.custom_server = self.parent_helper.config.get(\"interactsh_server\", None)\n self.token = self.parent_helper.config.get(\"interactsh_token\", None)\n self.poll_interval = poll_interval\n self._poll_task = None\n\n async def register(self, callback=None):\n \"\"\"\n Registers the instance with an interact.sh server and sets up polling.\n\n Generates RSA keys for secure communication, builds a correlation ID,\n and sends a POST request to an interact.sh server to register. Optionally,\n starts an asynchronous polling task to listen for interactions.\n\n Args:\n callback (callable, optional): A function to be called each time new interactions are received.\n\n Returns:\n str: The registered domain for out-of-band interactions.\n\n Raises:\n InteractshError: If registration with an interact.sh server fails.\n\n Examples:\n >>> interactsh_client = self.helpers.interactsh()\n >>> registered_domain = await interactsh_client.register()\n [INFO] Registering with interact.sh server: oast.me\n [INFO] Successfully registered to interactsh server oast.me with correlation_id rg99x2f860h5466ou3so [rg99x2f860h5466ou3so86i07n1m3013k.oast.me]\n \"\"\"\n rsa = RSA.generate(1024)\n\n self.public_key = rsa.publickey().exportKey()\n self.private_key = rsa.exportKey()\n\n encoded_public_key = base64.b64encode(self.public_key).decode(\"utf8\")\n\n uuid = uuid4().hex.ljust(33, \"a\")\n guid = \"\".join(i if i.isdigit() else chr(ord(i) + random.randint(0, 20)) for i in uuid)\n\n self.correlation_id = guid[:20]\n self.secret = str(uuid4())\n headers = {}\n\n if self.custom_server:\n if not self.token:\n log.verbose(\"Interact.sh token is not set\")\n else:\n headers[\"Authorization\"] = self.token\n self.server_list = [str(self.custom_server)]\n else:\n self.server_list = random.sample(server_list, k=len(server_list))\n for server in self.server_list:\n log.info(f\"Registering with interact.sh server: {server}\")\n data = {\n \"public-key\": encoded_public_key,\n \"secret-key\": self.secret,\n \"correlation-id\": self.correlation_id,\n }\n r = await self.parent_helper.request(\n f\"https://{server}/register\", headers=headers, json=data, method=\"POST\"\n )\n if r is None:\n continue\n try:\n msg = r.json().get(\"message\", \"\")\n assert \"registration successful\" in msg\n except Exception:\n log.debug(f\"Failed to register with interactsh server {self.server}\")\n continue\n self.server = server\n self.domain = f\"{guid}.{self.server}\"\n break\n\n if not self.server:\n raise InteractshError(\"Failed to register with an interactsh server\")\n\n log.info(\n f\"Successfully registered to interactsh server {self.server} with correlation_id {self.correlation_id} [{self.domain}]\"\n )\n\n if callable(callback):\n self._poll_task = asyncio.create_task(self.poll_loop(callback))\n\n return self.domain\n\n async def deregister(self):\n \"\"\"\n Deregisters the instance from the interact.sh server and cancels the polling task.\n\n Sends a POST request to the server to deregister, using the correlation ID\n and secret key generated during registration. Optionally, if a polling\n task was started, it is cancelled.\n\n Raises:\n InteractshError: If required information is missing or if deregistration fails.\n\n Examples:\n >>> await interactsh_client.deregister()\n \"\"\"\n if not self.server or not self.correlation_id or not self.secret:\n raise InteractshError(\"Missing required information to deregister\")\n\n headers = {}\n if self.token:\n headers[\"Authorization\"] = self.token\n\n data = {\"secret-key\": self.secret, \"correlation-id\": self.correlation_id}\n\n r = await self.parent_helper.request(\n f\"https://{self.server}/deregister\", headers=headers, json=data, method=\"POST\"\n )\n\n if self._poll_task is not None:\n self._poll_task.cancel()\n\n if \"success\" not in getattr(r, \"text\", \"\"):\n raise InteractshError(f\"Failed to de-register with interactsh server {self.server}\")\n\n async def poll(self):\n \"\"\"\n Polls the interact.sh server for interactions tied to the current instance.\n\n Sends a GET request to the server to fetch interactions associated with the\n current correlation_id and secret key. Returned interactions are decrypted\n using an AES key provided by the server response.\n\n Raises:\n InteractshError: If required information for polling is missing.\n\n Returns:\n list: A list of decrypted interaction data dictionaries.\n\n Examples:\n >>> data_list = await interactsh_client.poll()\n >>> print(data_list)\n [\n {\n \"protocol\": \"dns\",\n \"unique-id\": \"rg99x2f860h5466ou3so86i07n1m3013k\",\n ...\n },\n ...\n ]\n \"\"\"\n if not self.server or not self.correlation_id or not self.secret:\n raise InteractshError(\"Missing required information to poll\")\n\n headers = {}\n if self.token:\n headers[\"Authorization\"] = self.token\n\n try:\n r = await self.parent_helper.request(\n f\"https://{self.server}/poll?id={self.correlation_id}&secret={self.secret}\",\n headers=headers,\n timeout=15,\n )\n if r is None:\n raise InteractshError(\"Error polling interact.sh: No response from server\")\n\n ret = []\n data_list = r.json().get(\"data\", None)\n if data_list:\n aes_key = r.json()[\"aes_key\"]\n\n for data in data_list:\n decrypted_data = self._decrypt(aes_key, data)\n ret.append(decrypted_data)\n return ret\n except Exception as e:\n raise InteractshError(f\"Error polling interact.sh: {e}\")\n\n async def poll_loop(self, callback):\n \"\"\"\n Starts a polling loop to continuously check for interactions with the interact.sh server.\n\n Continuously polls the interact.sh server for interactions tied to the current instance,\n using the `poll` method. When interactions are received, it executes the given callback\n function with each interaction data.\n\n Parameters:\n callback (callable): The function to be called for every interaction received from the server.\n\n Returns:\n awaitable: An awaitable object that executes the internal `_poll_loop` method.\n\n Examples:\n >>> await interactsh_client.poll_loop(my_callback)\n \"\"\"\n async with self.parent_helper.scan._acatch(context=self._poll_loop):\n return await self._poll_loop(callback)\n\n async def _poll_loop(self, callback):\n while 1:\n if self.parent_helper.scan.stopping:\n await asyncio.sleep(1)\n continue\n data_list = []\n try:\n data_list = await self.poll()\n except InteractshError as e:\n log.warning(e)\n log.trace(traceback.format_exc())\n if not data_list:\n await asyncio.sleep(self.poll_interval)\n continue\n for data in data_list:\n if data:\n await self.parent_helper.execute_sync_or_async(callback, data)\n\n def _decrypt(self, aes_key, data):\n \"\"\"\n Decrypts and returns the data received from the interact.sh server.\n\n Uses RSA and AES for decrypting the data. RSA with PKCS1_OAEP and SHA256 is used to decrypt the AES key,\n and then AES (CTR mode) is used to decrypt the actual data payload.\n\n Parameters:\n aes_key (str): The AES key for decryption, encrypted with RSA and base64 encoded.\n data (str): The data payload to decrypt, which is base64 encoded and AES encrypted.\n\n Returns:\n dict: The decrypted data, loaded as a JSON object.\n\n Examples:\n >>> decrypted_data = self._decrypt(aes_key, data)\n \"\"\"\n private_key = RSA.importKey(self.private_key)\n cipher = PKCS1_OAEP.new(private_key, hashAlgo=SHA256)\n aes_plain_key = cipher.decrypt(base64.b64decode(aes_key))\n decode = base64.b64decode(data)\n bs = AES.block_size\n iv = decode[:bs]\n ciphertext = decode[bs:]\n cryptor = AES.new(key=aes_plain_key, mode=AES.MODE_CTR, nonce=b\"\", initial_value=iv)\n plain_text = cryptor.decrypt(ciphertext)\n return json.loads(plain_text)\n" }, { "path": "bbot/core/helpers/libmagic.py", "content": "import puremagic\n\n\ndef get_magic_info(file):\n magic_detections = puremagic.magic_file(file)\n if magic_detections:\n magic_detections.sort(key=lambda x: x.confidence, reverse=True)\n detection = magic_detections[0]\n return detection.extension, detection.mime_type, detection.name, detection.confidence\n return \"\", \"\", \"\", 0\n\n\ndef get_compression(mime_type):\n mime_type = mime_type.lower()\n # from https://github.com/cdgriffith/puremagic/blob/master/puremagic/magic_data.json\n compression_map = {\n \"application/arj\": \"arj\", # ARJ archive\n \"application/binhex\": \"binhex\", # BinHex encoded file\n \"application/epub+zip\": \"zip\", # EPUB book (Zip archive)\n \"application/fictionbook2+zip\": \"zip\", # FictionBook 2.0 (Zip)\n \"application/fictionbook3+zip\": \"zip\", # FictionBook 3.0 (Zip)\n \"application/gzip\": \"gzip\", # Gzip compressed file\n \"application/java-archive\": \"zip\", # Java Archive (JAR)\n \"application/pak\": \"pak\", # PAK archive\n \"application/vnd.android.package-archive\": \"zip\", # Android package (APK)\n \"application/vnd.comicbook-rar\": \"rar\", # Comic book archive (RAR)\n \"application/vnd.comicbook+zip\": \"zip\", # Comic book archive (Zip)\n \"application/vnd.ms-cab-compressed\": \"cab\", # Microsoft Cabinet archive\n \"application/vnd.palm\": \"palm\", # Palm OS data\n \"application/vnd.rar\": \"rar\", # RAR archive\n \"application/x-7z-compressed\": \"7z\", # 7-Zip archive\n \"application/x-ace\": \"ace\", # ACE archive\n \"application/x-alz\": \"alz\", # ALZip archive\n \"application/x-arc\": \"arc\", # ARC archive\n \"application/x-archive\": \"ar\", # Unix archive\n \"application/x-bzip2\": \"bzip2\", # Bzip2 compressed file\n \"application/x-compress\": \"compress\", # Unix compress file\n \"application/x-cpio\": \"cpio\", # CPIO archive\n \"application/x-gzip\": \"gzip\", # Gzip compressed file\n \"application/x-itunes-ipa\": \"zip\", # iOS application archive (IPA)\n \"application/x-java-pack200\": \"pack200\", # Java Pack200 archive\n \"application/x-lha\": \"lha\", # LHA archive\n \"application/x-lrzip\": \"lrzip\", # Long Range ZIP\n \"application/x-lz4-compressed-tar\": \"lz4\", # LZ4 compressed Tar archive\n \"application/x-lz4\": \"lz4\", # LZ4 compressed file\n \"application/x-lzip\": \"lzip\", # Lzip compressed file\n \"application/x-lzma\": \"lzma\", # LZMA compressed file\n \"application/x-par2\": \"par2\", # PAR2 recovery file\n \"application/x-qpress\": \"qpress\", # Qpress archive\n \"application/x-rar-compressed\": \"rar\", # RAR archive\n \"application/x-sit\": \"sit\", # StuffIt archive\n \"application/x-stuffit\": \"sit\", # StuffIt archive\n \"application/x-tar\": \"tar\", # Tar archive\n \"application/x-tgz\": \"tgz\", # Gzip compressed Tar archive\n \"application/x-webarchive\": \"zip\", # Web archive (Zip)\n \"application/x-xar\": \"xar\", # XAR archive\n \"application/x-xz\": \"xz\", # XZ compressed file\n \"application/x-zip-compressed-fb2\": \"zip\", # Zip archive (FB2)\n \"application/x-zoo\": \"zoo\", # Zoo archive\n \"application/x-zstd-compressed-tar\": \"zstd\", # Zstandard compressed Tar archive\n \"application/zip\": \"zip\", # Zip archive\n \"application/zstd\": \"zstd\", # Zstandard compressed file\n }\n\n return compression_map.get(mime_type, \"\")\n" }, { "path": "bbot/core/helpers/misc.py", "content": "import os\nimport sys\nimport copy\nimport json\nimport math\nimport random\nimport string\nimport asyncio\nimport logging\nimport ipaddress\nimport regex as re\nimport subprocess as sp\n\nfrom pathlib import Path\nfrom contextlib import suppress\nfrom unidecode import unidecode # noqa F401\nfrom asyncio import create_task, gather, sleep, wait_for # noqa\nfrom urllib.parse import urlparse, quote, unquote, urlunparse, urljoin # noqa F401\n\nfrom .git import * # noqa F401\nfrom .url import * # noqa F401\nfrom ... import errors\nfrom . import regexes as bbot_regexes\nfrom .names_generator import random_name, names, adjectives # noqa F401\n\nlog = logging.getLogger(\"bbot.core.helpers.misc\")\n\n\ndef is_domain(d):\n \"\"\"\n Check if the given input represents a domain without subdomains.\n\n This function takes an input string `d` and returns True if it represents a domain without any subdomains.\n Otherwise, it returns False.\n\n Args:\n d (str): The input string containing the domain.\n\n Returns:\n bool: True if the input is a domain without subdomains, False otherwise.\n\n Examples:\n >>> is_domain(\"evilcorp.co.uk\")\n True\n\n >>> is_domain(\"www.evilcorp.co.uk\")\n False\n\n Notes:\n - Port, if present in input, is ignored.\n \"\"\"\n d, _ = split_host_port(d)\n if is_ip(d):\n return False\n extracted = tldextract(d)\n if extracted.top_domain_under_public_suffix:\n if not extracted.subdomain:\n return True\n else:\n return d.count(\".\") == 1\n return False\n\n\ndef is_subdomain(d):\n \"\"\"\n Check if the given input represents a subdomain.\n\n This function takes an input string `d` and returns True if it represents a subdomain.\n Otherwise, it returns False.\n\n Args:\n d (str): The input string containing the domain or subdomain.\n\n Returns:\n bool: True if the input is a subdomain, False otherwise.\n\n Examples:\n >>> is_subdomain(\"www.evilcorp.co.uk\")\n True\n\n >>> is_subdomain(\"evilcorp.co.uk\")\n False\n\n Notes:\n - Port, if present in input, is ignored.\n \"\"\"\n d, _ = split_host_port(d)\n if is_ip(d):\n return False\n extracted = tldextract(d)\n if extracted.top_domain_under_public_suffix:\n if extracted.subdomain:\n return True\n else:\n return d.count(\".\") > 1\n return False\n\n\ndef is_ptr(d):\n \"\"\"\n Check if the given input represents a PTR record domain.\n\n This function takes an input string `d` and returns True if it matches the PTR record format.\n Otherwise, it returns False.\n\n Args:\n d (str): The input string potentially representing a PTR record domain.\n\n Returns:\n bool: True if the input matches PTR record format, False otherwise.\n\n Examples:\n >>> is_ptr(\"wsc-11-22-33-44.evilcorp.com\")\n True\n\n >>> is_ptr(\"www2.evilcorp.com\")\n False\n \"\"\"\n return bool(bbot_regexes.ptr_regex.search(str(d)))\n\n\ndef is_url(u):\n \"\"\"\n Check if the given input represents a valid URL.\n\n This function takes an input string `u` and returns True if it matches any of the predefined URL formats.\n Otherwise, it returns False.\n\n Args:\n u (str): The input string potentially representing a URL.\n\n Returns:\n bool: True if the input matches a valid URL format, False otherwise.\n\n Examples:\n >>> is_url(\"https://evilcorp.com\")\n True\n\n >>> is_url(\"not-a-url\")\n False\n \"\"\"\n u = str(u)\n for r in bbot_regexes.event_type_regexes[\"URL\"]:\n if r.match(u):\n return True\n return False\n\n\nuri_regex = re.compile(r\"^([a-z0-9]{2,20})://\", re.I)\n\n\ndef is_uri(u, return_scheme=False):\n \"\"\"\n Check if the given input represents a URI and optionally return its scheme.\n\n This function takes an input string `u` and returns True if it matches a URI format.\n When `return_scheme` is True, it returns the URI scheme instead of a boolean.\n\n Args:\n u (str): The input string potentially representing a URI.\n return_scheme (bool, optional): Whether to return the URI scheme. Defaults to False.\n\n Returns:\n Union[bool, str]: True if the input matches a URI format; the URI scheme if `return_scheme` is True.\n\n Examples:\n >>> is_uri(\"http://evilcorp.com\")\n True\n\n >>> is_uri(\"ftp://evilcorp.com\")\n True\n\n >>> is_uri(\"evilcorp.com\")\n False\n\n >>> is_uri(\"ftp://evilcorp.com\", return_scheme=True)\n \"ftp\"\n \"\"\"\n match = uri_regex.match(u)\n if return_scheme:\n if match:\n return match.groups()[0].lower()\n return \"\"\n return bool(match)\n\n\ndef split_host_port(d):\n \"\"\"\n Parse a string containing a host and port into a tuple.\n\n This function takes an input string `d` and returns a tuple containing the host and port.\n The host is converted to its appropriate IP address type if possible. The port is inferred\n based on the scheme if not provided.\n\n Args:\n d (str): The input string containing the host and possibly the port.\n\n Returns:\n Tuple[Union[IPv4Address, IPv6Address, str], Optional[int]]: Tuple containing the host and port.\n\n Examples:\n >>> split_host_port(\"evilcorp.com:443\")\n (\"evilcorp.com\", 443)\n\n >>> split_host_port(\"192.168.1.1:443\")\n (IPv4Address('192.168.1.1'), 443)\n\n >>> split_host_port(\"[dead::beef]:443\")\n (IPv6Address('dead::beef'), 443)\n\n Notes:\n - If port is not provided, it is inferred based on the scheme:\n - For \"https\" and \"wss\", port 443 is used.\n - For \"http\" and \"ws\", port 80 is used.\n \"\"\"\n d = str(d)\n host = None\n port = None\n scheme = None\n\n # first, try to parse as an IP address\n if is_ip(d):\n return make_ip_type(d), port\n\n # if not an IP address, try to parse as a host:port\n match = bbot_regexes.split_host_port_regex.match(d)\n if match is None:\n raise ValueError(f'split_host_port() failed to parse \"{d}\"')\n scheme = match.group(\"scheme\")\n netloc = match.group(\"netloc\")\n if netloc is None:\n raise ValueError(f'split_host_port() failed to parse \"{d}\"')\n\n match = bbot_regexes.extract_open_port_regex.match(netloc)\n if match is None:\n raise ValueError(f'split_host_port() failed to parse netloc \"{netloc}\" (original value: {d})')\n\n host = match.group(2)\n if host is None:\n host = match.group(1)\n if host is None:\n raise ValueError(f'split_host_port() failed to locate host in netloc \"{netloc}\" (original value: {d})')\n\n port = match.group(3)\n if port is None and scheme is not None:\n scheme = scheme.lower()\n if scheme in (\"https\", \"wss\"):\n port = 443\n elif scheme in (\"http\", \"ws\"):\n port = 80\n elif port is not None:\n with suppress(ValueError):\n port = int(port)\n\n return make_ip_type(host), port\n\n\ndef parent_domain(d):\n \"\"\"\n Retrieve the parent domain of a given subdomain string.\n\n This function takes an input string `d` representing a subdomain and returns its parent domain.\n If the input does not represent a subdomain, it returns the input as is.\n\n Args:\n d (str): The input string representing a subdomain or domain.\n\n Returns:\n str: The parent domain of the subdomain, or the original input if it is not a subdomain.\n\n Examples:\n >>> parent_domain(\"www.internal.evilcorp.co.uk\")\n \"internal.evilcorp.co.uk\"\n\n >>> parent_domain(\"www.internal.evilcorp.co.uk:8080\")\n \"internal.evilcorp.co.uk:8080\"\n\n >>> parent_domain(\"www.evilcorp.co.uk\")\n \"evilcorp.co.uk\"\n\n >>> parent_domain(\"evilcorp.co.uk\")\n \"evilcorp.co.uk\"\n\n Notes:\n - Port, if present in input, is preserved in the output.\n \"\"\"\n host, port = split_host_port(d)\n if is_subdomain(d):\n return make_netloc(\".\".join(str(host).split(\".\")[1:]), port)\n return d\n\n\ndef domain_parents(d, include_self=False):\n \"\"\"\n Generate a list of parent domains for a given domain string.\n\n This function takes an input string `d` and generates a list of parent domains in decreasing order of specificity.\n If `include_self` is set to True, the list will also include the input domain if it is not a top-level domain.\n\n Args:\n d (str): The input string representing a domain or subdomain.\n include_self (bool, optional): Whether to include the input domain itself. Defaults to False.\n\n Yields:\n str: Parent domains of the input string in decreasing order of specificity.\n\n Examples:\n >>> list(domain_parents(\"test.www.evilcorp.co.uk\"))\n [\"www.evilcorp.co.uk\", \"evilcorp.co.uk\"]\n\n Notes:\n - Port, if present in input, is preserved in the output.\n \"\"\"\n\n parent = str(d)\n if include_self and not is_domain(parent):\n yield parent\n while 1:\n parent = parent_domain(parent)\n if is_subdomain(parent):\n yield parent\n continue\n elif is_domain(parent):\n yield parent\n break\n\n\ndef subdomain_depth(d):\n \"\"\"\n Calculate the depth of subdomains within a given domain name.\n\n Args:\n d (str): The domain name to analyze.\n\n Returns:\n int: The depth of the subdomain. For example, a hostname \"5.4.3.2.1.evilcorp.com\"\n has a subdomain depth of 5.\n \"\"\"\n subdomain, domain = split_domain(d)\n if not subdomain:\n return 0\n return subdomain.count(\".\") + 1\n\n\ndef parent_url(u):\n \"\"\"\n Retrieve the parent URL of a given URL.\n\n This function takes an input string `u` representing a URL and returns its parent URL.\n If the input URL does not have a parent (i.e., it's already the top-level), it returns None.\n\n Args:\n u (str): The input string representing a URL.\n\n Returns:\n Union[str, None]: The parent URL of the input URL, or None if it has no parent.\n\n Examples:\n >>> parent_url(\"https://evilcorp.com/sub/path/\")\n \"https://evilcorp.com/sub/\"\n\n >>> parent_url(\"https://evilcorp.com/\")\n None\n\n Notes:\n - Only the path component of the URL is modified.\n - All other components like scheme, netloc, query, and fragment are preserved.\n \"\"\"\n parsed = urlparse(u)\n path = Path(parsed.path)\n if path.parent == path:\n return None\n else:\n return urlunparse(parsed._replace(path=str(path.parent), query=\"\"))\n\n\ndef url_parents(u):\n \"\"\"\n Generate a list of parent URLs for a given URL string.\n\n This function takes an input string `u` representing a URL and generates a list of its parent URLs in decreasing order of specificity.\n\n Args:\n u (str): The input string representing a URL.\n\n Returns:\n List[str]: A list of parent URLs of the input URL in decreasing order of specificity.\n\n Examples:\n >>> url_parents(\"http://www.evilcorp.co.uk/admin/tools/cmd.php\")\n [\"http://www.evilcorp.co.uk/admin/tools/\", \"http://www.evilcorp.co.uk/admin/\", \"http://www.evilcorp.co.uk/\"]\n\n Notes:\n - The list is generated by continuously calling `parent_url` until it returns None.\n - All components of the URL except for the path are preserved.\n \"\"\"\n parent_list = []\n while 1:\n parent = parent_url(u)\n if parent is None:\n return parent_list\n elif parent not in parent_list:\n parent_list.append(parent)\n u = parent\n\n\ndef best_http_status(code1, code2):\n \"\"\"\n Determine the better HTTP status code between two given codes.\n\n The 'better' status code is considered based on typical usage and priority in HTTP communication.\n Lower codes are generally better than higher codes. Within the same class (e.g., 2xx), a lower code is better.\n Between different classes, the order of preference is 2xx > 3xx > 1xx > 4xx > 5xx.\n\n Args:\n code1 (int): The first HTTP status code.\n code2 (int): The second HTTP status code.\n\n Returns:\n int: The better HTTP status code between the two provided codes.\n\n Examples:\n >>> better_http_status(200, 404)\n 200\n >>> better_http_status(500, 400)\n 400\n >>> better_http_status(301, 302)\n 301\n \"\"\"\n\n # Classify the codes into their respective categories (1xx, 2xx, 3xx, 4xx, 5xx)\n def classify_code(code):\n return int(code) // 100\n\n class1 = classify_code(code1)\n class2 = classify_code(code2)\n\n # Priority order for classes\n priority_order = {2: 1, 3: 2, 1: 3, 4: 4, 5: 5}\n\n # Compare based on class priority\n p1 = priority_order.get(class1, 10)\n p2 = priority_order.get(class2, 10)\n if p1 != p2:\n return code1 if p1 < p2 else code2\n\n # If in the same class, the lower code is better\n return min(code1, code2)\n\n\ndef tldextract(data):\n \"\"\"\n Extracts the subdomain, domain, and suffix from a URL string.\n\n Args:\n data (str): The URL string to be processed.\n\n Returns:\n ExtractResult: A named tuple containing the subdomain, domain, and suffix.\n\n Examples:\n >>> tldextract(\"www.evilcorp.co.uk\")\n ExtractResult(subdomain='www', domain='evilcorp', suffix='co.uk')\n\n Notes:\n - Utilizes `smart_decode` to preprocess the data.\n - Makes use of the `tldextract` library for extraction.\n \"\"\"\n import tldextract as _tldextract\n\n return _tldextract.extract(smart_decode(data))\n\n\ndef split_domain(hostname):\n \"\"\"\n Splits the hostname into its subdomain and registered domain components.\n\n Args:\n hostname (str): The full hostname to be split.\n\n Returns:\n tuple: A tuple containing the subdomain and registered domain.\n\n Examples:\n >>> split_domain(\"www.internal.evilcorp.co.uk\")\n (\"www.internal\", \"evilcorp.co.uk\")\n\n Notes:\n - Utilizes the `tldextract` function to first break down the hostname.\n \"\"\"\n if is_ip(hostname):\n return (\"\", hostname)\n parsed = tldextract(hostname)\n subdomain = parsed.subdomain\n domain = parsed.top_domain_under_public_suffix\n if not domain:\n split = hostname.split(\".\")\n subdomain = \".\".join(split[:-2])\n domain = \".\".join(split[-2:])\n return (subdomain, domain)\n\n\ndef domain_stem(domain):\n \"\"\"\n Returns an abbreviated representation of the hostname by removing the TLD (Top-Level Domain).\n\n Args:\n domain (str): The full domain name to be abbreviated.\n\n Returns:\n str: An abbreviated domain string without the TLD.\n\n Examples:\n >>> domain_stem(\"www.evilcorp.com\")\n \"www.evilcorp\"\n\n Notes:\n - Utilizes the `tldextract` function for domain parsing.\n \"\"\"\n parsed = tldextract(str(domain))\n return \".\".join(parsed.subdomain.split(\".\") + parsed.domain.split(\".\")).strip(\".\")\n\n\ndef ip_network_parents(i, include_self=False):\n \"\"\"\n Generates all parent IP networks for a given IP address or network, optionally including the network itself.\n\n Args:\n i (str or ipaddress.IPv4Network/ipaddress.IPv6Network): The IP address or network to find parents for.\n include_self (bool, optional): Whether to include the network itself in the result. Default is False.\n\n Yields:\n ipaddress.IPv4Network or ipaddress.IPv6Network: Parent IP networks in descending order of prefix length.\n\n Examples:\n >>> list(ip_network_parents(\"192.168.1.1\"))\n [ipaddress.IPv4Network('192.168.1.0/31'), ipaddress.IPv4Network('192.168.1.0/30'), ... , ipaddress.IPv4Network('0.0.0.0/0')]\n\n Notes:\n - Utilizes Python's built-in `ipaddress` module for network operations.\n \"\"\"\n net = ipaddress.ip_network(i, strict=False)\n for i in range(net.prefixlen - (0 if include_self else 1), -1, -1):\n yield ipaddress.ip_network(f\"{net.network_address}/{i}\", strict=False)\n\n\ndef is_port(p):\n \"\"\"\n Checks if the given string represents a valid port number.\n\n Args:\n p (str or int): The port number to check.\n\n Returns:\n bool: True if the port number is valid, False otherwise.\n\n Examples:\n >>> is_port('80')\n True\n >>> is_port('70000')\n False\n \"\"\"\n\n p = str(p)\n return p and p.isdigit() and 0 <= int(p) <= 65535\n\n\ndef is_dns_name(d):\n \"\"\"\n Determines if the given string is a valid DNS name.\n\n Args:\n d (str): The string to be checked.\n\n Returns:\n bool: True if the string is a valid DNS name, False otherwise.\n\n Examples:\n >>> is_dns_name('www.example.com')\n True\n >>> is_dns_name('localhost')\n True\n >>> is_dns_name('192.168.1.1')\n False\n \"\"\"\n if is_ip(d):\n return False\n d = smart_decode(d)\n if bbot_regexes.dns_name_validation_regex.match(d):\n return True\n return False\n\n\ndef is_ip(d, version=None, include_network=False):\n \"\"\"\n Checks if the given string or object represents a valid IP address.\n\n Args:\n d (str or ipaddress.IPvXAddress): The IP address to check.\n include_network (bool, optional): Whether to include network types (IPv4Network or IPv6Network). Defaults to False.\n version (int, optional): The IP version to validate (4 or 6). Default is None.\n\n Returns:\n bool: True if the string or object is a valid IP address, False otherwise.\n\n Examples:\n >>> is_ip('192.168.1.1')\n True\n >>> is_ip('bad::c0de', version=6)\n True\n >>> is_ip('bad::c0de', version=4)\n False\n >>> is_ip('evilcorp.com')\n False\n \"\"\"\n ip = None\n try:\n ip = ipaddress.ip_address(d)\n except Exception:\n if include_network:\n try:\n ip = ipaddress.ip_network(d, strict=False)\n except Exception:\n pass\n if ip is not None and (version is None or ip.version == version):\n return True\n return False\n\n\ndef is_ip_type(i, network=None):\n \"\"\"\n Checks if the given object is an instance of an IPv4 or IPv6 type from the ipaddress module.\n\n Args:\n i (ipaddress._BaseV4 or ipaddress._BaseV6): The IP object to check.\n network (bool, optional): Whether to restrict the check to network types (IPv4Network or IPv6Network). Defaults to False.\n\n Returns:\n bool: True if the object is an instance of ipaddress._BaseV4 or ipaddress._BaseV6, False otherwise.\n\n Examples:\n >>> is_ip_type(ipaddress.IPv6Address('dead::beef'))\n True\n >>> is_ip_type(ipaddress.IPv4Network('192.168.1.0/24'))\n True\n >>> is_ip_type(\"192.168.1.0/24\")\n False\n \"\"\"\n if network is not None:\n is_network = ipaddress._BaseNetwork in i.__class__.__mro__\n if network:\n return is_network\n else:\n return not is_network\n return ipaddress._IPAddressBase in i.__class__.__mro__\n\n\ndef make_ip_type(s):\n \"\"\"\n Convert a string to its corresponding IP address or network type.\n\n This function attempts to convert the input string `s` into either an IPv4 or IPv6 address object,\n or an IPv4 or IPv6 network object. If none of these conversions are possible, the original string is returned.\n\n Args:\n s (str): The input string to be converted.\n\n Returns:\n Union[IPv4Address, IPv6Address, IPv4Network, IPv6Network, str]: The converted object or original string.\n\n Examples:\n >>> make_ip_type(\"dead::beef\")\n IPv6Address('dead::beef')\n\n >>> make_ip_type(\"192.168.1.0/24\")\n IPv4Network('192.168.1.0/24')\n\n >>> make_ip_type(\"evilcorp.com\")\n 'evilcorp.com'\n \"\"\"\n if not s:\n raise ValueError(f'Invalid hostname: \"{s}\"')\n # IP address\n with suppress(Exception):\n return ipaddress.ip_address(s)\n # IP network\n with suppress(Exception):\n return ipaddress.ip_network(s, strict=False)\n return s\n\n\ndef sha1(data):\n \"\"\"\n Computes the SHA-1 hash of the given data.\n\n Args:\n data (str or dict): The data to hash. If a dictionary, it is first converted to a JSON string with sorted keys.\n\n Returns:\n hashlib.Hash: SHA-1 hash object of the input data.\n\n Examples:\n >>> sha1(\"asdf\").hexdigest()\n '3da541559918a808c2402bba5012f6c60b27661c'\n \"\"\"\n from hashlib import sha1 as hashlib_sha1\n\n if isinstance(data, dict):\n data = json.dumps(data, sort_keys=True)\n return hashlib_sha1(smart_encode(data))\n\n\ndef smart_decode(data):\n \"\"\"\n Decodes the input data to a UTF-8 string, silently ignoring errors.\n\n Args:\n data (str or bytes): The data to decode.\n\n Returns:\n str: The decoded string.\n\n Examples:\n >>> smart_decode(b\"asdf\")\n \"asdf\"\n >>> smart_decode(\"asdf\")\n \"asdf\"\n \"\"\"\n if isinstance(data, bytes):\n return data.decode(\"utf-8\", errors=\"ignore\")\n else:\n return str(data)\n\n\ndef smart_encode(data):\n \"\"\"\n Encodes the input data to bytes using UTF-8 encoding, silently ignoring errors.\n\n Args:\n data (str or bytes): The data to encode.\n\n Returns:\n bytes: The encoded bytes.\n\n Examples:\n >>> smart_encode(\"asdf\")\n b\"asdf\"\n >>> smart_encode(b\"asdf\")\n b\"asdf\"\n \"\"\"\n if isinstance(data, bytes):\n return data\n return str(data).encode(\"utf-8\", errors=\"ignore\")\n\n\nencoded_regex = re.compile(r\"%[0-9a-fA-F]{2}|\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}|\\\\[ntrbv]\")\nbackslash_regex = re.compile(r\"(?P\\\\+)(?P[ntrvb])\")\n\n\ndef ensure_utf8_compliant(text):\n return text.encode(\"utf-8\", errors=\"ignore\").decode(\"utf-8\")\n\n\ndef recursive_decode(data, max_depth=5):\n \"\"\"\n Recursively decodes doubly or triply-encoded strings to their original form.\n\n Supports both URL-encoding and backslash-escapes (including unicode)\n\n Args:\n data (str): The data to decode.\n max_depth (int, optional): Maximum recursion depth for decoding. Defaults to 5.\n\n Returns:\n str: The decoded string.\n\n Examples:\n >>> recursive_decode(\"Hello%20world%21\")\n \"Hello world!\"\n >>> recursive_decode(\"Hello%20%5Cu041f%5Cu0440%5Cu0438%5Cu0432%5Cu0435%5Cu0442\")\n \"Hello ПŅ€Đ¸Đ˛ĐĩŅ‚\"\n >>> recursive_dcode(\"%5Cu0020%5Cu041f%5Cu0440%5Cu0438%5Cu0432%5Cu0435%5Cu0442%5Cu0021\")\n \" ПŅ€Đ¸Đ˛ĐĩŅ‚!\"\n \"\"\"\n import codecs\n\n # Decode newline and tab escapes\n data = backslash_regex.sub(\n lambda match: {\"n\": \"\\n\", \"t\": \"\\t\", \"r\": \"\\r\", \"b\": \"\\b\", \"v\": \"\\v\"}.get(match.group(\"char\")), data\n )\n data = smart_decode(data)\n if max_depth == 0:\n return data\n # Decode URL encoding\n data = unquote(data, errors=\"ignore\")\n # Decode Unicode escapes\n with suppress(UnicodeEncodeError):\n data = ensure_utf8_compliant(codecs.decode(data, \"unicode_escape\", errors=\"ignore\"))\n # Check if there's still URL-encoded or Unicode-escaped content\n if encoded_regex.search(data):\n # If yes, continue decoding\n return recursive_decode(data, max_depth=max_depth - 1)\n return data\n\n\ndef rand_string(length=10, digits=True, numeric_only=False):\n \"\"\"\n Generates a random string of specified length.\n\n Args:\n length (int, optional): The length of the random string. Defaults to 10.\n digits (bool, optional): Whether to include digits in the string. Defaults to True.\n numeric_only (bool, optional): Whether to generate a numeric-only string. Defaults to False.\n\n Returns:\n str: A random string of the specified length.\n\n Examples:\n >>> rand_string()\n 'c4hp4i9jzx'\n >>> rand_string(20)\n 'ap4rsdtg5iw7ey7y3oa5'\n >>> rand_string(30, digits=False)\n 'xdmyxtglqfzqktngkesyulwbfrihva'\n >>> rand_string(15, numeric_only=True)\n '934857349857395'\n \"\"\"\n if numeric_only:\n pool = string.digits\n elif digits:\n pool = string.ascii_lowercase + string.digits\n else:\n pool = string.ascii_lowercase\n\n return \"\".join(random.choice(pool) for _ in range(length))\n\n\ndef truncate_string(s: str, n: int) -> str:\n if not isinstance(s, str):\n raise ValueError(f\"Expected string, got {type(s)}\")\n if len(s) > n:\n return s[: n - 3] + \"...\"\n else:\n return s\n\n\ndef extract_params_json(json_data, compare_mode=\"getparam\"):\n \"\"\"\n Extracts key-value pairs from a JSON object and returns them as a set of tuples. Used by the `paramminer_headers` module.\n\n Args:\n json_data (str): JSON-formatted string containing key-value pairs.\n\n Returns:\n set: A set of tuples containing the keys and their corresponding values present in the JSON object.\n\n Raises:\n Returns an empty set if JSONDecodeError occurs.\n\n Examples:\n >>> extract_params_json('{\"a\": 1, \"b\": {\"c\": 2}}')\n {('a', 1), ('b', {'c': 2}), ('c', 2)}\n \"\"\"\n try:\n data = json.loads(json_data)\n except json.JSONDecodeError:\n return set()\n\n key_value_pairs = set()\n stack = [(data, \"\")]\n\n while stack:\n current_data, path = stack.pop()\n if isinstance(current_data, dict):\n for key, value in current_data.items():\n full_key = f\"{path}.{key}\" if path else key\n if isinstance(value, dict):\n stack.append((value, full_key))\n elif isinstance(value, list):\n stack.append((value, full_key))\n else:\n if validate_parameter(full_key, compare_mode):\n key_value_pairs.add((full_key, value))\n elif isinstance(current_data, list):\n for item in current_data:\n if isinstance(item, (dict, list)):\n stack.append((item, path))\n return key_value_pairs\n\n\ndef extract_params_xml(xml_data, compare_mode=\"getparam\"):\n \"\"\"\n Extracts tags and their text values from an XML object and returns them as a set of tuples.\n\n Args:\n xml_data (str): XML-formatted string containing elements.\n\n Returns:\n set: A set of tuples containing the tags and their corresponding sanitized text values present in the XML object.\n\n Raises:\n Returns an empty set if ParseError occurs.\n\n Examples:\n >>> extract_params_xml('value')\n {('root', None), ('child1', None), ('child2', 'value')}\n \"\"\"\n import xml.etree.ElementTree as ET\n\n try:\n root = ET.fromstring(xml_data)\n except ET.ParseError:\n return set()\n\n tag_value_pairs = set()\n stack = [root]\n\n while stack:\n current_element = stack.pop()\n if validate_parameter(current_element.tag, compare_mode):\n # Sanitize the text value\n text_value = current_element.text.strip() if current_element.text else None\n sanitized_value = quote(text_value, safe=\"\") if text_value else None\n tag_value_pairs.add((current_element.tag, sanitized_value))\n for child in current_element:\n stack.append(child)\n return tag_value_pairs\n\n\n# Define valid characters for each mode based on RFCs\nvalid_chars_dict = {\n \"header\": {\n chr(c) for c in range(33, 127) if chr(c) in \"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_\"\n },\n \"getparam\": {chr(c) for c in range(33, 127) if chr(c) not in \":/?#[]@!$&'()*+,;=\"},\n \"postparam\": {chr(c) for c in range(33, 127) if chr(c) not in \":/?#[]@!$&'()*+,;=\"},\n \"cookie\": {chr(c) for c in range(33, 127) if chr(c) not in '()<>@,;:\"/[]?={} \\t'},\n \"bodyjson\": set(chr(c) for c in range(33, 127) if chr(c) not in \":/?#[]@!$&'()*+,;=\"),\n}\n\n\ndef validate_parameter(param, compare_mode):\n compare_mode = compare_mode.lower()\n if len(param) > 100:\n return False\n if compare_mode not in valid_chars_dict:\n raise ValueError(f\"Invalid compare_mode: {compare_mode}\")\n allowed_chars = valid_chars_dict[compare_mode]\n return set(param).issubset(allowed_chars)\n\n\ndef extract_words(data, acronyms=True, wordninja=True, model=None, max_length=100, word_regexes=None):\n \"\"\"Intelligently extracts words from given data.\n\n This function uses regular expressions and optionally wordninja to extract words\n from a given text string. Thanks to wordninja it can handle concatenated words intelligently.\n\n Args:\n data (str): The data from which words are to be extracted.\n acronyms (bool, optional): Whether to include acronyms. Defaults to True.\n wordninja (bool, optional): Whether to use the wordninja library to split concatenated words. Defaults to True.\n model (object, optional): A custom wordninja model for special types of data such as DNS names.\n max_length (int, optional): Maximum length for a word to be included. Defaults to 100.\n word_regexes (list, optional): A list of compiled regular expression objects for word extraction. Defaults to None.\n\n Returns:\n set: A set of extracted words.\n\n Examples:\n >>> extract_words('blacklanternsecurity')\n {'black', 'lantern', 'security', 'bls', 'blacklanternsecurity'}\n \"\"\"\n import wordninja as _wordninja\n\n if word_regexes is None:\n word_regexes = bbot_regexes.word_regexes\n words = set()\n data = smart_decode(data)\n for r in word_regexes:\n for word in set(r.findall(data)):\n # blacklanternsecurity\n if len(word) <= max_length:\n words.add(word)\n\n # blacklanternsecurity --> ['black', 'lantern', 'security']\n # max_slice_length = 3\n for word in list(words):\n if wordninja:\n if model is None:\n model = _wordninja\n subwords = model.split(word)\n for subword in subwords:\n words.add(subword)\n # this section generates compound words\n # it is interesting but currently disabled the quality of its output doesn't quite justify its quantity\n # blacklanternsecurity --> ['black', 'lantern', 'security', 'blacklantern', 'lanternsecurity']\n # for s, e in combinations(range(len(subwords) + 1), 2):\n # if e - s <= max_slice_length:\n # subword_slice = \"\".join(subwords[s:e])\n # words.add(subword_slice)\n # blacklanternsecurity --> bls\n if acronyms:\n if len(subwords) > 1:\n words.add(\"\".join([c[0] for c in subwords if len(c) > 0]))\n\n return words\n\n\ndef closest_match(s, choices, n=1, cutoff=0.0):\n \"\"\"Finds the closest matching strings from a list of choices based on a given string.\n\n This function uses the difflib library to find the closest matches to a given string `s` from a list of `choices`.\n It can return either the single best match or a list of the top `n` best matches.\n\n Args:\n s (str): The string for which to find the closest match.\n choices (list): A list of strings to compare against.\n n (int, optional): The number of best matches to return. Defaults to 1.\n cutoff (float, optional): A float value that defines the similarity threshold. Strings with similarity below this value are not considered. Defaults to 0.0.\n\n Returns:\n str or list: Either the closest matching string or a list of the `n` closest matching strings.\n\n Examples:\n >>> closest_match(\"asdf\", [\"asd\", \"fds\"])\n 'asd'\n >>> closest_match(\"asdf\", [\"asd\", \"fds\", \"asdff\"], n=3)\n ['asdff', 'asd', 'fds']\n \"\"\"\n import difflib\n\n matches = difflib.get_close_matches(s, choices, n=n, cutoff=cutoff)\n if not choices or not matches:\n return\n if n == 1:\n return matches[0]\n return matches\n\n\ndef get_closest_match(s, choices, msg=None):\n \"\"\"Finds the closest match from a list of choices for a given string.\n\n This function is particularly useful for CLI applications where you want to validate flags or modules.\n\n Args:\n s (str): The string for which to find the closest match.\n choices (list): A list of strings to compare against.\n msg (str, optional): Additional message to prepend in the warning message. Defaults to None.\n loglevel (str, optional): The log level to use for the warning message. Defaults to \"HUGEWARNING\".\n exitcode (int, optional): The exit code to use when exiting the program. Defaults to 2.\n\n Examples:\n >>> get_closest_match(\"some_module\", [\"some_mod\", \"some_other_mod\"], msg=\"module\")\n # Output: Could not find module \"some_module\". Did you mean \"some_mod\"?\n \"\"\"\n if msg is None:\n msg = \"\"\n else:\n msg += \" \"\n closest = closest_match(s, choices)\n return f'Could not find {msg}\"{s}\". Did you mean \"{closest}\"?'\n\n\ndef kill_children(parent_pid=None, sig=None):\n \"\"\"\n Forgive me father for I have sinned\n \"\"\"\n import psutil\n import signal\n\n if sig is None:\n sig = signal.SIGTERM\n\n try:\n parent = psutil.Process(parent_pid)\n except psutil.NoSuchProcess:\n log.debug(f\"No such PID: {parent_pid}\")\n return\n log.debug(f\"Killing children of process ID {parent.pid}\")\n children = parent.children(recursive=True)\n for child in children:\n log.debug(f\"Killing child with PID {child.pid}\")\n if child.name != \"python\":\n try:\n child.send_signal(sig)\n except psutil.NoSuchProcess:\n log.debug(f\"No such PID: {child.pid}\")\n except psutil.AccessDenied:\n log.debug(f\"Error killing PID: {child.pid} - access denied\")\n log.debug(f\"Finished killing children of process ID {parent.pid}\")\n\n\ndef str_or_file(s):\n \"\"\"Reads a string or file and yields its content line-by-line.\n\n This function tries to open the given string `s` as a file and yields its lines.\n If it fails to open `s` as a file, it treats `s` as a regular string and yields it as is.\n\n Args:\n s (str): The string or file path to read.\n\n Yields:\n str: Either lines from the file or the original string.\n\n Examples:\n >>> list(str_or_file(\"file.txt\"))\n ['file_line1', 'file_line2', 'file_line3']\n >>> list(str_or_file(\"not_a_file\"))\n ['not_a_file']\n \"\"\"\n try:\n with open(s, errors=\"ignore\") as f:\n for line in f:\n yield line.rstrip(\"\\r\\n\")\n except OSError:\n yield s\n\n\nsplit_regex = re.compile(r\"[\\s,]\")\n\n\ndef chain_lists(\n l,\n try_files=False,\n msg=None,\n remove_blank=True,\n validate=False,\n validate_chars='<>:\"/\\\\|?*)',\n):\n \"\"\"Chains together list elements, allowing for entries separated by commas.\n\n This function takes a list `l` and flattens it by splitting its entries on commas.\n It also allows you to optionally open entries as files and add their contents to the list.\n\n The order of entries is preserved, and deduplication is performed automatically.\n\n Args:\n l (list): The list of strings to chain together.\n try_files (bool, optional): Whether to try to open entries as files. Defaults to False.\n msg (str, optional): An optional message to log when reading from a file. Defaults to None.\n remove_blank (bool, optional): Whether to remove blank entries from the list. Defaults to True.\n validate (bool, optional): Whether to perform validation for undesirable characters. Defaults to False.\n validate_chars (str, optional): When performing validation, what additional set of characters to block (blocks non-printable ascii automatically). Defaults to '<>:\"/\\\\|?*)'\n\n Returns:\n list: The list of chained elements.\n\n Raises:\n ValueError: If the input string contains invalid characters, when enabled (off by default).\n\n Examples:\n >>> chain_lists([\"a\", \"b,c,d\"])\n ['a', 'b', 'c', 'd']\n\n >>> chain_lists([\"a,file.txt\", \"c,d\"], try_files=True)\n ['a', 'f_line1', 'f_line2', 'f_line3', 'c', 'd']\n \"\"\"\n if isinstance(l, str):\n l = [l]\n final_list = {}\n for entry in l:\n for s in split_regex.split(entry):\n f = s.strip()\n if validate:\n if any((c in validate_chars) or (ord(c) < 32 and c != \" \") for c in f):\n raise ValueError(f\"Invalid character in string: {f}\")\n f_path = Path(f).resolve()\n if try_files and f_path.is_file():\n if msg is not None:\n new_msg = str(msg).format(filename=f_path)\n log.info(new_msg)\n for line in str_or_file(f):\n final_list[line] = None\n else:\n final_list[f] = None\n\n ret = list(final_list)\n if remove_blank:\n ret = [r for r in ret if r]\n return ret\n\n\ndef list_files(directory, filter=lambda x: True):\n \"\"\"Lists files in a given directory that meet a specified filter condition.\n\n Args:\n directory (str): The directory where to list files.\n filter (callable, optional): A function to filter the files. Defaults to a lambda function that returns True for all files.\n\n Yields:\n Path: A Path object for each file that meets the filter condition.\n\n Examples:\n >>> list(list_files(\"/tmp/test\"))\n [Path('/tmp/test/file1.py'), Path('/tmp/test/file2.txt')]\n\n >>> list(list_files(\"/tmp/test\"), filter=lambda f: f.suffix == \".py\")\n [Path('/tmp/test/file1.py')]\n \"\"\"\n directory = Path(directory).resolve()\n if directory.is_dir():\n for file in directory.iterdir():\n if file.is_file() and filter(file):\n yield file\n\n\ndef rm_at_exit(path):\n \"\"\"Registers a file to be automatically deleted when the program exits.\n\n Args:\n path (str or Path): The path to the file to be deleted upon program exit.\n\n Examples:\n >>> rm_at_exit(\"/tmp/test/file1.txt\")\n \"\"\"\n import atexit\n\n atexit.register(delete_file, path)\n\n\ndef delete_file(path):\n \"\"\"Deletes a file at the given path.\n\n Args:\n path (str or Path): The path to the file to be deleted.\n\n Note:\n This function suppresses all exceptions to ensure that the program continues running even if the file could not be deleted.\n\n Examples:\n >>> delete_file(\"/tmp/test/file1.txt\")\n \"\"\"\n with suppress(Exception):\n Path(path).unlink(missing_ok=True)\n\n\ndef read_file(filename):\n \"\"\"Reads a file line by line and yields each line without line breaks.\n\n Args:\n filename (str or Path): The path to the file to read.\n\n Yields:\n str: A line from the file without the trailing line break.\n\n Examples:\n >>> for line in read_file(\"/tmp/file.txt\"):\n ... print(line)\n file_line1\n file_line2\n file_line3\n \"\"\"\n with open(filename, errors=\"ignore\") as f:\n for line in f:\n yield line.rstrip(\"\\r\\n\")\n\n\ndef gen_numbers(n, padding=2):\n \"\"\"Generates numbers with variable padding and returns them as a set of strings.\n\n Args:\n n (int): The upper limit of numbers to generate, exclusive.\n padding (int, optional): The maximum number of digits to pad the numbers with. Defaults to 2.\n\n Returns:\n set: A set of string representations of numbers with varying degrees of padding.\n\n Examples:\n >>> gen_numbers(5)\n {'0', '00', '01', '02', '03', '04', '1', '2', '3', '4'}\n\n >>> gen_numbers(3, padding=3)\n {'0', '00', '000', '001', '002', '01', '02', '1', '2'}\n\n >>> gen_numbers(5, padding=1)\n {'0', '1', '2', '3', '4'}\n \"\"\"\n results = set()\n for i in range(n):\n for p in range(1, padding + 1):\n results.add(str(i).zfill(p))\n return results\n\n\ndef make_netloc(host, port=None):\n \"\"\"Constructs a network location string from a given host and port.\n\n Args:\n host (str): The hostname or IP address.\n port (int, optional): The port number. If None, the port is omitted.\n\n Returns:\n str: A network location string in the form 'host' or 'host:port'.\n\n Examples:\n >>> make_netloc(\"192.168.1.1\", None)\n \"192.168.1.1\"\n\n >>> make_netloc(\"192.168.1.1\", 443)\n \"192.168.1.1:443\"\n\n >>> make_netloc(\"evilcorp.com\", 80)\n \"evilcorp.com:80\"\n\n >>> make_netloc(\"dead::beef\", None)\n \"[dead::beef]\"\n\n >>> make_netloc(\"dead::beef\", 443)\n \"[dead::beef]:443\"\n \"\"\"\n if is_ip(host, version=6):\n host = f\"[{host}]\"\n if port is None:\n return str(host)\n return f\"{host}:{port}\"\n\n\ndef which(*executables, path=None):\n \"\"\"Finds the full path of the first available executable from a list of executables.\n\n Args:\n *executables (str): One or more executable names to search for.\n\n Returns:\n str: The full path of the first available executable, or None if none are found.\n\n Examples:\n >>> which(\"python\", \"python3\")\n \"/usr/bin/python\"\n \"\"\"\n import shutil\n\n for e in executables:\n location = shutil.which(e, path=path)\n if location:\n # Resolve directory symlinks but preserve the binary name.\n # This fixes native 7zip on Fedora where /usr/sbin -> bin symlink\n # causes codec loading to fail when invoked as /usr/sbin/7z.\n resolved_dir = os.path.realpath(os.path.dirname(location))\n return os.path.join(resolved_dir, os.path.basename(location))\n\n\ndef search_dict_by_key(key, d):\n \"\"\"Search a nested dictionary or list of dictionaries by a key and yield all matching values.\n\n Args:\n key (str): The key to search for.\n d (Union[dict, list]): The dictionary or list of dictionaries to search.\n\n Yields:\n Any: Yields all values that match the provided key.\n\n Examples:\n >>> d = {'a': 1, 'b': {'c': 2, 'a': 3}, 'd': [{'a': 4}, {'e': 5}]}\n >>> list(search_dict_by_key('a', d))\n [1, 3, 4]\n \"\"\"\n if isinstance(d, dict):\n if key in d:\n yield d[key]\n for v in d.values():\n yield from search_dict_by_key(key, v)\n elif isinstance(d, list):\n for v in d:\n yield from search_dict_by_key(key, v)\n\n\ndef search_format_dict(d, **kwargs):\n \"\"\"Recursively format string values in a dictionary or list using the provided keyword arguments.\n\n Args:\n d (Union[dict, list, str]): The dictionary, list, or string to format.\n **kwargs: Arbitrary keyword arguments used for string formatting.\n\n Returns:\n Union[dict, list, str]: The formatted dictionary, list, or string.\n\n Examples:\n >>> search_format_dict({\"test\": \"#{name} is awesome\"}, name=\"keanu\")\n {\"test\": \"keanu is awesome\"}\n \"\"\"\n if isinstance(d, dict):\n return {k: search_format_dict(v, **kwargs) for k, v in d.items()}\n elif isinstance(d, list):\n return [search_format_dict(v, **kwargs) for v in d]\n elif isinstance(d, str):\n for find, replace in kwargs.items():\n find = \"#{\" + str(find) + \"}\"\n d = d.replace(find, replace)\n return d\n\n\ndef search_dict_values(d, *regexes):\n \"\"\"Recursively search a dictionary's values based on provided regex patterns.\n\n Args:\n d (Union[dict, list, str]): The dictionary, list, or string to search.\n *regexes: Arbitrary number of compiled regex patterns.\n\n Returns:\n Generator: Yields matching values based on the provided regex patterns.\n\n Examples:\n >>> dict_to_search = {\n ... \"key1\": {\n ... \"key2\": [\n ... {\n ... \"key3\": \"A URL: https://www.evilcorp.com\"\n ... }\n ... ]\n ... }\n ... }\n >>> url_regexes = re.compile(r'https?://[^\\\\s<>\"]+|www\\\\.[^\\\\s<>\"]+')\n >>> list(search_dict_values(dict_to_search, url_regexes))\n [\"https://www.evilcorp.com\"]\n \"\"\"\n\n results = set()\n if isinstance(d, str):\n for r in regexes:\n for match in r.finditer(d):\n result = match.group()\n h = hash(result)\n if h not in results:\n results.add(h)\n yield result\n elif isinstance(d, dict):\n for v in d.values():\n yield from search_dict_values(v, *regexes)\n elif isinstance(d, list):\n for v in d:\n yield from search_dict_values(v, *regexes)\n\n\ndef grouper(iterable, n):\n \"\"\"\n Grouper groups an iterable into chunks of a given size.\n\n Args:\n iterable (iterable): The iterable to be chunked.\n n (int): The size of each chunk.\n\n Returns:\n iterator: An iterator that produces lists of elements from the original iterable, each of length `n` or less.\n\n Examples:\n >>> list(grouper('ABCDEFG', 3))\n [['A', 'B', 'C'], ['D', 'E', 'F'], ['G']]\n \"\"\"\n from itertools import islice\n\n iterable = iter(iterable)\n return iter(lambda: list(islice(iterable, n)), [])\n\n\ndef split_list(alist, wanted_parts=2):\n \"\"\"\n Splits a list into a specified number of approximately equal parts.\n\n Args:\n alist (list): The list to be split.\n wanted_parts (int): The number of parts to split the list into.\n\n Returns:\n list: A list of lists, each containing a portion of the original list.\n\n Examples:\n >>> split_list([1, 2, 3, 4, 5])\n [[1, 2], [3, 4, 5]]\n \"\"\"\n length = len(alist)\n return [alist[i * length // wanted_parts : (i + 1) * length // wanted_parts] for i in range(wanted_parts)]\n\n\ndef mkdir(path, check_writable=True, raise_error=True):\n \"\"\"\n Creates a directory and optionally checks if it's writable.\n\n Args:\n path (str or Path): The directory to create.\n check_writable (bool, optional): Whether to check if the directory is writable. Default is True.\n raise_error (bool, optional): Whether to raise an error if the directory creation fails. Default is True.\n\n Returns:\n bool: True if the directory is successfully created (and writable, if check_writable=True); otherwise False.\n\n Raises:\n DirectoryCreationError: Raised if the directory cannot be created and `raise_error=True`.\n\n Examples:\n >>> mkdir(\"/tmp/new_dir\")\n True\n >>> mkdir(\"/restricted_dir\", check_writable=False, raise_error=False)\n False\n \"\"\"\n path = Path(path).resolve()\n touchfile = path / f\".{rand_string()}\"\n try:\n path.mkdir(exist_ok=True, parents=True)\n if check_writable:\n touchfile.touch()\n return True\n except Exception as e:\n if raise_error:\n raise errors.DirectoryCreationError(f\"Failed to create directory at {path}: {e}\")\n finally:\n with suppress(Exception):\n touchfile.unlink()\n return False\n\n\ndef make_date(d=None, microseconds=False):\n \"\"\"\n Generates a string representation of the current date and time, with optional microsecond precision.\n\n Args:\n d (datetime, optional): A datetime object to convert. Defaults to the current date and time.\n microseconds (bool, optional): Whether to include microseconds. Defaults to False.\n\n Returns:\n str: A string representation of the date and time, formatted as YYYYMMDD_HHMM_SS or YYYYMMDD_HHMM_SSFFFFFF if microseconds are included.\n\n Examples:\n >>> make_date()\n \"20220707_1325_50\"\n >>> make_date(microseconds=True)\n \"20220707_1330_35167617\"\n \"\"\"\n from datetime import datetime\n\n f = \"%Y%m%d_%H%M_%S\"\n if microseconds:\n f += \"%f\"\n if d is None:\n d = datetime.now()\n return d.strftime(f)\n\n\ndef error_and_exit(msg):\n print(f\"\\n[!!!] {msg}\\n\")\n sys.exit(2)\n\n\ndef get_file_extension(s):\n \"\"\"\n Extracts the file extension from a given string representing a URL or file path.\n\n Args:\n s (str): The string from which to extract the file extension.\n\n Returns:\n str: The file extension, or an empty string if no extension is found.\n\n Examples:\n >>> get_file_extension(\"https://evilcorp.com/api/test.php\")\n \"php\"\n >>> get_file_extension(\"/etc/test.conf\")\n \"conf\"\n >>> get_file_extension(\"/etc/passwd\")\n \"\"\n \"\"\"\n s = str(s).lower().strip()\n rightmost_section = s.rsplit(\"/\", 1)[-1]\n if \".\" in rightmost_section:\n extension = rightmost_section.rsplit(\".\", 1)[-1]\n return extension\n return \"\"\n\n\ndef backup_file(filename, max_backups=10):\n \"\"\"\n Renames a file by appending an iteration number as a backup. Recursively renames\n files up to a specified maximum number of backups.\n\n Args:\n filename (str or pathlib.Path): The file to backup.\n max_backups (int, optional): The maximum number of backups to keep. Defaults to 10.\n\n Returns:\n pathlib.Path: The new backup filepath.\n\n Examples:\n >>> backup_file(\"/tmp/test.txt\")\n PosixPath(\"/tmp/test.0.txt\")\n >>> backup_file(\"/tmp/test.0.txt\")\n PosixPath(\"/tmp/test.1.txt\")\n >>> backup_file(\"/tmp/test.1.txt\")\n PosixPath(\"/tmp/test.2.txt\")\n \"\"\"\n filename = Path(filename).resolve()\n suffixes = [s.strip(\".\") for s in filename.suffixes]\n iteration = 1\n with suppress(Exception):\n iteration = min(max_backups - 1, max(0, int(suffixes[0]))) + 1\n suffixes = suffixes[1:]\n stem = filename.stem.split(\".\")[0]\n destination = filename.parent / f\"{stem}.{iteration}.{'.'.join(suffixes)}\"\n if destination.exists() and iteration < max_backups:\n backup_file(destination)\n if filename.exists():\n filename.rename(destination)\n return destination\n\n\ndef latest_mtime(d):\n \"\"\"Get the latest modified time of any file or sub-directory in a given directory.\n\n This function takes a directory path as an argument and returns the latest modified time\n of any contained file or directory, recursively. It's useful for sorting directories by\n modified time for cleanup or other purposes.\n\n Args:\n d (str or Path): The directory path to search for the latest modified time.\n\n Returns:\n float: The latest modified time in Unix timestamp format.\n\n Examples:\n >>> latest_mtime(\"~/.bbot/scans/mushy_susan\")\n 1659016928.2848816\n \"\"\"\n d = Path(d).resolve()\n mtimes = [d.lstat().st_mtime]\n if d.is_dir():\n to_list = d.glob(\"**/*\")\n else:\n to_list = [d]\n for e in to_list:\n mtimes.append(e.lstat().st_mtime)\n return max(mtimes)\n\n\ndef filesize(f):\n \"\"\"Get the file size of a given file.\n\n This function takes a file path as an argument and returns its size in bytes. If the path\n does not point to a file, the function returns 0.\n\n Args:\n f (str or Path): The file path for which to get the size.\n\n Returns:\n int: The size of the file in bytes, or 0 if the path does not point to a file.\n\n Examples:\n >>> filesize(\"/path/to/file.txt\")\n 1024\n \"\"\"\n f = Path(f)\n if f.is_file():\n return f.stat().st_size\n return 0\n\n\ndef rm_rf(f, ignore_errors=False):\n \"\"\"Recursively delete a directory\n\n Args:\n f (str or Path): The directory path to delete.\n\n Examples:\n >>> rm_rf(\"/tmp/httpx98323849\")\n \"\"\"\n import shutil\n\n shutil.rmtree(f, ignore_errors=ignore_errors)\n\n\ndef clean_old(d, keep=10, filter=lambda x: True, key=latest_mtime, reverse=True, raise_error=False):\n \"\"\"Clean up old files and directories within a given directory based on various filtering and sorting options.\n\n This function removes the oldest files and directories in the provided directory 'd' that exceed a specified\n threshold ('keep'). The items to be deleted can be filtered using a lambda function 'filter', and they are\n sorted by a key function, defaulting to latest modification time.\n\n Args:\n d (str or Path): The directory path to clean up.\n keep (int): The number of items to keep. Ones beyond this count will be removed.\n filter (Callable): A lambda function for filtering which files or directories to consider.\n Defaults to a lambda function that returns True for all.\n key (Callable): A function to sort the files and directories. Defaults to latest modification time.\n reverse (bool): Whether to reverse the order of sorted items before removing. Defaults to True.\n raise_error (bool): Whether to raise an error if directory deletion fails. Defaults to False.\n\n Examples:\n >>> clean_old(\"~/.bbot/scans\", filter=lambda x: x.is_dir() and scan_name_regex.match(x.name))\n \"\"\"\n d = Path(d)\n if not d.is_dir():\n return\n paths = [x for x in d.iterdir() if filter(x)]\n paths.sort(key=key, reverse=reverse)\n for path in paths[keep:]:\n try:\n log.debug(f\"Removing {path}\")\n rm_rf(path)\n except Exception as e:\n msg = f\"Failed to delete directory: {path}, {e}\"\n if raise_error:\n raise errors.DirectoryDeletionError()\n log.warning(msg)\n\n\ndef extract_emails(s):\n \"\"\"\n Extract email addresses from a body of text\n\n This function takes in a string and yields all email addresses found in it.\n The emails are converted to lower case before yielding. It utilizes\n regular expressions for email pattern matching.\n\n Args:\n s (str): The input string from which to extract email addresses.\n\n Yields:\n str: Yields email addresses found in the input string, in lower case.\n\n Examples:\n >>> list(extract_emails(\"Contact us at info@evilcorp.com and support@evilcorp.com\"))\n ['info@evilcorp.com', 'support@evilcorp.com']\n \"\"\"\n for email in bbot_regexes.email_regex.findall(smart_decode(s)):\n yield email.lower()\n\n\ndef extract_host(s):\n \"\"\"\n Attempts to find and extract the host portion of a string.\n\n Args:\n s (str): The string from which to extract the host.\n\n Returns:\n tuple: A tuple containing three strings:\n (hostname (None if not found), string_before_hostname, string_after_hostname).\n\n Examples:\n >>> extract_host(\"evilcorp.com:80\")\n (\"evilcorp.com\", \"\", \":80\")\n\n >>> extract_host(\"http://evilcorp.com:80/asdf.php?a=b\")\n (\"evilcorp.com\", \"http://\", \":80/asdf.php?a=b\")\n\n >>> extract_host(\"bob@evilcorp.com\")\n (\"evilcorp.com\", \"bob@\", \"\")\n\n >>> extract_host(\"[dead::beef]:22\")\n (\"dead::beef\", \"[\", \"]:22\")\n\n >>> extract_host(\"ftp://username:password@my-ftp.com/my-file.csv\")\n (\n \"my-ftp.com\",\n \"ftp://username:password@\",\n \"/my-file.csv\",\n )\n \"\"\"\n s = smart_decode(s)\n match = bbot_regexes.extract_host_regex.search(s)\n\n if match:\n hostname = match.group(1)\n before = s[: match.start(1)]\n after = s[match.end(1) :]\n host, port = split_host_port(hostname)\n netloc = make_netloc(host, port)\n if netloc != hostname:\n # invalid host / port\n return (None, s, \"\")\n if host is not None:\n if port is not None:\n after = f\":{port}{after}\"\n if is_ip(host, version=6) and hostname.startswith(\"[\"):\n before = f\"{before}[\"\n after = f\"]{after}\"\n hostname = str(host)\n return (hostname, before, after)\n\n return (None, s, \"\")\n\n\ndef smart_encode_punycode(text: str) -> str:\n \"\"\"\n ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト --> xn--eckwd4c7c.xn--zckzah\n \"\"\"\n import idna\n\n host, before, after = extract_host(text)\n if host is None:\n return text\n\n try:\n host = idna.encode(host).decode(errors=\"ignore\")\n except UnicodeError:\n pass # If encoding fails, leave the host as it is\n\n return f\"{before}{host}{after}\"\n\n\ndef smart_decode_punycode(text: str) -> str:\n \"\"\"\n xn--eckwd4c7c.xn--zckzah --> ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\n \"\"\"\n import idna\n\n host, before, after = extract_host(text)\n if host is None:\n return text\n\n try:\n host = idna.decode(host)\n except UnicodeError:\n pass # If decoding fails, leave the host as it is\n\n return f\"{before}{host}{after}\"\n\n\ndef can_sudo_without_password():\n \"\"\"Check if the current user has passwordless sudo access.\n\n This function checks whether the current user can use sudo without entering a password.\n It runs a command with sudo and checks the return code to determine this.\n\n Returns:\n bool: True if the current user can use sudo without a password, False otherwise.\n\n Examples:\n >>> can_sudo_without_password()\n True\n \"\"\"\n if os.geteuid() != 0:\n env = dict(os.environ)\n env[\"SUDO_ASKPASS\"] = \"/bin/false\"\n try:\n sp.run([\"sudo\", \"-K\"], stderr=sp.DEVNULL, stdout=sp.DEVNULL, check=True, env=env)\n sp.run([\"sudo\", \"-An\", \"/bin/true\"], stderr=sp.DEVNULL, stdout=sp.DEVNULL, check=True, env=env)\n except sp.CalledProcessError:\n return False\n return True\n\n\ndef verify_sudo_password(sudo_pass):\n \"\"\"Verify if the given sudo password is correct.\n\n This function checks whether the sudo password provided is valid for the current user.\n It runs a command with sudo, feeding in the password via stdin, and checks the return code.\n\n Args:\n sudo_pass (str): The sudo password to verify.\n\n Returns:\n bool: True if the sudo password is correct, False otherwise.\n\n Examples:\n >>> verify_sudo_password(\"mysecretpassword\")\n True\n \"\"\"\n try:\n sp.run(\n [\"sudo\", \"-S\", \"-k\", \"true\"],\n input=smart_encode(sudo_pass),\n stderr=sp.DEVNULL,\n stdout=sp.DEVNULL,\n check=True,\n )\n except sp.CalledProcessError:\n return False\n return True\n\n\ndef make_table(rows, header, **kwargs):\n \"\"\"Generate a formatted table from the given rows and headers.\n\n This function uses the `tabulate` package to generate a table with formatting options.\n It can accept various input formats and table styles, which can be customized using optional arguments.\n\n Args:\n *args: Positional arguments to be passed to `tabulate.tabulate`.\n **kwargs: Keyword arguments to customize table formatting.\n - tablefmt (str, optional): Table format. Default is 'grid'.\n - disable_numparse (bool, optional): Disable automatic number parsing. Default is True.\n - maxcolwidths (int, optional): Maximum column width. Default is 40.\n\n Returns:\n str: A string representing the formatted table.\n\n Examples:\n >>> print(make_table([[\"row1\", \"row1\"], [\"row2\", \"row2\"]], [\"header1\", \"header2\"]))\n +-----------+-----------+\n | header1 | header2 |\n +===========+===========+\n | row1 | row1 |\n +-----------+-----------+\n | row2 | row2 |\n +-----------+-----------+\n \"\"\"\n\n from tabulate import tabulate\n\n # fix IndexError: list index out of range\n if not rows:\n rows = [[]]\n tablefmt = os.environ.get(\"BBOT_TABLE_FORMAT\", None)\n defaults = {\"tablefmt\": \"grid\", \"disable_numparse\": True, \"maxcolwidths\": None}\n if tablefmt is None:\n defaults.update({\"maxcolwidths\": 40})\n else:\n defaults.update({\"tablefmt\": tablefmt})\n for k, v in defaults.items():\n if k not in kwargs:\n kwargs[k] = v\n # don't wrap columns in markdown\n if tablefmt in (\"github\", \"markdown\"):\n kwargs.pop(\"maxcolwidths\")\n # escape problematic markdown characters in rows\n\n def markdown_escape(s):\n return str(s).replace(\"|\", \"|\")\n\n rows = [[markdown_escape(f) for f in row] for row in rows]\n header = [markdown_escape(h) for h in header]\n return tabulate(rows, header, **kwargs)\n\n\ndef human_timedelta(d):\n \"\"\"Convert a TimeDelta object into a human-readable string.\n\n This function takes a datetime.timedelta object and converts it into a string format that\n is easier to read and understand.\n\n Args:\n d (datetime.timedelta): The TimeDelta object to convert.\n\n Returns:\n str: A string representation of the TimeDelta object in human-readable form.\n\n Examples:\n >>> from datetime import datetime\n >>>\n >>> start_time = datetime.now()\n >>> end_time = datetime.now()\n >>> elapsed_time = end_time - start_time\n >>> human_timedelta(elapsed_time)\n '2 hours, 30 minutes, 15 seconds'\n \"\"\"\n hours, remainder = divmod(d.seconds, 3600)\n minutes, seconds = divmod(remainder, 60)\n result = []\n if hours:\n result.append(f\"{hours:,} hour\" + (\"s\" if hours > 1 else \"\"))\n if minutes:\n result.append(f\"{minutes:,} minute\" + (\"s\" if minutes > 1 else \"\"))\n if seconds:\n result.append(f\"{seconds:,} second\" + (\"s\" if seconds > 1 else \"\"))\n ret = \", \".join(result)\n if not ret:\n ret = \"0 seconds\"\n return ret\n\n\ndef bytes_to_human(_bytes):\n \"\"\"Convert a bytes size to a human-readable string.\n\n This function converts a numeric bytes value into a human-readable string format, complete\n with the appropriate unit symbol (B, KB, MB, GB, etc.).\n\n Args:\n _bytes (int): The number of bytes to convert.\n\n Returns:\n str: A string representing the number of bytes in a more readable format, rounded to two\n decimal places.\n\n Examples:\n >>> bytes_to_human(1234129384)\n '1.15GB'\n \"\"\"\n sizes = [\"B\", \"KB\", \"MB\", \"GB\", \"TB\", \"PB\", \"EB\", \"ZB\"]\n units = {}\n for count, size in enumerate(sizes):\n units[size] = pow(1024, count)\n for size in sizes:\n if abs(_bytes) < 1024.0:\n if size == sizes[0]:\n _bytes = str(int(_bytes))\n else:\n _bytes = f\"{_bytes:.2f}\"\n return f\"{_bytes}{size}\"\n _bytes /= 1024\n raise ValueError(f'Unable to convert \"{_bytes}\" to human filesize')\n\n\nfilesize_regex = re.compile(r\"(?P[0-9\\.]+)[\\s]*(?P[a-z])\", re.I)\n\n\ndef human_to_bytes(filesize):\n \"\"\"Convert a human-readable file size string to its bytes equivalent.\n\n This function takes a human-readable file size string, such as \"2.5GB\", and converts it\n to its equivalent number of bytes.\n\n Args:\n filesize (str or int): The human-readable file size string or integer bytes value to convert.\n\n Returns:\n int: The number of bytes equivalent to the input human-readable file size.\n\n Raises:\n ValueError: If the input string cannot be converted to bytes.\n\n Examples:\n >>> human_to_bytes(\"23.23gb\")\n 24943022571\n \"\"\"\n if isinstance(filesize, int):\n return filesize\n sizes = [\"B\", \"KB\", \"MB\", \"GB\", \"TB\", \"PB\", \"EB\", \"ZB\"]\n units = {}\n for count, size in enumerate(sizes):\n size_increment = pow(1024, count)\n units[size] = size_increment\n if len(size) == 2:\n units[size[0]] = size_increment\n match = filesize_regex.match(filesize)\n try:\n if match:\n num, size = match.groups()\n size = size.upper()\n size_increment = units[size]\n return int(float(num) * size_increment)\n except KeyError:\n pass\n raise ValueError(f'Unable to convert filesize \"{filesize}\" to bytes')\n\n\ndef integer_to_ordinal(n):\n \"\"\"\n Convert an integer to its ordinal representation.\n\n Args:\n n (int): The integer to convert.\n\n Returns:\n str: The ordinal representation of the integer.\n\n Examples:\n >>> integer_to_ordinal(1)\n '1st'\n >>> integer_to_ordinal(2)\n '2nd'\n >>> integer_to_ordinal(3)\n '3rd'\n >>> integer_to_ordinal(11)\n '11th'\n >>> integer_to_ordinal(21)\n '21st'\n >>> integer_to_ordinal(101)\n '101st'\n \"\"\"\n # Check the last digit\n last_digit = n % 10\n # Check the last two digits for special cases (11th, 12th, 13th)\n last_two_digits = n % 100\n\n if 10 <= last_two_digits <= 20:\n suffix = \"th\"\n else:\n if last_digit == 1:\n suffix = \"st\"\n elif last_digit == 2:\n suffix = \"nd\"\n elif last_digit == 3:\n suffix = \"rd\"\n else:\n suffix = \"th\"\n\n return f\"{n}{suffix}\"\n\n\ndef cpu_architecture():\n \"\"\"Return the CPU architecture of the current system.\n\n This function fetches and returns the architecture type of the CPU where the code is being executed.\n It maps common identifiers like \"x86_64\" to more general types like \"amd64\".\n\n Returns:\n str: A string representing the CPU architecture, such as \"amd64\", \"armv7\", or \"arm64\".\n\n Examples:\n >>> cpu_architecture()\n 'amd64'\n \"\"\"\n import platform\n\n uname = platform.uname()\n return uname.machine.lower()\n\n\ndef cpu_architecture_golang():\n \"\"\"\n CPU architecture for GoLang release binaries.\n \"\"\"\n arch = cpu_architecture()\n # golang uses \"arm64\" instead of \"aarch64\"\n if arch.startswith(\"aarch\"):\n return \"arm64\"\n # golang uses \"amd64\" instead of \"x86_64\"\n if arch == \"x86_64\":\n return \"amd64\"\n return arch\n\n\ndef cpu_architecture_rust():\n \"\"\"\n CPU architecture for Rust release binaries.\n \"\"\"\n arch = cpu_architecture()\n # rust uses \"arm64\" instead of \"aarch64\"\n if arch.startswith(\"aarch\"):\n return \"arm64\"\n return arch\n\n\ndef os_platform():\n \"\"\"Return the OS platform of the current system.\n\n This function fetches and returns the OS type where the code is being executed.\n It converts the platform identifier to lowercase.\n\n Returns:\n str: A string representing the OS platform, such as \"linux\", \"darwin\", or \"windows\".\n\n Examples:\n >>> os_platform()\n 'linux'\n \"\"\"\n import platform\n\n return platform.system().lower()\n\n\ndef os_platform_friendly():\n \"\"\"Return a human-friendly OS platform string, suitable for golang release binaries.\n\n This function fetches the OS platform and modifies it to a more human-readable format if necessary.\n Specifically, it changes \"darwin\" to \"macOS\".\n\n Returns:\n str: A string representing the human-friendly OS platform, such as \"macOS\", \"linux\", or \"windows\".\n\n Examples:\n >>> os_platform_friendly()\n 'macOS'\n \"\"\"\n p = os_platform()\n if p == \"darwin\":\n return \"macOS\"\n return p\n\n\ntag_filter_regex = re.compile(r\"[^a-z0-9]+\")\n\n\ndef tagify(s, delimiter=None, maxlen=None):\n \"\"\"Sanitize a string into a tag-friendly format.\n\n Converts a given string to lowercase and replaces all characters not matching\n [a-z0-9] with hyphens. Optionally truncates the result to 'maxlen' characters.\n\n Args:\n s (str): The input string to sanitize.\n maxlen (int, optional): The maximum length for the tag. Defaults to None.\n\n Returns:\n str: A sanitized, tag-friendly string.\n\n Examples:\n >>> tagify(\"HTTP Web Title\")\n 'http-web-title'\n >>> tagify(\"HTTP Web Title\", maxlen=8)\n 'http-web'\n \"\"\"\n if delimiter is None:\n delimiter = \"-\"\n ret = str(s).lower()\n return tag_filter_regex.sub(delimiter, ret)[:maxlen].strip(delimiter)\n\n\ndef memory_status():\n \"\"\"Return statistics on system memory consumption.\n\n The function returns a `psutil` named tuple that contains statistics on\n system virtual memory usage, such as total memory, used memory, available\n memory, and more.\n\n Returns:\n psutil._pslinux.svmem: A named tuple representing various statistics\n about system virtual memory usage.\n\n Examples:\n >>> mem = memory_status()\n >>> mem.available\n 13195399168\n\n >>> mem = memory_status()\n >>> mem.percent\n 79.0\n \"\"\"\n import psutil\n\n return psutil.virtual_memory()\n\n\ndef swap_status():\n \"\"\"Return statistics on swap memory consumption.\n\n The function returns a `psutil` named tuple that contains statistics on\n system swap memory usage, such as total swap, used swap, free swap, and more.\n\n Returns:\n psutil._common.sswap: A named tuple representing various statistics\n about system swap memory usage.\n\n Examples:\n >>> swap = swap_status()\n >>> swap.total\n 4294967296\n\n >>> swap = swap_status()\n >>> swap.used\n 2097152\n \"\"\"\n import psutil\n\n return psutil.swap_memory()\n\n\ndef get_size(obj, max_depth=5, seen=None):\n \"\"\"\n Roughly estimate the memory footprint of a Python object using recursion.\n\n Parameters:\n obj (any): The object whose size is to be determined.\n max_depth (int, optional): Maximum depth to which nested objects will be inspected. Defaults to 5.\n seen (set, optional): Objects that have already been accounted for, to avoid loops.\n\n Returns:\n int: Approximate memory footprint of the object in bytes.\n\n Examples:\n >>> get_size(my_list)\n 4200\n\n >>> get_size(my_dict, max_depth=3)\n 8400\n \"\"\"\n from collections.abc import Mapping\n\n # If seen is not provided, initialize an empty set\n if seen is None:\n seen = set()\n # Get the id of the object\n obj_id = id(obj)\n # Decrease the maximum depth for the next recursion\n new_max_depth = max_depth - 1\n # If the object has already been seen or we've reached the maximum recursion depth, return 0\n if obj_id in seen or new_max_depth <= 0:\n return 0\n # Get the size of the object\n size = sys.getsizeof(obj)\n # Add the object's id to the set of seen objects\n seen.add(obj_id)\n # If the object has a __dict__ attribute, we want to measure its size\n if hasattr(obj, \"__dict__\"):\n # Iterate over the Method Resolution Order (MRO) of the class of the object\n for cls in obj.__class__.__mro__:\n # If the class's __dict__ contains a __dict__ key\n if \"__dict__\" in cls.__dict__:\n for k, v in obj.__dict__.items():\n size += get_size(k, new_max_depth, seen)\n size += get_size(v, new_max_depth, seen)\n break\n # If the object is a mapping (like a dictionary), we want to measure the size of its items\n if isinstance(obj, Mapping):\n with suppress(StopIteration):\n k, v = next(iter(obj.items()))\n size += (get_size(k, new_max_depth, seen) + get_size(v, new_max_depth, seen)) * len(obj)\n # If the object is a container (like a list or tuple) but not a string or bytes-like object\n elif isinstance(obj, (list, tuple, set)):\n with suppress(StopIteration):\n size += get_size(next(iter(obj)), new_max_depth, seen) * len(obj)\n # If the object has __slots__, we want to measure the size of the attributes in __slots__\n if hasattr(obj, \"__slots__\"):\n size += sum(get_size(getattr(obj, s), new_max_depth, seen) for s in obj.__slots__ if hasattr(obj, s))\n return size\n\n\ndef is_file(f):\n \"\"\"\n Check if a path points to a file.\n\n Parameters:\n f (str): Path to the file.\n\n Returns:\n bool: True if the path is a file, False otherwise.\n\n Examples:\n >>> is_file(\"/etc/passwd\")\n True\n\n >>> is_file(\"/nonexistent\")\n False\n \"\"\"\n with suppress(Exception):\n return Path(f).is_file()\n return False\n\n\ndef is_async_function(f):\n \"\"\"\n Check if a given function is an asynchronous function.\n\n Args:\n f (function): The function to check.\n\n Returns:\n bool: True if the function is asynchronous, False otherwise.\n\n Examples:\n >>> async def foo():\n ... pass\n >>> is_async_function(foo)\n True\n \"\"\"\n import inspect\n\n return inspect.iscoroutinefunction(f)\n\n\nasync def execute_sync_or_async(callback, *args, **kwargs):\n \"\"\"\n Execute a function or coroutine, handling either synchronous or asynchronous invocation.\n\n Args:\n callback (Union[Callable, Coroutine]): The function or coroutine to execute.\n *args: Variable-length argument list to pass to the callback.\n **kwargs: Arbitrary keyword arguments to pass to the callback.\n\n Returns:\n Any: The return value from the executed function or coroutine.\n\n Examples:\n >>> async def foo_async(x):\n ... return x + 1\n >>> def foo_sync(x):\n ... return x + 1\n\n >>> asyncio.run(execute_sync_or_async(foo_async, 1))\n 2\n\n >>> asyncio.run(execute_sync_or_async(foo_sync, 1))\n 2\n \"\"\"\n if is_async_function(callback):\n return await callback(*args, **kwargs)\n else:\n return callback(*args, **kwargs)\n\n\ndef get_exception_chain(e):\n \"\"\"\n Retrieves the full chain of exceptions leading to the given exception.\n\n Args:\n e (BaseException): The exception for which to get the chain.\n\n Returns:\n list[BaseException]: List of exceptions in the chain, from the given exception back to the root cause.\n\n Examples:\n >>> try:\n ... raise ValueError(\"This is a value error\")\n ... except ValueError as e:\n ... exc_chain = get_exception_chain(e)\n ... for exc in exc_chain:\n ... print(exc)\n This is a value error\n \"\"\"\n exception_chain = []\n current_exception = e\n while current_exception is not None:\n exception_chain.append(current_exception)\n current_exception = getattr(current_exception, \"__context__\", None)\n return exception_chain\n\n\ndef in_exception_chain(e, exc_types):\n \"\"\"\n Given an Exception and a list of Exception types, returns whether any of the specified types are contained anywhere in the Exception chain.\n\n Args:\n e (BaseException): The exception to check\n exc_types (list[Exception]): Exception types to consider intentional cancellations. Default is KeyboardInterrupt\n\n Returns:\n bool: Whether the error is the result of an intentional cancellaion\n\n Examples:\n >>> try:\n ... raise ValueError(\"This is a value error\")\n ... except Exception as e:\n ... if not in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):\n ... raise\n \"\"\"\n return any(isinstance(_, exc_types) for _ in get_exception_chain(e))\n\n\ndef get_traceback_details(e):\n \"\"\"\n Retrieves detailed information from the traceback of an exception.\n\n Args:\n e (BaseException): The exception for which to get traceback details.\n\n Returns:\n tuple: A tuple containing filename (str), line number (int), and function name (str) where the exception was raised.\n\n Examples:\n >>> try:\n ... raise ValueError(\"This is a value error\")\n ... except ValueError as e:\n ... filename, lineno, funcname = get_traceback_details(e)\n ... print(f\"File: {filename}, Line: {lineno}, Function: {funcname}\")\n File: , Line: 2, Function: \n \"\"\"\n import traceback\n\n tb = traceback.extract_tb(e.__traceback__)\n last_frame = tb[-1] # Get the last frame in the traceback (the one where the exception was raised)\n filename = last_frame.filename\n lineno = last_frame.lineno\n funcname = last_frame.name\n return filename, lineno, funcname\n\n\nasync def cancel_tasks(tasks, ignore_errors=True):\n \"\"\"\n Asynchronously cancels a list of asyncio tasks.\n\n Args:\n tasks (list[Task]): A list of asyncio Task objects to cancel.\n ignore_errors (bool, optional): Whether to ignore errors other than asyncio.CancelledError. Defaults to True.\n\n Examples:\n >>> async def main():\n ... task1 = asyncio.create_task(async_function1())\n ... task2 = asyncio.create_task(async_function2())\n ... await cancel_tasks([task1, task2])\n ...\n >>> asyncio.run(main())\n\n Note:\n This function will not cancel the current task that it is called from.\n \"\"\"\n current_task = asyncio.current_task()\n tasks = [t for t in tasks if t != current_task]\n for task in tasks:\n # log.debug(f\"Cancelling task: {task}\")\n task.cancel()\n if ignore_errors:\n for task in tasks:\n try:\n await task\n except BaseException as e:\n if not isinstance(e, asyncio.CancelledError):\n import traceback\n\n log.trace(traceback.format_exc())\n\n\ndef cancel_tasks_sync(tasks):\n \"\"\"\n Synchronously cancels a list of asyncio tasks.\n\n Args:\n tasks (list[Task]): A list of asyncio Task objects to cancel.\n\n Examples:\n >>> loop = asyncio.get_event_loop()\n >>> task1 = loop.create_task(some_async_function1())\n >>> task2 = loop.create_task(some_async_function2())\n >>> cancel_tasks_sync([task1, task2])\n\n Note:\n This function will not cancel the current task from which it is called.\n \"\"\"\n current_task = asyncio.current_task()\n for task in tasks:\n if task != current_task:\n # log.debug(f\"Cancelling task: {task}\")\n task.cancel()\n\n\ndef weighted_shuffle(items, weights):\n \"\"\"\n Shuffles a list of items based on their corresponding weights.\n\n Args:\n items (list): The list of items to shuffle.\n weights (list): The list of weights corresponding to each item.\n\n Returns:\n list: A new list containing the shuffled items.\n\n Examples:\n >>> items = ['apple', 'banana', 'cherry']\n >>> weights = [0.4, 0.5, 0.1]\n >>> weighted_shuffle(items, weights)\n ['banana', 'apple', 'cherry']\n >>> weighted_shuffle(items, weights)\n ['apple', 'banana', 'cherry']\n >>> weighted_shuffle(items, weights)\n ['apple', 'banana', 'cherry']\n >>> weighted_shuffle(items, weights)\n ['banana', 'apple', 'cherry']\n\n Note:\n The sum of all weights does not have to be 1. They will be normalized internally.\n \"\"\"\n # Create a list of tuples where each tuple is (item, weight)\n pool = list(zip(items, weights))\n\n shuffled_items = []\n\n # While there are still items to be chosen...\n while pool:\n # Normalize weights\n total = sum(weight for item, weight in pool)\n weights = [weight / total for item, weight in pool]\n\n # Choose an index based on weight\n chosen_index = random.choices(range(len(pool)), weights=weights, k=1)[0]\n\n # Add the chosen item to the shuffled list\n chosen_item, chosen_weight = pool.pop(chosen_index)\n shuffled_items.append(chosen_item)\n\n return shuffled_items\n\n\ndef parse_port_string(port_string):\n \"\"\"\n Parses a string containing ports and port ranges into a list of individual ports.\n\n Args:\n port_string (str): The string containing individual ports and port ranges separated by commas.\n\n Returns:\n list: A list of individual ports parsed from the input string.\n\n Raises:\n ValueError: If the input string contains invalid ports or port ranges.\n\n Examples:\n >>> parse_port_string(\"22,80,1000-1002\")\n [22, 80, 1000, 1001, 1002]\n\n >>> parse_port_string(\"1-2,3-5\")\n [1, 2, 3, 4, 5]\n\n >>> parse_port_string(\"invalid\")\n ValueError: Invalid port or port range: invalid\n \"\"\"\n elements = str(port_string).split(\",\")\n ports = []\n\n for element in elements:\n if element.isdigit():\n port = int(element)\n if 1 <= port <= 65535:\n ports.append(port)\n else:\n raise ValueError(f\"Invalid port: {element}\")\n elif \"-\" in element:\n range_parts = element.split(\"-\")\n if len(range_parts) != 2 or not all(part.isdigit() for part in range_parts):\n raise ValueError(f\"Invalid port or port range: {element}\")\n start, end = map(int, range_parts)\n if not (1 <= start < end <= 65535):\n raise ValueError(f\"Invalid port range: {element}\")\n ports.extend(range(start, end + 1))\n else:\n raise ValueError(f\"Invalid port or port range: {element}\")\n\n return ports\n\n\nasync def as_completed(coros):\n \"\"\"\n Async generator that yields completed Tasks as they are completed.\n\n Args:\n coros (iterable): An iterable of coroutine objects or asyncio Tasks.\n\n Yields:\n asyncio.Task: A Task object that has completed its execution.\n\n Examples:\n >>> async def main():\n ... async for task in as_completed([coro1(), coro2(), coro3()]):\n ... result = task.result()\n ... print(f'Task completed with result: {result}')\n\n >>> asyncio.run(main())\n \"\"\"\n tasks = {coro if isinstance(coro, asyncio.Task) else asyncio.create_task(coro): coro for coro in coros}\n while tasks:\n done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED)\n for task in done:\n tasks.pop(task)\n yield task\n\n\ndef clean_dns_record(record):\n \"\"\"\n Cleans and formats a given DNS record for further processing.\n\n This static method converts the DNS record to text format if it's not already a string.\n It also removes any trailing dots and converts the record to lowercase.\n\n Args:\n record (str or dns.rdata.Rdata): The DNS record to clean.\n\n Returns:\n str: The cleaned and formatted DNS record.\n\n Examples:\n >>> clean_dns_record('www.evilcorp.com.')\n 'www.evilcorp.com'\n\n >>> from dns.rrset import from_text\n >>> record = from_text('www.evilcorp.com', 3600, 'IN', 'A', '1.2.3.4')[0]\n >>> clean_dns_record(record)\n '1.2.3.4'\n \"\"\"\n if not isinstance(record, str):\n record = str(record.to_text())\n return str(record).rstrip(\".\").lower()\n\n\ndef truncate_filename(file_path, max_length=255):\n \"\"\"\n Truncate the filename while preserving the file extension to ensure the total path length does not exceed the maximum length.\n\n Args:\n file_path (str): The original file path.\n max_length (int): The maximum allowed length for the total path. Default is 255.\n\n Returns:\n pathlib.Path: A new Path object with the truncated filename.\n\n Raises:\n ValueError: If the directory path is too long to accommodate any filename within the limit.\n\n Example:\n >>> truncate_filename('/path/to/example_long_filename.txt', 20)\n PosixPath('/path/to/example.txt')\n \"\"\"\n p = Path(file_path)\n directory, stem, suffix = p.parent, p.stem, p.suffix\n\n max_filename_length = max_length - len(str(directory)) - len(suffix) - 1 # 1 for the '/' separator\n\n if max_filename_length <= 0:\n raise ValueError(\"The directory path is too long to accommodate any filename within the limit.\")\n\n if len(stem) > max_filename_length:\n truncated_stem = stem[:max_filename_length]\n else:\n truncated_stem = stem\n\n new_path = directory / (truncated_stem + suffix)\n return new_path\n\n\ndef get_keys_in_dot_syntax(config):\n \"\"\"Retrieve all keys in an OmegaConf configuration in dot notation.\n\n This function converts an OmegaConf configuration into a list of keys\n represented in dot notation.\n\n Args:\n config (DictConfig): The OmegaConf configuration object.\n\n Returns:\n List[str]: A list of keys in dot notation.\n\n Examples:\n >>> config = OmegaConf.create({\n ... \"web\": {\n ... \"test\": True\n ... },\n ... \"db\": {\n ... \"host\": \"localhost\",\n ... \"port\": 5432\n ... }\n ... })\n >>> get_keys_in_dot_syntax(config)\n ['web.test', 'db.host', 'db.port']\n \"\"\"\n from omegaconf import OmegaConf\n\n container = OmegaConf.to_container(config, resolve=True)\n keys = []\n\n def recursive_keys(d, parent_key=\"\"):\n for k, v in d.items():\n full_key = f\"{parent_key}.{k}\" if parent_key else k\n if isinstance(v, dict):\n recursive_keys(v, full_key)\n else:\n keys.append(full_key)\n\n recursive_keys(container)\n return keys\n\n\ndef filter_dict(d, *key_names, fuzzy=False, exclude_keys=None, _prev_key=None):\n \"\"\"\n Recursively filter a dictionary based on key names.\n\n Args:\n d (dict): The input dictionary.\n *key_names: Names of keys to filter for.\n fuzzy (bool): Whether to perform fuzzy matching on keys.\n exclude_keys (list, None): List of keys to be excluded from the final dict.\n _prev_key (str, None): For internal recursive use; the previous key in the hierarchy.\n\n Returns:\n dict: A dictionary containing only the keys specified in key_names.\n\n Examples:\n >>> filter_dict({\"key1\": \"test\", \"key2\": \"asdf\"}, \"key2\")\n {\"key2\": \"asdf\"}\n >>> filter_dict({\"key1\": \"test\", \"key2\": {\"key3\": \"asdf\"}}, \"key1\", \"key3\", exclude_keys=\"key2\")\n {'key1': 'test'}\n \"\"\"\n if exclude_keys is None:\n exclude_keys = []\n if isinstance(exclude_keys, str):\n exclude_keys = [exclude_keys]\n ret = {}\n if isinstance(d, dict):\n for key in d:\n if key in key_names or (fuzzy and any(k in key for k in key_names)):\n if not any(k in exclude_keys for k in [key, _prev_key]):\n ret[key] = copy.deepcopy(d[key])\n elif isinstance(d[key], list) or isinstance(d[key], dict):\n child = filter_dict(d[key], *key_names, fuzzy=fuzzy, _prev_key=key, exclude_keys=exclude_keys)\n if child:\n ret[key] = child\n return ret\n\n\ndef clean_dict(d, *key_names, fuzzy=False, exclude_keys=None, _prev_key=None):\n \"\"\"\n Recursively clean unwanted keys from a dictionary.\n Useful for removing secrets from a config.\n\n Args:\n d (dict): The input dictionary.\n *key_names: Names of keys to remove.\n fuzzy (bool): Whether to perform fuzzy matching on keys.\n exclude_keys (list, None): List of keys to be excluded from removal.\n _prev_key (str, None): For internal recursive use; the previous key in the hierarchy.\n\n Returns:\n dict: A dictionary cleaned of the keys specified in key_names.\n\n \"\"\"\n if exclude_keys is None:\n exclude_keys = []\n if isinstance(exclude_keys, str):\n exclude_keys = [exclude_keys]\n d = copy.deepcopy(d)\n if isinstance(d, dict):\n for key, val in list(d.items()):\n if key in key_names or (fuzzy and any(k in key for k in key_names)):\n if _prev_key not in exclude_keys:\n d.pop(key)\n continue\n d[key] = clean_dict(val, *key_names, fuzzy=fuzzy, _prev_key=key, exclude_keys=exclude_keys)\n return d\n\n\ndef calculate_entropy(data):\n \"\"\"Calculate the Shannon entropy of a byte sequence\"\"\"\n if not data:\n return 0\n frequency = {}\n for byte in data:\n if byte in frequency:\n frequency[byte] += 1\n else:\n frequency[byte] = 1\n data_len = len(data)\n entropy = -sum((count / data_len) * math.log2(count / data_len) for count in frequency.values())\n return entropy\n\n\ntop_ports_cache = None\n\n\ndef top_tcp_ports(n, as_string=False):\n \"\"\"\n Returns the top *n* TCP ports as evaluated by nmap\n \"\"\"\n top_ports_file = Path(__file__).parent.parent.parent / \"wordlists\" / \"top_open_ports_nmap.txt\"\n\n global top_ports_cache\n if top_ports_cache is None:\n # Read the open ports from the file\n with open(top_ports_file, \"r\") as f:\n top_ports_cache = [int(line.strip()) for line in f]\n\n # If n is greater than the length of the ports list, add remaining ports from range(1, 65536)\n unique_ports = set(top_ports_cache)\n top_ports_cache.extend([port for port in range(1, 65536) if port not in unique_ports])\n\n top_ports = top_ports_cache[:n]\n if as_string:\n return \",\".join([str(s) for s in top_ports])\n return top_ports\n\n\nclass SafeDict(dict):\n def __missing__(self, key):\n return \"{\" + key + \"}\"\n\n\ndef safe_format(s, **kwargs):\n \"\"\"\n Format string while ignoring unused keys (prevents KeyError)\n \"\"\"\n return s.format_map(SafeDict(kwargs))\n\n\ndef get_python_constraints():\n req_regex = re.compile(r\"([^(]+)\\s*\\((.*)\\)\", re.IGNORECASE)\n\n def clean_requirement(req_string):\n # Extract package name and version constraints from format like \"package (>=1.0,<2.0)\"\n match = req_regex.match(req_string)\n if match:\n name, constraints = match.groups()\n return f\"{name.strip()}{constraints}\"\n\n return req_string\n\n from importlib.metadata import distribution\n\n dist = distribution(\"bbot\")\n return [clean_requirement(r) for r in dist.requires]\n\n\ndef is_printable(s):\n \"\"\"\n Check if a string is printable\n \"\"\"\n if not isinstance(s, str):\n raise ValueError(f\"Expected a string, got {type(s)}\")\n\n # Exclude control characters that break display/printing\n s = set(s)\n return all(ord(c) >= 32 or c in \"\\t\\n\\r\" for c in s)\n" }, { "path": "bbot/core/helpers/names_generator.py", "content": "import random\n\nadjectives = [\n \"abnormal\",\n \"accidental\",\n \"acoustic\",\n \"acrophobic\",\n \"adorable\",\n \"adversarial\",\n \"affectionate\",\n \"aggravated\",\n \"aggrieved\",\n \"agoraphobic\",\n \"almighty\",\n \"anal\",\n \"atrocious\",\n \"autistic\",\n \"awkward\",\n \"baby\",\n \"begrudged\",\n \"benevolent\",\n \"bewildered\",\n \"bighuge\",\n \"black\",\n \"blazed\",\n \"bloodshot\",\n \"brown\",\n \"cheeky\",\n \"childish\",\n \"chiseled\",\n \"cold\",\n \"condescending\",\n \"considerate\",\n \"constipated\",\n \"contentious\",\n \"corrupted\",\n \"cosmic\",\n \"crafty\",\n \"crazed\",\n \"creamy\",\n \"crispy\",\n \"crumbly\",\n \"cryptic\",\n \"cuddly\",\n \"cute\",\n \"dark\",\n \"dastardly\",\n \"decrypted\",\n \"deep\",\n \"delicious\",\n \"demented\",\n \"demonic\",\n \"demonstrative\",\n \"depraved\",\n \"depressed\",\n \"deranged\",\n \"derogatory\",\n \"despicable\",\n \"devilish\",\n \"devious\",\n \"diabolic\",\n \"diabolical\",\n \"difficult\",\n \"dilapidated\",\n \"dismal\",\n \"distilled\",\n \"disturbed\",\n \"dramatic\",\n \"drunk\",\n \"effeminate\",\n \"effervescent\",\n \"elden\",\n \"eldritch\",\n \"embarrassed\",\n \"encrypted\",\n \"enigmatic\",\n \"enlightened\",\n \"esoteric\",\n \"ethereal\",\n \"euphoric\",\n \"evil\",\n \"expired\",\n \"exquisite\",\n \"extreme\",\n \"fermented\",\n \"ferocious\",\n \"fiendish\",\n \"fierce\",\n \"flamboyant\",\n \"fleecy\",\n \"flirtatious\",\n \"flustered\",\n \"foreboding\",\n \"frank\",\n \"frenetic\",\n \"frolicking\",\n \"furry\",\n \"fuzzy\",\n \"gentle\",\n \"giddy\",\n \"glowering\",\n \"glutinous\",\n \"golden\",\n \"gothic\",\n \"grievous\",\n \"gummy\",\n \"hallucinogenic\",\n \"hammered\",\n \"harmful\",\n \"heated\",\n \"hectic\",\n \"heightened\",\n \"heinous\",\n \"hellish\",\n \"hideous\",\n \"hubristic\",\n \"hysterical\",\n \"imaginary\",\n \"immense\",\n \"immoral\",\n \"impulsive\",\n \"incomprehensible\",\n \"inebriated\",\n \"inexplicable\",\n \"infernal\",\n \"ingenious\",\n \"inquisitive\",\n \"insecure\",\n \"insidious\",\n \"insightful\",\n \"insolent\",\n \"insufferable\",\n \"intelligent\",\n \"intensified\",\n \"intensive\",\n \"intoxicated\",\n \"inventive\",\n \"irritable\",\n \"large\",\n \"liquid\",\n \"loveable\",\n \"lovely\",\n \"lucid\",\n \"malevolent\",\n \"malfunctioning\",\n \"malicious\",\n \"manic\",\n \"masochistic\",\n \"medicated\",\n \"mediocre\",\n \"melodramatic\",\n \"mighty\",\n \"moist\",\n \"molten\",\n \"monstrous\",\n \"muscular\",\n \"mushy\",\n \"mysterious\",\n \"nascent\",\n \"naughty\",\n \"nefarious\",\n \"negligent\",\n \"neurotic\",\n \"nihilistic\",\n \"normal\",\n \"overattached\",\n \"overcompensating\",\n \"overenthusiastic\",\n \"overmedicated\",\n \"overwhelming\",\n \"overzealous\",\n \"paranoid\",\n \"pasty\",\n \"peckish\",\n \"pedantic\",\n \"pensive\",\n \"pernicious\",\n \"perturbed\",\n \"perverted\",\n \"philosophical\",\n \"pillowy\",\n \"pink\",\n \"pissed\",\n \"pixilated\",\n \"plastered\",\n \"playful\",\n \"plump\",\n \"powerful\",\n \"premature\",\n \"pretentious\",\n \"profound\",\n \"promiscuous\",\n \"psychedelic\",\n \"psychic\",\n \"puffy\",\n \"pure\",\n \"questionable\",\n \"rabid\",\n \"raging\",\n \"rambunctious\",\n \"rapid_unscheduled\",\n \"raving\",\n \"reckless\",\n \"reductive\",\n \"ripped\",\n \"ruthless\",\n \"sadistic\",\n \"satanic\",\n \"saucy\",\n \"savvy\",\n \"scheming\",\n \"schizophrenic\",\n \"secretive\",\n \"sedated\",\n \"senile\",\n \"severe\",\n \"shaggy\",\n \"sinful\",\n \"sinister\",\n \"slippery\",\n \"sly\",\n \"sneaky\",\n \"soft\",\n \"sophisticated\",\n \"spasmodic\",\n \"spicy\",\n \"spiteful\",\n \"squishy\",\n \"steamy\",\n \"sticky\",\n \"stoned\",\n \"strained\",\n \"strenuous\",\n \"stricken\",\n \"stubborn\",\n \"stuffed\",\n \"stumped\",\n \"subtle\",\n \"sudden\",\n \"suggestive\",\n \"sunburned\",\n \"super\",\n \"surreal\",\n \"suspicious\",\n \"sweet\",\n \"swole\",\n \"sycophantic\",\n \"tense\",\n \"terrible\",\n \"terrific\",\n \"thick\",\n \"thoughtful\",\n \"ticklish\",\n \"tiny\",\n \"tricky\",\n \"twitchy\",\n \"ugly\",\n \"unabated\",\n \"unchained\",\n \"unexplained\",\n \"unhinged\",\n \"unholy\",\n \"unleashed\",\n \"unmedicated\",\n \"unmelted\",\n \"unmitigated\",\n \"unrelenting\",\n \"unrestrained\",\n \"unscheduled\",\n \"unworthy\",\n \"utmost\",\n \"vehement\",\n \"vicious\",\n \"vigorous\",\n \"vile\",\n \"violent\",\n \"vivid\",\n \"voluptuous\",\n \"wasted\",\n \"wet\",\n \"wheedling\",\n \"whimsical\",\n \"white\",\n \"wicked\",\n \"wild\",\n \"wispy\",\n \"witty\",\n \"woolly\",\n \"zesty\",\n]\n\nnames = [\n \"aaron\",\n \"abigail\",\n \"adam\",\n \"adeem\",\n \"alan\",\n \"albert\",\n \"alex\",\n \"alexander\",\n \"alexis\",\n \"alice\",\n \"allen\",\n \"allison\",\n \"alyssa\",\n \"amanda\",\n \"amber\",\n \"amir\",\n \"amy\",\n \"andrea\",\n \"andrew\",\n \"angela\",\n \"ann\",\n \"anna\",\n \"anne\",\n \"annie\",\n \"anthony\",\n \"antonio\",\n \"aragorn\",\n \"arthur\",\n \"arwen\",\n \"ashley\",\n \"audrey\",\n \"austin\",\n \"azathoth\",\n \"baggins\",\n \"bailey\",\n \"barbara\",\n \"bart\",\n \"bellatrix\",\n \"benjamin\",\n \"betty\",\n \"beverly\",\n \"bilbo\",\n \"billy\",\n \"bobby\",\n \"bombadil\",\n \"bonnie\",\n \"bonson\",\n \"boromir\",\n \"bradley\",\n \"brandon\",\n \"brandybuck\",\n \"brenda\",\n \"brian\",\n \"brianna\",\n \"brittany\",\n \"bruce\",\n \"bryan\",\n \"caitlyn\",\n \"caleb\",\n \"cameron\",\n \"carl\",\n \"carlos\",\n \"carol\",\n \"carolyn\",\n \"carrie\",\n \"catherine\",\n \"charles\",\n \"charlotte\",\n \"cheryl\",\n \"christian\",\n \"christina\",\n \"christine\",\n \"christopher\",\n \"cindy\",\n \"ciri\",\n \"clara\",\n \"clarence\",\n \"cody\",\n \"connie\",\n \"courtney\",\n \"craig\",\n \"crystal\",\n \"cthulu\",\n \"curtis\",\n \"cynthia\",\n \"dagon\",\n \"dale\",\n \"dandelion\",\n \"daniel\",\n \"danielle\",\n \"danny\",\n \"daryl\",\n \"data\",\n \"david\",\n \"dawn\",\n \"deborah\",\n \"debra\",\n \"deckard\",\n \"denethor\",\n \"denise\",\n \"dennis\",\n \"diana\",\n \"diane\",\n \"dobby\",\n \"donald\",\n \"donna\",\n \"dooku\",\n \"doris\",\n \"dorothy\",\n \"douglas\",\n \"draco\",\n \"dumbledore\",\n \"dylan\",\n \"earl\",\n \"edith\",\n \"edna\",\n \"edward\",\n \"elaine\",\n \"eleanor\",\n \"elendil\",\n \"elijah\",\n \"elizabeth\",\n \"ella\",\n \"ellen\",\n \"elrond\",\n \"emily\",\n \"emma\",\n \"eomer\",\n \"eomund\",\n \"eothain\",\n \"eowyn\",\n \"eric\",\n \"erin\",\n \"ernest\",\n \"esther\",\n \"ethan\",\n \"ethel\",\n \"eugene\",\n \"eva\",\n \"evan\",\n \"evelyn\",\n \"faramir\",\n \"florence\",\n \"fox\",\n \"frances\",\n \"francis\",\n \"frank\",\n \"fred\",\n \"frederick\",\n \"frodo\",\n \"gabriel\",\n \"galadriel\",\n \"gandalf\",\n \"gary\",\n \"geordi\",\n \"george\",\n \"gerald\",\n \"geralt\",\n \"gertrude\",\n \"gimli\",\n \"gladys\",\n \"glenn\",\n \"glorfindel\",\n \"gloria\",\n \"goldberry\",\n \"gollum\",\n \"grace\",\n \"gregory\",\n \"gus\",\n \"hagrid\",\n \"hank\",\n \"hannah\",\n \"harold\",\n \"harry\",\n \"hazel\",\n \"heather\",\n \"helen\",\n \"henry\",\n \"hermione\",\n \"homer\",\n \"howard\",\n \"hunter\",\n \"irene\",\n \"isaac\",\n \"isabella\",\n \"isildur\",\n \"jack\",\n \"jacob\",\n \"jacqueline\",\n \"james\",\n \"jamie\",\n \"jane\",\n \"janet\",\n \"janice\",\n \"jaskier\",\n \"jasmine\",\n \"jason\",\n \"jayce\",\n \"jean\",\n \"jean-luc\",\n \"jeffrey\",\n \"jennifer\",\n \"jeremy\",\n \"jerry\",\n \"jesse\",\n \"jessica\",\n \"jimmy\",\n \"jinx\",\n \"joan\",\n \"joe\",\n \"joel\",\n \"john\",\n \"johnny\",\n \"jonathan\",\n \"jordan\",\n \"jose\",\n \"joseph\",\n \"josephine\",\n \"josh\",\n \"joyce\",\n \"juan\",\n \"judith\",\n \"judy\",\n \"julia\",\n \"julie\",\n \"justin\",\n \"karen\",\n \"katherine\",\n \"kathleen\",\n \"kathryn\",\n \"kathy\",\n \"kayla\",\n \"keith\",\n \"kelly\",\n \"kenneth\",\n \"kenobi\",\n \"kerry\",\n \"kevin\",\n \"kimberly\",\n \"kronk\",\n \"kyle\",\n \"kylie\",\n \"lantern\",\n \"larry\",\n \"laura\",\n \"lauren\",\n \"lawrence\",\n \"legolas\",\n \"leia\",\n \"leonard\",\n \"leslie\",\n \"lillian\",\n \"linda\",\n \"lisa\",\n \"logan\",\n \"lois\",\n \"lori\",\n \"louis\",\n \"louise\",\n \"lucius\",\n \"luis\",\n \"luke\",\n \"lupin\",\n \"madison\",\n \"magnus\",\n \"marcus\",\n \"margaret\",\n \"maria\",\n \"marie\",\n \"marilyn\",\n \"marjorie\",\n \"mark\",\n \"martha\",\n \"martin\",\n \"marvin\",\n \"mary\",\n \"mason\",\n \"matthew\",\n \"megan\",\n \"melissa\",\n \"melvin\",\n \"merry\",\n \"michael\",\n \"micheal\",\n \"michelle\",\n \"mildred\",\n \"milhouse\",\n \"monica\",\n \"nancy\",\n \"natalie\",\n \"nathan\",\n \"nathaniel\",\n \"nazgul\",\n \"ned\",\n \"nelson\",\n \"nicholas\",\n \"nicole\",\n \"noah\",\n \"norma\",\n \"norman\",\n \"nyarlathotep\",\n \"obama\",\n \"olivia\",\n \"padme\",\n \"palpatine\",\n \"pamela\",\n \"patricia\",\n \"patrick\",\n \"paul\",\n \"paula\",\n \"peggy\",\n \"peter\",\n \"philip\",\n \"phillip\",\n \"phyllis\",\n \"pippin\",\n \"powder\",\n \"rachel\",\n \"radagast\",\n \"ralph\",\n \"randy\",\n \"raymond\",\n \"rebecca\",\n \"richard\",\n \"rita\",\n \"roach\",\n \"robert\",\n \"robin\",\n \"rodney\",\n \"rodriguez\",\n \"roger\",\n \"ron\",\n \"ronald\",\n \"rose\",\n \"ross\",\n \"roy\",\n \"ruby\",\n \"russell\",\n \"ruth\",\n \"ryan\",\n \"samantha\",\n \"samuel\",\n \"samwise\",\n \"sandra\",\n \"sara\",\n \"sarah\",\n \"saruman\",\n \"sauron\",\n \"scott\",\n \"sean\",\n \"shannon\",\n \"sharon\",\n \"shawn\",\n \"shelob\",\n \"shirley\",\n \"silco\",\n \"sirius\",\n \"skywalker\",\n \"snape\",\n \"sophia\",\n \"stanley\",\n \"stephanie\",\n \"stephen\",\n \"steven\",\n \"susan\",\n \"syrina\",\n \"tammy\",\n \"taylor\",\n \"teresa\",\n \"terry\",\n \"theoden\",\n \"theon\",\n \"theresa\",\n \"thomas\",\n \"tiffany\",\n \"timothy\",\n \"tina\",\n \"todd\",\n \"tony\",\n \"tracy\",\n \"travis\",\n \"treebeard\",\n \"trent\",\n \"triss\",\n \"tyler\",\n \"tyrell\",\n \"vader\",\n \"valerie\",\n \"vander\",\n \"vanessa\",\n \"vi\",\n \"victor\",\n \"victoria\",\n \"viktor\",\n \"vincent\",\n \"virginia\",\n \"voldemort\",\n \"wallace\",\n \"walter\",\n \"wanda\",\n \"wayne\",\n \"wendy\",\n \"william\",\n \"willie\",\n \"worf\",\n \"wormtongue\",\n \"xavier\",\n \"yennefer\",\n \"yoda\",\n \"zach\",\n \"zachary\",\n]\n\n\ndef random_name():\n name = random.choice(names)\n adjective = random.choice(adjectives)\n if adjective == \"unchained\":\n scan_name = f\"{name}_{adjective}\"\n else:\n scan_name = f\"{adjective}_{name}\"\n return scan_name\n" }, { "path": "bbot/core/helpers/ntlm.py", "content": "# Stolen from https://github.com/blacklanternsecurity/TREVORspray who stole it from https://github.com/byt3bl33d3r/SprayingToolkit/blob/master/core/utils/ntlmdecoder.py\n\nimport base64\nimport struct\nimport logging\nimport collections\n\nfrom bbot.errors import NTLMError\n\nlog = logging.getLogger(\"bbot.core.helpers.ntlm\")\n\n\nclass StrStruct(object):\n def __init__(self, pos_tup, raw):\n length, alloc, offset = pos_tup\n self.length = length\n self.alloc = alloc\n self.offset = offset\n self.raw = raw[offset : offset + length]\n\n if len(self.raw) >= 2 and self.raw[1] == \"\\0\":\n self.string = self.raw.decode(\"utf-16\")\n else:\n self.string = self.raw\n\n\ntarget_field_types = collections.defaultdict(lambda: \"UNKNOWN\")\ntarget_field_types[0] = \"TERMINATOR\"\ntarget_field_types[1] = \"NetBIOS_Computer_Name\"\ntarget_field_types[2] = \"NetBIOS_Domain_Name\"\ntarget_field_types[3] = \"FQDN\"\ntarget_field_types[4] = \"DNS_Domain_name\"\ntarget_field_types[5] = \"DNS_Tree_Name\"\ntarget_field_types[7] = \"Timestamp\"\n\n\ndef decode_ntlm_challenge(st):\n hdr_tup = struct.unpack(\">> rate_limiter = RateLimiter(100, \"web\")\n >>> async def rate_limited_request(url):\n ... async with rate_limiter:\n ... return await request(url)\n \"\"\"\n\n def __init__(self, rate, name):\n self.rate = rate / 10\n self.name = name\n self.log_interval = 10\n self.current_timestamp = time.time()\n self.count = 0\n self._lock = None\n self.last_notification = None\n\n @property\n def lock(self):\n if self._lock is None:\n self._lock = asyncio.Lock()\n return self._lock\n\n async def __aenter__(self):\n async with self.lock:\n while True:\n if time.time() - self.current_timestamp >= 0.1:\n # A new 0.1 second interval has begun, reset the count and timestamp\n self.current_timestamp = time.time()\n self.count = 1\n break\n elif self.count < self.rate:\n # Still within the rate limit for the current 0.1 second interval\n self.count += 1\n break\n else:\n now = time.time()\n if self.last_notification is None or now - self.last_notification >= self.log_interval:\n log.verbose(f\"{self.name} rate limit threshold ({self.rate * 10:.1f}/s) reached\")\n self.last_notification = now\n # Rate limit for the current 0.1 second interval has been reached, wait until the next interval\n await asyncio.sleep(self.current_timestamp + 0.1 - time.time())\n\n return self\n\n async def __aexit__(self, exc_type, exc_val, exc_tb):\n pass\n" }, { "path": "bbot/core/helpers/regex.py", "content": "import asyncio\nimport regex as re\nfrom . import misc\n\n\nclass RegexHelper:\n \"\"\"\n Class for misc CPU-intensive regex operations\n\n Offloads regex processing to other CPU cores via GIL release + thread pool\n\n For quick, one-off regexes, you don't need to use this helper.\n Only use this helper if you're searching large bodies of text\n or if your regex is CPU-intensive\n \"\"\"\n\n def __init__(self, parent_helper):\n self.parent_helper = parent_helper\n\n def ensure_compiled_regex(self, r):\n \"\"\"\n Make sure a regex has been compiled\n \"\"\"\n if not isinstance(r, re.Pattern):\n raise ValueError(\"Regex must be compiled first!\")\n\n def compile(self, *args, **kwargs):\n return re.compile(*args, **kwargs)\n\n async def search(self, compiled_regex, *args, **kwargs):\n self.ensure_compiled_regex(compiled_regex)\n return await self.parent_helper.run_in_executor(compiled_regex.search, *args, **kwargs)\n\n async def match(self, compiled_regex, *args, **kwargs):\n self.ensure_compiled_regex(compiled_regex)\n return await self.parent_helper.run_in_executor(compiled_regex.match, *args, **kwargs)\n\n async def sub(self, compiled_regex, *args, **kwargs):\n self.ensure_compiled_regex(compiled_regex)\n return await self.parent_helper.run_in_executor(compiled_regex.sub, *args, **kwargs)\n\n async def findall(self, compiled_regex, *args, **kwargs):\n self.ensure_compiled_regex(compiled_regex)\n return await self.parent_helper.run_in_executor(compiled_regex.findall, *args, **kwargs)\n\n async def findall_multi(self, compiled_regexes, *args, threads=10, **kwargs):\n \"\"\"\n Same as findall() but with multiple regexes\n \"\"\"\n if not isinstance(compiled_regexes, dict):\n raise ValueError('compiled_regexes must be a dictionary like this: {\"regex_name\": }')\n for v in compiled_regexes.values():\n self.ensure_compiled_regex(v)\n\n tasks = {}\n\n def new_task(regex_name, r):\n task = self.parent_helper.run_in_executor(r.findall, *args, **kwargs)\n tasks[task] = regex_name\n\n compiled_regexes = dict(compiled_regexes)\n for _ in range(threads): # Start initial batch of tasks\n if compiled_regexes: # Ensure there are args to process\n new_task(*compiled_regexes.popitem())\n\n while tasks: # While there are tasks pending\n # Wait for the first task to complete\n done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)\n\n for task in done:\n result = task.result()\n regex_name = tasks.pop(task)\n yield (regex_name, result)\n\n if compiled_regexes: # Start a new task for each one completed, if URLs remain\n new_task(*compiled_regexes.popitem())\n\n async def finditer(self, compiled_regex, *args, **kwargs):\n self.ensure_compiled_regex(compiled_regex)\n return await self.parent_helper.run_in_executor(self._finditer, compiled_regex, *args, **kwargs)\n\n async def finditer_multi(self, compiled_regexes, *args, **kwargs):\n \"\"\"\n Same as finditer() but with multiple regexes\n \"\"\"\n for r in compiled_regexes:\n self.ensure_compiled_regex(r)\n return await self.parent_helper.run_in_executor(self._finditer_multi, compiled_regexes, *args, **kwargs)\n\n def _finditer_multi(self, compiled_regexes, *args, **kwargs):\n matches = []\n for r in compiled_regexes:\n for m in r.finditer(*args, **kwargs):\n matches.append(m)\n return matches\n\n def _finditer(self, compiled_regex, *args, **kwargs):\n return list(compiled_regex.finditer(*args, **kwargs))\n\n async def extract_params_html(self, *args, **kwargs):\n return await self.parent_helper.run_in_executor(misc.extract_params_html, *args, **kwargs)\n\n async def extract_emails(self, *args, **kwargs):\n return await self.parent_helper.run_in_executor(misc.extract_emails, *args, **kwargs)\n\n async def search_dict_values(self, *args, **kwargs):\n def _search_dict_values(*_args, **_kwargs):\n return list(misc.search_dict_values(*_args, **_kwargs))\n\n return await self.parent_helper.run_in_executor(_search_dict_values, *args, **kwargs)\n\n async def recursive_decode(self, *args, **kwargs):\n return await self.parent_helper.run_in_executor(misc.recursive_decode, *args, **kwargs)\n" }, { "path": "bbot/core/helpers/regexes.py", "content": "import regex as re\nfrom collections import OrderedDict\n\n# for extracting words from strings\nword_regexes = [\n re.compile(r, re.I)\n for r in [\n # alpha\n r\"[a-z]{3,}\",\n # alphanum\n r\"[a-z0-9]{3,}\",\n # alpha, dash\n r\"[a-z][a-z-]+[a-z]\",\n # alpha, underscore\n r\"[a-z][a-z_]+[a-z]\",\n ]\n]\n\nword_regex = re.compile(r\"[^\\d\\W_]+\")\nword_num_regex = re.compile(r\"[^\\W_]+\")\nnum_regex = re.compile(r\"\\d+\")\n\n_ipv4_regex = r\"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}\"\nipv4_regex = re.compile(_ipv4_regex, re.I)\n\n# IPv6 regex breakdown:\n#\n# (?: # —— address body ——\n# We have to individually account for all possible variations of: \"N left hextets :: M right hextets\" with N+M ≤ 8 or fully expanded 8 hextets.\n# (?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4} # 8 hextets, no compression.\n# | (?:[A-F0-9]{1,4}:){1,7}: # 1–7 left, then \"::\" (0 right).\n# | (?:[A-F0-9]{1,4}:){1,6}:[A-F0-9]{1,4} # 1–6 left, \"::\", 1 right.\n# | (?:[A-F0-9]{1,4}:){1,5}(?::[A-F0-9]{1,4}){1,2} # 1–5 left, \"::\", 1–2 right.\n# | (?:[A-F0-9]{1,4}:){1,4}(?::[A-F0-9]{1,4}){1,3} # 1–4 left, \"::\", 1–3 right.\n# | (?:[A-F0-9]{1,4}:){1,3}(?::[A-F0-9]{1,4}){1,4} # 1–3 left, \"::\", 1–4 right.\n# | (?:[A-F0-9]{1,4}:){1,2}(?::[A-F0-9]{1,4}){1,5} # 1–2 left, \"::\", 1–5 right.\n# | [A-F0-9]{1,4}:(?::[A-F0-9]{1,4}){1,6} # 1 left, \"::\", 1–6 right.\n# | :(?::[A-F0-9]{1,4}){1,7} # 0 left, \"::\", 1–7 right.\n# | :: # all zeros.\n# )\n#\n# Notes:\n# - Does not match IPv4-embedded forms (e.g., ::ffff:192.0.2.1).\n# - Does not match zone IDs (e.g., %eth0).\n# - Pure syntax check; will not validate special ranges.\n\n_ipv6_regex = r\"(?:(?:[A-F0-9]{1,4}:){7}[A-F0-9]{1,4}|(?:[A-F0-9]{1,4}:){1,7}:|(?:[A-F0-9]{1,4}:){1,6}:[A-F0-9]{1,4}|(?:[A-F0-9]{1,4}:){1,5}(?::[A-F0-9]{1,4}){1,2}|(?:[A-F0-9]{1,4}:){1,4}(?::[A-F0-9]{1,4}){1,3}|(?:[A-F0-9]{1,4}:){1,3}(?::[A-F0-9]{1,4}){1,4}|(?:[A-F0-9]{1,4}:){1,2}(?::[A-F0-9]{1,4}){1,5}|[A-F0-9]{1,4}:(?::[A-F0-9]{1,4}){1,6}|:(?::[A-F0-9]{1,4}){1,7}|::)\"\nipv6_regex = re.compile(_ipv6_regex, re.I)\n\n_ip_range_regexes = (\n _ipv4_regex + r\"\\/[0-9]{1,2}\",\n _ipv6_regex + r\"\\/[0-9]{1,3}\",\n)\nip_range_regexes = [re.compile(r, re.I) for r in _ip_range_regexes]\n\n# all dns names including IP addresses and bare hostnames (e.g. \"localhost\")\n_dns_name_regex = r\"(?:\\w(?:[\\w-]{0,100}\\w)?\\.?)+(?:[xX][nN]--)?[^\\W_]{1,63}\\.?\"\n# dns names with periods (e.g. \"www.example.com\")\n_dns_name_regex_with_period = r\"(?:\\w(?:[\\w-]{0,100}\\w)?\\.)+(?:[xX][nN]--)?[^\\W_]{1,63}\\.?\"\n\ndns_name_extraction_regex = re.compile(_dns_name_regex_with_period, re.I)\ndns_name_validation_regex = re.compile(r\"^\" + _dns_name_regex + r\"$\", re.I)\n\n_email_regex = r\"(?:[^\\W_][\\w\\-\\.\\+']{,100})@\" + _dns_name_regex + r\"(?::[0-9]{1,5})?\"\nemail_regex = re.compile(_email_regex, re.I)\n\n_ptr_regex = r\"(?:[0-9]{1,3}[-_\\.]){3}[0-9]{1,3}\"\nptr_regex = re.compile(_ptr_regex)\n# uuid regex\n_uuid_regex = r\"[0-9a-f]{8}\\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\\b[0-9a-f]{12}\"\nuuid_regex = re.compile(_uuid_regex, re.I)\n# event uuid regex\n_event_uuid_regex = r\"[0-9A-Z_]+:[0-9a-f]{8}\\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\\b[0-9a-f]{12}\"\nevent_uuid_regex = re.compile(_event_uuid_regex, re.I)\n\n_open_port_regexes = (\n _dns_name_regex + r\":[0-9]{1,5}\",\n r\"\\[\" + _ipv6_regex + r\"\\]:[0-9]{1,5}\",\n)\nopen_port_regexes = [re.compile(r, re.I) for r in _open_port_regexes]\n\n_url_regexes = (\n r\"https?://\" + _dns_name_regex + r\"(?::[0-9]{1,5})?(?:(?:/|\\?).*)?\",\n r\"https?://\\[\" + _ipv6_regex + r\"\\](?::[0-9]{1,5})?(?:(?:/|\\?).*)?\",\n)\nurl_regexes = [re.compile(r, re.I) for r in _url_regexes]\n\n_double_slash_regex = r\"/{2,}\"\ndouble_slash_regex = re.compile(_double_slash_regex)\n\n# event type regexes, used throughout BBOT for autodetection of event types, validation, and excavation.\nevent_type_regexes = OrderedDict(\n (\n (k, tuple(re.compile(r, re.I) for r in regexes))\n for k, regexes in (\n (\n \"DNS_NAME\",\n (r\"^\" + _dns_name_regex + r\"$\",),\n ),\n (\n \"EMAIL_ADDRESS\",\n (r\"^\" + _email_regex + r\"$\",),\n ),\n (\n \"IP_ADDRESS\",\n (\n r\"^\" + _ipv4_regex + r\"$\",\n r\"^\" + _ipv6_regex + r\"$\",\n ),\n ),\n (\n \"IP_RANGE\",\n tuple(r\"^\" + r + r\"$\" for r in _ip_range_regexes),\n ),\n (\n \"OPEN_TCP_PORT\",\n tuple(r\"^\" + r + r\"$\" for r in _open_port_regexes),\n ),\n (\n \"URL\",\n tuple(r\"^\" + r + r\"$\" for r in _url_regexes),\n ),\n )\n )\n)\n\nscan_name_regex = re.compile(r\"[a-z]{3,20}_[a-z]{3,20}\")\n\n\n# For use with excavate parameters extractor\ninput_tag_regex = re.compile(\n r\"]*?\\sname=[\\\"\\']?([\\-\\._=+\\/\\w]+)[\\\"\\']?[^>]*?\\svalue=[\\\"\\']?([:%\\-\\._=+\\/\\w\\s]*)[\\\"\\']?[^>]*?>\"\n)\ninput_tag_regex2 = re.compile(\n r\"]*?\\svalue=[\\\"\\']?([:\\-%\\._=+\\/\\w\\s]*)[\\\"\\']?[^>]*?\\sname=[\\\"\\']?([\\-\\._=+\\/\\w]+)[\\\"\\']?[^>]*?>\"\n)\ninput_tag_novalue_regex = re.compile(r\"]*\\b\\svalue=)[^>]*?\\sname=[\\\"\\']?([\\-\\._=+\\/\\w]*)[\\\"\\']?[^>]*?>\")\n# jquery_get_regex = re.compile(r\"url:\\s?[\\\"\\'].+?\\?(\\w+)=\")\n# jquery_get_regex = re.compile(r\"\\$.get\\([\\'\\\"].+[\\'\\\"].+\\{(.+)\\}\")\n# jquery_post_regex = re.compile(r\"\\$.post\\([\\'\\\"].+[\\'\\\"].+\\{(.+)\\}\")\na_tag_regex = re.compile(r\"]*href=[\\\"\\']([^\\\"\\'?>]*)\\?([^&\\\"\\'=]+)=([^&\\\"\\'=]+)\")\nimg_tag_regex = re.compile(r\"]*src=[\\\"\\']([^\\\"\\'?>]*)\\?([^&\\\"\\'=]+)=([^&\\\"\\'=]+)\")\nget_form_regex = re.compile(\n r\"]*\\bmethod=[\\\"']?[gG][eE][tT][\\\"']?[^>]*\\baction=[\\\"']?([^\\s\\\"'<>]+)[\\\"']?[^>]*>([\\s\\S]*?)<\\/form>\",\n re.DOTALL,\n)\nget_form_regex2 = re.compile(\n r\"]*\\baction=[\\\"']?([^\\s\\\"'<>]+)[\\\"']?[^>]*\\bmethod=[\\\"']?[gG][eE][tT][\\\"']?[^>]*>([\\s\\S]*?)<\\/form>\",\n re.DOTALL,\n)\npost_form_regex = re.compile(\n r\"]*\\bmethod=[\\\"']?[pP][oO][sS][tT][\\\"']?[^>]*\\baction=[\\\"']?([^\\s\\\"'<>]+)[\\\"']?[^>]*>([\\s\\S]*?)<\\/form>\",\n re.DOTALL,\n)\npost_form_regex2 = re.compile(\n r\"]*\\baction=[\\\"']?([^\\s\\\"'<>]+)[\\\"']?[^>]*\\bmethod=[\\\"']?[pP][oO][sS][tT][\\\"']?[^>]*>([\\s\\S]*?)<\\/form>\",\n re.DOTALL,\n)\npost_form_regex_noaction = re.compile(\n r\"]*(?:\\baction=[\\\"']?([^\\s\\\"'<>]+)[\\\"']?)?[^>]*\\bmethod=[\\\"']?[pP][oO][sS][tT][\\\"']?[^>]*>([\\s\\S]*?)<\\/form>\",\n re.DOTALL,\n)\ngeneric_form_regex = re.compile(\n r\"]*\\bmethod=)[^>]+(?:\\baction=[\\\"']?([^\\s\\\"'<>]+)[\\\"']?)[^>]*>([\\s\\S]*?)<\\/form>\",\n re.IGNORECASE | re.DOTALL,\n)\n\nselect_tag_regex = re.compile(\n r\"]+?name=[\\\"\\']?([_\\-\\.\\w]+)[\\\"\\']?[^>]*>(?:\\s*]*?value=[\\\"\\']?([_\\.\\-\\w]*)[\\\"\\']?[^>]*>)?\",\n re.IGNORECASE | re.DOTALL,\n)\n\ntextarea_tag_regex = re.compile(\n r\"]*?\\sname=[\\\"\\']?([\\-\\._=+\\/\\w]+)[\\\"\\']?[^>]*?\\svalue=[\\\"\\']?([:%\\-\\._=+\\/\\w]*)[\\\"\\']?[^>]*?>\"\n)\ntextarea_tag_regex2 = re.compile(\n r\"]*?\\svalue=[\\\"\\']?([:\\-%\\._=+\\/\\w]*)[\\\"\\']?[^>]*?\\sname=[\\\"\\']?([\\-\\._=+\\/\\w]+)[\\\"\\']?[^>]*?>\"\n)\ntextarea_tag_novalue_regex = re.compile(\n r']*\\bname=[\"\\']?([_\\-\\.\\w]+)[\"\\']?[^>]*>(.*?)', re.IGNORECASE | re.DOTALL\n)\n\nbutton_tag_regex = re.compile(\n r\"]*?name=[\\\"\\']?([\\-\\._=+\\/\\w]+)[\\\"\\']?[^>]*?value=[\\\"\\']?([%\\-\\._=+\\/\\w]*)[\\\"\\']?[^>]*?>\"\n)\nbutton_tag_regex2 = re.compile(\n r\"]*?value=[\\\"\\']?([\\-%\\._=+\\/\\w]*)[\\\"\\']?[^>]*?name=[\\\"\\']?([\\-\\._=+\\/\\w]+)[\\\"\\']?[^>]*?>\"\n)\ntag_attribute_regex = re.compile(r\"<[^>]*(?:href|action|src)\\s*=\\s*[\\\"\\']?(?!mailto:)([^\\'\\\"\\>]+)[\\\"\\']?[^>]*>\")\n\n_invalid_netloc_chars = r\"\\s!@#$%^&()=/?\\\\'\\\";~`<>\"\n# first char must not be a colon, even though it's a valid char for a netloc\nvalid_netloc = r\"[^\" + (_invalid_netloc_chars + \":\") + r\"]{1}[^\" + _invalid_netloc_chars + \"]*\"\n\n_split_host_port_regex = r\"(?:(?P[a-z0-9]{1,20})://)?(?:[^?]*@)?(?P\" + valid_netloc + \")\"\nsplit_host_port_regex = re.compile(_split_host_port_regex, re.I)\n\n_extract_open_port_regex = r\"(?:(?:\\[([0-9a-f:]+)\\])|([^\\s:]+))(?::(\\d{1,5}))?\"\nextract_open_port_regex = re.compile(_extract_open_port_regex)\n\n_extract_host_regex = r\"(?:[a-z0-9]{1,20}://)?(?:[^?]*@)?(\" + valid_netloc + \")\"\nextract_host_regex = re.compile(_extract_host_regex, re.I)\n\n# for use in recursive_decode()\nencoded_regex = re.compile(r\"%[0-9a-fA-F]{2}|\\\\u[0-9a-fA-F]{4}|\\\\U[0-9a-fA-F]{8}|\\\\[ntrbv]\")\nbackslash_regex = re.compile(r\"(?P\\\\+)(?P[ntrvb])\")\n\nuuid_regex = re.compile(r\"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\")\n" }, { "path": "bbot/core/helpers/url.py", "content": "import uuid\nimport logging\nfrom contextlib import suppress\nfrom urllib.parse import urlparse, parse_qs, urlencode, ParseResult\n\nfrom .regexes import double_slash_regex\n\n\nlog = logging.getLogger(\"bbot.core.helpers.url\")\n\n\ndef parse_url(url):\n \"\"\"\n Parse the given URL string or ParseResult object and return a ParseResult.\n\n This function checks if the input is already a ParseResult object. If it is,\n it returns the object as-is. Otherwise, it parses the given URL string using\n `urlparse`.\n\n Args:\n url (Union[str, ParseResult]): The URL string or ParseResult object to be parsed.\n\n Returns:\n ParseResult: A named 6-tuple that contains the components of a URL.\n\n Examples:\n >>> parse_url('https://www.evilcorp.com')\n ParseResult(scheme='https', netloc='www.evilcorp.com', path='', params='', query='', fragment='')\n \"\"\"\n if isinstance(url, ParseResult):\n return url\n return urlparse(url)\n\n\ndef add_get_params(url, params, encode=True):\n def _no_encode_quote(s, safe=\"/\", encoding=None, errors=None):\n return s\n\n \"\"\"\n Add or update query parameters to the given URL.\n\n This function takes an existing URL and a dictionary of query parameters,\n updates or adds these parameters to the URL, and returns a new URL.\n\n Args:\n url (Union[str, ParseResult]): The original URL.\n params (Dict[str, Any]): A dictionary containing the query parameters to be added or updated.\n\n Returns:\n ParseResult: A named 6-tuple containing the components of the modified URL.\n\n Examples:\n >>> add_get_params('https://www.evilcorp.com?foo=1', {'bar': 2})\n ParseResult(scheme='https', netloc='www.evilcorp.com', path='', params='', query='foo=1&bar=2', fragment='')\n\n >>> add_get_params('https://www.evilcorp.com?foo=1', {'foo': 2})\n ParseResult(scheme='https', netloc='www.evilcorp.com', path='', params='', query='foo=2', fragment='')\n \"\"\"\n parsed = urlparse(url)\n query_params = parsed.query.split(\"&\")\n\n existing_params = {}\n for param in query_params:\n if \"=\" in param:\n k, v = param.split(\"=\", 1)\n existing_params[k] = v\n\n existing_params.update(params)\n\n if encode:\n new_query = urlencode(existing_params, doseq=True)\n else:\n new_query = urlencode(existing_params, doseq=True, quote_via=_no_encode_quote)\n\n return parsed._replace(query=new_query)\n\n\ndef get_get_params(url):\n \"\"\"\n Extract the query parameters from the given URL as a dictionary.\n\n Args:\n url (Union[str, ParseResult]): The URL from which to extract query parameters.\n\n Returns:\n Dict[str, List[str]]: A dictionary containing the query parameters and their values.\n\n Examples:\n >>> get_get_params('https://www.evilcorp.com?foo=1&bar=2')\n {'foo': ['1'], 'bar': ['2']}\n\n >>> get_get_params('https://www.evilcorp.com?foo=1&foo=2')\n {'foo': ['1', '2']}\n \"\"\"\n parsed = parse_url(url)\n return dict(parse_qs(parsed.query))\n\n\nCHAR_LOWER = 1\nCHAR_UPPER = 2\nCHAR_DIGIT = 4\nCHAR_SYMBOL = 8\n\n\ndef charset(p):\n \"\"\"\n Determine the character set of the given string based on the types of characters it contains.\n\n Args:\n p (str): The string whose character set is to be determined.\n\n Returns:\n int: A bitmask representing the types of characters present in the string.\n - CHAR_LOWER = 1: Lowercase alphabets\n - CHAR_UPPER = 2: Uppercase alphabets\n - CHAR_DIGIT = 4: Digits\n - CHAR_SYMBOL = 8: Symbols/Special characters\n\n Examples:\n >>> charset('abc')\n 1\n\n >>> charset('abcABC')\n 3\n\n >>> charset('abc123')\n 5\n\n >>> charset('!abc123')\n 13\n \"\"\"\n ret = 0\n for c in p:\n if c.islower():\n ret |= CHAR_LOWER\n elif c.isupper():\n ret |= CHAR_UPPER\n elif c.isnumeric():\n ret |= CHAR_DIGIT\n else:\n ret |= CHAR_SYMBOL\n return ret\n\n\ndef param_type(p):\n \"\"\"\n Evaluates the type of the given parameter.\n\n Args:\n p (str): The parameter whose type is to be evaluated.\n\n Returns:\n int: An integer representing the type of parameter.\n - 1: Integer\n - 2: UUID\n - 3: Other\n\n Examples:\n >>> param_type('123')\n 1\n\n >>> param_type('550e8400-e29b-41d4-a716-446655440000')\n 2\n\n >>> param_type('abc')\n 3\n \"\"\"\n try:\n int(p)\n return 1\n except Exception:\n with suppress(Exception):\n uuid.UUID(p)\n return 2\n return 3\n\n\ndef hash_url(url):\n \"\"\"\n Hashes a URL for the purpose of cleaning or collapsing similar URLs.\n\n Args:\n url (str): The URL to be hashed.\n\n Returns:\n int: The hash value of the cleaned URL.\n\n Examples:\n >>> hash_url('https://www.evilcorp.com')\n -7448777882396416944\n\n >>> hash_url('https://www.evilcorp.com/page/1')\n -8101275613229735915\n\n >>> hash_url('https://www.evilcorp.com/page/2')\n -8101275613229735915\n \"\"\"\n parsed = parse_url(url)\n parsed = parsed._replace(fragment=\"\", query=\"\")\n to_hash = [parsed.netloc]\n for segment in parsed.path.split(\"/\"):\n hash_segment = []\n hash_segment.append(charset(segment))\n hash_segment.append(param_type(segment))\n dot_split = segment.split(\".\")\n if len(dot_split) > 1:\n hash_segment.append(dot_split[-1])\n else:\n hash_segment.append(\"\")\n to_hash.append(tuple(hash_segment))\n return hash(tuple(to_hash))\n\n\ndef url_depth(url):\n \"\"\"\n Calculate the depth of the given URL based on its path components.\n\n Args:\n url (Union[str, ParseResult]): The URL whose depth is to be calculated.\n\n Returns:\n int: The depth of the URL, based on its path components.\n\n Examples:\n >>> url_depth('https://www.evilcorp.com/foo/bar/')\n 2\n\n >>> url_depth('https://www.evilcorp.com/foo//bar/baz/')\n 3\n \"\"\"\n parsed = parse_url(url)\n parsed = parsed._replace(path=double_slash_regex.sub(\"/\", parsed.path))\n split_path = str(parsed.path).strip(\"/\").split(\"/\")\n split_path = [e for e in split_path if e]\n return len(split_path)\n" }, { "path": "bbot/core/helpers/validators.py", "content": "import logging\nimport ipaddress\nfrom typing import Union\nfrom functools import wraps\nfrom contextlib import suppress\n\nfrom bbot.core.helpers import regexes\nfrom bbot.errors import ValidationError\nfrom bbot.core.helpers.url import parse_url, hash_url\nfrom bbot.core.helpers.misc import smart_encode_punycode, split_host_port, make_netloc, is_ip\n\nlog = logging.getLogger(\"bbot.core.helpers.validators\")\n\n\ndef validator(func):\n \"\"\"\n Decorator that squashes all errors raised by the wrapped function into a ValueError.\n\n Args:\n func (Callable): The function to be decorated.\n\n Returns:\n Callable: The wrapped function.\n\n Examples:\n >>> @validator\n ... def validate_port(port):\n ... return max(1, min(65535, int(str(port))))\n \"\"\"\n\n @wraps(func)\n def validate_wrapper(*args, **kwargs):\n try:\n return func(*args)\n except Exception as e:\n raise ValueError(f\"Validation failed for {args}, {kwargs}: {e}\")\n\n return validate_wrapper\n\n\n@validator\ndef validate_port(port: Union[str, int]):\n \"\"\"\n Validates and sanitizes a port number by ensuring it falls within the allowed range (1-65535).\n\n Args:\n port (int or str): The port number to validate.\n\n Returns:\n int: The sanitized port number.\n\n Raises:\n ValueError: If the port number cannot be converted to an integer or is out of range.\n\n Examples:\n >>> validate_port(22)\n 22\n\n >>> validate_port(70000)\n 65535\n\n >>> validate_port(-123)\n 1\n \"\"\"\n return max(1, min(65535, int(str(port))))\n\n\n@validator\ndef validate_open_port(open_port: Union[str, int]):\n host, port = split_host_port(open_port)\n port = validate_port(port)\n host = validate_host(host)\n if host and port:\n return make_netloc(host, port)\n\n\n@validator\ndef validate_host(host: Union[str, ipaddress.IPv4Address, ipaddress.IPv6Address]):\n \"\"\"\n Validates and sanitizes a host string. This function handles IPv4, IPv6, and domain names.\n\n It automatically strips ports, trailing periods, and clinging asterisks and dashes.\n\n Args:\n host (str): The host string to validate.\n\n Returns:\n str: The sanitized host string.\n\n Raises:\n ValidationError: If the host is invalid or does not conform to IPv4, IPv6, or DNS_NAME formats.\n\n Examples:\n >>> validate_host(\"2001:db8::ff00:42:8329\")\n '2001:db8::ff00:42:8329'\n\n >>> validate_host(\"192.168.0.1:443\")\n '192.168.0.1'\n\n >>> validate_host(\".*.eViLCoRP.com.\")\n 'evilcorp.com'\n\n >>> validate_host(\"Invalid<>Host\")\n ValueError: Validation failed for ('Invalid<>Host',), {}: Invalid hostname: \"invalid<>host\"\n \"\"\"\n # stringify, strip and lowercase\n host = str(host).strip().lower()\n # handle IPv6 netlocs\n if host.startswith(\"[\"):\n host = host.split(\"[\")[-1].split(\"]\")[0]\n try:\n # try IPv6 first\n ip = ipaddress.IPv6Address(host)\n return str(ip)\n except Exception:\n # if IPv6 fails, strip ports and root zone\n host = host.split(\":\")[0].rstrip(\".\")\n try:\n ip = ipaddress.IPv4Address(host)\n return str(ip)\n except Exception:\n # finally, try DNS_NAME\n host = smart_encode_punycode(host)\n # clean asterisks and clinging dashes\n host = host.strip(\"*.-\").replace(\"*\", \"\")\n for r in regexes.event_type_regexes[\"DNS_NAME\"]:\n if r.match(host):\n return host\n raise ValidationError(f'Invalid hostname: \"{host}\"')\n\n\n@validator\ndef validate_severity(severity: str):\n severity = str(severity).strip().upper()\n if severity not in (\"UNKNOWN\", \"INFO\", \"LOW\", \"MEDIUM\", \"HIGH\", \"CRITICAL\"):\n raise ValueError(f\"Invalid severity: {severity}\")\n return severity\n\n\n@validator\ndef validate_email(email: str):\n email = smart_encode_punycode(str(email).strip().lower())\n if any(r.match(email) for r in regexes.event_type_regexes[\"EMAIL_ADDRESS\"]):\n return email\n raise ValidationError(f'Invalid email: \"{email}\"')\n\n\ndef clean_url(url: str, url_querystring_remove=True):\n \"\"\"\n Cleans and normalizes a URL. This function removes the query string and fragment,\n lowercases the netloc, and removes redundant port numbers.\n\n Args:\n url (str): The URL string to clean.\n\n Returns:\n ParseResult: A ParseResult object containing the cleaned URL.\n\n Examples:\n >>> clean_url(\"http://evilcorp.com:80\")\n ParseResult(scheme='http', netloc='evilcorp.com', path='/', params='', query='', fragment='')\n\n >>> clean_url(\"http://eViLcORp.com/\")\n ParseResult(scheme='http', netloc='evilcorp.com', path='/', params='', query='', fragment='')\n\n >>> clean_url(\"http://evilcorp.com/api?user=bob#place\")\n ParseResult(scheme='http', netloc='evilcorp.com', path='/api', params='', query='', fragment='')\n \"\"\"\n parsed = parse_url(url)\n\n if url_querystring_remove:\n parsed = parsed._replace(netloc=str(parsed.netloc).lower(), fragment=\"\", query=\"\")\n else:\n parsed = parsed._replace(netloc=str(parsed.netloc).lower(), fragment=\"\")\n try:\n scheme = parsed.scheme\n except ValueError:\n scheme = \"https\"\n port = None\n with suppress(Exception):\n port = parsed.port\n if port is None:\n port = 80 if scheme == \"http\" else 443\n hostname = validate_host(parsed.hostname)\n # remove ports if they're redundant\n if (scheme == \"http\" and port == 80) or (scheme == \"https\" and port == 443):\n port = None\n # special case for IPv6 URLs\n netloc = make_netloc(hostname, port)\n # urlparse is special - it needs square brackets even if there's no port\n if is_ip(netloc, version=6):\n netloc = f\"[{netloc}]\"\n parsed = parsed._replace(netloc=netloc)\n # normalize double slashes\n parsed = parsed._replace(path=regexes.double_slash_regex.sub(\"/\", parsed.path))\n # append / if path is empty\n if parsed.path == \"\":\n parsed = parsed._replace(path=\"/\")\n return parsed\n\n\ndef collapse_urls(*args, **kwargs):\n return list(_collapse_urls(*args, **kwargs))\n\n\ndef _collapse_urls(urls, threshold=10):\n \"\"\"\n Collapses a list of URLs by deduping similar URLs based on a hashing mechanism.\n Useful for cleaning large lists of noisy URLs, such as those retrieved from wayback.\n\n Args:\n urls (list): The list of URL strings to collapse.\n threshold (int): The number of allowed duplicate URLs before collapsing.\n\n Yields:\n str: A deduped URL from the input list.\n\n Example:\n >>> list(collapse_urls([\"http://evilcorp.com/user/11111/info\", \"http://evilcorp.com/user/2222/info\"], threshold=1))\n [\"http://evilcorp.com/user/11111/info\"]\n\n \"\"\"\n log.verbose(f\"Collapsing {len(urls):,} URLs\")\n url_hashes = {}\n for url in urls:\n try:\n new_url = clean_url(url)\n except ValueError as e:\n log.verbose(f\"Failed to clean url {url}: {e}\")\n url_hash = hash_url(new_url)\n try:\n url_hashes[url_hash].add(new_url)\n except KeyError:\n url_hashes[url_hash] = {\n new_url,\n }\n\n for url_hash, new_urls in url_hashes.items():\n # if the number of URLs exceeds the threshold\n if len(new_urls) > threshold:\n # yield only one\n yield next(iter(new_urls))\n else:\n yield from new_urls\n\n\n@validator\ndef validate_url(url: str):\n return validate_url_parsed(url).geturl()\n\n\n@validator\ndef validate_url_parsed(url: str):\n url = str(url).strip()\n if not any(r.match(url) for r in regexes.event_type_regexes[\"URL\"]):\n raise ValidationError(f'Invalid URL: \"{url}\"')\n return clean_url(url)\n\n\ndef soft_validate(s, t):\n \"\"\"\n Softly validates a given string against a specified type. This function returns a boolean\n instead of raising an error.\n\n Args:\n s (str): The string to validate.\n t (str): The type to validate against, e.g., \"url\" or \"host\".\n\n Returns:\n bool: True if the string is valid, False otherwise.\n\n Raises:\n ValueError: If no validator for the specified type is found.\n\n Examples:\n >>> soft_validate(\"http://evilcorp.com\", \"url\")\n True\n >>> soft_validate(\"evilcorp.com\", \"url\")\n False\n >>> soft_validate(\"http://evilcorp\", \"wrong_type\")\n ValueError: No validator for type \"wrong_type\"\n \"\"\"\n try:\n validator_fn = globals()[f\"validate_{t.strip().lower()}\"]\n except KeyError:\n raise ValueError(f'No validator for type \"{t}\"')\n try:\n validator_fn(s)\n return True\n except ValueError:\n return False\n\n\ndef is_email(email):\n try:\n validate_email(email)\n return True\n except ValueError:\n return False\n\n\nclass Validators:\n def __init__(self, parent_helper):\n self.parent_helper = parent_helper\n\n def clean_url(self, url: str):\n url_querystring_remove = self.parent_helper.config.get(\"url_querystring_remove\", True)\n return clean_url(url, url_querystring_remove=url_querystring_remove)\n\n def validate_url_parsed(self, url: str):\n \"\"\"\n This version is necessary so that it can be config-aware when needed, to avoid a chicken-egg situation. Currently this is only used by the base event class to sanitize URLs\n \"\"\"\n url = str(url).strip()\n if not any(r.match(url) for r in regexes.event_type_regexes[\"URL\"]):\n raise ValidationError(f'Invalid URL: \"{url}\"')\n return self.clean_url(url)\n" }, { "path": "bbot/core/helpers/web/__init__.py", "content": "from .web import WebHelper # noqa\n" }, { "path": "bbot/core/helpers/web/client.py", "content": "import httpx\nimport logging\nfrom httpx._models import Cookies\n\nlog = logging.getLogger(\"bbot.core.helpers.web.client\")\n\n\nclass DummyCookies(Cookies):\n def extract_cookies(self, *args, **kwargs):\n pass\n\n\nclass BBOTAsyncClient(httpx.AsyncClient):\n \"\"\"\n A subclass of httpx.AsyncClient tailored with BBOT-specific configurations and functionalities.\n This class provides rate limiting, logging, configurable timeouts, user-agent customization, custom\n headers, and proxy settings. Additionally, it allows the disabling of cookies, making it suitable\n for use across an entire scan.\n\n Attributes:\n _bbot_scan (object): BBOT scan object containing configuration details.\n _persist_cookies (bool): Flag to determine whether cookies should be persisted across requests.\n\n Examples:\n >>> async with BBOTAsyncClient(_bbot_scan=bbot_scan_object) as client:\n >>> response = await client.request(\"GET\", \"https://example.com\")\n >>> print(response.status_code)\n 200\n \"\"\"\n\n @classmethod\n def from_config(cls, config, target, *args, **kwargs):\n kwargs[\"_config\"] = config\n kwargs[\"_target\"] = target\n web_config = config.get(\"web\", {})\n retries = kwargs.pop(\"retries\", web_config.get(\"http_retries\", 1))\n ssl_verify = web_config.get(\"ssl_verify\", False)\n if ssl_verify is False:\n from .ssl_context import ssl_context_noverify\n\n ssl_verify = ssl_context_noverify\n kwargs[\"transport\"] = httpx.AsyncHTTPTransport(retries=retries, verify=ssl_verify)\n kwargs[\"verify\"] = ssl_verify\n return cls(*args, **kwargs)\n\n def __init__(self, *args, **kwargs):\n self._config = kwargs.pop(\"_config\")\n self._target = kwargs.pop(\"_target\")\n\n self._web_config = self._config.get(\"web\", {})\n http_debug = self._web_config.get(\"debug\", None)\n if http_debug:\n log.trace(f\"Creating AsyncClient: {args}, {kwargs}\")\n\n self._persist_cookies = kwargs.pop(\"persist_cookies\", False)\n\n # timeout\n http_timeout = self._web_config.get(\"http_timeout\", 20)\n if \"timeout\" not in kwargs:\n kwargs[\"timeout\"] = http_timeout\n\n # headers\n headers = kwargs.get(\"headers\", None)\n if headers is None:\n headers = {}\n\n # cookies\n cookies = kwargs.get(\"cookies\", None)\n if cookies is None:\n cookies = {}\n\n # user agent\n user_agent = self._web_config.get(\"user_agent\", \"BBOT\")\n if \"User-Agent\" not in headers:\n headers[\"User-Agent\"] = user_agent\n kwargs[\"headers\"] = headers\n kwargs[\"cookies\"] = cookies\n # proxy\n proxies = self._web_config.get(\"http_proxy\", None)\n kwargs[\"proxy\"] = proxies\n\n log.verbose(f\"Creating httpx.AsyncClient({args}, {kwargs})\")\n super().__init__(*args, **kwargs)\n if not self._persist_cookies:\n self._cookies = DummyCookies()\n\n def build_request(self, *args, **kwargs):\n if args:\n url = args[0]\n kwargs[\"url\"] = url\n url = kwargs[\"url\"]\n\n target_in_scope = self._target.in_scope(str(url))\n\n if target_in_scope:\n if not kwargs.get(\"cookies\", None):\n kwargs[\"cookies\"] = {}\n for ck, cv in self._web_config.get(\"http_cookies\", {}).items():\n if ck not in kwargs[\"cookies\"]:\n kwargs[\"cookies\"][ck] = cv\n\n request = super().build_request(**kwargs)\n\n if target_in_scope:\n for hk, hv in self._web_config.get(\"http_headers\", {}).items():\n hv = str(hv)\n # don't clobber headers\n if hk not in request.headers:\n request.headers[hk] = hv\n return request\n\n def _merge_cookies(self, cookies):\n if self._persist_cookies:\n return super()._merge_cookies(cookies)\n return cookies\n\n @property\n def retries(self):\n return self._transport._pool._retries\n" }, { "path": "bbot/core/helpers/web/engine.py", "content": "import ssl\nimport anyio\nimport httpx\nimport asyncio\nimport logging\nimport traceback\nfrom socksio.exceptions import SOCKSError\nfrom contextlib import asynccontextmanager\n\nfrom bbot.core.engine import EngineServer\nfrom bbot.core.helpers.misc import bytes_to_human, human_to_bytes, get_exception_chain, truncate_string\n\nlog = logging.getLogger(\"bbot.core.helpers.web.engine\")\n\n\nclass HTTPEngine(EngineServer):\n CMDS = {\n 0: \"request\",\n 1: \"request_batch\",\n 2: \"request_custom_batch\",\n 3: \"download\",\n }\n\n client_only_options = (\n \"retries\",\n \"max_redirects\",\n )\n\n def __init__(self, socket_path, target, config={}, debug=False):\n super().__init__(socket_path, debug=debug)\n self.target = target\n self.config = config\n self.web_config = self.config.get(\"web\", {})\n self.http_debug = self.web_config.get(\"debug\", False)\n self._ssl_context_noverify = None\n self.web_clients = {}\n self.web_client = self.AsyncClient(persist_cookies=False)\n\n def AsyncClient(self, *args, **kwargs):\n # cache by retries to prevent unwanted accumulation of clients\n # (they are not garbage-collected)\n retries = kwargs.get(\"retries\", 1)\n try:\n return self.web_clients[retries]\n except KeyError:\n from .client import BBOTAsyncClient\n\n client = BBOTAsyncClient.from_config(self.config, self.target, *args, **kwargs)\n self.web_clients[client.retries] = client\n return client\n\n async def request(self, *args, **kwargs):\n raise_error = kwargs.pop(\"raise_error\", False)\n # TODO: use this\n cache_for = kwargs.pop(\"cache_for\", None) # noqa\n\n client = kwargs.get(\"client\", self.web_client)\n\n # allow vs follow, httpx why??\n allow_redirects = kwargs.pop(\"allow_redirects\", None)\n if allow_redirects is not None and \"follow_redirects\" not in kwargs:\n kwargs[\"follow_redirects\"] = allow_redirects\n\n # in case of URL only, assume GET request\n if len(args) == 1:\n kwargs[\"url\"] = args[0]\n args = []\n\n url = kwargs.get(\"url\", \"\")\n\n if not args and \"method\" not in kwargs:\n kwargs[\"method\"] = \"GET\"\n\n client_kwargs = {}\n for k in list(kwargs):\n if k in self.client_only_options:\n v = kwargs.pop(k)\n client_kwargs[k] = v\n\n if client_kwargs:\n client = self.AsyncClient(**client_kwargs)\n\n try:\n async with self._acatch(url, raise_error):\n if self.http_debug:\n log.trace(f\"Web request: {str(args)}, {str(kwargs)}\")\n response = await client.request(*args, **kwargs)\n if self.http_debug:\n log.trace(\n f\"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}\"\n )\n return response\n except httpx.HTTPError as e:\n if raise_error:\n _response = getattr(e, \"response\", None)\n return {\"_request_error\": str(e), \"_response\": _response}\n\n async def request_batch(self, urls, threads=10, **kwargs):\n async for (args, _, _), response in self.task_pool(\n self.request, args_kwargs=urls, threads=threads, global_kwargs=kwargs\n ):\n yield args[0], response\n\n async def request_custom_batch(self, urls_and_kwargs, threads=10, **kwargs):\n async for (args, kwargs, tracker), response in self.task_pool(\n self.request, args_kwargs=urls_and_kwargs, threads=threads, global_kwargs=kwargs\n ):\n yield args[0], kwargs, tracker, response\n\n async def download(self, url, **kwargs):\n warn = kwargs.pop(\"warn\", True)\n raise_error = kwargs.pop(\"raise_error\", False)\n filename = kwargs.pop(\"filename\")\n try:\n result = await self.stream_request(url, **kwargs)\n if result is None:\n raise httpx.HTTPError(f\"No response from {url}\")\n content, response = result\n log.debug(f\"Download result: HTTP {response.status_code}\")\n response.raise_for_status()\n with open(filename, \"wb\") as f:\n f.write(content)\n return filename\n except httpx.HTTPError as e:\n log_fn = log.verbose\n if warn:\n log_fn = log.warning\n log_fn(f\"Failed to download {url}: {e}\")\n if raise_error:\n _response = getattr(e, \"response\", None)\n return {\"_download_error\": str(e), \"_response\": _response}\n\n async def stream_request(self, url, **kwargs):\n follow_redirects = kwargs.pop(\"follow_redirects\", True)\n max_size = kwargs.pop(\"max_size\", None)\n raise_error = kwargs.pop(\"raise_error\", False)\n if max_size is not None:\n max_size = human_to_bytes(max_size)\n kwargs[\"follow_redirects\"] = follow_redirects\n if \"method\" not in kwargs:\n kwargs[\"method\"] = \"GET\"\n try:\n total_size = 0\n chunk_size = 8192\n chunks = []\n\n async with self._acatch(url, raise_error=True), self.web_client.stream(url=url, **kwargs) as response:\n agen = response.aiter_bytes(chunk_size=chunk_size)\n async for chunk in agen:\n _chunk_size = len(chunk)\n if max_size is not None and total_size + _chunk_size > max_size:\n log.verbose(\n f\"Size of response from {url} exceeds {bytes_to_human(max_size)}, file will be truncated\"\n )\n await agen.aclose()\n break\n total_size += _chunk_size\n chunks.append(chunk)\n return b\"\".join(chunks), response\n except httpx.HTTPError as e:\n self.log.debug(f\"Error requesting {url}: {e}\")\n if raise_error:\n raise\n\n def ssl_context_noverify(self):\n if self._ssl_context_noverify is None:\n ssl_context = ssl.create_default_context()\n ssl_context.check_hostname = False\n ssl_context.verify_mode = ssl.CERT_NONE\n ssl_context.options &= ~ssl.OP_NO_SSLv2 & ~ssl.OP_NO_SSLv3\n ssl_context.set_ciphers(\"ALL:@SECLEVEL=0\")\n ssl_context.options |= 0x4 # Add the OP_LEGACY_SERVER_CONNECT option\n self._ssl_context_noverify = ssl_context\n return self._ssl_context_noverify\n\n @asynccontextmanager\n async def _acatch(self, url, raise_error):\n \"\"\"\n Asynchronous context manager to handle various httpx errors during a request.\n\n Yields:\n None\n\n Note:\n This function is internal and should generally not be used directly.\n `url`, `args`, `kwargs`, and `raise_error` should be in the same context as this function.\n \"\"\"\n try:\n yield\n except httpx.TimeoutException:\n if raise_error:\n raise\n else:\n log.verbose(f\"HTTP timeout to URL: {url}\")\n except httpx.ConnectError:\n if raise_error:\n raise\n else:\n log.debug(f\"HTTP connect failed to URL: {url}\")\n except httpx.HTTPError as e:\n if raise_error:\n raise\n else:\n log.trace(f\"Error with request to URL: {url}: {e}\")\n log.trace(traceback.format_exc())\n except httpx.InvalidURL as e:\n if raise_error:\n raise\n else:\n log.warning(\n f\"Invalid URL (possibly due to dangerous redirect) on request to : {url}: {truncate_string(str(e), 200)}\"\n )\n log.trace(traceback.format_exc())\n except ssl.SSLError as e:\n msg = f\"SSL error with request to URL: {url}: {e}\"\n if raise_error:\n raise httpx.RequestError(msg)\n else:\n log.trace(msg)\n log.trace(traceback.format_exc())\n except anyio.EndOfStream as e:\n msg = f\"AnyIO error with request to URL: {url}: {e}\"\n if raise_error:\n raise httpx.RequestError(msg)\n else:\n log.trace(msg)\n log.trace(traceback.format_exc())\n except SOCKSError as e:\n msg = f\"SOCKS error with request to URL: {url}: {e}\"\n if raise_error:\n raise httpx.RequestError(msg)\n else:\n log.trace(msg)\n log.trace(traceback.format_exc())\n except BaseException as e:\n # don't log if the error is the result of an intentional cancellation\n if not any(isinstance(_e, asyncio.exceptions.CancelledError) for _e in get_exception_chain(e)):\n log.trace(f\"Unhandled exception with request to URL: {url}: {e}\")\n log.trace(traceback.format_exc())\n raise\n" }, { "path": "bbot/core/helpers/web/envelopes.py", "content": "import json\nimport base64\nimport binascii\nimport xmltodict\nfrom contextlib import suppress\nfrom urllib.parse import unquote, quote\nfrom xml.parsers.expat import ExpatError\n\nfrom bbot.core.helpers.misc import is_printable\n\n\n# TODO: This logic is perfect for extracting params. We should expand it outwards to include other higher-level envelopes:\n# - QueryStringEnvelope\n# - MultipartFormEnvelope\n# - HeaderEnvelope\n# - CookieEnvelope\n#\n# Once we start ingesting HTTP_REQUEST events, this will make them instantly fuzzable\n\n\nclass EnvelopeChildTracker(type):\n \"\"\"\n Keeps track of all the child envelope classes\n \"\"\"\n\n children = []\n\n def __new__(mcs, name, bases, class_dict):\n # Create the class\n cls = super().__new__(mcs, name, bases, class_dict)\n # Don't register the base class itself\n if bases and not name.startswith(\"Base\"): # Only register if it has base classes (i.e., is a child)\n EnvelopeChildTracker.children.append(cls)\n EnvelopeChildTracker.children.sort(key=lambda x: x.priority)\n return cls\n\n\nclass BaseEnvelope(metaclass=EnvelopeChildTracker):\n __slots__ = [\"subparams\", \"selected_subparam\", \"singleton\"]\n\n # determines the order of the envelope detection\n priority = 5\n # whether the envelope is the final format, e.g. raw text/binary\n end_format = False\n ignore_exceptions = (Exception,)\n envelope_classes = EnvelopeChildTracker.children\n # transparent envelopes (i.e. TextEnvelope) are not counted as envelopes or included in the finding descriptions\n transparent = False\n\n def __init__(self, s):\n unpacked_data = self.unpack(s)\n\n if self.end_format:\n inner_envelope = unpacked_data\n else:\n inner_envelope = self.detect(unpacked_data)\n\n self.selected_subparam = None\n # if we have subparams, our inner envelope will be a dictionary\n if isinstance(inner_envelope, dict):\n self.subparams = inner_envelope\n self.singleton = False\n # otherwise if we just have one value, we make a dictionary with a default key\n else:\n self.subparams = {\"__default__\": inner_envelope}\n self.singleton = True\n\n @property\n def final_envelope(self):\n try:\n return self.unpacked_data(recursive=False).final_envelope\n except AttributeError:\n return self\n\n @property\n def friendly_name(self):\n if self.friendly_name:\n return self.friendly_name\n else:\n return self.name\n\n def pack(self, data=None):\n if data is None:\n data = self.unpacked_data(recursive=False)\n with suppress(AttributeError):\n data = data.pack()\n return self._pack(data)\n\n def unpack(self, s):\n return self._unpack(s)\n\n def _pack(self, s):\n \"\"\"\n Encodes the string using the class's unique encoder (adds the outer envelope)\n \"\"\"\n raise NotImplementedError(\"Envelope.pack() must be implemented\")\n\n def _unpack(self, s):\n \"\"\"\n Decodes the string using the class's unique encoder (removes the outer envelope)\n \"\"\"\n raise NotImplementedError(\"Envelope.unpack() must be implemented\")\n\n def unpacked_data(self, recursive=True):\n try:\n unpacked = self.subparams[\"__default__\"]\n if recursive:\n with suppress(AttributeError):\n return unpacked.unpacked_data(recursive=recursive)\n return unpacked\n except KeyError:\n return self.subparams\n\n @classmethod\n def detect(cls, s):\n \"\"\"\n Detects the type of envelope used to encode the packed_data\n \"\"\"\n if not isinstance(s, str):\n raise ValueError(f\"Invalid data passed to detect(): {s} ({type(s)})\")\n # if the value is empty, we just return the text envelope\n if not s.strip():\n return TextEnvelope(s)\n for envelope_class in cls.envelope_classes:\n with suppress(*envelope_class.ignore_exceptions):\n envelope = envelope_class(s)\n if envelope is not False:\n # make sure the envelope is not just the original string, to prevent unnecessary envelope detection. For example, \"10\" is technically valid JSON, but nothing is being encapsulated\n if str(envelope.unpacked_data()) == s:\n return TextEnvelope(s)\n else:\n return envelope\n del envelope\n raise Exception(f\"No envelope detected for data: '{s}' ({type(s)})\")\n\n def get_subparams(self, key=None, data=None, recursive=True):\n if data is None:\n data = self.unpacked_data(recursive=recursive)\n if key is None:\n key = []\n\n if isinstance(data, dict):\n for k, v in data.items():\n full_key = key + [k]\n if isinstance(v, dict):\n yield from self.get_subparams(full_key, v)\n else:\n yield full_key, v\n else:\n yield [], data\n\n def get_subparam(self, key=None, recursive=True):\n if key is None:\n key = self.selected_subparam\n envelope = self\n if recursive:\n envelope = self.final_envelope\n data = envelope.unpacked_data(recursive=False)\n if key is None:\n if envelope.singleton:\n key = []\n else:\n raise ValueError(\"No subparam selected\")\n else:\n for segment in key:\n data = data[segment]\n return data\n\n def pack_value(self, value, key=None):\n \"\"\"\n Pack a value through the envelope chain WITHOUT modifying internal state.\n \"\"\"\n if key is None:\n key = self.selected_subparam\n\n inner = self.unpacked_data(recursive=False)\n\n if hasattr(inner, \"pack_value\"):\n # Inner is another envelope - delegate down the chain\n data = inner.pack_value(value, key)\n elif self.singleton:\n # At the leaf singleton - use the new value directly\n data = value\n else:\n # At the leaf non-singleton (JSON/XML) - copy the data and substitute\n import copy\n\n if key is None:\n raise ValueError(\"No subparam selected for non-singleton envelope\")\n data = copy.deepcopy(inner)\n # In the loop: Traverse all the way down to the parent of the target value (all segments except the last),\n target = data\n for segment in key[:-1]:\n target = target[segment]\n # Use the final segment to actually assign the value.\n target[key[-1]] = value\n return self._pack(data)\n\n def set_subparam(self, key=None, value=None, recursive=True):\n envelope = self\n if recursive:\n envelope = self.final_envelope\n\n # if there's only one value to set, we can just set it directly\n if envelope.singleton:\n envelope.subparams[\"__default__\"] = value\n return\n\n # if key isn't specified, use the selected subparam\n if key is None:\n key = self.selected_subparam\n if key is None:\n raise ValueError(f\"{self} -> {envelope}: No subparam selected\")\n\n data = envelope.unpacked_data(recursive=False)\n for segment in key[:-1]:\n data = data[segment]\n data[key[-1]] = value\n\n @property\n def name(self):\n return self.__class__.__name__\n\n @property\n def num_envelopes(self):\n num_envelopes = 0 if self.transparent else 1\n if self.end_format:\n return num_envelopes\n for envelope in self.subparams.values():\n with suppress(AttributeError):\n num_envelopes += envelope.num_envelopes\n return num_envelopes\n\n @property\n def summary(self):\n if self.transparent:\n return \"\"\n self_string = f\"{self.friendly_name}\"\n with suppress(AttributeError):\n child_envelope = self.unpacked_data(recursive=False)\n child_summary = child_envelope.summary\n if child_summary:\n self_string += f\" -> {child_summary}\"\n\n if self.selected_subparam:\n self_string += f\" [{'.'.join(self.selected_subparam)}]\"\n return self_string\n\n def to_dict(self):\n return self.summary\n\n def __str__(self):\n return self.summary\n\n __repr__ = __str__\n\n\nclass HexEnvelope(BaseEnvelope):\n \"\"\"\n Hexadecimal encoding\n \"\"\"\n\n friendly_name = \"Hexadecimal-Encoded\"\n\n ignore_exceptions = (ValueError, UnicodeDecodeError)\n\n def _pack(self, s):\n return s.encode().hex()\n\n def _unpack(self, s):\n return bytes.fromhex(s).decode()\n\n\nclass B64Envelope(BaseEnvelope):\n \"\"\"\n Base64 encoding\n \"\"\"\n\n friendly_name = \"Base64-Encoded\"\n\n ignore_exceptions = (binascii.Error, UnicodeDecodeError, ValueError)\n\n def unpack(self, s):\n # it's easy to have a small value that accidentally decodes to base64\n if len(s) < 8 and not s.endswith(\"=\"):\n raise ValueError(\"Data is too small to be sure\")\n return super().unpack(s)\n\n def _pack(self, s):\n return base64.b64encode(s.encode()).decode()\n\n def _unpack(self, s):\n return base64.b64decode(s).decode()\n\n\nclass URLEnvelope(BaseEnvelope):\n \"\"\"\n URL encoding\n \"\"\"\n\n friendly_name = \"URL-Encoded\"\n\n def unpack(self, s):\n unpacked = super().unpack(s)\n if unpacked == s:\n raise ValueError(\"Data is not URL-encoded\")\n return unpacked\n\n def _pack(self, s):\n return quote(s)\n\n def _unpack(self, s):\n return unquote(s)\n\n\nclass TextEnvelope(BaseEnvelope):\n \"\"\"\n Text encoding\n \"\"\"\n\n end_format = True\n # lowest priority means text is the ultimate fallback\n priority = 10\n transparent = True\n ignore_exceptions = ()\n\n def _pack(self, s):\n return s\n\n def _unpack(self, s):\n if not is_printable(s):\n raise ValueError(f\"Non-printable data detected in TextEnvelope: '{s}' ({type(s)})\")\n return s\n\n\n# class BinaryEnvelope(BaseEnvelope):\n# \"\"\"\n# Binary encoding\n# \"\"\"\n# end_format = True\n\n# def pack(self, s):\n# return s\n\n# def unpack(self, s):\n# if is_printable(s):\n# raise Exception(\"Non-binary data detected in BinaryEnvelope\")\n# return s\n\n\nclass JSONEnvelope(BaseEnvelope):\n \"\"\"\n JSON encoding\n \"\"\"\n\n friendly_name = \"JSON-formatted\"\n end_format = True\n priority = 8\n ignore_exceptions = (json.JSONDecodeError,)\n\n def _pack(self, s):\n return json.dumps(s)\n\n def _unpack(self, s):\n return json.loads(s)\n\n\nclass XMLEnvelope(BaseEnvelope):\n \"\"\"\n XML encoding\n \"\"\"\n\n friendly_name = \"XML-formatted\"\n end_format = True\n priority = 9\n ignore_exceptions = (ExpatError,)\n\n def _pack(self, s):\n return xmltodict.unparse(s)\n\n def _unpack(self, s):\n return xmltodict.parse(s)\n" }, { "path": "bbot/core/helpers/web/ssl_context.py", "content": "import ssl\n\nssl_context_noverify = ssl.create_default_context()\nssl_context_noverify.check_hostname = False\nssl_context_noverify.verify_mode = ssl.CERT_NONE\nssl_context_noverify.options &= ~ssl.OP_NO_SSLv2 & ~ssl.OP_NO_SSLv3\nssl_context_noverify.set_ciphers(\"ALL:@SECLEVEL=0\")\nssl_context_noverify.options |= 0x4 # Add the OP_LEGACY_SERVER_CONNECT option\n" }, { "path": "bbot/core/helpers/web/web.py", "content": "import logging\nimport warnings\nfrom pathlib import Path\nfrom bs4 import BeautifulSoup\n\nfrom bbot.core.engine import EngineClient\nfrom bbot.core.helpers.misc import truncate_filename\nfrom bbot.errors import WordlistError, CurlError, WebError\n\nfrom bs4 import MarkupResemblesLocatorWarning\nfrom bs4.builder import XMLParsedAsHTMLWarning\n\nfrom .engine import HTTPEngine\n\nwarnings.filterwarnings(\"ignore\", category=XMLParsedAsHTMLWarning)\nwarnings.filterwarnings(\"ignore\", category=MarkupResemblesLocatorWarning)\n\nlog = logging.getLogger(\"bbot.core.helpers.web\")\n\n\nclass WebHelper(EngineClient):\n SERVER_CLASS = HTTPEngine\n ERROR_CLASS = WebError\n\n \"\"\"\n Main utility class for managing HTTP operations in BBOT. It serves as a wrapper around the BBOTAsyncClient,\n which itself is a subclass of httpx.AsyncClient. The class provides functionalities to make HTTP requests,\n download files, and handle cached wordlists.\n\n Attributes:\n parent_helper (object): The parent helper object containing scan configurations.\n http_debug (bool): Flag to indicate whether HTTP debugging is enabled.\n ssl_verify (bool): Flag to indicate whether SSL verification is enabled.\n web_client (BBOTAsyncClient): An instance of BBOTAsyncClient for making HTTP requests.\n client_only_options (tuple): A tuple of options only applicable to the web client.\n\n Examples:\n Basic web request:\n >>> response = await self.helpers.request(\"https://www.evilcorp.com\")\n\n Download file:\n >>> filename = await self.helpers.download(\"https://www.evilcorp.com/passwords.docx\")\n\n Download wordlist (cached for 30 days by default):\n >>> filename = await self.helpers.wordlist(\"https://www.evilcorp.com/wordlist.txt\")\n \"\"\"\n\n def __init__(self, parent_helper):\n self.parent_helper = parent_helper\n self.preset = self.parent_helper.preset\n self.config = self.preset.config\n self.web_config = self.config.get(\"web\", {})\n self.web_spider_depth = self.web_config.get(\"spider_depth\", 1)\n self.web_spider_distance = self.web_config.get(\"spider_distance\", 0)\n self.web_clients = {}\n self.target = self.preset.target\n self.ssl_verify = self.config.get(\"ssl_verify\", False)\n engine_debug = self.config.get(\"engine\", {}).get(\"debug\", False)\n super().__init__(\n server_kwargs={\"config\": self.config, \"target\": self.parent_helper.preset.target},\n debug=engine_debug,\n )\n\n def AsyncClient(self, *args, **kwargs):\n # cache by retries to prevent unwanted accumulation of clients\n # (they are not garbage-collected)\n retries = kwargs.get(\"retries\", 1)\n try:\n return self.web_clients[retries]\n except KeyError:\n from .client import BBOTAsyncClient\n\n client = BBOTAsyncClient.from_config(self.config, self.target, *args, persist_cookies=False, **kwargs)\n self.web_clients[client.retries] = client\n return client\n\n async def request(self, *args, **kwargs):\n \"\"\"\n Asynchronous function for making HTTP requests, intended to be the most basic web request function\n used widely across BBOT and within this helper class. Handles various exceptions and timeouts\n that might occur during the request.\n\n This function automatically respects the scan's global timeout, proxy, headers, etc.\n Headers you specify will be merged with the scan's. Your arguments take ultimate precedence,\n meaning you can override the scan's values if you want.\n\n Args:\n url (str): The URL to send the request to.\n method (str, optional): The HTTP method to use for the request. Defaults to 'GET'.\n headers (dict, optional): Dictionary of HTTP headers to send with the request.\n params (dict, optional): Dictionary, list of tuples, or bytes to send in the query string.\n cookies (dict, optional): Dictionary or CookieJar object containing cookies.\n json (Any, optional): A JSON serializable Python object to send in the body.\n data (dict, optional): Dictionary, list of tuples, or bytes to send in the body.\n files (dict, optional): Dictionary of 'name': file-like-objects for multipart encoding upload.\n auth (tuple, optional): Auth tuple to enable Basic/Digest/Custom HTTP auth.\n timeout (float, optional): The maximum time to wait for the request to complete.\n proxy (str, optional): HTTP proxy URL.\n allow_redirects (bool, optional): Enables or disables redirection. Defaults to None.\n stream (bool, optional): Enables or disables response streaming.\n raise_error (bool, optional): Whether to raise exceptions for HTTP connect, timeout errors. Defaults to False.\n client (httpx.AsyncClient, optional): A specific httpx.AsyncClient to use for the request. Defaults to self.web_client.\n cache_for (int, optional): Time in seconds to cache the request. Not used currently. Defaults to None.\n\n Raises:\n httpx.TimeoutException: If the request times out.\n httpx.ConnectError: If the connection fails.\n httpx.RequestError: For other request-related errors.\n\n Returns:\n httpx.Response or None: The HTTP response object returned by the httpx library.\n\n Examples:\n >>> response = await self.helpers.request(\"https://www.evilcorp.com\")\n\n >>> response = await self.helpers.request(\"https://api.evilcorp.com/\", method=\"POST\", data=\"stuff\")\n\n Note:\n If the web request fails, it will return None unless `raise_error` is `True`.\n \"\"\"\n raise_error = kwargs.get(\"raise_error\", False)\n result = await self.run_and_return(\"request\", *args, **kwargs)\n if isinstance(result, dict) and \"_request_error\" in result:\n if raise_error:\n error_msg = result[\"_request_error\"]\n response = result[\"_response\"]\n error = self.ERROR_CLASS(error_msg)\n error.response = response\n raise error\n return result\n\n async def request_batch(self, urls, *args, **kwargs):\n \"\"\"\n Given a list of URLs, request them in parallel and yield responses as they come in.\n\n Args:\n urls (list[str]): List of URLs to visit\n *args: Positional arguments to pass through to httpx\n **kwargs: Keyword arguments to pass through to httpx\n\n Examples:\n >>> async for url, response in self.helpers.request_batch(urls, headers={\"X-Test\": \"Test\"}):\n >>> if response is not None and response.status_code == 200:\n >>> self.hugesuccess(response)\n \"\"\"\n agen = self.run_and_yield(\"request_batch\", urls, *args, **kwargs)\n while 1:\n try:\n yield await agen.__anext__()\n except (StopAsyncIteration, GeneratorExit):\n await agen.aclose()\n break\n\n async def request_custom_batch(self, urls_and_kwargs):\n \"\"\"\n Make web requests in parallel with custom options for each request. Yield responses as they come in.\n\n Similar to `request_batch` except it allows individual arguments for each URL.\n\n Args:\n urls_and_kwargs (list[tuple]): List of tuples in the format: (url, kwargs, custom_tracker)\n where custom_tracker is an optional value for your own internal use. You may use it to\n help correlate requests, etc.\n\n Examples:\n >>> urls_and_kwargs = [\n >>> (\"http://evilcorp.com/1\", {\"method\": \"GET\"}, \"request-1\"),\n >>> (\"http://evilcorp.com/2\", {\"method\": \"POST\"}, \"request-2\"),\n >>> ]\n >>> async for url, kwargs, custom_tracker, response in self.helpers.request_custom_batch(\n >>> urls_and_kwargs\n >>> ):\n >>> if response is not None and response.status_code == 200:\n >>> self.hugesuccess(response)\n \"\"\"\n agen = self.run_and_yield(\"request_custom_batch\", urls_and_kwargs)\n while 1:\n try:\n yield await agen.__anext__()\n except (StopAsyncIteration, GeneratorExit):\n await agen.aclose()\n break\n\n async def download(self, url, **kwargs):\n \"\"\"\n Asynchronous function for downloading files from a given URL. Supports caching with an optional\n time period in hours via the \"cache_hrs\" keyword argument. In case of successful download,\n returns the full path of the saved filename. If the download fails, returns None.\n\n Args:\n url (str): The URL of the file to download.\n filename (str, optional): The filename to save the downloaded file as.\n If not provided, will generate based on URL.\n max_size (str or int): Maximum filesize as a string (\"5MB\") or integer in bytes.\n cache_hrs (float, optional): The number of hours to cache the downloaded file.\n A negative value disables caching. Defaults to -1.\n method (str, optional): The HTTP method to use for the request, defaults to 'GET'.\n raise_error (bool, optional): Whether to raise exceptions for HTTP connect, timeout errors. Defaults to False.\n **kwargs: Additional keyword arguments to pass to the httpx request.\n\n Returns:\n Path or None: The full path of the downloaded file as a Path object if successful, otherwise None.\n\n Examples:\n >>> filepath = await self.helpers.download(\"https://www.evilcorp.com/passwords.docx\", cache_hrs=24)\n \"\"\"\n success = False\n raise_error = kwargs.get(\"raise_error\", False)\n filename = kwargs.pop(\"filename\", self.parent_helper.cache_filename(url))\n filename = truncate_filename(Path(filename).resolve())\n kwargs[\"filename\"] = filename\n max_size = kwargs.pop(\"max_size\", None)\n if max_size is not None:\n max_size = self.parent_helper.human_to_bytes(max_size)\n kwargs[\"max_size\"] = max_size\n cache_hrs = float(kwargs.pop(\"cache_hrs\", -1))\n if cache_hrs > 0 and self.parent_helper.is_cached(url):\n log.debug(f\"{url} is cached at {self.parent_helper.cache_filename(url)}\")\n success = True\n else:\n result = await self.run_and_return(\"download\", url, **kwargs)\n if isinstance(result, dict) and \"_download_error\" in result:\n if raise_error:\n error_msg = result[\"_download_error\"]\n response = result[\"_response\"]\n error = self.ERROR_CLASS(error_msg)\n error.response = response\n raise error\n elif result:\n success = True\n\n if success:\n return filename\n\n async def wordlist(self, path, lines=None, zip=False, zip_filename=None, **kwargs):\n \"\"\"\n Asynchronous function for retrieving wordlists, either from a local path or a URL.\n Allows for optional line-based truncation and caching. Returns the full path of the wordlist\n file or a truncated version of it.\n\n Args:\n path (str): The local or remote path of the wordlist.\n lines (int, optional): Number of lines to read from the wordlist.\n If specified, will return a truncated wordlist with this many lines.\n zip (bool, optional): Whether to unzip the file after downloading. Defaults to False.\n zip_filename (str, optional): The name of the file to extract from the ZIP archive.\n Required if zip is True.\n cache_hrs (float, optional): Number of hours to cache the downloaded wordlist.\n Defaults to 720 hours (30 days) for remote wordlists.\n **kwargs: Additional keyword arguments to pass to the 'download' function for remote wordlists.\n\n Returns:\n Path: The full path of the wordlist (or its truncated version) as a Path object.\n\n Raises:\n WordlistError: If the path is invalid or the wordlist could not be retrieved or found.\n\n Examples:\n Fetching full wordlist\n >>> wordlist_path = await self.helpers.wordlist(\"https://www.evilcorp.com/wordlist.txt\")\n\n Fetching and truncating to the first 100 lines\n >>> wordlist_path = await self.helpers.wordlist(\"/root/rockyou.txt\", lines=100)\n \"\"\"\n import zipfile\n\n if not path:\n raise WordlistError(f\"Invalid wordlist: {path}\")\n if \"cache_hrs\" not in kwargs:\n # 4320 hrs = 180 days = 6 months\n kwargs[\"cache_hrs\"] = 4320\n if self.parent_helper.is_url(path):\n filename = await self.download(str(path), **kwargs)\n if filename is None:\n raise WordlistError(f\"Unable to retrieve wordlist from {path}\")\n else:\n filename = Path(path).resolve()\n if not filename.is_file():\n raise WordlistError(f\"Unable to find wordlist at {path}\")\n\n if zip:\n if not zip_filename:\n raise WordlistError(\"zip_filename must be specified when zip is True\")\n try:\n with zipfile.ZipFile(filename, \"r\") as zip_ref:\n if zip_filename not in zip_ref.namelist():\n raise WordlistError(f\"File {zip_filename} not found in the zip archive {filename}\")\n zip_ref.extract(zip_filename, filename.parent)\n filename = filename.parent / zip_filename\n except Exception as e:\n raise WordlistError(f\"Error unzipping file {filename}: {e}\")\n\n if lines is None:\n return filename\n else:\n lines = int(lines)\n with open(filename) as f:\n read_lines = f.readlines()\n cache_key = f\"{filename}:{lines}\"\n truncated_filename = self.parent_helper.cache_filename(cache_key)\n with open(truncated_filename, \"w\") as f:\n for line in read_lines[:lines]:\n f.write(line)\n return truncated_filename\n\n async def curl(self, *args, **kwargs):\n \"\"\"\n An asynchronous function that runs a cURL command with specified arguments and options.\n\n This function constructs and executes a cURL command based on the provided parameters.\n It offers support for various cURL options such as headers, post data, and cookies.\n\n Args:\n *args: Variable length argument list for positional arguments. Unused in this function.\n url (str): The URL for the cURL request. Mandatory.\n raw_path (bool, optional): If True, activates '--path-as-is' in cURL. Defaults to False.\n headers (dict, optional): A dictionary of HTTP headers to include in the request.\n ignore_bbot_global_settings (bool, optional): If True, ignores the global settings of BBOT. Defaults to False.\n post_data (dict, optional): A dictionary containing data to be sent in the request body.\n method (str, optional): The HTTP method to use for the request (e.g., 'GET', 'POST').\n cookies (dict, optional): A dictionary of cookies to include in the request.\n path_override (str, optional): Overrides the request-target to use in the HTTP request line.\n head_mode (bool, optional): If True, includes '-I' to fetch headers only. Defaults to None.\n raw_body (str, optional): Raw string to be sent in the body of the request.\n **kwargs: Arbitrary keyword arguments that will be forwarded to the HTTP request function.\n\n Returns:\n str: The output of the cURL command.\n\n Raises:\n CurlError: If 'url' is not supplied.\n\n Examples:\n >>> output = await curl(url=\"https://example.com\", headers={\"X-Header\": \"Wat\"})\n >>> print(output)\n \"\"\"\n url = kwargs.get(\"url\", \"\")\n\n if not url:\n raise CurlError(\"No URL supplied to CURL helper\")\n\n curl_command = [\"curl\", url, \"-s\"]\n\n raw_path = kwargs.get(\"raw_path\", False)\n if raw_path:\n curl_command.append(\"--path-as-is\")\n\n # respect global ssl verify settings\n if self.ssl_verify is not True:\n curl_command.append(\"-k\")\n\n headers = kwargs.get(\"headers\", {})\n cookies = kwargs.get(\"cookies\", {})\n\n ignore_bbot_global_settings = kwargs.get(\"ignore_bbot_global_settings\", False)\n\n if ignore_bbot_global_settings:\n http_timeout = 20 # setting 20 as a worse-case setting\n log.debug(\"ignore_bbot_global_settings enabled. Global settings will not be applied\")\n else:\n http_timeout = self.parent_helper.web_config.get(\"http_timeout\", 20)\n user_agent = self.parent_helper.web_config.get(\"user_agent\", \"BBOT\")\n\n if \"User-Agent\" not in headers:\n headers[\"User-Agent\"] = user_agent\n\n # only add custom headers / cookies if the URL is in-scope\n if self.parent_helper.preset.in_scope(url):\n for hk, hv in self.web_config.get(\"http_headers\", {}).items():\n # Only add the header if it doesn't already exist in the headers dictionary\n if hk not in headers:\n headers[hk] = hv\n\n for ck, cv in self.web_config.get(\"http_cookies\", {}).items():\n # don't clobber cookies\n if ck not in cookies:\n cookies[ck] = cv\n\n # add the timeout\n if \"timeout\" not in kwargs:\n timeout = http_timeout\n\n curl_command.append(\"-m\")\n curl_command.append(str(timeout))\n\n for k, v in headers.items():\n if isinstance(v, list):\n for x in v:\n curl_command.append(\"-H\")\n curl_command.append(f\"{k}: {x}\")\n\n else:\n curl_command.append(\"-H\")\n curl_command.append(f\"{k}: {v}\")\n\n post_data = kwargs.get(\"post_data\", {})\n if len(post_data.items()) > 0:\n curl_command.append(\"-d\")\n post_data_str = \"\"\n for k, v in post_data.items():\n post_data_str += f\"&{k}={v}\"\n curl_command.append(post_data_str.lstrip(\"&\"))\n\n method = kwargs.get(\"method\", \"\")\n if method:\n curl_command.append(\"-X\")\n curl_command.append(method)\n\n cookies = kwargs.get(\"cookies\", \"\")\n if cookies:\n curl_command.append(\"-b\")\n cookies_str = \"\"\n for k, v in cookies.items():\n cookies_str += f\"{k}={v}; \"\n curl_command.append(f\"{cookies_str.rstrip(' ')}\")\n\n path_override = kwargs.get(\"path_override\", None)\n if path_override:\n curl_command.append(\"--request-target\")\n curl_command.append(f\"{path_override}\")\n\n head_mode = kwargs.get(\"head_mode\", None)\n if head_mode:\n curl_command.append(\"-I\")\n\n raw_body = kwargs.get(\"raw_body\", None)\n if raw_body:\n curl_command.append(\"-d\")\n curl_command.append(raw_body)\n log.verbose(f\"Running curl command: {curl_command}\")\n output = (await self.parent_helper.run(curl_command)).stdout\n return output\n\n def beautifulsoup(\n self,\n markup,\n features=\"html.parser\",\n builder=None,\n parse_only=None,\n from_encoding=None,\n exclude_encodings=None,\n element_classes=None,\n **kwargs,\n ):\n \"\"\"\n Naviate, Search, Modify, Parse, or PrettyPrint HTML Content.\n More information at https://beautiful-soup-4.readthedocs.io/en/latest/\n\n Args:\n markup: A string or a file-like object representing markup to be parsed.\n features: Desirable features of the parser to be used.\n This may be the name of a specific parser (\"lxml\",\n \"lxml-xml\", \"html.parser\", or \"html5lib\") or it may be\n the type of markup to be used (\"html\", \"html5\", \"xml\").\n Defaults to 'html.parser'.\n builder: A TreeBuilder subclass to instantiate (or instance to use)\n instead of looking one up based on `features`.\n parse_only: A SoupStrainer. Only parts of the document\n matching the SoupStrainer will be considered.\n from_encoding: A string indicating the encoding of the\n document to be parsed.\n exclude_encodings = A list of strings indicating\n encodings known to be wrong.\n element_classes = A dictionary mapping BeautifulSoup\n classes like Tag and NavigableString, to other classes you'd\n like to be instantiated instead as the parse tree is\n built.\n **kwargs = For backwards compatibility purposes.\n\n Returns:\n soup: An instance of the BeautifulSoup class\n\n Todo:\n - Write tests for this function\n\n Examples:\n >>> soup = self.helpers.beautifulsoup(event.data[\"body\"], \"html.parser\")\n Perform an html parse of the 'markup' argument and return a soup instance\n\n >>> email_type = soup.find(type=\"email\")\n Searches the soup instance for all occurrences of the passed in argument\n \"\"\"\n try:\n soup = BeautifulSoup(\n markup, features, builder, parse_only, from_encoding, exclude_encodings, element_classes, **kwargs\n )\n return soup\n except Exception as e:\n log.debug(f\"Error parsing beautifulsoup: {e}\")\n return False\n\n def response_to_json(self, response):\n \"\"\"\n Convert web response to JSON object, similar to the output of `httpx -irr -json`\n \"\"\"\n\n if response is None:\n return\n\n import mmh3\n from datetime import datetime\n from hashlib import md5, sha256\n from bbot.core.helpers.misc import tagify, urlparse, split_host_port, smart_decode\n\n request = response.request\n url = str(request.url)\n parsed_url = urlparse(url)\n netloc = parsed_url.netloc\n scheme = parsed_url.scheme.lower()\n host, port = split_host_port(f\"{scheme}://{netloc}\")\n\n raw_headers = \"\\r\\n\".join([f\"{k}: {v}\" for k, v in response.headers.items()])\n raw_headers_encoded = raw_headers.encode()\n\n headers = {}\n for k, v in response.headers.items():\n k = tagify(k, delimiter=\"_\")\n headers[k] = v\n\n j = {\n \"timestamp\": datetime.now().isoformat(),\n \"hash\": {\n \"body_md5\": md5(response.content).hexdigest(),\n \"body_mmh3\": mmh3.hash(response.content),\n \"body_sha256\": sha256(response.content).hexdigest(),\n # \"body_simhash\": \"TODO\",\n \"header_md5\": md5(raw_headers_encoded).hexdigest(),\n \"header_mmh3\": mmh3.hash(raw_headers_encoded),\n \"header_sha256\": sha256(raw_headers_encoded).hexdigest(),\n # \"header_simhash\": \"TODO\",\n },\n \"header\": headers,\n \"body\": smart_decode(response.content),\n \"content_type\": headers.get(\"content_type\", \"\").split(\";\")[0].strip(),\n \"url\": url,\n \"host\": str(host),\n \"port\": port,\n \"scheme\": scheme,\n \"method\": response.request.method,\n \"path\": parsed_url.path,\n \"raw_header\": raw_headers,\n \"status_code\": response.status_code,\n }\n\n return j\n" }, { "path": "bbot/core/helpers/wordcloud.py", "content": "import re\nimport csv\nimport string\nimport logging\nimport wordninja\nfrom pathlib import Path\nfrom contextlib import suppress\nfrom collections import OrderedDict\n\nfrom .misc import tldextract, extract_words\n\nlog = logging.getLogger(\"bbot.core.helpers.wordcloud\")\n\n\nclass WordCloud(dict):\n \"\"\"\n WordCloud is a specialized dictionary-like class for storing and aggregating\n words extracted from various data sources such as DNS names and URLs. The class\n is intended to facilitate the generation of target-specific wordlists and mutations.\n\n The WordCloud class can be accessed and manipulated like a standard Python dictionary.\n It also offers additional methods for generating mutations based on the words it contains.\n\n Attributes:\n parent_helper: The parent helper object that provides necessary utilities.\n devops_mutations: A set containing common devops-related mutations, loaded from a file.\n dns_mutator: An instance of the DNSMutator class for generating DNS-based mutations.\n\n Examples:\n >>> s = Scanner(\"www1.evilcorp.com\", \"www-test.evilcorp.com\")\n >>> s.start_without_generator()\n >>> print(s.helpers.word_cloud)\n {\n \"evilcorp\": 2,\n \"ec\": 2,\n \"www1\": 1,\n \"evil\": 2,\n \"www\": 2,\n \"w1\": 1,\n \"corp\": 2,\n \"1\": 1,\n \"wt\": 1,\n \"test\": 1,\n \"www-test\": 1\n }\n\n >>> s.helpers.word_cloud.mutations([\"word\"], cloud=True, numbers=0, devops=False, letters=False)\n [\n [\n \"1\",\n \"word\"\n ],\n [\n \"corp\",\n \"word\"\n ],\n [\n \"ec\",\n \"word\"\n ],\n [\n \"evil\",\n \"word\"\n ],\n ...\n ]\n\n >>> s.helpers.word_cloud.dns_mutator.mutations(\"word\")\n [\n \"word\",\n \"word-test\",\n \"word1\",\n \"wordtest\",\n \"www-word\",\n \"wwwword\"\n ]\n \"\"\"\n\n def __init__(self, parent_helper, *args, **kwargs):\n self.parent_helper = parent_helper\n\n devops_filename = self.parent_helper.wordlist_dir / \"devops_mutations.txt\"\n self.devops_mutations = set(self.parent_helper.read_file(devops_filename))\n\n self.dns_mutator = DNSMutator()\n\n super().__init__(*args, **kwargs)\n\n def mutations(\n self, words, devops=True, cloud=True, letters=True, numbers=5, number_padding=2, substitute_numbers=True\n ):\n \"\"\"\n Generate various mutations for the given list of words based on different criteria.\n\n Yields tuples of strings which can be joined on the desired delimiter, e.g. \"-\" or \"_\".\n\n Args:\n words (Union[str, Iterable[str]]): A single word or list of words to mutate.\n devops (bool): Whether to include devops-related mutations.\n cloud (bool): Whether to include mutations from the word cloud.\n letters (bool): Whether to include letter-based mutations.\n numbers (int): The maximum numeric mutations to include.\n number_padding (int): Padding for numeric mutations.\n substitute_numbers (bool): Whether to substitute numbers in mutations.\n\n Yields:\n tuple: A tuple containing each of the mutation segments.\n \"\"\"\n if isinstance(words, str):\n words = (words,)\n results = set()\n for word in words:\n h = hash(word)\n if h not in results:\n results.add(h)\n yield (word,)\n if numbers > 0:\n if substitute_numbers:\n for word in words:\n for number_mutation in self.get_number_mutations(word, n=numbers, padding=number_padding):\n h = hash(number_mutation)\n if h not in results:\n results.add(h)\n yield (number_mutation,)\n for word in words:\n for modifier in self.modifiers(\n devops=devops, cloud=cloud, letters=letters, numbers=numbers, number_padding=number_padding\n ):\n a = (word, modifier)\n b = (modifier, word)\n for _ in (a, b):\n h = hash(_)\n if h not in results:\n results.add(h)\n yield _\n\n def modifiers(self, devops=True, cloud=True, letters=True, numbers=5, number_padding=2):\n modifiers = set()\n if devops:\n modifiers.update(self.devops_mutations)\n if cloud:\n modifiers.update(set(self))\n if letters:\n modifiers.update(set(string.ascii_lowercase))\n if numbers > 0:\n modifiers.update(self.parent_helper.gen_numbers(numbers, number_padding))\n return modifiers\n\n def absorb_event(self, event):\n \"\"\"\n Absorbs an event from a BBOT scan into the word cloud.\n\n This method updates the word cloud by extracting words from the given event. It aims to avoid including PTR\n (Pointer) records, as they tend to produce unhelpful mutations in the word cloud.\n\n Args:\n event (Event): The event object containing the words to be absorbed into the word cloud.\n \"\"\"\n for word in event.words:\n self.add_word(word)\n if event.scope_distance == 0 and event.type.startswith(\"DNS_NAME\"):\n subdomain = tldextract(event.data).subdomain\n if subdomain and not self.parent_helper.is_ptr(subdomain):\n for s in subdomain.split(\".\"):\n self.dns_mutator.add_word(s)\n\n def absorb_word(self, word, wordninja=True):\n \"\"\"\n Absorbs a word into the word cloud after splitting it using a word extraction algorithm.\n\n This method splits the input word into smaller meaningful words using word extraction, and then adds each\n of them to the word cloud. The splitting is done using a predefined algorithm in the parent helper.\n\n Args:\n word (str): The word to be split and absorbed into the word cloud.\n wordninja (bool, optional): If True, word extraction is enabled. Defaults to True.\n\n Examples:\n >>> self.helpers.word_cloud.absorb_word(\"blacklantern\")\n >>> print(self.helpers.word_cloud)\n {\n \"blacklantern\": 1,\n \"black\": 1,\n \"bl\": 1,\n \"lantern\": 1\n }\n \"\"\"\n for w in self.parent_helper.extract_words(word, wordninja=wordninja):\n self.add_word(w)\n\n def add_word(self, word, lowercase=True):\n \"\"\"\n Adds a word to the word cloud.\n\n This method updates the word cloud by adding a given word. If the word already exists in the cloud,\n its frequency count is incremented by 1. Optionally, the word can be converted to lowercase before adding.\n\n Args:\n word (str): The word to be added to the word cloud.\n lowercase (bool, optional): If True, the word will be converted to lowercase before adding. Defaults to True.\n\n Examples:\n >>> self.helpers.word_cloud.add_word(\"Example\")\n >>> self.helpers.word_cloud.add_word(\"example\")\n >>> print(self.helpers.word_cloud)\n {'example': 2}\n \"\"\"\n if lowercase:\n word = word.lower()\n try:\n self[word] += 1\n except KeyError:\n self[word] = 1\n\n def get_number_mutations(self, base, n=5, padding=2):\n \"\"\"\n Generates mutations of a base string by modifying the numerical parts or appending numbers.\n\n This method detects existing numbers in the base string and tries incrementing and decrementing them within a\n specified range. It also appends numbers at the end or after each word to generate more mutations.\n\n Args:\n base (str): The base string to generate mutations from.\n n (int, optional): The range of numbers to use for incrementing/decrementing. Defaults to 5.\n padding (int, optional): Zero-pad numbers up to this length. Defaults to 2.\n\n Returns:\n set: A set of mutated strings based on the base input.\n\n Examples:\n >>> self.helpers.word_cloud.get_number_mutations(\"www2-test\", n=2)\n {\n \"www0-test\",\n \"www1-test\",\n \"www2-test\",\n \"www2-test0\",\n \"www2-test00\",\n \"www2-test01\",\n \"www2-test1\",\n \"www3-test\",\n \"www4-test\"\n }\n \"\"\"\n results = set()\n\n # detects numbers and increments/decrements them\n # e.g. for \"base2_p013\", we would try:\n # - \"base0_p013\" through \"base12_p013\"\n # - \"base2_p003\" through \"base2_p023\"\n # limited to three iterations for sanity's sake\n for match in list(self.parent_helper.regexes.num_regex.finditer(base))[-3:]:\n span = match.span()\n before = base[: span[0]]\n after = base[span[-1] :]\n number = base[span[0] : span[-1]]\n numlen = len(number)\n maxnum = min(int(\"9\" * numlen), int(number) + n)\n minnum = max(0, int(number) - n)\n for i in range(minnum, maxnum + 1):\n filled_num = str(i).zfill(numlen)\n results.add(f\"{before}{filled_num}{after}\")\n if not number.startswith(\"0\"):\n results.add(f\"{before}{i}{after}\")\n\n # appends numbers after each word\n # e.g., for \"base_www\", we would try:\n # - \"base1_www\", \"base2_www\", etc.\n # - \"base_www1\", \"base_www2\", etc.\n # limited to three iterations for sanity's sake\n number_suffixes = self.parent_helper.gen_numbers(n, padding)\n for match in list(self.parent_helper.regexes.word_regex.finditer(base))[-3:]:\n span = match.span()\n for suffix in number_suffixes:\n before = base[: span[-1]]\n after = base[span[-1] :]\n # skip if there's already a number\n if len(after) > 1 and not after[0].isdigit():\n results.add(f\"{before}{suffix}{after}\")\n # basic cases so we don't miss anything\n for s in number_suffixes:\n results.add(f\"{base}{s}\")\n results.add(base)\n\n return results\n\n def truncate(self, limit):\n \"\"\"\n Truncates the word cloud dictionary to retain only the top `limit` entries based on their occurrence frequencies.\n\n Args:\n limit (int): The maximum number of entries to retain in the word cloud.\n\n Examples:\n >>> self.helpers.word_cloud.update({\"apple\": 5, \"banana\": 2, \"cherry\": 8})\n >>> self.helpers.word_cloud.truncate(2)\n >>> self.helpers.word_cloud\n {'cherry': 8, 'apple': 5}\n \"\"\"\n new_self = dict(self.json(limit=limit))\n self.clear()\n self.update(new_self)\n\n def json(self, limit=None):\n \"\"\"\n Returns the word cloud as a sorted OrderedDict, optionally truncated to the top `limit` entries.\n\n Args:\n limit (int, optional): The maximum number of entries to include in the returned OrderedDict. If None, all entries are included.\n\n Returns:\n OrderedDict: A dictionary sorted by word frequencies, potentially truncated to the top `limit` entries.\n\n Examples:\n >>> self.helpers.word_cloud.update({\"apple\": 5, \"banana\": 2, \"cherry\": 8})\n >>> self.helpers.word_cloud.json(limit=2)\n OrderedDict([('cherry', 8), ('apple', 5)])\n \"\"\"\n cloud_sorted = sorted(self.items(), key=lambda x: x[-1], reverse=True)\n if limit is not None:\n cloud_sorted = cloud_sorted[:limit]\n return OrderedDict(cloud_sorted)\n\n @property\n def default_filename(self):\n return self.parent_helper.preset.scan.home / \"wordcloud.tsv\"\n\n def save(self, filename=None, limit=None):\n \"\"\"\n Saves the word cloud to a file. The cloud can optionally be truncated to the top `limit` entries.\n\n Args:\n filename (str, optional): The path to the file where the word cloud will be saved. If None, uses a default filename.\n limit (int, optional): The maximum number of entries to save to the file. If None, all entries are saved.\n\n Returns:\n tuple: A tuple containing a boolean indicating success or failure, and the resolved filename.\n\n Examples:\n >>> self.helpers.word_cloud.update({\"apple\": 5, \"banana\": 2, \"cherry\": 8})\n >>> self.helpers.word_cloud.save(filename=\"word_cloud.txt\", limit=2)\n (True, Path('word_cloud.txt'))\n \"\"\"\n if filename is None:\n filename = self.default_filename\n else:\n filename = Path(filename).resolve()\n try:\n if not self.parent_helper.mkdir(filename.parent):\n log.error(f\"Failure creating or error writing to {filename.parent} when saving word cloud\")\n return\n if len(self) > 0:\n log.debug(f\"Saving word cloud to {filename}\")\n with open(str(filename), mode=\"w\", newline=\"\") as f:\n c = csv.writer(f, delimiter=\"\\t\")\n for word, count in self.json(limit).items():\n c.writerow([count, word])\n log.debug(f\"Saved word cloud ({len(self):,} words) to {filename}\")\n return True, filename\n else:\n log.debug(\"No words to save\")\n except Exception as e:\n import traceback\n\n log.warning(f\"Failed to save word cloud to {filename}: {e}\")\n log.trace(traceback.format_exc())\n return False, filename\n\n def load(self, filename=None):\n \"\"\"\n Loads a word cloud from a file. The file can be either a standard wordlist with one entry per line\n or a .tsv (tab-separated) file where the first row is the count and the second row is the associated entry.\n\n Args:\n filename (str, optional): The path to the file from which to load the word cloud. If None, uses a default filename.\n \"\"\"\n if filename is None:\n wordcloud_path = self.default_filename\n else:\n wordcloud_path = Path(filename).resolve()\n log.verbose(f\"Loading word cloud from {wordcloud_path}\")\n try:\n with open(str(wordcloud_path), newline=\"\") as f:\n c = csv.reader(f, delimiter=\"\\t\")\n for row in c:\n if len(row) == 1:\n self.add_word(row[0])\n elif len(row) == 2:\n with suppress(Exception):\n count, word = row\n count = int(count)\n self[word] = count\n if len(self) > 0:\n log.success(f\"Loaded word cloud ({len(self):,} words) from {wordcloud_path}\")\n except Exception as e:\n import traceback\n\n log_fn = log.debug\n if filename is not None:\n log_fn = log.warning\n log_fn(f\"Failed to load word cloud from {wordcloud_path}: {e}\")\n if filename is not None:\n log.trace(traceback.format_exc())\n\n\nclass Mutator(dict):\n \"\"\"\n Base class for generating mutations from a list of words.\n It accumulates words and produces mutations from them.\n \"\"\"\n\n def mutations(self, words, max_mutations=None):\n mutations = self.top_mutations(max_mutations)\n ret = set()\n if isinstance(words, str):\n words = [words]\n for word in words:\n for m in self.mutate(word, mutations=mutations):\n ret.add(\"\".join(m))\n return ret\n\n def mutate(self, word, max_mutations=None, mutations=None):\n if mutations is None:\n mutations = self.top_mutations(max_mutations)\n for mutation in mutations.keys():\n ret = []\n for s in mutation:\n if s is not None:\n ret.append(s)\n else:\n ret.append(word)\n yield ret\n\n def top_mutations(self, n=None):\n if n is not None:\n return dict(sorted(self.items(), key=lambda x: x[-1], reverse=True)[:n])\n else:\n return dict(self)\n\n def _add_mutation(self, mutation):\n if None not in mutation:\n return\n mutation = tuple([m for m in mutation if m != \"\"])\n try:\n self[mutation] += 1\n except KeyError:\n self[mutation] = 1\n\n def add_word(self, word):\n pass\n\n\nclass DNSMutator(Mutator):\n \"\"\"\n DNS-specific mutator used by the `dnsbrute_mutations` module to generate target-specific subdomain mutations.\n\n This class extends the Mutator base class to add DNS-specific logic for generating\n subdomain mutations based on input words. It utilizes custom word extraction patterns\n and a wordninja model trained on DNS-specific data.\n\n Examples:\n >>> s = Scanner(\"www1.evilcorp.com\", \"www-test.evilcorp.com\")\n >>> s.start_without_generator()\n >>> s.helpers.word_cloud.dns_mutator.mutations(\"word\")\n [\n \"word\",\n \"word-test\",\n \"word1\",\n \"wordtest\",\n \"www-word\",\n \"wwwword\"\n ]\n \"\"\"\n\n extract_word_regexes = [\n re.compile(r, re.I)\n for r in [\n r\"[a-z]+\",\n r\"[a-z_-]+\",\n r\"[a-z0-9]+\",\n r\"[a-z0-9_-]+\",\n ]\n ]\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n wordlist_dir = Path(__file__).parent.parent.parent / \"wordlists\"\n wordninja_dns_wordlist = wordlist_dir / \"wordninja_dns.txt.gz\"\n self.model = wordninja.LanguageModel(wordninja_dns_wordlist)\n\n def mutations(self, words, max_mutations=None):\n if isinstance(words, str):\n words = [words]\n new_words = set()\n for word in words:\n for e in extract_words(word, acronyms=False, model=self.model, word_regexes=self.extract_word_regexes):\n new_words.add(e)\n return super().mutations(new_words, max_mutations=max_mutations)\n\n def add_word(self, word):\n spans = set()\n mutations = set()\n for r in self.extract_word_regexes:\n for match in r.finditer(word):\n span = match.span()\n if span not in spans:\n spans.add(span)\n for start, end in spans:\n match_str = word[start:end]\n # skip digits\n if match_str.isdigit():\n continue\n before = word[:start]\n after = word[end:]\n basic_mutation = (before, None, after)\n mutations.add(basic_mutation)\n match_str_split = self.model.split(match_str)\n if len(match_str_split) > 1:\n for i, s in enumerate(match_str_split):\n if s.isdigit():\n continue\n split_before = \"\".join(match_str_split[:i])\n split_after = \"\".join(match_str_split[i + 1 :])\n wordninja_mutation = (before + split_before, None, split_after + after)\n mutations.add(wordninja_mutation)\n for m in mutations:\n self._add_mutation(m)\n" }, { "path": "bbot/core/helpers/yara_helper.py", "content": "import yara\n\n\nclass YaraHelper:\n def __init__(self, parent_helper):\n self.parent_helper = parent_helper\n\n def compile_strings(self, strings: list[str], nocase=False):\n \"\"\"\n Compile a list of strings into a YARA rule\n \"\"\"\n # Format each string as a YARA string definition\n yara_strings = []\n for i, s in enumerate(strings):\n s = s.replace('\"', '\\\\\"')\n yara_string = f'$s{i} = \"{s}\"'\n if nocase:\n yara_string += \" nocase\"\n yara_strings.append(yara_string)\n yara_strings = \"\\n \".join(yara_strings)\n\n # Create the complete YARA rule\n yara_rule = f\"\"\"\nrule strings_match\n{{\n strings:\n {yara_strings}\n condition:\n any of them\n}}\n\"\"\"\n # Compile and return the rule\n return self.compile(source=yara_rule)\n\n def compile(self, *args, **kwargs):\n return yara.compile(*args, **kwargs)\n\n async def match(self, compiled_rules, text):\n \"\"\"\n Given a compiled YARA rule and a body of text, return a list of strings that match the rule\n \"\"\"\n matched_strings = []\n matches = await self.parent_helper.run_in_executor(compiled_rules.match, data=text)\n if matches:\n for match in matches:\n for string_match in match.strings:\n for instance in string_match.instances:\n matched_string = instance.matched_data.decode(\"utf-8\")\n matched_strings.append(matched_string)\n return matched_strings\n" }, { "path": "bbot/core/modules.py", "content": "import re\nimport ast\nimport sys\nimport atexit\nimport pickle\nimport logging\nimport importlib\nimport omegaconf\nimport traceback\nfrom copy import copy\nfrom pathlib import Path\nfrom omegaconf import OmegaConf\nfrom contextlib import suppress\n\nfrom bbot.core import CORE\nfrom bbot.errors import BBOTError\nfrom bbot.logger import log_to_stderr\n\nfrom .flags import flag_descriptions\nfrom .shared_deps import SHARED_DEPS\nfrom .helpers.misc import (\n list_files,\n sha1,\n search_dict_by_key,\n search_format_dict,\n make_table,\n os_platform,\n mkdir,\n)\n\n\nlog = logging.getLogger(\"bbot.module_loader\")\n\nbbot_code_dir = Path(__file__).parent.parent\n\n\nclass ModuleLoader:\n \"\"\"\n Main class responsible for preloading BBOT modules.\n\n This class is in charge of preloading modules to determine their dependencies.\n Once dependencies are identified, they are installed before the actual module is imported.\n This ensures that all requisite libraries and components are available for the module to function correctly.\n \"\"\"\n\n default_module_dir = bbot_code_dir / \"modules\"\n\n module_dir_regex = re.compile(r\"^[a-z][a-z0-9_]*$\")\n\n # if a module consumes these event types, automatically assume these dependencies\n default_module_deps = {\"HTTP_RESPONSE\": \"httpx\", \"URL\": \"httpx\", \"SOCIAL\": \"social\"}\n\n def __init__(self):\n self.core = CORE\n\n self._shared_deps = dict(SHARED_DEPS)\n\n self.__preloaded = {}\n self._configs = {}\n self.flag_choices = set()\n self.all_module_choices = set()\n self.scan_module_choices = set()\n self.output_module_choices = set()\n self.internal_module_choices = set()\n\n self._preload_cache = None\n\n self._module_dirs = set()\n self._module_dirs_preloaded = set()\n self.add_module_dir(self.default_module_dir)\n\n # save preload cache before exiting\n atexit.register(self.save_preload_cache)\n\n def copy(self):\n module_loader_copy = copy(self)\n module_loader_copy.__preloaded = dict(self.__preloaded)\n return module_loader_copy\n\n @property\n def preload_cache_file(self):\n return self.core.cache_dir / \"module_preload_cache\"\n\n @property\n def module_dirs(self):\n return self._module_dirs\n\n def add_module_dir(self, module_dir):\n module_dir = Path(module_dir).resolve()\n if module_dir in self._module_dirs:\n log.debug(f'Already added custom module dir \"{module_dir}\"')\n return\n if not module_dir.is_dir():\n log.warning(f'Failed to add custom module dir \"{module_dir}\", please make sure it exists')\n return\n new_module_dirs = set()\n for _module_dir in self.get_recursive_dirs(module_dir):\n _module_dir = Path(_module_dir).resolve()\n if _module_dir not in self._module_dirs:\n self._module_dirs.add(_module_dir)\n new_module_dirs.add(_module_dir)\n self.preload(module_dirs=new_module_dirs)\n\n def file_filter(self, file):\n file = file.resolve()\n for part in file.parts:\n if part.endswith(\"_submodules\") or part == \"templates\":\n return False\n return file.suffix.lower() == \".py\" and file.stem not in [\"base\", \"__init__\"]\n\n def preload(self, module_dirs=None):\n \"\"\"Preloads all BBOT modules.\n\n This function recursively iterates through each file in the module directories\n and preloads each BBOT module to gather its meta-information and dependencies.\n\n Args:\n module_dir (str or Path): Directory containing BBOT modules to be preloaded.\n\n Returns:\n dict: A dictionary where keys are the names of the preloaded modules and\n values are their respective preloaded data.\n\n Examples:\n >>> preload(\"/path/to/bbot_modules/\")\n {\n \"module1\": {...},\n \"module2\": {...},\n ...\n }\n \"\"\"\n new_modules = False\n if module_dirs is None:\n module_dirs = self.module_dirs\n\n for module_dir in module_dirs:\n if module_dir in self._module_dirs_preloaded:\n log.debug(f\"Already preloaded modules from {module_dir}\")\n continue\n\n log.debug(f\"Preloading modules from {module_dir}\")\n new_modules = True\n for module_file in list_files(module_dir, filter=self.file_filter):\n module_name = module_file.stem\n module_file = module_file.resolve()\n\n # try to load from cache\n module_cache_key = (str(module_file), tuple(module_file.stat()))\n preloaded = self.preload_cache.get(module_name, {})\n cache_key = preloaded.get(\"cache_key\", ())\n if preloaded and module_cache_key == cache_key:\n log.debug(f\"Preloading {module_name} from cache\")\n else:\n log.debug(f\"Preloading {module_name} from disk\")\n if module_dir.name == \"modules\":\n namespace = \"bbot.modules\"\n else:\n namespace = f\"bbot.modules.{module_dir.name}\"\n try:\n preloaded = self.preload_module(module_file)\n if preloaded is None:\n continue\n module_type = \"scan\"\n if module_dir.name in (\"output\", \"internal\"):\n module_type = str(module_dir.name)\n\n disable_auto_module_deps = preloaded.get(\"disable_auto_module_deps\", False)\n\n # derive module dependencies from watched event types (only for scan modules)\n if module_type == \"scan\" and not disable_auto_module_deps:\n for event_type in preloaded[\"watched_events\"]:\n if event_type in self.default_module_deps:\n deps_modules = set(preloaded.get(\"deps\", {}).get(\"modules\", []))\n deps_modules.add(self.default_module_deps[event_type])\n preloaded[\"deps\"][\"modules\"] = sorted(deps_modules)\n\n preloaded[\"type\"] = module_type\n preloaded[\"namespace\"] = namespace\n preloaded[\"cache_key\"] = module_cache_key\n\n except Exception:\n log_to_stderr(f\"Error preloading {module_file}\\n\\n{traceback.format_exc()}\", level=\"CRITICAL\")\n log_to_stderr(f\"Error in {module_file.name}\", level=\"CRITICAL\")\n sys.exit(1)\n\n self.all_module_choices.add(module_name)\n module_type = preloaded.get(\"type\", \"scan\")\n if module_type == \"scan\":\n self.scan_module_choices.add(module_name)\n elif module_type == \"output\":\n self.output_module_choices.add(module_name)\n elif module_type == \"internal\":\n self.internal_module_choices.add(module_name)\n\n flags = preloaded.get(\"flags\", [])\n self.flag_choices.update(set(flags))\n\n self.__preloaded[module_name] = preloaded\n config = OmegaConf.create(preloaded.get(\"config\", {}))\n self._configs[module_name] = config\n\n self._module_dirs_preloaded.add(module_dir)\n\n # update default config with module defaults\n module_config = omegaconf.OmegaConf.create(\n {\n \"modules\": self.configs(),\n }\n )\n self.core.merge_default(module_config)\n\n return new_modules\n\n @property\n def preload_cache(self):\n if self._preload_cache is None:\n self._preload_cache = {}\n if self.preload_cache_file.is_file():\n with suppress(Exception):\n with open(self.preload_cache_file, \"rb\") as f:\n self._preload_cache = pickle.load(f)\n return self._preload_cache\n\n @preload_cache.setter\n def preload_cache(self, value):\n self._preload_cache = value\n mkdir(self.preload_cache_file.parent)\n with open(self.preload_cache_file, \"wb\") as f:\n pickle.dump(self._preload_cache, f)\n\n def save_preload_cache(self):\n self.preload_cache = self.__preloaded\n\n @property\n def _preloaded(self):\n return self.__preloaded\n\n def get_recursive_dirs(self, *dirs):\n dirs = {Path(d).resolve() for d in dirs}\n for d in list(dirs):\n if not d.is_dir():\n continue\n for p in d.iterdir():\n if p.is_dir() and self.module_dir_regex.match(p.name):\n dirs.update(self.get_recursive_dirs(p))\n return dirs\n\n def preloaded(self, type=None):\n preloaded = {}\n if type is not None:\n preloaded = {k: v for k, v in self._preloaded.items() if self.check_type(k, type)}\n else:\n preloaded = dict(self._preloaded)\n return preloaded\n\n def configs(self, type=None):\n configs = {}\n if type is not None:\n configs = {k: v for k, v in self._configs.items() if self.check_type(k, type)}\n else:\n configs = dict(self._configs)\n return OmegaConf.create(configs)\n\n def find_and_replace(self, **kwargs):\n self.__preloaded = search_format_dict(self.__preloaded, **kwargs)\n self._shared_deps = search_format_dict(self._shared_deps, **kwargs)\n\n def check_type(self, module, type):\n return self._preloaded[module][\"type\"] == type\n\n def preload_module(self, module_file):\n \"\"\"\n Preloads a BBOT module to gather its meta-information and dependencies.\n\n This function reads a BBOT module file, extracts its attributes such as\n events watched and produced, flags, meta-information, and dependencies.\n\n Args:\n module_file (str): Path to the BBOT module file.\n\n Returns:\n dict: A dictionary containing meta-information and dependencies for the module.\n\n Examples:\n >>> preload_module(\"bbot/modules/wappalyzer.py\")\n {\n \"watched_events\": [\n \"HTTP_RESPONSE\"\n ],\n \"produced_events\": [\n \"TECHNOLOGY\"\n ],\n \"flags\": [\n \"active\",\n \"safe\",\n \"web-basic\",\n \"web-thorough\"\n ],\n \"meta\": {\n \"description\": \"Extract technologies from web responses\"\n },\n \"config\": {},\n \"options_desc\": {},\n \"hash\": \"d5a88dd3866c876b81939c920bf4959716e2a374\",\n \"deps\": {\n \"modules\": [\n \"httpx\"\n ]\n \"pip\": [\n \"python-Wappalyzer~=0.3.1\"\n ],\n \"pip_constraints\": [],\n \"shell\": [],\n \"apt\": [],\n \"ansible\": []\n },\n \"sudo\": false\n }\n \"\"\"\n watched_events = set()\n produced_events = set()\n flags = set()\n meta = {}\n deps_modules = set()\n deps_pip = []\n deps_pip_constraints = []\n deps_shell = []\n deps_apt = []\n deps_common = []\n ansible_tasks = []\n config = {}\n options_desc = {}\n disable_auto_module_deps = False\n python_code = open(module_file).read()\n # take a hash of the code so we can keep track of when it changes\n module_hash = sha1(python_code).hexdigest()\n parsed_code = ast.parse(python_code)\n\n # discard if the module isn't a valid BBOT module\n is_bbot_module = False\n for root_element in parsed_code.body:\n if type(root_element) == ast.ClassDef:\n for class_attr in root_element.body:\n if type(class_attr) == ast.Assign and any(\n target.id in (\"watched_events\", \"produced_events\") for target in class_attr.targets\n ):\n is_bbot_module = True\n break\n\n if not is_bbot_module:\n log.debug(f\"Skipping {module_file} as it is not a valid BBOT module\")\n return\n\n for root_element in parsed_code.body:\n # look for classes\n if type(root_element) == ast.ClassDef:\n for class_attr in root_element.body:\n if not type(class_attr) == ast.Assign:\n continue\n\n # class attributes that are dictionaries\n if type(class_attr.value) == ast.Dict:\n # module options\n if any(target.id == \"options\" for target in class_attr.targets):\n config.update(ast.literal_eval(class_attr.value))\n # module options\n elif any(target.id == \"options_desc\" for target in class_attr.targets):\n options_desc.update(ast.literal_eval(class_attr.value))\n # module metadata\n elif any(target.id == \"meta\" for target in class_attr.targets):\n meta = ast.literal_eval(class_attr.value)\n\n # class attributes that are lists\n if type(class_attr.value) == ast.List:\n # flags\n if any(target.id == \"flags\" for target in class_attr.targets):\n for flag in class_attr.value.elts:\n if type(flag.value) == str:\n flags.add(flag.value)\n # watched events\n elif any(target.id == \"watched_events\" for target in class_attr.targets):\n for event_type in class_attr.value.elts:\n if type(event_type.value) == str:\n watched_events.add(event_type.value)\n # produced events\n elif any(target.id == \"produced_events\" for target in class_attr.targets):\n for event_type in class_attr.value.elts:\n if type(event_type.value) == str:\n produced_events.add(event_type.value)\n\n # bbot module dependencies\n elif any(target.id == \"deps_modules\" for target in class_attr.targets):\n for dep_module in class_attr.value.elts:\n if type(dep_module.value) == str:\n deps_modules.add(dep_module.value)\n # python dependencies\n elif any(target.id == \"deps_pip\" for target in class_attr.targets):\n for dep_pip in class_attr.value.elts:\n if type(dep_pip.value) == str:\n deps_pip.append(dep_pip.value)\n elif any(target.id == \"deps_pip_constraints\" for target in class_attr.targets):\n for dep_pip in class_attr.value.elts:\n if type(dep_pip.value) == str:\n deps_pip_constraints.append(dep_pip.value)\n # apt dependencies\n elif any(target.id == \"deps_apt\" for target in class_attr.targets):\n for dep_apt in class_attr.value.elts:\n if type(dep_apt.value) == str:\n deps_apt.append(dep_apt.value)\n # bash dependencies\n elif any(target.id == \"deps_shell\" for target in class_attr.targets):\n for dep_shell in class_attr.value.elts:\n deps_shell.append(ast.literal_eval(dep_shell))\n # ansible playbook\n elif any(target.id == \"deps_ansible\" for target in class_attr.targets):\n ansible_tasks = ast.literal_eval(class_attr.value)\n # shared/common module dependencies\n elif any(target.id == \"deps_common\" for target in class_attr.targets):\n for dep_common in class_attr.value.elts:\n if type(dep_common.value) == str:\n deps_common.append(dep_common.value)\n\n # class attributes that are booleans\n if type(class_attr.value) == ast.Constant:\n if any(target.id == \"_disable_auto_module_deps\" for target in class_attr.targets):\n if type(class_attr.value.value) == bool:\n disable_auto_module_deps = class_attr.value.value\n\n for task in ansible_tasks:\n if \"become\" not in task:\n task[\"become\"] = False\n # don't sudo brew\n elif os_platform() == \"darwin\" and (\"package\" in task and task.get(\"become\", False) is True):\n task[\"become\"] = False\n\n preloaded_data = {\n \"path\": str(module_file.resolve()),\n \"watched_events\": sorted(watched_events),\n \"produced_events\": sorted(produced_events),\n \"flags\": sorted(flags),\n \"meta\": meta,\n \"config\": config,\n \"options_desc\": options_desc,\n \"hash\": module_hash,\n \"deps\": {\n \"modules\": sorted(deps_modules),\n \"pip\": deps_pip,\n \"pip_constraints\": deps_pip_constraints,\n \"shell\": deps_shell,\n \"apt\": deps_apt,\n \"ansible\": ansible_tasks,\n \"common\": deps_common,\n },\n \"sudo\": len(deps_apt) > 0,\n \"disable_auto_module_deps\": disable_auto_module_deps,\n }\n ansible_task_list = list(ansible_tasks)\n for dep_common in deps_common:\n try:\n ansible_task_list.extend(self._shared_deps[dep_common])\n except KeyError:\n common_choices = \",\".join(self._shared_deps)\n raise BBOTError(\n f'Error while preloading module \"{module_file}\": No shared dependency named \"{dep_common}\" (choices: {common_choices})'\n )\n for ansible_task in ansible_task_list:\n if any(x is True for x in search_dict_by_key(\"become\", ansible_task)) or any(\n x is True for x in search_dict_by_key(\"ansible_become\", ansible_tasks)\n ):\n preloaded_data[\"sudo\"] = True\n return preloaded_data\n\n def load_modules(self, module_names):\n modules = {}\n for module_name in module_names:\n try:\n module = self.load_module(module_name)\n except ModuleNotFoundError as e:\n raise BBOTError(\n f\"Error loading module {module_name}: {e}. You may have leftover artifacts from an older version of BBOT. Try deleting/renaming your '~/.bbot' directory.\"\n ) from e\n modules[module_name] = module\n return modules\n\n def load_module(self, module_name):\n \"\"\"Loads a BBOT module by its name.\n\n Imports the module from its namespace, locates its class, and returns it.\n Identifies modules based on the presence of `watched_events` and `produced_events` attributes.\n\n Args:\n module_name (str): The name of the module to load.\n\n Returns:\n object: The loaded module class object.\n\n Examples:\n >>> module = load_module(\"example_module\")\n >>> isinstance(module, object)\n True\n \"\"\"\n preloaded = self._preloaded[module_name]\n namespace = preloaded[\"namespace\"]\n try:\n module_path = preloaded[\"path\"]\n except KeyError:\n module_path = preloaded[\"cache_key\"][0]\n full_namespace = f\"{namespace}.{module_name}\"\n\n spec = importlib.util.spec_from_file_location(full_namespace, module_path)\n module = importlib.util.module_from_spec(spec)\n sys.modules[full_namespace] = module\n spec.loader.exec_module(module)\n\n # for every top-level variable in the .py file\n for variable in module.__dict__.keys():\n # get its value\n value = getattr(module, variable)\n with suppress(AttributeError):\n # if it has watched_events and produced_events\n if all(\n type(a) == list\n for a in (getattr(value, \"watched_events\", None), getattr(value, \"produced_events\", None))\n ):\n # and if its variable name matches its filename\n if value.__name__.lower() == module_name.lower():\n value._name = module_name\n # then we have a module\n return value\n\n def check_dependency(self, event_type, modname, produced):\n if event_type not in produced:\n return False\n if produced[event_type] == {modname}:\n return False\n return True\n\n @staticmethod\n def add_or_create(d, k, *items):\n try:\n d[k].update(set(items))\n except KeyError:\n d[k] = set(items)\n\n def modules_table(self, modules=None, mod_type=None, include_author=False, include_created_date=False):\n \"\"\"Generates a table of module information.\n\n Constructs a table to display information such as module name, type, and event details.\n\n Args:\n modules (list, optional): List of module names to include in the table.\n mod_type (str, optional): Type of modules to include ('scan', 'output', 'internal').\n\n Returns:\n str: A formatted table string.\n\n Examples:\n >>> print(modules_table([\"portscan\"]))\n +----------+--------+-----------------+------------------------------+-------------------------------+----------------------+-------------------+\n | Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events |\n +==========+========+=================+==============================+===============================+======================+===================+\n | portscan | scan | No | Execute port scans | active, aggressive, portscan, | DNS_NAME, IP_ADDRESS | OPEN_TCP_PORT |\n | | | | | web-thorough | | |\n +----------+--------+-----------------+------------------------------+-------------------------------+----------------------+-------------------+\n \"\"\"\n\n table = []\n header = [\"Module\", \"Type\", \"Needs API Key\", \"Description\", \"Flags\", \"Consumed Events\", \"Produced Events\"]\n if include_author:\n header.append(\"Author\")\n if include_created_date:\n header.append(\"Created Date\")\n maxcolwidths = [20, 10, 5, 30, 30, 20, 20]\n for module_name, preloaded in self.filter_modules(modules, mod_type):\n module_type = preloaded[\"type\"]\n consumed_events = sorted(preloaded.get(\"watched_events\", []))\n produced_events = sorted(preloaded.get(\"produced_events\", []))\n flags = sorted(preloaded.get(\"flags\", []))\n api_key_required = \"\"\n meta = preloaded.get(\"meta\", {})\n api_key_required = \"Yes\" if meta.get(\"auth_required\", False) else \"No\"\n description = meta.get(\"description\", \"\")\n row = [\n module_name,\n module_type,\n api_key_required,\n description,\n \", \".join(flags),\n \", \".join(consumed_events),\n \", \".join(produced_events),\n ]\n if include_author:\n author = meta.get(\"author\", \"\")\n row.append(author)\n if include_created_date:\n created_date = meta.get(\"created_date\", \"\")\n row.append(created_date)\n table.append(row)\n return make_table(table, header, maxcolwidths=maxcolwidths)\n\n def modules_options(self, modules=None, mod_type=None):\n \"\"\"\n Return a list of module options\n \"\"\"\n modules_options = {}\n for module_name, preloaded in self.filter_modules(modules, mod_type):\n modules_options[module_name] = []\n module_options = preloaded[\"config\"]\n module_options_desc = preloaded[\"options_desc\"]\n for k, v in sorted(module_options.items(), key=lambda x: x[0]):\n option_name = f\"modules.{module_name}.{k}\"\n option_type = type(v).__name__\n option_description = module_options_desc[k]\n modules_options[module_name].append((option_name, option_type, option_description, str(v)))\n return modules_options\n\n def modules_options_table(self, modules=None, mod_type=None):\n table = []\n header = [\"Config Option\", \"Type\", \"Description\", \"Default\"]\n for module_options in self.modules_options(modules, mod_type).values():\n table += module_options\n return make_table(table, header)\n\n def flags(self, flags=None):\n _flags = {}\n for module_name, preloaded in self.preloaded().items():\n for flag in preloaded.get(\"flags\", []):\n if not flags or flag in flags:\n try:\n _flags[flag].add(module_name)\n except KeyError:\n _flags[flag] = {module_name}\n\n _flags = sorted(_flags.items(), key=lambda x: x[0])\n _flags = sorted(_flags, key=lambda x: len(x[-1]), reverse=True)\n return _flags\n\n def flags_table(self, flags=None):\n table = []\n header = [\"Flag\", \"# Modules\", \"Description\", \"Modules\"]\n maxcolwidths = [20, 5, 40, 80]\n _flags = self.flags(flags=flags)\n for flag, modules in _flags:\n description = flag_descriptions.get(flag, \"\")\n table.append([flag, f\"{len(modules)}\", description, \", \".join(sorted(modules))])\n return make_table(table, header, maxcolwidths=maxcolwidths)\n\n def events(self):\n consuming_events = {}\n producing_events = {}\n for module_name, preloaded in self.preloaded().items():\n consumed = preloaded.get(\"watched_events\", [])\n produced = preloaded.get(\"produced_events\", [])\n for c in consumed:\n try:\n consuming_events[c].add(module_name)\n except KeyError:\n consuming_events[c] = {module_name}\n for c in produced:\n try:\n producing_events[c].add(module_name)\n except KeyError:\n producing_events[c] = {module_name}\n return consuming_events, producing_events\n\n def events_table(self):\n table = []\n header = [\"Event Type\", \"# Consuming Modules\", \"# Producing Modules\", \"Consuming Modules\", \"Producing Modules\"]\n consuming_events, producing_events = self.events()\n all_event_types = sorted(set(consuming_events).union(set(producing_events)))\n for e in all_event_types:\n consuming = sorted(consuming_events.get(e, []))\n producing = sorted(producing_events.get(e, []))\n table.append([e, len(consuming), len(producing), \", \".join(consuming), \", \".join(producing)])\n return make_table(table, header)\n\n def filter_modules(self, modules=None, mod_type=None):\n if modules is None:\n module_list = list(self.preloaded(type=mod_type).items())\n else:\n module_list = [(m, self._preloaded[m]) for m in modules]\n module_list.sort(key=lambda x: x[0])\n module_list.sort(key=lambda x: \"passive\" in x[-1][\"flags\"])\n module_list.sort(key=lambda x: x[-1][\"type\"], reverse=True)\n return module_list\n\n def ensure_config_files(self):\n files = self.core.files_config\n mkdir(files.config_dir)\n\n comment_notice = (\n \"# NOTICE: THESE ENTRIES ARE COMMENTED BY DEFAULT\\n\"\n + \"# Please be sure to uncomment when inserting API keys, etc.\\n\"\n )\n\n config_obj = OmegaConf.to_object(self.core.default_config)\n\n # ensure bbot.yml\n if not files.config_filename.exists():\n log_to_stderr(f\"Creating BBOT config at {files.config_filename}\")\n no_secrets_config = self.core.no_secrets_config(config_obj)\n yaml = OmegaConf.to_yaml(no_secrets_config)\n yaml = comment_notice + \"\\n\".join(f\"# {line}\" for line in yaml.splitlines())\n with open(str(files.config_filename), \"w\") as f:\n f.write(yaml)\n\n # ensure secrets.yml\n if not files.secrets_filename.exists():\n log_to_stderr(f\"Creating BBOT secrets at {files.secrets_filename}\")\n secrets_only_config = self.core.secrets_only_config(config_obj)\n yaml = OmegaConf.to_yaml(secrets_only_config)\n yaml = comment_notice + \"\\n\".join(f\"# {line}\" for line in yaml.splitlines())\n with open(str(files.secrets_filename), \"w\") as f:\n f.write(yaml)\n files.secrets_filename.chmod(0o600)\n\n\nMODULE_LOADER = ModuleLoader()\n" }, { "path": "bbot/core/multiprocess.py", "content": "import os\nimport atexit\nfrom contextlib import suppress\n\n\nclass SharedInterpreterState:\n \"\"\"\n A class to track the primary BBOT process.\n\n Used to prevent spawning multiple unwanted processes with multiprocessing.\n \"\"\"\n\n def __init__(self):\n self.main_process_var_name = \"_BBOT_MAIN_PID\"\n self.scan_process_var_name = \"_BBOT_SCAN_PID\"\n atexit.register(self.cleanup)\n\n @property\n def is_main_process(self):\n is_main_process = self.main_pid == os.getpid()\n return is_main_process\n\n @property\n def is_scan_process(self):\n is_scan_process = os.getpid() == self.scan_pid\n return is_scan_process\n\n @property\n def main_pid(self):\n main_pid = int(os.environ.get(self.main_process_var_name, 0))\n if main_pid == 0:\n main_pid = os.getpid()\n # if main PID is not set, set it to the current PID\n os.environ[self.main_process_var_name] = str(main_pid)\n return main_pid\n\n @property\n def scan_pid(self):\n scan_pid = int(os.environ.get(self.scan_process_var_name, 0))\n if scan_pid == 0:\n scan_pid = os.getpid()\n # if scan PID is not set, set it to the current PID\n os.environ[self.scan_process_var_name] = str(scan_pid)\n return scan_pid\n\n def update_scan_pid(self):\n os.environ[self.scan_process_var_name] = str(os.getpid())\n\n def cleanup(self):\n with suppress(Exception):\n if self.is_main_process:\n with suppress(KeyError):\n del os.environ[self.main_process_var_name]\n with suppress(KeyError):\n del os.environ[self.scan_process_var_name]\n\n\nSHARED_INTERPRETER_STATE = SharedInterpreterState()\n" }, { "path": "bbot/core/shared_deps.py", "content": "DEP_FFUF = [\n {\n \"name\": \"Download ffuf\",\n \"unarchive\": {\n \"src\": \"https://github.com/ffuf/ffuf/releases/download/v#{BBOT_DEPS_FFUF_VERSION}/ffuf_#{BBOT_DEPS_FFUF_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH_GOLANG}.tar.gz\",\n \"include\": \"ffuf\",\n \"dest\": \"#{BBOT_TOOLS}\",\n \"remote_src\": True,\n },\n }\n]\n\nDEP_DOCKER = [\n {\n \"name\": \"Check if Docker is already installed\",\n \"command\": \"docker --version\",\n \"register\": \"docker_installed\",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Install Docker (Non-Debian)\",\n \"package\": {\"name\": \"docker\", \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] != 'Debian' and docker_installed.rc != 0\",\n },\n {\n \"name\": \"Install Docker (Debian)\",\n \"package\": {\n \"name\": \"docker.io\",\n \"state\": \"present\",\n },\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Debian' and docker_installed.rc != 0\",\n },\n]\n\nDEP_MASSDNS = [\n {\n \"name\": \"install dev tools\",\n \"package\": {\"name\": [\"gcc\", \"git\", \"make\"], \"state\": \"present\"},\n \"become\": True,\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Download massdns source code\",\n \"git\": {\n \"repo\": \"https://github.com/blechschmidt/massdns.git\",\n \"dest\": \"#{BBOT_TEMP}/massdns\",\n \"single_branch\": True,\n \"version\": \"master\",\n },\n },\n {\n \"name\": \"Build massdns (Linux)\",\n \"command\": {\"chdir\": \"#{BBOT_TEMP}/massdns\", \"cmd\": \"make\", \"creates\": \"#{BBOT_TEMP}/massdns/bin/massdns\"},\n \"when\": \"ansible_facts['system'] == 'Linux'\",\n },\n {\n \"name\": \"Build massdns (non-Linux)\",\n \"command\": {\n \"chdir\": \"#{BBOT_TEMP}/massdns\",\n \"cmd\": \"make nolinux\",\n \"creates\": \"#{BBOT_TEMP}/massdns/bin/massdns\",\n },\n \"when\": \"ansible_facts['system'] != 'Linux'\",\n },\n {\n \"name\": \"Install massdns\",\n \"copy\": {\"src\": \"#{BBOT_TEMP}/massdns/bin/massdns\", \"dest\": \"#{BBOT_TOOLS}/\", \"mode\": \"u+x,g+x,o+x\"},\n },\n]\n\nDEP_CHROMIUM = [\n {\n \"name\": \"Install Chromium (Non-Debian)\",\n \"package\": {\"name\": \"chromium\", \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] != 'Debian'\",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Install Chromium dependencies (Ubuntu 24.04)\",\n \"package\": {\n \"name\": \"libasound2t64,libatk-bridge2.0-0,libatk1.0-0,libcairo2,libcups2,libdrm2,libgbm1,libnss3,libpango-1.0-0,libglib2.0-0,libxcomposite1,libxdamage1,libxfixes3,libxkbcommon0,libxrandr2\",\n \"state\": \"present\",\n },\n \"become\": True,\n \"when\": \"ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_version'] == '24.04'\",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Install Chromium dependencies (Other Debian-based)\",\n \"package\": {\n \"name\": \"libasound2,libatk-bridge2.0-0,libatk1.0-0,libcairo2,libcups2,libdrm2,libgbm1,libnss3,libpango-1.0-0,libglib2.0-0,libxcomposite1,libxdamage1,libxfixes3,libxkbcommon0,libxrandr2\",\n \"state\": \"present\",\n },\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Debian' and not (ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_version'] == '24.04')\",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Get latest Chromium version (Debian)\",\n \"uri\": {\n \"url\": \"https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2FLAST_CHANGE?alt=media\",\n \"return_content\": True,\n },\n \"register\": \"chromium_version\",\n \"when\": \"ansible_facts['os_family'] == 'Debian'\",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Get latest Chromium version (Darwin x86_64)\",\n \"uri\": {\n \"url\": \"https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2FLAST_CHANGE?alt=media\",\n \"return_content\": True,\n },\n \"register\": \"chromium_version_darwin_x86_64\",\n \"when\": \"ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'x86_64'\",\n },\n {\n \"name\": \"Get latest Chromium version (Darwin arm64)\",\n \"uri\": {\n \"url\": \"https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2FLAST_CHANGE?alt=media\",\n \"return_content\": True,\n },\n \"register\": \"chromium_version_darwin_arm64\",\n \"when\": \"ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'arm64'\",\n },\n {\n \"name\": \"Download Chromium (Debian)\",\n \"unarchive\": {\n \"src\": \"https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F{{ chromium_version.content }}%2Fchrome-linux.zip?alt=media\",\n \"remote_src\": True,\n \"dest\": \"#{BBOT_TOOLS}\",\n \"creates\": \"#{BBOT_TOOLS}/chrome-linux\",\n },\n \"when\": \"ansible_facts['os_family'] == 'Debian'\",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Download Chromium (Darwin x86_64)\",\n \"unarchive\": {\n \"src\": \"https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac%2F{{ chromium_version_darwin_x86_64.content }}%2Fchrome-mac.zip?alt=media\",\n \"remote_src\": True,\n \"dest\": \"#{BBOT_TOOLS}\",\n \"creates\": \"#{BBOT_TOOLS}/chrome-mac\",\n },\n \"when\": \"ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'x86_64'\",\n },\n {\n \"name\": \"Download Chromium (Darwin arm64)\",\n \"unarchive\": {\n \"src\": \"https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Mac_Arm%2F{{ chromium_version_darwin_arm64.content }}%2Fchrome-mac.zip?alt=media\",\n \"remote_src\": True,\n \"dest\": \"#{BBOT_TOOLS}\",\n \"creates\": \"#{BBOT_TOOLS}/chrome-mac\",\n },\n \"when\": \"ansible_facts['os_family'] == 'Darwin' and ansible_facts['architecture'] == 'arm64'\",\n },\n # Because Ubuntu is a special snowflake, we have to bend over backwards to fix the chrome sandbox\n # see https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md\n {\n \"name\": \"Chown chrome_sandbox to root:root\",\n \"command\": {\"cmd\": \"chown -R root:root #{BBOT_TOOLS}/chrome-linux/chrome_sandbox\"},\n \"when\": \"ansible_facts['os_family'] == 'Debian'\",\n \"become\": True,\n },\n {\n \"name\": \"Chmod chrome_sandbox to 4755\",\n \"command\": {\"cmd\": \"chmod -R 4755 #{BBOT_TOOLS}/chrome-linux/chrome_sandbox\"},\n \"when\": \"ansible_facts['os_family'] == 'Debian'\",\n \"become\": True,\n },\n]\n\nDEP_MASSCAN = [\n {\n \"name\": \"install os deps (Debian)\",\n \"package\": {\"name\": [\"gcc\", \"git\", \"make\", \"libpcap0.8-dev\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Debian'\",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"install dev tools (Non-Debian)\",\n \"package\": {\"name\": [\"gcc\", \"git\", \"make\", \"libpcap\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] != 'Debian'\",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Download masscan source code\",\n \"git\": {\n \"repo\": \"https://github.com/robertdavidgraham/masscan.git\",\n \"dest\": \"#{BBOT_TEMP}/masscan\",\n \"single_branch\": True,\n \"version\": \"master\",\n },\n },\n {\n \"name\": \"Build masscan\",\n \"command\": {\n \"chdir\": \"#{BBOT_TEMP}/masscan\",\n \"cmd\": \"make -j\",\n \"creates\": \"#{BBOT_TEMP}/masscan/bin/masscan\",\n },\n },\n {\n \"name\": \"Install masscan\",\n \"copy\": {\"src\": \"#{BBOT_TEMP}/masscan/bin/masscan\", \"dest\": \"#{BBOT_TOOLS}/\", \"mode\": \"u+x,g+x,o+x\"},\n },\n]\n\nDEP_JAVA = [\n {\n \"name\": \"Check if Java is installed\",\n \"command\": \"which java\",\n \"register\": \"java_installed\",\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Install latest JRE (Debian)\",\n \"package\": {\"name\": [\"default-jre\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Debian' and java_installed.rc != 0\",\n },\n {\n \"name\": \"Install latest JRE (Arch)\",\n \"package\": {\"name\": [\"jre-openjdk\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Archlinux' and java_installed.rc != 0\",\n },\n {\n \"name\": \"Install latest JRE (Fedora)\",\n \"package\": {\"name\": [\"which\", \"java-latest-openjdk-headless\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'RedHat' and java_installed.rc != 0\",\n },\n {\n \"name\": \"Install latest JRE (Alpine)\",\n \"package\": {\"name\": [\"openjdk11\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Alpine' and java_installed.rc != 0\",\n },\n]\n\n# shared module dependencies -- ffuf, massdns, chromium, etc.\nSHARED_DEPS = {}\nfor var, val in list(locals().items()):\n if var.startswith(\"DEP_\") and isinstance(val, list):\n var = var.split(\"_\", 1)[-1].lower()\n SHARED_DEPS[var] = val\n" }, { "path": "bbot/db/sql/models.py", "content": "# This file contains SQLModel (Pydantic + SQLAlchemy) models for BBOT events, scans, and targets.\n# Used by the SQL output modules, but portable for outside use.\n\nimport json\nimport logging\nfrom pydantic import ConfigDict\nfrom typing import List, Optional\nfrom datetime import datetime, timezone\nfrom typing_extensions import Annotated\nfrom pydantic.functional_validators import AfterValidator\nfrom sqlmodel import inspect, Column, Field, SQLModel, JSON, String, DateTime as SQLADateTime\n\n\nlog = logging.getLogger(\"bbot_server.models\")\n\n\ndef naive_datetime_validator(d: datetime):\n \"\"\"\n Converts all dates into UTC, then drops timezone information.\n\n This is needed to prevent inconsistencies in sqlite, because it is timezone-naive.\n \"\"\"\n # drop timezone info\n return d.replace(tzinfo=None)\n\n\nNaiveUTC = Annotated[datetime, AfterValidator(naive_datetime_validator)]\n\n\nclass CustomJSONEncoder(json.JSONEncoder):\n def default(self, obj):\n # handle datetime\n if isinstance(obj, datetime):\n return obj.isoformat()\n return super().default(obj)\n\n\nclass BBOTBaseModel(SQLModel):\n model_config = ConfigDict(extra=\"ignore\")\n\n def __init__(self, *args, **kwargs):\n self._validated = None\n super().__init__(*args, **kwargs)\n\n @property\n def validated(self):\n try:\n if self._validated is None:\n self._validated = self.__class__.model_validate(self)\n return self._validated\n except AttributeError:\n return self\n\n def to_json(self, **kwargs):\n return json.dumps(self.validated.model_dump(), sort_keys=True, cls=CustomJSONEncoder, **kwargs)\n\n @classmethod\n def _pk_column_names(cls):\n return [column.name for column in inspect(cls).primary_key]\n\n def __hash__(self):\n return hash(self.to_json())\n\n def __eq__(self, other):\n return hash(self) == hash(other)\n\n\n### EVENT ###\n\n\nclass Event(BBOTBaseModel, table=True):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n data = self._get_data(self.data, self.type)\n self.data = {self.type: data}\n if self.host:\n self.reverse_host = self.host[::-1]\n\n def get_data(self):\n return self._get_data(self.data, self.type)\n\n @staticmethod\n def _get_data(data, type):\n # handle SIEM-friendly format\n if isinstance(data, dict) and list(data) == [type]:\n return data[type]\n return data\n\n uuid: str = Field(\n primary_key=True,\n index=True,\n nullable=False,\n )\n id: str = Field(index=True)\n type: str = Field(index=True)\n scope_description: str\n data: dict = Field(sa_type=JSON)\n host: Optional[str]\n port: Optional[int]\n netloc: Optional[str]\n # store the host in reversed form for efficient lookups by domain\n reverse_host: Optional[str] = Field(default=\"\", exclude=True, index=True)\n resolved_hosts: List = Field(default=[], sa_type=JSON)\n dns_children: dict = Field(default={}, sa_type=JSON)\n web_spider_distance: int = 10\n scope_distance: int = Field(default=10, index=True)\n scan: str = Field(index=True)\n timestamp: NaiveUTC = Field(index=True)\n parent: str = Field(index=True)\n tags: List = Field(default=[], sa_type=JSON)\n module: str = Field(index=True)\n module_sequence: str\n discovery_context: str = \"\"\n discovery_path: List[str] = Field(default=[], sa_type=JSON)\n parent_chain: List[str] = Field(default=[], sa_type=JSON)\n inserted_at: NaiveUTC = Field(default_factory=lambda: datetime.now(timezone.utc))\n\n\n### SCAN ###\n\n\nclass Scan(BBOTBaseModel, table=True):\n id: str = Field(primary_key=True)\n name: str\n status: str\n started_at: NaiveUTC = Field(index=True)\n finished_at: Optional[NaiveUTC] = Field(default=None, sa_column=Column(SQLADateTime, nullable=True, index=True))\n duration_seconds: Optional[float] = Field(default=None)\n duration: Optional[str] = Field(default=None)\n target: dict = Field(sa_type=JSON)\n preset: dict = Field(sa_type=JSON)\n\n\n### TARGET ###\n\n\nclass Target(BBOTBaseModel, table=True):\n name: str = \"Default Target\"\n strict_scope: bool = False\n seeds: List = Field(default=[], sa_type=JSON)\n whitelist: List = Field(default=None, sa_type=JSON)\n blacklist: List = Field(default=[], sa_type=JSON)\n hash: str = Field(sa_column=Column(\"hash\", String(length=255), unique=True, primary_key=True, index=True))\n scope_hash: str = Field(sa_column=Column(\"scope_hash\", String(length=255), index=True))\n seed_hash: str = Field(sa_column=Column(\"seed_hashhash\", String(length=255), index=True))\n whitelist_hash: str = Field(sa_column=Column(\"whitelist_hash\", String(length=255), index=True))\n blacklist_hash: str = Field(sa_column=Column(\"blacklist_hash\", String(length=255), index=True))\n" }, { "path": "bbot/defaults.yml", "content": "### BASIC OPTIONS ###\n\n# BBOT working directory\nhome: ~/.bbot\n# How many scan results to keep before cleaning up the older ones\nkeep_scans: 20\n# Interval for displaying status messages\nstatus_frequency: 15\n# Include the raw data of files (i.e. PDFs, web screenshots) as base64 in the event\nfile_blobs: false\n# Include the raw data of directories (i.e. git repos) as tar.gz base64 in the event\nfolder_blobs: false\n\n### SCOPE ###\n\nscope:\n # strict scope means only exact DNS names are considered in-scope\n # subdomains are not included unless they are explicitly provided in the target list\n strict: false\n # Filter by scope distance which events are displayed in the output\n # 0 == show only in-scope events (affiliates are always shown)\n # 1 == show all events up to distance-1 (1 hop from target)\n report_distance: 0\n # How far out from the main scope to search\n # Do not change this setting unless you know what you're doing\n search_distance: 0\n\n### DNS ###\n\ndns:\n # Completely disable DNS resolution (careful if you have IP whitelists/blacklists, consider using minimal=true instead)\n disable: false\n # Speed up scan by not creating any new DNS events, and only resolving A and AAAA records\n minimal: false\n # How many instances of the dns module to run concurrently\n threads: 25\n # How many concurrent DNS resolvers to use when brute-forcing\n # (under the hood this is passed through directly to massdns -s)\n brute_threads: 1000\n # nameservers to use for DNS brute-forcing\n # default is updated weekly and contains ~10K high-quality public servers\n brute_nameservers: https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt\n # How far away from the main target to explore via DNS resolution (independent of scope.search_distance)\n # This is safe to change\n search_distance: 1\n # Limit how many DNS records can be followed in a row (stop malicious/runaway DNS records)\n runaway_limit: 5\n # DNS query timeout\n timeout: 5\n # How many times to retry DNS queries\n retries: 1\n # Completely disable BBOT's DNS wildcard detection\n wildcard_disable: False\n # Disable BBOT's DNS wildcard detection for select domains\n wildcard_ignore: []\n # How many sanity checks to make when verifying wildcard DNS\n # Increase this value if BBOT's wildcard detection isn't working\n wildcard_tests: 10\n # Skip DNS requests for a certain domain and rdtype after encountering this many timeouts or SERVFAILs\n # This helps prevent faulty DNS servers from hanging up the scan\n abort_threshold: 50\n # Don't show PTR records containing IP addresses\n filter_ptrs: true\n # Enable/disable debug messages for DNS queries\n debug: false\n # For performance reasons, always skip these DNS queries\n # Microsoft's DNS infrastructure is misconfigured so that certain queries to mail.protection.outlook.com always time out\n omit_queries:\n - SRV:mail.protection.outlook.com\n - CNAME:mail.protection.outlook.com\n - TXT:mail.protection.outlook.com\n\n### WEB ###\n\nweb:\n # HTTP proxy\n http_proxy:\n # Web user-agent\n user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.2151.97\n # Set the maximum number of HTTP links that can be followed in a row (0 == no spidering allowed)\n spider_distance: 0\n # Set the maximum directory depth for the web spider\n spider_depth: 1\n # Set the maximum number of links that can be followed per page\n spider_links_per_page: 25\n # HTTP timeout (for Python requests; API calls, etc.)\n http_timeout: 10\n # HTTP timeout (for httpx)\n httpx_timeout: 5\n # Custom HTTP headers (e.g. cookies, etc.)\n # in the format { \"Header-Key\": \"header_value\" }\n # These are attached to all in-scope HTTP requests\n # Note that some modules (e.g. github) may end up sending these to out-of-scope resources\n http_headers: {}\n # How many times to retry API requests\n # Note that this is a separate mechanism on top of HTTP retries\n # which will retry API requests that don't return a successful status code\n api_retries: 2\n # HTTP retries - try again if the raw connection fails\n http_retries: 1\n # HTTP retries (for httpx)\n httpx_retries: 1\n # Default sleep interval when rate limited by 429 (and retry-after isn't provided)\n 429_sleep_interval: 30\n # Maximum sleep interval when rate limited by 429 (and an excessive retry-after is provided)\n 429_max_sleep_interval: 60\n # Enable/disable debug messages for web requests/responses\n debug: false\n # Maximum number of HTTP redirects to follow\n http_max_redirects: 5\n # Whether to verify SSL certificates\n ssl_verify: false\n\n### ENGINE ###\n\nengine:\n debug: false\n\n# Tool dependencies\ndeps:\n ffuf:\n version: \"2.1.0\"\n # How to handle installation of module dependencies\n # Choices are:\n # - abort_on_failure (default) - if a module dependency fails to install, abort the scan\n # - retry_failed - try again to install failed dependencies\n # - ignore_failed - run the scan regardless of what happens with dependency installation\n # - disable - completely disable BBOT's dependency system (you are responsible for installing tools, pip packages, etc.)\n behavior: abort_on_failure\n\n### ADVANCED OPTIONS ###\n\n# Load BBOT modules from these custom paths\nmodule_dirs: []\n\n# maximum runtime in seconds for each module's handle_event() is 60 minutes\n# when the timeout is reached, the offending handle_event() will be cancelled and the module will move on to the next event\nmodule_handle_event_timeout: 3600\n# handle_batch() default timeout is 2 hours\nmodule_handle_batch_timeout: 7200\n\n# Infer certain events from others, e.g. IPs from IP ranges, DNS_NAMEs from URLs, etc.\nspeculate: True\n# Passively search event data for URLs, hostnames, emails, etc.\nexcavate: True\n# Summarize activity at the end of a scan\naggregate: True\n# DNS resolution, wildcard detection, etc.\ndnsresolve: True\n# Cloud provider tagging\ncloudcheck: True\n\n# Strip querystring from URLs by default\nurl_querystring_remove: True\n# When query string is retained, by default collapse parameter values down to a single value per parameter\nurl_querystring_collapse: True\n\n# Completely ignore URLs with these extensions\nurl_extension_blacklist:\n # images\n - png\n - jpg\n - bmp\n - ico\n - jpeg\n - gif\n - svg\n - webp\n # web/fonts\n - css\n - woff\n - woff2\n - ttf\n - eot\n - sass\n - scss\n # audio\n - mp3\n - m4a\n - wav\n - flac\n # video\n - mp4\n - mkv\n - avi\n - wmv\n - mov\n - flv\n - webm\n\n# URLs with these extensions are not distributed to modules unless the module opts in via `accept_url_special = True`\n# They are also excluded from output. If you want to see them in output, remove them from this list.\nurl_extension_special:\n - js\n\n# These url extensions are almost always static, so we exclude them from modules that fuzz things\nurl_extension_static:\n - pdf\n - doc\n - docx\n - xls\n - xlsx\n - ppt\n - pptx\n - txt\n - csv\n - xml\n - yaml\n - ini\n - log\n - conf\n - cfg\n - env\n - md\n - rtf\n - tiff\n - bmp\n - jpg\n - jpeg\n - png\n - gif\n - svg\n - ico\n - mp3\n - wav\n - flac\n - mp4\n - mov\n - avi\n - mkv\n - webm\n - zip\n - tar\n - gz\n - bz2\n - 7z\n - rar\n\nparameter_blacklist:\n - __VIEWSTATE\n - __EVENTARGUMENT\n - __EVENTVALIDATION\n - __EVENTTARGET\n - __EVENTARGUMENT\n - __VIEWSTATEGENERATOR\n - __SCROLLPOSITIONY\n - __SCROLLPOSITIONX\n - ASP.NET_SessionId\n - PHPSESSID\n - __cf_bm\n - f5_cspm\n\nparameter_blacklist_prefixes:\n - TS01\n - BIGipServer\n - incap_\n - visid_incap_\n - AWSALB\n - utm_\n - ApplicationGatewayAffinity\n - JSESSIONID\n - ARRAffinity\n\n# Don't output these types of events (they are still distributed to modules)\nomit_event_types:\n - HTTP_RESPONSE\n - RAW_TEXT\n - URL_UNVERIFIED\n - DNS_NAME_UNRESOLVED\n - FILESYSTEM\n - WEB_PARAMETER\n - RAW_DNS_RECORD\n # - IP_ADDRESS\n\n# Custom interactsh server settings\ninteractsh_server: null\ninteractsh_token: null\ninteractsh_disable: false\n" }, { "path": "bbot/errors.py", "content": "class BBOTError(Exception):\n pass\n\n\nclass ScanError(BBOTError):\n pass\n\n\nclass ValidationError(BBOTError):\n pass\n\n\nclass ConfigLoadError(BBOTError):\n pass\n\n\nclass HttpCompareError(BBOTError):\n pass\n\n\nclass DirectoryCreationError(BBOTError):\n pass\n\n\nclass DirectoryDeletionError(BBOTError):\n pass\n\n\nclass NTLMError(BBOTError):\n pass\n\n\nclass InteractshError(BBOTError):\n pass\n\n\nclass WordlistError(BBOTError):\n pass\n\n\nclass CurlError(BBOTError):\n pass\n\n\nclass PresetNotFoundError(BBOTError):\n pass\n\n\nclass EnableModuleError(BBOTError):\n pass\n\n\nclass EnableFlagError(BBOTError):\n pass\n\n\nclass BBOTArgumentError(BBOTError):\n pass\n\n\nclass PresetConditionError(BBOTError):\n pass\n\n\nclass PresetAbortError(PresetConditionError):\n pass\n\n\nclass BBOTEngineError(BBOTError):\n pass\n\n\nclass WebError(BBOTEngineError):\n pass\n\n\nclass DNSError(BBOTEngineError):\n pass\n\n\nclass ExcavateError(BBOTError):\n pass\n" }, { "path": "bbot/logger.py", "content": "import os\nimport sys\nimport logging.handlers\n\nloglevel_mapping = {\n \"DEBUG\": \"DBUG\",\n \"TRACE\": \"TRCE\",\n \"VERBOSE\": \"VERB\",\n \"HUGEVERBOSE\": \"VERB\",\n \"INFO\": \"INFO\",\n \"HUGEINFO\": \"INFO\",\n \"SUCCESS\": \"SUCC\",\n \"HUGESUCCESS\": \"SUCC\",\n \"WARNING\": \"WARN\",\n \"HUGEWARNING\": \"WARN\",\n \"ERROR\": \"ERRR\",\n \"CRITICAL\": \"CRIT\",\n}\ncolor_mapping = {\n \"DEBUG\": 242, # grey\n \"TRACE\": 242, # red\n \"VERBOSE\": 242, # grey\n \"INFO\": 69, # blue\n \"HUGEINFO\": 69, # blue\n \"SUCCESS\": 118, # green\n \"HUGESUCCESS\": 118, # green\n \"WARNING\": 208, # orange\n \"HUGEWARNING\": 208, # orange\n \"ERROR\": 196, # red\n \"CRITICAL\": 196, # red\n}\ncolor_prefix = \"\\033[1;38;5;\"\ncolor_suffix = \"\\033[0m\"\n\n\ndef colorize(s, level=\"INFO\"):\n seq = color_mapping.get(level, 15) # default white\n colored = f\"{color_prefix}{seq}m{s}{color_suffix}\"\n return colored\n\n\ndef log_to_stderr(msg, level=\"INFO\", logname=True):\n \"\"\"\n Print to stderr with BBOT logger colors\n \"\"\"\n levelname = level.upper()\n if not any(x in sys.argv for x in (\"-s\", \"--silent\")):\n levelshort = f\"[{loglevel_mapping.get(level, 'INFO')}]\"\n levelshort = f\"{colorize(levelshort, level=levelname)}\"\n if levelname == \"CRITICAL\" or levelname.startswith(\"HUGE\"):\n msg = colorize(msg, level=levelname)\n if logname:\n msg = f\"{levelshort} {msg}\"\n print(msg, file=sys.stderr)\n\n\nclass GzipRotatingFileHandler(logging.handlers.RotatingFileHandler):\n \"\"\"\n A rotating file handler that compresses rotated files with gzip.\n Checks file size only periodically to improve performance.\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self._msg_count = 0\n self._check_interval = 1000 # Check size every 1000 messages\n\n def rotation_filename(self, default_name):\n \"\"\"\n Modify the rotated filename to include .gz extension\n \"\"\"\n return default_name + \".gz\"\n\n def rotate(self, source, dest):\n \"\"\"\n Compress the source file and move it to the destination.\n \"\"\"\n import gzip\n\n with open(source, \"rb\") as f_in:\n with gzip.open(dest, \"wb\") as f_out:\n f_out.writelines(f_in)\n os.remove(source)\n\n def emit(self, record):\n \"\"\"\n Emit a record, checking for rollover only periodically using modulo.\n \"\"\"\n self._msg_count += 1\n\n # Only check for rollover periodically to save compute\n if self._msg_count % self._check_interval == 0:\n if self.shouldRollover(record):\n self.doRollover()\n\n # Continue with normal emit process\n super().emit(record)\n" }, { "path": "bbot/modules/__init__.py", "content": "" }, { "path": "bbot/modules/ajaxpro.py", "content": "import regex as re\nfrom urllib.parse import urlparse\nfrom bbot.modules.base import BaseModule\n\n\nclass ajaxpro(BaseModule):\n \"\"\"\n Reference: https://mogwailabs.de/en/blog/2022/01/vulnerability-spotlight-rce-in-ajax.net-professional/\n \"\"\"\n\n ajaxpro_regex = re.compile(r' self.dns_abort_depth:\n return True\n return False\n\n async def abort_if(self, event):\n # abort if dns name is unresolved\n if event.type == \"DNS_NAME_UNRESOLVED\":\n return True, \"DNS name is unresolved\"\n return await super().abort_if(event)\n\n async def parse_results(self, r, query):\n results = set()\n json = r.json()\n if json:\n for hostname in json:\n hostname = str(hostname).lower()\n in_scope = hostname.endswith(f\".{query}\")\n is_ptr = self.helpers.is_ptr(hostname)\n too_long = self.abort_if_pre(hostname)\n if in_scope and not is_ptr and not too_long:\n results.add(hostname)\n return sorted(results)[: self.config.get(\"limit\", 1000)]\n" }, { "path": "bbot/modules/apkpure.py", "content": "import re\nfrom pathlib import Path\nfrom bbot.modules.base import BaseModule\n\n\nclass apkpure(BaseModule):\n watched_events = [\"MOBILE_APP\"]\n produced_events = [\"FILESYSTEM\"]\n flags = [\"passive\", \"safe\", \"code-enum\", \"download\"]\n meta = {\n \"description\": \"Download android applications from apkpure.com\",\n \"created_date\": \"2024-10-11\",\n \"author\": \"@domwhewell-sage\",\n }\n options = {\"output_folder\": \"\"}\n options_desc = {\n \"output_folder\": \"Folder to download APKs to. If not specified, downloaded APKs will be deleted when the scan completes, to minimize disk usage.\"\n }\n\n async def setup(self):\n output_folder = self.config.get(\"output_folder\", \"\")\n if output_folder:\n self.output_dir = Path(output_folder) / \"apk_files\"\n else:\n self.output_dir = self.scan.temp_dir / \"apk_files\"\n self.helpers.mkdir(self.output_dir)\n return await super().setup()\n\n async def filter_event(self, event):\n if event.type == \"MOBILE_APP\":\n if \"android\" not in event.tags:\n return False, \"event is not an android app\"\n return True\n\n async def handle_event(self, event):\n app_id = event.data.get(\"id\", \"\")\n path = await self.download_apk(app_id)\n if path:\n await self.emit_event(\n {\"path\": str(path)},\n \"FILESYSTEM\",\n tags=[\"apk\", \"file\"],\n parent=event,\n context=f'{{module}} downloaded the apk \"{app_id}\" to: {path}',\n )\n\n async def download_apk(self, app_id):\n path = None\n url = f\"https://d.apkpure.com/b/XAPK/{app_id}?version=latest\"\n self.helpers.mkdir(self.output_dir / app_id)\n response = await self.helpers.request(url, allow_redirects=True)\n if response:\n attachment = response.headers.get(\"Content-Disposition\", \"\")\n if \"filename\" in attachment:\n match = re.search(r'filename=\"?([^\"]+)\"?', attachment)\n if match:\n filename = match.group(1)\n extension = filename.split(\".\")[-1]\n content = response.content\n file_destination = self.output_dir / app_id / f\"{app_id}.{extension}\"\n with open(file_destination, \"wb\") as f:\n f.write(content)\n self.info(f'Downloaded \"{app_id}\" from \"{url}\", saved to {file_destination}')\n path = file_destination\n return path\n" }, { "path": "bbot/modules/aspnet_bin_exposure.py", "content": "from bbot.modules.base import BaseModule\n\n\nclass aspnet_bin_exposure(BaseModule):\n watched_events = [\"URL\"]\n produced_events = [\"VULNERABILITY\"]\n flags = [\"active\", \"safe\", \"web-thorough\"]\n meta = {\n \"description\": \"Check for ASP.NET Security Feature Bypasses (CVE-2023-36899 and CVE-2023-36560)\",\n \"created_date\": \"2025-01-28\",\n \"author\": \"@liquidsec\",\n }\n\n in_scope_only = True\n test_dlls = [\n \"Telerik.Web.UI.dll\",\n \"Newtonsoft.Json.dll\",\n \"System.Net.Http.dll\",\n \"EntityFramework.dll\",\n \"AjaxControlToolkit.dll\",\n ]\n\n @staticmethod\n def normalize_url(url):\n return str(url.rstrip(\"/\") + \"/\").lower()\n\n def _incoming_dedup_hash(self, event):\n return hash(self.normalize_url(event.data))\n\n async def handle_event(self, event):\n normalized_url = self.normalize_url(event.data)\n for test_dll in self.test_dlls:\n for technique in [\"b/(S(X))in/###DLL_PLACEHOLDER###/(S(X))/\", \"(S(X))/b/(S(X))in/###DLL_PLACEHOLDER###\"]:\n test_url = f\"{normalized_url}{technique.replace('###DLL_PLACEHOLDER###', test_dll)}\"\n self.debug(f\"Sending test URL: [{test_url}]\")\n kwargs = {\"method\": \"GET\", \"allow_redirects\": False, \"timeout\": 10}\n test_result = await self.helpers.request(test_url, **kwargs)\n if test_result:\n if test_result.status_code == 200 and (\n \"content-type\" in test_result.headers\n and \"application/x-msdownload\" in test_result.headers[\"content-type\"]\n ):\n self.debug(\n f\"Got positive result for probe with test url: [{test_url}]. Status Code: [{test_result.status_code}] Content Length: [{len(test_result.content)}]\"\n )\n\n if test_result.status_code == 200 and (\n \"content-type\" in test_result.headers\n and \"application/x-msdownload\" in test_result.headers[\"content-type\"]\n ):\n confirm_url = (\n f\"{normalized_url}{technique.replace('###DLL_PLACEHOLDER###', 'oopsnotarealdll.dll')}\"\n )\n confirm_result = await self.helpers.request(confirm_url, **kwargs)\n\n if confirm_result and (\n confirm_result.status_code != 200\n or not (\n \"content-type\" in confirm_result.headers\n and \"application/x-msdownload\" in confirm_result.headers[\"content-type\"]\n )\n ):\n description = f\"IIS Bin Directory DLL Exposure. Detection Url: [{test_url}]\"\n await self.emit_event(\n {\n \"severity\": \"HIGH\",\n \"host\": str(event.host),\n \"url\": normalized_url,\n \"description\": description,\n },\n \"VULNERABILITY\",\n event,\n context=\"{module} detected IIS Bin Directory DLL Exposure vulnerability\",\n )\n return True\n\n async def filter_event(self, event):\n if \"dir\" in event.tags:\n return True\n return False\n" }, { "path": "bbot/modules/azure_realm.py", "content": "from .base import BaseModule\n\n\nclass azure_realm(BaseModule):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"URL_UNVERIFIED\"]\n flags = [\"affiliates\", \"subdomain-enum\", \"cloud-enum\", \"web-basic\", \"passive\", \"safe\"]\n meta = {\n \"description\": 'Retrieves the \"AuthURL\" from login.microsoftonline.com/getuserrealm',\n \"created_date\": \"2023-07-12\",\n \"author\": \"@TheTechromancer\",\n }\n\n async def setup(self):\n self.processed = set()\n return True\n\n async def handle_event(self, event):\n _, domain = self.helpers.split_domain(event.data)\n domain_hash = hash(domain)\n if domain_hash not in self.processed:\n self.processed.add(domain_hash)\n auth_url = await self.getuserrealm(domain)\n if auth_url:\n url_event = self.make_event(\n auth_url, \"URL_UNVERIFIED\", parent=event, tags=[\"affiliate\", \"ms-auth-url\"]\n )\n url_event.source_domain = domain\n await self.emit_event(\n url_event,\n context=\"{module} queried login.microsoftonline.com for user realm and found {event.type}: {event.data}\",\n )\n\n async def getuserrealm(self, domain):\n url = f\"https://login.microsoftonline.com/getuserrealm.srf?login=test@{domain}\"\n r = await self.helpers.request(url)\n if r is None:\n return\n try:\n json = r.json()\n except Exception:\n return\n if json and isinstance(json, dict):\n return json.get(\"AuthURL\", \"\")\n" }, { "path": "bbot/modules/azure_tenant.py", "content": "from bbot.modules.base import BaseModule\n\n\nclass azure_tenant(BaseModule):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"affiliates\", \"subdomain-enum\", \"cloud-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query Azure via azmap.dev for tenant sister domains\",\n \"created_date\": \"2024-07-04\",\n \"author\": \"@TheTechromancer\",\n }\n\n base_url = \"https://azmap.dev/api/tenant\"\n in_scope_only = True\n per_domain_only = True\n\n async def setup(self):\n self.processed = set()\n return True\n\n async def handle_event(self, event):\n _, query = self.helpers.split_domain(event.data)\n tenant_data = await self.query(query)\n\n if not tenant_data:\n return\n\n tenant_id = tenant_data.get(\"tenant_id\")\n tenant_name = tenant_data.get(\"tenant_name\")\n email_domains = tenant_data.get(\"email_domains\", [])\n\n if email_domains:\n self.verbose(\n f'Found {len(email_domains):,} domains under tenant for \"{query}\": {\", \".join(sorted(email_domains))}'\n )\n for domain in email_domains:\n if domain != query:\n await self.emit_event(\n domain,\n \"DNS_NAME\",\n parent=event,\n tags=[\"affiliate\", \"azure-tenant\"],\n context=f'{{module}} queried azmap.dev for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n\n # Build tenant names list (include the tenant name from the API)\n tenant_names = []\n if tenant_name:\n tenant_names.append(tenant_name)\n\n # Also extract tenant names from .onmicrosoft.com domains\n for domain in email_domains:\n if domain.lower().endswith(\".onmicrosoft.com\"):\n tenantname = domain.split(\".\")[0].lower()\n if tenantname and tenantname not in tenant_names:\n tenant_names.append(tenantname)\n\n event_data = {\"tenant-names\": tenant_names, \"domains\": sorted(email_domains)}\n tenant_names_str = \",\".join(tenant_names)\n if tenant_id:\n event_data[\"tenant-id\"] = tenant_id\n await self.emit_event(\n event_data,\n \"AZURE_TENANT\",\n parent=event,\n context=f'{{module}} queried azmap.dev for \"{query}\" and found {{event.type}}: {tenant_names_str}',\n )\n\n async def query(self, domain):\n url = f\"{self.base_url}?domain={domain}&extract=true\"\n\n self.debug(f\"Retrieving tenant domains at {url}\")\n\n r = await self.helpers.request(url)\n status_code = getattr(r, \"status_code\", 0)\n if status_code != 200:\n self.verbose(f'Error retrieving azure_tenant domains for \"{domain}\" (status code: {status_code})')\n return {}\n\n try:\n tenant_data = r.json()\n except Exception as e:\n self.warning(f'Error parsing JSON response for \"{domain}\": {e}')\n return {}\n\n # Absorb domains into word cloud\n email_domains = tenant_data.get(\"email_domains\", [])\n for d in email_domains:\n d = str(d).lower()\n _, query = self.helpers.split_domain(d)\n self.processed.add(hash(query))\n self.scan.word_cloud.absorb_word(d)\n\n return tenant_data\n" }, { "path": "bbot/modules/baddns.py", "content": "from baddns.base import get_all_modules\nfrom baddns.lib.loader import load_signatures\nfrom .base import BaseModule\n\nimport asyncio\nimport logging\n\n\nclass baddns(BaseModule):\n watched_events = [\"DNS_NAME\", \"DNS_NAME_UNRESOLVED\"]\n produced_events = [\"FINDING\", \"VULNERABILITY\"]\n flags = [\"active\", \"safe\", \"web-basic\", \"baddns\", \"cloud-enum\", \"subdomain-hijack\"]\n meta = {\n \"description\": \"Check hosts for domain/subdomain takeovers\",\n \"created_date\": \"2024-01-18\",\n \"author\": \"@liquidsec\",\n }\n options = {\"custom_nameservers\": [], \"only_high_confidence\": False, \"enabled_submodules\": []}\n options_desc = {\n \"custom_nameservers\": \"Force BadDNS to use a list of custom nameservers\",\n \"only_high_confidence\": \"Do not emit low-confidence or generic detections\",\n \"enabled_submodules\": \"A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only\",\n }\n module_threads = 8\n deps_pip = [\"baddns~=1.12.294\"]\n\n def select_modules(self):\n selected_submodules = []\n for m in get_all_modules():\n if m.name in self.enabled_submodules:\n selected_submodules.append(m)\n return selected_submodules\n\n def set_modules(self):\n self.enabled_submodules = self.config.get(\"enabled_submodules\", [])\n if self.enabled_submodules == []:\n self.enabled_submodules = [\"CNAME\", \"MX\", \"TXT\"]\n\n async def setup(self):\n self.preset.core.logger.include_logger(logging.getLogger(\"baddns\"))\n self.custom_nameservers = self.config.get(\"custom_nameservers\", []) or None\n if self.custom_nameservers:\n self.custom_nameservers = self.helpers.chain_lists(self.custom_nameservers)\n self.only_high_confidence = self.config.get(\"only_high_confidence\", False)\n self.signatures = load_signatures()\n self.set_modules()\n all_submodules_list = [m.name for m in get_all_modules()]\n for m in self.enabled_submodules:\n if m not in all_submodules_list:\n self.hugewarning(\n f\"Selected BadDNS submodule [{m}] does not exist. Available submodules: [{','.join(all_submodules_list)}]\"\n )\n return False\n self.debug(f\"Enabled BadDNS Submodules: [{','.join(self.enabled_submodules)}]\")\n return True\n\n async def handle_event(self, event):\n tasks = []\n for ModuleClass in self.select_modules():\n kwargs = {\n \"http_client_class\": self.scan.helpers.web.AsyncClient,\n \"dns_client\": self.scan.helpers.dns.resolver,\n \"custom_nameservers\": self.custom_nameservers,\n \"signatures\": self.signatures,\n }\n\n if ModuleClass.name == \"NS\":\n kwargs[\"raw_query_max_retries\"] = 1\n kwargs[\"raw_query_timeout\"] = 5.0\n kwargs[\"raw_query_retry_wait\"] = 0\n\n module_instance = ModuleClass(event.data, **kwargs)\n task = asyncio.create_task(module_instance.dispatch())\n tasks.append((module_instance, task))\n\n async for completed_task in self.helpers.as_completed([task for _, task in tasks]):\n module_instance = next((m for m, t in tasks if t == completed_task), None)\n try:\n task_result = await completed_task\n except Exception as e:\n self.warning(f\"Task for {module_instance} raised an error: {e}\")\n task_result = None\n\n if task_result:\n results = module_instance.analyze()\n if results and len(results) > 0:\n for r in results:\n r_dict = r.to_dict()\n\n confidence = r_dict[\"confidence\"]\n\n if confidence in [\"CONFIRMED\", \"PROBABLE\"]:\n data = {\n \"severity\": \"MEDIUM\",\n \"description\": f\"{r_dict['description']}. Confidence: [{confidence}] Signature: [{r_dict['signature']}] Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]\",\n \"host\": str(event.host),\n }\n await self.emit_event(\n data,\n \"VULNERABILITY\",\n event,\n tags=[f\"baddns-{module_instance.name.lower()}\"],\n context=f'{{module}}\\'s \"{r_dict[\"module\"]}\" module found {{event.type}}: {r_dict[\"description\"]}',\n )\n\n elif confidence in [\"UNLIKELY\", \"POSSIBLE\"]:\n if not self.only_high_confidence:\n data = {\n \"description\": f\"{r_dict['description']} Confidence: [{confidence}] Signature: [{r_dict['signature']}] Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]\",\n \"host\": str(event.host),\n }\n await self.emit_event(\n data,\n \"FINDING\",\n event,\n tags=[f\"baddns-{module_instance.name.lower()}\"],\n context=f'{{module}}\\'s \"{r_dict[\"module\"]}\" module found {{event.type}}: {r_dict[\"description\"]}',\n )\n else:\n self.debug(\n f\"Skipping low-confidence result due to only_high_confidence setting: {confidence}\"\n )\n\n else:\n self.warning(f\"Got unrecognized confidence level: {confidence}\")\n\n found_domains = r_dict.get(\"found_domains\", None)\n if found_domains:\n for found_domain in found_domains:\n await self.emit_event(\n found_domain,\n \"DNS_NAME\",\n event,\n tags=[f\"baddns-{module_instance.name.lower()}\"],\n context=f'{{module}}\\'s \"{r_dict[\"module\"]}\" module found {{event.type}}: {{event.data}}',\n )\n await module_instance.cleanup()\n" }, { "path": "bbot/modules/baddns_direct.py", "content": "from baddns.base import get_all_modules\nfrom baddns.lib.loader import load_signatures\nfrom .base import BaseModule\n\nimport logging\n\n\nclass baddns_direct(BaseModule):\n watched_events = [\"URL\", \"STORAGE_BUCKET\"]\n produced_events = [\"FINDING\", \"VULNERABILITY\"]\n flags = [\"active\", \"safe\", \"subdomain-enum\", \"baddns\", \"cloud-enum\"]\n meta = {\n \"description\": \"Check for unusual subdomain / service takeover edge cases that require direct detection\",\n \"created_date\": \"2024-01-29\",\n \"author\": \"@liquidsec\",\n }\n options = {\"custom_nameservers\": []}\n options_desc = {\n \"custom_nameservers\": \"Force BadDNS to use a list of custom nameservers\",\n }\n module_threads = 8\n deps_pip = [\"baddns~=1.12.294\"]\n\n scope_distance_modifier = 1\n\n async def setup(self):\n self.preset.core.logger.include_logger(logging.getLogger(\"baddns\"))\n self.custom_nameservers = self.config.get(\"custom_nameservers\", []) or None\n if self.custom_nameservers:\n self.custom_nameservers = self.helpers.chain_lists(self.custom_nameservers)\n self.only_high_confidence = self.config.get(\"only_high_confidence\", False)\n self.signatures = load_signatures()\n return True\n\n def select_modules(self):\n selected_modules = []\n for m in get_all_modules():\n if m.name in [\"CNAME\"]:\n selected_modules.append(m)\n return selected_modules\n\n async def handle_event(self, event):\n CNAME_direct_module = self.select_modules()[0]\n kwargs = {\n \"http_client_class\": self.scan.helpers.web.AsyncClient,\n \"dns_client\": self.scan.helpers.dns.resolver,\n \"custom_nameservers\": self.custom_nameservers,\n \"signatures\": self.signatures,\n \"direct_mode\": True,\n }\n\n CNAME_direct_instance = CNAME_direct_module(str(event.host), **kwargs)\n if await CNAME_direct_instance.dispatch():\n results = CNAME_direct_instance.analyze()\n if results and len(results) > 0:\n for r in results:\n r_dict = r.to_dict()\n\n data = {\n \"description\": f\"Possible [{r_dict['signature']}] via direct BadDNS analysis. Indicator: [{r_dict['indicator']}] Trigger: [{r_dict['trigger']}] baddns Module: [{r_dict['module']}]\",\n \"host\": str(event.host),\n }\n\n await self.emit_event(\n data,\n \"FINDING\",\n event,\n tags=[f\"baddns-{CNAME_direct_module.name.lower()}\"],\n context=f'{{module}}\\'s \"{r_dict[\"module\"]}\" module found {{event.type}}: {r_dict[\"description\"]}',\n )\n await CNAME_direct_instance.cleanup()\n\n async def filter_event(self, event):\n if event.type == \"STORAGE_BUCKET\":\n if str(event.module).startswith(\"bucket_\"):\n return False\n self.debug(f\"Processing STORAGE_BUCKET for {event.host}\")\n if event.type == \"URL\":\n if event.scope_distance > 0:\n self.debug(\n f\"Rejecting {event.host} due to not being in scope (scope distance: {event.scope_distance})\"\n )\n return False\n if \"cdn-cloudflare\" not in event.tags:\n self.debug(f\"Rejecting {event.host} due to not being behind CloudFlare\")\n return False\n if \"status-200\" in event.tags or \"status-301\" in event.tags:\n self.debug(f\"Rejecting {event.host} due to lack of non-standard status code\")\n return False\n\n self.debug(f\"Passed all checks and is processing {event.host}\")\n return True\n" }, { "path": "bbot/modules/baddns_zone.py", "content": "from .baddns import baddns as baddns_module\n\n\nclass baddns_zone(baddns_module):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"FINDING\", \"VULNERABILITY\"]\n flags = [\"active\", \"safe\", \"subdomain-enum\", \"baddns\", \"cloud-enum\"]\n meta = {\n \"description\": \"Check hosts for DNS zone transfers and NSEC walks\",\n \"created_date\": \"2024-01-29\",\n \"author\": \"@liquidsec\",\n }\n options = {\"custom_nameservers\": [], \"only_high_confidence\": False}\n options_desc = {\n \"custom_nameservers\": \"Force BadDNS to use a list of custom nameservers\",\n \"only_high_confidence\": \"Do not emit low-confidence or generic detections\",\n }\n module_threads = 8\n deps_pip = [\"baddns~=1.12.294\"]\n\n def set_modules(self):\n self.enabled_submodules = [\"NSEC\", \"zonetransfer\"]\n\n # minimize nsec records feeding back into themselves\n async def filter_event(self, event):\n if \"baddns-nsec\" in event.tags or \"baddns-nsec\" in event.parent.tags:\n return False\n return True\n" }, { "path": "bbot/modules/badsecrets.py", "content": "import multiprocessing\nfrom pathlib import Path\nfrom .base import BaseModule\nfrom badsecrets.base import carve_all_modules\n\n\nclass badsecrets(BaseModule):\n watched_events = [\"HTTP_RESPONSE\"]\n produced_events = [\"FINDING\", \"VULNERABILITY\", \"TECHNOLOGY\"]\n flags = [\"active\", \"safe\", \"web-basic\"]\n meta = {\n \"description\": \"Library for detecting known or weak secrets across many web frameworks\",\n \"created_date\": \"2022-11-19\",\n \"author\": \"@liquidsec\",\n }\n options = {\"custom_secrets\": None}\n options_desc = {\n \"custom_secrets\": \"Include custom secrets loaded from a local file\",\n }\n deps_pip = [\"badsecrets~=0.13.47\"]\n\n async def setup(self):\n self.custom_secrets = None\n custom_secrets = self.config.get(\"custom_secrets\", None)\n if custom_secrets:\n secrets_path = Path(custom_secrets).expanduser()\n if secrets_path.is_file():\n self.custom_secrets = custom_secrets\n self.info(f\"Successfully loaded secrets file [{custom_secrets}]\")\n else:\n self.warning(f\"custom secrets file [{custom_secrets}] is not valid\")\n return False, \"Custom secrets file not valid\"\n return True\n\n @property\n def _module_threads(self):\n return max(1, multiprocessing.cpu_count() - 1)\n\n async def handle_event(self, event):\n resp_body = event.data.get(\"body\", None)\n resp_headers = event.data.get(\"header\", None)\n resp_cookies = {}\n if resp_headers:\n resp_cookies_raw = resp_headers.get(\"set_cookie\", None)\n if resp_cookies_raw:\n if \",\" in resp_cookies_raw:\n resp_cookies_list = resp_cookies_raw.split(\",\")\n else:\n resp_cookies_list = [resp_cookies_raw]\n for c in resp_cookies_list:\n c2 = c.lstrip(\";\").strip().split(\";\")[0].split(\"=\")\n if len(c2) == 2:\n resp_cookies[c2[0]] = c2[1]\n if resp_body or resp_cookies:\n try:\n r_list = await self.helpers.run_in_executor_mp(\n carve_all_modules,\n body=resp_body,\n headers=resp_headers,\n cookies=resp_cookies,\n url=event.data.get(\"url\", None),\n custom_resource=self.custom_secrets,\n )\n except Exception as e:\n self.warning(f\"Error processing {event}: {e}\")\n return\n if r_list:\n for r in r_list:\n if r[\"type\"] == \"SecretFound\":\n data = {\n \"severity\": r[\"description\"][\"severity\"],\n \"description\": f\"Known Secret Found. Secret Type: [{r['description']['secret']}] Secret: [{r['secret']}] Product Type: [{r['description']['product']}] Product: [{self.helpers.truncate_string(r['product'], 2000)}] Detecting Module: [{r['detecting_module']}] Details: [{r['details']}]\",\n \"url\": event.data[\"url\"],\n \"host\": str(event.host),\n }\n await self.emit_event(\n data,\n \"VULNERABILITY\",\n event,\n context=f'{{module}}\\'s \"{r[\"detecting_module\"]}\" module found known {r[\"description\"][\"product\"]} secret ({{event.type}}): \"{r[\"secret\"]}\"',\n )\n elif r[\"type\"] == \"IdentifyOnly\":\n # There is little value to presenting a non-vulnerable asp.net viewstate, as it is not crackable without a Matrioshka brain. Just emit a technology instead.\n if r[\"detecting_module\"] == \"ASPNET_Viewstate\":\n technology = \"microsoft asp.net\"\n await self.emit_event(\n {\"technology\": technology, \"url\": event.data[\"url\"], \"host\": str(event.host)},\n \"TECHNOLOGY\",\n event,\n context=f\"{{module}} identified {{event.type}}: {technology}\",\n )\n else:\n data = {\n \"description\": f\"Cryptographic Product identified. Product Type: [{r['description']['product']}] Product: [{self.helpers.truncate_string(r['product'], 2000)}] Detecting Module: [{r['detecting_module']}]\",\n \"url\": event.data[\"url\"],\n \"host\": str(event.host),\n }\n await self.emit_event(\n data,\n \"FINDING\",\n event,\n context=f'{{module}} identified cryptographic product ({{event.type}}): \"{r[\"description\"][\"product\"]}\"',\n )\n" }, { "path": "bbot/modules/base.py", "content": "import asyncio\nimport logging\nimport traceback\nfrom sys import exc_info\nfrom contextlib import suppress\n\nfrom ..core.helpers.misc import get_size # noqa\nfrom ..errors import ValidationError, WebError\nfrom ..core.helpers.async_helpers import TaskCounter, ShuffleQueue\nfrom ..core.event import is_event\n\n\nclass BaseModule:\n \"\"\"The base class for all BBOT modules.\n\n Attributes:\n watched_events (List): Event types to watch.\n\n produced_events (List): Event types to produce.\n\n meta (Dict): Metadata about the module, such as whether authentication is required and a description.\n\n flags (List): Flags indicating the type of module (must have at least \"safe\" or \"aggressive\" and \"passive\" or \"active\").\n\n deps_modules (List): Other BBOT modules this module depends on. Empty list by default.\n\n deps_pip (List): Python dependencies to install via pip. Empty list by default.\n\n deps_apt (List): APT package dependencies to install. Empty list by default.\n\n deps_shell (List): Other dependencies installed via shell commands. Uses [ansible.builtin.shell](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/shell_module.html). Empty list by default.\n\n deps_ansible (List): Additional Ansible tasks for complex dependencies. Empty list by default.\n\n accept_dupes (bool): Whether to accept incoming duplicate events. Default is False.\n\n suppress_dupes (bool): Whether to suppress outgoing duplicate events. Default is True.\n\n per_host_only (bool): Limit the module to only scanning once per host. Default is False.\n\n per_hostport_only (bool): Limit the module to only scanning once per host:port. Default is False.\n\n per_domain_only (bool): Limit the module to only scanning once per domain. Default is False.\n\n scope_distance_modifier (int, None): Modifies scope distance acceptance for events. Default is 0.\n ```\n None == accept all events\n 2 == accept events up to and including the scan's configured search distance plus two\n 1 == accept events up to and including the scan's configured search distance plus one\n 0 == (DEFAULT) accept events up to and including the scan's configured search distance\n ```\n\n target_only (bool): Accept only the initial target event(s). Default is False.\n\n in_scope_only (bool): Accept only explicitly in-scope events, regardless of the scan's search distance. Default is False.\n\n accept_url_special (bool): Accept \"special\" URLs not typically distributed to web modules, e.g. JS URLs. Default is False.\n\n options (Dict): Customizable options for the module, e.g., {\"api_key\": \"\"}. Empty dict by default.\n\n options_desc (Dict): Descriptions for options, e.g., {\"api_key\": \"API Key\"}. Empty dict by default.\n\n module_threads (int): Maximum concurrent instances of handle_event() or handle_batch(). Default is 1.\n\n batch_size (int): Size of batches processed by handle_batch(). Default is 1.\n\n api_failure_abort_threshold (int): Threshold for setting error state after failed HTTP requests (only takes effect when `api_request()` is used. Default is 5.\n\n _preserve_graph (bool): When set to True, accept events that may be duplicates but are necessary for construction of complete graph. Typically only enabled for output modules that need to maintain full chains of events, e.g. `neo4j` and `json`. Default is False.\n\n _stats_exclude (bool): Whether to exclude this module from scan statistics. Default is False.\n\n _disable_auto_module_deps (bool): Whether to disable automatic module dependencies. This is useful e.g. if the module consumes URLs, but you don't want to automatically enable the httpx module. Default is False.\n\n _qsize (int): Outgoing queue size (0 for infinite). Default is 0.\n\n _priority (int): Priority level of the module. Lower values are higher priority. Default is 3.\n\n _name (str): Module name, overridden automatically. Default is 'base'.\n\n _type (str): Module type, for differentiating between normal and output modules. Default is 'scan'.\n \"\"\"\n\n watched_events = []\n produced_events = []\n meta = {\"auth_required\": False, \"description\": \"Base module\"}\n flags = []\n options = {}\n options_desc = {}\n\n deps_modules = []\n deps_pip = []\n deps_apt = []\n deps_shell = []\n deps_ansible = []\n\n accept_dupes = False\n suppress_dupes = True\n per_host_only = False\n per_hostport_only = False\n per_domain_only = False\n scope_distance_modifier = 0\n target_only = False\n in_scope_only = False\n accept_url_special = False\n _module_threads = 1\n _batch_size = 1\n\n # disable the module after this many failed attempts in a row\n _api_failure_abort_threshold = 3\n # whether to retry on 429s when first pinging the API at scan start\n _ping_retry_on_http_429 = False\n\n default_discovery_context = \"{module} discovered {event.type}: {event.data}\"\n\n _preserve_graph = False\n _stats_exclude = False\n _disable_auto_module_deps = False\n _qsize = 1000\n _priority = 3\n _name = \"base\"\n _type = \"scan\"\n _intercept = False\n _shuffle_incoming_queue = True\n\n def __init__(self, scan):\n \"\"\"Initializes a module instance.\n\n Args:\n scan: The BBOT scan object associated with this module instance.\n\n Attributes:\n scan: The scan object associated with this module.\n\n errored (bool): Whether the module has errored out. Default is False.\n \"\"\"\n self.scan = scan\n self.errored = False\n self._log = None\n self._incoming_event_queue = None\n self._outgoing_event_queue = None\n # track incoming events to prevent unwanted duplicates\n self._incoming_dup_tracker = set()\n # tracks which subprocesses are running under this module\n self._proc_tracker = set()\n # seconds since we've submitted a batch\n self._last_submitted_batch = None\n # additional callbacks to be executed alongside self.cleanup()\n self.cleanup_callbacks = []\n self._cleanedup = False\n self._watched_events = None\n\n self._task_counter = TaskCounter()\n\n # string constant\n self._custom_filter_criteria_msg = \"it did not meet custom filter criteria\"\n\n self._api_keys = []\n\n # track number of failures (for .api_request())\n self._api_request_failures = 0\n\n self._default_api_retries = self.scan.config.get(\"web\", {}).get(\"api_retries\", 2)\n\n self._tasks = []\n self._event_received = None\n # maximum runtime for each module's handle_event()\n self._default_handle_event_timeout = self.scan.config.get(\"module_handle_event_timeout\", 60 * 60) # 1 hour\n self._default_handle_batch_timeout = self.scan.config.get(\n \"module_handle_batch_timeout\", 60 * 60 * 2\n ) # 2 hours\n self._event_handler_watchdog_interval = self.event_handler_timeout / 10\n\n # used for optional \"per host\" tracking\n self._per_host_tracker = set()\n\n # 429 rate limit handling\n self._429_sleep_interval = self.scan.web_config.get(\"429_sleep_interval\", 30)\n self._429_max_sleep_interval = self.scan.web_config.get(\"429_max_sleep_interval\", 60)\n\n async def setup(self):\n \"\"\"\n Performs one-time setup tasks for the module.\n\n This method is responsible for preparing the module for its operation, which may include tasks\n such as downloading necessary resources, validating configuration parameters, or other preliminary\n checks.\n\n Returns:\n tuple:\n - bool or None: A status indicating the outcome of the setup process. Returns `True` if\n the setup was successful, `None` for a soft-fail where the module setup did not succeed\n but the scan will continue with the module disabled, and `False` for a hard-fail where\n the setup failure causes the scan to abort.\n - str, optional: A reason for the setup failure, provided only when the setup does not\n succeed (i.e., returns `None` or `False`).\n\n Examples:\n >>> async def setup(self):\n >>> if not self.config.get(\"api_key\"):\n >>> # Soft-fail: Configuration missing an API key\n >>> return None, \"No API key specified\"\n\n >>> async def setup(self):\n >>> try:\n >>> wordlist = await self.helpers.wordlist(\"https://raw.githubusercontent.com/user/wordlist.txt\")\n >>> except WordlistError as e:\n >>> # Hard-fail: Error retrieving wordlist\n >>> return False, f\"Error retrieving wordlist: {e}\"\n\n >>> async def setup(self):\n >>> self.timeout = self.config.get(\"timeout\", 5)\n >>> # Success: Setup completed without issues\n >>> return True\n \"\"\"\n\n return True\n\n async def setup_deps(self):\n \"\"\"\n Similar to setup(), but reserved for installing dependencies not covered by Ansible.\n\n This should always be used to install static dependencies like AI models, wordlists, etc.\n \"\"\"\n return True\n\n async def handle_event(self, event, **kwargs):\n \"\"\"Asynchronously handles incoming events that the module is configured to watch.\n\n This method is automatically invoked when an event that matches any in `watched_events` is encountered during a scan. Override this method to implement custom event-handling logic for your module.\n\n Args:\n event (Event): The event object containing details about the incoming event.\n\n Note:\n This method should be overridden if the `batch_size` attribute of the module is set to 1.\n\n Returns:\n None\n \"\"\"\n pass\n\n async def handle_batch(self, *events):\n \"\"\"Handles incoming events in batches for optimized processing.\n\n This method is automatically called when multiple events that match any in `watched_events` are encountered and the `batch_size` attribute is set to a value greater than 1. Override this method to implement custom batch event-handling logic for your module.\n\n Args:\n *events (Event): A variable number of Event objects to be processed in a batch.\n\n Note:\n This method should be overridden if the `batch_size` attribute of the module is set to a value greater than 1.\n\n Returns:\n None\n \"\"\"\n pass\n\n async def filter_event(self, event):\n \"\"\"Asynchronously filters incoming events based on custom criteria.\n\n Override this method for more granular control over which events are accepted by your module. This method is called automatically before `handle_event()` for each incoming event that matches any in `watched_events`.\n\n Args:\n event (Event): The incoming Event object to be filtered.\n\n Returns:\n tuple: A 2-tuple where the first value is a bool indicating whether the event should be accepted, and the second value is a string explaining the reason for its acceptance or rejection. By default, returns `(True, None)` to indicate acceptance without reason.\n\n Note:\n This method should be overridden if the module requires custom logic for event filtering.\n \"\"\"\n return True\n\n async def finish(self):\n \"\"\"Asynchronously performs final tasks as the scan nears completion.\n\n This method can be overridden to execute any necessary finalization logic. For example, if the module relies on a word cloud, you might wait for the scan to finish to ensure the word cloud is most complete before running an operation.\n\n Returns:\n None\n\n Warnings:\n This method may be called multiple times since it can raise events, which may re-trigger the \"finish\" phase of the scan. Optional to override.\n \"\"\"\n return\n\n async def report(self):\n \"\"\"Asynchronously executes a final task after the scan is complete but before cleanup.\n\n This method can be overridden to aggregate data and raise summary events at the end of the scan.\n\n Returns:\n None\n\n Note:\n This method is called only once per scan.\n \"\"\"\n return\n\n async def cleanup(self):\n \"\"\"Asynchronously performs final cleanup operations after the scan is complete.\n\n This method can be overridden to implement custom cleanup logic. It is called only once per scan and may not raise events.\n\n Returns:\n None\n\n Note:\n This method is called only once per scan and may not raise events.\n \"\"\"\n return\n\n async def require_api_key(self):\n \"\"\"\n Asynchronously checks if an API key is required and valid.\n\n Args:\n None\n\n Returns:\n bool or tuple: Returns True if API key is valid and ready.\n Returns a tuple (None, \"error message\") otherwise.\n\n Notes:\n - Fetches the API key from the configuration.\n - Calls the 'ping()' method to test API accessibility.\n - Sets the API key readiness status accordingly.\n \"\"\"\n self.api_key = self.config.get(\"api_key\", \"\")\n if self.auth_secret:\n try:\n await self.ping()\n self.hugesuccess(\"API is ready\")\n return True, \"\"\n except Exception as e:\n self.trace(traceback.format_exc())\n return None, f\"Error with API ({str(e).strip()})\"\n else:\n return None, \"No API key set\"\n\n @property\n def api_key(self):\n if self._api_keys:\n return self._api_keys[0]\n\n @api_key.setter\n def api_key(self, api_keys):\n if isinstance(api_keys, str):\n api_keys = [api_keys]\n self._api_keys = list(api_keys)\n\n def cycle_api_key(self):\n if len(self._api_keys) > 1:\n self.verbose(\"Cycling API key\")\n self._api_keys.insert(0, self._api_keys.pop())\n else:\n self.debug(\"No extra API keys to cycle\")\n\n @property\n def api_retries(self):\n return max(self._default_api_retries + 1, len(self._api_keys))\n\n @property\n def api_failure_abort_threshold(self):\n return (self.api_retries * self._api_failure_abort_threshold) + 1\n\n async def ping(self, url=None):\n \"\"\"Asynchronously checks the health of the configured API.\n\n This method is used in conjunction with require_api_key() to verify that the API is not just configured, but also responsive. It makes a test request to a known endpoint to validate the API's health.\n\n The method uses the `ping_url` attribute if defined, or falls back to a provided URL. If neither is available, no request is made.\n\n Args:\n url (str, optional): A specific URL to use for the ping request. If not provided, the method will use the `ping_url` attribute.\n\n Returns:\n None\n\n Raises:\n ValueError: If the API response is not successful (status code != 200).\n\n Example Usage:\n To use this method, simply define the `ping_url` attribute in your module:\n\n class MyModule(BaseModule):\n ping_url = \"https://api.example.com/ping\"\n\n Alternatively, you can override this method for more complex health checks:\n\n async def ping(self):\n r = await self.api_request(f\"{self.base_url}/complex-health-check\")\n if r.status_code != 200 or r.json().get('status') != 'healthy':\n raise ValueError(f\"API unhealthy: {r.text}\")\n \"\"\"\n if url is None:\n url = getattr(self, \"ping_url\", \"\")\n retry_on_http_429 = getattr(self, \"_ping_retry_on_http_429\", False)\n if url:\n r = await self.api_request(url, retry_on_http_429=retry_on_http_429)\n if getattr(r, \"status_code\", 0) != 200:\n response_text = getattr(r, \"text\", \"no response from server\")\n raise ValueError(response_text)\n\n @property\n def batch_size(self):\n batch_size = self.config.get(\"batch_size\", None)\n # only allow overriding the batch size if its default value is greater than 1\n # this prevents modules from being accidentally neutered by an incorrect batch_size setting\n if batch_size is None or self._batch_size == 1:\n batch_size = self._batch_size\n return batch_size\n\n @property\n def module_threads(self):\n module_threads = self.config.get(\"module_threads\", None)\n if module_threads is None:\n module_threads = self._module_threads\n return module_threads\n\n @property\n def event_handler_timeout(self):\n module_timeout = self.config.get(\"module_timeout\", None)\n if module_timeout is not None:\n return float(module_timeout)\n return self._default_handle_event_timeout if self.batch_size <= 1 else self._default_handle_batch_timeout\n\n @property\n def auth_secret(self):\n \"\"\"Indicates if the module is properly configured for authentication.\n\n This read-only property should be used to check whether all necessary attributes (e.g., API keys, tokens, etc.) are configured to perform authenticated requests in the module. Commonly used in setup or initialization steps.\n\n Returns:\n bool: True if the module is properly configured for authentication, otherwise False.\n \"\"\"\n return getattr(self, \"api_key\", \"\")\n\n @property\n def event_received(self):\n if self._event_received is None:\n self._event_received = asyncio.Condition()\n return self._event_received\n\n def get_watched_events(self):\n \"\"\"Retrieve the set of events that the module is interested in observing.\n\n Override this method if the set of events the module should watch needs to be determined dynamically, e.g., based on configuration options or other runtime conditions.\n\n Returns:\n set: The set of event types that this module will handle.\n \"\"\"\n if self._watched_events is None:\n self._watched_events = set(self.watched_events)\n return self._watched_events\n\n async def _handle_batch(self):\n \"\"\"\n Asynchronously handles a batch of events in the module.\n\n Args:\n None\n\n Returns:\n bool: True if events were submitted for processing, False otherwise.\n\n Notes:\n - The method is wrapped in a task counter to monitor asynchronous operations.\n - Checks if there are any events in the incoming queue and module is not in an error state.\n - Invokes '_events_waiting()' to fetch a batch of events.\n - Calls the module's 'handle_batch()' method to process these events.\n - If a \"FINISHED\" event is found, invokes 'finish()' method of the module.\n \"\"\"\n finish = False\n submitted = False\n if self.batch_size <= 1:\n return\n if self.num_incoming_events > 0:\n events, finish = await self._events_waiting()\n if events and not self.errored:\n self.verbose(f\"Handling batch of {len(events):,} events\")\n event_types = {}\n for e in events:\n event_types[e.type] = event_types.get(e.type, 0) + 1\n event_types_sorted = sorted(event_types.items(), key=lambda x: x[1], reverse=True)\n event_types_str = \", \".join(f\"{k}: {v}\" for k, v in event_types_sorted)\n submitted = True\n context = f\"{self.name}.handle_batch({event_types_str})\"\n try:\n await self.run_task(self.handle_batch(*events), context, n=len(events))\n except asyncio.CancelledError:\n self.debug(f\"{context} was cancelled\")\n self.verbose(f\"Finished handling batch of {len(events):,} events\")\n if finish:\n context = f\"{self.name}.finish()\"\n await self.run_task(self.finish(), context)\n return submitted\n\n def make_event(self, *args, **kwargs):\n \"\"\"Create an event for the scan.\n\n Raises a validation error if the event could not be created, unless raise_error is set to False.\n\n Args:\n *args: Positional arguments to be passed to the scan's make_event method.\n **kwargs: Keyword arguments to be passed to the scan's make_event method.\n raise_error (bool, optional): Whether to raise a validation error if the event could not be created. Defaults to False.\n\n Examples:\n >>> new_event = self.make_event(\"1.2.3.4\", parent=event)\n >>> await self.emit_event(new_event)\n\n Returns:\n Event or None: The created event, or None if a validation error occurred and raise_error was False.\n\n Raises:\n ValidationError: If the event could not be validated and raise_error is True.\n \"\"\"\n raise_error = kwargs.pop(\"raise_error\", False)\n module = kwargs.pop(\"module\", None)\n if module is None:\n if (not args) or getattr(args[0], \"module\", None) is None:\n kwargs[\"module\"] = self\n try:\n if args and is_event(args[0]):\n raise ValidationError(\n f\"{self.__class__.__name__}.make_event() does not accept an existing event \"\n f\"({type(args[0]).__name__}) as the first argument. \"\n \"Use update_event(event, ...) or emit_event(event, ...) instead.\"\n )\n event = self.scan.make_event(*args, **kwargs)\n except ValidationError as e:\n if raise_error:\n raise\n self.warning(f\"{e}\")\n return\n return event\n\n def update_event(self, event, **kwargs):\n \"\"\"Update an existing event for the scan.\n\n This is the counterpart to :meth:`make_event` for modifying an existing\n :class:`bbot.core.event.base.BaseEvent` instance.\n\n Raises a validation error if the update could not be applied, unless\n ``raise_error`` is set to False.\n\n Args:\n event: The event object to update.\n **kwargs: Keyword arguments to be passed to the scan's update_event method.\n raise_error (bool, optional): Whether to raise a validation error if the event could not be updated. Defaults to False.\n\n Returns:\n Event or None: The updated event, or None if a validation error occurred and raise_error was False.\n\n Raises:\n ValidationError: If the event could not be validated and raise_error is True.\n \"\"\"\n raise_error = kwargs.pop(\"raise_error\", False)\n module = kwargs.pop(\"module\", None)\n if module is None and getattr(event, \"module\", None) is None:\n kwargs[\"module\"] = self\n try:\n updated = self.scan.update_event(event, **kwargs)\n except ValidationError as e:\n if raise_error:\n raise\n self.warning(f\"{e}\")\n return\n return updated\n\n async def emit_event(self, *args, **kwargs):\n \"\"\"Emit an event to the event queue and distribute it to interested modules.\n\n This is how modules \"return\" data.\n\n The method first creates an event object by calling `self.make_event()` with the provided arguments.\n Then, the event is queued for outgoing distribution using `self.queue_outgoing_event()`.\n\n Args:\n *args: Positional arguments to be passed to `self.make_event()` for event creation.\n **kwargs: Keyword arguments to be passed for event creation or configuration of the emit action.\n ```markdown\n - on_success_callback: Optional callback function to execute upon successful event emission.\n - abort_if: Optional condition under which the event emission should be aborted.\n - quick: Optional flag to indicate whether the event should be processed quickly.\n ```\n\n Examples:\n >>> await self.emit_event(\"www.evilcorp.com\", parent=event, tags=[\"affiliate\"])\n\n >>> new_event = self.make_event(\"1.2.3.4\", parent=event)\n >>> await self.emit_event(new_event)\n\n Returns:\n None\n\n Raises:\n ValidationError: If the event cannot be validated (handled in `self.make_event()`).\n \"\"\"\n event_kwargs = dict(kwargs)\n emit_kwargs = {}\n for o in (\"on_success_callback\", \"abort_if\", \"quick\"):\n v = event_kwargs.pop(o, None)\n if v is not None:\n emit_kwargs[o] = v\n\n # Two entry points:\n # - emit_event(data, ...) -> create a new event via make_event()\n # - emit_event(existing_event, ...) -> update and re‑emit that event\n if args and is_event(args[0]):\n event, *rest = args\n if rest:\n self.warning(\n f\"emit_event() was called on {self.name} with an existing event and extra \"\n f\"positional args ({rest}); extra args are ignored. \"\n \"Pass only the event plus keyword arguments, or call make_event() explicitly.\"\n )\n # Update the existing event (e.g. tags/context/module) before emitting\n event = self.update_event(event, **event_kwargs)\n else:\n event = self.make_event(*args, **event_kwargs)\n\n if event is not None:\n children = event.children\n for e in [event] + children:\n await self.queue_outgoing_event(e, **emit_kwargs)\n return event\n\n async def _events_waiting(self, batch_size=None):\n \"\"\"\n Asynchronously fetches events from the incoming_event_queue, up to a specified batch size.\n\n Args:\n None\n\n Returns:\n tuple: A tuple containing two elements:\n - events (list): A list of acceptable events from the queue.\n - finish (bool): A flag indicating if a \"FINISHED\" event is encountered.\n\n Notes:\n - The method pulls events from incoming_event_queue using 'get_nowait()'.\n - Events go through '_event_postcheck()' for validation.\n - \"FINISHED\" events are handled differently and the finish flag is set to True.\n - If the queue is empty or the batch size is reached, the loop breaks.\n \"\"\"\n if batch_size is None:\n batch_size = self.batch_size\n events = []\n finish = False\n while self.incoming_event_queue:\n if batch_size != -1 and len(events) > self.batch_size:\n break\n try:\n event = self.incoming_event_queue.get_nowait()\n self.debug(f\"Got {event} from {getattr(event, 'module', 'unknown_module')}\")\n acceptable, reason = await self._event_postcheck(event)\n if acceptable:\n if event.type == \"FINISHED\":\n finish = True\n else:\n events.append(event)\n self.scan.stats.event_consumed(event, self)\n elif reason:\n self.debug(f\"Not accepting {event} because {reason}\")\n except asyncio.queues.QueueEmpty:\n break\n return events, finish\n\n @property\n def num_incoming_events(self):\n ret = 0\n if self.incoming_event_queue is not False:\n ret = self.incoming_event_queue.qsize()\n return ret\n\n def start(self):\n self._tasks = [\n asyncio.create_task(self._worker(), name=f\"{self.scan.name}.{self.name}._worker()\")\n for _ in range(self.module_threads)\n ]\n watchdog_task = asyncio.create_task(\n self._event_handler_watchdog(),\n name=f\"{self.scan.name}.{self.name}._event_handler_watchdog()\",\n )\n self._tasks.append(watchdog_task)\n\n async def _setup(self, deps_only=False):\n \"\"\" \"\"\"\n status_codes = {False: \"hard-fail\", None: \"soft-fail\", True: \"success\"}\n\n status = False\n self.debug(f\"Setting up module {self.name}\")\n try:\n funcs = [self.setup_deps]\n if not deps_only:\n funcs.append(self.setup)\n for func in funcs:\n self.debug(f\"Running {self.name}.{func.__name__}()\")\n result = await func()\n if type(result) == tuple and len(result) == 2:\n status, msg = result\n else:\n status = result\n msg = status_codes[status]\n if status is False:\n break\n self.debug(f\"Finished setting up module {self.name}\")\n except Exception as e:\n self.set_error_state(f\"Unexpected error during module setup: {e}\", critical=True)\n msg = f\"{e}\"\n self.trace()\n return self, status, str(msg)\n\n async def _worker(self):\n \"\"\"\n The core worker loop for the module, responsible for handling events from the incoming event queue.\n\n This method is a coroutine and is run asynchronously. Multiple instances can run simultaneously based on\n the 'module_threads' configuration. The worker dequeues events from 'incoming_event_queue', performs\n necessary prechecks, and passes the event to the appropriate handler function.\n\n Args:\n None\n\n Returns:\n None\n\n Raises:\n asyncio.CancelledError: If the worker is cancelled during its operation.\n\n Notes:\n - The worker is sensitive to the 'stopping' flag of the scan. It will terminate if this flag is set.\n - The worker handles backpressure by pausing when the outgoing event queue is full.\n - Batch processing is supported and is activated when 'batch_size' > 1.\n - Each event is subject to a post-check via '_event_postcheck()' to decide whether it should be handled.\n - Special 'FINISHED' events trigger the 'finish()' method of the module.\n \"\"\"\n async with self.scan._acatch(context=self._worker, unhandled_is_critical=True):\n try:\n while not self.scan.stopping and not self.errored:\n # if batch wasn't big enough, we wait for the next event before continuing\n if self.batch_size > 1:\n submitted = await self._handle_batch()\n if not submitted:\n async with self.event_received:\n await self.event_received.wait()\n\n else:\n try:\n if self.incoming_event_queue is not False:\n event = await self.incoming_event_queue.get()\n else:\n self.debug(\"Event queue is in bad state\")\n break\n except asyncio.queues.QueueEmpty:\n continue\n self.debug(f\"Got {event} from {getattr(event, 'module', 'unknown_module')}\")\n async with self._task_counter.count(f\"event_postcheck({event})\"):\n acceptable, reason = await self._event_postcheck(event)\n if acceptable:\n if event.type == \"FINISHED\":\n context = f\"{self.name}.finish()\"\n try:\n await self.run_task(self.finish(), context)\n except asyncio.CancelledError:\n self.debug(f\"{context} was cancelled\")\n continue\n else:\n context = f\"{self.name}.handle_event({event})\"\n self.scan.stats.event_consumed(event, self)\n self.debug(f\"Handling {event}\")\n try:\n await self.run_task(self.handle_event(event), context)\n except asyncio.CancelledError:\n self.debug(f\"{context} was cancelled\")\n continue\n self.debug(f\"Finished handling {event}\")\n else:\n self.debug(f\"Not accepting {event} because {reason}\")\n except asyncio.CancelledError:\n # this trace was used for debugging leaked CancelledErrors from inside httpx\n # self.log.trace(\"Worker cancelled\")\n raise\n except BaseException as e:\n if self.helpers.in_exception_chain(e, (KeyboardInterrupt,)):\n self.scan.stop()\n else:\n self.error(f\"Critical failure in module {self.name}: {e}\")\n self.error(traceback.format_exc())\n self.log.trace(\"Worker stopped\")\n\n @property\n def max_scope_distance(self):\n \"\"\"\n Maximum scope distance for events that are accepted by the module.\n \"\"\"\n if self.in_scope_only or self.target_only:\n return 0\n if self.scope_distance_modifier is None:\n return 999\n return max(0, self.scan.scope_search_distance + self.scope_distance_modifier)\n\n def _event_precheck(self, event):\n \"\"\"\n Pre-checks an event to determine if it should be accepted by the module for queuing.\n\n This method is called when an event is about to be enqueued into the module's incoming event queue.\n It applies various filters such as special signal event types, module error state, watched event types, and more\n to decide whether or not the event should be enqueued.\n\n Args:\n event (Event): The event object to check.\n\n Returns:\n tuple: A tuple (bool, str) where the bool indicates if the event should be accepted, and the str gives the reason.\n\n Examples:\n >>> result, reason = self._event_precheck(event)\n >>> if result:\n ... self.incoming_event_queue.put_nowait(event)\n ... else:\n ... self.debug(f\"Not accepting {event} because {reason}\")\n\n Notes:\n - The method considers special signal event types like \"FINISHED\".\n - Checks whether the module is in an error state.\n - Checks if the event type matches the types this module is interested in (`watched_events`).\n - Checks for events tagged as 'target' if the module has `target_only` flag set.\n - Applies specific filtering based on event type and module name.\n \"\"\"\n\n # special signal event types\n if event.type in (\"FINISHED\",):\n return True, \"its type is FINISHED\"\n if self.errored:\n return False, \"module is in error state\"\n # exclude non-watched types\n if not any(t in self.get_watched_events() for t in (\"*\", event.type)):\n return False, \"its type is not in watched_events\"\n if self.target_only:\n if \"target\" not in event.tags:\n return False, \"it did not meet target_only filter criteria\"\n\n # limit js URLs to modules that opt in to receive them\n if (not self.accept_url_special) and event.type.startswith(\"URL\"):\n extension = getattr(event, \"url_extension\", \"\")\n if extension in self.scan.url_extension_special:\n return (\n False,\n f\"it is a special URL (extension {extension}) but the module does not opt in to receive special URLs\",\n )\n\n return True, \"precheck succeeded\"\n\n async def _event_postcheck(self, event):\n \"\"\"\n A simple wrapper for dup tracking\n \"\"\"\n # special exception for \"FINISHED\" event\n if event.type in (\"FINISHED\",):\n return True, \"\"\n acceptable, reason = await self._event_postcheck_inner(event)\n if acceptable:\n # check duplicates\n is_incoming_duplicate, reason = self.is_incoming_duplicate(event, add=True)\n if is_incoming_duplicate and not self.accept_dupes:\n return False, \"module has already seen it\" + (f\" ({reason})\" if reason else \"\")\n\n return acceptable, reason\n\n async def _event_postcheck_inner(self, event):\n \"\"\"\n Post-checks an event to determine if it should be accepted by the module for handling.\n\n This method is called when an event is dequeued from the module's incoming event queue, right before it is actually processed.\n It applies various filters such as scope, custom filtering logic, and per-host tracking to decide the event's fate.\n\n Args:\n event (Event): The event object to check.\n\n Returns:\n tuple: A tuple (bool, str) where the bool indicates if the event should be accepted, and the str gives the reason.\n\n Notes:\n - Override the `filter_event` method for custom filtering logic.\n - This method also maintains host-based tracking when the `per_host_only` or similar flags are set.\n - The method will also update event production stats for output modules.\n \"\"\"\n # force-output certain events to the graph\n if self._is_graph_important(event):\n return True, \"event is critical to the graph\"\n\n # check scope distance\n filter_result, reason = self._scope_distance_check(event)\n if not filter_result:\n return filter_result, reason\n\n # custom filtering\n async with self.scan._acatch(context=self.filter_event):\n try:\n filter_result = await self.filter_event(event)\n except Exception as e:\n msg = f\"Unhandled exception in {self.name}.filter_event({event}): {e}\"\n self.error(msg)\n return False, msg\n msg = str(self._custom_filter_criteria_msg)\n with suppress(ValueError, TypeError):\n filter_result, reason = filter_result\n msg += f\": {reason}\"\n if not filter_result:\n return False, msg\n\n self.debug(f\"{event} passed post-check\")\n return True, \"\"\n\n def _scope_distance_check(self, event):\n if self.in_scope_only:\n if event.scope_distance > 0:\n return False, \"it did not meet in_scope_only filter criteria\"\n if self.scope_distance_modifier is not None:\n if event.scope_distance < 0:\n return False, f\"its scope_distance ({event.scope_distance}) is invalid.\"\n elif event.scope_distance > self.max_scope_distance:\n return (\n False,\n f\"its scope_distance ({event.scope_distance}) exceeds the maximum allowed by the scan ({self.scan.scope_search_distance}) + the module ({self.scope_distance_modifier}) == {self.max_scope_distance}\",\n )\n return True, \"\"\n\n async def _cleanup(self):\n if not self._cleanedup:\n self._cleanedup = True\n for callback in [self.cleanup] + self.cleanup_callbacks:\n context = f\"{self.name}.cleanup()\"\n if callable(callback):\n async with self.scan._acatch(context), self._task_counter.count(context):\n await self.helpers.execute_sync_or_async(callback)\n\n async def run_task(self, coro, name, n=1):\n \"\"\"\n Start a task while tracking it in the module's task counter.\n\n This lets us keep a detailed module status and selectively cancel tasks when needed, like when handle_event exceeds its max runtime.\n \"\"\"\n task = asyncio.create_task(coro)\n async with self.scan._acatch(context=name), self._task_counter.count(task_name=name, asyncio_task=task, n=n):\n return await task\n\n async def _event_handler_watchdog(self):\n \"\"\"\n Watches handle_event and handle_batch tasks and cancels them if they exceed their max runtime.\n \"\"\"\n while not self.scan.stopping and not self.errored:\n # if there are events in the outgoing queue, we leave the tasks alone\n if self.outgoing_event_queue.qsize() > 0:\n await self.helpers.sleep(self._event_handler_watchdog_interval)\n continue\n event_handler_tasks = [\n t for t in self._task_counter.tasks.values() if t.function_name in (\"handle_event\", \"handle_batch\")\n ]\n for task in event_handler_tasks:\n if task.running_for > self.event_handler_timeout:\n self.warning(\n f\"{self.name} Cancelling event handler task {task.task_name} because it's been running for {task.running_for:.1f}s (max timeout is {self.event_handler_timeout})\"\n )\n await task.cancel()\n await asyncio.sleep(self._event_handler_watchdog_interval)\n\n async def queue_event(self, event):\n \"\"\"\n Asynchronously queues an incoming event to the module's event queue for further processing.\n\n The function performs an initial check to see if the event is acceptable for queuing.\n If the event passes the check, it is put into the `incoming_event_queue`.\n\n Args:\n event: The event object to be queued.\n\n Returns:\n None: The function doesn't return anything but modifies the state of the `incoming_event_queue`.\n\n Examples:\n >>> await self.queue_event(some_event)\n\n Raises:\n AttributeError: If the module is not in an acceptable state to queue incoming events.\n \"\"\"\n async with self._task_counter.count(\"queue_event()\", _log=False):\n if self.incoming_event_queue is False:\n self.debug(\"Not in an acceptable state to queue incoming event\")\n return\n acceptable, reason = self._event_precheck(event)\n if not acceptable:\n if reason and reason != \"its type is not in watched_events\":\n self.debug(f\"Not queueing {event} because {reason}\")\n return\n else:\n self.debug(f\"Queueing {event} because {reason}\")\n try:\n self.incoming_event_queue.put_nowait(event)\n async with self.event_received:\n self.event_received.notify()\n if event.type != \"FINISHED\":\n self.scan._new_activity = True\n except AttributeError:\n self.debug(\"Not in an acceptable state to queue incoming event\")\n\n async def queue_outgoing_event(self, event, **kwargs):\n \"\"\"\n Queues an outgoing event to the module's outgoing event queue for further processing.\n\n The function attempts to put the event into the `outgoing_event_queue` immediately.\n If it's not possible due to the current state of the module, an AttributeError is raised, and a debug log is generated.\n\n Args:\n event: The event object to be queued.\n **kwargs: Additional keyword arguments to be associated with the event.\n\n Returns:\n None: The function doesn't return anything but modifies the state of the `outgoing_event_queue`.\n\n Examples:\n >>> self.queue_outgoing_event(some_outgoing_event, abort_if=lambda e: \"unresolved\" in e.tags)\n\n Raises:\n AttributeError: If the module is not in an acceptable state to queue outgoing events.\n \"\"\"\n try:\n await self.outgoing_event_queue.put((event, kwargs))\n except AttributeError:\n self.debug(\"Not in an acceptable state to queue outgoing event\")\n\n def set_error_state(self, message=None, clear_outgoing_queue=False, critical=False):\n \"\"\"\n Puts the module into an errored state where it cannot accept new events. Optionally logs a warning message.\n\n The function sets the module's `errored` attribute to True and logs a warning with the optional message.\n It also clears the incoming event queue to prevent further processing and updates its status to False.\n\n Args:\n message (str, optional): Additional message to be logged along with the warning.\n\n Returns:\n None: The function doesn't return anything but updates the `errored` state and clears the incoming event queue.\n\n Examples:\n >>> self.set_error_state()\n >>> self.set_error_state(\"Failed to connect to the server\")\n\n Notes:\n - The function sets `self._incoming_event_queue` to False to prevent its further use.\n - If the module was already in an errored state, the function will not reset the error state or the queue.\n \"\"\"\n if not self.errored:\n log_msg = \"Setting error state\"\n if message is not None:\n log_msg += f\": {message}\"\n if critical:\n log_fn = self.error\n else:\n log_fn = self.warning\n log_fn(log_msg)\n self.errored = True\n # clear incoming queue\n if self.incoming_event_queue is not False:\n self.debug(\"Emptying event_queue\")\n with suppress(asyncio.queues.QueueEmpty):\n while 1:\n self.incoming_event_queue.get_nowait()\n # set queue to None to prevent its use\n # if there are leftover objects in the queue, the scan will hang.\n self._incoming_event_queue = False\n\n if clear_outgoing_queue:\n with suppress(asyncio.queues.QueueEmpty):\n while 1:\n self.outgoing_event_queue.get_nowait()\n\n def is_incoming_duplicate(self, event, add=False):\n if event.type in (\"FINISHED\",):\n return False, \"\"\n reason = \"\"\n try:\n event_hash = self._incoming_dedup_hash(event)\n except Exception as e:\n msg = f\"Unhandled exception in {self.name}._incoming_dedup_hash({event}): {e}\"\n self.error(msg)\n return True, msg\n with suppress(TypeError, ValueError):\n event_hash, reason = event_hash\n is_dup = event_hash in self._incoming_dup_tracker\n if add:\n self._incoming_dup_tracker.add(event_hash)\n return is_dup, reason\n\n def _incoming_dedup_hash(self, event):\n \"\"\"\n Determines the criteria for what is considered to be a duplicate event if `accept_dupes` is False.\n \"\"\"\n if self.per_host_only:\n return self.get_per_host_hash(event), \"per_host_only=True\"\n if self.per_hostport_only:\n return self.get_per_hostport_hash(event), \"per_hostport_only=True\"\n elif self.per_domain_only:\n return self.get_per_domain_hash(event), \"per_domain_only=True\"\n return hash(event), \"\"\n\n def _outgoing_dedup_hash(self, event):\n \"\"\"\n Determines the criteria for what is considered to be a duplicate event if `suppress_dupes` is True.\n\n We take into account the `internal` attribute we don't want an internal event (which isn't distributed to output modules)\n to inadvertently suppress a non-internal event.\n \"\"\"\n return hash((event, self.name, event.internal, event.always_emit))\n\n def get_per_host_hash(self, event):\n \"\"\"\n Computes a per-host hash value for a given event. This method may be optionally overridden in subclasses.\n\n The function uses the event's `host` to create a string to be hashed.\n\n Args:\n event (Event): The event object containing host information.\n\n Returns:\n int: The hash value computed for the host.\n\n Examples:\n >>> event = self.make_event(\"https://example.com:8443\")\n >>> self.get_per_host_hash(event)\n \"\"\"\n return hash(event.host)\n\n def get_per_hostport_hash(self, event):\n \"\"\"\n Computes a per-host:port hash value for a given event. This method may be optionally overridden in subclasses.\n\n The function uses the event's `host`, `port`, and `scheme` (for URLs) to create a string to be hashed.\n The hash value is used for distinguishing events related to the same host.\n\n Args:\n event (Event): The event object containing host, port, or parsed URL information.\n\n Returns:\n int: The hash value computed for the host.\n\n Examples:\n >>> event = self.make_event(\"https://example.com:8443\")\n >>> self.get_per_hostport_hash(event)\n \"\"\"\n parsed = getattr(event, \"parsed_url\", None)\n if parsed is None:\n to_hash = self.helpers.make_netloc(event.host, event.port)\n else:\n to_hash = f\"{parsed.scheme}://{parsed.netloc}/\"\n return hash(to_hash)\n\n def get_per_domain_hash(self, event):\n \"\"\"\n Computes a per-domain hash value for a given event. This method may be optionally overridden in subclasses.\n\n Events with the same root domain will receive the same hash value.\n\n Args:\n event (Event): The event object containing host, port, or parsed URL information.\n\n Returns:\n int: The hash value computed for the domain.\n\n Examples:\n >>> event = self.make_event(\"https://www.example.com:8443\")\n >>> self.get_per_domain_hash(event)\n \"\"\"\n _, domain = self.helpers.split_domain(event.host)\n return hash(domain)\n\n @property\n def name(self):\n return str(self._name)\n\n @property\n def helpers(self):\n return self.scan.helpers\n\n @property\n def status(self):\n \"\"\"\n Provides the current status of the module as a dictionary.\n\n The dictionary contains the following keys:\n - 'events': A sub-dictionary with 'incoming' and 'outgoing' keys, representing the number of events in the respective queues.\n - 'tasks': The current value of the task counter.\n - 'errored': A boolean value indicating if the module is in an error state.\n - 'running': A boolean value indicating if the module is currently processing data.\n\n Returns:\n dict: A dictionary containing the current status of the module.\n\n Examples:\n >>> self.status\n {'events': {'incoming': 5, 'outgoing': 2}, 'tasks': 3, 'errored': False, 'running': True}\n \"\"\"\n status = {\n \"events\": {\"incoming\": self.num_incoming_events, \"outgoing\": self.outgoing_event_queue.qsize()},\n \"tasks\": self._task_counter.value,\n \"errored\": self.errored,\n }\n status[\"running\"] = self.running\n return status\n\n @property\n def running(self):\n \"\"\"Property indicating whether the module is currently processing data.\n\n This property checks if the task counter (`self._task_counter.value`) is greater than zero,\n indicating that there are ongoing tasks in the module.\n\n Returns:\n bool: True if the module is currently processing data, False otherwise.\n \"\"\"\n return self._task_counter.value > 0\n\n @property\n def finished(self):\n \"\"\"Property indicating whether the module has finished processing.\n\n This property checks three conditions to determine if the module is finished:\n 1. The module is not currently running (`self.running` is False).\n 2. The number of incoming events in the queue is zero or less (`self.num_incoming_events <= 0`).\n 3. The number of outgoing events in the queue is zero or less (`self.outgoing_event_queue.qsize() <= 0`).\n\n Returns:\n bool: True if the module has finished processing, False otherwise.\n \"\"\"\n return not self.running and self.num_incoming_events <= 0 and self.outgoing_event_queue.qsize() <= 0\n\n async def run_process(self, *args, **kwargs):\n kwargs[\"_proc_tracker\"] = self._proc_tracker\n return await self.helpers.run(*args, **kwargs)\n\n async def run_process_live(self, *args, **kwargs):\n kwargs[\"_proc_tracker\"] = self._proc_tracker\n async for line in self.helpers.run_live(*args, **kwargs):\n yield line\n\n def prepare_api_request(self, url, kwargs):\n \"\"\"\n Prepare an API request by adding the necessary authentication - header, bearer token, etc.\n \"\"\"\n if self.api_key:\n url = url.format(api_key=self.api_key)\n if \"headers\" not in kwargs:\n kwargs[\"headers\"] = {}\n kwargs[\"headers\"][\"Authorization\"] = f\"Bearer {self.api_key}\"\n return url, kwargs\n\n async def api_request(self, *args, **kwargs):\n \"\"\"\n Makes an HTTP request while automatically:\n - avoiding rate limits (sleep/retry)\n - cycling API keys\n - cancelling after too many failed attempts\n \"\"\"\n url = args[0] if args else kwargs.pop(\"url\", \"\")\n retry_on_http_429 = kwargs.pop(\"retry_on_http_429\", True)\n\n # loop until we have a successful request\n for _ in range(self.api_retries):\n if \"headers\" not in kwargs:\n kwargs[\"headers\"] = {}\n new_url, kwargs = self.prepare_api_request(url, kwargs)\n kwargs[\"url\"] = new_url\n\n r = await self.helpers.request(**kwargs)\n success = r is not None and self._api_response_is_success(r)\n\n if success:\n self._api_request_failures = 0\n else:\n status_code = getattr(r, \"status_code\", 0)\n response_text = getattr(r, \"text\", \"\")\n self.trace(f\"API response to {url} failed with status code {status_code}: {response_text}\")\n self._api_request_failures += 1\n if self._api_request_failures >= self.api_failure_abort_threshold:\n self.set_error_state(\n f\"Setting error state due to {self._api_request_failures:,} failed HTTP requests\"\n )\n else:\n # sleep for a bit if we're being rate limited\n retry_after = self._get_retry_after(r)\n if (retry_after or status_code == 429) and retry_on_http_429:\n sleep_interval = int(retry_after) if retry_after is not None else self._429_sleep_interval\n if retry_after and retry_after > self._429_max_sleep_interval:\n self.verbose(\n f\"Got an excessive retry-after header of {retry_after} from {new_url}, using {self._429_max_sleep_interval} instead\"\n )\n sleep_interval = self._429_max_sleep_interval\n self.verbose(\n f\"Sleeping for {sleep_interval:,} seconds due to rate limit (HTTP status: {status_code})\"\n )\n await asyncio.sleep(sleep_interval)\n elif self._api_keys:\n # if request failed, cycle API keys and try again\n self.cycle_api_key()\n continue\n break\n\n return r\n\n async def api_download(self, url, **kwargs):\n \"\"\"\n A wrapper around the `download()` web helper that incorporates API key cycling.\n \"\"\"\n error = None\n raise_error = kwargs.pop(\"raise_error\", False)\n for _ in range(self.api_retries):\n new_url, kwargs = self.prepare_api_request(url, kwargs)\n if \"raise_error\" not in kwargs:\n kwargs[\"raise_error\"] = True\n try:\n return await self.helpers.download(new_url, **kwargs)\n except WebError as e:\n error = e\n self.cycle_api_key()\n if raise_error:\n raise error\n\n def _get_retry_after(self, r):\n # try to get retry_after from headers first\n headers = getattr(r, \"headers\", {})\n retry_after = headers.get(\"Retry-After\", None)\n if retry_after is None:\n # then look in body json\n with suppress(Exception):\n body_json = r.json()\n if isinstance(body_json, dict):\n retry_after = body_json.get(\"retry_after\", None)\n if retry_after is not None:\n # we don't allow retry-after smaller than 1 second\n # this is to prevent cases where APIs erroneously return a retry-after value of 0\n # e.g. https://github.com/blacklanternsecurity/bbot/issues/2826\n return max(1.0, float(retry_after))\n\n def _prepare_api_iter_req(self, url, page, page_size, offset, **requests_kwargs):\n \"\"\"\n Default function for preparing an API request for iterating through paginated data.\n \"\"\"\n url = self.helpers.safe_format(url, page=page, page_size=page_size, offset=offset)\n return url, requests_kwargs\n\n def _api_response_is_success(self, r):\n # 404s typically indicate no data rather than an actual error with the API, so we don't want to retry them\n return getattr(r, \"is_success\", False) or getattr(r, \"status_code\", 0) == 404\n\n async def api_page_iter(self, url, page_size=100, _json=True, next_key=None, iter_key=None, **requests_kwargs):\n \"\"\"\n An asynchronous generator function for iterating through paginated API data.\n\n This function continuously makes requests to a specified API URL, incrementing the page number\n or applying a custom pagination function, and yields the received data one page at a time.\n It is well-suited for APIs that provide paginated results.\n\n Args:\n url (str): The initial API URL. Can contain placeholders for 'page', 'page_size', and 'offset'.\n page_size (int, optional): The number of items per page. Defaults to 100.\n json (bool, optional): If True, attempts to deserialize the response content to a JSON object. Defaults to True.\n next_key (callable, optional): A function that takes the last page's data and returns the URL for the next page. Defaults to None.\n iter_key (callable, optional): A function that builds each new request based on the page number, page size, and offset. Defaults to a simple implementation that autoreplaces {page} and {page_size} in the url.\n **requests_kwargs: Arbitrary keyword arguments that will be forwarded to the HTTP request function.\n\n Yields:\n dict or httpx.Response: If 'json' is True, yields a dictionary containing the parsed JSON data. Otherwise, yields the raw HTTP response.\n\n Note:\n The loop will continue indefinitely unless manually stopped. Make sure to break out of the loop once the last page has been received.\n\n Examples:\n >>> agen = api_page_iter('https://api.example.com/data?page={page}&page_size={page_size}')\n >>> try:\n >>> async for page in agen:\n >>> subdomains = page[\"subdomains\"]\n >>> self.hugesuccess(subdomains)\n >>> if not subdomains:\n >>> break\n >>> finally:\n >>> await agen.aclose()\n \"\"\"\n page = 1\n offset = 0\n result = None\n if iter_key is None:\n iter_key = self._prepare_api_iter_req\n while 1:\n if result and callable(next_key):\n try:\n new_url = next_key(result)\n except Exception as e:\n self.debug(f\"Failed to extract next page of results from {url}: {e}\")\n self.debug(traceback.format_exc())\n else:\n new_url, new_kwargs = iter_key(url, page, page_size, offset, **requests_kwargs)\n result = await self.api_request(new_url, **new_kwargs)\n if result is None:\n self.verbose(f\"api_page_iter() got no response for {new_url}\")\n break\n try:\n if _json:\n result = result.json()\n yield result\n except Exception:\n self.warning(f'Error in api_page_iter() for url: \"{new_url}\"')\n self.trace(traceback.format_exc())\n break\n finally:\n offset += page_size\n page += 1\n\n @property\n def preset(self):\n return self.scan.preset\n\n @property\n def config(self):\n \"\"\"Property that provides easy access to the module's configuration in the scan's config.\n\n This property serves as a shortcut to retrieve the module-specific configuration from\n `self.scan.config`. If no configuration is found for this module, an empty dictionary is returned.\n\n Returns:\n dict: The configuration dictionary specific to this module.\n \"\"\"\n config = self.scan.config.get(\"modules\", {}).get(self.name, {})\n if config is None:\n config = {}\n return config\n\n @property\n def incoming_event_queue(self):\n if self._incoming_event_queue is None:\n if self._shuffle_incoming_queue:\n self._incoming_event_queue = ShuffleQueue()\n else:\n self._incoming_event_queue = asyncio.Queue()\n return self._incoming_event_queue\n\n @property\n def outgoing_event_queue(self):\n if self._outgoing_event_queue is None:\n self._outgoing_event_queue = ShuffleQueue(self._qsize)\n return self._outgoing_event_queue\n\n @property\n def priority(self):\n \"\"\"\n Gets the priority level of the module as an integer.\n\n The priority level is constrained to be between 1 and 5, inclusive.\n A lower value indicates a higher priority.\n\n Returns:\n int: The priority level of the module, constrained between 1 and 5.\n\n Examples:\n >>> self.priority\n 3\n \"\"\"\n return int(max(1, min(5, self._priority)))\n\n @property\n def auth_required(self):\n return self.meta.get(\"auth_required\", False)\n\n @property\n def http_timeout(self):\n \"\"\"\n Convenience shortcut to `http_timeout` in the config\n \"\"\"\n return self.scan.web_config.get(\"http_timeout\", 10)\n\n @property\n def log(self):\n if getattr(self, \"_log\", None) is None:\n self._log = logging.getLogger(f\"bbot.modules.{self.name}\")\n return self._log\n\n @property\n def memory_usage(self):\n \"\"\"Property that calculates the current memory usage of the module in bytes.\n\n This property uses the `get_size` function to estimate the memory consumption\n of the module object. The depth of the object graph traversal is limited to 3 levels\n to avoid performance issues. Commonly shared objects like `self.scan`, `self.helpers`,\n are excluded from the calculation to prevent double-counting.\n\n Returns:\n int: The estimated memory usage of the module in bytes.\n \"\"\"\n seen = {self.scan, self.helpers, self.log} # noqa\n return get_size(self, max_depth=3, seen=seen)\n\n def __str__(self):\n return self.name\n\n def log_table(self, *args, **kwargs):\n \"\"\"Logs a table to the console and optionally writes it to a file.\n\n This function generates a table using `self.helpers.make_table`, then logs each line\n of the table as an info-level log. If a table_name is provided, it also writes the table to a file.\n\n Args:\n *args: Variable length argument list to be passed to `self.helpers.make_table`.\n **kwargs: Arbitrary keyword arguments. If 'table_name' is specified, the table will be written to a file.\n\n Returns:\n str: The generated table as a string.\n\n Examples:\n >>> self.log_table(['Header1', 'Header2'], [['row1col1', 'row1col2'], ['row2col1', 'row2col2']], table_name=\"my_table\")\n \"\"\"\n table_name = kwargs.pop(\"table_name\", None)\n max_log_entries = kwargs.pop(\"max_log_entries\", None)\n table = self.helpers.make_table(*args, **kwargs)\n lines_logged = 0\n for line in table.splitlines():\n if max_log_entries is not None and lines_logged > max_log_entries:\n break\n self.info(line)\n lines_logged += 1\n if table_name is not None:\n date = self.helpers.make_date()\n filename = self.scan.home / f\"{self.helpers.tagify(table_name)}-table-{date}.txt\"\n with open(filename, \"w\") as f:\n f.write(table)\n self.verbose(f\"Wrote {table_name} to {filename}\")\n return table\n\n def _is_graph_important(self, event):\n return self.preserve_graph and getattr(event, \"_graph_important\", False) and not getattr(event, \"_omit\", False)\n\n @property\n def preserve_graph(self):\n preserve_graph = self.config.get(\"preserve_graph\", None)\n if preserve_graph is None:\n preserve_graph = self._preserve_graph\n return preserve_graph\n\n def debug(self, *args, trace=False, **kwargs):\n \"\"\"Logs debug messages and optionally the stack trace of the most recent exception.\n\n Args:\n *args: Variable-length argument list to pass to the logger.\n trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to False.\n **kwargs: Arbitrary keyword arguments to pass to the logger.\n\n Examples:\n >>> self.debug(\"This is a debug message\")\n >>> self.debug(\"This is a debug message with a trace\", trace=True)\n \"\"\"\n self.log.debug(*args, extra={\"scan_id\": self.scan.id}, **kwargs)\n if trace:\n self.trace()\n\n def verbose(self, *args, trace=False, **kwargs):\n \"\"\"Logs messages and optionally the stack trace of the most recent exception.\n\n Args:\n *args: Variable-length argument list to pass to the logger.\n trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to False.\n **kwargs: Arbitrary keyword arguments to pass to the logger.\n\n Examples:\n >>> self.verbose(\"This is a verbose message\")\n >>> self.verbose(\"This is a verbose message with a trace\", trace=True)\n \"\"\"\n self.log.verbose(*args, extra={\"scan_id\": self.scan.id}, **kwargs)\n if trace:\n self.trace()\n\n def hugeverbose(self, *args, trace=False, **kwargs):\n \"\"\"Logs a whole message in emboldened white text, and optionally the stack trace of the most recent exception.\n\n Args:\n *args: Variable-length argument list to pass to the logger.\n trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to False.\n **kwargs: Arbitrary keyword arguments to pass to the logger.\n\n Examples:\n >>> self.hugeverbose(\"This is a huge verbose message\")\n >>> self.hugeverbose(\"This is a huge verbose message with a trace\", trace=True)\n \"\"\"\n self.log.hugeverbose(*args, extra={\"scan_id\": self.scan.id}, **kwargs)\n if trace:\n self.trace()\n\n def info(self, *args, trace=False, **kwargs):\n \"\"\"Logs informational messages and optionally the stack trace of the most recent exception.\n\n Args:\n *args: Variable-length argument list to pass to the logger.\n trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to False.\n **kwargs: Arbitrary keyword arguments to pass to the logger.\n\n Examples:\n >>> self.info(\"This is an informational message\")\n >>> self.info(\"This is an informational message with a trace\", trace=True)\n \"\"\"\n self.log.info(*args, extra={\"scan_id\": self.scan.id}, **kwargs)\n if trace:\n self.trace()\n\n def hugeinfo(self, *args, trace=False, **kwargs):\n \"\"\"Logs a whole message in emboldened blue text, and optionally the stack trace of the most recent exception.\n\n Args:\n *args: Variable-length argument list to pass to the logger.\n trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to False.\n **kwargs: Arbitrary keyword arguments to pass to the logger.\n\n Examples:\n >>> self.hugeinfo(\"This is a huge informational message\")\n >>> self.hugeinfo(\"This is a huge informational message with a trace\", trace=True)\n \"\"\"\n self.log.hugeinfo(*args, extra={\"scan_id\": self.scan.id}, **kwargs)\n if trace:\n self.trace()\n\n def success(self, *args, trace=False, **kwargs):\n \"\"\"Logs a success message, and optionally the stack trace of the most recent exception.\n\n Args:\n *args: Variable-length argument list to pass to the logger.\n trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to False.\n **kwargs: Arbitrary keyword arguments to pass to the logger.\n\n Examples:\n >>> self.success(\"Operation completed successfully\")\n >>> self.success(\"Operation completed with a trace\", trace=True)\n \"\"\"\n self.log.success(*args, extra={\"scan_id\": self.scan.id}, **kwargs)\n if trace:\n self.trace()\n\n def hugesuccess(self, *args, trace=False, **kwargs):\n \"\"\"Logs a whole message in emboldened green text, and optionally the stack trace of the most recent exception.\n\n Args:\n *args: Variable-length argument list to pass to the logger.\n trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to False.\n **kwargs: Arbitrary keyword arguments to pass to the logger.\n\n Examples:\n >>> self.hugesuccess(\"This is a huge success message\")\n >>> self.hugesuccess(\"This is a huge success message with a trace\", trace=True)\n \"\"\"\n self.log.hugesuccess(*args, extra={\"scan_id\": self.scan.id}, **kwargs)\n if trace:\n self.trace()\n\n def warning(self, *args, trace=True, **kwargs):\n \"\"\"Logs a warning message, and optionally the stack trace of the most recent exception.\n\n Args:\n *args: Variable-length argument list to pass to the logger.\n trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to True.\n **kwargs: Arbitrary keyword arguments to pass to the logger.\n\n Examples:\n >>> self.warning(\"This is a warning message\")\n >>> self.warning(\"This is a warning message with a trace\", trace=False)\n \"\"\"\n self.log.warning(*args, extra={\"scan_id\": self.scan.id}, **kwargs)\n if trace:\n self.trace()\n\n def hugewarning(self, *args, trace=True, **kwargs):\n \"\"\"Logs a whole message in emboldened orange text, and optionally the stack trace of the most recent exception.\n\n Args:\n *args: Variable-length argument list to pass to the logger.\n trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to True.\n **kwargs: Arbitrary keyword arguments to pass to the logger.\n\n Examples:\n >>> self.hugewarning(\"This is a huge warning message\")\n >>> self.hugewarning(\"This is a huge warning message with a trace\", trace=False)\n \"\"\"\n self.log.hugewarning(*args, extra={\"scan_id\": self.scan.id}, **kwargs)\n if trace:\n self.trace()\n\n def error(self, *args, trace=True, **kwargs):\n \"\"\"Logs an error message, and optionally the stack trace of the most recent exception.\n\n Args:\n *args: Variable-length argument list to pass to the logger.\n trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to True.\n **kwargs: Arbitrary keyword arguments to pass to the logger.\n\n Examples:\n >>> self.error(\"This is an error message\")\n >>> self.error(\"This is an error message with a trace\", trace=False)\n \"\"\"\n self.log.error(*args, extra={\"scan_id\": self.scan.id}, **kwargs)\n if trace:\n self.trace()\n\n def trace(self, msg=None):\n \"\"\"Logs the stack trace of the most recently caught exception.\n\n This method captures the type, value, and traceback of the most recent exception and logs it using the trace level. It is typically used for debugging purposes.\n\n Anything logged using this method will always be written to the scan's `debug.log`, even if debugging is not enabled.\n\n Examples:\n >>> try:\n >>> 1 / 0\n >>> except ZeroDivisionError:\n >>> self.trace()\n \"\"\"\n if msg is None:\n e_type, e_val, e_traceback = exc_info()\n if e_type is not None:\n self.log.trace(traceback.format_exc())\n else:\n self.log.trace(msg)\n\n def critical(self, *args, trace=True, **kwargs):\n \"\"\"Logs a whole message in emboldened red text, and optionally the stack trace of the most recent exception.\n\n Args:\n *args: Variable-length argument list to pass to the logger.\n trace (bool, optional): Whether to log the stack trace of the most recently caught exception. Defaults to True.\n **kwargs: Arbitrary keyword arguments to pass to the logger.\n\n Examples:\n >>> self.critical(\"This is a critical message\")\n >>> self.critical(\"This is a critical message with a trace\", trace=False)\n \"\"\"\n self.log.critical(*args, extra={\"scan_id\": self.scan.id}, **kwargs)\n if trace:\n self.trace()\n\n @classmethod\n def help_text(self):\n \"\"\"\n Returns a string containing help text for the module.\n This includes the module's description, metadata, events, flags, and available options.\n \"\"\"\n # Retrieve the module's metadata, options, events, and flags\n meta = getattr(self, \"meta\", {})\n options = getattr(self, \"options\", {})\n options_desc = getattr(self, \"options_desc\", {})\n watched_events = getattr(self, \"watched_events\", [])\n produced_events = getattr(self, \"produced_events\", [])\n flags = getattr(self, \"flags\", [])\n\n help_text = \"\\n\" + \"=\" * 40 + \"\\n\"\n help_text += f\"Module Help: {self.__name__}\\n\"\n help_text += \"=\" * 40 + \"\\n\\n\"\n\n for key, value in meta.items():\n help_text += f\"{key.replace('_', ' ').title()}: {value}\\n\"\n\n help_text += \"\\nWatched Events:\\n\"\n help_text += \" \" + \", \".join(watched_events) + \"\\n\" if watched_events else \" None\\n\"\n\n help_text += \"\\nProduced Events:\\n\"\n help_text += \" \" + \", \".join(produced_events) + \"\\n\" if produced_events else \" None\\n\"\n\n help_text += \"\\nFlags:\\n\"\n help_text += \" \" + \", \".join(flags) + \"\\n\" if flags else \" None\\n\"\n\n help_text += \"\\nOptions:\\n\"\n if options:\n for option, default_value in options.items():\n option_description = options_desc.get(option, \"No description available.\")\n help_text += f\" - {option}:\\n\"\n help_text += f\" Description: {option_description}\\n\"\n help_text += f\" Default: {default_value}\\n\"\n else:\n help_text += \" No options available.\"\n\n return help_text\n\n\nclass BaseInterceptModule(BaseModule):\n \"\"\"\n An Intercept Module is a special type of high-priority module that gets early access to events.\n\n If you want your module to tag or modify an event before it's distributed to the scan, it should\n probably be an intercept module.\n\n Examples of intercept modules include `dns` (for DNS resolution and wildcard detection)\n and `cloud` (for detection and tagging of cloud assets).\n \"\"\"\n\n accept_dupes = True\n accept_url_special = True\n _intercept = True\n\n async def _worker(self):\n async with self.scan._acatch(context=self._worker, unhandled_is_critical=True):\n try:\n while not self.scan.stopping and not self.errored:\n try:\n if self.incoming_event_queue is not False:\n incoming = await self.get_incoming_event()\n try:\n event, kwargs = incoming\n except ValueError:\n event = incoming\n kwargs = {}\n else:\n self.debug(\"Event queue is in bad state\")\n break\n except asyncio.queues.QueueEmpty:\n await asyncio.sleep(0.1)\n continue\n\n if event.type == \"FINISHED\":\n context = f\"{self.name}.finish()\"\n async with self.scan._acatch(context), self._task_counter.count(context):\n await self.finish()\n continue\n\n acceptable = True\n async with self._task_counter.count(f\"event_precheck({event})\"):\n precheck_pass, reason = self._event_precheck(event)\n if not precheck_pass:\n self.debug(f\"Not intercepting {event} because precheck failed ({reason})\")\n acceptable = False\n async with self._task_counter.count(f\"event_postcheck({event})\"):\n postcheck_pass, reason = await self._event_postcheck(event)\n if not postcheck_pass:\n self.debug(f\"Not intercepting {event} because postcheck failed ({reason})\")\n acceptable = False\n\n # whether to pass the event on to the rest of the scan\n # defaults to true, unless handle_event returns False\n forward_event = True\n forward_event_reason = \"\"\n\n if acceptable:\n context = f\"{self.name}.handle_event({event, kwargs})\"\n self.scan.stats.event_consumed(event, self)\n self.debug(f\"Intercepting {event}\")\n try:\n forward_event = await self.run_task(self.handle_event(event, **kwargs), context)\n except asyncio.CancelledError:\n self.debug(f\"{context} was cancelled\")\n continue\n with suppress(ValueError, TypeError):\n forward_event, forward_event_reason = forward_event\n\n if forward_event is False:\n self.debug(f\"Not forwarding {event} because {forward_event_reason}\")\n continue\n\n self.debug(f\"Forwarding {event}\")\n await self.forward_event(event, kwargs)\n\n except asyncio.CancelledError:\n # this trace was used for debugging leaked CancelledErrors from inside httpx\n # self.log.trace(\"Worker cancelled\")\n raise\n except BaseException as e:\n if self.helpers.in_exception_chain(e, (KeyboardInterrupt,)):\n self.scan.stop()\n else:\n self.critical(f\"Critical failure in intercept module {self.name}: {e}\")\n self.critical(traceback.format_exc())\n self.log.trace(\"Worker stopped\")\n\n async def get_incoming_event(self):\n \"\"\"\n Get an event from this module's incoming event queue\n \"\"\"\n return await self.incoming_event_queue.get()\n\n async def forward_event(self, event, kwargs):\n \"\"\"\n Used for forwarding the event on to the next intercept module\n \"\"\"\n await self.outgoing_event_queue.put((event, kwargs))\n\n async def queue_outgoing_event(self, event, **kwargs):\n \"\"\"\n Used by emit_event() to raise new events to the scan\n \"\"\"\n # if this was a normal module, we'd put it in the outgoing queue\n # but because it's an intercept module, we need to queue it at the scan's ingress\n await self.scan.ingress_module.queue_event(event, kwargs)\n\n async def queue_event(self, event, kwargs=None):\n \"\"\"\n Put an event in this module's incoming event queue\n \"\"\"\n if kwargs is None:\n kwargs = {}\n try:\n self.incoming_event_queue.put_nowait((event, kwargs))\n except AttributeError:\n self.debug(\"Not in an acceptable state to queue incoming event\")\n\n async def _event_postcheck(self, event):\n return await self._event_postcheck_inner(event)\n" }, { "path": "bbot/modules/bevigil.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass bevigil(subdomain_enum_apikey):\n \"\"\"\n Retrieve OSINT data from mobile applications using BeVigil\n \"\"\"\n\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\", \"URL_UNVERIFIED\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Retrieve OSINT data from mobile applications using BeVigil\",\n \"created_date\": \"2022-10-26\",\n \"author\": \"@alt-glitch\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\", \"urls\": False}\n options_desc = {\"api_key\": \"BeVigil OSINT API Key\", \"urls\": \"Emit URLs in addition to DNS_NAMEs\"}\n\n base_url = \"https://osint.bevigil.com/api\"\n\n async def setup(self):\n self.api_key = self.config.get(\"api_key\", \"\")\n self.urls = self.config.get(\"urls\", False)\n return await super().setup()\n\n def prepare_api_request(self, url, kwargs):\n kwargs[\"headers\"][\"X-Access-Token\"] = self.api_key\n return url, kwargs\n\n async def handle_event(self, event):\n query = self.make_query(event)\n subdomains = await self.query(query, request_fn=self.request_subdomains, parse_fn=self.parse_subdomains)\n if subdomains:\n for subdomain in subdomains:\n await self.emit_event(\n subdomain,\n \"DNS_NAME\",\n parent=event,\n context=f'{{module}} queried BeVigil\\'s API for \"{query}\" and discovered {{event.type}}: {{event.data}}',\n )\n\n if self.urls:\n urls = await self.query(query, request_fn=self.request_urls, parse_fn=self.parse_urls)\n if urls:\n for parsed_url in await self.helpers.run_in_executor_mp(self.helpers.validators.collapse_urls, urls):\n await self.emit_event(\n parsed_url.geturl(),\n \"URL_UNVERIFIED\",\n parent=event,\n context=f'{{module}} queried BeVigil\\'s API for \"{query}\" and discovered {{event.type}}: {{event.data}}',\n )\n\n async def request_subdomains(self, query):\n url = f\"{self.base_url}/{self.helpers.quote(query)}/subdomains/\"\n return await self.api_request(url)\n\n async def request_urls(self, query):\n url = f\"{self.base_url}/{self.helpers.quote(query)}/urls/\"\n return await self.api_request(url)\n\n async def parse_subdomains(self, r, query=None):\n results = set()\n subdomains = r.json().get(\"subdomains\")\n if subdomains:\n results.update(subdomains)\n return results\n\n async def parse_urls(self, r, query=None):\n results = set()\n urls = r.json().get(\"urls\")\n if urls:\n results.update(urls)\n return results\n" }, { "path": "bbot/modules/bucket_amazon.py", "content": "from bbot.modules.templates.bucket import bucket_template\n\n\nclass bucket_amazon(bucket_template):\n watched_events = [\"DNS_NAME\", \"STORAGE_BUCKET\"]\n produced_events = [\"STORAGE_BUCKET\", \"FINDING\"]\n flags = [\"active\", \"safe\", \"cloud-enum\", \"web-basic\"]\n meta = {\n \"description\": \"Check for S3 buckets related to target\",\n \"created_date\": \"2022-11-04\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"permutations\": False}\n options_desc = {\n \"permutations\": \"Whether to try permutations\",\n }\n scope_distance_modifier = 3\n\n cloudcheck_provider_name = \"Amazon\"\n delimiters = (\"\", \".\", \"-\")\n base_domains = [\"s3.amazonaws.com\"]\n regions = [None]\n supports_open_check = True\n" }, { "path": "bbot/modules/bucket_digitalocean.py", "content": "from bbot.modules.templates.bucket import bucket_template\n\n\nclass bucket_digitalocean(bucket_template):\n watched_events = [\"DNS_NAME\", \"STORAGE_BUCKET\"]\n produced_events = [\"STORAGE_BUCKET\", \"FINDING\"]\n flags = [\"active\", \"safe\", \"slow\", \"cloud-enum\", \"web-thorough\"]\n meta = {\n \"description\": \"Check for DigitalOcean spaces related to target\",\n \"created_date\": \"2022-11-08\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"permutations\": False}\n options_desc = {\n \"permutations\": \"Whether to try permutations\",\n }\n\n cloudcheck_provider_name = \"DigitalOcean\"\n delimiters = (\"\", \"-\")\n base_domains = [\"digitaloceanspaces.com\"]\n regions = [\"ams3\", \"fra1\", \"nyc3\", \"sfo2\", \"sfo3\", \"sgp1\"]\n\n def build_url(self, bucket_name, base_domain, region):\n return f\"https://{bucket_name}.{region}.{base_domain}/\"\n" }, { "path": "bbot/modules/bucket_file_enum.py", "content": "from bbot.modules.base import BaseModule\nimport xml.etree.ElementTree as ET\n\n\nclass bucket_file_enum(BaseModule):\n \"\"\"\n Enumerate files in public storage buckets\n\n Currently only supports AWS and DigitalOcean\n \"\"\"\n\n watched_events = [\"STORAGE_BUCKET\"]\n produced_events = [\"URL_UNVERIFIED\"]\n meta = {\n \"description\": \"Works in conjunction with the filedownload module to download files from open storage buckets. Currently supported cloud providers: AWS, DigitalOcean\",\n \"created_date\": \"2023-11-14\",\n \"author\": \"@TheTechromancer\",\n }\n flags = [\"passive\", \"safe\", \"cloud-enum\"]\n options = {\n \"file_limit\": 50,\n }\n options_desc = {\"file_limit\": \"Limit the number of files downloaded per bucket\"}\n scope_distance_modifier = 2\n\n async def setup(self):\n self.file_limit = self.config.get(\"file_limit\", 50)\n return True\n\n async def handle_event(self, event):\n cloud_tags = (t for t in event.tags if t.startswith(\"cloud-\"))\n if any(t.endswith(\"-amazon\") or t.endswith(\"-digitalocean\") for t in cloud_tags):\n await self.handle_aws(event)\n\n async def handle_aws(self, event):\n url = event.data[\"url\"]\n urls_emitted = 0\n response = await self.helpers.request(url)\n status_code = getattr(response, \"status_code\", 0)\n if status_code == 200:\n content = response.text\n root = ET.fromstring(content)\n namespace = {\"s3\": \"http://s3.amazonaws.com/doc/2006-03-01/\"}\n keys = [key.text for key in root.findall(\".//s3:Key\", namespace)]\n for key in keys:\n bucket_file = url + \"/\" + key\n file_extension = self.helpers.get_file_extension(key)\n if file_extension not in self.scan.url_extension_blacklist:\n extension_upper = file_extension.upper()\n await self.emit_event(\n bucket_file,\n \"URL_UNVERIFIED\",\n parent=event,\n tags=\"filedownload\",\n context=f\"{{module}} enumerate files in bucket and discovered {extension_upper} file at {{event.type}}: {{event.data}}\",\n )\n urls_emitted += 1\n if urls_emitted >= self.file_limit:\n return\n" }, { "path": "bbot/modules/bucket_firebase.py", "content": "from bbot.modules.templates.bucket import bucket_template\n\n\nclass bucket_firebase(bucket_template):\n watched_events = [\"DNS_NAME\", \"STORAGE_BUCKET\"]\n produced_events = [\"STORAGE_BUCKET\", \"FINDING\"]\n flags = [\"active\", \"safe\", \"cloud-enum\", \"web-basic\"]\n meta = {\n \"description\": \"Check for open Firebase databases related to target\",\n \"created_date\": \"2023-03-20\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"permutations\": False}\n options_desc = {\n \"permutations\": \"Whether to try permutations\",\n }\n\n cloudcheck_provider_name = \"Google\"\n delimiters = (\"\", \"-\")\n base_domains = [\"firebaseio.com\"]\n\n def filter_bucket(self, event):\n host = str(event.host)\n if not any(host.endswith(f\".{d}\") for d in self.base_domains):\n return False, \"bucket belongs to a different cloud provider\"\n return True, \"\"\n\n def build_url(self, bucket_name, base_domain, region):\n return f\"https://{bucket_name}.{base_domain}/.json\"\n\n async def check_bucket_open(self, bucket_name, url):\n url = url.strip(\"/\") + \"/.json\"\n response = await self.helpers.request(url)\n tags = self.gen_tags_exists(response)\n status_code = getattr(response, \"status_code\", 404)\n msg = \"\"\n if status_code == 200:\n msg = \"Open storage bucket\"\n return (msg, tags)\n" }, { "path": "bbot/modules/bucket_google.py", "content": "from bbot.modules.templates.bucket import bucket_template\n\n\nclass bucket_google(bucket_template):\n \"\"\"\n Adapted from https://github.com/RhinoSecurityLabs/GCPBucketBrute/blob/master/gcpbucketbrute.py\n \"\"\"\n\n watched_events = [\"DNS_NAME\", \"STORAGE_BUCKET\"]\n produced_events = [\"STORAGE_BUCKET\", \"FINDING\"]\n flags = [\"active\", \"safe\", \"cloud-enum\", \"web-basic\"]\n meta = {\n \"description\": \"Check for Google object storage related to target\",\n \"created_date\": \"2022-11-04\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"permutations\": False}\n options_desc = {\n \"permutations\": \"Whether to try permutations\",\n }\n\n cloudcheck_provider_name = \"Google\"\n delimiters = (\"\", \"-\", \".\", \"_\")\n base_domains = [\"storage.googleapis.com\"]\n bad_permissions = [\n \"storage.buckets.get\",\n \"storage.buckets.list\",\n \"storage.buckets.create\",\n \"storage.buckets.delete\",\n \"storage.buckets.setIamPolicy\",\n \"storage.objects.get\",\n \"storage.objects.list\",\n \"storage.objects.create\",\n \"storage.objects.delete\",\n \"storage.objects.setIamPolicy\",\n ]\n\n def filter_bucket(self, event):\n if not str(event.host).endswith(\".googleapis.com\"):\n return False, \"bucket belongs to a different cloud provider\"\n return True, \"\"\n\n def build_url(self, bucket_name, base_domain, region):\n return f\"https://www.googleapis.com/storage/v1/b/{bucket_name}\"\n\n async def check_bucket_open(self, bucket_name, url):\n bad_permissions = []\n try:\n list_permissions = \"&\".join([\"=\".join((\"permissions\", p)) for p in self.bad_permissions])\n url = f\"https://www.googleapis.com/storage/v1/b/{bucket_name}/iam/testPermissions?\" + list_permissions\n response = await self.helpers.request(url)\n permissions = response.json()\n if isinstance(permissions, dict):\n bad_permissions = list(permissions.get(\"permissions\", {}))\n except Exception as e:\n self.info(f'Failed to enumerate permissions for bucket \"{bucket_name}\": {e}')\n msg = \"\"\n if bad_permissions:\n perms_str = \",\".join(bad_permissions)\n msg = f\"Open permissions on storage bucket ({perms_str})\"\n return (msg, set())\n\n def check_bucket_exists(self, bucket_name, response):\n status_code = getattr(response, \"status_code\", 0)\n existent_bucket = status_code not in (0, 400, 404)\n return existent_bucket, set()\n" }, { "path": "bbot/modules/bucket_microsoft.py", "content": "from bbot.modules.templates.bucket import bucket_template\n\n\nclass bucket_microsoft(bucket_template):\n watched_events = [\"DNS_NAME\", \"STORAGE_BUCKET\"]\n produced_events = [\"STORAGE_BUCKET\", \"FINDING\"]\n flags = [\"active\", \"safe\", \"cloud-enum\", \"web-basic\"]\n meta = {\n \"description\": \"Check for Azure storage blobs related to target\",\n \"created_date\": \"2022-11-04\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"permutations\": False}\n options_desc = {\n \"permutations\": \"Whether to try permutations\",\n }\n\n cloudcheck_provider_name = \"Microsoft\"\n delimiters = (\"\", \"-\")\n base_domains = [\"blob.core.windows.net\"]\n # Dirbusting is required to know whether a bucket is public\n supports_open_check = False\n\n def build_bucket_request(self, bucket_name, base_domain, region):\n url = self.build_url(bucket_name, base_domain, region)\n url = url.strip(\"/\") + f\"/{bucket_name}?restype=container\"\n return url, {}\n\n def check_bucket_exists(self, bucket_name, response):\n status_code = getattr(response, \"status_code\", 0)\n existent_bucket = status_code != 0\n return existent_bucket, set()\n\n def clean_bucket_url(self, url):\n # only return root URL\n return \"/\".join(url.split(\"/\")[:3])\n" }, { "path": "bbot/modules/bufferoverrun.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass BufferOverrun(subdomain_enum_apikey):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query BufferOverrun's TLS API for subdomains\",\n \"created_date\": \"2024-10-23\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\", \"commercial\": False}\n options_desc = {\"api_key\": \"BufferOverrun API key\", \"commercial\": \"Use commercial API\"}\n\n base_url = \"https://tls.bufferover.run/dns\"\n commercial_base_url = \"https://bufferover-run-tls.p.rapidapi.com/ipv4/dns\"\n\n async def setup(self):\n self.commercial = self.config.get(\"commercial\", False)\n return await super().setup()\n\n def prepare_api_request(self, url, kwargs):\n if self.commercial:\n kwargs[\"headers\"][\"x-rapidapi-host\"] = \"bufferover-run-tls.p.rapidapi.com\"\n kwargs[\"headers\"][\"x-rapidapi-key\"] = self.api_key\n else:\n kwargs[\"headers\"][\"x-api-key\"] = self.api_key\n return url, kwargs\n\n async def request_url(self, query):\n url = f\"{self.commercial_base_url if self.commercial else self.base_url}?q=.{query}\"\n return await self.api_request(url)\n\n async def parse_results(self, r, query):\n j = r.json()\n subdomains_set = set()\n if isinstance(j, dict):\n results = j.get(\"Results\", [])\n for result in results:\n parts = result.split(\",\")\n if len(parts) > 4:\n subdomain = parts[4].strip()\n if subdomain and subdomain.endswith(f\".{query}\"):\n subdomains_set.add(subdomain)\n return subdomains_set\n" }, { "path": "bbot/modules/builtwith.py", "content": "############################################################\n# #\n# #\n# [-] Processing BuiltWith Domains Output #\n# #\n# [-] 2022.08.19 #\n# V05 #\n# Black Lantern Security (BLSOPS) #\n# #\n# #\n############################################################\n\nfrom bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass builtwith(subdomain_enum_apikey):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"affiliates\", \"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query Builtwith.com for subdomains\",\n \"created_date\": \"2022-08-23\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\", \"redirects\": True}\n options_desc = {\"api_key\": \"Builtwith API key\", \"redirects\": \"Also look up inbound and outbound redirects\"}\n base_url = \"https://api.builtwith.com\"\n\n async def handle_event(self, event):\n query = self.make_query(event)\n # domains\n subdomains = await self.query(query, parse_fn=self.parse_domains, request_fn=self.request_domains)\n if subdomains:\n for s in subdomains:\n # `s` is a hostname string; compare against the event's data, not the Event object itself.\n if s != event.data:\n await self.emit_event(\n s,\n \"DNS_NAME\",\n parent=event,\n context=f'{{module}} queried the BuiltWith API for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n # redirects\n if self.config.get(\"redirects\", True):\n redirects = await self.query(query, parse_fn=self.parse_redirects, request_fn=self.request_redirects)\n if redirects:\n for r in redirects:\n # `r` is a hostname string; compare against the event's data, not the Event object itself.\n if r != event.data:\n await self.emit_event(\n r,\n \"DNS_NAME\",\n parent=event,\n tags=[\"affiliate\"],\n context=f'{{module}} queried the BuiltWith redirect API for \"{query}\" and found redirect to {{event.type}}: {{event.data}}',\n )\n\n async def request_domains(self, query):\n url = f\"{self.base_url}/v20/api.json?KEY={{api_key}}&LOOKUP={query}&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes\"\n return await self.api_request(url)\n\n async def request_redirects(self, query):\n url = f\"{self.base_url}/redirect1/api.json?KEY={{api_key}}&LOOKUP={query}\"\n return await self.api_request(url)\n\n async def parse_domains(self, r, query):\n \"\"\"\n This method returns a set of subdomains.\n Each subdomain is an \"FQDN\" that was reported in the \"Detailed Technology Profile\" page on builtwith.com\n\n Parameters\n ----------\n r (requests Response): The raw requests response from the API\n query (string): The query used against the API\n \"\"\"\n results_set = set()\n json = r.json()\n if json and isinstance(json, dict):\n results = json.get(\"Results\", [])\n if results:\n for result in results:\n for chunk in result.get(\"Result\", {}).get(\"Paths\", []):\n domain = chunk.get(\"Domain\", \"\")\n subdomain = chunk.get(\"SubDomain\", \"\")\n if domain:\n if subdomain:\n domain = f\"{subdomain}.{domain}\"\n results_set.add(domain)\n else:\n errors = json.get(\"Errors\", [{}])\n if errors:\n error = errors[0].get(\"Message\", \"Unknown Error\")\n self.verbose(f\"No results for {query}: {error}\")\n return results_set\n\n async def parse_redirects(self, r, query):\n \"\"\"\n This method creates a set.\n Each entry in the set is either an Inbound or Outbound Redirect reported in the \"Redirect Profile\" page on builtwith.com\n\n Parameters\n ----------\n r (requests Response): The raw requests response from the API\n query (string): The query used against the API\n\n Returns\n -------\n results (set)\n \"\"\"\n results = set()\n json = r.json()\n if json and isinstance(json, dict):\n inbound = json.get(\"Inbound\", [])\n outbound = json.get(\"Outbound\", [])\n if inbound:\n for i in inbound:\n domain = i.get(\"Domain\", \"\")\n if domain:\n results.add(domain)\n if outbound:\n for o in outbound:\n domain = o.get(\"Domain\", \"\")\n if domain:\n results.add(domain)\n if not results:\n error = json.get(\"error\", \"\")\n if error:\n self.warning(f\"No results for {query}: {error}\")\n return results\n" }, { "path": "bbot/modules/bypass403.py", "content": "from bbot.errors import HttpCompareError\nfrom bbot.modules.base import BaseModule\n\n\"\"\"\nPort of https://github.com/iamj0ker/bypass-403/ and https://portswigger.net/bappstore/444407b96d9c4de0adb7aed89e826122\n\"\"\"\n\n# ([string]method,[string]path,[dictionary]header,[bool]strip trailing slash)\nsignatures = [\n (\"GET\", \"{scheme}://{netloc}/%2e/{path}\", None, False),\n (\"GET\", \"{scheme}://{netloc}/{path}\", {\"X-Original-URL\": \"{path}\"}, False),\n (\"GET\", \"{scheme}://{netloc}/{path}\", {\"X-Forwarded-For\": \"http://127.0.0.1\"}, False),\n (\"GET\", \"{scheme}://{netloc}/{path}\", {\"X-rewrite-url\": \"nonsense\"}, False),\n (\"GET\", \"{scheme}://{netloc}/{path}.html\", None, False),\n (\"GET\", \"{scheme}://{netloc}/{path}#\", None, False),\n (\"POST\", \"{scheme}://{netloc}/{path}\", {\"Content-Length\": \"0\"}, False),\n (\"GET\", \"{scheme}://{netloc}/{path}.php\", None, False),\n (\"GET\", \"{scheme}://{netloc}/{path}.json\", None, False),\n (\"TRACE\", \"{scheme}://{netloc}/{path}\", None, True),\n (\"GET\", \"{scheme}://{netloc}/(S(X))/{path}\", None, True), # ASPNET COOKIELESS URLS\n (\"GET\", \"{scheme}://{netloc}/(S(X))/../(S(X))/{path}\", None, True), # ASPNET COOKIELESS URLS\n]\n\n\nquery_payloads = [\n \"%09\",\n \"%20\",\n \"%23\",\n \"%2e\",\n \"%2f\",\n \".\",\n \"?\",\n \";\",\n \"..;\",\n \";%09\",\n \";%09..\",\n \";%09..;\",\n \";%2f..\",\n \"*\",\n \"/*\",\n \"..;/\",\n \";/\",\n \"/..;/\",\n \"/;/\",\n \"/./\",\n \"//\",\n \"/.\",\n \"/?anything\",\n]\n\nheader_payloads = {\n \"Client-IP\": \"127.0.0.1\",\n \"X-Real-Ip\": \"127.0.0.1\",\n \"Redirect\": \"127.0.0.1\",\n \"Referer\": \"127.0.0.1\",\n \"X-Client-IP\": \"127.0.0.1\",\n \"X-Custom-IP-Authorization\": \"127.0.0.1\",\n \"X-Forwarded-By\": \"127.0.0.1\",\n \"X-Forwarded-For\": \"127.0.0.1\",\n \"X-Forwarded-Host\": \"127.0.0.1\",\n \"X-Forwarded-Port\": \"80\",\n \"X-True-IP\": \"127.0.0.1\",\n \"X-Host\": \"127.0.0.1\",\n}\n\n# This is planned to be replaced in the future: https://github.com/blacklanternsecurity/bbot/issues/1068\nwaf_strings = [\"The requested URL was rejected\"]\n\nfor qp in query_payloads:\n signatures.append((\"GET\", \"{scheme}://{netloc}/{path}%s\" % qp, None, True))\n if \"?\" not in qp: # we only want to use \"?\" after the path\n signatures.append((\"GET\", \"{scheme}://{netloc}/%s/{path}\" % qp, None, True))\n\nfor hp_key in header_payloads.keys():\n signatures.append((\"GET\", \"{scheme}://{netloc}/{path}\", {hp_key: header_payloads[hp_key]}, False))\n\n\nclass bypass403(BaseModule):\n watched_events = [\"URL\"]\n produced_events = [\"FINDING\"]\n flags = [\"active\", \"aggressive\", \"web-thorough\"]\n meta = {\"description\": \"Check 403 pages for common bypasses\", \"created_date\": \"2022-07-05\", \"author\": \"@liquidsec\"}\n in_scope_only = True\n\n async def do_checks(self, compare_helper, event, collapse_threshold):\n results = set()\n error_count = 0\n\n for sig in signatures:\n if error_count > 3:\n self.warning(f\"Received too many errors for URL {event.data} aborting bypass403\")\n return None\n\n sig = self.format_signature(sig, event)\n if sig[2] is not None:\n headers = dict(sig[2])\n else:\n headers = None\n try:\n match, reasons, reflection, subject_response = await compare_helper.compare(\n sig[1], headers=headers, method=sig[0], allow_redirects=True\n )\n except HttpCompareError as e:\n error_count += 1\n self.debug(e)\n continue\n\n # In some cases WAFs will respond with a 200 code which causes a false positive\n if subject_response is not None:\n for ws in waf_strings:\n if ws in subject_response.text:\n self.debug(\"Rejecting result based on presence of WAF string\")\n return\n\n if match is False:\n if str(subject_response.status_code)[0] != \"4\":\n if sig[2]:\n added_header_tuple = next(iter(sig[2].items()))\n reported_signature = f\"Added Header: {added_header_tuple[0]}: {added_header_tuple[1]}\"\n else:\n reported_signature = f\"Modified URL: {sig[0]} {sig[1]}\"\n description = f\"403 Bypass Reasons: [{','.join(reasons)}] Sig: [{reported_signature}]\"\n results.add(description)\n if len(results) > collapse_threshold:\n return results\n else:\n self.debug(f\"Status code changed to {str(subject_response.status_code)}, ignoring\")\n return results\n\n async def handle_event(self, event):\n try:\n compare_helper = self.helpers.http_compare(event.data, allow_redirects=True)\n except HttpCompareError as e:\n self.debug(e)\n return\n\n collapse_threshold = 6\n results = await self.do_checks(compare_helper, event, collapse_threshold)\n if results is None:\n return\n if len(results) > collapse_threshold:\n await self.emit_event(\n {\n \"description\": f\"403 Bypass MULTIPLE SIGNATURES (exceeded threshold {str(collapse_threshold)})\",\n \"host\": str(event.host),\n \"url\": event.data,\n },\n \"FINDING\",\n parent=event,\n context=f\"{{module}} discovered multiple potential 403 bypasses ({{event.type}}) for {event.data}\",\n )\n else:\n for description in results:\n await self.emit_event(\n {\"description\": description, \"host\": str(event.host), \"url\": event.data},\n \"FINDING\",\n parent=event,\n context=f\"{{module}} discovered potential 403 bypass ({{event.type}}) for {event.data}\",\n )\n\n # When a WAF-check helper is available in the future, we will convert to HTTP_RESPONSE and check for the WAF string here.\n async def filter_event(self, event):\n if (\"status-403\" in event.tags) or (\"status-401\" in event.tags):\n return True\n return False\n\n def format_signature(self, sig, event):\n if sig[3] is True:\n cleaned_path = event.parsed_url.path.strip(\"/\")\n else:\n cleaned_path = event.parsed_url.path.lstrip(\"/\")\n kwargs = {\"scheme\": event.parsed_url.scheme, \"netloc\": event.parsed_url.netloc, \"path\": cleaned_path}\n formatted_url = sig[1].format(**kwargs)\n if sig[2] is not None:\n formatted_headers = {k: v.format(**kwargs) for k, v in sig[2].items()}\n else:\n formatted_headers = None\n return (sig[0], formatted_url, formatted_headers)\n" }, { "path": "bbot/modules/c99.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass c99(subdomain_enum_apikey):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query the C99 API for subdomains\",\n \"created_date\": \"2022-07-08\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"c99.nl API key\"}\n\n base_url = \"https://api.c99.nl\"\n ping_url = f\"{base_url}/randomnumber?key={{api_key}}&between=1,100&json\"\n\n async def ping(self):\n url = f\"{self.base_url}/randomnumber?key={{api_key}}&between=1,100&json\"\n response = await self.api_request(url, retry_on_http_429=False)\n assert response.json()[\"success\"] is True, getattr(response, \"text\", \"no response from server\")\n\n async def request_url(self, query):\n url = f\"{self.base_url}/subdomainfinder?key={{api_key}}&domain={self.helpers.quote(query)}&json\"\n return await self.api_request(url)\n\n async def parse_results(self, r, query):\n results = set()\n j = r.json()\n if isinstance(j, dict):\n subdomains = j.get(\"subdomains\", [])\n if subdomains:\n for s in subdomains:\n subdomain = s.get(\"subdomain\", \"\")\n if subdomain:\n results.add(subdomain)\n return results\n" }, { "path": "bbot/modules/censys_dns.py", "content": "from bbot.modules.templates.censys import censys\n\n\nclass censys_dns(censys):\n \"\"\"\n Query the Censys certificates API for subdomains.\n Thanks to https://github.com/owasp-amass/amass/blob/master/resources/scripts/cert/censys.ads\n \"\"\"\n\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query the Censys API for subdomains\",\n \"created_date\": \"2022-08-04\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\", \"max_pages\": 5}\n options_desc = {\n \"api_key\": \"Censys.io API Key in the format of 'key:secret'\",\n \"max_pages\": \"Maximum number of pages to fetch (100 results per page)\",\n }\n\n async def setup(self):\n self.max_pages = self.config.get(\"max_pages\", 5)\n return await super().setup()\n\n async def query(self, query):\n results = set()\n cursor = \"\"\n for i in range(self.max_pages):\n url = f\"{self.base_url}/v2/certificates/search\"\n json_data = {\n \"q\": f\"names: {query}\",\n \"per_page\": 100,\n }\n if cursor:\n json_data.update({\"cursor\": cursor})\n resp = await self.api_request(\n url,\n method=\"POST\",\n json=json_data,\n )\n\n if resp is None:\n break\n\n try:\n d = resp.json()\n except Exception as e:\n self.warning(f\"Failed to parse JSON from {url} (response: {resp}): {e}\")\n\n if resp.status_code < 200 or resp.status_code >= 400:\n if isinstance(d, dict):\n error = d.get(\"error\", \"\")\n if error:\n self.warning(error)\n self.verbose(f'Non-200 Status code: {resp.status_code} for query \"{query}\", page #{i + 1}')\n self.debug(f\"Response: {resp.text}\")\n break\n else:\n if d is None:\n break\n elif not isinstance(d, dict):\n break\n status = d.get(\"status\", \"\").lower()\n result = d.get(\"result\", {})\n hits = result.get(\"hits\", [])\n if status != \"ok\" or not hits:\n break\n\n for h in hits:\n names = h.get(\"names\", [])\n for n in names:\n results.add(n.strip(\".*\").lower())\n\n cursor = result.get(\"links\", {}).get(\"next\", \"\")\n if not cursor:\n break\n\n return results\n" }, { "path": "bbot/modules/censys_ip.py", "content": "from bbot.modules.templates.censys import censys\n\n\nclass censys_ip(censys):\n \"\"\"\n Query the Censys /v2/hosts/{ip} endpoint for associated hostnames, IPs, and URLs.\n \"\"\"\n\n watched_events = [\"IP_ADDRESS\"]\n produced_events = [\n \"IP_ADDRESS\",\n \"DNS_NAME\",\n \"URL_UNVERIFIED\",\n \"OPEN_TCP_PORT\",\n \"OPEN_UDP_PORT\",\n \"TECHNOLOGY\",\n \"PROTOCOL\",\n ]\n flags = [\"passive\", \"safe\"]\n meta = {\n \"description\": \"Query the Censys API for hosts by IP address\",\n \"created_date\": \"2026-01-26\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\", \"dns_names_limit\": 100, \"in_scope_only\": True}\n options_desc = {\n \"api_key\": \"Censys.io API Key in the format of 'key:secret'\",\n \"dns_names_limit\": \"Maximum number of DNS names to extract from dns.names (default 100)\",\n \"in_scope_only\": \"Only query in-scope IPs. If False, will query up to distance 1.\",\n }\n scope_distance_modifier = 1\n\n async def setup(self):\n self.dns_names_limit = self.config.get(\"dns_names_limit\", 100)\n self.warning(\n \"This module may consume a lot of API queries. Unless you specifically want to query on each individual IP, we recommend using the censys_dns module instead.\"\n )\n return await super().setup()\n\n async def filter_event(self, event):\n in_scope_only = self.config.get(\"in_scope_only\", True)\n max_scope_distance = 0 if in_scope_only else (self.scan.scope_search_distance + 1)\n if event.scope_distance > max_scope_distance:\n return False, \"event is not in scope\"\n return True\n\n async def handle_event(self, event):\n ip = str(event.host)\n url = f\"{self.base_url}/v2/hosts/{ip}\"\n\n resp = await self.api_request(url)\n if resp is None:\n self.debug(f\"No response for {ip}\")\n return\n\n if resp.status_code == 404:\n self.debug(f\"No data found for {ip}\")\n return\n\n if resp.status_code != 200:\n self.verbose(f\"Non-200 status code ({resp.status_code}) for {ip}\")\n return\n\n try:\n data = resp.json()\n except Exception as e:\n self.warning(f\"Failed to parse JSON response for {ip}: {e}\")\n return\n\n result = data.get(\"result\", {})\n if not result:\n return\n\n # Track what we've already emitted to avoid duplicates\n seen = set()\n\n # Extract data from services\n for service in result.get(\"services\", []):\n port = service.get(\"port\")\n transport = service.get(\"transport_protocol\", \"TCP\").upper()\n\n # Emit OPEN_TCP_PORT or OPEN_UDP_PORT for services with a port\n # QUIC uses UDP as transport, so treat it as UDP\n if port and (port, transport) not in seen:\n seen.add((port, transport))\n if transport in (\"UDP\", \"QUIC\"):\n event_type = \"OPEN_UDP_PORT\"\n else:\n event_type = \"OPEN_TCP_PORT\"\n await self.emit_event(\n self.helpers.make_netloc(ip, port),\n event_type,\n parent=event,\n context=\"{module} found open port on {event.parent.data}\",\n )\n\n # Emit PROTOCOL for non-HTTP services\n # Use extended_service_name (more specific) falling back to service_name\n # Also check transport_protocol for protocols like QUIC\n service_name = service.get(\"extended_service_name\") or service.get(\"service_name\", \"\")\n # If service_name is UNKNOWN but transport_protocol is meaningful, use that\n if service_name.upper() == \"UNKNOWN\" and transport and transport not in (\"TCP\", \"UDP\"):\n service_name = transport\n if service_name and service_name.upper() not in (\"HTTP\", \"HTTPS\", \"UNKNOWN\"):\n protocol_key = (\"protocol\", service_name.upper(), port)\n if protocol_key not in seen:\n seen.add(protocol_key)\n protocol_data = {\"host\": str(event.host), \"protocol\": service_name}\n if port:\n protocol_data[\"port\"] = port\n await self.emit_event(\n protocol_data,\n \"PROTOCOL\",\n parent=event,\n context=\"{module} found {event.type}: {event.data[protocol]} on {event.parent.data}\",\n )\n\n # Extract URLs from HTTP services\n http_data = service.get(\"http\", {})\n request = http_data.get(\"request\", {})\n uri = request.get(\"uri\")\n if uri and uri not in seen:\n seen.add(uri)\n await self.emit_event(\n uri,\n \"URL_UNVERIFIED\",\n parent=event,\n context=\"{module} found {event.data} in HTTP service of {event.parent.data}\",\n )\n\n # Extract TLS certificate data\n tls_data = service.get(\"tls\", {})\n certs = tls_data.get(\"certificates\", {})\n leaf_data = certs.get(\"leaf_data\", {})\n\n # Extract names from leaf_data.names\n for name in leaf_data.get(\"names\", []):\n await self._emit_host(name, event, seen, \"TLS certificate\")\n\n # Extract common_name from leaf_data.subject\n subject = leaf_data.get(\"subject\", {})\n for cn in subject.get(\"common_name\", []):\n await self._emit_host(cn, event, seen, \"TLS certificate subject\")\n\n # Extract software/technologies\n for software in service.get(\"software\", []):\n product = software.get(\"uniform_resource_identifier\", software.get(\"product\", \"\"))\n if product:\n await self.emit_event(\n {\"technology\": product, \"host\": str(event.host)},\n \"TECHNOLOGY\",\n parent=event,\n context=\"{module} found {event.type}: {event.data[technology]} on {event.parent.data}\",\n )\n\n # Extract dns.names (limit to configured max)\n dns_data = result.get(\"dns\", {})\n dns_names = dns_data.get(\"names\", [])\n for name in dns_names[: self.dns_names_limit]:\n await self._emit_host(name, event, seen, \"reverse DNS\")\n\n async def _emit_host(self, host, event, seen, source):\n \"\"\"Emit IP_ADDRESS or DNS_NAME for a host value.\"\"\"\n # Validate and emit as DNS_NAME\n try:\n validated = self.helpers.validators.validate_host(host)\n except ValueError as e:\n self.debug(f\"Error validating host {host} in {source}: {e}\")\n if validated and validated not in seen:\n seen.add(validated)\n await self.emit_event(\n validated,\n \"DNS_NAME\",\n parent=event,\n context=f\"{{module}} found {{event.data}} in {source} of {{event.parent.data}}\",\n )\n" }, { "path": "bbot/modules/certspotter.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass certspotter(subdomain_enum):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query Certspotter's API for subdomains\",\n \"created_date\": \"2022-07-28\",\n \"author\": \"@TheTechromancer\",\n }\n\n base_url = \"https://api.certspotter.com/v1\"\n\n def request_url(self, query):\n url = f\"{self.base_url}/issuances?domain={self.helpers.quote(query)}&include_subdomains=true&expand=dns_names\"\n return self.api_request(url)\n\n async def parse_results(self, r, query):\n results = set()\n json = r.json()\n if json:\n for r in json:\n for dns_name in r.get(\"dns_names\", []):\n results.add(dns_name.lstrip(\".*\").rstrip(\".\"))\n return results\n" }, { "path": "bbot/modules/chaos.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass chaos(subdomain_enum_apikey):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query ProjectDiscovery's Chaos API for subdomains\",\n \"created_date\": \"2022-08-14\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"Chaos API key\"}\n\n base_url = \"https://dns.projectdiscovery.io/dns\"\n ping_url = f\"{base_url}/example.com\"\n\n def prepare_api_request(self, url, kwargs):\n kwargs[\"headers\"][\"Authorization\"] = self.api_key\n return url, kwargs\n\n async def request_url(self, query):\n _, domain = self.helpers.split_domain(query)\n url = f\"{self.base_url}/{domain}/subdomains\"\n return await self.api_request(url)\n\n async def parse_results(self, r, query):\n results = set()\n j = r.json()\n subdomains_set = set()\n if isinstance(j, dict):\n domain = j.get(\"domain\", \"\")\n if domain:\n subdomains = j.get(\"subdomains\", [])\n for s in subdomains:\n s = s.lower().strip(\".*\")\n subdomains_set.add(s)\n for s in subdomains_set:\n full_subdomain = f\"{s}.{domain}\"\n if full_subdomain and full_subdomain.endswith(f\".{query}\"):\n results.add(full_subdomain)\n return results\n" }, { "path": "bbot/modules/code_repository.py", "content": "import re\nfrom bbot.modules.base import BaseModule\n\n\nclass code_repository(BaseModule):\n watched_events = [\"URL_UNVERIFIED\"]\n produced_events = [\"CODE_REPOSITORY\"]\n meta = {\n \"description\": \"Look for code repository links in webpages\",\n \"created_date\": \"2024-05-15\",\n \"author\": \"@domwhewell-sage\",\n }\n flags = [\"passive\", \"safe\", \"code-enum\"]\n\n # platform name : (regex, case_sensitive)\n code_repositories = {\n \"git\": [\n (r\"github.com/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+\", False),\n (r\"gitlab.(?:com|org)/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+\", False),\n ],\n \"docker\": (r\"hub.docker.com/r/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+\", False),\n \"postman\": (r\"www.postman.com/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+\", False),\n }\n\n scope_distance_modifier = 1\n\n async def setup(self):\n self.compiled_regexes = {}\n for k, v in self.code_repositories.items():\n if isinstance(v, list):\n self.compiled_regexes[k] = [(re.compile(pattern), c) for pattern, c in v]\n else:\n pattern, c = v\n self.compiled_regexes[k] = (re.compile(pattern), c)\n return True\n\n async def handle_event(self, event):\n for platform, regexes in self.compiled_regexes.items():\n if not isinstance(regexes, list):\n regexes = [regexes]\n for regex, case_sensitive in regexes:\n for match in regex.finditer(event.data):\n url = match.group()\n if not case_sensitive:\n url = url.lower()\n url = f\"https://{url}\"\n repo_event = self.make_event(\n {\"url\": url},\n \"CODE_REPOSITORY\",\n tags=platform,\n parent=event,\n )\n await self.emit_event(\n repo_event,\n context=f\"{{module}} detected {platform} {{event.type}} at {url}\",\n )\n" }, { "path": "bbot/modules/credshed.py", "content": "from contextlib import suppress\n\nfrom bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass credshed(subdomain_enum):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"PASSWORD\", \"HASHED_PASSWORD\", \"USERNAME\", \"EMAIL_ADDRESS\"]\n flags = [\"passive\", \"safe\"]\n meta = {\n \"description\": \"Send queries to your own credshed server to check for known credentials of your targets\",\n \"created_date\": \"2023-10-12\",\n \"author\": \"@SpamFaux\",\n \"auth_required\": True,\n }\n options = {\"username\": \"\", \"password\": \"\", \"credshed_url\": \"\"}\n options_desc = {\n \"username\": \"Credshed username\",\n \"password\": \"Credshed password\",\n \"credshed_url\": \"URL of credshed server\",\n }\n target_only = True\n\n async def setup(self):\n self.base_url = self.config.get(\"credshed_url\", \"\").rstrip(\"/\")\n self.username = self.config.get(\"username\", \"\")\n self.password = self.config.get(\"password\", \"\")\n\n # soft-fail if we don't have the necessary information to make queries\n if not (self.base_url and self.username and self.password):\n return None, \"Must set username, password, and credshed_url\"\n\n auth_setup = await self.helpers.request(\n f\"{self.base_url}/api/auth\", method=\"POST\", json={\"username\": self.username, \"password\": self.password}\n )\n self.auth_token = \"\"\n with suppress(Exception):\n self.auth_token = auth_setup.json().get(\"access_token\", \"\")\n # hard-fail if we didn't get an access token\n if not self.auth_token:\n return False, f\"Failed to retrieve credshed auth token from url: {self.base_url}\"\n\n return await super().setup()\n\n async def handle_event(self, event):\n query = self.make_query(event)\n cs_query = await self.helpers.request(\n f\"{self.base_url}/api/search\",\n method=\"POST\",\n cookies={\"access_token_cookie\": self.auth_token},\n json={\"query\": query},\n )\n\n if cs_query is not None and cs_query.status_code != 200:\n self.warning(\n f\"Error retrieving results from {self.base_url} (status code {cs_query.status_code}): {cs_query.text}\"\n )\n\n json_result = {}\n with suppress(Exception):\n json_result = cs_query.json()\n\n if not json_result:\n return\n\n accounts = json_result.get(\"accounts\", [])\n\n for i in accounts:\n email = i.get(\"e\", \"\")\n pw = i.get(\"p\", \"\")\n hashes = i.get(\"h\", [])\n user = i.get(\"u\", \"\")\n src = i.get(\"s\", [])\n src = [src[0] if src else \"\"]\n\n tags = []\n if src:\n tags = [f\"credshed-source-{src}\"]\n\n email_event = self.make_event(email, \"EMAIL_ADDRESS\", parent=event, tags=tags)\n if email_event is not None:\n await self.emit_event(\n email_event, context=f'{{module}} searched for \"{query}\" and found {{event.type}}: {{event.data}}'\n )\n if user:\n await self.emit_event(\n f\"{email}:{user}\",\n \"USERNAME\",\n parent=email_event,\n tags=tags,\n context=f\"{{module}} found {email} with {{event.type}}: {{event.data}}\",\n )\n if pw:\n await self.emit_event(\n f\"{email}:{pw}\",\n \"PASSWORD\",\n parent=email_event,\n tags=tags,\n context=f\"{{module}} found {email} with {{event.type}}: {{event.data}}\",\n )\n for h_pw in hashes:\n if h_pw:\n await self.emit_event(\n f\"{email}:{h_pw}\",\n \"HASHED_PASSWORD\",\n parent=email_event,\n tags=tags,\n context=f\"{{module}} found {email} with {{event.type}}: {{event.data}}\",\n )\n" }, { "path": "bbot/modules/crt.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass crt(subdomain_enum):\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n meta = {\n \"description\": \"Query crt.sh (certificate transparency) for subdomains\",\n \"created_date\": \"2022-05-13\",\n \"author\": \"@TheTechromancer\",\n }\n\n base_url = \"https://crt.sh\"\n reject_wildcards = False\n\n async def setup(self):\n self.cert_ids = set()\n return await super().setup()\n\n async def request_url(self, query):\n params = {\"q\": f\"%.{query}\", \"output\": \"json\"}\n url = self.helpers.add_get_params(self.base_url, params).geturl()\n return await self.api_request(url, timeout=self.http_timeout + 30)\n\n async def parse_results(self, r, query):\n results = set()\n j = r.json()\n for cert_info in j:\n if not type(cert_info) == dict:\n continue\n cert_id = cert_info.get(\"id\")\n if cert_id:\n if hash(cert_id) not in self.cert_ids:\n self.cert_ids.add(hash(cert_id))\n domain = cert_info.get(\"name_value\")\n if domain:\n for d in domain.splitlines():\n results.add(d.lower())\n return results\n" }, { "path": "bbot/modules/crt_db.py", "content": "import time\nimport asyncpg\n\nfrom bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass crt_db(subdomain_enum):\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n meta = {\n \"description\": \"Query crt.sh (certificate transparency) for subdomains via PostgreSQL\",\n \"created_date\": \"2025-03-27\",\n \"author\": \"@TheTechromancer\",\n }\n\n deps_pip = [\"asyncpg\"]\n\n db_host = \"crt.sh\"\n db_port = 5432\n db_user = \"guest\"\n db_name = \"certwatch\"\n reject_wildcards = False\n\n async def setup(self):\n self.db_conn = None\n return await super().setup()\n\n async def request_url(self, query):\n if not self.db_conn:\n self.db_conn = await asyncpg.connect(\n host=self.db_host,\n port=self.db_port,\n user=self.db_user,\n database=self.db_name,\n statement_cache_size=0, # Disable automatic statement preparation\n )\n\n sql = \"\"\"\n WITH ci AS (\n SELECT array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES\n FROM (\n SELECT DISTINCT cai.CERTIFICATE, cai.NAME_VALUE\n FROM certificate_and_identities cai\n WHERE plainto_tsquery('certwatch', $1) @@ identities(cai.CERTIFICATE)\n AND cai.NAME_VALUE ILIKE ('%.' || $1)\n LIMIT 50000\n ) sub\n GROUP BY sub.CERTIFICATE\n )\n SELECT DISTINCT unnest(NAME_VALUES) as name_value FROM ci;\n \"\"\"\n start = time.time()\n results = await self.db_conn.fetch(sql, query)\n end = time.time()\n self.verbose(f\"SQL query executed in: {end - start} seconds with {len(results):,} results\")\n return results\n\n async def parse_results(self, results, query):\n domains = set()\n for row in results:\n domain = row[\"name_value\"]\n if domain:\n for d in domain.splitlines():\n domains.add(d.lower())\n return domains\n\n async def cleanup(self):\n if self.db_conn:\n await self.db_conn.close()\n" }, { "path": "bbot/modules/deadly/legba.py", "content": "import json\nfrom pathlib import Path\nfrom bbot.errors import WordlistError\nfrom bbot.modules.base import BaseModule\n\n# key: value: \n# List with `legba -L`\nPROTOCOL_LEGBA_PLUGIN_MAP = {\n \"postgresql\": \"pgsql\",\n}\n\n\n# Maps common protocol names to Legba protocol plugin names\ndef map_protocol_to_legba_plugin_name(common_protocol_name: str) -> str:\n return PROTOCOL_LEGBA_PLUGIN_MAP.get(common_protocol_name, common_protocol_name)\n\n\nclass legba(BaseModule):\n watched_events = [\"PROTOCOL\"]\n produced_events = [\"FINDING\"]\n flags = [\"active\", \"aggressive\", \"deadly\"]\n per_hostport_only = True\n meta = {\n \"description\": \"Credential bruteforcing supporting various services.\",\n \"created_date\": \"2025-07-18\",\n \"author\": \"@christianfl, @fuzikowski\",\n }\n _module_threads = 25\n scope_distance_modifier = None\n\n options = {\n \"ssh_wordlist\": \"https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/ssh-betterdefaultpasslist.txt\",\n \"ftp_wordlist\": \"https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/ftp-betterdefaultpasslist.txt\",\n \"telnet_wordlist\": \"https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/telnet-betterdefaultpasslist.txt\",\n \"vnc_wordlist\": \"https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/vnc-betterdefaultpasslist.txt\",\n \"mssql_wordlist\": \"https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/mssql-betterdefaultpasslist.txt\",\n \"mysql_wordlist\": \"https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/mysql-betterdefaultpasslist.txt\",\n \"postgresql_wordlist\": \"https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/postgres-betterdefaultpasslist.txt\",\n \"concurrency\": 3,\n \"rate_limit\": 3,\n \"version\": \"1.1.1\",\n }\n\n options_desc = {\n \"ssh_wordlist\": \"Wordlist URL for SSH combined username:password wordlist, newline separated\",\n \"ftp_wordlist\": \"Wordlist URL for FTP combined username:password wordlist, newline separated\",\n \"telnet_wordlist\": \"Wordlist URL for TELNET combined username:password wordlist, newline separated\",\n \"vnc_wordlist\": \"Wordlist URL for VNC password wordlist, newline separated\",\n \"mssql_wordlist\": \"Wordlist URL for MSSQL combined username:password wordlist, newline separated\",\n \"mysql_wordlist\": \"Wordlist URL for MySQL combined username:password wordlist, newline separated\",\n \"postgresql_wordlist\": \"Wordlist URL for PostgreSQL combined username:password wordlist, newline separated\",\n \"concurrency\": \"Number of concurrent workers, gets overridden for SSH\",\n \"rate_limit\": \"Limit the number of requests per second, gets overridden for SSH\",\n \"version\": \"legba version\",\n }\n\n deps_ansible = [\n {\n \"name\": \"Download legba\",\n \"unarchive\": {\n \"src\": \"https://github.com/evilsocket/legba/releases/download/#{BBOT_MODULES_LEGBA_VERSION}/legba-#{BBOT_MODULES_LEGBA_VERSION}-#{BBOT_OS}-#{BBOT_CPU_ARCH_RUST}.tar.gz\",\n \"dest\": \"#{BBOT_TEMP}\",\n \"include\": \"legba-#{BBOT_MODULES_LEGBA_VERSION}-#{BBOT_OS}-#{BBOT_CPU_ARCH_RUST}/legba\",\n \"remote_src\": True,\n \"mode\": \"u+x,g+x,o+x\",\n },\n }\n ]\n\n async def setup(self):\n self.output_dir = Path(self.scan.temp_dir / \"legba-output\")\n self.helpers.mkdir(self.output_dir)\n if not \"fingerprintx\" in self.scan.modules:\n self.warning(\"Enabling 'fingerprintx' module is recommended for discovery of PROTOCOL events\")\n\n return True\n\n async def filter_event(self, event):\n handled_protocols = [\"ssh\", \"ftp\", \"telnet\", \"vnc\", \"mssql\", \"mysql\", \"postgresql\"]\n\n protocol = event.data[\"protocol\"].lower()\n if not protocol in handled_protocols:\n return False, f\"service {protocol} is currently not supported or can't be bruteforced by Legba\"\n\n return True\n\n async def handle_event(self, event):\n host = str(event.host)\n port = str(event.port)\n protocol = event.data[\"protocol\"].lower()\n\n command_data = await self.construct_command(host, port, protocol)\n\n if not command_data:\n self.warning(f\"Skipping {host}:{port} ({protocol}) due to errors while constructing the command\")\n return\n\n command, output_path = command_data\n\n await self.run_process(command)\n\n async for finding_event in self.parse_output(output_path, event):\n await self.emit_event(finding_event)\n\n async def parse_output(self, output_filepath, event):\n protocol = event.data[\"protocol\"].lower()\n\n try:\n with open(output_filepath) as file:\n for line in file:\n # example line (ssh):\n # {\"found_at\":\"2025-07-18T06:28:08.969812152+01:00\",\"target\":\"localhost:22\",\"plugin\":\"ssh\",\"data\":{\"username\":\"user\",\"password\":\"pass\"},\"partial\":false}\n line = line.strip()\n\n try:\n data = json.loads(line)[\"data\"]\n username = data.get(\"username\", \"\")\n password = data.get(\"password\", \"\")\n\n if username and password:\n message_addition = f\"{username}:{password}\"\n elif username:\n message_addition = username\n elif password:\n message_addition = password\n except Exception as e:\n self.warning(f\"Failed to parse Legba output ({line}), using raw output instead: {e}\")\n message_addition = f\"raw output: {line}\"\n\n yield self.make_event(\n {\n \"severity\": \"CRITICAL\",\n \"confidence\": \"CONFIRMED\",\n \"host\": str(event.host),\n \"port\": str(event.port),\n \"description\": f\"Valid {protocol} credentials found - {message_addition}\",\n },\n \"FINDING\",\n parent=event,\n )\n except FileNotFoundError:\n self.info(\n f\"Could not open Legba output file {output_filepath}. File is missing if no valid credentials could be found\"\n )\n except Exception as e:\n self.warning(f\"Error processing Legba output file {output_filepath}: {e}\")\n else:\n self.helpers.delete_file(output_filepath)\n\n async def construct_command(self, host, port, protocol):\n # -C Combo wordlist delimited by ':'\n # -P Passwordlist\n # --target Target (allowed: host, url, IP address, CIDR, @filename)\n # --output-format Output file format\n # --output Save results to this file\n # -Q Do not report statistics\n #\n # --wait Wait time in milliseconds per login attempt\n # --rate-limit Limit the number of requests per second\n # --concurrency Number of concurrent workers\n\n # Example command to bruteforce SSH:\n #\n # legba ssh -C combolist.txt --target 127.0.0.1:22 --output-format jsonl --output out.txt -Q --wait 4000 --rate-limit 1 --concurrency 1\n\n try:\n wordlist_path = await self.helpers.wordlist(self.config.get(f\"{protocol}_wordlist\"))\n except WordlistError as e:\n self.warning(f\"Error retrieving wordlist for protocol {protocol}: {e}\")\n return None\n except Exception as e:\n self.warning(f\"Unexpected error during wordlist loading for protocol {protocol}: {e}\")\n return None\n\n protocol_plugin_name = map_protocol_to_legba_plugin_name(protocol)\n output_path = Path(self.output_dir) / f\"{host}_{port}.json\"\n\n cmd = [\n \"legba\",\n protocol_plugin_name,\n ]\n\n if protocol == \"vnc\":\n # use only passwords, not combinations\n cmd += [\"-P\"]\n\n else:\n # use combinations\n cmd += [\"-C\"]\n\n # wrap IPv6 addresses in square brackets\n if self.helpers.is_ip(host, version=6):\n host = f\"[{host}]\"\n\n cmd += [\n wordlist_path,\n \"--target\",\n f\"{host}:{port}\",\n \"--output-format\",\n \"jsonl\",\n \"--output\",\n output_path,\n \"-Q\",\n ]\n\n if protocol == \"ssh\":\n # With OpenSSH 9.8, the sshd_config option \"PerSourcePenalties\" was introduced (on by default)\n # The penalty \"authfail\" defaults to 5 seconds, so bruteforcing fast will block access.\n # Legba is not able to check that by itself, so the wait time is set to 5 s, rate limit to 1 and concurrency to 1 with SSH.\n # See https://www.openssh.com/txt/release-9.8\n cmd += [\n \"--wait\",\n \"5000\",\n \"--rate-limit\",\n \"1\",\n \"--concurrency\",\n \"1\",\n ]\n else:\n cmd += [\"--rate-limit\", self.config.rate_limit, \"--concurrency\", self.config.concurrency]\n\n return cmd, output_path\n" }, { "path": "bbot/modules/dehashed.py", "content": "from contextlib import suppress\n\nfrom bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass dehashed(subdomain_enum):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"PASSWORD\", \"HASHED_PASSWORD\", \"USERNAME\", \"EMAIL_ADDRESS\"]\n flags = [\"passive\", \"safe\", \"email-enum\"]\n meta = {\n \"description\": \"Execute queries against dehashed.com for exposed credentials\",\n \"created_date\": \"2023-10-12\",\n \"author\": \"@SpamFaux\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"DeHashed API Key\"}\n target_only = True\n\n base_url = \"https://api.dehashed.com/v2/search\"\n\n async def setup(self):\n self.api_key = self.config.get(\"api_key\", \"\")\n self.headers = {\n \"Accept\": \"application/json\",\n \"Content-Type\": \"application/json\",\n \"Dehashed-Api-Key\": self.api_key,\n }\n\n # soft-fail if we don't have the necessary information to make queries\n if not self.api_key:\n return None, \"No API key set\"\n\n return await super().setup()\n\n async def handle_event(self, event):\n query = self.make_query(event)\n async for entries in self.query(query):\n for entry in entries:\n # we have to clean up the email field because dehashed does a poor job of it\n emails = []\n for email in entry.get(\"email\", []):\n email_str = email.replace(\"\\\\\", \"\")\n found_emails = list(await self.helpers.re.extract_emails(email_str))\n if not found_emails:\n self.debug(f\"Invalid email from dehashed.com: {email_str}\")\n continue\n emails += found_emails\n\n users = entry.get(\"username\", [])\n pws = entry.get(\"password\", [])\n h_pws = entry.get(\"hashed_password\", [])\n db_name = entry.get(\"database_name\", \"\")\n\n tags = []\n if db_name:\n tags = [f\"db-{db_name}\"]\n for email in emails:\n email_event = self.make_event(email, \"EMAIL_ADDRESS\", parent=event, tags=tags)\n if email_event is not None:\n await self.emit_event(\n email_event,\n context=f'{{module}} searched API for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n for user in users:\n await self.emit_event(\n f\"{email}:{user}\",\n \"USERNAME\",\n parent=email_event,\n tags=tags,\n context=f\"{{module}} found {email} with {{event.type}}: {{event.data}}\",\n )\n for pw in pws:\n await self.emit_event(\n f\"{email}:{pw}\",\n \"PASSWORD\",\n parent=email_event,\n tags=tags,\n context=f\"{{module}} found {email} with {{event.type}}: {{event.data}}\",\n )\n for h_pw in h_pws:\n await self.emit_event(\n f\"{email}:{h_pw}\",\n \"HASHED_PASSWORD\",\n parent=email_event,\n tags=tags,\n context=f\"{{module}} found {email} with {{event.type}}: {{event.data}}\",\n )\n\n async def query(self, domain):\n url = self.base_url\n json = {\n \"query\": \"\",\n \"page\": 1,\n \"size\": 10000, # The maximum permitted size and pagination.\n }\n json[\"query\"] = f\"domain:{domain}\"\n json[\"page\"] = 1\n max_pages = 1\n agen = self.api_page_iter(url=url, headers=self.headers, _json=False, method=\"POST\", json=json)\n async for result in agen:\n result_json = {}\n with suppress(Exception):\n result_json = result.json()\n total = result_json.get(\"total\", 0)\n entries = result_json.get(\"entries\", [])\n json[\"page\"] += 1\n if result is not None and result.status_code != 200:\n self.warning(\n f\"Error retrieving results from dehashed.com (status code {result.status_code}): {result.text}\"\n )\n elif (json[\"page\"] > max_pages) and (total > (json[\"size\"] * max_pages)):\n self.info(\n f\"{domain} has {total:,} results in Dehashed. The API can only process the first 10,000 results. Please check dehashed.com to get the remaining results.\"\n )\n if entries:\n yield entries\n if not entries or json[\"page\"] > max_pages:\n await agen.aclose()\n break\n" }, { "path": "bbot/modules/digitorus.py", "content": "import re\n\nfrom bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass digitorus(subdomain_enum):\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n meta = {\n \"description\": \"Query certificatedetails.com for subdomains\",\n \"created_date\": \"2023-07-25\",\n \"author\": \"@TheTechromancer\",\n }\n\n base_url = \"https://certificatedetails.com\"\n\n async def request_url(self, query):\n url = f\"{self.base_url}/{self.helpers.quote(query)}\"\n return await self.helpers.request(url)\n\n async def parse_results(self, r, query):\n results = set()\n content = getattr(r, \"text\", \"\")\n extract_regex = re.compile(r\"[\\w.-]+\\.\" + query, re.I)\n if content:\n for match in extract_regex.finditer(content):\n subdomain = match.group().lower()\n if subdomain and subdomain.endswith(f\".{query}\"):\n results.add(subdomain)\n return results\n" }, { "path": "bbot/modules/dnsbimi.py", "content": "# bimi.py\n#\n# Checks for and parses common BIMI DNS TXT records, e.g. default._bimi.target.domain\n#\n# Example TXT record: \"v=BIMI1; l=https://example.com/brand/logo.svg; a=https://example.com/brand/certificate.pem\"\n#\n# BIMI records may contain a link to an SVG format brand authorised image, which may be useful for:\n# 1. Sub-domain or otherwise unknown content hosting locations\n# 2. Brand impersonation\n# 3. May not be formatted/stripped of metadata correctly leading to some (low value probably) information exposure\n#\n# BIMI records may also contain a link to a PEM format X.509 VMC certificate, which may be similarly useful.\n#\n# We simply extract any URL's as URL_UNVERIFIED, no further parsing or download is done by this module in order to remain passive.\n#\n# The domain portion of any URL's is also passively checked and added as appropriate, for additional inspection by other modules.\n#\n# Files may be downloaded by other modules which respond to URL_UNVERIFIED events, if you have configured bbot to do so.\n#\n# NOTE: .svg file extensions are filtered from inclusion by default, modify \"url_extension_blacklist\" appropriately if you want the .svg image to be considered for download.\n#\n# NOTE: use the \"filedownload\" module if you to download .svg and .pem files. .pem will be downloaded by default, .svg will require a customised configuration for that module.\n#\n# The domain portion of any URL_UNVERIFIED's will be extracted by the various internal modules if .svg is not filtered.\n#\n\nfrom bbot.modules.base import BaseModule\nfrom bbot.core.helpers.dns.helpers import service_record\n\nimport re\n\n# Handle \"v=BIMI1; l=; a=;\" == RFC conformant explicit declination to publish, e.g. useful on a sub-domain if you don't want the sub-domain to have a BIMI logo, yet your registered domain does?\n# Handle \"v=BIMI1; l=; a=\" == RFC non-conformant explicit declination to publish\n# Handle \"v=BIMI1; l=;\" == RFC non-conformant explicit declination to publish\n# Handle \"v=BIMI1; l=\" == RFC non-conformant explicit declination to publish\n# Handle \"v=BIMI1;\" == RFC non-conformant explicit declination to publish\n# Handle \"v=BIMI1\" == RFC non-conformant explicit declination to publish\n# Handle \"v=BIMI1;l=https://bimi.entrust.net/example.com/logo.svg;\"\n# Handle \"v=BIMI1; l=https://bimi.entrust.net/example.com/logo.svg;\"\n# Handle \"v=BIMI1;l=https://bimi.entrust.net/example.com/logo.svg;a=https://bimi.entrust.net/example.com/certchain.pem\"\n# Handle \"v=BIMI1; l=https://bimi.entrust.net/example.com/logo.svg;a=https://bimi.entrust.net/example.com/certchain.pem;\"\n_bimi_regex = r\"^v=(?PBIMI1);\\s?(?:l=(?Phttps?://[^;\\s]{1,255})?)?;?(?:\\s?a=(?Phttps://[^;\\s]{1,255})?;?)?$\"\nbimi_regex = re.compile(_bimi_regex, re.I)\n\n\nclass dnsbimi(BaseModule):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"URL_UNVERIFIED\", \"RAW_DNS_RECORD\"]\n flags = [\"subdomain-enum\", \"cloud-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Check DNS_NAME's for BIMI records to find image and certificate hosting URL's\",\n \"author\": \"@colin-stubbs\",\n \"created_date\": \"2024-11-15\",\n }\n options = {\n \"emit_raw_dns_records\": False,\n \"emit_urls\": True,\n \"selectors\": \"default,email,mail,bimi\",\n }\n options_desc = {\n \"emit_raw_dns_records\": \"Emit RAW_DNS_RECORD events\",\n \"emit_urls\": \"Emit URL_UNVERIFIED events\",\n \"selectors\": \"CSV list of BIMI selectors to check\",\n }\n\n async def setup(self):\n self.emit_raw_dns_records = self.config.get(\"emit_raw_dns_records\", False)\n self.emit_urls = self.config.get(\"emit_urls\", True)\n self._selectors = self.config.get(\"selectors\", \"\").replace(\", \", \",\").split(\",\")\n\n return await super().setup()\n\n def _incoming_dedup_hash(self, event):\n # dedupe by parent\n parent_domain = self.helpers.parent_domain(event.data)\n return hash(parent_domain), \"already processed parent domain\"\n\n async def filter_event(self, event):\n if \"_wildcard\" in str(event.host).split(\".\"):\n return False, \"event is wildcard\"\n\n # there's no value in inspecting service records\n if service_record(event.host) is True:\n return False, \"service record detected\"\n\n return True\n\n async def inspectBIMI(self, event, domain):\n parent_domain = self.helpers.parent_domain(event.data)\n rdtype = \"TXT\"\n\n for selector in self._selectors:\n tags = [\"bimi-record\", f\"bimi-{selector}\"]\n hostname = f\"{selector}._bimi.{parent_domain}\"\n\n r = await self.helpers.resolve_raw(hostname, type=rdtype)\n\n if r:\n raw_results, errors = r\n\n for answer in raw_results:\n if self.emit_raw_dns_records:\n await self.emit_event(\n {\n \"host\": hostname,\n \"type\": rdtype,\n \"answer\": answer.to_text(),\n },\n \"RAW_DNS_RECORD\",\n parent=event,\n tags=tags.append(f\"{rdtype.lower()}-record\"),\n context=f\"{rdtype} lookup on {hostname} produced {{event.type}}\",\n )\n\n # we need to strip surrounding quotes and whitespace, as well as fix TXT data that may have been split across two different rdata's\n # e.g. we will get a single string, but within that string we may have two parts such as:\n # answer = '\"part 1 that was really long\" \"part 2 that did not fit in part 1\"'\n s = answer.to_text().strip('\"').strip().replace('\" \"', \"\")\n\n bimi_match = bimi_regex.search(s)\n\n if bimi_match and bimi_match.group(\"v\") and \"bimi\" in bimi_match.group(\"v\").lower():\n if bimi_match.group(\"l\") and bimi_match.group(\"l\") != \"\":\n if self.emit_urls:\n await self.emit_event(\n bimi_match.group(\"l\"),\n \"URL_UNVERIFIED\",\n parent=event,\n tags=tags.append(\"bimi-location\"),\n )\n\n if bimi_match.group(\"a\") and bimi_match.group(\"a\") != \"\":\n if self.emit_urls:\n await self.emit_event(\n bimi_match.group(\"a\"),\n \"URL_UNVERIFIED\",\n parent=event,\n tags=tags.append(\"bimi-authority\"),\n )\n\n async def handle_event(self, event):\n await self.inspectBIMI(event, event.host)\n" }, { "path": "bbot/modules/dnsbrute.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass dnsbrute(subdomain_enum):\n flags = [\"subdomain-enum\", \"active\", \"aggressive\"]\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n meta = {\n \"description\": \"Brute-force subdomains with massdns + static wordlist\",\n \"author\": \"@TheTechromancer\",\n \"created_date\": \"2024-04-24\",\n }\n options = {\n \"wordlist\": \"https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt\",\n \"max_depth\": 5,\n }\n options_desc = {\n \"wordlist\": \"Subdomain wordlist URL\",\n \"max_depth\": \"How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com\",\n }\n deps_common = [\"massdns\"]\n reject_wildcards = \"strict\"\n dedup_strategy = \"lowest_parent\"\n _qsize = 10000\n\n async def setup_deps(self):\n self.subdomain_file = await self.helpers.wordlist(self.config.get(\"wordlist\"))\n # tell the dnsbrute helper to fetch the resolver file\n await self.helpers.dns.brute.resolver_file()\n return True\n\n async def setup(self):\n self.max_depth = max(1, self.config.get(\"max_depth\", 5))\n self.subdomain_list = set(self.helpers.read_file(self.subdomain_file))\n self.wordlist_size = len(self.subdomain_list)\n return await super().setup()\n\n async def filter_event(self, event):\n eligible, reason = await super().filter_event(event)\n query = self.make_query(event)\n\n # limit brute force depth\n subdomain_depth = self.helpers.subdomain_depth(query) + 1\n if subdomain_depth > self.max_depth:\n eligible = False\n reason = f\"subdomain depth of *.{query} ({subdomain_depth}) > max_depth ({self.max_depth})\"\n\n # don't brute-force things that look like autogenerated PTRs\n if self.helpers.dns.brute.has_excessive_digits(query):\n eligible = False\n reason = f'\"{query}\" looks like an autogenerated PTR'\n\n return eligible, reason\n\n async def handle_event(self, event):\n query = self.make_query(event)\n self.info(f\"Brute-forcing {self.wordlist_size:,} subdomains for {query} (source: {event.data})\")\n for hostname in await self.helpers.dns.brute(self, query, self.subdomain_list):\n await self.emit_event(\n hostname,\n \"DNS_NAME\",\n parent=event,\n context=f'{{module}} tried {self.wordlist_size:,} subdomains against \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n" }, { "path": "bbot/modules/dnsbrute_mutations.py", "content": "import time\n\nfrom bbot.modules.base import BaseModule\n\n\nclass dnsbrute_mutations(BaseModule):\n flags = [\"subdomain-enum\", \"active\", \"aggressive\", \"slow\"]\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n meta = {\n \"description\": \"Brute-force subdomains with massdns + target-specific mutations\",\n \"author\": \"@TheTechromancer\",\n \"created_date\": \"2024-04-25\",\n }\n options = {\n \"max_mutations\": 100,\n }\n options_desc = {\n \"max_mutations\": \"Maximum number of target-specific mutations to try per subdomain\",\n }\n deps_common = [\"massdns\"]\n _qsize = 10000\n\n async def setup(self):\n self.found = {}\n self.parent_events = {}\n self.max_mutations = self.config.get(\"max_mutations\", 500)\n # 800M bits == 100MB bloom filter == 10M entries before false positives start emerging\n self.mutations_tried = self.helpers.bloom_filter(800000000)\n self._mutation_run_counter = {}\n return True\n\n async def handle_event(self, event):\n # here we don't brute-force, we just add the subdomain to our end-of-scan\n host = str(event.host)\n self.parent_events[host] = event\n if self.helpers.is_subdomain(host):\n subdomain, domain = host.split(\".\", 1)\n if not self.helpers.dns.brute.has_excessive_digits(subdomain):\n try:\n self.found[domain].add(subdomain)\n except KeyError:\n self.found[domain] = {subdomain}\n\n async def get_parent_event(self, subdomain):\n start = time.time()\n parent_host = await self.helpers.run_in_executor(self.helpers.closest_match, subdomain, self.parent_events)\n elapsed = time.time() - start\n self.trace(f\"{subdomain}: got closest match among {len(self.parent_events):,} parent events in {elapsed:.2f}s\")\n return self.parent_events[parent_host]\n\n async def finish(self):\n \"\"\"\n TODO: speed up this loop.\n We should see if we can combine multiple runs together instead of running them each individually.\n \"\"\"\n found = sorted(self.found.items(), key=lambda x: len(x[-1]), reverse=True)\n # if we have a lot of rounds to make, don't try mutations on less-populated domains\n trimmed_found = []\n if found:\n avg_subdomains = sum([len(subdomains) for domain, subdomains in found[:50]]) / len(found[:50])\n for i, (domain, subdomains) in enumerate(found):\n # accept domains that are in the top 50 or have more than 5 percent of the average number of subdomains\n if i < 50 or (len(subdomains) > 1 and len(subdomains) >= (avg_subdomains * 0.05)):\n trimmed_found.append((domain, subdomains))\n else:\n self.verbose(\n f\"Skipping mutations on {domain} because it only has {len(subdomains):,} subdomain(s) (avg: {avg_subdomains:,})\"\n )\n\n base_mutations = set()\n try:\n for i, (domain, subdomains) in enumerate(trimmed_found):\n self.verbose(f\"{domain} has {len(subdomains):,} subdomains\")\n # keep looping as long as we're finding things\n while 1:\n query = domain\n\n mutations = set(base_mutations)\n\n def add_mutation(m):\n h = f\"{m}.{domain}\"\n if h not in self.mutations_tried:\n self.mutations_tried.add(h)\n mutations.add(m)\n\n # try every subdomain everywhere else\n for _domain, _subdomains in found:\n if _domain == domain:\n continue\n for s in _subdomains:\n first_segment = s.split(\".\")[0]\n # skip stuff with lots of numbers (e.g. PTRs)\n if self.helpers.dns.brute.has_excessive_digits(first_segment):\n continue\n add_mutation(first_segment)\n for word in self.helpers.extract_words(\n first_segment, word_regexes=self.helpers.word_cloud.dns_mutator.extract_word_regexes\n ):\n add_mutation(word)\n\n # numbers + devops mutations\n for mutation in self.helpers.word_cloud.mutations(\n subdomains, cloud=False, numbers=3, number_padding=1\n ):\n for delimiter in (\"\", \".\", \"-\"):\n m = delimiter.join(mutation).lower()\n add_mutation(m)\n\n # special dns mutator\n for subdomain in self.helpers.word_cloud.dns_mutator.mutations(\n subdomains, max_mutations=self.max_mutations\n ):\n add_mutation(subdomain)\n\n # skip if there's hardly any mutations\n if len(mutations) < 10:\n self.verbose(\n f\"Skipping {len(mutations):,} mutations against {domain} because there are less than 10\"\n )\n break\n\n if mutations:\n self.info(\n f\"Trying {len(mutations):,} mutations against {domain} ({i + 1}/{len(trimmed_found)})\"\n )\n results = await self.helpers.dns.brute(self, query, mutations)\n try:\n mutation_run = self._mutation_run_counter[domain]\n except KeyError:\n self._mutation_run_counter[domain] = mutation_run = 1\n self._mutation_run_counter[domain] += 1\n for hostname in results:\n parent_event = await self.get_parent_event(hostname)\n mutation_run_ordinal = self.helpers.integer_to_ordinal(mutation_run)\n await self.emit_event(\n hostname,\n \"DNS_NAME\",\n parent=parent_event,\n tags=[f\"mutation-{mutation_run}\"],\n abort_if=self.abort_if,\n context=f'{{module}} found a mutated subdomain of \"{parent_event.host}\" on its {mutation_run_ordinal} run: {{event.type}}: {{event.data}}',\n )\n if results:\n continue\n break\n except AssertionError as e:\n self.warning(e)\n\n def abort_if(self, event):\n if not event.scope_distance == 0:\n return True, \"event is not in scope\"\n if \"wildcard\" in event.tags:\n return True, \"event is a wildcard\"\n if \"unresolved\" in event.tags:\n return True, \"event is unresolved\"\n return False, \"\"\n" }, { "path": "bbot/modules/dnscaa.py", "content": "# dnscaa.py\n#\n# Checks for and parses CAA DNS TXT records for IODEF reporting destination email addresses and/or URL's.\n#\n# NOTE: when the target domain is initially resolved basic \"dns_name_extraction_regex\" matched targets will be extracted so we do not perform that again here.\n#\n# Example CAA records,\n# 0 iodef \"mailto:dnsadmin@example.com\"\n# 0 iodef \"mailto:contact_pki@example.com\"\n# 0 iodef \"mailto:ipladmin@example.com\"\n# 0 iodef \"https://example.com/caa\"\n# 0 iodef \"https://203.0.113.1/caa\" <<< unlikely but possible?\n# 0 iodef \"https://[2001:db8::1]/caa\" <<< unlikely but possible?\n#\n# We simply extract any URL's as URL_UNVERIFIED, no further activity against URL's is performed by this module in order to remain passive.\n#\n# Other modules which respond to URL_UNVERIFIED events may do so if you have configured bbot appropriately.\n#\n# The domain/IP portion of any URL_UNVERIFIED's should be extracted by the various internal modules.\n#\n\nfrom bbot.modules.base import BaseModule\n\nimport re\n\nfrom bbot.core.helpers.regexes import dns_name_extraction_regex, email_regex, url_regexes\n\n# Handle '0 iodef \"mailto:support@hcaptcha.com\"'\n# Handle '1 iodef \"https://some.host.tld/caa;\"'\n# Handle '0 issue \"pki.goog; cansignhttpexchanges=yes; somethingelse=1\"'\n# Handle '1 issue \";\"' == explicit denial for any wildcard issuance.\n# Handle '128 issuewild \"comodoca.com\"'\n# Handle '128 issuewild \";\"' == explicit denial for any wildcard issuance.\n_caa_regex = r\"^(?P[0-9]+) +(?P\\w+) +\\\"(?P[^;\\\"]*);* *(?P[^\\\"]*)\\\"$\"\ncaa_regex = re.compile(_caa_regex)\n\n_caa_extensions_kvp_regex = r\"(?P\\w+)=(?P[^;]+)\"\ncaa_extensions_kvp_regex = re.compile(_caa_extensions_kvp_regex)\n\n\nclass dnscaa(BaseModule):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\", \"EMAIL_ADDRESS\", \"URL_UNVERIFIED\"]\n flags = [\"subdomain-enum\", \"email-enum\", \"passive\", \"safe\"]\n meta = {\"description\": \"Check for CAA records\", \"author\": \"@colin-stubbs\", \"created_date\": \"2024-05-26\"}\n options = {\n \"in_scope_only\": True,\n \"dns_names\": True,\n \"emails\": True,\n \"urls\": True,\n }\n options_desc = {\n \"in_scope_only\": \"Only check in-scope domains\",\n \"dns_names\": \"emit DNS_NAME events\",\n \"emails\": \"emit EMAIL_ADDRESS events\",\n \"urls\": \"emit URL_UNVERIFIED events\",\n }\n # accept DNS_NAMEs out to 2 hops if in_scope_only is False\n scope_distance_modifier = 2\n\n async def setup(self):\n self.in_scope_only = self.config.get(\"in_scope_only\", True)\n self._dns_names = self.config.get(\"dns_names\", True)\n self._emails = self.config.get(\"emails\", True)\n self._urls = self.config.get(\"urls\", True)\n return await super().setup()\n\n async def filter_event(self, event):\n if \"_wildcard\" in str(event.host).split(\".\"):\n return False, \"event is wildcard\"\n\n # scope filtering\n if event.scope_distance > 0 and self.in_scope_only:\n return False, \"event is not in scope\"\n\n return True\n\n async def handle_event(self, event):\n tags = [\"caa-record\"]\n\n r = await self.helpers.resolve_raw(event.host, type=\"caa\")\n\n if r:\n raw_results, errors = r\n\n for answer in raw_results:\n s = answer.to_text().strip().replace('\" \"', \"\")\n\n # validate CAA record vi regex so that we can determine what to do with it.\n caa_match = caa_regex.search(s)\n\n if caa_match and caa_match.group(\"flags\") and caa_match.group(\"property\") and caa_match.group(\"text\"):\n # it's legit.\n if caa_match.group(\"property\").lower() == \"iodef\":\n if self._emails:\n for match in email_regex.finditer(caa_match.group(\"text\")):\n start, end = match.span()\n email = caa_match.group(\"text\")[start:end]\n\n await self.emit_event(email, \"EMAIL_ADDRESS\", tags=tags, parent=event)\n\n if self._urls:\n for url_regex in url_regexes:\n for match in url_regex.finditer(caa_match.group(\"text\")):\n start, end = match.span()\n url = caa_match.group(\"text\")[start:end].strip('\"').strip()\n\n await self.emit_event(url, \"URL_UNVERIFIED\", tags=tags, parent=event)\n\n elif caa_match.group(\"property\").lower().startswith(\"issue\"):\n if self._dns_names:\n for match in dns_name_extraction_regex.finditer(caa_match.group(\"text\")):\n start, end = match.span()\n name = caa_match.group(\"text\")[start:end]\n\n await self.emit_event(name, \"DNS_NAME\", tags=tags, parent=event)\n\n\n# EOF\n" }, { "path": "bbot/modules/dnscommonsrv.py", "content": "from bbot.core.helpers.dns.helpers import common_srvs\nfrom bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass dnscommonsrv(subdomain_enum):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"active\", \"safe\"]\n meta = {\"description\": \"Check for common SRV records\", \"created_date\": \"2022-05-15\", \"author\": \"@TheTechromancer\"}\n dedup_strategy = \"lowest_parent\"\n deps_common = [\"massdns\"]\n\n options = {\"max_depth\": 2}\n options_desc = {\"max_depth\": \"The maximum subdomain depth to brute-force SRV records\"}\n\n async def setup(self):\n self.max_subdomain_depth = self.config.get(\"max_depth\", 2)\n self.num_srvs = len(common_srvs)\n return True\n\n async def filter_event(self, event):\n subdomain_depth = self.helpers.subdomain_depth(event.host)\n if subdomain_depth > self.max_subdomain_depth:\n return False, f\"its subdomain depth ({subdomain_depth}) exceeds max_depth={self.max_subdomain_depth}\"\n return True\n\n async def handle_event(self, event):\n query = self.make_query(event)\n self.verbose(f'Brute-forcing {self.num_srvs:,} SRV records for \"{query}\"')\n for hostname in await self.helpers.dns.brute(self, query, common_srvs, type=\"SRV\"):\n await self.emit_event(\n hostname,\n \"DNS_NAME\",\n parent=event,\n context=f'{{module}} tried {self.num_srvs:,} common SRV records against \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n" }, { "path": "bbot/modules/dnsdumpster.py", "content": "import json\n\nfrom bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass dnsdumpster(subdomain_enum):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query dnsdumpster for subdomains\",\n \"created_date\": \"2022-03-12\",\n \"author\": \"@TheTechromancer\",\n }\n\n base_url = \"https://dnsdumpster.com\"\n\n async def setup(self):\n self.apikey_regex = self.helpers.re.compile(r']*data-form-id=\"mainform\"[^>]*hx-headers=\\'([^\\']*)\\'')\n return True\n\n async def query(self, domain):\n ret = []\n # first, get the JWT token from the main page\n res1 = await self.api_request(self.base_url)\n status_code = getattr(res1, \"status_code\", 0)\n if status_code not in [200]:\n self.verbose(f'Bad response code \"{status_code}\" from DNSDumpster')\n return ret\n\n # Extract JWT token from the form's hx-headers attribute using regex\n jwt_token = None\n try:\n # Look for the form with data-form-id=\"mainform\" and extract hx-headers\n form_match = await self.helpers.re.search(self.apikey_regex, res1.text)\n if form_match:\n headers_json = form_match.group(1)\n headers_data = json.loads(headers_json)\n jwt_token = headers_data.get(\"Authorization\")\n except (AttributeError, json.JSONDecodeError, KeyError):\n self.log.warning(\"Error obtaining JWT token\")\n return ret\n\n # Abort if we didn't get the JWT token\n if not jwt_token:\n self.verbose(\"Error obtaining JWT token\")\n self.errorState = True\n return ret\n else:\n self.debug(\"Successfully obtained JWT token\")\n\n if self.scan.stopping:\n return ret\n\n # Query the API with the JWT token\n res2 = await self.api_request(\n \"https://api.dnsdumpster.com/htmld/\",\n method=\"POST\",\n data={\"target\": str(domain).lower()},\n headers={\n \"Authorization\": jwt_token,\n \"Content-Type\": \"application/x-www-form-urlencoded\",\n \"Origin\": \"https://dnsdumpster.com\",\n \"Referer\": \"https://dnsdumpster.com/\",\n \"HX-Request\": \"true\",\n \"HX-Target\": \"results\",\n \"HX-Current-URL\": \"https://dnsdumpster.com/\",\n },\n )\n status_code = getattr(res2, \"status_code\", 0)\n if status_code not in [200]:\n self.verbose(f'Bad response code \"{status_code}\" from DNSDumpster API')\n return ret\n\n return await self.scan.extract_in_scope_hostnames(res2.text)\n" }, { "path": "bbot/modules/dnstlsrpt.py", "content": "# dnstlsrpt.py\n#\n# Checks for and parses common TLS-RPT TXT records, e.g. _smtp._tls.target.domain\n#\n# TLS-RPT policies may contain email addresses or URL's for reporting destinations, typically the email addresses are software processed inboxes, but they may also be to individual humans or team inboxes.\n#\n# The domain portion of any email address or URL is also passively checked and added as appropriate, for additional inspection by other modules.\n#\n# Example records,\n# _smtp._tls.example.com TXT \"v=TLSRPTv1;rua=https://tlsrpt.azurewebsites.net/report\"\n# _smtp._tls.example.net TXT \"v=TLSRPTv1; rua=mailto:sts-reports@example.net;\"\n#\n# TODO: extract %{UNIQUE_ID}% from hosted services as ORG_STUB ?\n# e.g. %{UNIQUE_ID}%@tlsrpt.hosted.service.provider is usually a tenant specific ID.\n# e.g. tlsrpt@%{UNIQUE_ID}%.hosted.service.provider is usually a tenant specific ID.\n\nfrom bbot.modules.base import BaseModule\nfrom bbot.core.helpers.dns.helpers import service_record\n\nimport re\n\nfrom bbot.core.helpers.regexes import email_regex, url_regexes\n\n_tlsrpt_regex = r\"^v=(?PTLSRPTv[0-9]+); *(?P.*)$\"\ntlsrpt_regex = re.compile(_tlsrpt_regex, re.I)\n\n_tlsrpt_kvp_regex = r\"(?P\\w+)=(?P[^;]+);*\"\ntlsrpt_kvp_regex = re.compile(_tlsrpt_kvp_regex)\n\n_csul = r\"(?P[^, ]+)\"\ncsul = re.compile(_csul)\n\n\nclass dnstlsrpt(BaseModule):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"EMAIL_ADDRESS\", \"URL_UNVERIFIED\", \"RAW_DNS_RECORD\"]\n flags = [\"subdomain-enum\", \"cloud-enum\", \"email-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Check for TLS-RPT records\",\n \"author\": \"@colin-stubbs\",\n \"created_date\": \"2024-07-26\",\n }\n options = {\n \"emit_emails\": True,\n \"emit_raw_dns_records\": False,\n \"emit_urls\": True,\n }\n options_desc = {\n \"emit_emails\": \"Emit EMAIL_ADDRESS events\",\n \"emit_raw_dns_records\": \"Emit RAW_DNS_RECORD events\",\n \"emit_urls\": \"Emit URL_UNVERIFIED events\",\n }\n\n async def setup(self):\n self.emit_emails = self.config.get(\"emit_emails\", True)\n self.emit_raw_dns_records = self.config.get(\"emit_raw_dns_records\", False)\n self.emit_urls = self.config.get(\"emit_urls\", True)\n return await super().setup()\n\n def _incoming_dedup_hash(self, event):\n # dedupe by parent\n parent_domain = self.helpers.parent_domain(event.data)\n return hash(parent_domain), \"already processed parent domain\"\n\n async def filter_event(self, event):\n if \"_wildcard\" in str(event.host).split(\".\"):\n return False, \"event is wildcard\"\n\n # there's no value in inspecting service records\n if service_record(event.host) is True:\n return False, \"service record detected\"\n\n return True\n\n async def handle_event(self, event):\n rdtype = \"TXT\"\n tags = [\"tlsrpt-record\"]\n hostname = f\"_smtp._tls.{event.host}\"\n\n r = await self.helpers.resolve_raw(hostname, type=rdtype)\n\n if r:\n raw_results, errors = r\n for answer in raw_results:\n if self.emit_raw_dns_records:\n await self.emit_event(\n {\"host\": hostname, \"type\": rdtype, \"answer\": answer.to_text()},\n \"RAW_DNS_RECORD\",\n parent=event,\n tags=tags.append(f\"{rdtype.lower()}-record\"),\n context=f\"{rdtype} lookup on {hostname} produced {{event.type}}\",\n )\n\n # we need to fix TXT data that may have been split across two different rdata's\n # e.g. we will get a single string, but within that string we may have two parts such as:\n # answer = '\"part 1 that was really long\" \"part 2 that did not fit in part 1\"'\n # NOTE: the leading and trailing double quotes are essential as part of a raw DNS TXT record, or another record type that contains a free form text string as a component.\n s = answer.to_text().strip('\"').replace('\" \"', \"\")\n\n # validate TLSRPT record, tag appropriately\n tlsrpt_match = tlsrpt_regex.search(s)\n\n if (\n tlsrpt_match\n and tlsrpt_match.group(\"v\")\n and tlsrpt_match.group(\"kvps\")\n and tlsrpt_match.group(\"kvps\") != \"\"\n ):\n for kvp_match in tlsrpt_kvp_regex.finditer(tlsrpt_match.group(\"kvps\")):\n key = kvp_match.group(\"k\").lower()\n\n if key == \"rua\":\n for csul_match in csul.finditer(kvp_match.group(\"v\")):\n if csul_match.group(\"uri\"):\n for match in email_regex.finditer(csul_match.group(\"uri\")):\n start, end = match.span()\n email = csul_match.group(\"uri\")[start:end]\n\n if self.emit_emails:\n await self.emit_event(\n email,\n \"EMAIL_ADDRESS\",\n tags=tags.append(f\"tlsrpt-record-{key}\"),\n parent=event,\n )\n\n for url_regex in url_regexes:\n for match in url_regex.finditer(csul_match.group(\"uri\")):\n start, end = match.span()\n url = csul_match.group(\"uri\")[start:end]\n\n if self.emit_urls:\n await self.emit_event(\n url,\n \"URL_UNVERIFIED\",\n tags=tags.append(f\"tlsrpt-record-{key}\"),\n parent=event,\n )\n" }, { "path": "bbot/modules/docker_pull.py", "content": "import io\nimport json\nimport tarfile\nfrom pathlib import Path\nfrom bbot.modules.base import BaseModule\n\n\nclass docker_pull(BaseModule):\n watched_events = [\"CODE_REPOSITORY\"]\n produced_events = [\"FILESYSTEM\"]\n flags = [\"passive\", \"safe\", \"slow\", \"code-enum\", \"download\"]\n meta = {\n \"description\": \"Download images from a docker repository\",\n \"created_date\": \"2024-03-24\",\n \"author\": \"@domwhewell-sage\",\n }\n options = {\"all_tags\": False, \"output_folder\": \"\"}\n options_desc = {\n \"all_tags\": \"Download all tags from each registry (Default False)\",\n \"output_folder\": \"Folder to download docker repositories to. If not specified, downloaded docker images will be deleted when the scan completes, to minimize disk usage.\",\n }\n\n scope_distance_modifier = 2\n\n async def setup(self):\n self.headers = {\n \"Accept\": \",\".join(\n [\n \"application/vnd.docker.distribution.manifest.v2+json\",\n \"application/vnd.docker.distribution.manifest.list.v2+json\",\n \"application/vnd.docker.distribution.manifest.v1+json\",\n \"application/vnd.oci.image.manifest.v1+json\",\n ]\n )\n }\n self.all_tags = self.config.get(\"all_tags\", True)\n output_folder = self.config.get(\"output_folder\", \"\")\n if output_folder:\n self.output_dir = Path(output_folder) / \"docker_images\"\n else:\n self.output_dir = self.scan.temp_dir / \"docker_images\"\n self.helpers.mkdir(self.output_dir)\n return await super().setup()\n\n async def filter_event(self, event):\n if event.type == \"CODE_REPOSITORY\":\n if \"docker\" not in event.tags:\n return False, \"event is not a docker repository\"\n return True\n\n async def handle_event(self, event):\n repo_url = event.data.get(\"url\")\n repo_path = await self.download_docker_repo(repo_url)\n if repo_path:\n self.verbose(f\"Downloaded docker repository {repo_url} to {repo_path}\")\n codebase_event = self.make_event(\n {\"path\": str(repo_path), \"description\": f\"Docker image repository: {repo_url}\"},\n \"FILESYSTEM\",\n tags=[\"docker\", \"tarball\"],\n parent=event,\n )\n if codebase_event:\n await self.emit_event(\n codebase_event, context=f\"{{module}} downloaded Docker image to {{event.type}}: {repo_path}\"\n )\n\n def get_registry_and_repository(self, repository_url):\n \"\"\"Function to get the registry and repository from a html repository URL.\"\"\"\n if repository_url.startswith(\"https://hub.docker.com/r/\"):\n registry = \"https://registry-1.docker.io\"\n repository = repository_url.replace(\"https://hub.docker.com/r/\", \"\")\n else:\n repository = \"/\".join(repository_url.split(\"/\")[-2:])\n registry = repository_url.replace(repository, \"\")\n return registry, repository\n\n async def docker_api_request(self, url: str):\n \"\"\"Make a request to the URL if that fails try to obtain an authentication token and try again.\"\"\"\n for _ in range(2):\n response = await self.helpers.request(url, headers=self.headers, follow_redirects=True)\n if response is not None and response.status_code != 401:\n return response\n try:\n www_authenticate_headers = response.headers.get(\"www-authenticate\", \"\")\n realm = www_authenticate_headers.split('realm=\"')[1].split('\"')[0]\n service = www_authenticate_headers.split('service=\"')[1].split('\"')[0]\n scope = www_authenticate_headers.split('scope=\"')[1].split('\"')[0]\n except (KeyError, IndexError):\n self.log.warning(f\"Could not obtain realm, service or scope from {url}\")\n break\n auth_url = f\"{realm}?service={service}&scope={scope}\"\n auth_response = await self.helpers.request(auth_url)\n if not auth_response:\n self.log.warning(f\"Could not obtain token from {auth_url}\")\n break\n auth_json = auth_response.json()\n token = auth_json[\"token\"]\n self.headers.update({\"Authorization\": f\"Bearer {token}\"})\n return None\n\n async def get_tags(self, registry, repository):\n url = f\"{registry}/v2/{repository}/tags/list\"\n r = await self.docker_api_request(url)\n if r is None or r.status_code != 200:\n self.log.warning(f\"Could not retrieve all tags for {repository} assuming tag:latest only.\")\n self.log.debug(f\"Response: {r}\")\n return [\"latest\"]\n try:\n tags = r.json().get(\"tags\", [\"latest\"])\n self.debug(f\"Tags for {repository}: {tags}\")\n if self.all_tags:\n return tags\n else:\n if \"latest\" in tags:\n return [\"latest\"]\n else:\n return [tags[-1]]\n except (KeyError, IndexError):\n self.log.warning(f\"Could not retrieve tags for {repository}.\")\n return [\"latest\"]\n\n async def get_manifest(self, registry, repository, tag):\n url = f\"{registry}/v2/{repository}/manifests/{tag}\"\n r = await self.docker_api_request(url)\n if r is None or r.status_code != 200:\n self.log.warning(f\"Could not retrieve manifest for {repository}:{tag}.\")\n self.log.debug(f\"Response: {r}\")\n return {}\n response_json = r.json()\n if response_json.get(\"manifests\", []):\n for manifest in response_json[\"manifests\"]:\n if manifest[\"platform\"][\"os\"] == \"linux\" and manifest[\"platform\"][\"architecture\"] == \"amd64\":\n return await self.get_manifest(registry, repository, manifest[\"digest\"])\n return response_json\n\n async def get_layers(self, manifest):\n schema_version = manifest.get(\"schemaVersion\", 2)\n if schema_version == 1:\n return [l[\"blobSum\"] for l in manifest.get(\"fsLayers\", [])]\n elif schema_version == 2:\n return [l[\"digest\"] for l in manifest.get(\"layers\", [])]\n else:\n return []\n\n async def download_blob(self, registry, repository, digest):\n url = f\"{registry}/v2/{repository}/blobs/{digest}\"\n r = await self.docker_api_request(url)\n if r is None or r.status_code != 200:\n return None\n else:\n return r.content\n\n async def create_local_manifest(self, config, repository, tag, layers):\n manifest = [{\"Config\": config, \"RepoTags\": [f\"{repository}:{tag}\"], \"Layers\": layers}]\n return json.dumps(manifest).encode()\n\n async def download_and_get_filename(self, registry, repository, digest):\n if \":\" not in digest:\n return None, None\n blob = await self.download_blob(registry, repository, digest)\n hash_func = digest.split(\":\")[0]\n digest = digest.split(\":\")[1]\n filename = f\"blobs/{hash_func}/{digest}\"\n return blob, filename\n\n async def write_file_to_tar(self, tar, filename, file_content):\n if filename and file_content:\n file_io = io.BytesIO(file_content)\n file_info = tarfile.TarInfo(name=filename)\n file_info.size = len(file_io.getvalue())\n file_io.seek(0)\n tar.addfile(file_info, file_io)\n\n async def download_docker_repo(self, repository_url):\n registry, repository = self.get_registry_and_repository(repository_url)\n tags = await self.get_tags(registry, repository)\n for tag in tags:\n self.info(f\"Downloading {repository}:{tag}\")\n tar_file = await self.download_and_write_to_tar(registry, repository, tag)\n return tar_file\n\n async def download_and_write_to_tar(self, registry, repository, tag):\n output_tar = self.output_dir / f\"{repository.replace('/', '_')}_{tag}.tar\"\n with tarfile.open(output_tar, mode=\"w\") as tar:\n manifest = await self.get_manifest(registry, repository, tag)\n config_file, config_filename = await self.download_and_get_filename(\n registry, repository, manifest.get(\"config\", {}).get(\"digest\", \"\")\n )\n await self.write_file_to_tar(tar, config_filename, config_file)\n\n layer_filenames = []\n layer_digests = await self.get_layers(manifest)\n for i, layer_digest in enumerate(layer_digests):\n self.verbose(f\"Downloading layer {i + 1}/{len(layer_digests)} from {repository}:{tag}\")\n blob, layer_filename = await self.download_and_get_filename(registry, repository, layer_digest)\n layer_filenames.append(layer_filename)\n await self.write_file_to_tar(tar, layer_filename, blob)\n\n manifest_json = await self.create_local_manifest(config_filename, repository, tag, layer_filenames)\n await self.write_file_to_tar(tar, \"manifest.json\", manifest_json)\n return output_tar\n" }, { "path": "bbot/modules/dockerhub.py", "content": "from bbot.modules.base import BaseModule\n\n\nclass dockerhub(BaseModule):\n watched_events = [\"SOCIAL\", \"ORG_STUB\"]\n produced_events = [\"SOCIAL\", \"CODE_REPOSITORY\", \"URL_UNVERIFIED\"]\n flags = [\"passive\", \"safe\", \"code-enum\"]\n meta = {\n \"description\": \"Search for docker repositories of discovered orgs/usernames\",\n \"created_date\": \"2024-03-12\",\n \"author\": \"@domwhewell-sage\",\n }\n\n site_url = \"https://hub.docker.com\"\n api_url = f\"{site_url}/v2\"\n\n scope_distance_modifier = 2\n\n async def filter_event(self, event):\n if event.type == \"SOCIAL\":\n if event.data[\"platform\"] != \"docker\":\n return False, \"platform is not docker\"\n return True\n\n async def handle_event(self, event):\n if event.type == \"ORG_STUB\":\n await self.handle_org_stub(event)\n elif event.type == \"SOCIAL\":\n await self.handle_social(event)\n\n async def handle_org_stub(self, event):\n profile_name = event.data\n # docker usernames are case sensitive, so if there are capitalizations we also try a lowercase variation\n profiles_to_check = {profile_name, profile_name.lower()}\n for p in profiles_to_check:\n api_url = f\"{self.api_url}/users/{p}\"\n api_result = await self.helpers.request(api_url, follow_redirects=True)\n status_code = getattr(api_result, \"status_code\", 0)\n if status_code == 200:\n site_url = f\"{self.site_url}/u/{p}\"\n # emit social event\n await self.emit_event(\n {\"platform\": \"docker\", \"url\": site_url, \"profile_name\": p},\n \"SOCIAL\",\n parent=event,\n context=f\"{{module}} tried {event.type} {event.data} and found docker profile ({{event.type}}) at {p}\",\n )\n\n async def handle_social(self, event):\n username = event.data.get(\"profile_name\", \"\")\n if not username:\n return\n self.verbose(f\"Searching for docker images belonging to {username}\")\n repos = await self.get_repos(username)\n for repo in repos:\n await self.emit_event(\n {\"url\": repo},\n \"CODE_REPOSITORY\",\n tags=\"docker\",\n parent=event,\n context=f\"{{module}} found docker image {{event.type}}: {repo}\",\n )\n\n async def get_repos(self, username):\n repos = []\n url = f\"{self.api_url}/repositories/{username}?page_size=25&page=\" + \"{page}\"\n agen = self.api_page_iter(url, _json=False)\n try:\n async for r in agen:\n if r is None:\n break\n status_code = getattr(r, \"status_code\", 0)\n if status_code != 200:\n break\n try:\n j = r.json()\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n break\n if not j:\n break\n for item in j.get(\"results\", []):\n image_name = item.get(\"name\", \"\")\n namespace = item.get(\"namespace\", \"\")\n if image_name and namespace:\n repos.append(\"https://hub.docker.com/r/\" + namespace + \"/\" + image_name)\n finally:\n await agen.aclose()\n return repos\n" }, { "path": "bbot/modules/dotnetnuke.py", "content": "from bbot.errors import InteractshError\nfrom bbot.modules.base import BaseModule\n\n\nclass dotnetnuke(BaseModule):\n DNN_signatures_body = [\n \" CODE_REPOSITORY)\n repos = []\n if is_org:\n if in_scope:\n self.verbose(f\"Searching for repos belonging to organization {user}\")\n repos = await self.query_org_repos(user)\n else:\n self.verbose(f\"Organization {user} does not appear to be in-scope\")\n elif \"github-org-member\" in event.tags:\n self.verbose(f\"Searching for repos belonging to user {user}\")\n repos = await self.query_user_repos(user)\n for repo_url in repos:\n repo_event = self.make_event({\"url\": repo_url}, \"CODE_REPOSITORY\", tags=\"git\", parent=event)\n if not repo_event:\n continue\n await self.emit_event(\n repo_event,\n context=f\"{{module}} listed repos for GitHub profile and discovered {{event.type}}: {repo_url}\",\n )\n\n # find members from org (SOCIAL --> SOCIAL)\n if is_org and self.include_members:\n self.verbose(f\"Searching for any members belonging to {user}\")\n org_members = await self.query_org_members(user)\n for member in org_members:\n member_url = f\"https://github.com/{member}\"\n event_data = {\"platform\": \"github\", \"profile_name\": member, \"url\": member_url}\n member_event = self.make_event(event_data, \"SOCIAL\", tags=\"github-org-member\", parent=event)\n if member_event:\n await self.emit_event(\n member_event,\n context=f\"{{module}} listed members of GitHub organization and discovered {{event.type}}: {member_url}\",\n )\n\n # find valid orgs from stub (ORG_STUB --> SOCIAL)\n elif event.type == \"ORG_STUB\":\n user = event.data\n self.verbose(f\"Validating whether the organization {user} is within our scope...\")\n is_org, in_scope = await self.validate_org(user)\n if \"target\" in event.tags:\n in_scope = True\n if not is_org or not in_scope:\n self.verbose(f\"Unable to validate that {user} is in-scope, skipping...\")\n return\n\n user_url = f\"https://github.com/{user}\"\n event_data = {\"platform\": \"github\", \"profile_name\": user, \"url\": user_url}\n github_org_event = self.make_event(event_data, \"SOCIAL\", tags=\"github-org\", parent=event)\n if github_org_event:\n await self.emit_event(\n github_org_event,\n context=f'{{module}} tried \"{user}\" as GitHub profile and discovered {{event.type}}: {user_url}',\n )\n\n async def query_org_repos(self, query):\n repos = []\n url = f\"{self.base_url}/orgs/{self.helpers.quote(query)}/repos?per_page=100&page=\" + \"{page}\"\n agen = self.api_page_iter(url, _json=False)\n try:\n async for r in agen:\n if r is None:\n break\n status_code = getattr(r, \"status_code\", 0)\n if status_code == 403:\n self.warning(\"Github is rate-limiting us (HTTP status: 403)\")\n break\n if status_code != 200:\n break\n try:\n j = r.json()\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n break\n if not j:\n break\n for item in j:\n html_url = item.get(\"html_url\", \"\")\n repos.append(html_url)\n finally:\n await agen.aclose()\n return repos\n\n async def query_org_members(self, query):\n members = []\n url = f\"{self.base_url}/orgs/{self.helpers.quote(query)}/members?per_page=100&page=\" + \"{page}\"\n agen = self.api_page_iter(url, _json=False)\n try:\n async for r in agen:\n if r is None:\n break\n status_code = getattr(r, \"status_code\", 0)\n if status_code == 403:\n self.warning(\"Github is rate-limiting us (HTTP status: 403)\")\n break\n if status_code != 200:\n break\n try:\n j = r.json()\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n break\n if not j:\n break\n for item in j:\n login = item.get(\"login\", \"\")\n members.append(login)\n finally:\n await agen.aclose()\n return members\n\n async def query_user_repos(self, query):\n repos = []\n url = f\"{self.base_url}/users/{self.helpers.quote(query)}/repos?per_page=100&page=\" + \"{page}\"\n agen = self.api_page_iter(url, _json=False)\n try:\n async for r in agen:\n if r is None:\n break\n status_code = getattr(r, \"status_code\", 0)\n if status_code == 403:\n self.warning(\"Github is rate-limiting us (HTTP status: 403)\")\n break\n if status_code != 200:\n break\n try:\n j = r.json()\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n break\n if not j:\n break\n for item in j:\n html_url = item.get(\"html_url\", \"\")\n repos.append(html_url)\n finally:\n await agen.aclose()\n return repos\n\n async def validate_org(self, org):\n is_org = False\n in_scope = False\n url = f\"{self.base_url}/orgs/{org}\"\n r = await self.api_request(url)\n if r is None:\n return is_org, in_scope\n status_code = getattr(r, \"status_code\", 0)\n if status_code == 403:\n self.warning(\"Github is rate-limiting us (HTTP status: 403)\")\n return is_org, in_scope\n if status_code == 200:\n is_org = True\n in_scope_hosts = await self.scan.extract_in_scope_hostnames(getattr(r, \"text\", \"\"))\n if in_scope_hosts:\n self.verbose(\n f'Found in-scope hostname(s): \"{in_scope_hosts}\" for github org: {org}, it appears to be in-scope'\n )\n in_scope = True\n return is_org, in_scope\n" }, { "path": "bbot/modules/github_usersearch.py", "content": "from bbot.modules.templates.github import github\nfrom bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass github_usersearch(github, subdomain_enum):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"SOCIAL\", \"EMAIL_ADDRESS\"]\n flags = [\"passive\", \"safe\", \"code-enum\"]\n meta = {\n \"description\": \"Query Github's API for users with emails matching in scope domains that may not be discoverable by listing members of the organization.\",\n \"created_date\": \"2025-05-10\",\n \"author\": \"@domwhewell-sage\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"Github token\"}\n\n async def handle_event(self, event):\n self.verbose(\"Searching for users with emails matching in scope domains\")\n query = self.make_query(event)\n users = await self.query_users(query)\n for user, email in users:\n user_url = f\"https://github.com/{user}\"\n event_data = {\"platform\": \"github\", \"profile_name\": user, \"url\": user_url}\n user_event = self.make_event(event_data, \"SOCIAL\", tags=\"github-org-member\", parent=event)\n if user_event:\n await self.emit_event(\n user_event,\n context=f\"{{module}} searched for users with {{DNS_NAME}} in the profile and discovered {{event.type}}: {user_url}\",\n )\n if email:\n await self.emit_event(\n email,\n \"EMAIL_ADDRESS\",\n parent=event,\n context=f\"{{module}} found an {{event.type}} on the github profile {user_url}: {{event.data}}\",\n )\n\n async def query_users(self, query):\n users = []\n graphql_query = f\"\"\"query search_users {{\n search(query: \"{query}\", type: USER, first: 100, after: \"{{NEXT_KEY}}\") {{\n userCount\n pageInfo {{\n hasNextPage\n endCursor\n }}\n edges {{\n node {{\n ... on User {{\n login\n # bio Commented out as user can add arbritrary domains to their bio\n email # Email is verified by github\n websiteUrl # Website is not verified by github\n }}\n }}\n }}\n }}\n }}\"\"\"\n async for data in self.github_graphql_request(graphql_query, \"search\"):\n if data:\n user_count = data.get(\"userCount\", 0)\n self.verbose(f\"Found {user_count} users with the query {query}, verifying if they are in-scope...\")\n edges = data.get(\"edges\", [])\n for node in edges:\n user = node.get(\"node\", {})\n in_scope_hosts = await self.scan.extract_in_scope_hostnames(str(user))\n if in_scope_hosts:\n login = user.get(\"login\", \"\")\n email = user.get(\"email\", None)\n self.verbose(\n f'Found in-scope hostname(s): \"{in_scope_hosts}\" in the profile https://github.com/{login}, the profile appears to be in-scope'\n )\n users.append((login, email))\n return users\n" }, { "path": "bbot/modules/github_workflows.py", "content": "import zipfile\nimport fnmatch\nfrom pathlib import Path\n\nfrom bbot.modules.templates.github import github\n\n\nclass github_workflows(github):\n watched_events = [\"CODE_REPOSITORY\"]\n produced_events = [\"FILESYSTEM\"]\n flags = [\"passive\", \"safe\", \"code-enum\", \"download\"]\n meta = {\n \"description\": \"Download a github repositories workflow logs and workflow artifacts\",\n \"created_date\": \"2024-04-29\",\n \"author\": \"@domwhewell-sage\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\", \"num_logs\": 1, \"output_folder\": \"\"}\n options_desc = {\n \"api_key\": \"Github token\",\n \"num_logs\": \"For each workflow fetch the last N successful runs logs (max 100)\",\n \"output_folder\": \"Folder to download workflow logs and artifacts to\",\n }\n\n scope_distance_modifier = 2\n\n async def setup(self):\n self.num_logs = int(self.config.get(\"num_logs\", 1))\n if self.num_logs > 100:\n self.log.error(\"num_logs option is capped at 100\")\n return False\n output_folder = self.config.get(\"output_folder\", \"\")\n if output_folder:\n self.output_dir = Path(output_folder) / \"workflow_logs\"\n else:\n self.output_dir = self.scan.home / \"workflow_logs\"\n self.helpers.mkdir(self.output_dir)\n return await super().setup()\n\n def _api_response_is_success(self, r):\n # we allow 404s because they're normal\n return r.is_success or getattr(r, \"status_code\", 0) == 404\n\n async def filter_event(self, event):\n if \"git\" not in event.tags:\n return False, \"event is not a git repository\"\n elif \"github.com\" not in event.data.get(\"url\", \"\"):\n return False, \"event is not a github repository\"\n return True\n\n async def handle_event(self, event):\n repo_url = event.data.get(\"url\")\n owner = repo_url.split(\"/\")[-2]\n repo = repo_url.split(\"/\")[-1]\n for workflow in await self.get_workflows(owner, repo):\n workflow_name = workflow.get(\"name\")\n workflow_id = workflow.get(\"id\")\n self.log.debug(f\"Looking up runs for {workflow_name} in {owner}/{repo}\")\n for run in await self.get_workflow_runs(owner, repo, workflow_id):\n run_id = run.get(\"id\")\n workflow_url = f\"https://github.com/{owner}/{repo}/actions/runs/{run_id}\"\n self.log.debug(f\"Downloading logs for {workflow_name}/{run_id} in {owner}/{repo}\")\n for log in await self.download_run_logs(owner, repo, run_id):\n logfile_event = self.make_event(\n {\n \"path\": str(log),\n \"description\": f\"Workflow run logs from {workflow_url}\",\n },\n \"FILESYSTEM\",\n tags=[\"textfile\"],\n parent=event,\n )\n await self.emit_event(\n logfile_event,\n context=f\"{{module}} downloaded workflow run logs from {workflow_url} to {{event.type}}: {log}\",\n )\n artifacts = await self.get_run_artifacts(owner, repo, run_id)\n if artifacts:\n for artifact in artifacts:\n artifact_id = artifact.get(\"id\")\n artifact_name = artifact.get(\"name\")\n expired = artifact.get(\"expired\")\n if not expired:\n filepath = await self.download_run_artifacts(owner, repo, artifact_id, artifact_name)\n if filepath:\n artifact_event = self.make_event(\n {\n \"path\": str(filepath),\n \"description\": f\"Workflow run artifact from {workflow_url}\",\n },\n \"FILESYSTEM\",\n tags=[\"zipfile\"],\n parent=event,\n )\n await self.emit_event(\n artifact_event,\n context=f\"{{module}} downloaded workflow run artifact from {workflow_url} to {{event.type}}: {filepath}\",\n )\n\n async def get_workflows(self, owner, repo):\n workflows = []\n url = f\"{self.base_url}/repos/{owner}/{repo}/actions/workflows?per_page=100&page=\" + \"{page}\"\n agen = self.api_page_iter(url, _json=False)\n try:\n async for r in agen:\n if r is None:\n break\n status_code = getattr(r, \"status_code\", 0)\n if status_code == 403:\n self.warning(\"Github is rate-limiting us (HTTP status: 403)\")\n break\n if status_code != 200:\n break\n try:\n j = r.json().get(\"workflows\", [])\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n break\n if not j:\n break\n for item in j:\n workflows.append(item)\n finally:\n await agen.aclose()\n return workflows\n\n async def get_workflow_runs(self, owner, repo, workflow_id):\n runs = []\n url = f\"{self.base_url}/repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs?status=success&per_page={self.num_logs}\"\n r = await self.api_request(url)\n if r is None:\n return runs\n status_code = getattr(r, \"status_code\", 0)\n if status_code == 403:\n self.warning(\"Github is rate-limiting us (HTTP status: 403)\")\n return runs\n if status_code != 200:\n return runs\n try:\n j = r.json().get(\"workflow_runs\", [])\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n return runs\n if not j:\n return runs\n for item in j:\n runs.append(item)\n return runs\n\n async def download_run_logs(self, owner, repo, run_id):\n folder = self.output_dir / owner / repo\n self.helpers.mkdir(folder)\n filename = f\"run_{run_id}.zip\"\n file_destination = folder / filename\n try:\n await self.api_download(\n f\"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/logs\",\n filename=file_destination,\n headers=self.headers,\n raise_error=True,\n warn=False,\n )\n self.info(f\"Downloaded logs for {owner}/{repo}/{run_id} to {file_destination}\")\n except Exception as e:\n file_destination = None\n response = getattr(e, \"response\", None)\n status_code = getattr(response, \"status_code\", 0)\n if status_code == 403:\n self.warning(\n f\"The current access key does not have access to workflow {owner}/{repo}/{run_id}, The API key must have the 'repo' scope or read 'Actions' repository permissions (status: {status_code})\"\n )\n else:\n self.info(\n f\"The logs for {owner}/{repo}/{run_id} have expired and are no longer available (status: {status_code})\"\n )\n # Secrets are duplicated in the individual workflow steps so just extract the main log files from the top folder\n if file_destination:\n main_logs = []\n with zipfile.ZipFile(file_destination, \"r\") as logzip:\n for name in logzip.namelist():\n if fnmatch.fnmatch(name, \"*.txt\") and \"/\" not in name:\n logzip.extract(name, folder)\n main_logs.append(folder / name)\n return main_logs\n else:\n return []\n\n async def get_run_artifacts(self, owner, repo, run_id):\n artifacts = []\n url = f\"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/artifacts\"\n r = await self.api_request(url)\n if r is None:\n return artifacts\n status_code = getattr(r, \"status_code\", 0)\n if status_code == 403:\n self.warning(\"Github is rate-limiting us (HTTP status: 403)\")\n return artifacts\n if status_code != 200:\n return artifacts\n try:\n j = r.json().get(\"artifacts\", [])\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n return artifacts\n if not j:\n return artifacts\n for item in j:\n artifacts.append(item)\n return artifacts\n\n async def download_run_artifacts(self, owner, repo, artifact_id, artifact_name):\n folder = self.output_dir / owner / repo\n self.helpers.mkdir(folder)\n file_destination = folder / artifact_name\n try:\n await self.api_download(\n f\"{self.base_url}/repos/{owner}/{repo}/actions/artifacts/{artifact_id}/zip\",\n filename=file_destination,\n headers=self.headers,\n raise_error=True,\n warn=False,\n )\n self.info(\n f\"Downloaded workflow artifact {owner}/{repo}/{artifact_id}/{artifact_name} to {file_destination}\"\n )\n except Exception as e:\n file_destination = None\n response = getattr(e, \"response\", None)\n status_code = getattr(response, \"status_code\", 0)\n if status_code == 403:\n self.warning(\n f\"The current access key does not have access to workflow artifacts {owner}/{repo}/{artifact_id}, The API key must have the 'repo' scope or read 'Actions' repository permissions (status: {status_code})\"\n )\n return file_destination\n" }, { "path": "bbot/modules/gitlab_com.py", "content": "from bbot.modules.templates.gitlab import GitLabBaseModule\n\n\nclass gitlab_com(GitLabBaseModule):\n watched_events = [\"SOCIAL\"]\n produced_events = [\n \"CODE_REPOSITORY\",\n ]\n flags = [\"active\", \"safe\", \"code-enum\"]\n meta = {\n \"description\": \"Enumerate GitLab SaaS (gitlab.com/org) for projects and groups\",\n \"created_date\": \"2024-03-11\",\n \"author\": \"@TheTechromancer\",\n }\n\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"GitLab access token (for gitlab.com/org only)\"}\n\n # This is needed because we are consuming SOCIAL events, which aren't in scope\n scope_distance_modifier = 2\n\n async def handle_event(self, event):\n await self.handle_social(event)\n\n async def filter_event(self, event):\n if event.data[\"platform\"] != \"gitlab\":\n return False, \"platform is not gitlab\"\n _, domain = self.helpers.split_domain(event.host)\n if domain not in self.saas_domains:\n return False, \"gitlab instance is not gitlab.com/org\"\n return True\n" }, { "path": "bbot/modules/gitlab_onprem.py", "content": "from bbot.modules.templates.gitlab import GitLabBaseModule\n\n\nclass gitlab_onprem(GitLabBaseModule):\n watched_events = [\"HTTP_RESPONSE\", \"TECHNOLOGY\", \"SOCIAL\"]\n produced_events = [\n \"TECHNOLOGY\",\n \"SOCIAL\",\n \"CODE_REPOSITORY\",\n \"FINDING\",\n ]\n flags = [\"active\", \"safe\", \"code-enum\"]\n meta = {\n \"description\": \"Detect self-hosted GitLab instances and query them for repositories\",\n \"created_date\": \"2024-03-11\",\n \"author\": \"@TheTechromancer\",\n }\n\n # Optional GitLab access token (only required for gitlab.com, but still\n # supported for on-prem installations that expose private projects).\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"GitLab access token (for self-hosted instances only)\"}\n\n # Allow accepting events slightly beyond configured max distance so we can\n # discover repos on neighbouring infrastructure.\n scope_distance_modifier = 2\n\n async def handle_event(self, event):\n if event.type == \"HTTP_RESPONSE\":\n await self.handle_http_response(event)\n elif event.type == \"TECHNOLOGY\":\n await self.handle_technology(event)\n elif event.type == \"SOCIAL\":\n await self.handle_social(event)\n\n async def filter_event(self, event):\n # only accept out-of-scope SOCIAL events\n if event.type == \"HTTP_RESPONSE\":\n if event.scope_distance > self.scan.scope_search_distance:\n return False, \"event is out of scope distance\"\n elif event.type == \"TECHNOLOGY\":\n if not event.data[\"technology\"].lower().startswith(\"gitlab\"):\n return False, \"technology is not gitlab\"\n if not self.helpers.is_ip(event.host) and self.helpers.tldextract(event.host).domain == \"gitlab\":\n return False, \"gitlab instance is not self-hosted\"\n elif event.type == \"SOCIAL\":\n if event.data[\"platform\"] != \"gitlab\":\n return False, \"platform is not gitlab\"\n _, domain = self.helpers.split_domain(event.host)\n if domain in self.saas_domains:\n return False, \"gitlab instance is not self-hosted\"\n return True\n\n async def handle_http_response(self, event):\n \"\"\"Identify GitLab servers from HTTP responses.\"\"\"\n headers = event.data.get(\"header\", {})\n if \"x_gitlab_meta\" in headers:\n url = event.parsed_url._replace(path=\"/\").geturl()\n await self.emit_event(\n {\"host\": str(event.host), \"technology\": \"GitLab\", \"url\": url},\n \"TECHNOLOGY\",\n parent=event,\n context=f\"{{module}} detected {{event.type}}: GitLab at {url}\",\n )\n description = f\"GitLab server at {event.host}\"\n await self.emit_event(\n {\"host\": str(event.host), \"description\": description},\n \"FINDING\",\n parent=event,\n context=f\"{{module}} detected {{event.type}}: {description}\",\n )\n\n async def handle_technology(self, event):\n \"\"\"Enumerate projects & groups once we know a host is GitLab.\"\"\"\n base_url = self.get_base_url(event)\n\n # Projects owned by the authenticated user (or public projects if no\n # authentication).\n projects_url = self.helpers.urljoin(base_url, \"api/v4/projects?simple=true\")\n await self.handle_projects_url(projects_url, event)\n\n # Group enumeration.\n groups_url = self.helpers.urljoin(base_url, \"api/v4/groups?simple=true\")\n await self.handle_groups_url(groups_url, event)\n" }, { "path": "bbot/modules/google_playstore.py", "content": "from bbot.modules.base import BaseModule\n\n\nclass google_playstore(BaseModule):\n watched_events = [\"ORG_STUB\", \"CODE_REPOSITORY\"]\n produced_events = [\"MOBILE_APP\"]\n flags = [\"passive\", \"safe\", \"code-enum\"]\n meta = {\n \"description\": \"Search for android applications on play.google.com\",\n \"created_date\": \"2024-10-08\",\n \"author\": \"@domwhewell-sage\",\n }\n\n base_url = \"https://play.google.com\"\n\n async def setup(self):\n self.app_link_regex = self.helpers.re.compile(r\"/store/apps/details\\?id=([a-zA-Z0-9._-]+)\")\n return True\n\n async def filter_event(self, event):\n if event.type == \"CODE_REPOSITORY\":\n if \"android\" not in event.tags:\n return False, \"event is not an android repository\"\n return True\n\n async def handle_event(self, event):\n if event.type == \"CODE_REPOSITORY\":\n await self.handle_url(event)\n elif event.type == \"ORG_STUB\":\n await self.handle_org_stub(event)\n\n async def handle_url(self, event):\n repo_url = event.data.get(\"url\")\n app_id = repo_url.split(\"id=\")[1].split(\"&\")[0]\n await self.emit_event(\n {\"id\": app_id, \"url\": repo_url},\n \"MOBILE_APP\",\n tags=\"android\",\n parent=event,\n context=f'{{module}} extracted the mobile app name \"{app_id}\" from: {repo_url}',\n )\n\n async def handle_org_stub(self, event):\n org_name = event.data\n self.verbose(f\"Searching for any android applications for {org_name}\")\n for apk_name in await self.query(org_name):\n valid_apk = await self.validate_apk(apk_name)\n if valid_apk:\n self.verbose(f\"Got {apk_name} from playstore\")\n await self.emit_event(\n {\"id\": apk_name, \"url\": f\"{self.base_url}/store/apps/details?id={apk_name}\"},\n \"MOBILE_APP\",\n tags=\"android\",\n parent=event,\n context=f'{{module}} searched play.google.com for apps belonging to \"{org_name}\" and found \"{apk_name}\" to be in scope',\n )\n else:\n self.debug(f\"Got {apk_name} from playstore app details does not contain any in-scope URLs or Emails\")\n\n async def query(self, query):\n app_links = []\n url = f\"{self.base_url}/store/search?q={self.helpers.quote(query)}&c=apps\"\n r = await self.helpers.request(url)\n if r is None:\n return app_links\n status_code = getattr(r, \"status_code\", 0)\n try:\n html_content = r.content.decode(\"utf-8\")\n # Use regex to find all app links\n app_links = await self.helpers.re.findall(self.app_link_regex, html_content)\n except Exception as e:\n self.warning(f\"Failed to parse html response from {r.url} (HTTP status: {status_code}): {e}\")\n return app_links\n return app_links\n\n async def validate_apk(self, apk_name):\n \"\"\"\n Check the app details page the \"App support\" section will include URLs or Emails to the app developer\n \"\"\"\n in_scope = False\n url = f\"{self.base_url}/store/apps/details?id={apk_name}\"\n r = await self.helpers.request(url)\n if r is None:\n return in_scope\n status_code = getattr(r, \"status_code\", 0)\n if status_code == 200:\n html = r.text\n in_scope_hosts = await self.scan.extract_in_scope_hostnames(html)\n if in_scope_hosts:\n in_scope = True\n else:\n self.warning(f\"Failed to fetch {url} (HTTP status: {status_code})\")\n return in_scope\n" }, { "path": "bbot/modules/gowitness.py", "content": "import os\nimport asyncio\nimport aiosqlite\nimport multiprocessing\nimport platform\nfrom pathlib import Path\nfrom contextlib import suppress\nfrom shutil import copyfile, copymode\n\nfrom bbot.modules.base import BaseModule\n\n\nclass gowitness(BaseModule):\n watched_events = [\"URL\", \"SOCIAL\"]\n produced_events = [\"WEBSCREENSHOT\", \"URL\", \"URL_UNVERIFIED\", \"TECHNOLOGY\"]\n flags = [\"active\", \"safe\", \"web-screenshots\"]\n meta = {\"description\": \"Take screenshots of webpages\", \"created_date\": \"2022-07-08\", \"author\": \"@TheTechromancer\"}\n options = {\n \"version\": \"3.0.5\",\n \"threads\": 0,\n \"timeout\": 10,\n \"resolution_x\": 1440,\n \"resolution_y\": 900,\n \"output_path\": \"\",\n \"social\": False,\n \"idle_timeout\": 1800,\n \"chrome_path\": \"\",\n }\n options_desc = {\n \"version\": \"Gowitness version\",\n \"threads\": \"How many gowitness threads to spawn (default is number of CPUs x 2)\",\n \"timeout\": \"Preflight check timeout\",\n \"resolution_x\": \"Screenshot resolution x\",\n \"resolution_y\": \"Screenshot resolution y\",\n \"output_path\": \"Where to save screenshots\",\n \"social\": \"Whether to screenshot social media webpages\",\n \"idle_timeout\": \"Skip the current gowitness batch if it stalls for longer than this many seconds\",\n \"chrome_path\": \"Path to chrome executable\",\n }\n deps_common = [\"chromium\"]\n deps_pip = [\"aiosqlite\"]\n deps_ansible = [\n {\n \"name\": \"Download gowitness\",\n \"get_url\": {\n \"url\": \"https://github.com/sensepost/gowitness/releases/download/#{BBOT_MODULES_GOWITNESS_VERSION}/gowitness-#{BBOT_MODULES_GOWITNESS_VERSION}-#{BBOT_OS_PLATFORM}-#{BBOT_CPU_ARCH_GOLANG}\",\n \"dest\": \"#{BBOT_TOOLS}/gowitness\",\n \"mode\": \"755\",\n },\n },\n ]\n _batch_size = 100\n # gowitness accepts SOCIAL events up to distance 2, otherwise it is in-scope-only\n scope_distance_modifier = 2\n\n async def setup(self):\n num_cpus = multiprocessing.cpu_count()\n default_thread_count = min(20, num_cpus * 2)\n self.timeout = self.config.get(\"timeout\", 10)\n self.idle_timeout = self.config.get(\"idle_timeout\", 1800)\n self.threads = self.config.get(\"threads\", 0)\n if not self.threads:\n self.threads = default_thread_count\n self.proxy = self.scan.web_config.get(\"http_proxy\", \"\")\n self.resolution_x = self.config.get(\"resolution_x\")\n self.resolution_y = self.config.get(\"resolution_y\")\n self.visit_social = self.config.get(\"social\", True)\n output_path = self.config.get(\"output_path\")\n if output_path:\n self.base_path = Path(output_path) / \"gowitness\"\n else:\n self.base_path = self.scan.home / \"gowitness\"\n\n self.chrome_path = None\n config_chrome_path = self.config.get(\"chrome_path\")\n if config_chrome_path:\n config_chrome_path = Path(config_chrome_path)\n if not config_chrome_path.is_file():\n return False, f\"Could not find custom Chrome path at {config_chrome_path}\"\n self.chrome_path = config_chrome_path\n else:\n if platform.system() == \"Darwin\":\n bbot_chrome_path = (\n self.helpers.tools_dir / \"chrome-mac\" / \"Chromium.app\" / \"Contents\" / \"MacOS\" / \"Chromium\"\n )\n else:\n bbot_chrome_path = self.helpers.tools_dir / \"chrome-linux\" / \"chrome\"\n if bbot_chrome_path.is_file():\n self.chrome_path = bbot_chrome_path\n\n # make sure our chrome path works\n chrome_test_pass = False\n if self.chrome_path and self.chrome_path.is_file():\n chrome_test_proc = await self.run_process([str(self.chrome_path), \"--version\"])\n if getattr(chrome_test_proc, \"returncode\", 1) == 0:\n self.verbose(f\"Found chrome executable at {self.chrome_path}\")\n chrome_test_pass = True\n\n if not chrome_test_pass:\n # last resort - try to find a working chrome install\n for binary in (\"Google Chrome\", \"chrome\", \"chromium\", \"chromium-browser\"):\n binary_path = self.helpers.which(binary)\n if binary_path and Path(binary_path).is_file():\n chrome_test_proc = await self.run_process([str(binary_path), \"--version\"])\n if getattr(chrome_test_proc, \"returncode\", 1) == 0:\n self.verbose(f\"Found chrome executable at {binary_path}\")\n chrome_test_pass = True\n break\n\n if not chrome_test_pass:\n return (\n False,\n \"Failed to set up Google chrome. Please install manually and set `chrome_path`, or try again with --force-deps.\",\n )\n\n # fix ubuntu-specific sandbox bug\n chrome_devel_sandbox = self.helpers.tools_dir / \"chrome-linux\" / \"chrome_sandbox\"\n if chrome_devel_sandbox.is_file():\n os.environ[\"CHROME_DEVEL_SANDBOX\"] = str(chrome_devel_sandbox)\n\n self.db_path = self.base_path / \"gowitness.sqlite3\"\n self.screenshot_path = self.base_path / \"screenshots\"\n self.command = self.construct_command()\n self.prepped = False\n self.screenshots_taken = {}\n self.connections_logged = set()\n self.technologies_found = set()\n return True\n\n def prep(self):\n if not self.prepped:\n self.helpers.mkdir(self.screenshot_path)\n self.db_path.touch()\n with suppress(Exception):\n copyfile(self.helpers.tools_dir / \"gowitness\", self.base_path / \"gowitness\")\n copymode(self.helpers.tools_dir / \"gowitness\", self.base_path / \"gowitness\")\n self.prepped = True\n\n async def filter_event(self, event):\n # Ignore URLs that are redirects\n if any(t.startswith(\"status-30\") for t in event.tags):\n return False, \"URL is a redirect\"\n # ignore events from self\n if event.type == \"URL\" and event.module == self:\n return False, \"event is from self\"\n if event.type == \"SOCIAL\":\n if not self.visit_social:\n return False, \"visit_social=False\"\n else:\n # Accept out-of-scope SOCIAL pages, but not URLs\n if event.scope_distance > 0:\n return False, \"event is not in-scope\"\n return True\n\n async def handle_batch(self, *events):\n self.prep()\n event_dict = {}\n for e in events:\n key = e.data\n if e.type == \"SOCIAL\":\n key = e.data[\"url\"]\n event_dict[key] = e\n stdin = \"\\n\".join(list(event_dict))\n\n try:\n async for line in self.run_process_live(self.command, input=stdin, idle_timeout=self.idle_timeout):\n self.debug(line)\n except asyncio.exceptions.TimeoutError:\n urls_str = \",\".join(event_dict)\n self.warning(f\"Gowitness timed out while visiting the following URLs: {urls_str}\", trace=False)\n return\n\n # emit web screenshots\n new_screenshots = await self.get_new_screenshots()\n for filename, screenshot in new_screenshots.items():\n url = screenshot[\"url\"]\n url = self.helpers.clean_url(url).geturl()\n final_url = screenshot[\"final_url\"]\n filename = self.screenshot_path / screenshot[\"filename\"]\n filename = filename.relative_to(self.scan.home)\n # NOTE: this prevents long filenames from causing problems in BBOT, but gowitness will still fail to save it.\n filename = self.helpers.truncate_filename(filename)\n webscreenshot_data = {\"path\": str(filename), \"url\": final_url}\n parent_event = event_dict[url]\n await self.emit_event(\n webscreenshot_data,\n \"WEBSCREENSHOT\",\n parent=parent_event,\n context=f\"{{module}} visited {final_url} and saved {{event.type}} to {filename}\",\n )\n\n # emit URLs\n new_network_logs = await self.get_new_network_logs()\n for url, row in new_network_logs.items():\n ip = row[\"remote_ip\"]\n status_code = row[\"status_code\"]\n tags = [f\"status-{status_code}\", f\"ip-{ip}\", \"spider-danger\"]\n\n _id = row[\"result_id\"]\n parent_url = self.screenshots_taken[_id]\n parent_event = event_dict[parent_url]\n if url and url.startswith(\"http\"):\n await self.emit_event(\n url,\n \"URL_UNVERIFIED\",\n parent=parent_event,\n tags=tags,\n context=f\"{{module}} visited {{event.type}}: {url}\",\n )\n\n # emit technologies\n new_technologies = await self.get_new_technologies()\n for row in new_technologies.values():\n parent_id = row[\"result_id\"]\n parent_url = self.screenshots_taken[parent_id]\n parent_event = event_dict[parent_url]\n technology = row[\"value\"]\n tech_data = {\"technology\": technology, \"url\": parent_url, \"host\": str(parent_event.host)}\n await self.emit_event(\n tech_data,\n \"TECHNOLOGY\",\n parent=parent_event,\n context=f\"{{module}} visited {parent_url} and found {{event.type}}: {technology}\",\n )\n\n def construct_command(self):\n # base executable\n command = [\"gowitness\", \"scan\"]\n # chrome path\n if self.chrome_path is not None:\n command += [\"--chrome-path\", str(self.chrome_path)]\n # db path\n command += [\"--write-db\"]\n command += [\"--write-db-uri\", f\"sqlite://{self.db_path}\"]\n # screenshot path\n command += [\"--screenshot-path\", str(self.screenshot_path)]\n # user agent\n command += [\"--chrome-user-agent\", f\"{self.scan.useragent}\"]\n # proxy\n if self.proxy:\n command += [\"--chrome-proxy\", str(self.proxy)]\n # resolution\n command += [\"--chrome-window-x\", str(self.resolution_x)]\n command += [\"--chrome-window-y\", str(self.resolution_y)]\n # threads\n command += [\"--threads\", str(self.threads)]\n # timeout\n command += [\"--timeout\", str(self.timeout)]\n # input\n command += [\"file\", \"-f\", \"-\"]\n return command\n\n async def get_new_screenshots(self):\n screenshots = {}\n if self.db_path.is_file():\n async with aiosqlite.connect(str(self.db_path)) as con:\n con.row_factory = aiosqlite.Row\n con.text_factory = self.helpers.smart_decode\n async with con.execute(\"SELECT * FROM results\") as cur:\n async for row in cur:\n row = dict(row)\n _id = row[\"id\"]\n if _id not in self.screenshots_taken:\n self.screenshots_taken[_id] = row[\"url\"]\n screenshots[_id] = row\n return screenshots\n\n async def get_new_network_logs(self):\n network_logs = {}\n if self.db_path.is_file():\n async with aiosqlite.connect(str(self.db_path)) as con:\n con.row_factory = aiosqlite.Row\n async with con.execute(\"SELECT * FROM network_logs\") as cur:\n async for row in cur:\n row = dict(row)\n url = row[\"url\"]\n if url not in self.connections_logged:\n self.connections_logged.add(url)\n network_logs[url] = row\n return network_logs\n\n async def get_new_technologies(self):\n technologies = {}\n if self.db_path.is_file():\n async with aiosqlite.connect(str(self.db_path)) as con:\n con.row_factory = aiosqlite.Row\n async with con.execute(\"SELECT * FROM technologies\") as cur:\n async for row in cur:\n _id = row[\"id\"]\n if _id not in self.technologies_found:\n self.technologies_found.add(_id)\n row = dict(row)\n technologies[_id] = row\n return technologies\n\n async def cur_execute(self, cur, query):\n try:\n return await cur.execute(query)\n except aiosqlite.OperationalError as e:\n self.warning(f\"Error executing query: {query}: {e}\")\n return []\n\n async def report(self):\n if self.screenshots_taken:\n self.success(f\"{len(self.screenshots_taken):,} web screenshots captured. To view:\")\n self.success(\" - Start gowitness\")\n self.success(f\" - cd {self.base_path} && ./gowitness server\")\n self.success(\" - Browse to http://localhost:7171\")\n else:\n self.info(\"No web screenshots captured\")\n" }, { "path": "bbot/modules/graphql_introspection.py", "content": "import json\nfrom pathlib import Path\nfrom bbot.modules.base import BaseModule\n\n\nclass graphql_introspection(BaseModule):\n watched_events = [\"URL\"]\n produced_events = [\"FINDING\"]\n flags = [\"safe\", \"active\", \"web-basic\"]\n meta = {\n \"description\": \"Perform GraphQL introspection on a target\",\n \"created_date\": \"2025-07-01\",\n \"author\": \"@mukesh-dream11\",\n }\n options = {\n \"graphql_endpoint_urls\": [\"/\", \"/graphql\", \"/v1/graphql\"],\n \"output_folder\": \"\",\n }\n options_desc = {\n \"graphql_endpoint_urls\": \"List of GraphQL endpoint to suffix to the target URL\",\n \"output_folder\": \"Folder to save the GraphQL schemas to\",\n }\n\n async def setup(self):\n output_folder = self.config.get(\"output_folder\", \"\")\n if output_folder:\n self.output_dir = Path(output_folder) / \"graphql-schemas\"\n else:\n self.output_dir = self.scan.home / \"graphql-schemas\"\n return True\n\n async def filter_event(self, event):\n # Dedup by the base URL\n base_url = event.parsed_url._replace(path=\"/\", query=\"\", fragment=\"\").geturl()\n return hash(base_url)\n\n async def handle_event(self, event):\n base_url = event.parsed_url._replace(path=\"/\", query=\"\", fragment=\"\").geturl().rstrip(\"/\")\n for endpoint_url in self.config.get(\"graphql_endpoint_urls\", []):\n url = f\"{base_url}{endpoint_url}\"\n request_args = {\n \"url\": url,\n \"method\": \"POST\",\n \"json\": {\n \"query\": \"\"\"\\\nquery IntrospectionQuery {\n __schema {\n queryType {\n name\n }\n mutationType {\n name\n }\n types {\n name\n kind\n description\n fields(includeDeprecated: true) {\n name\n description\n type {\n ... TypeRef\n }\n isDeprecated\n deprecationReason\n }\n interfaces {\n ... TypeRef\n }\n possibleTypes {\n ... TypeRef\n }\n enumValues(includeDeprecated: true) {\n name\n description\n isDeprecated\n deprecationReason\n }\n ofType {\n ... TypeRef\n }\n }\n }\n}\n\nfragment TypeRef on __Type {\n kind\n name\n ofType {\n kind\n name\n ofType {\n kind\n name\n ofType {\n kind\n name\n ofType {\n kind\n name\n ofType {\n kind\n name\n ofType {\n kind\n name\n ofType {\n kind\n name\n }\n }\n }\n }\n }\n }\n }\n}\"\"\"\n },\n }\n response = await self.helpers.request(**request_args)\n if not response or response.status_code != 200:\n self.debug(\n f\"Failed to get GraphQL schema for {url} \"\n f\"{f'(status code {response.status_code})' if response else ''}\"\n )\n continue\n try:\n response_json = response.json()\n except json.JSONDecodeError:\n self.debug(f\"Failed to parse JSON for {url}\")\n continue\n if response_json.get(\"data\", {}).get(\"__schema\", {}).get(\"types\", []):\n self.helpers.mkdir(self.output_dir)\n filename = f\"schema-{self.helpers.tagify(url)}.json\"\n filename = self.output_dir / filename\n with open(filename, \"w\") as f:\n json.dump(response_json, f)\n await self.emit_event(\n {\"url\": url, \"description\": \"GraphQL schema\", \"path\": str(filename.relative_to(self.scan.home))},\n \"FINDING\",\n event,\n context=f\"{{module}} found GraphQL schema at {url}\",\n )\n # return, because we only want to find one schema per target\n return\n" }, { "path": "bbot/modules/hackertarget.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass hackertarget(subdomain_enum):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query the hackertarget.com API for subdomains\",\n \"created_date\": \"2022-07-28\",\n \"author\": \"@TheTechromancer\",\n }\n\n base_url = \"https://api.hackertarget.com\"\n\n async def request_url(self, query):\n url = f\"{self.base_url}/hostsearch/?q={self.helpers.quote(query)}\"\n response = await self.api_request(url)\n return response\n\n async def parse_results(self, r, query):\n results = set()\n for line in r.text.splitlines():\n host = line.split(\",\")[0]\n try:\n self.helpers.validators.validate_host(host)\n results.add(host)\n except ValueError:\n self.debug(f\"Error validating API result: {line}\")\n continue\n return results\n" }, { "path": "bbot/modules/host_header.py", "content": "from bbot.errors import InteractshError\nfrom bbot.modules.base import BaseModule\n\n\nclass host_header(BaseModule):\n watched_events = [\"HTTP_RESPONSE\"]\n produced_events = [\"FINDING\"]\n flags = [\"active\", \"aggressive\", \"web-thorough\"]\n meta = {\n \"description\": \"Try common HTTP Host header spoofing techniques\",\n \"created_date\": \"2022-07-27\",\n \"author\": \"@liquidsec\",\n }\n\n in_scope_only = True\n per_hostport_only = True\n\n deps_apt = [\"curl\"]\n\n async def setup(self):\n self.subdomain_tags = {}\n if self.scan.config.get(\"interactsh_disable\", False) is False:\n try:\n self.interactsh_instance = self.helpers.interactsh()\n self.domain = await self.interactsh_instance.register(callback=self.interactsh_callback)\n except InteractshError as e:\n self.warning(f\"Interactsh failure: {e}\")\n return False\n else:\n self.warning(\"Interactsh is disabled globally. Interaction based detections will be disabled.\")\n self.domain = f\"{self.rand_string(12, digits=False)}.com\"\n return True\n\n def rand_string(self, *args, **kwargs):\n return self.helpers.rand_string(*args, **kwargs)\n\n async def interactsh_callback(self, r):\n full_id = r.get(\"full-id\", None)\n if full_id:\n if \".\" in full_id:\n match = self.subdomain_tags.get(full_id.split(\".\")[0])\n if match is None:\n return\n matched_event = match[0]\n matched_technique = match[1]\n\n protocol = r.get(\"protocol\").upper()\n await self.emit_event(\n {\n \"host\": str(matched_event.host),\n \"url\": matched_event.data[\"url\"],\n \"description\": f\"Spoofed Host header ({matched_technique}) [{protocol}] interaction\",\n },\n \"FINDING\",\n matched_event,\n context=f\"{{module}} spoofed host header and induced {{event.type}}: {protocol} interaction\",\n )\n else:\n # this is likely caused by something trying to resolve the base domain first and can be ignored\n self.debug(\"skipping results because subdomain tag was missing\")\n\n async def finish(self):\n if self.scan.config.get(\"interactsh_disable\", False) is False:\n await self.helpers.sleep(5)\n try:\n for r in await self.interactsh_instance.poll():\n await self.interactsh_callback(r)\n except InteractshError as e:\n self.debug(f\"Error in interact.sh: {e}\")\n\n async def cleanup(self):\n if self.scan.config.get(\"interactsh_disable\", False) is False:\n try:\n await self.interactsh_instance.deregister()\n self.debug(\n f\"successfully deregistered interactsh session with correlation_id {self.interactsh_instance.correlation_id}\"\n )\n except InteractshError as e:\n self.warning(f\"Interactsh failure: {e}\")\n\n async def handle_event(self, event):\n # get any set-cookie responses from the response and add them to the request\n url = event.data[\"url\"]\n\n added_cookies = {}\n\n for header_values in event.data[\"header-dict\"].values():\n for header_value in header_values:\n if header_value.lower() == \"set-cookie\":\n header_split = header_value.split(\"=\")\n try:\n added_cookies = {header_split[0]: header_split[1]}\n except IndexError:\n self.debug(f\"failed to parse cookie from string {header_value}\")\n\n domain_reflections = []\n\n # host header replacement\n technique_description = \"standard\"\n self.debug(f\"Performing {technique_description} case\")\n subdomain_tag = self.rand_string(4, digits=False)\n self.subdomain_tags[subdomain_tag] = (event, technique_description)\n output = await self.helpers.curl(\n url=url,\n headers={\"Host\": f\"{subdomain_tag}.{self.domain}\"},\n ignore_bbot_global_settings=True,\n cookies=added_cookies,\n )\n if self.domain in output:\n domain_reflections.append(technique_description)\n\n # absolute URL / Host header transposition\n technique_description = \"absolute URL transposition\"\n self.debug(f\"Performing {technique_description} case\")\n subdomain_tag = self.rand_string(4, digits=False)\n self.subdomain_tags[subdomain_tag] = (event, technique_description)\n output = await self.helpers.curl(\n url=url,\n path_override=url,\n cookies=added_cookies,\n )\n\n if self.domain in output:\n domain_reflections.append(technique_description)\n\n # duplicate host header tolerance\n technique_description = \"duplicate host header tolerance\"\n output = await self.helpers.curl(\n url=url,\n # Sending a blank HOST first as a hack to trick curl. This makes it no longer an \"internal header\", thereby allowing for duplicates\n # The fact that it's accepting two host headers is rare enough to note on its own, and not too noisy. Having the 3rd header be an interactsh would result in false negatives for the slightly less interesting cases.\n headers={\"Host\": [\"\", str(event.host), str(event.host)]},\n cookies=added_cookies,\n head_mode=True,\n )\n\n split_output = output.split(\"\\n\")\n if \" 4\" in split_output:\n description = \"Duplicate Host Header Tolerated\"\n await self.emit_event(\n {\n \"host\": str(event.host),\n \"url\": url,\n \"description\": description,\n },\n \"FINDING\",\n event,\n context=f\"{{module}} scanned {event.data['url']} and identified {{event.type}}: {description}\",\n )\n\n # host header overrides\n technique_description = \"host override headers\"\n self.verbose(f\"Performing {technique_description} case\")\n subdomain_tag = self.rand_string(4, digits=False)\n self.subdomain_tags[subdomain_tag] = (event, technique_description)\n\n override_headers_list = [\n \"X-Host\",\n \"X-Forwarded-Server\",\n \"X-Forwarded-Host\",\n \"X-Original-Host\",\n \"X-Forwarded-For\",\n \"X-Host\",\n \"X-HTTP-Host-Override\",\n \"Forwarded\",\n ]\n override_headers = {}\n for oh in override_headers_list:\n override_headers[oh] = f\"{subdomain_tag}.{self.domain}\"\n\n output = await self.helpers.curl(\n url=url,\n headers=override_headers,\n cookies=added_cookies,\n )\n if self.domain in output:\n domain_reflections.append(technique_description)\n\n # emit all the domain reflections we found\n for dr in domain_reflections:\n description = f\"Possible Host header injection. Injection technique: {dr}\"\n await self.emit_event(\n {\n \"host\": str(event.host),\n \"url\": url,\n \"description\": description,\n },\n \"FINDING\",\n event,\n context=f\"{{module}} scanned {url} and identified {{event.type}}: {description}\",\n )\n" }, { "path": "bbot/modules/httpx.py", "content": "import re\nimport orjson\nimport tempfile\nimport subprocess\nfrom pathlib import Path\nfrom http.cookies import SimpleCookie\n\nfrom bbot.modules.base import BaseModule\n\n\nclass httpx(BaseModule):\n watched_events = [\"OPEN_TCP_PORT\", \"URL_UNVERIFIED\", \"URL\"]\n produced_events = [\"URL\", \"HTTP_RESPONSE\"]\n flags = [\"active\", \"safe\", \"web-basic\", \"social-enum\", \"subdomain-enum\", \"cloud-enum\"]\n meta = {\n \"description\": \"Visit webpages. Many other modules rely on httpx\",\n \"created_date\": \"2022-07-08\",\n \"author\": \"@TheTechromancer\",\n }\n\n options = {\n \"threads\": 50,\n \"in_scope_only\": True,\n \"version\": \"1.2.5\",\n \"max_response_size\": 5242880,\n \"store_responses\": False,\n \"probe_all_ips\": False,\n }\n options_desc = {\n \"threads\": \"Number of httpx threads to use\",\n \"in_scope_only\": \"Only visit web reparents that are in scope.\",\n \"version\": \"httpx version\",\n \"max_response_size\": \"Max response size in bytes\",\n \"store_responses\": \"Save raw HTTP responses to scan folder\",\n \"probe_all_ips\": \"Probe all the ips associated with same host\",\n }\n deps_ansible = [\n {\n \"name\": \"Download httpx\",\n \"unarchive\": {\n \"src\": \"https://github.com/projectdiscovery/httpx/releases/download/v#{BBOT_MODULES_HTTPX_VERSION}/httpx_#{BBOT_MODULES_HTTPX_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH_GOLANG}.zip\",\n \"include\": \"httpx\",\n \"dest\": \"#{BBOT_TOOLS}\",\n \"remote_src\": True,\n },\n }\n ]\n\n scope_distance_modifier = 2\n _shuffle_incoming_queue = False\n _batch_size = 500\n _priority = 2\n # accept Javascript URLs\n accept_url_special = True\n\n async def setup(self):\n self.threads = self.config.get(\"threads\", 50)\n self.max_response_size = self.config.get(\"max_response_size\", 5242880)\n self.store_responses = self.config.get(\"store_responses\", False)\n self.probe_all_ips = self.config.get(\"probe_all_ips\", False)\n self.httpx_tempdir_regex = re.compile(r\"^httpx\\d+$\")\n return True\n\n async def filter_event(self, event):\n if \"_wildcard\" in str(event.host).split(\".\"):\n return False, \"event is wildcard\"\n\n if \"unresolved\" in event.tags:\n return False, \"event is unresolved\"\n\n if event.module == self:\n return False, \"event is from self\"\n\n if \"spider-max\" in event.tags:\n return False, \"event exceeds spidering limits\"\n\n # scope filtering\n in_scope_only = self.config.get(\"in_scope_only\", True)\n if \"httpx-safe\" in event.tags:\n return True\n max_scope_distance = 0 if in_scope_only else (self.scan.scope_search_distance + 1)\n if event.scope_distance > max_scope_distance:\n return False, \"event is not in scope\"\n return True\n\n def make_url_metadata(self, event):\n has_spider_max = \"spider-max\" in event.tags\n url_hash = None\n if event.type.startswith(\"URL\"):\n # we NEED the port, otherwise httpx will try HTTPS even for HTTP URLs\n url = event.with_port().geturl()\n if event.parsed_url.path == \"/\":\n url_hash = hash((event.host, event.port, has_spider_max))\n else:\n url = str(event.data)\n url_hash = hash((event.host, event.port, has_spider_max))\n if url_hash is None:\n url_hash = hash((url, has_spider_max))\n return url, url_hash\n\n def _incoming_dedup_hash(self, event):\n url, url_hash = self.make_url_metadata(event)\n return url_hash\n\n async def handle_batch(self, *events):\n stdin = {}\n\n for event in events:\n url, url_hash = self.make_url_metadata(event)\n stdin[url] = event\n\n if not stdin:\n return\n\n command = [\n \"httpx\",\n \"-silent\",\n \"-json\",\n \"-include-response\",\n \"-threads\",\n self.threads,\n \"-timeout\",\n self.scan.httpx_timeout,\n \"-retries\",\n self.scan.httpx_retries,\n \"-header\",\n f\"User-Agent: {self.scan.useragent}\",\n \"-response-size-to-read\",\n f\"{self.max_response_size}\",\n ]\n\n if self.store_responses:\n response_dir = self.scan.home / \"httpx\"\n self.helpers.mkdir(response_dir)\n command += [\"-srd\", str(response_dir)]\n\n dns_resolvers = \",\".join(self.helpers.system_resolvers)\n if dns_resolvers:\n command += [\"-r\", dns_resolvers]\n\n if self.probe_all_ips:\n command += [\"-probe-all-ips\"]\n\n # Add custom HTTP headers\n for hk, hv in self.scan.custom_http_headers.items():\n command += [\"-header\", f\"{hk}: {hv}\"]\n\n # Add custom HTTP cookies as a single header\n if self.scan.custom_http_cookies:\n cookie = SimpleCookie()\n for ck, cv in self.scan.custom_http_cookies.items():\n cookie[ck] = cv\n\n # Build the cookie header\n cookie_header = f\"Cookie: {cookie.output(header='', sep='; ').strip()}\"\n command += [\"-header\", cookie_header]\n\n proxy = self.scan.http_proxy\n if proxy:\n command += [\"-http-proxy\", proxy]\n async for line in self.run_process_live(command, text=False, input=list(stdin), stderr=subprocess.DEVNULL):\n try:\n j = await self.helpers.run_in_executor(orjson.loads, line)\n except orjson.JSONDecodeError:\n self.warning(f\"httpx failed to decode line: {line}\")\n continue\n\n url = j.get(\"url\", \"\")\n status_code = int(j.get(\"status_code\", 0))\n if status_code == 0:\n self.debug(f'No HTTP status code for \"{url}\"')\n continue\n\n parent_event = stdin.get(j.get(\"input\", \"\"), None)\n\n if parent_event is None:\n self.warning(f\"Unable to correlate parent event from: {line}\")\n continue\n\n # discard 404s from unverified URLs\n path = j.get(\"path\", \"/\")\n if parent_event.type == \"URL_UNVERIFIED\" and status_code in (404,) and path != \"/\":\n self.debug(f'Discarding 404 from \"{url}\"')\n continue\n\n # main URL\n tags = [f\"status-{status_code}\"]\n httpx_ip = j.get(\"host\", \"\")\n if httpx_ip:\n tags.append(f\"ip-{httpx_ip}\")\n # grab title\n title = self.helpers.tagify(j.get(\"title\", \"\"), maxlen=30)\n if title:\n tags.append(f\"http-title-{title}\")\n\n url_context = \"{module} visited {event.parent.data} and got status code {event.http_status}\"\n if parent_event.type == \"OPEN_TCP_PORT\":\n url_context += \" at {event.data}\"\n\n url_event = self.make_event(\n url,\n \"URL\",\n parent_event,\n tags=tags,\n context=url_context,\n )\n if url_event:\n if url_event != parent_event:\n await self.emit_event(url_event)\n # HTTP response\n content_type = j.get(\"header\", {}).get(\"content_type\", \"unspecified\").split(\";\")[0]\n content_length = j.get(\"content_length\", 0)\n content_length = self.helpers.bytes_to_human(content_length)\n await self.emit_event(\n j,\n \"HTTP_RESPONSE\",\n url_event,\n tags=url_event.tags,\n context=f\"HTTP_RESPONSE was {content_length} with {content_type} content type\",\n )\n\n for tempdir in Path(tempfile.gettempdir()).iterdir():\n if tempdir.is_dir() and self.httpx_tempdir_regex.match(tempdir.name):\n self.helpers.rm_rf(tempdir)\n\n async def cleanup(self):\n resume_file = self.helpers.current_dir / \"resume.cfg\"\n resume_file.unlink(missing_ok=True)\n" }, { "path": "bbot/modules/hunt.py", "content": "# adapted from https://github.com/bugcrowd/HUNT\n\nfrom bbot.modules.base import BaseModule\n\nhunt_param_dict = {\n \"Command Injection\": [\n \"daemon\",\n \"host\",\n \"upload\",\n \"dir\",\n \"execute\",\n \"download\",\n \"log\",\n \"ip\",\n \"cli\",\n \"cmd\",\n \"exec\",\n \"command\",\n \"func\",\n \"code\",\n \"update\",\n \"shell\",\n \"eval\",\n ],\n \"Debug\": [\n \"access\",\n \"admin\",\n \"dbg\",\n \"debug\",\n \"edit\",\n \"grant\",\n \"test\",\n \"alter\",\n \"clone\",\n \"create\",\n \"delete\",\n \"disable\",\n \"enable\",\n \"exec\",\n \"execute\",\n \"load\",\n \"make\",\n \"modify\",\n \"rename\",\n \"reset\",\n \"shell\",\n \"toggle\",\n \"adm\",\n \"root\",\n \"cfg\",\n \"config\",\n ],\n \"Directory Traversal\": [\n \"entry\",\n \"download\",\n \"attachment\",\n \"basepath\",\n \"path\",\n \"file\",\n \"source\",\n \"dest\",\n ],\n \"Local File Include\": [\n \"file\",\n \"document\",\n \"folder\",\n \"root\",\n \"path\",\n \"pg\",\n \"style\",\n \"pdf\",\n \"template\",\n \"php_path\",\n \"doc\",\n \"lang\",\n \"include\",\n \"img\",\n \"view\",\n \"layout\",\n \"export\",\n \"log\",\n \"configFile\",\n \"stylesheet\",\n \"configFileUrl\",\n ],\n \"Insecure Direct Object Reference\": [\n \"id\",\n \"user\",\n \"account\",\n \"number\",\n \"order\",\n \"no\",\n \"doc\",\n \"key\",\n \"email\",\n \"group\",\n \"profile\",\n \"edit\",\n \"report\",\n \"docId\",\n \"accountId\",\n \"customerId\",\n \"reportId\",\n \"jobId\",\n \"sessionId\",\n \"api_key\",\n \"instance\",\n \"identifier\",\n \"access\",\n ],\n \"SQL Injection\": [\n \"id\",\n \"select\",\n \"report\",\n \"role\",\n \"update\",\n \"query\",\n \"user\",\n \"name\",\n \"sort\",\n \"where\",\n \"search\",\n \"params\",\n \"category\",\n \"process\",\n \"row\",\n \"view\",\n \"table\",\n \"from\",\n \"sel\",\n \"results\",\n \"sleep\",\n \"fetch\",\n \"order\",\n \"keyword\",\n \"column\",\n \"field\",\n \"delete\",\n \"string\",\n \"number\",\n \"filter\",\n \"limit\",\n \"offset\",\n \"item\",\n \"input\",\n \"date\",\n \"value\",\n \"orderBy\",\n \"groupBy\",\n \"pageNum\",\n \"pageSize\",\n \"tag\",\n \"author\",\n \"postId\",\n \"parentId\",\n \"d\",\n ],\n \"Server-side Request Forgery\": [\n \"dest\",\n \"redirect\",\n \"uri\",\n \"path\",\n \"continue\",\n \"url\",\n \"window\",\n \"next\",\n \"data\",\n \"reference\",\n \"site\",\n \"html\",\n \"val\",\n \"validate\",\n \"domain\",\n \"callback\",\n \"return\",\n \"page\",\n \"feed\",\n \"host\",\n \"port\",\n \"to\",\n \"out\",\n \"view\",\n \"dir\",\n \"show\",\n \"navigation\",\n \"open\",\n \"proxy\",\n \"target\",\n \"server\",\n \"domain\",\n \"connect\",\n \"fetch\",\n \"apiEndpoint\",\n ],\n \"Server-Side Template Injection\": [\n \"template\",\n \"preview\",\n \"id\",\n \"view\",\n \"activity\",\n \"name\",\n \"content\",\n \"redirect\",\n \"expression\",\n \"statement\",\n \"tpl\",\n \"render\",\n \"format\",\n \"engine\",\n ],\n \"XML external entity injection\": [\n \"xml\",\n \"dtd\",\n \"xsd\",\n \"xmlDoc\",\n \"xmlData\",\n \"entityType\",\n \"entity\",\n \"xmlUrl\",\n \"schema\",\n \"xmlFile\",\n \"xmlPath\",\n \"xmlSource\",\n \"xmlEndpoint\",\n \"xslt\",\n \"xmlConfig\",\n \"xmlCallback\",\n \"attributeName\",\n \"wsdl\",\n \"xmlDocUrl\",\n ],\n \"Insecure Cryptography\": [\n \"encrypted\",\n \"cipher\",\n \"iv\",\n \"checksum\",\n \"hash\",\n \"salt\",\n \"hmac\",\n \"secret\",\n \"key\",\n \"signatureAlgorithm\",\n \"keyId\",\n \"sharedSecret\",\n \"privateKeyId\",\n \"privateKey\",\n \"publicKey\",\n \"publicKeyId\",\n \"encryptedData\",\n \"encryptedMessage\",\n \"encryptedPayload\",\n \"encryptedFile\",\n \"cipherText\",\n \"cipherAlgorithm\",\n \"keySize\",\n \"keyPair\",\n \"keyDerivation\",\n \"encryptionMethod\",\n \"decryptionKey\",\n ],\n \"Unsafe Deserialization\": [\n \"serialized\",\n \"object\",\n \"dataObject\",\n \"serialization\",\n \"payload\",\n \"encoded\",\n \"marshalled\",\n \"pickled\",\n \"jsonData\",\n \"state\",\n \"sessionData\",\n \"cache\",\n \"tokenData\",\n \"serializedSession\",\n \"objectState\",\n \"jsonDataPayload\",\n ],\n}\n\n\nclass hunt(BaseModule):\n watched_events = [\"WEB_PARAMETER\"]\n produced_events = [\"FINDING\"]\n flags = [\"active\", \"safe\", \"web-thorough\"]\n meta = {\n \"description\": \"Watch for commonly-exploitable HTTP parameters\",\n \"author\": \"@liquidsec\",\n \"created_date\": \"2022-07-20\",\n }\n\n async def handle_event(self, event):\n p = event.data[\"name\"]\n matching_categories = []\n\n # Collect all matching categories\n for k in hunt_param_dict.keys():\n if p.lower() in hunt_param_dict[k]:\n matching_categories.append(k)\n\n if matching_categories:\n # Create a comma-separated string of categories\n category_str = \", \".join(matching_categories)\n description = f\"Found potentially interesting parameter. Name: [{p}] Parameter Type: [{event.data['type']}] Categories: [{category_str}]\"\n\n if (\n \"original_value\" in event.data.keys()\n and event.data[\"original_value\"] != \"\"\n and event.data[\"original_value\"] is not None\n ):\n description += (\n f\" Original Value: [{self.helpers.truncate_string(str(event.data['original_value']), 200)}]\"\n )\n\n data = {\"host\": str(event.host), \"description\": description}\n url = event.data.get(\"url\", \"\")\n if url:\n data[\"url\"] = url\n await self.emit_event(data, \"FINDING\", event)\n" }, { "path": "bbot/modules/hunterio.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass hunterio(subdomain_enum_apikey):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"EMAIL_ADDRESS\", \"DNS_NAME\", \"URL_UNVERIFIED\"]\n flags = [\"passive\", \"email-enum\", \"subdomain-enum\", \"safe\"]\n meta = {\n \"description\": \"Query hunter.io for emails\",\n \"created_date\": \"2022-04-25\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"Hunter.IO API key\"}\n\n base_url = \"https://api.hunter.io/v2\"\n ping_url = f\"{base_url}/account?api_key={{api_key}}\"\n limit = 100\n\n async def handle_event(self, event):\n query = self.make_query(event)\n for entry in await self.query(query):\n email = entry.get(\"value\", \"\")\n sources = entry.get(\"sources\", [])\n if email:\n email_event = self.make_event(email, \"EMAIL_ADDRESS\", event)\n if email_event:\n await self.emit_event(\n email_event,\n context=f'{{module}} queried Hunter.IO API for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n for source in sources:\n domain = source.get(\"domain\", \"\")\n if domain:\n await self.emit_event(\n domain,\n \"DNS_NAME\",\n email_event,\n context=f\"{{module}} originally found {email} at {{event.type}}: {{event.data}}\",\n )\n url = source.get(\"uri\", \"\")\n if url:\n await self.emit_event(\n url,\n \"URL_UNVERIFIED\",\n email_event,\n context=f\"{{module}} originally found {email} at {{event.type}}: {{event.data}}\",\n )\n\n async def query(self, query):\n emails = []\n url = (\n f\"{self.base_url}/domain-search?domain={query}&api_key={{api_key}}\" + \"&limit={page_size}&offset={offset}\"\n )\n agen = self.api_page_iter(url, page_size=self.limit)\n try:\n async for j in agen:\n new_emails = j.get(\"data\", {}).get(\"emails\", [])\n if not new_emails:\n break\n emails += new_emails\n finally:\n await agen.aclose()\n return emails\n" }, { "path": "bbot/modules/iis_shortnames.py", "content": "import re\n\nfrom bbot.modules.base import BaseModule\n\nvalid_chars = \"ETAONRISHDLFCMUGYPWBVKJXQZ0123456789_-$~()&!#%'@^`{}]]\"\n\n\ndef encode_all(string):\n return \"\".join(\"%{0:0>2}\".format(format(ord(char), \"x\")) for char in string)\n\n\nclass IISShortnamesError(Exception):\n pass\n\n\nclass iis_shortnames(BaseModule):\n watched_events = [\"URL\"]\n produced_events = [\"URL_HINT\"]\n flags = [\"active\", \"safe\", \"web-basic\", \"iis-shortnames\"]\n meta = {\n \"description\": \"Check for IIS shortname vulnerability\",\n \"created_date\": \"2022-04-15\",\n \"author\": \"@liquidsec\",\n }\n options = {\"detect_only\": True, \"max_node_count\": 50, \"speculate_magic_urls\": True}\n options_desc = {\n \"detect_only\": \"Only detect the vulnerability and do not run the shortname scanner\",\n \"max_node_count\": \"Limit how many nodes to attempt to resolve on any given recursion branch\",\n \"speculate_magic_urls\": \"Attempt to discover iis 'magic' special folders\",\n }\n in_scope_only = True\n\n _module_threads = 8\n\n async def detect(self, target):\n technique = None\n detections = []\n random_string = self.helpers.rand_string(8)\n control_url = f\"{target}{random_string}*~1*/a.aspx\"\n test_url = f\"{target}*~1*/a.aspx\"\n\n for method in [\"GET\", \"POST\", \"OPTIONS\", \"DEBUG\", \"HEAD\", \"TRACE\"]:\n kwargs = {\"method\": method, \"allow_redirects\": False, \"timeout\": 10}\n confirmations = 0\n iterations = 5 # one failed detection is tolerated, as long as its not the first run\n while iterations > 0:\n control_result = await self.helpers.request(control_url, **kwargs)\n test_result = await self.helpers.request(test_url, **kwargs)\n if control_result and test_result:\n if control_result.status_code != test_result.status_code:\n confirmations += 1\n self.debug(f\"New detection on {target}, number of confirmations: [{str(confirmations)}]\")\n if confirmations > 3:\n technique = f\"{str(control_result.status_code)}/{str(test_result.status_code)} HTTP Code\"\n detections.append((method, test_result.status_code, technique))\n break\n elif (\"Error Code0x80070002\" in control_result.text) and (\n \"Error Code0x00000000\" in test_result.text\n ):\n confirmations += 1\n if confirmations > 3:\n detections.append((method, 0, technique))\n technique = \"HTTP Body Error Message\"\n break\n iterations -= 1\n if confirmations == 0:\n break\n return detections\n\n async def setup(self):\n self.scanned_tracker = set()\n return True\n\n @staticmethod\n def normalize_url(url):\n return str(url.rstrip(\"/\") + \"/\").lower()\n\n async def directory_confirm(self, target, method, url_hint, affirmative_status_code):\n payload = encode_all(f\"{url_hint}\")\n url = f\"{target}{payload}\"\n directory_confirm_result = await self.helpers.request(\n method=method, url=url, allow_redirects=False, retries=2, timeout=10\n )\n if directory_confirm_result is not None:\n if directory_confirm_result.status_code == affirmative_status_code:\n return True\n return False\n\n async def duplicate_check(self, target, method, url_hint, affirmative_status_code):\n duplicates = []\n count = 2\n base_hint = re.sub(r\"~\\d\", \"\", url_hint)\n suffix = \"/a.aspx\"\n\n while 1:\n payload = encode_all(f\"{base_hint}~{str(count)}*\")\n url = f\"{target}{payload}{suffix}\"\n\n duplicate_check_results = await self.helpers.request(\n method=method, url=url, allow_redirects=False, retries=2, timeout=10\n )\n\n if not duplicate_check_results:\n self.debug(\"duplicate check produced NoneType sample\")\n break\n\n if duplicate_check_results.status_code != affirmative_status_code:\n break\n else:\n duplicates.append(f\"{base_hint}~{str(count)}\")\n count += 1\n\n if count > 5:\n self.warning(\"Found more than 5 files with the same shortname. Will stop further duplicate checking.\")\n break\n\n return duplicates\n\n async def solve_valid_chars(self, method, target, affirmative_status_code):\n confirmed_chars = []\n confirmed_exts = []\n suffix = \"/a.aspx\"\n\n urls_and_kwargs = []\n kwargs = {\"method\": method, \"allow_redirects\": False, \"retries\": 2, \"timeout\": 10}\n for c in valid_chars:\n for file_part in (\"stem\", \"ext\"):\n if file_part == \"stem\":\n payload = encode_all(f\"*{c}*~1*\")\n elif file_part == \"ext\":\n payload = encode_all(f\"*~1*{c}*\")\n url = f\"{target}{payload}{suffix}\"\n urls_and_kwargs.append((url, kwargs, (c, file_part)))\n\n async for url, kwargs, (c, file_part), response in self.helpers.request_custom_batch(urls_and_kwargs):\n if response is not None:\n if response.status_code == affirmative_status_code:\n if file_part == \"stem\":\n confirmed_chars.append(c)\n elif file_part == \"ext\":\n confirmed_exts.append(c)\n\n return confirmed_chars, confirmed_exts\n\n async def solve_shortname_recursive(\n self,\n safety_counter,\n method,\n target,\n prefix,\n affirmative_status_code,\n char_list,\n ext_char_list,\n extension_mode=False,\n node_count=0,\n ):\n url_hint_list = []\n found_results = False\n\n cl = ext_char_list if extension_mode is True else char_list\n\n self.debug(\n f\"Solving shortname recursive for {target} with prefix {prefix} and extension mode {extension_mode}\"\n )\n\n urls_and_kwargs = []\n\n for c in cl:\n suffix = \"/a.aspx\"\n wildcard = \"*\" if extension_mode else \"*~1*\"\n payload = encode_all(f\"{prefix}{c}{wildcard}\")\n url = f\"{target}{payload}{suffix}\"\n kwargs = {\"method\": method}\n urls_and_kwargs.append((url, kwargs, c))\n\n async for url, kwargs, c, response in self.helpers.request_custom_batch(urls_and_kwargs):\n if response is not None:\n if response.status_code == affirmative_status_code:\n found_results = True\n node_count += 1\n safety_counter.counter += 1\n if safety_counter.counter > 1500:\n raise IISShortnamesError(f\"Exceeded safety counter threshold ({safety_counter.counter})\")\n self.verbose(f\"node_count: {str(node_count)} for node: {target}\")\n if node_count > self.config.get(\"max_node_count\"):\n self.verbose(\n f\"iis_shortnames: max_node_count ({str(self.config.get('max_node_count'))}) exceeded for node: {target}. Affected branch will be terminated.\"\n )\n return url_hint_list\n\n # check to make sure the file isn't shorter than 6 characters\n wildcard = \"~1*\"\n payload = encode_all(f\"{prefix}{c}{wildcard}\")\n url = f\"{target}{payload}{suffix}\"\n r = await self.helpers.request(\n method=method, url=url, allow_redirects=False, retries=2, timeout=10\n )\n if r is not None:\n if r.status_code == affirmative_status_code:\n url_hint_list.append(f\"{prefix}{c}\")\n\n url_hint_list += await self.solve_shortname_recursive(\n safety_counter,\n method,\n target,\n f\"{prefix}{c}\",\n affirmative_status_code,\n char_list,\n ext_char_list,\n extension_mode,\n node_count=node_count,\n )\n if len(prefix) > 0 and found_results is False:\n url_hint_list.append(f\"{prefix}\")\n self.verbose(f\"Found new (possibly partial) URL_HINT: {prefix} from node {target}\")\n return url_hint_list\n\n async def handle_event(self, event):\n class safety_counter_obj:\n counter = 0\n\n normalized_url = self.normalize_url(event.data)\n self.scanned_tracker.add(normalized_url)\n\n detections = await self.detect(normalized_url)\n\n technique_strings = []\n if detections:\n for detection in detections:\n method, affirmative_status_code, technique = detection\n technique_strings.append(f\"{method} ({technique})\")\n\n description = f\"IIS Shortname Vulnerability Detected. Potentially Vulnerable Method/Techniques: [{','.join(technique_strings)}]\"\n await self.emit_event(\n {\"severity\": \"LOW\", \"host\": str(event.host), \"url\": normalized_url, \"description\": description},\n \"VULNERABILITY\",\n event,\n context=\"{module} detected low {event.type}: IIS shortname enumeration\",\n )\n\n if self.config.get(\"speculate_magic_urls\") and \"iis-magic-url\" not in event.tags:\n magic_url_bin = f\"{normalized_url}bin::$INDEX_ALLOCATION/\"\n self.debug(f\"making IIS magic URL: {magic_url_bin}\")\n magic_url_event = self.make_event(\n magic_url_bin, \"URL\", parent=event, tags=[\"iis-magic-url\", \"status-403\"]\n )\n await self.scan.modules[\"iis_shortnames\"].incoming_event_queue.put(magic_url_event)\n\n if not self.config.get(\"detect_only\"):\n for detection in detections:\n safety_counter = safety_counter_obj()\n\n method, affirmative_status_code, technique = detection\n valid_method_confirmed = False\n\n if valid_method_confirmed:\n break\n confirmed_chars, confirmed_exts = await self.solve_valid_chars(\n method, normalized_url, affirmative_status_code\n )\n\n if len(confirmed_chars) >= len(valid_chars) - 4:\n self.debug(\n f\"Detected [{len(confirmed_chars)}] characters (out of {len(valid_chars)}) as valid. This is likely a false positive\"\n )\n continue\n\n if len(confirmed_chars) > 0:\n valid_method_confirmed = True\n else:\n continue\n\n self.verbose(f\"Confirmed character list: {','.join(confirmed_chars)}\")\n self.verbose(f\"Confirmed ext character list: {','.join(confirmed_exts)}\")\n try:\n file_name_hints = list(\n set(\n await self.solve_shortname_recursive(\n safety_counter,\n method,\n normalized_url,\n \"\",\n affirmative_status_code,\n confirmed_chars,\n confirmed_exts,\n )\n )\n )\n except IISShortnamesError as e:\n self.warning(f\"Aborted Shortname Run for URL [{normalized_url}] due to Error: [{e}]\")\n return\n\n file_name_hints = [f\"{x}~1\" for x in file_name_hints]\n url_hint_list = []\n\n file_name_hints_dedupe = file_name_hints[:]\n\n for x in file_name_hints_dedupe:\n duplicates = await self.duplicate_check(normalized_url, method, x, affirmative_status_code)\n if duplicates:\n file_name_hints += duplicates\n\n # check for the case of a folder and file with the same filename\n for d in file_name_hints:\n if await self.directory_confirm(normalized_url, method, d, affirmative_status_code):\n self.verbose(f\"Confirmed Directory URL_HINT: {d} from node {normalized_url}\")\n url_hint_list.append(d)\n\n for y in file_name_hints:\n try:\n file_name_extension_hints = await self.solve_shortname_recursive(\n safety_counter,\n method,\n normalized_url,\n f\"{y}.\",\n affirmative_status_code,\n confirmed_chars,\n confirmed_exts,\n extension_mode=True,\n )\n except IISShortnamesError as e:\n self.warning(f\"Aborted Shortname Run for URL {normalized_url} due to Error: [{e}]\")\n return\n\n for z in file_name_extension_hints:\n if z.endswith(\".\"):\n z = z.rstrip(\".\")\n self.verbose(f\"Found new file URL_HINT: {z} from node {normalized_url}\")\n url_hint_list.append(z)\n\n for url_hint in url_hint_list:\n if \".\" in url_hint:\n hint_type = \"shortname-endpoint\"\n # Check if it's a ZIP file\n if url_hint.lower().endswith(\".zip\"):\n await self.emit_event(\n {\n \"host\": str(event.host),\n \"url\": event.data,\n \"description\": f\"Possible backup file (zip) in web root: {normalized_url}{url_hint}\",\n },\n \"FINDING\",\n event,\n context=f\"{{module}} discovered possible backup file in web root: {url_hint}\",\n )\n else:\n hint_type = \"shortname-directory\"\n\n tags = [hint_type]\n if \"iis-magic-url\" in event.tags:\n tags.append(\"iis-magic-url\")\n await self.emit_event(\n f\"{normalized_url}/{url_hint}\",\n \"URL_HINT\",\n event,\n tags=tags,\n context=f\"{{module}} enumerated shortnames at {normalized_url} and found {{event.type}}: {url_hint}\",\n )\n\n async def filter_event(self, event):\n if \"dir\" in event.tags:\n if self.normalize_url(event.data) not in self.scanned_tracker:\n return True\n return False\n return False\n" }, { "path": "bbot/modules/internal/__init__.py", "content": "" }, { "path": "bbot/modules/internal/aggregate.py", "content": "from bbot.modules.report.base import BaseReportModule\n\n\nclass aggregate(BaseReportModule):\n watched_events = []\n flags = [\"passive\", \"safe\"]\n meta = {\n \"description\": \"Summarize statistics at the end of a scan\",\n \"created_date\": \"2022-07-25\",\n \"author\": \"@TheTechromancer\",\n }\n\n async def report(self):\n self.log_table(*self.scan.stats._make_table(), table_name=\"scan-stats\")\n" }, { "path": "bbot/modules/internal/base.py", "content": "import logging\n\nfrom bbot.modules.base import BaseModule\n\n\nclass BaseInternalModule(BaseModule):\n in_scope_only = False\n _type = \"internal\"\n # Priority, 1-5, lower numbers == higher priority\n _priority = 3\n\n @property\n def log(self):\n if self._log is None:\n self._log = logging.getLogger(f\"bbot.modules.internal.{self.name}\")\n return self._log\n" }, { "path": "bbot/modules/internal/cloudcheck.py", "content": "import asyncio\nimport regex as re\nfrom contextlib import suppress\n\nfrom bbot.modules.base import BaseInterceptModule\n\n\nclass CloudCheck(BaseInterceptModule):\n watched_events = [\"*\"]\n meta = {\n \"description\": \"Tag events by cloud provider, identify cloud resources like storage buckets\",\n \"created_date\": \"2024-07-07\",\n \"author\": \"@TheTechromancer\",\n }\n # tag events up to and including distance-2\n scope_distance_modifier = 2\n _priority = 3\n\n async def setup(self):\n self._cloud_hostname_regexes = None\n self._cloud_hostname_regexes_lock = asyncio.Lock()\n # perform a test lookup during setup to force signature update\n await self.helpers.cloudcheck.lookup(\"8.8.8.8\")\n return True\n\n async def filter_event(self, event):\n if (not event.host) or (event.type in (\"IP_RANGE\",)):\n return False, \"event does not have host attribute\"\n return True\n\n async def handle_event(self, event, **kwargs):\n # cloud tagging by hosts\n hosts_to_check = set(event.resolved_hosts)\n with suppress(KeyError):\n hosts_to_check.remove(event.host_original)\n hosts_to_check = [str(event.host_original)] + list(hosts_to_check)\n\n for i, host in enumerate(hosts_to_check):\n host_is_ip = self.helpers.is_ip(host)\n try:\n cloudcheck_results = await self.helpers.cloudcheck.lookup(host)\n except Exception as e:\n self.warning(f\"Error running cloudcheck against {event} (host: {host}): {e}\")\n continue\n for provider in cloudcheck_results:\n provider_name = provider[\"name\"].lower()\n tags = provider.get(\"tags\", [])\n for tag in tags:\n event.add_tag(tag)\n event.add_tag(f\"{tag}-{provider_name}\")\n if host_is_ip:\n event.add_tag(f\"{provider_name}-ip\")\n else:\n # if the original hostname is a cloud domain, tag it as such\n if i == 0:\n event.add_tag(f\"{provider_name}-domain\")\n # any children are tagged as CNAMEs\n else:\n event.add_tag(f\"{provider_name}-cname\")\n\n # we only generate storage buckets off of in-scope or distance-1 events\n if event.scope_distance >= self.max_scope_distance:\n return\n\n # see if any of our hosts are storage buckets, etc.\n regexes = await self.cloud_hostname_regexes()\n regexes = regexes.get(\"STORAGE_BUCKET_HOSTNAME\", [])\n for regex_name, regex in regexes.items():\n for host in hosts_to_check:\n if match := regex.match(host):\n try:\n bucket_name, bucket_domain = match.groups()\n except Exception as e:\n self.error(\n f\"Bucket regex {regex_name} ({regex}) is not formatted correctly to extract bucket name and domain: {e}\"\n )\n continue\n bucket_name, bucket_domain = match.groups()\n bucket_url = f\"https://{bucket_name}.{bucket_domain}\"\n await self.emit_event(\n {\n \"name\": bucket_name,\n \"url\": bucket_url,\n \"context\": f\"{{module}} analyzed {event.type} and found {{event.type}}: {bucket_url}\",\n },\n \"STORAGE_BUCKET\",\n parent=event,\n )\n\n async def cloud_hostname_regexes(self):\n async with self._cloud_hostname_regexes_lock:\n if not self._cloud_hostname_regexes:\n storage_bucket_regexes = {}\n self._cloud_hostname_regexes = {\"STORAGE_BUCKET_HOSTNAME\": storage_bucket_regexes}\n from cloudcheck import providers\n\n for attr in dir(providers):\n if attr.startswith(\"_\"):\n continue\n provider = getattr(providers, attr)\n provider_regexes = getattr(provider, \"regexes\", {})\n for regex_name, regexes in provider_regexes.items():\n for i, regex in enumerate(regexes):\n if not regex_name in (\"STORAGE_BUCKET_HOSTNAME\"):\n continue\n try:\n storage_bucket_regexes[f\"{attr}-{regex_name}-{i}\"] = re.compile(regex)\n except Exception as e:\n self.error(f\"Error compiling regex for {attr}-{regex_name}: {e}\")\n continue\n return self._cloud_hostname_regexes\n" }, { "path": "bbot/modules/internal/dnsresolve.py", "content": "import ipaddress\nfrom contextlib import suppress\n\nfrom bbot.errors import ValidationError\nfrom bbot.core.helpers.dns.engine import all_rdtypes\nfrom bbot.core.helpers.dns.helpers import extract_targets\nfrom bbot.modules.base import BaseInterceptModule, BaseModule\n\n\nclass DNSResolve(BaseInterceptModule):\n watched_events = [\"*\"]\n produced_events = [\"DNS_NAME\", \"IP_ADDRESS\", \"RAW_DNS_RECORD\"]\n meta = {\"description\": \"Perform DNS resolution\", \"created_date\": \"2022-04-08\", \"author\": \"@TheTechromancer\"}\n _priority = 1\n scope_distance_modifier = None\n\n class HostModule(BaseModule):\n _name = \"host\"\n _type = \"internal\"\n\n @property\n def module_threads(self):\n return self.dns_config.get(\"threads\", 25)\n\n async def setup(self):\n self.dns_config = self.scan.config.get(\"dns\", {})\n self.dns_disable = self.dns_config.get(\"disable\", False)\n if self.dns_disable:\n return None, \"DNS resolution is disabled in the config\"\n\n self.minimal = self.dns_config.get(\"minimal\", False)\n self.minimal_rdtypes = (\"A\", \"AAAA\", \"CNAME\")\n if self.minimal:\n self.non_minimal_rdtypes = ()\n else:\n self.non_minimal_rdtypes = tuple([t for t in all_rdtypes if t not in self.minimal_rdtypes])\n self.dns_search_distance = max(0, int(self.dns_config.get(\"search_distance\", 1)))\n self._emit_raw_records = None\n\n self.host_module = self.HostModule(self.scan)\n self.children_emitted = set()\n self.children_emitted_raw = set()\n self.hosts_resolved = set()\n\n return True\n\n async def filter_event(self, event):\n if (not event.host) or (event.type in (\"IP_RANGE\",)):\n return False, \"event does not have host attribute\"\n return True\n\n async def handle_event(self, event, **kwargs):\n event_is_ip = self.helpers.is_ip(event.host)\n if event_is_ip:\n minimal_rdtypes = (\"PTR\",)\n non_minimal_rdtypes = ()\n else:\n minimal_rdtypes = self.minimal_rdtypes\n non_minimal_rdtypes = self.non_minimal_rdtypes\n\n # first, we find or create the main DNS_NAME or IP_ADDRESS associated with this event\n main_host_event, whitelisted, blacklisted, new_event = self.get_dns_parent(event)\n original_tags = set(event.tags)\n\n # minimal resolution - first, we resolve A/AAAA records for scope purposes\n if new_event or event is main_host_event:\n await self.resolve_event(main_host_event, types=minimal_rdtypes)\n # are any of its IPs whitelisted/blacklisted?\n whitelisted, blacklisted = self.check_scope(main_host_event)\n if whitelisted and event.scope_distance > 0:\n self.debug(f\"Making {main_host_event} in-scope because it resolves to an in-scope resource (A/AAAA)\")\n main_host_event.scope_distance = 0\n\n # abort if the event resolves to something blacklisted\n if blacklisted:\n return False, \"it has a blacklisted DNS record\"\n\n # DNS resolution for hosts that aren't IPs\n if not event_is_ip:\n # if the event is within our dns search distance, resolve the rest of our records\n if main_host_event.scope_distance < self._dns_search_distance:\n await self.resolve_event(main_host_event, types=non_minimal_rdtypes)\n # check for wildcards if the event is within the scan's search distance\n if new_event and main_host_event.scope_distance <= self.scan.scope_search_distance:\n event_data_changed = await self.handle_wildcard_event(main_host_event)\n if event_data_changed:\n # since data has changed, we check again whether it's a duplicate\n if event.type == \"DNS_NAME\" and self.scan.ingress_module.is_incoming_duplicate(\n event, add=True\n ):\n if not event._graph_important:\n return (\n False,\n \"it's a DNS wildcard, and its module already emitted a similar wildcard event\",\n )\n else:\n self.debug(\n f\"Event {event} was already emitted by its module, but it's graph-important so it gets a pass\"\n )\n\n # if there weren't any DNS children and it's not an IP address, tag as unresolved\n if not main_host_event.raw_dns_records and not event_is_ip:\n main_host_event.add_tag(\"unresolved\")\n main_host_event.type = \"DNS_NAME_UNRESOLVED\"\n\n # main_host_event.add_tag(f\"resolve-distance-{main_host_event.dns_resolve_distance}\")\n\n dns_tags = main_host_event.tags.difference(original_tags)\n\n dns_resolve_distance = getattr(main_host_event, \"dns_resolve_distance\", 0)\n runaway_dns = dns_resolve_distance >= self.helpers.dns.runaway_limit\n if runaway_dns:\n # kill runaway DNS chains\n self.debug(\n f\"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.helpers.dns.runaway_limit})\"\n )\n main_host_event.add_tag(f\"runaway-dns-{dns_resolve_distance}\")\n else:\n # emit dns children\n await self.emit_dns_children_raw(main_host_event, dns_tags)\n if not self.minimal:\n await self.emit_dns_children(main_host_event)\n\n # emit the main DNS_NAME or IP_ADDRESS\n if (\n new_event\n and event is not main_host_event\n and main_host_event.scope_distance <= self._dns_search_distance\n ):\n await self.emit_event(main_host_event)\n\n # transfer scope distance to event\n event.scope_distance = main_host_event.scope_distance\n event._resolved_hosts = main_host_event.resolved_hosts\n\n async def handle_wildcard_event(self, event):\n rdtypes = tuple(event.raw_dns_records)\n wildcard_rdtypes = await self.helpers.is_wildcard(\n event.host, rdtypes=rdtypes, raw_dns_records=event.raw_dns_records\n )\n for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items():\n if is_wildcard is False:\n continue\n elif is_wildcard is True:\n event.add_tag(\"wildcard\")\n wildcard_tag = \"wildcard\"\n else:\n event.add_tag(f\"wildcard-{is_wildcard}\")\n wildcard_tag = f\"wildcard-{is_wildcard}\"\n event.add_tag(f\"{rdtype}-{wildcard_tag}\")\n\n # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com)\n if wildcard_rdtypes and \"target\" not in event.tags:\n # these are the rdtypes that have wildcards\n wildcard_rdtypes_set = set(wildcard_rdtypes)\n # consider the event a full wildcard if all its records are wildcards\n event_is_wildcard = False\n if wildcard_rdtypes_set:\n event_is_wildcard = all(r[0] is True for r in wildcard_rdtypes.values())\n\n if event_is_wildcard:\n if event.type in (\"DNS_NAME\",) and \"_wildcard\" not in event.data.split(\".\"):\n wildcard_parent = self.helpers.parent_domain(event.host)\n for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items():\n if _is_wildcard:\n wildcard_parent = _parent_domain\n break\n wildcard_data = f\"_wildcard.{wildcard_parent}\"\n if wildcard_data != event.data:\n self.debug(f'Wildcard detected, changing event.data \"{event.data}\" --> \"{wildcard_data}\"')\n event.data = wildcard_data\n return True\n return False\n\n async def emit_dns_children(self, event):\n for rdtype, children in event.dns_children.items():\n module = self._make_dummy_module(rdtype)\n for child_host in children:\n try:\n child_event = self.scan.make_event(\n child_host,\n \"DNS_NAME\",\n module=module,\n parent=event,\n context=f\"{rdtype} record for {event.host} contains {{event.type}}: {{event.host}}\",\n )\n except ValidationError as e:\n self.warning(f'Event validation failed for DNS child of {event}: \"{child_host}\" ({rdtype}): {e}')\n continue\n\n child_hash = hash(f\"{event.host}:{module}:{child_host}\")\n # if we haven't emitted this one before\n if child_hash not in self.children_emitted:\n # and it's either in-scope or inside our dns search distance\n if self.preset.in_scope(child_host) or child_event.scope_distance <= self._dns_search_distance:\n self.children_emitted.add(child_hash)\n # if it's a hostname and it's only one hop away, mark it as affiliate\n if child_event.type == \"DNS_NAME\" and child_event.scope_distance == 1:\n child_event.add_tag(\"affiliate\")\n self.debug(f\"Queueing DNS child for {event}: {child_event}\")\n await self.emit_event(child_event)\n\n async def emit_dns_children_raw(self, event, dns_tags):\n for rdtype, answers in event.raw_dns_records.items():\n rdtype_lower = rdtype.lower()\n tags = {t for t in dns_tags if rdtype_lower in t.split(\"-\")}\n if self.emit_raw_records and rdtype not in (\"A\", \"AAAA\", \"CNAME\", \"PTR\"):\n for answer in answers:\n text_answer = answer.to_text()\n child_hash = hash(f\"{event.host}:{rdtype}:{text_answer}\")\n if child_hash not in self.children_emitted_raw:\n self.children_emitted_raw.add(child_hash)\n await self.emit_event(\n {\"host\": str(event.host), \"type\": rdtype, \"answer\": text_answer},\n \"RAW_DNS_RECORD\",\n parent=event,\n tags=tags,\n context=f\"{rdtype} lookup on {{event.parent.host}} produced {{event.type}}\",\n )\n\n def check_scope(self, event):\n whitelisted = False\n blacklisted = False\n dns_children = getattr(event, \"dns_children\", {})\n for rdtype in (\"A\", \"AAAA\", \"CNAME\"):\n hosts = dns_children.get(rdtype, [])\n # update resolved hosts\n event.resolved_hosts.update(hosts)\n for host in hosts:\n # having a CNAME to an in-scope host doesn't make you in-scope\n if rdtype != \"CNAME\":\n if not whitelisted:\n with suppress(ValidationError):\n if self.scan.whitelisted(host):\n whitelisted = True\n event.add_tag(f\"dns-whitelisted-{rdtype}\")\n # but a CNAME to a blacklisted host means you're blacklisted\n if not blacklisted:\n with suppress(ValidationError):\n if self.scan.blacklisted(host):\n blacklisted = True\n event.add_tag(\"blacklisted\")\n event.add_tag(f\"dns-blacklisted-{rdtype}\")\n if blacklisted:\n whitelisted = False\n return whitelisted, blacklisted\n\n async def resolve_event(self, event, types):\n if not types:\n return\n event_host = str(event.host)\n queries = [(event_host, rdtype) for rdtype in types]\n dns_errors = {}\n async for (query, rdtype), (answers, errors) in self.helpers.dns.resolve_raw_batch(queries):\n # errors\n try:\n dns_errors[rdtype].update(errors)\n except KeyError:\n dns_errors[rdtype] = set(errors)\n for answer in answers:\n event.add_tag(f\"{rdtype}-record\")\n # raw dnspython answers\n try:\n event.raw_dns_records[rdtype].add(answer)\n except KeyError:\n event.raw_dns_records[rdtype] = {answer}\n # hosts\n for _rdtype, host in extract_targets(answer):\n try:\n event.dns_children[_rdtype].add(host)\n except KeyError:\n event.dns_children[_rdtype] = {host}\n # check for private IPs\n try:\n ip = ipaddress.ip_address(host)\n if ip.is_private:\n event.add_tag(\"private-ip\")\n except ValueError:\n continue\n\n # tag event with errors\n for rdtype, errors in dns_errors.items():\n # only consider it an error if there weren't any results for that rdtype\n if errors and rdtype not in event.dns_children:\n event.add_tag(f\"{rdtype}-error\")\n\n def get_dns_parent(self, event):\n \"\"\"\n Get the first parent DNS_NAME / IP_ADDRESS of an event. If one isn't found, create it.\n \"\"\"\n for parent in event.get_parents(include_self=True):\n if parent.host == event.host and parent.type in (\"IP_ADDRESS\", \"DNS_NAME\", \"DNS_NAME_UNRESOLVED\"):\n blacklisted = any(t.startswith(\"dns-blacklisted-\") for t in parent.tags)\n whitelisted = any(t.startswith(\"dns-whitelisted-\") for t in parent.tags)\n new_event = parent is event\n return parent, whitelisted, blacklisted, new_event\n tags = set()\n if \"target\" in event.tags:\n tags.add(\"target\")\n return (\n self.scan.make_event(\n event.host,\n \"DNS_NAME\",\n module=self.host_module,\n parent=event,\n context=\"{event.parent.type} has host {event.type}: {event.host}\",\n tags=tags,\n ),\n None,\n None,\n True,\n )\n\n @property\n def emit_raw_records(self):\n if self._emit_raw_records is None:\n watching_raw_records = any(\"RAW_DNS_RECORD\" in m.get_watched_events() for m in self.scan.modules.values())\n omitted_event_types = self.scan.config.get(\"omit_event_types\", [])\n omit_raw_records = \"RAW_DNS_RECORD\" in omitted_event_types\n self._emit_raw_records = watching_raw_records or not omit_raw_records\n return self._emit_raw_records\n\n @property\n def _dns_search_distance(self):\n return max(self.scan.scope_search_distance, self.dns_search_distance)\n\n def _make_dummy_module(self, name):\n try:\n dummy_module = self.scan.dummy_modules[name]\n except KeyError:\n dummy_module = self.scan._make_dummy_module(name=name, _type=\"DNS\")\n dummy_module._priority = 4\n dummy_module.suppress_dupes = False\n self.scan.dummy_modules[name] = dummy_module\n return dummy_module\n" }, { "path": "bbot/modules/internal/excavate.py", "content": "import yara\nimport json\nimport html\nimport time\nimport inspect\nimport regex as re\nfrom pathlib import Path\nfrom bbot.errors import ExcavateError, ValidationError\nimport bbot.core.helpers.regexes as bbot_regexes\nfrom bbot.modules.base import BaseInterceptModule\nfrom bbot.modules.internal.base import BaseInternalModule\nfrom urllib.parse import urlparse, urljoin, parse_qs, urlunparse, urldefrag\n\n\ndef find_subclasses(obj, base_class):\n \"\"\"\n Finds and returns subclasses of a specified base class within an object.\n\n Parameters:\n obj : object\n The object to inspect for subclasses.\n base_class : type\n The base class to find subclasses of.\n\n Returns:\n list\n A list of subclasses found within the object.\n\n Example:\n >>> class A: pass\n >>> class B(A): pass\n >>> class C(A): pass\n >>> find_subclasses(locals(), A)\n [, ]\n \"\"\"\n subclasses = []\n for name, member in inspect.getmembers(obj):\n if inspect.isclass(member) and issubclass(member, base_class) and member is not base_class:\n subclasses.append(member)\n return subclasses\n\n\ndef _exclude_key(original_dict, key_to_exclude):\n \"\"\"\n Returns a new dictionary excluding the specified key from the original dictionary.\n\n Parameters:\n original_dict : dict\n The dictionary to exclude the key from.\n key_to_exclude : hashable\n The key to exclude.\n\n Returns:\n dict\n A new dictionary without the specified key.\n\n Example:\n >>> original = {'a': 1, 'b': 2, 'c': 3}\n >>> _exclude_key(original, 'b')\n {'a': 1, 'c': 3}\n \"\"\"\n return {key: value for key, value in original_dict.items() if key != key_to_exclude}\n\n\ndef extract_params_url(parsed_url):\n \"\"\"\n Yields query parameters from a parsed URL.\n\n Args:\n parsed_url (ParseResult): The URL to extract parameters from.\n\n Yields:\n tuple: Contains the hardcoded HTTP method ('GET'), parsed URL, parameter name,\n original value, source (hardcoded to 'direct_url'), and additional parameters\n (all parameters excluding the current one).\n \"\"\"\n params = parse_qs(parsed_url.query)\n flat_params = {k: v[0] for k, v in params.items()}\n\n for p, p_value in flat_params.items():\n yield \"GET\", parsed_url, p, p_value, \"direct_url\", _exclude_key(flat_params, p)\n\n\ndef extract_params_location(location_header_value, original_parsed_url):\n \"\"\"\n Extracts parameters from a location header, yielding them one at a time.\n\n Args:\n location_header_value (dict): Contents of location header\n original_url: The original parsed URL the header was received from (urllib.parse.ParseResult)\n\n Yields:\n method(str), parsed_url(urllib.parse.ParseResult), parameter_name(str), original_value(str), regex_name(str), additional_params(dict): The HTTP method associated with the parameter (GET, POST, None), A urllib.parse.ParseResult object representing the endpoint associated with the parameter, the parameter found in the location header, its original value (if available), the name of the detecting regex, a dict of additional params if any\n \"\"\"\n if location_header_value.startswith(\"http://\") or location_header_value.startswith(\"https://\"):\n parsed_url = urlparse(location_header_value)\n else:\n parsed_url = urlparse(f\"{original_parsed_url.scheme}://{original_parsed_url.netloc}{location_header_value}\")\n\n params = parse_qs(parsed_url.query)\n flat_params = {k: v[0] for k, v in params.items()}\n\n for p, p_value in flat_params.items():\n yield \"GET\", parsed_url, p, p_value, \"location_header\", _exclude_key(flat_params, p)\n\n\nclass YaraRuleSettings:\n def __init__(self, description, tags, emit_match):\n self.description = description\n self.tags = tags\n self.emit_match = emit_match\n\n\nclass ExcavateRule:\n \"\"\"\n The BBOT Regex Commandments:\n\n 1) Thou shalt employ YARA regexes in place of Python regexes, save when necessity doth compel otherwise.\n 2) Thou shalt ne'er wield a Python regex against a vast expanse of text.\n 3) Whensoever it be possible, thou shalt favor string matching o'er regexes.\n\n Amen.\n \"\"\"\n\n yara_rules = {}\n\n def __init__(self, excavate):\n self.excavate = excavate\n self.helpers = excavate.helpers\n self.name = \"\"\n\n async def preprocess(self, r, event, discovery_context):\n \"\"\"\n Preprocesses YARA rule results, extracts meta tags, and configures a YaraRuleSettings object.\n\n This method retrieves optional meta tags from YARA rules and uses them to configure a YaraRuleSettings object.\n It formats the results from the YARA engine into a suitable format for the process() method and initiates\n a call to process(), passing on the pre-processed YARA results, event data, YARA rule settings, and discovery context.\n\n This should typically NOT be overridden.\n\n Parameters:\n r : YaraMatch\n The YARA match object containing the rule and meta information.\n event : Event\n The event data associated with the YARA match.\n discovery_context : DiscoveryContext\n The context in which the discovery is made.\n\n Returns:\n None\n \"\"\"\n description = \"\"\n tags = []\n emit_match = False\n\n if \"description\" in r.meta.keys():\n description = r.meta[\"description\"]\n if \"tags\" in r.meta.keys():\n tags = self.excavate.helpers.chain_lists(r.meta[\"tags\"])\n if \"emit_match\" in r.meta.keys():\n emit_match = True\n\n yara_rule_settings = YaraRuleSettings(description, tags, emit_match)\n yara_results = {}\n for h in r.strings:\n yara_results[h.identifier.lstrip(\"$\")] = sorted(\n {i.matched_data.decode(\"utf-8\", errors=\"ignore\") for i in h.instances}\n )\n await self.process(yara_results, event, yara_rule_settings, discovery_context)\n\n async def process(self, yara_results, event, yara_rule_settings, discovery_context):\n \"\"\"\n Processes YARA rule results and reports events with enriched data.\n\n This method iterates over the provided YARA rule results and constructs event data for each match.\n It enriches the event data with host, URL, and description information, and conditionally includes\n matched data based on the YaraRuleSettings. Finally, it reports the constructed event data.\n\n Override when custom processing and/or validation is needed on data before reporting.\n\n Parameters:\n yara_results : dict\n A dictionary where keys are YARA rule identifiers and values are lists of matched data strings.\n event : Event\n The event data associated with the YARA match.\n yara_rule_settings : YaraRuleSettings\n The settings configured from YARA rule meta tags, including description, tags, and emit_match flag.\n discovery_context : DiscoveryContext\n The context in which the discovery is made.\n\n Returns:\n None\n \"\"\"\n for results in yara_results.values():\n for result in results:\n event_data = {\"description\": f\"{discovery_context} {yara_rule_settings.description}\"}\n if yara_rule_settings.emit_match:\n event_data[\"description\"] += f\" [{result}]\"\n await self.report(event_data, event, yara_rule_settings, discovery_context)\n\n async def report_prep(self, event_data, event_type, event, tags):\n \"\"\"\n Prepares an event draft for reporting by creating and tagging the event.\n\n This method creates an event draft using the provided event data and type, associating it with a parent event.\n It tags the event draft with the provided tags and returns the draft. If event creation fails, it returns None.\n\n Override when an event needs to be modified before it is emitted - for example, custom tags need to be conditionally added.\n\n Parameters:\n event_data : dict\n The data to be included in the event.\n event_type : str\n The type of the event being reported.\n event : Event\n The parent event to which this event draft is related.\n tags : list\n A list of tags to be associated with the event draft.\n\n Returns:\n EventDraft or None\n \"\"\"\n event_draft = self.excavate.make_event(event_data, event_type, parent=event)\n if not event_draft:\n return None\n event_draft.add_tags(tags)\n return event_draft\n\n async def report(\n self, event_data, event, yara_rule_settings, discovery_context, event_type=\"FINDING\", abort_if=None, **kwargs\n ):\n \"\"\"\n Reports an event by preparing an event draft and emitting it.\n\n Processes the provided event data, sets a default description if needed, prepares the event draft, and emits it.\n It constructs a context string for the event and uses the report_prep method to create the event draft. If the draft is successfully\n created, it emits the event.\n\n Typically not overridden, but might need to be if custom logic is needed to build description/context, etc.\n\n Parameters:\n event_data : dict\n The data to be included in the event.\n event : Event\n The parent event to which this event is related.\n yara_rule_settings : YaraRuleSettings\n The settings configured from YARA rule meta tags, including description and tags.\n discovery_context : DiscoveryContext\n The context in which the discovery is made.\n event_type : str, optional\n The type of the event being reported, default is \"FINDING\".\n abort_if : callable, optional\n A callable that determines if the event emission should be aborted.\n **kwargs : dict\n Additional keyword arguments to pass to the report_prep method.\n\n Returns:\n None\n \"\"\"\n\n # If a description is not set and is needed, provide a basic one\n if event_type == \"FINDING\" and \"description\" not in event_data.keys():\n event_data[\"description\"] = f\"{discovery_context} {yara_rule_settings['self.description']}\"\n subject = \"\"\n if isinstance(event_data, str):\n subject = f\" {event_data}\"\n context = f\"Excavate's {self.__class__.__name__} emitted {event_type}{subject}, because {discovery_context} {yara_rule_settings.description}\"\n tags = yara_rule_settings.tags\n event_draft = await self.report_prep(event_data, event_type, event, tags, **kwargs)\n if event_draft:\n await self.excavate.emit_event(event_draft, context=context, abort_if=abort_if)\n\n\nclass CustomExtractor(ExcavateRule):\n description = \"Enables custom, user-defined YARA rules.\"\n\n def __init__(self, excavate):\n super().__init__(excavate)\n\n async def process(self, yara_results, event, yara_rule_settings, discovery_context):\n for identifier, results in yara_results.items():\n for result in results:\n event_data = {}\n description_string = (\n f\" with description: [{yara_rule_settings.description}]\" if yara_rule_settings.description else \"\"\n )\n event_data[\"description\"] = (\n f\"Custom Yara Rule [{self.name}]{description_string} Matched via identifier [{identifier}]\"\n )\n if yara_rule_settings.emit_match:\n event_data[\"description\"] += f\" and extracted [{result}]\"\n await self.report(event_data, event, yara_rule_settings, discovery_context)\n\n\nclass excavate(BaseInternalModule, BaseInterceptModule):\n \"\"\"\n Example (simple) Excavate Rules:\n\n class excavateTestRule(ExcavateRule):\n yara_rules = {\n \"SearchForText\": 'rule SearchForText { meta: description = \"Contains the text AAAABBBBCCCC\" strings: $text = \"AAAABBBBCCCC\" condition: $text }',\n \"SearchForText2\": 'rule SearchForText2 { meta: description = \"Contains the text DDDDEEEEFFFF\" strings: $text2 = \"DDDDEEEEFFFF\" condition: $text2 }',\n }\n \"\"\"\n\n watched_events = [\"HTTP_RESPONSE\", \"RAW_TEXT\"]\n produced_events = [\"URL_UNVERIFIED\", \"WEB_PARAMETER\"]\n flags = [\"passive\"]\n meta = {\n \"description\": \"Passively extract juicy tidbits from scan data\",\n \"created_date\": \"2022-06-27\",\n \"author\": \"@liquidsec\",\n }\n\n options = {\n \"yara_max_match_data\": 2000,\n \"custom_yara_rules\": \"\",\n \"speculate_params\": False,\n }\n options_desc = {\n \"yara_max_match_data\": \"Sets the maximum amount of text that can extracted from a YARA regex\",\n \"custom_yara_rules\": \"Include custom Yara rules\",\n \"speculate_params\": \"Enable speculative parameter extraction from JSON and XML content\",\n }\n scope_distance_modifier = None\n accept_dupes = False\n\n _module_threads = 8\n\n yara_rule_name_regex = re.compile(r\"rule\\s(\\w+)\\s{\")\n yara_rule_regex = re.compile(r\"(?s)((?:rule\\s+\\w+\\s*{[^{}]*(?:{[^{}]*}[^{}]*)*[^{}]*(?:/\\S*?}[^/]*?/)*)*})\")\n\n def in_bl(self, value):\n # Check if the value is in the blacklist or starts with a blacklisted prefix.\n lower_value = value.lower()\n\n if lower_value in self.parameter_blacklist:\n return True\n\n for bl_param_prefix in self.parameter_blacklist_prefixes:\n if lower_value.startswith(bl_param_prefix.lower()):\n return True\n\n return False\n\n def url_unparse(self, param_type, parsed_url):\n # Reconstructs a URL, optionally omitting the query string based on remove_querystring configuration value.\n if param_type == \"GETPARAM\":\n querystring = \"\"\n else:\n querystring = parsed_url.query\n\n return urlunparse(\n (\n parsed_url.scheme,\n parsed_url.netloc,\n parsed_url.path,\n \"\",\n \"\" if self.remove_querystring else querystring,\n \"\",\n )\n )\n\n class ParameterExtractor(ExcavateRule):\n description = \"Extracts web parameters. Enabled if any modules are enabled that emit WEB_PARAMETER events.\"\n yara_rules = {}\n\n class ParameterExtractorRule:\n name = \"\"\n\n async def extract(self):\n pass\n\n def __init__(self, excavate, result):\n self.excavate = excavate\n self.result = result\n\n class GetJquery(ParameterExtractorRule):\n name = \"GET jquery\"\n discovery_regex = r\"/\\$.get\\([^\\)].+\\)/ nocase\"\n extraction_regex = re.compile(r\"\\$.get\\([\\'\\\"](.+)[\\'\\\"].+(\\{.+\\})\\)\")\n output_type = \"GETPARAM\"\n\n async def extract(self):\n extracted_results = await self.excavate.helpers.re.findall(self.extraction_regex, str(self.result))\n if extracted_results:\n for action, extracted_parameters in extracted_results:\n extracted_parameters_dict = await self.convert_to_dict(extracted_parameters)\n for parameter_name, original_value in extracted_parameters_dict.items():\n yield (\n self.output_type,\n parameter_name,\n original_value.strip(),\n action,\n _exclude_key(extracted_parameters_dict, parameter_name),\n )\n\n async def convert_to_dict(self, extracted_str):\n extracted_str = extracted_str.replace(\"'\", '\"')\n extracted_str = await self.excavate.helpers.re.sub(\n re.compile(r\"(\\w+):\"), r'\"\\1\":', extracted_str\n ) # Quote keys\n\n try:\n return json.loads(extracted_str)\n except json.JSONDecodeError as e:\n self.excavate.debug(f\"Failed to decode JSON: {e}\")\n return None\n\n class PostJquery(GetJquery):\n name = \"POST jquery\"\n discovery_regex = r\"/\\$.post\\([^\\)].+\\)/ nocase\"\n extraction_regex = re.compile(r\"\\$.post\\([\\'\\\"](.+)[\\'\\\"].+(\\{.+\\})\\)\")\n output_type = \"POSTPARAM\"\n\n class HtmlTags(ParameterExtractorRule):\n name = \"HTML Tags\"\n discovery_regex = r'/<[^>]+(href|src|action)=[\"\\']?[^\"\\'>\\s]*[\"\\']?[^>]*>/ nocase'\n extraction_regex = bbot_regexes.tag_attribute_regex\n output_type = \"GETPARAM\"\n\n async def extract(self):\n urls = await self.excavate.helpers.re.findall(self.extraction_regex, str(self.result))\n for url in urls:\n parsed_url = urlparse(url)\n query_strings = parse_qs(html.unescape(parsed_url.query))\n query_strings_dict = {k: v[0] if isinstance(v, list) else v for k, v in query_strings.items()}\n for parameter_name, original_value in query_strings_dict.items():\n yield (\n self.output_type,\n parameter_name,\n original_value.strip(),\n url,\n _exclude_key(query_strings_dict, parameter_name),\n )\n\n class AjaxJquery(ParameterExtractorRule):\n name = \"JQuery Extractor\"\n discovery_regex = r\"/\\$\\.ajax\\(\\{[^\\<$\\$]*\\}\\)/s nocase\"\n extraction_regex = None\n output_type = \"BODYJSON\"\n ajax_content_regexes = {\n \"url\": re.compile(r\"url\\s*:\\s*['\\\"](.*?)['\\\"]\"),\n \"type\": re.compile(r\"type\\s*:\\s*['\\\"](.*?)['\\\"]\"),\n \"content_type\": re.compile(r\"contentType\\s*:\\s*['\\\"](.*?)['\\\"]\"),\n \"data\": re.compile(r\"data:.*(\\{[^}]*\\})\"),\n }\n\n async def extract(self):\n # Iterate through each regex in ajax_content_regexes\n extracted_values = {}\n for key, pattern in self.ajax_content_regexes.items():\n match = await self.excavate.helpers.re.search(pattern, self.result)\n if match:\n # Store the matched value in the dictionary\n extracted_values[key] = match.group(1)\n\n # Check to see if the format is defined as JSON\n if (\n \"content_type\" in extracted_values.keys()\n and extracted_values[\"content_type\"] == \"application/json\"\n ):\n form_parameters = {}\n\n # If we can't figure out the parameter names, there is no point in continuing\n if \"data\" in extracted_values.keys():\n form_url = extracted_values.get(\"url\", None)\n\n try:\n s = extracted_values[\"data\"]\n s = await self.excavate.helpers.re.sub(re.compile(r\"(\\w+)\\s*:\"), r'\"\\1\":', s) # Quote keys\n s = await self.excavate.helpers.re.sub(\n re.compile(r\":\\s*(\\w+)\"), r': \"\\1\"', s\n ) # Quote values if they are unquoted\n data = json.loads(s)\n except (ValueError, SyntaxError):\n data = None\n\n if data:\n for p in data.keys():\n form_parameters[p] = None\n\n for parameter_name in form_parameters:\n yield (\n \"BODYJSON\",\n parameter_name,\n None,\n form_url,\n _exclude_key(form_parameters, parameter_name),\n )\n\n class GetForm(ParameterExtractorRule):\n name = \"GET Form\"\n discovery_regex = r'/]*\\bmethod=[\"\\']?get[\"\\']?[^>]*>.*<\\/form>/s nocase'\n form_content_regexes = {\n \"input_tag_regex\": bbot_regexes.input_tag_regex,\n \"input_tag_regex2\": bbot_regexes.input_tag_regex2,\n \"select_tag_regex\": bbot_regexes.select_tag_regex,\n \"textarea_tag_regex\": bbot_regexes.textarea_tag_regex,\n \"textarea_tag_regex2\": bbot_regexes.textarea_tag_regex2,\n \"textarea_tag_novalue_regex\": bbot_regexes.textarea_tag_novalue_regex,\n \"button_tag_regex\": bbot_regexes.button_tag_regex,\n \"button_tag_regex2\": bbot_regexes.button_tag_regex2,\n \"_input_tag_novalue_regex\": bbot_regexes.input_tag_novalue_regex,\n }\n extraction_regex = bbot_regexes.get_form_regex\n output_type = \"GETPARAM\"\n\n async def extract(self):\n forms = await self.excavate.helpers.re.findall(self.extraction_regex, str(self.result))\n for form_action, form_content in forms:\n if not form_action or form_action == \"#\":\n form_action = None\n\n elif form_action.startswith(\"./\"):\n form_action = form_action.lstrip(\".\")\n\n form_parameters = {}\n for form_content_regex_name, form_content_regex in self.form_content_regexes.items():\n input_tags = await self.excavate.helpers.re.findall(form_content_regex, form_content)\n if input_tags:\n # Normalize each input_tag to be a tuple of two elements\n input_tags = [(tag if isinstance(tag, tuple) else (tag, None)) for tag in input_tags]\n\n if form_content_regex_name in [\n \"input_tag_regex2\",\n \"button_tag_regex2\",\n \"textarea_tag_regex2\",\n ]:\n # Swap elements if needed\n input_tags = [(b, a) for a, b in input_tags]\n for parameter_name, original_value in input_tags:\n form_parameters.setdefault(\n parameter_name, original_value.strip() if original_value else None\n )\n\n for parameter_name, original_value in form_parameters.items():\n yield (\n self.output_type,\n parameter_name,\n original_value,\n form_action,\n _exclude_key(form_parameters, parameter_name),\n )\n\n class GetForm2(GetForm):\n extraction_regex = bbot_regexes.get_form_regex2\n\n class PostForm(GetForm):\n name = \"POST Form\"\n discovery_regex = r'/]*\\bmethod=[\"\\']?post[\"\\']?[^>]*>.*<\\/form>/s nocase'\n extraction_regex = bbot_regexes.post_form_regex\n output_type = \"POSTPARAM\"\n\n class PostForm2(PostForm):\n extraction_regex = bbot_regexes.post_form_regex2\n\n class PostForm_NoAction(PostForm):\n name = \"POST Form (no action)\"\n extraction_regex = bbot_regexes.post_form_regex_noaction\n\n # underscore ensure generic forms runs last, so it doesn't cause dedupe to stop full form detection\n class _GenericForm(GetForm):\n name = \"Generic Form\"\n discovery_regex = r\"/]*>.*<\\/form>/s nocase\"\n\n extraction_regex = bbot_regexes.generic_form_regex\n output_type = \"GETPARAM\"\n\n def __init__(self, excavate):\n super().__init__(excavate)\n self.parameterExtractorCallbackDict = {}\n regexes_component_list = []\n parameterExtractorRules = find_subclasses(self, self.ParameterExtractorRule)\n for r in parameterExtractorRules:\n self.excavate.verbose(f\"Including ParameterExtractor Submodule: {r.__name__}\")\n self.parameterExtractorCallbackDict[r.__name__] = r\n regexes_component_list.append(f\"${r.__name__} = {r.discovery_regex}\")\n regexes_component = \" \".join(regexes_component_list)\n self.yara_rules[\"parameter_extraction\"] = (\n rf'rule parameter_extraction {{meta: description = \"contains Parameter\" strings: {regexes_component} condition: any of them}}'\n )\n\n async def process(self, yara_results, event, yara_rule_settings, discovery_context):\n for identifier, results in yara_results.items():\n for result in results:\n if identifier not in self.parameterExtractorCallbackDict.keys():\n raise ExcavateError(\"ParameterExtractor YaraRule identified reference non-existent submodule\")\n parameterExtractorSubModule = self.parameterExtractorCallbackDict[identifier](\n self.excavate, result\n )\n\n # Use async for to iterate over the async generator\n async for (\n parameter_type,\n parameter_name,\n original_value,\n endpoint,\n additional_params,\n ) in parameterExtractorSubModule.extract():\n self.excavate.debug(\n f\"Found Parameter [{parameter_name}] in [{parameterExtractorSubModule.name}] ParameterExtractor Submodule\"\n )\n\n # account for the case where the action is html encoded\n if endpoint and (\n endpoint.startswith(\"https://\")\n or endpoint.startswith(\"http://\")\n ):\n endpoint = html.unescape(endpoint)\n\n # If we have a full URL, leave it as-is\n if endpoint and endpoint.startswith((\"http://\", \"https://\")):\n url = endpoint\n\n # The endpoint is usually a form action - we should use it if we have it. If not, default to URL.\n else:\n # Use the original URL as the base and resolve the endpoint correctly in case of relative paths\n base_url = f\"{event.parsed_url.scheme}://{event.parsed_url.netloc}{event.parsed_url.path}\"\n if not self.excavate.remove_querystring and len(event.parsed_url.query) > 0:\n base_url += f\"?{event.parsed_url.query}\"\n url = urljoin(base_url, endpoint)\n\n try:\n # Validate the URL before using it\n parsed_url = self.excavate.helpers.validators.validate_url_parsed(url)\n except (ValidationError, ValueError) as e:\n self.excavate.debug(f\"Invalid URL [{url}]: {e}\")\n continue\n\n if self.excavate.helpers.validate_parameter(parameter_name, parameter_type):\n if self.excavate.in_bl(parameter_name) is False:\n description = f\"HTTP Extracted Parameter [{parameter_name}] ({parameterExtractorSubModule.name} Submodule)\"\n data = {\n \"host\": parsed_url.hostname,\n \"type\": parameter_type,\n \"name\": parameter_name,\n \"original_value\": original_value,\n \"url\": self.excavate.url_unparse(parameter_type, parsed_url),\n \"additional_params\": additional_params,\n \"assigned_cookies\": self.excavate.assigned_cookies,\n \"description\": description,\n }\n await self.report(\n data, event, yara_rule_settings, discovery_context, event_type=\"WEB_PARAMETER\"\n )\n else:\n self.excavate.debug(f\"blocked parameter [{parameter_name}] due to BL match\")\n else:\n self.excavate.debug(f\"blocked parameter [{parameter_name}] due to validation failure\")\n\n class CSPExtractor(ExcavateRule):\n description = \"Extracts domains from CSP headers.\"\n\n yara_rules = {\n \"csp\": r'rule csp { meta: tags = \"affiliate\" description = \"contains CSP Header\" strings: $csp = /Content-Security-Policy:[^\\r\\n]+/ nocase condition: $csp }',\n }\n\n async def process(self, yara_results, event, yara_rule_settings, discovery_context):\n for identifier in yara_results.keys():\n for csp_str in yara_results[identifier]:\n domains = await self.excavate.scan.extract_in_scope_hostnames(csp_str)\n for domain in domains:\n await self.report(domain, event, yara_rule_settings, discovery_context, event_type=\"DNS_NAME\")\n\n class EmailExtractor(ExcavateRule):\n description = \"Extract email addresses.\"\n\n yara_rules = {\n \"email\": 'rule email { meta: description = \"contains email address\" strings: $email = /[^\\\\W_][\\\\w\\\\-\\\\.\\\\+\\']{0,100}@[a-zA-Z0-9\\\\-]{1,100}(\\\\.[a-zA-Z0-9\\\\-]{1,100})*\\\\.[a-zA-Z]{2,63}/ nocase fullword condition: $email }',\n }\n\n async def process(self, yara_results, event, yara_rule_settings, discovery_context):\n for identifier in yara_results.keys():\n for email_str in yara_results[identifier]:\n await self.report(\n email_str, event, yara_rule_settings, discovery_context, event_type=\"EMAIL_ADDRESS\"\n )\n\n # Future Work: Emit a JWT Object, and make a new Module to ingest it.\n class JWTExtractor(ExcavateRule):\n description = \"Extracts JSON Web Tokens.\"\n yara_rules = {\n \"jwt\": r'rule jwt { meta: emit_match = \"True\" description = \"contains JSON Web Token (JWT)\" strings: $jwt = /\\beyJ[_a-zA-Z0-9\\/+]*\\.[_a-zA-Z0-9\\/+]*\\.[_a-zA-Z0-9\\/+]*/ nocase condition: $jwt }',\n }\n\n class ErrorExtractor(ExcavateRule):\n description = \"Identifies error messages from various platforms.\"\n signatures = {\n \"PHP_1\": r\"/\\.php on line [0-9]+/\",\n \"PHP_2\": r\"/\\.php<\\/b> on line [0-9]+/\",\n \"PHP_3\": '\"Fatal error:\"',\n \"Microsoft_SQL_Server_1\": r\"/\\[(ODBC SQL Server Driver|SQL Server|ODBC Driver Manager)\\]/\",\n \"Microsoft_SQL_Server_2\": '\"You have an error in your SQL syntax; check the manual\"',\n \"Java_1\": r\"/\\.java:[0-9]+/\",\n \"Java_2\": r\"/\\.java\\((Inlined )?Compiled Code\\)/\",\n \"Perl\": r\"/at (\\/[A-Za-z0-9\\._]+)*\\.pm line [0-9]+/\",\n \"Python\": r\"/File \\\"[A-Za-z0-9\\-_\\.\\/]*\\\", line [0-9]+, in/\",\n \"Ruby\": r\"/\\.rb:[0-9]+:in/\",\n \"ASPNET_1\": '\"Exception of type\"',\n \"ASPNET_2\": '\"--- End of inner exception stack trace ---\"',\n \"ASPNET_3\": '\"Microsoft OLE DB Provider\"',\n \"ASPNET_4\": r\"/Error ([\\d-]+) \\([\\dA-F]+\\)/\",\n }\n yara_rules = {}\n\n def __init__(self, excavate):\n super().__init__(excavate)\n signature_component_list = []\n for signature_name, signature in self.signatures.items():\n signature_component_list.append(rf\"${signature_name} = {signature}\")\n signature_component = \" \".join(signature_component_list)\n self.yara_rules[\"error_detection\"] = (\n f'rule error_detection {{meta: description = \"contains a verbose error message\" strings: {signature_component} condition: any of them}}'\n )\n\n async def process(self, yara_results, event, yara_rule_settings, discovery_context):\n for identifier in yara_results.keys():\n for findings in yara_results[identifier]:\n event_data = {\n \"description\": f\"{discovery_context} {yara_rule_settings.description} ({identifier})\"\n }\n await self.report(event_data, event, yara_rule_settings, discovery_context, event_type=\"FINDING\")\n\n class SerializationExtractor(ExcavateRule):\n description = \"Identifies serialized objects from various platforms.\"\n regexes = {\n \"Java\": re.compile(r\"[^a-zA-Z0-9\\/+][\\\"']?rO0[a-zA-Z0-9+\\/]+={0,2}\"),\n \"Ruby\": re.compile(r\"[^a-zA-Z0-9\\/+][\\\"']?BAh[a-zA-Z0-9+\\/]+={0,2}\"),\n \"DOTNET\": re.compile(r\"[^a-zA-Z0-9\\/+][\\\"']?AAEAAAD\\/\\/[a-zA-Z0-9\\/+]+={0,2}\"),\n \"PHP_Array\": re.compile(r\"[^a-zA-Z0-9\\/+][\\\"']?YTo[xyz0123456][a-zA-Z0-9+\\/]+={0,2}\"),\n \"PHP_String\": re.compile(r\"[^a-zA-Z0-9\\/+][\\\"']?czo[xyz0123456][a-zA-Z0-9+\\/]+={0,2}\"),\n \"PHP_Object\": re.compile(r\"[^a-zA-Z0-9\\/+][\\\"']?Tzo[xyz0123456][a-zA-Z0-9+\\/]+={0,2}\"),\n \"Possible_Compressed\": re.compile(r\"[^a-zA-Z0-9\\/+][\\\"']?H4sIAAAA[a-zA-Z0-9+\\/]+={0,2}\"),\n }\n yara_rules = {}\n\n def __init__(self, excavate):\n super().__init__(excavate)\n regexes_component_list = []\n for regex_name, regex in self.regexes.items():\n regexes_component_list.append(rf\"${regex_name} = /\\b{regex.pattern}/\")\n regexes_component = \" \".join(regexes_component_list)\n self.yara_rules[\"serialization_detection\"] = (\n f'rule serialization_detection {{meta: description = \"contains a possible serialized object\" strings: {regexes_component} condition: any of them}}'\n )\n\n async def process(self, yara_results, event, yara_rule_settings, discovery_context):\n for identifier in yara_results.keys():\n for findings in yara_results[identifier]:\n event_data = {\n \"description\": f\"{discovery_context} {yara_rule_settings.description} ({identifier})\"\n }\n await self.report(event_data, event, yara_rule_settings, discovery_context, event_type=\"FINDING\")\n\n class FunctionalityExtractor(ExcavateRule):\n description = \"Detects potentially exploitable functionality and attack surface in web applications.\"\n yara_rules = {\n \"File_Upload_Functionality\": r'rule File_Upload_Functionality { meta: description = \"contains file upload functionality\" strings: $fileuploadfunc = /]+type=[\"\\']?file[\"\\']?[^>]+>/ nocase condition: $fileuploadfunc }',\n \"Web_Service_WSDL\": r'rule Web_Service_WSDL { meta: emit_match = \"True\" description = \"contains a web service WSDL URL\" strings: $wsdl = /https?:\\/\\/[^\\s]*\\.(wsdl)/ nocase condition: $wsdl }',\n }\n\n class NonHttpSchemeExtractor(ExcavateRule):\n description = \"Detects URIs with non-HTTP schemes.\"\n yara_rules = {\n \"Non_HTTP_Scheme\": r'rule Non_HTTP_Scheme { meta: description = \"contains non-http scheme URL\" strings: $nonhttpscheme = /\\b\\w{2,35}:\\/\\/[\\w.-]+(:\\d+)?\\b/ nocase fullword condition: $nonhttpscheme }'\n }\n\n scheme_blacklist = [\"javascript\", \"mailto\", \"tel\", \"data\", \"vbscript\", \"about\", \"file\"]\n\n async def process(self, yara_results, event, yara_rule_settings, discovery_context):\n for results in yara_results.values():\n for url_str in results:\n scheme = url_str.split(\"://\")[0]\n if scheme in self.scheme_blacklist:\n continue\n if scheme not in self.excavate.valid_schemes:\n continue\n try:\n parsed_url = urlparse(url_str)\n except Exception as e:\n self.excavate.debug(f\"Error parsing URI {url_str}: {e}\")\n continue\n netloc = getattr(parsed_url, \"netloc\", None)\n if netloc is None:\n continue\n try:\n host, port = self.excavate.helpers.split_host_port(parsed_url.netloc)\n except ValueError as e:\n self.excavate.debug(f\"Failed to parse netloc: {e}\")\n continue\n if parsed_url.scheme in [\"http\", \"https\"]:\n continue\n\n def abort_if(e):\n return e.scope_distance > 0\n\n finding_data = {\"host\": str(host), \"description\": f\"Non-HTTP URI: {parsed_url.geturl()}\"}\n await self.report(finding_data, event, yara_rule_settings, discovery_context, abort_if=abort_if)\n protocol_data = {\"protocol\": parsed_url.scheme, \"host\": str(host)}\n if port:\n protocol_data[\"port\"] = port\n await self.report(\n protocol_data,\n event,\n yara_rule_settings,\n discovery_context,\n event_type=\"PROTOCOL\",\n abort_if=abort_if,\n )\n\n class URLExtractor(ExcavateRule):\n description = \"Extracts URLs.\"\n yara_rules = {\n \"url_full\": (\n r\"\"\"\n rule url_full {\n meta:\n tags = \"spider-danger\"\n description = \"contains full URL\"\n strings:\n $url_full = /https?:\\/\\/([\\w\\.-]+)(:\\d{1,5})?([\\/\\w\\.-]*)/\n condition:\n $url_full\n }\n \"\"\"\n ),\n \"url_attr\": (\n r\"\"\"\n rule url_attr {\n meta:\n tags = \"spider-danger\"\n description = \"contains tag with src or href attribute\"\n strings:\n $url_attr = /<[^>]+(href|src|action)=[\"\\']?[^\"\\']*[\"\\']?[^>]*>/\n condition:\n $url_attr\n }\n \"\"\"\n ),\n }\n full_url_regex = re.compile(r\"(https?)://(\\w(?:[\\w-]+\\.?)+(?::\\d{1,5})?(?:/[-\\w\\.\\(\\)]*[-\\w\\.]+)*/?)\")\n full_url_regex_strict = re.compile(r\"^(https?):\\/\\/([\\w.-]+)(?::\\d{1,5})?(\\/[\\w\\/\\.-]*)?(\\?[^\\s]+)?$\")\n tag_attribute_regex = bbot_regexes.tag_attribute_regex\n\n async def process(self, yara_results, event, yara_rule_settings, discovery_context):\n for identifier, results in yara_results.items():\n urls_found = 0\n final_url = \"\"\n for url_str in results:\n try:\n if identifier == \"url_full\":\n if not await self.helpers.re.search(self.full_url_regex, url_str):\n self.excavate.debug(\n f\"Rejecting potential full URL [{url_str}] as did not match full_url_regex\"\n )\n continue\n final_url = url_str\n self.excavate.debug(f\"Discovered Full URL [{final_url}]\")\n\n elif identifier == \"url_attr\" and hasattr(event, \"parsed_url\"):\n m = await self.helpers.re.search(self.tag_attribute_regex, url_str)\n if not m:\n self.excavate.debug(\n f\"Rejecting potential attribute URL [{url_str}] as did not match tag_attribute_regex\"\n )\n continue\n unescaped_url = html.unescape(m.group(1))\n source_url = event.parsed_url.geturl()\n final_url = urldefrag(urljoin(source_url, unescaped_url)).url\n if not await self.helpers.re.search(self.full_url_regex_strict, final_url):\n self.excavate.debug(\n f\"Rejecting reconstructed URL [{final_url}] as did not match full_url_regex_strict\"\n )\n continue\n self.excavate.debug(\n f\"Reconstructed Full URL [{final_url}] from extracted relative URL [{unescaped_url}] \"\n )\n\n if final_url:\n # Validate the URL before using it\n self.excavate.helpers.validators.validate_url_parsed(final_url)\n if self.excavate.scan.in_scope(final_url):\n urls_found += 1\n await self.report(\n final_url,\n event,\n yara_rule_settings,\n discovery_context,\n event_type=\"URL_UNVERIFIED\",\n urls_found=urls_found,\n )\n except (ValidationError, ValueError) as e:\n self.excavate.debug(f\"Invalid URL [{url_str if not final_url else final_url}]: {e}\")\n continue\n\n async def report_prep(self, event_data, event_type, event, tags, **kwargs):\n event_draft = self.excavate.make_event(event_data, event_type, parent=event)\n if not event_draft:\n return None\n url_in_scope = self.excavate.scan.in_scope(event_draft.host_filterable)\n urls_found = kwargs.get(\"urls_found\", None)\n if urls_found:\n exceeds_max_links = urls_found > self.excavate.scan.web_spider_links_per_page and url_in_scope\n if exceeds_max_links:\n tags.append(\"spider-max\")\n event_draft.add_tags(tags)\n return event_draft\n\n class HostnameExtractor(ExcavateRule):\n description = \"DNS name discovery, based on the scan target.\"\n\n yara_rules = {}\n\n def __init__(self, excavate):\n super().__init__(excavate)\n self.yara_rules.update(excavate.scan.dns_yara_rules_uncompiled)\n\n async def process(self, yara_results, event, yara_rule_settings, discovery_context):\n for identifier in yara_results.keys():\n for domain_str in yara_results[identifier]:\n await self.report(domain_str, event, yara_rule_settings, discovery_context, event_type=\"DNS_NAME\")\n\n class LoginPageExtractor(ExcavateRule):\n description = \"Detects login pages with username and password fields.\"\n yara_rules = {\n \"login_page\": r\"\"\"\n rule login_page {\n meta:\n description = \"Detects login pages with username and password fields\"\n strings:\n $username_field = /]+name=[\"']?(user|login|email)/ nocase\n $password_field = /]+name=[\"']?passw?/ nocase\n condition:\n $username_field and $password_field\n }\n \"\"\"\n }\n\n async def process(self, yara_results, event, yara_rule_settings, discovery_context):\n if yara_results:\n event.add_tag(\"login-page\")\n\n def add_yara_rule(self, rule_name, rule_content, rule_instance):\n rule_instance.name = rule_name\n self.yara_rules_dict[rule_name] = rule_content\n self.yara_preprocess_dict[rule_name] = rule_instance.preprocess\n\n async def extract_yara_rules(self, rules_content):\n for r in await self.helpers.re.findall(self.yara_rule_regex, rules_content):\n yield r\n\n async def emit_web_parameter(\n self, host, param_type, name, original_value, url, description, additional_params, event, context\n ):\n data = {\n \"host\": host,\n \"type\": param_type,\n \"name\": name,\n \"original_value\": original_value,\n \"url\": url,\n \"description\": description,\n \"additional_params\": additional_params,\n }\n await self.emit_event(data, \"WEB_PARAMETER\", event, context=context)\n\n async def emit_custom_parameters(self, event, config_key, param_type, description_suffix):\n # Emits WEB_PARAMETER events for custom headers and cookies from the configuration.\n custom_params = self.scan.web_config.get(config_key, {})\n for param_name, param_value in custom_params.items():\n await self.emit_web_parameter(\n host=event.parsed_url.hostname,\n param_type=param_type,\n name=param_name,\n original_value=param_value,\n url=self.url_unparse(param_type, event.parsed_url),\n description=f\"HTTP Extracted Parameter [{param_name}] ({description_suffix})\",\n additional_params=_exclude_key(custom_params, param_name),\n event=event,\n context=f\"Excavate saw a custom {param_type.lower()} set [{param_name}], and emitted a WEB_PARAMETER for it\",\n )\n\n async def setup(self):\n self.yara_rules_dict = {}\n self.yara_preprocess_dict = {}\n\n modules_WEB_PARAMETER = [\n module_name\n for module_name, module in self.scan.modules.items()\n if \"WEB_PARAMETER\" in module.watched_events\n ]\n\n self.parameter_extraction = bool(modules_WEB_PARAMETER)\n self.speculate_params = bool(self.config.get(\"speculate_params\", False))\n self.remove_querystring = self.scan.config.get(\"url_querystring_remove\", True)\n\n for module in self.scan.modules.values():\n if not str(module).startswith(\"_\"):\n ExcavateRules = find_subclasses(module, ExcavateRule)\n for e in ExcavateRules:\n self.debug(f\"Including Submodule {e.__name__}\")\n if e.__name__ == \"ParameterExtractor\":\n message = (\n \"Parameter Extraction disabled because no modules consume WEB_PARAMETER events\"\n if not self.parameter_extraction\n else f\"Parameter Extraction enabled because the following modules consume WEB_PARAMETER events: [{', '.join(modules_WEB_PARAMETER)}]\"\n )\n self.debug(message) if not self.parameter_extraction else self.hugeinfo(message)\n # do not add parameter extraction yara rules if it's disabled\n if not self.parameter_extraction:\n continue\n excavateRule = e(self)\n for rule_name, rule_content in excavateRule.yara_rules.items():\n self.add_yara_rule(rule_name, rule_content, excavateRule)\n\n self.parameter_blacklist = set(p.lower() for p in self.scan.config.get(\"parameter_blacklist\", []))\n self.parameter_blacklist_prefixes = set(self.scan.config.get(\"parameter_blacklist_prefixes\", []))\n\n self.custom_yara_rules = str(self.config.get(\"custom_yara_rules\", \"\"))\n if self.custom_yara_rules:\n custom_rules_count = 0\n if Path(self.custom_yara_rules).is_file():\n with open(self.custom_yara_rules) as f:\n rules_content = f.read()\n self.debug(f\"Successfully loaded custom yara rules file [{self.custom_yara_rules}]\")\n else:\n self.debug(\"Custom yara rules file is NOT a file. Will attempt to treat it as rule content\")\n rules_content = self.custom_yara_rules\n\n self.debug(f\"Final combined yara rule contents: {rules_content}\")\n custom_yara_rule_processed = self.extract_yara_rules(rules_content)\n async for rule_content in custom_yara_rule_processed:\n try:\n yara.compile(source=rule_content)\n except yara.SyntaxError as e:\n return False, f\"Custom Yara rule failed to compile: {e}\"\n\n rule_match = await self.helpers.re.search(self.yara_rule_name_regex, rule_content)\n if not rule_match:\n return False, \"Custom Yara formatted incorrectly: could not find rule name\"\n\n rule_name = rule_match.groups(1)[0]\n c = CustomExtractor(self)\n self.add_yara_rule(rule_name, rule_content, c)\n custom_rules_count += 1\n if custom_rules_count > 0:\n self.hugeinfo(f\"Successfully added {str(custom_rules_count)} custom Yara rule(s)\")\n\n yara_max_match_data = self.config.get(\"yara_max_match_data\", 2000)\n\n yara.set_config(max_match_data=yara_max_match_data)\n yara_rules_combined = \"\\n\".join(self.yara_rules_dict.values())\n try:\n start = time.time()\n self.verbose(f\"Compiling {len(self.yara_rules_dict):,} YARA rules\")\n for rule_name, rule_content in self.yara_rules_dict.items():\n self.debug(f\" - {rule_name}\")\n self.yara_rules = yara.compile(source=yara_rules_combined)\n self.verbose(f\"{len(self.yara_rules_dict):,} YARA rules compiled in {time.time() - start:.2f} seconds\")\n except yara.SyntaxError as e:\n self.debug(yara_rules_combined)\n return False, f\"Yara Rules failed to compile with error: [{e}]\"\n\n # pre-load valid URL schemes\n valid_schemes_filename = self.helpers.wordlist_dir / \"valid_url_schemes.txt\"\n self.valid_schemes = set(self.helpers.read_file(valid_schemes_filename))\n\n self.url_querystring_remove = self.scan.config.get(\"url_querystring_remove\", True)\n\n return True\n\n async def search(self, data, event, content_type, discovery_context=\"HTTP response\"):\n if not data:\n return None\n decoded_data = await self.helpers.re.recursive_decode(data)\n\n if self.parameter_extraction and self.speculate_params:\n content_type_lower = content_type.lower() if content_type else \"\"\n extraction_map = {\n \"json\": self.helpers.extract_params_json,\n \"xml\": self.helpers.extract_params_xml,\n }\n\n for source_type, extract_func in extraction_map.items():\n if source_type in content_type_lower:\n results = extract_func(data)\n if results:\n for parameter_name, original_value in results:\n await self.emit_web_parameter(\n host=str(event.host),\n param_type=\"SPECULATIVE\",\n name=parameter_name,\n original_value=original_value,\n url=str(event.data[\"url\"]),\n description=f\"HTTP Extracted Parameter (speculative from {source_type} content) [{parameter_name}]\",\n additional_params={},\n event=event,\n context=f\"excavate's Parameter extractor found a speculative WEB_PARAMETER: {parameter_name} by parsing {source_type} data from {str(event.host)}\",\n )\n return\n\n # Initialize the list of data items to process\n data_items = []\n\n # Check if data and decoded_data are identical\n if data == decoded_data:\n data_items.append((\"data\", data)) # Add only one since both are the same\n else:\n data_items.append((\"data\", data))\n data_items.append((\"decoded_data\", decoded_data))\n\n for label, data_instance in data_items:\n # Your existing processing code\n for result in self.yara_rules.match(data=f\"{data_instance}\"):\n rule_name = result.rule\n\n # Skip specific operations for 'parameter_extraction' rule on decoded_data\n if label == \"decoded_data\" and rule_name == \"parameter_extraction\":\n continue\n\n # Check if rule processing function exists\n if rule_name in self.yara_preprocess_dict:\n try:\n await self.yara_preprocess_dict[rule_name](result, event, discovery_context)\n except ValidationError as e:\n self.debug(f\"ValidationError in rule {rule_name} for result {result}: {e}\")\n else:\n self.hugewarning(f\"YARA Rule {rule_name} not found in pre-compiled rules\")\n\n async def handle_event(self, event, **kwargs):\n if event.type == \"HTTP_RESPONSE\":\n if self.parameter_extraction is True:\n # if parameter extraction is enabled, and we have custom cookies or headers, emit them as WEB_PARAMETER events\n await self.emit_custom_parameters(event, \"http_cookies\", \"COOKIE\", \"Custom Cookie\")\n await self.emit_custom_parameters(event, \"http_headers\", \"HEADER\", \"Custom Header\")\n\n # if parameter extraction is enabled, and querystring removal is disabled, and the event is directly from the TARGET, create a WEB\n if self.url_querystring_remove is False and str(event.parent.parent.module) == \"TARGET\":\n self.debug(f\"Processing target URL [{urlunparse(event.parsed_url)}] for GET parameters\")\n for (\n method,\n parsed_url,\n parameter_name,\n original_value,\n regex_name,\n additional_params,\n ) in extract_params_url(event.parsed_url):\n if self.in_bl(parameter_name) is False:\n await self.emit_web_parameter(\n host=parsed_url.hostname,\n param_type=\"GETPARAM\",\n name=parameter_name,\n original_value=original_value,\n url=self.url_unparse(\"GETPARAM\", parsed_url),\n description=f\"HTTP Extracted Parameter [{parameter_name}] (Target URL)\",\n additional_params=additional_params,\n event=event,\n context=f\"Excavate parsed a URL directly from the scan target for parameters and found [GETPARAM] Parameter Name: [{parameter_name}] and emitted a WEB_PARAMETER for it\",\n )\n\n # process response data\n body = event.data.get(\"body\", \"\")\n headers = event.data.get(\"header-dict\", {})\n if body == \"\" and headers == {}:\n return\n\n self.assigned_cookies = {}\n content_type = None\n reported_location_header = False\n\n for header, header_values in headers.items():\n for header_value in header_values:\n # Process 'set-cookie' headers to extract and emit cookies as WEB_PARAMETER events.\n if header.lower() == \"set-cookie\" and self.parameter_extraction:\n if \"=\" not in header_value:\n self.debug(f\"Cookie found without '=': {header_value}\")\n continue\n else:\n cookie_name, _, remainder = header_value.partition(\"=\")\n cookie_value = remainder.split(\";\")[0]\n\n if self.in_bl(cookie_name) is False:\n self.assigned_cookies[cookie_name] = cookie_value\n await self.emit_web_parameter(\n host=str(event.host),\n param_type=\"COOKIE\",\n name=cookie_name,\n original_value=cookie_value,\n url=self.url_unparse(\"COOKIE\", event.parsed_url),\n description=f\"Set-Cookie Assigned Cookie [{cookie_name}]\",\n additional_params={},\n event=event,\n context=f\"Excavate noticed a set-cookie header for cookie [{cookie_name}] and emitted a WEB_PARAMETER for it\",\n )\n else:\n self.debug(f\"blocked cookie parameter [{cookie_name}] due to BL match\")\n # Handle 'location' headers to process and emit redirect URLs as URL_UNVERIFIED events.\n if header.lower() == \"location\":\n redirect_location = getattr(event, \"redirect_location\", \"\")\n if redirect_location:\n scheme = self.helpers.is_uri(redirect_location, return_scheme=True)\n if scheme in (\"http\", \"https\"):\n web_spider_distance = getattr(event, \"web_spider_distance\", 0)\n num_redirects = max(getattr(event, \"num_redirects\", 0), web_spider_distance)\n if num_redirects <= self.scan.web_max_redirects:\n # we do not want to allow the web_spider_distance to be incremented on redirects, so we do not add spider-danger tag\n url_event = self.make_event(\n redirect_location, \"URL_UNVERIFIED\", event, tags=\"affiliate\"\n )\n if url_event is not None:\n reported_location_header = True\n await self.emit_event(\n url_event,\n context=f'excavate looked in \"Location\" header and found {url_event.type}: {url_event.data}',\n )\n\n # Try to extract parameters from the redirect URL\n if self.parameter_extraction:\n for (\n method,\n parsed_url,\n parameter_name,\n original_value,\n regex_name,\n additional_params,\n ) in extract_params_location(header_value, event.parsed_url):\n if self.in_bl(parameter_name) is False:\n await self.emit_web_parameter(\n host=parsed_url.hostname,\n param_type=\"GETPARAM\",\n name=parameter_name,\n original_value=original_value,\n url=self.url_unparse(\"GETPARAM\", parsed_url),\n description=f\"HTTP Extracted Parameter [{parameter_name}] (Location Header)\",\n additional_params=additional_params,\n event=event,\n context=f\"Excavate parsed a location header for parameters and found [GETPARAM] Parameter Name: [{parameter_name}] and emitted a WEB_PARAMETER for it\",\n )\n else:\n self.warning(\"location header found but missing redirect_location in HTTP_RESPONSE\")\n if header.lower() == \"content-type\":\n content_type = headers[\"content-type\"][0]\n\n # skip PDF responses -- running YARA/regex on raw PDF bytes produces false positives and wastes time.\n # PDFs are still processed correctly via the filedownload → extractous → RAW_TEXT pipeline,\n # which extracts readable text and feeds it back to excavate as a RAW_TEXT event (handled separately below).\n # TODO: remove this in favor of a proper categorization system for text vs non-text (i.e. to-be-extracted) content\n if content_type and \"application/pdf\" in content_type.lower():\n self.debug(f\"Skipping PDF response: {event.data.get('url', 'unknown')}\")\n return\n\n await self.search(\n body,\n event,\n content_type,\n discovery_context=\"HTTP response (body)\",\n )\n\n if reported_location_header:\n # Location header should be removed if we already found and emitted a result.\n # Failure to do so results in a race against the same URL extracted by the URLExtractor submodule\n # If the extracted URL wins, it will cause the manual one to be a dupe, but it will have a higher web_spider_distance.\n headers.pop(\"location\")\n headers_str = \"\\n\".join(f\"{k}: {v}\" for k, values in headers.items() for v in values)\n\n await self.search(\n headers_str,\n event,\n content_type,\n discovery_context=\"HTTP response (headers)\",\n )\n else:\n await self.search(\n event.data,\n event,\n content_type=\"\",\n discovery_context=\"Parsed file content\",\n )\n\n @classmethod\n def help_text(self):\n # Call the base class help_text method\n base_help_text = super().help_text()\n\n # Import the current module to inspect its classes\n import sys\n\n current_module = sys.modules[self.__module__]\n\n # Function to recursively find subclasses of ExcavateRule\n def find_subclasses(cls):\n subclasses = []\n for name, obj in vars(cls).items():\n if isinstance(obj, type) and issubclass(obj, ExcavateRule) and obj is not ExcavateRule:\n description = getattr(obj, \"description\", \"No description available.\")\n subclasses.append((name, description))\n # Recursively check for nested classes\n if isinstance(obj, type):\n subclasses.extend(find_subclasses(obj))\n return subclasses\n\n # Find all classes in the module that inherit from ExcavateRule\n submodules = find_subclasses(current_module)\n\n # Format submodules information\n submodules_info = \"\\nSubmodules:\\n\"\n if submodules:\n for submodule, description in submodules:\n submodules_info += f\" - {submodule}: {description}\\n\"\n else:\n submodules_info += \" No submodules available.\\n\"\n\n # Combine the base help text with the submodules information\n return base_help_text + submodules_info\n" }, { "path": "bbot/modules/internal/speculate.py", "content": "import random\nimport ipaddress\n\nfrom bbot.core.helpers import validators\nfrom bbot.modules.internal.base import BaseInternalModule\n\n\nclass speculate(BaseInternalModule):\n \"\"\"\n Bridge the gap between ranges and ips, or ips and open ports\n in situations where e.g. a port scanner isn't enabled\n \"\"\"\n\n watched_events = [\n \"IP_RANGE\",\n \"URL\",\n \"URL_UNVERIFIED\",\n \"DNS_NAME\",\n \"DNS_NAME_UNRESOLVED\",\n \"IP_ADDRESS\",\n \"HTTP_RESPONSE\",\n \"STORAGE_BUCKET\",\n \"SOCIAL\",\n \"AZURE_TENANT\",\n \"USERNAME\",\n ]\n produced_events = [\"DNS_NAME\", \"OPEN_TCP_PORT\", \"IP_ADDRESS\", \"FINDING\", \"ORG_STUB\"]\n flags = [\"passive\"]\n meta = {\n \"description\": \"Derive certain event types from others by common sense\",\n \"created_date\": \"2022-05-03\",\n \"author\": \"@liquidsec\",\n }\n\n options = {\"max_hosts\": 65536, \"ports\": \"80,443\", \"essential_only\": False}\n options_desc = {\n \"max_hosts\": \"Max number of IP_RANGE hosts to convert into IP_ADDRESS events\",\n \"ports\": \"The set of ports to speculate on\",\n \"essential_only\": \"Only enable essential speculate features (no extra discovery)\",\n }\n scope_distance_modifier = 1\n _priority = 4\n\n default_discovery_context = \"speculated {event.type}: {event.data}\"\n\n async def setup(self):\n scan_modules = [m for m in self.scan.modules.values() if m._type == \"scan\"]\n self.open_port_consumers = any(\"OPEN_TCP_PORT\" in m.watched_events for m in scan_modules)\n # only consider active portscanners (still speculate if only passive ones are enabled)\n self.portscanner_enabled = any(\n \"portscan\" in m.flags and \"active\" in m.flags for m in self.scan.modules.values()\n )\n self.emit_open_ports = self.open_port_consumers and not self.portscanner_enabled\n self.range_to_ip = True\n self.dns_disable = self.scan.config.get(\"dns\", {}).get(\"disable\", False)\n self.essential_only = self.config.get(\"essential_only\", False)\n self.org_stubs_seen = set()\n\n port_string = self.config.get(\"ports\", \"80,443\")\n try:\n self.ports = self.helpers.parse_port_string(str(port_string))\n except ValueError as e:\n return False, f\"Error parsing ports: {e}\"\n\n if not self.portscanner_enabled:\n self.info(f\"No portscanner enabled. Assuming open ports: {', '.join(str(x) for x in self.ports)}\")\n\n target_len = len(self.scan.target.seeds)\n if target_len > self.config.get(\"max_hosts\", 65536):\n if not self.portscanner_enabled:\n self.hugewarning(\n f\"Selected target ({target_len:,} hosts) is too large, skipping IP_RANGE --> IP_ADDRESS speculation\"\n )\n self.hugewarning('Enabling the \"portscan\" module is highly recommended')\n self.range_to_ip = False\n\n return True\n\n async def handle_event(self, event):\n ### BEGIN ESSENTIAL SPECULATION ###\n # These features are required for smooth operation of bbot\n # I.e. they are not \"osinty\" or intended to discover anything, they only compliment other modules\n\n # we speculate on distance-1 stuff too, because distance-1 open ports are needed by certain modules like sslcert\n event_in_scope_distance = event.scope_distance <= (self.scan.scope_search_distance + 1)\n speculate_open_ports = self.emit_open_ports and event_in_scope_distance\n\n # generate individual IP addresses from IP range\n if event.type == \"IP_RANGE\" and self.range_to_ip:\n net = ipaddress.ip_network(event.data)\n ips = list(net)\n random.shuffle(ips)\n for ip in ips:\n await self.emit_event(\n ip,\n \"IP_ADDRESS\",\n parent=event,\n internal=True,\n context=f\"speculate converted range into individual IP_ADDRESS: {ip}\",\n )\n\n # IP_ADDRESS / DNS_NAME --> OPEN_TCP_PORT\n if speculate_open_ports:\n # don't act on unresolved DNS_NAMEs\n usable_dns = False\n if event.type == \"DNS_NAME\":\n if self.dns_disable or event.resolved_hosts:\n usable_dns = True\n\n if event.type == \"IP_ADDRESS\" or usable_dns:\n for port in self.ports:\n await self.emit_event(\n self.helpers.make_netloc(event.data, port),\n \"OPEN_TCP_PORT\",\n parent=event,\n internal=True,\n context=\"speculated {event.type}: {event.data}\",\n )\n\n ### END ESSENTIAL SPECULATION ###\n if self.essential_only:\n return\n\n # parent domains\n if event.type.startswith(\"DNS_NAME\"):\n parent = self.helpers.parent_domain(event.host_original)\n if parent != event.data:\n await self.emit_event(\n parent, \"DNS_NAME\", parent=event, context=\"speculated parent {event.type}: {event.data}\"\n )\n\n # URL --> OPEN_TCP_PORT\n event_is_url = event.type == \"URL\"\n if event_is_url or (event.type == \"URL_UNVERIFIED\" and self.open_port_consumers):\n # only speculate port from a URL if it wouldn't be speculated naturally from the host\n if event.host and (event.port not in self.ports or not speculate_open_ports):\n await self.emit_event(\n self.helpers.make_netloc(event.host, event.port),\n \"OPEN_TCP_PORT\",\n parent=event,\n internal=not event_is_url, # if the URL is verified, the port is definitely open\n context=f\"speculated {{event.type}} from {event.type}: {{event.data}}\",\n )\n\n # speculate sub-directory URLS from URLS\n if event.type == \"URL\":\n url_parents = self.helpers.url_parents(event.data)\n for up in url_parents:\n url_event = self.make_event(f\"{up}/\", \"URL_UNVERIFIED\", parent=event)\n if url_event is not None:\n # inherit web spider distance from parent (don't increment)\n parent_web_spider_distance = getattr(event, \"web_spider_distance\", 0)\n url_event.web_spider_distance = parent_web_spider_distance\n await self.emit_event(url_event, context=\"speculated web sub-directory {event.type}: {event.data}\")\n\n # speculate URL_UNVERIFIED from URL or any event with \"url\" attribute\n event_is_url = event.type == \"URL\"\n event_has_url = isinstance(event.data, dict) and \"url\" in event.data\n event_tags = [\"httpx-safe\"] if event.type in (\"CODE_REPOSITORY\", \"SOCIAL\") else []\n if event_is_url or event_has_url:\n if event_is_url:\n url = event.data\n else:\n url = event.data[\"url\"]\n # only emit the url if it's not already in the event's history\n if not any(e.type == \"URL_UNVERIFIED\" and e.data == url for e in event.get_parents()):\n await self.emit_event(\n url,\n \"URL_UNVERIFIED\",\n tags=event_tags,\n parent=event,\n context=\"speculated {event.type}: {event.data}\",\n )\n\n # ORG_STUB from TLD, SOCIAL, AZURE_TENANT\n org_stubs = set()\n if event.type == \"DNS_NAME\" and event.scope_distance == 0:\n tldextracted = self.helpers.tldextract(event.data)\n top_domain_under_public_suffix = getattr(tldextracted, \"top_domain_under_public_suffix\", \"\")\n if top_domain_under_public_suffix:\n tld_stub = getattr(tldextracted, \"domain\", \"\")\n if tld_stub:\n decoded_tld_stub = self.helpers.smart_decode_punycode(tld_stub)\n org_stubs.add(decoded_tld_stub)\n org_stubs.add(self.helpers.unidecode(decoded_tld_stub))\n elif event.type == \"SOCIAL\":\n stub = event.data.get(\"stub\", \"\")\n if stub:\n org_stubs.add(stub.lower())\n elif event.type == \"AZURE_TENANT\":\n tenant_names = event.data.get(\"tenant-names\", [])\n org_stubs.update(set(tenant_names))\n for stub in org_stubs:\n stub_hash = hash(stub)\n if stub_hash not in self.org_stubs_seen:\n self.org_stubs_seen.add(stub_hash)\n stub_event = self.make_event(stub, \"ORG_STUB\", parent=event)\n if stub_event:\n await self.emit_event(stub_event, context=\"speculated {event.type}: {event.data}\")\n\n # USERNAME --> EMAIL\n if event.type == \"USERNAME\":\n email = event.data.split(\":\", 1)[-1]\n if validators.soft_validate(email, \"email\"):\n email_event = self.make_event(email, \"EMAIL_ADDRESS\", parent=event, tags=[\"affiliate\"])\n if email_event:\n await self.emit_event(email_event, context=\"detected {event.type}: {event.data}\")\n" }, { "path": "bbot/modules/internal/unarchive.py", "content": "from pathlib import Path\nfrom contextlib import suppress\nfrom bbot.modules.internal.base import BaseInternalModule\nfrom bbot.core.helpers.libmagic import get_magic_info, get_compression\n\n\nclass unarchive(BaseInternalModule):\n watched_events = [\"FILESYSTEM\"]\n produced_events = [\"FILESYSTEM\"]\n flags = [\"passive\", \"safe\"]\n meta = {\n \"description\": \"Extract different types of files into folders on the filesystem\",\n \"created_date\": \"2024-12-08\",\n \"author\": \"@domwhewell-sage\",\n }\n\n async def setup(self):\n self.ignore_compressions = [\"application/java-archive\", \"application/vnd.android.package-archive\"]\n self.compression_methods = {\n \"zip\": [\"7z\", \"x\", \"-aoa\", \"{filename}\", \"-o{extract_dir}/\"],\n \"bzip2\": [\"tar\", \"--overwrite\", \"-xvjf\", \"{filename}\", \"-C\", \"{extract_dir}/\"],\n \"xz\": [\"tar\", \"--overwrite\", \"-xvJf\", \"{filename}\", \"-C\", \"{extract_dir}/\"],\n \"7z\": [\"7z\", \"x\", \"-aoa\", \"{filename}\", \"-o{extract_dir}/\"],\n # \"rar\": [\"7z\", \"x\", \"-aoa\", \"{filename}\", \"-o{extract_dir}/\"],\n # \"lzma\": [\"7z\", \"x\", \"-aoa\", \"{filename}\", \"-o{extract_dir}/\"],\n \"tar\": [\"tar\", \"--overwrite\", \"-xvf\", \"{filename}\", \"-C\", \"{extract_dir}/\"],\n \"gzip\": [\"tar\", \"--overwrite\", \"-xvzf\", \"{filename}\", \"-C\", \"{extract_dir}/\"],\n }\n return True\n\n async def filter_event(self, event):\n if \"file\" in event.tags:\n magic_mime_type = event.data.get(\"magic_mime_type\", \"\")\n if magic_mime_type in self.ignore_compressions:\n return False, f\"Ignoring file type: {magic_mime_type}, {event.data['path']}\"\n if \"compression\" in event.data:\n if not event.data[\"compression\"] in self.compression_methods:\n return (\n False,\n f\"Extract unable to handle file type: {event.data['compression']}, {event.data['path']}\",\n )\n else:\n return False, f\"Event is not a compressed file: {event.data['path']}\"\n else:\n return False, \"Event is not a file\"\n return True\n\n async def handle_event(self, event):\n path = Path(event.data[\"path\"])\n output_dir = path.parent / path.name.replace(\".\", \"_\")\n\n # Use the appropriate extraction method based on the file type\n self.info(f\"Extracting {path} to {output_dir}\")\n success = await self.extract_file(path, output_dir)\n\n # If the extraction was successful, emit the event\n if success:\n await self.emit_event(\n {\"path\": str(output_dir)},\n \"FILESYSTEM\",\n tags=[\"folder\", \"unarchived-folder\"],\n parent=event,\n context=f'extracted \"{path}\" to: {output_dir}',\n )\n else:\n with suppress(OSError):\n output_dir.rmdir()\n\n async def extract_file(self, path, output_dir):\n extension, mime_type, description, confidence = get_magic_info(path)\n compression_format = get_compression(mime_type)\n cmd_list = self.compression_methods.get(compression_format, [])\n if cmd_list:\n # output dir must not already exist\n try:\n output_dir.mkdir(exist_ok=False)\n except FileExistsError:\n self.warning(f\"Destination directory {output_dir} already exists, aborting unarchive for {path}\")\n return False\n command = [s.format(filename=path, extract_dir=output_dir) for s in cmd_list]\n try:\n await self.run_process(command, check=True)\n for item in output_dir.iterdir():\n if item.is_file():\n await self.extract_file(item, output_dir / item.stem)\n except Exception as e:\n self.warning(f\"Error extracting {path}. Error: {e}\")\n return False\n return True\n" }, { "path": "bbot/modules/ip2location.py", "content": "from bbot.modules.base import BaseModule\n\n\nclass IP2Location(BaseModule):\n \"\"\"\n IP2Location.io Geolocation API.\n \"\"\"\n\n watched_events = [\"IP_ADDRESS\"]\n produced_events = [\"GEOLOCATION\"]\n flags = [\"passive\", \"safe\"]\n meta = {\n \"description\": \"Query IP2location.io's API for geolocation information. \",\n \"created_date\": \"2023-09-12\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\", \"lang\": \"\"}\n options_desc = {\n \"api_key\": \"IP2location.io API Key\",\n \"lang\": \"Translation information(ISO639-1). The translation is only applicable for continent, country, region and city name.\",\n }\n scope_distance_modifier = 1\n _priority = 2\n suppress_dupes = False\n\n base_url = \"http://api.ip2location.io\"\n\n async def setup(self):\n self.lang = self.config.get(\"lang\", \"\")\n return await self.require_api_key()\n\n async def ping(self):\n url = self.build_url(\"8.8.8.8\")\n await super().ping(url)\n\n def build_url(self, data):\n url = f\"{self.base_url}/?key={{api_key}}&ip={data}&format=json&source=bbot\"\n if self.lang:\n url = f\"{url}&lang={self.lang}\"\n return url\n\n async def handle_event(self, event):\n try:\n url = self.build_url(event.data)\n result = await self.api_request(url)\n if result:\n geo_data = result.json()\n if not geo_data:\n self.verbose(f\"No JSON response from {url}\")\n else:\n self.verbose(f\"No response from {url}\")\n except Exception:\n self.verbose(f\"Error retrieving results for {event.data}\", trace=True)\n return\n\n geo_data = {k: v for k, v in geo_data.items() if v is not None}\n if \"error\" in geo_data:\n error_msg = geo_data.get(\"error\").get(\"error_message\", \"\")\n if error_msg:\n self.warning(error_msg)\n elif geo_data:\n country = geo_data.get(\"country_name\", \"unknown country\")\n region = geo_data.get(\"region_name\", \"unknown region\")\n city = geo_data.get(\"city_name\", \"unknown city\")\n lat = geo_data.get(\"latitude\", \"\")\n long = geo_data.get(\"longitude\", \"\")\n description = f\"{city}, {region}, {country} ({lat}, {long})\"\n await self.emit_event(\n geo_data,\n \"GEOLOCATION\",\n event,\n context=f'{{module}} queried IP2Location API for \"{event.data}\" and found {{event.type}}: {description}',\n )\n" }, { "path": "bbot/modules/ipneighbor.py", "content": "import ipaddress\n\nfrom bbot.modules.base import BaseModule\n\n\nclass ipneighbor(BaseModule):\n watched_events = [\"IP_ADDRESS\"]\n produced_events = [\"IP_ADDRESS\"]\n flags = [\"passive\", \"subdomain-enum\", \"aggressive\"]\n meta = {\n \"description\": \"Look beside IPs in their surrounding subnet\",\n \"created_date\": \"2022-06-08\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"num_bits\": 4}\n options_desc = {\"num_bits\": \"Netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts)\"}\n scope_distance_modifier = 1\n\n async def setup(self):\n self.processed = set()\n self.num_bits = max(1, int(self.config.get(\"num_bits\", 4)))\n return True\n\n async def filter_event(self, event):\n if str(event.module) in (\"speculate\", \"ipneighbor\"):\n return False\n return True\n\n async def handle_event(self, event):\n main_ip = event.host\n netmask = main_ip.max_prefixlen - min(main_ip.max_prefixlen, self.num_bits)\n network = ipaddress.ip_network(f\"{main_ip}/{netmask}\", strict=False)\n subnet_hash = hash(network)\n if subnet_hash not in self.processed:\n self.processed.add(subnet_hash)\n for ip in network:\n if ip != main_ip:\n ip_event = self.make_event(str(ip), \"IP_ADDRESS\", event, internal=True)\n if ip_event:\n await self.emit_event(\n ip_event,\n context=\"{module} produced {event.type}: {event.data}\",\n )\n" }, { "path": "bbot/modules/ipstack.py", "content": "from bbot.modules.base import BaseModule\n\n\nclass Ipstack(BaseModule):\n \"\"\"\n Ipstack GeoIP\n Leverages the ipstack.com API to geolocate a host by IP address.\n \"\"\"\n\n watched_events = [\"IP_ADDRESS\"]\n produced_events = [\"GEOLOCATION\"]\n flags = [\"passive\", \"safe\"]\n meta = {\n \"description\": \"Query IPStack's GeoIP API\",\n \"created_date\": \"2022-11-26\",\n \"author\": \"@tycoonslive\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"IPStack GeoIP API Key\"}\n scope_distance_modifier = 1\n _priority = 2\n suppress_dupes = False\n\n base_url = \"http://api.ipstack.com\"\n ping_url = f\"{base_url}/check?access_key={{api_key}}\"\n\n async def setup(self):\n return await self.require_api_key()\n\n async def handle_event(self, event):\n try:\n url = f\"{self.base_url}/{event.data}?access_key={{api_key}}\"\n result = await self.api_request(url)\n if result:\n geo_data = result.json()\n if not geo_data:\n self.verbose(f\"No JSON response from {url}\")\n else:\n self.verbose(f\"No response from {url}\")\n except Exception:\n self.verbose(f\"Error retrieving results for {event.data}\", trace=True)\n return\n geo_data = {k: v for k, v in geo_data.items() if v is not None}\n if \"error\" in geo_data:\n error_msg = geo_data.get(\"error\").get(\"info\", \"\")\n if error_msg:\n self.warning(error_msg)\n elif geo_data:\n country = geo_data.get(\"country_name\", \"unknown country\")\n region = geo_data.get(\"region_name\", \"unknown region\")\n city = geo_data.get(\"city\", \"unknown city\")\n lat = geo_data.get(\"latitude\", \"\")\n long = geo_data.get(\"longitude\", \"\")\n description = f\"{city}, {region}, {country} ({lat}, {long})\"\n await self.emit_event(\n geo_data,\n \"GEOLOCATION\",\n event,\n context=f'{{module}} queried ipstack.com\\'s API for \"{event.data}\" and found {{event.type}}: {description}',\n )\n" }, { "path": "bbot/modules/jadx.py", "content": "from pathlib import Path\nfrom subprocess import CalledProcessError\nfrom bbot.modules.internal.base import BaseModule\n\n\nclass jadx(BaseModule):\n watched_events = [\"FILESYSTEM\"]\n produced_events = [\"FILESYSTEM\"]\n flags = [\"passive\", \"safe\", \"code-enum\"]\n meta = {\n \"description\": \"Decompile APKs and XAPKs using JADX\",\n \"created_date\": \"2024-11-04\",\n \"author\": \"@domwhewell-sage\",\n }\n options = {\n \"threads\": 4,\n }\n options_desc = {\n \"threads\": \"Maximum jadx threads for extracting apk's, default: 4\",\n }\n deps_common = [\"java\"]\n deps_ansible = [\n {\n \"name\": \"Create jadx directory\",\n \"file\": {\"path\": \"#{BBOT_TOOLS}/jadx\", \"state\": \"directory\", \"mode\": \"0755\"},\n },\n {\n \"name\": \"Download jadx\",\n \"unarchive\": {\n \"src\": \"https://github.com/skylot/jadx/releases/download/v1.5.0/jadx-1.5.0.zip\",\n \"include\": [\"lib/jadx-1.5.0-all.jar\", \"bin/jadx\"],\n \"dest\": \"#{BBOT_TOOLS}/jadx\",\n \"remote_src\": True,\n },\n },\n ]\n\n allowed_file_types = [\"java archive\", \"android application package\"]\n\n async def setup(self):\n self.threads = self.config.get(\"threads\", 4)\n return True\n\n async def filter_event(self, event):\n if \"file\" in event.tags:\n if event.data[\"magic_description\"].lower() not in self.allowed_file_types:\n return False, f\"Jadx is not able to decompile this file type: {event.data['magic_description']}\"\n else:\n return False, \"Event is not a file\"\n return True\n\n async def handle_event(self, event):\n path = Path(event.data[\"path\"])\n output_dir = path.parent / path.name.replace(\".\", \"_\")\n self.helpers.mkdir(output_dir)\n success = await self.decompile_apk(path, output_dir)\n\n # If jadx was able to decompile the java archive, emit an event\n if success:\n await self.emit_event(\n {\"path\": str(output_dir)},\n \"FILESYSTEM\",\n tags=[\"folder\", \"unarchived-folder\"],\n parent=event,\n context=f'extracted \"{path}\" to: {output_dir}',\n )\n else:\n output_dir.rmdir()\n\n async def decompile_apk(self, path, output_dir):\n command = [\n f\"{self.scan.helpers.tools_dir}/jadx/bin/jadx\",\n \"--threads-count\",\n self.threads,\n \"--output-dir\",\n str(output_dir),\n str(path),\n ]\n try:\n output = await self.run_process(command, check=True)\n except CalledProcessError as e:\n self.warning(f\"Error decompiling {path}. STDOUT: {e.stdout} STDERR: {repr(e.stderr)}\")\n return False\n if not (output_dir / \"resources\").exists() and not (output_dir / \"sources\").exists():\n self.warning(f\"JADX was unable to decompile {path}: (STDOUT: {output.stdout} STDERR: {output.stderr})\")\n return False\n return True\n" }, { "path": "bbot/modules/leakix.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass leakix(subdomain_enum_apikey):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n options = {\"api_key\": \"\"}\n # NOTE: API key is not required (but having one will get you more results)\n options_desc = {\"api_key\": \"LeakIX API Key\"}\n meta = {\n \"description\": \"Query leakix.net for subdomains\",\n \"created_date\": \"2022-07-11\",\n \"author\": \"@TheTechromancer\",\n }\n\n base_url = \"https://leakix.net\"\n ping_url = f\"{base_url}/host/1.1.1.1\"\n\n async def setup(self):\n ret = await super(subdomain_enum_apikey, self).setup()\n self.api_key = self.config.get(\"api_key\", \"\")\n if self.api_key:\n return await self.require_api_key()\n return ret\n\n def prepare_api_request(self, url, kwargs):\n if self.api_key:\n kwargs[\"headers\"][\"api-key\"] = self.api_key\n kwargs[\"headers\"][\"Accept\"] = \"application/json\"\n return url, kwargs\n\n async def request_url(self, query):\n url = f\"{self.base_url}/api/subdomains/{self.helpers.quote(query)}\"\n response = await self.api_request(url)\n return response\n\n async def parse_results(self, r, query=None):\n results = set()\n json = r.json()\n if json:\n for entry in json:\n subdomain = entry.get(\"subdomain\", \"\")\n if subdomain:\n results.add(subdomain)\n return results\n" }, { "path": "bbot/modules/lightfuzz/lightfuzz.py", "content": "import importlib\nfrom bbot.modules.base import BaseModule\n\nfrom bbot.errors import InteractshError\n\n\nclass lightfuzz(BaseModule):\n watched_events = [\"URL\", \"WEB_PARAMETER\"]\n produced_events = [\"FINDING\", \"VULNERABILITY\"]\n flags = [\"active\", \"aggressive\", \"web-thorough\", \"deadly\"]\n\n options = {\n \"force_common_headers\": False,\n \"enabled_submodules\": [\"sqli\", \"cmdi\", \"xss\", \"path\", \"ssti\", \"crypto\", \"serial\", \"esi\"],\n \"disable_post\": False,\n \"try_post_as_get\": False,\n \"try_get_as_post\": False,\n \"avoid_wafs\": True,\n }\n options_desc = {\n \"force_common_headers\": \"Force emit commonly exploitable parameters that may be difficult to detect\",\n \"enabled_submodules\": \"A list of submodules to enable. Empty list enabled all modules.\",\n \"disable_post\": \"Disable processing of POST parameters, avoiding form submissions.\",\n \"try_post_as_get\": \"For each POSTPARAM, also fuzz it as a GETPARAM (in addition to normal POST fuzzing).\",\n \"try_get_as_post\": \"For each GETPARAM, also fuzz it as a POSTPARAM (in addition to normal GET fuzzing).\",\n \"avoid_wafs\": \"Avoid running against confirmed WAFs, which are likely to block lightfuzz requests\",\n }\n\n meta = {\n \"description\": \"Find Web Parameters and Lightly Fuzz them using a heuristic based scanner\",\n \"author\": \"@liquidsec\",\n \"created_date\": \"2024-06-28\",\n }\n common_headers = [\"x-forwarded-for\", \"user-agent\"]\n in_scope_only = True\n\n _module_threads = 4\n\n async def setup(self):\n self.event_dict = {}\n self.interactsh_subdomain_tags = {}\n self.interactsh_instance = None\n self.interactsh_domain = None\n self.disable_post = self.config.get(\"disable_post\", False)\n self.try_post_as_get = self.config.get(\"try_post_as_get\", False)\n self.try_get_as_post = self.config.get(\"try_get_as_post\", False)\n self.enabled_submodules = self.config.get(\"enabled_submodules\")\n self.interactsh_disable = self.scan.config.get(\"interactsh_disable\", False)\n self.avoid_wafs = self.scan.config.get(\"avoid_wafs\", True)\n self.submodules = {}\n\n if not self.enabled_submodules:\n return False, \"Lightfuzz enabled without any submodules. Must enable at least one submodule.\"\n\n for submodule_name in self.enabled_submodules:\n try:\n submodule_module = importlib.import_module(f\"bbot.modules.lightfuzz.submodules.{submodule_name}\")\n submodule_class = getattr(submodule_module, submodule_name)\n except ImportError:\n return False, f\"Invalid Lightfuzz submodule ({submodule_name}) specified in enabled_modules\"\n self.submodules[submodule_name] = submodule_class\n\n interactsh_needed = any(submodule.uses_interactsh for submodule in self.submodules.values())\n if interactsh_needed and not self.interactsh_disable:\n try:\n self.interactsh_instance = self.helpers.interactsh()\n self.interactsh_domain = await self.interactsh_instance.register(callback=self.interactsh_callback)\n if not self.interactsh_domain:\n self.warning(\"Interactsh failure: No domain returned from self.interactsh_instance.register()\")\n self.interactsh_instance = None\n except InteractshError as e:\n self.warning(f\"Interactsh failure: {e}\")\n self.interactsh_instance = None\n return True\n\n async def interactsh_callback(self, r):\n full_id = r.get(\"full-id\", None)\n if full_id:\n if \".\" in full_id:\n details = self.interactsh_subdomain_tags.get(full_id.split(\".\")[0])\n if not details[\"event\"]:\n return\n # currently, this is only used by the cmdi submodule. Later, when other modules use it, we will need to store description data in the interactsh_subdomain_tags dictionary\n await self.emit_event(\n {\n \"severity\": \"CRITICAL\",\n \"host\": str(details[\"event\"].host),\n \"url\": details[\"event\"].data[\"url\"],\n \"description\": f\"OS Command Injection (OOB Interaction) Type: [{details['type']}] Parameter Name: [{details['name']}] Probe: [{details['probe']}]\",\n },\n \"VULNERABILITY\",\n details[\"event\"],\n )\n else:\n # this is likely caused by something trying to resolve the base domain first and can be ignored\n self.debug(\"skipping result because subdomain tag was missing\")\n\n def _outgoing_dedup_hash(self, event):\n return hash(\n (\n \"lightfuzz\",\n str(event.host),\n event.data[\"url\"],\n event.data[\"description\"],\n event.data.get(\"type\", \"\"),\n event.data.get(\"name\", \"\"),\n )\n )\n\n async def run_submodule(self, submodule, event):\n submodule_instance = submodule(self, event)\n await submodule_instance.fuzz()\n if len(submodule_instance.results) > 0:\n for r in submodule_instance.results:\n event_data = {\"host\": str(event.host), \"url\": event.data[\"url\"], \"description\": r[\"description\"]}\n\n envelopes = getattr(event, \"envelopes\", None)\n envelope_summary = getattr(envelopes, \"summary\", None)\n if envelope_summary:\n # Append the envelope summary to the description\n event_data[\"description\"] += f\" Envelopes: [{envelope_summary}]\"\n\n if r[\"type\"] == \"VULNERABILITY\":\n event_data[\"severity\"] = r[\"severity\"]\n await self.emit_event(\n event_data,\n r[\"type\"],\n event,\n )\n\n async def handle_event(self, event):\n if event.type == \"URL\":\n if self.config.get(\"force_common_headers\", False) is False:\n return False\n\n # If force_common_headers is True, we force the emission of a WEB_PARAMETER for each of the common headers to force fuzzing against them\n for h in self.common_headers:\n description = f\"Speculative (Forced) Header [{h}]\"\n data = {\n \"host\": str(event.host),\n \"type\": \"HEADER\",\n \"name\": h,\n \"original_value\": None,\n \"url\": event.data,\n \"description\": description,\n }\n await self.emit_event(data, \"WEB_PARAMETER\", event)\n\n elif event.type == \"WEB_PARAMETER\":\n # check connectivity to url\n connectivity_test = await self.helpers.request(event.data[\"url\"], timeout=10)\n\n if connectivity_test:\n original_type = event.data[\"type\"]\n\n # Normal fuzzing pass (skipped for POSTPARAM if disable_post is True)\n if not (self.disable_post and original_type == \"POSTPARAM\"):\n for submodule_name, submodule in self.submodules.items():\n self.debug(f\"Starting {submodule_name} fuzz()\")\n await self.run_submodule(submodule, event)\n\n # Additional pass: try POSTPARAM as GETPARAM\n if self.try_post_as_get and original_type == \"POSTPARAM\":\n event.data[\"type\"] = \"GETPARAM\"\n event.data[\"converted_from_post\"] = True\n for submodule_name, submodule in self.submodules.items():\n self.debug(f\"Starting {submodule_name} fuzz() (try_post_as_get)\")\n await self.run_submodule(submodule, event)\n\n # Additional pass: try GETPARAM as POSTPARAM\n if self.try_get_as_post and original_type == \"GETPARAM\":\n event.data[\"type\"] = \"POSTPARAM\"\n event.data[\"converted_from_get\"] = True\n for submodule_name, submodule in self.submodules.items():\n self.debug(f\"Starting {submodule_name} fuzz() (try_get_as_post)\")\n await self.run_submodule(submodule, event)\n else:\n self.debug(f\"WEB_PARAMETER URL {event.data['url']} failed connectivity test, aborting\")\n\n async def cleanup(self):\n if self.interactsh_instance:\n try:\n await self.interactsh_instance.deregister()\n self.debug(\n f\"successfully deregistered interactsh session with correlation_id {self.interactsh_instance.correlation_id}\"\n )\n except InteractshError as e:\n self.warning(f\"Interactsh failure: {e}\")\n\n async def finish(self):\n if self.interactsh_instance:\n await self.helpers.sleep(5)\n try:\n for r in await self.interactsh_instance.poll():\n await self.interactsh_callback(r)\n except InteractshError as e:\n self.debug(f\"Error in interact.sh: {e}\")\n\n async def filter_event(self, event):\n # Unless configured specifically to do so, avoid running against confirmed WAFs\n if self.avoid_wafs and \"waf\" in event.tags:\n # Use parsed_url.geturl() for both URL and WEB_PARAMETER events\n parsed_url = getattr(event, \"parsed_url\", None)\n url = parsed_url.geturl() if parsed_url else \"unknown\"\n self.debug(f\"Skipping {event.type} because it is likely to be blocked by a WAF. URL: {url}\")\n return False\n\n # If we've disabled fuzzing POST parameters, back out of POSTPARAM WEB_PARAMETER events as quickly as possible\n if event.type == \"WEB_PARAMETER\" and self.disable_post and event.data[\"type\"] == \"POSTPARAM\":\n if not self.try_post_as_get:\n return False, \"POST parameter disabled in lightfuzz module\"\n return True\n\n @classmethod\n def help_text(self):\n # Call the base class help_text method\n base_help_text = super().help_text()\n\n import importlib\n\n submodules = {}\n for submodule_name in self.options.get(\"enabled_submodules\", []):\n try:\n submodule_module = importlib.import_module(f\"bbot.modules.lightfuzz.submodules.{submodule_name}\")\n submodule_class = getattr(submodule_module, submodule_name)\n submodules[submodule_name] = submodule_class\n except ImportError:\n continue\n\n # Find all submodules\n submodules_info = \"\\nLightfuzz Submodules:\\n\"\n for submodule_name, submodule_class in submodules.items():\n try:\n friendly_name = getattr(submodule_class, \"friendly_name\", submodule_name)\n description = (\n submodule_class.__doc__.strip() if submodule_class.__doc__ else \"No description available\"\n )\n indented_description = \" \" + description.replace(\"\\n\", \"\\n \")\n submodules_info += f\" - {submodule_name} ({friendly_name}):\\n\"\n submodules_info += f\"{indented_description}\\n\\n\"\n except AttributeError:\n continue\n\n # Combine the base help text with the submodules information\n return base_help_text + submodules_info\n" }, { "path": "bbot/modules/lightfuzz/submodules/__init__.py", "content": "" }, { "path": "bbot/modules/lightfuzz/submodules/base.py", "content": "import copy\nimport base64\nimport binascii\nfrom urllib.parse import quote\n\n\nclass BaseLightfuzz:\n friendly_name = \"\"\n uses_interactsh = False\n\n def __init__(self, lightfuzz, event):\n self.lightfuzz = lightfuzz\n self.event = event\n self.results = []\n self.parameter_name = self.event.data[\"name\"]\n\n @staticmethod\n def is_hex(s):\n try:\n bytes.fromhex(s)\n return True\n except ValueError:\n return False\n\n @staticmethod\n def is_base64(s):\n try:\n if base64.b64encode(base64.b64decode(s)).decode() == s:\n return True\n except (binascii.Error, UnicodeDecodeError):\n return False\n return False\n\n # WEB_PARAMETER event may contain additional_params (e.g. other parameters in the same form or query string). These will be sent unchanged along with the probe.\n def additional_params_process(self, additional_params, additional_params_populate_empty):\n \"\"\"\n Processes additional parameters by populating blank or empty values with random strings if specified.\n\n Parameters:\n - additional_params (dict): A dictionary of additional parameters to process.\n - additional_params_populate_blank_empty (bool): If True, populates blank or empty parameter values with random numeric strings.\n\n Returns:\n - dict: A dictionary with processed additional parameters, where blank or empty values are replaced with random strings if specified.\n\n The function iterates over the provided additional parameters and replaces any blank or empty values with a random numeric string\n of length 10, if the flag is set to True. Otherwise, it returns the parameters unchanged.\n \"\"\"\n if not additional_params or not additional_params_populate_empty:\n return additional_params\n\n return {\n k: self.lightfuzz.helpers.rand_string(10, numeric_only=True) if v in (\"\", None) else v\n for k, v in additional_params.items()\n }\n\n def conditional_urlencode(self, probe, event_type, skip_urlencoding=False):\n \"\"\"Conditionally url-encodes the probe if the event type requires it and encoding is not skipped by the submodule.\n We also don't encode if any envelopes are present.\n \"\"\"\n if event_type in [\"GETPARAM\", \"COOKIE\"] and not skip_urlencoding and getattr(self.event, \"envelopes\", None):\n # Exclude '&' from being encoded since we are operating on full query strings\n return quote(probe, safe=\"&\")\n return probe\n\n def build_query_string(self, probe, parameter_name, additional_params=None):\n \"\"\"Constructs a URL with query parameters from the given probe and additional parameters.\"\"\"\n url = f\"{self.event.data['url']}?{parameter_name}={probe}\"\n if additional_params:\n url = self.lightfuzz.helpers.add_get_params(url, additional_params, encode=False).geturl()\n return url\n\n def prepare_request(\n self,\n event_type,\n probe,\n cookies,\n additional_params=None,\n speculative_mode=\"GETPARAM\",\n parameter_name_suffix=\"\",\n additional_params_populate_empty=False,\n skip_urlencoding=False,\n ):\n \"\"\"\n Prepares the request parameters by processing the probe and constructing the request based on the event type.\n \"\"\"\n\n if parameter_name_suffix:\n parameter_name = f\"{self.parameter_name}{parameter_name_suffix}\"\n else:\n parameter_name = self.parameter_name\n additional_params = self.additional_params_process(additional_params, additional_params_populate_empty)\n\n # Transparently pack the probe value into the envelopes, if present\n probe = self.outgoing_probe_value(probe)\n\n # URL Encode the probe if the event type is GETPARAM or COOKIE, if there are no envelopes, and the submodule did not opt-out with skip_urlencoding\n probe = self.conditional_urlencode(probe, event_type, skip_urlencoding)\n\n if event_type == \"SPECULATIVE\":\n event_type = speculative_mode\n\n # Construct request parameters based on the event type\n if event_type == \"GETPARAM\":\n url = self.build_query_string(probe, parameter_name, additional_params)\n return {\"method\": \"GET\", \"cookies\": cookies, \"url\": url}\n elif event_type == \"COOKIE\":\n cookies_probe = {parameter_name: probe}\n return {\"method\": \"GET\", \"cookies\": {**cookies, **cookies_probe}, \"url\": self.event.data[\"url\"]}\n elif event_type == \"HEADER\":\n headers = {parameter_name: probe}\n return {\"method\": \"GET\", \"headers\": headers, \"cookies\": cookies, \"url\": self.event.data[\"url\"]}\n elif event_type in [\"POSTPARAM\", \"BODYJSON\"]:\n # Prepare data for POSTPARAM and BODYJSON event types\n data = {parameter_name: probe}\n if additional_params:\n data.update(additional_params)\n if event_type == \"BODYJSON\":\n return {\"method\": \"POST\", \"json\": data, \"cookies\": cookies, \"url\": self.event.data[\"url\"]}\n else:\n return {\"method\": \"POST\", \"data\": data, \"cookies\": cookies, \"url\": self.event.data[\"url\"]}\n\n def compare_baseline(\n self,\n event_type,\n probe,\n cookies,\n additional_params_populate_empty=False,\n speculative_mode=\"GETPARAM\",\n skip_urlencoding=False,\n parameter_name_suffix=\"\",\n parameter_name_suffix_additional_params=\"\",\n ):\n \"\"\"\n Compares the baseline using prepared request parameters.\n \"\"\"\n additional_params = copy.deepcopy(self.event.data.get(\"additional_params\", {}))\n\n if additional_params and parameter_name_suffix_additional_params:\n # Add suffix to each key in additional_params\n additional_params = {\n f\"{k}{parameter_name_suffix_additional_params}\": v for k, v in additional_params.items()\n }\n\n request_params = self.prepare_request(\n event_type,\n probe,\n cookies,\n additional_params,\n speculative_mode,\n parameter_name_suffix,\n additional_params_populate_empty,\n skip_urlencoding,\n )\n request_params.update({\"include_cache_buster\": False})\n return self.lightfuzz.helpers.http_compare(**request_params)\n\n async def baseline_probe(self, cookies):\n \"\"\"\n Executes a baseline probe to establish a baseline for comparison.\n \"\"\"\n if self.event.data.get(\"eventtype\") in [\"POSTPARAM\", \"BODYJSON\"]:\n method = \"POST\"\n else:\n method = \"GET\"\n\n return await self.lightfuzz.helpers.request(\n method=method,\n cookies=cookies,\n url=self.event.data.get(\"url\"),\n allow_redirects=False,\n retries=1,\n timeout=10,\n )\n\n async def compare_probe(\n self,\n http_compare,\n event_type,\n probe,\n cookies,\n additional_params_populate_empty=False,\n additional_params_override={},\n speculative_mode=\"GETPARAM\",\n skip_urlencoding=False,\n parameter_name_suffix=\"\",\n parameter_name_suffix_additional_params=\"\",\n ):\n # Deep copy to avoid modifying original additional_params\n additional_params = copy.deepcopy(self.event.data.get(\"additional_params\", {}))\n\n # Override additional parameters if provided\n additional_params.update(additional_params_override)\n\n if additional_params and parameter_name_suffix_additional_params:\n # Add suffix to each key in additional_params\n additional_params = {\n f\"{k}{parameter_name_suffix_additional_params}\": v for k, v in additional_params.items()\n }\n\n # Prepare request parameters\n request_params = self.prepare_request(\n event_type,\n probe,\n cookies,\n additional_params,\n speculative_mode,\n parameter_name_suffix,\n additional_params_populate_empty,\n skip_urlencoding,\n )\n # Perform the comparison using the constructed request parameters\n url = request_params.pop(\"url\")\n return await http_compare.compare(url, **request_params)\n\n async def standard_probe(\n self,\n event_type,\n cookies,\n probe,\n timeout=10,\n additional_params_populate_empty=False,\n speculative_mode=\"GETPARAM\",\n allow_redirects=False,\n skip_urlencoding=False,\n ):\n request_params = self.prepare_request(\n event_type,\n probe,\n cookies,\n self.event.data.get(\"additional_params\"),\n speculative_mode,\n \"\",\n additional_params_populate_empty,\n skip_urlencoding,\n )\n request_params.update({\"allow_redirects\": allow_redirects, \"retries\": 0, \"timeout\": timeout})\n self.debug(f\"standard_probe requested URL: [{request_params['url']}]\")\n return await self.lightfuzz.helpers.request(**request_params)\n\n def conversion_note(self):\n if self.event.data.get(\"converted_from_post\", False):\n return \" (converted from POSTPARAM)\"\n elif self.event.data.get(\"converted_from_get\", False):\n return \" (converted from GETPARAM)\"\n return \"\"\n\n def metadata(self):\n metadata_string = f\"Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}]{self.conversion_note()}\"\n if self.event.data[\"original_value\"] != \"\" and self.event.data[\"original_value\"] is not None:\n metadata_string += (\n f\" Original Value: [{self.lightfuzz.helpers.truncate_string(self.event.data['original_value'], 200)}]\"\n )\n return metadata_string\n\n def incoming_probe_value(self, populate_empty=True):\n \"\"\"\n Transparently modifies the incoming probe value (the original value of the WEB_PARAMETER), given any envelopes that may have been identified, so that fuzzing within the envelopes can occur.\n \"\"\"\n envelopes = getattr(self.event, \"envelopes\", None)\n probe_value = \"\"\n if envelopes is not None:\n probe_value = envelopes.get_subparam()\n self.debug(f\"incoming_probe_value (after unpacking): {probe_value} with envelopes [{envelopes}]\")\n if not probe_value:\n if populate_empty is True:\n probe_value = self.lightfuzz.helpers.rand_string(10, numeric_only=True)\n else:\n probe_value = \"\"\n probe_value = str(probe_value)\n return probe_value\n\n def outgoing_probe_value(self, outgoing_probe_value):\n \"\"\"\n Transparently packs the outgoing probe value (fuzz probe being sent to the target) through\n any envelopes that may have been identified, so that fuzzing within the envelopes can occur.\n\n Uses pack_value() to avoid mutating the envelope's internal state, preventing cross-contamination\n between submodules that share the same event/envelope object.\n \"\"\"\n self.debug(f\"outgoing_probe_value (before packing): {outgoing_probe_value} / {self.event}\")\n envelopes = getattr(self.event, \"envelopes\", None)\n if envelopes is not None:\n outgoing_probe_value = envelopes.pack_value(outgoing_probe_value)\n self.debug(\n f\"outgoing_probe_value (after packing): {outgoing_probe_value} with envelopes [{envelopes}] / {self.event}\"\n )\n return outgoing_probe_value\n\n def get_submodule_name(self):\n \"\"\"Extracts the submodule name from the class name.\"\"\"\n return self.__class__.__name__.replace(\"Lightfuzz\", \"\").lower()\n\n def log(self, level, message, *args, **kwargs):\n submodule_name = self.get_submodule_name()\n prefixed_message = f\"[{submodule_name}] {message}\"\n log_method = getattr(self.lightfuzz, level)\n log_method(prefixed_message, *args, **kwargs)\n\n def debug(self, message, *args, **kwargs):\n self.log(\"debug\", message, *args, **kwargs)\n\n def verbose(self, message, *args, **kwargs):\n self.log(\"verbose\", message, *args, **kwargs)\n\n def info(self, message, *args, **kwargs):\n self.log(\"info\", message, *args, **kwargs)\n\n def hugeinfo(self, message, *args, **kwargs):\n self.log(\"hugeinfo\", message, *args, **kwargs)\n\n def warning(self, message, *args, **kwargs):\n self.log(\"warning\", message, *args, **kwargs)\n\n def hugewarning(self, message, *args, **kwargs):\n self.log(\"hugewarning\", message, *args, **kwargs)\n\n def error(self, message, *args, **kwargs):\n self.log(\"error\", message, *args, **kwargs)\n\n def critical(self, message, *args, **kwargs):\n self.log(\"critical\", message, *args, **kwargs)\n" }, { "path": "bbot/modules/lightfuzz/submodules/cmdi.py", "content": "from bbot.errors import HttpCompareError\nfrom .base import BaseLightfuzz\n\nimport urllib.parse\n\n\nclass cmdi(BaseLightfuzz):\n \"\"\"\n Detects command injection vulnerabilities.\n\n Techniques:\n\n * Echo Canary Detection:\n - Injects command delimiters (;, &&, ||, &, |) along with an echo command\n - Checks if the echoed canary appears in the response without the \"echo\" itself\n - Uses a false positive probe to validate findings\n\n * Blind Command Injection:\n - Injects nslookup commands with unique subdomain tags\n - Detects command execution through DNS resolution via Interactsh\n \"\"\"\n\n friendly_name = \"Command Injection\"\n uses_interactsh = True\n\n async def fuzz(self):\n cookies = self.event.data.get(\n \"assigned_cookies\", {}\n ) # Retrieve assigned cookies from WEB_PARAMETER event data, if present\n probe_value = self.incoming_probe_value()\n\n canary = self.lightfuzz.helpers.rand_string(10, numeric_only=True)\n http_compare = self.compare_baseline(\n self.event.data[\"type\"], probe_value, cookies\n ) # Initialize the http_compare object and establish a baseline HTTP response\n\n cmdi_probe_strings = [\n \"AAAA\", # False positive probe\n \";\",\n \"&&\",\n \"||\",\n \"&\",\n \"|\",\n ]\n\n positive_detections = []\n for p in cmdi_probe_strings:\n try:\n # add \"echo\" to the cmdi probe value to construct the command to be executed\n echo_probe = f\"{probe_value}{p} echo {canary} {p}\"\n # we have to handle our own URL-encoding here, because our payloads include the & character\n if self.event.data[\"type\"] == \"GETPARAM\":\n echo_probe = urllib.parse.quote(echo_probe.encode(), safe=\"\")\n\n # send cmdi probe and compare with baseline response\n cmdi_probe = await self.compare_probe(\n http_compare, self.event.data[\"type\"], echo_probe, cookies, skip_urlencoding=True\n )\n\n # ensure we received an HTTP response\n if cmdi_probe[3]:\n # check if the canary is in the response and the word \"echo\" is NOT in the response text, ruling out mere reflection of the entire probe value without execution\n if canary in cmdi_probe[3].text and \"echo\" not in cmdi_probe[3].text:\n self.debug(f\"canary [{canary}] found in response when sending probe [{p}]\")\n if p == \"AAAA\": # Handle detection false positive probe\n self.warning(\n f\"False Postive Probe appears to have been triggered for {self.event.data['url']}, aborting remaining detection\"\n )\n return\n positive_detections.append(p) # Add detected probes to positive detections\n except HttpCompareError as e:\n self.debug(e)\n continue\n if len(positive_detections) > 0:\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"POSSIBLE OS Command Injection. {self.metadata()} Detection Method: [echo canary] CMD Probe Delimeters: [{' '.join(positive_detections)}]\",\n }\n )\n\n # Blind OS Command Injection\n if self.lightfuzz.interactsh_instance:\n self.lightfuzz.event_dict[self.event.data[\"url\"]] = self.event # Store the event associated with the URL\n for p in cmdi_probe_strings:\n # generate a random subdomain tag and associate it with the event, type, name, and probe\n subdomain_tag = self.lightfuzz.helpers.rand_string(4, digits=False)\n self.lightfuzz.interactsh_subdomain_tags[subdomain_tag] = {\n \"event\": self.event,\n \"type\": self.event.data[\"type\"],\n \"name\": self.event.data[\"name\"],\n \"probe\": p,\n }\n # payload is an nslookup command that includes the interactsh domain prepended the previously generated subdomain tag\n interactsh_probe = f\"{p} nslookup {subdomain_tag}.{self.lightfuzz.interactsh_domain} {p}\"\n # we have to handle our own URL-encoding here, because our payloads include the & character\n if self.event.data[\"type\"] == \"GETPARAM\":\n interactsh_probe = urllib.parse.quote(interactsh_probe.encode(), safe=\"\")\n # we send the probe here, and any positive detections are processed in the interactsh_callback defined in lightfuzz.py\n await self.standard_probe(\n self.event.data[\"type\"],\n cookies,\n f\"{probe_value}{interactsh_probe}\",\n timeout=15,\n skip_urlencoding=True,\n )\n" }, { "path": "bbot/modules/lightfuzz/submodules/crypto.py", "content": "import base64\nimport hashlib\nfrom .base import BaseLightfuzz\nfrom bbot.errors import HttpCompareError\nfrom urllib.parse import unquote, quote\n\n\n# Global cache for compiled YARA rules\n_compiled_rules_cache = None\n\n\nclass crypto(BaseLightfuzz):\n \"\"\"\n Detects the use of cryptography in web parameters, and probes for some cryptographic vulnerabilities\n\n * Cryptographic Error Detection:\n - Detects known cryptographic error messages in server responses.\n\n * Cryptographic Parameter Value Detection:\n - Detects use of cryptography in web parameter values.\n - Validates by attempting to manipulate the value regardless of its encoding.\n\n * Length Extension Attack Detection:\n - Identifies parameters which may be expecting hash digests for values, and any linked parameters which invalidate them.\n\n * Padding Oracle Vulnerabilities:\n - Identifies the presence of cryptographic oracles that could be exploited to arbitrary decrypt or encrypt data for the parameter value.\n\n\n \"\"\"\n\n friendly_name = \"Cryptography Probe\"\n\n # Although we have an envelope system to detect hex and base64 encoded parameter values, those are only assigned when they decode to a valid string.\n # Since crypto values (and serialized objects) will not decode properly, we need a more concise check here to determine how to process them.\n\n @staticmethod\n def is_hex(s):\n try:\n bytes.fromhex(s)\n return True\n except ValueError:\n return False\n\n @staticmethod\n def is_base64(s):\n try:\n if base64.b64encode(base64.b64decode(s)).decode() == s:\n return True\n except Exception:\n return False\n return False\n\n # A list of YARA rules for detecting cryptographic error messages\n crypto_error_strings = [\n \"invalid mac\",\n \"padding is invalid\",\n \"bad data\",\n \"length of the data to decrypt is invalid\",\n \"specify a valid key size\",\n \"invalid algorithm specified\",\n \"object already exists\",\n \"key does not exist\",\n \"the parameter is incorrect\",\n \"cryptography exception\",\n \"access denied\",\n \"unknown error\",\n \"invalid provider type\",\n \"no valid cert found\",\n \"cannot find the original signer\",\n \"signature description could not be created\",\n \"crypto operation failed\",\n \"OpenSSL Error\",\n ]\n\n @property\n def compiled_rules(self):\n \"\"\"\n We need to cache the compiled YARA rule globally since lightfuzz submodules are recreated for every handle_event\n \"\"\"\n global _compiled_rules_cache\n if _compiled_rules_cache is None:\n _compiled_rules_cache = self.lightfuzz.helpers.yara.compile_strings(self.crypto_error_strings, nocase=True)\n return _compiled_rules_cache\n\n @staticmethod\n def format_agnostic_decode(input_string, urldecode=False):\n \"\"\"\n Decodes a string from either hex or base64 (without knowing which first), and optionally URL-decoding it first.\n\n Parameters:\n - input_string (str): The string to decode.\n - urldecode (bool): If True, URL-decodes the input first.\n\n Returns:\n - tuple: (decoded data, encoding type: 'hex', 'base64', or 'unknown').\n \"\"\"\n encoding = \"unknown\"\n if urldecode:\n input_string = unquote(input_string)\n if BaseLightfuzz.is_hex(input_string):\n data = bytes.fromhex(input_string)\n encoding = \"hex\"\n elif BaseLightfuzz.is_base64(input_string):\n data = base64.b64decode(input_string)\n encoding = \"base64\"\n else:\n data = str\n return data, encoding\n\n @staticmethod\n def format_agnostic_encode(data, encoding, urlencode=False):\n \"\"\"\n Encodes data into hex or base64, with optional URL-encoding.\n\n Parameters:\n - data (bytes): The data to encode.\n - encoding (str): The encoding type ('hex' or 'base64').\n - urlencode (bool): If True, URL-encodes the result.\n\n Returns:\n - str: The encoded data as a string.\n\n Raises:\n - ValueError: If an unsupported encoding type is specified.\n \"\"\"\n if encoding == \"hex\":\n encoded_data = data.hex()\n elif encoding == \"base64\":\n encoded_data = base64.b64encode(data).decode(\"utf-8\") # base64 encoding returns bytes, decode to string\n else:\n raise ValueError(\"Unsupported encoding type specified\")\n if urlencode:\n return quote(encoded_data)\n return encoded_data\n\n @staticmethod\n def modify_string(input_string, action=\"truncate\", position=None, extension_length=1):\n \"\"\"\n Modifies a cryptographic string by either truncating it, mutating a byte at a specified position, or extending it with null bytes.\n\n Parameters:\n - input_string (str): The string to modify.\n - action (str): The action to perform ('truncate', 'mutate', 'extend').\n - position (int): The position to mutate (only used if action is 'mutate').\n - extension_length (int): The number of null bytes to add if action is 'extend'.\n\n Returns:\n - str: The modified string.\n \"\"\"\n if not isinstance(input_string, str):\n input_string = str(input_string)\n\n data, encoding = crypto.format_agnostic_decode(input_string)\n if encoding != \"base64\" and encoding != \"hex\":\n raise ValueError(\"Input must be either hex or base64 encoded\")\n\n if action == \"truncate\":\n modified_data = data[:-1] # Remove the last byte\n elif action == \"mutate\":\n if not position:\n position = len(data) // 2\n if position < 0 or position >= len(data):\n raise ValueError(\"Position out of range\")\n byte_list = list(data)\n byte_list[position] = (byte_list[position] + 1) % 256\n modified_data = bytes(byte_list)\n elif action == \"extend\":\n modified_data = data + (b\"\\x00\" * extension_length)\n elif action == \"flip\":\n if not position:\n position = len(data) // 2\n if position < 0 or position >= len(data):\n raise ValueError(\"Position out of range\")\n byte_list = list(data)\n byte_list[position] ^= 0xFF # Flip all bits in the byte at the specified position\n modified_data = bytes(byte_list)\n else:\n raise ValueError(\"Unsupported action\")\n return crypto.format_agnostic_encode(modified_data, encoding)\n\n # Check if the entropy of the data is greater than the threshold, indicating it is likely encrypted\n def is_likely_encrypted(self, data, threshold=4.5):\n entropy = self.lightfuzz.helpers.calculate_entropy(data)\n return entropy >= threshold\n\n # Perform basic cryptanalysis on the input string, attempting to determine if it is likely encrypted and if it is a block cipher\n def cryptanalysis(self, input_string):\n likely_crypto = False\n possible_block_cipher = False\n data, encoding = self.format_agnostic_decode(input_string)\n likely_crypto = self.is_likely_encrypted(data)\n data_length = len(data)\n if data_length % 8 == 0:\n possible_block_cipher = True\n return likely_crypto, possible_block_cipher\n\n # Determine possible block sizes for a given ciphertext length\n @staticmethod\n def possible_block_sizes(ciphertext_length):\n potential_block_sizes = [8, 16]\n possible_sizes = []\n for block_size in potential_block_sizes:\n num_blocks = ciphertext_length // block_size\n if ciphertext_length % block_size == 0 and num_blocks >= 2:\n possible_sizes.append(block_size)\n return possible_sizes\n\n async def padding_oracle_execute(self, original_data, encoding, block_size, cookies, possible_first_byte=True):\n \"\"\"\n Execute the padding oracle attack for a given block size.\n The goal here is not actual exploitation (arbitrary encryption or decryption), but rather to definitively confirm whether padding oracle vulnerability exists and is exploitable.\n\n Parameters:\n - original_data (bytes): The original ciphertext data.\n - encoding (str): The encoding type ('hex' or 'base64').\n - block_size (int): The block size to use for the padding oracle attack.\n - cookies (dict): Cookies to include, if any\n - possible_first_byte (bool): If True, use the first byte as the baseline byte.\n\n Returns:\n - bool: True if the padding oracle attack is successful.\n \"\"\"\n ivblock = b\"\\x00\" * block_size # initialize the IV block with null bytes\n paddingblock = b\"\\x00\" * block_size # initialize the padding block with null bytes\n datablock = original_data[-block_size:] # extract the last block of the original data\n\n # This handling the 1/255 chance that the first byte is correct padding which would cause a false negative.\n if possible_first_byte:\n baseline_byte = b\"\\xff\" # set the baseline byte to 0xff\n starting_pos = 0 # set the starting position to 0\n else:\n baseline_byte = b\"\\x00\" # set the baseline byte to 0x00\n starting_pos = 1 # set the starting position to 1\n\n baseline_probe_value = self.format_agnostic_encode(\n ivblock + paddingblock[:-1] + baseline_byte + datablock, encoding\n )\n baseline = self.compare_baseline(\n self.event.data[\"type\"],\n baseline_probe_value,\n cookies,\n )\n differ_count = 0\n # for each possible byte value, send a probe and check if the response is different\n for i in range(starting_pos, starting_pos + 254):\n byte = bytes([i])\n probe_value = self.format_agnostic_encode(ivblock + paddingblock[:-1] + byte + datablock, encoding)\n oracle_probe = await self.compare_probe(\n baseline,\n self.event.data[\"type\"],\n probe_value,\n cookies,\n )\n # oracle_probe[0] will be false if the response is different - oracle_probe[1] stores what aspect of the response is different (headers, body, code)\n if oracle_probe[0] is False and \"body\" in oracle_probe[1]:\n # When the server reflects submitted values or reveals decrypted data, every probe will differ in the body. Strip the known probe values from both responses and re-compare.\n stripped_baseline = baseline.baseline.text\n stripped_probe = oracle_probe[3].text\n for encoded_baseline, encoded_probe in [\n (baseline_probe_value, probe_value),\n (baseline_probe_value.replace(\"+\", \" \"), probe_value.replace(\"+\", \" \")),\n (quote(baseline_probe_value), quote(probe_value)),\n ]:\n stripped_baseline = stripped_baseline.replace(encoded_baseline, \"\")\n stripped_probe = stripped_probe.replace(encoded_probe, \"\")\n if stripped_baseline == stripped_probe:\n continue\n # If the server reveals decrypted data, the response may differ by only a few bytes (the varying decrypted byte). Tolerate small character-level differences.\n if len(stripped_baseline) == len(stripped_probe):\n char_diffs = sum(1 for a, b in zip(stripped_baseline, stripped_probe) if a != b)\n if char_diffs <= 5:\n continue\n differ_count += 1\n self.debug(f\"padding_oracle_execute: finished loop. differ_count={differ_count}\")\n # A padding oracle vulnerability can produce a small number of different responses.\n # The correct \\x01 padding byte always differs, but also, multi-byte padding values (\\x02\\x02, \\x03\\x03\\x03, etc.) can also produce valid padding if the intermediate state randomly aligns. At most 'block_size' of such values are possible.\n if 1 <= differ_count <= block_size:\n return True\n # If too many probes differ, the baseline byte may have been the correct padding byte (1/255 chance).\n # In that case, the baseline response represents \"valid padding\" and nearly all probes appear different.\n # Retry with a different baseline byte to rule this out.\n if possible_first_byte and differ_count > block_size:\n return None\n return False\n\n async def padding_oracle(self, probe_value, cookies):\n data, encoding = self.format_agnostic_decode(probe_value)\n possible_block_sizes = self.possible_block_sizes(\n len(data)\n ) # determine possible block sizes for the ciphertext\n\n for block_size in possible_block_sizes:\n padding_oracle_result = await self.padding_oracle_execute(data, encoding, block_size, cookies)\n # if we get a negative result first, theres a 1/255 change it's a false negative. To rule that out, we must retry again with possible_first_byte set to false\n if padding_oracle_result is None:\n self.debug(\"still could be in a possible_first_byte situation - retrying with different first byte\")\n padding_oracle_result = await self.padding_oracle_execute(\n data, encoding, block_size, cookies, possible_first_byte=False\n )\n\n if padding_oracle_result is True:\n context = f\"Lightfuzz Cryptographic Probe Submodule detected a probable padding oracle vulnerability after manipulating parameter: [{self.event.data['name']}]\"\n self.results.append(\n {\n \"type\": \"VULNERABILITY\",\n \"severity\": \"HIGH\",\n \"description\": f\"Padding Oracle Vulnerability. Block size: [{str(block_size)}] {self.metadata()}\",\n \"context\": context,\n }\n )\n\n async def error_string_search(self, text_dict, baseline_text):\n \"\"\"\n Search for cryptographic error strings using YARA rules in the provided text dictionary and baseline text.\n \"\"\"\n matching_techniques = set()\n matching_strings = set()\n\n # Check each manipulation technique\n for label, text in text_dict.items():\n matches = await self.lightfuzz.helpers.yara.match(self.compiled_rules, text)\n if matches:\n matching_techniques.add(label)\n for matched_string in matches:\n matching_strings.add(matched_string)\n\n # Check for false positives by scanning baseline text\n context = f\"Lightfuzz Cryptographic Probe Submodule detected a cryptographic error after manipulating parameter: [{self.event.data['name']}]\"\n if matching_strings:\n baseline_matches = await self.lightfuzz.helpers.yara.match(self.compiled_rules, baseline_text)\n baseline_strings = set()\n for matched_string in baseline_matches:\n baseline_strings.add(matched_string)\n\n # Only report strings that weren't in the baseline\n unique_matches = matching_strings - baseline_strings\n if unique_matches:\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"Possible Cryptographic Error. {self.metadata()} Strings: [{','.join(unique_matches)}] Detection Technique(s): [{','.join(matching_techniques)}]\",\n \"context\": context,\n }\n )\n\n else:\n self.debug(\n f\"Aborting cryptographic error reporting - baseline_text already contained detected string(s) ({','.join(baseline_strings)})\"\n )\n\n # Identify the hash function based on the length of the hash\n @staticmethod\n def identify_hash_function(hash_bytes):\n hash_length = len(hash_bytes)\n hash_functions = {\n 16: hashlib.md5,\n 20: hashlib.sha1,\n 32: hashlib.sha256,\n 48: hashlib.sha384,\n 64: hashlib.sha512,\n }\n\n if hash_length in hash_functions:\n return hash_functions[hash_length]\n\n async def fuzz(self):\n cookies = self.event.data.get(\"assigned_cookies\", {})\n probe_value = self.incoming_probe_value(populate_empty=False)\n\n if not probe_value:\n self.debug(\n f\"The Cryptography Probe Submodule requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]\"\n )\n return\n\n # obtain the baseline probe to compare against\n baseline_probe = await self.baseline_probe(cookies)\n if not baseline_probe:\n self.verbose(f\"Couldn't get baseline_probe for url {self.event.data['url']}, aborting\")\n return\n\n # perform the manipulation techniques\n try:\n truncate_probe_value = self.modify_string(probe_value, action=\"truncate\")\n mutate_probe_value = self.modify_string(probe_value, action=\"mutate\")\n except ValueError as e:\n self.debug(f\"Encountered error modifying value for parameter [{self.event.data['name']}]: {e} , aborting\")\n return\n\n # Basic crypanalysis\n likely_crypto, possible_block_cipher = self.cryptanalysis(probe_value)\n\n # if the value is not likely to be cryptographic, we can skip the rest of the tests\n if not likely_crypto:\n self.debug(\"Parameter value does not appear to be cryptographic, aborting tests\")\n return\n\n # Cryptographic Response Divergence Test\n\n http_compare = self.compare_baseline(self.event.data[\"type\"], probe_value, cookies)\n try:\n arbitrary_probe = await self.compare_probe(http_compare, self.event.data[\"type\"], \"AAAAAAA\", cookies) #\n truncate_probe = await self.compare_probe(\n http_compare, self.event.data[\"type\"], truncate_probe_value, cookies\n ) # manipulate the value by truncating a byte\n mutate_probe = await self.compare_probe(\n http_compare, self.event.data[\"type\"], mutate_probe_value, cookies\n ) # manipulate the value by mutating a byte in place\n except HttpCompareError as e:\n self.verbose(f\"Encountered HttpCompareError Sending Compare Probe: {e}\")\n return\n\n confirmed_techniques = []\n # mutate_probe[0] will be false if the response is different - mutate_probe[1] stores what aspect of the response is different (headers, body, code)\n # ensure the difference is in the body and not the headers or code\n # if the body is different and not empty, we have confirmed that single-byte mutation affected the response body\n if mutate_probe[0] is False and \"body\" in mutate_probe[1]:\n if (http_compare.compare_body(mutate_probe[3].text, arbitrary_probe[3].text) is False) or mutate_probe[\n 3\n ].text == \"\":\n confirmed_techniques.append(\"Single-byte Mutation\")\n\n # if the body is different and not empty, we have confirmed that byte truncation affected the response body\n if truncate_probe[0] is False and \"body\" in truncate_probe[1]:\n if (http_compare.compare_body(truncate_probe[3].text, arbitrary_probe[3].text) is False) or truncate_probe[\n 3\n ].text == \"\":\n confirmed_techniques.append(\"Data Truncation\")\n\n if confirmed_techniques:\n context = f\"Lightfuzz Cryptographic Probe Submodule detected a parameter ({self.event.data['name']}) to appears to drive a cryptographic operation\"\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"Probable Cryptographic Parameter. {self.metadata()} Detection Technique(s): [{', '.join(confirmed_techniques)}]\",\n \"context\": context,\n }\n )\n\n # Cryptographic Error String Test\n # Check if cryptographic error strings are present in the response after performing the manipulation techniques\n await self.error_string_search(\n {\"truncate value\": truncate_probe[3].text, \"mutate value\": mutate_probe[3].text}, baseline_probe.text\n )\n # if we have any confirmed techniques, or the word \"padding\" is in the response, we need to check for a padding oracle\n if confirmed_techniques or (\n \"padding\" in truncate_probe[3].text.lower() or \"padding\" in mutate_probe[3].text.lower()\n ):\n # Padding Oracle Test\n if possible_block_cipher:\n self.debug(\n \"Attempting padding oracle exploit since it looks like a block cipher and we have confirmed crypto\"\n )\n await self.padding_oracle(probe_value, cookies)\n\n # Hash identification / Potential Length extension attack\n data, encoding = crypto.format_agnostic_decode(probe_value)\n # see if its possible that a given value is a hash, and if so, which one\n hash_function = self.identify_hash_function(data)\n if hash_function:\n hash_instance = hash_function()\n # if there are any hash functions which match the length, we check the additional parameters to see if they cause identical changes\n # this would indicate they are being used to generate the hash\n if (\n hash_function\n and \"additional_params\" in self.event.data.keys()\n and self.event.data[\"additional_params\"]\n ):\n # for each additional parameter, we send a probe and check if it causes the same change in the response as the original probe\n for additional_param_name, additional_param_value in self.event.data[\"additional_params\"].items():\n try:\n additional_param_probe = await self.compare_probe(\n http_compare,\n self.event.data[\"type\"],\n probe_value,\n cookies,\n additional_params_override={additional_param_name: additional_param_value + \"A\"},\n )\n except HttpCompareError as e:\n self.verbose(f\"Encountered HttpCompareError Sending Compare Probe: {e}\")\n continue\n # the additional parameter affects the potential hash parameter (suggesting its calculated in the hash)\n # This is a potential length extension attack\n if additional_param_probe[0] is False and (additional_param_probe[1] == mutate_probe[1]):\n context = f\"Lightfuzz Cryptographic Probe Submodule detected a parameter ({self.event.data['name']}) that is a likely a hash, which is connected to another parameter {additional_param_name})\"\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"Possible {self.event.data['type']} parameter with {hash_instance.name.upper()} Hash as value. {self.metadata()}, linked to additional parameter [{additional_param_name}]\",\n \"context\": context,\n }\n )\n" }, { "path": "bbot/modules/lightfuzz/submodules/esi.py", "content": "from .base import BaseLightfuzz\n\n\nclass esi(BaseLightfuzz):\n \"\"\"\n Detects Edge Side Includes (ESI) processing vulnerabilities.\n\n Tests if the server processes ESI tags by sending a payload containing ESI tags\n and checking if the tags are processed (removed) in the response.\n \"\"\"\n\n # Technique lifted from https://github.com/PortSwigger/active-scan-plus-plus\n\n friendly_name = \"Edge Side Includes\"\n\n async def check_probe(self, cookies, probe, match):\n \"\"\"\n Sends the probe and checks if the expected match string is found in the response.\n \"\"\"\n probe_result = await self.standard_probe(self.event.data[\"type\"], cookies, probe)\n if probe_result and match in probe_result.text:\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"Edge Side Include. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}]{self.conversion_note()}\",\n }\n )\n return True\n return False\n\n async def fuzz(self):\n \"\"\"\n Main fuzzing method that sends the ESI test payload and checks for processing.\n \"\"\"\n cookies = self.event.data.get(\"assigned_cookies\", {})\n\n # ESI test payload: if ESI is processed, will be removed\n # leaving AABBCC in the response\n payload = \"AABBCC\"\n detection_string = \"AABBCC\"\n\n await self.check_probe(cookies, payload, detection_string)\n" }, { "path": "bbot/modules/lightfuzz/submodules/path.py", "content": "from .base import BaseLightfuzz\nfrom bbot.errors import HttpCompareError\n\nfrom urllib.parse import quote\n\n\nclass path(BaseLightfuzz):\n \"\"\"\n Detects path traversal and local file inclusion vulnerabilities\n\n Techniques:\n\n * Relative Path Traversal:\n - Tests various relative path traversal patterns (../, ./, .../, etc.)\n - Uses multiple encoding variations (URL encoding, double encoding)\n - Attempts various path validation bypass techniques\n\n * Absolute Path Traversal:\n - Tests absolute paths for Windows (c:\\\\windows\\\\win.ini)\n - Tests absolute paths for Unix (/etc/passwd)\n - Tests null byte injection for extension bypass (%00)\n\n Results are validated using multiple confirmations and WAF response filtering to eliminate false positives.\n \"\"\"\n\n friendly_name = \"Path Traversal\"\n\n async def fuzz(self):\n cookies = self.event.data.get(\"assigned_cookies\", {})\n probe_value = self.incoming_probe_value(populate_empty=False)\n if not probe_value:\n self.debug(\n f\"Path Traversal detection requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]\"\n )\n return\n\n # Single dot traversal tolerance test\n path_techniques = {\n \"single-dot traversal tolerance (no-encoding)\": {\n \"singledot_payload\": f\"./a/../{probe_value}\",\n \"doubledot_payload\": f\"../a/../{probe_value}\",\n },\n \"single-dot traversal tolerance (no-encoding, leading slash)\": {\n \"singledot_payload\": f\"/./a/../{probe_value}\",\n \"doubledot_payload\": f\"/../a/../{probe_value}\",\n },\n \"single-dot traversal tolerance (url-encoding)\": {\n \"singledot_payload\": quote(f\"./a/../{probe_value}\".encode(), safe=\"\"),\n \"doubledot_payload\": quote(f\"../a/../{probe_value}\".encode(), safe=\"\"),\n },\n \"single-dot traversal tolerance (url-encoding, leading slash)\": {\n \"singledot_payload\": quote(f\"/./a/../{probe_value}\".encode(), safe=\"\"),\n \"doubledot_payload\": quote(f\"/../a/../{probe_value}\".encode(), safe=\"\"),\n },\n \"single-dot traversal tolerance (non-recursive stripping)\": {\n \"singledot_payload\": f\"...//a/....//{probe_value}\",\n \"doubledot_payload\": f\"....//a/....//{probe_value}\",\n },\n \"single-dot traversal tolerance (non-recursive stripping, leading slash)\": {\n \"singledot_payload\": f\"/...//a/....//{probe_value}\",\n \"doubledot_payload\": f\"/....//a/....//{probe_value}\",\n },\n \"single-dot traversal tolerance (double url-encoding)\": {\n \"singledot_payload\": f\".%252fa%252f..%252f{probe_value}\",\n \"doubledot_payload\": f\"..%252fa%252f..%252f{probe_value}\",\n },\n \"single-dot traversal tolerance (double url-encoding, leading slash)\": {\n \"singledot_payload\": f\"%252f.%252fa%252f..%252f{probe_value}\",\n \"doubledot_payload\": f\"%252f..%252fa%252f..%252f{probe_value}\",\n },\n }\n\n compiled_regex = self.lightfuzz.helpers.re.compile(r\"/(?:[\\w-]+/)*[\\w-]+\\.\\w+\")\n linux_path_regex = await self.lightfuzz.helpers.re.match(compiled_regex, probe_value)\n if linux_path_regex is not None:\n original_path_only = \"/\".join(probe_value.split(\"/\")[:-1])\n original_filename_only = probe_value.split(\"/\")[-1]\n # Some servers validate the start of the path, so we construct our payload with the original path and filename\n path_techniques[\"single-dot traversal tolerance (start of path validation)\"] = {\n \"singledot_payload\": f\"{original_path_only}/./{original_filename_only}\",\n \"doubledot_payload\": f\"{original_path_only}/../{original_filename_only}\",\n }\n\n for path_technique, payloads in path_techniques.items():\n iterations = 5 # one failed detection is tolerated, as long as its not the first run\n confirmations = 0\n while iterations > 0:\n try:\n http_compare = self.compare_baseline(\n self.event.data[\"type\"], probe_value, cookies, skip_urlencoding=True\n )\n singledot_probe = await self.compare_probe(\n http_compare,\n self.event.data[\"type\"],\n payloads[\"singledot_payload\"],\n cookies,\n skip_urlencoding=True,\n )\n doubledot_probe = await self.compare_probe(\n http_compare,\n self.event.data[\"type\"],\n payloads[\"doubledot_payload\"],\n cookies,\n skip_urlencoding=True,\n )\n # if singledot_probe[0] is true, the response is the same as the baseline. This indicates adding a single dot did not break the functionality\n # next, if doubledot_probe[0] is false, the response is different from the baseline. This further indicates that a real path is being manipulated\n # if doubledot_probe[3] is not None, the response is not empty.\n # if doubledot_probe[1] is not [\"header\"], the response is not JUST a header change.\n # \"The requested URL was rejected\" is a very common WAF error message which appears on 200 OK response, confusing detections\n if (\n singledot_probe[0] is True\n and doubledot_probe[0] is False\n and doubledot_probe[3] is not None\n and doubledot_probe[1] != [\"header\"]\n and \"The requested URL was rejected\" not in doubledot_probe[3].text\n ):\n confirmations += 1\n self.verbose(f\"Got possible Path Traversal detection: [{str(confirmations)}] Confirmations\")\n # only report if we have 3 confirmations\n if confirmations > 3:\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"POSSIBLE Path Traversal. {self.metadata()} Detection Method: [{path_technique}]\",\n }\n )\n # no need to report both techniques if they both work\n break\n except HttpCompareError as e:\n iterations -= 1\n self.debug(e)\n continue\n\n iterations -= 1\n if confirmations == 0:\n break\n\n # Absolute path test, covering Windows and Linux\n absolute_paths = {\n r\"c:\\\\windows\\\\win.ini\": \"; for 16-bit app support\",\n \"/etc/passwd\": \"daemon:x:\",\n \"../../../../../etc/passwd%00.png\": \"daemon:x:\",\n }\n\n for path, trigger in absolute_paths.items():\n r = await self.standard_probe(self.event.data[\"type\"], cookies, path, skip_urlencoding=True)\n if r and trigger in r.text:\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"POSSIBLE Path Traversal. {self.metadata()} Detection Method: [Absolute Path: {path}]\",\n }\n )\n" }, { "path": "bbot/modules/lightfuzz/submodules/serial.py", "content": "from .base import BaseLightfuzz\nfrom bbot.errors import HttpCompareError\n\n\nclass serial(BaseLightfuzz):\n \"\"\"Finds parameters where serialized objects might be being deserialized.\n It starts by performing a baseline with a specially-crafted non-serialized payload, separated by type (base64, hex, php raw).\n This is designed to coax out an error that's not related to the decoding process.\n\n After performing the baseline (Which by design may contain an error), we check for two possible deserialization cases:\n\n 1) Replacing the payload with a serialized object changes the status code to 200 (minus some string signatures to help prevent false positives)\n\n 2) If the first case doesn't match, we check for a telltale error string like \"java.io.optionaldataexception\" in the response.\n \"\"\"\n\n friendly_name = \"Unsafe Deserialization\"\n\n # Class-level constants\n CONTROL_PAYLOAD_HEX = \"f56124208220432ec767646acd2e6c6bc9622a62c5656f2eeb616e2f\"\n CONTROL_PAYLOAD_BASE64 = \"4Wt5fYx5Y3rELn5myS5oa996Ji7IZ28uwGdha4x6YmuMfG992CA=\"\n CONTROL_PAYLOAD_PHP_RAW = \"z:0:{}\"\n\n BASE64_SERIALIZATION_PAYLOADS = {\n \"php_base64\": \"YToxOntpOjA7aToxO30=\",\n \"java_base64\": \"rO0ABXNyABFqYXZhLmxhbmcuQm9vbGVhbs0gcoDVnPruAgABWgAFdmFsdWV4cAA=\",\n \"java_base64_string_error\": \"rO0ABXQABHRlc3Q=\",\n \"java_base64_OptionalDataException\": \"rO0ABXcEAAAAAAEAAAABc3IAEGphdmEudXRpbC5IYXNoTWFwAAAAAAAAAAECAAJMAARrZXkxYgABAAAAAAAAAAJ4cHcBAAAAB3QABHRlc3Q=\",\n \"dotnet_base64\": \"AAEAAAD/////AQAAAAAAAAAGAQAAAAdndXN0YXZvCw==\",\n \"ruby_base64\": \"BAh7BjoKbE1FAAVJsg==\",\n }\n\n HEX_SERIALIZATION_PAYLOADS = {\n \"java_hex\": \"ACED00057372000E6A6176612E6C616E672E426F6F6C65616ECD207EC0D59CF6EE02000157000576616C7565787000\",\n \"java_hex_OptionalDataException\": \"ACED0005737200106A6176612E7574696C2E486173684D617000000000000000012000014C00046B6579317A00010000000000000278707000000774000474657374\",\n \"dotnet_hex\": \"0001000000ffffffff01000000000000000601000000076775737461766f0b\",\n }\n\n PHP_RAW_SERIALIZATION_PAYLOADS = {\n \"php_raw\": \"a:0:{}\",\n }\n\n SERIALIZATION_ERRORS = [\n \"invalid user\",\n \"cannot cast java.lang.string\",\n \"dump format error\",\n \"java.io.optionaldataexception\",\n ]\n\n GENERAL_ERRORS = [\n \"Internal Error\",\n \"Internal Server Error\",\n \"The requested URL was rejected\",\n ]\n\n def is_possibly_serialized(self, value):\n # Use the is_base64 method from BaseLightfuzz via self\n if self.is_base64(value):\n return True\n\n # Use the is_hex method from BaseLightfuzz via self\n if self.is_hex(value):\n return True\n\n # List of common PHP serialized data prefixes\n php_serialized_prefixes = [\n \"a:\", # Array\n \"O:\", # Object\n \"s:\", # String\n \"i:\", # Integer\n \"d:\", # Double\n \"b:\", # Boolean\n \"N;\", # Null\n ]\n\n # Check if the value starts with any of the PHP serialized prefixes\n if any(value.startswith(prefix) for prefix in php_serialized_prefixes):\n return True\n return False\n\n async def fuzz(self):\n cookies = self.event.data.get(\"assigned_cookies\", {})\n control_payload_hex = self.CONTROL_PAYLOAD_HEX\n control_payload_base64 = self.CONTROL_PAYLOAD_BASE64\n control_payload_php_raw = self.CONTROL_PAYLOAD_PHP_RAW\n\n base64_serialization_payloads = self.BASE64_SERIALIZATION_PAYLOADS\n hex_serialization_payloads = self.HEX_SERIALIZATION_PAYLOADS\n php_raw_serialization_payloads = self.PHP_RAW_SERIALIZATION_PAYLOADS\n\n serialization_errors = self.SERIALIZATION_ERRORS\n general_errors = self.GENERAL_ERRORS\n\n probe_value = self.incoming_probe_value(populate_empty=False)\n if probe_value:\n if self.is_possibly_serialized(probe_value):\n self.debug(\n f\"Existing value is not ruled out for being a serialized object, proceeding [{self.event.data['type']}] [{self.event.data['name']}]\"\n )\n else:\n self.debug(\n f\"The Serialization Submodule only operates when there is no original value, or when the original value could potentially be a serialized object, aborting [{self.event.data['type']}] [{self.event.data['name']}]\"\n )\n return\n\n try:\n http_compare_hex = self.compare_baseline(self.event.data[\"type\"], control_payload_hex, cookies)\n http_compare_base64 = self.compare_baseline(self.event.data[\"type\"], control_payload_base64, cookies)\n http_compare_php_raw = self.compare_baseline(self.event.data[\"type\"], control_payload_php_raw, cookies)\n except HttpCompareError as e:\n self.debug(f\"HttpCompareError encountered: {e}\")\n return\n\n # Proceed with payload probes\n for payload_set, payload_baseline in [\n (base64_serialization_payloads, http_compare_base64),\n (hex_serialization_payloads, http_compare_hex),\n (php_raw_serialization_payloads, http_compare_php_raw),\n ]:\n for type, payload in payload_set.items():\n try:\n matches_baseline, diff_reasons, reflection, response = await self.compare_probe(\n payload_baseline, self.event.data[\"type\"], payload, cookies\n )\n except HttpCompareError as e:\n self.debug(f\"HttpCompareError encountered: {e}\")\n continue\n\n if matches_baseline:\n self.debug(f\"Payload {type} matches baseline, skipping\")\n continue\n\n self.debug(f\"Probe result for {type}: {response}\")\n\n status_code = getattr(response, \"status_code\", 0)\n if status_code == 0:\n continue\n\n if diff_reasons == [\"header\"]:\n self.debug(f\"Only header diffs found for {type}, skipping\")\n continue\n\n if status_code not in (200, 500):\n self.debug(f\"Status code {status_code} not in (200, 500), skipping\")\n continue\n\n # if the status code changed to 200, and the response doesn't match our general error exclusions, we have a finding\n self.debug(f\"Potential finding detected for {type}, needs confirmation\")\n if (\n status_code == 200\n and \"code\" in diff_reasons\n and not any(\n error in response.text for error in general_errors\n ) # ensure the 200 is not actually an error\n ):\n\n def get_title(text):\n soup = self.lightfuzz.helpers.beautifulsoup(text, \"html.parser\")\n if soup and soup.title and soup.title.string:\n return f\"'{self.lightfuzz.helpers.truncate_string(soup.title.string, 50)}'\"\n return \"\"\n\n baseline_title = get_title(payload_baseline.baseline.text)\n probe_title = get_title(response.text)\n\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"POSSIBLE Unsafe Deserialization. {self.metadata()} Technique: [Error Resolution (Baseline: [{payload_baseline.baseline.status_code}] {baseline_title} -> Probe: [{status_code}] {probe_title})] Serialization Payload: [{type}]\",\n }\n )\n # if the first case doesn't match, we check for a telltale error string like \"java.io.optionaldataexception\" in the response.\n # but only if the response is a 500, or a 200 with a body diff\n elif status_code == 500 or (status_code == 200 and diff_reasons == [\"body\"]):\n self.debug(f\"500 status code or body match for {type}\")\n for serialization_error in serialization_errors:\n # check for the error string, but also ensure the error string isn't just always present in the response\n if (\n serialization_error in response.text.lower()\n and serialization_error not in payload_baseline.baseline.text.lower()\n ):\n self.debug(f\"Error string '{serialization_error}' found in response for {type}\")\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"POSSIBLE Unsafe Deserialization. {self.metadata()} Technique: [Differential Error Analysis] Error-String: [{serialization_error}] Payload: [{type}]\",\n }\n )\n break\n" }, { "path": "bbot/modules/lightfuzz/submodules/sqli.py", "content": "from .base import BaseLightfuzz\nfrom bbot.errors import HttpCompareError\n\nimport statistics\n\n\nclass sqli(BaseLightfuzz):\n \"\"\"\n Detects SQL injection vulnerabilities.\n\n Techniques:\n\n * Error-based Detection:\n - Injects single quotes and observes error responses\n - Tests quote escape sequence variations\n - Matches against known SQL error patterns\n\n * Time-based Blind Detection:\n - Uses vendor-specific time delay payloads\n - Confirms delays with statistical analysis\n - Requires multiple confirmations to eliminate false positives\n \"\"\"\n\n friendly_name = \"SQL Injection\"\n\n expected_delay = 5\n # These are common error strings that strongly indicate SQL injection\n sqli_error_strings = [\n \"Unterminated string literal\",\n \"Failed to parse string literal\",\n \"error in your SQL syntax\",\n \"syntax error at or near\",\n \"Unknown column\",\n \"unterminated quoted string\",\n \"Unclosed quotation mark\",\n \"Incorrect syntax near\",\n \"SQL command not properly ended\",\n \"string not properly terminated\",\n ]\n\n def evaluate_delay(self, mean_baseline, measured_delay):\n \"\"\"\n Evaluates if a measured delay falls within an expected range, indicating potential SQL injection.\n\n Parameters:\n - mean_baseline (float): The average baseline delay measured from non-injected requests.\n - measured_delay (float): The delay measured from a potentially injected request.\n\n Returns:\n - bool: True if the measured delay is within the expected range or exactly twice the expected delay, otherwise False.\n\n The function checks if the measured delay is within a margin of the expected delay or twice the expected delay,\n accounting for cases where the injected statement might be executed twice.\n \"\"\"\n margin = 1.5\n if (\n mean_baseline + self.expected_delay - margin\n <= measured_delay\n <= mean_baseline + self.expected_delay + margin\n ):\n return True\n # check for exactly twice the delay, in case the statement gets placed in the query twice (a common occurrence)\n elif (\n mean_baseline + (self.expected_delay * 2) - margin\n <= measured_delay\n <= mean_baseline + (self.expected_delay * 2) + margin\n ):\n return True\n else:\n return False\n\n async def fuzz(self):\n cookies = self.event.data.get(\"assigned_cookies\", {})\n probe_value = self.incoming_probe_value(populate_empty=True)\n http_compare = self.compare_baseline(\n self.event.data[\"type\"], probe_value, cookies, additional_params_populate_empty=True\n )\n\n try:\n # send the with a single quote, and then another with two single quotes\n single_quote = await self.compare_probe(\n http_compare,\n self.event.data[\"type\"],\n f\"{probe_value}'\",\n cookies,\n additional_params_populate_empty=True,\n )\n double_single_quote = await self.compare_probe(\n http_compare,\n self.event.data[\"type\"],\n f\"{probe_value}''\",\n cookies,\n additional_params_populate_empty=True,\n )\n # if the single quote probe response is different from the baseline\n if single_quote[0] is False:\n # check for common SQL error strings in the response\n for sqli_error_string in self.sqli_error_strings:\n if sqli_error_string.lower() in single_quote[3].text.lower():\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"Possible SQL Injection. {self.metadata()} Detection Method: [SQL Error Detection] Detected String: [{sqli_error_string}]\",\n }\n )\n break\n # if both probes were successful (and had a response)\n if single_quote[3] and double_single_quote[3]:\n # Ensure none of the status codes are \"429\"\n if (\n single_quote[3].status_code != 429\n and double_single_quote[3].status_code != 429\n and http_compare.baseline.status_code != 429\n and http_compare.baseline.status_code != 403 # Ensure the baseline status code is not 403\n ): # prevent false positives from rate limiting\n # if the code changed in the single quote probe, and the code is NOT the same between that and the double single quote probe, SQL injection is indicated\n if \"code\" in single_quote[1] and (\n single_quote[3].status_code != double_single_quote[3].status_code\n ):\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"Possible SQL Injection. {self.metadata()} Detection Method: [Single Quote/Two Single Quote, Code Change ({http_compare.baseline.status_code}->{single_quote[3].status_code}->{double_single_quote[3].status_code})]\",\n }\n )\n else:\n self.debug(\"Failed to get responses for both single_quote and double_single_quote\")\n except HttpCompareError as e:\n self.verbose(f\"Encountered HttpCompareError Sending Compare Probe: {e}\")\n\n # These are common SQL injection payloads for inducing an intentional delay across several different SQL database types\n standard_probe_strings = [\n f\"'||pg_sleep({str(self.expected_delay)})--\", # postgres\n f\"1' AND (SLEEP({str(self.expected_delay)})) AND '\", # mysql\n f\"' AND (SELECT FROM DBMS_LOCK.SLEEP({str(self.expected_delay)})) AND '1'='1\" # oracle (not tested)\n f\"; WAITFOR DELAY '00:00:{str(self.expected_delay)}'--\", # mssql (not tested)\n ]\n\n baseline_1 = await self.standard_probe(\n self.event.data[\"type\"], cookies, probe_value, additional_params_populate_empty=True\n )\n baseline_2 = await self.standard_probe(\n self.event.data[\"type\"], cookies, probe_value, additional_params_populate_empty=True\n )\n\n # get a baseline from two different probes. We will average them to establish a mean baseline\n if baseline_1 and baseline_2:\n baseline_1_delay = baseline_1.elapsed.total_seconds()\n baseline_2_delay = baseline_2.elapsed.total_seconds()\n mean_baseline = statistics.mean([baseline_1_delay, baseline_2_delay])\n\n for p in standard_probe_strings:\n confirmations = 0\n for i in range(0, 3):\n # send the probe 3 times, and check if the delay is within the detection threshold\n r = await self.standard_probe(\n self.event.data[\"type\"],\n cookies,\n f\"{probe_value}{p}\",\n additional_params_populate_empty=True,\n timeout=20,\n )\n if not r:\n self.debug(\"delay measure request failed\")\n break\n\n d = r.elapsed.total_seconds()\n self.debug(f\"measured delay: {str(d)}\")\n if self.evaluate_delay(\n mean_baseline, d\n ): # decide if the delay is within the detection threshold and constitutes a successful sleep execution\n confirmations += 1\n self.debug(\n f\"{self.event.data['url']}:{self.event.data['name']}:{self.event.data['type']} Increasing confirmations, now: {str(confirmations)} \"\n )\n else:\n break\n\n if confirmations == 3:\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"Possible Blind SQL Injection. {self.metadata()} Detection Method: [Delay Probe ({p})]\",\n }\n )\n\n else:\n self.debug(\"Could not get baseline for time-delay tests\")\n" }, { "path": "bbot/modules/lightfuzz/submodules/ssti.py", "content": "from .base import BaseLightfuzz\n\n\nclass ssti(BaseLightfuzz):\n \"\"\"\n Detects server-side template injection vulnerabilities.\n\n Techniques:\n\n * Arithmetic Evaluation:\n - Injects encoded and unencoded multiplication expressions to detect evaluation\n \"\"\"\n\n friendly_name = \"Server-side Template Injection\"\n\n async def fuzz(self):\n cookies = self.event.data.get(\"assigned_cookies\", {})\n # These are common SSTI payloads, each attempting to trigger an integer multiplication which would produce an expected value\n ssti_probes = [\n \"<%25%3d%201337*1337%20%25>\",\n \"<%= 1337*1337 %>\",\n \"${1337*1337}\",\n \"%24%7b1337*1337%7d\",\n \"1,787{{z}},569\",\n ]\n for probe_value in ssti_probes:\n r = await self.standard_probe(\n self.event.data[\"type\"], cookies, probe_value, allow_redirects=True, skip_urlencoding=True\n )\n\n # look for the expected value in the response\n if r and (\"1787569\" in r.text or \"1,787,569\" in r.text):\n self.results.append(\n {\n \"type\": \"FINDING\",\n \"description\": f\"POSSIBLE Server-side Template Injection. {self.metadata()} Detection Method: [Integer Multiplication] Payload: [{probe_value}]\",\n }\n )\n break\n" }, { "path": "bbot/modules/lightfuzz/submodules/xss.py", "content": "from .base import BaseLightfuzz\n\nimport regex as re\n\n\nclass xss(BaseLightfuzz):\n \"\"\"\n Detects Reflected Cross-Site Scripting vulnerabilities across multiple contexts and techniques\n\n * Context Detection:\n - Between HTML Tags: injection\n - Within Tag Attributes: \n - Inside JavaScript: \n\n * Context-Specific Testing:\n - Between Tags: Tests basic HTML injection and tag creation\n - Tag Attributes: Tests quote escaping and JavaScript event handlers\n - JavaScript Context: Tests string delimiter breaking and script tag termination\n - Handles both single and double quote contexts in JavaScript\n\n Can often detect through WAFs, since it does not attempt to construct an exploitation payload\n \"\"\"\n\n friendly_name = \"Cross-Site Scripting\"\n\n async def determine_context(self, cookies, html, random_string):\n \"\"\"\n Determines the context of the random string in the HTML response.\n With XSS, the context is what kind part of the page the injection is occuring in, which determine what payloads might be successful\n\n https://portswigger.net/web-security/cross-site-scripting/contexts\n \"\"\"\n between_tags = False\n in_tag_attribute = False\n in_javascript = False\n\n between_tags_regex = re.compile(\n rf\"<(\\/?\\w+)[^>]*>.*?{random_string}.*?<\\/?\\w+>\"\n ) # The between tags context is when the injection occurs between HTML tags\n in_tag_attribute_regex = re.compile(\n rf'<(\\w+)\\s+[^>]*?(\\w+)=\"([^\"]*?{random_string}[^\"]*?)\"[^>]*>'\n ) # The in tag attribute context is when the injection occurs in an attribute of an HTML tag\n in_javascript_regex = re.compile(\n rf\"]*>[^<]*(?:<(?!\\/script>)[^<]*)*{random_string}[^<]*(?:<(?!\\/script>)[^<]*)*<\\/script>\"\n ) # The in javascript context is when the injection occurs within a \"\n result = await self.check_probe(\n cookies, in_javascript_probe, in_javascript_probe, \"In Javascript\"\n ) # After reflection in the HTTP response, did the script tags survive without url-encoding or other sanitization/escaping?\n if result is False:\n # To attempt this technique, we need to determine the type of quote we are within\n quote_context = await self.determine_javascript_quote_context(\n random_string, reflection_probe_result.text\n )\n\n # Skip the test if the context is outside\n if quote_context == \"outside\":\n return\n\n # Update probes based on the quote context\n if quote_context == \"single\":\n in_javascript_escape_probe = rf\"a\\';zzzzz({random_string})\\\\\"\n in_javascript_escape_match = rf\"a\\\\';zzzzz({random_string})\\\\\"\n elif quote_context == \"double\":\n in_javascript_escape_probe = rf\"a\\\";zzzzz({random_string})\\\\\"\n in_javascript_escape_match = rf'a\\\\\";zzzzz({random_string})\\\\'\n\n await self.check_probe(\n cookies,\n in_javascript_escape_probe,\n in_javascript_escape_match,\n f\"In Javascript (escaping the escape character, {quote_context} quote)\",\n )\n" }, { "path": "bbot/modules/medusa.py", "content": "import re\nfrom bbot.modules.base import BaseModule\n\n\nclass medusa(BaseModule):\n watched_events = [\"PROTOCOL\"]\n produced_events = [\"VULNERABILITY\"]\n flags = [\"active\", \"aggressive\", \"deadly\"]\n per_host_only = True\n meta = {\n \"description\": \"Medusa SNMP bruteforcing with v1, v2c and R/W check.\",\n \"created_date\": \"2025-05-16\",\n \"author\": \"@christianfl\",\n }\n scope_distance_modifier = None\n\n options = {\n \"snmp_wordlist\": \"https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/common-snmp-community-strings.txt\",\n \"snmp_versions\": [\"1\", \"2C\"], # Only 1 and 2C are available with medusa 2.3.\n \"wait_microseconds\": 200,\n \"timeout_s\": 5,\n \"threads\": 5,\n }\n\n options_desc = {\n \"snmp_wordlist\": \"Wordlist url for SNMP community strings, newline separated (default https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/snmp.txt)\",\n \"snmp_versions\": \"List of SNMP versions to attempt against the SNMP server (default ['1', '2C'])\",\n \"wait_microseconds\": \"Wait time after every SNMP request in microseconds (default 200)\",\n \"timeout_s\": \"Wait time for the SNMP response(s) once at the end of all attempts (default 5)\",\n \"threads\": \"Number of communities to be tested concurrently (default 5)\",\n }\n\n deps_ansible = [\n {\n \"name\": \"Install build dependencies\",\n \"package\": {\n \"name\": [\n \"autoconf\",\n \"automake\",\n \"libtool\",\n \"gcc\",\n \"make\",\n ],\n \"state\": \"present\",\n },\n \"become\": True,\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Get medusa repo\",\n \"git\": {\n \"repo\": \"https://github.com/jmk-foofus/medusa\",\n \"dest\": \"#{BBOT_TEMP}/medusa/gitrepo\",\n \"version\": \"2.3\", # Newest stable, 2025-05-15\n },\n },\n {\n # The git repo will be copied because during build, files and subfolders get created. That prevents the Ansible git module to cache the repo.\n \"name\": \"Copy medusa repo\",\n \"copy\": {\n \"src\": \"#{BBOT_TEMP}/medusa/gitrepo/\",\n \"dest\": \"#{BBOT_TEMP}/medusa/workdir/\",\n },\n },\n {\n \"name\": \"Build medusa: autoreconf\",\n \"command\": {\n \"chdir\": \"#{BBOT_TEMP}/medusa/workdir\",\n \"cmd\": \"autoreconf -f -i\",\n },\n },\n {\n \"name\": \"Build medusa: configure\",\n \"command\": {\n \"chdir\": \"#{BBOT_TEMP}/medusa/workdir\",\n \"cmd\": \"./configure --prefix=#{BBOT_TEMP}/medusa/build\",\n },\n },\n {\n \"name\": \"Build medusa: make\",\n \"command\": {\n \"chdir\": \"#{BBOT_TEMP}/medusa/workdir\",\n \"cmd\": \"make\",\n },\n },\n {\n \"name\": \"Build medusa: make install\",\n \"command\": {\n \"chdir\": \"#{BBOT_TEMP}/medusa/workdir\",\n \"cmd\": \"make install\",\n \"creates\": \"#{BBOT_TEMP}/medusa/build/bin/medusa\",\n },\n },\n {\n \"name\": \"Install medusa\",\n \"copy\": {\n \"src\": \"#{BBOT_TEMP}/medusa/build/bin/medusa\",\n \"dest\": \"#{BBOT_TOOLS}/\",\n \"mode\": \"u+x,g+x,o+x\",\n },\n },\n ]\n\n async def setup_deps(self):\n self.snmp_wordlist_path = await self.helpers.wordlist(self.config.get(\"snmp_wordlist\"))\n return True\n\n async def setup(self):\n self.password_match_regex = re.compile(r\"Password:\\s*(\\S+)\")\n self.success_indicator_match_regex = re.compile(r\"\\[([^\\]]+)\\]\\s*$\")\n\n return True\n\n async def filter_event(self, event):\n handled_protocols = [\"snmp\"] # Could be extended later\n\n protocol = event.data[\"protocol\"].lower()\n if not protocol in handled_protocols:\n return False, f\"service {protocol} is currently not supported. Only SNMP.\"\n\n return True\n\n async def handle_event(self, event):\n host = str(event.host)\n port = str(event.port)\n protocol = event.data[\"protocol\"].lower()\n\n if protocol == \"snmp\":\n snmp_versions = self.config.get(\"snmp_versions\")\n\n # Medusa must be called for each SNMP version separately after each run finished.\n for snmp_version in snmp_versions:\n command = await self.construct_command(host, port, protocol, snmp_version)\n\n result = await self.run_process(command)\n\n if result.stderr:\n # Medusa outputs to stderr if a readonly community was found in WRITE mode\n # That's intended behavior\n self.info(f\"Medusa stderr: {result.stderr}\")\n\n async for message in self.parse_output(result.stdout, snmp_version):\n vuln_event = self.create_vuln_event(\"CRITICAL\", message, event)\n await self.emit_event(vuln_event)\n\n # else: Medusa supports various protocols which could in theory be implemented later on.\n\n async def parse_output(self, output, protocol_version):\n for line in output.splitlines():\n # Print original Medusa output\n self.info(line)\n\n if \"FOUND\" in line:\n # Some credential was guessed\n password_match = self.password_match_regex.search(line)\n password = password_match.group(1) if password_match else None\n\n success_indicator_match = self.success_indicator_match_regex.search(line)\n success_indicator = success_indicator_match.group(1) if success_indicator_match else None\n\n # Medusa in WRITE mode shows \"ERROR\" if a readonly community was found. Replace with \"READ\"\n mode = \"R/W\" if success_indicator == \"success\" else \"READ\" if success_indicator == \"ERROR\" else \"MODE?\"\n\n message = f\"VALID [SNMPV{protocol_version}] CREDENTIALS FOUND: {password} [{mode}]\"\n\n yield message\n\n async def construct_command(self, host, port, protocol, protocol_version):\n # -b Suppress startup banner\n # -v Set verbosity level (4 = Show only errors and credentials)\n # -R Number of attempted retries\n # -M Medusa module to execute (SNMP)\n # -T Number of concurrent hosts\n # -t Number of concurrent login attempts\n # -h Target hostname or ip address\n # -u Username to test (Empty for SNMP)\n # -P Wordlist for passwords\n # -m Module specific parameters:\n # TIMEOUT: Sets the number of seconds to wait for the UDP responses (default: 5 sec).\n # SEND_DELAY: Sets the number of microseconds to wait between sending queries (default: 200 usec).\n # VERSION:<1|2C> Set the SNMP client version.\n # ACCESS: Set level of access to test for with the community string. (\"WRITE\" does include \"READ\")\n\n # Example command to bruteforce SNMP:\n #\n # medusa -b -v 4 -R 1 -M snmp -T 1 -t 1 -h 127.0.0.1 -u '' -P communities.txt -m VERSION:2C -m SEND_DELAY:1000000 -m ACCESS:WRITE -m TIMEOUT:10\n\n cmd = [\n \"medusa\",\n \"-b\",\n \"-v\",\n 4,\n \"-R\",\n 1,\n \"-M\",\n protocol,\n \"-T\",\n 1,\n \"-t\",\n self.config.get(\"threads\"),\n \"-h\",\n host,\n \"-u\",\n \"''\",\n \"-P\",\n self.snmp_wordlist_path,\n \"-m\",\n f\"VERSION:{protocol_version}\",\n \"-m\",\n f\"SEND_DELAY:{self.config.get('wait_microseconds')}\",\n \"-m\",\n \"ACCESS:WRITE\",\n \"-m\",\n f\"TIMEOUT:{self.config.get('timeout_s')}\",\n ]\n\n return cmd\n\n def create_vuln_event(self, severity, description, source_event):\n host = str(source_event.host)\n port = str(source_event.port)\n\n return self.make_event(\n {\n \"severity\": severity,\n \"host\": host,\n \"port\": port,\n \"description\": description,\n },\n \"VULNERABILITY\",\n source_event,\n )\n" }, { "path": "bbot/modules/myssl.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass myssl(subdomain_enum):\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n meta = {\n \"description\": \"Query myssl.com's API for subdomains\",\n \"created_date\": \"2023-07-10\",\n \"author\": \"@TheTechromancer\",\n }\n\n base_url = \"https://myssl.com/api/v1/discover_sub_domain\"\n\n async def request_url(self, query):\n url = f\"{self.base_url}?domain={self.helpers.quote(query)}\"\n return await self.api_request(url)\n\n async def parse_results(self, r, query):\n results = set()\n json = r.json()\n if json and isinstance(json, dict):\n data = json.get(\"data\", [])\n for d in data:\n hostname = d.get(\"domain\", \"\").lower()\n if hostname:\n results.add(hostname)\n return results\n" }, { "path": "bbot/modules/newsletters.py", "content": "# Created a new module called 'newsletters' that will scrape the websites (or recursive websites,\n# thanks to BBOT's sub-domain enumeration) looking for the presence of an 'email type' that also\n# contains a 'placeholder'. The combination of these two HTML items usually signify the presence\n# of an \"Enter Your Email Here\" type Newsletter Subscription service. This module could be used\n# to find newsletters for a future email bombing attack.\n\nfrom .base import BaseModule\nimport re\n\n# Known Websites with Newsletters\n# https://futureparty.com/\n# https://www.marketingbrew.com/\n# https://buffer.com/\n# https://www.milkkarten.net/\n# https://geekout.mattnavarra.com/\n\n\nclass newsletters(BaseModule):\n watched_events = [\"HTTP_RESPONSE\"]\n produced_events = [\"FINDING\"]\n flags = [\"active\", \"safe\"]\n meta = {\n \"description\": \"Searches for Newsletter Submission Entry Fields on Websites\",\n \"created_date\": \"2024-02-02\",\n \"author\": \"@stryker2k2\",\n }\n\n # Parse through Website to find a Text Entry Box of 'type = email'\n # and ensure that there is placeholder text within it.\n def find_type(self, soup):\n email_type = soup.find(type=\"email\")\n if email_type:\n regex = re.compile(r\"placeholder\")\n if regex.search(str(email_type)):\n return True\n return False\n\n async def handle_event(self, event):\n _event = event\n\n # Call find_type Function if Webpage return Status Code 200 && \"body\" is found in event.data\n # Ex: 'bbot -m httpx newsletters -t https://apf-api.eng.vn.cloud.tesla.com' returns\n # Status Code 200 but does NOT have event.data[\"body\"]\n if _event.data[\"status_code\"] == 200:\n if \"body\" in _event.data:\n body = _event.data[\"body\"]\n soup = self.helpers.beautifulsoup(body, \"html.parser\")\n if soup is False:\n self.debug(\"BeautifulSoup returned False\")\n return\n result = self.find_type(soup)\n if result:\n description = \"Found a Newsletter Submission Form that could be used for email bombing attacks\"\n data = {\"host\": str(_event.host), \"description\": description, \"url\": _event.data[\"url\"]}\n await self.emit_event(\n data,\n \"FINDING\",\n _event,\n context=\"{module} searched HTTP_RESPONSE and identified {event.type}: a Newsletter Submission Form that could be used for email bombing attacks\",\n )\n" }, { "path": "bbot/modules/ntlm.py", "content": "from bbot.errors import NTLMError\nfrom bbot.modules.base import BaseModule\n\nntlm_discovery_endpoints = [\n \"\",\n \"autodiscover/autodiscover.xml\",\n \"ecp/\",\n \"ews/\",\n \"ews/exchange.asmx\",\n \"exchange/\",\n \"exchweb/\",\n \"oab/\",\n \"owa/\",\n \"_windows/default.aspx?ReturnUrl=/\",\n \"abs/\",\n \"adfs/ls/wia\",\n \"adfs/services/trust/2005/windowstransport\",\n \"aspnet_client/\",\n \"autodiscover/\",\n \"autoupdate/\",\n \"certenroll/\",\n \"certprov/\",\n \"certsrv/\",\n \"conf/\",\n \"debug/\",\n \"deviceupdatefiles_ext/\",\n \"deviceupdatefiles_int/\",\n \"dialin/\",\n \"etc/\",\n \"groupexpansion/\",\n \"hybridconfig/\",\n \"iwa/authenticated.aspx\",\n \"iwa/iwa_test.aspx\",\n \"mcx/\",\n \"mcx/mcxservice.svc\",\n \"meet/\",\n \"meeting/\",\n \"microsoft-server-activesync/\",\n \"ocsp/\",\n \"persistentchat/\",\n \"phoneconferencing/\",\n \"powershell/\",\n \"public/\",\n \"reach/sip.svc\",\n \"remoteDesktopGateway/\",\n \"requesthandler/\",\n \"requesthandlerext/\",\n \"rgs/\",\n \"rgsclients/\",\n \"rpc/\",\n \"rpcwithcert/\",\n \"scheduler/\",\n \"ucwa/\",\n \"unifiedmessaging/\",\n \"webticket/\",\n \"webticket/webticketservice.svc\",\n]\n\nNTLM_test_header = {\"Authorization\": \"NTLM TlRMTVNTUAABAAAAl4II4gAAAAAAAAAAAAAAAAAAAAAKAGFKAAAADw==\"}\n\n\nclass ntlm(BaseModule):\n \"\"\"\n Todo:\n Cancel pending requests and break out of loop when valid endpoint is found\n (waiting on https://github.com/encode/httpcore/discussions/783/ to be fixed first)\n \"\"\"\n\n watched_events = [\"URL\", \"HTTP_RESPONSE\"]\n produced_events = [\"FINDING\", \"DNS_NAME\"]\n flags = [\"active\", \"safe\", \"web-basic\"]\n meta = {\n \"description\": \"Watch for HTTP endpoints that support NTLM authentication\",\n \"created_date\": \"2022-07-25\",\n \"author\": \"@liquidsec\",\n }\n options = {\"try_all\": False}\n options_desc = {\"try_all\": \"Try every NTLM endpoint\"}\n\n in_scope_only = True\n\n async def setup(self):\n self.found = set()\n self.try_all = self.config.get(\"try_all\", False)\n return True\n\n async def handle_event(self, event):\n found_hash = hash(f\"{event.host}:{event.port}\")\n if event.type == \"URL\":\n url = event.data\n else:\n url = event.data[\"url\"]\n if found_hash in self.found:\n return\n\n urls = {url}\n if self.try_all:\n for endpoint in ntlm_discovery_endpoints:\n urls.add(f\"{event.parsed_url.scheme}://{event.parsed_url.netloc}/{endpoint}\")\n\n num_urls = len(urls)\n agen = self.helpers.request_batch(\n urls, headers=NTLM_test_header, allow_redirects=False, timeout=self.http_timeout\n )\n async for url, response in agen:\n ntlm_resp = response.headers.get(\"WWW-Authenticate\", \"\")\n if not ntlm_resp:\n continue\n ntlm_resp_b64 = max(ntlm_resp.split(\",\"), key=lambda x: len(x)).split()[-1]\n try:\n ntlm_resp_decoded = self.helpers.ntlm.ntlmdecode(ntlm_resp_b64)\n if not ntlm_resp_decoded:\n continue\n\n await agen.aclose()\n self.found.add(found_hash)\n fqdn = ntlm_resp_decoded.get(\"FQDN\", \"\")\n await self.emit_event(\n {\n \"host\": str(event.host),\n \"url\": url,\n \"description\": f\"NTLM AUTH: {ntlm_resp_decoded}\",\n },\n \"FINDING\",\n parent=event,\n context=f\"{{module}} tried {num_urls:,} NTLM endpoints against {url} and identified NTLM auth ({{event.type}}): {fqdn}\",\n )\n fqdn = ntlm_resp_decoded.get(\"FQDN\", \"\")\n if fqdn:\n await self.emit_event(fqdn, \"DNS_NAME\", parent=event)\n break\n\n except NTLMError as e:\n self.verbose(str(e))\n\n async def filter_event(self, event):\n if self.try_all:\n return True\n if event.type == \"HTTP_RESPONSE\":\n if \"www-authenticate\" in event.data[\"header-dict\"]:\n header_value = event.data[\"header-dict\"][\"www-authenticate\"][0].lower()\n if \"ntlm\" in header_value or \"negotiate\" in header_value:\n return True\n return False\n" }, { "path": "bbot/modules/nuclei.py", "content": "import json\nimport yaml\nfrom itertools import islice\nfrom bbot.modules.base import BaseModule\n\n\nclass nuclei(BaseModule):\n watched_events = [\"URL\"]\n produced_events = [\"FINDING\", \"VULNERABILITY\", \"TECHNOLOGY\"]\n flags = [\"active\", \"aggressive\", \"deadly\"]\n meta = {\n \"description\": \"Fast and customisable vulnerability scanner\",\n \"created_date\": \"2022-03-12\",\n \"author\": \"@TheTechromancer\",\n }\n\n options = {\n \"version\": \"3.7.1\",\n \"tags\": \"\",\n \"templates\": \"\",\n \"severity\": \"\",\n \"ratelimit\": 150,\n \"concurrency\": 25,\n \"mode\": \"manual\",\n \"etags\": \"\",\n \"budget\": 1,\n \"silent\": False,\n \"directory_only\": True,\n \"retries\": 0,\n \"batch_size\": 200,\n \"module_timeout\": 21600, # 6 hours\n }\n options_desc = {\n \"version\": \"nuclei version\",\n \"tags\": \"execute a subset of templates that contain the provided tags\",\n \"templates\": \"template or template directory paths to include in the scan\",\n \"severity\": \"Filter based on severity field available in the template.\",\n \"ratelimit\": \"maximum number of requests to send per second (default 150)\",\n \"concurrency\": \"maximum number of templates to be executed in parallel (default 25)\",\n \"mode\": \"manual | technology | severe | budget. Technology: Only activate based on technology events that match nuclei tags (nuclei -as mode). Manual (DEFAULT): Fully manual settings. Severe: Only critical and high severity templates without intrusive. Budget: Limit Nuclei to a specified number of HTTP requests\",\n \"etags\": \"tags to exclude from the scan\",\n \"budget\": \"Used in budget mode to set the number of allowed requests per host\",\n \"silent\": \"Don't display nuclei's banner or status messages\",\n \"directory_only\": \"Filter out 'file' URL event (default True)\",\n \"retries\": \"number of times to retry a failed request (default 0)\",\n \"batch_size\": \"Number of targets to send to Nuclei per batch (default 200)\",\n \"module_timeout\": \"Max time in seconds to spend handling each batch of events\",\n }\n deps_ansible = [\n {\n \"name\": \"Download nuclei\",\n \"unarchive\": {\n \"src\": \"https://github.com/projectdiscovery/nuclei/releases/download/v#{BBOT_MODULES_NUCLEI_VERSION}/nuclei_#{BBOT_MODULES_NUCLEI_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH_GOLANG}.zip\",\n \"include\": \"nuclei\",\n \"dest\": \"#{BBOT_TOOLS}\",\n \"remote_src\": True,\n },\n }\n ]\n deps_pip = [\"pyyaml~=6.0\"]\n in_scope_only = True\n _batch_size = 200\n\n async def setup(self):\n # attempt to update nuclei templates\n self.nuclei_templates_dir = self.helpers.tools_dir / \"nuclei-templates\"\n self.info(\"Updating Nuclei templates\")\n update_results = await self.run_process(\n [\"nuclei\", \"-update-template-dir\", self.nuclei_templates_dir, \"-update-templates\"]\n )\n if update_results.stderr:\n if \"Successfully downloaded nuclei-templates\" in update_results.stderr:\n self.success(\"Successfully updated nuclei templates\")\n elif \"No new updates found for nuclei templates\" in update_results.stderr:\n self.info(\"Nuclei templates already up-to-date\")\n else:\n self.warning(f\"Failure while updating nuclei templates: {update_results.stderr}\")\n else:\n self.warning(\"Error running nuclei template update command\")\n self.proxy = self.scan.web_config.get(\"http_proxy\", \"\")\n self.mode = self.config.get(\"mode\", \"severe\").lower()\n self.ratelimit = int(self.config.get(\"ratelimit\", 150))\n self.concurrency = int(self.config.get(\"concurrency\", 25))\n self.budget = int(self.config.get(\"budget\", 1))\n self.silent = self.config.get(\"silent\", False)\n self.templates = self.config.get(\"templates\")\n if self.templates:\n self.info(f\"Using custom template(s) at: [{self.templates}]\")\n self.tags = self.config.get(\"tags\")\n if self.tags:\n self.info(f\"Setting the following nuclei tags: [{self.tags}]\")\n self.etags = self.config.get(\"etags\")\n if self.etags:\n self.info(f\"Excluding the following nuclei tags: [{self.etags}]\")\n self.severity = self.config.get(\"severity\")\n if self.mode != \"severe\" and self.severity != \"\":\n self.info(f\"Limiting nuclei templates to the following severities: [{self.severity}]\")\n self.iserver = self.scan.config.get(\"interactsh_server\", None)\n self.itoken = self.scan.config.get(\"interactsh_token\", None)\n self.retries = int(self.config.get(\"retries\", 0))\n\n if self.mode not in (\"technology\", \"severe\", \"manual\", \"budget\"):\n self.warning(f\"Unable to initialize nuclei: invalid mode selected: [{self.mode}]\")\n return False\n\n if self.mode == \"technology\":\n self.info(\n \"Running nuclei in TECHNOLOGY mode. Scans will only be performed with the --automatic-scan flag set. This limits the templates used to those that match wappalyzer signatures\"\n )\n self.tags = \"\"\n\n if self.mode == \"severe\":\n self.info(\n \"Running nuclei in SEVERE mode. Only critical and high severity templates will be used. Tag setting will be IGNORED.\"\n )\n self.severity = \"critical,high\"\n self.tags = \"\"\n\n if self.mode == \"manual\":\n self.info(\n \"Running nuclei in MANUAL mode. Settings will be passed directly into nuclei with no modification\"\n )\n\n if self.mode == \"budget\":\n self.info(\n f\"Running nuclei in BUDGET mode. This mode calculates which nuclei templates can be used, constrained by your 'budget' of number of requests. Current budget is set to: {self.budget}\"\n )\n\n self.info(\"Processing nuclei templates to perform budget calculations...\")\n\n self.nucleibudget = NucleiBudget(self)\n self.budget_templates_file = self.helpers.tempfile(self.nucleibudget.collapsible_templates, pipe=False)\n\n self.info(\n f\"Loaded [{str(sum(self.nucleibudget.severity_stats.values()))}] templates based on a budget of [{str(self.budget)}] request(s)\"\n )\n self.info(\n f\"Template Severity: Critical [{self.nucleibudget.severity_stats['critical']}] High [{self.nucleibudget.severity_stats['high']}] Medium [{self.nucleibudget.severity_stats['medium']}] Low [{self.nucleibudget.severity_stats['low']}] Info [{self.nucleibudget.severity_stats['info']}] Unknown [{self.nucleibudget.severity_stats['unknown']}]\"\n )\n\n return True\n\n async def handle_batch(self, *events):\n temp_target = self.helpers.make_target()\n for e in events:\n temp_target.add(e.data, e)\n nuclei_input = [str(e.data) for e in events]\n async for severity, template, tags, host, url, name, extracted_results in self.execute_nuclei(nuclei_input):\n # this is necessary because sometimes nuclei is inconsistent about the data returned in the host field\n cleaned_host = temp_target.get(host)\n parent_event = self.correlate_event(events, cleaned_host)\n\n if not parent_event:\n continue\n\n if url == \"\":\n url = str(parent_event.data)\n\n if severity == \"INFO\" and \"tech\" in tags:\n await self.emit_event(\n {\"technology\": str(name).lower(), \"url\": url, \"host\": str(parent_event.host)},\n \"TECHNOLOGY\",\n parent_event,\n context=f\"{{module}} scanned {url} and identified {{event.type}}: {str(name).lower()}\",\n )\n continue\n\n description_string = f\"template: [{template}], name: [{name}]\"\n if len(extracted_results) > 0:\n description_string += f\" Extracted Data: [{','.join(extracted_results)}]\"\n\n if severity in [\"INFO\", \"UNKNOWN\"]:\n await self.emit_event(\n {\n \"host\": str(parent_event.host),\n \"url\": url,\n \"description\": description_string,\n },\n \"FINDING\",\n parent_event,\n context=f\"{{module}} scanned {url} and identified {{event.type}}: {description_string}\",\n )\n else:\n await self.emit_event(\n {\n \"severity\": severity,\n \"host\": str(parent_event.host),\n \"url\": url,\n \"description\": description_string,\n },\n \"VULNERABILITY\",\n parent_event,\n context=f\"{{module}} scanned {url} and identified {severity.lower()} {{event.type}}: {description_string}\",\n )\n\n def correlate_event(self, events, host):\n for event in events:\n if host in event:\n return event\n self.verbose(f\"Failed to correlate nuclei result for {host}. Possible parent events:\")\n for event in events:\n self.verbose(f\" - {event.data}\")\n\n async def execute_nuclei(self, nuclei_input):\n command = [\n \"nuclei\",\n \"-jsonl\",\n \"-update-template-dir\",\n self.nuclei_templates_dir,\n \"-rate-limit\",\n self.ratelimit,\n \"-concurrency\",\n self.concurrency,\n \"-disable-update-check\",\n \"-stats-json\",\n \"-retries\",\n self.retries,\n ]\n\n if self.helpers.system_resolvers:\n command += [\"-r\", self.helpers.resolver_file]\n\n for hk, hv in self.scan.custom_http_headers.items():\n command += [\"-H\", f\"{hk}: {hv}\"]\n\n for cli_option in (\"severity\", \"templates\", \"iserver\", \"itoken\", \"tags\", \"etags\"):\n option = getattr(self, cli_option)\n\n if option:\n command.append(f\"-{cli_option}\")\n command.append(option)\n\n if self.scan.config.get(\"interactsh_disable\") is True:\n self.info(\"Disabling interactsh in accordance with global settings\")\n command.append(\"-no-interactsh\")\n\n if self.mode == \"technology\":\n command.append(\"-as\")\n\n if self.mode == \"budget\":\n command.append(\"-t\")\n command.append(self.budget_templates_file)\n\n if self.proxy:\n command.append(\"-proxy\")\n command.append(f\"{self.proxy}\")\n\n stats_file = self.helpers.tempfile_tail(callback=self.log_nuclei_status)\n try:\n with open(stats_file, \"w\") as stats_fh:\n async for line in self.run_process_live(command, input=nuclei_input, stderr=stats_fh):\n try:\n j = json.loads(line)\n except json.decoder.JSONDecodeError:\n self.debug(f\"Failed to decode line: {line}\")\n continue\n\n template = j.get(\"template-id\", \"\")\n\n # try to get the specific matcher name\n name = j.get(\"matcher-name\", \"\")\n\n info = j.get(\"info\", {})\n\n # fall back to regular name\n if not name:\n self.debug(\n f\"Couldn't get matcher-name from nuclei json, falling back to regular name. Template: [{template}]\"\n )\n name = info.get(\"name\", \"\")\n severity = info.get(\"severity\", \"\").upper()\n tags = info.get(\"tags\", [])\n host = j.get(\"host\", \"\")\n url = j.get(\"matched-at\", \"\")\n if not self.helpers.is_url(url):\n url = \"\"\n\n extracted_results = j.get(\"extracted-results\", [])\n\n if template and name and severity:\n yield (severity, template, tags, host, url, name, extracted_results)\n else:\n self.debug(\"Nuclei result missing one or more required elements, not reporting. JSON: ({j})\")\n finally:\n stats_file.unlink(missing_ok=True)\n\n def log_nuclei_status(self, line):\n if self.silent:\n return\n try:\n line = json.loads(line)\n except Exception:\n self.info(str(line))\n return\n duration = line.get(\"duration\", \"\")\n errors = line.get(\"errors\", \"\")\n hosts = line.get(\"hosts\", \"\")\n matched = line.get(\"matched\", \"\")\n percent = line.get(\"percent\", \"\")\n requests = line.get(\"requests\", \"\")\n rps = line.get(\"rps\", \"\")\n templates = line.get(\"templates\", \"\")\n total = line.get(\"total\", \"\")\n status = f\"[{duration}] | Templates: {templates} | Hosts: {hosts} | RPS: {rps} | Matched: {matched} | Errors: {errors} | Requests: {requests}/{total} ({percent}%)\"\n self.info(status)\n\n async def cleanup(self):\n resume_file = self.helpers.current_dir / \"resume.cfg\"\n resume_file.unlink(missing_ok=True)\n\n async def filter_event(self, event):\n if self.config.get(\"directory_only\", True):\n if \"endpoint\" in event.tags:\n self.debug(\n f\"rejecting URL [{str(event.data)}] because directory_only is true and event has endpoint tag\"\n )\n return False\n return True\n\n\nclass NucleiBudget:\n def __init__(self, nuclei_module):\n self.parent = nuclei_module\n self._yaml_files = {}\n self.templates_dir = nuclei_module.nuclei_templates_dir\n self.yaml_list = self.get_yaml_list()\n self.budget_paths = self.find_budget_paths(nuclei_module.budget)\n self.collapsible_templates, self.severity_stats = self.find_collapsible_templates()\n\n def get_yaml_list(self):\n return list(self.templates_dir.rglob(\"*.yaml\"))\n\n # Given the current budget setting, scan all of the templates for paths, sort them by frequency and select the first N (budget) items\n def find_budget_paths(self, budget):\n path_frequency = {}\n for yf in self.yaml_list:\n if yf:\n for paths in self.get_yaml_request_attr(yf, \"path\"):\n for path in paths:\n if path in path_frequency.keys():\n path_frequency[path] += 1\n else:\n path_frequency[path] = 1\n\n sorted_dict = dict(sorted(path_frequency.items(), key=lambda item: item[1], reverse=True))\n return list(dict(islice(sorted_dict.items(), budget)).keys())\n\n def get_yaml_request_attr(self, yf, attr):\n p = self.parse_yaml(yf)\n requests = p.get(\"http\", [])\n for r in requests:\n raw = r.get(\"raw\")\n if not raw:\n res = r.get(attr)\n if res is not None:\n yield res\n\n def get_yaml_info_attr(self, yf, attr):\n p = self.parse_yaml(yf)\n info = p.get(\"info\", [])\n res = info.get(attr)\n if res is not None:\n yield res\n\n # Parse through all templates and locate those which match the conditions necessary to collapse down to the budget setting\n def find_collapsible_templates(self):\n collapsible_templates = []\n severity_dict = {}\n for yf in self.yaml_list:\n valid = True\n if yf:\n for paths in self.get_yaml_request_attr(yf, \"path\"):\n if set(paths).issubset(self.budget_paths):\n headers = self.get_yaml_request_attr(yf, \"headers\")\n for header in headers:\n if header:\n valid = False\n\n method = self.get_yaml_request_attr(yf, \"method\")\n for m in method:\n if m != \"GET\":\n valid = False\n\n max_redirects = self.get_yaml_request_attr(yf, \"max-redirects\")\n for mr in max_redirects:\n if mr:\n valid = False\n\n redirects = self.get_yaml_request_attr(yf, \"redirects\")\n for rd in redirects:\n if rd:\n valid = False\n\n cookie_reuse = self.get_yaml_request_attr(yf, \"cookie-reuse\")\n for c in cookie_reuse:\n if c:\n valid = False\n\n if valid:\n collapsible_templates.append(str(yf))\n severity_gen = self.get_yaml_info_attr(yf, \"severity\")\n severity = next(severity_gen)\n if severity in severity_dict.keys():\n severity_dict[severity] += 1\n else:\n severity_dict[severity] = 1\n return collapsible_templates, severity_dict\n\n def parse_yaml(self, yamlfile):\n if yamlfile not in self._yaml_files:\n with open(yamlfile, \"r\") as stream:\n try:\n y = yaml.safe_load(stream)\n self._yaml_files[yamlfile] = y\n except yaml.YAMLError as e:\n self.parent.warning(f\"failed to load yaml file: {e}\")\n return {}\n return self._yaml_files[yamlfile]\n" }, { "path": "bbot/modules/oauth.py", "content": "from bbot.core.helpers.regexes import url_regexes\n\nfrom .base import BaseModule\n\n\nclass OAUTH(BaseModule):\n watched_events = [\"DNS_NAME\", \"URL_UNVERIFIED\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"affiliates\", \"subdomain-enum\", \"cloud-enum\", \"web-basic\", \"active\", \"safe\"]\n meta = {\n \"description\": \"Enumerate OAUTH and OpenID Connect services\",\n \"created_date\": \"2023-07-12\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"try_all\": False}\n options_desc = {\"try_all\": \"Check for OAUTH/IODC on every subdomain and URL.\"}\n\n in_scope_only = False\n scope_distance_modifier = 1\n _module_threads = 2\n\n async def setup(self):\n self.processed = set()\n self.regexes = list(url_regexes) + list(self.scan.dns_regexes)\n self.try_all = self.config.get(\"try_all\", False)\n return True\n\n async def filter_event(self, event):\n if event.module == self or any(t in event.tags for t in (\"target\", \"domain\", \"ms-auth-url\")):\n return True\n elif self.try_all and event.scope_distance == 0:\n return True\n return False\n\n async def handle_event(self, event):\n _, domain = self.helpers.split_domain(event.data)\n source_domain = getattr(event, \"source_domain\", domain)\n if not self.scan.in_scope(source_domain):\n return\n\n oidc_tasks = []\n if event.scope_distance == 0:\n domain_hash = hash(domain)\n if domain_hash not in self.processed:\n self.processed.add(domain_hash)\n oidc_tasks.append(self.helpers.create_task(self.getoidc(f\"https://login.windows.net/{domain}\")))\n\n if event.type == \"URL_UNVERIFIED\":\n url = event.data\n else:\n url = f\"https://{event.data}\"\n\n oauth_tasks = []\n if self.try_all or any(t in event.tags for t in (\"oauth-token-endpoint\",)):\n oauth_tasks.append(self.helpers.create_task(self.getoauth(url)))\n if self.try_all or any(t in event.tags for t in (\"ms-auth-url\",)):\n for u in self.url_and_base(url):\n oidc_tasks.append(self.helpers.create_task(self.getoidc(u)))\n\n for oidc_task in oidc_tasks:\n url, token_endpoint, oidc_results = await oidc_task\n if token_endpoint:\n finding_event = self.make_event(\n {\n \"description\": f\"OpenID Connect Endpoint (domain: {source_domain}) found at {url}\",\n \"host\": event.host,\n \"url\": url,\n },\n \"FINDING\",\n parent=event,\n )\n if finding_event:\n finding_event.source_domain = source_domain\n await self.emit_event(\n finding_event,\n context=f'{{module}} identified {{event.type}}: OpenID Connect Endpoint for \"{source_domain}\" at {url}',\n )\n url_event = self.make_event(\n token_endpoint, \"URL_UNVERIFIED\", parent=event, tags=[\"affiliate\", \"oauth-token-endpoint\"]\n )\n if url_event:\n url_event.source_domain = source_domain\n await self.emit_event(\n url_event,\n context=f'{{module}} identified OpenID Connect Endpoint for \"{source_domain}\" at {{event.type}}: {url}',\n )\n for result in oidc_results:\n if result not in (domain, event.data):\n event_type = \"URL_UNVERIFIED\" if self.helpers.is_url(result) else \"DNS_NAME\"\n await self.emit_event(\n result,\n event_type,\n parent=event,\n tags=[\"affiliate\"],\n context=f'{{module}} analyzed OpenID configuration for \"{source_domain}\" and found {{event.type}}: {{event.data}}',\n )\n\n for oauth_task in oauth_tasks:\n url = await oauth_task\n if url:\n description = f\"Potentially Sprayable OAUTH Endpoint (domain: {source_domain}) at {url}\"\n oauth_finding = self.make_event(\n {\n \"description\": description,\n \"host\": event.host,\n \"url\": url,\n },\n \"FINDING\",\n parent=event,\n )\n if oauth_finding:\n oauth_finding.source_domain = source_domain\n await self.emit_event(\n oauth_finding,\n context=f\"{{module}} identified {{event.type}}: {description}\",\n )\n\n def url_and_base(self, url):\n yield url\n parsed = self.helpers.urlparse(url)\n baseurl = f\"{parsed.scheme}://{parsed.netloc}/\"\n if baseurl != url:\n yield baseurl\n\n async def getoidc(self, url):\n results = set()\n if not url.endswith(\"openid-configuration\"):\n url = url.strip(\"/\") + \"/.well-known/openid-configuration\"\n url_hash = hash(\"OIDC:\" + url)\n token_endpoint = \"\"\n if url_hash not in self.processed:\n self.processed.add(url_hash)\n r = await self.helpers.request(url)\n if r is None:\n return url, token_endpoint, results\n try:\n json = r.json()\n except Exception:\n return url, token_endpoint, results\n if json and isinstance(json, dict):\n token_endpoint = json.get(\"token_endpoint\", \"\")\n for found in await self.helpers.re.search_dict_values(json, *self.regexes):\n results.add(found)\n results -= {token_endpoint}\n return url, token_endpoint, results\n\n async def getoauth(self, url):\n data = {\n \"grant_type\": \"authorization_code\",\n \"client_id\": \"xxx\",\n \"redirect_uri\": \"https://example.com\",\n \"code\": \"xxx\",\n \"client_secret\": \"xxx\",\n }\n url_hash = hash(\"OAUTH:\" + url)\n if url_hash not in self.processed:\n self.processed.add(url_hash)\n r = await self.helpers.request(url, method=\"POST\", data=data)\n if r is None:\n return\n if r.status_code in (400, 401):\n if \"json\" in r.headers.get(\"content-type\", \"\").lower():\n if any(x in r.text.lower() for x in (\"invalid_grant\", \"invalid_client\")):\n return url\n" }, { "path": "bbot/modules/otx.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass otx(subdomain_enum_apikey):\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n meta = {\n \"description\": \"Query otx.alienvault.com for subdomains\",\n \"created_date\": \"2022-08-24\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"OTX API key\"}\n\n base_url = \"https://otx.alienvault.com\"\n\n def prepare_api_request(self, url, kwargs):\n kwargs[\"headers\"][\"X-OTX-API-KEY\"] = self.api_key\n return url, kwargs\n\n def request_url(self, query):\n url = f\"{self.base_url}/api/v1/indicators/domain/{self.helpers.quote(query)}/passive_dns\"\n return self.api_request(url)\n\n async def parse_results(self, r, query):\n results = set()\n j = r.json()\n if isinstance(j, dict):\n for entry in j.get(\"passive_dns\", []):\n subdomain = entry.get(\"hostname\", \"\")\n if subdomain:\n results.add(subdomain)\n return results\n" }, { "path": "bbot/modules/output/__init__.py", "content": "" }, { "path": "bbot/modules/output/asset_inventory.py", "content": "import csv\nimport ipaddress\nfrom contextlib import suppress\n\nfrom .csv import CSV\nfrom bbot.core.helpers.misc import make_ip_type, is_ip, is_port, best_http_status\n\nseverity_map = {\n \"INFO\": 0,\n 0: \"N/A\",\n 1: \"LOW\",\n 2: \"MEDIUM\",\n 3: \"HIGH\",\n 4: \"CRITICAL\",\n \"N/A\": 0,\n \"LOW\": 1,\n \"MEDIUM\": 2,\n \"HIGH\": 3,\n \"CRITICAL\": 4,\n}\n\n\nclass asset_inventory(CSV):\n watched_events = [\n \"OPEN_TCP_PORT\",\n \"DNS_NAME\",\n \"URL\",\n \"FINDING\",\n \"VULNERABILITY\",\n \"TECHNOLOGY\",\n \"IP_ADDRESS\",\n \"WAF\",\n \"HTTP_RESPONSE\",\n ]\n produced_events = [\"IP_ADDRESS\", \"OPEN_TCP_PORT\"]\n meta = {\n \"description\": \"Merge hosts, open ports, technologies, findings, etc. into a single asset inventory CSV\",\n \"created_date\": \"2022-09-30\",\n \"author\": \"@liquidsec\",\n }\n options = {\"output_file\": \"\", \"use_previous\": False, \"recheck\": False, \"summary_netmask\": 16}\n options_desc = {\n \"output_file\": \"Set a custom output file\",\n \"use_previous\": \"Emit previous asset inventory as new events (use in conjunction with -n )\",\n \"recheck\": \"When use_previous=True, don't retain past details like open ports or findings. Instead, allow them to be rediscovered by the new scan\",\n \"summary_netmask\": \"Subnet mask to use when summarizing IP addresses at end of scan\",\n }\n\n header_row = [\n \"Host\",\n \"Provider\",\n \"IP (External)\",\n \"IP (Internal)\",\n \"Open Ports\",\n \"HTTP Status\",\n \"HTTP Title\",\n \"Risk Rating\",\n \"Findings\",\n \"Technologies\",\n \"WAF\",\n \"DNS Records\",\n ]\n filename = \"asset-inventory.csv\"\n\n async def setup(self):\n self.assets = {}\n self.use_previous = self.config.get(\"use_previous\", False)\n self.recheck = self.config.get(\"recheck\", False)\n self.summary_netmask = self.config.get(\"summary_netmask\", 16)\n self.emitted_contents = False\n self._ran_hooks = False\n ret = await super().setup()\n return ret\n\n async def filter_event(self, event):\n if event._internal:\n return False, \"event is internal\"\n if event.type not in self.watched_events:\n return False, \"event type is not in watched_events\"\n if not self.scan.in_scope(event.host):\n return False, \"event is not in scope\"\n if \"unresolved\" in event.tags:\n return False, \"event is unresolved\"\n return True, \"\"\n\n async def handle_event(self, event):\n if (await self.filter_event(event))[0]:\n hostkey = _make_hostkey(event.host, event.resolved_hosts)\n if hostkey not in self.assets:\n self.assets[hostkey] = Asset(event.host, self.recheck)\n self.assets[hostkey].absorb_event(event)\n\n async def report(self):\n stats = {}\n totals = {}\n\n def increment_stat(stat, value):\n try:\n totals[stat] += 1\n except KeyError:\n totals[stat] = 1\n if stat not in stats:\n stats[stat] = {}\n try:\n stats[stat][value] += 1\n except KeyError:\n stats[stat][value] = 1\n\n def sort_key(asset):\n host = str(asset.host)\n is_digit = False\n with suppress(IndexError):\n is_digit = host[0].isdigit()\n return (is_digit, host)\n\n for asset in sorted(self.assets.values(), key=sort_key):\n findings_and_vulns = asset.findings.union(asset.vulnerabilities)\n ports = getattr(asset, \"ports\", set())\n ports = [str(p) for p in sorted([int(p) for p in asset.ports])]\n ips_all = getattr(asset, \"ip_addresses\", [])\n ips_external = sorted([str(ip) for ip in [i for i in ips_all if not i.is_private]])\n ips_internal = sorted([str(ip) for ip in [i for i in ips_all if i.is_private]])\n host = self.helpers.make_ip_type(getattr(asset, \"host\", \"\"))\n if host and isinstance(host, str):\n _, domain = self.helpers.split_domain(host)\n if domain:\n increment_stat(\"Domains\", domain)\n for ip in ips_all:\n net = ipaddress.ip_network(f\"{ip}/{self.summary_netmask}\", strict=False)\n increment_stat(\"IP Addresses\", str(net))\n for port in ports:\n increment_stat(\"Open Ports\", port)\n row = {\n \"Host\": host,\n \"Provider\": getattr(asset, \"provider\", \"\"),\n \"IP (External)\": \", \".join(ips_external),\n \"IP (Internal)\": \", \".join(ips_internal),\n \"Open Ports\": \", \".join(ports),\n \"HTTP Status\": asset.http_status_full,\n \"HTTP Title\": str(getattr(asset, \"http_title\", \"\")),\n \"Risk Rating\": severity_map[getattr(asset, \"risk_rating\", \"\")],\n \"Findings\": \"\\n\".join(findings_and_vulns),\n \"Technologies\": \"\\n\".join(str(x) for x in getattr(asset, \"technologies\", set())),\n \"WAF\": getattr(asset, \"waf\", \"\"),\n \"DNS Records\": \", \".join(sorted([str(r) for r in getattr(asset, \"dns_records\", [])])),\n }\n row.update(asset.custom_fields)\n self.writerow(row)\n\n for header in (\"Domains\", \"IP Addresses\", \"Open Ports\"):\n table_header = [header, \"\"]\n if header in stats:\n table = []\n stats_sorted = sorted(stats[header].items(), key=lambda x: x[-1], reverse=True)\n total = totals[header]\n for k, v in stats_sorted:\n table.append([str(k), f\"{v:,}/{total} ({v / total * 100:.1f}%)\"])\n self.log_table(table, table_header, table_name=f\"asset-inventory-{header}\")\n\n if self._file is not None:\n self.info(f\"Saved asset-inventory output to {self.output_file}\")\n\n async def finish(self):\n if self.use_previous and not self.emitted_contents:\n self.emitted_contents = True\n if self.output_file.is_file():\n self.info(f\"Emitting previous results from {self.output_file}\")\n with open(self.output_file, newline=\"\") as f:\n c = csv.DictReader(f)\n for row in c:\n # yield to event loop to make sure we don't hold up the scan\n await self.helpers.sleep(0)\n host = row.get(\"Host\", \"\").strip()\n ips = row.get(\"IP (External)\", \"\") + \",\" + row.get(\"IP (Internal)\", \"\")\n if not host or not ips:\n continue\n hostkey = _make_hostkey(host, ips)\n asset = self.assets.get(hostkey, None)\n if asset is None:\n asset = Asset(host, self.recheck)\n self.assets[hostkey] = asset\n asset.absorb_csv_row(row)\n self.add_custom_headers(list(asset.custom_fields))\n if not is_ip(asset.host):\n host_event = self.make_event(\n asset.host, \"DNS_NAME\", parent=self.scan.root_event, raise_error=True\n )\n await self.emit_event(\n host_event, context=\"{module} emitted previous result: {event.type}: {event.data}\"\n )\n for port in asset.ports:\n netloc = self.helpers.make_netloc(asset.host, port)\n open_port_event = self.make_event(netloc, \"OPEN_TCP_PORT\", parent=host_event)\n if open_port_event:\n await self.emit_event(\n open_port_event,\n context=\"{module} emitted previous result: {event.type}: {event.data}\",\n )\n else:\n for ip in asset.ip_addresses:\n ip_event = self.make_event(\n ip, \"IP_ADDRESS\", parent=self.scan.root_event, raise_error=True\n )\n await self.emit_event(\n ip_event, context=\"{module} emitted previous result: {event.type}: {event.data}\"\n )\n for port in asset.ports:\n netloc = self.helpers.make_netloc(ip, port)\n open_port_event = self.make_event(netloc, \"OPEN_TCP_PORT\", parent=ip_event)\n if open_port_event:\n await self.emit_event(\n open_port_event,\n context=\"{module} emitted previous result: {event.type}: {event.data}\",\n )\n else:\n self.warning(\n f\"use_previous=True was set but no previous asset inventory was found at {self.output_file}\"\n )\n else:\n self._run_hooks()\n\n def _run_hooks(self):\n \"\"\"\n modules can use self.asset_inventory_hook() to add custom functionality to asset_inventory\n the asset inventory module is passed in as the first argument to the method.\n \"\"\"\n if not self._ran_hooks:\n self._ran_hooks = True\n for module in self.scan.modules.values():\n hook = getattr(module, \"asset_inventory_hook\", None)\n if hook is not None and callable(hook):\n hook(self)\n\n\nclass Asset:\n def __init__(self, host, recheck):\n self.host = host\n self.ip_addresses = set()\n self.dns_records = set()\n self.ports = set()\n self.findings = set()\n self.vulnerabilities = set()\n self.status = \"UNKNOWN\"\n self.risk_rating = 0\n self.provider = \"\"\n self.waf = \"\"\n self.technologies = set()\n self.custom_fields = {}\n self.http_status = 0\n self.http_title = \"\"\n self.redirect_location = \"\"\n self.recheck = recheck\n\n def absorb_csv_row(self, row):\n # host\n host = make_ip_type(row.get(\"Host\", \"\").strip())\n if host and not is_ip(host):\n self.host = host\n # ips\n self.ip_addresses = set(_make_ip_list(row.get(\"IP (External)\", \"\")))\n self.ip_addresses.update(set(_make_ip_list(row.get(\"IP (Internal)\", \"\"))))\n # If user requests a recheck dont import the following fields to force them to be rechecked\n if not self.recheck:\n # ports\n ports = [i.strip() for i in row.get(\"Open Ports\", \"\").split(\",\")]\n self.ports.update({i for i in ports if i and is_port(i)})\n # findings\n findings = [i.strip() for i in row.get(\"Findings\", \"\").splitlines()]\n self.findings.update({i for i in findings if i})\n # technologies\n technologies = [i.strip() for i in row.get(\"Technologies\", \"\").splitlines()]\n self.technologies.update({i for i in technologies if i})\n # risk rating\n risk_rating = row.get(\"Risk Rating\", \"\").strip()\n if risk_rating and risk_rating.isdigit() and int(risk_rating) > self.risk_rating:\n self.risk_rating = int(risk_rating)\n # provider\n provider = row.get(\"Provider\", \"\").strip()\n if provider:\n self.provider = provider\n # custom fields\n for k, v in row.items():\n v = str(v)\n # update the custom field if it doesn't clash with our main fields\n # and if the new value isn't blank\n if v and k not in asset_inventory.header_row:\n self.custom_fields[k] = v\n\n def absorb_event(self, event):\n if not is_ip(event.host):\n self.host = event.host\n\n dns_children = getattr(event, \"_dns_children\", {})\n for rdtype, records in sorted(dns_children.items(), key=lambda x: x[0]):\n for record in sorted([str(r) for r in records]):\n self.dns_records.add(f\"{rdtype}:{record}\")\n\n http_status = getattr(event, \"http_status\", 0)\n update_http_status = bool(http_status) and best_http_status(http_status, self.http_status) == http_status\n if update_http_status:\n self.http_status = http_status\n if str(http_status).startswith(\"3\"):\n if event.type == \"HTTP_RESPONSE\":\n redirect_location = getattr(event, \"redirect_location\", \"\")\n if redirect_location:\n self.redirect_location = redirect_location\n else:\n self.redirect_location = \"\"\n\n if event.resolved_hosts:\n self.ip_addresses.update(set(_make_ip_list(event.resolved_hosts)))\n\n if event.port:\n self.ports.add(str(event.port))\n\n if event.type == \"FINDING\":\n location = event.data.get(\"url\", event.data.get(\"host\", \"\"))\n if location:\n self.findings.add(f\"{location}:{event.data['description']}\")\n\n if event.type == \"VULNERABILITY\":\n location = event.data.get(\"url\", event.data.get(\"host\", \"\"))\n if location:\n self.findings.add(f\"{location}:{event.data['description']}:{event.data['severity']}\")\n severity_int = severity_map.get(event.data.get(\"severity\", \"N/A\"), 0)\n if severity_int > self.risk_rating:\n self.risk_rating = severity_int\n\n if event.type == \"TECHNOLOGY\":\n self.technologies.add(event.data[\"technology\"])\n\n if event.type == \"WAF\":\n if waf := event.data.get(\"waf\", \"\"):\n if update_http_status or not self.waf:\n self.waf = waf\n\n if event.type == \"HTTP_RESPONSE\":\n if title := event.data.get(\"title\", \"\"):\n if update_http_status or not self.http_title:\n self.http_title = title\n\n for tag in event.tags:\n if tag.startswith(\"cdn-\") or tag.startswith(\"cloud-\"):\n self.provider = tag\n break\n\n @property\n def hostkey(self):\n return _make_hostkey(self.host, self.ip_addresses)\n\n @property\n def http_status_full(self):\n return str(self.http_status) + (f\" -> {self.redirect_location}\" if self.redirect_location else \"\")\n\n\ndef _make_hostkey(host, ips):\n \"\"\"\n We handle public and private IPs differently\n If the IPs are public, we dedupe by host\n If they're private, we dedupe by the IPs themselves\n \"\"\"\n ips = _make_ip_list(ips)\n is_private = ips and all(is_ip(i) and i.is_private for i in ips)\n if is_private:\n return \",\".join(sorted([str(i) for i in ips]))\n return str(host)\n\n\ndef _make_ip_list(ips):\n if isinstance(ips, str):\n ips = [i.strip() for i in ips.split(\",\")]\n ips = [make_ip_type(i) for i in ips if i and is_ip(i)]\n return ips\n" }, { "path": "bbot/modules/output/base.py", "content": "import logging\nfrom pathlib import Path\nfrom bbot.modules.base import BaseModule\n\n\nclass BaseOutputModule(BaseModule):\n accept_dupes = True\n _type = \"output\"\n scope_distance_modifier = None\n _stats_exclude = True\n _shuffle_incoming_queue = False\n\n def human_event_str(self, event):\n event_type = f\"[{event.type}]\"\n event_tags = \"\"\n if getattr(event, \"tags\", []):\n event_tags = f\"\\t({', '.join(sorted(getattr(event, 'tags', [])))})\"\n event_str = f\"{event_type:<20}\\t{event.data_human}\\t{event.module_sequence}{event_tags}\"\n return event_str\n\n def _event_precheck(self, event):\n reason = \"precheck succeeded\"\n # special signal event types\n if event.type in (\"FINISHED\",):\n return True, \"its type is FINISHED\"\n if self.errored:\n return False, \"module is in error state\"\n # exclude non-watched types\n if not any(t in self.get_watched_events() for t in (\"*\", event.type)):\n return False, \"its type is not in watched_events\"\n if self.target_only:\n if \"target\" not in event.tags:\n return False, \"it did not meet target_only filter criteria\"\n\n ### begin output-module specific ###\n\n # force-output certain events to the graph\n if self._is_graph_important(event):\n return True, \"event is critical to the graph\"\n\n # omit certain event types\n if event._omit:\n if event.type in self.get_watched_events():\n reason = \"its type is explicitly in watched_events\"\n self.debug(f\"Allowing omitted event: {event} because {reason}\")\n else:\n return False, \"its type is omitted in the config\"\n\n # internal events like those from speculate, ipneighbor\n # or events that are over our report distance\n if event._internal:\n return False, \"event is internal and output modules don't accept internal events\"\n\n return True, reason\n\n async def _event_postcheck(self, event):\n acceptable, reason = await super()._event_postcheck(event)\n if acceptable and not event._stats_recorded and event.type not in (\"FINISHED\",):\n event._stats_recorded = True\n self.scan.stats.event_produced(event)\n return acceptable, reason\n\n def is_incoming_duplicate(self, event, add=False):\n is_incoming_duplicate, reason = super().is_incoming_duplicate(event, add=add)\n # make exception for graph-important events\n if self._is_graph_important(event):\n return False, \"event is graph-important\"\n return is_incoming_duplicate, reason\n\n def _prep_output_dir(self, filename):\n self.output_file = self.config.get(\"output_file\", \"\")\n if self.output_file:\n self.output_file = Path(self.output_file)\n else:\n self.output_file = self.scan.home / str(filename)\n self.helpers.mkdir(self.output_file.parent)\n self._file = None\n\n def _scope_distance_check(self, event):\n return True, \"\"\n\n @property\n def file(self):\n if getattr(self, \"_file\", None) is None:\n self._file = open(self.output_file, mode=\"a\")\n return self._file\n\n @property\n def log(self):\n if self._log is None:\n self._log = logging.getLogger(f\"bbot.modules.output.{self.name}\")\n return self._log\n" }, { "path": "bbot/modules/output/csv.py", "content": "import csv\nfrom contextlib import suppress\n\nfrom bbot.modules.output.base import BaseOutputModule\n\n\nclass CSV(BaseOutputModule):\n watched_events = [\"*\"]\n meta = {\"description\": \"Output to CSV\", \"created_date\": \"2022-04-07\", \"author\": \"@TheTechromancer\"}\n options = {\"output_file\": \"\"}\n options_desc = {\"output_file\": \"Output to CSV file\"}\n\n header_row = [\n \"Event type\",\n \"Event data\",\n \"IP Address\",\n \"Source Module\",\n \"Scope Distance\",\n \"Event Tags\",\n \"Discovery Path\",\n ]\n filename = \"output.csv\"\n accept_dupes = False\n\n async def setup(self):\n self.custom_headers = []\n self._headers_set = set()\n self._writer = None\n self._prep_output_dir(self.filename)\n return True\n\n @property\n def writer(self):\n if self._writer is None:\n self._writer = csv.DictWriter(self.file, fieldnames=self.fieldnames)\n self._writer.writeheader()\n return self._writer\n\n @property\n def file(self):\n if self._file is None:\n if self.output_file.is_file():\n self.helpers.backup_file(self.output_file)\n self._file = open(self.output_file, mode=\"a\", newline=\"\")\n return self._file\n\n @property\n def fieldnames(self):\n return self.header_row + list(self.custom_headers)\n\n def writerow(self, row):\n self.writer.writerow(row)\n self.file.flush()\n\n async def handle_event(self, event):\n # [\"Event type\", \"Event data\", \"IP Address\", \"Source Module\", \"Scope Distance\", \"Event Tags\"]\n discovery_path = getattr(event, \"discovery_path\", [])\n self.writerow(\n {\n \"Event type\": getattr(event, \"type\", \"\"),\n \"Event data\": getattr(event, \"data\", \"\"),\n \"IP Address\": \",\".join(\n str(x) for x in getattr(event, \"resolved_hosts\", set()) if self.helpers.is_ip(x)\n ),\n \"Source Module\": str(getattr(event, \"module_sequence\", \"\")),\n \"Scope Distance\": str(getattr(event, \"scope_distance\", \"\")),\n \"Event Tags\": \",\".join(sorted(getattr(event, \"tags\", []))),\n \"Discovery Path\": \" --> \".join(discovery_path),\n }\n )\n\n async def cleanup(self):\n if getattr(self, \"_file\", None) is not None:\n with suppress(Exception):\n self.file.close()\n\n async def report(self):\n if self._file is not None:\n self.info(f\"Saved CSV output to {self.output_file}\")\n\n def add_custom_headers(self, headers):\n if isinstance(headers, str):\n headers = [headers]\n for header in headers:\n if header not in self._headers_set:\n self._headers_set.add(header)\n self.custom_headers.append(header)\n" }, { "path": "bbot/modules/output/discord.py", "content": "from bbot.modules.templates.webhook import WebhookOutputModule\n\n\nclass Discord(WebhookOutputModule):\n watched_events = [\"*\"]\n meta = {\n \"description\": \"Message a Discord channel when certain events are encountered\",\n \"created_date\": \"2023-08-14\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"webhook_url\": \"\", \"event_types\": [\"VULNERABILITY\", \"FINDING\"], \"min_severity\": \"LOW\", \"retries\": 10}\n options_desc = {\n \"webhook_url\": \"Discord webhook URL\",\n \"event_types\": \"Types of events to send\",\n \"min_severity\": \"Only allow VULNERABILITY events of this severity or higher\",\n \"retries\": \"Number of times to retry sending the message before skipping the event\",\n }\n" }, { "path": "bbot/modules/output/emails.py", "content": "from bbot.modules.output.txt import TXT\nfrom bbot.modules.base import BaseModule\n\n\nclass Emails(TXT):\n watched_events = [\"EMAIL_ADDRESS\"]\n flags = [\"email-enum\"]\n meta = {\n \"description\": \"Output any email addresses found belonging to the target domain\",\n \"created_date\": \"2023-12-23\",\n \"author\": \"@domwhewell-sage\",\n }\n options = {\"output_file\": \"\"}\n options_desc = {\"output_file\": \"Output to file\"}\n in_scope_only = True\n accept_dupes = False\n\n output_filename = \"emails.txt\"\n\n async def setup(self):\n self.emails_written = 0\n return await super().setup()\n\n def _scope_distance_check(self, event):\n return BaseModule._scope_distance_check(self, event)\n\n async def handle_event(self, event):\n if self.file is not None:\n self.emails_written += 1\n self.file.write(f\"{event.data}\\n\")\n self.file.flush()\n\n async def report(self):\n if getattr(self, \"_file\", None) is not None:\n self.info(f\"Saved {self.emails_written:,} email addresses to {self.output_file}\")\n" }, { "path": "bbot/modules/output/http.py", "content": "from bbot.modules.output.base import BaseOutputModule\n\n\nclass HTTP(BaseOutputModule):\n watched_events = [\"*\"]\n meta = {\n \"description\": \"Send every event to a custom URL via a web request\",\n \"created_date\": \"2022-04-13\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\n \"url\": \"\",\n \"method\": \"POST\",\n \"bearer\": \"\",\n \"username\": \"\",\n \"password\": \"\",\n \"timeout\": 10,\n \"siem_friendly\": False,\n }\n options_desc = {\n \"url\": \"Web URL\",\n \"method\": \"HTTP method\",\n \"bearer\": \"Authorization Bearer token\",\n \"username\": \"Username (basic auth)\",\n \"password\": \"Password (basic auth)\",\n \"timeout\": \"HTTP timeout\",\n \"siem_friendly\": \"Format JSON in a SIEM-friendly way for ingestion into Elastic, Splunk, etc.\",\n }\n\n async def setup(self):\n self.url = self.config.get(\"url\", \"\")\n self.method = self.config.get(\"method\", \"POST\")\n self.timeout = self.config.get(\"timeout\", 10)\n self.siem_friendly = self.config.get(\"siem_friendly\", False)\n self.headers = {}\n bearer = self.config.get(\"bearer\", \"\")\n if bearer:\n self.headers[\"Authorization\"] = f\"Bearer {bearer}\"\n username = self.config.get(\"username\", \"\")\n password = self.config.get(\"password\", \"\")\n self.auth = None\n if username:\n self.auth = (username, password)\n if not self.url:\n self.warning(\"Must set URL\")\n return False\n if not self.method:\n self.warning(\"Must set HTTP method\")\n return False\n return True\n\n async def handle_event(self, event):\n while 1:\n response = await self.helpers.request(\n url=self.url,\n method=self.method,\n auth=self.auth,\n headers=self.headers,\n json=event.json(siem_friendly=self.siem_friendly),\n )\n is_success = False if response is None else response.is_success\n if not is_success:\n status_code = getattr(response, \"status_code\", 0)\n self.warning(f\"Error sending {event} (HTTP status code: {status_code}), retrying...\")\n body = getattr(response, \"text\", \"\")\n self.debug(body)\n if status_code == 429:\n sleep_interval = 10\n else:\n sleep_interval = 1\n await self.helpers.sleep(sleep_interval)\n continue\n break\n" }, { "path": "bbot/modules/output/json.py", "content": "import json\nfrom contextlib import suppress\n\nfrom bbot.modules.output.base import BaseOutputModule\n\n\nclass JSON(BaseOutputModule):\n watched_events = [\"*\"]\n meta = {\n \"description\": \"Output to Newline-Delimited JSON (NDJSON)\",\n \"created_date\": \"2022-04-07\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"output_file\": \"\", \"siem_friendly\": False}\n options_desc = {\n \"output_file\": \"Output to file\",\n \"siem_friendly\": \"Output JSON in a SIEM-friendly format for ingestion into Elastic, Splunk, etc.\",\n }\n _preserve_graph = True\n\n async def setup(self):\n self._prep_output_dir(\"output.json\")\n self.siem_friendly = self.config.get(\"siem_friendly\", False)\n return True\n\n async def handle_event(self, event):\n event_json = event.json(siem_friendly=self.siem_friendly)\n event_str = json.dumps(event_json)\n if self.file is not None:\n self.file.write(event_str + \"\\n\")\n self.file.flush()\n\n async def cleanup(self):\n if getattr(self, \"_file\", None) is not None:\n with suppress(Exception):\n self.file.close()\n\n async def report(self):\n if self._file is not None:\n self.info(f\"Saved JSON output to {self.output_file}\")\n" }, { "path": "bbot/modules/output/mysql.py", "content": "from bbot.modules.templates.sql import SQLTemplate\n\n\nclass MySQL(SQLTemplate):\n watched_events = [\"*\"]\n meta = {\n \"description\": \"Output scan data to a MySQL database\",\n \"created_date\": \"2024-11-13\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\n \"username\": \"root\",\n \"password\": \"bbotislife\",\n \"host\": \"localhost\",\n \"port\": 3306,\n \"database\": \"bbot\",\n }\n options_desc = {\n \"username\": \"The username to connect to MySQL\",\n \"password\": \"The password to connect to MySQL\",\n \"host\": \"The server running MySQL\",\n \"port\": \"The port to connect to MySQL\",\n \"database\": \"The database name to connect to\",\n }\n deps_pip = [\"sqlmodel\", \"aiomysql\"]\n protocol = \"mysql+aiomysql\"\n\n async def create_database(self):\n from sqlalchemy import text\n from sqlalchemy.ext.asyncio import create_async_engine\n\n # Create the engine for the initial connection to the server\n initial_engine = create_async_engine(self.connection_string().rsplit(\"/\", 1)[0])\n\n async with initial_engine.connect() as conn:\n # Check if the database exists\n result = await conn.execute(text(f\"SHOW DATABASES LIKE '{self.database}'\"))\n database_exists = result.scalar() is not None\n\n # Create the database if it does not exist\n if not database_exists:\n # Use aiomysql directly to create the database\n import aiomysql\n\n raw_conn = await aiomysql.connect(\n user=self.username,\n password=self.password,\n host=self.host,\n port=self.port,\n )\n try:\n async with raw_conn.cursor() as cursor:\n await cursor.execute(f\"CREATE DATABASE {self.database}\")\n finally:\n await raw_conn.ensure_closed()\n" }, { "path": "bbot/modules/output/neo4j.py", "content": "import json\nimport logging\nfrom contextlib import suppress\nfrom neo4j import AsyncGraphDatabase\n\nfrom bbot.modules.output.base import BaseOutputModule\n\n\n# silence annoying neo4j logger\nlogging.getLogger(\"neo4j\").setLevel(logging.CRITICAL)\n\n\nclass neo4j(BaseOutputModule):\n \"\"\"\n # start Neo4j in the background with docker\n docker run -d -p 7687:7687 -p 7474:7474 -v \"$(pwd)/neo4j/:/data/\" -e NEO4J_AUTH=neo4j/bbotislife neo4j\n\n # view all running docker containers\n > docker ps\n\n # view all docker containers\n > docker ps -a\n\n # stop a docker container\n > docker stop \n\n # remove a docker container\n > docker remove \n\n # start a stopped container\n > docker start \n \"\"\"\n\n watched_events = [\"*\"]\n meta = {\"description\": \"Output to Neo4j\", \"created_date\": \"2022-04-07\", \"author\": \"@TheTechromancer\"}\n options = {\"uri\": \"bolt://localhost:7687\", \"username\": \"neo4j\", \"password\": \"bbotislife\"}\n options_desc = {\n \"uri\": \"Neo4j server + port\",\n \"username\": \"Neo4j username\",\n \"password\": \"Neo4j password\",\n }\n deps_pip = [\"neo4j\"]\n _batch_size = 500\n _preserve_graph = True\n\n async def setup(self):\n try:\n self.driver = AsyncGraphDatabase.driver(\n uri=self.config.get(\"uri\", self.options[\"uri\"]),\n auth=(\n self.config.get(\"username\", self.options[\"username\"]),\n self.config.get(\"password\", self.options[\"password\"]),\n ),\n )\n self.session = self.driver.session()\n await self.session.run(\"Match () Return 1 Limit 1\")\n except Exception as e:\n return False, f\"Error setting up Neo4j: {e}\"\n return True\n\n async def handle_batch(self, *all_events):\n # group events by type, since cypher doesn't allow dynamic labels\n events_by_type = {}\n parents_by_type = {}\n relationships = []\n for event in all_events:\n parent = event.get_parent()\n try:\n events_by_type[event.type].append(event)\n except KeyError:\n events_by_type[event.type] = [event]\n try:\n parents_by_type[parent.type].append(parent)\n except KeyError:\n parents_by_type[parent.type] = [parent]\n\n module = str(event.module)\n timestamp = event.timestamp\n relationships.append((parent, module, timestamp, event))\n\n all_ids = {}\n for event_type, events in events_by_type.items():\n self.debug(f\"{len(events):,} events of type {event_type}\")\n all_ids.update(await self.merge_events(events, event_type))\n for event_type, parents in parents_by_type.items():\n self.debug(f\"{len(parents):,} parents of type {event_type}\")\n all_ids.update(await self.merge_events(parents, event_type, id_only=True))\n\n rel_ids = []\n for parent, module, timestamp, event in relationships:\n try:\n src_id = all_ids[parent.id]\n dst_id = all_ids[event.id]\n except KeyError as e:\n self.error(f'Error \"{e}\" correlating {parent.id}:{parent.data} --> {event.id}:{event.data}')\n continue\n rel_ids.append((src_id, module, timestamp, dst_id))\n\n await self.merge_relationships(rel_ids)\n\n async def merge_events(self, events, event_type, id_only=False):\n if id_only:\n insert_data = [{\"data\": str(e.data), \"type\": e.type, \"id\": e.id} for e in events]\n else:\n insert_data = []\n for e in events:\n event_json = e.json(mode=\"graph\")\n # we pop the timestamp because it belongs on the relationship\n event_json.pop(\"timestamp\")\n # nested data types aren't supported in neo4j\n for key in (\"dns_children\", \"discovery_path\"):\n if key in event_json:\n event_json[key] = json.dumps(event_json[key])\n insert_data.append(event_json)\n\n cypher = f\"\"\"UNWIND $events AS event\n MERGE (_:{event_type} {{ id: event.id }})\n SET _ += properties(event)\n RETURN event.data as event_data, event.id as event_id, elementId(_) as neo4j_id\"\"\"\n neo4j_ids = {}\n # insert events\n try:\n results = await self.session.run(cypher, events=insert_data)\n # get Neo4j ids\n for result in await results.data():\n event_id = result[\"event_id\"]\n neo4j_id = result[\"neo4j_id\"]\n neo4j_ids[event_id] = neo4j_id\n except Exception as e:\n self.error(f\"Error inserting Neo4j nodes (label:{event_type}): {e}\")\n self.trace(insert_data)\n self.trace(cypher)\n return neo4j_ids\n\n async def merge_relationships(self, relationships):\n rels_by_module = {}\n # group by module\n for src_id, module, timestamp, dst_id in relationships:\n data = {\"src_id\": src_id, \"timestamp\": timestamp, \"dst_id\": dst_id}\n try:\n rels_by_module[module].append(data)\n except KeyError:\n rels_by_module[module] = [data]\n\n for module, rels in rels_by_module.items():\n self.debug(f\"{len(rels):,} relationships of type {module}\")\n cypher = f\"\"\"\n UNWIND $rels AS rel\n MATCH (a) WHERE elementId(a) = rel.src_id\n MATCH (b) WHERE elementId(b) = rel.dst_id\n MERGE (a)-[_:{module}]->(b)\n SET _.timestamp = rel.timestamp\"\"\"\n try:\n await self.session.run(cypher, rels=rels)\n except Exception as e:\n self.error(f\"Error inserting Neo4j relationship (label:{module}): {e}\")\n self.trace(cypher)\n\n async def cleanup(self):\n with suppress(Exception):\n await self.session.close()\n with suppress(Exception):\n await self.driver.close()\n" }, { "path": "bbot/modules/output/nmap_xml.py", "content": "import sys\nfrom xml.dom import minidom\nfrom datetime import datetime\nfrom xml.etree.ElementTree import Element, SubElement, tostring\n\nfrom bbot import __version__\nfrom bbot.modules.output.base import BaseOutputModule\n\n\nclass NmapHost:\n __slots__ = [\"hostnames\", \"open_ports\"]\n\n def __init__(self):\n self.hostnames = set()\n # a dict of {port: {protocol: banner}}\n self.open_ports = dict()\n\n\nclass Nmap_XML(BaseOutputModule):\n watched_events = [\"OPEN_TCP_PORT\", \"DNS_NAME\", \"IP_ADDRESS\", \"PROTOCOL\", \"HTTP_RESPONSE\"]\n meta = {\"description\": \"Output to Nmap XML\", \"created_date\": \"2024-11-16\", \"author\": \"@TheTechromancer\"}\n output_filename = \"output.nmap.xml\"\n in_scope_only = True\n\n async def setup(self):\n self.hosts = {}\n self._prep_output_dir(self.output_filename)\n return True\n\n async def handle_event(self, event):\n event_host = event.host\n\n # we always record by IP\n ips = []\n for ip in event.resolved_hosts:\n try:\n ips.append(self.helpers.make_ip_type(ip))\n except ValueError:\n continue\n if not ips and self.helpers.is_ip(event_host):\n ips = [event_host]\n\n for ip in ips:\n try:\n nmap_host = self.hosts[ip]\n except KeyError:\n nmap_host = NmapHost()\n self.hosts[ip] = nmap_host\n\n event_port = getattr(event, \"port\", None)\n if event.type == \"OPEN_TCP_PORT\":\n if event_port not in nmap_host.open_ports:\n nmap_host.open_ports[event.port] = {}\n elif event.type in (\"PROTOCOL\", \"HTTP_RESPONSE\"):\n if event_port is not None:\n try:\n existing_services = nmap_host.open_ports[event.port]\n except KeyError:\n existing_services = {}\n nmap_host.open_ports[event.port] = existing_services\n if event.type == \"PROTOCOL\":\n protocol = event.data[\"protocol\"].lower()\n banner = event.data.get(\"banner\", None)\n elif event.type == \"HTTP_RESPONSE\":\n protocol = event.parsed_url.scheme.lower()\n banner = event.http_title\n if protocol not in existing_services:\n existing_services[protocol] = banner\n\n if self.helpers.is_ip(event_host):\n if str(event.module) == \"PTR\":\n nmap_host.hostnames.add(event.parent.data)\n else:\n nmap_host.hostnames.add(event_host)\n\n async def report(self):\n scan_start_time = str(int(self.scan.start_time.timestamp()))\n scan_start_time_str = self.scan.start_time.strftime(\"%a %b %d %H:%M:%S %Y\")\n scan_end_time = datetime.now()\n scan_end_time_str = scan_end_time.strftime(\"%a %b %d %H:%M:%S %Y\")\n scan_end_time_timestamp = str(scan_end_time.timestamp())\n scan_duration = scan_end_time - self.scan.start_time\n num_hosts_up = len(self.hosts)\n\n # Create the root element\n nmaprun = Element(\n \"nmaprun\",\n {\n \"scanner\": \"bbot\",\n \"args\": \" \".join(sys.argv),\n \"start\": scan_start_time,\n \"startstr\": scan_start_time_str,\n \"version\": str(__version__),\n \"xmloutputversion\": \"1.05\",\n },\n )\n\n ports_scanned = []\n speculate_module = self.scan.modules.get(\"speculate\", None)\n if speculate_module is not None:\n ports_scanned = speculate_module.ports\n portscan_module = self.scan.modules.get(\"portscan\", None)\n if portscan_module is not None:\n ports_scanned = self.helpers.parse_port_string(str(portscan_module.ports))\n num_ports_scanned = len(sorted(ports_scanned))\n ports_scanned = \",\".join(str(x) for x in sorted(ports_scanned))\n\n # Add scaninfo\n SubElement(\n nmaprun,\n \"scaninfo\",\n {\"type\": \"syn\", \"protocol\": \"tcp\", \"numservices\": str(num_ports_scanned), \"services\": ports_scanned},\n )\n\n # Add host information\n for ip, nmap_host in self.hosts.items():\n hostnames = sorted(nmap_host.hostnames)\n ports = sorted(nmap_host.open_ports)\n\n host_elem = SubElement(nmaprun, \"host\")\n SubElement(host_elem, \"status\", {\"state\": \"up\", \"reason\": \"user-set\", \"reason_ttl\": \"0\"})\n SubElement(host_elem, \"address\", {\"addr\": str(ip), \"addrtype\": f\"ipv{ip.version}\"})\n\n if hostnames:\n hostnames_elem = SubElement(host_elem, \"hostnames\")\n for hostname in hostnames:\n SubElement(hostnames_elem, \"hostname\", {\"name\": hostname, \"type\": \"user\"})\n\n ports = SubElement(host_elem, \"ports\")\n for port, protocols in nmap_host.open_ports.items():\n port_elem = SubElement(ports, \"port\", {\"protocol\": \"tcp\", \"portid\": str(port)})\n SubElement(port_elem, \"state\", {\"state\": \"open\", \"reason\": \"syn-ack\", \"reason_ttl\": \"0\"})\n # \n for protocol, banner in protocols.items():\n attrs = {\"name\": protocol, \"method\": \"probed\", \"conf\": \"10\"}\n if banner is not None:\n attrs[\"product\"] = banner\n attrs[\"extrainfo\"] = banner\n SubElement(port_elem, \"service\", attrs)\n\n # Add runstats\n runstats = SubElement(nmaprun, \"runstats\")\n SubElement(\n runstats,\n \"finished\",\n {\n \"time\": scan_end_time_timestamp,\n \"timestr\": scan_end_time_str,\n \"summary\": f\"BBOT done at {scan_end_time_str}; {num_hosts_up} scanned in {scan_duration} seconds\",\n \"elapsed\": str(scan_duration.total_seconds()),\n \"exit\": \"success\",\n },\n )\n SubElement(runstats, \"hosts\", {\"up\": str(num_hosts_up), \"down\": \"0\", \"total\": str(num_hosts_up)})\n\n # make backup of the file\n self.helpers.backup_file(self.output_file)\n\n # Pretty-format the XML\n rough_string = tostring(nmaprun, encoding=\"utf-8\")\n reparsed = minidom.parseString(rough_string)\n\n # Create a new document with the doctype\n doctype = minidom.DocumentType(\"nmaprun\")\n reparsed.insertBefore(doctype, reparsed.documentElement)\n\n pretty_xml = reparsed.toprettyxml(indent=\" \")\n\n with open(self.output_file, \"w\") as f:\n f.write(pretty_xml)\n self.info(f\"Saved Nmap XML output to {self.output_file}\")\n" }, { "path": "bbot/modules/output/postgres.py", "content": "from bbot.modules.templates.sql import SQLTemplate\n\n\nclass Postgres(SQLTemplate):\n watched_events = [\"*\"]\n meta = {\n \"description\": \"Output scan data to a SQLite database\",\n \"created_date\": \"2024-11-08\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\n \"username\": \"postgres\",\n \"password\": \"bbotislife\",\n \"host\": \"localhost\",\n \"port\": 5432,\n \"database\": \"bbot\",\n }\n options_desc = {\n \"username\": \"The username to connect to Postgres\",\n \"password\": \"The password to connect to Postgres\",\n \"host\": \"The server running Postgres\",\n \"port\": \"The port to connect to Postgres\",\n \"database\": \"The database name to connect to\",\n }\n deps_pip = [\"sqlmodel\", \"asyncpg\"]\n protocol = \"postgresql+asyncpg\"\n\n async def create_database(self):\n import asyncpg\n from sqlalchemy import text\n from sqlalchemy.ext.asyncio import create_async_engine\n\n # Create the engine for the initial connection to the server\n initial_engine = create_async_engine(self.connection_string().rsplit(\"/\", 1)[0])\n\n async with initial_engine.connect() as conn:\n # Check if the database exists\n result = await conn.execute(text(f\"SELECT 1 FROM pg_database WHERE datname = '{self.database}'\"))\n database_exists = result.scalar() is not None\n\n # Create the database if it does not exist\n if not database_exists:\n # Use asyncpg directly to create the database\n raw_conn = await asyncpg.connect(\n user=self.username,\n password=self.password,\n host=self.host,\n port=self.port,\n )\n try:\n await raw_conn.execute(f\"CREATE DATABASE {self.database}\")\n finally:\n await raw_conn.close()\n" }, { "path": "bbot/modules/output/python.py", "content": "from bbot.modules.output.base import BaseOutputModule\n\n\nclass python(BaseOutputModule):\n watched_events = [\"*\"]\n meta = {\"description\": \"Output via Python API\", \"created_date\": \"2022-09-13\", \"author\": \"@TheTechromancer\"}\n\n async def _worker(self):\n pass\n" }, { "path": "bbot/modules/output/slack.py", "content": "import yaml\n\nfrom bbot.modules.templates.webhook import WebhookOutputModule\n\n\nclass Slack(WebhookOutputModule):\n watched_events = [\"*\"]\n meta = {\n \"description\": \"Message a Slack channel when certain events are encountered\",\n \"created_date\": \"2023-08-14\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"webhook_url\": \"\", \"event_types\": [\"VULNERABILITY\", \"FINDING\"], \"min_severity\": \"LOW\", \"retries\": 10}\n options_desc = {\n \"webhook_url\": \"Discord webhook URL\",\n \"event_types\": \"Types of events to send\",\n \"min_severity\": \"Only allow VULNERABILITY events of this severity or higher\",\n \"retries\": \"Number of times to retry sending the message before skipping the event\",\n }\n content_key = \"text\"\n\n def format_message_str(self, event):\n event_tags = \",\".join(sorted(event.tags))\n return f\"`[{event.type}]`\\t*`{event.data}`*\\t`{event_tags}`\"\n\n def format_message_other(self, event):\n event_yaml = yaml.dump(event.data)\n event_type = f\"*`[{event.type}]`*\"\n if event.type in (\"VULNERABILITY\", \"FINDING\"):\n event_str, color = self.get_severity_color(event)\n event_type = f\"{color} `{event_str}` {color}\"\n return f\"\"\"*{event_type}*\\n```\\n{event_yaml}```\"\"\"\n" }, { "path": "bbot/modules/output/splunk.py", "content": "from bbot.errors import WebError\nfrom bbot.modules.output.base import BaseOutputModule\n\n\nclass Splunk(BaseOutputModule):\n watched_events = [\"*\"]\n meta = {\n \"description\": \"Send every event to a splunk instance through HTTP Event Collector\",\n \"created_date\": \"2024-02-17\",\n \"author\": \"@w0Tx\",\n }\n options = {\n \"url\": \"\",\n \"hectoken\": \"\",\n \"index\": \"\",\n \"source\": \"\",\n \"timeout\": 10,\n }\n options_desc = {\n \"url\": \"Web URL\",\n \"hectoken\": \"HEC Token\",\n \"index\": \"Index to send data to\",\n \"source\": \"Source path to be added to the metadata\",\n \"timeout\": \"HTTP timeout\",\n }\n\n async def setup(self):\n self.url = self.config.get(\"url\", \"\")\n self.source = self.config.get(\"source\", \"bbot\")\n self.index = self.config.get(\"index\", \"main\")\n self.timeout = self.config.get(\"timeout\", 10)\n self.headers = {}\n\n hectoken = self.config.get(\"hectoken\", \"\")\n if hectoken:\n self.headers[\"Authorization\"] = f\"Splunk {hectoken}\"\n if not self.url:\n return False, \"Must set URL\"\n if not self.source:\n self.warning(\"Please provide a source\")\n return True\n\n async def handle_event(self, event):\n while 1:\n try:\n data = {\n \"index\": self.index,\n \"source\": self.source,\n \"sourcetype\": \"_json\",\n \"event\": event.json(),\n }\n await self.helpers.request(\n url=self.url,\n method=\"POST\",\n headers=self.headers,\n json=data,\n raise_error=True,\n )\n break\n except WebError as e:\n self.warning(f\"Error sending {event}: {e}, retrying...\")\n await self.helpers.sleep(1)\n" }, { "path": "bbot/modules/output/sqlite.py", "content": "from pathlib import Path\n\nfrom bbot.modules.templates.sql import SQLTemplate\n\n\nclass SQLite(SQLTemplate):\n watched_events = [\"*\"]\n meta = {\n \"description\": \"Output scan data to a SQLite database\",\n \"created_date\": \"2024-11-07\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\n \"database\": \"\",\n }\n options_desc = {\n \"database\": \"The path to the sqlite database file\",\n }\n deps_pip = [\"sqlmodel\", \"aiosqlite\"]\n\n async def setup(self):\n db_file = self.config.get(\"database\", \"\")\n if not db_file:\n db_file = self.scan.home / \"output.sqlite\"\n db_file = Path(db_file)\n if not db_file.is_absolute():\n db_file = self.scan.home / db_file\n self.db_file = db_file\n self.db_file.parent.mkdir(parents=True, exist_ok=True)\n return await super().setup()\n\n def connection_string(self, mask_password=False):\n return f\"sqlite+aiosqlite:///{self.db_file}\"\n" }, { "path": "bbot/modules/output/stdout.py", "content": "import json\n\nfrom bbot.logger import log_to_stderr\nfrom bbot.modules.output.base import BaseOutputModule\n\n\nclass Stdout(BaseOutputModule):\n watched_events = [\"*\"]\n meta = {\"description\": \"Output to text\", \"created_date\": \"2024-04-03\", \"author\": \"@TheTechromancer\"}\n options = {\"format\": \"text\", \"event_types\": [], \"event_fields\": [], \"in_scope_only\": False, \"accept_dupes\": True}\n options_desc = {\n \"format\": \"Which text format to display, choices: text,json\",\n \"event_types\": \"Which events to display, default all event types\",\n \"event_fields\": \"Which event fields to display\",\n \"in_scope_only\": \"Whether to only show in-scope events\",\n \"accept_dupes\": \"Whether to show duplicate events, default True\",\n }\n vuln_severity_map = {\"LOW\": \"HUGEWARNING\", \"MEDIUM\": \"HUGEWARNING\", \"HIGH\": \"CRITICAL\", \"CRITICAL\": \"CRITICAL\"}\n format_choices = [\"text\", \"json\"]\n\n async def setup(self):\n self.text_format = self.config.get(\"format\", \"text\").strip().lower()\n if self.text_format not in self.format_choices:\n return (\n False,\n f'Invalid text format choice, \"{self.text_format}\" (choices: {\",\".join(self.format_choices)})',\n )\n self.accept_event_types = [str(s).upper() for s in self.config.get(\"event_types\", [])]\n self.show_event_fields = [str(s) for s in self.config.get(\"event_fields\", [])]\n self.in_scope_only = self.config.get(\"in_scope_only\", False)\n self.accept_dupes = self.config.get(\"accept_dupes\", False)\n return True\n\n async def filter_event(self, event):\n if self.accept_event_types:\n if event.type not in self.accept_event_types:\n return False, f'Event type \"{event.type}\" is not in the allowed event_types'\n return True\n\n async def handle_event(self, event):\n json_mode = \"human\" if self.text_format == \"text\" else \"json\"\n event_json = event.json(mode=json_mode)\n if self.show_event_fields:\n event_json = {k: str(event_json.get(k, \"\")) for k in self.show_event_fields}\n\n if self.text_format == \"text\":\n await self.handle_text(event, event_json)\n elif self.text_format == \"json\":\n await self.handle_json(event, event_json)\n\n async def handle_text(self, event, event_json):\n if self.show_event_fields:\n event_str = \"\\t\".join([str(s) for s in event_json.values()])\n else:\n event_str = self.human_event_str(event)\n\n # log vulnerabilities in vivid colors\n if event.type == \"VULNERABILITY\":\n severity = event.data.get(\"severity\", \"INFO\")\n if severity in self.vuln_severity_map:\n loglevel = self.vuln_severity_map[severity]\n log_to_stderr(event_str, level=loglevel, logname=False)\n elif event.type == \"FINDING\":\n log_to_stderr(event_str, level=\"HUGEINFO\", logname=False)\n\n print(event_str)\n\n async def handle_json(self, event, event_json):\n print(json.dumps(event_json))\n" }, { "path": "bbot/modules/output/subdomains.py", "content": "from bbot.modules.output.txt import TXT\nfrom bbot.modules.base import BaseModule\n\n\nclass Subdomains(TXT):\n watched_events = [\"DNS_NAME\", \"DNS_NAME_UNRESOLVED\"]\n flags = [\"subdomain-enum\"]\n meta = {\n \"description\": \"Output only resolved, in-scope subdomains\",\n \"created_date\": \"2023-07-31\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"output_file\": \"\", \"include_unresolved\": False}\n options_desc = {\"output_file\": \"Output to file\", \"include_unresolved\": \"Include unresolved subdomains in output\"}\n accept_dupes = False\n in_scope_only = True\n\n output_filename = \"subdomains.txt\"\n\n async def setup(self):\n self.include_unresolved = self.config.get(\"include_unresolved\", False)\n self.subdomains_written = 0\n return await super().setup()\n\n async def filter_event(self, event):\n if event.type == \"DNS_NAME_UNRESOLVED\" and not self.include_unresolved:\n return False, \"Not accepting unresolved subdomain (include_unresolved=False)\"\n return True\n\n def _scope_distance_check(self, event):\n return BaseModule._scope_distance_check(self, event)\n\n async def handle_event(self, event):\n if self.file is not None:\n self.subdomains_written += 1\n self.file.write(f\"{event.data}\\n\")\n self.file.flush()\n\n async def report(self):\n if getattr(self, \"_file\", None) is not None:\n self.info(f\"Saved {self.subdomains_written:,} subdomains to {self.output_file}\")\n" }, { "path": "bbot/modules/output/teams.py", "content": "from bbot.modules.templates.webhook import WebhookOutputModule\n\n\nclass Teams(WebhookOutputModule):\n watched_events = [\"*\"]\n meta = {\n \"description\": \"Message a Teams channel when certain events are encountered\",\n \"created_date\": \"2023-08-14\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"webhook_url\": \"\", \"event_types\": [\"VULNERABILITY\", \"FINDING\"], \"min_severity\": \"LOW\", \"retries\": 10}\n options_desc = {\n \"webhook_url\": \"Teams webhook URL\",\n \"event_types\": \"Types of events to send\",\n \"min_severity\": \"Only allow VULNERABILITY events of this severity or higher\",\n \"retries\": \"Number of times to retry sending the message before skipping the event\",\n }\n\n async def handle_event(self, event):\n data = self.format_message(event)\n await self.api_request(\n url=self.webhook_url,\n method=\"POST\",\n json=data,\n )\n\n def trim_message(self, message):\n if len(message) > self.message_size_limit:\n message = message[: self.message_size_limit - 3] + \"...\"\n return message\n\n def format_message_str(self, event):\n items = []\n msg = self.trim_message(event.data)\n items.append({\"type\": \"TextBlock\", \"text\": f\"{msg}\", \"wrap\": True})\n items.append({\"type\": \"FactSet\", \"facts\": [{\"title\": \"Tags:\", \"value\": \", \".join(event.tags)}]})\n return items\n\n def format_message_other(self, event):\n items = [{\"type\": \"FactSet\", \"facts\": []}]\n for key, value in event.data.items():\n if key != \"severity\":\n msg = self.trim_message(str(value))\n items[0][\"facts\"].append({\"title\": f\"{key}:\", \"value\": msg})\n return items\n\n def get_severity_color(self, event):\n color = \"Accent\"\n if event.type == \"VULNERABILITY\":\n severity = event.data.get(\"severity\", \"INFO\")\n if severity == \"CRITICAL\":\n color = \"Attention\"\n elif severity == \"HIGH\":\n color = \"Attention\"\n elif severity == \"MEDIUM\":\n color = \"Warning\"\n elif severity == \"LOW\":\n color = \"Good\"\n return color\n\n def format_message(self, event):\n adaptive_card = {\n \"type\": \"message\",\n \"attachments\": [\n {\n \"contentType\": \"application/vnd.microsoft.card.adaptive\",\n \"contentUrl\": None,\n \"content\": {\n \"$schema\": \"http://adaptivecards.io/schemas/adaptive-card.json\",\n \"type\": \"AdaptiveCard\",\n \"version\": \"1.2\",\n \"msteams\": {\"width\": \"full\"},\n \"body\": [],\n },\n }\n ],\n }\n heading = {\"type\": \"TextBlock\", \"text\": f\"{event.type}\", \"wrap\": True, \"size\": \"Large\", \"style\": \"heading\"}\n body = adaptive_card[\"attachments\"][0][\"content\"][\"body\"]\n body.append(heading)\n if event.type in (\"VULNERABILITY\", \"FINDING\"):\n subheading = {\n \"type\": \"TextBlock\",\n \"text\": event.data.get(\"severity\", \"INFO\"),\n \"spacing\": \"None\",\n \"size\": \"Large\",\n \"wrap\": True,\n }\n subheading[\"color\"] = self.get_severity_color(event)\n body.append(subheading)\n main_text = {\n \"type\": \"ColumnSet\",\n \"separator\": True,\n \"spacing\": \"Medium\",\n \"columns\": [\n {\n \"type\": \"Column\",\n \"width\": \"stretch\",\n \"items\": [],\n }\n ],\n }\n if isinstance(event.data, str):\n items = self.format_message_str(event)\n else:\n items = self.format_message_other(event)\n main_text[\"columns\"][0][\"items\"] = items\n body.append(main_text)\n return adaptive_card\n" }, { "path": "bbot/modules/output/txt.py", "content": "from contextlib import suppress\n\nfrom bbot.modules.output.base import BaseOutputModule\n\n\nclass TXT(BaseOutputModule):\n watched_events = [\"*\"]\n meta = {\"description\": \"Output to text\", \"created_date\": \"2024-04-03\", \"author\": \"@TheTechromancer\"}\n options = {\"output_file\": \"\"}\n options_desc = {\"output_file\": \"Output to file\"}\n\n output_filename = \"output.txt\"\n\n async def setup(self):\n self._prep_output_dir(self.output_filename)\n return True\n\n async def handle_event(self, event):\n event_str = self.human_event_str(event)\n\n if self.file is not None:\n self.file.write(event_str + \"\\n\")\n self.file.flush()\n\n async def cleanup(self):\n if getattr(self, \"_file\", None) is not None:\n with suppress(Exception):\n self.file.close()\n\n async def report(self):\n if getattr(self, \"_file\", None) is not None:\n self.info(f\"Saved TXT output to {self.output_file}\")\n" }, { "path": "bbot/modules/output/web_parameters.py", "content": "from contextlib import suppress\nfrom collections import defaultdict\n\nfrom bbot.modules.output.base import BaseOutputModule\n\n\nclass Web_parameters(BaseOutputModule):\n watched_events = [\"WEB_PARAMETER\"]\n meta = {\n \"description\": \"Output WEB_PARAMETER names to a file\",\n \"created_date\": \"2025-01-25\",\n \"author\": \"@liquidsec\",\n }\n options = {\"output_file\": \"\", \"include_count\": False}\n options_desc = {\n \"output_file\": \"Output to file\",\n \"include_count\": \"Include the count of each parameter in the output\",\n }\n\n output_filename = \"web_parameters.txt\"\n\n async def setup(self):\n self._prep_output_dir(self.output_filename)\n self.parameter_counts = defaultdict(int)\n return True\n\n async def handle_event(self, event):\n parameter_name = event.data.get(\"name\", \"\")\n if parameter_name:\n self.parameter_counts[parameter_name] += 1\n\n async def cleanup(self):\n if getattr(self, \"_file\", None) is not None:\n with suppress(Exception):\n self.file.close()\n\n async def report(self):\n include_count = self.config.get(\"include_count\", False)\n\n # Sort behavior:\n # - If include_count is True, sort by count (descending) and then alphabetically by name\n # - If include_count is False, sort alphabetically by name only\n sorted_parameters = sorted(\n self.parameter_counts.items(), key=lambda x: (-x[1], x[0]) if include_count else x[0]\n )\n for param, count in sorted_parameters:\n if include_count:\n # Include the count of each parameter in the output\n self.file.write(f\"{count}\\t{param}\\n\")\n else:\n # Only include the parameter name, effectively deduplicating by name\n self.file.write(f\"{param}\\n\")\n self.file.flush()\n if getattr(self, \"_file\", None) is not None:\n self.info(f\"Saved web parameters to {self.output_file}\")\n" }, { "path": "bbot/modules/output/web_report.py", "content": "from bbot.modules.output.base import BaseOutputModule\nimport markdown\nimport html\n\n\nclass web_report(BaseOutputModule):\n watched_events = [\"URL\", \"TECHNOLOGY\", \"FINDING\", \"VULNERABILITY\", \"VHOST\"]\n meta = {\n \"description\": \"Create a markdown report with web assets\",\n \"created_date\": \"2023-02-08\",\n \"author\": \"@liquidsec\",\n }\n options = {\n \"output_file\": \"\",\n \"css_theme_file\": \"https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css\",\n }\n options_desc = {\"output_file\": \"Output to file\", \"css_theme_file\": \"CSS theme URL for HTML output\"}\n deps_pip = [\"markdown~=3.4.3\"]\n\n async def setup(self):\n html_css_file = self.config.get(\"css_theme_file\", \"\")\n\n self.html_header = f\"\"\"\n \n \n \n \n \n \n \"\"\"\n\n self.html_footer = \"\"\n self.web_assets = {}\n self.markdown = \"\"\n\n self._prep_output_dir(\"web_report.html\")\n return True\n\n async def handle_event(self, event):\n if event.type == \"URL\":\n parsed = event.parsed_url\n host = f\"{parsed.scheme}://{parsed.netloc}/\"\n if host not in self.web_assets.keys():\n self.web_assets[host] = {\"URL\": []}\n parent_chain = []\n\n current_parent = event.parent\n while not current_parent.type == \"SCAN\":\n parent_chain.append(\n f\" ({current_parent.module})---> [{current_parent.type}]:{html.escape(current_parent.pretty_string)}\"\n )\n current_parent = current_parent.parent\n\n parent_chain.reverse()\n parent_chain_text = (\n \"\".join(parent_chain)\n + f\" ({event.module})---> \"\n + f\"[{event.type}]:{html.escape(event.pretty_string)}\"\n )\n self.web_assets[host][\"URL\"].append(f\"**{html.escape(event.data)}**: {parent_chain_text}\")\n\n else:\n current_parent = event.parent\n parsed = None\n while 1:\n if current_parent.type == \"URL\":\n parsed = current_parent.parsed_url\n break\n current_parent = current_parent.parent\n if current_parent.parent.type == \"SCAN\":\n break\n if parsed:\n host = f\"{parsed.scheme}://{parsed.netloc}/\"\n if host not in self.web_assets.keys():\n self.web_assets[host] = {\"URL\": []}\n if event.type not in self.web_assets[host].keys():\n self.web_assets[host][event.type] = [html.escape(event.pretty_string)]\n else:\n self.web_assets[host][event.type].append(html.escape(event.pretty_string))\n\n async def report(self):\n for host in self.web_assets.keys():\n self.markdown += f\"# {host}\\n\\n\"\n\n for event_type in self.web_assets[host].keys():\n self.markdown += f\"### {event_type}\\n\"\n dedupe = []\n for e in self.web_assets[host][event_type]:\n if e in dedupe:\n continue\n dedupe.append(e)\n self.markdown += f\"\\n* {e}\\n\"\n self.markdown += \"\\n\"\n\n if self.file is not None:\n self.file.write(self.html_header)\n self.file.write(markdown.markdown(self.markdown))\n self.file.write(self.html_footer)\n self.file.flush()\n self.info(f\"Web Report saved to {self.output_file}\")\n" }, { "path": "bbot/modules/output/websocket.py", "content": "import json\nimport asyncio\nimport ssl\nimport websockets\n\nfrom bbot.modules.output.base import BaseOutputModule\n\n\nclass Websocket(BaseOutputModule):\n watched_events = [\"*\"]\n meta = {\"description\": \"Output to websockets\", \"created_date\": \"2022-04-15\", \"author\": \"@TheTechromancer\"}\n options = {\"url\": \"\", \"token\": \"\", \"preserve_graph\": True, \"ignore_ssl\": False}\n options_desc = {\n \"url\": \"Web URL\",\n \"token\": \"Authorization Bearer token\",\n \"preserve_graph\": \"Preserve full chains of events in the graph (prevents orphans)\",\n \"ignore_ssl\": \"Ignores all Websocket SSL related errors (like Self-Signed Certificates, etc.)\",\n }\n\n async def setup(self):\n self.url = self.config.get(\"url\", \"\")\n if not self.url:\n return False, \"Must set URL\"\n self.token = self.config.get(\"token\", \"\")\n self._ws = None\n return True\n\n async def handle_event(self, event):\n event_json = event.json()\n await self.send(event_json)\n\n async def ws(self, rebuild=False):\n if self._ws is None or rebuild:\n kwargs = {\"close_timeout\": 0.5}\n if self.token:\n kwargs.update({\"additional_headers\": {\"Authorization\": f\"Bearer {self.token}\"}})\n verbs = (\"Building\", \"Built\") if not rebuild else (\"Rebuilding\", \"Rebuilt\")\n self.debug(f\"{verbs[0]} websocket connection to {self.url}\")\n if self.config.get(\"ignore_ssl\", False):\n ssl_context = ssl.create_default_context()\n ssl_context.check_hostname = False\n ssl_context.verify_mode = ssl.CERT_NONE\n self._ws = await websockets.connect(self.url, ssl=ssl_context, **kwargs)\n else:\n self._ws = await websockets.connect(self.url, **kwargs)\n self.debug(f\"{verbs[1]} websocket connection to {self.url}\")\n return self._ws\n\n async def send(self, message):\n rebuild = False\n while not self.scan.stopped:\n try:\n ws = await self.ws(rebuild=rebuild)\n message_str = json.dumps(message)\n self.debug(f\"Sending message of length {len(message_str)}\")\n await ws.send(message_str)\n rebuild = False\n break\n except Exception as e:\n self.warning(f\"Error sending message: {e}, retrying\")\n await asyncio.sleep(1)\n rebuild = True\n\n async def cleanup(self):\n if self._ws is not None:\n self.debug(f\"Closing connection to {self.url}\")\n await self._ws.close()\n self.debug(f\"Closed connection to {self.url}\")\n self._ws = None\n" }, { "path": "bbot/modules/paramminer_cookies.py", "content": "from .paramminer_headers import paramminer_headers\n\n\nclass paramminer_cookies(paramminer_headers):\n \"\"\"\n Inspired by https://github.com/PortSwigger/param-miner\n \"\"\"\n\n watched_events = [\"HTTP_RESPONSE\", \"WEB_PARAMETER\"]\n produced_events = [\"WEB_PARAMETER\"]\n produced_events = [\"FINDING\"]\n flags = [\"active\", \"aggressive\", \"slow\", \"web-paramminer\"]\n meta = {\n \"description\": \"Smart brute-force to check for common HTTP cookie parameters\",\n \"created_date\": \"2022-06-27\",\n \"author\": \"@liquidsec\",\n }\n options = {\n \"wordlist\": \"\", # default is defined within setup function\n \"recycle_words\": False,\n \"skip_boring_words\": True,\n }\n options_desc = {\n \"wordlist\": \"Define the wordlist to be used to derive headers\",\n \"recycle_words\": \"Attempt to use words found during the scan on all other endpoints\",\n \"skip_boring_words\": \"Remove commonly uninteresting words from the wordlist\",\n }\n options_desc = {\"wordlist\": \"Define the wordlist to be used to derive cookies\"}\n scanned_hosts = []\n boring_words = set()\n _module_threads = 12\n in_scope_only = True\n compare_mode = \"cookie\"\n default_wordlist = \"paramminer_parameters.txt\"\n\n async def check_batch(self, compare_helper, url, cookie_list):\n cookies = {p: self.rand_string(14) for p in cookie_list}\n return await compare_helper.compare(url, cookies=cookies, check_reflection=(len(cookie_list) == 1))\n\n def gen_count_args(self, url):\n cookie_count = 40\n while 1:\n if cookie_count < 0:\n break\n fake_cookies = {self.rand_string(14): self.rand_string(14) for _ in range(0, cookie_count)}\n yield cookie_count, (url,), {\"cookies\": fake_cookies}\n cookie_count -= 5\n" }, { "path": "bbot/modules/paramminer_getparams.py", "content": "from .paramminer_headers import paramminer_headers\n\n\nclass paramminer_getparams(paramminer_headers):\n \"\"\"\n Inspired by https://github.com/PortSwigger/param-miner\n \"\"\"\n\n watched_events = [\"HTTP_RESPONSE\", \"WEB_PARAMETER\"]\n produced_events = [\"WEB_PARAMETER\"]\n produced_events = [\"FINDING\"]\n flags = [\"active\", \"aggressive\", \"slow\", \"web-paramminer\"]\n meta = {\n \"description\": \"Use smart brute-force to check for common HTTP GET parameters\",\n \"created_date\": \"2022-06-28\",\n \"author\": \"@liquidsec\",\n }\n scanned_hosts = []\n options = {\n \"wordlist\": \"\", # default is defined within setup function\n \"recycle_words\": False,\n \"skip_boring_words\": True,\n }\n options_desc = {\n \"wordlist\": \"Define the wordlist to be used to derive headers\",\n \"recycle_words\": \"Attempt to use words found during the scan on all other endpoints\",\n \"skip_boring_words\": \"Remove commonly uninteresting words from the wordlist\",\n }\n boring_words = {\"utm_source\", \"utm_campaign\", \"utm_medium\", \"utm_term\", \"utm_content\"}\n in_scope_only = True\n compare_mode = \"getparam\"\n default_wordlist = \"paramminer_parameters.txt\"\n\n async def check_batch(self, compare_helper, url, getparam_list):\n test_getparams = {p: self.rand_string(14) for p in getparam_list}\n return await compare_helper.compare(\n self.helpers.add_get_params(url, test_getparams).geturl(), check_reflection=(len(getparam_list) == 1)\n )\n\n def gen_count_args(self, url):\n getparam_count = 40\n while 1:\n if getparam_count < 0:\n break\n fake_getparams = {self.rand_string(14): self.rand_string(14) for _ in range(0, getparam_count)}\n yield getparam_count, (self.helpers.add_get_params(url, fake_getparams).geturl(),), {}\n getparam_count -= 5\n" }, { "path": "bbot/modules/paramminer_headers.py", "content": "import re\n\nfrom bbot.errors import HttpCompareError\nfrom bbot.modules.base import BaseModule\n\n\nclass paramminer_headers(BaseModule):\n \"\"\"\n Inspired by https://github.com/PortSwigger/param-miner\n \"\"\"\n\n watched_events = [\"HTTP_RESPONSE\", \"WEB_PARAMETER\"]\n produced_events = [\"WEB_PARAMETER\"]\n flags = [\"active\", \"aggressive\", \"slow\", \"web-paramminer\"]\n meta = {\n \"description\": \"Use smart brute-force to check for common HTTP header parameters\",\n \"created_date\": \"2022-04-15\",\n \"author\": \"@liquidsec\",\n }\n options = {\n \"wordlist\": \"\", # default is defined within setup function\n \"recycle_words\": False,\n \"skip_boring_words\": True,\n }\n options_desc = {\n \"wordlist\": \"Define the wordlist to be used to derive headers\",\n \"recycle_words\": \"Attempt to use words found during the scan on all other endpoints\",\n \"skip_boring_words\": \"Remove commonly uninteresting words from the wordlist\",\n }\n scanned_hosts = []\n boring_words = {\n \"accept\",\n \"accept-encoding\",\n \"accept-language\",\n \"action\",\n \"authorization\",\n \"cf-connecting-ip\",\n \"connection\",\n \"content-encoding\",\n \"content-length\",\n \"content-range\",\n \"content-type\",\n \"cookie\",\n \"date\",\n \"expect\",\n \"host\",\n \"if\",\n \"if-match\",\n \"if-modified-since\",\n \"if-none-match\",\n \"if-unmodified-since\",\n \"javascript\",\n \"keep-alive\",\n \"label\",\n \"max-forwards\",\n \"negotiate\",\n \"proxy\",\n \"range\",\n \"referer\",\n \"start\",\n \"trailer\",\n \"transfer-encoding\",\n \"upgrade\",\n \"user-agent\",\n \"vary\",\n \"waf-stuff-below\",\n \"x-scanner\",\n \"x_alto_ajax_key\",\n \"zaccess-control-request-headers\",\n \"zaccess-control-request-method\",\n \"zmax-forwards\",\n \"zorigin\",\n \"zreferrer\",\n \"zvia\",\n \"zx-request-id\",\n \"zx-timer\",\n }\n _module_threads = 12\n in_scope_only = True\n compare_mode = \"header\"\n default_wordlist = \"paramminer_headers.txt\"\n\n header_regex = re.compile(r\"^[!#$%&\\'*+\\-.^_`|~0-9a-zA-Z]+: [^\\r\\n]+$\")\n\n async def setup_deps(self):\n wordlist = self.config.get(\"wordlist\", \"\")\n if not wordlist:\n wordlist = f\"{self.helpers.wordlist_dir}/{self.default_wordlist}\"\n self.wordlist_file = await self.helpers.wordlist(wordlist)\n self.debug(f\"Using wordlist: [{wordlist}]\")\n return True\n\n async def setup(self):\n self.recycle_words = self.config.get(\"recycle_words\", True)\n self.event_dict = {}\n self.already_checked = set()\n\n self.wl = {\n h.strip().lower() for h in self.helpers.read_file(self.wordlist_file) if len(h) > 0 and \"%\" not in h\n }\n\n # check against the boring list (if the option is set)\n if self.config.get(\"skip_boring_words\", True):\n self.wl -= self.boring_words\n self.extracted_words_master = set()\n\n return True\n\n def rand_string(self, *args, **kwargs):\n return self.helpers.rand_string(*args, **kwargs)\n\n async def do_mining(self, wl, url, batch_size, compare_helper):\n for i in wl:\n if i not in self.wl:\n h = hash(i + url)\n self.already_checked.add(h)\n\n results = set()\n abort_threshold = 15\n try:\n for group in self.helpers.grouper(wl, batch_size):\n async for result, reasons, reflection in self.binary_search(compare_helper, url, group):\n results.add((result, \",\".join(reasons), reflection))\n if len(results) >= abort_threshold:\n self.warning(\n f\"Abort threshold ({abort_threshold}) reached, too many {self.compare_mode}s found for url: {url}\"\n )\n results.clear()\n assert False\n except AssertionError:\n pass\n return results\n\n async def process_results(self, event, results):\n url = event.data.get(\"url\")\n for result, reasons, reflection in results:\n paramtype = self.compare_mode.upper()\n if paramtype == \"HEADER\":\n if self.header_regex.match(result):\n self.debug(\"rejecting parameter as it is not a valid header\")\n continue\n tags = []\n if reflection:\n tags = [\"http_reflection\"]\n description = f\"[Paramminer] {self.compare_mode.capitalize()}: [{result}] Reasons: [{reasons}] Reflection: [{str(reflection)}]\"\n reflected = \"reflected \" if reflection else \"\"\n\n await self.emit_event(\n {\n \"host\": str(event.host),\n \"url\": url,\n \"type\": paramtype,\n \"description\": description,\n \"name\": result,\n \"original_value\": None,\n },\n \"WEB_PARAMETER\",\n event,\n tags=tags,\n context=f'{{module}} scanned {url} and identified {{event.type}}: {reflected}{self.compare_mode} parameter: \"{result}\"',\n )\n\n async def handle_event(self, event):\n # If recycle words is enabled, we will collect WEB_PARAMETERS we find to build our list in finish()\n # We also collect any parameters of type \"SPECULATIVE\"\n if event.type == \"WEB_PARAMETER\":\n parameter_name = event.data.get(\"name\")\n if self.recycle_words or (event.data.get(\"type\") == \"SPECULATIVE\"):\n if self.config.get(\"skip_boring_words\", True) and parameter_name in self.boring_words:\n return\n if parameter_name not in self.wl: # Ensure it's not already in the wordlist\n self.debug(f\"Adding {parameter_name} to wordlist\")\n self.extracted_words_master.add(parameter_name)\n\n elif event.type == \"HTTP_RESPONSE\":\n url = event.data.get(\"url\")\n try:\n compare_helper = self.helpers.http_compare(url)\n except HttpCompareError as e:\n self.debug(f\"Error initializing compare helper: {e}\")\n return\n batch_size = await self.count_test(url)\n if batch_size is None or batch_size <= 0:\n self.debug(f\"Failed to get baseline max {self.compare_mode} count, aborting\")\n return\n self.debug(f\"Resolved batch_size at {str(batch_size)}\")\n\n self.event_dict[url] = (event, batch_size)\n try:\n if not await compare_helper.canary_check(url, mode=self.compare_mode):\n raise HttpCompareError(\"failed canary check\")\n except HttpCompareError as e:\n self.verbose(f'Aborting \"{url}\" ({e})')\n return\n\n try:\n results = await self.do_mining(self.wl, url, batch_size, compare_helper)\n except HttpCompareError as e:\n self.debug(f\"Encountered HttpCompareError: [{e}] for URL [{event.data}]\")\n await self.process_results(event, results)\n\n async def count_test(self, url):\n baseline = await self.helpers.request(url)\n if baseline is None:\n return\n if str(baseline.status_code)[0] in {\"4\", \"5\"}:\n return\n for count, args, kwargs in self.gen_count_args(url):\n r = await self.helpers.request(*args, **kwargs)\n if r is not None and str(r.status_code)[0] not in {\"4\", \"5\"}:\n return count\n\n def gen_count_args(self, url):\n header_count = 95\n while 1:\n if header_count < 0:\n break\n fake_headers = {}\n for i in range(0, header_count):\n fake_headers[self.rand_string(14)] = self.rand_string(14)\n yield header_count, (url,), {\"headers\": fake_headers}\n header_count -= 5\n\n async def binary_search(self, compare_helper, url, group, reasons=None, reflection=False):\n if reasons is None:\n reasons = []\n self.debug(f\"Entering recursive binary_search with {len(group):,} sized group\")\n if len(group) == 1 and len(reasons) > 0:\n yield group[0], reasons, reflection\n elif len(group) > 1 or (len(group) == 1 and len(reasons) == 0):\n for group_slice in self.helpers.split_list(group):\n match, reasons, reflection, subject_response = await self.check_batch(compare_helper, url, group_slice)\n if match is False:\n async for r in self.binary_search(compare_helper, url, group_slice, reasons, reflection):\n yield r\n else:\n self.debug(\n f\"binary_search() failed to start with group of size {str(len(group))} and {str(len(reasons))} length reasons\"\n )\n\n async def check_batch(self, compare_helper, url, header_list):\n rand = self.rand_string()\n test_headers = {}\n for header in header_list:\n test_headers[header] = rand\n return await compare_helper.compare(url, headers=test_headers, check_reflection=(len(header_list) == 1))\n\n async def finish(self):\n for url, (event, batch_size) in list(self.event_dict.items()):\n try:\n compare_helper = self.helpers.http_compare(url)\n except HttpCompareError as e:\n self.debug(f\"Error initializing compare helper: {e}\")\n continue\n words_to_process = {\n i for i in self.extracted_words_master.copy() if hash(i + url) not in self.already_checked\n }\n try:\n results = await self.do_mining(words_to_process, url, batch_size, compare_helper)\n except HttpCompareError as e:\n self.debug(f\"Encountered HttpCompareError: [{e}] for URL [{url}]\")\n continue\n await self.process_results(event, results)\n\n async def filter_event(self, event):\n # Filter out static endpoints\n if event.data.get(\"url\").endswith(tuple(f\".{ext}\" for ext in self.config.get(\"url_extension_static\", []))):\n return False\n\n # We don't need to look at WEB_PARAMETERS that we produced\n if str(event.module).startswith(\"paramminer\"):\n return False\n\n return True\n" }, { "path": "bbot/modules/passivetotal.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass passivetotal(subdomain_enum_apikey):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query the PassiveTotal API for subdomains\",\n \"created_date\": \"2022-08-08\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"PassiveTotal API Key in the format of 'username:api_key'\"}\n\n base_url = \"https://api.passivetotal.org/v2\"\n\n async def setup(self):\n return await super().setup()\n\n async def ping(self):\n url = f\"{self.base_url}/account/quota\"\n j = (await self.api_request(url, retry_on_http_429=False)).json()\n limit = j[\"user\"][\"limits\"][\"search_api\"]\n used = j[\"user\"][\"counts\"][\"search_api\"]\n assert used < limit, \"No quota remaining\"\n\n def prepare_api_request(self, url, kwargs):\n api_username, api_key = self.api_key.split(\":\", 1)\n kwargs[\"auth\"] = (api_username, api_key)\n return url, kwargs\n\n async def abort_if(self, event):\n # RiskIQ is famous for their junk data\n return await super().abort_if(event) or \"unresolved\" in event.tags\n\n async def request_url(self, query):\n url = f\"{self.base_url}/enrichment/subdomains?query={self.helpers.quote(query)}\"\n return await self.api_request(url)\n\n async def parse_results(self, r, query):\n results = set()\n for subdomain in r.json().get(\"subdomains\", []):\n results.add(f\"{subdomain}.{query}\")\n return results\n" }, { "path": "bbot/modules/pgp.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass pgp(subdomain_enum):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"EMAIL_ADDRESS\"]\n flags = [\"passive\", \"email-enum\", \"safe\"]\n meta = {\n \"description\": \"Query common PGP servers for email addresses\",\n \"created_date\": \"2022-08-10\",\n \"author\": \"@TheTechromancer\",\n }\n # TODO: scan for Web Key Directory (/.well-known/openpgpkey/)\n options = {\n \"search_urls\": [\n \"https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=\",\n \"http://the.earth.li:11371/pks/lookup?fingerprint=on&op=vindex&search=\",\n \"https://pgpkeys.eu/pks/lookup?search=&op=index\",\n \"https://pgp.mit.edu/pks/lookup?search=&op=index\",\n ]\n }\n options_desc = {\"search_urls\": \"PGP key servers to search\"}\n\n async def handle_event(self, event):\n query = self.make_query(event)\n results = await self.query(query)\n if results:\n for email, keyserver in results:\n await self.emit_event(\n email,\n \"EMAIL_ADDRESS\",\n event,\n abort_if=self.abort_if,\n context=f'{{module}} queried PGP keyserver {keyserver} for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n\n async def query(self, query):\n results = set()\n urls = self.config.get(\"search_urls\", [])\n urls = [url.replace(\"\", self.helpers.quote(query)) for url in urls]\n async for url, response in self.helpers.request_batch(urls):\n keyserver = self.helpers.urlparse(url).netloc\n response = await self.helpers.request(url)\n if response is not None:\n for email in await self.helpers.re.extract_emails(response.text):\n email = email.lower()\n if email.endswith(query):\n results.add((email, keyserver))\n return results\n" }, { "path": "bbot/modules/portfilter.py", "content": "from bbot.modules.base import BaseInterceptModule\n\n\nclass portfilter(BaseInterceptModule):\n watched_events = [\"OPEN_TCP_PORT\", \"URL_UNVERIFIED\", \"URL\"]\n flags = [\"passive\", \"safe\"]\n meta = {\n \"description\": \"Filter out unwanted open ports from cloud/CDN targets\",\n \"created_date\": \"2025-01-06\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\n \"cdn_tags\": \"cdn-\",\n \"allowed_cdn_ports\": \"80,443\",\n }\n options_desc = {\n \"cdn_tags\": \"Comma-separated list of tags to skip, e.g. 'cdn,cloud'\",\n \"allowed_cdn_ports\": \"Comma-separated list of ports that are allowed to be scanned for CDNs\",\n }\n\n _priority = 4\n # we consume URLs but we don't want to automatically enable httpx\n _disable_auto_module_deps = True\n\n async def setup(self):\n self.cdn_tags = [t.strip() for t in self.config.get(\"cdn_tags\", \"\").split(\",\")]\n self.allowed_cdn_ports = self.config.get(\"allowed_cdn_ports\", \"\").strip()\n if self.allowed_cdn_ports:\n try:\n self.allowed_cdn_ports = [int(p.strip()) for p in self.allowed_cdn_ports.split(\",\")]\n except Exception as e:\n return False, f\"Error parsing allowed CDN ports '{self.allowed_cdn_ports}': {e}\"\n return True\n\n async def handle_event(self, event, **kwargs):\n # if the port isn't in our list of allowed CDN ports\n if event.port not in self.allowed_cdn_ports:\n for cdn_tag in self.cdn_tags:\n # and if any of the event's tags match our CDN filter\n if any(t.startswith(str(cdn_tag)) for t in event.tags):\n return (\n False,\n f\"one of the event's tags matches the tag '{cdn_tag}' and the port is not in the allowed list\",\n )\n return True\n" }, { "path": "bbot/modules/portscan.py", "content": "import json\nimport ipaddress\nfrom contextlib import suppress\nfrom radixtarget import RadixTarget, host_size_key\n\nfrom bbot.modules.base import BaseModule\n\n\n# TODO: this module is getting big. It should probably be two modules: one for ping and one for SYN.\n\n\nclass portscan(BaseModule):\n flags = [\"active\", \"portscan\", \"safe\"]\n watched_events = [\"IP_ADDRESS\", \"IP_RANGE\", \"DNS_NAME\"]\n produced_events = [\"OPEN_TCP_PORT\"]\n meta = {\n \"description\": \"Port scan with masscan. By default, scans top 100 ports.\",\n \"created_date\": \"2024-05-15\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\n \"top_ports\": 100,\n \"ports\": \"\",\n # ping scan at 600 packets/s ~= private IP space in 8 hours\n \"rate\": 300,\n \"wait\": 5,\n \"ping_first\": False,\n \"ping_only\": False,\n \"adapter\": \"\",\n \"adapter_ip\": \"\",\n \"adapter_mac\": \"\",\n \"router_mac\": \"\",\n \"module_timeout\": 259200, # 3 days\n }\n options_desc = {\n \"top_ports\": \"Top ports to scan (default 100) (to override, specify 'ports')\",\n \"ports\": \"Ports to scan\",\n \"rate\": \"Rate in packets per second\",\n \"wait\": \"Seconds to wait for replies after scan is complete\",\n \"ping_first\": \"Only portscan hosts that reply to pings\",\n \"ping_only\": \"Ping sweep only, no portscan\",\n \"adapter\": 'Manually specify a network interface, such as \"eth0\" or \"tun0\". If not specified, the first network interface found with a default gateway will be used.',\n \"adapter_ip\": \"Send packets using this IP address. Not needed unless masscan's autodetection fails\",\n \"adapter_mac\": \"Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails\",\n \"router_mac\": \"Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails\",\n \"module_timeout\": \"Max time in seconds to spend handling each batch of events\",\n }\n deps_common = [\"masscan\"]\n batch_size = 1000000\n _shuffle_incoming_queue = False\n\n async def setup(self):\n self.top_ports = self.config.get(\"top_ports\", 100)\n self.rate = self.config.get(\"rate\", 300)\n self.wait = self.config.get(\"wait\", 10)\n self.ping_first = self.config.get(\"ping_first\", False)\n self.ping_only = self.config.get(\"ping_only\", False)\n self.ping_scan = self.ping_first or self.ping_only\n self.adapter = self.config.get(\"adapter\", \"\")\n self.adapter_ip = self.config.get(\"adapter_ip\", \"\")\n self.adapter_mac = self.config.get(\"adapter_mac\", \"\")\n self.router_mac = self.config.get(\"router_mac\", \"\")\n self.ports = self.config.get(\"ports\", \"\")\n if self.ports:\n try:\n self.helpers.parse_port_string(self.ports)\n except ValueError as e:\n return False, f\"Error parsing ports '{self.ports}': {e}\"\n\n # keeps track of individual scanned IPs and their open ports\n # this is necessary because we may encounter more hosts with the same IP\n # and we want to avoid scanning them again\n self.open_port_cache = {}\n # keeps track of which IPs/subnets have already been scanned\n self.syn_scanned = self.helpers.make_target(acl_mode=True)\n self.ping_scanned = self.helpers.make_target(acl_mode=True)\n self.prep_blacklist()\n self.helpers.depsinstaller.ensure_root(message=\"Masscan requires root privileges\")\n # check if we're set up for IPv6\n self.ipv6_support = True\n dry_run_command = self._build_masscan_command(target_file=self.helpers.tempfile([\"::1\"], pipe=False), wait=0)\n ipv6_result = await self.run_process(\n dry_run_command,\n sudo=True,\n _log_stderr=False,\n )\n if ipv6_result is None:\n return False, \"Masscan failed to run\"\n returncode = getattr(ipv6_result, \"returncode\", 0)\n if returncode and \"failed to detect IPv6 address\" in ipv6_result.stderr:\n self.warning(\"It looks like you are not set up for IPv6. IPv6 targets will not be scanned.\")\n self.ipv6_support = False\n return True\n\n async def handle_batch(self, *events):\n # ping scan\n if self.ping_scan:\n ping_targets, ping_correlator = await self.make_targets(events, self.ping_scanned)\n ping_events = []\n async for alive_host, _, parent_event in self.masscan(ping_targets, ping_correlator, ping=True):\n # port 0 means icmp ping response\n ping_event = await self.emit_open_port(alive_host, 0, parent_event)\n ping_events.append(ping_event)\n syn_targets, syn_correlator = await self.make_targets(ping_events, self.syn_scanned)\n else:\n syn_targets, syn_correlator = await self.make_targets(events, self.syn_scanned)\n\n # TCP SYN scan\n if not self.ping_only:\n async for ip, port, parent_event in self.masscan(syn_targets, syn_correlator):\n await self.emit_open_port(ip, port, parent_event)\n else:\n self.debug(\"Only ping sweep was requested, skipping TCP SYN scan\")\n\n async def masscan(self, targets, correlator, ping=False):\n scan_type = \"ping\" if ping else \"SYN\"\n self.debug(f\"Starting masscan {scan_type} scan\")\n if not targets:\n self.debug(\"No targets specified, aborting.\")\n return\n\n target_file = self.helpers.tempfile(targets, pipe=False)\n command = self._build_masscan_command(target_file, ping=ping)\n stats_file = self.helpers.tempfile_tail(callback=self.log_masscan_status)\n try:\n with open(stats_file, \"w\") as stats_fh:\n async for line in self.run_process_live(command, sudo=True, stderr=stats_fh):\n for ip, port in self.parse_json_line(line):\n parent_events = correlator.search(ip)\n # masscan gets the occasional junk result. this is harmless and\n # seems to be a side effect of it having its own TCP stack\n # see https://github.com/robertdavidgraham/masscan/issues/397\n if parent_events is None:\n self.debug(f\"Failed to correlate {ip} to targets\")\n continue\n emitted_hosts = set()\n for parent_event in parent_events:\n if parent_event.type == \"DNS_NAME\":\n host = parent_event.host\n else:\n host = ip\n if host not in emitted_hosts:\n yield host, port, parent_event\n emitted_hosts.add(host)\n finally:\n for file in (stats_file, target_file):\n file.unlink(missing_ok=True)\n\n async def make_targets(self, events, scanned_tracker):\n \"\"\"\n Convert events into a list of targets, skipping ones that have already been scanned\n \"\"\"\n correlator = RadixTarget()\n targets = set()\n for event in sorted(events, key=lambda e: host_size_key(e.host)):\n # skip events without host\n if not event.host:\n continue\n ips = set()\n try:\n # first assume it's an ip address / ip range\n ips.add(ipaddress.ip_network(event.host, strict=False))\n except Exception:\n # if it's a hostname, get its IPs from resolved_hosts\n for h in event.resolved_hosts:\n try:\n ips.add(ipaddress.ip_network(h, strict=False))\n except Exception:\n continue\n\n for ip in ips:\n # remove IPv6 addresses if we're not scanning IPv6\n if not self.ipv6_support and ip.version == 6:\n self.debug(f\"Not scanning IPv6 address {ip} because we aren't set up for IPv6\")\n continue\n\n # check if we already found open ports on this IP\n if event.type != \"IP_RANGE\":\n ip_hash = hash(ip.network_address)\n already_found_ports = self.open_port_cache.get(ip_hash, None)\n if already_found_ports is not None:\n # if so, emit them\n for port in already_found_ports:\n await self.emit_open_port(event.host, port, event)\n\n # build a correlation from the IP back to its original parent event\n events_set = correlator.search(ip)\n if events_set is None:\n correlator.insert(ip, {event})\n else:\n events_set.add(event)\n\n # has this IP already been scanned?\n if not scanned_tracker.get(ip):\n # if not, add it to targets!\n scanned_tracker.add(ip)\n targets.add(ip)\n else:\n self.debug(f\"Skipping {ip} because it's already been scanned\")\n\n return targets, correlator\n\n async def emit_open_port(self, ip, port, parent_event):\n parent_is_dns_name = parent_event.type == \"DNS_NAME\"\n if parent_is_dns_name:\n host = parent_event.host\n else:\n host = ip\n\n if port == 0:\n event_data = host\n event_type = \"DNS_NAME\" if parent_is_dns_name else \"IP_ADDRESS\"\n scan_type = \"ping\"\n else:\n event_data = self.helpers.make_netloc(host, port)\n event_type = \"OPEN_TCP_PORT\"\n scan_type = \"TCP SYN\"\n\n event = self.make_event(\n event_data,\n event_type,\n parent=parent_event,\n context=f\"{{module}} executed a {scan_type} scan against {parent_event.data} and found: {{event.type}}: {{event.data}}\",\n )\n\n await self.emit_event(event)\n return event\n\n def parse_json_line(self, line):\n try:\n j = json.loads(line)\n except Exception:\n return\n ip = j.get(\"ip\", \"\")\n if not ip:\n return\n ip = self.helpers.make_ip_type(ip)\n ip_hash = hash(ip)\n ports = j.get(\"ports\", [])\n if not ports:\n return\n for p in ports:\n proto = p.get(\"proto\", \"\")\n port_number = p.get(\"port\", 0)\n try:\n self.open_port_cache[ip_hash].add(port_number)\n except KeyError:\n self.open_port_cache[ip_hash] = {port_number}\n if proto == \"\" or port_number == \"\":\n continue\n yield ip, port_number\n\n def prep_blacklist(self):\n exclude = []\n for t in self.scan.blacklist:\n t = self.helpers.make_ip_type(t.data)\n if not isinstance(t, str):\n if self.helpers.is_ip(t):\n exclude.append(str(ipaddress.ip_network(t)))\n else:\n exclude.append(str(t))\n if not exclude:\n exclude = [\"255.255.255.255/32\"]\n self.exclude_file = self.helpers.tempfile(exclude, pipe=False)\n\n def _build_masscan_command(self, target_file=None, ping=False, dry_run=False, wait=None):\n if wait is None:\n wait = self.wait\n command = (\n \"masscan\",\n \"--excludefile\",\n str(self.exclude_file),\n \"--rate\",\n self.rate,\n \"--wait\",\n wait,\n \"--open-only\",\n \"-oJ\",\n \"-\",\n )\n if target_file is not None:\n command += (\"-iL\", str(target_file))\n if dry_run:\n command += (\"-p1\", \"--wait\", \"0\")\n else:\n if self.adapter:\n command += (\"--adapter\", self.adapter)\n if self.adapter_ip:\n command += (\"--adapter-ip\", self.adapter_ip)\n if self.adapter_mac:\n command += (\"--adapter-mac\", self.adapter_mac)\n if self.router_mac:\n command += (\"--router-mac\", self.router_mac)\n if ping:\n command += (\"--ping\",)\n else:\n if self.ports:\n command += (\"-p\", self.ports)\n else:\n command += (\"-p\", self.helpers.top_tcp_ports(self.top_ports, as_string=True))\n return command\n\n def log_masscan_status(self, s):\n if \"FAIL\" in s:\n self.warning(s)\n self.warning(\n 'Masscan failed to detect interface. Recommend passing \"adapter_ip\", \"adapter_mac\", and \"router_mac\" config options to portscan module.'\n )\n else:\n self.verbose(s)\n\n async def cleanup(self):\n with suppress(Exception):\n self.exclude_file.unlink()\n" }, { "path": "bbot/modules/postman.py", "content": "from bbot.modules.templates.postman import postman\n\n\nclass postman(postman):\n watched_events = [\"ORG_STUB\", \"SOCIAL\"]\n produced_events = [\"CODE_REPOSITORY\"]\n flags = [\"passive\", \"subdomain-enum\", \"safe\", \"code-enum\"]\n meta = {\n \"description\": \"Query Postman's API for related workspaces, collections, requests and download them\",\n \"created_date\": \"2024-09-07\",\n \"author\": \"@domwhewell-sage\",\n }\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"Postman API Key\"}\n reject_wildcards = False\n\n async def handle_event(self, event):\n # Handle postman profile\n if event.type == \"SOCIAL\":\n owner = event.data.get(\"profile_name\", \"\")\n in_scope_workspaces = await self.process_workspaces(user=owner)\n elif event.type == \"ORG_STUB\":\n owner = event.data\n in_scope_workspaces = await self.process_workspaces(org=owner)\n if in_scope_workspaces:\n for workspace in in_scope_workspaces:\n repo_url = workspace[\"url\"]\n repo_name = workspace[\"repo_name\"]\n if event.type == \"SOCIAL\":\n context = f'{{module}} searched postman.com for workspaces belonging to \"{owner}\" and found \"{repo_name}\" at {{event.type}}: {repo_url}'\n elif event.type == \"ORG_STUB\":\n context = f'{{module}} searched postman.com for \"{owner}\" and found matching workspace \"{repo_name}\" at {{event.type}}: {repo_url}'\n await self.emit_event(\n {\"url\": repo_url},\n \"CODE_REPOSITORY\",\n tags=\"postman\",\n parent=event,\n context=context,\n )\n\n async def process_workspaces(self, user=None, org=None):\n in_scope_workspaces = []\n owner = user or org\n if owner:\n self.verbose(f\"Searching for postman workspaces, collections, requests for {owner}\")\n for item in await self.query(owner):\n workspace = item[\"document\"]\n slug = workspace[\"slug\"]\n profile = workspace[\"publisherHandle\"]\n repo_url = f\"{self.html_url}/{profile}/{slug}\"\n workspace_id = await self.get_workspace_id(repo_url)\n if (org and workspace_id) or (user and owner.lower() == profile.lower()):\n self.verbose(f\"Found workspace ID {workspace_id} for {repo_url}\")\n data = await self.request_workspace(workspace_id)\n in_scope = await self.validate_workspace(\n data[\"workspace\"], data[\"environments\"], data[\"collections\"]\n )\n if in_scope:\n in_scope_workspaces.append({\"url\": repo_url, \"repo_name\": slug})\n else:\n self.verbose(\n f\"Failed to validate {repo_url} is in our scope as it does not contain any in-scope dns_names / emails\"\n )\n return in_scope_workspaces\n\n async def query(self, query):\n def api_page_iter(url, page, page_size, offset, **kwargs):\n kwargs[\"json\"][\"body\"][\"from\"] = offset\n return url, kwargs\n\n data = []\n url = f\"{self.base_url}/ws/proxy\"\n json = {\n \"service\": \"search\",\n \"method\": \"POST\",\n \"path\": \"/search-all\",\n \"body\": {\n \"queryIndices\": [\n \"collaboration.workspace\",\n ],\n \"queryText\": self.helpers.quote(query),\n \"size\": 25,\n \"from\": 0,\n \"clientTraceId\": \"\",\n \"requestOrigin\": \"srp\",\n \"mergeEntities\": \"true\",\n \"nonNestedRequests\": \"true\",\n \"domain\": \"public\",\n },\n }\n\n agen = self.api_page_iter(\n url, page_size=25, method=\"POST\", iter_key=api_page_iter, json=json, _json=False, headers=self.headers\n )\n async for r in agen:\n status_code = getattr(r, \"status_code\", 0)\n if status_code != 200:\n self.debug(f\"Reached end of postman search results (url: {r.url}) with status code {status_code}\")\n break\n try:\n data.extend(r.json().get(\"data\", []))\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n return None\n\n return data\n" }, { "path": "bbot/modules/postman_download.py", "content": "import zipfile\nimport json\nfrom pathlib import Path\nfrom bbot.modules.templates.postman import postman\n\n\nclass postman_download(postman):\n watched_events = [\"CODE_REPOSITORY\"]\n produced_events = [\"FILESYSTEM\"]\n flags = [\"passive\", \"subdomain-enum\", \"safe\", \"code-enum\", \"download\"]\n meta = {\n \"description\": \"Download workspaces, collections, requests from Postman\",\n \"created_date\": \"2024-09-07\",\n \"author\": \"@domwhewell-sage\",\n }\n options = {\"output_folder\": \"\", \"api_key\": \"\"}\n options_desc = {\n \"output_folder\": \"Folder to download postman workspaces to. If not specified, downloaded workspaces will be deleted when the scan completes, to minimize disk usage.\",\n \"api_key\": \"Postman API Key\",\n }\n scope_distance_modifier = 2\n\n async def setup(self):\n output_folder = self.config.get(\"output_folder\", \"\")\n if output_folder:\n self.output_dir = Path(output_folder) / \"postman_workspaces\"\n else:\n self.output_dir = self.scan.temp_dir / \"postman_workspaces\"\n self.helpers.mkdir(self.output_dir)\n return await super().setup()\n\n async def filter_event(self, event):\n if event.type == \"CODE_REPOSITORY\":\n if \"postman\" not in event.tags:\n return False, \"event is not a postman workspace\"\n return True\n\n async def handle_event(self, event):\n repo_url = event.data.get(\"url\")\n workspace_id = await self.get_workspace_id(repo_url)\n if workspace_id:\n self.verbose(f\"Found workspace ID {workspace_id} for {repo_url}\")\n data = await self.request_workspace(workspace_id)\n workspace = data[\"workspace\"]\n environments = data[\"environments\"]\n collections = data[\"collections\"]\n workspace_path = self.save_workspace(workspace, environments, collections)\n if workspace_path:\n self.verbose(f\"Downloaded workspace from {repo_url} to {workspace_path}\")\n codebase_event = self.make_event(\n {\"path\": str(workspace_path)}, \"FILESYSTEM\", tags=[\"postman\", \"workspace\"], parent=event\n )\n await self.emit_event(\n codebase_event,\n context=f\"{{module}} downloaded postman workspace at {repo_url} to {{event.type}}: {workspace_path}\",\n )\n\n def save_workspace(self, workspace, environments, collections):\n zip_path = None\n # Create a folder for the workspace\n name = workspace[\"name\"]\n id = workspace[\"id\"]\n folder = self.output_dir / name\n self.helpers.mkdir(folder)\n zip_path = folder / f\"{id}.zip\"\n\n # Main Workspace\n self.add_json_to_zip(zip_path, workspace, f\"{name}.postman_workspace.json\")\n\n # Workspace Environments\n if environments:\n for environment in environments:\n environment_id = environment[\"id\"]\n self.add_json_to_zip(zip_path, environment, f\"{environment_id}.postman_environment.json\")\n\n # Workspace Collections\n if collections:\n for collection in collections:\n collection_name = collection[\"info\"][\"name\"]\n self.add_json_to_zip(zip_path, collection, f\"{collection_name}.postman_collection.json\")\n return zip_path\n\n def add_json_to_zip(self, zip_path, data, filename):\n with zipfile.ZipFile(zip_path, \"a\") as zipf:\n json_content = json.dumps(data, indent=4)\n zipf.writestr(filename, json_content)\n" }, { "path": "bbot/modules/rapiddns.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass rapiddns(subdomain_enum):\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n meta = {\n \"description\": \"Query rapiddns.io for subdomains\",\n \"created_date\": \"2022-08-24\",\n \"author\": \"@TheTechromancer\",\n }\n\n base_url = \"https://rapiddns.io\"\n\n async def request_url(self, query):\n url = f\"{self.base_url}/subdomain/{self.helpers.quote(query)}?full=1#result\"\n response = await self.api_request(url, timeout=self.http_timeout + 10)\n return response\n\n async def parse_results(self, r, query):\n text = getattr(r, \"text\", \"\")\n return await self.scan.extract_in_scope_hostnames(text)\n" }, { "path": "bbot/modules/reflected_parameters.py", "content": "from bbot.modules.base import BaseModule\n\n\nclass reflected_parameters(BaseModule):\n watched_events = [\"WEB_PARAMETER\"]\n produced_events = [\"FINDING\"]\n flags = [\"active\", \"safe\", \"web-thorough\"]\n meta = {\n \"description\": \"Highlight parameters that reflect their contents in response body\",\n \"author\": \"@liquidsec\",\n \"created_date\": \"2024-10-29\",\n }\n\n async def handle_event(self, event):\n url = event.data.get(\"url\")\n reflection_detected = await self.detect_reflection(event, url)\n\n if reflection_detected:\n param_type = event.data.get(\"type\", \"UNKNOWN\")\n description = (\n f\"[{param_type}] Parameter value reflected in response body. Name: [{event.data['name']}] \"\n f\"Source Module: [{str(event.module)}]\"\n )\n if event.data.get(\"original_value\"):\n description += (\n f\" Original Value: [{self.helpers.truncate_string(str(event.data['original_value']), 200)}]\"\n )\n data = {\"host\": str(event.host), \"description\": description, \"url\": url}\n await self.emit_event(data, \"FINDING\", event)\n\n async def detect_reflection(self, event, url):\n \"\"\"Detects reflection by sending a probe with a random value and a canary parameter.\"\"\"\n probe_parameter_name = event.data[\"name\"]\n probe_parameter_value = self.helpers.rand_string()\n canary_parameter_value = self.helpers.rand_string()\n probe_response = await self.send_probe_with_canary(\n event,\n probe_parameter_name,\n probe_parameter_value,\n canary_parameter_value,\n cookies=event.data.get(\"assigned_cookies\", {}),\n timeout=10,\n )\n\n # Check if the probe parameter value is reflected AND the canary is not\n if probe_response:\n response_text = probe_response.text\n reflection_result = probe_parameter_value in response_text and canary_parameter_value not in response_text\n return reflection_result\n return False\n\n async def send_probe_with_canary(self, event, parameter_name, parameter_value, canary_value, cookies, timeout=10):\n method = \"GET\"\n url = event.data[\"url\"]\n headers = {}\n data = None\n json_data = None\n params = {parameter_name: parameter_value, \"c4n4ry\": canary_value}\n\n if event.data[\"type\"] == \"GETPARAM\":\n url = f\"{url}?{parameter_name}={parameter_value}&c4n4ry={canary_value}\"\n elif event.data[\"type\"] == \"COOKIE\":\n cookies.update(params)\n elif event.data[\"type\"] == \"HEADER\":\n headers.update(params)\n elif event.data[\"type\"] == \"POSTPARAM\":\n method = \"POST\"\n data = params\n elif event.data[\"type\"] == \"BODYJSON\":\n method = \"POST\"\n json_data = params\n\n self.debug(\n f\"Sending {method} request to {url} with headers: {headers}, cookies: {cookies}, data: {data}, json: {json_data}\"\n )\n\n response = await self.helpers.request(\n method=method, url=url, headers=headers, cookies=cookies, data=data, json=json_data, timeout=timeout\n )\n return response\n" }, { "path": "bbot/modules/report/affiliates.py", "content": "from bbot.modules.report.base import BaseReportModule\n\n\nclass affiliates(BaseReportModule):\n watched_events = [\"*\"]\n produced_events = []\n flags = [\"passive\", \"safe\", \"affiliates\"]\n meta = {\n \"description\": \"Summarize affiliate domains at the end of a scan\",\n \"created_date\": \"2022-07-25\",\n \"author\": \"@TheTechromancer\",\n }\n scope_distance_modifier = None\n accept_dupes = True\n\n async def setup(self):\n self.affiliates = {}\n return True\n\n async def handle_event(self, event):\n self.add_affiliate(event)\n\n async def report(self):\n affiliates = sorted(self.affiliates.items(), key=lambda x: x[-1][\"weight\"], reverse=True)\n header = [\"Affiliate\", \"Score\", \"Count\"]\n table = []\n for domain, stats in affiliates:\n count = stats[\"count\"]\n weight = stats[\"weight\"]\n table.append([domain, f\"{weight:.2f}\", f\"{count:,}\"])\n self.log_table(table, header, table_name=\"affiliates\", max_log_entries=50)\n\n def add_affiliate(self, event):\n if event.scope_distance > 0 and event.host and isinstance(event.host, str):\n subdomain, domain = self.helpers.split_domain(event.host)\n weight = (1 / event.scope_distance) + (1 if \"affiliate\" in event.tags else 0)\n if domain and not self.scan.in_scope(domain):\n try:\n self.affiliates[domain][\"weight\"] += weight\n self.affiliates[domain][\"count\"] += 1\n except KeyError:\n self.affiliates[domain] = {}\n self.affiliates[domain][\"weight\"] = weight\n self.affiliates[domain][\"count\"] = 1\n" }, { "path": "bbot/modules/report/asn.py", "content": "from bbot.modules.report.base import BaseReportModule\n\n\nclass asn(BaseReportModule):\n watched_events = [\"IP_ADDRESS\"]\n produced_events = [\"ASN\"]\n flags = [\"passive\", \"subdomain-enum\", \"safe\"]\n meta = {\n \"description\": \"Query ripe and bgpview.io for ASNs\",\n \"created_date\": \"2022-07-25\",\n \"author\": \"@TheTechromancer\",\n }\n scope_distance_modifier = 1\n # we accept dupes to avoid missing data\n # because sometimes IP addresses are re-emitted with lower scope distances\n accept_dupes = True\n\n async def setup(self):\n self.asn_counts = {}\n self.asn_cache = {}\n self.ripe_cache = {}\n self.sources = [\"bgpview\", \"ripe\"]\n self.unknown_asn = {\n \"asn\": \"UNKNOWN\",\n \"subnet\": \"0.0.0.0/32\",\n \"name\": \"unknown\",\n \"description\": \"unknown\",\n \"country\": \"\",\n }\n return True\n\n async def filter_event(self, event):\n if str(event.module) == \"ipneighbor\":\n return False\n if getattr(event.host, \"is_private\", False):\n return False\n return True\n\n async def handle_event(self, event):\n host = event.host\n if self.cache_get(host) is False:\n asns, source = await self.get_asn(host)\n if not asns:\n self.cache_put(self.unknown_asn)\n else:\n for asn in asns:\n emails = asn.pop(\"emails\", [])\n self.cache_put(asn)\n asn_event = self.make_event(asn, \"ASN\", parent=event)\n asn_number = asn.get(\"asn\", \"\")\n asn_desc = asn.get(\"description\", \"\")\n asn_name = asn.get(\"name\", \"\")\n asn_subnet = asn.get(\"subnet\", \"\")\n if not asn_event:\n continue\n await self.emit_event(\n asn_event,\n context=f\"{{module}} checked {event.data} against {source} API and got {{event.type}}: AS{asn_number} ({asn_name}, {asn_desc}, {asn_subnet})\",\n )\n for email in emails:\n await self.emit_event(\n email,\n \"EMAIL_ADDRESS\",\n parent=asn_event,\n context=f\"{{module}} retrieved details for AS{asn_number} and found {{event.type}}: {{event.data}}\",\n )\n\n async def report(self):\n asn_data = sorted(self.asn_cache.items(), key=lambda x: self.asn_counts[x[0]], reverse=True)\n if not asn_data:\n return\n header = [\"ASN\", \"Subnet\", \"Host Count\", \"Name\", \"Description\", \"Country\"]\n table = []\n for subnet, asn in asn_data:\n count = self.asn_counts[subnet]\n number = asn[\"asn\"]\n if number != \"UNKNOWN\":\n number = \"AS\" + number\n name = asn[\"name\"]\n country = asn[\"country\"]\n description = asn[\"description\"]\n table.append([number, str(subnet), f\"{count:,}\", name, description, country])\n self.log_table(table, header, table_name=\"asns\")\n\n def cache_put(self, asn):\n asn = dict(asn)\n subnet = self.helpers.make_ip_type(asn.pop(\"subnet\"))\n self.asn_cache[subnet] = asn\n try:\n self.asn_counts[subnet] += 1\n except KeyError:\n self.asn_counts[subnet] = 1\n\n def cache_get(self, ip):\n ret = False\n for p in self.helpers.ip_network_parents(ip):\n try:\n self.asn_counts[p] += 1\n if ret is False:\n ret = p\n except KeyError:\n continue\n return ret\n\n async def get_asn(self, ip, retries=1):\n \"\"\"\n Takes in an IP\n returns a list of ASNs, e.g.:\n [{'asn': '54113', 'subnet': '2606:50c0:8000::/48', 'name': 'FASTLY', 'description': 'Fastly', 'country': 'US', 'emails': []}, {'asn': '54113', 'subnet': '2606:50c0:8000::/46', 'name': 'FASTLY', 'description': 'Fastly', 'country': 'US', 'emails': []}]\n \"\"\"\n for attempt in range(retries + 1):\n for i, source in enumerate(list(self.sources)):\n get_asn_fn = getattr(self, f\"get_asn_{source}\")\n res = await get_asn_fn(ip)\n if res is False:\n # demote the current source to lowest priority since it just failed\n self.sources.append(self.sources.pop(i))\n self.verbose(f\"Failed to contact {source}, retrying\")\n continue\n return res, source\n self.warning(f\"Error retrieving ASN for {ip}\")\n return [], \"\"\n\n async def get_asn_ripe(self, ip):\n url = f\"https://stat.ripe.net/data/network-info/data.json?resource={ip}\"\n response = await self.get_url(url, \"ASN\")\n asns = []\n if response is False:\n return False\n data = response.get(\"data\", {})\n if not data:\n data = {}\n prefix = data.get(\"prefix\", \"\")\n asn_numbers = data.get(\"asns\", [])\n if not prefix or not asn_numbers:\n return []\n if not asn_numbers:\n asn_numbers = []\n for number in asn_numbers:\n asn = await self.get_asn_metadata_ripe(number)\n if asn is False:\n return False\n asn[\"subnet\"] = prefix\n asns.append(asn)\n return asns\n\n async def get_asn_metadata_ripe(self, asn_number):\n try:\n return self.ripe_cache[asn_number]\n except KeyError:\n metadata_keys = {\n \"name\": [\"ASName\", \"OrgId\"],\n \"description\": [\"OrgName\", \"OrgTechName\", \"RTechName\"],\n \"country\": [\"Country\"],\n }\n url = f\"https://stat.ripe.net/data/whois/data.json?resource={asn_number}\"\n response = await self.get_url(url, \"ASN Metadata\", cache=True)\n if response is False:\n return False\n data = response.get(\"data\", {})\n if not data:\n data = {}\n records = data.get(\"records\", [])\n if not records:\n records = []\n emails = set()\n asn = {k: \"\" for k in metadata_keys.keys()}\n for record in records:\n for item in record:\n key = item.get(\"key\", \"\")\n value = item.get(\"value\", \"\")\n for email in await self.helpers.re.extract_emails(value):\n emails.add(email.lower())\n if not key:\n continue\n if value:\n for keyname, keyvals in metadata_keys.items():\n if key in keyvals and not asn.get(keyname, \"\"):\n asn[keyname] = value\n asn[\"emails\"] = list(emails)\n asn[\"asn\"] = str(asn_number)\n self.ripe_cache[asn_number] = asn\n return asn\n\n async def get_asn_bgpview(self, ip):\n url = f\"https://api.bgpview.io/ip/{ip}\"\n data = await self.get_url(url, \"ASN\")\n asns = []\n asns_tried = set()\n if data is False:\n return False\n data = data.get(\"data\", {})\n prefixes = data.get(\"prefixes\", [])\n for prefix in prefixes:\n details = prefix.get(\"asn\", {})\n asn = str(details.get(\"asn\", \"\"))\n subnet = prefix.get(\"prefix\", \"\")\n if not (asn or subnet):\n continue\n name = details.get(\"name\") or prefix.get(\"name\") or \"\"\n description = details.get(\"description\") or prefix.get(\"description\") or \"\"\n country = details.get(\"country_code\") or prefix.get(\"country_code\") or \"\"\n emails = []\n if asn not in asns_tried:\n emails = await self.get_emails_bgpview(asn)\n if emails is False:\n return False\n asns_tried.add(asn)\n asns.append(\n {\n \"asn\": asn,\n \"subnet\": subnet,\n \"name\": name,\n \"description\": description,\n \"country\": country,\n \"emails\": emails,\n }\n )\n if not asns:\n self.debug(f'No results for \"{ip}\"')\n return asns\n\n async def get_emails_bgpview(self, asn):\n contacts = []\n url = f\"https://api.bgpview.io/asn/{asn}\"\n data = await self.get_url(url, \"ASN metadata\", cache=True)\n if data is False:\n return False\n data = data.get(\"data\", {})\n if not data:\n self.debug(f'No results for \"{asn}\"')\n return\n email_contacts = data.get(\"email_contacts\", [])\n abuse_contacts = data.get(\"abuse_contacts\", [])\n contacts = [l.strip().lower() for l in email_contacts + abuse_contacts]\n return list(set(contacts))\n\n async def get_url(self, url, data_type, cache=False):\n kwargs = {}\n if cache:\n kwargs[\"cache_for\"] = 60 * 60 * 24\n r = await self.helpers.request(url, **kwargs)\n data = {}\n try:\n j = r.json()\n if not isinstance(j, dict):\n return data\n return j\n except Exception as e:\n self.verbose(f\"Error retrieving {data_type} at {url}: {e}\", trace=True)\n self.debug(f\"Got data: {getattr(r, 'content', '')}\")\n return False\n" }, { "path": "bbot/modules/report/base.py", "content": "from bbot.modules.base import BaseModule\n\n\nclass BaseReportModule(BaseModule):\n _stats_exclude = True\n" }, { "path": "bbot/modules/retirejs.py", "content": "import json\nfrom enum import IntEnum\nfrom bbot.modules.base import BaseModule\n\n\nclass RetireJSSeverity(IntEnum):\n NONE = 0\n LOW = 1\n MEDIUM = 2\n HIGH = 3\n CRITICAL = 4\n\n @classmethod\n def from_string(cls, severity_str):\n try:\n return cls[severity_str.upper()]\n except (KeyError, AttributeError):\n return cls.NONE\n\n\nclass retirejs(BaseModule):\n watched_events = [\"URL_UNVERIFIED\"]\n produced_events = [\"FINDING\"]\n flags = [\"active\", \"safe\", \"web-thorough\"]\n meta = {\n \"description\": \"Detect vulnerable/out-of-date JavaScript libraries\",\n \"created_date\": \"2025-08-19\",\n \"author\": \"@liquidsec\",\n }\n options = {\n \"version\": \"5.3.0\",\n \"node_version\": \"18.19.1\",\n \"severity\": \"medium\",\n }\n options_desc = {\n \"version\": \"retire.js version\",\n \"node_version\": \"Node.js version to install locally\",\n \"severity\": \"Minimum severity level to report (none, low, medium, high, critical)\",\n }\n\n deps_ansible = [\n # Download Node.js binary (Linux x64)\n {\n \"name\": \"Download Node.js binary (Linux x64)\",\n \"get_url\": {\n \"url\": \"https://nodejs.org/dist/v#{BBOT_MODULES_RETIREJS_NODE_VERSION}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64.tar.xz\",\n \"dest\": \"#{BBOT_TEMP}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64.tar.xz\",\n \"mode\": \"0644\",\n },\n },\n # Extract Node.js binary (x64)\n {\n \"name\": \"Extract Node.js binary (x64)\",\n \"unarchive\": {\n \"src\": \"#{BBOT_TEMP}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64.tar.xz\",\n \"dest\": \"#{BBOT_TOOLS}\",\n \"remote_src\": True,\n },\n },\n # Remove existing node directory if it exists\n {\n \"name\": \"Remove existing node directory\",\n \"file\": {\"path\": \"#{BBOT_TOOLS}/node\", \"state\": \"absent\"},\n },\n # Rename extracted directory to 'node' (x64)\n {\n \"name\": \"Rename Node.js directory (x64)\",\n \"command\": \"mv #{BBOT_TOOLS}/node-v#{BBOT_MODULES_RETIREJS_NODE_VERSION}-linux-x64 #{BBOT_TOOLS}/node\",\n },\n # Set permissions on entire Node.js bin directory\n {\n \"name\": \"Set permissions on Node.js bin directory\",\n \"file\": {\"path\": \"#{BBOT_TOOLS}/node/bin\", \"mode\": \"0755\", \"recurse\": \"yes\"},\n },\n # Make Node.js binary executable\n {\n \"name\": \"Make Node.js binary executable\",\n \"file\": {\"path\": \"#{BBOT_TOOLS}/node/bin/node\", \"mode\": \"0755\"},\n },\n # Remove existing retirejs directory if it exists\n {\n \"name\": \"Remove existing retirejs directory\",\n \"file\": {\"path\": \"#{BBOT_TOOLS}/retirejs\", \"state\": \"absent\"},\n },\n # Create retire.js local directory\n {\n \"name\": \"Create retire.js directory in BBOT_TOOLS\",\n \"file\": {\"path\": \"#{BBOT_TOOLS}/retirejs\", \"state\": \"directory\", \"mode\": \"0755\"},\n },\n # Install retire.js locally using local Node.js\n {\n \"name\": \"Install retire.js locally\",\n \"shell\": \"cd #{BBOT_TOOLS}/retirejs && #{BBOT_TOOLS}/node/bin/node #{BBOT_TOOLS}/node/lib/node_modules/npm/bin/npm-cli.js install --prefix . retire@#{BBOT_MODULES_RETIREJS_VERSION} --no-fund --no-audit --silent --no-optional\",\n \"args\": {\"creates\": \"#{BBOT_TOOLS}/retirejs/node_modules/.bin/retire\"},\n \"timeout\": 600,\n \"ignore_errors\": False,\n },\n # Make retire script executable\n {\n \"name\": \"Make retire script executable\",\n \"file\": {\"path\": \"#{BBOT_TOOLS}/retirejs/node_modules/.bin/retire\", \"mode\": \"0755\"},\n },\n # Create retire cache directory\n {\n \"name\": \"Create retire cache directory\",\n \"file\": {\"path\": \"#{BBOT_CACHE}/retire_cache\", \"state\": \"directory\", \"mode\": \"0755\"},\n },\n ]\n\n accept_url_special = True\n scope_distance_modifier = 1\n _module_threads = 4\n\n async def setup(self):\n excavate_enabled = self.scan.config.get(\"excavate\")\n if not excavate_enabled:\n return None, \"retirejs will not function without excavate enabled\"\n\n # Validate severity level\n valid_severities = [\"none\", \"low\", \"medium\", \"high\", \"critical\"]\n configured_severity = self.config.get(\"severity\", \"medium\").lower()\n if configured_severity not in valid_severities:\n return (\n False,\n f\"Invalid severity level '{configured_severity}'. Valid options are: {', '.join(valid_severities)}\",\n )\n\n self.repofile = await self.helpers.download(\n \"https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/jsrepository-v4.json\", cache_hrs=24\n )\n if not self.repofile:\n return False, \"failed to download retire.js repository file\"\n return True\n\n async def handle_event(self, event):\n js_file = await self.helpers.request(event.data)\n if js_file:\n js_file_body = js_file.text\n if js_file_body:\n js_file_body_saved = self.helpers.tempfile(js_file_body, pipe=False, extension=\"js\")\n results = await self.execute_retirejs(js_file_body_saved)\n if not results:\n self.warning(\"no output from retire.js\")\n return\n results_json = json.loads(results)\n if results_json.get(\"data\"):\n for file_result in results_json[\"data\"]:\n for component_result in file_result.get(\"results\", []):\n component = component_result.get(\"component\", \"unknown\")\n version = component_result.get(\"version\", \"unknown\")\n vulnerabilities = component_result.get(\"vulnerabilities\", [])\n for vuln in vulnerabilities:\n severity = vuln.get(\"severity\", \"unknown\")\n\n # Filter by minimum severity level\n min_severity = RetireJSSeverity.from_string(self.config.get(\"severity\", \"medium\"))\n vuln_severity = RetireJSSeverity.from_string(severity)\n if vuln_severity < min_severity:\n self.debug(\n f\"Skipping vulnerability with severity '{severity}' (below minimum '{min_severity.name.lower()}')\"\n )\n continue\n\n identifiers = vuln.get(\"identifiers\", {})\n summary = identifiers.get(\"summary\", \"Unknown vulnerability\")\n cves = identifiers.get(\"CVE\", [])\n description_parts = [\n f\"Vulnerable JavaScript library detected: {component} v{version}\",\n f\"Severity: {severity.upper()}\",\n f\"Summary: {summary}\",\n f\"JavaScript URL: {event.data}\",\n ]\n if cves:\n description_parts.append(f\"CVE(s): {', '.join(cves)}\")\n\n below_version = vuln.get(\"below\", \"\")\n at_or_above = vuln.get(\"atOrAbove\", \"\")\n if at_or_above and below_version:\n description_parts.append(f\"Affected versions: [{at_or_above} to {below_version})\")\n elif below_version:\n description_parts.append(f\"Affected versions: [< {below_version}]\")\n elif at_or_above:\n description_parts.append(f\"Affected versions: [>= {at_or_above}]\")\n description = \" \".join(description_parts)\n data = {\n \"description\": description,\n \"severity\": severity,\n \"component\": component,\n \"url\": event.parent.data[\"url\"],\n }\n await self.emit_event(\n data,\n \"FINDING\",\n parent=event,\n context=f\"{{module}} identified vulnerable JavaScript library {component} v{version} ({severity} severity)\",\n )\n\n async def filter_event(self, event):\n url_extension = getattr(event, \"url_extension\", \"\")\n if url_extension != \"js\":\n return False, f\"it is a {url_extension} URL but retirejs only accepts js URLs\"\n return True\n\n async def execute_retirejs(self, js_file):\n cache_dir = self.helpers.cache_dir / \"retire_cache\"\n retire_dir = self.scan.helpers.tools_dir / \"retirejs\"\n local_node_dir = self.scan.helpers.tools_dir / \"node\"\n\n # Use the retire binary directly with our local Node.js\n retire_binary_path = retire_dir / \"node_modules\" / \".bin\" / \"retire\"\n command = [\n str(local_node_dir / \"bin\" / \"node\"),\n str(retire_binary_path),\n \"--outputformat\",\n \"json\",\n \"--cachedir\",\n str(cache_dir),\n \"--path\",\n js_file,\n \"--jsrepo\",\n str(self.repofile),\n ]\n\n proxy = self.scan.web_config.get(\"http_proxy\")\n if proxy:\n command.extend([\"--proxy\", proxy])\n\n self.verbose(f\"Running retire.js on {js_file}\")\n self.verbose(f\"retire.js command: {command}\")\n\n result = await self.run_process(command)\n return result.stdout\n" }, { "path": "bbot/modules/robots.py", "content": "from bbot.modules.base import BaseModule\n\n\nclass robots(BaseModule):\n watched_events = [\"URL\"]\n produced_events = [\"URL_UNVERIFIED\"]\n flags = [\"active\", \"safe\", \"web-basic\"]\n meta = {\"description\": \"Look for and parse robots.txt\", \"created_date\": \"2023-02-01\", \"author\": \"@liquidsec\"}\n\n options = {\"include_sitemap\": False, \"include_allow\": True, \"include_disallow\": True}\n options_desc = {\n \"include_sitemap\": \"Include 'sitemap' entries\",\n \"include_allow\": \"Include 'Allow' Entries\",\n \"include_disallow\": \"Include 'Disallow' Entries\",\n }\n\n in_scope_only = True\n per_hostport_only = True\n\n async def setup(self):\n return True\n\n async def handle_event(self, event):\n host = f\"{event.parsed_url.scheme}://{event.parsed_url.netloc}/\"\n result = None\n url = f\"{host}robots.txt\"\n result = await self.helpers.request(url)\n if result:\n body = result.text\n\n if body:\n lines = body.split(\"\\n\")\n for l in lines:\n if len(l) > 0:\n split_l = l.split(\": \")\n if (split_l[0].lower() == \"allow\" and self.config.get(\"include_allow\") is True) or (\n split_l[0].lower() == \"disallow\" and self.config.get(\"include_disallow\") is True\n ):\n unverified_url = f\"{host}{split_l[1].lstrip('/')}\".replace(\n \"*\", self.helpers.rand_string(4)\n )\n\n elif split_l[0].lower() == \"sitemap\" and self.config.get(\"include_sitemap\") is True:\n unverified_url = split_l[1]\n else:\n continue\n await self.emit_event(\n unverified_url,\n \"URL_UNVERIFIED\",\n parent=event,\n tags=[\"spider-danger\"],\n context=f\"{{module}} found robots.txt at {url} and extracted {{event.type}}: {{event.data}}\",\n )\n" }, { "path": "bbot/modules/securitytrails.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass securitytrails(subdomain_enum_apikey):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query the SecurityTrails API for subdomains\",\n \"created_date\": \"2022-07-03\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"SecurityTrails API key\"}\n\n base_url = \"https://api.securitytrails.com/v1\"\n ping_url = f\"{base_url}/ping?apikey={{api_key}}\"\n\n async def setup(self):\n self.limit = 100\n return await super().setup()\n\n async def request_url(self, query):\n url = f\"{self.base_url}/domain/{query}/subdomains?apikey={{api_key}}\"\n response = await self.api_request(url)\n return response\n\n async def parse_results(self, r, query):\n results = set()\n j = r.json()\n if isinstance(j, dict):\n for host in j.get(\"subdomains\", []):\n results.add(f\"{host}.{query}\")\n return results\n" }, { "path": "bbot/modules/securitytxt.py", "content": "# securitytxt.py\n#\n# Checks for/parses https://target.domain/.well-known/security.txt\n#\n# Refer to: https://securitytxt.org/\n#\n# security.txt may contain email addresses and URL's, and possibly IP addresses.\n#\n# Example security.txt:\n#\n# Contact: mailto:security.reports@example.com\n# Expires: 2028-05-31T14:00:00.000Z\n# Encryption: https://example.com/security.pgp\n# Preferred-Languages: en, es\n# Canonical: https://example.com/.well-known/security.txt\n# Canonical: https://www.example.com/.well-known/security.txt\n# Policy: https://example.com/security-policy.html\n# Hiring: https://example.com/jobs.html\n#\n# Example security.txt with PGP signature:\n#\n# -----BEGIN PGP SIGNED MESSAGE-----\n# Hash: SHA512\n#\n# Contact: https://vdp.example.com\n# Expires: 2025-01-01T00:00:00.000Z\n# Preferred-Languages: fr, en\n# Canonical: https://example.com/.well-known/security.txt\n# Policy: https://example.com/cert\n# Hiring: https://www.careers.example.com\n# -----BEGIN PGP SIGNATURE-----\n#\n# iQIzBAEBCgAdFiEELC1a63jHPhyV60KPsvWy9dDkrigFAmJBypcACgkQsvWy9dDk\n# rijXHQ//Qya3hUSy5PYW+fI3eFP1+ak6gYq3Cbzkf57cqiBhxGetIGIGNJ6mxgjS\n# KAuvXLMUWgZD73r//fjZ5v1lpuWmpt54+ecat4DgcVCvFKYpaH+KBlay8SX7XtQH\n# 9T2NXMcez353TMR3EUOdLwdBzGZprf0Ekg9EzaHKMk0k+A4D9CnSb8Y6BKDPC7wr\n# eadwDIR9ESo0va4sjjcllCG9MF5hqK25SfsKriCSEAMhse2FToEBbw8ImkPKowMN\n# whJ4MIVlBxybu6XoIyk3n7HRRduijywy7uV80pAkhk/hL6wiW3M956FiahfRI6ad\n# +Gky/Ri5TjwAE/x5DhUH8O2toPsn71DeIE4geKfz5d/v41K0yncdrHjzbj0CAHu3\n# wVWLKnEp8RVqTlOR8jU0HqQUQy8iZk4LY91ROv+QjG/jUTWlwun8Ljh+YUeJTMRp\n# MGftCdCrrYjIy5aEQqWztt+dXKac/9e1plq3yyfuW1L+wG3zS7X+NpIJgygMvEwT\n# L3dqfQf63sjk8kWIZMVnicHBlc6BiLqUn020l+pkIOr4MuuJmIlByhlnfqH7YM8k\n# VShwDx7rs4Hj08C7NVCYIySaM2jM4eNKGt9V5k1F1sklCVfYaT8OqOhJrzhcisOC\n# YcQDhjt/iZTR8SzrHO7kFZbaskIp2P7JMaPax2fov15AnNHQQq8=\n# =8vfR\n# -----END PGP SIGNATURE-----\n\nfrom bbot.modules.base import BaseModule\n\nimport re\n\nfrom bbot.core.helpers.regexes import email_regex, url_regexes\n\n_securitytxt_regex = r\"^(?P\\w+): *(?P.*)$\"\nsecuritytxt_regex = re.compile(_securitytxt_regex, re.I | re.M)\n\n\nclass securitytxt(BaseModule):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"EMAIL_ADDRESS\", \"URL_UNVERIFIED\"]\n flags = [\"subdomain-enum\", \"cloud-enum\", \"active\", \"web-basic\", \"safe\"]\n meta = {\n \"description\": \"Check for security.txt content\",\n \"author\": \"@colin-stubbs\",\n \"created_date\": \"2024-05-26\",\n }\n options = {\n \"emails\": True,\n \"urls\": True,\n }\n options_desc = {\n \"emails\": \"emit EMAIL_ADDRESS events\",\n \"urls\": \"emit URL_UNVERIFIED events\",\n }\n\n async def setup(self):\n self._emails = self.config.get(\"emails\", True)\n self._urls = self.config.get(\"urls\", True)\n return await super().setup()\n\n def _incoming_dedup_hash(self, event):\n # dedupe by parent\n parent_domain = self.helpers.parent_domain(event.data)\n return hash(parent_domain), \"already processed parent domain\"\n\n async def filter_event(self, event):\n if \"_wildcard\" in str(event.host).split(\".\"):\n return False, \"event is wildcard\"\n return True\n\n async def handle_event(self, event):\n tags = [\"securitytxt-policy\"]\n url = f\"https://{event.host}/.well-known/security.txt\"\n\n r = await self.helpers.request(url, method=\"GET\")\n\n if r is None or r.status_code != 200:\n # it doesn't look like we got a valid response...\n return\n\n try:\n s = r.text\n except Exception:\n s = \"\"\n\n # avoid parsing the response unless it looks, at a very basic level, like an actual security.txt\n s_lower = s.lower()\n if \"contact: \" in s_lower or \"expires: \" in s_lower:\n for securitytxt_match in securitytxt_regex.finditer(s):\n v = securitytxt_match.group(\"v\")\n\n for match in email_regex.finditer(v):\n start, end = match.span()\n email = v[start:end]\n\n if self._emails:\n await self.emit_event(email, \"EMAIL_ADDRESS\", parent=event, tags=tags)\n\n for url_regex in url_regexes:\n for match in url_regex.finditer(v):\n start, end = match.span()\n found_url = v[start:end]\n\n if found_url != url and self._urls is True:\n await self.emit_event(found_url, \"URL_UNVERIFIED\", parent=event, tags=tags)\n" }, { "path": "bbot/modules/shodan_dns.py", "content": "from bbot.modules.templates.shodan import shodan\n\n\nclass shodan_dns(shodan):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query Shodan for subdomains\",\n \"created_date\": \"2022-07-03\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"Shodan API key\"}\n\n base_url = \"https://api.shodan.io\"\n\n async def handle_event(self, event):\n await self.handle_event_paginated(event)\n\n def make_url(self, query):\n return f\"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}&page={{page}}\"\n\n async def parse_results(self, json, query):\n return [f\"{sub}.{query}\" for sub in json.get(\"subdomains\", [])]\n" }, { "path": "bbot/modules/shodan_idb.py", "content": "from bbot.modules.base import BaseModule\nimport time\n\n\nclass shodan_idb(BaseModule):\n \"\"\"\n Query IP in Shodan InternetDB, returning open ports, discovered technologies, and findings/vulnerabilities\n\n InternetDB is especially nice because it doesn't require an API key\n\n API reference: https://internetdb.shodan.io/docs\n\n Example API response:\n\n {\n \"cpes\": [\n \"cpe:/a:microsoft:internet_information_services\",\n \"cpe:/a:microsoft:outlook_web_access:15.0.1367\",\n ],\n \"hostnames\": [\n \"autodiscover.evilcorp.com\",\n \"mail.evilcorp.com\",\n ],\n \"ip\": \"1.2.3.4\",\n \"ports\": [\n 25,\n 80,\n 443,\n ],\n \"tags\": [\n \"starttls\",\n \"self-signed\",\n \"eol-os\"\n ],\n \"vulns\": [\n \"CVE-2021-26857\",\n \"CVE-2021-26855\"\n ]\n }\n \"\"\"\n\n watched_events = [\"IP_ADDRESS\", \"DNS_NAME\"]\n produced_events = [\"TECHNOLOGY\", \"VULNERABILITY\", \"FINDING\", \"OPEN_TCP_PORT\", \"DNS_NAME\"]\n flags = [\"passive\", \"safe\", \"portscan\", \"subdomain-enum\"]\n meta = {\n \"description\": \"Query Shodan's InternetDB for open ports, hostnames, technologies, and vulnerabilities\",\n \"created_date\": \"2023-12-22\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"retries\": None}\n options_desc = {\n \"retries\": \"How many times to retry API requests (e.g. after a 429 error). Overrides the global web.api_retries setting.\"\n }\n\n # we typically don't want to abort this module\n _api_failure_abort_threshold = 9999999999\n\n # since there are rate limits, we set a lower qsize\n # this way when our queue is full, we can give the API a break\n _qsize = 100\n\n base_url = \"https://internetdb.shodan.io\"\n\n async def setup(self):\n await super().setup()\n self.last_request_time = 0\n return True\n\n def _incoming_dedup_hash(self, event):\n return hash(self.get_ip(event))\n\n @property\n def api_retries(self):\n # allow the module to override global retry setting\n return self.config.get(\"retries\", None) or super().api_retries\n\n async def handle_event(self, event):\n ip = self.get_ip(event)\n if ip is None:\n return\n url = f\"{self.base_url}/{ip}\"\n\n # Rate limiting: ensure at least 1 second between requests\n current_time = time.time()\n time_since_last = current_time - self.last_request_time\n if time_since_last < 1:\n await self.helpers.sleep(1 - time_since_last)\n\n # Update the last request time\n self.last_request_time = time.time()\n\n r = await self.api_request(url)\n if r is None:\n self.debug(f\"No response for {event.data}\")\n return\n try:\n data = r.json()\n except Exception as e:\n self.verbose(f\"Error parsing JSON response from {url}: {e}\")\n self.trace()\n return\n if data:\n if r.status_code == 200:\n await self._parse_response(data=data, event=event, ip=ip)\n elif r.status_code == 404:\n detail = data.get(\"detail\", \"\")\n if detail:\n self.debug(f\"404 response for {url}: {detail}\")\n else:\n err_data = data.get(\"type\", \"\")\n err_msg = data.get(\"msg\", \"\")\n self.verbose(f\"Shodan error for {ip}: {err_data}: {err_msg}\")\n\n async def _parse_response(self, data: dict, event, ip):\n \"\"\"Handles emitting events from returned JSON\"\"\"\n data: dict # has keys: cpes, hostnames, ip, ports, tags, vulns\n ip = str(ip)\n query_host = ip if event.data == ip else f\"{event.data} ({ip})\"\n # ip is a string, ports is a list of ports, the rest is a list of strings\n for hostname in data.get(\"hostnames\", []):\n if hostname != event.data:\n await self.emit_event(\n hostname,\n \"DNS_NAME\",\n parent=event,\n context=f'{{module}} queried Shodan\\'s InternetDB API for \"{query_host}\" and found {{event.type}}: {{event.data}}',\n )\n for cpe in data.get(\"cpes\", []):\n await self.emit_event(\n {\"technology\": cpe, \"host\": str(event.host)},\n \"TECHNOLOGY\",\n parent=event,\n context=f'{{module}} queried Shodan\\'s InternetDB API for \"{query_host}\" and found {{event.type}}: {{event.data}}',\n )\n for port in data.get(\"ports\", []):\n await self.emit_event(\n self.helpers.make_netloc(event.data, port),\n \"OPEN_TCP_PORT\",\n parent=event,\n context=f'{{module}} queried Shodan\\'s InternetDB API for \"{query_host}\" and found {{event.type}}: {{event.data}}',\n )\n vulns = data.get(\"vulns\", [])\n if vulns:\n vulns_str = \", \".join([str(v) for v in vulns])\n await self.emit_event(\n {\"description\": f\"Shodan reported possible vulnerabilities: {vulns_str}\", \"host\": str(event.host)},\n \"FINDING\",\n parent=event,\n context=f'{{module}} queried Shodan\\'s InternetDB API for \"{query_host}\" and found potential {{event.type}}: {vulns_str}',\n )\n\n def get_ip(self, event):\n \"\"\"\n Get the first available IP address from an event (IP_ADDRESS or DNS_NAME)\n \"\"\"\n if event.type == \"IP_ADDRESS\":\n return event.host\n elif event.type == \"DNS_NAME\":\n # always try IPv4 first\n ipv6 = []\n ips = [h for h in event.resolved_hosts if self.helpers.is_ip(h)]\n for ip in sorted([str(ip) for ip in ips]):\n if self.helpers.is_ip(ip, version=4):\n return ip\n elif self.helpers.is_ip(ip, version=6):\n ipv6.append(ip)\n for ip in ipv6:\n return ip\n" }, { "path": "bbot/modules/sitedossier.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass sitedossier(subdomain_enum):\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n meta = {\n \"description\": \"Query sitedossier.com for subdomains\",\n \"created_date\": \"2023-08-04\",\n \"author\": \"@TheTechromancer\",\n }\n\n base_url = \"http://www.sitedossier.com/parentdomain\"\n max_pages = 10\n\n async def handle_event(self, event):\n query = self.make_query(event)\n async for hostname in self.query(query):\n try:\n hostname = self.helpers.validators.validate_host(hostname)\n except ValueError as e:\n self.verbose(e)\n continue\n if hostname and hostname.endswith(f\".{query}\") and not hostname == event.data:\n await self.emit_event(\n hostname,\n \"DNS_NAME\",\n event,\n abort_if=self.abort_if,\n context=f'{{module}} searched sitedossier.com for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n\n async def query(self, query, parse_fn=None, request_fn=None):\n results = set()\n base_url = f\"{self.base_url}/{self.helpers.quote(query)}\"\n url = str(base_url)\n for i, page in enumerate(range(1, 100 * self.max_pages + 2, 100)):\n self.verbose(f\"Fetching page #{i + 1} for {query}\")\n if page > 1:\n url = f\"{base_url}/{page}\"\n response = await self.helpers.request(url)\n if response is None:\n self.info(f'Query \"{query}\" failed (no response)')\n break\n if response.status_code == 302:\n self.verbose(\"Hit rate limit captcha\")\n break\n for match in await self.helpers.re.finditer_multi(self.scan.dns_regexes, response.text):\n hostname = match.group().lower()\n if hostname and hostname not in results:\n results.add(hostname)\n yield hostname\n if '= last_page:\n break\n\n for i, r in enumerate(responses):\n for email in await self.helpers.re.extract_emails(r.text):\n await self.emit_event(\n email,\n \"EMAIL_ADDRESS\",\n parent=event,\n context=f'{{module}} searched skymem.info for \"{query}\" and found {{event.type}} on page {i + 1}: {{event.data}}',\n )\n" }, { "path": "bbot/modules/smuggler.py", "content": "import sys\n\nfrom bbot.modules.base import BaseModule\n\n\n\"\"\"\nwrapper for https://github.com/defparam/smuggler.git\n\"\"\"\n\n\nclass smuggler(BaseModule):\n watched_events = [\"URL\"]\n produced_events = [\"FINDING\"]\n flags = [\"active\", \"aggressive\", \"slow\", \"web-thorough\"]\n meta = {\"description\": \"Check for HTTP smuggling\", \"created_date\": \"2022-07-06\", \"author\": \"@liquidsec\"}\n\n in_scope_only = True\n per_hostport_only = True\n\n deps_ansible = [\n {\n \"name\": \"Get smuggler repo\",\n \"git\": {\"repo\": \"https://github.com/defparam/smuggler.git\", \"dest\": \"#{BBOT_TOOLS}/smuggler\"},\n }\n ]\n\n async def handle_event(self, event):\n command = [\n sys.executable,\n f\"{self.scan.helpers.tools_dir}/smuggler/smuggler.py\",\n \"--no-color\",\n \"-q\",\n \"-u\",\n event.data,\n ]\n async for line in self.run_process_live(command):\n for f in line.split(\"\\r\"):\n if \"Issue Found\" in f:\n technique = f.split(\":\")[0].rstrip()\n text = f.split(\":\")[1].split(\"-\")[0].strip()\n description = f\"[HTTP SMUGGLER] [{text}] Technique: {technique}\"\n await self.emit_event(\n {\"host\": str(event.host), \"url\": event.data, \"description\": description},\n \"FINDING\",\n parent=event,\n context=f\"{{module}} scanned {event.data} and found HTTP smuggling ({{event.type}}): {text}\",\n )\n" }, { "path": "bbot/modules/social.py", "content": "import re\nfrom bbot.modules.base import BaseModule\n\n\nclass social(BaseModule):\n watched_events = [\"URL_UNVERIFIED\"]\n produced_events = [\"SOCIAL\"]\n meta = {\n \"description\": \"Look for social media links in webpages\",\n \"created_date\": \"2023-03-28\",\n \"author\": \"@TheTechromancer\",\n }\n flags = [\"passive\", \"safe\", \"social-enum\"]\n\n # platform name : (regex, case_sensitive)\n social_media_platforms = {\n \"linkedin\": (r\"linkedin.com/(?:in|company)/([a-zA-Z0-9-]+)\", False),\n \"facebook\": (r\"facebook.com/([a-zA-Z0-9.]+)\", False),\n \"twitter\": (r\"twitter.com/([a-zA-Z0-9_]{1,15})\", False),\n \"github\": (r\"github.com/([a-zA-Z0-9_-]+)\", False),\n \"instagram\": (r\"instagram.com/([a-zA-Z0-9_.]+)\", False),\n \"youtube\": (r\"youtube.com/@([a-zA-Z0-9_]+)\", False),\n \"bitbucket\": (r\"bitbucket.org/([a-zA-Z0-9_-]+)\", False),\n \"gitlab\": (r\"gitlab.(?:com|org)/([a-zA-Z0-9_-]+)\", False),\n \"discord\": (r\"discord.gg/([a-zA-Z0-9_-]+)\", True),\n \"docker\": (r\"hub.docker.com/[ru]/([a-zA-Z0-9_-]+)\", False),\n \"huggingface\": (r\"huggingface.co/([a-zA-Z0-9_-]+)\", False),\n \"postman\": (r\"www.postman.com/([a-zA-Z0-9_-]+)\", False),\n }\n\n scope_distance_modifier = 1\n\n async def setup(self):\n self.compiled_regexes = {k: (re.compile(v), c) for k, (v, c) in self.social_media_platforms.items()}\n return True\n\n async def handle_event(self, event):\n for platform, (regex, case_sensitive) in self.compiled_regexes.items():\n for match in regex.finditer(event.data):\n url = match.group()\n profile_name = match.groups()[0]\n if not case_sensitive:\n url = url.lower()\n profile_name = profile_name.lower()\n url = f\"https://{url}\"\n event_data = {\"platform\": platform, \"url\": url, \"profile_name\": profile_name}\n # only emit if the same event isn't already in the parent chain\n if not any(e.type == \"SOCIAL\" and e.data == event_data for e in event.get_parents()):\n social_event = self.make_event(\n event_data,\n \"SOCIAL\",\n parent=event,\n )\n await self.emit_event(\n social_event,\n context=f\"{{module}} detected {platform} {{event.type}} at {url}\",\n )\n" }, { "path": "bbot/modules/sslcert.py", "content": "import asyncio\nfrom OpenSSL import crypto\nfrom contextlib import suppress\n\nfrom bbot.errors import ValidationError\nfrom bbot.modules.base import BaseModule\nfrom bbot.core.helpers.async_helpers import NamedLock\nfrom bbot.core.helpers.web.ssl_context import ssl_context_noverify\n\n\nclass sslcert(BaseModule):\n watched_events = [\"OPEN_TCP_PORT\"]\n produced_events = [\"DNS_NAME\", \"EMAIL_ADDRESS\"]\n flags = [\"affiliates\", \"subdomain-enum\", \"email-enum\", \"active\", \"safe\", \"web-basic\"]\n meta = {\n \"description\": \"Visit open ports and retrieve SSL certificates\",\n \"created_date\": \"2022-03-30\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"timeout\": 5.0, \"skip_non_ssl\": True}\n options_desc = {\"timeout\": \"Socket connect timeout in seconds\", \"skip_non_ssl\": \"Don't try common non-SSL ports\"}\n deps_apt = [\"openssl\"]\n deps_pip = [\"pyOpenSSL~=25.3.0\"]\n _module_threads = 25\n scope_distance_modifier = 1\n _priority = 2\n\n async def setup(self):\n self.timeout = self.config.get(\"timeout\", 5.0)\n self.skip_non_ssl = self.config.get(\"skip_non_ssl\", True)\n self.non_ssl_ports = (22, 53, 80)\n\n # sometimes we run into a server with A LOT of SANs\n # these are usually stupid and useless, so we abort based on a different threshold\n # depending on whether the parent event is in scope\n self.in_scope_abort_threshold = 50\n self.out_of_scope_abort_threshold = 10\n\n self.hosts_visited = set()\n self.ip_lock = NamedLock()\n return True\n\n async def filter_event(self, event):\n if self.skip_non_ssl and event.port in self.non_ssl_ports:\n return False, f\"Port {event.port} doesn't typically use SSL\"\n return True\n\n async def handle_event(self, event):\n _host = event.host\n if event.port:\n port = event.port\n else:\n port = 443\n\n # turn hostnames into IP address(es)\n if self.helpers.is_ip(_host):\n hosts = [_host]\n else:\n hosts = list(await self.helpers.resolve(_host))\n\n if event.scope_distance == 0:\n abort_threshold = self.in_scope_abort_threshold\n else:\n abort_threshold = self.out_of_scope_abort_threshold\n\n tasks = [self.visit_host(host, port) for host in hosts]\n async for task in self.helpers.as_completed(tasks):\n result = await task\n if not isinstance(result, tuple) or not len(result) == 3:\n continue\n dns_names, emails, (host, port) = result\n if len(dns_names) > abort_threshold:\n netloc = self.helpers.make_netloc(host, port)\n self.verbose(\n f\"Skipping Subject Alternate Names (SANs) on {netloc} because number of hostnames ({len(dns_names):,}) exceeds threshold ({abort_threshold})\"\n )\n dns_names = dns_names[:1] + [n for n in dns_names[1:] if self.scan.in_scope(n)]\n for event_type, results in ((\"DNS_NAME\", set(dns_names)), (\"EMAIL_ADDRESS\", emails)):\n for event_data in results:\n if event_data is not None and event_data != event.data:\n self.debug(f\"Discovered new {event_type} via SSL certificate parsing: [{event_data}]\")\n try:\n ssl_event = self.make_event(event_data, event_type, parent=event, raise_error=True)\n parent_event = ssl_event.get_parent()\n if parent_event.scope_distance == 0:\n tags = [\"affiliate\"]\n else:\n tags = None\n if ssl_event:\n await self.emit_event(\n ssl_event,\n tags=tags,\n context=f\"{{module}} parsed SSL certificate at {event.data} and found {{event.type}}: {{event.data}}\",\n )\n except ValidationError as e:\n self.hugeinfo(f'Malformed {event_type} \"{event_data}\" at {event.data}')\n self.debug(f\"Invalid data at {host}:{port}: {e}\")\n\n def on_success_callback(self, event):\n parent_scope_distance = event.get_parent().scope_distance\n if parent_scope_distance == 0 and event.scope_distance > 0:\n event.add_tag(\"affiliate\")\n\n async def visit_host(self, host, port):\n host = self.helpers.make_ip_type(host)\n netloc = self.helpers.make_netloc(host, port)\n host_hash = hash((host, port))\n dns_names = []\n emails = set()\n async with self.ip_lock.lock(host_hash):\n if host_hash in self.hosts_visited:\n self.debug(f\"Already processed {host} on port {port}, skipping\")\n return [], [], (host, port)\n else:\n self.hosts_visited.add(host_hash)\n\n host = str(host)\n\n # Connect to the host\n try:\n transport, _ = await asyncio.wait_for(\n self.helpers.loop.create_connection(\n lambda: asyncio.Protocol(), host, port, ssl=ssl_context_noverify\n ),\n timeout=self.timeout,\n )\n except asyncio.TimeoutError:\n self.debug(f\"Timed out after {self.timeout} seconds while connecting to {netloc}\")\n return [], [], (host, port)\n except Exception as e:\n log_fn = self.warning\n if isinstance(e, OSError):\n log_fn = self.debug\n log_fn(f\"Error connecting to {netloc}: {e}\")\n return [], [], (host, port)\n finally:\n with suppress(Exception):\n transport.close()\n\n # Get the SSL object\n try:\n ssl_object = transport.get_extra_info(\"ssl_object\")\n except Exception as e:\n self.verbose(f\"Error getting ssl_object: {e}\", trace=True)\n return [], [], (host, port)\n\n # Get the certificate\n try:\n der = ssl_object.getpeercert(binary_form=True)\n except Exception as e:\n self.verbose(f\"Error getting peer cert: {e}\", trace=True)\n return [], [], (host, port)\n try:\n cert = crypto.load_certificate(crypto.FILETYPE_ASN1, der)\n except Exception as e:\n self.verbose(f\"Error loading certificate: {e}\", trace=True)\n return [], [], (host, port)\n issuer = cert.get_issuer()\n if issuer.emailAddress and self.helpers.regexes.email_regex.match(issuer.emailAddress):\n emails.add(issuer.emailAddress)\n subject = cert.get_subject()\n if subject.emailAddress and self.helpers.regexes.email_regex.match(subject.emailAddress):\n emails.add(subject.emailAddress)\n common_name = str(subject.commonName).lstrip(\"*.\").lower()\n dns_names = set(self.get_cert_sans(cert))\n with suppress(KeyError):\n dns_names.remove(common_name)\n dns_names = [common_name] + list(dns_names)\n return dns_names, list(emails), (host, port)\n\n @staticmethod\n def get_cert_sans(cert):\n sans = []\n raw_sans = None\n ext_count = cert.get_extension_count()\n for i in range(0, ext_count):\n ext = cert.get_extension(i)\n short_name = str(ext.get_short_name())\n if \"subjectAltName\" in short_name:\n raw_sans = str(ext)\n if raw_sans is not None:\n for raw_san in raw_sans.split(\",\"):\n hostname = raw_san.split(\":\", 1)[-1].strip().lower()\n # IPv6 addresses\n if hostname.startswith(\"[\") and hostname.endswith(\"]\"):\n hostname = hostname.strip(\"[]\")\n hostname = hostname.lstrip(\"*.\")\n sans.append(hostname)\n return sans\n" }, { "path": "bbot/modules/subdomaincenter.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass subdomaincenter(subdomain_enum):\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n meta = {\n \"description\": \"Query subdomain.center's API for subdomains\",\n \"created_date\": \"2023-07-26\",\n \"author\": \"@TheTechromancer\",\n }\n\n base_url = \"https://api.subdomain.center\"\n\n async def request_url(self, query):\n url = f\"{self.base_url}/?domain={self.helpers.quote(query)}\"\n response = await self.api_request(url)\n return response\n\n async def parse_results(self, r, query):\n results = set()\n json = r.json()\n if json and isinstance(json, list):\n results = set(json)\n return results\n" }, { "path": "bbot/modules/subdomainradar.py", "content": "import time\nimport asyncio\n\nfrom bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass SubdomainRadar(subdomain_enum_apikey):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query the Subdomain API for subdomains\",\n \"created_date\": \"2022-07-08\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\", \"group\": \"fast\", \"timeout\": 120}\n options_desc = {\n \"api_key\": \"SubDomainRadar.io API key\",\n \"group\": \"The enumeration group to use. Choose from fast, medium, deep\",\n \"timeout\": \"Timeout in seconds\",\n }\n\n base_url = \"https://api.subdomainradar.io\"\n ping_url = f\"{base_url}/profile\"\n group_choices = (\"fast\", \"medium\", \"deep\")\n\n # set this really high so the poll loop finishes as soon as possible\n _qsize = 9999999\n\n async def setup(self):\n self.group = self.config.get(\"group\", \"fast\").strip().lower()\n self.timeout = self.config.get(\"timeout\", 120)\n if self.group not in self.group_choices:\n return False, f'Invalid group: \"{self.group}\", please choose from {\",\".join(self.group_choices)}'\n success, reason = await self.require_api_key()\n if not success:\n return success, reason\n # convert groups to enumerators\n enumerators = {}\n response = await self.api_request(f\"{self.base_url}/enumerators/groups\")\n status_code = getattr(response, \"status_code\", 0)\n if status_code != 200:\n return False, f\"Failed to get enumerators: (HTTP status code: {status_code})\"\n else:\n try:\n j = response.json()\n except Exception:\n return False, \"Failed to get enumerators: failed to parse response as JSON\"\n for group in j:\n group_name = group.get(\"name\", \"\").strip().lower()\n if group_name:\n group_enumerators = []\n for enumerator in group.get(\"enumerators\", []):\n enumerator_name = enumerator.get(\"display_name\", \"\")\n if enumerator_name:\n group_enumerators.append(enumerator_name)\n if group_enumerators:\n enumerators[group_name] = group_enumerators\n\n self.enumerators = enumerators.get(self.group, [])\n if not self.enumerators:\n return False, f'No enumerators found for group: \"{self.group}\" ({self.enumerators})'\n\n self.enum_tasks = {}\n self.poll_task = asyncio.create_task(self.task_poll_loop())\n\n return True\n\n def prepare_api_request(self, url, kwargs):\n if self.api_key:\n kwargs[\"headers\"] = {\"Authorization\": f\"Bearer {self.api_key}\"}\n return url, kwargs\n\n async def handle_event(self, event):\n query = self.make_query(event)\n # start enumeration task\n url = f\"{self.base_url}/enumerate\"\n response = await self.api_request(\n url, method=\"POST\", json={\"domains\": [query], \"enumerators\": self.enumerators}\n )\n try:\n j = response.json()\n except Exception:\n self.warning(f\"Failed to parse response as JSON: {getattr(response, 'text', '')}\")\n return\n task_id = j.get(\"tasks\", {}).get(query, \"\")\n if not task_id:\n self.warning(f\"Failed to start enumeration for {query}\")\n return\n self.enum_tasks[query] = (task_id, time.time(), event)\n self.debug(f\"Started enumeration task for {query}; task id: {task_id}\")\n\n async def task_poll_loop(self):\n # async with self._task_counter.count(f\"{self.name}.task_poll_loop()\"):\n while 1:\n for query, (task_id, start_time, event) in list(self.enum_tasks.items()):\n url = f\"{self.base_url}/tasks/{task_id}\"\n response = await self.api_request(url)\n if getattr(response, \"status_code\", 0) == 200:\n finished = await self.parse_response(response, query, event)\n if finished:\n self.enum_tasks.pop(query)\n continue\n # if scan is finishing, consider timeout\n if self.scan.status == \"FINISHING\":\n if start_time + self.timeout < time.time():\n self.enum_tasks.pop(query)\n self.info(f\"Enumeration task for {query} timed out\")\n\n if self.scan.status == \"FINISHING\" and not self.enum_tasks:\n break\n await self.helpers.sleep(5)\n\n async def parse_response(self, response, query, event):\n j = response.json()\n status = j.get(\"status\", \"\")\n if status.lower() == \"completed\":\n for subdomain in j.get(\"subdomains\", []):\n hostname = subdomain.get(\"subdomain\", \"\")\n if hostname and hostname.endswith(f\".{query}\") and not hostname == event.data:\n await self.emit_event(\n hostname,\n \"DNS_NAME\",\n event,\n abort_if=self.abort_if,\n context=f'{{module}} searched SubDomainRadar.io API for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n return True\n return False\n\n async def finish(self):\n start_time = time.time()\n while self.enum_tasks and not self.poll_task.done():\n elapsed_time = time.time() - start_time\n if elapsed_time >= self.timeout:\n self.warning(f\"Timed out waiting for the following tasks to finish: {self.enum_tasks}\")\n for query, (task_id, _, _) in list(self.enum_tasks.items()):\n url = f\"{self.base_url}/tasks/{task_id}\"\n self.warning(f\" - {query} ({url})\")\n break\n\n self.verbose(\n f\"Waiting for enumeration task poll loop to finish ({int(elapsed_time)}/{self.timeout} seconds)\"\n )\n\n try:\n # Wait for the task to complete or for 10 seconds, whichever comes first\n await asyncio.wait_for(asyncio.shield(self.poll_task), timeout=10)\n except asyncio.TimeoutError:\n # This just means our 10-second check has elapsed, not that the task failed\n pass\n\n # Cancel the poll_task if it's still running\n if not self.poll_task.done():\n self.poll_task.cancel()\n try:\n await self.poll_task\n except asyncio.CancelledError:\n pass\n" }, { "path": "bbot/modules/telerik.py", "content": "from sys import executable\n\nfrom bbot.modules.base import BaseModule\n\n\nclass telerik(BaseModule):\n \"\"\"\n Test for endpoints associated with Telerik.Web.UI.dll\n\n Telerik.Web.UI.WebResource.axd (CVE-2017-11317)\n Telerik.Web.UI.DialogHandler.aspx (CVE-2017-9248)\n Telerik.Web.UI.SpellCheckHandler.axd (associated with CVE-2017-9248)\n ChartImage.axd (CVE-2019-19790)\n\n For the Telerik Report Server vulnerability (CVE-2024-4358) Use the Nuclei Template: (https://github.com/projectdiscovery/nuclei-templates/blob/main/http/cves/2024/CVE-2024-4358.yaml)\n\n With exploit_RAU_crypto enabled, the module will attempt to exploit CVE-2017-11317. THIS WILL UPLOAD A (benign) FILE IF SUCCESSFUL.\n\n Will dedupe to host by default (running against first received URL). With include_subdirs enabled, will run against every directory.\n \"\"\"\n\n watched_events = [\"URL\", \"HTTP_RESPONSE\"]\n produced_events = [\"VULNERABILITY\", \"FINDING\"]\n flags = [\"active\", \"aggressive\", \"web-thorough\"]\n meta = {\n \"description\": \"Scan for critical Telerik vulnerabilities\",\n \"created_date\": \"2022-04-10\",\n \"author\": \"@liquidsec\",\n }\n\n telerikVersions = [\n \"2007.1423\",\n \"2007.1521\",\n \"2007.1626\",\n \"2007.2918\",\n \"2007.21010\",\n \"2007.21107\",\n \"2007.31218\",\n \"2007.31314\",\n \"2007.31425\",\n \"2008.1415\",\n \"2008.1515\",\n \"2008.1619\",\n \"2008.2723\",\n \"2008.2826\",\n \"2008.21001\",\n \"2008.31105\",\n \"2008.31125\",\n \"2008.31314\",\n \"2009.1311\",\n \"2009.1402\",\n \"2009.1527\",\n \"2009.2701\",\n \"2009.2826\",\n \"2009.31103\",\n \"2009.31208\",\n \"2009.31314\",\n \"2010.1309\",\n \"2010.1415\",\n \"2010.1519\",\n \"2010.2713\",\n \"2010.2826\",\n \"2010.2929\",\n \"2010.31109\",\n \"2010.31215\",\n \"2010.31317\",\n \"2011.1315\",\n \"2011.1413\",\n \"2011.1519\",\n \"2011.2712\",\n \"2011.2915\",\n \"2011.31115\",\n \"2011.3.1305\",\n \"2012.1.215\",\n \"2012.1.411\",\n \"2012.2.607\",\n \"2012.2.724\",\n \"2012.2.912\",\n \"2012.3.1016\",\n \"2012.3.1205\",\n \"2012.3.1308\",\n \"2013.1.220\",\n \"2013.1.403\",\n \"2013.1.417\",\n \"2013.2.611\",\n \"2013.2.717\",\n \"2013.3.1015\",\n \"2013.3.1114\",\n \"2013.3.1324\",\n \"2014.1.225\",\n \"2014.1.403\",\n \"2014.2.618\",\n \"2014.2.724\",\n \"2014.3.1024\",\n \"2015.1.204\",\n \"2015.1.225\",\n \"2015.2.604\",\n \"2015.2.623\",\n \"2015.2.729\",\n \"2015.2.826\",\n \"2015.3.930\",\n \"2015.3.1111\",\n \"2016.1.113\",\n \"2016.1.225\",\n \"2016.2.504\",\n \"2016.2.607\",\n \"2016.3.914\",\n \"2016.3.1018\",\n \"2016.3.1027\",\n \"2016.1.1213\",\n \"2017.1.118\",\n \"2017.1.228\",\n \"2017.2.503\",\n \"2017.2.621\",\n \"2017.2.711\",\n \"2017.3.913\",\n ]\n\n DialogHandlerUrls = [\n \"Telerik.Web.UI.DialogHandler.aspx\",\n \"Telerik.Web.UI.DialogHandler.axd\",\n \"Admin/ServerSide/Telerik.Web.UI.DialogHandler.aspx\",\n \"App_Master/Telerik.Web.UI.DialogHandler.aspx\",\n \"AsiCommon/Controls/ContentManagement/ContentDesigner/Telerik.Web.UI.DialogHandler.aspx\",\n \"cms/portlets/telerik.web.ui.dialoghandler.aspx\",\n \"common/admin/Calendar/Telerik.Web.UI.DialogHandler.aspx\",\n \"common/admin/Jobs2/Telerik.Web.UI.DialogHandler.aspx\",\n \"common/admin/PhotoGallery2/Telerik.Web.UI.DialogHandler.aspx\",\n \"dashboard/UserControl/CMS/Page/Telerik.Web.UI.DialogHandler.aspx\",\n \"DesktopModule/UIQuestionControls/UIAskQuestion/Telerik.Web.UI.DialogHandler.aspx\",\n \"Desktopmodules/Admin/dnnWerk.Users/DialogHandler.aspx\",\n \"DesktopModules/Admin/RadEditorProvider/DialogHandler.aspx\",\n \"desktopmodules/base/editcontrols/telerik.web.ui.dialoghandler.aspx\",\n \"desktopmodules/dnnwerk.radeditorprovider/dialoghandler.aspx\",\n \"DesktopModules/RadEditorProvider/telerik.web.ui.dialoghandler.aspx\",\n \"desktopmodules/tcmodules/tccategory/telerik.web.ui.dialoghandler.aspx\",\n \"desktopmodules/telerikwebui/radeditorprovider/telerik.web.ui.dialoghandler.aspx\",\n \"DesktopModules/TNComments/Telerik.Web.UI.DialogHandler.aspx\",\n \"dotnetnuke/DesktopModules/Admin/RadEditorProvider/DialogHandler.aspx\",\n \"Modules/CMS/Telerik.Web.UI.DialogHandler.aspx\",\n \"modules/shop/manage/telerik.web.ui.dialoghandler.aspx\",\n \"portal/channels/fa/Cms_HtmlText_Manage/Telerik.Web.UI.DialogHandler.aspx\",\n \"providers/htmleditorproviders/telerik/telerik.web.ui.dialoghandler.aspx\",\n \"Resources/Telerik.Web.UI.DialogHandler.aspx\",\n \"sitecore/shell/applications/contentmanager/telerik.web.ui.dialoghandler.aspx\",\n \"sitecore/shell/Controls/RichTextEditor/Telerik.Web.UI.DialogHandler.aspx\",\n \"Sitefinity/ControlTemplates/Blogs/Telerik.Web.UI.DialogHandler.aspx\",\n \"SiteTemplates/Telerik.Web.UI.DialogHandler.aspx\",\n \"static/usercontrols/Telerik.Web.UI.DialogHandler.aspx\",\n \"system/providers/htmleditor/Telerik.Web.UI.DialogHandler.aspx\",\n \"WebUIDialogs/Telerik.Web.UI.DialogHandler.aspx\",\n ]\n\n RAUConfirmed = []\n\n options = {\"exploit_RAU_crypto\": False, \"include_subdirs\": False}\n options_desc = {\n \"exploit_RAU_crypto\": \"Attempt to confirm any RAU AXD detections are vulnerable\",\n \"include_subdirs\": \"Include subdirectories in the scan (off by default)\", # will create many finding events if used in conjunction with web spider or ffuf\n }\n\n in_scope_only = True\n\n deps_pip = [\"pycryptodome~=3.23.0\"]\n\n deps_ansible = [\n {\"name\": \"Create telerik dir\", \"file\": {\"state\": \"directory\", \"path\": \"#{BBOT_TOOLS}/telerik/\"}},\n {\"file\": {\"state\": \"touch\", \"path\": \"#{BBOT_TOOLS}/telerik/testfile.txt\"}},\n {\n \"name\": \"Download RAU_crypto\",\n \"unarchive\": {\n \"src\": \"https://github.com/bao7uo/RAU_crypto/archive/refs/heads/master.zip\",\n \"include\": \"RAU_crypto-master/RAU_crypto.py\",\n \"dest\": \"#{BBOT_TOOLS}/telerik/\",\n \"remote_src\": True,\n },\n },\n ]\n\n _module_threads = 5\n\n @staticmethod\n def normalize_url(url):\n return str(url.rstrip(\"/\") + \"/\").lower()\n\n def _incoming_dedup_hash(self, event):\n if event.type == \"URL\":\n if self.config.get(\"include_subdirs\") is True:\n return hash(f\"{event.type}{self.normalize_url(event.data)}\")\n else:\n return hash(f\"{event.type}{event.netloc}\")\n else: # HTTP_RESPONSE\n return hash(f\"{event.type}{event.data['url']}\")\n\n async def handle_event(self, event):\n if event.type == \"URL\":\n if self.config.get(\"include_subdirs\"):\n base_url = self.normalize_url(event.data) # Use the entire URL including subdirectories\n\n else:\n base_url = f\"{event.parsed_url.scheme}://{event.parsed_url.netloc}/\" # path will be omitted\n\n # Check for RAU AXD Handler\n webresource = \"Telerik.Web.UI.WebResource.axd?type=rau\"\n result, _ = await self.test_detector(base_url, webresource)\n if result:\n if \"RadAsyncUpload handler is registered succesfully\" in result.text:\n self.verbose(\"Detected Telerik instance (Telerik.Web.UI.WebResource.axd?type=rau)\")\n\n probe_data = {\n \"rauPostData\": (\n None,\n \"mQheol55IDiQWWSxl+Atkc68JXWUJ6QSirwLhEwleMiw3vN4cwABE74V2fWsLGg8CFXHOP6np90M+sLrLDqFACGNvonxmgT8aBsTZPWbXErewMGNWBP34aX0DmMvXVyTEpQ6FkFhZi19cTtdYfRLI8Uc04uNSsdWnltDMQ2CX/sSLOXUFNnZdAwAXgUuprYhU28Zwh/GdgYh447ksXfAC2fuPqEJqKDDwBlltxsS/zSq8ipIg326ymB2dmOpH/P3hcAmTKOyzB0dW6a6pmJvqNVU+50DlrUC00RbBbTJwlV6Xm4s4XTvgXLvMQ6czz2OAYY18HI+HYX5uvajctj/25UR8edwu68ZCgedsD7EZHRSSthjxohxfAyrfshjcu1LnhCEd0ClowKxBS4eiaLxVxhJAdB7XcbbXxIS9WWKa7gtRMNc/jUAOlIpvOZ3N+bOQ6rsNMHv7TZk1g0bxPl99yBn9qvtAwDMNPDoADxoBSisAkIIl9mImKv7y7nAiKoj7ukApdu5XQuVo10SxwkLkqHcvEEgjxTrOlCbEbxK2/du9TgXxD9iqKyaPLHPzNZsnzCsG6qNXv0fNkeASP9tZAyvi/y1eLrpScE+J7blfT+kBkGPTTFc6Z4z6lN7GqSHofq/CDHC2S2+qdoRdC3C25V74j+Ae6MkpSfqYx4KZYNtxBAxjf9Uf3JVSiZh3X2W/7aFeimFft0h/liybSjJTzO+AwNJluI4kXqemFoHnjVFfUQViaIuk4UP0D861kCU6KIGLZLpOaa0g0KM8hmu3OjwVOy8QVXYtbx5lOmSX9h3imRzMDFRTXK25YpUJgD0/LFMgCeZLA8SCYzkThyN2d8f8n5l8iOScR47o8i8sqCp/fd3JTogSbwD7LxnHudpiw2W/OfpMGipgc6loQFoX4klQaYwKkA4w+GUzahfAJmIiukZuTLOPCPQvX4wKtLqw1YiHtuaLHvLYq2/F66QQXNrZ4SucUNED0p5TUVTvHGUbuA0zxAyYSfYVgTNZjXGguQBY7DsN1SkpCa/ltvIiGtCbHQR86OrvjJMACe0wdpMCqEg7JiGym3RrLqvmjpS&sbZRwxJ96gmXFBSbSvT0ve7jpvDoieqd6RbG+GIP0H7sO5/0ZnvheosB9jQAifuMabY7lW4UzZgr5o2iqE0tBl4SGhfWyYW7iCFXnd3aIuCnUvhT58Rp8g7kGkA/eU/s68E66KOBXNuBnokZR9cIsjE0Tt3Jfxrk018+CmVcXpjXp/RmhRwCJTgEAXQuNplb/KdkLxqDn519iRtbiU6aLZX8YctdFQBqyKVgkk8WYXxcXQ8wYnxtpEtGuBcsndUi1iPp4Od8rYY1HPWg+FIquW17YPHjfP4gO4dhZe4sd7gH0ARyGDjiYVj7ODDE0wGmwmFVdQTrDX5AaxKuJy0NbQ==\",\n ),\n \"file\": (\"blob\", b\"e1daf48a\", \"application/octet-stream\"),\n \"fileName\": (None, \"df8dbc7a\"),\n \"contentType\": (None, \"text/html\"),\n \"lastModifiedDate\": (None, \"2020-01-02T08:02:01.067Z\"),\n \"metadata\": (\n None,\n '{\"TotalChunks\":1,\"ChunkIndex\":0,\"TotalFileSize\":1,\"UploadID\":\"3ea7b19db6c5.txt\"}',\n ),\n }\n\n version = \"unknown\"\n verbose_errors = False\n # send probe\n probe_response = await self.helpers.request(\n f\"{event.data}{webresource}\", method=\"POST\", files=probe_data\n )\n\n if probe_response:\n if \"Exception Details: \" in probe_response.text:\n verbose_errors = True\n if (\n \"Telerik.Web.UI.CryptoExceptionThrower.ThrowGenericCryptoException\"\n in probe_response.text\n ):\n version = \"Post-2020 (Encrypt-Then-Mac Enabled, with Generic Crypto Failure Message)\"\n elif \"Padding is invalid and cannot be removed\" in probe_response.text:\n version = \"<= 2019 (Either Pre-2017 (vulnerable), or 2017-2019 w/ Encrypt-Then-Mac)\"\n\n description = f\"Telerik RAU AXD Handler detected. Verbose Errors Enabled: [{str(verbose_errors)}] Version Guess: [{version}]\"\n await self.emit_event(\n {\"host\": str(event.host), \"url\": f\"{base_url}{webresource}\", \"description\": description},\n \"FINDING\",\n event,\n context=f\"{{module}} scanned {base_url} and identified {{event.type}}: Telerik RAU AXD Handler\",\n )\n if self.config.get(\"exploit_RAU_crypto\") is True:\n if base_url not in self.RAUConfirmed:\n self.RAUConfirmed.append(base_url)\n root_tool_path = self.scan.helpers.tools_dir / \"telerik\"\n self.debug(root_tool_path)\n\n for version in self.telerikVersions:\n command = [\n executable,\n str(root_tool_path / \"RAU_crypto-master/RAU_crypto.py\"),\n \"-P\",\n \"C:\\\\\\\\Windows\\\\\\\\Temp\",\n version,\n str(root_tool_path / \"testfile.txt\"),\n result.url,\n ]\n\n # Add proxy if set in the scan config\n if self.scan.http_proxy:\n command.append(self.scan.http_proxy)\n\n output = await self.run_process(command)\n description = f\"[CVE-2017-11317] [{str(version)}] {webresource}\"\n if \"fileInfo\" in output.stdout:\n self.debug(f\"Confirmed Vulnerable Telerik (version: {str(version)}\")\n await self.emit_event(\n {\n \"severity\": \"CRITICAL\",\n \"description\": description,\n \"host\": str(event.host),\n \"url\": f\"{base_url}{webresource}\",\n },\n \"VULNERABILITY\",\n event,\n context=f\"{{module}} scanned {base_url} and identified critical {{event.type}}: {description}\",\n )\n break\n\n urls = {}\n for dh in self.DialogHandlerUrls:\n url = self.create_url(base_url, f\"{dh}?dp=1\")\n urls[url] = dh\n\n gen = self.helpers.request_batch(list(urls))\n fail_count = 0\n async for url, response in gen:\n # cancel if we run into timeouts etc.\n if response is None:\n fail_count += 1\n\n # tolerate some random errors\n if fail_count < 2:\n continue\n self.debug(f\"Cancelling run against {base_url} due to failed request\")\n await gen.aclose()\n else:\n if \"Cannot deserialize dialog parameters\" in response.text:\n self.debug(f\"Detected Telerik UI instance ({dh})\")\n description = \"Telerik DialogHandler detected\"\n await self.emit_event(\n {\"host\": str(event.host), \"url\": f\"{base_url}{dh}\", \"description\": description},\n \"FINDING\",\n event,\n )\n # Once we have a match we need to stop, because the basic handler (Telerik.Web.UI.DialogHandler.aspx) usually works with a path wildcard\n await gen.aclose()\n\n spellcheckhandler = \"Telerik.Web.UI.SpellCheckHandler.axd\"\n result, _ = await self.test_detector(base_url, spellcheckhandler)\n status_code = getattr(result, \"status_code\", 0)\n # The standard behavior for the spellcheck handler without parameters is a 500\n if status_code == 500:\n # Sometimes webapps will just return 500 for everything, so rule out the false positive\n validate_result, _ = await self.test_detector(base_url, f\"{self.helpers.rand_string()}.axd\")\n self.debug(validate_result)\n validate_status_code = getattr(validate_result, \"status_code\", 0)\n if validate_status_code not in (0, 500):\n self.debug(\"Detected Telerik UI instance (Telerik.Web.UI.SpellCheckHandler.axd)\")\n description = \"Telerik SpellCheckHandler detected\"\n await self.emit_event(\n {\n \"host\": str(event.host),\n \"url\": f\"{base_url}{spellcheckhandler}\",\n \"description\": description,\n },\n \"FINDING\",\n event,\n context=f\"{{module}} scanned {base_url} and identified {{event.type}}: Telerik SpellCheckHandler\",\n )\n\n chartimagehandler = \"ChartImage.axd?ImageName=bqYXJAqm315eEd6b%2bY4%2bGqZpe7a1kY0e89gfXli%2bjFw%3d\"\n result, _ = await self.test_detector(base_url, chartimagehandler)\n status_code = getattr(result, \"status_code\", 0)\n if status_code == 200:\n chartimagehandler_error = \"ChartImage.axd?ImageName=\"\n result_error, _ = await self.test_detector(base_url, chartimagehandler_error)\n error_status_code = getattr(result_error, \"status_code\", 0)\n if error_status_code not in (0, 200):\n await self.emit_event(\n {\n \"host\": str(event.host),\n \"url\": f\"{base_url}{chartimagehandler}\",\n \"description\": \"Telerik ChartImage AXD Handler Detected\",\n },\n \"FINDING\",\n event,\n context=f\"{{module}} scanned {base_url} and identified {{event.type}}: Telerik ChartImage AXD Handler\",\n )\n\n elif event.type == \"HTTP_RESPONSE\":\n resp_body = event.data.get(\"body\", None)\n url = event.data[\"url\"]\n if resp_body:\n if '\":{\"SerializedParameters\":\"' in resp_body:\n await self.emit_event(\n {\n \"host\": str(event.host),\n \"url\": url,\n \"description\": \"Telerik DialogHandler [SerializedParameters] Detected in HTTP Response\",\n },\n \"FINDING\",\n event,\n context=\"{module} searched HTTP_RESPONSE and identified {event.type}: Telerik ChartImage AXD Handler\",\n )\n elif '\"_serializedConfiguration\":\"' in resp_body:\n await self.emit_event(\n {\n \"host\": str(event.host),\n \"url\": url,\n \"description\": \"Telerik AsyncUpload [serializedConfiguration] Detected in HTTP Response\",\n },\n \"FINDING\",\n event,\n context=\"{module} searched HTTP_RESPONSE and identified {event.type}: Telerik AsyncUpload\",\n )\n\n def create_url(self, baseurl, detector):\n return f\"{baseurl}{detector}\"\n\n async def test_detector(self, baseurl, detector):\n result = None\n url = self.create_url(baseurl, detector)\n result = await self.helpers.request(url, timeout=self.scan.httpx_timeout)\n return result, detector\n\n async def filter_event(self, event):\n if event.type == \"URL\" and \"endpoint\" in event.tags:\n return False\n else:\n return True\n" }, { "path": "bbot/modules/templates/bucket.py", "content": "import importlib\nimport regex as re\nfrom functools import cached_property\nfrom bbot.modules.base import BaseModule\n\n\nclass bucket_template(BaseModule):\n watched_events = [\"DNS_NAME\", \"STORAGE_BUCKET\"]\n produced_events = [\"STORAGE_BUCKET\", \"FINDING\"]\n flags = [\"active\", \"safe\", \"cloud-enum\", \"web-basic\"]\n options = {\"permutations\": False}\n options_desc = {\n \"permutations\": \"Whether to try permutations\",\n }\n scope_distance_modifier = 3\n\n cloudcheck_provider_name = \"Amazon|Google|DigitalOcean|etc\"\n delimiters = (\"\", \".\", \"-\")\n base_domains = [\"s3.amazonaws.com|digitaloceanspaces.com|etc\"]\n regions = [None]\n supports_open_check = True\n\n async def setup(self):\n self.buckets_tried = set()\n self.permutations = self.config.get(\"permutations\", False)\n cloudcheck_import_path = \"cloudcheck.providers\"\n try:\n self.cloudcheck_provider = getattr(\n importlib.import_module(cloudcheck_import_path), self.cloudcheck_provider_name\n )\n except (ImportError, AttributeError) as e:\n return False, f\"cloud helper at {cloudcheck_import_path} not found: {e}\"\n return True\n\n async def filter_event(self, event):\n if event.type == \"DNS_NAME\" and event.scope_distance > 0:\n return False, \"only accepts in-scope DNS_NAMEs\"\n if event.type == \"STORAGE_BUCKET\":\n filter_result, reason = self.filter_bucket(event)\n if not filter_result:\n return (filter_result, reason)\n return True\n\n def filter_bucket(self, event):\n if not any(t.endswith(f\"-{self.cloudcheck_provider_name.lower()}\") for t in event.tags):\n return False, \"bucket belongs to a different cloud provider\"\n return True, \"\"\n\n async def handle_event(self, event):\n if event.type == \"DNS_NAME\":\n await self.handle_dns_name(event)\n elif event.type == \"STORAGE_BUCKET\":\n await self.handle_storage_bucket(event)\n\n async def handle_dns_name(self, event):\n buckets = set()\n base = self.helpers.unidecode(self.helpers.smart_decode_punycode(event.data))\n stem = self.helpers.domain_stem(base)\n for b in [base, stem]:\n split = b.split(\".\")\n for d in self.delimiters:\n bucket_name = d.join(split)\n buckets.add(bucket_name)\n async for bucket_name, url, tags, num_buckets in self.brute_buckets(buckets, permutations=self.permutations):\n await self.emit_storage_bucket(\n {\"name\": bucket_name, \"url\": url},\n \"STORAGE_BUCKET\",\n parent=event,\n tags=tags,\n context=f\"{{module}} tried {num_buckets:,} bucket variations of {event.data} and found {{event.type}} at {url}\",\n )\n\n async def handle_storage_bucket(self, event):\n url = event.data[\"url\"]\n bucket_name = event.data[\"name\"]\n if self.supports_open_check:\n description, tags = await self._check_bucket_open(bucket_name, url)\n if description:\n event_data = {\"host\": event.host, \"url\": url, \"description\": description}\n await self.emit_event(\n event_data,\n \"FINDING\",\n parent=event,\n tags=tags,\n context=f\"{{module}} scanned {event.type} and identified {{event.type}}: {description}\",\n )\n\n async for bucket_name, new_url, tags, num_buckets in self.brute_buckets(\n [bucket_name], permutations=self.permutations, omit_base=True\n ):\n await self.emit_storage_bucket(\n {\"name\": bucket_name, \"url\": new_url},\n \"STORAGE_BUCKET\",\n parent=event,\n tags=tags,\n context=f\"{{module}} tried {num_buckets:,} variations of {url} and found {{event.type}} at {new_url}\",\n )\n\n async def emit_storage_bucket(self, event_data, event_type, parent, tags, context):\n event_data[\"url\"] = self.clean_bucket_url(event_data[\"url\"])\n await self.emit_event(\n event_data,\n event_type,\n parent=parent,\n tags=tags,\n context=context,\n )\n\n async def brute_buckets(self, buckets, permutations=False, omit_base=False):\n buckets = set(buckets)\n new_buckets = set(buckets)\n if permutations:\n for b in buckets:\n for mutation in self.helpers.word_cloud.mutations(b, cloud=False):\n for d in self.delimiters:\n new_buckets.add(d.join(mutation))\n if omit_base:\n new_buckets = new_buckets - buckets\n new_buckets = [b for b in new_buckets if self.valid_bucket_name(b)]\n num_buckets = len(new_buckets)\n bucket_urls_kwargs = []\n for base_domain in self.base_domains:\n for region in self.regions:\n for bucket_name in new_buckets:\n url, kwargs = self.build_bucket_request(bucket_name, base_domain, region)\n bucket_urls_kwargs.append((url, kwargs, (bucket_name, base_domain, region)))\n async for url, kwargs, (bucket_name, base_domain, region), response in self.helpers.request_custom_batch(\n bucket_urls_kwargs\n ):\n existent_bucket, tags = self._check_bucket_exists(bucket_name, response)\n if existent_bucket:\n yield bucket_name, url, tags, num_buckets\n\n def clean_bucket_url(self, url):\n # if needed, modify the bucket url before emitting it\n return url\n\n def build_bucket_request(self, bucket_name, base_domain, region):\n url = self.build_url(bucket_name, base_domain, region)\n return url, {}\n\n def _check_bucket_exists(self, bucket_name, response):\n self.debug(f'Checking if bucket exists: \"{bucket_name}\"')\n return self.check_bucket_exists(bucket_name, response)\n\n def check_bucket_exists(self, bucket_name, response):\n tags = self.gen_tags_exists(response)\n status_code = getattr(response, \"status_code\", 404)\n existent_bucket = status_code != 404\n return (existent_bucket, tags)\n\n async def _check_bucket_open(self, bucket_name, url):\n self.debug(f'Checking if bucket is misconfigured: \"{bucket_name}\"')\n return await self.check_bucket_open(bucket_name, url)\n\n async def check_bucket_open(self, bucket_name, url):\n response = await self.helpers.request(url)\n tags = self.gen_tags_exists(response)\n status_code = getattr(response, \"status_code\", 404)\n content = getattr(response, \"text\", \"\")\n open_bucket = status_code == 200 and \"Contents\" in content\n msg = \"\"\n if open_bucket:\n msg = \"Open storage bucket\"\n return (msg, tags)\n\n def valid_bucket_name(self, bucket_name):\n valid = self.is_valid_bucket_name(bucket_name)\n if valid and not self.helpers.is_ip(bucket_name):\n bucket_hash = hash(bucket_name)\n if bucket_hash not in self.buckets_tried:\n self.buckets_tried.add(bucket_hash)\n return True\n return False\n\n def is_valid_bucket_name(self, bucket_name):\n return any(regex.match(bucket_name) for regex in self.bucket_name_regexes)\n\n @cached_property\n def bucket_name_regexes(self):\n return [re.compile(regex) for regex in self.cloudcheck_provider.regexes[\"STORAGE_BUCKET_NAME\"]]\n\n # @cached_property\n # def bucket_hostname_regexes(self):\n # return [re.compile(regex) for regex in self.cloudcheck_provider.regexes[\"STORAGE_BUCKET_HOSTNAME\"]]\n\n def build_url(self, bucket_name, base_domain, region):\n return f\"https://{bucket_name}.{base_domain}/\"\n\n def gen_tags_exists(self, response):\n return set()\n\n def gen_tags_open(self, response):\n return set()\n" }, { "path": "bbot/modules/templates/censys.py", "content": "import traceback\n\nfrom bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass censys(subdomain_enum_apikey):\n \"\"\"\n Base template for Censys API modules.\n Provides common authentication and API request handling.\n \"\"\"\n\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"Censys.io API Key in the format of 'key:secret'\"}\n\n base_url = \"https://search.censys.io/api\"\n\n async def setup(self):\n await super().setup()\n api_keys = set()\n for module_name in (\"censys\", \"censys_dns\", \"censys_ip\"):\n module_config = self.scan.config.get(\"modules\", {}).get(module_name, {})\n api_key = module_config.get(\"api_key\", \"\")\n if isinstance(api_key, str):\n api_key = [api_key]\n for key in api_key:\n key = key.strip()\n if key:\n api_keys.add(key)\n if not api_keys:\n if self.auth_required:\n return None, \"No API key set\"\n self.api_key = api_keys.pop() if api_keys else \"\"\n try:\n await self.ping()\n self.hugesuccess(\"API is ready\")\n return True\n except Exception as e:\n self.trace(traceback.format_exc())\n return None, f\"Error with API ({str(e).strip()})\"\n\n async def ping(self):\n url = f\"{self.base_url}/v1/account\"\n resp = await self.api_request(url, retry_on_http_429=False)\n d = resp.json()\n assert isinstance(d, dict), f\"Invalid response from {url}: {resp}\"\n quota = d.get(\"quota\", {})\n used = int(quota.get(\"used\", 0))\n allowance = int(quota.get(\"allowance\", 0))\n assert used < allowance, \"No quota remaining\"\n\n def prepare_api_request(self, url, kwargs):\n api_id, api_secret = self.api_key.split(\":\", 1)\n kwargs[\"auth\"] = (api_id, api_secret)\n return url, kwargs\n" }, { "path": "bbot/modules/templates/github.py", "content": "import traceback\n\nfrom bbot.modules.base import BaseModule\n\n\nclass github(BaseModule):\n \"\"\"\n A template module for use of the GitHub API\n Inherited by several other github modules.\n \"\"\"\n\n _qsize = 1\n base_url = \"https://api.github.com\"\n ping_url = f\"{base_url}/zen\"\n\n def prepare_api_request(self, url, kwargs):\n kwargs[\"headers\"][\"Authorization\"] = f\"token {self.api_key}\"\n return url, kwargs\n\n async def setup(self):\n await super().setup()\n self.headers = {}\n api_keys = set()\n modules_config = self.scan.config.get(\"modules\", {})\n git_modules = [m for m in modules_config if str(m).startswith(\"git\")]\n for module_name in git_modules:\n module_config = modules_config.get(module_name, {})\n api_key = module_config.get(\"api_key\", \"\")\n if isinstance(api_key, str):\n api_key = [api_key]\n for key in api_key:\n key = key.strip()\n if key:\n api_keys.add(key)\n if not api_keys:\n if self.auth_required:\n return None, \"No API key set\"\n self.api_key = api_keys\n try:\n await self.ping()\n self.hugesuccess(\"API is ready\")\n return True\n except Exception as e:\n self.trace(traceback.format_exc())\n return None, f\"Error with API ({str(e).strip()})\"\n return True\n\n async def github_graphql_request(self, graphql_query, resp_key):\n url = f\"{self.base_url}/graphql\"\n next_key = \"\"\n has_next_page = True\n\n while has_next_page:\n query = graphql_query.replace(\"{NEXT_KEY}\", next_key)\n r = await self.api_request(url, method=\"POST\", json={\"query\": query})\n if r is None:\n break\n status_code = getattr(r, \"status_code\", 0)\n if status_code == 403:\n self.warning(\"Github is rate-limiting us (HTTP status: 403)\")\n break\n try:\n json = r.json()\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n break\n\n data = json.get(\"data\", {}).get(resp_key, {})\n yield data\n\n # Update pagination variables\n page_info = data.get(\"pageInfo\", {})\n has_next_page = page_info.get(\"hasNextPage\", False)\n next_key = page_info.get(\"endCursor\", \"\")\n" }, { "path": "bbot/modules/templates/gitlab.py", "content": "from bbot.modules.base import BaseModule\n\n\nclass GitLabBaseModule(BaseModule):\n \"\"\"Common functionality for interacting with GitLab instances.\n\n This template is intended to be inherited by two concrete modules:\n 1. ``gitlab_com`` – Handles public SaaS instances (gitlab.com / gitlab.org).\n 2. ``gitlab_onprem`` – Handles self-hosted, on-premises GitLab servers.\n\n Both child modules share identical behaviour when talking to the GitLab\n REST API; they only differ in which events they are willing to accept.\n \"\"\"\n\n # domains owned by GitLab\n saas_domains = [\"gitlab.com\", \"gitlab.org\"]\n\n async def setup(self):\n if self.options.get(\"api_key\") is not None:\n await self.require_api_key()\n return True\n\n async def handle_social(self, event):\n \"\"\"Enumerate projects belonging to a user or group profile.\"\"\"\n username = event.data.get(\"profile_name\", \"\")\n if not username:\n return\n base_url = self.get_base_url(event)\n urls = [\n # User-owned projects\n self.helpers.urljoin(base_url, f\"api/v4/users/{username}/projects?simple=true\"),\n # Group-owned projects\n self.helpers.urljoin(base_url, f\"api/v4/groups/{username}/projects?simple=true\"),\n ]\n for url in urls:\n await self.handle_projects_url(url, event)\n\n async def handle_projects_url(self, projects_url, event):\n for project in await self.gitlab_json_request(projects_url):\n project_url = project.get(\"web_url\", \"\")\n if project_url:\n code_event = self.make_event({\"url\": project_url}, \"CODE_REPOSITORY\", tags=\"git\", parent=event)\n await self.emit_event(\n code_event,\n context=f\"{{module}} enumerated projects and found {{event.type}} at {project_url}\",\n )\n namespace = project.get(\"namespace\", {})\n if namespace:\n await self.handle_namespace(namespace, event)\n\n async def handle_groups_url(self, groups_url, event):\n for group in await self.gitlab_json_request(groups_url):\n await self.handle_namespace(group, event)\n\n async def gitlab_json_request(self, url):\n \"\"\"Helper that performs an HTTP request and safely returns JSON list.\"\"\"\n response = await self.api_request(url)\n if response is not None:\n try:\n json_data = response.json()\n except Exception:\n return []\n if json_data and isinstance(json_data, list):\n return json_data\n return []\n\n async def handle_namespace(self, namespace, event):\n namespace_name = namespace.get(\"path\", \"\")\n namespace_url = namespace.get(\"web_url\", \"\")\n namespace_path = namespace.get(\"full_path\", \"\")\n\n if not (namespace_name and namespace_url and namespace_path):\n return\n\n namespace_url = self.helpers.parse_url(namespace_url)._replace(path=f\"/{namespace_path}\").geturl()\n\n social_event = self.make_event(\n {\n \"platform\": \"gitlab\",\n \"profile_name\": namespace_path,\n \"url\": namespace_url,\n },\n \"SOCIAL\",\n parent=event,\n )\n await self.emit_event(\n social_event,\n context=f'{{module}} found GitLab namespace ({{event.type}}) \"{namespace_name}\" at {namespace_url}',\n )\n\n # ------------------------------------------------------------------\n # Utility helpers\n # ------------------------------------------------------------------\n def get_base_url(self, event):\n base_url = event.data.get(\"url\", \"\")\n if not base_url:\n base_url = f\"https://{event.host}\"\n return self.helpers.urlparse(base_url)._replace(path=\"/\").geturl()\n" }, { "path": "bbot/modules/templates/postman.py", "content": "from bbot.modules.base import BaseModule\n\n\nclass postman(BaseModule):\n \"\"\"\n A template module for use of the GitHub API\n Inherited by several other github modules.\n \"\"\"\n\n base_url = \"https://www.postman.com/_api\"\n api_url = \"https://api.getpostman.com\"\n html_url = \"https://www.postman.com\"\n ping_url = f\"{api_url}/me\"\n\n headers = {\n \"Content-Type\": \"application/json\",\n \"X-App-Version\": \"11.27.4-250109-2338\",\n \"X-Entity-Team-Id\": \"0\",\n \"Origin\": \"https://www.postman.com\",\n \"Referer\": \"https://www.postman.com/search?q=&scope=public&type=all\",\n }\n auth_required = True\n\n async def setup(self):\n await super().setup()\n self.headers = {}\n api_keys = set()\n modules_config = self.scan.config.get(\"modules\", {})\n postman_modules = [m for m in modules_config if str(m).startswith(\"postman\")]\n for module_name in postman_modules:\n module_config = modules_config.get(module_name, {})\n api_key = module_config.get(\"api_key\", \"\")\n if isinstance(api_key, str):\n api_key = [api_key]\n for key in api_key:\n key = key.strip()\n if key:\n api_keys.add(key)\n if not api_keys:\n if self.auth_required:\n return None, \"No API key set\"\n self.api_key = api_keys\n if self.api_key:\n try:\n await self.ping()\n self.hugesuccess(\"API is ready\")\n return True\n except Exception as e:\n self.trace()\n return None, f\"Error with API ({str(e).strip()})\"\n return True\n\n def prepare_api_request(self, url, kwargs):\n if self.api_key:\n kwargs[\"headers\"][\"X-Api-Key\"] = self.api_key\n return url, kwargs\n\n async def get_workspace_id(self, repo_url):\n workspace_id = \"\"\n profile = repo_url.split(\"/\")[-2]\n name = repo_url.split(\"/\")[-1]\n url = f\"{self.base_url}/ws/proxy\"\n json = {\n \"service\": \"workspaces\",\n \"method\": \"GET\",\n \"path\": f\"/workspaces?handle={profile}&slug={name}\",\n }\n r = await self.helpers.request(url, method=\"POST\", json=json, headers=self.headers)\n if r is None:\n return workspace_id\n status_code = getattr(r, \"status_code\", 0)\n try:\n json = r.json()\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n return workspace_id\n data = json.get(\"data\", [])\n if len(data) == 1:\n workspace_id = data[0][\"id\"]\n return workspace_id\n\n async def request_workspace(self, id):\n data = {\"workspace\": {}, \"environments\": [], \"collections\": []}\n workspace = await self.get_workspace(id)\n if workspace:\n # Main Workspace\n name = workspace[\"name\"]\n data[\"workspace\"] = workspace\n\n # Workspace global variables\n self.verbose(f\"Searching globals for workspace {name}\")\n globals = await self.get_globals(id)\n data[\"environments\"].append(globals)\n\n # Workspace Environments\n workspace_environments = workspace.get(\"environments\", [])\n if workspace_environments:\n self.verbose(f\"Searching environments for workspace {name}\")\n for _ in workspace_environments:\n environment_id = _[\"uid\"]\n environment = await self.get_environment(environment_id)\n data[\"environments\"].append(environment)\n\n # Workspace Collections\n workspace_collections = workspace.get(\"collections\", [])\n if workspace_collections:\n self.verbose(f\"Searching collections for workspace {name}\")\n for _ in workspace_collections:\n collection_id = _[\"uid\"]\n collection = await self.get_collection(collection_id)\n data[\"collections\"].append(collection)\n return data\n\n async def get_workspace(self, workspace_id):\n workspace = {}\n workspace_url = f\"{self.api_url}/workspaces/{workspace_id}\"\n r = await self.api_request(workspace_url)\n if r is None:\n return workspace\n status_code = getattr(r, \"status_code\", 0)\n try:\n json = r.json()\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n return workspace\n workspace = json.get(\"workspace\", {})\n return workspace\n\n async def get_globals(self, workspace_id):\n globals = {}\n globals_url = f\"{self.base_url}/workspace/{workspace_id}/globals\"\n r = await self.helpers.request(globals_url, headers=self.headers)\n if r is None:\n return globals\n status_code = getattr(r, \"status_code\", 0)\n try:\n json = r.json()\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n return globals\n globals = json.get(\"data\", {})\n return globals\n\n async def get_environment(self, environment_id):\n environment = {}\n environment_url = f\"{self.api_url}/environments/{environment_id}\"\n r = await self.api_request(environment_url)\n if r is None:\n return environment\n status_code = getattr(r, \"status_code\", 0)\n try:\n json = r.json()\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n return environment\n environment = json.get(\"environment\", {})\n return environment\n\n async def get_collection(self, collection_id):\n collection = {}\n collection_url = f\"{self.api_url}/collections/{collection_id}\"\n r = await self.api_request(collection_url)\n if r is None:\n return collection\n status_code = getattr(r, \"status_code\", 0)\n try:\n json = r.json()\n except Exception as e:\n self.warning(f\"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}\")\n return collection\n collection = json.get(\"collection\", {})\n return collection\n\n async def validate_workspace(self, workspace, environments, collections):\n name = workspace.get(\"name\", \"\")\n full_wks = str([workspace, environments, collections])\n in_scope_hosts = await self.scan.extract_in_scope_hostnames(full_wks)\n if in_scope_hosts:\n self.verbose(\n f'Found in-scope hostname(s): \"{in_scope_hosts}\" in workspace {name}, it appears to be in-scope'\n )\n return True\n return False\n" }, { "path": "bbot/modules/templates/shodan.py", "content": "import traceback\n\nfrom bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass shodan(subdomain_enum):\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"Shodan API key\"}\n\n base_url = \"https://api.shodan.io\"\n ping_url = f\"{base_url}/api-info?key={{api_key}}\"\n\n async def setup(self):\n await super().setup()\n api_keys = set()\n for module_name in (\"shodan\", \"shodan_dns\", \"shodan_port\"):\n module_config = self.scan.config.get(\"modules\", {}).get(module_name, {})\n api_key = module_config.get(\"api_key\", \"\")\n if isinstance(api_key, str):\n api_key = [api_key]\n for key in api_key:\n key = key.strip()\n if key:\n api_keys.add(key)\n if not api_keys:\n if self.auth_required:\n return None, \"No API key set\"\n self.api_key = api_keys\n try:\n await self.ping()\n self.hugesuccess(\"API is ready\")\n return True\n except Exception as e:\n self.trace(traceback.format_exc())\n return None, f\"Error with API ({str(e).strip()})\"\n" }, { "path": "bbot/modules/templates/sql.py", "content": "from contextlib import suppress\nfrom sqlmodel import SQLModel\nfrom sqlalchemy.orm import sessionmaker\nfrom sqlalchemy.ext.asyncio import create_async_engine, AsyncSession\n\nfrom bbot.db.sql.models import Event, Scan, Target\nfrom bbot.modules.output.base import BaseOutputModule\n\n\nclass SQLTemplate(BaseOutputModule):\n meta = {\"description\": \"SQL output module template\"}\n options = {\n \"database\": \"bbot\",\n \"username\": \"\",\n \"password\": \"\",\n \"host\": \"127.0.0.1\",\n \"port\": 0,\n }\n options_desc = {\n \"database\": \"The database to use\",\n \"username\": \"The username to use to connect to the database\",\n \"password\": \"The password to use to connect to the database\",\n \"host\": \"The host to use to connect to the database\",\n \"port\": \"The port to use to connect to the database\",\n }\n\n protocol = \"\"\n\n async def setup(self):\n self.database = self.config.get(\"database\", \"bbot\")\n self.username = self.config.get(\"username\", \"\")\n self.password = self.config.get(\"password\", \"\")\n self.host = self.config.get(\"host\", \"127.0.0.1\")\n self.port = self.config.get(\"port\", 0)\n\n await self.init_database()\n return True\n\n async def handle_event(self, event):\n event_obj = Event(**event.json()).validated\n\n async with self.async_session() as session:\n async with session.begin():\n # insert event\n session.add(event_obj)\n\n # if it's a SCAN event, create/update the scan and target\n if event_obj.type == \"SCAN\":\n event_data = event_obj.get_data()\n if not isinstance(event_data, dict):\n raise ValueError(f\"Invalid data for SCAN event: {event_data}\")\n scan = Scan(**event_data).validated\n await session.merge(scan) # Insert or update scan\n\n target_data = event_data.get(\"target\", {})\n if not isinstance(target_data, dict):\n raise ValueError(f\"Invalid target for SCAN event: {target_data}\")\n target = Target(**target_data).validated\n await session.merge(target) # Insert or update target\n\n await session.commit()\n\n async def create_database(self):\n pass\n\n async def init_database(self):\n await self.create_database()\n\n # Now create the engine for the actual database\n self.engine = create_async_engine(self.connection_string())\n # Create a session factory bound to the engine\n self.async_session = sessionmaker(self.engine, expire_on_commit=False, class_=AsyncSession)\n\n # Use the engine directly to create all tables\n async with self.engine.begin() as conn:\n await conn.run_sync(SQLModel.metadata.create_all)\n\n def connection_string(self, mask_password=False):\n connection_string = f\"{self.protocol}://\"\n if self.username:\n password = self.password\n if mask_password:\n password = \"****\"\n connection_string += f\"{self.username}:{password}\"\n if self.host:\n connection_string += f\"@{self.host}\"\n if self.port:\n connection_string += f\":{self.port}\"\n if self.database:\n connection_string += f\"/{self.database}\"\n return connection_string\n\n async def cleanup(self):\n with suppress(Exception):\n await self.engine.dispose()\n" }, { "path": "bbot/modules/templates/subdomain_enum.py", "content": "from bbot.modules.base import BaseModule\n\n\nclass subdomain_enum(BaseModule):\n \"\"\"\n A typical free API-based subdomain enumeration module\n Inherited by many other modules including sublist3r, dnsdumpster, etc.\n \"\"\"\n\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\"description\": \"Query an API for subdomains\"}\n\n base_url = \"https://api.example.com\"\n\n # set module error state after this many failed requests in a row\n abort_after_failures = 5\n\n # whether to reject wildcard DNS_NAMEs\n reject_wildcards = \"strict\"\n\n # set qsize to 10. this helps combat rate limiting by ensuring the next query doesn't execute\n # until the result from the previous queue have been consumed by the scan\n # we don't use 1 because it causes delays due to the asyncio.sleep; 10 gives us reasonable buffer room\n _qsize = 10\n\n # how to deduplicate incoming events\n # options:\n # \"highest_parent\": dedupe by highest parent (highest parent of www.api.test.evilcorp.com is evilcorp.com)\n # \"lowest_parent\": dedupe by lowest parent (lowest parent of www.api.test.evilcorp.com is api.test.evilcorp.com)\n dedup_strategy = \"highest_parent\"\n\n # how many results to request per API call\n page_size = 100\n # arguments to pass to api_page_iter\n api_page_iter_kwargs = {}\n\n @property\n def source_pretty_name(self):\n return f\"{self.__class__.__name__} API\"\n\n def _incoming_dedup_hash(self, event):\n \"\"\"\n Determines the criteria for what is considered to be a duplicate event if `accept_dupes` is False.\n \"\"\"\n return hash(self.make_query(event)), f\"dedup_strategy={self.dedup_strategy}\"\n\n async def handle_event(self, event):\n query = self.make_query(event)\n results = await self.query(query)\n if results:\n for hostname in set(results):\n if hostname:\n try:\n hostname = self.helpers.validators.validate_host(hostname)\n except ValueError as e:\n self.verbose(e)\n continue\n if hostname and hostname.endswith(f\".{query}\") and not hostname == event.data:\n await self.emit_event(\n hostname,\n \"DNS_NAME\",\n event,\n abort_if=self.abort_if,\n context=f'{{module}} searched {self.source_pretty_name} for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n\n async def handle_event_paginated(self, event):\n query = self.make_query(event)\n async for result_batch in self.query_paginated(query):\n for hostname in set(result_batch):\n try:\n hostname = self.helpers.validators.validate_host(hostname)\n except ValueError as e:\n self.verbose(e)\n continue\n if hostname and hostname.endswith(f\".{query}\") and not hostname == event.data:\n await self.emit_event(\n hostname,\n \"DNS_NAME\",\n event,\n abort_if=self.abort_if,\n context=f'{{module}} searched {self.source_pretty_name} for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n\n async def request_url(self, query):\n url = self.make_url(query)\n return await self.api_request(url)\n\n def make_url(self, query):\n return f\"{self.base_url}/subdomains/{self.helpers.quote(query)}\"\n\n def make_query(self, event):\n query = event.data\n parents = list(self.helpers.domain_parents(event.data))\n if self.dedup_strategy == \"highest_parent\":\n parents = list(reversed(parents))\n elif self.dedup_strategy == \"lowest_parent\":\n pass\n else:\n raise ValueError('self.dedup_strategy attribute must be set to either \"highest_parent\" or \"lowest_parent\"')\n for p in parents:\n if self.scan.in_scope(p):\n query = p\n break\n return \".\".join([s for s in query.split(\".\") if s != \"_wildcard\"])\n\n async def parse_results(self, r, query=None):\n json = r.json()\n if json:\n for hostname in json:\n yield hostname\n\n async def query(self, query, request_fn=None, parse_fn=None):\n if request_fn is None:\n request_fn = self.request_url\n if parse_fn is None:\n parse_fn = self.parse_results\n try:\n response = await request_fn(query)\n if response is None:\n self.info(f'Query \"{query}\" failed (no response)')\n return []\n try:\n results = list(await parse_fn(response, query))\n except Exception as e:\n if response:\n self.info(\n f'Error parsing results for query \"{query}\" (status code {response.status_code})', trace=True\n )\n self.log.trace(repr(response.text))\n else:\n self.info(f'Error parsing results for \"{query}\": {e}', trace=True)\n return\n if results:\n return results\n self.debug(f'No results for \"{query}\"')\n except Exception as e:\n self.info(f\"Error retrieving results for {query}: {e}\", trace=True)\n\n async def query_paginated(self, query):\n url = self.make_url(query)\n agen = self.api_page_iter(url, page_size=self.page_size, **self.api_page_iter_kwargs)\n try:\n async for response in agen:\n subdomains = await self.parse_results(response, query)\n self.verbose(f'Got {len(subdomains):,} subdomains for \"{query}\"')\n if not subdomains:\n break\n yield subdomains\n finally:\n await agen.aclose()\n\n async def _is_wildcard(self, query):\n rdtypes = (\"A\", \"AAAA\", \"CNAME\")\n if self.helpers.is_dns_name(query):\n for wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query, rdtypes=rdtypes)).values():\n if any(t in wildcard_rdtypes for t in rdtypes):\n return True\n return False\n\n async def filter_event(self, event):\n query = self.make_query(event)\n # check if wildcard\n is_wildcard = await self._is_wildcard(query)\n # check if cloud\n is_cloud = False\n if any(t.startswith(\"cloud-\") for t in event.tags):\n is_cloud = True\n # reject if it's a cloud resource and not in our target\n if is_cloud and event not in self.scan.target.whitelist:\n return False, \"Event is a cloud resource and not a direct target\"\n # optionally reject events with wildcards / errors\n if self.reject_wildcards:\n if any(t in event.tags for t in (\"a-error\", \"aaaa-error\")):\n return False, \"Event has a DNS resolution error\"\n if self.reject_wildcards == \"strict\":\n if is_wildcard:\n return False, \"Event is a wildcard domain\"\n elif self.reject_wildcards == \"cloud_only\":\n if is_wildcard and is_cloud:\n return False, \"Event is both a cloud resource and a wildcard domain\"\n return True, \"\"\n\n async def abort_if(self, event):\n # this helps weed out unwanted results when scanning IP_RANGES and wildcard domains\n if \"in-scope\" not in event.tags:\n return True\n return False\n\n\nclass subdomain_enum_apikey(subdomain_enum):\n \"\"\"\n A typical module for authenticated, API-based subdomain enumeration\n Inherited by several other modules including securitytrails, c99.nl, etc.\n \"\"\"\n\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\"description\": \"Query API for subdomains\", \"auth_required\": True}\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"API key\"}\n\n async def setup(self):\n await super().setup()\n return await self.require_api_key()\n" }, { "path": "bbot/modules/templates/webhook.py", "content": "import yaml\n\nfrom bbot.modules.output.base import BaseOutputModule\n\n\nclass WebhookOutputModule(BaseOutputModule):\n \"\"\"\n A template for webhook output modules such as Discord, Teams, and Slack\n \"\"\"\n\n accept_dupes = False\n message_size_limit = 2000\n content_key = \"content\"\n vuln_severities = [\"UNKNOWN\", \"LOW\", \"MEDIUM\", \"HIGH\", \"CRITICAL\"]\n\n # abort module after 10 failed requests (not including retries)\n _api_failure_abort_threshold = 10\n # retry each request up to 10 times, respecting the Retry-After header\n _default_api_retries = 10\n\n async def setup(self):\n self.webhook_url = self.config.get(\"webhook_url\", \"\")\n self.min_severity = self.config.get(\"min_severity\", \"LOW\").strip().upper()\n assert self.min_severity in self.vuln_severities, (\n f\"min_severity must be one of the following: {','.join(self.vuln_severities)}\"\n )\n self.allowed_severities = self.vuln_severities[self.vuln_severities.index(self.min_severity) :]\n if not self.webhook_url:\n self.warning(\"Must set Webhook URL\")\n return False\n return await super().setup()\n\n @property\n def api_retries(self):\n return self.config.get(\"retries\", self._default_api_retries)\n\n async def handle_event(self, event):\n message = self.format_message(event)\n data = {self.content_key: message}\n await self.api_request(\n url=self.webhook_url,\n method=\"POST\",\n json=data,\n )\n\n def get_watched_events(self):\n if self._watched_events is None:\n event_types = self.config.get(\"event_types\", [\"VULNERABILITY\"])\n if isinstance(event_types, str):\n event_types = [event_types]\n self._watched_events = set(event_types)\n return self._watched_events\n\n async def filter_event(self, event):\n if event.type == \"VULNERABILITY\":\n severity = event.data.get(\"severity\", \"UNKNOWN\")\n if severity not in self.allowed_severities:\n return False, f\"{severity} is below min_severity threshold\"\n return True\n\n def format_message_str(self, event):\n event_tags = \",\".join(event.tags)\n return f\"`[{event.type}]`\\t**`{event.data}`**\\ttags:{event_tags}\"\n\n def format_message_other(self, event):\n event_yaml = yaml.dump(event.data)\n event_type = f\"**`[{event.type}]`**\"\n if event.type in (\"VULNERABILITY\", \"FINDING\"):\n event_str, color = self.get_severity_color(event)\n event_type = f\"{color} {event_str} {color}\"\n return f\"\"\"**`{event_type}`**\\n```yaml\\n{event_yaml}```\"\"\"\n\n def get_severity_color(self, event):\n if event.type == \"VULNERABILITY\":\n severity = event.data.get(\"severity\", \"UNKNOWN\")\n return f\"{event.type} ({severity})\", event.severity_colors[severity]\n else:\n return event.type, \"đŸŸĻ\"\n\n def format_message(self, event):\n if isinstance(event.data, str):\n msg = self.format_message_str(event)\n else:\n msg = self.format_message_other(event)\n if len(msg) > self.message_size_limit:\n msg = msg[: self.message_size_limit - 3] + \"...\"\n return msg\n\n def evaluate_response(self, response):\n return getattr(response, \"is_success\", False)\n" }, { "path": "bbot/modules/trickest.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass Trickest(subdomain_enum_apikey):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"affiliates\", \"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query Trickest's API for subdomains\",\n \"author\": \"@amiremami\",\n \"created_date\": \"2024-07-27\",\n \"auth_required\": True,\n }\n options = {\n \"api_key\": \"\",\n }\n options_desc = {\n \"api_key\": \"Trickest API key\",\n }\n\n base_url = \"https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be\"\n ping_url = f\"{base_url}/dataset\"\n dataset_id = \"a0a49ca9-03bb-45e0-aa9a-ad59082ebdfc\"\n page_size = 50\n\n def prepare_api_request(self, url, kwargs):\n kwargs[\"headers\"][\"Authorization\"] = f\"Token {self.api_key}\"\n return url, kwargs\n\n async def handle_event(self, event):\n await self.handle_event_paginated(event)\n\n def make_url(self, query):\n url = f\"{self.base_url}/view?q=hostname%20~%20%22.{self.helpers.quote(query)}%22\"\n url += f\"&dataset_id={self.dataset_id}\"\n url += \"&limit={page_size}&offset={offset}&select=hostname&orderby=hostname\"\n return url\n\n async def parse_results(self, j, query):\n results = j.get(\"results\", [])\n subdomains = set()\n for item in results:\n hostname = item.get(\"hostname\", \"\")\n if hostname:\n subdomains.add(hostname)\n return subdomains\n" }, { "path": "bbot/modules/trufflehog.py", "content": "import json\nfrom functools import partial\nfrom bbot.modules.base import BaseModule\n\n\nclass trufflehog(BaseModule):\n watched_events = [\"CODE_REPOSITORY\", \"FILESYSTEM\", \"HTTP_RESPONSE\", \"RAW_TEXT\"]\n produced_events = [\"FINDING\", \"VULNERABILITY\"]\n flags = [\"passive\", \"safe\", \"code-enum\"]\n meta = {\n \"description\": \"TruffleHog is a tool for finding credentials\",\n \"created_date\": \"2024-03-12\",\n \"author\": \"@domwhewell-sage\",\n }\n\n options = {\n \"version\": \"3.90.8\",\n \"config\": \"\",\n \"only_verified\": True,\n \"concurrency\": 8,\n \"deleted_forks\": False,\n }\n options_desc = {\n \"version\": \"trufflehog version\",\n \"config\": \"File path or URL to YAML trufflehog config\",\n \"only_verified\": \"Only report credentials that have been verified\",\n \"concurrency\": \"Number of concurrent workers\",\n \"deleted_forks\": \"Scan for deleted github forks. WARNING: This is SLOW. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours.\",\n }\n deps_ansible = [\n {\n \"name\": \"Download trufflehog\",\n \"unarchive\": {\n \"src\": \"https://github.com/trufflesecurity/trufflehog/releases/download/v#{BBOT_MODULES_TRUFFLEHOG_VERSION}/trufflehog_#{BBOT_MODULES_TRUFFLEHOG_VERSION}_#{BBOT_OS_PLATFORM}_#{BBOT_CPU_ARCH_GOLANG}.tar.gz\",\n \"include\": \"trufflehog\",\n \"dest\": \"#{BBOT_TOOLS}\",\n \"remote_src\": True,\n },\n }\n ]\n\n scope_distance_modifier = 2\n\n async def setup_deps(self):\n self.config_file = self.config.get(\"config\", \"\")\n if self.config_file:\n self.config_file = await self.helpers.wordlist(self.config_file)\n return True\n\n async def setup(self):\n self.verified = self.config.get(\"only_verified\", True)\n self.concurrency = int(self.config.get(\"concurrency\", 8))\n\n self.deleted_forks = self.config.get(\"deleted_forks\", False)\n self.github_token = \"\"\n if self.deleted_forks:\n self.warning(\n \"Deleted forks is enabled. Scanning for deleted forks is slooooooowwwww. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours.\"\n )\n for module_name in (\"github\", \"github_codesearch\", \"github_org\", \"git_clone\"):\n module_config = self.scan.config.get(\"modules\", {}).get(module_name, {})\n api_key = module_config.get(\"api_key\", \"\")\n if api_key:\n self.github_token = api_key\n break\n\n # soft-fail if we don't have a github token as well\n if not self.github_token:\n self.deleted_forks = False\n return None, \"A github api_key must be provided to the github modules for deleted forks to be scanned\"\n return True\n\n async def filter_event(self, event):\n if event.type == \"CODE_REPOSITORY\":\n if self.deleted_forks:\n if \"git\" not in event.tags:\n return False, \"Module only accepts git CODE_REPOSITORY events\"\n if \"github\" not in event.data[\"url\"]:\n return False, \"Module only accepts github CODE_REPOSITORY events\"\n else:\n return False, \"Deleted forks is not enabled\"\n else:\n if \"unarchived-folder\" in event.tags:\n return False, \"Not accepting unarchived-folder events\"\n return True\n\n async def handle_event(self, event):\n description = \"\"\n if isinstance(event.data, dict):\n description = event.data.get(\"description\", \"\")\n\n if event.type == \"CODE_REPOSITORY\":\n path = event.data[\"url\"]\n module = \"github-experimental\"\n elif event.type == \"FILESYSTEM\":\n path = event.data[\"path\"]\n if \"git\" in event.tags:\n module = \"git\"\n elif \"docker\" in event.tags:\n module = \"docker\"\n elif \"postman\" in event.tags:\n module = \"postman\"\n else:\n module = \"filesystem\"\n elif event.type in (\"HTTP_RESPONSE\", \"RAW_TEXT\"):\n module = \"filesystem\"\n file_data = event.raw_response if event.type == \"HTTP_RESPONSE\" else event.data\n # write the response to a tempfile\n # this is necessary because trufflehog doesn't yet support reading from stdin\n # https://github.com/trufflesecurity/trufflehog/issues/162\n path = self.helpers.tempfile(file_data, pipe=False)\n\n if event.type == \"CODE_REPOSITORY\":\n host = event.host\n else:\n host = str(event.parent.host)\n async for (\n decoder_name,\n detector_name,\n raw_result,\n rawv2_result,\n verified,\n source_metadata,\n ) in self.execute_trufflehog(module, path):\n verified_str = \"Verified\" if verified else \"Possible\"\n finding_type = \"VULNERABILITY\" if verified else \"FINDING\"\n data = {\n \"description\": f\"{verified_str} Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Details: [{source_metadata}]\",\n }\n if host:\n data[\"host\"] = host\n if finding_type == \"VULNERABILITY\":\n data[\"severity\"] = \"High\"\n if description:\n data[\"description\"] += f\" Description: [{description}]\"\n data[\"description\"] += f\" Raw result: [{raw_result}]\"\n if rawv2_result:\n data[\"description\"] += f\" RawV2 result: [{rawv2_result}]\"\n await self.emit_event(\n data,\n finding_type,\n event,\n context=f'{{module}} searched {event.type} using \"{module}\" method and found {verified_str.lower()} secret ({{event.type}}): {raw_result}',\n )\n\n # clean up the tempfile when we're done with it\n if event.type in (\"HTTP_RESPONSE\", \"RAW_TEXT\"):\n path.unlink(missing_ok=True)\n\n async def execute_trufflehog(self, module, path=None, string=None):\n command = [\n \"trufflehog\",\n \"--json\",\n \"--no-update\",\n ]\n if self.verified:\n command.append(\"--only-verified\")\n if self.config_file:\n command.append(\"--config=\" + str(self.config_file))\n command.append(\"--concurrency=\" + str(self.concurrency))\n if module == \"git\":\n command.append(\"git\")\n command.append(\"file://\" + path)\n elif module == \"docker\":\n command.append(\"docker\")\n command.append(\"--image=file://\" + path)\n elif module == \"postman\":\n command.append(\"postman\")\n command.append(\"--workspace-paths=\" + path)\n elif module == \"filesystem\":\n command.append(\"filesystem\")\n command.append(path)\n elif module == \"github-experimental\":\n command.append(\"github-experimental\")\n command.append(\"--repo=\" + path)\n command.append(\"--object-discovery\")\n command.append(\"--delete-cached-data\")\n command.append(\"--token=\" + self.github_token)\n\n stats_file = self.helpers.tempfile_tail(callback=partial(self.log_trufflehog_status, path))\n try:\n with open(stats_file, \"w\") as stats_fh:\n async for line in self.helpers.run_live(command, stderr=stats_fh):\n try:\n j = json.loads(line)\n except json.decoder.JSONDecodeError:\n self.debug(f\"Failed to decode line: {line}\")\n continue\n\n decoder_name = j.get(\"DecoderName\", \"\")\n\n detector_name = j.get(\"DetectorName\", \"\")\n\n raw_result = j.get(\"Raw\", \"\")\n\n rawv2_result = j.get(\"RawV2\", \"\")\n\n verified = j.get(\"Verified\", False)\n\n source_metadata = j.get(\"SourceMetadata\", {})\n\n yield (decoder_name, detector_name, raw_result, rawv2_result, verified, source_metadata)\n finally:\n stats_file.unlink(missing_ok=True)\n\n def log_trufflehog_status(self, path, line):\n try:\n line = json.loads(line)\n except Exception:\n self.info(str(line))\n return\n message = line.get(\"msg\", \"\")\n ts = line.get(\"ts\", \"\")\n status = f\"Message: {message} | Timestamp: {ts}\"\n self.verbose(f\"Current scan target: {path}\")\n self.verbose(status)\n" }, { "path": "bbot/modules/url_manipulation.py", "content": "from bbot.errors import HttpCompareError\nfrom bbot.modules.base import BaseModule\n\n\nclass url_manipulation(BaseModule):\n watched_events = [\"URL\"]\n produced_events = [\"FINDING\"]\n flags = [\"active\", \"aggressive\", \"web-thorough\"]\n meta = {\n \"description\": \"Attempt to identify URL parsing/routing based vulnerabilities\",\n \"created_date\": \"2022-09-27\",\n \"author\": \"@liquidsec\",\n }\n in_scope_only = True\n\n options = {\"allow_redirects\": True}\n options_desc = {\n \"allow_redirects\": \"Allowing redirects will sometimes create false positives. Disallowing will sometimes create false negatives. Allowed by default.\"\n }\n\n async def setup(self):\n # ([string]method,[string]path,[bool]strip trailing slash)\n self.signatures = []\n\n self.rand_string = self.helpers.rand_string()\n\n # Test for abuse of extension based routing\n extensions = [\n \".css\",\n \".js\",\n \".xls\",\n \".png\",\n \".jpg\",\n \".swf\",\n \".xml\",\n \".pdf\",\n \".gif\",\n ]\n for ext in extensions:\n self.signatures.append((\"GET\", \"{scheme}://{netloc}/{path}?%s=%s\" % (self.rand_string, ext), False))\n\n self.allow_redirects = self.config.get(\"allow_redirects\", True)\n return True\n\n async def handle_event(self, event):\n try:\n compare_helper = self.helpers.http_compare(\n event.data, allow_redirects=self.allow_redirects, include_cache_buster=False\n )\n except HttpCompareError as e:\n self.debug(e)\n return\n\n try:\n if not await compare_helper.canary_check(event.data, mode=\"getparam\"):\n raise HttpCompareError()\n except HttpCompareError:\n self.verbose(f'Aborting \"{event.data}\" due to failed canary check')\n return\n\n for sig in self.signatures:\n sig = self.format_signature(sig, event)\n try:\n match, reasons, reflection, subject_response = await compare_helper.compare(\n sig[1], method=sig[0], allow_redirects=self.allow_redirects\n )\n except HttpCompareError as e:\n self.debug(f\"Encountered HttpCompareError: [{e}] for URL [{event.data}]\")\n\n if subject_response:\n subject_content = \"\".join([str(x) for x in subject_response.headers])\n if subject_response.text is not None:\n subject_content += subject_response.text\n\n if self.rand_string not in subject_content:\n if match is False:\n if str(subject_response.status_code).startswith(\"2\"):\n if \"body\" in reasons:\n reported_signature = f\"Modified URL: {sig[1]}\"\n description = f\"Url Manipulation: [{','.join(reasons)}] Sig: [{reported_signature}]\"\n await self.emit_event(\n {\"description\": description, \"host\": str(event.host), \"url\": event.data},\n \"FINDING\",\n parent=event,\n context=f\"{{module}} probed {event.data} and identified {{event.type}}: {description}\",\n )\n else:\n self.debug(f\"Status code changed to {str(subject_response.status_code)}, ignoring\")\n else:\n self.debug(\"Ignoring positive result due to presence of parameter name in result\")\n\n async def filter_event(self, event):\n accepted_status_codes = [\"200\", \"301\", \"302\"]\n\n for c in accepted_status_codes:\n if f\"status-{c}\" in event.tags:\n return True\n return False\n\n def format_signature(self, sig, event):\n if sig[2] is True:\n cleaned_path = event.parsed_url.path.strip(\"/\")\n else:\n cleaned_path = event.parsed_url.path.lstrip(\"/\")\n\n kwargs = {\"scheme\": event.parsed_url.scheme, \"netloc\": event.parsed_url.netloc, \"path\": cleaned_path}\n formatted_url = sig[1].format(**kwargs)\n return (sig[0], formatted_url)\n" }, { "path": "bbot/modules/urlscan.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass urlscan(subdomain_enum):\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\", \"URL_UNVERIFIED\"]\n meta = {\n \"description\": \"Query urlscan.io for subdomains\",\n \"created_date\": \"2022-06-09\",\n \"author\": \"@TheTechromancer\",\n }\n options = {\"urls\": False}\n options_desc = {\"urls\": \"Emit URLs in addition to DNS_NAMEs\"}\n\n base_url = \"https://urlscan.io/api/v1\"\n\n async def setup(self):\n self.urls = self.config.get(\"urls\", False)\n return await super().setup()\n\n async def handle_event(self, event):\n query = self.make_query(event)\n for domain, url in await self.query(query):\n parent_event = event\n if domain and domain != query:\n domain_event = self.make_event(domain, \"DNS_NAME\", parent=event)\n if domain_event:\n if str(domain_event.host).endswith(query) and not str(domain_event.host) == str(event.host):\n await self.emit_event(\n domain_event,\n abort_if=self.abort_if,\n context=f'{{module}} searched urlscan.io API for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n parent_event = domain_event\n if url:\n url_event = self.make_event(url, \"URL_UNVERIFIED\", parent=parent_event)\n if url_event:\n if str(url_event.host).endswith(query):\n if self.urls:\n await self.emit_event(\n url_event,\n abort_if=self.abort_if,\n context=f'{{module}} searched urlscan.io API for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n else:\n await self.emit_event(\n str(url_event.host),\n \"DNS_NAME\",\n parent=event,\n abort_if=self.abort_if,\n context=f'{{module}} searched urlscan.io API for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n else:\n self.debug(f\"{url_event.host} does not match {query}\")\n\n async def query(self, query):\n results = set()\n url = f\"{self.base_url}/search/?q={self.helpers.quote(query)}\"\n r = await self.helpers.request(url)\n try:\n json = r.json()\n if json and type(json) == dict:\n for result in json.get(\"results\", []):\n if result and type(result) == dict:\n task = result.get(\"task\", {})\n if task and type(task) == dict:\n domain = task.get(\"domain\", \"\")\n url = task.get(\"url\", \"\")\n if domain or url:\n results.add((domain, url))\n page = result.get(\"page\", {})\n if page and type(page) == dict:\n domain = page.get(\"domain\", \"\")\n url = page.get(\"url\", \"\")\n if domain or url:\n results.add((domain, url))\n else:\n self.debug(f'No results for \"{query}\"')\n except Exception:\n self.verbose(\"Error retrieving urlscan results\")\n return results\n" }, { "path": "bbot/modules/vhost.py", "content": "import base64\nfrom urllib.parse import urlparse\n\nfrom bbot.modules.ffuf import ffuf\n\n\nclass vhost(ffuf):\n watched_events = [\"URL\"]\n produced_events = [\"VHOST\", \"DNS_NAME\"]\n flags = [\"active\", \"aggressive\", \"slow\", \"deadly\"]\n meta = {\"description\": \"Fuzz for virtual hosts\", \"created_date\": \"2022-05-02\", \"author\": \"@liquidsec\"}\n\n special_vhost_list = [\"127.0.0.1\", \"localhost\", \"host.docker.internal\"]\n options = {\n \"wordlist\": \"https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt\",\n \"force_basehost\": \"\",\n \"lines\": 5000,\n }\n options_desc = {\n \"wordlist\": \"Wordlist containing subdomains\",\n \"force_basehost\": \"Use a custom base host (e.g. evilcorp.com) instead of the default behavior of using the current URL\",\n \"lines\": \"take only the first N lines from the wordlist when finding directories\",\n }\n\n deps_common = [\"ffuf\"]\n banned_characters = {\" \", \".\"}\n\n in_scope_only = True\n\n async def setup(self):\n self.scanned_hosts = {}\n self.wordcloud_tried_hosts = set()\n return await super().setup()\n\n async def handle_event(self, event):\n if not self.helpers.is_ip(event.host) or self.config.get(\"force_basehost\"):\n host = f\"{event.parsed_url.scheme}://{event.parsed_url.netloc}\"\n if host in self.scanned_hosts.keys():\n return\n else:\n self.scanned_hosts[host] = event\n\n # subdomain vhost check\n self.verbose(\"Main vhost bruteforce\")\n if self.config.get(\"force_basehost\"):\n basehost = self.config.get(\"force_basehost\")\n else:\n basehost = self.helpers.parent_domain(event.parsed_url.netloc)\n\n self.debug(f\"Using basehost: {basehost}\")\n async for vhost in self.ffuf_vhost(host, f\".{basehost}\", event):\n self.verbose(f\"Starting mutations check for {vhost}\")\n async for vhost in self.ffuf_vhost(host, f\".{basehost}\", event, wordlist=self.mutations_check(vhost)):\n pass\n\n # check existing host for mutations\n self.verbose(\"Checking for vhost mutations on main host\")\n async for vhost in self.ffuf_vhost(\n host, f\".{basehost}\", event, wordlist=self.mutations_check(event.parsed_url.netloc.split(\".\")[0])\n ):\n pass\n\n # special vhost list\n self.verbose(\"Checking special vhost list\")\n async for vhost in self.ffuf_vhost(\n host,\n \"\",\n event,\n wordlist=self.helpers.tempfile(self.special_vhost_list, pipe=False),\n skip_dns_host=True,\n ):\n pass\n\n async def ffuf_vhost(self, host, basehost, event, wordlist=None, skip_dns_host=False):\n filters = await self.baseline_ffuf(f\"{host}/\", exts=[\"\"], suffix=basehost, mode=\"hostheader\")\n self.debug(\"Baseline completed and returned these filters:\")\n self.debug(filters)\n if not wordlist:\n wordlist = self.tempfile\n async for r in self.execute_ffuf(\n wordlist, host, exts=[\"\"], suffix=basehost, filters=filters, mode=\"hostheader\"\n ):\n found_vhost_b64 = r[\"input\"][\"FUZZ\"]\n vhost_str = base64.b64decode(found_vhost_b64).decode()\n vhost_dict = {\"host\": str(event.host), \"url\": host, \"vhost\": vhost_str}\n if f\"{vhost_dict['vhost']}{basehost}\" != event.parsed_url.netloc:\n await self.emit_event(\n vhost_dict,\n \"VHOST\",\n parent=event,\n context=f\"{{module}} brute-forced virtual hosts for {event.data} and found {{event.type}}: {vhost_str}\",\n )\n if skip_dns_host is False:\n await self.emit_event(\n f\"{vhost_dict['vhost']}{basehost}\",\n \"DNS_NAME\",\n parent=event,\n tags=[\"vhost\"],\n context=f\"{{module}} brute-forced virtual hosts for {event.data} and found {{event.type}}: {{event.data}}\",\n )\n\n yield vhost_dict[\"vhost\"]\n\n def mutations_check(self, vhost):\n mutations_list = []\n for mutation in self.helpers.word_cloud.mutations(vhost):\n for i in [\"\", \"-\"]:\n mutations_list.append(i.join(mutation))\n mutations_list_file = self.helpers.tempfile(mutations_list, pipe=False)\n return mutations_list_file\n\n async def finish(self):\n # check existing hosts with wordcloud\n tempfile = self.helpers.tempfile(list(self.helpers.word_cloud.keys()), pipe=False)\n\n for host, event in self.scanned_hosts.items():\n if host not in self.wordcloud_tried_hosts:\n event.parsed_url = urlparse(host)\n\n self.verbose(\"Checking main host with wordcloud\")\n if self.config.get(\"force_basehost\"):\n basehost = self.config.get(\"force_basehost\")\n else:\n basehost = self.helpers.parent_domain(event.parsed_url.netloc)\n\n async for vhost in self.ffuf_vhost(host, f\".{basehost}\", event, wordlist=tempfile):\n pass\n\n self.wordcloud_tried_hosts.add(host)\n" }, { "path": "bbot/modules/viewdns.py", "content": "import re\n\nfrom bbot.modules.base import BaseModule\n\n\nclass viewdns(BaseModule):\n \"\"\"\n Todo: Also retrieve registrar?\n \"\"\"\n\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"affiliates\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query viewdns.info's reverse whois for related domains\",\n \"created_date\": \"2022-07-04\",\n \"author\": \"@TheTechromancer\",\n }\n base_url = \"https://viewdns.info\"\n in_scope_only = True\n per_domain_only = True\n _qsize = 1\n\n async def setup(self):\n self.date_regex = re.compile(r\"\\d{4}-\\d{2}-\\d{2}\")\n return True\n\n async def handle_event(self, event):\n _, query = self.helpers.split_domain(event.data)\n for domain, _ in await self.query(query):\n await self.emit_event(\n domain,\n \"DNS_NAME\",\n parent=event,\n tags=[\"affiliate\"],\n context=f'{{module}} searched viewdns.info for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n\n async def query(self, query):\n results = set()\n url = f\"{self.base_url}/reversewhois/?q={query}\"\n r = await self.helpers.request(url)\n status_code = getattr(r, \"status_code\", 0)\n if status_code not in (200,):\n self.verbose(f\"Error retrieving reverse whois results (status code: {status_code})\")\n\n content = getattr(r, \"content\", b\"\")\n\n html = self.helpers.beautifulsoup(content, \"html.parser\")\n if html is False:\n self.debug(\"BeautifulSoup returned False\")\n return results\n found = set()\n for table_row in html.findAll(\"tr\"):\n table_cells = table_row.findAll(\"td\")\n # make double-sure we're in the right table by checking the date field\n try:\n if self.date_regex.match(table_cells[1].text.strip()):\n # domain == first cell\n domain = table_cells[0].text.strip().lower()\n # registrar == last cell\n registrar = table_cells[-1].text.strip()\n if domain and not domain == query:\n result = (domain, registrar)\n result_hash = hash(result)\n if result_hash not in found:\n found.add(result_hash)\n results.add(result)\n except IndexError:\n self.debug(f\"Invalid row {str(table_row)[:40]}...\")\n continue\n return results\n" }, { "path": "bbot/modules/virustotal.py", "content": "from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey\n\n\nclass virustotal(subdomain_enum_apikey):\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"DNS_NAME\"]\n flags = [\"subdomain-enum\", \"passive\", \"safe\"]\n meta = {\n \"description\": \"Query VirusTotal's API for subdomains\",\n \"created_date\": \"2022-08-25\",\n \"author\": \"@TheTechromancer\",\n \"auth_required\": True,\n }\n options = {\"api_key\": \"\"}\n options_desc = {\"api_key\": \"VirusTotal API Key\"}\n\n base_url = \"https://www.virustotal.com/api/v3\"\n api_page_iter_kwargs = {\"json\": False, \"next_key\": lambda r: r.json().get(\"links\", {}).get(\"next\", \"\")}\n\n def make_url(self, query):\n return f\"{self.base_url}/domains/{self.helpers.quote(query)}/subdomains\"\n\n def prepare_api_request(self, url, kwargs):\n kwargs[\"headers\"][\"x-apikey\"] = self.api_key\n return url, kwargs\n\n async def parse_results(self, r, query):\n text = getattr(r, \"text\", \"\")\n return await self.scan.extract_in_scope_hostnames(text)\n" }, { "path": "bbot/modules/wafw00f.py", "content": "from bbot.modules.base import BaseModule\nfrom wafw00f import main as wafw00f_main\n\n# disable wafw00f logging\nimport logging\n\nwafw00f_logger = logging.getLogger(\"wafw00f\")\nwafw00f_logger.setLevel(logging.CRITICAL + 100)\n\n\nclass wafw00f(BaseModule):\n \"\"\"\n https://github.com/EnableSecurity/wafw00f\n \"\"\"\n\n watched_events = [\"URL\"]\n produced_events = [\"WAF\"]\n flags = [\"active\", \"aggressive\"]\n meta = {\n \"description\": \"Web Application Firewall Fingerprinting Tool\",\n \"created_date\": \"2023-02-15\",\n \"author\": \"@liquidsec\",\n }\n\n deps_pip = [\"wafw00f~=2.3.1\"]\n\n options = {\"generic_detect\": True}\n options_desc = {\"generic_detect\": \"When no specific WAF detections are made, try to perform a generic detect\"}\n\n in_scope_only = True\n per_hostport_only = True\n\n async def filter_event(self, event):\n http_status = getattr(event, \"http_status\", 0)\n if not http_status or str(http_status).startswith(\"3\"):\n return False, f\"Invalid HTTP status code: {http_status}\"\n return True, \"\"\n\n def _incoming_dedup_hash(self, event):\n return hash(f\"{event.parsed_url.scheme}://{event.parsed_url.netloc}/\")\n\n async def handle_event(self, event):\n url = f\"{event.parsed_url.scheme}://{event.parsed_url.netloc}/\"\n WW = await self.helpers.run_in_executor(wafw00f_main.WAFW00F, url, followredirect=False)\n waf_detections, url = await self.helpers.run_in_executor(WW.identwaf)\n if waf_detections:\n for waf in waf_detections:\n await self.emit_event(\n {\"host\": str(event.host), \"url\": url, \"waf\": waf},\n \"WAF\",\n parent=event,\n context=f\"{{module}} scanned {url} and identified {{event.type}}: {waf}\",\n )\n else:\n if self.config.get(\"generic_detect\") is True:\n generic = await self.helpers.run_in_executor(WW.genericdetect)\n if generic:\n waf = \"generic detection\"\n await self.emit_event(\n {\n \"host\": str(event.host),\n \"url\": url,\n \"waf\": waf,\n \"info\": WW.knowledge[\"generic\"][\"reason\"],\n },\n \"WAF\",\n parent=event,\n context=f\"{{module}} scanned {url} and identified {{event.type}}: {waf}\",\n )\n" }, { "path": "bbot/modules/wayback.py", "content": "from datetime import datetime\n\nfrom bbot.modules.templates.subdomain_enum import subdomain_enum\n\n\nclass wayback(subdomain_enum):\n flags = [\"passive\", \"subdomain-enum\", \"safe\"]\n watched_events = [\"DNS_NAME\"]\n produced_events = [\"URL_UNVERIFIED\", \"DNS_NAME\"]\n meta = {\n \"description\": \"Query archive.org's API for subdomains\",\n \"created_date\": \"2022-04-01\",\n \"author\": \"@liquidsec\",\n }\n options = {\"urls\": False, \"garbage_threshold\": 10}\n options_desc = {\n \"urls\": \"emit URLs in addition to DNS_NAMEs\",\n \"garbage_threshold\": \"Dedupe similar urls if they are in a group of this size or higher (lower values == less garbage data)\",\n }\n in_scope_only = True\n\n base_url = \"http://web.archive.org\"\n\n async def setup(self):\n self.urls = self.config.get(\"urls\", False)\n self.garbage_threshold = self.config.get(\"garbage_threshold\", 10)\n return await super().setup()\n\n async def handle_event(self, event):\n query = self.make_query(event)\n for result, event_type in await self.query(query):\n await self.emit_event(\n result,\n event_type,\n event,\n abort_if=self.abort_if,\n context=f'{{module}} queried archive.org for \"{query}\" and found {{event.type}}: {{event.data}}',\n )\n\n async def query(self, query):\n results = set()\n waybackurl = f\"{self.base_url}/cdx/search/cdx?url={self.helpers.quote(query)}&matchType=domain&output=json&fl=original&collapse=original\"\n r = await self.helpers.request(waybackurl, timeout=self.http_timeout + 10)\n if not r:\n self.warning(f'Error connecting to archive.org for query \"{query}\"')\n return results\n try:\n j = r.json()\n assert type(j) == list\n except Exception:\n self.warning(f'Error JSON-decoding archive.org response for query \"{query}\"')\n return results\n\n urls = []\n for result in j[1:]:\n try:\n url = result[0]\n urls.append(url)\n except KeyError:\n continue\n\n self.verbose(f\"Found {len(urls):,} URLs for {query}\")\n\n dns_names = set()\n collapsed_urls = 0\n start_time = datetime.now()\n # we consolidate URLs to cut down on garbage data\n # this is CPU-intensive, so we do it in its own core.\n parsed_urls = await self.helpers.run_in_executor_mp(\n self.helpers.validators.collapse_urls,\n urls,\n threshold=self.garbage_threshold,\n )\n for parsed_url in parsed_urls:\n collapsed_urls += 1\n if not self.urls:\n dns_name = parsed_url.hostname\n h = hash(dns_name)\n if h not in dns_names:\n dns_names.add(h)\n results.add((dns_name, \"DNS_NAME\"))\n else:\n results.add((parsed_url.geturl(), \"URL_UNVERIFIED\"))\n end_time = datetime.now()\n duration = self.helpers.human_timedelta(end_time - start_time)\n self.verbose(f\"Collapsed {len(urls):,} -> {collapsed_urls:,} URLs in {duration}\")\n return results\n" }, { "path": "bbot/modules/wpscan.py", "content": "import json\nfrom bbot.modules.base import BaseModule\n\n\nclass wpscan(BaseModule):\n watched_events = [\"HTTP_RESPONSE\", \"TECHNOLOGY\"]\n produced_events = [\"URL_UNVERIFIED\", \"FINDING\", \"VULNERABILITY\", \"TECHNOLOGY\"]\n flags = [\"active\", \"aggressive\"]\n meta = {\n \"description\": \"Wordpress security scanner. Highly recommended to use an API key for better results.\",\n \"created_date\": \"2024-05-29\",\n \"author\": \"@domwhewell-sage\",\n }\n\n options = {\n \"api_key\": \"\",\n \"enumerate\": \"vp,vt,cb,dbe\",\n \"threads\": 5,\n \"request_timeout\": 5,\n \"connection_timeout\": 2,\n \"disable_tls_checks\": True,\n \"force\": False,\n }\n options_desc = {\n \"api_key\": \"WPScan API Key\",\n \"enumerate\": \"Enumeration Process see wpscan help documentation (default: vp,vt,cb,dbe)\",\n \"threads\": \"How many wpscan threads to spawn (default is 5)\",\n \"request_timeout\": \"The request timeout in seconds (default 5)\",\n \"connection_timeout\": \"The connection timeout in seconds (default 2)\",\n \"disable_tls_checks\": \"Disables the SSL/TLS certificate verification (Default True)\",\n \"force\": \"Do not check if the target is running WordPress or returns a 403\",\n }\n deps_apt = [\"curl\", \"make\", \"gcc\"]\n deps_ansible = [\n {\n \"name\": \"Install Ruby Deps (Debian)\",\n \"package\": {\"name\": [\"ruby-rubygems\", \"ruby-dev\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Debian'\",\n },\n {\n \"name\": \"Install Ruby Deps (Arch)\",\n \"package\": {\"name\": [\"rubygems\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Archlinux'\",\n },\n {\n \"name\": \"Install Ruby Deps (Fedora)\",\n \"package\": {\"name\": [\"rubygems\", \"ruby-devel\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'RedHat'\",\n },\n {\n \"name\": \"Install Ruby Deps (Alpine)\",\n \"package\": {\"name\": [\"ruby-dev\", \"ruby-bundler\"], \"state\": \"present\"},\n \"become\": True,\n \"when\": \"ansible_facts['os_family'] == 'Alpine'\",\n },\n {\n \"name\": \"Install wpscan gem\",\n \"gem\": {\"name\": \"wpscan\", \"state\": \"latest\", \"user_install\": False},\n \"become\": True,\n },\n ]\n\n async def setup(self):\n self.processed = set()\n self.ignore_events = [\"xmlrpc\", \"readme\"]\n self.api_key = self.config.get(\"api_key\", \"\")\n self.enumerate = self.config.get(\"enumerate\", \"vp,vt,cb,dbe\")\n self.proxy = self.scan.web_config.get(\"http_proxy\", \"\")\n self.threads = self.config.get(\"threads\", 5)\n self.request_timeout = self.config.get(\"request_timeout\", 5)\n self.connection_timeout = self.config.get(\"connection_timeout\", 2)\n self.disable_tls_checks = self.config.get(\"disable_tls_checks\", True)\n self.force = self.config.get(\"force\", False)\n return True\n\n async def filter_event(self, event):\n host_hash = hash(event.host)\n if host_hash in self.processed:\n return False, \"Host has already been processed\"\n if event.type == \"HTTP_RESPONSE\":\n is_redirect = str(event.data[\"status_code\"]).startswith(\"30\")\n if is_redirect:\n return False, \"URL is a redirect\"\n elif event.type == \"TECHNOLOGY\":\n if not event.data[\"technology\"].lower().startswith(\"wordpress\"):\n return False, \"technology is not wordpress\"\n self.processed.add(host_hash)\n return True\n\n async def handle_event(self, event):\n if event.type == \"HTTP_RESPONSE\":\n await self.handle_http_response(event)\n elif event.type == \"TECHNOLOGY\":\n await self.handle_technology(event)\n\n async def handle_http_response(self, source_event):\n url = source_event.parsed_url._replace(path=\"/\").geturl()\n command = self.construct_command(url)\n output = await self.run_process(command)\n for new_event in self.parse_wpscan_output(output.stdout, url, source_event):\n await self.emit_event(new_event)\n\n async def handle_technology(self, source_event):\n url = self.get_base_url(source_event)\n command = self.construct_command(url)\n output = await self.run_process(command)\n for new_event in self.parse_wpscan_output(output.stdout, url, source_event):\n await self.emit_event(new_event)\n\n def construct_command(self, url):\n # base executable\n command = [\"wpscan\", \"--url\", url]\n # proxy\n if self.proxy:\n command += [\"--proxy\", str(self.proxy)]\n # user agent\n command += [\"--user-agent\", f\"'{self.scan.useragent}'\"]\n # threads\n command += [\"--max-threads\", str(self.threads)]\n # request timeout\n command += [\"--request-timeout\", str(self.request_timeout)]\n # connection timeout\n command += [\"--connect-timeout\", str(self.connection_timeout)]\n # api key\n if self.api_key:\n command += [\"--api-token\", f\"{self.api_key}\"]\n # enumerate\n command += [\"--enumerate\", self.enumerate]\n # disable tls checks\n if self.disable_tls_checks:\n command += [\"--disable-tls-checks\"]\n # force\n if self.force:\n command += [\"--force\"]\n # output format\n command += [\"--format\", \"json\"]\n return command\n\n def parse_wpscan_output(self, output, base_url, source_event):\n json_output = json.loads(output)\n interesting_json = json_output.get(\"interesting_findings\", {}) or {}\n version_json = json_output.get(\"version\", {}) or {}\n theme_json = json_output.get(\"main_theme\", {}) or {}\n plugins_json = json_output.get(\"plugins\", {}) or {}\n if interesting_json:\n yield from self.parse_wp_misc(interesting_json, base_url, source_event)\n if version_json:\n yield from self.parse_wp_version(version_json, base_url, source_event)\n if theme_json:\n yield from self.parse_wp_themes(theme_json, base_url, source_event)\n if plugins_json:\n yield from self.parse_wp_plugins(plugins_json, base_url, source_event)\n\n def parse_wp_misc(self, interesting_json, base_url, source_event):\n for finding in interesting_json:\n url = finding.get(\"url\", base_url)\n type = finding[\"type\"]\n if type in self.ignore_events:\n continue\n description_string = finding[\"to_s\"]\n interesting_entries = finding[\"interesting_entries\"]\n if type == \"headers\":\n for header in interesting_entries:\n yield self.make_event(\n {\"technology\": str(header).lower(), \"url\": url, \"host\": str(source_event.host)},\n \"TECHNOLOGY\",\n source_event,\n )\n else:\n url_event = self.make_event(url, \"URL_UNVERIFIED\", parent=source_event, tags=[\"httpx-safe\"])\n if url_event:\n yield url_event\n yield self.make_event(\n {\"description\": description_string, \"url\": url, \"host\": str(source_event.host)},\n \"FINDING\",\n source_event,\n )\n\n def parse_wp_version(self, version_json, url, source_event):\n version = version_json.get(\"number\", \"\")\n if version:\n technology = f\"wordpress {version}\"\n else:\n technology = \"wordpress detect\"\n yield self.make_event(\n {\"technology\": str(technology).lower(), \"url\": url, \"host\": str(source_event.host)},\n \"TECHNOLOGY\",\n source_event,\n )\n for wp_vuln in version_json.get(\"vulnerabilities\", []):\n yield self.make_event(\n {\n \"severity\": \"HIGH\",\n \"host\": str(source_event.host),\n \"url\": url,\n \"description\": self.vulnerability_to_s(wp_vuln),\n },\n \"VULNERABILITY\",\n source_event,\n )\n\n def parse_wp_themes(self, theme_json, url, source_event):\n name = theme_json.get(\"slug\", \"\")\n version = theme_json.get(\"version\", {}).get(\"number\", \"\")\n if name:\n if version:\n technology = f\"{name} v{version}\"\n else:\n technology = name\n yield self.make_event(\n {\"technology\": str(technology).lower(), \"url\": url, \"host\": str(source_event.host)},\n \"TECHNOLOGY\",\n source_event,\n )\n for theme_vuln in theme_json.get(\"vulnerabilities\", []):\n yield self.make_event(\n {\n \"severity\": \"HIGH\",\n \"host\": str(source_event.host),\n \"url\": url,\n \"description\": self.vulnerability_to_s(theme_vuln),\n },\n \"VULNERABILITY\",\n source_event,\n )\n\n def parse_wp_plugins(self, plugins_json, base_url, source_event):\n for name, plugin in plugins_json.items():\n url = plugin.get(\"location\", base_url)\n if url != base_url:\n url_event = self.make_event(url, \"URL_UNVERIFIED\", parent=source_event, tags=[\"httpx-safe\"])\n if url_event:\n yield url_event\n version = plugin.get(\"version\", {}).get(\"number\", \"\")\n if version:\n technology = f\"{name} {version}\"\n else:\n technology = name\n yield self.make_event(\n {\"technology\": str(technology).lower(), \"url\": url, \"host\": str(source_event.host)},\n \"TECHNOLOGY\",\n source_event,\n )\n for vuln in plugin.get(\"vulnerabilities\", []):\n yield self.make_event(\n {\n \"severity\": \"HIGH\",\n \"host\": str(source_event.host),\n \"url\": url,\n \"description\": self.vulnerability_to_s(vuln),\n },\n \"VULNERABILITY\",\n source_event,\n )\n\n def vulnerability_to_s(self, vuln_json):\n string = []\n title = vuln_json.get(\"title\", \"\")\n string.append(f\"Title: {title}\")\n fixed_in = vuln_json.get(\"fixed_in\", \"\")\n string.append(f\"Fixed in: {fixed_in}\")\n references = vuln_json.get(\"references\", {})\n if references:\n cves = references.get(\"cve\", [])\n urls = references.get(\"url\", [])\n youtube_urls = references.get(\"youtube\", [])\n cves_list = []\n for cve in cves:\n cves_list.append(f\"CVE-{cve}\")\n if cves_list:\n string.append(f\"CVEs: [{', '.join(cves_list)}]\")\n if urls:\n string.append(f\"References: [{', '.join(urls)}]\")\n if youtube_urls:\n string.append(f\"Youtube Links: [{', '.join(youtube_urls)}]\")\n return \" \".join(string)\n\n def get_base_url(self, event):\n base_url = event.data.get(\"url\", \"\")\n if not base_url:\n base_url = f\"https://{event.host}\"\n return self.helpers.urlparse(base_url)._replace(path=\"/\").geturl()\n" }, { "path": "bbot/presets/baddns-intense.yml", "content": "description: Run all baddns modules and submodules.\n\n\nmodules:\n - baddns\n - baddns_zone\n - baddns_direct\n\nconfig:\n modules:\n baddns:\n enabled_submodules: [CNAME,references,MX,NS,TXT]\n" }, { "path": "bbot/presets/cloud-enum.yml", "content": "description: Enumerate cloud resources such as storage buckets, etc.\n\ninclude:\n - subdomain-enum\n\nflags:\n - cloud-enum\n" }, { "path": "bbot/presets/code-enum.yml", "content": "description: Enumerate Git repositories, Docker images, etc.\n\nflags:\n - code-enum\n" }, { "path": "bbot/presets/email-enum.yml", "content": "description: Enumerate email addresses from APIs, web crawling, etc.\n\nflags:\n - email-enum\n\noutput_modules:\n - emails\n" }, { "path": "bbot/presets/fast.yml", "content": "description: Scan only the provided targets as fast as possible - no extra discovery\n\nexclude_modules:\n - excavate\n\nconfig:\n # only scan the exact targets specified\n scope:\n strict: true\n # speed up dns resolution by doing A/AAAA only - not MX/NS/SRV/etc\n dns:\n minimal: true\n # essential speculation only\n modules:\n speculate:\n essential_only: true\n" }, { "path": "bbot/presets/kitchen-sink.yml", "content": "description: Everything everywhere all at once\n\ninclude:\n - subdomain-enum\n - cloud-enum\n - code-enum\n - email-enum\n - spider\n - web-basic\n - paramminer\n - dirbust-light\n - web-screenshots\n - baddns-intense\n\nconfig:\n modules:\n baddns:\n enable_references: True\n" }, { "path": "bbot/presets/nuclei/nuclei-budget.yml", "content": "description: Run nuclei scans against all discovered targets, using budget mode to look for low hanging fruit with greatly reduced number of requests\n\nmodules:\n - httpx\n - nuclei\n - portfilter\n\nconfig:\n modules:\n nuclei:\n mode: budget\n budget: 10\n directory_only: true # Do not run nuclei on individual non-directory URLs\n\nconditions:\n - |\n {% if config.web.spider_distance != 0 %}\n {{ warn(\"Running nuclei with spider enabled is generally not recommended. Consider removing 'spider' preset.\") }}\n {% endif %}\n" }, { "path": "bbot/presets/nuclei/nuclei-intense.yml", "content": "description: Run nuclei scans against all discovered targets, allowing for spidering, against ALL URLs, and with additional discovery modules.\n\nmodules:\n - httpx\n - nuclei\n - robots\n - urlscan\n - portfilter\n - wayback\n\nconfig:\n modules:\n nuclei:\n directory_only: False # Will run nuclei on ALL discovered URLs - Be careful!\n wayback:\n urls: true\n\nconditions:\n - |\n {% if config.web.spider_distance == 0 and config.modules.nuclei.directory_only == False %}\n {{ warn(\"The 'nuclei-intense' preset turns the 'directory_only' limitation off on the nuclei module. To make the best use of this, you may want to enable spidering with 'spider' or 'spider-intense' preset.\") }}\n {% endif %}\n\n\n# Example for also running a dirbust\n\n#include:\n# - dirbust-light" }, { "path": "bbot/presets/nuclei/nuclei-technology.yml", "content": "description: Run nuclei scans against all discovered targets, running templates which match discovered technologies\n\nmodules:\n - httpx\n - nuclei\n - portfilter\n\nconfig:\n modules:\n nuclei:\n mode: technology\n directory_only: True # Do not run nuclei on individual non-directory URLs. This is less unsafe to disable with technology mode.\n\nconditions:\n - |\n {% if config.web.spider_distance != 0 %}\n {{ warn(\"Running nuclei with spider enabled is generally not recommended. Consider removing 'spider' preset.\") }}\n {% endif %}\n\n# Example for also running a dirbust\n\n#include:\n# - dirbust-light\n" }, { "path": "bbot/presets/nuclei/nuclei.yml", "content": "description: Run nuclei scans against all discovered targets\n\nmodules:\n - httpx\n - nuclei\n - portfilter\n\nconfig:\n modules:\n nuclei:\n directory_only: True # Do not run nuclei on individual non-directory URLs\n\n\nconditions:\n - |\n {% if config.web.spider_distance != 0 %}\n {{ warn(\"Running nuclei with spider enabled is generally not recommended. Consider removing 'spider' preset.\") }}\n {% endif %}\n\n\n\n# Additional Examples:\n\n# Slowing Down Scan\n\n#config:\n# modules:\n# nuclei:\n# ratelimit: 10\n# concurrency: 5\n\n\n\n\n" }, { "path": "bbot/presets/spider-intense.yml", "content": "description: Recursive web spider with more aggressive settings\n\ninclude:\n - spider\n \nconfig:\n web:\n # how many links to follow in a row\n spider_distance: 4\n # don't follow links whose directory depth is higher than 6\n spider_depth: 6\n # maximum number of links to follow per page\n spider_links_per_page: 50\n" }, { "path": "bbot/presets/spider.yml", "content": "description: Recursive web spider\n\nmodules:\n - httpx\n\nblacklist:\n # Prevent spider from invalidating sessions by logging out\n - \"RE:/.*(sign|log)[_-]?out\"\n\nconfig:\n web:\n # how many links to follow in a row\n spider_distance: 2\n # don't follow links whose directory depth is higher than 4\n spider_depth: 4\n # maximum number of links to follow per page\n spider_links_per_page: 25\n" }, { "path": "bbot/presets/subdomain-enum.yml", "content": "description: Enumerate subdomains via APIs, brute-force\n\nflags:\n # enable every module with the subdomain-enum flag\n - subdomain-enum\n\noutput_modules:\n # output unique subdomains to TXT file\n - subdomains\n\nconfig:\n dns:\n threads: 25\n brute_threads: 1000\n # put your API keys here\n # modules:\n # github:\n # api_key: \"\"\n # chaos:\n # api_key: \"\"\n # securitytrails:\n # api_key: \"\"\n" }, { "path": "bbot/presets/tech-detect.yml", "content": "description: Detect technologies via Nuclei, and FingerprintX\n\nmodules:\n - nuclei\n - fingerprintx\n\nconfig:\n modules:\n nuclei:\n tags: tech\n" }, { "path": "bbot/presets/web/dirbust-heavy.yml", "content": "description: Recursive web directory brute-force (aggressive)\n\ninclude:\n - spider\n\nflags:\n - iis-shortnames\n\nmodules:\n - ffuf\n - wayback\n\nconfig:\n modules:\n iis_shortnames:\n # we exploit the shortnames vulnerability to produce URL_HINTs which are consumed by ffuf_shortnames\n detect_only: False\n ffuf:\n depth: 3\n lines: 5000\n extensions:\n - php\n - asp\n - aspx\n - ashx\n - asmx\n - jsp\n - jspx\n - cfm\n - zip\n - conf\n - config\n - xml\n - json\n - yml\n - yaml\n # emit URLs from wayback\n wayback:\n urls: True\n" }, { "path": "bbot/presets/web/dirbust-light.yml", "content": "description: Basic web directory brute-force (surface-level directories only)\n\ninclude:\n - iis-shortnames\n\nmodules:\n - ffuf\n\nconfig:\n modules:\n ffuf:\n # wordlist size = 1000\n lines: 1000\n" }, { "path": "bbot/presets/web/dotnet-audit.yml", "content": "description: Comprehensive scan for all IIS/.NET specific modules and module settings\n\n\ninclude:\n - iis-shortnames\n\nmodules:\n - httpx\n - badsecrets\n - ffuf_shortnames\n - ffuf\n - telerik\n - ajaxpro\n - dotnetnuke\n - aspnet_bin_exposure\n\nconfig:\n modules:\n ffuf:\n extensions: asp,aspx,ashx,asmx,ascx\n extensions_ignore_case: True\n ffuf_shortnames:\n find_subwords: True\n telerik:\n exploit_RAU_crypto: True\n include_subdirs: True # Run against every directory, not the default first received URL per-host\n" }, { "path": "bbot/presets/web/iis-shortnames.yml", "content": "description: Recursively enumerate IIS shortnames\n\nflags:\n - iis-shortnames\n\nconfig:\n modules:\n iis_shortnames:\n # exploit the vulnerability\n detect_only: false\n" }, { "path": "bbot/presets/web/lightfuzz-heavy.yml", "content": "description: Discover web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques, including POST parameters, which are more invasive. Uses all lightfuzz modules, and adds paramminer modules for parameter discovery. Avoids running against confirmed WAFs.\n\ninclude:\n - lightfuzz-medium\n\nflags:\n - web-paramminer\n\nmodules:\n - robots\n\nconfig:\n modules:\n lightfuzz:\n enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss,esi]\n disable_post: False\n try_post_as_get: True\n try_get_as_post: True\n" }, { "path": "bbot/presets/web/lightfuzz-light.yml", "content": "description: Discover web parameters and lightly fuzz them for vulnerabilities, with only the most common vulnerabilities and minimal extra modules. Safest to run alongside larger scans.\n\nmodules:\n - httpx\n - lightfuzz\n - portfilter\n \nconfig:\n url_querystring_remove: False # don't strip off the querystring (BBOT normally does this; but lightfuzz needs it)\n url_querystring_collapse: True # in cases where the same parameter has multiple values, collapse them into a single parameter to save on fuzzing attempts\n modules:\n lightfuzz:\n enabled_submodules: [path,sqli,xss] # only look for the most common vulnerabilities\n disable_post: True # don't send POST requests (less aggressive)\n avoid_wafs: True\n\nconditions:\n- |\n {% if config.web.spider_distance == 0 %}\n {{ warn(\"Lightfuzz works much better with spider enabled! Consider adding 'spider' or 'spider-intense' preset.\") }}\n {% endif %}\n" }, { "path": "bbot/presets/web/lightfuzz-medium.yml", "content": "description: Discover web parameters and lightly fuzz them for vulnerabilities. Uses all lightfuzz modules, without some of the more intense discovery techniques. Does not send POST requests. This is the default lightfuzz preset; if you're not sure which one to use, this is a good starting point. Avoids running against confirmed WAFs.\n\ninclude:\n - lightfuzz-light\n\nmodules:\n - badsecrets\n - hunt\n - reflected_parameters\n \nconfig:\n modules:\n lightfuzz:\n enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss,esi]\n try_post_as_get: True\n" }, { "path": "bbot/presets/web/lightfuzz-superheavy.yml", "content": "description: Discover web parameters and lightly fuzz them for vulnerabilities, with the most intense discovery techniques, including POST parameters, which are more invasive. Uses all lightfuzz modules, adds paramminer modules for parameter discovery, and tests each unique parameter-value instance individually.\n\ninclude:\n - lightfuzz-heavy\n\nconfig:\n url_querystring_collapse: False # in cases where the same parameter is observed multiple times, fuzz them individually instead of collapsing them into a single parameter\n modules:\n lightfuzz:\n force_common_headers: True # Fuzz common headers like X-Forwarded-For even if they're not observed on the target\n enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss,esi]\n avoid_wafs: False\n excavate:\n speculate_params: True # speculate potential parameters extracted from JSON/XML web responses\n" }, { "path": "bbot/presets/web/lightfuzz-xss.yml", "content": "description: Discover web parameters and lightly fuzz them, limited to just GET-based xss vulnerabilities. Avoids running against confirmed WAFs. This is an example of a custom lightfuzz preset, selectively enabling a single lightfuzz module.\n\nmodules:\n - httpx\n - lightfuzz\n - paramminer_getparams\n - reflected_parameters\n - portfilter\n \nconfig:\n url_querystring_remove: False\n url_querystring_collapse: False\n modules:\n lightfuzz:\n enabled_submodules: [xss]\n disable_post: True\n\nconditions:\n - |\n {% if config.web.spider_distance == 0 %}\n {{ warn(\"The lightfuzz-xss preset works much better with spider enabled! Consider adding 'spider' or 'spider-intense' preset.\") }}\n {% endif %}\n" }, { "path": "bbot/presets/web/paramminer.yml", "content": "description: Discover new web parameters via brute-force, and analyze them with additional modules\n\nflags:\n - web-paramminer\n\nmodules:\n - httpx\n - reflected_parameters\n - hunt\n\nconditions:\n - |\n {% if config.web.spider_distance == 0 %}\n {{ warn(\"The paramminer preset works much better with spider enabled! Consider adding 'spider' or 'spider-intense' preset.\") }}\n {% endif %}" }, { "path": "bbot/presets/web-basic.yml", "content": "description: Quick web scan\n\ninclude:\n - iis-shortnames\n\nflags:\n - web-basic\n" }, { "path": "bbot/presets/web-screenshots.yml", "content": "description: Take screenshots of webpages\n\nflags:\n - web-screenshots\n\nconfig:\n modules:\n gowitness:\n resolution_x: 1440\n resolution_y: 900\n # folder to output web screenshots (default is inside ~/.bbot/scans/scan_name)\n output_path: \"\"\n # whether to take screenshots of social media pages\n social: True\n" }, { "path": "bbot/presets/web-thorough.yml", "content": "description: Aggressive web scan\n\ninclude:\n # include the web-basic preset\n - web-basic\n\nflags:\n - web-thorough\n" }, { "path": "bbot/scanner/__init__.py", "content": "from .preset import Preset\nfrom .scanner import Scanner\n\n__all__ = [\"Preset\", \"Scanner\"]\n" }, { "path": "bbot/scanner/dispatcher.py", "content": "import logging\nimport traceback\n\nlog = logging.getLogger(\"bbot.scanner.dispatcher\")\n\n\nclass Dispatcher:\n \"\"\"\n Enables custom hooks/callbacks on certain scan events\n \"\"\"\n\n def set_scan(self, scan):\n self.scan = scan\n\n async def on_start(self, scan):\n return\n\n async def on_finish(self, scan):\n return\n\n async def on_status(self, status, scan_id):\n \"\"\"\n Execute an event when the scan's status is updated\n \"\"\"\n self.scan.debug(f\"Setting scan status to {status}\")\n\n async def catch(self, callback, *args, **kwargs):\n try:\n return await callback(*args, **kwargs)\n except Exception as e:\n log.error(f\"Error in {callback.__qualname__}(): {e}\")\n log.trace(traceback.format_exc())\n" }, { "path": "bbot/scanner/manager.py", "content": "import asyncio\nfrom contextlib import suppress\nfrom radixtarget.helpers import host_size_key\n\nfrom bbot.modules.base import BaseInterceptModule\n\n\nclass ScanIngress(BaseInterceptModule):\n \"\"\"\n This is always the first intercept module in the chain, responsible for basic scope checks\n\n It has its own incoming queue, but will also pull events from modules' outgoing queues\n \"\"\"\n\n watched_events = [\"*\"]\n # accept all events regardless of scope distance\n scope_distance_modifier = None\n _name = \"_scan_ingress\"\n _qsize = -1\n\n @property\n def priority(self):\n # we are the highest priority\n return -99\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self._module_priority_weights = None\n self._non_intercept_modules = None\n # track incoming duplicates module-by-module (for `suppress_dupes` attribute of modules)\n self.incoming_dup_tracker = set()\n\n async def init_events(self, event_seeds=None):\n \"\"\"\n Initializes events by seeding the scanner with target events and distributing them for further processing.\n\n Notes:\n - This method populates the event queue with initial target events.\n - It also marks the Scan object as finished with initialization by setting `_finished_init` to True.\n \"\"\"\n async with (\n self.scan._acatch(self.init_events, unhandled_is_critical=True),\n self._task_counter.count(self.init_events),\n ):\n if event_seeds is None:\n event_seeds = self.scan.target.seeds.event_seeds\n root_event = self.scan.root_event\n event_seeds = sorted(event_seeds, key=lambda e: (host_size_key(str(e.host)), e.data))\n # queue root scan event\n await self.queue_event(root_event, {})\n target_module = self.scan._make_dummy_module(name=\"TARGET\", _type=\"TARGET\")\n # queue each target in turn\n for event_seed in event_seeds:\n event = self.scan.make_event(\n event_seed.data,\n event_seed.type,\n parent=root_event,\n module=target_module,\n context=f\"Scan {self.scan.name} seeded with \" + \"{event.type}: {event.data}\",\n tags=[\"target\"],\n )\n self.verbose(f\"Target: {event}\")\n # don't fill up the queue with too many events\n while self.incoming_event_queue.qsize() > 100:\n await asyncio.sleep(0.2)\n await self.queue_event(event, {})\n await asyncio.sleep(0.1)\n self.scan._finished_init = True\n\n async def handle_event(self, event, **kwargs):\n # don't accept dummy events\n if event._dummy:\n return False, \"cannot emit dummy event\"\n\n # don't accept events with self as parent\n if not event.type == \"SCAN\":\n if event == event.get_parent():\n return False, \"event's parent is itself\"\n if not event.discovery_context:\n self.warning(f\"Event {event} has no discovery context\")\n\n # don't accept duplicates\n if self.is_incoming_duplicate(event, add=True):\n if not event._graph_important:\n return False, \"event was already emitted by its module\"\n else:\n self.debug(\n f\"Event {event} was already emitted by its module, but it's graph-important so it gets a pass\"\n )\n\n # update event's scope distance based on its parent\n event.scope_distance = event.parent.scope_distance + 1\n\n # special handling of URL extensions\n url_extension = getattr(event, \"url_extension\", None)\n if url_extension is not None:\n # blacklist by extension\n if url_extension in self.scan.url_extension_blacklist:\n self.debug(\n f\"Blacklisting {event} because its extension (.{url_extension}) is blacklisted in the config\"\n )\n event.add_tag(\"blacklisted\")\n\n # main scan blacklist\n host_filterable = getattr(event, \"host_filterable\", None)\n event_blacklisted = False\n if host_filterable:\n event_blacklisted = self.scan.blacklisted(host_filterable)\n\n # reject all blacklisted events\n if event_blacklisted or \"blacklisted\" in event.tags:\n return False, \"event is blacklisted\"\n\n # Scope shepherding\n # here is where we make sure in-scope events are set to their proper scope distance\n if event.host:\n event_whitelisted = self.scan.whitelisted(event)\n if event_whitelisted:\n self.debug(f\"Making {event} in-scope because its main host matches the scan target\")\n event.scope_distance = 0\n\n # nerf event's priority if it's not in scope\n event.module_priority += event.scope_distance\n\n @property\n def non_intercept_modules(self):\n if self._non_intercept_modules is None:\n self._non_intercept_modules = [m for m in self.scan.modules.values() if not m._intercept]\n return self._non_intercept_modules\n\n @property\n def incoming_queues(self):\n queues = [self.incoming_event_queue] + [m.outgoing_event_queue for m in self.non_intercept_modules]\n return [q for q in queues if q is not False]\n\n @property\n def module_priority_weights(self):\n if not self._module_priority_weights:\n # we subtract from six because lower priorities == higher weights\n priorities = [5] + [6 - m.priority for m in self.non_intercept_modules]\n self._module_priority_weights = priorities\n return self._module_priority_weights\n\n async def get_incoming_event(self):\n for q in self.helpers.weighted_shuffle(self.incoming_queues, self.module_priority_weights):\n try:\n return q.get_nowait()\n except (asyncio.queues.QueueEmpty, AttributeError):\n continue\n raise asyncio.queues.QueueEmpty()\n\n def is_incoming_duplicate(self, event, add=False):\n \"\"\"\n Calculate whether an event is a duplicate in the context of the module that emitted it\n This will return True if the event's parent module has raised the event before.\n \"\"\"\n try:\n event_hash = event.module._outgoing_dedup_hash(event)\n except AttributeError:\n module_name = str(getattr(event, \"module\", \"\"))\n event_hash = hash((event, module_name))\n is_dup = event_hash in self.incoming_dup_tracker\n if add:\n self.incoming_dup_tracker.add(event_hash)\n suppress_dupes = getattr(event.module, \"suppress_dupes\", True)\n if suppress_dupes and is_dup:\n return True\n return False\n\n\nclass ScanEgress(BaseInterceptModule):\n \"\"\"\n This is always the last intercept module in the chain, responsible for executing and acting on the\n `abort_if` and `on_success_callback` functions.\n \"\"\"\n\n watched_events = [\"*\"]\n # accept all events regardless of scope distance\n scope_distance_modifier = None\n _name = \"_scan_egress\"\n\n @property\n def priority(self):\n # we are the lowest priority\n return 99\n\n async def handle_event(self, event, **kwargs):\n abort_if = kwargs.pop(\"abort_if\", None)\n on_success_callback = kwargs.pop(\"on_success_callback\", None)\n\n # mark omitted event types\n # we could do this all in the output module's filter_event(), but we mark it here permanently so the events' .get_parent() can factor in the omission, and skip over omitted parents\n omitted_event_type = event.type in self.scan.omitted_event_types\n is_target = \"target\" in event.tags\n if omitted_event_type and not is_target:\n self.debug(f\"Making {event} omitted because its type is omitted in the config\")\n event._omit = True\n\n # make event internal if it's above our configured report distance\n event_in_report_distance = event.scope_distance <= self.scan.scope_report_distance\n event_will_be_output = event.always_emit or event_in_report_distance\n\n # if an event isn't being re-emitted for output, we may want to make it internal\n if not event._graph_important:\n if not event_will_be_output and not event.internal:\n self.debug(\n f\"Making {event} internal because its scope_distance ({event.scope_distance}) > scope_report_distance ({self.scan.scope_report_distance})\"\n )\n event.internal = True\n\n # mark special URLs (e.g. Javascript) as internal so they don't get output except when they're critical to the graph\n if event.type.startswith(\"URL\") and not event.internal:\n extension = getattr(event, \"url_extension\", \"\")\n if extension in self.scan.url_extension_special:\n self.debug(f\"Making {event} internal because it is a special URL (extension {extension})\")\n event.internal = True\n\n # custom callback - abort event emission if it returns true\n abort_result = False\n if callable(abort_if):\n async with self.scan._acatch(context=abort_if):\n abort_result = await self.scan.helpers.execute_sync_or_async(abort_if, event)\n msg = f\"{event.module}: not raising event {event} due to custom criteria in abort_if()\"\n with suppress(ValueError, TypeError):\n abort_result, reason = abort_result\n msg += f\": {reason}\"\n if abort_result:\n return False, msg\n\n if event._suppress_chain_dupes:\n for parent in event.get_parents():\n if parent == event:\n return False, f\"an identical parent {event} was found, and _suppress_chain_dupes=True\"\n\n # if we discovered something interesting from an internal event,\n # make sure we preserve its chain of parents\n # here we retroactively resurrect any interesting internal events that led to this discovery\n # \"interesting\" meaning any event types that aren't omitted in the config\n # (by using .get_parent() instead of .parent, we're intentionally skipping over omitted events)\n parent = event.get_parent()\n event_is_graph_worthy = (not event.internal) or event._graph_important\n parent_is_graph_worthy = (not parent.internal) or parent._graph_important\n if event_is_graph_worthy and not parent_is_graph_worthy:\n parent_in_report_distance = parent.scope_distance <= self.scan.scope_report_distance\n self.debug(f\"parent {parent} in report distance: {parent_in_report_distance}\")\n if parent_in_report_distance:\n self.debug(f\"setting parent {parent} internal to False\")\n parent.internal = False\n if not parent._graph_important:\n self.debug(f\"Re-queuing internal event {parent} with parent {event} to prevent graph orphan\")\n parent._graph_important = True\n await self.emit_event(parent)\n\n # run success callback before distributing event (so it can add tags, etc.)\n if callable(on_success_callback):\n async with self.scan._acatch(context=on_success_callback):\n await self.scan.helpers.execute_sync_or_async(on_success_callback, event)\n\n async def forward_event(self, event, kwargs):\n \"\"\"\n Queue event with modules\n \"\"\"\n # absorb event into the word cloud if it's in scope\n if -1 < event.scope_distance < 1:\n self.scan.word_cloud.absorb_event(event)\n\n for mod in self.scan.modules.values():\n # don't distribute events to intercept modules\n if not mod._intercept:\n await mod.queue_event(event)\n" }, { "path": "bbot/scanner/preset/__init__.py", "content": "from .preset import Preset\n\n__all__ = [\"Preset\"]\n" }, { "path": "bbot/scanner/preset/args.py", "content": "import re\nimport logging\nimport argparse\nfrom omegaconf import OmegaConf\n\nfrom bbot.errors import *\nfrom bbot.core.helpers.misc import chain_lists, get_closest_match, get_keys_in_dot_syntax\n\nlog = logging.getLogger(\"bbot.presets.args\")\n\n\nuniversal_module_options = {\n \"batch_size\": \"The number of events to process in a single batch (only applies to batch modules)\",\n \"module_threads\": \"How many event handlers to run in parallel\",\n \"module_timeout\": \"Max time in seconds to spend handling each event or batch of events\",\n}\n\n\nclass BBOTArgs:\n # module config options to exclude from validation\n exclude_from_validation = re.compile(\n r\".*modules\\.[a-z0-9_]+\\.(?:\" + \"|\".join(universal_module_options.keys()) + \")$\"\n )\n\n scan_examples = [\n (\n \"Subdomains\",\n \"Perform a full subdomain enumeration on evilcorp.com\",\n \"bbot -t evilcorp.com -p subdomain-enum\",\n ),\n (\n \"Subdomains (passive only)\",\n \"Perform a passive-only subdomain enumeration on evilcorp.com\",\n \"bbot -t evilcorp.com -p subdomain-enum -rf passive\",\n ),\n (\n \"Subdomains + port scan + web screenshots\",\n \"Port-scan every subdomain, screenshot every webpage, output to current directory\",\n \"bbot -t evilcorp.com -p subdomain-enum -m portscan gowitness -n my_scan -o .\",\n ),\n (\n \"Subdomains + basic web scan\",\n \"A basic web scan includes robots.txt, storage buckets, IIS shortnames, and other non-intrusive web modules\",\n \"bbot -t evilcorp.com -p subdomain-enum web-basic\",\n ),\n (\n \"Web spider\",\n \"Crawl www.evilcorp.com up to a max depth of 2, automatically extracting emails, secrets, etc.\",\n \"bbot -t www.evilcorp.com -p spider -c web.spider_distance=2 web.spider_depth=2\",\n ),\n (\n \"Everything everywhere all at once\",\n \"Subdomains, emails, cloud buckets, port scan, basic web, web screenshots, nuclei\",\n \"bbot -t evilcorp.com -p kitchen-sink\",\n ),\n ]\n\n usage_examples = [\n (\n \"List modules\",\n \"\",\n \"bbot -l\",\n ),\n (\n \"List output modules\",\n \"\",\n \"bbot -lo\",\n ),\n (\n \"List presets\",\n \"\",\n \"bbot -lp\",\n ),\n (\n \"List flags\",\n \"\",\n \"bbot -lf\",\n ),\n (\n \"Show help for a specific module\",\n \"\",\n \"bbot -mh \",\n ),\n ]\n\n epilog = \"EXAMPLES\\n\"\n for example in (scan_examples, usage_examples):\n for title, description, command in example:\n epilog += f\"\\n {title}:\\n {command}\\n\"\n\n def __init__(self, preset):\n self.preset = preset\n self._config = None\n\n self.parser = self.create_parser()\n self._parsed = None\n\n @property\n def parsed(self):\n if self._parsed is None:\n self._parsed = self.parser.parse_args()\n self.sanitize_args()\n return self._parsed\n\n def preset_from_args(self):\n # the order here is important\n # first we make the preset\n args_preset = self.preset.__class__(\n *self.parsed.targets,\n whitelist=self.parsed.whitelist,\n blacklist=self.parsed.blacklist,\n name=\"args_preset\",\n )\n\n # then we load requested preset\n # this is important so we can load custom module directories, pull in custom flags, module config options, etc.\n for preset_arg in self.parsed.preset:\n try:\n args_preset.include_preset(preset_arg)\n except BBOTArgumentError:\n raise\n except Exception as e:\n raise BBOTArgumentError(f'Error parsing preset \"{preset_arg}\": {e}')\n\n # then we set verbosity levels (so if the user enables -d they can see debug output)\n if self.parsed.silent:\n args_preset.silent = True\n if self.parsed.verbose:\n args_preset.verbose = True\n if self.parsed.debug:\n args_preset.debug = True\n\n # modules + flags\n args_preset.exclude_modules.update(set(self.parsed.exclude_modules))\n args_preset.exclude_flags.update(set(self.parsed.exclude_flags))\n args_preset.require_flags.update(set(self.parsed.require_flags))\n args_preset.explicit_scan_modules.update(set(self.parsed.modules))\n args_preset.explicit_output_modules.update(set(self.parsed.output_modules))\n args_preset.flags.update(set(self.parsed.flags))\n\n # output\n if self.parsed.json:\n args_preset.core.merge_custom({\"modules\": {\"stdout\": {\"format\": \"json\"}}})\n if self.parsed.brief:\n args_preset.core.merge_custom(\n {\"modules\": {\"stdout\": {\"event_fields\": [\"type\", \"scope_description\", \"data\"]}}}\n )\n if self.parsed.event_types:\n args_preset.core.merge_custom({\"modules\": {\"stdout\": {\"event_types\": self.parsed.event_types}}})\n if self.parsed.exclude_cdn:\n args_preset.explicit_scan_modules.add(\"portfilter\")\n\n # dependencies\n deps_config = args_preset.core.custom_config.get(\"deps\", {})\n if self.parsed.retry_deps:\n deps_config[\"behavior\"] = \"retry_failed\"\n elif self.parsed.force_deps:\n deps_config[\"behavior\"] = \"force_install\"\n elif self.parsed.no_deps:\n deps_config[\"behavior\"] = \"disable\"\n elif self.parsed.ignore_failed_deps:\n deps_config[\"behavior\"] = \"ignore_failed\"\n if deps_config:\n args_preset.core.merge_custom({\"deps\": deps_config})\n\n # other scan options\n if self.parsed.name is not None:\n args_preset.scan_name = self.parsed.name\n if self.parsed.output_dir is not None:\n args_preset.output_dir = self.parsed.output_dir\n if self.parsed.force:\n args_preset.force_start = self.parsed.force\n\n if self.parsed.proxy:\n args_preset.core.merge_custom({\"web\": {\"http_proxy\": self.parsed.proxy}})\n\n if self.parsed.custom_headers:\n args_preset.core.merge_custom({\"web\": {\"http_headers\": self.parsed.custom_headers}})\n\n if self.parsed.custom_cookies:\n args_preset.core.merge_custom({\"web\": {\"http_cookies\": self.parsed.custom_cookies}})\n\n if self.parsed.custom_yara_rules:\n args_preset.core.merge_custom(\n {\"modules\": {\"excavate\": {\"custom_yara_rules\": self.parsed.custom_yara_rules}}}\n )\n\n # Check if both user_agent and user_agent_suffix are set. If so combine them and merge into the config\n if self.parsed.user_agent and self.parsed.user_agent_suffix:\n modified_user_agent = f\"{self.parsed.user_agent} {self.parsed.user_agent_suffix}\"\n args_preset.core.merge_custom({\"web\": {\"user_agent\": modified_user_agent}})\n\n # If only user_agent_suffix is set, retrieve the existing user_agent from the merged config and append the suffix\n elif self.parsed.user_agent_suffix:\n existing_user_agent = args_preset.core.config.get(\"web\", {}).get(\"user_agent\", \"\")\n modified_user_agent = f\"{existing_user_agent} {self.parsed.user_agent_suffix}\"\n args_preset.core.merge_custom({\"web\": {\"user_agent\": modified_user_agent}})\n\n # If only user_agent is set, merge it directly\n elif self.parsed.user_agent:\n args_preset.core.merge_custom({\"web\": {\"user_agent\": self.parsed.user_agent}})\n\n # CLI config options (dot-syntax)\n for config_arg in self.parsed.config:\n try:\n # if that fails, try to parse as key=value syntax\n args_preset.core.merge_custom(OmegaConf.from_cli([config_arg]))\n except Exception as e:\n raise BBOTArgumentError(f'Error parsing command-line config option: \"{config_arg}\": {e}')\n\n # strict scope\n if self.parsed.strict_scope:\n args_preset.core.merge_custom({\"scope\": {\"strict\": True}})\n\n return args_preset\n\n def create_parser(self, *args, **kwargs):\n kwargs.update(\n {\n \"description\": \"Bighuge BLS OSINT Tool\",\n \"formatter_class\": argparse.RawTextHelpFormatter,\n \"epilog\": self.epilog,\n }\n )\n p = argparse.ArgumentParser(*args, **kwargs)\n\n target = p.add_argument_group(title=\"Target\")\n target.add_argument(\n \"-t\", \"--targets\", nargs=\"+\", default=[], help=\"Targets to seed the scan\", metavar=\"TARGET\"\n )\n target.add_argument(\n \"-w\",\n \"--whitelist\",\n nargs=\"+\",\n default=None,\n help=\"What's considered in-scope (by default it's the same as --targets)\",\n )\n target.add_argument(\"-b\", \"--blacklist\", nargs=\"+\", default=[], help=\"Don't touch these things\")\n target.add_argument(\n \"--strict-scope\",\n action=\"store_true\",\n help=\"Don't consider subdomains of target/whitelist to be in-scope\",\n )\n presets = p.add_argument_group(title=\"Presets\")\n presets.add_argument(\n \"-p\",\n \"--preset\",\n nargs=\"*\",\n help=\"Enable BBOT preset(s)\",\n metavar=\"PRESET\",\n default=[],\n )\n presets.add_argument(\n \"-c\",\n \"--config\",\n nargs=\"*\",\n help=\"Custom config options in key=value format: e.g. 'modules.shodan.api_key=1234'\",\n metavar=\"CONFIG\",\n default=[],\n )\n presets.add_argument(\"-lp\", \"--list-presets\", action=\"store_true\", help=\"List available presets.\")\n\n modules = p.add_argument_group(title=\"Modules\")\n modules.add_argument(\n \"-m\",\n \"--modules\",\n nargs=\"+\",\n default=[],\n help=f\"Modules to enable. Choices: {','.join(sorted(self.preset.module_loader.scan_module_choices))}\",\n metavar=\"MODULE\",\n )\n modules.add_argument(\"-l\", \"--list-modules\", action=\"store_true\", help=\"List available modules.\")\n modules.add_argument(\n \"-lmo\", \"--list-module-options\", action=\"store_true\", help=\"Show all module config options\"\n )\n modules.add_argument(\n \"-em\", \"--exclude-modules\", nargs=\"+\", default=[], help=\"Exclude these modules.\", metavar=\"MODULE\"\n )\n modules.add_argument(\n \"-f\",\n \"--flags\",\n nargs=\"+\",\n default=[],\n help=f\"Enable modules by flag. Choices: {','.join(sorted(self.preset.module_loader.flag_choices))}\",\n metavar=\"FLAG\",\n )\n modules.add_argument(\"-lf\", \"--list-flags\", action=\"store_true\", help=\"List available flags.\")\n modules.add_argument(\n \"-rf\",\n \"--require-flags\",\n nargs=\"+\",\n default=[],\n help=\"Only enable modules with these flags (e.g. -rf passive)\",\n metavar=\"FLAG\",\n )\n modules.add_argument(\n \"-ef\",\n \"--exclude-flags\",\n nargs=\"+\",\n default=[],\n help=\"Disable modules with these flags. (e.g. -ef aggressive)\",\n metavar=\"FLAG\",\n )\n modules.add_argument(\"--allow-deadly\", action=\"store_true\", help=\"Enable the use of highly aggressive modules\")\n\n scan = p.add_argument_group(title=\"Scan\")\n scan.add_argument(\"-n\", \"--name\", help=\"Name of scan (default: random)\", metavar=\"SCAN_NAME\")\n scan.add_argument(\"-v\", \"--verbose\", action=\"store_true\", help=\"Be more verbose\")\n scan.add_argument(\"-d\", \"--debug\", action=\"store_true\", help=\"Enable debugging\")\n scan.add_argument(\"-s\", \"--silent\", action=\"store_true\", help=\"Be quiet\")\n scan.add_argument(\n \"--force\",\n action=\"store_true\",\n help=\"Run scan even in the case of condition violations or failed module setups\",\n )\n scan.add_argument(\"-y\", \"--yes\", action=\"store_true\", help=\"Skip scan confirmation prompt\")\n scan.add_argument(\n \"--fast-mode\",\n action=\"store_true\",\n help=\"Scan only the provided targets as fast as possible, with no extra discovery\",\n )\n scan.add_argument(\"--dry-run\", action=\"store_true\", help=\"Abort before executing scan\")\n scan.add_argument(\n \"--current-preset\",\n action=\"store_true\",\n help=\"Show the current preset in YAML format\",\n )\n scan.add_argument(\n \"--current-preset-full\",\n action=\"store_true\",\n help=\"Show the current preset in its full form, including defaults\",\n )\n\n scan.add_argument(\n \"-mh\",\n \"--module-help\",\n default=None,\n help=\"Show help for a specific module\",\n metavar=\"MODULE\",\n )\n\n output = p.add_argument_group(title=\"Output\")\n output.add_argument(\n \"-o\",\n \"--output-dir\",\n help=\"Directory to output scan results\",\n metavar=\"DIR\",\n )\n output.add_argument(\n \"-om\",\n \"--output-modules\",\n nargs=\"+\",\n default=[],\n help=f\"Output module(s). Choices: {','.join(sorted(self.preset.module_loader.output_module_choices))}\",\n metavar=\"MODULE\",\n )\n output.add_argument(\"-lo\", \"--list-output-modules\", action=\"store_true\", help=\"List available output modules\")\n output.add_argument(\"--json\", \"-j\", action=\"store_true\", help=\"Output scan data in JSON format\")\n output.add_argument(\"--brief\", \"-br\", action=\"store_true\", help=\"Output only the data itself\")\n output.add_argument(\"--event-types\", nargs=\"+\", default=[], help=\"Choose which event types to display\")\n output.add_argument(\n \"--exclude-cdn\",\n \"-ec\",\n action=\"store_true\",\n help=\"Filter out unwanted open ports on CDNs/WAFs (80,443 only)\",\n )\n\n deps = p.add_argument_group(\n title=\"Module dependencies\", description=\"Control how modules install their dependencies\"\n )\n g2 = deps.add_mutually_exclusive_group()\n g2.add_argument(\"--no-deps\", action=\"store_true\", help=\"Don't install module dependencies\")\n g2.add_argument(\"--force-deps\", action=\"store_true\", help=\"Force install all module dependencies\")\n g2.add_argument(\"--retry-deps\", action=\"store_true\", help=\"Try again to install failed module dependencies\")\n g2.add_argument(\n \"--ignore-failed-deps\", action=\"store_true\", help=\"Run modules even if they have failed dependencies\"\n )\n g2.add_argument(\"--install-all-deps\", action=\"store_true\", help=\"Install dependencies for all modules\")\n\n misc = p.add_argument_group(title=\"Misc\")\n misc.add_argument(\"--version\", action=\"store_true\", help=\"show BBOT version and exit\")\n misc.add_argument(\"--proxy\", help=\"Use this proxy for all HTTP requests\", metavar=\"HTTP_PROXY\")\n misc.add_argument(\n \"-H\",\n \"--custom-headers\",\n nargs=\"+\",\n default=[],\n help=\"List of custom headers as key value pairs (header=value).\",\n )\n misc.add_argument(\n \"-C\",\n \"--custom-cookies\",\n nargs=\"+\",\n default=[],\n help=\"List of custom cookies as key value pairs (cookie=value).\",\n )\n misc.add_argument(\"--custom-yara-rules\", \"-cy\", help=\"Add custom yara rules to excavate\")\n\n misc.add_argument(\"--user-agent\", \"-ua\", help=\"Set the user-agent for all HTTP requests\")\n misc.add_argument(\"--user-agent-suffix\", \"-uas\", help=argparse.SUPPRESS, metavar=\"SUFFIX\", default=None)\n return p\n\n def sanitize_args(self):\n # silent implies -y\n if self.parsed.silent:\n self.parsed.yes = True\n # chain_lists allows either comma-separated or space-separated lists\n self.parsed.modules = chain_lists(self.parsed.modules)\n self.parsed.exclude_modules = chain_lists(self.parsed.exclude_modules)\n self.parsed.output_modules = chain_lists(self.parsed.output_modules)\n self.parsed.targets = chain_lists(\n self.parsed.targets, try_files=True, msg=\"Reading targets from file: {filename}\"\n )\n if self.parsed.whitelist is not None:\n self.parsed.whitelist = chain_lists(\n self.parsed.whitelist, try_files=True, msg=\"Reading whitelist from file: {filename}\"\n )\n self.parsed.blacklist = chain_lists(\n self.parsed.blacklist, try_files=True, msg=\"Reading blacklist from file: {filename}\"\n )\n self.parsed.flags = chain_lists(self.parsed.flags)\n self.parsed.exclude_flags = chain_lists(self.parsed.exclude_flags)\n self.parsed.require_flags = chain_lists(self.parsed.require_flags)\n self.parsed.event_types = [t.upper() for t in chain_lists(self.parsed.event_types)]\n\n # Custom Header Parsing / Validation\n custom_headers_dict = {}\n custom_header_example = \"Example: --custom-headers foo=bar foo2=bar2\"\n\n for i in self.parsed.custom_headers:\n parts = i.split(\"=\", 1)\n if len(parts) != 2:\n raise ValidationError(f\"Custom headers not formatted correctly (missing '='). {custom_header_example}\")\n k, v = parts\n if not k or not v:\n raise ValidationError(\n f\"Custom headers not formatted correctly (missing header name or value). {custom_header_example}\"\n )\n custom_headers_dict[k] = v\n self.parsed.custom_headers = custom_headers_dict\n\n # Custom Cookie Parsing / Validation\n custom_cookies_dict = {}\n custom_cookie_example = \"Example: --custom-cookies foo=bar foo2=bar2\"\n\n for i in self.parsed.custom_cookies:\n parts = i.split(\"=\", 1)\n if len(parts) != 2:\n raise ValidationError(f\"Custom cookies not formatted correctly (missing '='). {custom_cookie_example}\")\n k, v = parts\n if not k or not v:\n raise ValidationError(\n f\"Custom cookies not formatted correctly (missing cookie name or value). {custom_cookie_example}\"\n )\n custom_cookies_dict[k] = v\n self.parsed.custom_cookies = custom_cookies_dict\n\n # --fast-mode\n if self.parsed.fast_mode:\n self.parsed.preset += [\"fast\"]\n\n def validate(self):\n # validate config options\n sentinel = object()\n all_options = set(get_keys_in_dot_syntax(self.preset.core.default_config))\n for c in self.parsed.config:\n c = c.split(\"=\")[0].strip()\n v = OmegaConf.select(self.preset.core.default_config, c, default=sentinel)\n # if option isn't in the default config\n if v is sentinel:\n # skip if it's excluded from validation\n if self.exclude_from_validation.match(c):\n continue\n # otherwise, ensure it exists as a module option\n raise ValidationError(get_closest_match(c, all_options, msg=\"config option\"))\n" }, { "path": "bbot/scanner/preset/conditions.py", "content": "import logging\n\nfrom bbot.errors import *\n\nlog = logging.getLogger(\"bbot.preset.conditions\")\n\nJINJA_ENV = None\n\n\nclass ConditionEvaluator:\n def __init__(self, preset):\n self.preset = preset\n\n @property\n def context(self):\n return {\n \"preset\": self.preset,\n \"config\": self.preset.config,\n \"abort\": self.abort,\n \"warn\": self.warn,\n }\n\n def abort(self, message):\n if not self.preset.force_start:\n raise PresetAbortError(message)\n\n def warn(self, message):\n log.warning(message)\n\n def evaluate(self):\n context = self.context\n already_evaluated = set()\n for preset_name, condition in self.preset.conditions:\n condition_str = str(condition)\n if condition_str not in already_evaluated:\n already_evaluated.add(condition_str)\n try:\n self.check_condition(condition_str, context)\n except PresetAbortError as e:\n raise PresetAbortError(f'Preset \"{preset_name}\" requested abort: {e} (--force to override)')\n\n @property\n def jinja_env(self):\n from jinja2.sandbox import SandboxedEnvironment\n\n global JINJA_ENV\n if JINJA_ENV is None:\n JINJA_ENV = SandboxedEnvironment()\n return JINJA_ENV\n\n def check_condition(self, condition_str, context):\n log.debug(f'Evaluating condition \"{repr(condition_str)}\"')\n template = self.jinja_env.from_string(condition_str)\n template.render(context)\n" }, { "path": "bbot/scanner/preset/environ.py", "content": "import os\nimport sys\nimport omegaconf\nfrom pathlib import Path\n\nfrom bbot.core.helpers.misc import (\n cpu_architecture,\n cpu_architecture_golang,\n cpu_architecture_rust,\n os_platform,\n os_platform_friendly,\n)\n\n\nREQUESTS_PATCHED = False\n\n\ndef increase_limit(new_limit):\n try:\n import resource\n\n # Get current limit\n soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)\n\n new_limit = min(new_limit, hard_limit)\n\n # Attempt to set new limit\n resource.setrlimit(resource.RLIMIT_NOFILE, (new_limit, hard_limit))\n except Exception as e:\n sys.stderr.write(f\"Failed to set new ulimit: {e}\\n\")\n\n\nincrease_limit(65535)\n\n\n# Custom custom omegaconf resolver to get environment variables\ndef env_resolver(env_name, default=None):\n return os.getenv(env_name, default)\n\n\ndef add_to_path(v, k=\"PATH\", environ=None):\n \"\"\"\n Add an entry to a colon-separated PATH variable.\n If it's already contained in the value, shift it to be in first position.\n \"\"\"\n if environ is None:\n environ = os.environ\n var_list = os.environ.get(k, \"\").split(\":\")\n deduped_var_list = []\n for _ in var_list:\n if _ != v and _ not in deduped_var_list:\n deduped_var_list.append(_)\n deduped_var_list = [v] + deduped_var_list\n new_var_str = \":\".join(deduped_var_list).strip(\":\")\n environ[k] = new_var_str\n\n\n# if we're running in a virtual environment, make sure to include its /bin in PATH\nif sys.prefix != sys.base_prefix:\n bin_dir = str(Path(sys.prefix) / \"bin\")\n add_to_path(bin_dir)\n\n# add ~/.local/bin to PATH\nlocal_bin_dir = str(Path.home() / \".local\" / \"bin\")\nadd_to_path(local_bin_dir)\n\n\n# Register the new resolver\n# this allows you to substitute environment variables in your config like \"${env:PATH}\"\"\nomegaconf.OmegaConf.register_new_resolver(\"env\", env_resolver)\n\n\nclass BBOTEnviron:\n def __init__(self, preset):\n self.preset = preset\n\n def flatten_config(self, config, base=\"bbot\"):\n \"\"\"\n Flatten a JSON-like config into a list of environment variables:\n {\"modules\": [{\"httpx\": {\"timeout\": 5}}]} --> \"BBOT_MODULES_HTTPX_TIMEOUT=5\"\n \"\"\"\n if type(config) == omegaconf.dictconfig.DictConfig:\n for k, v in config.items():\n new_base = f\"{base}_{k}\"\n if type(v) == omegaconf.dictconfig.DictConfig:\n yield from self.flatten_config(v, base=new_base)\n elif type(v) != omegaconf.listconfig.ListConfig:\n yield (new_base.upper(), str(v))\n\n def prepare(self):\n \"\"\"\n Sync config to OS environment variables\n \"\"\"\n environ = dict(os.environ)\n\n # ensure bbot_tools\n environ[\"BBOT_TOOLS\"] = str(self.preset.core.tools_dir)\n add_to_path(str(self.preset.core.tools_dir), environ=environ)\n # ensure bbot_cache\n environ[\"BBOT_CACHE\"] = str(self.preset.core.cache_dir)\n # ensure bbot_temp\n environ[\"BBOT_TEMP\"] = str(self.preset.core.temp_dir)\n # ensure bbot_lib\n environ[\"BBOT_LIB\"] = str(self.preset.core.lib_dir)\n # export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:~/.bbot/lib/\n add_to_path(str(self.preset.core.lib_dir), k=\"LD_LIBRARY_PATH\", environ=environ)\n\n # platform variables\n environ[\"BBOT_OS_PLATFORM\"] = os_platform()\n environ[\"BBOT_OS\"] = os_platform_friendly()\n environ[\"BBOT_CPU_ARCH\"] = cpu_architecture()\n environ[\"BBOT_CPU_ARCH_GOLANG\"] = cpu_architecture_golang()\n environ[\"BBOT_CPU_ARCH_RUST\"] = cpu_architecture_rust()\n\n # copy config to environment\n bbot_environ = self.flatten_config(self.preset.config)\n environ.update(bbot_environ)\n\n # handle HTTP proxy\n http_proxy = self.preset.config.get(\"web\", {}).get(\"http_proxy\", \"\")\n if http_proxy:\n environ[\"HTTP_PROXY\"] = http_proxy\n environ[\"HTTPS_PROXY\"] = http_proxy\n else:\n environ.pop(\"HTTP_PROXY\", None)\n environ.pop(\"HTTPS_PROXY\", None)\n\n # ssl verification\n import urllib3\n\n urllib3.disable_warnings()\n ssl_verify = self.preset.config.get(\"ssl_verify\", False)\n\n global REQUESTS_PATCHED\n if not ssl_verify and not REQUESTS_PATCHED:\n REQUESTS_PATCHED = True\n import requests\n import functools\n\n requests.adapters.BaseAdapter.send = functools.partialmethod(\n requests.adapters.BaseAdapter.send, verify=False\n )\n requests.adapters.HTTPAdapter.send = functools.partialmethod(\n requests.adapters.HTTPAdapter.send, verify=False\n )\n requests.Session.request = functools.partialmethod(requests.Session.request, verify=False)\n requests.request = functools.partial(requests.request, verify=False)\n\n return environ\n" }, { "path": "bbot/scanner/preset/path.py", "content": "import logging\nfrom pathlib import Path\n\nfrom bbot.errors import *\n\nlog = logging.getLogger(\"bbot.presets.path\")\n\nDEFAULT_PRESET_PATH = Path(__file__).parent.parent.parent / \"presets\"\nDEFAULT_PRESET_PATH = DEFAULT_PRESET_PATH.expanduser().resolve()\n\n\nclass PresetPath:\n \"\"\"\n Keeps track of where to look for preset .yaml files\n \"\"\"\n\n def __init__(self):\n self.paths = [DEFAULT_PRESET_PATH]\n\n def find(self, filename):\n filename_path = Path(filename).expanduser()\n extension = filename_path.suffix.lower()\n file_candidates = set()\n extension_candidates = {\".yaml\", \".yml\"}\n if extension:\n extension_candidates.add(extension.lower())\n else:\n file_candidates.add(filename_path.stem)\n for ext in extension_candidates:\n file_candidates.add(f\"{filename_path.stem}{ext}\")\n file_candidates = sorted(file_candidates)\n file_candidates_str = \",\".join([str(s) for s in file_candidates])\n if \"/\" in str(filename):\n self.add_path(filename_path.parent)\n log.debug(f\"Searching for {file_candidates_str} in {[str(p) for p in self.paths]}\")\n for path in self.paths:\n for candidate in file_candidates:\n for file in path.rglob(f\"**/{candidate}\"):\n if file.is_file():\n log.verbose(f'Found preset matching \"{filename}\" at {file}')\n self.add_path(file.parent)\n return file\n raise ValidationError(\n f'Could not find preset at \"{filename}\" - file does not exist. Use -lp to list available presets'\n )\n\n def __str__(self):\n return \":\".join([str(s) for s in self.paths])\n\n def add_path(self, path):\n path = Path(path).expanduser().resolve()\n # skip if already in paths\n if path in self.paths:\n return\n # skip if path is a subdirectory of any path in paths\n if any(path.is_relative_to(p) for p in self.paths):\n return\n # skip if path is not a directory\n if not path.is_dir():\n log.debug(f'Path \"{path.resolve()}\" is not a directory')\n return\n # preemptively remove any paths that are subdirectories of the new path\n self.paths = [p for p in self.paths if not p.is_relative_to(path)]\n self.paths.insert(0, path)\n\n def __iter__(self):\n yield from self.paths\n\n\nPRESET_PATH = PresetPath()\n" }, { "path": "bbot/scanner/preset/preset.py", "content": "import os\nimport yaml\nimport logging\nimport omegaconf\nimport traceback\nfrom copy import copy\nfrom pathlib import Path\nfrom contextlib import suppress\n\nfrom .path import PRESET_PATH\n\nfrom bbot.errors import *\nfrom bbot.core import CORE\nfrom bbot.core.helpers.misc import make_table, mkdir, get_closest_match\n\n\nlog = logging.getLogger(\"bbot.presets\")\n\n\n_preset_cache = {}\n\n\n# cache default presets to prevent having to reload from disk\nDEFAULT_PRESETS = None\n\n\nclass BasePreset(type):\n def __call__(cls, *args, include=None, presets=None, name=None, description=None, _exclude=None, **kwargs):\n \"\"\"\n Handles loading of \"included\" presets, while preserving the proper load order\n\n Overriding __call__() allows us to reuse the logic from .merge() without duplicating functionality in __init__().\n \"\"\"\n include_preset = None\n\n # \"presets\" is alias to \"include\"\n if presets and include:\n raise ValueError(\n 'Cannot use both \"presets\" and \"include\" args at the same time (presets is an alias to include). Please pick one or the other :)'\n )\n if presets and not include:\n include = presets\n # include other presets\n if include and not isinstance(include, (list, tuple, set)):\n include = [include]\n\n main_preset = type.__call__(cls, *args, name=name, description=description, _exclude=_exclude, **kwargs)\n\n if include:\n include_preset = type.__call__(cls, name=name, description=description, _exclude=_exclude)\n for included_preset in include:\n include_preset.include_preset(included_preset)\n include_preset.merge(main_preset)\n return include_preset\n\n return main_preset\n\n\nclass Preset(metaclass=BasePreset):\n \"\"\"\n A preset is the central config for a BBOT scan. It contains everything a scan needs to run --\n targets, modules, flags, config options like API keys, etc.\n\n You can create a preset manually and pass it into `Scanner(preset=preset)`.\n Or, you can pass `Preset`'s kwargs into `Scanner()` and it will create the preset for you implicitly.\n\n Presets can include other presets (which can in turn include other presets, and so on).\n This works by merging each preset in turn using `Preset.merge()`.\n The order matters. In case of a conflict, the last preset to be merged wins priority.\n\n Presets can be loaded from or saved to YAML. BBOT has a number of ready-made presets for common tasks like\n subdomain enumeration, web spidering, dirbusting, etc.\n\n Presets are highly customizable via `conditions`, which use the Jinja2 templating engine.\n Using `conditions`, you can define custom logic to inspect the final preset before the scan starts, and change it if need be.\n Based on the state of the preset, you can print a warning message, abort the scan, enable/disable modules, etc..\n\n Attributes:\n target (Target): Target(s) of scan.\n whitelist (Target): Scan whitelist (by default this is the same as `target`).\n blacklist (Target): Scan blacklist (this takes ultimate precedence).\n helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc.\n output_dir (pathlib.Path): Output directory for scan.\n scan_name (str): Name of scan. Defaults to random value, e.g. \"demonic_jimmy\".\n name (str): Human-friendly name of preset. Used mainly for logging purposes.\n description (str): Description of preset.\n modules (set): Combined modules to enable for the scan. Includes scan modules, internal modules, and output modules.\n scan_modules (set): Modules to enable for the scan.\n output_modules (set): Output modules to enable for the scan. (note: if no output modules are specified, this is not populated until .bake())\n internal_modules (set): Internal modules for the scan. (note: not populated until .bake())\n exclude_modules (set): Modules to exclude from the scan. When set, automatically removes excluded modules.\n flags (set): Flags to enable for the scan. When set, automatically enables modules.\n require_flags (set): Require modules to have these flags. When set, automatically removes offending modules.\n exclude_flags (set): Exclude modules that have any of these flags. When set, automatically removes offending modules.\n module_dirs (set): Custom directories from which to load modules (alias to `self.module_loader.module_dirs`). When set, automatically preloads contained modules.\n config (omegaconf.dictconfig.DictConfig): BBOT config (alias to `core.config`)\n core (BBOTCore): Local copy of BBOTCore object.\n verbose (bool): Whether log level is currently set to verbose. When set, updates log level for all BBOT log handlers.\n debug (bool): Whether log level is currently set to debug. When set, updates log level for all BBOT log handlers.\n silent (bool): Whether logging is currently disabled. When set to True, silences all stderr.\n\n Examples:\n >>> preset = Preset(\n \"evilcorp.com\",\n \"1.2.3.0/24\",\n flags=[\"subdomain-enum\"],\n modules=[\"nuclei\"],\n config={\"web\": {\"http_proxy\": \"http://127.0.0.1\"}}\n )\n >>> scan = Scanner(preset=preset)\n\n >>> preset = Preset.from_yaml_file(\"my_preset.yml\")\n >>> scan = Scanner(preset=preset)\n \"\"\"\n\n def __init__(\n self,\n *targets,\n whitelist=None,\n blacklist=None,\n modules=None,\n output_modules=None,\n exclude_modules=None,\n flags=None,\n require_flags=None,\n exclude_flags=None,\n config=None,\n module_dirs=None,\n output_dir=None,\n name=None,\n description=None,\n scan_name=None,\n conditions=None,\n force_start=False,\n verbose=False,\n debug=False,\n silent=False,\n _exclude=None,\n _log=True,\n ):\n \"\"\"\n Initializes the Preset class.\n\n Args:\n *targets (str): Target(s) to scan. Types supported: hostnames, IPs, CIDRs, emails, open ports.\n whitelist (list, optional): Whitelisted target(s) to scan. Defaults to the same as `targets`.\n blacklist (list, optional): Blacklisted target(s). Takes ultimate precedence. Defaults to empty.\n modules (list[str], optional): List of scan modules to enable for the scan. Defaults to empty list.\n output_modules (list[str], optional): List of output modules to use. Defaults to csv, human, and json.\n exclude_modules (list[str], optional): List of modules to exclude from the scan.\n require_flags (list[str], optional): Only enable modules if they have these flags.\n exclude_flags (list[str], optional): Don't enable modules if they have any of these flags.\n module_dirs (list[str], optional): additional directories to load modules from.\n config (dict, optional): Additional scan configuration settings.\n include (list[str], optional): names or filenames of other presets to include.\n presets (list[str], optional): an alias for `include`.\n output_dir (str or Path, optional): Directory to store scan output. Defaults to BBOT home directory (`~/.bbot`).\n scan_name (str, optional): Human-readable name of the scan. If not specified, it will be random, e.g. \"demonic_jimmy\".\n name (str, optional): Human-readable name of the preset. Used mainly for logging.\n description (str, optional): Description of the preset.\n conditions (list[str], optional): Custom conditions to be executed before scan start. Written in Jinja2.\n force_start (bool, optional): If True, ignore conditional aborts and failed module setups. Just run the scan!\n verbose (bool, optional): Set the BBOT logger to verbose mode.\n debug (bool, optional): Set the BBOT logger to debug mode.\n silent (bool, optional): Silence all stderr (effectively disables the BBOT logger).\n _exclude (list[Path], optional): Preset filenames to exclude from inclusion. Used internally to prevent infinite recursion in circular or self-referencing presets.\n _log (bool, optional): Whether to enable logging for the preset. This will record which modules/flags are enabled, etc.\n \"\"\"\n # internal variables\n self._cli = False\n self._log = _log\n self.scan = None\n self._args = None\n self._environ = None\n self._helpers = None\n self._module_loader = None\n self._yaml_str = \"\"\n self._baked = False\n\n self._default_output_modules = None\n self._default_internal_modules = None\n\n # modules / flags\n self.modules = set()\n self.exclude_modules = set()\n self.flags = set()\n self.exclude_flags = set()\n self.require_flags = set()\n\n # modules + flags\n if modules is None:\n modules = []\n if isinstance(modules, str):\n modules = [modules]\n if output_modules is None:\n output_modules = []\n if isinstance(output_modules, str):\n output_modules = [output_modules]\n if exclude_modules is None:\n exclude_modules = []\n if isinstance(exclude_modules, str):\n exclude_modules = [exclude_modules]\n if flags is None:\n flags = []\n if isinstance(flags, str):\n flags = [flags]\n if exclude_flags is None:\n exclude_flags = []\n if isinstance(exclude_flags, str):\n exclude_flags = [exclude_flags]\n if require_flags is None:\n require_flags = []\n if isinstance(require_flags, str):\n require_flags = [require_flags]\n\n # these are used only for preserving the modules as specified in the original preset\n # this is to ensure the preset looks the same when reserialized\n self.explicit_scan_modules = set() if modules is None else set(modules)\n self.explicit_output_modules = set() if output_modules is None else set(output_modules)\n\n # whether to force-start the scan (ignoring conditional aborts and failed module setups)\n self.force_start = force_start\n\n # scan output directory\n self.output_dir = output_dir\n # name of scan\n self.scan_name = scan_name\n\n # name of preset, default blank\n self.name = name or \"\"\n # preset description, default blank\n self.description = description or \"\"\n\n # custom conditions, evaluated during .bake()\n self.conditions = []\n if conditions is not None:\n for condition in conditions:\n self.conditions.append((self.name, condition))\n\n # keeps track of loaded preset files to prevent infinite circular inclusions\n self._preset_files_loaded = set()\n if _exclude is not None:\n for _filename in _exclude:\n self._preset_files_loaded.add(Path(_filename).resolve())\n\n # bbot core config\n self.core = CORE.copy()\n if config is None:\n config = omegaconf.OmegaConf.create({})\n # merge custom configs if specified by the user\n self.core.merge_custom(config)\n\n # log verbosity\n # actual log verbosity isn't set until .bake()\n self.verbose = verbose\n self.debug = debug\n self.silent = silent\n\n # custom module directories\n self._module_dirs = set()\n self.module_dirs = module_dirs\n\n # target / whitelist / blacklist\n # these are temporary receptacles until they all get .baked() together\n self._seeds = set(targets if targets else [])\n self._whitelist = set(whitelist) if whitelist else whitelist\n self._blacklist = set(blacklist if blacklist else [])\n\n self._target = None\n\n # we don't fill self.modules yet (that happens in .bake())\n self.explicit_scan_modules.update(set(modules))\n self.explicit_output_modules.update(set(output_modules))\n self.exclude_modules.update(set(exclude_modules))\n self.flags.update(set(flags))\n self.exclude_flags.update(set(exclude_flags))\n self.require_flags.update(set(require_flags))\n\n @property\n def bbot_home(self):\n return Path(self.config.get(\"home\", \"~/.bbot\")).expanduser().resolve()\n\n @property\n def target(self):\n if self._target is None:\n raise ValueError(\"Cannot access target before preset is baked (use ._seeds instead)\")\n return self._target\n\n @property\n def seeds(self):\n if self._seeds is None:\n raise ValueError(\"Cannot access target before preset is baked (use ._seeds instead)\")\n return self.target.seeds\n\n @property\n def whitelist(self):\n if self._target is None:\n raise ValueError(\"Cannot access whitelist before preset is baked (use ._whitelist instead)\")\n return self.target.whitelist\n\n @property\n def blacklist(self):\n if self._target is None:\n raise ValueError(\"Cannot access blacklist before preset is baked (use ._blacklist instead)\")\n return self.target.blacklist\n\n @property\n def preset_dir(self):\n return (self.bbot_home / \"presets\").expanduser().resolve()\n\n @property\n def default_output_modules(self):\n if self._default_output_modules is not None:\n output_modules = self._default_output_modules\n else:\n output_modules = [\"python\", \"csv\", \"txt\", \"json\"]\n if self._cli:\n output_modules.append(\"stdout\")\n return output_modules\n\n @property\n def default_internal_modules(self):\n preloaded_internal = self.module_loader.preloaded(type=\"internal\")\n if self._default_internal_modules is not None:\n internal_modules = self._default_internal_modules\n else:\n internal_modules = list(preloaded_internal)\n return {k: preloaded_internal[k] for k in internal_modules}\n\n def merge(self, other):\n \"\"\"\n Merge another preset into this one.\n\n If there are any config conflicts, `other` will win over `self`.\n\n Args:\n other (Preset): The preset to merge into this one.\n\n Examples:\n >>> preset1 = Preset(modules=[\"portscan\"])\n >>> preset1.scan_modules\n ['portscan']\n >>> preset2 = Preset(modules=[\"sslcert\"])\n >>> preset2.scan_modules\n ['sslcert']\n >>> preset1.merge(preset2)\n >>> preset1.scan_modules\n ['portscan', 'sslcert']\n \"\"\"\n self.log_debug(f'Merging preset \"{other.name}\" into \"{self.name}\"')\n\n # config\n self.core.merge_custom(other.core.custom_config)\n self.module_loader.core = self.core\n # module dirs\n # modules + flags\n # establish requirements / exclusions first\n self.exclude_modules.update(other.exclude_modules)\n self.require_flags.update(other.require_flags)\n self.exclude_flags.update(other.exclude_flags)\n # then it's okay to start enabling modules\n self.explicit_scan_modules.update(other.explicit_scan_modules)\n self.explicit_output_modules.update(other.explicit_output_modules)\n self.flags.update(other.flags)\n\n # target / scope\n self._seeds.update(other._seeds)\n # leave whitelist as None until we encounter one\n if other._whitelist is not None:\n if self._whitelist is None:\n self._whitelist = set(other._whitelist)\n else:\n self._whitelist.update(other._whitelist)\n self._blacklist.update(other._blacklist)\n\n # module dirs\n self.module_dirs = self.module_dirs.union(other.module_dirs)\n\n # log verbosity\n if other.silent:\n self.silent = other.silent\n if other.verbose:\n self.verbose = other.verbose\n if other.debug:\n self.debug = other.debug\n # scan name\n if other.scan_name is not None:\n self.scan_name = other.scan_name\n if other.output_dir is not None:\n self.output_dir = other.output_dir\n # conditions\n if other.conditions:\n self.conditions.extend(other.conditions)\n # misc\n self.force_start = self.force_start | other.force_start\n self._cli = self._cli | other._cli\n # transfer args\n if other._args is not None:\n self._args = other._args\n\n def bake(self, scan=None):\n \"\"\"\n Return a \"baked\" copy of this preset, ready for use by a BBOT scan.\n\n Baking a preset finalizes it by populating `preset.modules` based on flags,\n performing final validations, and substituting environment variables in preloaded modules.\n It also evaluates custom `conditions` as specified in the preset.\n\n This function is automatically called in Scanner.__init__(). There is no need to call it manually.\n \"\"\"\n self.log_debug(\"Getting baked\")\n # create a copy of self\n baked_preset = copy(self)\n\n # copy core\n baked_preset.core = self.core.copy()\n\n if scan is not None:\n baked_preset.scan = scan\n # copy module loader\n baked_preset._module_loader = self.module_loader.copy()\n # prepare os environment\n os_environ = baked_preset.environ.prepare()\n # find and replace preloaded modules with os environ\n # this is different from the config variable substitution because it modifies\n # the preloaded modules, i.e. their ansible playbooks\n baked_preset.module_loader.find_and_replace(**os_environ)\n # update os environ\n os.environ.clear()\n os.environ.update(os_environ)\n\n # assign baked preset to our scan\n scan.preset = baked_preset\n\n # validate log level options\n baked_preset.apply_log_level(apply_core=scan is not None)\n\n # validate flags, config options\n baked_preset.validate()\n\n # now that our requirements / exclusions are validated, we can start enabling modules\n # enable scan modules\n for module in baked_preset.explicit_scan_modules:\n baked_preset.add_module(module, module_type=\"scan\")\n\n # enable output modules\n output_modules_to_enable = set(baked_preset.explicit_output_modules)\n default_output_modules = self.default_output_modules\n output_module_override = any(m in default_output_modules for m in output_modules_to_enable)\n # if none of the default output modules have been explicitly specified, enable them all\n if not output_module_override:\n output_modules_to_enable.update(self.default_output_modules)\n for module in output_modules_to_enable:\n baked_preset.add_module(module, module_type=\"output\", raise_error=False)\n\n # enable internal modules\n for internal_module, preloaded in self.default_internal_modules.items():\n is_enabled = baked_preset.config.get(internal_module, True)\n is_excluded = internal_module in baked_preset.exclude_modules\n if is_enabled and not is_excluded:\n baked_preset.add_module(internal_module, module_type=\"internal\", raise_error=False)\n\n # disable internal modules if requested\n for internal_module in baked_preset.internal_modules:\n if baked_preset.config.get(internal_module, True) is False:\n baked_preset.exclude_modules.add(internal_module)\n\n # enable modules by flag\n for flag in baked_preset.flags:\n for module, preloaded in baked_preset.module_loader.preloaded().items():\n module_flags = preloaded.get(\"flags\", [])\n module_type = preloaded.get(\"type\", \"scan\")\n if flag in module_flags:\n self.log_debug(f'Enabling module \"{module}\" because it has flag \"{flag}\"')\n baked_preset.add_module(module, module_type, raise_error=False)\n\n # ensure we have output modules\n if not baked_preset.output_modules:\n for output_module in self.default_output_modules:\n baked_preset.add_module(output_module, module_type=\"output\", raise_error=False)\n\n # create target object\n from bbot.scanner.target import BBOTTarget\n\n baked_preset._target = BBOTTarget(\n *list(self._seeds),\n whitelist=self._whitelist,\n blacklist=self._blacklist,\n strict_scope=self.strict_scope,\n )\n\n if scan is not None:\n # evaluate conditions\n if baked_preset.conditions:\n from .conditions import ConditionEvaluator\n\n evaluator = ConditionEvaluator(baked_preset)\n evaluator.evaluate()\n\n self._baked = True\n return baked_preset\n\n def parse_args(self):\n \"\"\"\n Parse CLI arguments, and merge them into this preset.\n\n Used in `cli.py`.\n \"\"\"\n self._cli = True\n self.merge(self.args.preset_from_args())\n\n @property\n def module_dirs(self):\n return self.module_loader.module_dirs\n\n @module_dirs.setter\n def module_dirs(self, module_dirs):\n if module_dirs:\n if isinstance(module_dirs, str):\n module_dirs = [module_dirs]\n for m in module_dirs:\n self.module_loader.add_module_dir(m)\n self._module_dirs.add(m)\n\n @property\n def scan_modules(self):\n return [m for m in self.modules if self.preloaded_module(m).get(\"type\", \"scan\") == \"scan\"]\n\n @property\n def output_modules(self):\n return [m for m in self.modules if self.preloaded_module(m).get(\"type\", \"scan\") == \"output\"]\n\n @property\n def internal_modules(self):\n return [m for m in self.modules if self.preloaded_module(m).get(\"type\", \"scan\") == \"internal\"]\n\n def add_module(self, module_name, module_type=\"scan\", raise_error=True):\n self.log_debug(f'Adding module \"{module_name}\" of type \"{module_type}\"')\n is_valid, reason, preloaded = self._is_valid_module(module_name, module_type, raise_error=raise_error)\n if not is_valid:\n self.log_debug(f'Unable to add {module_type} module \"{module_name}\": {reason}')\n return\n self.modules.add(module_name)\n for module_dep in preloaded.get(\"deps\", {}).get(\"modules\", []):\n if module_dep != module_name and module_dep not in self.modules:\n self.log_verbose(f'Adding module \"{module_dep}\" because {module_name} depends on it')\n self.add_module(module_dep, raise_error=False)\n\n def preloaded_module(self, module):\n return self.module_loader.preloaded()[module]\n\n @property\n def config(self):\n return self.core.config\n\n @property\n def web_config(self):\n return self.core.config.get(\"web\", {})\n\n @property\n def scope_config(self):\n return self.config.get(\"scope\", {})\n\n @property\n def strict_scope(self):\n return self.scope_config.get(\"strict\", False)\n\n def apply_log_level(self, apply_core=False):\n \"\"\"\n Apply the log level to the preset.\n\n Args:\n apply_core (bool, optional): If True, apply the log level to the core logger.\n \"\"\"\n # silent takes precedence\n if self.silent:\n self.verbose = False\n self.debug = False\n if apply_core:\n self.core.logger.log_level = \"CRITICAL\"\n for key in (\"verbose\", \"debug\"):\n with suppress(omegaconf.errors.ConfigKeyError):\n del self.core.custom_config[key]\n else:\n # then debug\n if self.debug:\n self.verbose = False\n if apply_core:\n self.core.logger.log_level = \"DEBUG\"\n with suppress(omegaconf.errors.ConfigKeyError):\n del self.core.custom_config[\"verbose\"]\n else:\n # finally verbose\n if self.verbose and apply_core:\n self.core.logger.log_level = \"VERBOSE\"\n\n @property\n def helpers(self):\n if self._helpers is None:\n from bbot.core.helpers.helper import ConfigAwareHelper\n\n self._helpers = ConfigAwareHelper(preset=self)\n return self._helpers\n\n @property\n def module_loader(self):\n self.environ\n if self._module_loader is None:\n from bbot.core.modules import MODULE_LOADER\n\n self._module_loader = MODULE_LOADER\n self._module_loader.ensure_config_files()\n\n return self._module_loader\n\n @property\n def environ(self):\n if self._environ is None:\n from .environ import BBOTEnviron\n\n self._environ = BBOTEnviron(self)\n return self._environ\n\n @property\n def args(self):\n if self._args is None:\n from .args import BBOTArgs\n\n self._args = BBOTArgs(self)\n return self._args\n\n def in_scope(self, host):\n return self.target.in_scope(host)\n\n def blacklisted(self, host):\n return self.target.blacklisted(host)\n\n def whitelisted(self, host):\n return self.target.whitelisted(host)\n\n @classmethod\n def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False):\n \"\"\"\n Create a preset from a Python dictionary object.\n\n Args:\n preset_dict (dict): Preset in dictionary form\n name (str, optional): Name of preset\n _exclude (list[Path], optional): Preset filenames to exclude from inclusion. Used internally to prevent infinite recursion in circular or self-referencing presets.\n _log (bool, optional): Whether to enable logging for the preset. This will record which modules/flags are enabled, etc.\n\n Returns:\n Preset: The loaded preset\n\n Examples:\n >>> preset = Preset.from_dict({\"target\": [\"evilcorp.com\"], \"modules\": [\"portscan\"]})\n \"\"\"\n new_preset = cls(\n *preset_dict.get(\"target\", []),\n whitelist=preset_dict.get(\"whitelist\"),\n blacklist=preset_dict.get(\"blacklist\"),\n modules=preset_dict.get(\"modules\"),\n output_modules=preset_dict.get(\"output_modules\"),\n exclude_modules=preset_dict.get(\"exclude_modules\"),\n flags=preset_dict.get(\"flags\"),\n require_flags=preset_dict.get(\"require_flags\"),\n exclude_flags=preset_dict.get(\"exclude_flags\"),\n verbose=preset_dict.get(\"verbose\", False),\n debug=preset_dict.get(\"debug\", False),\n silent=preset_dict.get(\"silent\", False),\n config=preset_dict.get(\"config\"),\n module_dirs=preset_dict.get(\"module_dirs\", []),\n include=list(preset_dict.get(\"include\", [])),\n scan_name=preset_dict.get(\"scan_name\"),\n output_dir=preset_dict.get(\"output_dir\"),\n name=preset_dict.get(\"name\", name),\n description=preset_dict.get(\"description\"),\n conditions=preset_dict.get(\"conditions\", []),\n _exclude=_exclude,\n _log=_log,\n )\n return new_preset\n\n def include_preset(self, filename):\n \"\"\"\n Load a preset from a yaml file and merge it into this one.\n\n If the full path is not specified, BBOT will look in all the usual places for it.\n\n The file extension is optional.\n\n Args:\n filename (Path): The preset YAML file to merge\n\n Examples:\n >>> preset.include_preset(\"/home/user/my_preset.yml\")\n \"\"\"\n self.log_debug(f'Including preset \"{filename}\"')\n preset_from_yaml = self.from_yaml_file(filename, _exclude=self._preset_files_loaded)\n if preset_from_yaml is not False:\n self.merge(preset_from_yaml)\n self._preset_files_loaded.add(preset_from_yaml.filename)\n\n @classmethod\n def from_yaml_file(cls, filename, _exclude=None, _log=False):\n \"\"\"\n Create a preset from a YAML file. If the full path is not specified, BBOT will look in all the usual places for it.\n\n The file extension is optional.\n\n Examples:\n >>> preset = Preset.from_yaml_file(\"/home/user/my_preset.yml\")\n \"\"\"\n filename = PRESET_PATH.find(filename)\n try:\n return _preset_cache[filename]\n except KeyError:\n if _exclude is None:\n _exclude = set()\n if _exclude is not None and filename in _exclude:\n log.debug(f\"Not loading {filename} because it was already loaded {_exclude}\")\n return False\n log.debug(f\"Loading {filename} because it's not in excluded list ({_exclude})\")\n _exclude = set(_exclude)\n _exclude.add(filename)\n try:\n yaml_str = open(filename).read()\n except FileNotFoundError:\n raise PresetNotFoundError(f'Could not find preset at \"{filename}\" - file does not exist')\n preset = cls.from_dict(\n omegaconf.OmegaConf.create(yaml_str), name=filename.stem, _exclude=_exclude, _log=_log\n )\n preset._yaml_str = yaml_str\n preset.filename = filename\n _preset_cache[filename] = preset\n return preset\n\n @classmethod\n def from_yaml_string(cls, yaml_preset):\n \"\"\"\n Create a preset from a YAML string.\n\n The file extension is optional.\n\n Examples:\n >>> yaml_string = '''\n >>> target:\n >>> - evilcorp.com\n >>> modules:\n >>> - portscan'''\n >>> preset = Preset.from_yaml_string(yaml_string)\n \"\"\"\n return cls.from_dict(omegaconf.OmegaConf.create(yaml_preset))\n\n def to_dict(self, include_target=False, full_config=False, redact_secrets=False):\n \"\"\"\n Convert this preset into a Python dictionary.\n\n Args:\n include_target (bool, optional): If True, include target, whitelist, and blacklist in the dictionary\n full_config (bool, optional): If True, include the entire config, not just what's changed from the defaults.\n\n Returns:\n dict: The preset in dictionary form\n\n Examples:\n >>> preset = Preset(flags=[\"subdomain-enum\"], modules=[\"portscan\"])\n >>> preset.to_dict()\n {\"flags\": [\"subdomain-enum\"], \"modules\": [\"portscan\"]}\n \"\"\"\n preset_dict = {}\n\n if self.description:\n preset_dict[\"description\"] = self.description\n\n # config\n if full_config:\n config = self.core.config\n else:\n config = self.core.custom_config\n config = omegaconf.OmegaConf.to_object(config)\n if redact_secrets:\n config = self.core.no_secrets_config(config)\n if config:\n preset_dict[\"config\"] = config\n\n # scope\n if include_target:\n target = sorted(self.target.seeds.inputs)\n whitelist = []\n if self.target.whitelist is not None:\n whitelist = sorted(self.target.whitelist.inputs)\n blacklist = sorted(self.target.blacklist.inputs)\n if target:\n preset_dict[\"target\"] = target\n if whitelist and whitelist != target:\n preset_dict[\"whitelist\"] = whitelist\n if blacklist:\n preset_dict[\"blacklist\"] = blacklist\n\n # flags + modules\n if self.require_flags:\n preset_dict[\"require_flags\"] = sorted(self.require_flags)\n if self.exclude_flags:\n preset_dict[\"exclude_flags\"] = sorted(self.exclude_flags)\n if self.exclude_modules:\n preset_dict[\"exclude_modules\"] = sorted(self.exclude_modules)\n if self.flags:\n preset_dict[\"flags\"] = sorted(self.flags)\n if self.explicit_scan_modules:\n preset_dict[\"modules\"] = sorted(self.explicit_scan_modules)\n if self.explicit_output_modules:\n preset_dict[\"output_modules\"] = sorted(self.explicit_output_modules)\n\n # log verbosity\n if self.verbose:\n preset_dict[\"verbose\"] = True\n if self.debug:\n preset_dict[\"debug\"] = True\n if self.silent:\n preset_dict[\"silent\"] = True\n\n # misc scan options\n if self.scan_name:\n preset_dict[\"scan_name\"] = self.scan_name\n if self.scan_name and self.output_dir is not None:\n preset_dict[\"output_dir\"] = self.output_dir\n\n # conditions\n if self.conditions:\n preset_dict[\"conditions\"] = [c[-1] for c in self.conditions]\n\n return preset_dict\n\n def to_yaml(self, include_target=False, full_config=False, sort_keys=False):\n \"\"\"\n Return the preset in the form of a YAML string.\n\n Args:\n include_target (bool, optional): If True, include target, whitelist, and blacklist in the dictionary\n full_config (bool, optional): If True, include the entire config, not just what's changed from the defaults.\n sort_keys (bool, optional): If True, sort YAML keys alphabetically\n\n Returns:\n str: The preset in the form of a YAML string\n\n Examples:\n >>> preset = Preset(flags=[\"subdomain-enum\"], modules=[\"portscan\"])\n >>> print(preset.to_yaml())\n flags:\n - subdomain-enum\n modules:\n - portscan\n \"\"\"\n preset_dict = self.to_dict(include_target=include_target, full_config=full_config)\n return yaml.dump(preset_dict, sort_keys=sort_keys)\n\n def _is_valid_module(self, module, module_type, name_only=False, raise_error=True):\n if module_type == \"scan\":\n module_choices = self.module_loader.scan_module_choices\n elif module_type == \"output\":\n module_choices = self.module_loader.output_module_choices\n elif module_type == \"internal\":\n module_choices = self.module_loader.internal_module_choices\n else:\n raise ValidationError(f'Unknown module type \"{module}\"')\n\n if module not in module_choices:\n raise ValidationError(get_closest_match(module, module_choices, msg=f\"{module_type} module\"))\n\n try:\n preloaded = self.module_loader.preloaded()[module]\n except KeyError:\n raise ValidationError(f'Unknown module \"{module}\"')\n\n if name_only:\n return True, \"\", preloaded\n\n if module in self.exclude_modules:\n reason = \"the module has been excluded\"\n return False, reason, {}\n\n module_flags = preloaded.get(\"flags\", [])\n _module_type = preloaded.get(\"type\", \"scan\")\n if module_type:\n if _module_type != module_type:\n reason = f'its type ({_module_type}) is not \"{module_type}\"'\n if raise_error:\n raise ValidationError(f'Unable to add {module_type} module \"{module}\" because {reason}')\n return False, reason, preloaded\n\n if _module_type == \"scan\":\n if self.exclude_flags:\n for f in module_flags:\n if f in self.exclude_flags:\n return False, f'it has excluded flag, \"{f}\"', preloaded\n if self.require_flags and not all(f in module_flags for f in self.require_flags):\n return False, f\"it doesn't have the required flags ({','.join(self.require_flags)})\", preloaded\n\n return True, \"\", preloaded\n\n def validate(self):\n \"\"\"\n Validate module/flag exclusions/requirements, and CLI config options if applicable.\n \"\"\"\n if self._cli:\n self.args.validate()\n\n # validate excluded modules\n for excluded_module in self.exclude_modules:\n if excluded_module not in self.module_loader.all_module_choices:\n raise ValidationError(\n get_closest_match(excluded_module, self.module_loader.all_module_choices, msg=\"module\")\n )\n # validate excluded flags\n for excluded_flag in self.exclude_flags:\n if excluded_flag not in self.module_loader.flag_choices:\n raise ValidationError(get_closest_match(excluded_flag, self.module_loader.flag_choices, msg=\"flag\"))\n # validate required flags\n for required_flag in self.require_flags:\n if required_flag not in self.module_loader.flag_choices:\n raise ValidationError(get_closest_match(required_flag, self.module_loader.flag_choices, msg=\"flag\"))\n # validate flags\n for flag in self.flags:\n if flag not in self.module_loader.flag_choices:\n raise ValidationError(get_closest_match(flag, self.module_loader.flag_choices, msg=\"flag\"))\n\n @property\n def all_presets(self):\n \"\"\"\n Recursively find all the presets and return them as a dictionary\n \"\"\"\n # first, add local preset dir to PRESET_PATH\n PRESET_PATH.add_path(self.preset_dir)\n\n # ensure local preset directory exists\n mkdir(self.preset_dir)\n\n global DEFAULT_PRESETS\n if DEFAULT_PRESETS is None:\n presets = {}\n for preset_path in PRESET_PATH:\n for ext in (\"yml\", \"yaml\"):\n # for every yaml file\n for original_filename in preset_path.rglob(f\"**/*.{ext}\"):\n # not including symlinks\n if original_filename.is_symlink():\n continue\n\n # try to load it as a preset\n try:\n loaded_preset = self.from_yaml_file(original_filename, _log=True)\n if loaded_preset is False:\n continue\n except Exception as e:\n log.warning(f'Failed to load preset at \"{original_filename}\": {e}')\n log.trace(traceback.format_exc())\n continue\n\n # category is the parent folder(s), if any\n category = str(original_filename.relative_to(preset_path).parent)\n if category == \".\":\n category = \"\"\n\n local_preset = original_filename\n # populate symlinks in local preset dir\n if not original_filename.is_relative_to(self.preset_dir):\n relative_preset = original_filename.relative_to(preset_path)\n local_preset = self.preset_dir / relative_preset\n mkdir(local_preset.parent, check_writable=False)\n if not local_preset.exists():\n local_preset.symlink_to(original_filename)\n\n presets[local_preset.stem] = (loaded_preset, category, preset_path, original_filename)\n\n # sort by name\n DEFAULT_PRESETS = dict(sorted(presets.items(), key=lambda x: x[-1][0].name))\n return DEFAULT_PRESETS\n\n def presets_table(self, include_modules=True):\n \"\"\"\n Return a table of all the presets in the form of a string\n \"\"\"\n table = []\n header = [\"Preset\", \"Category\", \"Description\", \"# Modules\"]\n if include_modules:\n header.append(\"Modules\")\n for loaded_preset, category, preset_path, original_file in self.all_presets.values():\n loaded_preset = loaded_preset.bake()\n num_modules = f\"{len(loaded_preset.scan_modules):,}\"\n row = [loaded_preset.name, category, loaded_preset.description, num_modules]\n if include_modules:\n row.append(\", \".join(sorted(loaded_preset.scan_modules)))\n table.append(row)\n return make_table(table, header)\n\n def log_verbose(self, msg):\n if self._log:\n log.verbose(f\"Preset {self.name}: {msg}\")\n\n def log_debug(self, msg):\n if self._log:\n log.debug(f\"Preset {self.name}: {msg}\")\n" }, { "path": "bbot/scanner/scanner.py", "content": "import sys\nimport asyncio\nimport logging\nimport traceback\nimport contextlib\nimport regex as re\nfrom pathlib import Path\nfrom sys import exc_info\nfrom datetime import datetime\nfrom collections import OrderedDict\n\nfrom bbot import __version__\nfrom bbot.core.event import make_event, update_event\nfrom .manager import ScanIngress, ScanEgress\nfrom bbot.core.helpers.misc import sha1, rand_string\nfrom bbot.core.helpers.names_generator import random_name\nfrom bbot.core.config.logger import GzipRotatingFileHandler\nfrom bbot.core.multiprocess import SHARED_INTERPRETER_STATE\nfrom bbot.core.helpers.async_helpers import async_to_sync_gen\nfrom bbot.errors import BBOTError, ScanError, ValidationError\n\nlog = logging.getLogger(\"bbot.scanner\")\n\n\nclass Scanner:\n \"\"\"A class representing a single BBOT scan\n\n Examples:\n Create scan with multiple targets:\n >>> my_scan = Scanner(\"evilcorp.com\", \"1.2.3.0/24\", modules=[\"portscan\", \"sslcert\", \"httpx\"])\n\n Create scan with custom config:\n >>> config = {\"http_proxy\": \"http://127.0.0.1:8080\", \"modules\": {\"portscan\": {\"top_ports\": 2000}}}\n >>> my_scan = Scanner(\"www.evilcorp.com\", modules=[\"portscan\", \"httpx\"], config=config)\n\n Start the scan, iterating over events as they're discovered (synchronous):\n >>> for event in my_scan.start():\n >>> print(event)\n\n Start the scan, iterating over events as they're discovered (asynchronous):\n >>> async for event in my_scan.async_start():\n >>> print(event)\n\n Start the scan without consuming events (synchronous):\n >>> my_scan.start_without_generator()\n\n Start the scan without consuming events (asynchronous):\n >>> await my_scan.async_start_without_generator()\n\n Attributes:\n status (str): Status of scan, representing its current state. It can take on the following string values, each of which is mapped to an integer code in `_status_codes`:\n ```markdown\n - \"NOT_STARTED\" (0): Initial status before the scan starts.\n - \"STARTING\" (1): Status when the scan is initializing.\n - \"RUNNING\" (2): Status when the scan is in progress.\n - \"FINISHING\" (3): Status when the scan is in the process of finalizing.\n - \"CLEANING_UP\" (4): Status when the scan is cleaning up resources.\n - \"ABORTING\" (5): Status when the scan is in the process of being aborted.\n - \"ABORTED\" (6): Status when the scan has been aborted.\n - \"FAILED\" (7): Status when the scan has encountered a failure.\n - \"FINISHED\" (8): Status when the scan has successfully completed.\n ```\n _status_code (int): The numerical representation of the current scan status, stored for internal use. It is mapped according to the values in `_status_codes`.\n target (Target): Target of scan (alias to `self.preset.target`).\n preset (Preset): The main scan Preset in its baked form.\n config (omegaconf.dictconfig.DictConfig): BBOT config (alias to `self.preset.config`).\n whitelist (Target): Scan whitelist (by default this is the same as `target`) (alias to `self.preset.whitelist`).\n blacklist (Target): Scan blacklist (this takes ultimate precedence) (alias to `self.preset.blacklist`).\n helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc. (alias to `self.preset.helpers`).\n output_dir (pathlib.Path): Output directory for scan (alias to `self.preset.output_dir`).\n name (str): Name of scan (alias to `self.preset.scan_name`).\n dispatcher (Dispatcher): Triggers certain events when the scan `status` changes.\n modules (dict): Holds all loaded modules in this format: `{\"module_name\": Module()}`.\n stats (ScanStats): Holds high-level scan statistics such as how many events have been produced and consumed by each module.\n home (pathlib.Path): Base output directory of the scan (default: `~/.bbot/scans/`).\n running (bool): Whether the scan is currently running.\n stopping (bool): Whether the scan is currently stopping.\n stopped (bool): Whether the scan is currently stopped.\n aborting (bool): Whether the scan is aborted or currently aborting.\n\n Notes:\n - The status is read-only once set to \"ABORTING\" until it transitions to \"ABORTED.\"\n - Invalid statuses are logged but not applied.\n - Setting a status will trigger the `on_status` event in the dispatcher.\n \"\"\"\n\n _status_codes = {\n \"NOT_STARTED\": 0,\n \"STARTING\": 1,\n \"RUNNING\": 2,\n \"FINISHING\": 3,\n \"CLEANING_UP\": 4,\n \"ABORTING\": 5,\n \"ABORTED\": 6,\n \"FAILED\": 7,\n \"FINISHED\": 8,\n }\n\n def __init__(\n self,\n *targets,\n name=None,\n scan_id=None,\n dispatcher=None,\n **kwargs,\n ):\n \"\"\"\n Initializes the Scanner class.\n\n If a premade `preset` is specified, it will be used for the scan.\n Otherwise, `Scan` accepts the same arguments as `Preset`, which are passed through and used to create a new preset.\n\n Args:\n *targets (list[str], optional): Scan targets (passed through to `Preset`).\n preset (Preset, optional): Preset to use for the scan.\n scan_id (str, optional): Unique identifier for the scan. Auto-generates if None.\n dispatcher (Dispatcher, optional): Dispatcher object to use. Defaults to new Dispatcher.\n **kwargs (list[str], optional): Additional keyword arguments (passed through to `Preset`).\n \"\"\"\n self._root_event = None\n self._finish_event = None\n self.start_time = None\n self.end_time = None\n self.duration = None\n self.duration_human = None\n self.duration_seconds = None\n\n self._success = False\n self._scan_finish_status_message = None\n\n if scan_id is not None:\n self.id = str(scan_id)\n else:\n self.id = f\"SCAN:{sha1(rand_string(20)).hexdigest()}\"\n\n custom_preset = kwargs.pop(\"preset\", None)\n kwargs[\"_log\"] = True\n\n from .preset import Preset\n\n if name is not None:\n kwargs[\"scan_name\"] = name\n\n base_preset = Preset(*targets, **kwargs)\n\n if custom_preset is not None:\n if not isinstance(custom_preset, Preset):\n raise ValidationError(f'Preset must be of type Preset, not \"{type(custom_preset).__name__}\"')\n base_preset.merge(custom_preset)\n\n self.preset = base_preset.bake(self)\n\n # scan name\n if self.preset.scan_name is None:\n tries = 0\n while 1:\n if tries > 5:\n scan_name = f\"{rand_string(4)}_{rand_string(4)}\"\n break\n scan_name = random_name()\n if self.preset.output_dir is not None:\n home_path = Path(self.preset.output_dir).resolve() / scan_name\n else:\n home_path = self.preset.bbot_home / \"scans\" / scan_name\n if not home_path.exists():\n break\n tries += 1\n else:\n scan_name = str(self.preset.scan_name)\n self.name = scan_name.replace(\"/\", \"_\")\n\n # make sure the preset has a description\n if not self.preset.description:\n self.preset.description = self.name\n\n # scan output dir\n if self.preset.output_dir is not None:\n self.home = Path(self.preset.output_dir).resolve() / self.name\n else:\n self.home = self.preset.bbot_home / \"scans\" / self.name\n\n # scan temp dir\n self.temp_dir = self.home / \"temp\"\n self.helpers.mkdir(self.temp_dir)\n\n self._status = \"NOT_STARTED\"\n self._status_code = 0\n\n self.modules = OrderedDict({})\n self._modules_loaded = False\n self.dummy_modules = {}\n\n if dispatcher is None:\n from .dispatcher import Dispatcher\n\n self.dispatcher = Dispatcher()\n else:\n self.dispatcher = dispatcher\n self.dispatcher.set_scan(self)\n\n # scope distance\n self.scope_config = self.config.get(\"scope\", {})\n self.scope_search_distance = max(0, int(self.scope_config.get(\"search_distance\", 0)))\n self.scope_report_distance = int(self.scope_config.get(\"report_distance\", 1))\n\n # web config\n self.web_config = self.config.get(\"web\", {})\n self.web_spider_distance = self.web_config.get(\"spider_distance\", 0)\n self.web_spider_depth = self.web_config.get(\"spider_depth\", 1)\n self.web_spider_links_per_page = self.web_config.get(\"spider_links_per_page\", 20)\n max_redirects = self.web_config.get(\"http_max_redirects\", 5)\n self.web_max_redirects = max(max_redirects, self.web_spider_distance)\n self.http_proxy = self.web_config.get(\"http_proxy\", \"\")\n self.http_timeout = self.web_config.get(\"http_timeout\", 10)\n self.httpx_timeout = self.web_config.get(\"httpx_timeout\", 5)\n self.http_retries = self.web_config.get(\"http_retries\", 1)\n self.httpx_retries = self.web_config.get(\"httpx_retries\", 1)\n self.useragent = self.web_config.get(\"user_agent\", \"BBOT\")\n # custom HTTP headers warning\n self.custom_http_headers = self.web_config.get(\"http_headers\", {})\n if self.custom_http_headers:\n self.warning(\n \"You have enabled custom HTTP headers. These will be attached to all in-scope requests and all requests made by httpx.\"\n )\n # custom HTTP cookies warning\n self.custom_http_cookies = self.web_config.get(\"http_cookies\", {})\n if self.custom_http_cookies:\n self.warning(\n \"You have enabled custom HTTP cookies. These will be attached to all in-scope requests and all requests made by httpx.\"\n )\n\n # url file extensions\n self.url_extension_special = {e.lower() for e in self.config.get(\"url_extension_special\", [])}\n self.url_extension_blacklist = {e.lower() for e in self.config.get(\"url_extension_blacklist\", [])}\n\n # url querystring behavior\n self.url_querystring_remove = self.config.get(\"url_querystring_remove\", True)\n\n # blob inclusion\n self._file_blobs = self.config.get(\"file_blobs\", False)\n self._folder_blobs = self.config.get(\"folder_blobs\", False)\n\n # how often to print scan status\n self.status_frequency = self.config.get(\"status_frequency\", 15)\n\n from .stats import ScanStats\n\n self.stats = ScanStats(self)\n\n self._prepped = False\n self._finished_init = False\n self._new_activity = False\n self._cleanedup = False\n self._omitted_event_types = None\n\n self.init_events_task = None\n self.ticker_task = None\n self.dispatcher_tasks = []\n\n self._stopping = False\n\n self._dns_strings = None\n self._dns_regexes = None\n self._dns_regexes_yara = None\n self._dns_yara_rules_uncompiled = None\n self._dns_yara_rules = None\n\n self.__log_handlers = None\n self._log_handler_backup = []\n\n async def _prep(self):\n \"\"\"\n Creates the scan's output folder, loads its modules, and calls their .setup() methods.\n \"\"\"\n\n # update the master PID\n SHARED_INTERPRETER_STATE.update_scan_pid()\n\n self.helpers.mkdir(self.home)\n if not self._prepped:\n # save scan preset\n with open(self.home / \"preset.yml\", \"w\") as f:\n f.write(self.preset.to_yaml())\n\n # log scan overview\n start_msg = f\"Scan seeded with {len(self.seeds):,} targets\"\n details = []\n if self.whitelist != self.target:\n details.append(f\"{len(self.whitelist):,} in whitelist\")\n if self.blacklist:\n details.append(f\"{len(self.blacklist):,} in blacklist\")\n if details:\n start_msg += f\" ({', '.join(details)})\"\n self.hugeinfo(start_msg)\n\n # load scan modules (this imports and instantiates them)\n # up to this point they were only preloaded\n await self.load_modules()\n\n # run each module's .setup() method\n succeeded, hard_failed, soft_failed = await self.setup_modules()\n\n # intercept modules get sewn together like human centipede\n self.intercept_modules = [m for m in self.modules.values() if m._intercept]\n self.intercept_modules.sort(key=lambda x: x.priority)\n for i, intercept_module in enumerate(self.intercept_modules[1:]):\n prev_intercept_module = self.intercept_modules[i]\n self.debug(\n f\"Setting intercept module {intercept_module.name}._incoming_event_queue to previous intercept module {prev_intercept_module.name}.outgoing_event_queue\"\n )\n interqueue = asyncio.Queue()\n intercept_module._incoming_event_queue = interqueue\n prev_intercept_module._outgoing_event_queue = interqueue\n\n # abort if there are no output modules\n num_output_modules = len([m for m in self.modules.values() if m._type == \"output\"])\n if num_output_modules < 1:\n raise ScanError(\"Failed to load output modules. Aborting.\")\n # abort if any of the module .setup()s hard-failed (i.e. they errored or returned False)\n total_failed = len(hard_failed + soft_failed)\n if hard_failed:\n msg = f\"Setup hard-failed for {len(hard_failed):,} modules ({','.join(hard_failed)})\"\n self._fail_setup(msg)\n\n total_modules = total_failed + len(self.modules)\n success_msg = f\"Setup succeeded for {len(self.modules):,}/{total_modules:,} modules.\"\n\n self.success(success_msg)\n self._prepped = True\n\n def start(self):\n for event in async_to_sync_gen(self.async_start()):\n yield event\n\n def start_without_generator(self):\n for event in async_to_sync_gen(self.async_start()):\n pass\n\n async def async_start_without_generator(self):\n async for event in self.async_start():\n pass\n\n async def async_start(self):\n \"\"\" \"\"\"\n self.start_time = datetime.now()\n self.root_event.data[\"started_at\"] = self.start_time.isoformat()\n try:\n await self._prep()\n\n self._start_log_handlers()\n self.trace(f\"Ran BBOT {__version__} at {self.start_time}, command: {' '.join(sys.argv)}\")\n self.trace(f\"Target: {self.preset.target.json}\")\n self.trace(f\"Preset: {self.preset.to_dict(redact_secrets=True)}\")\n\n if not self.target:\n self.warning(\"No scan targets specified\")\n\n # start status ticker\n self.ticker_task = asyncio.create_task(\n self._status_ticker(self.status_frequency), name=f\"{self.name}._status_ticker()\"\n )\n\n self.status = \"STARTING\"\n\n if not self.modules:\n self.error(\"No modules loaded\")\n self.status = \"FAILED\"\n return\n else:\n self.hugesuccess(f\"Starting scan {self.name}\")\n\n await self.dispatcher.on_start(self)\n\n self.status = \"RUNNING\"\n self._start_modules()\n self.verbose(f\"{len(self.modules):,} modules started\")\n\n # distribute seed events\n self.init_events_task = asyncio.create_task(\n self.ingress_module.init_events(self.target.seeds.event_seeds),\n name=f\"{self.name}.ingress_module.init_events()\",\n )\n\n # main scan loop\n while 1:\n # abort if we're aborting\n if self.aborting:\n self._drain_queues()\n break\n\n # yield events as they come (async for event in scan.async_start())\n if \"python\" in self.modules:\n events, finish = await self.modules[\"python\"]._events_waiting(batch_size=-1)\n for e in events:\n yield e\n if events:\n continue\n\n # break if initialization finished and the scan is no longer active\n if self._finished_init and self.modules_finished:\n new_activity = await self.finish()\n if not new_activity:\n self._success = True\n scan_finish_event = await self._mark_finished()\n yield scan_finish_event\n break\n\n await asyncio.sleep(0.1)\n\n self._success = True\n\n except BaseException as e:\n if self.helpers.in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):\n self.stop()\n self._success = True\n else:\n try:\n raise\n except ScanError as e:\n self.error(f\"{e}\")\n\n except BBOTError as e:\n self.critical(f\"Error during scan: {e}\")\n\n except Exception:\n self.critical(f\"Unexpected error during scan:\\n{traceback.format_exc()}\")\n\n finally:\n tasks = self._cancel_tasks()\n self.debug(f\"Awaiting {len(tasks):,} tasks\")\n for task in tasks:\n # self.debug(f\"Awaiting {task}\")\n with contextlib.suppress(BaseException):\n await asyncio.wait_for(task, timeout=0.1)\n self.debug(f\"Awaited {len(tasks):,} tasks\")\n await self._report()\n await self._cleanup()\n\n await self.dispatcher.on_finish(self)\n\n self._stop_log_handlers()\n\n if self._scan_finish_status_message:\n log_fn = self.hugesuccess\n if self.status.startswith(\"ABORT\"):\n log_fn = self.hugewarning\n elif not self._success:\n log_fn = self.critical\n log_fn(self._scan_finish_status_message)\n\n async def _mark_finished(self):\n if self.status == \"ABORTING\":\n status = \"ABORTED\"\n elif not self._success:\n status = \"FAILED\"\n else:\n status = \"FINISHED\"\n\n self.end_time = datetime.now()\n self.duration = self.end_time - self.start_time\n self.duration_seconds = self.duration.total_seconds()\n self.duration_human = self.helpers.human_timedelta(self.duration)\n\n self._scan_finish_status_message = f\"Scan {self.name} completed in {self.duration_human} with status {status}\"\n\n scan_finish_event = self.finish_event(self._scan_finish_status_message, status)\n\n # queue final scan event with output modules\n output_modules = [m for m in self.modules.values() if m._type == \"output\" and m.name != \"python\"]\n for m in output_modules:\n await m.queue_event(scan_finish_event)\n # wait until output modules are flushed\n while 1:\n modules_finished = all(m.finished for m in output_modules)\n if modules_finished:\n break\n await asyncio.sleep(0.05)\n\n self.status = status\n return scan_finish_event\n\n def _start_modules(self):\n self.verbose(\"Starting module worker loops\")\n for module in self.modules.values():\n module.start()\n\n async def setup_modules(self, remove_failed=True, deps_only=False):\n \"\"\"Asynchronously initializes all loaded modules by invoking their `setup()` methods.\n\n Args:\n remove_failed (bool): Flag indicating whether to remove modules that fail setup.\n\n Returns:\n tuple:\n succeeded - List of modules that successfully set up.\n hard_failed - List of modules that encountered a hard failure during setup.\n soft_failed - List of modules that encountered a soft failure during setup.\n\n Raises:\n ScanError: If no output modules could be loaded.\n\n Notes:\n Hard-failed modules are set to an error state and removed if `remove_failed` is True.\n Soft-failed modules are not set to an error state but are also removed if `remove_failed` is True.\n \"\"\"\n await self.load_modules()\n self.verbose(\"Setting up modules\")\n succeeded = []\n hard_failed = []\n soft_failed = []\n\n async for task in self.helpers.as_completed([m._setup(deps_only=deps_only) for m in self.modules.values()]):\n module, status, msg = await task\n if status is True:\n self.debug(f\"Setup succeeded for {module.name} ({msg})\")\n succeeded.append(module.name)\n elif status is False:\n self.warning(f\"Setup hard-failed for {module.name}: {msg}\")\n self.modules[module.name].set_error_state()\n hard_failed.append(module.name)\n else:\n self.info(f\"Setup soft-failed for {module.name}: {msg}\")\n soft_failed.append(module.name)\n if (not status) and (module._intercept or remove_failed):\n # if a intercept module fails setup, we always remove it\n self.modules.pop(module.name)\n\n return succeeded, hard_failed, soft_failed\n\n async def load_modules(self):\n \"\"\"Asynchronously import and instantiate all scan modules, including internal and output modules.\n\n This method is automatically invoked by `setup_modules()`. It performs several key tasks in the following sequence:\n\n 1. Install dependencies for each module via `self.helpers.depsinstaller.install()`.\n 2. Load scan modules and updates the `modules` dictionary.\n 3. Load internal modules and updates the `modules` dictionary.\n 4. Load output modules and updates the `modules` dictionary.\n 5. Sorts modules based on their `_priority` attribute.\n\n If any modules fail to load or their dependencies fail to install, a ScanError will be raised (unless `self.force_start` is True).\n\n Attributes:\n succeeded, failed (tuple): A tuple containing lists of modules that succeeded or failed during the dependency installation.\n loaded_modules, loaded_internal_modules, loaded_output_modules (dict): Dictionaries of successfully loaded modules.\n failed, failed_internal, failed_output (list): Lists of module names that failed to load.\n\n Raises:\n ScanError: If any module dependencies fail to install or modules fail to load, and if `self.force_start` is False.\n\n Returns:\n None\n\n Note:\n After all modules are loaded, they are sorted by `_priority` and stored in the `modules` dictionary.\n \"\"\"\n if not self._modules_loaded:\n if not self.preset.modules:\n self.warning(\"No modules to load\")\n return\n\n if not self.preset.scan_modules:\n self.warning(\"No scan modules to load\")\n\n # install module dependencies\n succeeded, failed = await self.helpers.depsinstaller.install(*self.preset.modules)\n if failed:\n msg = f\"Failed to install dependencies for {len(failed):,} modules: {','.join(failed)}\"\n self._fail_setup(msg)\n modules = sorted([m for m in self.preset.scan_modules if m in succeeded])\n output_modules = sorted([m for m in self.preset.output_modules if m in succeeded])\n internal_modules = sorted([m for m in self.preset.internal_modules if m in succeeded])\n\n # Load scan modules\n self.verbose(f\"Loading {len(modules):,} scan modules: {','.join(modules)}\")\n loaded_modules, failed = self._load_modules(modules)\n self.modules.update(loaded_modules)\n if len(failed) > 0:\n msg = f\"Failed to load {len(failed):,} scan modules: {','.join(failed)}\"\n self._fail_setup(msg)\n if loaded_modules:\n self.info(\n f\"Loaded {len(loaded_modules):,}/{len(self.preset.scan_modules):,} scan modules ({','.join(loaded_modules)})\"\n )\n\n # Load internal modules\n self.verbose(f\"Loading {len(internal_modules):,} internal modules: {','.join(internal_modules)}\")\n loaded_internal_modules, failed_internal = self._load_modules(internal_modules)\n self.modules.update(loaded_internal_modules)\n if len(failed_internal) > 0:\n msg = f\"Failed to load {len(loaded_internal_modules):,} internal modules: {','.join(loaded_internal_modules)}\"\n self._fail_setup(msg)\n if loaded_internal_modules:\n self.info(\n f\"Loaded {len(loaded_internal_modules):,}/{len(self.preset.internal_modules):,} internal modules ({','.join(loaded_internal_modules)})\"\n )\n\n # Load output modules\n self.verbose(f\"Loading {len(output_modules):,} output modules: {','.join(output_modules)}\")\n loaded_output_modules, failed_output = self._load_modules(output_modules)\n self.modules.update(loaded_output_modules)\n if len(failed_output) > 0:\n msg = f\"Failed to load {len(failed_output):,} output modules: {','.join(failed_output)}\"\n self._fail_setup(msg)\n if loaded_output_modules:\n self.info(\n f\"Loaded {len(loaded_output_modules):,}/{len(self.preset.output_modules):,} output modules, ({','.join(loaded_output_modules)})\"\n )\n\n # builtin intercept modules\n self.ingress_module = ScanIngress(self)\n self.egress_module = ScanEgress(self)\n self.modules[self.ingress_module.name] = self.ingress_module\n self.modules[self.egress_module.name] = self.egress_module\n\n # sort modules by priority\n self.modules = OrderedDict(sorted(self.modules.items(), key=lambda x: getattr(x[-1], \"priority\", 3)))\n\n self._modules_loaded = True\n\n @property\n def modules_finished(self):\n finished_modules = [m.finished for m in self.modules.values()]\n return all(finished_modules)\n\n def kill_module(self, module_name, message=None):\n from signal import SIGINT\n\n module = self.modules[module_name]\n if module._intercept:\n self.warning(f'Cannot kill module \"{module_name}\" because it is critical to the scan')\n return\n module.set_error_state(message=message, clear_outgoing_queue=True)\n for proc in module._proc_tracker:\n with contextlib.suppress(Exception):\n proc.send_signal(SIGINT)\n self.helpers.cancel_tasks_sync(module._tasks)\n\n @property\n def incoming_event_queues(self):\n return self.ingress_module.incoming_queues\n\n @property\n def num_queued_events(self):\n total = 0\n for q in self.incoming_event_queues:\n total += len(q._queue)\n return total\n\n def modules_status(self, _log=False):\n finished = True\n status = {\"modules\": {}}\n\n sorted_modules = []\n for module_name, module in self.modules.items():\n if module_name.startswith(\"_\"):\n continue\n sorted_modules.append(module)\n mod_status = module.status\n if mod_status[\"running\"]:\n finished = False\n status[\"modules\"][module_name] = mod_status\n\n # sort modules by name\n sorted_modules.sort(key=lambda m: m.name)\n\n status[\"finished\"] = finished\n\n modules_errored = [m for m, s in status[\"modules\"].items() if s[\"errored\"]]\n\n max_mem_percent = 90\n mem_status = self.helpers.memory_status()\n # abort if we don't have the memory\n mem_percent = mem_status.percent\n if mem_percent > max_mem_percent:\n free_memory = mem_status.available\n free_memory_human = self.helpers.bytes_to_human(free_memory)\n self.warning(f\"System memory is at {mem_percent:.1f}% ({free_memory_human} remaining)\")\n\n if _log:\n modules_status = []\n for m, s in status[\"modules\"].items():\n running = s[\"running\"]\n incoming = s[\"events\"][\"incoming\"]\n outgoing = s[\"events\"][\"outgoing\"]\n tasks = s[\"tasks\"]\n total = sum([incoming, outgoing, tasks])\n if running or total > 0:\n modules_status.append((m, running, incoming, outgoing, tasks, total))\n modules_status.sort(key=lambda x: x[-1], reverse=True)\n\n if modules_status:\n modules_status_str = \", \".join([f\"{m}({i:,}:{t:,}:{o:,})\" for m, r, i, o, t, _ in modules_status])\n self.info(f\"{self.name}: Modules running (incoming:processing:outgoing) {modules_status_str}\")\n else:\n self.info(f\"{self.name}: No modules running\")\n event_type_summary = sorted(self.stats.events_emitted_by_type.items(), key=lambda x: x[-1], reverse=True)\n if event_type_summary:\n self.info(\n f\"{self.name}: Events produced so far: {', '.join([f'{k}: {v}' for k, v in event_type_summary])}\"\n )\n else:\n self.info(f\"{self.name}: No events produced yet\")\n\n if modules_errored:\n self.verbose(\n f\"{self.name}: Modules errored: {len(modules_errored):,} ({', '.join(list(modules_errored))})\"\n )\n\n num_queued_events = self.num_queued_events\n if num_queued_events:\n self.info(\n f\"{self.name}: {num_queued_events:,} events in queue ({self.stats.speedometer.speed:,} processed in the past {self.status_frequency} seconds)\"\n )\n else:\n self.info(\n f\"{self.name}: No events in queue ({self.stats.speedometer.speed:,} processed in the past {self.status_frequency} seconds)\"\n )\n\n if self.log_level <= logging.DEBUG:\n # status debugging\n scan_active_status = []\n scan_active_status.append(f\"scan._finished_init: {self._finished_init}\")\n scan_active_status.append(f\"scan.modules_finished: {self.modules_finished}\")\n for m in sorted_modules:\n running = m.running\n scan_active_status.append(f\" {m}:\")\n # scan_active_status.append(f\" running: {running}\")\n if running:\n # scan_active_status.append(f\" tasks:\")\n for task in list(m._task_counter.tasks.values()):\n scan_active_status.append(f\" - {task}:\")\n # scan_active_status.append(f\" incoming_queue_size: {m.num_incoming_events}\")\n # scan_active_status.append(f\" outgoing_queue_size: {m.outgoing_event_queue.qsize()}\")\n\n for line in scan_active_status:\n self.debug(line)\n\n # log module memory usage\n module_memory_usage = []\n for module in sorted_modules:\n memory_usage = module.memory_usage\n module_memory_usage.append((module.name, memory_usage))\n module_memory_usage.sort(key=lambda x: x[-1], reverse=True)\n self.debug(\"MODULE MEMORY USAGE:\")\n for module_name, usage in module_memory_usage:\n self.debug(f\" - {module_name}: {self.helpers.bytes_to_human(usage)}\")\n\n status.update({\"modules_errored\": len(modules_errored)})\n\n return status\n\n def stop(self):\n \"\"\"Stops the in-progress scan and performs necessary cleanup.\n\n This method sets the scan's status to \"ABORTING,\" cancels any pending tasks, and drains event queues. It also kills child processes spawned during the scan.\n\n Returns:\n None\n \"\"\"\n if not self._stopping:\n self._stopping = True\n self.status = \"ABORTING\"\n self.hugewarning(\"Aborting scan\")\n self.trace()\n self._cancel_tasks()\n self._drain_queues()\n self.helpers.kill_children()\n self._drain_queues()\n self.helpers.kill_children()\n self.debug(\"Finished aborting scan\")\n\n async def finish(self):\n \"\"\"Finalizes the scan by invoking the `finished()` method on all active modules if new activity is detected.\n\n The method is idempotent and will return False if no new activity has been recorded since the last invocation.\n\n Returns:\n bool: True if new activity has been detected and the `finished()` method is invoked on all modules.\n False if no new activity has been detected since the last invocation.\n\n Notes:\n This method alters the scan's status to \"FINISHING\" if new activity is detected.\n \"\"\"\n # if new events were generated since last time we were here\n if self._new_activity:\n self._new_activity = False\n self.status = \"FINISHING\"\n # Trigger .finished() on every module and start over\n log.info(\"Finishing scan\")\n for module in self.modules.values():\n finished_event = self.make_event(\"FINISHED\", \"FINISHED\", dummy=True, tags={module.name})\n await module.queue_event(finished_event)\n self.verbose(\"Completed finish()\")\n return True\n self.verbose(\"Completed final finish()\")\n # Return False if no new events were generated since last time\n return False\n\n def _drain_queues(self):\n \"\"\"Empties all the event queues for each loaded module and the manager's incoming event queue.\n\n This method iteratively empties both the incoming and outgoing event queues of each module, as well as the incoming event queue of the scan manager.\n\n Returns:\n None\n \"\"\"\n self.debug(\"Draining queues\")\n for module in self.modules.values():\n with contextlib.suppress(asyncio.queues.QueueEmpty):\n while 1:\n if module.incoming_event_queue not in (None, False):\n module.incoming_event_queue.get_nowait()\n with contextlib.suppress(asyncio.queues.QueueEmpty):\n while 1:\n if module.outgoing_event_queue not in (None, False):\n module.outgoing_event_queue.get_nowait()\n self.debug(\"Finished draining queues\")\n\n def _cancel_tasks(self):\n \"\"\"Cancels all asynchronous tasks and shuts down the process pool.\n\n This method collects all pending tasks from each module, the dispatcher,\n and the scan manager. After collecting these tasks, it cancels them synchronously\n using a helper function. Finally, it shuts down the process pool, canceling any\n pending futures.\n\n Returns:\n None\n \"\"\"\n self.debug(\"Cancelling all scan tasks\")\n tasks = []\n # module workers\n for m in self.modules.values():\n tasks += getattr(m, \"_tasks\", [])\n # init events\n if self.init_events_task:\n tasks.append(self.init_events_task)\n # ticker\n if self.ticker_task:\n tasks.append(self.ticker_task)\n # dispatcher\n tasks += self.dispatcher_tasks\n self.helpers.cancel_tasks_sync(tasks)\n # process pool\n self.helpers.process_pool.shutdown(cancel_futures=True)\n self.debug(\"Finished cancelling all scan tasks\")\n return tasks\n\n async def _report(self):\n \"\"\"Asynchronously executes the `report()` method for each module in the scan.\n\n This method is called once at the end of each scan and is responsible for\n triggering the `report()` function for each module. It executes irrespective\n of whether the scan was aborted or completed successfully. The method makes\n use of an asynchronous context manager (`_acatch`) to handle exceptions and\n a task counter to keep track of the task's context.\n\n Returns:\n None\n \"\"\"\n for mod in self.modules.values():\n context = f\"{mod.name}.report()\"\n async with self._acatch(context), mod._task_counter.count(context):\n await mod.report()\n\n async def _cleanup(self):\n \"\"\"Asynchronously executes the `cleanup()` method for each module in the scan.\n\n This method is called once at the end of the scan to perform resource cleanup\n tasks. It is executed regardless of whether the scan was aborted or completed\n successfully. The scan status is set to \"CLEANING_UP\" during the execution.\n After calling the `cleanup()` method for each module, it performs additional\n cleanup tasks such as removing the scan's home directory if empty and cleaning\n old scans.\n\n Returns:\n None\n \"\"\"\n # clean up self\n if not self._cleanedup:\n self._cleanedup = True\n self.status = \"CLEANING_UP\"\n # clean up dns engine\n if self.helpers._dns is not None:\n await self.helpers.dns.shutdown()\n # clean up web engine\n if self.helpers._web is not None:\n await self.helpers.web.shutdown()\n # clean up modules\n for mod in self.modules.values():\n await mod._cleanup()\n with contextlib.suppress(Exception):\n self.home.rmdir()\n self.helpers.rm_rf(self.temp_dir, ignore_errors=True)\n self.helpers.clean_old_scans()\n\n def in_scope(self, *args, **kwargs):\n return self.preset.in_scope(*args, **kwargs)\n\n def whitelisted(self, *args, **kwargs):\n return self.preset.whitelisted(*args, **kwargs)\n\n def blacklisted(self, *args, **kwargs):\n return self.preset.blacklisted(*args, **kwargs)\n\n @property\n def core(self):\n return self.preset.core\n\n @property\n def config(self):\n return self.preset.core.config\n\n @property\n def target(self):\n return self.preset.target\n\n @property\n def seeds(self):\n return self.preset.seeds\n\n @property\n def whitelist(self):\n return self.preset.whitelist\n\n @property\n def blacklist(self):\n return self.preset.blacklist\n\n @property\n def helpers(self):\n return self.preset.helpers\n\n @property\n def force_start(self):\n return self.preset.force_start\n\n @property\n def word_cloud(self):\n return self.helpers.word_cloud\n\n @property\n def stopping(self):\n return not self.running\n\n @property\n def stopped(self):\n return self._status_code > 5\n\n @property\n def running(self):\n return 0 < self._status_code < 4\n\n @property\n def aborting(self):\n return 5 <= self._status_code <= 6\n\n @property\n def status(self):\n return self._status\n\n @property\n def omitted_event_types(self):\n if self._omitted_event_types is None:\n self._omitted_event_types = self.config.get(\"omit_event_types\", [])\n return self._omitted_event_types\n\n @status.setter\n def status(self, status):\n \"\"\"\n Block setting after status has been aborted\n \"\"\"\n status = str(status).strip().upper()\n if status in self._status_codes:\n if self.status == \"ABORTING\" and not status == \"ABORTED\":\n self.debug(f'Attempt to set invalid status \"{status}\" on aborted scan')\n else:\n if status != self._status:\n self._status = status\n self._status_code = self._status_codes[status]\n self.dispatcher_tasks.append(\n asyncio.create_task(\n self.dispatcher.catch(self.dispatcher.on_status, self._status, self.id),\n name=f\"{self.name}.dispatcher.on_status({status})\",\n )\n )\n else:\n self.debug(f'Scan status is already \"{status}\"')\n else:\n self.debug(f'Attempt to set invalid status \"{status}\" on scan')\n\n def make_event(self, *args, **kwargs):\n kwargs[\"scan\"] = self\n event = make_event(*args, **kwargs)\n return event\n\n def update_event(self, event, **kwargs):\n kwargs[\"scan\"] = self\n return update_event(event, **kwargs)\n\n @property\n def root_event(self):\n \"\"\"\n The root scan event, e.g.:\n ```json\n {\n \"type\": \"SCAN\",\n \"id\": \"SCAN:1188928d942ace8e3befae0bdb9c3caa22705f54\",\n \"data\": \"pixilated_kathryn (SCAN:1188928d942ace8e3befae0bdb9c3caa22705f54)\",\n \"scope_distance\": 0,\n \"scan\": \"SCAN:1188928d942ace8e3befae0bdb9c3caa22705f54\",\n \"timestamp\": 1694548779.616255,\n \"parent\": \"SCAN:1188928d942ace8e3befae0bdb9c3caa22705f54\",\n \"tags\": [\n \"distance-0\"\n ],\n \"module\": \"TARGET\",\n \"module_sequence\": \"TARGET\"\n }\n ```\n \"\"\"\n if self._root_event is None:\n self._root_event = self.make_root_event(f\"Scan {self.name} started at {self.start_time}\")\n self._root_event.data[\"status\"] = self.status\n return self._root_event\n\n def finish_event(self, context=None, status=None):\n if self._finish_event is None:\n if context is None or status is None:\n raise ValueError(\"Must specify context and status\")\n self._finish_event = self.make_root_event(context)\n self._finish_event.data[\"status\"] = status\n return self._finish_event\n\n def make_root_event(self, context):\n root_event = self.make_event(data=self.json, event_type=\"SCAN\", dummy=True, context=context)\n root_event._id = self.id\n root_event.scope_distance = 0\n root_event.parent = root_event\n root_event._dummy = False\n root_event.module = self._make_dummy_module(name=\"TARGET\", _type=\"TARGET\")\n return root_event\n\n @property\n def dns_strings(self):\n \"\"\"\n A list of DNS hostname strings generated from the scan target\n \"\"\"\n if self._dns_strings is None:\n dns_whitelist = {t.host for t in self.whitelist if t.host and isinstance(t.host, str)}\n dns_whitelist = sorted(dns_whitelist, key=len)\n dns_whitelist_set = set()\n dns_strings = []\n for t in dns_whitelist:\n if not any(x in dns_whitelist_set for x in self.helpers.domain_parents(t, include_self=True)):\n dns_whitelist_set.add(t)\n dns_strings.append(t)\n self._dns_strings = dns_strings\n return self._dns_strings\n\n def _generate_dns_regexes(self, pattern):\n \"\"\"\n Generates a list of compiled DNS hostname regexes based on the provided pattern.\n This method centralizes the regex compilation to avoid redundancy in the dns_regexes and dns_regexes_yara methods.\n\n Args:\n pattern (str):\n Returns:\n list[re.Pattern]: A list of compiled regex patterns if enabled, otherwise an empty list.\n \"\"\"\n\n dns_regexes = []\n for t in self.dns_strings:\n regex_pattern = re.compile(f\"{pattern}{re.escape(t)})\", re.I)\n log.debug(f\"Generated Regex [{regex_pattern.pattern}] for domain {t}\")\n dns_regexes.append(regex_pattern)\n return dns_regexes\n\n @property\n def dns_regexes(self):\n \"\"\"\n A list of DNS hostname regexes generated from the scan target\n For the purpose of extracting hostnames\n\n Examples:\n Extract hostnames from text:\n >>> for regex in scan.dns_regexes:\n ... for match in regex.finditer(response.text):\n ... hostname = match.group().lower()\n \"\"\"\n if self._dns_regexes is None:\n self._dns_regexes = self._generate_dns_regexes(r\"((?:(?:[\\w-]+)\\.)+\")\n return self._dns_regexes\n\n @property\n def dns_regexes_yara(self):\n \"\"\"\n Returns a list of DNS hostname regexes formatted specifically for compatibility with YARA rules.\n \"\"\"\n if self._dns_regexes_yara is None:\n self._dns_regexes_yara = self._generate_dns_regexes(r\"(([a-z0-9-]+\\.)*\")\n return self._dns_regexes_yara\n\n @property\n def dns_yara_rules_uncompiled(self):\n if self._dns_yara_rules_uncompiled is None:\n regexes_component_list = []\n for i, r in enumerate(self.dns_regexes_yara):\n regexes_component_list.append(rf\"$dns_name_{i} = /\\b{r.pattern}/ nocase\")\n\n # Chunk the regexes into groups of 10,000\n chunk_size = 10000\n rules = {}\n for chunk_index in range(0, len(regexes_component_list), chunk_size):\n chunk = regexes_component_list[chunk_index : chunk_index + chunk_size]\n if chunk:\n regexes_component = \" \".join(chunk)\n rule_name = f\"hostname_extraction_{chunk_index // chunk_size}\"\n rule = f'rule {rule_name} {{meta: description = \"matches DNS hostname pattern derived from target(s)\" strings: {regexes_component} condition: any of them}}'\n rules[rule_name] = rule\n\n self._dns_yara_rules_uncompiled = rules\n return self._dns_yara_rules_uncompiled\n\n async def dns_yara_rules(self):\n if self._dns_yara_rules is None:\n if self.dns_yara_rules_uncompiled is not None:\n import yara\n\n self._dns_yara_rules = await self.helpers.run_in_executor(\n yara.compile, source=\"\\n\".join(self.dns_yara_rules_uncompiled.values())\n )\n return self._dns_yara_rules\n\n async def extract_in_scope_hostnames(self, s):\n \"\"\"\n Given a string, uses yara to extract hostnames matching scan targets\n\n Examples:\n >>> await self.scan.extract_in_scope_hostnames(\"http://www.evilcorp.com\")\n ... {\"www.evilcorp.com\"}\n \"\"\"\n matches = set()\n dns_yara_rules = await self.dns_yara_rules()\n if dns_yara_rules is not None:\n for match in await self.helpers.run_in_executor(dns_yara_rules.match, data=s):\n for string in match.strings:\n for instance in string.instances:\n matches.add(str(instance))\n return matches\n\n @property\n def json(self):\n \"\"\"\n A dictionary representation of the scan including its name, ID, targets, whitelist, blacklist, and modules\n \"\"\"\n j = {}\n for i in (\"id\", \"name\"):\n v = getattr(self, i, \"\")\n if v:\n j.update({i: v})\n j[\"target\"] = self.preset.target.json\n j[\"preset\"] = self.preset.to_dict(redact_secrets=True)\n if self.start_time is not None:\n j[\"started_at\"] = self.start_time.isoformat()\n if self.end_time is not None:\n j[\"finished_at\"] = self.end_time.isoformat()\n if self.duration is not None:\n j[\"duration_seconds\"] = self.duration_seconds\n if self.duration_human is not None:\n j[\"duration\"] = self.duration_human\n return j\n\n def debug(self, *args, trace=False, **kwargs):\n log.debug(*args, extra={\"scan_id\": self.id}, **kwargs)\n if trace:\n self.trace()\n\n def verbose(self, *args, trace=False, **kwargs):\n log.verbose(*args, extra={\"scan_id\": self.id}, **kwargs)\n if trace:\n self.trace()\n\n def hugeverbose(self, *args, trace=False, **kwargs):\n log.hugeverbose(*args, extra={\"scan_id\": self.id}, **kwargs)\n if trace:\n self.trace()\n\n def info(self, *args, trace=False, **kwargs):\n log.info(*args, extra={\"scan_id\": self.id}, **kwargs)\n if trace:\n self.trace()\n\n def hugeinfo(self, *args, trace=False, **kwargs):\n log.hugeinfo(*args, extra={\"scan_id\": self.id}, **kwargs)\n if trace:\n self.trace()\n\n def success(self, *args, trace=False, **kwargs):\n log.success(*args, extra={\"scan_id\": self.id}, **kwargs)\n if trace:\n self.trace()\n\n def hugesuccess(self, *args, trace=False, **kwargs):\n log.hugesuccess(*args, extra={\"scan_id\": self.id}, **kwargs)\n if trace:\n self.trace()\n\n def warning(self, *args, trace=True, **kwargs):\n log.warning(*args, extra={\"scan_id\": self.id}, **kwargs)\n if trace:\n self.trace()\n\n def hugewarning(self, *args, trace=True, **kwargs):\n log.hugewarning(*args, extra={\"scan_id\": self.id}, **kwargs)\n if trace:\n self.trace()\n\n def error(self, *args, trace=True, **kwargs):\n log.error(*args, extra={\"scan_id\": self.id}, **kwargs)\n if trace:\n self.trace()\n\n def trace(self, msg=None):\n if msg is None:\n e_type, e_val, e_traceback = exc_info()\n if e_type is not None:\n log.trace(traceback.format_exc())\n else:\n log.trace(msg)\n\n def critical(self, *args, trace=True, **kwargs):\n log.critical(*args, extra={\"scan_id\": self.id}, **kwargs)\n if trace:\n self.trace()\n\n @property\n def log_level(self):\n \"\"\"\n Return the current log level, e.g. logging.INFO\n \"\"\"\n return self.core.logger.log_level\n\n @property\n def _log_handlers(self):\n if self.__log_handlers is None:\n self.helpers.mkdir(self.home)\n main_handler = GzipRotatingFileHandler(\n str(self.home / \"scan.log\"), maxBytes=1024 * 1024 * 100, backupCount=100\n )\n main_handler.addFilter(lambda x: x.levelno != logging.TRACE and x.levelno >= logging.VERBOSE)\n debug_handler = GzipRotatingFileHandler(\n str(self.home / \"debug.log\"), maxBytes=1024 * 1024 * 100, backupCount=100\n )\n debug_handler.addFilter(lambda x: x.levelno >= logging.DEBUG)\n error_handler = GzipRotatingFileHandler(\n str(self.home / \"error.log\"), maxBytes=1024 * 1024 * 100, backupCount=100\n )\n error_handler.addFilter(lambda x: x.levelno == logging.TRACE or x.levelno >= logging.ERROR)\n self.__log_handlers = [main_handler, debug_handler, error_handler]\n return self.__log_handlers\n\n def _start_log_handlers(self):\n # add log handlers\n for handler in self._log_handlers:\n self.core.logger.add_log_handler(handler)\n # temporarily disable main ones\n for handler_name in (\"file_main\", \"file_debug\"):\n handler = self.core.logger.log_handlers.get(handler_name, None)\n if handler is not None and handler not in self._log_handler_backup:\n self._log_handler_backup.append(handler)\n self.core.logger.remove_log_handler(handler)\n\n def _stop_log_handlers(self):\n # remove log handlers\n for handler in self._log_handlers:\n self.core.logger.remove_log_handler(handler)\n # restore main ones\n for handler in self._log_handler_backup:\n self.core.logger.add_log_handler(handler)\n\n def _fail_setup(self, msg):\n msg = str(msg)\n if self.force_start:\n self.error(msg)\n else:\n msg += \" (--force to run module anyway)\"\n raise ScanError(msg)\n\n def _load_modules(self, modules):\n modules = [str(m) for m in modules]\n loaded_modules = {}\n failed = set()\n for module_name, module_class in self.preset.module_loader.load_modules(modules).items():\n if module_class:\n try:\n loaded_modules[module_name] = module_class(self)\n self.verbose(f'Loaded module \"{module_name}\"')\n continue\n except Exception:\n self.warning(f\"Failed to load module {module_class}\")\n else:\n self.warning(f'Failed to load unknown module \"{module_name}\"')\n failed.add(module_name)\n return loaded_modules, failed\n\n async def _status_ticker(self, interval=15):\n async with self._acatch():\n while 1:\n await asyncio.sleep(interval)\n self.modules_status(_log=True)\n\n @contextlib.asynccontextmanager\n async def _acatch(self, context=\"scan\", finally_callback=None, unhandled_is_critical=False):\n \"\"\"\n Async version of catch()\n\n async with catch():\n await do_stuff()\n \"\"\"\n try:\n yield\n except BaseException as e:\n try:\n self._handle_exception(e, context=context, unhandled_is_critical=unhandled_is_critical)\n except Exception as e2:\n self.log.critical(f\"Error in exception handler: {e2} {traceback.format_exc()}\")\n raise\n\n def _handle_exception(self, e, context=\"scan\", finally_callback=None, unhandled_is_critical=False):\n if callable(context):\n context = f\"{context.__qualname__}()\"\n filename, lineno, funcname = self.helpers.get_traceback_details(e)\n if self.helpers.in_exception_chain(e, (KeyboardInterrupt,)):\n log.debug(\"Interrupted\")\n self.stop()\n elif isinstance(e, BrokenPipeError):\n log.debug(f\"BrokenPipeError in {filename}:{lineno}:{funcname}(): {e}\")\n elif isinstance(e, asyncio.CancelledError):\n raise\n elif isinstance(e, Exception):\n traceback_str = getattr(e, \"engine_traceback\", None)\n if traceback_str is None:\n traceback_str = traceback.format_exc()\n if unhandled_is_critical:\n log.critical(f\"Error in {context}: {filename}:{lineno}:{funcname}(): {e}\")\n log.critical(traceback_str)\n else:\n log.error(f\"Error in {context}: {filename}:{lineno}:{funcname}(): {e}\")\n log.trace(traceback_str)\n if callable(finally_callback):\n finally_callback(e)\n\n def _make_dummy_module(self, name, _type=\"scan\"):\n \"\"\"\n Construct a dummy module, for attachment to events\n \"\"\"\n try:\n return self.dummy_modules[name]\n except KeyError:\n dummy = DummyModule(scan=self, name=name, _type=_type)\n self.dummy_modules[name] = dummy\n return dummy\n\n\nfrom bbot.modules.base import BaseModule\n\n\nclass DummyModule(BaseModule):\n _priority = 4\n\n def __init__(self, *args, **kwargs):\n self._name = kwargs.pop(\"name\")\n self._type = kwargs.pop(\"_type\")\n super().__init__(*args, **kwargs)\n" }, { "path": "bbot/scanner/stats.py", "content": "import time\nimport logging\nfrom collections import deque\n\nlog = logging.getLogger(\"bbot.scanner.stats\")\n\n\ndef _increment(d, k):\n try:\n d[k] += 1\n except KeyError:\n d[k] = 1\n\n\nclass SpeedCounter:\n \"\"\"\n A simple class for keeping a rolling tally of the number of events inside a specific time window\n \"\"\"\n\n def __init__(self, window=60):\n self.timestamps = deque()\n self.window = window\n\n def tick(self):\n current_time = time.time()\n self.timestamps.append(current_time)\n self.remove_old_timestamps(current_time)\n\n def remove_old_timestamps(self, current_time):\n while self.timestamps and current_time - self.timestamps[0] > self.window:\n self.timestamps.popleft()\n\n @property\n def speed(self):\n self.remove_old_timestamps(time.time())\n return len(self.timestamps)\n\n\nclass ScanStats:\n def __init__(self, scan):\n self.scan = scan\n self.module_stats = {}\n self.events_emitted_by_type = {}\n self.speedometer = SpeedCounter(scan.status_frequency)\n\n def event_produced(self, event):\n _increment(self.events_emitted_by_type, event.type)\n module_stat = self.get(event.module)\n if module_stat is not None:\n module_stat.increment_produced(event)\n\n def event_consumed(self, event, module):\n self.speedometer.tick()\n # skip ingress/egress modules, etc.\n if module.name.startswith(\"_\"):\n return\n module_stat = self.get(module)\n if module_stat is not None:\n module_stat.increment_consumed(event)\n\n def get(self, module):\n try:\n module_stat = self.module_stats[module.name]\n except KeyError:\n module_stat = ModuleStat(module)\n self.module_stats[module.name] = module_stat\n except AttributeError:\n module_stat = None\n return module_stat\n\n def table(self):\n header = [\"Module\", \"Produced\", \"Consumed\"]\n table = []\n for mname, mstat in self.module_stats.items():\n if mname == \"TARGET\" or mstat.module._stats_exclude:\n continue\n table_row = []\n table_row.append(mname)\n produced_str = f\"{mstat.produced_total:,}\"\n produced = sorted(mstat.produced.items(), key=lambda x: x[0])\n if produced:\n produced_str += \" (\" + \", \".join(f\"{c:,} {t}\" for t, c in produced) + \")\"\n table_row.append(produced_str)\n consumed_str = f\"{mstat.consumed_total:,}\"\n consumed = sorted(mstat.consumed.items(), key=lambda x: x[0])\n if consumed:\n consumed_str += \" (\" + \", \".join(f\"{c:,} {t}\" for t, c in consumed) + \")\"\n table_row.append(consumed_str)\n table.append(table_row)\n table.sort(key=lambda x: self.module_stats[x[0]].produced_total, reverse=True)\n return [header] + table\n\n def _make_table(self):\n table = self.table()\n if len(table) == 1:\n table += [[\"None\", \"None\", \"None\"]]\n return table[1:], table[0]\n\n\nclass ModuleStat:\n def __init__(self, module):\n self.module = module\n self.produced = {}\n self.produced_total = 0\n self.consumed = {}\n self.consumed_total = 0\n\n def increment_produced(self, event):\n self.produced_total += 1\n _increment(self.produced, event.type)\n\n def increment_consumed(self, event):\n if event.type not in (\"FINISHED\",):\n self.consumed_total += 1\n _increment(self.consumed, event.type)\n" }, { "path": "bbot/scanner/target.py", "content": "import logging\nimport regex as re\nfrom hashlib import sha1\nfrom radixtarget import RadixTarget\nfrom radixtarget.helpers import host_size_key\n\nfrom bbot.errors import *\nfrom bbot.core.event import is_event\nfrom bbot.core.event.helpers import EventSeed, BaseEventSeed\nfrom bbot.core.helpers.misc import is_dns_name, is_ip, is_ip_type\n\nlog = logging.getLogger(\"bbot.core.target\")\n\n\nclass BaseTarget(RadixTarget):\n \"\"\"\n A collection of BBOT events that represent a scan target.\n\n The purpose of this class is to hold a potentially huge target list in a space-efficient way,\n while allowing lightning fast scope lookups.\n\n This class is inherited by all three components of the BBOT target:\n - Whitelist\n - Blacklist\n - Seeds\n \"\"\"\n\n accept_target_types = [\"TARGET\"]\n\n def __init__(self, *targets, **kwargs):\n # ignore blank targets (sometimes happens as a symptom of .splitlines())\n targets = [stripped for t in targets if (stripped := (t.strip() if isinstance(t, str) else t))]\n self.event_seeds = set()\n super().__init__(*targets, **kwargs)\n\n @property\n def inputs(self):\n return set(e.input for e in self.event_seeds)\n\n def get(self, event, **kwargs):\n \"\"\"\n Here we override RadixTarget's get() method, which normally only accepts hosts, to also accept events for convenience.\n \"\"\"\n host = None\n raise_error = kwargs.get(\"raise_error\", False)\n # if it's already an event or event seed, use its host\n if is_event(event) or isinstance(event, BaseEventSeed):\n host = event.host\n # save resources by checking if the event is an IP or DNS name\n elif is_ip(event, include_network=True) or is_dns_name(event):\n host = event\n # if it's a string, autodetect its type and parse out its host\n elif isinstance(event, str):\n event_seed = self._make_event_seed(event, raise_error=raise_error)\n host = event_seed.host\n if not host:\n return\n else:\n raise ValueError(f\"Invalid target type for {self.__class__.__name__}: {type(event)}\")\n if not host:\n msg = f\"Host not found: '{event}'\"\n if raise_error:\n raise KeyError(msg)\n else:\n log.warning(msg)\n return\n results = super().get(host, **kwargs)\n return results\n\n def _make_event_seed(self, target, raise_error=False):\n try:\n return EventSeed(target)\n except ValidationError:\n msg = f\"Invalid target: '{target}'\"\n if raise_error:\n raise KeyError(msg)\n else:\n log.warning(msg)\n\n def add(self, targets, data=None):\n if not isinstance(targets, (list, set, tuple)):\n targets = [targets]\n event_seeds = set()\n for target in targets:\n event_seed = EventSeed(target)\n if not event_seed._target_type in self.accept_target_types:\n log.warning(f\"Invalid target type for {self.__class__.__name__}: {event_seed.type}\")\n continue\n event_seeds.add(event_seed)\n\n # sort by host size to ensure consistency\n event_seeds = sorted(event_seeds, key=lambda e: (0, 0) if not e.host else host_size_key(e.host))\n for event_seed in event_seeds:\n self.event_seeds.add(event_seed)\n self._add(event_seed.host, data=(event_seed if data is None else data))\n\n def __iter__(self):\n yield from self.event_seeds\n\n\nclass ScanSeeds(BaseTarget):\n \"\"\"\n Initial events used to seed a scan.\n\n These are the targets specified by the user, e.g. via `-t` on the CLI.\n \"\"\"\n\n def get(self, event, single=True, **kwargs):\n results = super().get(event, **kwargs)\n if results and single:\n return next(iter(results))\n return results\n\n def _add(self, host, data):\n \"\"\"\n Overrides the base method to enable having multiple events for the same host.\n\n The \"data\" attribute of the node is now a set of events.\n\n This is useful for seeds, because it lets us have both evilcorp.com:80 and https://evilcorp.com\n as separate events even though they have the same host.\n \"\"\"\n if host:\n try:\n event_set = self.get(host, raise_error=True, single=False)\n event_set.add(data)\n except KeyError:\n event_set = {data}\n super()._add(host, data=event_set)\n\n def _hash_value(self):\n # seeds get hashed by event data\n return sorted(str(e.data).encode() for e in self.event_seeds)\n\n\nclass ACLTarget(BaseTarget):\n def __init__(self, *args, **kwargs):\n # ACL mode dedupes by host (and skips adding already-contained hosts) for efficiency\n kwargs[\"acl_mode\"] = True\n super().__init__(*args, **kwargs)\n\n\nclass ScanWhitelist(ACLTarget):\n \"\"\"\n A collection of BBOT events that represent a scan's whitelist.\n \"\"\"\n\n pass\n\n\nclass ScanBlacklist(ACLTarget):\n \"\"\"\n A collection of BBOT events that represent a scan's blacklist.\n \"\"\"\n\n accept_target_types = [\"TARGET\", \"BLACKLIST\"]\n\n def __init__(self, *args, **kwargs):\n self.blacklist_regexes = set()\n super().__init__(*args, **kwargs)\n\n def get(self, host, **kwargs):\n \"\"\"\n Blacklists only accept IPs or strings. This is cleaner since we need to search for regex patterns.\n \"\"\"\n if not (is_ip_type(host) or isinstance(host, str)):\n raise ValueError(f\"Invalid target type for {self.__class__.__name__}: {type(host)}\")\n raise_error = kwargs.get(\"raise_error\", False)\n # first, check event's host against blacklist\n try:\n event_seed = self._make_event_seed(host, raise_error=raise_error)\n host = event_seed.host\n to_match = event_seed.data\n except ValidationError:\n to_match = str(host)\n try:\n event_result = super().get(host, raise_error=True)\n except KeyError:\n event_result = None\n if event_result is not None:\n return event_result\n # next, check event's host against regexes\n for regex in self.blacklist_regexes:\n if regex.search(to_match):\n return host\n if raise_error:\n raise KeyError(f\"Host not found: '{host}'\")\n return None\n\n def _add(self, host, data):\n if getattr(data, \"type\", \"\") == \"BLACKLIST_REGEX\":\n self.blacklist_regexes.add(re.compile(data.data))\n if host is not None:\n super()._add(host, data)\n\n def _hash_value(self):\n # regexes are included in blacklist hash\n regex_patterns = [str(r.pattern).encode() for r in self.blacklist_regexes]\n hosts = [str(h).encode() for h in self.sorted_hosts]\n return hosts + regex_patterns\n\n def __len__(self):\n return super().__len__() + len(self.blacklist_regexes)\n\n def __bool__(self):\n return bool(len(self))\n\n\nclass BBOTTarget:\n \"\"\"\n A convenient abstraction of a scan target that contains three subtargets:\n - seeds\n - whitelist\n - blacklist\n\n Provides high-level functions like in_scope(), which includes both whitelist and blacklist checks.\n \"\"\"\n\n def __init__(self, *seeds, whitelist=None, blacklist=None, strict_scope=False):\n self.strict_scope = strict_scope\n self.seeds = ScanSeeds(*seeds, strict_dns_scope=strict_scope)\n if whitelist is None:\n whitelist = self.seeds.hosts\n self.whitelist = ScanWhitelist(*whitelist, strict_dns_scope=strict_scope)\n if blacklist is None:\n blacklist = []\n self.blacklist = ScanBlacklist(*blacklist)\n\n @property\n def json(self):\n return {\n \"seeds\": sorted(self.seeds.inputs),\n \"whitelist\": sorted(self.whitelist.inputs),\n \"blacklist\": sorted(self.blacklist.inputs),\n \"strict_scope\": self.strict_scope,\n \"hash\": self.hash.hex(),\n \"seed_hash\": self.seeds.hash.hex(),\n \"whitelist_hash\": self.whitelist.hash.hex(),\n \"blacklist_hash\": self.blacklist.hash.hex(),\n \"scope_hash\": self.scope_hash.hex(),\n }\n\n @property\n def hash(self):\n sha1_hash = sha1()\n for target_hash in [t.hash for t in (self.seeds, self.whitelist, self.blacklist)]:\n sha1_hash.update(target_hash)\n return sha1_hash.digest()\n\n @property\n def scope_hash(self):\n sha1_hash = sha1()\n # Consider only the hash values of the whitelist and blacklist\n for target_hash in [t.hash for t in (self.whitelist, self.blacklist)]:\n sha1_hash.update(target_hash)\n return sha1_hash.digest()\n\n def in_scope(self, host):\n \"\"\"\n Check whether a hostname, url, IP, etc. is in scope.\n Accepts either events or string data.\n\n Checks whitelist and blacklist.\n If `host` is an event and its scope distance is zero, it will automatically be considered in-scope.\n\n Examples:\n Check if a URL is in scope:\n >>> preset.in_scope(\"http://www.evilcorp.com\")\n True\n \"\"\"\n blacklisted = self.blacklisted(host)\n whitelisted = self.whitelisted(host)\n return whitelisted and not blacklisted\n\n def blacklisted(self, host):\n \"\"\"\n Check whether a hostname, url, IP, etc. is blacklisted.\n\n Note that `host` can be a hostname, IP address, CIDR, email address, or any BBOT `Event` with the `host` attribute.\n\n Args:\n host (str or IPAddress or Event): The host to check against the blacklist\n\n Examples:\n Check if a URL's host is blacklisted:\n >>> preset.blacklisted(\"http://www.evilcorp.com\")\n True\n \"\"\"\n return host in self.blacklist\n\n def whitelisted(self, host):\n \"\"\"\n Check whether a hostname, url, IP, etc. is whitelisted.\n\n Note that `host` can be a hostname, IP address, CIDR, email address, or any BBOT `Event` with the `host` attribute.\n\n Args:\n host (str or IPAddress or Event): The host to check against the whitelist\n\n Examples:\n Check if a URL's host is whitelisted:\n >>> preset.whitelisted(\"http://www.evilcorp.com\")\n True\n \"\"\"\n return host in self.whitelist\n\n def __eq__(self, other):\n return self.hash == other.hash\n" }, { "path": "bbot/scripts/benchmark_report.py", "content": "#!/usr/bin/env python3\n\"\"\"\nBranch-based benchmark comparison tool for BBOT performance tests.\n\nThis script takes two git branches, runs benchmarks on each, and generates\na comparison report showing performance differences between them.\n\"\"\"\n\nimport json\nimport argparse\nimport subprocess\nimport tempfile\nfrom pathlib import Path\nfrom typing import Dict, List, Any, Tuple\n\n\ndef run_command(cmd: List[str], cwd: Path = None, capture_output: bool = True) -> subprocess.CompletedProcess:\n \"\"\"Run a shell command and return the result.\"\"\"\n try:\n result = subprocess.run(cmd, cwd=cwd, capture_output=capture_output, text=True, check=True)\n return result\n except subprocess.CalledProcessError as e:\n print(f\"Command failed: {' '.join(cmd)}\")\n print(f\"Exit code: {e.returncode}\")\n print(f\"Error output: {e.stderr}\")\n raise\n\n\ndef get_current_branch() -> str:\n \"\"\"Get the current git branch name.\"\"\"\n result = run_command([\"git\", \"branch\", \"--show-current\"])\n return result.stdout.strip()\n\n\ndef checkout_branch(branch: str, repo_path: Path = None):\n \"\"\"Checkout a git branch.\"\"\"\n print(f\"Checking out branch: {branch}\")\n run_command([\"git\", \"checkout\", branch], cwd=repo_path)\n\n\ndef run_benchmarks(output_file: Path, repo_path: Path = None) -> bool:\n \"\"\"Run benchmarks and save results to JSON file.\"\"\"\n print(f\"Running benchmarks, saving to {output_file}\")\n\n # Check if benchmarks directory exists\n benchmarks_dir = repo_path / \"bbot/test/benchmarks\" if repo_path else Path(\"bbot/test/benchmarks\")\n if not benchmarks_dir.exists():\n print(f\"Benchmarks directory not found: {benchmarks_dir}\")\n print(\"This branch likely doesn't have benchmark tests yet.\")\n return False\n\n try:\n cmd = [\n \"poetry\",\n \"run\",\n \"python\",\n \"-m\",\n \"pytest\",\n \"bbot/test/benchmarks/\",\n \"--benchmark-only\",\n f\"--benchmark-json={output_file}\",\n \"-q\",\n ]\n run_command(cmd, cwd=repo_path, capture_output=False)\n return True\n except subprocess.CalledProcessError:\n print(\"Benchmarks failed for current state\")\n return False\n\n\ndef load_benchmark_data(filepath: Path) -> Dict[str, Any]:\n \"\"\"Load benchmark data from JSON file.\"\"\"\n try:\n with open(filepath, \"r\") as f:\n return json.load(f)\n except FileNotFoundError:\n print(f\"Warning: Benchmark file not found: {filepath}\")\n return {}\n except json.JSONDecodeError:\n print(f\"Warning: Could not parse JSON from {filepath}\")\n return {}\n\n\ndef format_time(seconds: float) -> str:\n \"\"\"Format time in human-readable format.\"\"\"\n if seconds < 0.000001: # Less than 1 microsecond\n return f\"{seconds * 1000000000:.0f}ns\" # Show as nanoseconds with no decimal\n elif seconds < 0.001: # Less than 1 millisecond\n return f\"{seconds * 1000000:.2f}Âĩs\" # Show as microseconds with 2 decimal places\n elif seconds < 1: # Less than 1 second\n return f\"{seconds * 1000:.2f}ms\" # Show as milliseconds with 2 decimal places\n else:\n return f\"{seconds:.3f}s\" # Show as seconds with 3 decimal places\n\n\ndef format_ops(ops: float) -> str:\n \"\"\"Format operations per second.\"\"\"\n if ops > 1000:\n return f\"{ops / 1000:.1f}K ops/sec\"\n else:\n return f\"{ops:.1f} ops/sec\"\n\n\ndef calculate_change_percentage(old_value: float, new_value: float) -> Tuple[float, str]:\n \"\"\"Calculate percentage change and return emoji indicator.\"\"\"\n if old_value == 0:\n return 0, \"🆕\"\n\n change = ((new_value - old_value) / old_value) * 100\n\n if change > 10:\n return change, \"⚠ī¸\" # Regression (slower)\n elif change < -10:\n return change, \"🚀\" # Improvement (faster)\n else:\n return change, \"✅\" # No significant change\n\n\ndef generate_benchmark_table(benchmarks: List[Dict[str, Any]], title: str = \"Results\") -> str:\n \"\"\"Generate markdown table for benchmark results.\"\"\"\n if not benchmarks:\n return f\"### {title}\\nNo benchmark data available.\\n\"\n\n table = f\"\"\"### {title}\n\n| Test Name | Mean Time | Ops/sec | Min | Max |\n|-----------|-----------|---------|-----|-----|\n\"\"\"\n\n for bench in benchmarks:\n stats = bench.get(\"stats\", {})\n name = bench.get(\"name\", \"Unknown\")\n # Generic test name cleanup - just remove 'test_' prefix and format nicely\n test_name = name.replace(\"test_\", \"\").replace(\"_\", \" \").title()\n\n mean = format_time(stats.get(\"mean\", 0))\n ops = format_ops(stats.get(\"ops\", 0))\n min_time = format_time(stats.get(\"min\", 0))\n max_time = format_time(stats.get(\"max\", 0))\n\n table += f\"| {test_name} | {mean} | {ops} | {min_time} | {max_time} |\\n\"\n\n return table + \"\\n\"\n\n\ndef generate_comparison_table(current_data: Dict, base_data: Dict, current_branch: str, base_branch: str) -> str:\n \"\"\"Generate comparison table between current and base benchmark results.\"\"\"\n if not current_data or not base_data:\n return \"\"\n\n current_benchmarks = current_data.get(\"benchmarks\", [])\n base_benchmarks = base_data.get(\"benchmarks\", [])\n\n # Create lookup for base benchmarks\n base_lookup = {bench[\"name\"]: bench for bench in base_benchmarks}\n\n if not current_benchmarks:\n return \"\"\n\n # Count changes for summary\n improvements = 0\n regressions = 0\n no_change = 0\n\n table = f\"\"\"## 📊 Performance Benchmark Report\n\n> Comparing **`{base_branch}`** (baseline) vs **`{current_branch}`** (current)\n\n
\n📈 Detailed Results (All Benchmarks)\n\n> 📋 **Complete results for all benchmarks** - includes both significant and insignificant changes\n\n| đŸ§Ē Test Name | 📏 Base | 📏 Current | 📈 Change | đŸŽ¯ Status |\n|--------------|---------|------------|-----------|-----------|\"\"\"\n\n significant_changes = []\n performance_summary = []\n\n for current_bench in current_benchmarks:\n name = current_bench.get(\"name\", \"Unknown\")\n # Generic test name cleanup - just remove 'test_' prefix and format nicely\n test_name = name.replace(\"test_\", \"\").replace(\"_\", \" \").title()\n\n current_stats = current_bench.get(\"stats\", {})\n current_mean = current_stats.get(\"mean\", 0)\n # For multi-item benchmarks, calculate correct ops/sec\n if \"excavate\" in name:\n current_ops = 100 / current_mean # 100 segments per test\n elif \"event_validation\" in name and \"small\" in name:\n current_ops = 100 / current_mean # 100 targets per test\n elif \"event_validation\" in name and \"large\" in name:\n current_ops = 1000 / current_mean # 1000 targets per test\n elif \"make_event\" in name and \"small\" in name:\n current_ops = 100 / current_mean # 100 items per test\n elif \"make_event\" in name and \"large\" in name:\n current_ops = 1000 / current_mean # 1000 items per test\n elif \"ip\" in name:\n current_ops = 1000 / current_mean # 1000 IPs per test\n elif \"bloom_filter\" in name:\n if \"dns_mutation\" in name:\n current_ops = 2500 / current_mean # 2500 operations per test\n else:\n current_ops = 13000 / current_mean # 13000 operations per test\n else:\n current_ops = 1 / current_mean # Default: single operation\n\n base_bench = base_lookup.get(name)\n if base_bench:\n base_stats = base_bench.get(\"stats\", {})\n base_mean = base_stats.get(\"mean\", 0)\n # For multi-item benchmarks, calculate correct ops/sec\n if \"excavate\" in name:\n base_ops = 100 / base_mean # 100 segments per test\n elif \"event_validation\" in name and \"small\" in name:\n base_ops = 100 / base_mean # 100 targets per test\n elif \"event_validation\" in name and \"large\" in name:\n base_ops = 1000 / base_mean # 1000 targets per test\n elif \"make_event\" in name and \"small\" in name:\n base_ops = 100 / base_mean # 100 items per test\n elif \"make_event\" in name and \"large\" in name:\n base_ops = 1000 / base_mean # 1000 items per test\n elif \"ip\" in name:\n base_ops = 1000 / base_mean # 1000 IPs per test\n elif \"bloom_filter\" in name:\n if \"dns_mutation\" in name:\n base_ops = 2500 / base_mean # 2500 operations per test\n else:\n base_ops = 13000 / base_mean # 13000 operations per test\n else:\n base_ops = 1 / base_mean # Default: single operation\n\n change_percent, emoji = calculate_change_percentage(base_mean, current_mean)\n\n # Create visual change indicator\n if abs(change_percent) > 20:\n change_bar = \"🔴🔴🔴\" if change_percent > 0 else \"đŸŸĸđŸŸĸđŸŸĸ\"\n elif abs(change_percent) > 10:\n change_bar = \"🟡🟡\" if change_percent > 0 else \"đŸŸĸđŸŸĸ\"\n else:\n change_bar = \"âšĒ\"\n\n table += f\"\\n| **{test_name}** | `{format_time(base_mean)}` | `{format_time(current_mean)}` | **{change_percent:+.1f}%** {change_bar} | {emoji} |\"\n\n # Track significant changes\n if abs(change_percent) > 10:\n direction = \"🐌 slower\" if change_percent > 0 else \"🚀 faster\"\n significant_changes.append(f\"- **{test_name}**: {abs(change_percent):.1f}% {direction}\")\n if change_percent > 0:\n regressions += 1\n else:\n improvements += 1\n else:\n no_change += 1\n\n # Add to performance summary\n ops_change = ((current_ops - base_ops) / base_ops) * 100 if base_ops > 0 else 0\n performance_summary.append(\n {\n \"name\": test_name,\n \"time_change\": change_percent,\n \"ops_change\": ops_change,\n \"current_ops\": current_ops,\n }\n )\n else:\n table += f\"\\n| **{test_name}** | `-` | `{format_time(current_mean)}` | **New** 🆕 | 🆕 |\"\n significant_changes.append(\n f\"- **{test_name}**: New test 🆕 ({format_time(current_mean)}, {format_ops(current_ops)})\"\n )\n\n table += \"\\n\\n
\\n\\n\"\n\n # Add performance summary\n table += \"## đŸŽ¯ Performance Summary\\n\\n\"\n\n if improvements > 0 or regressions > 0:\n table += \"```diff\\n\"\n if improvements > 0:\n table += f\"+ {improvements} improvement{'s' if improvements != 1 else ''} 🚀\\n\"\n if regressions > 0:\n table += f\"! {regressions} regression{'s' if regressions != 1 else ''} ⚠ī¸\\n\"\n if no_change > 0:\n table += f\" {no_change} unchanged ✅\\n\"\n table += \"```\\n\\n\"\n else:\n table += \"✅ **No significant performance changes detected** (all changes <10%)\\n\\n\"\n\n # Add significant changes section\n if significant_changes:\n table += \"### 🔍 Significant Changes (>10%)\\n\\n\"\n for change in significant_changes:\n table += f\"{change}\\n\"\n table += \"\\n\"\n\n return table\n\n\ndef generate_report(current_data: Dict, base_data: Dict, current_branch: str, base_branch: str) -> str:\n \"\"\"Generate complete benchmark comparison report.\"\"\"\n\n if not current_data:\n report = \"\"\"## 🚀 Performance Benchmark Report\n\n> ⚠ī¸ **No current benchmark data available**\n> \n> This might be because:\n> - Benchmarks failed to run\n> - No benchmark tests found\n> - Dependencies missing\n\n\"\"\"\n return report\n\n if not base_data:\n report = f\"\"\"## 🚀 Performance Benchmark Report\n\n> ℹī¸ **No baseline benchmark data available**\n> \n> Showing current results for **{current_branch}** only.\n\n\"\"\"\n current_benchmarks = current_data.get(\"benchmarks\", [])\n if current_benchmarks:\n report += f\"\"\"
\n📊 Current Results ({current_branch}) - Click to expand\n\n{generate_benchmark_table(current_benchmarks, \"Results\")}\n
\"\"\"\n else:\n # Add comparison\n comparison = generate_comparison_table(current_data, base_data, current_branch, base_branch)\n if comparison:\n report = comparison\n else:\n # Fallback if no comparison data\n report = f\"\"\"## 🚀 Performance Benchmark Report\n\n> ℹī¸ **No baseline benchmark data available**\n> \n> Showing current results for **{current_branch}** only.\n\n\"\"\"\n\n # Get Python version info\n machine_info = current_data.get(\"machine_info\", {})\n python_version = machine_info.get(\"python_version\", \"Unknown\")\n\n report += f\"\\n\\n---\\n\\n🐍 Python Version {python_version}\"\n\n return report\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Compare benchmark performance between git branches\")\n parser.add_argument(\"--base\", required=True, help=\"Base branch name (e.g., 'main', 'dev')\")\n parser.add_argument(\"--current\", required=True, help=\"Current branch name (e.g., 'feature-branch', 'HEAD')\")\n parser.add_argument(\"--output\", type=Path, help=\"Output markdown file (default: stdout)\")\n parser.add_argument(\"--keep-results\", action=\"store_true\", help=\"Keep intermediate JSON files\")\n\n args = parser.parse_args()\n\n # Get current working directory\n repo_path = Path.cwd()\n\n # Save original branch to restore later\n try:\n original_branch = get_current_branch()\n print(f\"Current branch: {original_branch}\")\n except subprocess.CalledProcessError:\n print(\"Warning: Could not determine current branch\")\n original_branch = None\n\n # Create temporary files for benchmark results\n with tempfile.TemporaryDirectory() as temp_dir:\n temp_path = Path(temp_dir)\n base_results_file = temp_path / \"base_results.json\"\n current_results_file = temp_path / \"current_results.json\"\n\n base_data = {}\n current_data = {}\n\n base_data = {}\n current_data = {}\n\n try:\n # Run benchmarks on base branch\n print(f\"\\n=== Running benchmarks on base branch: {args.base} ===\")\n checkout_branch(args.base, repo_path)\n if run_benchmarks(base_results_file, repo_path):\n base_data = load_benchmark_data(base_results_file)\n\n # Run benchmarks on current branch\n print(f\"\\n=== Running benchmarks on current branch: {args.current} ===\")\n checkout_branch(args.current, repo_path)\n if run_benchmarks(current_results_file, repo_path):\n current_data = load_benchmark_data(current_results_file)\n\n # Generate report\n print(\"\\n=== Generating comparison report ===\")\n report = generate_report(current_data, base_data, args.current, args.base)\n\n # Output report\n if args.output:\n with open(args.output, \"w\") as f:\n f.write(report)\n print(f\"Report written to {args.output}\")\n else:\n print(\"\\n\" + \"=\" * 80)\n print(report)\n\n # Keep results if requested\n if args.keep_results:\n if base_data:\n with open(\"base_benchmark_results.json\", \"w\") as f:\n json.dump(base_data, f, indent=2)\n if current_data:\n with open(\"current_benchmark_results.json\", \"w\") as f:\n json.dump(current_data, f, indent=2)\n print(\"Benchmark result files saved.\")\n\n finally:\n # Restore original branch\n if original_branch:\n print(f\"\\nRestoring original branch: {original_branch}\")\n try:\n checkout_branch(original_branch, repo_path)\n except subprocess.CalledProcessError:\n print(f\"Warning: Could not restore original branch {original_branch}\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "bbot/scripts/docs.py", "content": "#!/usr/bin/env python3\n\nimport os\nimport re\nimport json\nimport yaml\nfrom pathlib import Path\n\nfrom bbot import Preset\nfrom bbot.core.modules import MODULE_LOADER\n\n\nDEFAULT_PRESET = Preset()\n\nos.environ[\"NO_COLOR\"] = \"1\"\nos.environ[\"BBOT_TABLE_FORMAT\"] = \"github\"\n\n\n# Make a regex pattern which will match any group of non-space characters that include a blacklisted character\nblacklist_chars = [\"<\", \">\"]\nblacklist_re = re.compile(r\"\\|([^|]*[\" + re.escape(\"\".join(blacklist_chars)) + r\"][^|]*)\\|\")\n\nbbot_code_dir = Path(__file__).parent.parent.parent\n\n\ndef gen_chord_data():\n # This function generates the dataset for the chord graph in the documentation\n # showing relationships between BBOT modules and their consumed/produced event types\n preloaded_mods = sorted(MODULE_LOADER.preloaded().items(), key=lambda x: x[0])\n\n entity_lookup_table = {}\n rels = []\n entities = {}\n entity_counter = 1\n\n def add_entity(entity, parent_id):\n if entity not in entity_lookup_table:\n nonlocal entity_counter\n e_id = entity_counter\n entity_counter += 1\n entity_lookup_table[entity] = e_id\n entity_lookup_table[e_id] = entity\n entities[e_id] = {\"id\": e_id, \"name\": entity, \"parent\": parent_id, \"consumes\": [], \"produces\": []}\n return entity_lookup_table[entity]\n\n # create entities for all the modules and event types\n for module, preloaded in preloaded_mods:\n watched = [e for e in preloaded[\"watched_events\"] if e != \"*\"]\n produced = [e for e in preloaded[\"produced_events\"] if e != \"*\"]\n if watched or produced:\n m_id = add_entity(module, 99999999)\n for event_type in watched:\n e_id = add_entity(event_type, 88888888)\n entities[m_id][\"consumes\"].append(e_id)\n entities[e_id][\"consumes\"].append(m_id)\n for event_type in produced:\n e_id = add_entity(event_type, 88888888)\n entities[m_id][\"produces\"].append(e_id)\n entities[e_id][\"produces\"].append(m_id)\n\n def add_rel(incoming, outgoing, t):\n if incoming == \"*\" or outgoing == \"*\":\n return\n i_id = entity_lookup_table[incoming]\n o_id = entity_lookup_table[outgoing]\n rels.append({\"source\": i_id, \"target\": o_id, \"type\": t})\n\n # create all the module <--> event type relationships\n for module, preloaded in preloaded_mods:\n for event_type in preloaded[\"watched_events\"]:\n add_rel(module, event_type, \"consumes\")\n for event_type in preloaded[\"produced_events\"]:\n add_rel(event_type, module, \"produces\")\n\n # write them to JSON files\n data_dir = Path(__file__).parent.parent.parent / \"docs\" / \"data\" / \"chord_graph\"\n data_dir.mkdir(parents=True, exist_ok=True)\n entity_file = data_dir / \"entities.json\"\n rels_file = data_dir / \"rels.json\"\n\n entities = [\n {\"id\": 77777777, \"name\": \"root\"},\n {\"id\": 99999999, \"name\": \"module\", \"parent\": 77777777},\n {\"id\": 88888888, \"name\": \"event_type\", \"parent\": 77777777},\n ] + sorted(entities.values(), key=lambda x: x[\"name\"])\n\n with open(entity_file, \"w\") as f:\n json.dump(entities, f, indent=4)\n\n with open(rels_file, \"w\") as f:\n json.dump(rels, f, indent=4)\n\n\ndef homedir_collapseuser(f):\n f = Path(f)\n home_dir = Path.home()\n if f.is_relative_to(home_dir):\n return Path(\"~\") / f.relative_to(home_dir)\n return f\n\n\ndef enclose_tags(text):\n # Use re.sub() to replace matched words with the same words enclosed in backticks\n result = blacklist_re.sub(r\"|`\\1`|\", text)\n return result\n\n\ndef find_replace_markdown(content, keyword, replace):\n begin_re = re.compile(r\"\", re.I)\n end_re = re.compile(r\"\", re.I)\n\n begin_match = begin_re.search(content)\n end_match = end_re.search(content)\n\n new_content = str(content)\n if begin_match and end_match:\n start_index = begin_match.span()[-1] + 1\n end_index = end_match.span()[0] - 1\n new_content = new_content[:start_index] + enclose_tags(replace) + new_content[end_index:]\n return new_content\n\n\ndef find_replace_file(file, keyword, replace):\n with open(file) as f:\n content = f.read()\n new_content = find_replace_markdown(content, keyword, replace)\n if new_content != content:\n if \"BBOT_TESTING\" not in os.environ:\n with open(file, \"w\") as f:\n f.write(new_content)\n\n\ndef update_docs():\n md_files = [p for p in bbot_code_dir.glob(\"**/*.md\") if p.is_file()]\n\n def update_md_files(keyword, s):\n for file in md_files:\n find_replace_file(file, keyword, s)\n\n def update_individual_module_options():\n regex = re.compile(\"BBOT MODULE OPTIONS ([A-Z_]+)\")\n for file in md_files:\n with open(file) as f:\n content = f.read()\n for match in regex.finditer(content):\n module_name = match.groups()[0].lower()\n bbot_module_options_table = DEFAULT_PRESET.module_loader.modules_options_table(modules=[module_name])\n find_replace_file(file, f\"BBOT MODULE OPTIONS {module_name.upper()}\", bbot_module_options_table)\n\n # Example commands\n bbot_example_commands = []\n for title, description, command in DEFAULT_PRESET.args.scan_examples:\n example = \"\"\n example += f\"**{title}:**\\n\\n\"\n # example += f\"{description}\\n\"\n example += f\"```bash\\n# {description}\\n{command}\\n```\"\n bbot_example_commands.append(example)\n bbot_example_commands = \"\\n\\n\".join(bbot_example_commands)\n assert len(bbot_example_commands.splitlines()) > 10\n update_md_files(\"BBOT EXAMPLE COMMANDS\", bbot_example_commands)\n\n # Help output\n bbot_help_output = DEFAULT_PRESET.args.parser.format_help().replace(\"docs.py\", \"bbot\")\n bbot_help_output = f\"```text\\n{bbot_help_output}\\n```\"\n assert len(bbot_help_output.splitlines()) > 50\n update_md_files(\"BBOT HELP OUTPUT\", bbot_help_output)\n\n # BBOT events\n bbot_event_table = DEFAULT_PRESET.module_loader.events_table()\n assert len(bbot_event_table.splitlines()) > 10\n update_md_files(\"BBOT EVENTS\", bbot_event_table)\n\n # BBOT modules\n bbot_module_table = DEFAULT_PRESET.module_loader.modules_table(include_author=True, include_created_date=True)\n assert len(bbot_module_table.splitlines()) > 50\n update_md_files(\"BBOT MODULES\", bbot_module_table)\n\n # BBOT output modules\n bbot_output_module_table = DEFAULT_PRESET.module_loader.modules_table(\n mod_type=\"output\", include_author=True, include_created_date=True\n )\n assert len(bbot_output_module_table.splitlines()) > 10\n update_md_files(\"BBOT OUTPUT MODULES\", bbot_output_module_table)\n\n # BBOT universal module options\n from bbot.scanner.preset.args import universal_module_options\n\n universal_module_options_table = \"\"\n for option, description in universal_module_options.items():\n universal_module_options_table += f\"**{option}**: {description}\\n\"\n update_md_files(\"BBOT UNIVERSAL MODULE OPTIONS\", universal_module_options_table)\n\n # BBOT module options\n bbot_module_options_table = DEFAULT_PRESET.module_loader.modules_options_table()\n assert len(bbot_module_options_table.splitlines()) > 100\n update_md_files(\"BBOT MODULE OPTIONS\", bbot_module_options_table)\n update_individual_module_options()\n\n # BBOT module flags\n bbot_module_flags_table = DEFAULT_PRESET.module_loader.flags_table()\n assert len(bbot_module_flags_table.splitlines()) > 10\n update_md_files(\"BBOT MODULE FLAGS\", bbot_module_flags_table)\n\n # BBOT presets\n bbot_presets_table = DEFAULT_PRESET.presets_table(include_modules=True)\n assert len(bbot_presets_table.splitlines()) > 5\n update_md_files(\"BBOT PRESETS\", bbot_presets_table)\n\n # BBOT presets\n for _, (loaded_preset, category, preset_path, original_filename) in DEFAULT_PRESET.all_presets.items():\n str_category = \"\" if not category else f\"/{category}\"\n filename = f\"~/.bbot/presets{str_category}/{original_filename.name}\"\n preset_yaml = f\"\"\"\n```yaml title={filename}\n{loaded_preset._yaml_str}\n```\n\"\"\"\n preset_yaml_expandable = f\"\"\"\n
\n{original_filename.name}\n\n```yaml\n{loaded_preset._yaml_str}\n```\n\n
\n\"\"\"\n update_md_files(f\"BBOT {loaded_preset.name.upper()} PRESET\", preset_yaml)\n update_md_files(f\"BBOT {loaded_preset.name.upper()} PRESET EXPANDABLE\", preset_yaml_expandable)\n\n content = []\n for _, (loaded_preset, category, preset_path, original_filename) in DEFAULT_PRESET.all_presets.items():\n yaml_str = loaded_preset._yaml_str\n indent = \" \" * 4\n yaml_str = f\"\\n{indent}\".join(yaml_str.splitlines())\n str_category = \"\" if not category else f\"/{category}\"\n filename = f\"~/.bbot/presets{str_category}/{original_filename.name}\"\n\n num_modules = len(loaded_preset.scan_modules)\n modules = \", \".join(sorted([f\"`{m}`\" for m in loaded_preset.scan_modules]))\n category = f\"Category: {category}\" if category else \"\"\n\n content.append(\n f\"\"\"## **{loaded_preset.name}**\n\n{loaded_preset.description}\n\n??? note \"`{original_filename.name}`\"\n ```yaml title=\"{filename}\"\n {yaml_str}\n ```\n\n{category}\n\nModules: [{num_modules:,}](\"{modules}\")\"\"\"\n )\n assert len(content) > 5\n update_md_files(\"BBOT PRESET YAML\", \"\\n\\n\".join(content))\n\n # Default config\n default_config_file = bbot_code_dir / \"bbot\" / \"defaults.yml\"\n with open(default_config_file) as f:\n default_config_yml = f.read()\n default_config_yml = f'```yaml title=\"defaults.yml\"\\n{default_config_yml}\\n```'\n assert len(default_config_yml.splitlines()) > 20\n update_md_files(\"BBOT DEFAULT CONFIG\", default_config_yml)\n\n # Table of Contents\n base_url = \"https://www.blacklanternsecurity.com/bbot/Stable\"\n\n def format_section(section_title, section_path):\n path = section_path.split(\"index.md\")[0]\n path = path.split(\".md\")[0]\n return f\"- [{section_title}]({base_url}/{path})\\n\"\n\n bbot_docs_toc = \"\"\n\n def update_toc(section, level=0):\n nonlocal bbot_docs_toc\n indent = \" \" * 4 * level\n if isinstance(section, dict):\n for section_title, subsections in section.items():\n if isinstance(subsections, str):\n bbot_docs_toc += f\"{indent}{format_section(section_title, subsections)}\"\n else:\n bbot_docs_toc += f\"{indent}- **{section_title}**\\n\"\n for subsection in subsections:\n update_toc(subsection, level=level + 1)\n\n mkdocs_yml_file = bbot_code_dir / \"mkdocs.yml\"\n yaml.SafeLoader.add_constructor(\n \"tag:yaml.org,2002:python/name:pymdownx.superfences.fence_code_format\", lambda x, y: {}\n )\n\n with open(mkdocs_yml_file, \"r\") as f:\n mkdocs_yaml = yaml.safe_load(f)\n nav = mkdocs_yaml[\"nav\"]\n for section in nav:\n update_toc(section)\n bbot_docs_toc = bbot_docs_toc.strip()\n # assert len(bbot_docs_toc.splitlines()) == 2\n update_md_files(\"BBOT DOCS TOC\", bbot_docs_toc)\n\n # generate data for chord graph\n gen_chord_data()\n\n\nupdate_docs()\n" }, { "path": "bbot/test/__init__.py", "content": "" }, { "path": "bbot/test/bbot_fixtures.py", "content": "import os # noqa\nimport sys # noqa\nimport pytest\nimport shutil # noqa\nimport asyncio # noqa\nimport logging\nimport tldextract\nimport pytest_httpserver\nfrom pathlib import Path\nfrom omegaconf import OmegaConf # noqa\n\nfrom werkzeug.wrappers import Request\n\nfrom bbot.errors import * # noqa: F401\nfrom bbot.core import CORE\nfrom bbot.scanner import Preset\nfrom bbot.core.helpers.misc import mkdir, rand_string\nfrom bbot.core.helpers.async_helpers import get_event_loop\n\n\nlog = logging.getLogger(\"bbot.test.fixtures\")\n\n\nbbot_test_dir = Path(\"/tmp/.bbot_test\")\nmkdir(bbot_test_dir)\n\n\nDEFAULT_PRESET = Preset()\n\navailable_modules = list(DEFAULT_PRESET.module_loader.configs(type=\"scan\"))\navailable_output_modules = list(DEFAULT_PRESET.module_loader.configs(type=\"output\"))\navailable_internal_modules = list(DEFAULT_PRESET.module_loader.configs(type=\"internal\"))\n\n\ndef tempwordlist(content):\n filename = bbot_test_dir / f\"{rand_string(8)}\"\n with open(filename, \"w\", errors=\"ignore\") as f:\n for c in content:\n line = f\"{c}\\n\"\n f.write(line)\n return filename\n\n\ndef tempapkfile():\n current_dir = Path(__file__).parent\n with open(current_dir / \"owasp_mastg.apk\", \"rb\") as f:\n apk_file = f.read()\n return apk_file\n\n\n@pytest.fixture\ndef clean_default_config(monkeypatch):\n clean_config = OmegaConf.merge(\n CORE.files_config.get_default_config(), {\"modules\": DEFAULT_PRESET.module_loader.configs()}\n )\n with monkeypatch.context() as m:\n m.setattr(\"bbot.core.core.DEFAULT_CONFIG\", clean_config)\n yield\n\n\nclass SubstringRequestMatcher(pytest_httpserver.httpserver.RequestMatcher):\n def match_data(self, request: Request) -> bool:\n if self.data is None:\n return True\n return self.data in request.data\n\n\npytest_httpserver.httpserver.RequestMatcher = SubstringRequestMatcher\n\n# silence pytest_httpserver\nlog = logging.getLogger(\"werkzeug\")\nlog.setLevel(logging.CRITICAL)\n\ntldextract.extract(\"www.evilcorp.com\")\n\n\n@pytest.fixture\ndef bbot_scanner():\n from bbot.scanner import Scanner\n\n return Scanner\n\n\n@pytest.fixture\ndef scan():\n from bbot.scanner import Scanner\n\n bbot_scan = Scanner(\"127.0.0.1\", modules=[\"ipneighbor\"])\n yield bbot_scan\n\n loop = get_event_loop()\n loop.run_until_complete(bbot_scan._cleanup())\n\n\n@pytest.fixture\ndef helpers(scan):\n return scan.helpers\n\n\nhttpx_response = {\n \"timestamp\": \"2022-11-14T12:14:27.377566416-05:00\",\n \"hash\": {\n \"body_md5\": \"84238dfc8092e5d9c0dac8ef93371a07\",\n \"body_mmh3\": \"-1139337416\",\n \"body_sha256\": \"ea8fac7c65fb589b0d53560f5251f74f9e9b243478dcb6b3ea79b5e36449c8d9\",\n \"body_simhash\": \"9899951357530060719\",\n \"header_md5\": \"6e483c85c3b9b96f0e33d84237ca651e\",\n \"header_mmh3\": \"-957156428\",\n \"header_sha256\": \"5a809d8a53aded843179237365bb6dd069fba75ff8603ac2f6dc6c05d6bf0e76\",\n \"header_simhash\": \"15614709017155972941\",\n },\n \"port\": \"80\",\n \"url\": \"http://example.com:80\",\n \"input\": \"http://example.com:80\",\n \"title\": \"Example Domain\",\n \"scheme\": \"http\",\n \"webserver\": \"ECS (agb/A445)\",\n \"body\": '\\n\\n\\n Example Domain\\n\\n \\n \\n \\n \\n\\n\\n\\n\\n\\n\\n',\n \"content_type\": \"text/html\",\n \"method\": \"GET\",\n \"host\": \"93.184.216.34\",\n \"path\": \"/\",\n \"header\": {\n \"age\": \"526111\",\n \"cache_control\": \"max-age=604800\",\n \"content_type\": \"text/html; charset=UTF-8\",\n \"date\": \"Mon, 14 Nov 2022 17:14:27 GMT\",\n \"etag\": '\"3147526947+ident+gzip\"',\n \"expires\": \"Mon, 21 Nov 2022 17:14:27 GMT\",\n \"last_modified\": \"Thu, 17 Oct 2019 07:18:26 GMT\",\n \"server\": \"ECS (agb/A445)\",\n \"vary\": \"Accept-Encoding\",\n \"x_cache\": \"HIT\",\n },\n \"raw_header\": 'HTTP/1.1 200 OK\\r\\nConnection: close\\r\\nAge: 526111\\r\\nCache-Control: max-age=604800\\r\\nContent-Type: text/html; charset=UTF-8\\r\\nDate: Mon, 14 Nov 2022 17:14:27 GMT\\r\\nEtag: \"3147526947+ident+gzip\"\\r\\nExpires: Mon, 21 Nov 2022 17:14:27 GMT\\r\\nLast-Modified: Thu, 17 Oct 2019 07:18:26 GMT\\r\\nServer: ECS (agb/A445)\\r\\nVary: Accept-Encoding\\r\\nX-Cache: HIT\\r\\n\\r\\n',\n \"request\": \"GET / HTTP/1.1\\r\\nHost: example.com\\r\\nUser-Agent: Mozilla/5.0 (SymbianOS/9.1; U; de) AppleWebKit/413 (KHTML, like Gecko) Safari/413\\r\\nAccept-Charset: utf-8\\r\\nAccept-Encoding: gzip\\r\\n\\r\\n\",\n \"time\": \"112.128324ms\",\n \"a\": [\"93.184.216.34\", \"2606:2800:220:1:248:1893:25c8:1946\"],\n \"words\": 298,\n \"lines\": 47,\n \"status_code\": 200,\n \"content_length\": 1256,\n \"failed\": False,\n}\n\n\n@pytest.fixture\ndef events(scan):\n class bbot_events:\n localhost = scan.make_event(\"127.0.0.1\", parent=scan.root_event)\n ipv4 = scan.make_event(\"8.8.8.8\", parent=scan.root_event)\n netv4 = scan.make_event(\"8.8.8.8/30\", parent=scan.root_event)\n ipv6 = scan.make_event(\"2001:4860:4860::8888\", parent=scan.root_event)\n netv6 = scan.make_event(\"2001:4860:4860::8888/126\", parent=scan.root_event)\n domain = scan.make_event(\"publicAPIs.org\", parent=scan.root_event)\n subdomain = scan.make_event(\"api.publicAPIs.org\", parent=scan.root_event)\n email = scan.make_event(\"bob@evilcorp.co.uk\", \"EMAIL_ADDRESS\", parent=scan.root_event)\n open_port = scan.make_event(\"api.publicAPIs.org:443\", parent=scan.root_event)\n protocol = scan.make_event(\n {\"host\": \"api.publicAPIs.org\", \"port\": 443, \"protocol\": \"HTTP\"}, \"PROTOCOL\", parent=scan.root_event\n )\n ipv4_open_port = scan.make_event(\"8.8.8.8:443\", parent=scan.root_event)\n ipv6_open_port = scan.make_event(\"[2001:4860:4860::8888]:443\", \"OPEN_TCP_PORT\", parent=scan.root_event)\n url_unverified = scan.make_event(\"https://api.publicAPIs.org:443/hellofriend\", parent=scan.root_event)\n ipv4_url_unverified = scan.make_event(\"https://8.8.8.8:443/hellofriend\", parent=scan.root_event)\n ipv6_url_unverified = scan.make_event(\"https://[2001:4860:4860::8888]:443/hellofriend\", parent=scan.root_event)\n url = scan.make_event(\n \"https://api.publicAPIs.org:443/hellofriend\", \"URL\", tags=[\"status-200\"], parent=scan.root_event\n )\n ipv4_url = scan.make_event(\n \"https://8.8.8.8:443/hellofriend\", \"URL\", tags=[\"status-200\"], parent=scan.root_event\n )\n ipv6_url = scan.make_event(\n \"https://[2001:4860:4860::8888]:443/hellofriend\", \"URL\", tags=[\"status-200\"], parent=scan.root_event\n )\n url_hint = scan.make_event(\"https://api.publicAPIs.org:443/hello.ash\", \"URL_HINT\", parent=url)\n vulnerability = scan.make_event(\n {\"host\": \"evilcorp.com\", \"severity\": \"INFO\", \"description\": \"asdf\"},\n \"VULNERABILITY\",\n parent=scan.root_event,\n )\n finding = scan.make_event({\"host\": \"evilcorp.com\", \"description\": \"asdf\"}, \"FINDING\", parent=scan.root_event)\n vhost = scan.make_event({\"host\": \"evilcorp.com\", \"vhost\": \"www.evilcorp.com\"}, \"VHOST\", parent=scan.root_event)\n http_response = scan.make_event(httpx_response, \"HTTP_RESPONSE\", parent=scan.root_event)\n storage_bucket = scan.make_event(\n {\"name\": \"storage\", \"url\": \"https://storage.blob.core.windows.net\"},\n \"STORAGE_BUCKET\",\n parent=scan.root_event,\n )\n emoji = scan.make_event(\"💩\", \"WHERE_IS_YOUR_GOD_NOW\", parent=scan.root_event)\n\n bbot_events.all = [ # noqa: F841\n bbot_events.localhost,\n bbot_events.ipv4,\n bbot_events.netv4,\n bbot_events.ipv6,\n bbot_events.netv6,\n bbot_events.domain,\n bbot_events.subdomain,\n bbot_events.email,\n bbot_events.open_port,\n bbot_events.protocol,\n bbot_events.ipv4_open_port,\n bbot_events.ipv6_open_port,\n bbot_events.url_unverified,\n bbot_events.ipv4_url_unverified,\n bbot_events.ipv6_url_unverified,\n bbot_events.url,\n bbot_events.ipv4_url,\n bbot_events.ipv6_url,\n bbot_events.url_hint,\n bbot_events.vulnerability,\n bbot_events.finding,\n bbot_events.vhost,\n bbot_events.http_response,\n bbot_events.storage_bucket,\n bbot_events.emoji,\n ]\n\n for e in bbot_events.all:\n e.scope_distance = 0\n\n return bbot_events\n\n\n# @pytest.fixture(scope=\"session\", autouse=True)\n# def install_all_python_deps():\n# deps_pip = set()\n# for module in DEFAULT_PRESET.module_loader.preloaded().values():\n# deps_pip.update(set(module.get(\"deps\", {}).get(\"pip\", [])))\n\n# constraint_file = tempwordlist(get_python_constraints())\n\n# subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"--constraint\", constraint_file] + list(deps_pip))\n" }, { "path": "bbot/test/benchmarks/__init__.py", "content": "# Benchmark tests for BBOT performance monitoring\n# These tests measure performance of critical code paths\n" }, { "path": "bbot/test/benchmarks/test_bloom_filter_benchmarks.py", "content": "import pytest\nimport string\nimport random\nfrom bbot.scanner import Scanner\n\n\nclass TestBloomFilterBenchmarks:\n \"\"\"\n Benchmark tests for Bloom Filter operations.\n\n These tests measure the performance of bloom filter operations which are\n critical for DNS brute-forcing efficiency in BBOT.\n \"\"\"\n\n def setup_method(self):\n \"\"\"Setup common test data\"\"\"\n self.scan = Scanner()\n\n # Generate test data of different sizes\n self.items_small = self._generate_random_strings(1000) # 1K items\n self.items_medium = self._generate_random_strings(10000) # 10K items\n\n def _generate_random_strings(self, n, length=10):\n \"\"\"Generate a list of n random strings.\"\"\"\n # Slightly longer strings for testing performance difference\n length = length + 2 # Make strings 2 chars longer\n return [\"\".join(random.choices(string.ascii_letters + string.digits, k=length)) for _ in range(n)]\n\n @pytest.mark.benchmark(group=\"bloom_filter_operations\")\n def test_bloom_filter_dns_mutation_tracking_performance(self, benchmark):\n \"\"\"Benchmark comprehensive bloom filter operations (add, check, mixed) for DNS brute-forcing\"\"\"\n\n def comprehensive_bloom_operations():\n bloom_filter = self.scan.helpers.bloom_filter(size=8000000) # 8M bits\n\n # Phase 1: Add operations (simulating storing tried DNS mutations)\n for item in self.items_small:\n bloom_filter.add(item)\n\n # Phase 2: Check operations (simulating lookup of existing mutations)\n found_count = 0\n for item in self.items_small:\n if item in bloom_filter:\n found_count += 1\n\n # Phase 3: Mixed operations (realistic DNS brute-force simulation)\n # Add new items while checking existing ones\n for i, item in enumerate(self.items_medium[:500]): # Smaller subset for mixed ops\n bloom_filter.add(item)\n # Every few additions, check some existing items\n if i % 10 == 0:\n for check_item in self.items_small[i : i + 5]:\n if check_item in bloom_filter:\n found_count += 1\n\n return {\n \"items_added\": len(self.items_small) + 500,\n \"items_checked\": found_count,\n \"bloom_size\": bloom_filter.size,\n }\n\n result = benchmark(comprehensive_bloom_operations)\n assert result[\"items_added\"] > 1000\n assert result[\"items_checked\"] > 0\n\n @pytest.mark.benchmark(group=\"bloom_filter_scalability\")\n def test_bloom_filter_large_scale_dns_brute_force(self, benchmark):\n \"\"\"Benchmark bloom filter performance with large-scale DNS brute-force simulation\"\"\"\n\n def large_scale_simulation():\n bloom_filter = self.scan.helpers.bloom_filter(size=8000000) # 8M bits\n\n # Simulate a large DNS brute-force session\n mutations_tried = 0\n duplicate_attempts = 0\n\n # Add all medium dataset (simulating 10K DNS mutations)\n for item in self.items_medium:\n bloom_filter.add(item)\n mutations_tried += 1\n\n # Simulate checking for duplicates during brute-force\n for item in self.items_medium[:2000]: # Check subset for duplicates\n if item in bloom_filter:\n duplicate_attempts += 1\n\n # Simulate adding more mutations with duplicate checking\n for item in self.items_small:\n if item not in bloom_filter: # Only add if not already tried\n bloom_filter.add(item)\n mutations_tried += 1\n else:\n duplicate_attempts += 1\n\n return {\n \"total_mutations_tried\": mutations_tried,\n \"duplicates_avoided\": duplicate_attempts,\n \"efficiency_ratio\": mutations_tried / (mutations_tried + duplicate_attempts)\n if duplicate_attempts > 0\n else 1.0,\n }\n\n result = benchmark(large_scale_simulation)\n assert result[\"total_mutations_tried\"] > 10000\n assert result[\"efficiency_ratio\"] > 0\n" }, { "path": "bbot/test/benchmarks/test_closest_match_benchmarks.py", "content": "import pytest\nimport random\nfrom bbot.core.helpers.misc import closest_match\n\n\nclass TestClosestMatchBenchmarks:\n \"\"\"\n Benchmark tests for closest_match operations.\n\n This function is critical for BBOT's DNS brute forcing, where it finds the best\n matching parent event among thousands of choices. Performance here directly impacts\n scan throughput and DNS mutation efficiency.\n \"\"\"\n\n def setup_method(self):\n \"\"\"Setup common test data\"\"\"\n # Set deterministic seed for consistent benchmark results\n random.seed(42) # Fixed seed for reproducible results\n\n # Generate test data for benchmarks\n self.large_closest_match_choices = self._generate_large_closest_match_choices()\n self.realistic_closest_match_choices = self._generate_realistic_closest_match_choices()\n\n def _generate_large_closest_match_choices(self):\n \"\"\"Generate large closest match dataset (stress test with many parent events)\"\"\"\n choices = []\n for i in range(10000):\n # Generate realistic domain names with more variety\n tld = random.choice([\"com\", \"net\", \"org\", \"io\", \"co\", \"dev\"])\n domain = f\"subdomain{i}.example{i % 100}.{tld}\"\n choices.append(domain)\n return choices\n\n def _generate_realistic_closest_match_choices(self):\n \"\"\"Generate realistic closest match parent event choices (like actual BBOT usage)\"\"\"\n choices = []\n\n # Common TLDs\n tlds = [\"com\", \"net\", \"org\", \"io\", \"co\", \"dev\", \"test\", \"local\"]\n\n # Generate parent domains with realistic patterns\n for i in range(5000):\n # Base domain patterns\n if i % 10 == 0:\n # Simple domains\n domain = f\"example{i}.{random.choice(tlds)}\"\n elif i % 5 == 0:\n # Multi-level domains\n domain = f\"sub{i}.example{i}.{random.choice(tlds)}\"\n else:\n # Complex domains\n domain = f\"level1{i}.level2{i}.example{i}.{random.choice(tlds)}\"\n\n choices.append(domain)\n\n return choices\n\n @pytest.mark.benchmark(group=\"closest_match\")\n def test_large_closest_match_lookup(self, benchmark):\n \"\"\"Benchmark closest_match with large closest match workload (many parent events)\"\"\"\n\n def find_large_closest_match():\n return closest_match(\"subdomain5678.example50.com\", self.large_closest_match_choices)\n\n result = benchmark.pedantic(find_large_closest_match, iterations=50, rounds=10)\n assert result is not None\n\n @pytest.mark.benchmark(group=\"closest_match\")\n def test_realistic_closest_match_workload(self, benchmark):\n \"\"\"Benchmark closest_match with realistic BBOT closest match parent event choices\"\"\"\n\n def find_realistic_closest_match():\n return closest_match(\"subdomain123.example5.com\", self.realistic_closest_match_choices)\n\n result = benchmark.pedantic(find_realistic_closest_match, iterations=50, rounds=10)\n assert result is not None\n" }, { "path": "bbot/test/benchmarks/test_event_validation_benchmarks.py", "content": "import pytest\nimport random\nimport string\nfrom bbot.scanner import Scanner\nfrom bbot.core.event.base import make_event\n\n\nclass TestEventValidationBenchmarks:\n def setup_method(self):\n \"\"\"Setup minimal scanner configuration for benchmarking event validation\"\"\"\n # Set deterministic random seed for reproducible benchmarks\n random.seed(42)\n\n # Create a minimal scanner with no modules to isolate event validation performance\n self.scanner_config = {\n \"modules\": None, # No modules to avoid overhead\n \"output_modules\": None, # No output modules\n \"dns\": {\"disable\": True}, # Disable DNS to avoid network calls\n \"web\": {\"http_timeout\": 1}, # Minimal timeouts\n }\n\n def _generate_diverse_targets(self, count=1000):\n \"\"\"Generate a diverse set of targets that will trigger different event type auto-detection\"\"\"\n # Use deterministic random state for reproducible target generation\n rng = random.Random(42)\n targets = []\n\n # DNS Names (various formats)\n subdomains = [\"www\", \"api\", \"mail\", \"ftp\", \"admin\", \"test\", \"dev\", \"staging\", \"blog\"]\n tlds = [\"com\", \"org\", \"net\", \"io\", \"co.uk\", \"de\", \"fr\", \"jp\"]\n\n for _ in range(count // 10):\n # Standard domains\n targets.append(\n f\"{rng.choice(subdomains)}.{rng.choice(['example', 'test', 'evilcorp'])}.{rng.choice(tlds)}\"\n )\n # Bare domains\n targets.append(f\"{rng.choice(['example', 'test', 'company'])}.{rng.choice(tlds)}\")\n\n # IP Addresses (IPv4 and IPv6)\n for _ in range(count // 15):\n # IPv4\n targets.append(f\"{rng.randint(1, 254)}.{rng.randint(1, 254)}.{rng.randint(1, 254)}.{rng.randint(1, 254)}\")\n # IPv6\n targets.append(f\"2001:db8::{rng.randint(1, 9999):x}:{rng.randint(1, 9999):x}\")\n\n # IP Ranges\n for _ in range(count // 20):\n targets.append(f\"192.168.{rng.randint(1, 254)}.0/24\")\n targets.append(f\"10.0.{rng.randint(1, 254)}.0/24\")\n\n # URLs (only supported schemes: http, https)\n url_schemes = [\"http\", \"https\"] # Only schemes supported by BBOT auto-detection\n url_paths = [\"\", \"/\", \"/admin\", \"/api/v1\", \"/login.php\", \"/index.html\"]\n for _ in range(count // 8):\n scheme = rng.choice(url_schemes)\n domain = f\"{rng.choice(subdomains)}.example.{rng.choice(tlds)}\"\n path = rng.choice(url_paths)\n port = rng.choice([\"\", \":8080\", \":443\", \":80\", \":8443\"])\n targets.append(f\"{scheme}://{domain}{port}{path}\")\n\n # Open Ports\n ports = [80, 443, 22, 21, 25, 53, 110, 143, 993, 995, 8080, 8443, 3389]\n for _ in range(count // 12):\n domain = f\"example.{rng.choice(tlds)}\"\n port = rng.choice(ports)\n targets.append(f\"{domain}:{port}\")\n # IPv4 with port\n ip = f\"{rng.randint(1, 254)}.{rng.randint(1, 254)}.{rng.randint(1, 254)}.{rng.randint(1, 254)}\"\n targets.append(f\"{ip}:{port}\")\n\n # Email Addresses\n email_domains = [\"example.com\", \"test.org\", \"company.net\"]\n email_users = [\"admin\", \"test\", \"info\", \"contact\", \"support\", \"sales\"]\n for _ in range(count // 15):\n user = rng.choice(email_users)\n domain = rng.choice(email_domains)\n targets.append(f\"{user}@{domain}\")\n # Plus addressing\n targets.append(f\"{user}+{rng.randint(1, 999)}@{domain}\")\n\n # Mixed/Edge cases that should trigger auto-detection logic\n edge_cases = [\n # Localhost variants\n \"localhost\",\n \"127.0.0.1\",\n \"::1\",\n # Punycode domains\n \"xn--e1afmkfd.xn--p1ai\",\n \"xn--fiqs8s.xn--0zwm56d\",\n # Long domains (shortened to avoid issues)\n \"very-long-subdomain-name-for-testing.test.com\",\n # IP with ports\n \"192.168.1.1\",\n \"10.0.0.1:80\",\n # URLs with parameters\n \"https://example.com/search?q=test&limit=10\",\n \"http://api.example.com:8080/v1/users?format=json\",\n # More standard domains for better compatibility\n \"api.test.com\",\n \"mail.example.org\",\n \"secure.company.net\",\n ]\n targets.extend(edge_cases)\n\n # Fill remainder with random variations\n remaining = count - len(targets)\n if remaining > 0:\n for _ in range(remaining):\n choice = rng.randint(1, 4)\n if choice == 1:\n # Random domain\n targets.append(f\"{''.join(rng.choices(string.ascii_lowercase, k=8))}.com\")\n elif choice == 2:\n # Random IP\n targets.append(\n f\"{rng.randint(1, 254)}.{rng.randint(1, 254)}.{rng.randint(1, 254)}.{rng.randint(1, 254)}\"\n )\n elif choice == 3:\n # Random URL\n targets.append(f\"https://{''.join(rng.choices(string.ascii_lowercase, k=8))}.com/path\")\n else:\n # Random email\n targets.append(f\"{''.join(rng.choices(string.ascii_lowercase, k=8))}@example.com\")\n\n # Ensure we have exactly the requested count by removing duplicates and filling as needed\n unique_targets = list(set(targets))\n\n # If we have too few unique targets, generate more\n while len(unique_targets) < count:\n additional_target = f\"filler{len(unique_targets)}.example.com\"\n if additional_target not in unique_targets:\n unique_targets.append(additional_target)\n\n # Return exactly the requested number of unique targets\n return unique_targets[:count]\n\n def _generate_diverse_event_data(self, count=1000):\n \"\"\"Generate diverse event data that will trigger different auto-detection paths in make_event\"\"\"\n # Use deterministic random state for reproducible data generation\n rng = random.Random(42)\n event_data = []\n\n # DNS Names (various formats)\n subdomains = [\"www\", \"api\", \"mail\", \"ftp\", \"admin\", \"test\", \"dev\", \"staging\", \"blog\"]\n tlds = [\"com\", \"org\", \"net\", \"io\", \"co.uk\", \"de\", \"fr\", \"jp\"]\n\n for _ in range(count // 10):\n # Standard domains\n event_data.append(\n f\"{rng.choice(subdomains)}.{rng.choice(['example', 'test', 'evilcorp'])}.{rng.choice(tlds)}\"\n )\n # Bare domains\n event_data.append(f\"{rng.choice(['example', 'test', 'company'])}.{rng.choice(tlds)}\")\n\n # IP Addresses (IPv4 and IPv6)\n for _ in range(count // 15):\n # IPv4\n event_data.append(\n f\"{rng.randint(1, 254)}.{rng.randint(1, 254)}.{rng.randint(1, 254)}.{rng.randint(1, 254)}\"\n )\n # IPv6\n event_data.append(f\"2001:db8::{rng.randint(1, 9999):x}:{rng.randint(1, 9999):x}\")\n\n # IP Ranges\n for _ in range(count // 20):\n event_data.append(f\"192.168.{rng.randint(1, 254)}.0/24\")\n event_data.append(f\"10.0.{rng.randint(1, 254)}.0/24\")\n\n # URLs (HTTP/HTTPS)\n url_schemes = [\"http\", \"https\"]\n url_paths = [\"\", \"/\", \"/admin\", \"/api/v1\", \"/login.php\", \"/index.html\"]\n for _ in range(count // 8):\n scheme = rng.choice(url_schemes)\n domain = f\"{rng.choice(subdomains)}.example.{rng.choice(tlds)}\"\n path = rng.choice(url_paths)\n port = rng.choice([\"\", \":8080\", \":443\", \":80\", \":8443\"])\n event_data.append(f\"{scheme}://{domain}{port}{path}\")\n\n # Open Ports\n ports = [80, 443, 22, 21, 25, 53, 110, 143, 993, 995, 8080, 8443, 3389]\n for _ in range(count // 12):\n domain = f\"example.{rng.choice(tlds)}\"\n port = rng.choice(ports)\n event_data.append(f\"{domain}:{port}\")\n # IPv4 with port\n ip = f\"{rng.randint(1, 254)}.{rng.randint(1, 254)}.{rng.randint(1, 254)}.{rng.randint(1, 254)}\"\n event_data.append(f\"{ip}:{port}\")\n\n # Email Addresses\n email_domains = [\"example.com\", \"test.org\", \"company.net\"]\n email_users = [\"admin\", \"test\", \"info\", \"contact\", \"support\", \"sales\"]\n for _ in range(count // 15):\n user = rng.choice(email_users)\n domain = rng.choice(email_domains)\n event_data.append(f\"{user}@{domain}\")\n # Plus addressing\n event_data.append(f\"{user}+{rng.randint(1, 999)}@{domain}\")\n\n # Mixed/Edge cases that test auto-detection logic\n edge_cases = [\n # Localhost variants\n \"localhost\",\n \"127.0.0.1\",\n \"::1\",\n # Punycode domains\n \"xn--e1afmkfd.xn--p1ai\",\n \"xn--fiqs8s.xn--0zwm56d\",\n # Long domains\n \"very-long-subdomain-name-for-testing.test.com\",\n # IP with ports\n \"192.168.1.1\",\n \"10.0.0.1:80\",\n # URLs with parameters\n \"https://example.com/search?q=test&limit=10\",\n \"http://api.example.com:8080/v1/users?format=json\",\n # Standard domains for better compatibility\n \"api.test.com\",\n \"mail.example.org\",\n \"secure.company.net\",\n ]\n event_data.extend(edge_cases)\n\n # Fill remainder with random variations\n remaining = count - len(event_data)\n if remaining > 0:\n for _ in range(remaining):\n choice = rng.randint(1, 4)\n if choice == 1:\n # Random domain\n event_data.append(f\"{''.join(rng.choices(string.ascii_lowercase, k=8))}.com\")\n elif choice == 2:\n # Random IP\n event_data.append(\n f\"{rng.randint(1, 254)}.{rng.randint(1, 254)}.{rng.randint(1, 254)}.{rng.randint(1, 254)}\"\n )\n elif choice == 3:\n # Random URL\n event_data.append(f\"https://{''.join(rng.choices(string.ascii_lowercase, k=8))}.com/path\")\n else:\n # Random email\n event_data.append(f\"{''.join(rng.choices(string.ascii_lowercase, k=8))}@example.com\")\n\n # Ensure we have exactly the requested count by removing duplicates and filling as needed\n unique_data = list(set(event_data))\n\n # If we have too few unique entries, generate more\n while len(unique_data) < count:\n additional_data = f\"filler{len(unique_data)}.example.com\"\n if additional_data not in unique_data:\n unique_data.append(additional_data)\n\n # Return exactly the requested number of unique data items\n return unique_data[:count]\n\n @pytest.mark.benchmark(group=\"event_validation_scan_startup_small\")\n def test_event_validation_full_scan_startup_small_batch(self, benchmark):\n \"\"\"Benchmark full scan startup event validation with small batch (100 targets) for quick iteration\"\"\"\n targets = self._generate_diverse_targets(100)\n\n def validate_event_batch():\n scan = Scanner(*targets, config=self.scanner_config)\n # Count successful event creations and types detected\n event_counts = {}\n total_events = 0\n\n for event_seed in scan.target.seeds:\n event_type = event_seed.type\n event_counts[event_type] = event_counts.get(event_type, 0) + 1\n total_events += 1\n\n return {\n \"total_events_processed\": total_events,\n \"unique_event_types\": len(event_counts),\n \"event_type_breakdown\": event_counts,\n \"targets_input\": len(targets),\n }\n\n result = benchmark(validate_event_batch)\n assert result[\"total_events_processed\"] == result[\"targets_input\"] # Should process ALL targets\n assert result[\"unique_event_types\"] >= 3 # Should detect at least DNS_NAME, IP_ADDRESS, URL\n\n @pytest.mark.benchmark(group=\"event_validation_scan_startup_large\")\n def test_event_validation_full_scan_startup_large_batch(self, benchmark):\n \"\"\"Benchmark full scan startup event validation with large batch (1000 targets) for comprehensive testing\"\"\"\n targets = self._generate_diverse_targets(1000)\n\n def validate_large_batch():\n scan = Scanner(*targets, config=self.scanner_config)\n\n # Comprehensive analysis of validation pipeline performance\n validation_metrics = {\n \"targets_input\": len(targets),\n \"events_created\": 0,\n \"validation_errors\": 0,\n \"auto_detection_success\": 0,\n \"type_distribution\": {},\n \"processing_efficiency\": 0.0,\n }\n\n try:\n for event_seed in scan.target.seeds:\n validation_metrics[\"events_created\"] += 1\n event_type = event_seed.type\n\n if event_type not in validation_metrics[\"type_distribution\"]:\n validation_metrics[\"type_distribution\"][event_type] = 0\n validation_metrics[\"type_distribution\"][event_type] += 1\n\n # If we got a valid event type, auto-detection succeeded\n if event_type and event_type != \"UNKNOWN\":\n validation_metrics[\"auto_detection_success\"] += 1\n\n except Exception:\n validation_metrics[\"validation_errors\"] += 1\n\n # Calculate efficiency ratio\n if validation_metrics[\"targets_input\"] > 0:\n validation_metrics[\"processing_efficiency\"] = (\n validation_metrics[\"events_created\"] / validation_metrics[\"targets_input\"]\n )\n\n return validation_metrics\n\n result = benchmark(validate_large_batch)\n assert result[\"events_created\"] == result[\"targets_input\"] # Should process ALL targets successfully\n assert result[\"processing_efficiency\"] == 1.0 # 100% success rate\n assert len(result[\"type_distribution\"]) >= 5 # Should detect multiple event types\n\n @pytest.mark.benchmark(group=\"make_event_small\")\n def test_make_event_autodetection_small(self, benchmark):\n \"\"\"Benchmark make_event with auto-detection for small batch (100 items)\"\"\"\n event_data = self._generate_diverse_event_data(100)\n\n def create_events_with_autodetection():\n events_created = []\n type_distribution = {}\n validation_errors = 0\n\n for data in event_data:\n try:\n # Test auto-detection by not providing event_type\n event = make_event(data, dummy=True)\n events_created.append(event)\n\n event_type = event.type\n type_distribution[event_type] = type_distribution.get(event_type, 0) + 1\n\n except Exception:\n validation_errors += 1\n\n return {\n \"events_created\": len(events_created),\n \"type_distribution\": type_distribution,\n \"validation_errors\": validation_errors,\n \"autodetection_success_rate\": len(events_created) / len(event_data) if event_data else 0,\n }\n\n result = benchmark.pedantic(create_events_with_autodetection, iterations=50, rounds=10)\n assert result[\"events_created\"] == len(event_data) # Should create events for all data\n assert result[\"validation_errors\"] == 0 # Should have no validation errors\n assert len(result[\"type_distribution\"]) >= 3 # Should detect multiple event types\n assert result[\"autodetection_success_rate\"] == 1.0 # 100% success rate\n\n @pytest.mark.benchmark(group=\"make_event_large\")\n def test_make_event_autodetection_large(self, benchmark):\n \"\"\"Benchmark make_event with auto-detection for large batch (1000 items)\"\"\"\n event_data = self._generate_diverse_event_data(1000)\n\n def create_large_event_batch():\n performance_metrics = {\n \"total_processed\": len(event_data),\n \"events_created\": 0,\n \"autodetection_failures\": 0,\n \"type_distribution\": {},\n \"processing_efficiency\": 0.0,\n }\n\n for data in event_data:\n try:\n # Use dummy=True for performance (no scan/parent validation)\n event = make_event(data, dummy=True)\n performance_metrics[\"events_created\"] += 1\n\n event_type = event.type\n if event_type not in performance_metrics[\"type_distribution\"]:\n performance_metrics[\"type_distribution\"][event_type] = 0\n performance_metrics[\"type_distribution\"][event_type] += 1\n\n except Exception:\n performance_metrics[\"autodetection_failures\"] += 1\n\n # Calculate efficiency ratio\n performance_metrics[\"processing_efficiency\"] = (\n performance_metrics[\"events_created\"] / performance_metrics[\"total_processed\"]\n )\n\n return performance_metrics\n\n result = benchmark.pedantic(create_large_event_batch, iterations=50, rounds=10)\n assert result[\"events_created\"] == result[\"total_processed\"] # Should process all successfully\n assert result[\"autodetection_failures\"] == 0 # Should have no failures\n assert result[\"processing_efficiency\"] == 1.0 # 100% efficiency\n assert len(result[\"type_distribution\"]) >= 5 # Should detect multiple event types\n\n @pytest.mark.benchmark(group=\"make_event_explicit_types\")\n def test_make_event_explicit_types(self, benchmark):\n \"\"\"Benchmark make_event when event types are explicitly provided (no auto-detection)\"\"\"\n # Create data with explicit type mappings to bypass auto-detection\n test_cases = [\n (\"example.com\", \"DNS_NAME\"),\n (\"192.168.1.1\", \"IP_ADDRESS\"),\n (\"https://example.com\", \"URL\"),\n (\"admin@example.com\", \"EMAIL_ADDRESS\"),\n (\"example.com:80\", \"OPEN_TCP_PORT\"),\n ] * 20 # 100 total cases\n\n def create_events_explicit_types():\n events_created = []\n type_distribution = {}\n\n for data, event_type in test_cases:\n # Explicitly provide event_type to skip auto-detection\n event = make_event(data, event_type=event_type, dummy=True)\n events_created.append(event)\n\n type_distribution[event_type] = type_distribution.get(event_type, 0) + 1\n\n return {\n \"events_created\": len(events_created),\n \"type_distribution\": type_distribution,\n \"bypass_autodetection\": True,\n }\n\n result = benchmark.pedantic(create_events_explicit_types, iterations=50, rounds=10)\n assert result[\"events_created\"] == len(test_cases) # Should create all events\n assert result[\"bypass_autodetection\"] # Confirms we bypassed auto-detection\n assert len(result[\"type_distribution\"]) == 5 # Should have exactly 5 types\n" }, { "path": "bbot/test/benchmarks/test_excavate_benchmarks.py", "content": "import pytest\nimport asyncio\nfrom bbot.scanner import Scanner\n\n\nclass TestExcavateDirectBenchmarks:\n \"\"\"\n Direct benchmark tests for Excavate module operations.\n\n These tests measure the performance of excavate's core YARA processing\n by calling the excavate.search() method directly with specific text sizes\n in both single-threaded and parallel asyncio tasks to test the GIL sidestep feature of YARA.\n \"\"\"\n\n # Number of text segments per test\n TEXT_SEGMENTS_COUNT = 100\n\n # Prescribed sizes for deterministic benchmarking (in bytes)\n SMALL_SIZE = 4096 # 4KB\n LARGE_SIZE = 5242880 # 5MB\n\n def _generate_text_segments(self, target_size, count):\n \"\"\"Generate a list of text segments of the specified size\"\"\"\n segments = []\n\n for i in range(count):\n # Generate realistic content that excavate can work with\n base_content = self._generate_realistic_content(i)\n\n # Pad to the exact target size with deterministic content\n remaining_size = target_size - len(base_content)\n if remaining_size > 0:\n # Use deterministic padding pattern\n padding_pattern = \"Lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. \"\n padding_repeats = (remaining_size // len(padding_pattern)) + 1\n padding = (padding_pattern * padding_repeats)[:remaining_size]\n content = base_content + padding\n else:\n content = base_content[:target_size]\n\n segments.append(content)\n\n return segments\n\n def _generate_realistic_content(self, index):\n \"\"\"Generate realistic content that excavate can extract from\"\"\"\n return f\"\"\"\n \n \n Test Content {index}\n \n \n \n

Page {index}

\n \n \n Link {index}\n CDN {index}\n \n \n \n
\n \n \n \n
\n \n \n \n \n \n

FTP: ftp://ftp{index}.example.com:21/files/

\n

SSH: ssh://server{index}.example.com:22/

\n

Email: contact{index}@example.com

\n \n \n \n \n \n \n \n \n \n \"\"\"\n\n async def _run_excavate_single_thread(self, text_segments):\n \"\"\"Run excavate processing in single thread\"\"\"\n # Create scanner and initialize excavate\n scan = Scanner(\"example.com\", modules=[\"httpx\"], config={\"excavate\": True})\n await scan._prep()\n excavate_module = scan.modules.get(\"excavate\")\n\n if not excavate_module:\n raise RuntimeError(\"Excavate module not found\")\n\n # Track events emitted by excavate\n emitted_events = []\n\n async def track_emit_event(event_data, *args, **kwargs):\n emitted_events.append(event_data)\n\n excavate_module.emit_event = track_emit_event\n\n # Process all text segments sequentially\n results = []\n for i, text_segment in enumerate(text_segments):\n # Create a mock HTTP_RESPONSE event\n mock_event = scan.make_event(\n {\n \"url\": f\"https://example.com/test/{i}\",\n \"method\": \"GET\",\n \"body\": text_segment,\n \"header-dict\": {\"Content-Type\": [\"text/html\"]},\n \"raw_header\": \"HTTP/1.1 200 OK\\r\\nContent-Type: text/html\\r\\n\\r\\n\",\n \"status_code\": 200,\n },\n \"HTTP_RESPONSE\",\n parent=scan.root_event,\n )\n\n # Process with excavate\n await excavate_module.search(text_segment, mock_event, \"text/html\", f\"Single thread benchmark {i}\")\n results.append(f\"processed_{i}\")\n\n return results, emitted_events\n\n async def _run_excavate_parallel_tasks(self, text_segments):\n \"\"\"Run excavate processing with parallel asyncio tasks\"\"\"\n # Create scanner and initialize excavate\n scan = Scanner(\"example.com\", modules=[\"httpx\"], config={\"excavate\": True})\n await scan._prep()\n excavate_module = scan.modules.get(\"excavate\")\n\n if not excavate_module:\n raise RuntimeError(\"Excavate module not found\")\n\n # Define async task to process a single text segment\n async def process_segment(segment_index, text_segment):\n mock_event = scan.make_event(\n {\n \"url\": f\"https://example.com/parallel/{segment_index}\",\n \"method\": \"GET\",\n \"body\": text_segment,\n \"header-dict\": {\"Content-Type\": [\"text/html\"]},\n \"raw_header\": \"HTTP/1.1 200 OK\\r\\nContent-Type: text/html\\r\\n\\r\\n\",\n \"status_code\": 200,\n },\n \"HTTP_RESPONSE\",\n parent=scan.root_event,\n )\n\n await excavate_module.search(\n text_segment, mock_event, \"text/html\", f\"Parallel benchmark task {segment_index}\"\n )\n return f\"processed_{segment_index}\"\n\n # Create all tasks and run them concurrently\n tasks = [process_segment(i, text_segment) for i, text_segment in enumerate(text_segments)]\n\n # Run all tasks in parallel\n results = await asyncio.gather(*tasks)\n return results\n\n # Single Thread Tests\n @pytest.mark.benchmark(group=\"excavate_single_small\")\n def test_excavate_single_thread_small(self, benchmark):\n \"\"\"Benchmark excavate single thread processing with small (4KB) segments\"\"\"\n text_segments = self._generate_text_segments(self.SMALL_SIZE, self.TEXT_SEGMENTS_COUNT)\n\n def run_test():\n return asyncio.run(self._run_excavate_single_thread(text_segments))\n\n result, events = benchmark(run_test)\n\n assert len(result) == self.TEXT_SEGMENTS_COUNT\n total_size_mb = (self.SMALL_SIZE * self.TEXT_SEGMENTS_COUNT) / (1024 * 1024)\n\n # Count events by type\n total_events = len(events)\n url_events = len([e for e in events if e.type == \"URL_UNVERIFIED\"])\n dns_events = len([e for e in events if e.type == \"DNS_NAME\"])\n email_events = len([e for e in events if e.type == \"EMAIL_ADDRESS\"])\n protocol_events = len([e for e in events if e.type == \"PROTOCOL\"])\n finding_events = len([e for e in events if e.type == \"FINDING\"])\n\n print(\"\\n✅ Single-thread small segments benchmark completed\")\n print(f\"📊 Processed {len(result):,} segments of {self.SMALL_SIZE / 1024:.0f}KB each\")\n print(f\"📊 Total size processed: {total_size_mb:.1f} MB\")\n print(f\"📊 Total events: {total_events}\")\n print(f\"📊 URL events: {url_events}\")\n print(f\"📊 DNS events: {dns_events}\")\n print(f\"📊 Email events: {email_events}\")\n print(f\"📊 Protocol events: {protocol_events}\")\n print(f\"📊 Finding events: {finding_events}\")\n\n # Validate that excavate actually found and processed content\n assert total_events > 0, \"Expected to find some events from excavate\"\n assert url_events > 0 or dns_events > 0 or protocol_events > 0, (\n \"Expected excavate to find URLs, DNS names, or protocols\"\n )\n\n @pytest.mark.benchmark(group=\"excavate_single_large\")\n def test_excavate_single_thread_large(self, benchmark):\n \"\"\"Benchmark excavate single thread processing with large (10MB) segments\"\"\"\n text_segments = self._generate_text_segments(self.LARGE_SIZE, self.TEXT_SEGMENTS_COUNT)\n\n def run_test():\n return asyncio.run(self._run_excavate_single_thread(text_segments))\n\n result, events = benchmark(run_test)\n\n assert len(result) == self.TEXT_SEGMENTS_COUNT\n total_size_mb = (self.LARGE_SIZE * self.TEXT_SEGMENTS_COUNT) / (1024 * 1024)\n\n # Count events by type\n total_events = len(events)\n url_events = len([e for e in events if e.type == \"URL_UNVERIFIED\"])\n dns_events = len([e for e in events if e.type == \"DNS_NAME\"])\n email_events = len([e for e in events if e.type == \"EMAIL_ADDRESS\"])\n protocol_events = len([e for e in events if e.type == \"PROTOCOL\"])\n finding_events = len([e for e in events if e.type == \"FINDING\"])\n\n print(\"\\n✅ Single-thread large segments benchmark completed\")\n print(f\"📊 Processed {len(result):,} segments of {self.LARGE_SIZE / (1024 * 1024):.0f}MB each\")\n print(f\"📊 Total size processed: {total_size_mb:.1f} MB\")\n print(f\"📊 Total events: {total_events}\")\n print(f\"📊 URL events: {url_events}\")\n print(f\"📊 DNS events: {dns_events}\")\n print(f\"📊 Email events: {email_events}\")\n print(f\"📊 Protocol events: {protocol_events}\")\n print(f\"📊 Finding events: {finding_events}\")\n\n # Validate that excavate actually found and processed content\n assert total_events > 0, \"Expected to find some events from excavate\"\n assert url_events > 0 or dns_events > 0 or protocol_events > 0, (\n \"Expected excavate to find URLs, DNS names, or protocols\"\n )\n\n # Parallel Tests\n @pytest.mark.benchmark(group=\"excavate_parallel_small\")\n def test_excavate_parallel_tasks_small(self, benchmark):\n \"\"\"Benchmark excavate parallel processing with small (4KB) segments\"\"\"\n text_segments = self._generate_text_segments(self.SMALL_SIZE, self.TEXT_SEGMENTS_COUNT)\n\n def run_test():\n return asyncio.run(self._run_excavate_parallel_tasks(text_segments))\n\n result = benchmark(run_test)\n\n assert len(result) == self.TEXT_SEGMENTS_COUNT\n total_size_mb = (self.SMALL_SIZE * self.TEXT_SEGMENTS_COUNT) / (1024 * 1024)\n print(\"\\n✅ Parallel small segments benchmark completed\")\n print(f\"📊 Processed {len(result):,} segments of {self.SMALL_SIZE / 1024:.0f}KB each in parallel\")\n print(f\"📊 Total size processed: {total_size_mb:.1f} MB\")\n print(\"📊 Tasks executed concurrently to test YARA GIL sidestep\")\n\n # Basic assertion that excavate is actually working (should find URLs in our test content)\n assert len(result) > 0, \"Expected excavate to process all segments\"\n\n @pytest.mark.benchmark(group=\"excavate_parallel_large\")\n def test_excavate_parallel_tasks_large(self, benchmark):\n \"\"\"Benchmark excavate parallel processing with large (10MB) segments to test YARA GIL sidestep\"\"\"\n text_segments = self._generate_text_segments(self.LARGE_SIZE, self.TEXT_SEGMENTS_COUNT)\n\n def run_test():\n return asyncio.run(self._run_excavate_parallel_tasks(text_segments))\n\n result = benchmark(run_test)\n\n assert len(result) == self.TEXT_SEGMENTS_COUNT\n total_size_mb = (self.LARGE_SIZE * self.TEXT_SEGMENTS_COUNT) / (1024 * 1024)\n print(\"\\n✅ Parallel large segments benchmark completed\")\n print(f\"📊 Processed {len(result):,} segments of {self.LARGE_SIZE / (1024 * 1024):.0f}MB each in parallel\")\n print(f\"📊 Total size processed: {total_size_mb:.1f} MB\")\n print(\"📊 Tasks executed concurrently to test YARA GIL sidestep\")\n\n # Basic assertion that excavate is actually working (should find URLs in our test content)\n assert len(result) > 0, \"Expected excavate to process all segments\"\n" }, { "path": "bbot/test/benchmarks/test_ipaddress_benchmarks.py", "content": "import pytest\nimport random\nimport string\nfrom bbot.core.helpers.misc import make_ip_type, is_ip\n\n\nclass TestIPAddressBenchmarks:\n \"\"\"\n Benchmark tests for IP address processing operations.\n\n These tests measure the performance of BBOT-level IP functions which are\n critical for network scanning efficiency and could benefit from different\n underlying implementations.\n \"\"\"\n\n def setup_method(self):\n \"\"\"Setup common test data\"\"\"\n # Set deterministic seed for consistent benchmark results\n random.seed(42) # Fixed seed for reproducible results\n\n # Generate test data of different types and sizes\n self.valid_ips = self._generate_valid_ips()\n self.invalid_ips = self._generate_invalid_ips()\n self.mixed_data = self._generate_mixed_data()\n\n def _generate_valid_ips(self):\n \"\"\"Generate valid IP addresses for testing\"\"\"\n valid_ips = []\n\n # IPv4 addresses\n for i in range(1000):\n valid_ips.append(\n f\"{random.randint(1, 223)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(1, 254)}\"\n )\n\n # IPv6 addresses\n for i in range(500):\n ipv6_parts = []\n for j in range(8):\n ipv6_parts.append(f\"{random.randint(0, 65535):x}\")\n valid_ips.append(\":\".join(ipv6_parts))\n\n # Network addresses\n for i in range(500):\n base_ip = f\"{random.randint(1, 223)}.{random.randint(0, 255)}.{random.randint(0, 255)}.0\"\n valid_ips.append(f\"{base_ip}/{random.randint(8, 30)}\")\n\n # IP ranges\n for i in range(200):\n start_ip = (\n f\"{random.randint(1, 223)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(1, 200)}\"\n )\n end_ip = f\"{random.randint(1, 223)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(201, 254)}\"\n valid_ips.append(f\"{start_ip}-{end_ip}\")\n\n return valid_ips\n\n def _generate_invalid_ips(self):\n \"\"\"Generate invalid IP addresses for testing\"\"\"\n invalid_ips = []\n\n # Malformed IPv4\n for i in range(500):\n invalid_ips.append(\n f\"{random.randint(256, 999)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}\"\n )\n invalid_ips.append(f\"{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}\")\n invalid_ips.append(\n f\"{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}.{random.randint(0, 255)}\"\n )\n\n # Malformed IPv6\n for i in range(300):\n ipv6_parts = []\n for j in range(random.randint(5, 10)): # Wrong number of parts\n ipv6_parts.append(f\"{random.randint(0, 65535):x}\")\n invalid_ips.append(\":\".join(ipv6_parts))\n\n # Random strings\n for i in range(200):\n length = random.randint(5, 20)\n invalid_ips.append(\"\".join(random.choices(string.ascii_letters + string.digits, k=length)))\n\n return invalid_ips\n\n def _generate_mixed_data(self):\n \"\"\"Generate mixed valid/invalid data for realistic testing\"\"\"\n mixed = []\n mixed.extend(self.valid_ips[:500]) # First 500 valid\n mixed.extend(self.invalid_ips[:500]) # First 500 invalid\n # Use deterministic shuffle with fixed seed for consistent results\n random.seed(42) # Reset seed before shuffle\n random.shuffle(mixed) # Shuffle for realistic distribution\n return mixed\n\n @pytest.mark.benchmark(group=\"ip_validation\")\n def test_is_ip_performance(self, benchmark):\n \"\"\"Benchmark IP validation performance with mixed data\"\"\"\n\n def validate_ips():\n valid_count = 0\n for ip in self.mixed_data:\n if is_ip(ip):\n valid_count += 1\n return valid_count\n\n result = benchmark(validate_ips)\n assert result > 0\n\n @pytest.mark.benchmark(group=\"ip_type_detection\")\n def test_make_ip_type_performance(self, benchmark):\n \"\"\"Benchmark IP type detection performance\"\"\"\n\n def detect_ip_types():\n type_count = 0\n for ip in self.valid_ips:\n try:\n make_ip_type(ip)\n type_count += 1\n except Exception:\n pass\n return type_count\n\n result = benchmark(detect_ip_types)\n assert result > 0\n\n @pytest.mark.benchmark(group=\"ip_processing\")\n def test_mixed_ip_operations(self, benchmark):\n \"\"\"Benchmark combined IP validation + type detection\"\"\"\n\n def process_ips():\n processed = 0\n for ip in self.mixed_data:\n if is_ip(ip):\n try:\n make_ip_type(ip)\n processed += 1\n except Exception:\n pass\n return processed\n\n result = benchmark(process_ips)\n assert result > 0\n" }, { "path": "bbot/test/benchmarks/test_weighted_shuffle_benchmarks.py", "content": "import pytest\nimport random\nfrom bbot.core.helpers.misc import weighted_shuffle\n\n\nclass TestWeightedShuffleBenchmarks:\n \"\"\"\n Benchmark tests for weighted_shuffle operations.\n\n This function is critical for BBOT's queue management, where it shuffles\n incoming queues based on module priority weights. Performance here directly\n impacts scan throughput and responsiveness.\n \"\"\"\n\n def setup_method(self):\n \"\"\"Setup common test data\"\"\"\n # Set deterministic seed for consistent benchmark results\n random.seed(42) # Fixed seed for reproducible results\n\n # Generate test data of different sizes and complexity\n self.small_data = self._generate_small_dataset()\n self.medium_data = self._generate_medium_dataset()\n self.large_data = self._generate_large_dataset()\n self.priority_weights = self._generate_priority_weights()\n\n def _generate_small_dataset(self):\n \"\"\"Generate small dataset (like few modules)\"\"\"\n return {\"items\": [\"module_a\", \"module_b\", \"module_c\"], \"weights\": [0.6, 0.3, 0.1]}\n\n def _generate_medium_dataset(self):\n \"\"\"Generate medium dataset (like typical scan)\"\"\"\n items = [f\"module_{i}\" for i in range(20)]\n weights = [random.uniform(0.1, 1.0) for _ in range(20)]\n return {\"items\": items, \"weights\": weights}\n\n def _generate_large_dataset(self):\n \"\"\"Generate large dataset (like complex scan with many modules)\"\"\"\n items = [f\"module_{i}\" for i in range(100)]\n weights = [random.uniform(0.1, 1.0) for _ in range(100)]\n return {\"items\": items, \"weights\": weights}\n\n def _generate_priority_weights(self):\n \"\"\"Generate realistic priority weights (like BBOT module priorities)\"\"\"\n # BBOT uses priorities 1-5, where lower priority = higher weight\n # Weights are calculated as [5] + [6 - m.priority for m in modules]\n priorities = [5] + [6 - p for p in [1, 2, 3, 4, 5]] * 20 # 5 + 5*20 = 105 items\n items = [f\"queue_{i}\" for i in range(len(priorities))]\n return {\"items\": items, \"weights\": priorities}\n\n @pytest.mark.benchmark(group=\"weighted_shuffle\")\n def test_typical_queue_shuffle(self, benchmark):\n \"\"\"Benchmark weighted shuffle with typical BBOT scan workload\"\"\"\n\n def shuffle_typical():\n return weighted_shuffle(self.medium_data[\"items\"], self.medium_data[\"weights\"])\n\n result = benchmark(shuffle_typical)\n assert len(result) == 20\n assert all(item in result for item in self.medium_data[\"items\"])\n\n @pytest.mark.benchmark(group=\"weighted_shuffle\")\n def test_priority_queue_shuffle(self, benchmark):\n \"\"\"Benchmark weighted shuffle with realistic BBOT priority weights\"\"\"\n\n def shuffle_priorities():\n return weighted_shuffle(self.priority_weights[\"items\"], self.priority_weights[\"weights\"])\n\n result = benchmark(shuffle_priorities)\n assert len(result) == len(self.priority_weights[\"items\"])\n assert all(item in result for item in self.priority_weights[\"items\"])\n" }, { "path": "bbot/test/conftest.py", "content": "import os\nimport ssl\nimport time\nimport pytest\nimport shutil\nimport asyncio\nimport logging\nfrom pathlib import Path\nfrom contextlib import suppress\nfrom omegaconf import OmegaConf\nfrom pytest_httpserver import HTTPServer\n\nfrom bbot.core import CORE\nfrom bbot.core.helpers.misc import execute_sync_or_async\nfrom bbot.core.helpers.interactsh import server_list as interactsh_servers\n\n# silence stdout + trace\nroot_logger = logging.getLogger()\npytest_debug_file = Path(__file__).parent.parent.parent / \"pytest_debug.log\"\ndebug_handler = logging.FileHandler(pytest_debug_file)\ndebug_handler.setLevel(logging.DEBUG)\ndebug_format = logging.Formatter(\"%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s\")\ndebug_handler.setFormatter(debug_format)\nroot_logger.addHandler(debug_handler)\n\ntest_config = OmegaConf.load(Path(__file__).parent / \"test.conf\")\n\nos.environ[\"BBOT_DEBUG\"] = \"True\"\nCORE.logger.log_level = logging.DEBUG\n\n# silence all stderr output:\nstderr_handler = CORE.logger.log_handlers[\"stderr\"]\nstderr_handler.setLevel(logging.CRITICAL)\nhandlers = list(CORE.logger.listener.handlers)\nhandlers.remove(stderr_handler)\nCORE.logger.listener.handlers = tuple(handlers)\n\nfor h in root_logger.handlers:\n h.addFilter(lambda x: x.levelname not in (\"STDOUT\", \"TRACE\"))\n\n\nCORE.merge_default(test_config)\n\n\n@pytest.fixture\ndef assert_all_responses_were_requested() -> bool:\n return False\n\n\n@pytest.fixture(autouse=True)\ndef silence_live_logging():\n for handler in logging.getLogger().handlers:\n if type(handler).__name__ == \"_LiveLoggingStreamHandler\":\n handler.setLevel(logging.CRITICAL)\n\n\ndef stop_server(server):\n server.stop()\n while server.is_running():\n time.sleep(0.1) # Wait a bit before checking again\n\n\n@pytest.fixture\ndef bbot_httpserver():\n server = HTTPServer(host=\"127.0.0.1\", port=8888, threaded=True)\n server.start()\n\n yield server\n\n server.clear()\n stop_server(server) # Ensure the server is fully stopped\n\n server.check_assertions()\n server.clear()\n\n\n@pytest.fixture\ndef bbot_httpserver_ssl():\n context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)\n current_dir = Path(__file__).parent\n keyfile = str(current_dir / \"testsslkey.pem\")\n certfile = str(current_dir / \"testsslcert.pem\")\n context.load_cert_chain(certfile, keyfile)\n server = HTTPServer(host=\"127.0.0.1\", port=9999, ssl_context=context, threaded=True)\n server.start()\n\n yield server\n\n server.clear()\n stop_server(server) # Ensure the server is fully stopped\n\n server.check_assertions()\n server.clear()\n\n\ndef should_mock(request):\n return request.url.host not in [\"127.0.0.1\", \"localhost\", \"raw.githubusercontent.com\"] + interactsh_servers\n\n\ndef pytest_collection_modifyitems(config, items):\n # make sure all tests have the httpx_mock marker\n for item in items:\n item.add_marker(\n pytest.mark.httpx_mock(\n should_mock=should_mock,\n assert_all_requests_were_expected=False,\n assert_all_responses_were_requested=False,\n can_send_already_matched_responses=True,\n )\n )\n\n\n@pytest.fixture\ndef bbot_httpserver_allinterfaces():\n server = HTTPServer(host=\"0.0.0.0\", port=5556, threaded=True)\n server.start()\n\n yield server\n\n server.clear()\n if server.is_running():\n server.stop()\n server.check_assertions()\n server.clear()\n\n\nclass Interactsh_mock:\n def __init__(self, name):\n self.name = name\n self.log = logging.getLogger(f\"bbot.interactsh.{self.name}\")\n self.interactions = asyncio.Queue() # Use an asyncio queue for async access\n self.correlation_id = \"deadbeef-dead-beef-dead-beefdeadbeef\"\n self.stop = False\n self.poll_task = None\n\n def mock_interaction(self, subdomain_tag, msg=None):\n self.log.info(f\"Mocking interaction to subdomain tag: {subdomain_tag}\")\n if msg is not None:\n self.log.info(msg)\n self.interactions.put_nowait(subdomain_tag) # Add to the thread-safe queue\n\n async def register(self, callback=None):\n if callable(callback):\n self.poll_task = asyncio.create_task(self.poll_loop(callback))\n return \"fakedomain.fakeinteractsh.com\"\n\n async def deregister(self, callback=None):\n await asyncio.sleep(1)\n self.stop = True\n if self.poll_task is not None:\n self.poll_task.cancel()\n with suppress(asyncio.CancelledError):\n await self.poll_task\n\n async def poll_loop(self, callback=None):\n while not self.stop:\n data_list = await self.poll(callback)\n if not data_list:\n await asyncio.sleep(0.5)\n continue\n await asyncio.sleep(1)\n await self.poll(callback)\n\n async def poll(self, callback=None):\n poll_results = []\n while not self.interactions.empty():\n subdomain_tag = await self.interactions.get() # Get the first element from the asyncio queue\n for protocol in [\"HTTP\", \"DNS\"]:\n result = {\"full-id\": f\"{subdomain_tag}.fakedomain.fakeinteractsh.com\", \"protocol\": protocol}\n poll_results.append(result)\n if callback is not None:\n await execute_sync_or_async(callback, result)\n await asyncio.sleep(0.1)\n return poll_results\n\n\nimport threading\nimport http.server\nimport socketserver\nimport urllib.request\n\n\nclass Proxy(http.server.SimpleHTTPRequestHandler):\n protocol_version = \"HTTP/1.0\"\n server_version = \"Proxy\"\n urls = []\n\n def do_GET(self):\n self.urls.append(self.path)\n\n # Extract host and port from path\n netloc = urllib.parse.urlparse(self.path).netloc\n host, _, port = netloc.partition(\":\")\n\n # Fetch the content\n conn = http.client.HTTPConnection(host, port if port else 80)\n conn.request(\"GET\", self.path, headers=self.headers)\n response = conn.getresponse()\n\n # Send the response back to the client\n self.send_response(response.status)\n for header, value in response.getheaders():\n self.send_header(header, value)\n self.end_headers()\n self.copyfile(response, self.wfile)\n\n response.close()\n conn.close()\n\n\n@pytest.fixture\ndef proxy_server():\n # Set up an HTTP server that acts as a simple proxy.\n server = socketserver.ThreadingTCPServer((\"localhost\", 0), Proxy)\n\n # Start the server in a new thread.\n server_thread = threading.Thread(target=server.serve_forever, daemon=True)\n server_thread.start()\n\n yield server\n\n # Stop the server.\n server.shutdown()\n server_thread.join()\n\n\ndef pytest_terminal_summary(terminalreporter, exitstatus, config): # pragma: no cover\n RED = \"\\033[1;31m\"\n GREEN = \"\\033[1;32m\"\n YELLOW = \"\\033[1;33m\"\n BLUE = \"\\033[1;34m\"\n CYAN = \"\\033[1;36m\"\n RESET = \"\\033[0m\"\n stats = terminalreporter.stats\n total_tests = len(terminalreporter._session.items)\n passed = len(stats.get(\"passed\", []))\n skipped = len(stats.get(\"skipped\", []))\n errors = len(stats.get(\"error\", []))\n failed = stats.get(\"failed\", [])\n\n terminalreporter.write(\"\\nTest Session Summary:\")\n terminalreporter.write(f\"\\nTotal tests run: {total_tests}\")\n terminalreporter.write(\n f\"\\n{GREEN}Passed: {passed}{RESET}, {RED}Failed: {len(failed)}{RESET}, {YELLOW}Skipped: {skipped}{RESET}, Errors: {errors}\"\n )\n\n if failed:\n terminalreporter.write(f\"\\n{RED}Detailed failed test report:{RESET}\")\n for item in failed:\n test_name = item.nodeid.split(\"::\")[-1] if \"::\" in item.nodeid else item.nodeid\n file_and_line = f\"{item.location[0]}:{item.location[1]}\" # File path and line number\n terminalreporter.write(f\"\\n{BLUE}Test Name: {test_name}{RESET} {CYAN}({file_and_line}){RESET}\")\n terminalreporter.write(f\"\\n{RED}Location: {item.nodeid} at {item.location[0]}:{item.location[1]}{RESET}\")\n terminalreporter.write(f\"\\n{RED}Failure details:\\n{item.longreprtext}{RESET}\")\n\n\n# BELOW: debugging for frozen/hung tests\nimport psutil\nimport traceback\nimport inspect\n\n\ndef _print_detailed_info(): # pragma: no cover\n \"\"\"\n Debugging pytests hanging\n \"\"\"\n print(\"=== Detailed Thread and Process Information ===\\n\")\n try:\n print(\"=== Threads ===\")\n for thread in threading.enumerate():\n print(f\"Thread Name: {thread.name}\")\n print(f\"Thread ID: {thread.ident}\")\n print(f\"Is Alive: {thread.is_alive()}\")\n print(f\"Daemon: {thread.daemon}\")\n\n if hasattr(thread, \"_target\"):\n target = thread._target\n if target:\n qualname = (\n f\"{target.__module__}.{target.__qualname__}\"\n if hasattr(target, \"__qualname__\")\n else str(target)\n )\n print(f\"Target Function: {qualname}\")\n\n if hasattr(thread, \"_args\"):\n args = thread._args\n kwargs = thread._kwargs if hasattr(thread, \"_kwargs\") else {}\n arg_spec = inspect.getfullargspec(target)\n\n all_args = list(args) + [f\"{k}={v}\" for k, v in kwargs.items()]\n\n if inspect.ismethod(target) and arg_spec.args[0] == \"self\":\n arg_spec.args.pop(0)\n\n named_args = list(zip(arg_spec.args, all_args))\n if arg_spec.varargs:\n named_args.extend((f\"*{arg_spec.varargs}\", arg) for arg in all_args[len(arg_spec.args) :])\n\n print(\"Arguments:\")\n for name, value in named_args:\n print(f\" {name}: {value}\")\n else:\n print(\"Target Function: None\")\n else:\n print(\"Target Function: Unknown\")\n\n print()\n\n print(\"=== Processes ===\")\n current_process = psutil.Process()\n for child in current_process.children(recursive=True):\n print(f\"Process ID: {child.pid}\")\n print(f\"Name: {child.name()}\")\n print(f\"Status: {child.status()}\")\n print(f\"CPU Times: {child.cpu_times()}\")\n print(f\"Memory Info: {child.memory_info()}\")\n print()\n\n print(\"=== Current Process ===\")\n print(f\"Process ID: {current_process.pid}\")\n print(f\"Name: {current_process.name()}\")\n print(f\"Status: {current_process.status()}\")\n print(f\"CPU Times: {current_process.cpu_times()}\")\n print(f\"Memory Info: {current_process.memory_info()}\")\n print()\n\n except Exception as e:\n print(f\"An error occurred: {str(e)}\")\n print(\"Traceback:\")\n traceback.print_exc()\n\n\n@pytest.hookimpl(tryfirst=True, hookwrapper=True)\ndef pytest_sessionfinish(session, exitstatus):\n # Remove handlers from all loggers to prevent logging errors at exit\n loggers = [logging.getLogger(\"bbot\")] + list(logging.Logger.manager.loggerDict.values())\n for logger in loggers:\n handlers = getattr(logger, \"handlers\", [])\n for handler in handlers:\n logger.removeHandler(handler)\n\n # Wipe out BBOT home dir\n shutil.rmtree(\"/tmp/.bbot_test\", ignore_errors=True)\n\n yield\n\n # temporarily suspend stdout capture and print detailed thread info\n capmanager = session.config.pluginmanager.get_plugin(\"capturemanager\")\n if capmanager:\n capmanager.suspend_global_capture(in_=True)\n\n _print_detailed_info()\n\n if capmanager:\n capmanager.resume_global_capture()\n" }, { "path": "bbot/test/coverage.cfg", "content": "[coverage:run]\nparallel = true\n" }, { "path": "bbot/test/fastapi_test.py", "content": "from typing import List\nfrom bbot import Scanner\nfrom fastapi import FastAPI, Query\n\napp = FastAPI()\n\n\n@app.get(\"/start\")\nasync def start(targets: List[str] = Query(...)):\n scanner = Scanner(*targets, modules=[\"httpx\"])\n events = [e async for e in scanner.async_start()]\n return [e.json() for e in events]\n\n\n@app.get(\"/ping\")\nasync def ping():\n return {\"status\": \"ok\"}\n" }, { "path": "bbot/test/run_tests.sh", "content": "#!/bin/bash\n\nbbot_dir=\"$( realpath \"$(dirname \"$(dirname \"${BASH_SOURCE[0]}\")\")\")\"\necho -e \"[+] BBOT dir: $bbot_dir\\n\"\n\necho \"[+] Checking code formatting with ruff\"\necho \"=======================================\"\nruff format \"$bbot_dir\" || exit 1\necho\n\necho \"[+] Linting with ruff\"\necho \"=======================\"\nruff check \"$bbot_dir\" || exit 1\necho\n\nif [ \"${1}x\" != \"x\" ] ; then\n MODULES=`echo ${1} | sed -e 's/,/ /g'`\n for MODULE in ${MODULES} ; do\n echo \"[+] Testing ${MODULE} with pytest\"\n pytest --exitfirst --disable-warnings --log-cli-level=ERROR \"$bbot_dir\" --cov=bbot/test/test_step_2/test_cli.py --cov-report=\"term-missing\" --cov-config=\"$bbot_dir/test/coverage.cfg\" -k ${MODULE}\n done\nelse\n echo \"[+] Testing all modules with pytest\"\n pytest --exitfirst --disable-warnings --log-cli-level=ERROR \"$bbot_dir\" --cov=bbot/test/test_step_2/test_cli.py --cov-report=\"term-missing\" --cov-config=\"$bbot_dir/test/coverage.cfg\"\nfi\n" }, { "path": "bbot/test/test.conf", "content": "home: /tmp/.bbot_test\nmodules:\n massdns:\n wordlist: https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/deepmagic.com-prefixes-top500.txt\n ffuf:\n prefix_busting: true\n http:\n url: http://127.0.0.1:11111\n username: username\n password: password\n bearer: bearer\n websocket:\n url: ws://127.0.0.1/ws:11111\n token: asdf\nweb:\n http_proxy:\n http_headers: { \"test\": \"header\" }\n ssl_verify: false\n user_agent: \"BBOT Test User-Agent\"\n debug: false\nscope:\n search_distance: 0\n report_distance: 0\ndns:\n disable: false\n minimal: true\n search_distance: 1\n debug: false\n timeout: 1\n wildcard_ignore:\n - blacklanternsecurity.com\n - fakedomain\n - notreal\n - google\n - google.com\n - example.com\n - evilcorp.com\n - one\ndeps:\n behavior: retry_failed\nengine:\n debug: true\nagent_url: ws://127.0.0.1:8765\nagent_token: test\nspeculate: false\nexcavate: false\naggregate: false\ncloudcheck: false\nomit_event_types: []\ndebug: true\n" }, { "path": "bbot/test/test_output.ndjson", "content": "[\n {\n \"ip\": \"8.8.8.8\",\n \"name\": \"dns.google.\",\n \"as_number\": 15169,\n \"as_org\": \"GOOGLE\",\n \"country_id\": \"US\",\n \"city\": \"\",\n \"version\": \"\",\n \"error\": \"\",\n \"dnssec\": true,\n \"reliability\": 1,\n \"checked_at\": \"2022-04-17T08:03:50.419919Z\",\n \"created_at\": \"2020-07-16T14:19:04.514857Z\"\n }\n]\n" }, { "path": "bbot/test/test_step_1/__init__.py", "content": "" }, { "path": "bbot/test/test_step_1/test__module__tests.py", "content": "import logging\nimport importlib\nfrom pathlib import Path\n\nfrom bbot import Preset\nfrom ..test_step_2.module_tests.base import ModuleTestBase\n\nlog = logging.getLogger(\"bbot.test.modules\")\n\nmodule_tests_dir = Path(__file__).parent.parent / \"test_step_2\" / \"module_tests\"\n\n_module_test_files = list(module_tests_dir.glob(\"test_module_*.py\"))\n_module_test_files.sort(key=lambda p: p.name)\nmodule_test_files = [m.name.split(\"test_module_\")[-1].split(\".\")[0] for m in _module_test_files]\n\n\ndef test__module__tests():\n preset = Preset()\n\n # make sure each module has a .py file\n for module_name, preloaded in preset.module_loader.preloaded().items():\n module_name = module_name.lower()\n assert module_name in module_test_files, f'No test file found for module \"{module_name}\"'\n\n # make sure each test file has a test class\n for file in _module_test_files:\n module_name = file.stem\n import_path = f\"bbot.test.test_step_2.module_tests.{module_name}\"\n module_test_variables = importlib.import_module(import_path, \"bbot\")\n module_pass = False\n for var_name in dir(module_test_variables):\n if var_name.startswith(\"Test\"):\n test_class = getattr(module_test_variables, var_name)\n if ModuleTestBase in getattr(test_class, \"__mro__\", ()):\n module_pass = True\n break\n assert module_pass, f\"Couldn't find a test class for {module_name} in {file}\"\n" }, { "path": "bbot/test/test_step_1/test_bbot_fastapi.py", "content": "import time\nimport httpx\nimport multiprocessing\nfrom pathlib import Path\nfrom subprocess import Popen\nfrom contextlib import suppress\n\ncwd = Path(__file__).parent.parent.parent\n\n\ndef run_bbot_multiprocess(queue):\n from bbot import Scanner\n\n scan = Scanner(\"http://127.0.0.1:8888\", \"blacklanternsecurity.com\", modules=[\"httpx\"])\n events = [e.json() for e in scan.start()]\n queue.put(events)\n\n\ndef test_bbot_multiprocess(bbot_httpserver):\n bbot_httpserver.expect_request(\"/\").respond_with_data(\"test@blacklanternsecurity.com\")\n\n queue = multiprocessing.Queue()\n events_process = multiprocessing.Process(target=run_bbot_multiprocess, args=(queue,))\n events_process.start()\n events_process.join(timeout=300)\n events = queue.get(timeout=10)\n assert len(events) >= 3\n scan_events = [e for e in events if e[\"type\"] == \"SCAN\"]\n assert len(scan_events) == 2\n assert any(e[\"data\"] == \"test@blacklanternsecurity.com\" for e in events)\n\n\ndef test_bbot_fastapi(bbot_httpserver):\n bbot_httpserver.expect_request(\"/\").respond_with_data(\"test@blacklanternsecurity.com\")\n fastapi_process = start_fastapi_server()\n\n try:\n # wait for the server to start with a timeout of 60 seconds\n start_time = time.time()\n while True:\n try:\n response = httpx.get(\"http://127.0.0.1:8978/ping\")\n response.raise_for_status()\n break\n except httpx.HTTPError:\n if time.time() - start_time > 60:\n raise TimeoutError(\"Server did not start within 60 seconds.\")\n time.sleep(0.1)\n continue\n\n # run a scan\n response = httpx.get(\n \"http://127.0.0.1:8978/start\",\n params={\"targets\": [\"http://127.0.0.1:8888\", \"blacklanternsecurity.com\"]},\n timeout=100,\n )\n events = response.json()\n assert len(events) >= 3\n scan_events = [e for e in events if e[\"type\"] == \"SCAN\"]\n assert len(scan_events) == 2\n assert any(e[\"data\"] == \"test@blacklanternsecurity.com\" for e in events)\n\n finally:\n with suppress(Exception):\n fastapi_process.terminate()\n\n\ndef start_fastapi_server():\n import os\n import sys\n\n env = os.environ.copy()\n with suppress(KeyError):\n del env[\"BBOT_TESTING\"]\n python_executable = str(sys.executable)\n process = Popen(\n [python_executable, \"-m\", \"uvicorn\", \"bbot.test.fastapi_test:app\", \"--port\", \"8978\"], cwd=cwd, env=env\n )\n return process\n" }, { "path": "bbot/test/test_step_1/test_bloom_filter.py", "content": "import time\nimport pytest\nimport string\nimport random\n\n\n@pytest.mark.asyncio\nasync def test_bloom_filter():\n def generate_random_strings(n, length=10):\n \"\"\"Generate a list of n random strings.\"\"\"\n return [\"\".join(random.choices(string.ascii_letters + string.digits, k=length)) for _ in range(n)]\n\n from bbot.scanner import Scanner\n\n scan = Scanner()\n\n n_items_to_add = 100000\n n_items_to_test = 100000\n bloom_filter_size = 8000000\n\n # Initialize the simple bloom filter and the set\n bloom_filter = scan.helpers.bloom_filter(size=bloom_filter_size)\n\n test_set = set()\n\n # Generate random strings to add\n print(f\"Generating {n_items_to_add:,} items to add\")\n items_to_add = set(generate_random_strings(n_items_to_add))\n\n # Generate random strings to test\n print(f\"Generating {n_items_to_test:,} items to test\")\n items_to_test = generate_random_strings(n_items_to_test)\n\n print(\"Adding items\")\n start = time.time()\n for item in items_to_add:\n bloom_filter.add(item)\n test_set.add(hash(item))\n end = time.time()\n elapsed = end - start\n print(f\"elapsed: {elapsed:.2f} ({int(n_items_to_test / elapsed)}/s)\")\n # this shouldn't take longer than 5 seconds\n assert elapsed < 5\n\n # make sure we have 100% accuracy\n start = time.time()\n for item in items_to_add:\n assert item in bloom_filter\n end = time.time()\n elapsed = end - start\n print(f\"elapsed: {elapsed:.2f} ({int(n_items_to_test / elapsed)}/s)\")\n # this shouldn't take longer than 5 seconds\n assert elapsed < 5\n\n print(\"Measuring false positives\")\n # Check for false positives\n false_positives = 0\n for item in items_to_test:\n if bloom_filter.check(item) and hash(item) not in test_set:\n false_positives += 1\n false_positive_percent = false_positives / len(items_to_test) * 100\n\n print(f\"False positive rate: {false_positive_percent:.2f}% ({false_positives}/{len(items_to_test)})\")\n\n # ensure false positives are less than .02 percent\n assert false_positive_percent < 0.02\n\n bloom_filter.close()\n\n await scan._cleanup()\n" }, { "path": "bbot/test/test_step_1/test_cli.py", "content": "import yaml\n\nfrom ..bbot_fixtures import *\n\nfrom bbot import cli\n\n\n@pytest.mark.asyncio\nasync def test_cli_scope(monkeypatch, capsys):\n import json\n\n monkeypatch.setattr(sys, \"exit\", lambda *args, **kwargs: True)\n monkeypatch.setattr(os, \"_exit\", lambda *args, **kwargs: True)\n\n # basic target without whitelist\n monkeypatch.setattr(\n \"sys.argv\",\n [\"bbot\", \"-t\", \"one.one.one.one\", \"-c\", \"scope.report_distance=10\", \"dns.minimal=false\", \"--json\"],\n )\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is True\n lines = [json.loads(l) for l in out.splitlines()]\n dns_events = [l for l in lines if l[\"type\"] == \"DNS_NAME\" and l[\"data\"] == \"one.one.one.one\"]\n assert dns_events\n assert all(l[\"scope_distance\"] == 0 and \"in-scope\" in l[\"tags\"] for l in dns_events)\n assert 1 == len(\n [\n l\n for l in dns_events\n if l[\"module\"] == \"TARGET\"\n and l[\"scope_distance\"] == 0\n and \"in-scope\" in l[\"tags\"]\n and \"target\" in l[\"tags\"]\n ]\n )\n ip_events = [l for l in lines if l[\"type\"] == \"IP_ADDRESS\" and l[\"data\"] == \"1.1.1.1\"]\n assert ip_events\n assert all(l[\"scope_distance\"] == 1 and \"distance-1\" in l[\"tags\"] for l in ip_events)\n ip_events = [l for l in lines if l[\"type\"] == \"IP_ADDRESS\" and l[\"data\"] == \"1.0.0.1\"]\n assert ip_events\n assert all(l[\"scope_distance\"] == 1 and \"distance-1\" in l[\"tags\"] for l in ip_events)\n\n # with whitelist\n monkeypatch.setattr(\n \"sys.argv\",\n [\n \"bbot\",\n \"-t\",\n \"one.one.one.one\",\n \"-w\",\n \"192.168.0.1\",\n \"-c\",\n \"scope.report_distance=10\",\n \"dns.minimal=false\",\n \"dns.search_distance=2\",\n \"--json\",\n ],\n )\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is True\n lines = [json.loads(l) for l in out.splitlines()]\n lines = [l for l in lines if l[\"type\"] != \"SCAN\"]\n assert lines\n assert not any(l[\"scope_distance\"] == 0 for l in lines)\n dns_events = [l for l in lines if l[\"type\"] == \"DNS_NAME\" and l[\"data\"] == \"one.one.one.one\"]\n assert dns_events\n assert all(l[\"scope_distance\"] == 1 and \"distance-1\" in l[\"tags\"] for l in dns_events)\n assert 1 == len(\n [\n l\n for l in dns_events\n if l[\"module\"] == \"TARGET\"\n and l[\"scope_distance\"] == 1\n and \"distance-1\" in l[\"tags\"]\n and \"target\" in l[\"tags\"]\n ]\n )\n ip_events = [l for l in lines if l[\"type\"] == \"IP_ADDRESS\" and l[\"data\"] == \"1.1.1.1\"]\n assert ip_events\n assert all(l[\"scope_distance\"] == 2 and \"distance-2\" in l[\"tags\"] for l in ip_events)\n ip_events = [l for l in lines if l[\"type\"] == \"IP_ADDRESS\" and l[\"data\"] == \"1.0.0.1\"]\n assert ip_events\n assert all(l[\"scope_distance\"] == 2 and \"distance-2\" in l[\"tags\"] for l in ip_events)\n\n\n@pytest.mark.asyncio\nasync def test_cli_scan(monkeypatch):\n monkeypatch.setattr(sys, \"exit\", lambda *args, **kwargs: True)\n monkeypatch.setattr(os, \"_exit\", lambda *args, **kwargs: True)\n\n scans_home = bbot_test_dir / \"scans\"\n\n # basic scan\n monkeypatch.setattr(\n sys,\n \"argv\",\n [\"bbot\", \"-y\", \"-t\", \"127.0.0.1\", \"www.example.com\", \"-n\", \"test_cli_scan\", \"-c\", \"dns.disable=true\"],\n )\n result = await cli._main()\n assert result is True\n\n scan_home = scans_home / \"test_cli_scan\"\n assert (scan_home / \"preset.yml\").is_file(), \"preset.yml not found\"\n assert (scan_home / \"wordcloud.tsv\").is_file(), \"wordcloud.tsv not found\"\n assert (scan_home / \"output.txt\").is_file(), \"output.txt not found\"\n assert (scan_home / \"output.csv\").is_file(), \"output.csv not found\"\n assert (scan_home / \"output.json\").is_file(), \"output.json not found\"\n\n with open(scan_home / \"preset.yml\") as f:\n text = f.read()\n assert \" dns:\\n disable: true\" in text\n\n with open(scan_home / \"output.csv\") as f:\n lines = f.readlines()\n assert lines[0] == \"Event type,Event data,IP Address,Source Module,Scope Distance,Event Tags,Discovery Path\\n\"\n assert len(lines) > 1, \"output.csv is not long enough\"\n\n ip_success = False\n dns_success = False\n output_filename = scan_home / \"output.txt\"\n with open(output_filename) as f:\n lines = f.read().splitlines()\n for line in lines:\n if \"[IP_ADDRESS] \\t127.0.0.1\\tTARGET\" in line:\n ip_success = True\n if \"[DNS_NAME] \\twww.example.com\\tTARGET\" in line:\n dns_success = True\n assert ip_success and dns_success, \"IP_ADDRESS and/or DNS_NAME are not present in output.txt\"\n\n # Check for gzipped scan log file\n scan_log = scan_home / \"scan.log\"\n assert scan_log.is_file(), \"scan.log not found\"\n assert \"[INFO]\" in open(scan_log).read()\n\n\n@pytest.mark.asyncio\nasync def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):\n caplog.set_level(logging.INFO)\n\n monkeypatch.setattr(sys, \"exit\", lambda *args, **kwargs: True)\n monkeypatch.setattr(os, \"_exit\", lambda *args, **kwargs: True)\n\n # show version\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--version\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert len(out.splitlines()) == 1\n assert out.count(\".\") > 1\n\n # deps behavior\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-n\", \"depstest\", \"--retry-deps\", \"--current-preset\"])\n result = await cli._main()\n assert result is None\n out, err = capsys.readouterr()\n print(out)\n # parse YAML output\n preset = yaml.safe_load(out)\n assert preset == {\n \"description\": \"depstest\",\n \"scan_name\": \"depstest\",\n \"config\": {\"deps\": {\"behavior\": \"retry_failed\"}},\n }\n\n # list modules\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--list-modules\"])\n result = await cli._main()\n assert result is None\n out, err = capsys.readouterr()\n # internal modules\n assert \"| excavate \" in out\n # no output modules\n assert not \"| csv \" in out\n # scan modules\n assert \"| wayback \" in out\n\n # list output modules\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--list-output-modules\"])\n result = await cli._main()\n assert result == None\n out, err = capsys.readouterr()\n # no internal modules\n assert not \"| excavate \" in out\n # output modules\n assert \"| csv \" in out\n # no scan modules\n assert not \"| wayback \" in out\n\n # output dir and scan name\n output_dir = bbot_test_dir / \"bbot_cli_args_output\"\n scan_name = \"bbot_cli_args_scan_name\"\n scan_dir = output_dir / scan_name\n if output_dir.exists():\n shutil.rmtree(output_dir)\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-o\", str(output_dir), \"-n\", scan_name, \"-y\"])\n result = await cli._main()\n assert result is True\n assert output_dir.is_dir()\n assert scan_dir.is_dir()\n assert \"[SCAN]\" in open(scan_dir / \"output.txt\").read()\n\n # Check for gzipped scan log file\n scan_log = scan_dir / \"scan.log\"\n assert scan_log.is_file(), \"scan.log not found\"\n assert \"[INFO]\" in open(scan_log).read()\n shutil.rmtree(output_dir)\n\n # list module options\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--list-module-options\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert \"| modules.wayback.urls\" in out\n assert \"| bool\" in out\n assert \"| emit URLs in addition to DNS_NAMEs\" in out\n assert \"| False\" in out\n assert \"| modules.dnsbrute.wordlist\" in out\n assert \"| modules.robots.include_allow\" in out\n\n # list module options by flag\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-f\", \"subdomain-enum\", \"--list-module-options\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert \"| modules.wayback.urls\" in out\n assert \"| bool\" in out\n assert \"| emit URLs in addition to DNS_NAMEs\" in out\n assert \"| False\" in out\n assert \"| modules.dnsbrute.wordlist\" in out\n assert \"| modules.robots.include_allow\" not in out\n\n # list module options by module\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-m\", \"dnsbrute\", \"-lmo\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert out.count(\"modules.\") == out.count(\"modules.dnsbrute.\")\n assert \"| modules.wayback.urls\" not in out\n assert \"| modules.dnsbrute.wordlist\" in out\n assert \"| modules.robots.include_allow\" not in out\n\n # list output module options by module\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-om\", \"stdout\", \"-lmo\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert out.count(\"modules.\") == out.count(\"modules.stdout.\")\n\n # list flags\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--list-flags\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert \"| safe \" in out\n assert \"| Non-intrusive, safe to run \" in out\n assert \"| active \" in out\n assert \"| passive \" in out\n\n # list only a single flag\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-f\", \"active\", \"--list-flags\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert \"| safe \" not in out\n assert \"| active \" in out\n assert \"| passive \" not in out\n\n # list multiple flags\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-f\", \"active\", \"safe\", \"--list-flags\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert \"| safe \" in out\n assert \"| active \" in out\n assert \"| passive \" not in out\n\n # no args\n monkeypatch.setattr(\"sys.argv\", [\"bbot\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert \"-t TARGET [TARGET ...]\" in out\n\n # list modules\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-l\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert \"| dnsbrute \" in out\n assert \"| httpx \" in out\n assert \"| robots \" in out\n\n # list modules by flag\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-f\", \"subdomain-enum\", \"-l\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert \"| dnsbrute \" in out\n assert \"| httpx \" in out\n assert \"| robots \" not in out\n\n # list modules by flag + required flag\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-f\", \"subdomain-enum\", \"-rf\", \"passive\", \"-l\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert \"| chaos \" in out\n assert \"| httpx \" not in out\n\n # list modules by flag + excluded flag\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-f\", \"subdomain-enum\", \"-ef\", \"active\", \"-l\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert \"| chaos \" in out\n assert \"| httpx \" not in out\n\n # list modules by flag + excluded module\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-f\", \"subdomain-enum\", \"-em\", \"dnsbrute\", \"-l\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is None\n assert \"| dnsbrute \" not in out\n assert \"| httpx \" in out\n\n # output modules override\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-om\", \"csv,json\", \"-y\"])\n result = await cli._main()\n assert result is True\n assert \"Loaded 2/2 output modules, (csv,json)\" in caplog.text\n caplog.clear()\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-em\", \"csv,json\", \"-y\"])\n result = await cli._main()\n assert result is True\n assert \"Loaded 3/3 output modules, (python,stdout,txt)\" in caplog.text\n\n # output modules override\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-om\", \"subdomains\", \"-y\"])\n result = await cli._main()\n assert result is True\n assert \"Loaded 6/6 output modules, (csv,json,python,stdout,subdomains,txt)\" in caplog.text\n\n # internal modules override\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-y\"])\n result = await cli._main()\n assert result is True\n assert \"Loaded 6/6 internal modules (aggregate,cloudcheck,dnsresolve,excavate,speculate,unarchive)\" in caplog.text\n caplog.clear()\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-em\", \"excavate\", \"speculate\", \"-y\"])\n result = await cli._main()\n assert result is True\n assert \"Loaded 4/4 internal modules (aggregate,cloudcheck,dnsresolve,unarchive)\" in caplog.text\n caplog.clear()\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-c\", \"speculate=false\", \"-y\"])\n result = await cli._main()\n assert result is True\n assert \"Loaded 5/5 internal modules (aggregate,cloudcheck,dnsresolve,excavate,unarchive)\" in caplog.text\n\n # custom target type\n out, err = capsys.readouterr()\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-t\", \"ORG:evilcorp\", \"-y\"])\n result = await cli._main()\n out, err = capsys.readouterr()\n assert result is True\n assert \"[ORG_STUB] \tevilcorp\tTARGET\" in out\n\n # activate modules by flag\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-f\", \"passive\"])\n result = await cli._main()\n assert result is True\n\n # unconsoleable output module\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-om\", \"web_report\"])\n result = await cli._main()\n assert result is True\n\n # python dependency\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-m\", \"baddns\"])\n result = await cli._main()\n assert result is True\n\n # require flags\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-f\", \"active\", \"-rf\", \"passive\"])\n result = await cli._main()\n assert result is True\n\n # excluded flags\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-f\", \"active\", \"-ef\", \"active\"])\n result = await cli._main()\n assert result is True\n\n # slow modules\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-m\", \"bucket_digitalocean\"])\n result = await cli._main()\n assert result is True\n\n # deadly modules\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-m\", \"nuclei\"])\n result = await cli._main()\n assert result is False, \"-m nuclei ran without --allow-deadly\"\n assert \"Please specify --allow-deadly to continue\" in caplog.text\n\n # --allow-deadly\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-m\", \"nuclei\", \"--allow-deadly\"])\n result = await cli._main()\n assert result is True, \"-m nuclei failed to run with --allow-deadly\"\n\n # install all deps\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--install-all-deps\"])\n success = await cli._main()\n assert success is True, \"--install-all-deps failed for at least one module\"\n\n\n@pytest.mark.asyncio\nasync def test_cli_customheaders(monkeypatch, caplog, capsys):\n monkeypatch.setattr(sys, \"exit\", lambda *args, **kwargs: True)\n monkeypatch.setattr(os, \"_exit\", lambda *args, **kwargs: True)\n\n # test custom headers\n monkeypatch.setattr(\n \"sys.argv\", [\"bbot\", \"--custom-headers\", \"foo=bar\", \"foo2=bar2\", \"foo3=bar=3\", \"--current-preset\"]\n )\n success = await cli._main()\n assert success is None, \"setting custom headers on command line failed\"\n captured = capsys.readouterr()\n stdout_preset = yaml.safe_load(captured.out)\n assert stdout_preset[\"config\"][\"web\"][\"http_headers\"] == {\"foo\": \"bar\", \"foo2\": \"bar2\", \"foo3\": \"bar=3\"}\n\n # test custom headers invalid (no \"=\")\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--custom-headers\", \"justastring\", \"--current-preset\"])\n result = await cli._main()\n assert result is None\n assert \"Custom headers not formatted correctly (missing '=')\" in caplog.text\n caplog.clear()\n\n # test custom headers invalid (missing key)\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--custom-headers\", \"=nokey\", \"--current-preset\"])\n result = await cli._main()\n assert result is None\n assert \"Custom headers not formatted correctly (missing header name or value)\" in caplog.text\n caplog.clear()\n\n # test custom headers invalid (missing value)\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--custom-headers\", \"missingvalue=\", \"--current-preset\"])\n result = await cli._main()\n assert result is None\n assert \"Custom headers not formatted correctly (missing header name or value)\" in caplog.text\n\n\n@pytest.mark.asyncio\nasync def test_cli_module_help(monkeypatch, capsys):\n monkeypatch.setattr(sys, \"exit\", lambda *args, **kwargs: True)\n monkeypatch.setattr(os, \"_exit\", lambda *args, **kwargs: True)\n\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--module-help\", \"excavate\"])\n success = await cli._main()\n assert success is None, \"module help failed to execute\"\n captured = capsys.readouterr()\n\n assert \"Extracts domains from CSP headers\" in captured.out\n assert \"Module Help:\" in captured.out\n\n\ndef test_cli_config_validation(monkeypatch, caplog):\n monkeypatch.setattr(sys, \"exit\", lambda *args, **kwargs: True)\n monkeypatch.setattr(os, \"_exit\", lambda *args, **kwargs: True)\n\n # incorrect module option\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-c\", \"modules.ipnegibhor.num_bits=4\"])\n cli.main()\n assert 'Could not find config option \"modules.ipnegibhor.num_bits\"' in caplog.text\n assert 'Did you mean \"modules.ipneighbor.num_bits\"?' in caplog.text\n\n # incorrect global option\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-c\", \"web_spier_distance=4\"])\n cli.main()\n assert 'Could not find config option \"web_spier_distance\"' in caplog.text\n assert 'Did you mean \"web.spider_distance\"?' in caplog.text\n\n\ndef test_cli_module_validation(monkeypatch, caplog):\n monkeypatch.setattr(sys, \"exit\", lambda *args, **kwargs: True)\n monkeypatch.setattr(os, \"_exit\", lambda *args, **kwargs: True)\n\n # incorrect module\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-m\", \"dnsbrutes\"])\n cli.main()\n assert 'Could not find scan module \"dnsbrutes\"' in caplog.text\n assert 'Did you mean \"dnsbrute\"?' in caplog.text\n\n # incorrect excluded module\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-em\", \"dnsbrutes\"])\n cli.main()\n assert 'Could not find module \"dnsbrutes\"' in caplog.text\n assert 'Did you mean \"dnsbrute\"?' in caplog.text\n\n # incorrect output module\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-om\", \"neoo4j\"])\n cli.main()\n assert 'Could not find output module \"neoo4j\"' in caplog.text\n assert 'Did you mean \"neo4j\"?' in caplog.text\n\n # output module setup failed\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-om\", \"websocket\", \"-c\", \"modules.websocket.url=\", \"-y\"])\n cli.main()\n lines = caplog.text.splitlines()\n assert \"Loaded 6/6 output modules, (csv,json,python,stdout,txt,websocket)\" in caplog.text\n assert 1 == len(\n [\n l\n for l in lines\n if l.startswith(\"WARNING bbot.scanner:scanner.py\")\n and l.endswith(\"Setup hard-failed for websocket: Must set URL\")\n ]\n )\n assert 1 == len(\n [\n l\n for l in lines\n if l.startswith(\"WARNING bbot.modules.output.websocket:base.py\") and l.endswith(\"Setting error state\")\n ]\n )\n assert 1 == len(\n [\n l\n for l in lines\n if l.startswith(\"ERROR bbot.cli:cli.py\")\n and l.endswith(\"Setup hard-failed for 1 modules (websocket) (--force to run module anyway)\")\n ]\n )\n\n # only output module setup failed\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\n \"sys.argv\",\n [\"bbot\", \"-om\", \"websocket\", \"-em\", \"python,stdout,csv,json,txt\", \"-c\", \"modules.websocket.url=\", \"-y\"],\n )\n cli.main()\n lines = caplog.text.splitlines()\n assert \"Loaded 1/1 output modules, (websocket)\" in caplog.text\n assert 1 == len(\n [\n l\n for l in lines\n if l.startswith(\"WARNING bbot.scanner:scanner.py\")\n and l.endswith(\"Setup hard-failed for websocket: Must set URL\")\n ]\n )\n assert 1 == len(\n [\n l\n for l in lines\n if l.startswith(\"WARNING bbot.modules.output.websocket:base.py\") and l.endswith(\"Setting error state\")\n ]\n )\n assert 1 == len(\n [\n l\n for l in lines\n if l.startswith(\"ERROR bbot.cli:cli.py\") and l.endswith(\"Failed to load output modules. Aborting.\")\n ]\n )\n\n # bad target\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-t\", \"asdf:::sdf\"])\n cli.main()\n assert 'Unable to autodetect data type from \"asdf:::sdf\"' in caplog.text\n\n # incorrect flag\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-f\", \"subdomainenum\"])\n cli.main()\n assert 'Could not find flag \"subdomainenum\"' in caplog.text\n assert 'Did you mean \"subdomain-enum\"?' in caplog.text\n\n # incorrect excluded flag\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-ef\", \"subdomainenum\"])\n cli.main()\n assert 'Could not find flag \"subdomainenum\"' in caplog.text\n assert 'Did you mean \"subdomain-enum\"?' in caplog.text\n\n # incorrect required flag\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-rf\", \"subdomainenum\"])\n cli.main()\n assert 'Could not find flag \"subdomainenum\"' in caplog.text\n assert 'Did you mean \"subdomain-enum\"?' in caplog.text\n\n\ndef test_cli_presets(monkeypatch, capsys, caplog):\n import yaml\n\n monkeypatch.setattr(sys, \"exit\", lambda *args, **kwargs: True)\n monkeypatch.setattr(os, \"_exit\", lambda *args, **kwargs: True)\n\n # show current preset\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-c\", \"web.http_proxy=currentpresettest\", \"--current-preset\"])\n cli.main()\n captured = capsys.readouterr()\n assert \" http_proxy: currentpresettest\" in captured.out\n\n # show current preset (full)\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-cmodules.c99.api_key=asdf\", \"--current-preset-full\"])\n cli.main()\n captured = capsys.readouterr()\n assert \" api_key: asdf\" in captured.out\n\n preset_dir = bbot_test_dir / \"test_cli_presets\"\n preset_dir.mkdir(exist_ok=True)\n\n preset1_file = preset_dir / \"cli_preset1.conf\"\n with open(preset1_file, \"w\") as f:\n f.write(\n \"\"\"\nconfig:\n web:\n http_proxy: http://proxy1\n \"\"\"\n )\n\n preset2_file = preset_dir / \"cli_preset2.yml\"\n with open(preset2_file, \"w\") as f:\n f.write(\n \"\"\"\nconfig:\n web:\n http_proxy: http://proxy2\n \"\"\"\n )\n\n # test reading single preset\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-p\", str(preset1_file.resolve()), \"--current-preset\"])\n cli.main()\n captured = capsys.readouterr()\n stdout_preset = yaml.safe_load(captured.out)\n assert stdout_preset[\"config\"][\"web\"][\"http_proxy\"] == \"http://proxy1\"\n\n # preset overrides preset\n monkeypatch.setattr(\n \"sys.argv\", [\"bbot\", \"-p\", str(preset2_file.resolve()), str(preset1_file.resolve()), \"--current-preset\"]\n )\n cli.main()\n captured = capsys.readouterr()\n stdout_preset = yaml.safe_load(captured.out)\n assert stdout_preset[\"config\"][\"web\"][\"http_proxy\"] == \"http://proxy1\"\n\n # override other way\n monkeypatch.setattr(\n \"sys.argv\", [\"bbot\", \"-p\", str(preset1_file.resolve()), str(preset2_file.resolve()), \"--current-preset\"]\n )\n cli.main()\n captured = capsys.readouterr()\n stdout_preset = yaml.safe_load(captured.out)\n assert stdout_preset[\"config\"][\"web\"][\"http_proxy\"] == \"http://proxy2\"\n\n # --fast-mode\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--current-preset\"])\n cli.main()\n captured = capsys.readouterr()\n stdout_preset = yaml.safe_load(captured.out)\n assert list(stdout_preset) == [\"description\"]\n\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--fast\", \"--current-preset\"])\n cli.main()\n captured = capsys.readouterr()\n stdout_preset = yaml.safe_load(captured.out)\n stdout_preset.pop(\"description\")\n assert stdout_preset == {\n \"config\": {\n \"scope\": {\"strict\": True},\n \"dns\": {\"minimal\": True},\n \"modules\": {\"speculate\": {\"essential_only\": True}},\n },\n \"exclude_modules\": [\"excavate\"],\n }\n\n # --proxy\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"--proxy\", \"http://127.0.0.1:8080\", \"--current-preset\"])\n cli.main()\n captured = capsys.readouterr()\n stdout_preset = yaml.safe_load(captured.out)\n stdout_preset.pop(\"description\")\n assert stdout_preset == {\"config\": {\"web\": {\"http_proxy\": \"http://127.0.0.1:8080\"}}}\n\n # cli config overrides all presets\n monkeypatch.setattr(\n \"sys.argv\",\n [\n \"bbot\",\n \"-p\",\n str(preset1_file.resolve()),\n str(preset2_file.resolve()),\n \"-c\",\n \"web.http_proxy=asdf\",\n \"--current-preset\",\n ],\n )\n cli.main()\n captured = capsys.readouterr()\n stdout_preset = yaml.safe_load(captured.out)\n assert stdout_preset[\"config\"][\"web\"][\"http_proxy\"] == \"asdf\"\n\n # invalid preset\n caplog.clear()\n assert not caplog.text\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-p\", \"asdfasdfasdf\", \"-y\"])\n cli.main()\n assert \"file does not exist. Use -lp to list available presets\" in caplog.text\n\n preset1_file.unlink()\n preset2_file.unlink()\n\n # test output dir preset\n output_dir_preset_file = bbot_test_dir / \"output_dir_preset.yml\"\n scan_name = \"cli_output_dir_test\"\n output_dir = bbot_test_dir / \"cli_output_dir_preset\"\n scan_dir = output_dir / scan_name\n output_file = scan_dir / \"output.txt\"\n\n with open(output_dir_preset_file, \"w\") as f:\n f.write(\n f\"\"\"\noutput_dir: {output_dir}\nscan_name: {scan_name}\n \"\"\"\n )\n\n assert not output_dir.exists()\n assert not scan_dir.exists()\n assert not output_file.exists()\n\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-p\", str(output_dir_preset_file.resolve()), \"--current-preset\"])\n cli.main()\n captured = capsys.readouterr()\n stdout_preset = yaml.safe_load(captured.out)\n assert stdout_preset[\"output_dir\"] == str(output_dir)\n assert stdout_preset[\"scan_name\"] == scan_name\n\n shutil.rmtree(output_dir, ignore_errors=True)\n shutil.rmtree(scan_dir, ignore_errors=True)\n shutil.rmtree(output_file, ignore_errors=True)\n\n assert not output_dir.exists()\n assert not scan_dir.exists()\n assert not output_file.exists()\n\n monkeypatch.setattr(\"sys.argv\", [\"bbot\", \"-p\", str(output_dir_preset_file.resolve())])\n cli.main()\n captured = capsys.readouterr()\n assert output_dir.is_dir()\n assert scan_dir.is_dir()\n assert output_file.is_file()\n\n shutil.rmtree(output_dir, ignore_errors=True)\n shutil.rmtree(scan_dir, ignore_errors=True)\n shutil.rmtree(output_file, ignore_errors=True)\n output_dir_preset_file.unlink()\n" }, { "path": "bbot/test/test_step_1/test_command.py", "content": "import time\nfrom ..bbot_fixtures import *\nfrom subprocess import CalledProcessError\n\n\n@pytest.mark.asyncio\nasync def test_command(bbot_scanner):\n scan1 = bbot_scanner()\n\n # test timeouts\n command = [\"sleep\", \"3\"]\n start = time.time()\n with pytest.raises(asyncio.exceptions.TimeoutError):\n await scan1.helpers.run(command, idle_timeout=1)\n end = time.time()\n elapsed = end - start\n assert 0 < elapsed < 2\n\n start = time.time()\n with pytest.raises(asyncio.exceptions.TimeoutError):\n async for line in scan1.helpers.run_live(command, idle_timeout=1):\n print(line)\n end = time.time()\n elapsed = end - start\n assert 0 < elapsed < 2\n\n # run\n assert \"plumbus\\n\" == (await scan1.helpers.run([\"echo\", \"plumbus\"])).stdout\n assert b\"plumbus\\n\" == (await scan1.helpers.run([\"echo\", \"plumbus\"], text=False)).stdout\n result = (await scan1.helpers.run([\"cat\"], input=\"some\\nrandom\\nstdin\")).stdout\n assert result.splitlines() == [\"some\", \"random\", \"stdin\"]\n result = (await scan1.helpers.run([\"cat\"], input=b\"some\\nrandom\\nstdin\", text=False)).stdout\n assert result.splitlines() == [b\"some\", b\"random\", b\"stdin\"]\n result = (await scan1.helpers.run([\"cat\"], input=[\"some\", \"random\", \"stdin\"])).stdout\n assert result.splitlines() == [\"some\", \"random\", \"stdin\"]\n result = (await scan1.helpers.run([\"cat\"], input=[b\"some\", b\"random\", b\"stdin\"], text=False)).stdout\n assert result.splitlines() == [b\"some\", b\"random\", b\"stdin\"]\n\n # test overflow - run\n tmpfile_path = Path(\"/tmp/test_bigfile\")\n with open(tmpfile_path, \"w\") as f:\n # write 2MB\n f.write(\"A\" * 1024 * 1024 * 2)\n result = (await scan1.helpers.run([\"cat\", str(tmpfile_path)], limit=1024 * 64, text=False)).stdout\n assert len(result) == 1024 * 1024 * 2\n tmpfile_path.unlink(missing_ok=True)\n # test overflow - run_live\n tmpfile_path = Path(\"/tmp/test_bigfile\")\n with open(tmpfile_path, \"w\") as f:\n # write 2MB\n f.write(\"A\" * 10 + \"\\n\")\n f.write(\"B\" * 1024 * 1024 * 2 + \"\\n\")\n f.write(\"C\" * 10 + \"\\n\")\n lines = []\n async for line in scan1.helpers.run_live([\"cat\", str(tmpfile_path)], limit=1024 * 64):\n lines.append(line)\n # only a small bit of the overflowed line survives, that's okay.\n assert lines == [\"AAAAAAAAAA\", \"BBBBBBBBBBB\", \"CCCCCCCCCC\"]\n tmpfile_path.unlink(missing_ok=True)\n\n # run_live\n lines = []\n async for line in scan1.helpers.run_live([\"echo\", \"plumbus\"]):\n lines.append(line)\n assert lines == [\"plumbus\"]\n lines = []\n async for line in scan1.helpers.run_live([\"echo\", \"plumbus\"], text=False):\n lines.append(line)\n assert lines == [b\"plumbus\"]\n lines = []\n async for line in scan1.helpers.run_live([\"cat\"], input=\"some\\nrandom\\nstdin\"):\n lines.append(line)\n assert lines == [\"some\", \"random\", \"stdin\"]\n lines = []\n async for line in scan1.helpers.run_live([\"cat\"], input=[\"some\", \"random\", \"stdin\"]):\n lines.append(line)\n assert lines == [\"some\", \"random\", \"stdin\"]\n\n # test check=True\n with pytest.raises(CalledProcessError) as excinfo:\n lines = [line async for line in scan1.helpers.run_live([\"ls\", \"/aslkdjflasdkfsd\"], check=True)]\n assert \"No such file or directory\" in excinfo.value.stderr\n with pytest.raises(CalledProcessError) as excinfo:\n lines = [line async for line in scan1.helpers.run_live([\"ls\", \"/aslkdjflasdkfsd\"], check=True, text=False)]\n assert b\"No such file or directory\" in excinfo.value.stderr\n with pytest.raises(CalledProcessError) as excinfo:\n await scan1.helpers.run([\"ls\", \"/aslkdjflasdkfsd\"], check=True)\n assert \"No such file or directory\" in excinfo.value.stderr\n with pytest.raises(CalledProcessError) as excinfo:\n await scan1.helpers.run([\"ls\", \"/aslkdjflasdkfsd\"], check=True, text=False)\n assert b\"No such file or directory\" in excinfo.value.stderr\n\n # test piping\n lines = []\n async for line in scan1.helpers.run_live(\n [\"cat\"], input=scan1.helpers.run_live([\"echo\", \"-en\", r\"some\\nrandom\\nstdin\"])\n ):\n lines.append(line)\n assert lines == [\"some\", \"random\", \"stdin\"]\n lines = []\n async for line in scan1.helpers.run_live(\n [\"cat\"], input=scan1.helpers.run_live([\"echo\", \"-en\", r\"some\\nrandom\\nstdin\"], text=False), text=False\n ):\n lines.append(line)\n assert lines == [b\"some\", b\"random\", b\"stdin\"]\n\n # test missing executable\n result = await scan1.helpers.run([\"sgkjlskdfsdf\"])\n assert result is None\n lines = [l async for l in scan1.helpers.run_live([\"ljhsdghsdf\"])]\n assert not lines\n # test stderr\n result = await scan1.helpers.run([\"ls\", \"/sldikgjasldkfsdf\"])\n assert \"No such file or directory\" in result.stderr\n lines = [l async for l in scan1.helpers.run_live([\"ls\", \"/sldikgjasldkfsdf\"])]\n assert not lines\n\n # test sudo + existence of environment variables\n await scan1.load_modules()\n path_parts = os.environ.get(\"PATH\", \"\").split(\":\")\n assert \"/tmp/.bbot_test/tools\" in path_parts\n run_lines = (await scan1.helpers.run([\"env\"])).stdout.splitlines()\n assert \"BBOT_WEB_USER_AGENT=BBOT Test User-Agent\" in run_lines\n for line in run_lines:\n if line.startswith(\"PATH=\"):\n path_parts = line.split(\"=\", 1)[-1].split(\":\")\n assert \"/tmp/.bbot_test/tools\" in path_parts\n run_lines_sudo = (await scan1.helpers.run([\"env\"], sudo=True)).stdout.splitlines()\n assert \"BBOT_WEB_USER_AGENT=BBOT Test User-Agent\" in run_lines_sudo\n for line in run_lines_sudo:\n if line.startswith(\"PATH=\"):\n path_parts = line.split(\"=\", 1)[-1].split(\":\")\n assert \"/tmp/.bbot_test/tools\" in path_parts\n run_live_lines = [l async for l in scan1.helpers.run_live([\"env\"])]\n assert \"BBOT_WEB_USER_AGENT=BBOT Test User-Agent\" in run_live_lines\n for line in run_live_lines:\n if line.startswith(\"PATH=\"):\n path_parts = line.strip().split(\"=\", 1)[-1].split(\":\")\n assert \"/tmp/.bbot_test/tools\" in path_parts\n run_live_lines_sudo = [l async for l in scan1.helpers.run_live([\"env\"], sudo=True)]\n assert \"BBOT_WEB_USER_AGENT=BBOT Test User-Agent\" in run_live_lines_sudo\n for line in run_live_lines_sudo:\n if line.startswith(\"PATH=\"):\n path_parts = line.strip().split(\"=\", 1)[-1].split(\":\")\n assert \"/tmp/.bbot_test/tools\" in path_parts\n\n await scan1._cleanup()\n" }, { "path": "bbot/test/test_step_1/test_config.py", "content": "from ..bbot_fixtures import * # noqa: F401\n\n\n@pytest.mark.asyncio\nasync def test_config(bbot_scanner):\n config = OmegaConf.create(\n {\n \"plumbus\": \"asdf\",\n \"speculate\": True,\n \"modules\": {\n \"ipneighbor\": {\"test_option\": \"ipneighbor\"},\n \"python\": {\"test_option\": \"asdf\"},\n \"speculate\": {\"test_option\": \"speculate\"},\n },\n }\n )\n scan1 = bbot_scanner(\"127.0.0.1\", modules=[\"ipneighbor\"], config=config)\n await scan1.load_modules()\n assert scan1.config.web.user_agent == \"BBOT Test User-Agent\"\n assert scan1.config.plumbus == \"asdf\"\n assert scan1.modules[\"ipneighbor\"].config.test_option == \"ipneighbor\"\n assert scan1.modules[\"python\"].config.test_option == \"asdf\"\n assert scan1.modules[\"speculate\"].config.test_option == \"speculate\"\n\n await scan1._cleanup()\n" }, { "path": "bbot/test/test_step_1/test_depsinstaller.py", "content": "from ..bbot_fixtures import *\n\n\n@pytest.mark.asyncio\nasync def test_depsinstaller(monkeypatch, bbot_scanner):\n scan = bbot_scanner(\n \"127.0.0.1\",\n )\n\n # test shell\n test_file = Path(\"/tmp/test_file\")\n test_file.unlink(missing_ok=True)\n scan.helpers.depsinstaller.shell(module=\"plumbus\", commands=[f\"touch {test_file}\"])\n assert test_file.is_file()\n test_file.unlink(missing_ok=True)\n\n # test tasks\n scan.helpers.depsinstaller.tasks(\n module=\"plumbus\",\n tasks=[{\"name\": \"test task execution\", \"ansible.builtin.shell\": {\"cmd\": f\"touch {test_file}\"}}],\n )\n assert test_file.is_file()\n test_file.unlink(missing_ok=True)\n\n await scan._cleanup()\n" }, { "path": "bbot/test/test_step_1/test_dns.py", "content": "from ..bbot_fixtures import *\n\nfrom bbot.core.helpers.dns.helpers import extract_targets, service_record, common_srvs\n\n\nmock_records = {\n \"one.one.one.one\": {\n \"A\": [\"1.1.1.1\", \"1.0.0.1\"],\n \"AAAA\": [\"2606:4700:4700::1111\", \"2606:4700:4700::1001\"],\n \"TXT\": [\n '\"v=spf1 ip4:103.151.192.0/23 ip4:185.12.80.0/22 ip4:188.172.128.0/20 ip4:192.161.144.0/20 ip4:216.198.0.0/18 ~all\"'\n ],\n },\n \"1.1.1.1.in-addr.arpa\": {\"PTR\": [\"one.one.one.one.\"]},\n}\n\n\n@pytest.mark.asyncio\nasync def test_dns_engine(bbot_scanner):\n scan = bbot_scanner()\n await scan.helpers._mock_dns(\n {\"one.one.one.one\": {\"A\": [\"1.1.1.1\"]}, \"1.1.1.1.in-addr.arpa\": {\"PTR\": [\"one.one.one.one\"]}}\n )\n result = await scan.helpers.resolve(\"one.one.one.one\")\n assert \"1.1.1.1\" in result\n assert \"2606:4700:4700::1111\" not in result\n\n results = [_ async for _ in scan.helpers.resolve_batch((\"one.one.one.one\", \"1.1.1.1\"))]\n pass_1 = False\n pass_2 = False\n for query, result in results:\n if query == \"one.one.one.one\" and \"1.1.1.1\" in result:\n pass_1 = True\n elif query == \"1.1.1.1\" and \"one.one.one.one\" in result:\n pass_2 = True\n assert pass_1 and pass_2\n\n results = [_ async for _ in scan.helpers.resolve_raw_batch(((\"one.one.one.one\", \"A\"), (\"1.1.1.1\", \"PTR\")))]\n pass_1 = False\n pass_2 = False\n for (query, rdtype), (answers, errors) in results:\n results = []\n for answer in answers:\n for t in extract_targets(answer):\n results.append(t[1])\n if query == \"one.one.one.one\" and \"1.1.1.1\" in results:\n pass_1 = True\n elif query == \"1.1.1.1\" and \"one.one.one.one\" in results:\n pass_2 = True\n assert pass_1 and pass_2\n\n from bbot.core.helpers.dns.mock import MockResolver\n\n # ensure dns records are being properly cleaned\n mockresolver = MockResolver({\"evilcorp.com\": {\"MX\": [\"0 .\"]}})\n mx_records = await mockresolver.resolve(\"evilcorp.com\", rdtype=\"MX\")\n results = set()\n for r in mx_records:\n results.update(extract_targets(r))\n assert not results\n\n await scan._cleanup()\n\n\n@pytest.mark.asyncio\nasync def test_dns_resolution(bbot_scanner):\n scan = bbot_scanner(\"1.1.1.1\")\n\n from bbot.core.helpers.dns.engine import DNSEngine\n\n dnsengine = DNSEngine(None)\n await dnsengine._mock_dns(mock_records)\n\n # lowest level functions\n a_responses = await dnsengine._resolve_hostname(\"one.one.one.one\")\n aaaa_responses = await dnsengine._resolve_hostname(\"one.one.one.one\", rdtype=\"AAAA\")\n ip_responses = await dnsengine._resolve_ip(\"1.1.1.1\")\n assert a_responses[0].response.answer[0][0].address in (\"1.1.1.1\", \"1.0.0.1\")\n assert aaaa_responses[0].response.answer[0][0].address in (\"2606:4700:4700::1111\", \"2606:4700:4700::1001\")\n assert ip_responses[0].response.answer[0][0].target.to_text() in (\"one.one.one.one.\",)\n\n # mid level functions\n answers, errors = await dnsengine.resolve_raw(\"one.one.one.one\", type=\"A\")\n responses = []\n for answer in answers:\n responses += list(extract_targets(answer))\n assert (\"A\", \"1.1.1.1\") in responses\n assert (\"AAAA\", \"2606:4700:4700::1111\") not in responses\n answers, errors = await dnsengine.resolve_raw(\"one.one.one.one\", type=\"AAAA\")\n responses = []\n for answer in answers:\n responses += list(extract_targets(answer))\n assert (\"A\", \"1.1.1.1\") not in responses\n assert (\"AAAA\", \"2606:4700:4700::1111\") in responses\n answers, errors = await dnsengine.resolve_raw(\"1.1.1.1\")\n responses = []\n for answer in answers:\n responses += list(extract_targets(answer))\n assert (\"PTR\", \"one.one.one.one\") in responses\n\n await dnsengine._shutdown()\n\n # high level functions\n dnsengine = DNSEngine(None)\n assert \"1.1.1.1\" in await dnsengine.resolve(\"one.one.one.one\")\n assert \"2606:4700:4700::1111\" in await dnsengine.resolve(\"one.one.one.one\", type=\"AAAA\")\n assert \"one.one.one.one\" in await dnsengine.resolve(\"1.1.1.1\")\n for rdtype in (\"NS\", \"SOA\", \"MX\", \"TXT\"):\n results = await dnsengine.resolve(\"google.com\", type=rdtype)\n assert len(results) > 0\n\n # batch resolution\n batch_results = [r async for r in dnsengine.resolve_batch([\"1.1.1.1\", \"one.one.one.one\"])]\n assert len(batch_results) == 2\n batch_results = dict(batch_results)\n assert any(x in batch_results[\"one.one.one.one\"] for x in (\"1.1.1.1\", \"1.0.0.1\"))\n assert \"one.one.one.one\" in batch_results[\"1.1.1.1\"]\n\n # custom batch resolution\n batch_results = [r async for r in dnsengine.resolve_raw_batch([(\"1.1.1.1\", \"PTR\"), (\"one.one.one.one\", \"A\")])]\n batch_results_new = []\n for query, (answers, errors) in batch_results:\n for answer in answers:\n batch_results_new.append((answer.to_text(), answer.rdtype.name))\n assert len(batch_results_new) == 3\n assert any(answer == \"1.0.0.1\" and rdtype == \"A\" for answer, rdtype in batch_results_new)\n assert any(answer == \"one.one.one.one.\" and rdtype == \"PTR\" for answer, rdtype in batch_results_new)\n\n # dns cache\n dnsengine._dns_cache.clear()\n assert hash((\"1.1.1.1\", \"PTR\")) not in dnsengine._dns_cache\n assert hash((\"one.one.one.one\", \"A\")) not in dnsengine._dns_cache\n assert hash((\"one.one.one.one\", \"AAAA\")) not in dnsengine._dns_cache\n await dnsengine.resolve(\"1.1.1.1\", use_cache=False)\n await dnsengine.resolve(\"one.one.one.one\", use_cache=False)\n assert hash((\"1.1.1.1\", \"PTR\")) not in dnsengine._dns_cache\n assert hash((\"one.one.one.one\", \"A\")) not in dnsengine._dns_cache\n assert hash((\"one.one.one.one\", \"AAAA\")) not in dnsengine._dns_cache\n\n await dnsengine.resolve(\"1.1.1.1\")\n assert hash((\"1.1.1.1\", \"PTR\")) in dnsengine._dns_cache\n await dnsengine.resolve(\"one.one.one.one\", type=\"A\")\n assert hash((\"one.one.one.one\", \"A\")) in dnsengine._dns_cache\n assert hash((\"one.one.one.one\", \"AAAA\")) not in dnsengine._dns_cache\n dnsengine._dns_cache.clear()\n await dnsengine.resolve(\"one.one.one.one\", type=\"AAAA\")\n assert hash((\"one.one.one.one\", \"AAAA\")) in dnsengine._dns_cache\n assert hash((\"one.one.one.one\", \"A\")) not in dnsengine._dns_cache\n\n await dnsengine._shutdown()\n\n # Ensure events with hosts have resolved_hosts attribute populated\n await scan._prep()\n resolved_hosts_event1 = scan.make_event(\"one.one.one.one\", \"DNS_NAME\", parent=scan.root_event)\n resolved_hosts_event2 = scan.make_event(\"http://one.one.one.one/\", \"URL_UNVERIFIED\", parent=scan.root_event)\n dnsresolve = scan.modules[\"dnsresolve\"]\n await dnsresolve.handle_event(resolved_hosts_event1)\n await dnsresolve.handle_event(resolved_hosts_event2)\n assert \"1.1.1.1\" in resolved_hosts_event2.resolved_hosts\n # URL event should not have dns_children\n assert not resolved_hosts_event2.dns_children\n assert resolved_hosts_event1.resolved_hosts == resolved_hosts_event2.resolved_hosts\n # DNS_NAME event should have dns_children\n assert \"1.1.1.1\" in resolved_hosts_event1.dns_children[\"A\"]\n assert \"A\" in resolved_hosts_event1.raw_dns_records\n assert \"AAAA\" in resolved_hosts_event1.raw_dns_records\n assert \"a-record\" in resolved_hosts_event1.tags\n assert \"a-record\" not in resolved_hosts_event2.tags\n\n scan2 = bbot_scanner(\"evilcorp.com\", config={\"dns\": {\"minimal\": False}})\n await scan2.helpers.dns._mock_dns(\n {\n \"evilcorp.com\": {\"TXT\": ['\"v=spf1 include:cloudprovider.com ~all\"']},\n \"cloudprovider.com\": {\"A\": [\"1.2.3.4\"]},\n },\n )\n events = [e async for e in scan2.async_start()]\n assert 1 == len(\n [e for e in events if e.type == \"DNS_NAME\" and e.data == \"cloudprovider.com\" and \"affiliate\" in e.tags]\n )\n\n await scan._cleanup()\n await scan2._cleanup()\n\n\n@pytest.mark.asyncio\nasync def test_wildcards(bbot_scanner):\n scan = bbot_scanner(\"1.1.1.1\")\n helpers = scan.helpers\n\n from bbot.core.helpers.dns.engine import DNSEngine, all_rdtypes\n\n dnsengine = DNSEngine(None, debug=True)\n\n # is_wildcard_domain\n wildcard_domains = await dnsengine.is_wildcard_domain(\"asdf.github.io\", all_rdtypes)\n assert len(dnsengine._wildcard_cache) == len(all_rdtypes) + (len(all_rdtypes) - 2)\n for rdtype in all_rdtypes:\n assert hash((\"github.io\", rdtype)) in dnsengine._wildcard_cache\n if rdtype not in (\"A\", \"AAAA\"):\n assert hash((\"asdf.github.io\", rdtype)) in dnsengine._wildcard_cache\n assert \"github.io\" in wildcard_domains\n assert \"A\" in wildcard_domains[\"github.io\"]\n assert \"SRV\" not in wildcard_domains[\"github.io\"]\n assert wildcard_domains[\"github.io\"][\"A\"] and all(helpers.is_ip(r) for r in wildcard_domains[\"github.io\"][\"A\"][0])\n dnsengine._wildcard_cache.clear()\n\n # is_wildcard\n for test_domain in (\"blacklanternsecurity.github.io\", \"asdf.asdf.asdf.github.io\"):\n wildcard_rdtypes = await dnsengine.is_wildcard(test_domain, all_rdtypes)\n assert \"A\" in wildcard_rdtypes\n assert \"SRV\" not in wildcard_rdtypes\n assert wildcard_rdtypes[\"A\"] == (True, \"github.io\")\n assert wildcard_rdtypes[\"AAAA\"] == (True, \"github.io\")\n assert len(dnsengine._wildcard_cache) == 2\n for rdtype in (\"A\", \"AAAA\"):\n assert hash((\"github.io\", rdtype)) in dnsengine._wildcard_cache\n assert len(dnsengine._wildcard_cache[hash((\"github.io\", rdtype))]) == 2\n assert len(dnsengine._wildcard_cache[hash((\"github.io\", rdtype))][0]) > 0\n assert len(dnsengine._wildcard_cache[hash((\"github.io\", rdtype))][1]) > 0\n dnsengine._wildcard_cache.clear()\n\n ### wildcard TXT record ###\n\n custom_lookup = \"\"\"\ndef custom_lookup(query, rdtype):\n if rdtype == \"TXT\" and query.strip(\".\").endswith(\"test.evilcorp.com\"):\n return {\"\"}\n\"\"\"\n\n mock_data = {\n \"evilcorp.com\": {\"A\": [\"127.0.0.1\"]},\n \"test.evilcorp.com\": {\"A\": [\"127.0.0.2\"]},\n \"www.test.evilcorp.com\": {\"AAAA\": [\"dead::beef\"]},\n }\n\n # basic sanity checks\n\n await dnsengine._mock_dns(mock_data, custom_lookup_fn=custom_lookup)\n\n a_result = await dnsengine.resolve(\"evilcorp.com\")\n assert a_result == {\"127.0.0.1\"}\n aaaa_result = await dnsengine.resolve(\"www.test.evilcorp.com\", type=\"AAAA\")\n assert aaaa_result == {\"dead::beef\"}\n txt_result = await dnsengine.resolve(\"asdf.www.test.evilcorp.com\", type=\"TXT\")\n assert txt_result == set()\n txt_result_raw, errors = await dnsengine.resolve_raw(\"asdf.www.test.evilcorp.com\", type=\"TXT\")\n txt_result_raw = list(txt_result_raw)\n assert txt_result_raw\n\n await dnsengine._shutdown()\n\n # first, we check with wildcard detection disabled\n\n scan = bbot_scanner(\n \"bbot.fdsa.www.test.evilcorp.com\",\n whitelist=[\"evilcorp.com\"],\n config={\n \"dns\": {\"minimal\": False, \"disable\": False, \"search_distance\": 5, \"wildcard_ignore\": [\"evilcorp.com\"]},\n \"speculate\": True,\n },\n )\n await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)\n\n events = [e async for e in scan.async_start()]\n assert len(events) == 12\n assert len([e for e in events if e.type == \"DNS_NAME\"]) == 5\n assert len([e for e in events if e.type == \"RAW_DNS_RECORD\"]) == 4\n assert sorted([e.data for e in events if e.type == \"DNS_NAME\"]) == [\n \"bbot.fdsa.www.test.evilcorp.com\",\n \"evilcorp.com\",\n \"fdsa.www.test.evilcorp.com\",\n \"test.evilcorp.com\",\n \"www.test.evilcorp.com\",\n ]\n\n dns_names_by_host = {e.host: e for e in events if e.type == \"DNS_NAME\"}\n assert dns_names_by_host[\"evilcorp.com\"].tags == {\"domain\", \"private-ip\", \"in-scope\", \"a-record\"}\n assert dns_names_by_host[\"evilcorp.com\"].resolved_hosts == {\"127.0.0.1\"}\n assert dns_names_by_host[\"test.evilcorp.com\"].tags == {\n \"subdomain\",\n \"private-ip\",\n \"in-scope\",\n \"a-record\",\n \"txt-record\",\n }\n assert dns_names_by_host[\"test.evilcorp.com\"].resolved_hosts == {\"127.0.0.2\"}\n assert dns_names_by_host[\"www.test.evilcorp.com\"].tags == {\"subdomain\", \"in-scope\", \"aaaa-record\", \"txt-record\"}\n assert dns_names_by_host[\"www.test.evilcorp.com\"].resolved_hosts == {\"dead::beef\"}\n assert dns_names_by_host[\"fdsa.www.test.evilcorp.com\"].tags == {\"subdomain\", \"in-scope\", \"txt-record\"}\n assert dns_names_by_host[\"fdsa.www.test.evilcorp.com\"].resolved_hosts == set()\n assert dns_names_by_host[\"bbot.fdsa.www.test.evilcorp.com\"].tags == {\n \"target\",\n \"subdomain\",\n \"in-scope\",\n \"txt-record\",\n }\n assert dns_names_by_host[\"bbot.fdsa.www.test.evilcorp.com\"].resolved_hosts == set()\n\n raw_records_by_host = {e.host: e for e in events if e.type == \"RAW_DNS_RECORD\"}\n assert raw_records_by_host[\"test.evilcorp.com\"].tags == {\"subdomain\", \"in-scope\", \"txt-record\"}\n assert raw_records_by_host[\"test.evilcorp.com\"].resolved_hosts == {\"127.0.0.2\"}\n assert raw_records_by_host[\"www.test.evilcorp.com\"].tags == {\"subdomain\", \"in-scope\", \"txt-record\"}\n assert raw_records_by_host[\"www.test.evilcorp.com\"].resolved_hosts == {\"dead::beef\"}\n assert raw_records_by_host[\"fdsa.www.test.evilcorp.com\"].tags == {\"subdomain\", \"in-scope\", \"txt-record\"}\n assert raw_records_by_host[\"fdsa.www.test.evilcorp.com\"].resolved_hosts == set()\n assert raw_records_by_host[\"bbot.fdsa.www.test.evilcorp.com\"].tags == {\"subdomain\", \"in-scope\", \"txt-record\"}\n assert raw_records_by_host[\"bbot.fdsa.www.test.evilcorp.com\"].resolved_hosts == set()\n\n # then we run it again with wildcard detection enabled\n\n scan = bbot_scanner(\n \"bbot.fdsa.www.test.evilcorp.com\",\n whitelist=[\"evilcorp.com\"],\n config={\n \"dns\": {\"minimal\": False, \"disable\": False, \"search_distance\": 5, \"wildcard_ignore\": []},\n \"speculate\": True,\n },\n )\n await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)\n\n events = [e async for e in scan.async_start()]\n assert len(events) == 12\n assert len([e for e in events if e.type == \"DNS_NAME\"]) == 5\n assert len([e for e in events if e.type == \"RAW_DNS_RECORD\"]) == 4\n assert sorted([e.data for e in events if e.type == \"DNS_NAME\"]) == [\n \"_wildcard.test.evilcorp.com\",\n \"bbot.fdsa.www.test.evilcorp.com\",\n \"evilcorp.com\",\n \"test.evilcorp.com\",\n \"www.test.evilcorp.com\",\n ]\n\n dns_names_by_host = {e.host: e for e in events if e.type == \"DNS_NAME\"}\n assert dns_names_by_host[\"evilcorp.com\"].tags == {\"domain\", \"private-ip\", \"in-scope\", \"a-record\"}\n assert dns_names_by_host[\"evilcorp.com\"].resolved_hosts == {\"127.0.0.1\"}\n assert dns_names_by_host[\"test.evilcorp.com\"].tags == {\n \"subdomain\",\n \"private-ip\",\n \"in-scope\",\n \"a-record\",\n \"txt-record\",\n }\n assert dns_names_by_host[\"test.evilcorp.com\"].resolved_hosts == {\"127.0.0.2\"}\n assert dns_names_by_host[\"_wildcard.test.evilcorp.com\"].tags == {\n \"subdomain\",\n \"in-scope\",\n \"txt-record\",\n \"txt-wildcard\",\n \"wildcard\",\n }\n assert dns_names_by_host[\"_wildcard.test.evilcorp.com\"].resolved_hosts == set()\n assert dns_names_by_host[\"www.test.evilcorp.com\"].tags == {\n \"subdomain\",\n \"in-scope\",\n \"aaaa-record\",\n \"txt-record\",\n \"txt-wildcard\",\n \"wildcard\",\n }\n assert dns_names_by_host[\"www.test.evilcorp.com\"].resolved_hosts == {\"dead::beef\"}\n assert dns_names_by_host[\"bbot.fdsa.www.test.evilcorp.com\"].tags == {\n \"target\",\n \"subdomain\",\n \"in-scope\",\n \"txt-record\",\n \"txt-wildcard\",\n \"wildcard\",\n }\n assert dns_names_by_host[\"bbot.fdsa.www.test.evilcorp.com\"].resolved_hosts == set()\n\n raw_records_by_host = {e.host: e for e in events if e.type == \"RAW_DNS_RECORD\"}\n assert raw_records_by_host[\"test.evilcorp.com\"].tags == {\"subdomain\", \"in-scope\", \"txt-record\"}\n assert raw_records_by_host[\"test.evilcorp.com\"].resolved_hosts == {\"127.0.0.2\"}\n assert raw_records_by_host[\"www.test.evilcorp.com\"].tags == {\"subdomain\", \"in-scope\", \"txt-record\", \"txt-wildcard\"}\n assert raw_records_by_host[\"www.test.evilcorp.com\"].resolved_hosts == {\"dead::beef\"}\n assert raw_records_by_host[\"_wildcard.test.evilcorp.com\"].tags == {\n \"subdomain\",\n \"in-scope\",\n \"txt-record\",\n \"txt-wildcard\",\n }\n assert raw_records_by_host[\"_wildcard.test.evilcorp.com\"].resolved_hosts == set()\n assert raw_records_by_host[\"bbot.fdsa.www.test.evilcorp.com\"].tags == {\n \"subdomain\",\n \"in-scope\",\n \"txt-record\",\n \"txt-wildcard\",\n }\n assert raw_records_by_host[\"bbot.fdsa.www.test.evilcorp.com\"].resolved_hosts == set()\n\n ### runaway SRV wildcard ###\n\n custom_lookup = \"\"\"\ndef custom_lookup(query, rdtype):\n if rdtype == \"SRV\" and query.strip(\".\").endswith(\"evilcorp.com\"):\n return {f\"0 100 389 test.{query}\"}\n\"\"\"\n\n mock_data = {\n \"evilcorp.com\": {\"A\": [\"127.0.0.1\"]},\n \"test.evilcorp.com\": {\"AAAA\": [\"dead::beef\"]},\n }\n\n scan = bbot_scanner(\n \"evilcorp.com\",\n config={\n \"dns\": {\n \"minimal\": False,\n \"disable\": False,\n \"search_distance\": 5,\n \"wildcard_ignore\": [],\n \"runaway_limit\": 3,\n },\n },\n )\n await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)\n\n events = [e async for e in scan.async_start()]\n\n assert len(events) == 11\n assert len([e for e in events if e.type == \"DNS_NAME\"]) == 5\n assert len([e for e in events if e.type == \"RAW_DNS_RECORD\"]) == 4\n assert sorted([e.data for e in events if e.type == \"DNS_NAME\"]) == [\n \"evilcorp.com\",\n \"test.evilcorp.com\",\n \"test.test.evilcorp.com\",\n \"test.test.test.evilcorp.com\",\n \"test.test.test.test.evilcorp.com\",\n ]\n\n dns_names_by_host = {e.host: e for e in events if e.type == \"DNS_NAME\"}\n assert dns_names_by_host[\"evilcorp.com\"].tags == {\n \"target\",\n \"a-record\",\n \"in-scope\",\n \"domain\",\n \"srv-record\",\n \"private-ip\",\n }\n assert dns_names_by_host[\"test.evilcorp.com\"].tags == {\n \"in-scope\",\n \"srv-record\",\n \"aaaa-record\",\n \"srv-wildcard-possible\",\n \"wildcard-possible\",\n \"subdomain\",\n }\n assert dns_names_by_host[\"test.test.evilcorp.com\"].tags == {\n \"in-scope\",\n \"srv-record\",\n \"srv-wildcard-possible\",\n \"wildcard-possible\",\n \"subdomain\",\n }\n assert dns_names_by_host[\"test.test.test.evilcorp.com\"].tags == {\n \"in-scope\",\n \"srv-record\",\n \"srv-wildcard-possible\",\n \"wildcard-possible\",\n \"subdomain\",\n }\n assert dns_names_by_host[\"test.test.test.test.evilcorp.com\"].tags == {\n \"in-scope\",\n \"srv-record\",\n \"srv-wildcard-possible\",\n \"wildcard-possible\",\n \"subdomain\",\n \"runaway-dns-3\",\n }\n\n raw_records_by_host = {e.host: e for e in events if e.type == \"RAW_DNS_RECORD\"}\n assert raw_records_by_host[\"evilcorp.com\"].tags == {\"in-scope\", \"srv-record\", \"domain\"}\n assert raw_records_by_host[\"test.evilcorp.com\"].tags == {\n \"in-scope\",\n \"srv-record\",\n \"srv-wildcard-possible\",\n \"subdomain\",\n }\n assert raw_records_by_host[\"test.test.evilcorp.com\"].tags == {\n \"in-scope\",\n \"srv-record\",\n \"srv-wildcard-possible\",\n \"subdomain\",\n }\n assert raw_records_by_host[\"test.test.test.evilcorp.com\"].tags == {\n \"in-scope\",\n \"srv-record\",\n \"srv-wildcard-possible\",\n \"subdomain\",\n }\n\n scan = bbot_scanner(\"1.1.1.1\")\n helpers = scan.helpers\n\n # event resolution\n wildcard_event1 = scan.make_event(\"wat.asdf.fdsa.github.io\", \"DNS_NAME\", parent=scan.root_event)\n wildcard_event1.scope_distance = 0\n wildcard_event2 = scan.make_event(\"wats.asd.fdsa.github.io\", \"DNS_NAME\", parent=scan.root_event)\n wildcard_event2.scope_distance = 0\n wildcard_event3 = scan.make_event(\"github.io\", \"DNS_NAME\", parent=scan.root_event)\n wildcard_event3.scope_distance = 0\n\n await scan._prep()\n dnsresolve = scan.modules[\"dnsresolve\"]\n await dnsresolve.handle_event(wildcard_event1)\n await dnsresolve.handle_event(wildcard_event2)\n await dnsresolve.handle_event(wildcard_event3)\n assert \"wildcard\" in wildcard_event1.tags\n assert \"a-wildcard\" in wildcard_event1.tags\n assert \"srv-wildcard\" not in wildcard_event1.tags\n assert \"wildcard\" in wildcard_event2.tags\n assert \"a-wildcard\" in wildcard_event2.tags\n assert \"srv-wildcard\" not in wildcard_event2.tags\n assert wildcard_event1.data == \"_wildcard.github.io\"\n assert wildcard_event2.data == \"_wildcard.github.io\"\n assert wildcard_event3.data == \"github.io\"\n\n # dns resolve distance\n event_distance_0 = scan.make_event(\n \"8.8.8.8\", module=scan.modules[\"dnsresolve\"]._make_dummy_module(\"PTR\"), parent=scan.root_event\n )\n assert event_distance_0.dns_resolve_distance == 0\n event_distance_1 = scan.make_event(\n \"evilcorp.com\", module=scan.modules[\"dnsresolve\"]._make_dummy_module(\"A\"), parent=event_distance_0\n )\n assert event_distance_1.dns_resolve_distance == 1\n event_distance_2 = scan.make_event(\n \"1.2.3.4\", module=scan.modules[\"dnsresolve\"]._make_dummy_module(\"PTR\"), parent=event_distance_1\n )\n assert event_distance_2.dns_resolve_distance == 1\n event_distance_3 = scan.make_event(\n \"evilcorp.org\", module=scan.modules[\"dnsresolve\"]._make_dummy_module(\"A\"), parent=event_distance_2\n )\n assert event_distance_3.dns_resolve_distance == 2\n\n await scan._cleanup()\n\n from bbot.scanner import Scanner\n\n # test with full scan\n scan2 = Scanner(\"asdfl.gashdgkjsadgsdf.github.io\", whitelist=[\"github.io\"], config={\"dns\": {\"minimal\": False}})\n await scan2._prep()\n other_event = scan2.make_event(\n \"lkjg.sdfgsg.jgkhajshdsadf.github.io\", module=scan2.modules[\"dnsresolve\"], parent=scan2.root_event\n )\n await scan2.ingress_module.queue_event(other_event, {})\n events = [e async for e in scan2.async_start()]\n assert len(events) == 4\n assert 2 == len([e for e in events if e.type == \"SCAN\"])\n unmodified_wildcard_events = [\n e for e in events if e.type == \"DNS_NAME\" and e.data == \"asdfl.gashdgkjsadgsdf.github.io\"\n ]\n assert len(unmodified_wildcard_events) == 1\n assert unmodified_wildcard_events[0].tags.issuperset(\n {\n \"a-record\",\n \"target\",\n \"aaaa-wildcard\",\n \"in-scope\",\n \"subdomain\",\n \"aaaa-record\",\n \"wildcard\",\n \"a-wildcard\",\n }\n )\n modified_wildcard_events = [e for e in events if e.type == \"DNS_NAME\" and e.data == \"_wildcard.github.io\"]\n assert len(modified_wildcard_events) == 1\n assert modified_wildcard_events[0].tags.issuperset(\n {\n \"a-record\",\n \"aaaa-wildcard\",\n \"in-scope\",\n \"subdomain\",\n \"aaaa-record\",\n \"wildcard\",\n \"a-wildcard\",\n }\n )\n assert modified_wildcard_events[0].host_original == \"lkjg.sdfgsg.jgkhajshdsadf.github.io\"\n\n # test with full scan (wildcard detection disabled for domain)\n scan2 = Scanner(\n \"asdfl.gashdgkjsadgsdf.github.io\",\n whitelist=[\"github.io\"],\n config={\"dns\": {\"wildcard_ignore\": [\"github.io\"]}},\n exclude_modules=[\"cloudcheck\"],\n )\n await scan2._prep()\n other_event = scan2.make_event(\n \"lkjg.sdfgsg.jgkhajshdsadf.github.io\", module=scan2.modules[\"dnsresolve\"], parent=scan2.root_event\n )\n await scan2.ingress_module.queue_event(other_event, {})\n events = [e async for e in scan2.async_start()]\n assert len(events) == 4\n assert 2 == len([e for e in events if e.type == \"SCAN\"])\n unmodified_wildcard_events = [e for e in events if e.type == \"DNS_NAME\" and \"_wildcard\" not in e.data]\n assert len(unmodified_wildcard_events) == 2\n assert 1 == len(\n [\n e\n for e in unmodified_wildcard_events\n if e.data == \"asdfl.gashdgkjsadgsdf.github.io\"\n and e.tags.issuperset(\n {\n \"target\",\n \"a-record\",\n \"in-scope\",\n \"subdomain\",\n \"aaaa-record\",\n }\n )\n ]\n )\n assert 1 == len(\n [\n e\n for e in unmodified_wildcard_events\n if e.data == \"lkjg.sdfgsg.jgkhajshdsadf.github.io\"\n and e.tags.issuperset(\n {\n \"a-record\",\n \"in-scope\",\n \"subdomain\",\n \"aaaa-record\",\n }\n )\n ]\n )\n modified_wildcard_events = [e for e in events if e.type == \"DNS_NAME\" and e.data == \"_wildcard.github.io\"]\n assert len(modified_wildcard_events) == 0\n\n\n@pytest.mark.asyncio\nasync def test_wildcard_deduplication(bbot_scanner):\n custom_lookup = \"\"\"\ndef custom_lookup(query, rdtype):\n if rdtype == \"TXT\" and query.strip(\".\").endswith(\"evilcorp.com\"):\n return {\"\"}\n\"\"\"\n\n mock_data = {\n \"evilcorp.com\": {\"A\": [\"127.0.0.1\"]},\n }\n\n from bbot.modules.base import BaseModule\n\n class DummyModule(BaseModule):\n watched_events = [\"DNS_NAME\"]\n per_domain_only = True\n\n async def handle_event(self, event):\n for i in range(30):\n await self.emit_event(f\"www{i}.evilcorp.com\", \"DNS_NAME\", parent=event)\n\n # scan without omitted event type\n scan = bbot_scanner(\n \"evilcorp.com\", config={\"dns\": {\"minimal\": False, \"wildcard_ignore\": []}, \"omit_event_types\": []}\n )\n await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)\n dummy_module = DummyModule(scan)\n scan.modules[\"dummy_module\"] = dummy_module\n events = [e async for e in scan.async_start()]\n dns_name_events = [e for e in events if e.type == \"DNS_NAME\"]\n assert len(dns_name_events) == 2\n assert 1 == len([e for e in dns_name_events if e.data == \"_wildcard.evilcorp.com\"])\n\n\n@pytest.mark.asyncio\nasync def test_dns_raw_records(bbot_scanner):\n from bbot.modules.base import BaseModule\n\n class DummyModule(BaseModule):\n watched_events = [\"*\"]\n\n async def setup(self):\n self.events = []\n return True\n\n async def handle_event(self, event):\n self.events.append(event)\n\n # scan without omitted event type\n scan = bbot_scanner(\"one.one.one.one\", \"1.1.1.1\", config={\"dns\": {\"minimal\": False}, \"omit_event_types\": []})\n await scan.helpers.dns._mock_dns(mock_records)\n dummy_module = DummyModule(scan)\n scan.modules[\"dummy_module\"] = dummy_module\n events = [e async for e in scan.async_start()]\n assert 1 == len([e for e in events if e.type == \"RAW_DNS_RECORD\"])\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"RAW_DNS_RECORD\"\n and e.host == \"one.one.one.one\"\n and e.data[\"host\"] == \"one.one.one.one\"\n and e.data[\"type\"] == \"TXT\"\n and e.data[\"answer\"]\n == '\"v=spf1 ip4:103.151.192.0/23 ip4:185.12.80.0/22 ip4:188.172.128.0/20 ip4:192.161.144.0/20 ip4:216.198.0.0/18 ~all\"'\n and e.discovery_context == \"TXT lookup on one.one.one.one produced RAW_DNS_RECORD\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in dummy_module.events\n if e.type == \"RAW_DNS_RECORD\"\n and e.host == \"one.one.one.one\"\n and e.data[\"host\"] == \"one.one.one.one\"\n and e.data[\"type\"] == \"TXT\"\n and e.data[\"answer\"]\n == '\"v=spf1 ip4:103.151.192.0/23 ip4:185.12.80.0/22 ip4:188.172.128.0/20 ip4:192.161.144.0/20 ip4:216.198.0.0/18 ~all\"'\n and e.discovery_context == \"TXT lookup on one.one.one.one produced RAW_DNS_RECORD\"\n ]\n )\n # scan with omitted event type\n scan = bbot_scanner(\"one.one.one.one\", config={\"dns\": {\"minimal\": False}, \"omit_event_types\": [\"RAW_DNS_RECORD\"]})\n await scan.helpers.dns._mock_dns(mock_records)\n dummy_module = DummyModule(scan)\n scan.modules[\"dummy_module\"] = dummy_module\n events = [e async for e in scan.async_start()]\n # no raw records should be emitted\n assert 0 == len([e for e in events if e.type == \"RAW_DNS_RECORD\"])\n assert 0 == len([e for e in dummy_module.events if e.type == \"RAW_DNS_RECORD\"])\n\n # scan with watching module\n DummyModule.watched_events = [\"RAW_DNS_RECORD\"]\n scan = bbot_scanner(\"one.one.one.one\", config={\"dns\": {\"minimal\": False}, \"omit_event_types\": [\"RAW_DNS_RECORD\"]})\n await scan.helpers.dns._mock_dns(mock_records)\n dummy_module = DummyModule(scan)\n scan.modules[\"dummy_module\"] = dummy_module\n events = [e async for e in scan.async_start()]\n # no raw records should be output\n assert 0 == len([e for e in events if e.type == \"RAW_DNS_RECORD\"])\n # but they should still make it to the module\n assert 1 == len(\n [\n e\n for e in dummy_module.events\n if e.type == \"RAW_DNS_RECORD\"\n and e.host == \"one.one.one.one\"\n and e.data[\"host\"] == \"one.one.one.one\"\n and e.data[\"type\"] == \"TXT\"\n and e.data[\"answer\"]\n == '\"v=spf1 ip4:103.151.192.0/23 ip4:185.12.80.0/22 ip4:188.172.128.0/20 ip4:192.161.144.0/20 ip4:216.198.0.0/18 ~all\"'\n and e.discovery_context == \"TXT lookup on one.one.one.one produced RAW_DNS_RECORD\"\n ]\n )\n\n\n@pytest.mark.asyncio\nasync def test_dns_graph_structure(bbot_scanner):\n scan = bbot_scanner(\"https://evilcorp.com\", config={\"dns\": {\"search_distance\": 1, \"minimal\": False}})\n await scan.helpers.dns._mock_dns(\n {\n \"evilcorp.com\": {\n \"CNAME\": [\n \"www.evilcorp.com\",\n ]\n },\n \"www.evilcorp.com\": {\"CNAME\": [\"test.evilcorp.com\"]},\n \"test.evilcorp.com\": {\"A\": [\"127.0.0.1\"]},\n }\n )\n events = [e async for e in scan.async_start()]\n assert len(events) == 6\n non_scan_events = [e for e in events if e.type != \"SCAN\"]\n assert sorted([e.type for e in non_scan_events]) == [\"DNS_NAME\", \"DNS_NAME\", \"DNS_NAME\", \"URL_UNVERIFIED\"]\n events_by_data = {e.data: e for e in non_scan_events}\n assert set(events_by_data) == {\"https://evilcorp.com/\", \"evilcorp.com\", \"www.evilcorp.com\", \"test.evilcorp.com\"}\n assert events_by_data[\"test.evilcorp.com\"].parent.data == \"www.evilcorp.com\"\n assert str(events_by_data[\"test.evilcorp.com\"].module) == \"CNAME\"\n assert events_by_data[\"www.evilcorp.com\"].parent.data == \"evilcorp.com\"\n assert str(events_by_data[\"www.evilcorp.com\"].module) == \"CNAME\"\n assert events_by_data[\"evilcorp.com\"].parent.data == \"https://evilcorp.com/\"\n assert str(events_by_data[\"evilcorp.com\"].module) == \"host\"\n\n\n@pytest.mark.asyncio\nasync def test_hostname_extraction(bbot_scanner):\n scan = bbot_scanner(\"evilcorp.com\", config={\"dns\": {\"minimal\": False}})\n await scan.helpers.dns._mock_dns(\n {\n \"evilcorp.com\": {\n \"A\": [\"127.0.0.1\"],\n \"TXT\": [\n \"v=spf1 include:spf-a.evilcorp.com include:spf-b.evilcorp.com include:icpbounce.com include:shops.shopify.com include:_spf.qemailserver.com include:spf.mandrillapp.com include:spf.protection.office365.us include:spf-003ea501.gpphosted.com 127.0.0.1 -all\"\n ],\n }\n }\n )\n events = [e async for e in scan.async_start()]\n dns_name_events = [e for e in events if e.type == \"DNS_NAME\"]\n main_dns_event = [e for e in dns_name_events if e.data == \"evilcorp.com\"]\n assert len(main_dns_event) == 1\n main_dns_event = main_dns_event[0]\n dns_children = main_dns_event.dns_children\n assert dns_children[\"A\"] == {\"127.0.0.1\"}\n assert dns_children[\"TXT\"] == {\n \"spf-a.evilcorp.com\",\n \"spf-b.evilcorp.com\",\n \"icpbounce.com\",\n \"shops.shopify.com\",\n \"_spf.qemailserver.com\",\n \"spf.mandrillapp.com\",\n \"spf.protection.office365.us\",\n \"spf-003ea501.gpphosted.com\",\n \"127.0.0.1\",\n }\n\n\n@pytest.mark.asyncio\nasync def test_dns_helpers(bbot_scanner):\n assert service_record(\"\") is False\n assert service_record(\"localhost\") is False\n assert service_record(\"www.example.com\") is False\n assert service_record(\"www.example.com\", \"SRV\") is True\n assert service_record(\"_custom._service.example.com\", \"SRV\") is True\n assert service_record(\"_custom._service.example.com\", \"A\") is False\n # top 100 most common SRV records\n for srv_record in common_srvs[:100]:\n hostname = f\"{srv_record}.example.com\"\n assert service_record(hostname) is True\n\n # make sure system nameservers are excluded from use by DNS brute force\n brute_nameservers = tempwordlist([\"1.2.3.4\", \"8.8.4.4\", \"4.3.2.1\", \"8.8.8.8\"])\n scan = bbot_scanner(config={\"dns\": {\"brute_nameservers\": brute_nameservers}})\n scan.helpers.dns.system_resolvers = [\"8.8.8.8\", \"8.8.4.4\"]\n resolver_file = await scan.helpers.dns.brute.resolver_file()\n resolvers = set(scan.helpers.read_file(resolver_file))\n assert resolvers == {\"1.2.3.4\", \"4.3.2.1\"}\n" }, { "path": "bbot/test/test_step_1/test_docs.py", "content": "def test_docs():\n from bbot.scripts.docs import update_docs\n\n update_docs()\n" }, { "path": "bbot/test/test_step_1/test_engine.py", "content": "from ..bbot_fixtures import *\n\n\n@pytest.mark.asyncio\nasync def test_engine():\n from bbot.core.engine import EngineClient, EngineServer\n\n counter = 0\n yield_cancelled = False\n yield_errored = False\n return_started = False\n return_finished = False\n return_cancelled = False\n return_errored = False\n\n class TestEngineServer(EngineServer):\n CMDS = {\n 0: \"return_thing\",\n 1: \"yield_stuff\",\n }\n\n async def return_thing(self, n):\n nonlocal return_started\n nonlocal return_finished\n nonlocal return_cancelled\n nonlocal return_errored\n try:\n return_started = True\n await asyncio.sleep(n)\n return_finished = True\n return f\"thing{n}\"\n except asyncio.CancelledError:\n return_cancelled = True\n raise\n except Exception:\n return_errored = True\n raise\n\n async def yield_stuff(self, n):\n nonlocal counter\n nonlocal yield_cancelled\n nonlocal yield_errored\n try:\n for i in range(n):\n yield f\"thing{i}\"\n counter += 1\n await asyncio.sleep(0.1)\n except asyncio.CancelledError:\n yield_cancelled = True\n raise\n except Exception:\n yield_errored = True\n raise\n\n class TestEngineClient(EngineClient):\n SERVER_CLASS = TestEngineServer\n\n async def return_thing(self, n):\n return await self.run_and_return(\"return_thing\", n)\n\n async def yield_stuff(self, n):\n async for _ in self.run_and_yield(\"yield_stuff\", n):\n yield _\n\n test_engine = TestEngineClient()\n\n # test return functionality\n return_res = await test_engine.return_thing(1)\n assert return_res == \"thing1\"\n\n # test async generator\n assert counter == 0\n assert yield_cancelled is False\n yield_res = [r async for r in test_engine.yield_stuff(13)]\n assert yield_res == [f\"thing{i}\" for i in range(13)]\n assert len(yield_res) == 13\n assert counter == 13\n\n # test async generator with cancellation\n counter = 0\n yield_cancelled = False\n yield_errored = False\n agen = test_engine.yield_stuff(1000)\n async for r in agen:\n if counter > 10:\n await agen.aclose()\n break\n await asyncio.sleep(5)\n assert yield_cancelled is True\n assert yield_errored is False\n assert counter < 15\n\n # test async generator with error\n yield_cancelled = False\n yield_errored = False\n agen = test_engine.yield_stuff(None)\n with pytest.raises(BBOTEngineError):\n async for _ in agen:\n pass\n assert yield_cancelled is False\n assert yield_errored is True\n\n # test return with cancellation\n return_started = False\n return_finished = False\n return_cancelled = False\n return_errored = False\n task = asyncio.create_task(test_engine.return_thing(2))\n await asyncio.sleep(1)\n task.cancel()\n with pytest.raises(asyncio.CancelledError):\n await task\n await asyncio.sleep(0.1)\n assert return_started is True\n assert return_finished is False\n assert return_cancelled is True\n assert return_errored is False\n\n # test return with late cancellation\n return_started = False\n return_finished = False\n return_cancelled = False\n return_errored = False\n task = asyncio.create_task(test_engine.return_thing(1))\n await asyncio.sleep(2)\n task.cancel()\n result = await task\n assert result == \"thing1\"\n assert return_started is True\n assert return_finished is True\n assert return_cancelled is False\n assert return_errored is False\n\n # test return with error\n return_started = False\n return_finished = False\n return_cancelled = False\n return_errored = False\n with pytest.raises(BBOTEngineError):\n result = await test_engine.return_thing(None)\n assert return_started is True\n assert return_finished is False\n assert return_cancelled is False\n assert return_errored is True\n\n await test_engine.shutdown()\n" }, { "path": "bbot/test/test_step_1/test_event_seeds.py", "content": "import pytest\nimport ipaddress\nfrom bbot.errors import ValidationError\nfrom bbot.core.event.helpers import EventSeed\n\n\ndef test_event_seeds():\n # DNS_NAME\n dns_seed = EventSeed(\"evilcOrp.com.\")\n assert dns_seed.type == \"DNS_NAME\"\n assert dns_seed.data == \"evilcorp.com\"\n assert dns_seed.host == \"evilcorp.com\"\n assert dns_seed.input == \"evilcorp.com\"\n assert dns_seed._target_type == \"TARGET\"\n\n # IP_ADDRESS (IPv4)\n ipv4_seed = EventSeed(\"192.168.1.1\")\n assert ipv4_seed.type == \"IP_ADDRESS\"\n assert ipv4_seed.data == \"192.168.1.1\"\n assert ipv4_seed.host == ipaddress.ip_address(\"192.168.1.1\")\n assert ipv4_seed.input == \"192.168.1.1\"\n\n # Test various IPv6 formats\n ipv6_formats = [\n \"2001:db8::ff00:42:8329\", # Standard format\n \"2001:0db8:0000:0000:0000:ff00:0042:8329\", # Full format\n \"2001:db8:0:0:0:ff00:42:8329\", # Mixed format\n \"::1\", # Loopback\n \"::ffff:192.168.1.1\", # IPv4-mapped\n \"2001:db8::\", # Subnet prefix\n \"fe80::1ff:fe23:4567:890a\", # Link-local\n ]\n\n # IP_ADDRESS (IPv6)\n for ipv6 in ipv6_formats:\n ipv6_seed = EventSeed(ipv6)\n normalized_ipv6 = str(ipaddress.IPv6Address(ipv6))\n assert ipv6_seed.type == \"IP_ADDRESS\"\n assert ipv6_seed.data == normalized_ipv6\n assert ipv6_seed.host == ipaddress.ip_address(ipv6)\n assert ipv6_seed.input == normalized_ipv6\n\n # IP_RANGE (IPv4)\n ipv4_range_seed = EventSeed(\"192.168.1.1/24\")\n assert ipv4_range_seed.type == \"IP_RANGE\"\n assert ipv4_range_seed.data == \"192.168.1.0/24\"\n assert ipv4_range_seed.host == ipaddress.ip_network(\"192.168.1.0/24\")\n assert ipv4_range_seed.input == \"192.168.1.0/24\"\n\n # IP_RANGE (IPv6)\n ipv6_range_seed = EventSeed(\"2001:db8::ff00:42:8329/64\")\n assert ipv6_range_seed.type == \"IP_RANGE\"\n assert ipv6_range_seed.data == \"2001:db8::/64\"\n assert ipv6_range_seed.host == ipaddress.ip_network(\"2001:db8::/64\")\n assert ipv6_range_seed.input == \"2001:db8::/64\"\n\n # OPEN_TCP_PORT (DNS)\n open_port_dns_seed = EventSeed(\"evilcOrp.com:80\")\n assert open_port_dns_seed.type == \"OPEN_TCP_PORT\"\n assert open_port_dns_seed.data == \"evilcorp.com:80\"\n assert open_port_dns_seed.host == \"evilcorp.com\"\n assert open_port_dns_seed.port == 80\n assert open_port_dns_seed.input == \"evilcorp.com:80\"\n\n # OPEN_TCP_PORT (IPv4)\n open_port_ipv4_seed = EventSeed(\"192.168.1.1:80\")\n assert open_port_ipv4_seed.type == \"OPEN_TCP_PORT\"\n assert open_port_ipv4_seed.data == \"192.168.1.1:80\"\n assert open_port_ipv4_seed.host == ipaddress.ip_address(\"192.168.1.1\")\n assert open_port_ipv4_seed.port == 80\n assert open_port_ipv4_seed.input == \"192.168.1.1:80\"\n\n # OPEN_TCP_PORT (IPv6)\n open_port_ipv6_seed = EventSeed(\"[2001:db8::42]:80\")\n assert open_port_ipv6_seed.type == \"OPEN_TCP_PORT\"\n assert open_port_ipv6_seed.data == \"[2001:db8::42]:80\"\n assert open_port_ipv6_seed.host == ipaddress.ip_address(\"2001:db8::42\")\n assert open_port_ipv6_seed.port == 80\n assert open_port_ipv6_seed.input == \"[2001:db8::42]:80\"\n\n # URL (DNS_NAME)\n url_dns_seed = EventSeed(\"http://evilcOrp.com./index.html?a=b#c\")\n assert url_dns_seed.type == \"URL_UNVERIFIED\"\n assert url_dns_seed.data == \"http://evilcorp.com/index.html?a=b\"\n assert url_dns_seed.host == \"evilcorp.com\"\n assert url_dns_seed.port == 80\n assert url_dns_seed.input == \"http://evilcorp.com/index.html?a=b\"\n\n # URL (IPv4)\n url_ipv4_seed = EventSeed(\"https://192.168.1.1/index.html?a=b#c\")\n assert url_ipv4_seed.type == \"URL_UNVERIFIED\"\n assert url_ipv4_seed.data == \"https://192.168.1.1/index.html?a=b\"\n assert url_ipv4_seed.host == ipaddress.ip_address(\"192.168.1.1\")\n assert url_ipv4_seed.port == 443\n assert url_ipv4_seed.input == \"https://192.168.1.1/index.html?a=b\"\n\n # URL (IPv6)\n url_ipv6_seed = EventSeed(\"https://[2001:db8::42]:8080/index.html?a=b#c\")\n assert url_ipv6_seed.type == \"URL_UNVERIFIED\"\n assert url_ipv6_seed.data == \"https://[2001:db8::42]:8080/index.html?a=b\"\n assert url_ipv6_seed.host == ipaddress.ip_address(\"2001:db8::42\")\n assert url_ipv6_seed.port == 8080\n assert url_ipv6_seed.input == \"https://[2001:db8::42]:8080/index.html?a=b\"\n\n # EMAIL_ADDRESS\n email_seed = EventSeed(\"john.doe@evilcOrp.com\")\n assert email_seed.type == \"EMAIL_ADDRESS\"\n assert email_seed.data == \"john.doe@evilcorp.com\"\n assert email_seed.host == \"evilcorp.com\"\n assert email_seed.port == None\n assert email_seed.input == \"john.doe@evilcorp.com\"\n\n email_seed_ipv4 = EventSeed(\"john.doe@192.168.1.1:80\")\n assert email_seed_ipv4.type == \"EMAIL_ADDRESS\"\n assert email_seed_ipv4.data == \"john.doe@192.168.1.1:80\"\n assert email_seed_ipv4.host == ipaddress.ip_address(\"192.168.1.1\")\n assert email_seed_ipv4.port == 80\n assert email_seed_ipv4.input == \"john.doe@192.168.1.1:80\"\n\n # ORG_STUB\n org_stub_seed = EventSeed(\"ORG:evilcorp\")\n assert org_stub_seed.type == \"ORG_STUB\"\n assert org_stub_seed.data == \"evilcorp\"\n assert org_stub_seed.host == None\n assert org_stub_seed.input == \"ORG_STUB:evilcorp\"\n\n # USERNAME\n username_seed = EventSeed(\"USER:john.doe\")\n assert username_seed.type == \"USERNAME\"\n assert username_seed.data == \"john.doe\"\n assert username_seed.host == None\n assert username_seed.input == \"USERNAME:john.doe\"\n\n # FILESYSTEM\n filesystem_seed = EventSeed(\"FILE:/home/john/documents\")\n assert filesystem_seed.type == \"FILESYSTEM\"\n assert filesystem_seed.data == {\"path\": \"/home/john/documents\"}\n assert filesystem_seed.host == None\n assert filesystem_seed.input == \"FILESYSTEM:/home/john/documents\"\n\n # MOBILE_APP\n mobile_app_seed = EventSeed(\"APK:https://play.google.com/store/apps/details?id=com.evilcorp.app\")\n assert mobile_app_seed.type == \"MOBILE_APP\"\n assert mobile_app_seed.data == {\"url\": \"https://play.google.com/store/apps/details?id=com.evilcorp.app\"}\n assert mobile_app_seed.host == None\n assert mobile_app_seed.input == \"MOBILE_APP:https://play.google.com/store/apps/details?id=com.evilcorp.app\"\n\n with pytest.raises(ValidationError):\n EventSeed(\"INVALID:INVALID\")\n\n with pytest.raises(ValidationError):\n EventSeed(\"^@#$^@#$\")\n\n # BLACKLIST_REGEX\n blacklist_regex_seed = EventSeed(\"RE:evil[0-9]{3}\")\n assert blacklist_regex_seed.type == \"BLACKLIST_REGEX\"\n assert blacklist_regex_seed.data == \"evil[0-9]{3}\"\n assert blacklist_regex_seed.host == None\n assert blacklist_regex_seed.input == \"REGEX:evil[0-9]{3}\"\n assert blacklist_regex_seed._target_type == \"BLACKLIST\"\n" }, { "path": "bbot/test/test_step_1/test_events.py", "content": "import json\nimport random\nimport ipaddress\n\nfrom ..bbot_fixtures import *\nfrom bbot.scanner import Scanner\nfrom bbot.core.helpers.regexes import event_uuid_regex\n\n\n@pytest.mark.asyncio\nasync def test_events(events, helpers):\n scan = Scanner()\n await scan._prep()\n\n assert events.ipv4.type == \"IP_ADDRESS\"\n assert events.ipv4.netloc == \"8.8.8.8\"\n assert events.ipv4.port is None\n assert events.ipv6.type == \"IP_ADDRESS\"\n assert events.ipv6.netloc == \"[2001:4860:4860::8888]\"\n assert events.ipv6.port is None\n assert events.ipv6_open_port.netloc == \"[2001:4860:4860::8888]:443\"\n assert events.netv4.type == \"IP_RANGE\"\n assert events.netv4.netloc is None\n assert \"netloc\" not in events.netv4.json()\n assert events.netv6.type == \"IP_RANGE\"\n assert events.domain.type == \"DNS_NAME\"\n assert events.domain.netloc == \"publicapis.org\"\n assert events.domain.port is None\n assert \"domain\" in events.domain.tags\n assert events.subdomain.type == \"DNS_NAME\"\n assert \"subdomain\" in events.subdomain.tags\n assert events.open_port.type == \"OPEN_TCP_PORT\"\n assert events.url_unverified.type == \"URL_UNVERIFIED\"\n assert events.ipv4_url_unverified.type == \"URL_UNVERIFIED\"\n assert events.ipv6_url_unverified.type == \"URL_UNVERIFIED\"\n assert \"\" not in events.ipv4\n assert None not in events.ipv4\n assert 1 not in events.ipv4\n assert False not in events.ipv4\n\n # ip tests\n assert events.ipv4 == scan.make_event(\"8.8.8.8\", dummy=True)\n assert \"8.8.8.8\" in events.ipv4\n assert events.ipv4.host_filterable == \"8.8.8.8\"\n assert events.ipv4.data == \"8.8.8.8\"\n assert \"8.8.8.8\" in events.netv4\n assert \"8.8.8.9\" not in events.ipv4\n assert \"8.8.9.8\" not in events.netv4\n assert \"8.8.8.8/31\" in events.netv4\n assert \"8.8.8.8/30\" in events.netv4\n assert \"8.8.8.8/29\" not in events.netv4\n assert \"2001:4860:4860::8888\" in events.ipv6\n assert \"2001:4860:4860::8888\" in events.netv6\n assert \"2001:4860:4860::8889\" not in events.ipv6\n assert \"2002:4860:4860::8888\" not in events.netv6\n assert \"2001:4860:4860::8888/127\" in events.netv6\n assert \"2001:4860:4860::8888/126\" in events.netv6\n assert \"2001:4860:4860::8888/125\" not in events.netv6\n assert events.emoji not in events.ipv4\n assert events.emoji not in events.netv6\n assert events.netv6 not in events.emoji\n ipv6_event = scan.make_event(\" [DEaD::c0De]:88\", \"DNS_NAME\", dummy=True)\n assert ipv6_event.data == \"dead::c0de\"\n assert ipv6_event.host_filterable == \"dead::c0de\"\n range_to_ip = scan.make_event(\"1.2.3.4/32\", dummy=True)\n assert range_to_ip.type == \"IP_ADDRESS\"\n range_to_ip = scan.make_event(\"dead::beef/128\", dummy=True)\n assert range_to_ip.type == \"IP_ADDRESS\"\n\n # hostname tests\n assert events.domain.host == \"publicapis.org\"\n assert events.domain.host_filterable == \"publicapis.org\"\n assert events.subdomain.host == \"api.publicapis.org\"\n assert events.subdomain.host_filterable == \"api.publicapis.org\"\n assert events.domain.host_stem == \"publicapis\"\n assert events.subdomain.host_stem == \"api.publicapis\"\n assert \"api.publicapis.org\" in events.domain\n assert \"api.publicapis.org\" in events.subdomain\n assert \"fsocie.ty\" not in events.domain\n assert \"fsocie.ty\" not in events.subdomain\n assert events.subdomain in events.domain\n assert events.domain not in events.subdomain\n assert events.ipv4 not in events.domain\n assert events.netv6 not in events.domain\n assert events.emoji not in events.domain\n assert events.domain not in events.emoji\n open_port_event = scan.make_event(\" eViLcorp.COM.:88\", \"DNS_NAME\", dummy=True)\n dns_event = scan.make_event(\"evilcorp.com.\", \"DNS_NAME\", dummy=True)\n for e in (open_port_event, dns_event):\n assert e.data == \"evilcorp.com\"\n assert e.netloc == \"evilcorp.com\"\n assert e.json()[\"netloc\"] == \"evilcorp.com\"\n assert e.port is None\n assert \"port\" not in e.json()\n\n # url tests\n url_no_trailing_slash = scan.make_event(\"http://evilcorp.com\", dummy=True)\n url_trailing_slash = scan.make_event(\"http://evilcorp.com/\", dummy=True)\n assert url_no_trailing_slash == url_trailing_slash\n assert url_no_trailing_slash.host_filterable == \"http://evilcorp.com/\"\n assert url_trailing_slash.host_filterable == \"http://evilcorp.com/\"\n assert events.url_unverified.host == \"api.publicapis.org\"\n assert events.url_unverified in events.domain\n assert events.url_unverified in events.subdomain\n assert \"api.publicapis.org:443\" in events.url_unverified\n assert \"publicapis.org\" not in events.url_unverified\n assert events.ipv4_url_unverified in events.ipv4\n assert events.ipv4_url_unverified.netloc == \"8.8.8.8:443\"\n assert events.ipv4_url_unverified.port == 443\n assert events.ipv4_url_unverified.json()[\"port\"] == 443\n assert events.ipv4_url_unverified in events.netv4\n assert events.ipv6_url_unverified in events.ipv6\n assert events.ipv6_url_unverified.netloc == \"[2001:4860:4860::8888]:443\"\n assert events.ipv6_url_unverified.port == 443\n assert events.ipv6_url_unverified.json()[\"port\"] == 443\n assert events.ipv6_url_unverified in events.netv6\n assert events.emoji not in events.url_unverified\n assert events.emoji not in events.ipv6_url_unverified\n assert events.url_unverified not in events.emoji\n\n # URL normalization tests – compare against normalized event.data / .with_port().geturl()\n assert scan.make_event(\"https://evilcorp.com:443\", dummy=True).data == \"https://evilcorp.com/\"\n assert scan.make_event(\"http://evilcorp.com:80\", dummy=True).data == \"http://evilcorp.com/\"\n assert \"http://evilcorp.com:80/asdf.js\" in scan.make_event(\"http://evilcorp.com/asdf.js\", dummy=True)\n assert \"http://evilcorp.com/asdf.js\" in scan.make_event(\"http://evilcorp.com:80/asdf.js\", dummy=True)\n assert scan.make_event(\"https://evilcorp.com\", dummy=True).data == \"https://evilcorp.com/\"\n assert scan.make_event(\"http://evilcorp.com\", dummy=True).data == \"http://evilcorp.com/\"\n assert scan.make_event(\"https://evilcorp.com:80\", dummy=True).data == \"https://evilcorp.com:80/\"\n assert scan.make_event(\"http://evilcorp.com:443\", dummy=True).data == \"http://evilcorp.com:443/\"\n assert scan.make_event(\"https://evilcorp.com\", dummy=True).with_port().geturl() == \"https://evilcorp.com:443/\"\n assert scan.make_event(\"https://evilcorp.com:666\", dummy=True).with_port().geturl() == \"https://evilcorp.com:666/\"\n assert scan.make_event(\"https://evilcorp.com.:666\", dummy=True).data == \"https://evilcorp.com:666/\"\n assert scan.make_event(\"https://[bad::c0de]\", dummy=True).with_port().geturl() == \"https://[bad::c0de]:443/\"\n assert scan.make_event(\"https://[bad::c0de]:666\", dummy=True).with_port().geturl() == \"https://[bad::c0de]:666/\"\n url_event = scan.make_event(\"https://evilcorp.com\", \"URL\", events.ipv4_url, tags=[\"status-200\"])\n assert \"status-200\" in url_event.tags\n assert url_event.http_status == 200\n with pytest.raises(ValidationError, match=\".*status tag.*\"):\n scan.make_event(\"https://evilcorp.com\", \"URL\", events.ipv4_url)\n\n # http response\n assert events.http_response.host == \"example.com\"\n assert events.http_response.port == 80\n assert events.http_response.parsed_url.scheme == \"http\"\n assert events.http_response.with_port().geturl() == \"http://example.com:80/\"\n assert events.http_response.host_filterable == \"http://example.com/\"\n\n http_response = scan.make_event(\n {\n \"port\": \"80\",\n \"title\": \"HTTP%20RESPONSE\",\n \"url\": \"http://www.evilcorp.com:80\",\n \"input\": \"http://www.evilcorp.com:80\",\n \"raw_header\": \"HTTP/1.1 301 Moved Permanently\\r\\nLocation: http://www.evilcorp.com/asdf\\r\\n\\r\\n\",\n \"location\": \"/asdf\",\n \"status_code\": 301,\n },\n \"HTTP_RESPONSE\",\n dummy=True,\n )\n assert http_response.http_status == 301\n assert http_response.http_title == \"HTTP RESPONSE\"\n assert http_response.redirect_location == \"http://www.evilcorp.com/asdf\"\n\n # http response url validation\n http_response_2 = scan.make_event(\n {\n \"port\": \"80\",\n \"url\": \"http://evilcorp.com:80/asdf\",\n \"raw_header\": \"HTTP/1.1 301 Moved Permanently\\r\\nLocation: http://www.evilcorp.com/asdf\\r\\n\\r\\n\",\n },\n \"HTTP_RESPONSE\",\n dummy=True,\n )\n assert http_response_2.data[\"url\"] == \"http://evilcorp.com/asdf\"\n\n # open port tests\n assert events.open_port in events.domain\n assert \"api.publicapis.org:443\" in events.open_port\n assert \"bad.publicapis.org:443\" not in events.open_port\n assert \"publicapis.org:443\" not in events.open_port\n assert events.ipv4_open_port in events.ipv4\n assert events.ipv4_open_port in events.netv4\n assert \"8.8.8.9\" not in events.ipv4_open_port\n assert events.ipv6_open_port in events.ipv6\n assert events.ipv6_open_port in events.netv6\n assert \"2002:4860:4860::8888\" not in events.ipv6_open_port\n assert events.emoji not in events.ipv6_open_port\n assert events.ipv6_open_port not in events.emoji\n\n # attribute tests\n assert events.ipv4.host == ipaddress.ip_address(\"8.8.8.8\")\n assert events.ipv4.port is None\n assert events.ipv6.host == ipaddress.ip_address(\"2001:4860:4860::8888\")\n assert events.ipv6.port is None\n assert events.domain.port is None\n assert events.subdomain.port is None\n assert events.open_port.host == \"api.publicapis.org\"\n assert events.open_port.port == 443\n assert events.ipv4_open_port.host == ipaddress.ip_address(\"8.8.8.8\")\n assert events.ipv4_open_port.port == 443\n assert events.ipv6_open_port.host == ipaddress.ip_address(\"2001:4860:4860::8888\")\n assert events.ipv6_open_port.port == 443\n assert events.url_unverified.host == \"api.publicapis.org\"\n assert events.url_unverified.port == 443\n assert events.ipv4_url_unverified.host == ipaddress.ip_address(\"8.8.8.8\")\n assert events.ipv4_url_unverified.port == 443\n assert events.ipv6_url_unverified.host == ipaddress.ip_address(\"2001:4860:4860::8888\")\n assert events.ipv6_url_unverified.port == 443\n\n javascript_event = scan.make_event(\"http://evilcorp.com/asdf/a.js?b=c#d\", \"URL_UNVERIFIED\", parent=scan.root_event)\n assert \"extension-js\" in javascript_event.tags\n await scan.ingress_module.handle_event(javascript_event)\n\n # scope distance\n event1 = scan.make_event(\"1.2.3.4\", dummy=True)\n assert event1._scope_distance is None\n event1.scope_distance = 0\n assert event1._scope_distance == 0\n event2 = scan.make_event(\"2.3.4.5\", parent=event1)\n assert event2._scope_distance == 1\n event3 = scan.make_event(\"3.4.5.6\", parent=event2)\n assert event3._scope_distance == 2\n event4 = scan.make_event(\"3.4.5.6\", parent=event3)\n assert event4._scope_distance == 2\n event5 = scan.make_event(\"4.5.6.7\", parent=event4)\n assert event5._scope_distance == 3\n\n url_1 = scan.make_event(\"https://127.0.0.1/asdf\", \"URL_UNVERIFIED\", parent=scan.root_event)\n assert url_1.scope_distance == 1\n url_2 = scan.make_event(\"https://127.0.0.1/test\", \"URL_UNVERIFIED\", parent=url_1)\n assert url_2.scope_distance == 1\n url_3 = scan.make_event(\"https://127.0.0.2/asdf\", \"URL_UNVERIFIED\", parent=url_1)\n assert url_3.scope_distance == 2\n\n org_stub_1 = scan.make_event(\"STUB1\", \"ORG_STUB\", parent=scan.root_event)\n org_stub_1.scope_distance == 1\n assert org_stub_1.netloc is None\n assert \"netloc\" not in org_stub_1.json()\n org_stub_2 = scan.make_event(\"STUB2\", \"ORG_STUB\", parent=org_stub_1)\n org_stub_2.scope_distance == 2\n\n # internal event tracking\n root_event = scan.make_event(\"0.0.0.0\", dummy=True)\n root_event.scope_distance = 0\n internal_event1 = scan.make_event(\"1.2.3.4\", parent=root_event, internal=True)\n assert internal_event1._internal is True\n assert \"internal\" in internal_event1.tags\n\n # tag inheritance\n for tag in (\"affiliate\", \"mutation-1\"):\n affiliate_event = scan.make_event(\"1.2.3.4\", parent=root_event, tags=tag)\n assert tag in affiliate_event.tags\n affiliate_event2 = scan.make_event(\"1.2.3.4:88\", parent=affiliate_event)\n affiliate_event3 = scan.make_event(\"4.3.2.1:88\", parent=affiliate_event)\n assert tag in affiliate_event2.tags\n assert tag not in affiliate_event3.tags\n\n # discovery context\n event = scan.make_event(\n \"127.0.0.1\", parent=scan.root_event, context=\"something discovered {event.type}: {event.data}\"\n )\n assert event.discovery_context == \"something discovered IP_ADDRESS: 127.0.0.1\"\n\n # updating an already-created event with update_event()\n # updating tags\n event1 = scan.make_event(\"127.0.0.1\", parent=scan.root_event)\n updated_event = scan.update_event(event1, tags=\"asdf\")\n # assert \"asdf\" not in event1.tags # why was this test added? why is it important the original event stays untouched? 🤔\n assert \"asdf\" in updated_event.tags\n # updating parent\n event2 = scan.make_event(\"127.0.0.1\", parent=scan.root_event)\n updated_event = scan.update_event(event2, parent=event1)\n # assert event2.parent == scan.root_event\n assert updated_event.parent == event1\n # updating module/internal flag\n event3 = scan.make_event(\"127.0.0.1\", parent=scan.root_event)\n updated_event = scan.update_event(event3, internal=True)\n # assert event3.internal is False\n assert updated_event.internal is True\n\n # event sorting\n parent1 = scan.make_event(\"127.0.0.1\", parent=scan.root_event)\n parent2 = scan.make_event(\"127.0.0.1\", parent=scan.root_event)\n parent2_child1 = scan.make_event(\"127.0.0.1\", parent=parent2)\n parent1_child1 = scan.make_event(\"127.0.0.1\", parent=parent1)\n parent1_child2 = scan.make_event(\"127.0.0.1\", parent=parent1)\n parent1_child2_child1 = scan.make_event(\"127.0.0.1\", parent=parent1_child2)\n parent1_child2_child2 = scan.make_event(\"127.0.0.1\", parent=parent1_child2)\n parent1_child1_child1 = scan.make_event(\"127.0.0.1\", parent=parent1_child1)\n parent2_child2 = scan.make_event(\"127.0.0.1\", parent=parent2)\n parent1_child2_child1_child1 = scan.make_event(\"127.0.0.1\", parent=parent1_child2_child1)\n\n sortable_events = {\n \"parent1\": parent1,\n \"parent2\": parent2,\n \"parent2_child1\": parent2_child1,\n \"parent1_child1\": parent1_child1,\n \"parent1_child2\": parent1_child2,\n \"parent1_child2_child1\": parent1_child2_child1,\n \"parent1_child2_child2\": parent1_child2_child2,\n \"parent1_child1_child1\": parent1_child1_child1,\n \"parent2_child2\": parent2_child2,\n \"parent1_child2_child1_child1\": parent1_child2_child1_child1,\n }\n\n ordered_list = [\n parent1,\n parent1_child1,\n parent1_child1_child1,\n parent1_child2,\n parent1_child2_child1,\n parent1_child2_child1_child1,\n parent1_child2_child2,\n parent2,\n parent2_child1,\n parent2_child2,\n ]\n\n shuffled_list = list(sortable_events.values())\n random.shuffle(shuffled_list)\n\n sorted_events = sorted(shuffled_list)\n assert sorted_events == ordered_list\n\n # test validation\n corrected_event1 = scan.make_event(\"asdf@asdf.com\", \"DNS_NAME\", dummy=True)\n assert corrected_event1.type == \"EMAIL_ADDRESS\"\n corrected_event2 = scan.make_event(\"127.0.0.1\", \"DNS_NAME\", dummy=True)\n assert corrected_event2.type == \"IP_ADDRESS\"\n corrected_event3 = scan.make_event(\"wat.asdf.com\", \"IP_ADDRESS\", dummy=True)\n assert corrected_event3.type == \"DNS_NAME\"\n\n corrected_event4 = scan.make_event(\"bob@evilcorp.com\", \"USERNAME\", dummy=True)\n assert corrected_event4.type == \"EMAIL_ADDRESS\"\n assert \"affiliate\" in corrected_event4.tags\n\n test_vuln = scan.make_event(\n {\"host\": \"EVILcorp.com\", \"severity\": \"iNfo \", \"description\": \"asdf\"}, \"VULNERABILITY\", dummy=True\n )\n assert test_vuln.data[\"host\"] == \"evilcorp.com\"\n assert test_vuln.data[\"severity\"] == \"INFO\"\n test_vuln2 = scan.make_event(\n {\"host\": \"192.168.1.1\", \"severity\": \"iNfo \", \"description\": \"asdf\"}, \"VULNERABILITY\", dummy=True\n )\n assert json.loads(test_vuln2.data_human)[\"severity\"] == \"INFO\"\n assert test_vuln2.host.is_private\n with pytest.raises(ValidationError, match=\".*validation error.*\\nseverity\\n.*Field required.*\"):\n test_vuln = scan.make_event({\"host\": \"evilcorp.com\", \"description\": \"asdf\"}, \"VULNERABILITY\", dummy=True)\n with pytest.raises(ValidationError, match=\".*host.*\\n.*Invalid host.*\"):\n test_vuln = scan.make_event(\n {\"host\": \"!@#$\", \"severity\": \"INFO\", \"description\": \"asdf\"}, \"VULNERABILITY\", dummy=True\n )\n with pytest.raises(ValidationError, match=\".*severity.*\\n.*Invalid severity.*\"):\n test_vuln = scan.make_event(\n {\"host\": \"evilcorp.com\", \"severity\": \"WACK\", \"description\": \"asdf\"}, \"VULNERABILITY\", dummy=True\n )\n\n # test tagging\n ip_event_1 = scan.make_event(\"8.8.8.8\", dummy=True)\n assert \"private-ip\" not in ip_event_1.tags\n ip_event_2 = scan.make_event(\"192.168.0.1\", dummy=True)\n assert \"private-ip\" in ip_event_2.tags\n dns_event_1 = scan.make_event(\"evilcorp.com\", dummy=True)\n assert \"domain\" in dns_event_1.tags\n dns_event_2 = scan.make_event(\"www.evilcorp.com\", dummy=True)\n assert \"subdomain\" in dns_event_2.tags\n\n # punycode - event type detection\n\n # japanese\n assert scan.make_event(\"ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\", dummy=True).type == \"DNS_NAME\"\n assert scan.make_event(\"bob@ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\", dummy=True).type == \"EMAIL_ADDRESS\"\n assert scan.make_event(\"テ゚ト@ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\", dummy=True).type == \"EMAIL_ADDRESS\"\n assert scan.make_event(\"ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト:80\", dummy=True).type == \"OPEN_TCP_PORT\"\n assert scan.make_event(\"http://ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト:80\", dummy=True).type == \"URL_UNVERIFIED\"\n assert scan.make_event(\"http://ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト:80/テ゚ト\", dummy=True).type == \"URL_UNVERIFIED\"\n\n assert scan.make_event(\"xn--eckwd4c7c.xn--zckzah\", dummy=True).type == \"DNS_NAME\"\n assert scan.make_event(\"bob@xn--eckwd4c7c.xn--zckzah\", dummy=True).type == \"EMAIL_ADDRESS\"\n assert scan.make_event(\"テ゚ト@xn--eckwd4c7c.xn--zckzah\", dummy=True).type == \"EMAIL_ADDRESS\"\n assert scan.make_event(\"xn--eckwd4c7c.xn--zckzah:80\", dummy=True).type == \"OPEN_TCP_PORT\"\n assert scan.make_event(\"http://xn--eckwd4c7c.xn--zckzah:80\", dummy=True).type == \"URL_UNVERIFIED\"\n assert scan.make_event(\"http://xn--eckwd4c7c.xn--zckzah:80/テ゚ト\", dummy=True).type == \"URL_UNVERIFIED\"\n\n # thai\n assert scan.make_event(\"āš€ā¸Ŗā¸˛āš€ā¸—ā¸ĩāšˆā¸ĸā¸§ā¸”āš‰ā¸§ā¸ĸā¸ā¸ąā¸™.com\", dummy=True).type == \"DNS_NAME\"\n assert scan.make_event(\"bob@āš€ā¸Ŗā¸˛āš€ā¸—ā¸ĩāšˆā¸ĸā¸§ā¸”āš‰ā¸§ā¸ĸā¸ā¸ąā¸™.com\", dummy=True).type == \"EMAIL_ADDRESS\"\n assert scan.make_event(\"ā¸—ā¸”ā¸Ēā¸­ā¸š@āš€ā¸Ŗā¸˛āš€ā¸—ā¸ĩāšˆā¸ĸā¸§ā¸”āš‰ā¸§ā¸ĸā¸ā¸ąā¸™.com\", dummy=True).type == \"EMAIL_ADDRESS\"\n assert scan.make_event(\"āš€ā¸Ŗā¸˛āš€ā¸—ā¸ĩāšˆā¸ĸā¸§ā¸”āš‰ā¸§ā¸ĸā¸ā¸ąā¸™.com:80\", dummy=True).type == \"OPEN_TCP_PORT\"\n assert scan.make_event(\"http://āš€ā¸Ŗā¸˛āš€ā¸—ā¸ĩāšˆā¸ĸā¸§ā¸”āš‰ā¸§ā¸ĸā¸ā¸ąā¸™.com:80\", dummy=True).type == \"URL_UNVERIFIED\"\n assert scan.make_event(\"http://āš€ā¸Ŗā¸˛āš€ā¸—ā¸ĩāšˆā¸ĸā¸§ā¸”āš‰ā¸§ā¸ĸā¸ā¸ąā¸™.com:80/ā¸—ā¸”ā¸Ēā¸­ā¸š\", dummy=True).type == \"URL_UNVERIFIED\"\n\n assert scan.make_event(\"xn--12c1bik6bbd8ab6hd1b5jc6jta.com\", dummy=True).type == \"DNS_NAME\"\n assert scan.make_event(\"bob@xn--12c1bik6bbd8ab6hd1b5jc6jta.com\", dummy=True).type == \"EMAIL_ADDRESS\"\n assert scan.make_event(\"ā¸—ā¸”ā¸Ēā¸­ā¸š@xn--12c1bik6bbd8ab6hd1b5jc6jta.com\", dummy=True).type == \"EMAIL_ADDRESS\"\n assert scan.make_event(\"xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80\", dummy=True).type == \"OPEN_TCP_PORT\"\n assert scan.make_event(\"http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80\", dummy=True).type == \"URL_UNVERIFIED\"\n assert scan.make_event(\"http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80/ā¸—ā¸”ā¸Ēā¸­ā¸š\", dummy=True).type == \"URL_UNVERIFIED\"\n\n # punycode - encoding / decoding tests\n\n # japanese\n assert scan.make_event(\"xn--eckwd4c7c.xn--zckzah\", dummy=True).data == \"xn--eckwd4c7c.xn--zckzah\"\n assert scan.make_event(\"bob@xn--eckwd4c7c.xn--zckzah\", dummy=True).data == \"bob@xn--eckwd4c7c.xn--zckzah\"\n assert scan.make_event(\"テ゚ト@xn--eckwd4c7c.xn--zckzah\", dummy=True).data == \"テ゚ト@xn--eckwd4c7c.xn--zckzah\"\n assert scan.make_event(\"xn--eckwd4c7c.xn--zckzah:80\", dummy=True).data == \"xn--eckwd4c7c.xn--zckzah:80\"\n assert scan.make_event(\"http://xn--eckwd4c7c.xn--zckzah:80\", dummy=True).data == \"http://xn--eckwd4c7c.xn--zckzah/\"\n assert (\n scan.make_event(\"http://xn--eckwd4c7c.xn--zckzah:80/テ゚ト\", dummy=True).data\n == \"http://xn--eckwd4c7c.xn--zckzah/テ゚ト\"\n )\n\n assert scan.make_event(\"ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\", dummy=True).data == \"xn--eckwd4c7c.xn--zckzah\"\n assert scan.make_event(\"bob@ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\", dummy=True).data == \"bob@xn--eckwd4c7c.xn--zckzah\"\n assert scan.make_event(\"テ゚ト@ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\", dummy=True).data == \"テ゚ト@xn--eckwd4c7c.xn--zckzah\"\n assert scan.make_event(\"ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト:80\", dummy=True).data == \"xn--eckwd4c7c.xn--zckzah:80\"\n assert scan.make_event(\"http://ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト:80\", dummy=True).data == \"http://xn--eckwd4c7c.xn--zckzah/\"\n assert (\n scan.make_event(\"http://ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト:80/テ゚ト\", dummy=True).data\n == \"http://xn--eckwd4c7c.xn--zckzah/テ゚ト\"\n )\n # thai\n assert (\n scan.make_event(\"xn--12c1bik6bbd8ab6hd1b5jc6jta.com\", dummy=True).data == \"xn--12c1bik6bbd8ab6hd1b5jc6jta.com\"\n )\n assert (\n scan.make_event(\"bob@xn--12c1bik6bbd8ab6hd1b5jc6jta.com\", dummy=True).data\n == \"bob@xn--12c1bik6bbd8ab6hd1b5jc6jta.com\"\n )\n assert (\n scan.make_event(\"ā¸—ā¸”ā¸Ēā¸­ā¸š@xn--12c1bik6bbd8ab6hd1b5jc6jta.com\", dummy=True).data\n == \"ā¸—ā¸”ā¸Ēā¸­ā¸š@xn--12c1bik6bbd8ab6hd1b5jc6jta.com\"\n )\n assert (\n scan.make_event(\"xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80\", dummy=True).data\n == \"xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80\"\n )\n assert (\n scan.make_event(\"http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80\", dummy=True).data\n == \"http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/\"\n )\n assert (\n scan.make_event(\"http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80/ā¸—ā¸”ā¸Ēā¸­ā¸š\", dummy=True).data\n == \"http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/ā¸—ā¸”ā¸Ēā¸­ā¸š\"\n )\n\n assert scan.make_event(\"āš€ā¸Ŗā¸˛āš€ā¸—ā¸ĩāšˆā¸ĸā¸§ā¸”āš‰ā¸§ā¸ĸā¸ā¸ąā¸™.com\", dummy=True).data == \"xn--12c1bik6bbd8ab6hd1b5jc6jta.com\"\n assert scan.make_event(\"bob@āš€ā¸Ŗā¸˛āš€ā¸—ā¸ĩāšˆā¸ĸā¸§ā¸”āš‰ā¸§ā¸ĸā¸ā¸ąā¸™.com\", dummy=True).data == \"bob@xn--12c1bik6bbd8ab6hd1b5jc6jta.com\"\n assert scan.make_event(\"ā¸—ā¸”ā¸Ēā¸­ā¸š@āš€ā¸Ŗā¸˛āš€ā¸—ā¸ĩāšˆā¸ĸā¸§ā¸”āš‰ā¸§ā¸ĸā¸ā¸ąā¸™.com\", dummy=True).data == \"ā¸—ā¸”ā¸Ēā¸­ā¸š@xn--12c1bik6bbd8ab6hd1b5jc6jta.com\"\n assert scan.make_event(\"āš€ā¸Ŗā¸˛āš€ā¸—ā¸ĩāšˆā¸ĸā¸§ā¸”āš‰ā¸§ā¸ĸā¸ā¸ąā¸™.com:80\", dummy=True).data == \"xn--12c1bik6bbd8ab6hd1b5jc6jta.com:80\"\n assert (\n scan.make_event(\"http://āš€ā¸Ŗā¸˛āš€ā¸—ā¸ĩāšˆā¸ĸā¸§ā¸”āš‰ā¸§ā¸ĸā¸ā¸ąā¸™.com:80\", dummy=True).data == \"http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/\"\n )\n assert (\n scan.make_event(\"http://āš€ā¸Ŗā¸˛āš€ā¸—ā¸ĩāšˆā¸ĸā¸§ā¸”āš‰ā¸§ā¸ĸā¸ā¸ąā¸™.com:80/ā¸—ā¸”ā¸Ēā¸­ā¸š\", dummy=True).data\n == \"http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/ā¸—ā¸”ā¸Ēā¸­ā¸š\"\n )\n\n # test event uuid\n import uuid\n\n parent_event1 = scan.make_event(\"evilcorp.com\", parent=scan.root_event, context=\"test context\")\n parent_event2 = scan.make_event(\"evilcorp.com\", parent=scan.root_event, context=\"test context\")\n\n event1 = scan.make_event(\"evilcorp.com:80\", parent=parent_event1, context=\"test context\")\n assert hasattr(event1, \"_uuid\")\n assert hasattr(event1, \"uuid\")\n assert isinstance(event1._uuid, uuid.UUID)\n assert isinstance(event1.uuid, str)\n assert event1.uuid == f\"{event1.type}:{event1._uuid}\"\n event2 = scan.make_event(\"evilcorp.com:80\", parent=parent_event2, context=\"test context\")\n assert hasattr(event2, \"_uuid\")\n assert hasattr(event2, \"uuid\")\n assert isinstance(event2._uuid, uuid.UUID)\n assert isinstance(event2.uuid, str)\n assert event2.uuid == f\"{event2.type}:{event2._uuid}\"\n # ids should match because the event type + data is the same\n assert event1.id == event2.id\n # but uuids should be unique!\n assert event1.uuid != event2.uuid\n # parent ids should match\n assert event1.parent_id == event2.parent_id == parent_event1.id == parent_event2.id\n # uuids should not\n assert event1.parent_uuid == parent_event1.uuid\n assert event2.parent_uuid == parent_event2.uuid\n assert event1.parent_uuid != event2.parent_uuid\n\n # test event serialization\n from bbot.core.event import event_from_json\n\n db_event = scan.make_event(\"evilcorp.com:80\", parent=scan.root_event, context=\"test context\")\n assert db_event.parent == scan.root_event\n assert db_event.parent is scan.root_event\n db_event._resolved_hosts = {\"127.0.0.1\"}\n db_event.scope_distance = 1\n assert db_event.discovery_context == \"test context\"\n assert db_event.discovery_path == [\"test context\"]\n assert len(db_event.parent_chain) == 1\n assert all(event_uuid_regex.match(u) for u in db_event.parent_chain)\n assert db_event.parent_chain[0] == str(db_event.uuid)\n assert db_event.parent.uuid == scan.root_event.uuid\n assert db_event.parent_uuid == scan.root_event.uuid\n timestamp = db_event.timestamp.isoformat()\n json_event = db_event.json()\n assert isinstance(json_event[\"uuid\"], str)\n assert json_event[\"uuid\"] == str(db_event.uuid)\n assert json_event[\"parent_uuid\"] == str(scan.root_event.uuid)\n assert json_event[\"scope_distance\"] == 1\n assert json_event[\"data\"] == \"evilcorp.com:80\"\n assert json_event[\"type\"] == \"OPEN_TCP_PORT\"\n assert json_event[\"host\"] == \"evilcorp.com\"\n assert json_event[\"timestamp\"] == timestamp\n assert json_event[\"discovery_context\"] == \"test context\"\n assert json_event[\"discovery_path\"] == [\"test context\"]\n assert json_event[\"parent_chain\"] == db_event.parent_chain\n assert json_event[\"parent_chain\"][0] == str(db_event.uuid)\n reconstituted_event = event_from_json(json_event)\n assert isinstance(reconstituted_event._uuid, uuid.UUID)\n assert str(reconstituted_event.uuid) == json_event[\"uuid\"]\n assert str(reconstituted_event.parent_uuid) == json_event[\"parent_uuid\"]\n assert reconstituted_event.uuid == db_event.uuid\n assert reconstituted_event.parent_uuid == scan.root_event.uuid\n assert reconstituted_event.scope_distance == 1\n assert reconstituted_event.timestamp.isoformat() == timestamp\n assert reconstituted_event.data == \"evilcorp.com:80\"\n assert reconstituted_event.type == \"OPEN_TCP_PORT\"\n assert reconstituted_event.host == \"evilcorp.com\"\n assert reconstituted_event.discovery_context == \"test context\"\n assert reconstituted_event.discovery_path == [\"test context\"]\n assert reconstituted_event.parent_chain == db_event.parent_chain\n assert \"127.0.0.1\" in reconstituted_event.resolved_hosts\n hostless_event = scan.make_event(\"asdf\", \"ASDF\", dummy=True)\n hostless_event_json = hostless_event.json()\n assert hostless_event_json[\"type\"] == \"ASDF\"\n assert hostless_event_json[\"data\"] == \"asdf\"\n assert \"host\" not in hostless_event_json\n\n # SIEM-friendly serialize/deserialize\n json_event_siemfriendly = db_event.json(siem_friendly=True)\n assert json_event_siemfriendly[\"scope_distance\"] == 1\n assert json_event_siemfriendly[\"data\"] == {\"OPEN_TCP_PORT\": \"evilcorp.com:80\"}\n assert json_event_siemfriendly[\"type\"] == \"OPEN_TCP_PORT\"\n assert json_event_siemfriendly[\"host\"] == \"evilcorp.com\"\n assert json_event_siemfriendly[\"timestamp\"] == timestamp\n reconstituted_event2 = event_from_json(json_event_siemfriendly, siem_friendly=True)\n assert reconstituted_event2.scope_distance == 1\n assert reconstituted_event2.timestamp.isoformat() == timestamp\n assert reconstituted_event2.data == \"evilcorp.com:80\"\n assert reconstituted_event2.type == \"OPEN_TCP_PORT\"\n assert reconstituted_event2.host == \"evilcorp.com\"\n assert \"127.0.0.1\" in reconstituted_event2.resolved_hosts\n\n http_response = scan.make_event(httpx_response, \"HTTP_RESPONSE\", parent=scan.root_event)\n assert http_response.parent_id == scan.root_event.id\n assert http_response.data[\"input\"] == \"http://example.com:80\"\n assert (\n http_response.raw_response\n == 'HTTP/1.1 200 OK\\r\\nConnection: close\\r\\nAge: 526111\\r\\nCache-Control: max-age=604800\\r\\nContent-Type: text/html; charset=UTF-8\\r\\nDate: Mon, 14 Nov 2022 17:14:27 GMT\\r\\nEtag: \"3147526947+ident+gzip\"\\r\\nExpires: Mon, 21 Nov 2022 17:14:27 GMT\\r\\nLast-Modified: Thu, 17 Oct 2019 07:18:26 GMT\\r\\nServer: ECS (agb/A445)\\r\\nVary: Accept-Encoding\\r\\nX-Cache: HIT\\r\\n\\r\\n\\n\\n\\n Example Domain\\n\\n \\n \\n \\n \\n\\n\\n\\n
\\n

Example Domain

\\n

This domain is for use in illustrative examples in documents. You may use this\\n domain in literature without prior coordination or asking for permission.

\\n

More information...

\\n
\\n\\n\\n'\n )\n json_event = http_response.json(mode=\"graph\")\n assert isinstance(json_event[\"data\"], str)\n json_event = http_response.json()\n assert isinstance(json_event[\"data\"], dict)\n assert json_event[\"type\"] == \"HTTP_RESPONSE\"\n assert json_event[\"host\"] == \"example.com\"\n assert json_event[\"parent\"] == scan.root_event.id\n reconstituted_event = event_from_json(json_event)\n assert isinstance(reconstituted_event.data, dict)\n assert reconstituted_event.data[\"input\"] == \"http://example.com:80\"\n assert reconstituted_event.host == \"example.com\"\n assert reconstituted_event.type == \"HTTP_RESPONSE\"\n assert reconstituted_event.parent_id == scan.root_event.id\n\n event_1 = scan.make_event(\"127.0.0.1\", parent=scan.root_event)\n event_2 = scan.make_event(\"127.0.0.2\", parent=event_1)\n event_3 = scan.make_event(\"127.0.0.3\", parent=event_2)\n event_3._omit = True\n event_4 = scan.make_event(\"127.0.0.4\", parent=event_3)\n event_5 = scan.make_event(\"127.0.0.5\", parent=event_4)\n assert event_5.get_parents() == [event_4, event_3, event_2, event_1, scan.root_event]\n assert event_5.get_parents(omit=True) == [event_4, event_2, event_1, scan.root_event]\n assert event_5.get_parents(include_self=True) == [event_5, event_4, event_3, event_2, event_1, scan.root_event]\n\n # test host backup\n host_event = scan.make_event(\"asdf.evilcorp.com\", \"DNS_NAME\", parent=scan.root_event)\n assert host_event.host_original == \"asdf.evilcorp.com\"\n host_event.host = \"_wildcard.evilcorp.com\"\n assert host_event.host == \"_wildcard.evilcorp.com\"\n assert host_event.host_original == \"asdf.evilcorp.com\"\n\n # test storage bucket validation\n bucket_event = scan.make_event(\n {\"name\": \"ASDF.s3.amazonaws.com\", \"url\": \"https://ASDF.s3.amazonaws.com\"},\n \"STORAGE_BUCKET\",\n parent=scan.root_event,\n )\n assert bucket_event.data[\"name\"] == \"asdf.s3.amazonaws.com\"\n assert bucket_event.data[\"url\"] == \"https://asdf.s3.amazonaws.com/\"\n\n # test module sequence\n module = scan._make_dummy_module(\"mymodule\")\n parent_event_1 = scan.make_event(\"127.0.0.1\", module=module, parent=scan.root_event)\n assert str(parent_event_1.module) == \"mymodule\"\n assert str(parent_event_1.module_sequence) == \"mymodule\"\n parent_event_2 = scan.make_event(\"127.0.0.2\", module=module, parent=parent_event_1)\n assert str(parent_event_2.module) == \"mymodule\"\n assert str(parent_event_2.module_sequence) == \"mymodule\"\n parent_event_3 = scan.make_event(\"127.0.0.3\", module=module, parent=parent_event_2)\n assert str(parent_event_3.module) == \"mymodule\"\n assert str(parent_event_3.module_sequence) == \"mymodule\"\n\n module = scan._make_dummy_module(\"mymodule\")\n parent_event_1 = scan.make_event(\"127.0.0.1\", module=module, parent=scan.root_event)\n parent_event_1._omit = True\n assert str(parent_event_1.module) == \"mymodule\"\n assert str(parent_event_1.module_sequence) == \"mymodule\"\n parent_event_2 = scan.make_event(\"127.0.0.2\", module=module, parent=parent_event_1)\n parent_event_2._omit = True\n assert str(parent_event_2.module) == \"mymodule\"\n assert str(parent_event_2.module_sequence) == \"mymodule->mymodule\"\n parent_event_3 = scan.make_event(\"127.0.0.3\", module=module, parent=parent_event_2)\n assert str(parent_event_3.module) == \"mymodule\"\n assert str(parent_event_3.module_sequence) == \"mymodule->mymodule->mymodule\"\n\n # event with no data\n with pytest.raises(ValidationError):\n event = scan.make_event(None, \"DNS_NAME\", parent=scan.root_event)\n\n await scan._cleanup()\n\n\n@pytest.mark.asyncio\nasync def test_event_discovery_context():\n from bbot.modules.base import BaseModule\n\n scan = Scanner(\"evilcorp.com\")\n await scan.helpers.dns._mock_dns(\n {\n \"evilcorp.com\": {\"A\": [\"1.2.3.4\"]},\n \"one.evilcorp.com\": {\"A\": [\"1.2.3.4\"]},\n \"two.evilcorp.com\": {\"A\": [\"1.2.3.4\"]},\n \"three.evilcorp.com\": {\"A\": [\"1.2.3.4\"]},\n \"four.evilcorp.com\": {\"A\": [\"1.2.3.4\"]},\n }\n )\n await scan._prep()\n\n dummy_module_1 = scan._make_dummy_module(\"module_1\")\n dummy_module_2 = scan._make_dummy_module(\"module_2\")\n\n class DummyModule(BaseModule):\n watched_events = [\"DNS_NAME\"]\n _name = \"dummy_module\"\n\n async def handle_event(self, event):\n new_event = None\n if event.data == \"evilcorp.com\":\n new_event = scan.make_event(\n \"one.evilcorp.com\",\n \"DNS_NAME\",\n event,\n context=\"{module} invoked forbidden magick to discover {event.type} {event.data}\",\n module=dummy_module_1,\n )\n elif event.data == \"one.evilcorp.com\":\n new_event = scan.make_event(\n \"two.evilcorp.com\",\n \"DNS_NAME\",\n event,\n context=\"{module} pledged its allegiance to cthulu and was awarded {event.type} {event.data}\",\n module=dummy_module_1,\n )\n elif event.data == \"two.evilcorp.com\":\n new_event = scan.make_event(\n \"three.evilcorp.com\",\n \"DNS_NAME\",\n event,\n context=\"{module} asked nicely and was given {event.type} {event.data}\",\n module=dummy_module_2,\n )\n elif event.data == \"three.evilcorp.com\":\n new_event = scan.make_event(\n \"four.evilcorp.com\",\n \"DNS_NAME\",\n event,\n context=\"{module} used brute force to obtain {event.type} {event.data}\",\n module=dummy_module_2,\n )\n if new_event is not None:\n await self.emit_event(new_event)\n\n dummy_module = DummyModule(scan)\n\n scan.modules[\"dummy_module\"] = dummy_module\n\n # test discovery context\n test_event = dummy_module.make_event(\"evilcorp.com\", \"DNS_NAME\", parent=scan.root_event)\n assert test_event.discovery_context == \"dummy_module discovered DNS_NAME: evilcorp.com\"\n\n test_event2 = dummy_module.make_event(\n \"evilcorp.com\", \"DNS_NAME\", parent=scan.root_event, context=\"{module} {found} {event.host}\"\n )\n assert test_event2.discovery_context == \"dummy_module {found} evilcorp.com\"\n # jank input\n test_event3 = dummy_module.make_event(\n \"http://evilcorp.com/{http://evilcorp.org!@#%@#$:,,,}\", \"URL_UNVERIFIED\", parent=scan.root_event\n )\n assert (\n test_event3.discovery_context\n == \"dummy_module discovered URL_UNVERIFIED: http://evilcorp.com/{http:/evilcorp.org!@\"\n )\n\n events = [e async for e in scan.async_start()]\n assert len(events) == 7\n\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"evilcorp.com\"\n and e.discovery_context == f\"Scan {scan.name} seeded with DNS_NAME: evilcorp.com\"\n and e.discovery_path == [f\"Scan {scan.name} seeded with DNS_NAME: evilcorp.com\"]\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"one.evilcorp.com\"\n and e.discovery_context == \"module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com\"\n and e.discovery_path\n == [\n f\"Scan {scan.name} seeded with DNS_NAME: evilcorp.com\",\n \"module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com\",\n ]\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"two.evilcorp.com\"\n and e.discovery_context\n == \"module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com\"\n and e.discovery_path\n == [\n f\"Scan {scan.name} seeded with DNS_NAME: evilcorp.com\",\n \"module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com\",\n \"module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com\",\n ]\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"three.evilcorp.com\"\n and e.discovery_context == \"module_2 asked nicely and was given DNS_NAME three.evilcorp.com\"\n and e.discovery_path\n == [\n f\"Scan {scan.name} seeded with DNS_NAME: evilcorp.com\",\n \"module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com\",\n \"module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com\",\n \"module_2 asked nicely and was given DNS_NAME three.evilcorp.com\",\n ]\n ]\n )\n final_path = [\n f\"Scan {scan.name} seeded with DNS_NAME: evilcorp.com\",\n \"module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com\",\n \"module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com\",\n \"module_2 asked nicely and was given DNS_NAME three.evilcorp.com\",\n \"module_2 used brute force to obtain DNS_NAME four.evilcorp.com\",\n ]\n final_event = [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"four.evilcorp.com\"\n and e.discovery_context == \"module_2 used brute force to obtain DNS_NAME four.evilcorp.com\"\n and e.discovery_path == final_path\n ]\n assert 1 == len(final_event)\n j = final_event[0].json()\n assert j[\"discovery_path\"] == final_path\n\n await scan._cleanup()\n\n # test to make sure this doesn't come back\n # https://github.com/blacklanternsecurity/bbot/issues/1498\n scan = Scanner(\"http://blacklanternsecurity.com\", config={\"dns\": {\"minimal\": False}})\n await scan.helpers.dns._mock_dns(\n {\"blacklanternsecurity.com\": {\"TXT\": [\"blsops.com\"], \"A\": [\"127.0.0.1\"]}, \"blsops.com\": {\"A\": [\"127.0.0.1\"]}}\n )\n events = [e async for e in scan.async_start()]\n blsops_event = [e for e in events if e.type == \"DNS_NAME\" and e.data == \"blsops.com\"]\n assert len(blsops_event) == 1\n assert blsops_event[0].discovery_path[1] == \"URL_UNVERIFIED has host DNS_NAME: blacklanternsecurity.com\"\n\n await scan._cleanup()\n\n\n@pytest.mark.asyncio\nasync def test_event_web_spider_distance(bbot_scanner):\n # make sure web spider distance inheritance works as intended\n # and we don't have any runaway situations with SOCIAL events + URLs\n\n # URL_UNVERIFIED events should not increment web spider distance\n scan = bbot_scanner(config={\"web\": {\"spider_distance\": 1}})\n url_event_1 = scan.make_event(\"http://www.evilcorp.com/test1\", \"URL_UNVERIFIED\", parent=scan.root_event)\n assert url_event_1.web_spider_distance == 0\n url_event_2 = scan.make_event(\"http://www.evilcorp.com/test2\", \"URL_UNVERIFIED\", parent=url_event_1)\n assert url_event_2.web_spider_distance == 0\n url_event_3 = scan.make_event(\n \"http://www.evilcorp.com/test3\", \"URL_UNVERIFIED\", parent=url_event_2, tags=[\"spider-danger\"]\n )\n assert url_event_3.web_spider_distance == 0\n assert \"spider-danger\" in url_event_3.tags\n assert \"spider-max\" not in url_event_3.tags\n\n # URL events should increment web spider distance\n scan = bbot_scanner(config={\"web\": {\"spider_distance\": 1}})\n url_event_1 = scan.make_event(\"http://www.evilcorp.com/test1\", \"URL\", parent=scan.root_event, tags=\"status-200\")\n assert url_event_1.web_spider_distance == 0\n url_event_2 = scan.make_event(\"http://www.evilcorp.com/test2\", \"URL\", parent=url_event_1, tags=\"status-200\")\n assert url_event_2.web_spider_distance == 0\n url_event_3 = scan.make_event(\n \"http://www.evilcorp.com/test3\", \"URL_UNVERIFIED\", parent=url_event_2, tags=[\"spider-danger\"]\n )\n assert url_event_3.web_spider_distance == 1\n assert \"spider-danger\" in url_event_3.tags\n assert \"spider-max\" not in url_event_3.tags\n\n # SOCIAL events should inherit spider distance\n social_event = scan.make_event(\n {\"platform\": \"github\", \"url\": \"http://www.evilcorp.com/test4\"}, \"SOCIAL\", parent=url_event_3\n )\n assert social_event.web_spider_distance == 1\n assert \"spider-danger\" in social_event.tags\n url_event_4 = scan.make_event(\"http://www.evilcorp.com/test4\", \"URL_UNVERIFIED\", parent=social_event)\n assert url_event_4.web_spider_distance == 2\n assert \"spider-danger\" in url_event_4.tags\n assert \"spider-max\" in url_event_4.tags\n social_event_2 = scan.make_event(\n {\"platform\": \"github\", \"url\": \"http://www.evilcorp.com/test5\"}, \"SOCIAL\", parent=url_event_4\n )\n assert social_event_2.web_spider_distance == 2\n assert \"spider-danger\" in social_event_2.tags\n assert \"spider-max\" in social_event_2.tags\n url_event_5 = scan.make_event(\"http://www.evilcorp.com/test5\", \"URL_UNVERIFIED\", parent=social_event_2)\n assert url_event_5.web_spider_distance == 3\n assert \"spider-danger\" in url_event_5.tags\n assert \"spider-max\" in url_event_5.tags\n\n url_event = scan.make_event(\"http://www.evilcorp.com\", \"URL_UNVERIFIED\", parent=scan.root_event)\n assert url_event.web_spider_distance == 0\n assert \"spider-danger\" not in url_event.tags\n assert \"spider-max\" not in url_event.tags\n url_event_2 = scan.make_event(\n \"http://www.evilcorp.com\", \"URL_UNVERIFIED\", parent=scan.root_event, tags=\"spider-danger\"\n )\n url_event_2b = scan.make_event(\"http://www.evilcorp.com\", \"URL\", parent=url_event_2, tags=\"status-200\")\n assert url_event_2b.web_spider_distance == 0\n assert \"spider-danger\" in url_event_2b.tags\n assert \"spider-max\" not in url_event_2b.tags\n url_event_3 = scan.make_event(\n \"http://www.evilcorp.com/3\", \"URL_UNVERIFIED\", parent=url_event_2b, tags=\"spider-danger\"\n )\n assert url_event_3.web_spider_distance == 1\n assert \"spider-danger\" in url_event_3.tags\n assert \"spider-max\" not in url_event_3.tags\n url_event_4 = scan.make_event(\"http://evilcorp.com\", \"URL\", parent=url_event_3, tags=\"status-200\")\n assert url_event_4.web_spider_distance == 0\n assert \"spider-danger\" not in url_event_4.tags\n assert \"spider-max\" not in url_event_4.tags\n url_event_4.add_tag(\"spider-danger\")\n assert url_event_4.web_spider_distance == 0\n assert \"spider-danger\" in url_event_4.tags\n assert \"spider-max\" not in url_event_4.tags\n url_event_4.remove_tag(\"spider-danger\")\n assert url_event_4.web_spider_distance == 0\n assert \"spider-danger\" not in url_event_4.tags\n assert \"spider-max\" not in url_event_4.tags\n url_event_5 = scan.make_event(\"http://evilcorp.com/5\", \"URL_UNVERIFIED\", parent=url_event_4)\n assert url_event_5.web_spider_distance == 0\n assert \"spider-danger\" not in url_event_5.tags\n assert \"spider-max\" not in url_event_5.tags\n url_event_5.add_tag(\"spider-danger\")\n # if host is the same as parent, web spider distance should auto-increment after adding spider-danger tag\n assert url_event_5.web_spider_distance == 1\n assert \"spider-danger\" in url_event_5.tags\n assert \"spider-max\" not in url_event_5.tags\n\n\ndef test_event_confidence():\n scan = Scanner()\n # default 100\n event1 = scan.make_event(\"evilcorp.com\", \"DNS_NAME\", dummy=True)\n assert event1.confidence == 100\n assert event1.cumulative_confidence == 100\n # custom confidence\n event2 = scan.make_event(\"evilcorp.com\", \"DNS_NAME\", confidence=90, dummy=True)\n assert event2.confidence == 90\n assert event2.cumulative_confidence == 90\n # max 100\n event3 = scan.make_event(\"evilcorp.com\", \"DNS_NAME\", confidence=999, dummy=True)\n assert event3.confidence == 100\n assert event3.cumulative_confidence == 100\n # min 1\n event4 = scan.make_event(\"evilcorp.com\", \"DNS_NAME\", confidence=0, dummy=True)\n assert event4.confidence == 1\n assert event4.cumulative_confidence == 1\n # first event in chain\n event5 = scan.make_event(\"evilcorp.com\", \"DNS_NAME\", confidence=90, parent=scan.root_event)\n assert event5.confidence == 90\n assert event5.cumulative_confidence == 90\n # compounding confidence\n event6 = scan.make_event(\"evilcorp.com\", \"DNS_NAME\", confidence=50, parent=event5)\n assert event6.confidence == 50\n assert event6.cumulative_confidence == 45\n event7 = scan.make_event(\"evilcorp.com\", \"DNS_NAME\", confidence=50, parent=event6)\n assert event7.confidence == 50\n assert event7.cumulative_confidence == 22\n # 100 confidence resets\n event8 = scan.make_event(\"evilcorp.com\", \"DNS_NAME\", confidence=100, parent=event7)\n assert event8.confidence == 100\n assert event8.cumulative_confidence == 100\n\n\ndef test_event_closest_host():\n scan = Scanner()\n # first event has a host\n event1 = scan.make_event(\"evilcorp.com\", \"DNS_NAME\", parent=scan.root_event)\n assert event1.host == \"evilcorp.com\"\n # second event has a host + url\n event2 = scan.make_event(\n {\n \"method\": \"GET\",\n \"url\": \"http://www.evilcorp.com/asdf\",\n \"hash\": {\"header_mmh3\": \"1\", \"body_mmh3\": \"2\"},\n \"raw_header\": \"HTTP/1.1 301 Moved Permanently\\r\\nLocation: http://www.evilcorp.com/asdf\\r\\n\\r\\n\",\n },\n \"HTTP_RESPONSE\",\n parent=event1,\n )\n assert event2.host == \"www.evilcorp.com\"\n # third event has a path\n event3 = scan.make_event({\"path\": \"/tmp/asdf.txt\"}, \"FILESYSTEM\", parent=event2)\n assert not event3.host\n # finding automatically uses the host from the second event\n finding = scan.make_event({\"description\": \"test\"}, \"FINDING\", parent=event3)\n assert finding.data[\"host\"] == \"www.evilcorp.com\"\n assert finding.data[\"url\"] == \"http://www.evilcorp.com/asdf\"\n assert finding.data[\"path\"] == \"/tmp/asdf.txt\"\n assert finding.host == \"www.evilcorp.com\"\n # same with vuln\n vuln = scan.make_event({\"description\": \"test\", \"severity\": \"HIGH\"}, \"VULNERABILITY\", parent=event3)\n assert vuln.data[\"host\"] == \"www.evilcorp.com\"\n assert vuln.data[\"url\"] == \"http://www.evilcorp.com/asdf\"\n assert vuln.data[\"path\"] == \"/tmp/asdf.txt\"\n assert vuln.host == \"www.evilcorp.com\"\n\n # no host and no path == not allowed\n event3 = scan.make_event(\"wat\", \"ASDF\", parent=scan.root_event)\n assert not event3.host\n with pytest.raises(ValueError):\n finding = scan.make_event({\"description\": \"test\"}, \"FINDING\", parent=event3)\n finding = scan.make_event({\"path\": \"/tmp/asdf.txt\", \"description\": \"test\"}, \"FINDING\", parent=event3)\n assert finding is not None\n finding = scan.make_event({\"host\": \"evilcorp.com\", \"description\": \"test\"}, \"FINDING\", parent=event3)\n assert finding is not None\n with pytest.raises(ValueError):\n vuln = scan.make_event({\"description\": \"test\", \"severity\": \"HIGH\"}, \"VULNERABILITY\", parent=event3)\n vuln = scan.make_event(\n {\"path\": \"/tmp/asdf.txt\", \"description\": \"test\", \"severity\": \"HIGH\"}, \"VULNERABILITY\", parent=event3\n )\n assert vuln is not None\n vuln = scan.make_event(\n {\"host\": \"evilcorp.com\", \"description\": \"test\", \"severity\": \"HIGH\"}, \"VULNERABILITY\", parent=event3\n )\n assert vuln is not None\n\n\ndef test_event_magic():\n from bbot.core.helpers.libmagic import get_magic_info, get_compression\n\n import base64\n\n zip_base64 = \"UEsDBAoDAAAAAOMmZ1lR4FaHBQAAAAUAAAAIAAAAYXNkZi50eHRhc2RmClBLAQI/AwoDAAAAAOMmZ1lR4FaHBQAAAAUAAAAIACQAAAAAAAAAIICkgQAAAABhc2RmLnR4dAoAIAAAAAAAAQAYAICi2B77MNsBgKLYHvsw2wGAotge+zDbAVBLBQYAAAAAAQABAFoAAAArAAAAAAA=\"\n zip_bytes = base64.b64decode(zip_base64)\n zip_file = Path(\"/tmp/.bbottestzipasdkfjalsdf.zip\")\n with open(zip_file, \"wb\") as f:\n f.write(zip_bytes)\n\n # test magic helpers\n extension, mime_type, description, confidence = get_magic_info(zip_file)\n assert extension == \".zip\"\n assert mime_type == \"application/zip\"\n assert description == \"PKZIP Archive file\"\n assert confidence > 0\n assert get_compression(mime_type) == \"zip\"\n\n # test filesystem event - file\n scan = Scanner()\n event = scan.make_event({\"path\": zip_file}, \"FILESYSTEM\", parent=scan.root_event)\n assert event.data == {\n \"path\": \"/tmp/.bbottestzipasdkfjalsdf.zip\",\n \"magic_extension\": \".zip\",\n \"magic_mime_type\": \"application/zip\",\n \"magic_description\": \"PKZIP Archive file\",\n \"magic_confidence\": 0.9,\n \"compression\": \"zip\",\n }\n assert event.tags == {\"file\", \"zip-archive\", \"compressed\"}\n\n # test filesystem event - folder\n scan = Scanner()\n event = scan.make_event({\"path\": \"/tmp\"}, \"FILESYSTEM\", parent=scan.root_event)\n assert event.data == {\"path\": \"/tmp\"}\n assert event.tags == {\"folder\"}\n\n zip_file.unlink()\n\n\n@pytest.mark.asyncio\nasync def test_mobile_app():\n scan = Scanner()\n with pytest.raises(ValidationError):\n scan.make_event(\"com.evilcorp.app\", \"MOBILE_APP\", parent=scan.root_event)\n with pytest.raises(ValidationError):\n scan.make_event({\"id\": \"com.evilcorp.app\"}, \"MOBILE_APP\", parent=scan.root_event)\n with pytest.raises(ValidationError):\n scan.make_event({\"url\": \"https://play.google.com/store/apps/details\"}, \"MOBILE_APP\", parent=scan.root_event)\n mobile_app = scan.make_event(\n {\"url\": \"https://play.google.com/store/apps/details?id=com.evilcorp.app\"}, \"MOBILE_APP\", parent=scan.root_event\n )\n assert sorted(mobile_app.data.items()) == [\n (\"id\", \"com.evilcorp.app\"),\n (\"url\", \"https://play.google.com/store/apps/details?id=com.evilcorp.app\"),\n ]\n\n scan = Scanner(\"MOBILE_APP:https://play.google.com/store/apps/details?id=com.evilcorp.app\")\n events = [e async for e in scan.async_start()]\n assert len(events) == 3\n mobile_app_event = [e for e in events if e.type == \"MOBILE_APP\"][0]\n assert mobile_app_event.type == \"MOBILE_APP\"\n assert sorted(mobile_app_event.data.items()) == [\n (\"id\", \"com.evilcorp.app\"),\n (\"url\", \"https://play.google.com/store/apps/details?id=com.evilcorp.app\"),\n ]\n\n\n@pytest.mark.asyncio\nasync def test_filesystem():\n scan = Scanner(\"FILESYSTEM:/tmp/asdfasdgasdfasdfddsdf\")\n events = [e async for e in scan.async_start()]\n assert len(events) == 3\n filesystem_events = [e for e in events if e.type == \"FILESYSTEM\"]\n assert len(filesystem_events) == 1\n assert filesystem_events[0].type == \"FILESYSTEM\"\n assert filesystem_events[0].data == {\"path\": \"/tmp/asdfasdgasdfasdfddsdf\"}\n\n\ndef test_event_hashing():\n scan = Scanner(\"example.com\")\n url_event = scan.make_event(\"https://api.example.com/\", \"URL_UNVERIFIED\", parent=scan.root_event)\n host_event_1 = scan.make_event(\"www.example.com\", \"DNS_NAME\", parent=url_event)\n host_event_2 = scan.make_event(\"test.example.com\", \"DNS_NAME\", parent=url_event)\n finding_data = {\"description\": \"Custom Yara Rule [find_string] Matched via identifier [str1]\"}\n finding1 = scan.make_event(finding_data, \"FINDING\", parent=host_event_1)\n finding2 = scan.make_event(finding_data, \"FINDING\", parent=host_event_2)\n finding3 = scan.make_event(finding_data, \"FINDING\", parent=host_event_2)\n\n assert finding1.data == {\n \"description\": \"Custom Yara Rule [find_string] Matched via identifier [str1]\",\n \"host\": \"www.example.com\",\n }\n assert finding2.data == {\n \"description\": \"Custom Yara Rule [find_string] Matched via identifier [str1]\",\n \"host\": \"test.example.com\",\n }\n assert finding3.data == {\n \"description\": \"Custom Yara Rule [find_string] Matched via identifier [str1]\",\n \"host\": \"test.example.com\",\n }\n assert finding1.id != finding2.id\n assert finding2.id == finding3.id\n assert finding1.data_id != finding2.data_id\n assert finding2.data_id == finding3.data_id\n assert finding1.data_hash != finding2.data_hash\n assert finding2.data_hash == finding3.data_hash\n assert hash(finding1) != hash(finding2)\n assert hash(finding2) == hash(finding3)\n" }, { "path": "bbot/test/test_step_1/test_files.py", "content": "import asyncio\n\nfrom ..bbot_fixtures import *\n\n\n@pytest.mark.asyncio\nasync def test_files(bbot_scanner):\n scan1 = bbot_scanner()\n\n # tempfile\n tempfile = scan1.helpers.tempfile((\"line1\", \"line2\"), pipe=False)\n assert list(scan1.helpers.read_file(tempfile)) == [\"line1\", \"line2\"]\n tempfile = scan1.helpers.tempfile((\"line1\", \"line2\"), pipe=True)\n assert list(scan1.helpers.read_file(tempfile)) == [\"line1\", \"line2\"]\n\n # tempfile tail\n results = []\n tempfile = scan1.helpers.tempfile_tail(callback=lambda x: results.append(x))\n with open(tempfile, \"w\") as f:\n f.write(\"asdf\\n\")\n await asyncio.sleep(0.1)\n assert \"asdf\" in results\n\n await scan1._cleanup()\n" }, { "path": "bbot/test/test_step_1/test_helpers.py", "content": "import asyncio\nimport datetime\nimport ipaddress\n\nfrom ..bbot_fixtures import *\n\n\n@pytest.mark.asyncio\nasync def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):\n ### URL ###\n bad_urls = (\n \"http://e.co/index.html\",\n \"http://e.co/u/1111/info\",\n \"http://e.co/u/2222/info\",\n \"http://e.co/u/3333/info\",\n \"http://e.co/u/4444/info\",\n \"http://e.co/u/5555/info\",\n )\n new_urls = tuple(helpers.validators.collapse_urls(bad_urls, threshold=4))\n assert len(new_urls) == 2\n new_urls = tuple(sorted([u.geturl() for u in helpers.validators.collapse_urls(bad_urls, threshold=5)]))\n assert new_urls == bad_urls\n\n new_url = helpers.add_get_params(\"http://evilcorp.com/a?p=1&q=2\", {\"r\": 3, \"s\": \"asdf\"}).geturl()\n query = dict(s.split(\"=\") for s in new_url.split(\"?\")[-1].split(\"&\"))\n query = tuple(sorted(query.items(), key=lambda x: x[0]))\n assert query == (\n (\"p\", \"1\"),\n (\"q\", \"2\"),\n (\"r\", \"3\"),\n (\"s\", \"asdf\"),\n )\n assert tuple(sorted(helpers.get_get_params(\"http://evilcorp.com/a?p=1&q=2#frag\").items())) == (\n (\"p\", [\"1\"]),\n (\"q\", [\"2\"]),\n )\n\n assert helpers.validators.clean_url(\"http://evilcorp.com:80\").geturl() == \"http://evilcorp.com/\"\n assert helpers.validators.clean_url(\"http://evilcorp.com/asdf?a=asdf#frag\").geturl() == \"http://evilcorp.com/asdf\"\n assert helpers.validators.clean_url(\"http://evilcorp.com//asdf\").geturl() == \"http://evilcorp.com/asdf\"\n assert helpers.validators.clean_url(\"http://evilcorp.com.\").geturl() == \"http://evilcorp.com/\"\n with pytest.raises(ValueError):\n helpers.validators.clean_url(\"http://evilcorp,com\")\n\n assert helpers.url_depth(\"http://evilcorp.com/asdf/user/\") == 2\n assert helpers.url_depth(\"http://evilcorp.com/asdf/user\") == 2\n assert helpers.url_depth(\"http://evilcorp.com/asdf/\") == 1\n assert helpers.url_depth(\"http://evilcorp.com/asdf\") == 1\n assert helpers.url_depth(\"http://evilcorp.com/\") == 0\n assert helpers.url_depth(\"http://evilcorp.com\") == 0\n\n assert helpers.parent_url(\"http://evilcorp.com/subdir1/subdir2?foo=bar\") == \"http://evilcorp.com/subdir1\"\n\n ### MISC ###\n assert helpers.is_domain(\"evilcorp.co.uk\")\n assert not helpers.is_domain(\"www.evilcorp.co.uk\")\n assert helpers.is_domain(\"evilcorp.notreal\")\n assert not helpers.is_domain(\"asdf.evilcorp.notreal\")\n assert not helpers.is_domain(\"notreal\")\n assert helpers.is_subdomain(\"www.evilcorp.co.uk\")\n assert not helpers.is_subdomain(\"evilcorp.co.uk\")\n assert helpers.is_subdomain(\"www.evilcorp.notreal\")\n assert not helpers.is_subdomain(\"evilcorp.notreal\")\n assert not helpers.is_subdomain(\"notreal\")\n assert helpers.is_url(\"http://evilcorp.co.uk/asdf?a=b&c=d#asdf\")\n assert helpers.is_url(\"https://evilcorp.co.uk/asdf?a=b&c=d#asdf\")\n assert helpers.is_uri(\"ftp://evilcorp.co.uk\") is True\n assert helpers.is_uri(\"http://evilcorp.co.uk\") is True\n assert helpers.is_uri(\"evilcorp.co.uk\", return_scheme=True) == \"\"\n assert helpers.is_uri(\"ftp://evilcorp.co.uk\", return_scheme=True) == \"ftp\"\n assert helpers.is_uri(\"FTP://evilcorp.co.uk\", return_scheme=True) == \"ftp\"\n assert not helpers.is_url(\"https:/evilcorp.co.uk/asdf?a=b&c=d#asdf\")\n assert not helpers.is_url(\"/evilcorp.co.uk/asdf?a=b&c=d#asdf\")\n assert not helpers.is_url(\"ftp://evilcorp.co.uk\")\n assert helpers.parent_domain(\"www.evilcorp.co.uk\") == \"evilcorp.co.uk\"\n assert helpers.parent_domain(\"evilcorp.co.uk\") == \"evilcorp.co.uk\"\n assert helpers.parent_domain(\"localhost\") == \"localhost\"\n assert helpers.parent_domain(\"www.evilcorp.notreal\") == \"evilcorp.notreal\"\n assert helpers.parent_domain(\"evilcorp.notreal\") == \"evilcorp.notreal\"\n assert helpers.parent_domain(\"notreal\") == \"notreal\"\n assert list(helpers.domain_parents(\"test.www.evilcorp.co.uk\")) == [\"www.evilcorp.co.uk\", \"evilcorp.co.uk\"]\n assert list(helpers.domain_parents(\"www.evilcorp.co.uk\", include_self=True)) == [\n \"www.evilcorp.co.uk\",\n \"evilcorp.co.uk\",\n ]\n assert list(helpers.domain_parents(\"evilcorp.co.uk\", include_self=True)) == [\"evilcorp.co.uk\"]\n assert list(helpers.ip_network_parents(\"0.0.0.0/2\")) == [\n ipaddress.ip_network(\"0.0.0.0/1\"),\n ipaddress.ip_network(\"0.0.0.0/0\"),\n ]\n assert list(helpers.ip_network_parents(\"0.0.0.0/1\", include_self=True)) == [\n ipaddress.ip_network(\"0.0.0.0/1\"),\n ipaddress.ip_network(\"0.0.0.0/0\"),\n ]\n assert helpers.is_ip(\"127.0.0.1\")\n assert helpers.is_ip(\"127.0.0.1\", include_network=True)\n assert helpers.is_ip(\"127.0.0.1\", version=4)\n assert not helpers.is_ip(\"127.0.0.1\", version=6)\n assert not helpers.is_ip(\"127.0.0.0.1\")\n\n assert helpers.is_ip(\"dead::beef\")\n assert helpers.is_ip(\"dead::beef\", include_network=True)\n assert not helpers.is_ip(\"dead::beef\", version=4)\n assert helpers.is_ip(\"dead::beef\", version=6)\n assert not helpers.is_ip(\"dead:::beef\")\n\n assert not helpers.is_ip(\"1.2.3.4/24\")\n assert helpers.is_ip(\"1.2.3.4/24\", include_network=True)\n assert not helpers.is_ip(\"1.2.3.4/24\", version=4)\n assert helpers.is_ip(\"1.2.3.4/24\", include_network=True, version=4)\n assert not helpers.is_ip(\"1.2.3.4/24\", include_network=True, version=6)\n\n assert not helpers.is_ip_type(\"127.0.0.1\")\n assert helpers.is_ip_type(ipaddress.ip_address(\"127.0.0.1\"))\n assert not helpers.is_ip_type(ipaddress.ip_address(\"127.0.0.1\"), network=True)\n assert helpers.is_ip_type(ipaddress.ip_address(\"127.0.0.1\"), network=False)\n assert helpers.is_ip_type(ipaddress.ip_network(\"127.0.0.0/8\"))\n assert helpers.is_ip_type(ipaddress.ip_network(\"127.0.0.0/8\"), network=True)\n assert not helpers.is_ip_type(ipaddress.ip_network(\"127.0.0.0/8\"), network=False)\n\n assert helpers.is_dns_name(\"evilcorp.com\")\n assert not helpers.is_dns_name(\"evilcorp.com:80\")\n assert not helpers.is_dns_name(\"http://evilcorp.com:80\")\n assert helpers.is_dns_name(\"evilcorp\")\n assert helpers.is_dns_name(\"evilcorp.\")\n assert helpers.is_dns_name(\"ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\")\n assert not helpers.is_dns_name(\"127.0.0.1\")\n assert not helpers.is_dns_name(\"dead::beef\")\n assert not helpers.is_dns_name(\"bob@evilcorp.com\")\n\n assert helpers.domain_stem(\"evilcorp.co.uk\") == \"evilcorp\"\n assert helpers.domain_stem(\"www.evilcorp.co.uk\") == \"www.evilcorp\"\n\n assert tuple(await helpers.re.extract_emails(\"asdf@asdf.com\\nT@t.Com&a=a@a.com__ b@b.com\")) == (\n \"asdf@asdf.com\",\n \"t@t.com\",\n \"a@a.com\",\n \"b@b.com\",\n )\n\n assert helpers.extract_host(\"evilcorp.com:80\") == (\"evilcorp.com\", \"\", \":80\")\n assert helpers.extract_host(\"http://evilcorp.com:80/asdf.php?a=b\") == (\n \"evilcorp.com\",\n \"http://\",\n \":80/asdf.php?a=b\",\n )\n assert helpers.extract_host(\"http://evilcorp.com:80/asdf.php?a=b@a.com\") == (\n \"evilcorp.com\",\n \"http://\",\n \":80/asdf.php?a=b@a.com\",\n )\n assert helpers.extract_host(\"bob@evilcorp.com\") == (\"evilcorp.com\", \"bob@\", \"\")\n assert helpers.extract_host(\"[dead::beef]:22\") == (\"dead::beef\", \"[\", \"]:22\")\n assert helpers.extract_host(\"scp://[dead::beef]:22\") == (\"dead::beef\", \"scp://[\", \"]:22\")\n assert helpers.extract_host(\"https://[dead::beef]:22?a=b\") == (\"dead::beef\", \"https://[\", \"]:22?a=b\")\n assert helpers.extract_host(\"https://[dead::beef]/?a=b\") == (\"dead::beef\", \"https://[\", \"]/?a=b\")\n assert helpers.extract_host(\"https://[dead::beef]?a=b\") == (\"dead::beef\", \"https://[\", \"]?a=b\")\n assert helpers.extract_host(\"https://[::1]\") == (\"::1\", \"https://[\", \"]\")\n assert helpers.extract_host(\"ftp://username:password@my-ftp.com/my-file.csv\") == (\n \"my-ftp.com\",\n \"ftp://username:password@\",\n \"/my-file.csv\",\n )\n assert helpers.extract_host(\"ftp://username:p@ssword@my-ftp.com/my-file.csv\") == (\n \"my-ftp.com\",\n \"ftp://username:p@ssword@\",\n \"/my-file.csv\",\n )\n assert helpers.extract_host(\"ftp://username:password:/@my-ftp.com/my-file.csv\") == (\n \"my-ftp.com\",\n \"ftp://username:password:/@\",\n \"/my-file.csv\",\n )\n assert helpers.extract_host(\"ftp://username:password:/@dead::beef/my-file.csv\") == (\n None,\n \"ftp://username:password:/@dead::beef/my-file.csv\",\n \"\",\n )\n assert helpers.extract_host(\"ftp://username:password:/@[dead::beef]/my-file.csv\") == (\n \"dead::beef\",\n \"ftp://username:password:/@[\",\n \"]/my-file.csv\",\n )\n assert helpers.extract_host(\"ftp://username:password:/@[dead::beef]:22/my-file.csv\") == (\n \"dead::beef\",\n \"ftp://username:password:/@[\",\n \"]:22/my-file.csv\",\n )\n\n assert helpers.best_http_status(200, 404) == 200\n assert helpers.best_http_status(500, 400) == 400\n assert helpers.best_http_status(301, 302) == 301\n assert helpers.best_http_status(0, 302) == 302\n assert helpers.best_http_status(500, 0) == 500\n\n assert helpers.split_domain(\"www.evilcorp.co.uk\") == (\"www\", \"evilcorp.co.uk\")\n assert helpers.split_domain(\"asdf.www.test.notreal\") == (\"asdf.www\", \"test.notreal\")\n assert helpers.split_domain(\"www.test.notreal\") == (\"www\", \"test.notreal\")\n assert helpers.split_domain(\"test.notreal\") == (\"\", \"test.notreal\")\n assert helpers.split_domain(\"notreal\") == (\"\", \"notreal\")\n assert helpers.split_domain(\"192.168.0.1\") == (\"\", \"192.168.0.1\")\n assert helpers.split_domain(\"dead::beef\") == (\"\", \"dead::beef\")\n\n assert helpers.subdomain_depth(\"a.s.d.f.evilcorp.co.uk\") == 4\n assert helpers.subdomain_depth(\"a.s.d.f.evilcorp.com\") == 4\n assert helpers.subdomain_depth(\"evilcorp.com\") == 0\n assert helpers.subdomain_depth(\"a.evilcorp.com\") == 1\n assert helpers.subdomain_depth(\"a.s.d.f.evilcorp.notreal\") == 4\n\n assert helpers.split_host_port(\"http://evilcorp.co.uk\") == (\"evilcorp.co.uk\", 80)\n assert helpers.split_host_port(\"https://evilcorp.co.uk\") == (\"evilcorp.co.uk\", 443)\n assert helpers.split_host_port(\"ws://evilcorp.co.uk\") == (\"evilcorp.co.uk\", 80)\n assert helpers.split_host_port(\"wss://evilcorp.co.uk\") == (\"evilcorp.co.uk\", 443)\n assert helpers.split_host_port(\"WSS://evilcorp.co.uk\") == (\"evilcorp.co.uk\", 443)\n assert helpers.split_host_port(\"http://evilcorp.co.uk:666\") == (\"evilcorp.co.uk\", 666)\n assert helpers.split_host_port(\"evilcorp.co.uk:666\") == (\"evilcorp.co.uk\", 666)\n assert helpers.split_host_port(\"evilcorp.co.uk\") == (\"evilcorp.co.uk\", None)\n assert helpers.split_host_port(\"192.168.0.1\") == (ipaddress.ip_address(\"192.168.0.1\"), None)\n assert helpers.split_host_port(\"192.168.0.1:80\") == (ipaddress.ip_address(\"192.168.0.1\"), 80)\n assert helpers.split_host_port(\"[e]:80\") == (\"e\", 80)\n assert helpers.split_host_port(\"d://wat:wat\") == (\"wat\", None)\n assert helpers.split_host_port(\"https://[dead::beef]:8338\") == (ipaddress.ip_address(\"dead::beef\"), 8338)\n assert helpers.split_host_port(\"[dead::beef]\") == (ipaddress.ip_address(\"dead::beef\"), None)\n assert helpers.split_host_port(\"dead::beef\") == (ipaddress.ip_address(\"dead::beef\"), None)\n extracted_words = helpers.extract_words(\"blacklanternsecurity\")\n assert \"black\" in extracted_words\n # assert \"blacklantern\" in extracted_words\n # assert \"lanternsecurity\" in extracted_words\n # assert \"blacklanternsecurity\" in extracted_words\n assert \"bls\" in extracted_words\n\n choices = [\"asdf.fdsa\", \"asdf.1234\", \"4321.5678\"]\n best_match = helpers.closest_match(\"asdf.123a\", choices)\n assert best_match == \"asdf.1234\"\n best_matches = helpers.closest_match(\"asdf.123a\", choices, n=2)\n assert len(best_matches) == 2\n assert best_matches[0] == \"asdf.1234\"\n assert best_matches[1] == \"asdf.fdsa\"\n\n ipv4_netloc = helpers.make_netloc(\"192.168.1.1\", 80)\n assert ipv4_netloc == \"192.168.1.1:80\"\n assert helpers.make_netloc(\"192.168.1.1\") == \"192.168.1.1\"\n assert helpers.make_netloc(ipaddress.ip_address(\"192.168.1.1\"), None) == \"192.168.1.1\"\n assert helpers.make_netloc(\"dead::beef\", \"443\") == \"[dead::beef]:443\"\n assert helpers.make_netloc(ipaddress.ip_address(\"dead::beef\"), 443) == \"[dead::beef]:443\"\n assert helpers.make_netloc(\"dead::beef\", None) == \"[dead::beef]\"\n assert helpers.make_netloc(ipaddress.ip_address(\"dead::beef\"), None) == \"[dead::beef]\"\n\n assert helpers.get_file_extension(\"https://evilcorp.com/evilcorp.com/test/asdf.TXT\") == \"txt\"\n assert helpers.get_file_extension(\"/etc/conf/test.tar.gz\") == \"gz\"\n assert helpers.get_file_extension(\"/etc/passwd\") == \"\"\n\n assert helpers.tagify(\"HttP -_Web Title-- \") == \"http-web-title\"\n tagged_event = scan.make_event(\"127.0.0.1\", parent=scan.root_event, tags=[\"HttP web -__- title \"])\n assert \"http-web-title\" in tagged_event.tags\n tagged_event.remove_tag(\"http-web-title\")\n assert \"http-web-title\" not in tagged_event.tags\n tagged_event.add_tag(\"Another tag \")\n assert \"another-tag\" in tagged_event.tags\n tagged_event.tags = [\"Some other tag \"]\n assert isinstance(tagged_event._tags, set)\n assert \"another-tag\" not in tagged_event.tags\n assert \"some-other-tag\" in tagged_event.tags\n\n assert list(helpers.search_dict_by_key(\"asdf\", {\"asdf\": \"fdsa\", 4: [{\"asdf\": 5}]})) == [\"fdsa\", 5]\n assert list(helpers.search_dict_by_key(\"asdf\", {\"wat\": {\"asdf\": \"fdsa\"}})) == [\"fdsa\"]\n assert list(helpers.search_dict_by_key(\"asdf\", [{\"wat\": {\"nope\": 1}}, {\"wat\": [{\"asdf\": \"fdsa\"}]}])) == [\"fdsa\"]\n assert not list(helpers.search_dict_by_key(\"asdf\", [{\"wat\": {\"nope\": 1}}, {\"wat\": [{\"fdsa\": \"asdf\"}]}]))\n assert not list(helpers.search_dict_by_key(\"asdf\", \"asdf\"))\n\n from bbot.core.helpers.regexes import url_regexes\n\n dict_to_search = {\n \"key1\": {\n \"key2\": [{\"key3\": \"A url of some kind: https://www.evilcorp.com/asdf\"}],\n \"key4\": \"A url of some kind: https://www.evilcorp.com/fdsa\",\n }\n }\n assert set(helpers.search_dict_values(dict_to_search, *url_regexes)) == {\n \"https://www.evilcorp.com/asdf\",\n \"https://www.evilcorp.com/fdsa\",\n }\n\n replaced = helpers.search_format_dict(\n {\"asdf\": [{\"wat\": {\"here\": \"#{replaceme}!\"}}, {500: True}]}, replaceme=\"asdf\"\n )\n assert replaced[\"asdf\"][1][500] is True\n assert replaced[\"asdf\"][0][\"wat\"][\"here\"] == \"asdf!\"\n\n filtered_dict = helpers.filter_dict(\n {\"modules\": {\"c99\": {\"api_key\": \"1234\", \"filterme\": \"asdf\"}, \"ipneighbor\": {\"test\": \"test\"}}}, \"api_key\"\n )\n assert \"api_key\" in filtered_dict[\"modules\"][\"c99\"]\n assert \"filterme\" not in filtered_dict[\"modules\"][\"c99\"]\n assert \"ipneighbor\" not in filtered_dict[\"modules\"]\n\n filtered_dict2 = helpers.filter_dict(\n {\"modules\": {\"c99\": {\"api_key\": \"1234\", \"filterme\": \"asdf\"}, \"ipneighbor\": {\"test\": \"test\"}}}, \"c99\"\n )\n assert \"api_key\" in filtered_dict2[\"modules\"][\"c99\"]\n assert \"filterme\" in filtered_dict2[\"modules\"][\"c99\"]\n assert \"ipneighbor\" not in filtered_dict2[\"modules\"]\n\n filtered_dict3 = helpers.filter_dict(\n {\"modules\": {\"c99\": {\"api_key\": \"1234\", \"filterme\": \"asdf\"}, \"ipneighbor\": {\"test\": \"test\"}}},\n \"key\",\n fuzzy=True,\n )\n assert \"api_key\" in filtered_dict3[\"modules\"][\"c99\"]\n assert \"filterme\" not in filtered_dict3[\"modules\"][\"c99\"]\n assert \"ipneighbor\" not in filtered_dict3[\"modules\"]\n\n filtered_dict4 = helpers.filter_dict(\n {\"modules\": {\"secrets_db\": {\"api_key\": \"1234\"}, \"ipneighbor\": {\"secret\": \"test\", \"asdf\": \"1234\"}}},\n \"secret\",\n fuzzy=True,\n exclude_keys=\"modules\",\n )\n assert \"secrets_db\" not in filtered_dict4[\"modules\"]\n assert \"ipneighbor\" in filtered_dict4[\"modules\"]\n assert \"secret\" in filtered_dict4[\"modules\"][\"ipneighbor\"]\n assert \"asdf\" not in filtered_dict4[\"modules\"][\"ipneighbor\"]\n\n cleaned_dict = helpers.clean_dict(\n {\"modules\": {\"c99\": {\"api_key\": \"1234\", \"filterme\": \"asdf\"}, \"ipneighbor\": {\"test\": \"test\"}}}, \"api_key\"\n )\n assert \"api_key\" not in cleaned_dict[\"modules\"][\"c99\"]\n assert \"filterme\" in cleaned_dict[\"modules\"][\"c99\"]\n assert \"ipneighbor\" in cleaned_dict[\"modules\"]\n\n cleaned_dict2 = helpers.clean_dict(\n {\"modules\": {\"c99\": {\"api_key\": \"1234\", \"filterme\": \"asdf\"}, \"ipneighbor\": {\"test\": \"test\"}}}, \"c99\"\n )\n assert \"c99\" not in cleaned_dict2[\"modules\"]\n assert \"ipneighbor\" in cleaned_dict2[\"modules\"]\n\n cleaned_dict3 = helpers.clean_dict(\n {\"modules\": {\"c99\": {\"api_key\": \"1234\", \"filterme\": \"asdf\"}, \"ipneighbor\": {\"test\": \"test\"}}},\n \"key\",\n fuzzy=True,\n )\n assert \"api_key\" not in cleaned_dict3[\"modules\"][\"c99\"]\n assert \"filterme\" in cleaned_dict3[\"modules\"][\"c99\"]\n assert \"ipneighbor\" in cleaned_dict3[\"modules\"]\n\n cleaned_dict4 = helpers.clean_dict(\n {\"modules\": {\"secrets_db\": {\"api_key\": \"1234\"}, \"ipneighbor\": {\"secret\": \"test\", \"asdf\": \"1234\"}}},\n \"secret\",\n fuzzy=True,\n exclude_keys=\"modules\",\n )\n assert \"secrets_db\" in cleaned_dict4[\"modules\"]\n assert \"ipneighbor\" in cleaned_dict4[\"modules\"]\n assert \"secret\" not in cleaned_dict4[\"modules\"][\"ipneighbor\"]\n assert \"asdf\" in cleaned_dict4[\"modules\"][\"ipneighbor\"]\n\n assert helpers.split_list([1, 2, 3, 4, 5]) == [[1, 2], [3, 4, 5]]\n assert list(helpers.grouper(\"ABCDEFG\", 3)) == [[\"A\", \"B\", \"C\"], [\"D\", \"E\", \"F\"], [\"G\"]]\n\n assert len(helpers.rand_string(3)) == 3\n assert len(helpers.rand_string(1)) == 1\n assert len(helpers.rand_string(0)) == 0\n assert type(helpers.rand_string(0)) == str\n\n test_file = Path(scan.config[\"home\"]) / \"testfile.asdf\"\n test_file.touch()\n\n assert test_file.is_file()\n backup = helpers.backup_file(test_file)\n assert backup.name == \"testfile.1.asdf\"\n assert not test_file.exists()\n assert backup.is_file()\n test_file.touch()\n backup2 = helpers.backup_file(test_file)\n assert backup2.name == \"testfile.1.asdf\"\n assert not test_file.exists()\n assert backup2.is_file()\n older_backup = Path(scan.config[\"home\"]) / \"testfile.2.asdf\"\n assert older_backup.is_file()\n older_backup.unlink()\n backup.unlink()\n\n with open(test_file, \"w\") as f:\n f.write(\"asdf\\nfdsa\")\n\n assert \"asdf\" in helpers.str_or_file(str(test_file))\n assert \"nope\" in helpers.str_or_file(\"nope\")\n assert tuple(helpers.chain_lists([str(test_file), \"nope\"], try_files=True)) == (\"asdf\", \"fdsa\", \"nope\")\n assert tuple(helpers.chain_lists(\"one, two\", try_files=True)) == (\"one\", \"two\")\n assert tuple(helpers.chain_lists(\"one, two three ,four five\")) == (\"one\", \"two\", \"three\", \"four\", \"five\")\n assert test_file.is_file()\n\n with pytest.raises(DirectoryCreationError, match=\"Failed to create.*\"):\n helpers.mkdir(test_file)\n\n helpers.delete_file(test_file)\n assert not test_file.exists()\n\n timedelta = datetime.timedelta(hours=1, minutes=2, seconds=3)\n assert helpers.human_timedelta(timedelta) == \"1 hour, 2 minutes, 3 seconds\"\n timedelta = datetime.timedelta(hours=3, seconds=1)\n assert helpers.human_timedelta(timedelta) == \"3 hours, 1 second\"\n timedelta = datetime.timedelta(seconds=2)\n assert helpers.human_timedelta(timedelta) == \"2 seconds\"\n\n ### VALIDATORS ###\n # hosts\n assert helpers.validators.validate_host(\" evilCorp.COM.\") == \"evilcorp.com\"\n assert helpers.validators.validate_host(\"LOCALHOST \") == \"localhost\"\n assert helpers.validators.validate_host(\" 192.168.1.1\") == \"192.168.1.1\"\n assert helpers.validators.validate_host(\" Dead::c0dE \") == \"dead::c0de\"\n assert helpers.validators.validate_host(\".*.wildcard.evilcorp.com\") == \"wildcard.evilcorp.com\"\n assert helpers.validators.soft_validate(\" evilCorp.COM\", \"host\") is True\n assert helpers.validators.soft_validate(\"!@#$\", \"host\") is False\n with pytest.raises(ValueError):\n assert helpers.validators.validate_host(\"!@#$\")\n # ports\n assert helpers.validators.validate_port(666) == 666\n assert helpers.validators.validate_port(666666) == 65535\n assert helpers.validators.soft_validate(666, \"port\") is True\n assert helpers.validators.soft_validate(\"!@#$\", \"port\") is False\n with pytest.raises(ValueError):\n helpers.validators.validate_port(\"asdf\")\n # top tcp ports\n top_tcp_ports = helpers.top_tcp_ports(100)\n assert len(top_tcp_ports) == 100\n assert len(set(top_tcp_ports)) == 100\n top_tcp_ports = helpers.top_tcp_ports(800000)\n assert top_tcp_ports[:10] == [80, 23, 443, 21, 22, 25, 3389, 110, 445, 139]\n assert top_tcp_ports[-10:] == [65526, 65527, 65528, 65529, 65530, 65531, 65532, 65533, 65534, 65535]\n assert len(top_tcp_ports) == 65535\n assert len(set(top_tcp_ports)) == 65535\n assert all(isinstance(i, int) for i in top_tcp_ports)\n top_tcp_ports = helpers.top_tcp_ports(10, as_string=True)\n assert top_tcp_ports == \"80,23,443,21,22,25,3389,110,445,139\"\n # urls\n assert helpers.validators.validate_url(\" httP://evilcorP.com/asdf?a=b&c=d#e\") == \"http://evilcorp.com/asdf\"\n assert (\n helpers.validators.validate_url_parsed(\" httP://evilcorP.com/asdf?a=b&c=d#e\").geturl()\n == \"http://evilcorp.com/asdf\"\n )\n assert helpers.validators.soft_validate(\" httP://evilcorP.com/asdf?a=b&c=d#e\", \"url\") is True\n assert helpers.validators.soft_validate(\"!@#$\", \"url\") is False\n with pytest.raises(ValueError):\n helpers.validators.validate_url(\"!@#$\")\n # severities\n assert helpers.validators.validate_severity(\" iNfo\") == \"INFO\"\n assert helpers.validators.soft_validate(\" iNfo\", \"severity\") is True\n assert helpers.validators.soft_validate(\"NOPE\", \"severity\") is False\n with pytest.raises(ValueError):\n helpers.validators.validate_severity(\"NOPE\")\n # emails\n assert helpers.validators.validate_email(\" bOb@eViLcorp.COM\") == \"bob@evilcorp.com\"\n assert helpers.validators.soft_validate(\" bOb@eViLcorp.COM\", \"email\") is True\n assert helpers.validators.soft_validate(\"!@#$\", \"email\") is False\n with pytest.raises(ValueError):\n helpers.validators.validate_email(\"!@#$\")\n\n assert type(helpers.make_date()) == str\n\n # string formatter\n s = \"asdf {unused} {used}\"\n assert helpers.safe_format(s, used=\"fdsa\") == \"asdf {unused} fdsa\"\n\n # is_printable\n assert helpers.is_printable(\"asdf\") is True\n assert helpers.is_printable(r\"\"\"~!@#$^&*()_+=-<>:\"?,./;'[]\\{}|\"\"\") is True\n assert helpers.is_printable(\"ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\") is True\n assert helpers.is_printable(\"4\") is True\n assert helpers.is_printable(\"asdf\\x00\") is False\n\n # punycode\n assert helpers.smart_encode_punycode(\"ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\") == \"xn--eckwd4c7c.xn--zckzah\"\n assert helpers.smart_decode_punycode(\"xn--eckwd4c7c.xn--zckzah\") == \"ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\"\n assert helpers.smart_encode_punycode(\"evilcorp.com\") == \"evilcorp.com\"\n assert helpers.smart_decode_punycode(\"evilcorp.com\") == \"evilcorp.com\"\n assert helpers.smart_encode_punycode(\"bob_smith@ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\") == \"bob_smith@xn--eckwd4c7c.xn--zckzah\"\n assert helpers.smart_decode_punycode(\"bob_smith@xn--eckwd4c7c.xn--zckzah\") == \"bob_smith@ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト\"\n assert helpers.smart_encode_punycode(\"ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト:80\") == \"xn--eckwd4c7c.xn--zckzah:80\"\n assert helpers.smart_decode_punycode(\"xn--eckwd4c7c.xn--zckzah:80\") == \"ãƒ‰ãƒĄã‚¤ãƒŗ.テ゚ト:80\"\n\n assert await helpers.re.recursive_decode(\"Hello%20world%21\") == \"Hello world!\"\n assert (\n await helpers.re.recursive_decode(\"Hello%20%5Cu041f%5Cu0440%5Cu0438%5Cu0432%5Cu0435%5Cu0442\") == \"Hello ПŅ€Đ¸Đ˛ĐĩŅ‚\"\n )\n assert (\n await helpers.re.recursive_decode(\"%5Cu0020%5Cu041f%5Cu0440%5Cu0438%5Cu0432%5Cu0435%5Cu0442%5Cu0021\")\n == \" ПŅ€Đ¸Đ˛ĐĩŅ‚!\"\n )\n assert await helpers.re.recursive_decode(\"Hello%2520world%2521\") == \"Hello world!\"\n assert (\n await helpers.re.recursive_decode(\n \"Hello%255Cu0020%255Cu041f%255Cu0440%255Cu0438%255Cu0432%255Cu0435%255Cu0442\"\n )\n == \"Hello ПŅ€Đ¸Đ˛ĐĩŅ‚\"\n )\n assert (\n await helpers.re.recursive_decode(\n \"%255Cu0020%255Cu041f%255Cu0440%255Cu0438%255Cu0432%255Cu0435%255Cu0442%255Cu0021\"\n )\n == \" ПŅ€Đ¸Đ˛ĐĩŅ‚!\"\n )\n assert (\n await helpers.re.recursive_decode(r\"Hello\\\\nWorld\\\\\\tGreetings\\\\\\\\nMore\\nText\")\n == \"Hello\\nWorld\\tGreetings\\nMore\\nText\"\n )\n\n ### CACHE ###\n helpers.cache_put(\"string\", \"wat\")\n helpers.cache_put(\"binary\", b\"wat\")\n assert helpers.cache_get(\"string\") == \"wat\"\n assert helpers.cache_get(\"binary\") == \"wat\"\n assert helpers.cache_get(\"binary\", text=False) == b\"wat\"\n cache_filename = helpers.cache_filename(\"string\")\n (m, i, d, n, u, g, sz, atime, mtime, ctime) = os.stat(str(cache_filename))\n # change modified time to be 10 days in the past\n os.utime(str(cache_filename), times=(atime, mtime - (3600 * 24 * 10)))\n assert helpers.cache_get(\"string\", cache_hrs=24 * 7) is None\n assert helpers.cache_get(\"string\", cache_hrs=24 * 14) == \"wat\"\n\n test_file = Path(scan.config[\"home\"]) / \"testfile.asdf\"\n with open(test_file, \"w\") as f:\n for i in range(100):\n f.write(f\"{i}\\n\")\n assert len(list(open(test_file).readlines())) == 100\n assert (await helpers.wordlist(test_file)).is_file()\n truncated_file = await helpers.wordlist(test_file, lines=10)\n assert truncated_file.is_file()\n assert len(list(open(truncated_file).readlines())) == 10\n with pytest.raises(WordlistError):\n await helpers.wordlist(\"/tmp/a9pseoysadf/asdkgjaosidf\")\n test_file.unlink()\n\n # filename truncation\n super_long_filename = \"/tmp/\" + (\"a\" * 1024) + \".txt\"\n with pytest.raises(OSError):\n with open(super_long_filename, \"w\") as f:\n f.write(\"wat\")\n truncated_filename = helpers.truncate_filename(super_long_filename)\n with open(truncated_filename, \"w\") as f:\n f.write(\"wat\")\n truncated_filename.unlink()\n\n # misc DNS helpers\n assert helpers.is_ptr(\"wsc-11-22-33-44-wat.evilcorp.com\") is True\n assert helpers.is_ptr(\"wsc-11-22-33-wat.evilcorp.com\") is False\n assert helpers.is_ptr(\"11wat.evilcorp.com\") is False\n\n ## NTLM\n testheader = \"TlRMTVNTUAACAAAAHgAeADgAAAAVgorilwL+bvnVipUAAAAAAAAAAJgAmABWAAAACgBjRQAAAA9XAEkATgAtAFMANAAyAE4ATwBCAEQAVgBUAEsAOAACAB4AVwBJAE4ALQBTADQAMgBOAE8AQgBEAFYAVABLADgAAQAeAFcASQBOAC0AUwA0ADIATgBPAEIARABWAFQASwA4AAQAHgBXAEkATgAtAFMANAAyAE4ATwBCAEQAVgBUAEsAOAADAB4AVwBJAE4ALQBTADQAMgBOAE8AQgBEAFYAVABLADgABwAIAHUwOZlfoNgBAAAAAA==\"\n decoded = helpers.ntlm.ntlmdecode(testheader)\n assert decoded[\"NetBIOS_Domain_Name\"] == \"WIN-S42NOBDVTK8\"\n assert decoded[\"NetBIOS_Computer_Name\"] == \"WIN-S42NOBDVTK8\"\n assert decoded[\"DNS_Domain_name\"] == \"WIN-S42NOBDVTK8\"\n assert decoded[\"FQDN\"] == \"WIN-S42NOBDVTK8\"\n assert decoded[\"Timestamp\"] == b\"u09\\x99_\\xa0\\xd8\\x01\"\n with pytest.raises(NTLMError):\n helpers.ntlm.ntlmdecode(\"asdf\")\n\n test_filesize = bbot_test_dir / \"test_filesize\"\n test_filesize.touch()\n assert test_filesize.is_file()\n assert helpers.filesize(test_filesize) == 0\n assert helpers.filesize(bbot_test_dir / \"glkasjdlgksadlkfsdf\") == 0\n\n # memory stuff\n int(helpers.memory_status().available)\n int(helpers.swap_status().total)\n\n assert helpers.bytes_to_human(459819198709) == \"428.24GB\"\n assert helpers.human_to_bytes(\"428.24GB\") == 459819198709\n\n # ordinals\n assert helpers.integer_to_ordinal(1) == \"1st\"\n assert helpers.integer_to_ordinal(2) == \"2nd\"\n assert helpers.integer_to_ordinal(3) == \"3rd\"\n assert helpers.integer_to_ordinal(4) == \"4th\"\n assert helpers.integer_to_ordinal(11) == \"11th\"\n assert helpers.integer_to_ordinal(12) == \"12th\"\n assert helpers.integer_to_ordinal(13) == \"13th\"\n assert helpers.integer_to_ordinal(21) == \"21st\"\n assert helpers.integer_to_ordinal(22) == \"22nd\"\n assert helpers.integer_to_ordinal(23) == \"23rd\"\n assert helpers.integer_to_ordinal(101) == \"101st\"\n assert helpers.integer_to_ordinal(111) == \"111th\"\n assert helpers.integer_to_ordinal(112) == \"112th\"\n assert helpers.integer_to_ordinal(113) == \"113th\"\n assert helpers.integer_to_ordinal(0) == \"0th\"\n\n await scan._cleanup()\n\n scan1 = bbot_scanner(modules=\"ipneighbor\")\n await scan1.load_modules()\n assert int(helpers.get_size(scan1.modules[\"ipneighbor\"])) > 0\n\n await scan1._cleanup()\n\n # weighted shuffle (used for module queues)\n items = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n first_frequencies = {i: 0 for i in items}\n weights = [1, 2, 3, 4, 5]\n for i in range(10000):\n shuffled = helpers.weighted_shuffle(items, weights)\n first = shuffled[0]\n first_frequencies[first] += 1\n assert (\n first_frequencies[\"a\"]\n < first_frequencies[\"b\"]\n < first_frequencies[\"c\"]\n < first_frequencies[\"d\"]\n < first_frequencies[\"e\"]\n )\n\n # error handling helpers\n test_ran = False\n try:\n try:\n raise KeyboardInterrupt(\"asdf\")\n except KeyboardInterrupt:\n raise ValueError(\"asdf\")\n except Exception as e:\n assert len(helpers.get_exception_chain(e)) == 2\n assert len([_ for _ in helpers.get_exception_chain(e) if isinstance(_, KeyboardInterrupt)]) == 1\n assert len([_ for _ in helpers.get_exception_chain(e) if isinstance(_, ValueError)]) == 1\n assert helpers.in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)) is True\n assert helpers.in_exception_chain(e, (TypeError, OSError)) is False\n test_ran = True\n assert test_ran\n test_ran = False\n try:\n try:\n raise AttributeError(\"asdf\")\n except AttributeError:\n raise ValueError(\"asdf\")\n except Exception as e:\n assert len(helpers.get_exception_chain(e)) == 2\n assert len([_ for _ in helpers.get_exception_chain(e) if isinstance(_, AttributeError)]) == 1\n assert len([_ for _ in helpers.get_exception_chain(e) if isinstance(_, ValueError)]) == 1\n assert helpers.in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)) is False\n assert helpers.in_exception_chain(e, (KeyboardInterrupt, AttributeError)) is True\n assert helpers.in_exception_chain(e, (AttributeError,)) is True\n test_ran = True\n assert test_ran\n\n\n@pytest.mark.asyncio\nasync def test_word_cloud(helpers, bbot_scanner):\n number_mutations = helpers.word_cloud.get_number_mutations(\"base2_p013\", n=5, padding=2)\n assert \"base0_p013\" in number_mutations\n assert \"base7_p013\" in number_mutations\n assert \"base8_p013\" not in number_mutations\n assert \"base2_p008\" in number_mutations\n assert \"base2_p007\" not in number_mutations\n assert \"base2_p018\" in number_mutations\n assert \"base2_p0134\" in number_mutations\n assert \"base2_p0135\" not in number_mutations\n\n permutations = helpers.word_cloud.mutations(\"_base\", numbers=1)\n assert (\"_base\", \"dev\") in permutations\n assert (\"dev\", \"_base\") in permutations\n\n # saving and loading\n scan1 = bbot_scanner(\"127.0.0.1\")\n word_cloud = scan1.helpers.word_cloud\n word_cloud.add_word(\"lantern\")\n word_cloud.add_word(\"black\")\n word_cloud.add_word(\"black\")\n word_cloud.save()\n with open(word_cloud.default_filename) as f:\n word_cloud_content = [l.rstrip() for l in f.read().splitlines()]\n assert len(word_cloud_content) == 2\n assert \"2\\tblack\" in word_cloud_content\n assert \"1\\tlantern\" in word_cloud_content\n word_cloud.save(limit=1)\n with open(word_cloud.default_filename) as f:\n word_cloud_content = [l.rstrip() for l in f.read().splitlines()]\n assert len(word_cloud_content) == 1\n assert \"2\\tblack\" in word_cloud_content\n assert \"1\\tlantern\" not in word_cloud_content\n word_cloud.clear()\n with open(word_cloud.default_filename, \"w\") as f:\n f.write(\"plumbus\\nrumbus\")\n word_cloud.load()\n assert word_cloud[\"plumbus\"] == 1\n assert word_cloud[\"rumbus\"] == 1\n\n # mutators\n from bbot.core.helpers.wordcloud import DNSMutator\n\n m = DNSMutator()\n m.add_word(\"blacklantern-security237\")\n mutations = set(m)\n assert mutations == {\n (None,),\n (None, \"237\"),\n (None, \"-security237\"),\n (None, \"lanternsecurity237\"),\n (None, \"lantern-security237\"),\n (\"blacklantern-\", None),\n (\"blacklantern\", None, \"237\"),\n (\"blacklantern-\", None, \"237\"),\n (\"black\", None, \"security237\"),\n (\"black\", None, \"-security237\"),\n }\n\n m = DNSMutator()\n m.add_word(\"blacklantern-security\")\n m.add_word(\"sec\")\n m.add_word(\"sec2\")\n m.add_word(\"black2\")\n mutations = sorted(m.mutations(\"whitebasket\"))\n assert mutations == sorted(\n [\n \"basket\",\n \"basket-security\",\n \"basket2\",\n \"basketlantern-security\",\n \"basketlanternsecurity\",\n \"blackbasket-security\",\n \"blackbasketsecurity\",\n \"blacklantern-basket\",\n \"blacklantern-white\",\n \"blacklantern-whitebasket\",\n \"blacklanternbasket\",\n \"blacklanternwhite\",\n \"blacklanternwhitebasket\",\n \"blackwhite-security\",\n \"blackwhitebasket-security\",\n \"blackwhitebasketsecurity\",\n \"blackwhitesecurity\",\n \"white\",\n \"white-security\",\n \"white2\",\n \"whitebasket\",\n \"whitebasket-security\",\n \"whitebasket2\",\n \"whitebasketlantern-security\",\n \"whitebasketlanternsecurity\",\n \"whitelantern-security\",\n \"whitelanternsecurity\",\n ]\n )\n top_mutations = sorted(m.top_mutations().items(), key=lambda x: x[-1], reverse=True)\n assert top_mutations[:2] == [((None,), 4), ((None, \"2\"), 2)]\n\n await scan1._cleanup()\n\n\ndef test_names(helpers):\n assert helpers.names == sorted(helpers.names)\n assert helpers.adjectives == sorted(helpers.adjectives)\n\n\n@pytest.mark.asyncio\nasync def test_ratelimiter(helpers):\n from bbot.core.helpers.ratelimiter import RateLimiter\n\n results = []\n\n async def web_request(r):\n async with r:\n await asyncio.sleep(0.12345)\n results.append(None)\n\n # allow 10 requests per second\n r = RateLimiter(10, \"Test\")\n tasks = []\n # start 500 requests\n for i in range(500):\n tasks.append(asyncio.create_task(web_request(r)))\n # sleep for 5 seconds\n await asyncio.sleep(5)\n await helpers.cancel_tasks(tasks)\n # 5 seconds * 10 requests per second == 50\n assert 45 <= len(results) <= 55\n\n\ndef test_sync_to_async():\n from bbot.core.helpers.async_helpers import async_to_sync_gen\n\n # async to sync generator converter\n async def async_gen():\n for i in range(5):\n await asyncio.sleep(0.1)\n yield i\n\n sync_gen = async_to_sync_gen(async_gen())\n\n l = []\n while 1:\n try:\n l.append(next(sync_gen))\n except StopIteration:\n break\n assert l == [0, 1, 2, 3, 4]\n\n\n@pytest.mark.asyncio\nasync def test_async_helpers():\n import random\n from bbot.core.helpers.misc import as_completed\n\n async def do_stuff(r):\n await asyncio.sleep(r)\n return r\n\n random_ints = [random.random() for _ in range(1000)]\n tasks = [do_stuff(r) for r in random_ints]\n results = set()\n async for t in as_completed(tasks):\n results.add(await t)\n assert len(results) == 1000\n assert sorted(random_ints) == sorted(results)\n\n\ndef test_portparse(helpers):\n assert helpers.parse_port_string(\"80,443,22\") == [80, 443, 22]\n assert helpers.parse_port_string(80) == [80]\n\n assert helpers.parse_port_string(\"80,443,22,1000-1002\") == [80, 443, 22, 1000, 1001, 1002]\n\n with pytest.raises(ValueError) as e:\n helpers.parse_port_string(\"80,443,22,70000\")\n assert str(e.value) == \"Invalid port: 70000\"\n\n with pytest.raises(ValueError) as e:\n helpers.parse_port_string(\"80,443,22,1000-70000\")\n assert str(e.value) == \"Invalid port range: 1000-70000\"\n\n with pytest.raises(ValueError) as e:\n helpers.parse_port_string(\"80,443,22,1000-1001-1002\")\n assert str(e.value) == \"Invalid port or port range: 1000-1001-1002\"\n\n with pytest.raises(ValueError) as e:\n helpers.parse_port_string(\"80,443,22,1002-1000\")\n assert str(e.value) == \"Invalid port range: 1002-1000\"\n\n with pytest.raises(ValueError) as e:\n helpers.parse_port_string(\"80,443,22,foo\")\n assert str(e.value) == \"Invalid port or port range: foo\"\n\n\n# test chain_lists helper\n\n\ndef test_liststring_valid_strings(helpers):\n assert helpers.chain_lists(\"hello,world,bbot\") == [\"hello\", \"world\", \"bbot\"]\n\n\ndef test_liststring_invalid_string(helpers):\n with pytest.raises(ValueError) as e:\n helpers.chain_lists(\"hello,world,\\x01\", validate=True)\n assert str(e.value) == \"Invalid character in string: \\x01\"\n\n\ndef test_liststring_singleitem(helpers):\n assert helpers.chain_lists(\"hello\") == [\"hello\"]\n\n\ndef test_liststring_invalidfnchars(helpers):\n with pytest.raises(ValueError) as e:\n helpers.chain_lists(\"hello,world,bbot|test\", validate=True)\n assert str(e.value) == \"Invalid character in string: bbot|test\"\n\n\n# test parameter validation\n@pytest.mark.asyncio\nasync def test_parameter_validation(helpers):\n getparam_valid_params = {\n \"name\",\n \"age\",\n \"valid_name\",\n \"valid-name\",\n \"session_token\",\n \"user.id\",\n \"user-name\",\n \"client.id\",\n \"auth-token\",\n \"access_token\",\n \"abcd\",\n \"jqueryget\",\n \"\n \n\n \n \n \n \n \"\"\"\n\n async def setup_before_prep(self, module_test):\n # Simulate HTTP_RESPONSE detection\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": self.http_response_data}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n ajaxpro_httpresponse_detection = False\n for e in events:\n if e.type == \"TECHNOLOGY\" and e.data[\"technology\"] == \"ajaxpro\":\n ajaxpro_httpresponse_detection = True\n assert ajaxpro_httpresponse_detection, \"Ajaxpro HTTP_RESPONSE detection failed\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_anubisdb.py", "content": "from .base import ModuleTestBase\n\n\nclass TestAnubisdb(ModuleTestBase):\n async def setup_after_prep(self, module_test):\n module_test.module.abort_if = lambda e: False\n module_test.httpx_mock.add_response(\n url=\"https://jldc.me/anubis/subdomains/blacklanternsecurity.com\",\n json=[\"asdf.blacklanternsecurity.com\", \"zzzz.blacklanternsecurity.com\"],\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_apkpure.py", "content": "from pathlib import Path\nfrom .base import ModuleTestBase, tempapkfile\nfrom bbot.test.bbot_fixtures import bbot_test_dir\n\n\nclass TestAPKPure(ModuleTestBase):\n modules_overrides = [\"apkpure\", \"google_playstore\", \"speculate\"]\n config_overrides = {\"modules\": {\"apkpure\": {\"output_folder\": str(bbot_test_dir / \"test_apkpure_files\")}}}\n apk_file = tempapkfile()\n\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns({\"blacklanternsecurity.com\": {\"A\": [\"127.0.0.99\"]}})\n module_test.httpx_mock.add_response(\n url=\"https://play.google.com/store/search?q=blacklanternsecurity&c=apps\",\n text=\"\"\"\n \n \n \"blacklanternsecurity\" - Android Apps on Google Play\n \n \n \n \n \"\"\",\n )\n module_test.httpx_mock.add_response(\n url=\"https://play.google.com/store/apps/details?id=com.bbot.test\",\n text=\"\"\"\n \n \n BBOT\n \n \n \n \n \n \n \"\"\",\n )\n module_test.httpx_mock.add_response(\n url=\"https://d.apkpure.com/b/XAPK/com.bbot.test?version=latest\",\n content=self.apk_file,\n headers={\n \"Content-Type\": \"application/vnd.android.package-archive\",\n \"Content-Disposition\": \"attachment; filename=com.bbot.test.apk\",\n },\n )\n\n def check(self, module_test, events):\n assert len(events) == 6\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\" and e.data == \"blacklanternsecurity.com\" and e.scope_distance == 0\n ]\n ), \"Failed to emit target DNS_NAME\"\n assert 1 == len(\n [e for e in events if e.type == \"ORG_STUB\" and e.data == \"blacklanternsecurity\" and e.scope_distance == 0]\n ), \"Failed to find ORG_STUB\"\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"MOBILE_APP\"\n and \"android\" in e.tags\n and e.data[\"id\"] == \"com.bbot.test\"\n and e.data[\"url\"] == \"https://play.google.com/store/apps/details?id=com.bbot.test\"\n ]\n ), \"Failed to find bbot android app\"\n filesystem_event = [e for e in events if e.type == \"FILESYSTEM\" and \"com.bbot.test.apk\" in e.data[\"path\"]]\n assert 1 == len(filesystem_event), \"Failed to download apk\"\n file = Path(filesystem_event[0].data[\"path\"])\n assert file.is_file(), \"Destination apk doesn't exist\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_asn.py", "content": "from .base import ModuleTestBase\n\n\nclass TestASNBGPView(ModuleTestBase):\n targets = [\"8.8.8.8\"]\n module_name = \"asn\"\n config_overrides = {\"scope\": {\"report_distance\": 2}}\n\n response_get_asn_bgpview = {\n \"status\": \"ok\",\n \"status_message\": \"Query was successful\",\n \"data\": {\n \"ip\": \"8.8.8.8\",\n \"ptr_record\": \"dns.google\",\n \"prefixes\": [\n {\n \"prefix\": \"8.8.8.0/24\",\n \"ip\": \"8.8.8.0\",\n \"cidr\": 24,\n \"asn\": {\"asn\": 15169, \"name\": \"GOOGLE\", \"description\": \"Google LLC\", \"country_code\": \"US\"},\n \"name\": \"LVLT-GOGL-8-8-8\",\n \"description\": \"Google LLC\",\n \"country_code\": \"US\",\n }\n ],\n \"rir_allocation\": {\n \"rir_name\": \"ARIN\",\n \"country_code\": None,\n \"ip\": \"8.0.0.0\",\n \"cidr\": 9,\n \"prefix\": \"8.0.0.0/9\",\n \"date_allocated\": \"1992-12-01 00:00:00\",\n \"allocation_status\": \"allocated\",\n },\n \"iana_assignment\": {\n \"assignment_status\": \"legacy\",\n \"description\": \"Administered by ARIN\",\n \"whois_server\": \"whois.arin.net\",\n \"date_assigned\": None,\n },\n \"maxmind\": {\"country_code\": None, \"city\": None},\n },\n \"@meta\": {\"time_zone\": \"UTC\", \"api_version\": 1, \"execution_time\": \"567.18 ms\"},\n }\n response_get_emails_bgpview = {\n \"status\": \"ok\",\n \"status_message\": \"Query was successful\",\n \"data\": {\n \"asn\": 15169,\n \"name\": \"GOOGLE\",\n \"description_short\": \"Google LLC\",\n \"description_full\": [\"Google LLC\"],\n \"country_code\": \"US\",\n \"website\": \"https://about.google/intl/en/\",\n \"email_contacts\": [\"network-abuse@google.com\", \"arin-contact@google.com\"],\n \"abuse_contacts\": [\"network-abuse@google.com\"],\n \"looking_glass\": None,\n \"traffic_estimation\": None,\n \"traffic_ratio\": \"Mostly Outbound\",\n \"owner_address\": [\"1600 Amphitheatre Parkway\", \"Mountain View\", \"CA\", \"94043\", \"US\"],\n \"rir_allocation\": {\n \"rir_name\": \"ARIN\",\n \"country_code\": \"US\",\n \"date_allocated\": \"2000-03-30 00:00:00\",\n \"allocation_status\": \"assigned\",\n },\n \"iana_assignment\": {\n \"assignment_status\": None,\n \"description\": None,\n \"whois_server\": None,\n \"date_assigned\": None,\n },\n \"date_updated\": \"2023-02-07 06:39:11\",\n },\n \"@meta\": {\"time_zone\": \"UTC\", \"api_version\": 1, \"execution_time\": \"56.55 ms\"},\n }\n\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.bgpview.io/ip/8.8.8.8\", json=self.response_get_asn_bgpview\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.bgpview.io/asn/15169\", json=self.response_get_emails_bgpview\n )\n module_test.module.sources = [\"bgpview\"]\n\n def check(self, module_test, events):\n assert any(e.type == \"ASN\" for e in events)\n assert any(e.type == \"EMAIL_ADDRESS\" for e in events)\n\n\nclass TestASNRipe(ModuleTestBase):\n targets = [\"8.8.8.8\"]\n module_name = \"asn\"\n config_overrides = {\"scope\": {\"report_distance\": 2}}\n\n response_get_asn_ripe = {\n \"messages\": [],\n \"see_also\": [],\n \"version\": \"1.1\",\n \"data_call_name\": \"network-info\",\n \"data_call_status\": \"supported\",\n \"cached\": False,\n \"data\": {\"asns\": [\"15169\"], \"prefix\": \"8.8.8.0/24\"},\n \"query_id\": \"20230217212133-f278ff23-d940-4634-8115-a64dee06997b\",\n \"process_time\": 5,\n \"server_id\": \"app139\",\n \"build_version\": \"live.2023.2.1.142\",\n \"status\": \"ok\",\n \"status_code\": 200,\n \"time\": \"2023-02-17T21:21:33.428469\",\n }\n response_get_asn_metadata_ripe = {\n \"messages\": [],\n \"see_also\": [],\n \"version\": \"4.1\",\n \"data_call_name\": \"whois\",\n \"data_call_status\": \"supported - connecting to ursa\",\n \"cached\": False,\n \"data\": {\n \"records\": [\n [\n {\"key\": \"ASNumber\", \"value\": \"15169\", \"details_link\": None},\n {\"key\": \"ASName\", \"value\": \"GOOGLE\", \"details_link\": None},\n {\"key\": \"ASHandle\", \"value\": \"15169\", \"details_link\": \"https://stat.ripe.net/AS15169\"},\n {\"key\": \"RegDate\", \"value\": \"2000-03-30\", \"details_link\": None},\n {\n \"key\": \"Ref\",\n \"value\": \"https://rdap.arin.net/registry/autnum/15169\",\n \"details_link\": \"https://rdap.arin.net/registry/autnum/15169\",\n },\n {\"key\": \"source\", \"value\": \"ARIN\", \"details_link\": None},\n ],\n [\n {\"key\": \"OrgAbuseHandle\", \"value\": \"ABUSE5250-ARIN\", \"details_link\": None},\n {\"key\": \"OrgAbuseName\", \"value\": \"Abuse\", \"details_link\": None},\n {\"key\": \"OrgAbusePhone\", \"value\": \"+1-650-253-0000\", \"details_link\": None},\n {\n \"key\": \"OrgAbuseEmail\",\n \"value\": \"network-abuse@google.com\",\n \"details_link\": \"mailto:network-abuse@google.com\",\n },\n {\n \"key\": \"OrgAbuseRef\",\n \"value\": \"https://rdap.arin.net/registry/entity/ABUSE5250-ARIN\",\n \"details_link\": \"https://rdap.arin.net/registry/entity/ABUSE5250-ARIN\",\n },\n {\"key\": \"source\", \"value\": \"ARIN\", \"details_link\": None},\n ],\n [\n {\"key\": \"OrgName\", \"value\": \"Google LLC\", \"details_link\": None},\n {\"key\": \"OrgId\", \"value\": \"GOGL\", \"details_link\": None},\n {\"key\": \"Address\", \"value\": \"1600 Amphitheatre Parkway\", \"details_link\": None},\n {\"key\": \"City\", \"value\": \"Mountain View\", \"details_link\": None},\n {\"key\": \"StateProv\", \"value\": \"CA\", \"details_link\": None},\n {\"key\": \"PostalCode\", \"value\": \"94043\", \"details_link\": None},\n {\"key\": \"Country\", \"value\": \"US\", \"details_link\": None},\n {\"key\": \"RegDate\", \"value\": \"2000-03-30\", \"details_link\": None},\n {\n \"key\": \"Comment\",\n \"value\": \"Please note that the recommended way to file abuse complaints are located in the following links.\",\n \"details_link\": None,\n },\n {\n \"key\": \"Comment\",\n \"value\": \"To report abuse and illegal activity: https://www.google.com/contact/\",\n \"details_link\": None,\n },\n {\n \"key\": \"Comment\",\n \"value\": \"For legal requests: http://support.google.com/legal\",\n \"details_link\": None,\n },\n {\"key\": \"Comment\", \"value\": \"Regards,\", \"details_link\": None},\n {\"key\": \"Comment\", \"value\": \"The Google Team\", \"details_link\": None},\n {\n \"key\": \"Ref\",\n \"value\": \"https://rdap.arin.net/registry/entity/GOGL\",\n \"details_link\": \"https://rdap.arin.net/registry/entity/GOGL\",\n },\n {\"key\": \"source\", \"value\": \"ARIN\", \"details_link\": None},\n ],\n [\n {\"key\": \"OrgTechHandle\", \"value\": \"ZG39-ARIN\", \"details_link\": None},\n {\"key\": \"OrgTechName\", \"value\": \"Google LLC\", \"details_link\": None},\n {\"key\": \"OrgTechPhone\", \"value\": \"+1-650-253-0000\", \"details_link\": None},\n {\n \"key\": \"OrgTechEmail\",\n \"value\": \"arin-contact@google.com\",\n \"details_link\": \"mailto:arin-contact@google.com\",\n },\n {\n \"key\": \"OrgTechRef\",\n \"value\": \"https://rdap.arin.net/registry/entity/ZG39-ARIN\",\n \"details_link\": \"https://rdap.arin.net/registry/entity/ZG39-ARIN\",\n },\n {\"key\": \"source\", \"value\": \"ARIN\", \"details_link\": None},\n ],\n [\n {\"key\": \"RTechHandle\", \"value\": \"ZG39-ARIN\", \"details_link\": None},\n {\"key\": \"RTechName\", \"value\": \"Google LLC\", \"details_link\": None},\n {\"key\": \"RTechPhone\", \"value\": \"+1-650-253-0000\", \"details_link\": None},\n {\"key\": \"RTechEmail\", \"value\": \"arin-contact@google.com\", \"details_link\": None},\n {\n \"key\": \"RTechRef\",\n \"value\": \"https://rdap.arin.net/registry/entity/ZG39-ARIN\",\n \"details_link\": None,\n },\n {\"key\": \"source\", \"value\": \"ARIN\", \"details_link\": None},\n ],\n ],\n \"irr_records\": [],\n \"authorities\": [\"arin\"],\n \"resource\": \"15169\",\n \"query_time\": \"2023-02-17T21:25:00\",\n },\n \"query_id\": \"20230217212529-75f57efd-59f4-473f-8bdd-803062e94290\",\n \"process_time\": 268,\n \"server_id\": \"app143\",\n \"build_version\": \"live.2023.2.1.142\",\n \"status\": \"ok\",\n \"status_code\": 200,\n \"time\": \"2023-02-17T21:25:29.417812\",\n }\n\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://stat.ripe.net/data/network-info/data.json?resource=8.8.8.8\",\n json=self.response_get_asn_ripe,\n )\n module_test.httpx_mock.add_response(\n url=\"https://stat.ripe.net/data/whois/data.json?resource=15169\",\n json=self.response_get_asn_metadata_ripe,\n )\n module_test.module.sources = [\"ripe\"]\n\n def check(self, module_test, events):\n assert any(e.type == \"ASN\" for e in events)\n assert any(e.type == \"EMAIL_ADDRESS\" for e in events)\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_aspnet_bin_exposure.py", "content": "from .base import ModuleTestBase\nimport re\n\n\nclass TestAspnetBinExposure(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"aspnet_bin_exposure\"]\n config_overrides = {\n \"modules\": {\n \"aspnet_bin_exposure\": {\n \"test_dlls\": [\n \"Newtonsoft.Json.dll\",\n ]\n }\n }\n }\n\n async def setup_before_prep(self, module_test):\n # Simulate successful DLL exposure\n expect_args = {\n \"method\": \"GET\",\n \"uri\": \"/b/(S(X))in/Newtonsoft.Json.dll/(S(X))/\",\n }\n respond_args = {\n \"status\": 200,\n \"headers\": {\"content-type\": \"application/x-msdownload\"},\n \"response_data\": b\"MZ\\x90\\x00\\x03\\x00\\x00\\x00\",\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Simulate failed DLL exposure (confirmation test)\n expect_args = {\n \"method\": \"GET\",\n \"uri\": \"/b/(S(X))in/oopsnotarealdll.dll/(S(X))/\",\n }\n respond_args = {\"status\": 404}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Simulate alternative technique\n expect_args = {\n \"method\": \"GET\",\n \"uri\": \"/(S(X))/b/(S(X))in/Newtonsoft.Json.dll\",\n }\n respond_args = {\n \"status\": 200,\n \"headers\": {\"content-type\": \"application/x-msdownload\"},\n \"response_data\": b\"MZ\\x90\\x00\\x03\\x00\\x00\\x00\",\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Simulate failed alternative technique (confirmation test)\n expect_args = {\n \"method\": \"GET\",\n \"uri\": \"/(S(X))/b/(S(X))in/oopsnotarealdll.dll\",\n }\n respond_args = {\"status\": 404}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Fallback for any other requests\n expect_args = {\"uri\": re.compile(r\"^/.*$\")}\n respond_args = {\"status\": 404}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n vulnerability_found = False\n for e in events:\n if e.type == \"VULNERABILITY\" and \"IIS Bin Directory DLL Exposure\" in e.data[\"description\"]:\n vulnerability_found = True\n assert e.data[\"severity\"] == \"HIGH\", \"Vulnerability severity should be HIGH\"\n assert \"Detection Url\" in e.data[\"description\"], \"Description should include detection URL\"\n break\n\n assert vulnerability_found, \"No vulnerability event was found\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_asset_inventory.py", "content": "from .base import ModuleTestBase\n\n\nclass TestAsset_Inventory(ModuleTestBase):\n targets = [\"127.0.0.1\", \"bbottest.notreal\"]\n scan_name = \"asset_inventory_test\"\n config_overrides = {\"dns\": {\"minimal\": False}, \"modules\": {\"portscan\": {\"ports\": \"9999\"}}}\n modules_overrides = [\"asset_inventory\", \"portscan\", \"sslcert\"]\n\n masscan_output = \"\"\"{ \"ip\": \"127.0.0.1\", \"timestamp\": \"1680197558\", \"ports\": [ {\"port\": 9999, \"proto\": \"tcp\", \"status\": \"open\", \"reason\": \"syn-ack\", \"ttl\": 54} ] }\"\"\"\n\n async def setup_before_prep(self, module_test):\n async def run_masscan(command, *args, **kwargs):\n if \"masscan\" in command[:2]:\n targets = open(command[11]).read().splitlines()\n yield \"[\"\n for l in self.masscan_output.splitlines():\n if \"127.0.0.1/32\" in targets:\n yield self.masscan_output\n yield \"]\"\n else:\n async for l in module_test.scan.helpers.run_live(command, *args, **kwargs):\n yield l\n\n module_test.monkeypatch.setattr(module_test.scan.helpers, \"run_live\", run_masscan)\n\n await module_test.mock_dns(\n {\n \"1.0.0.127.in-addr.arpa\": {\"PTR\": [\"www.bbottest.notreal\"]},\n \"www.bbottest.notreal\": {\"A\": [\"127.0.0.1\"]},\n }\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"127.0.0.1:9999\" for e in events), \"No open port found\"\n assert any(e.data == \"www.bbottest.notreal\" for e in events), \"No DNS name found\"\n filename = next(module_test.scan.home.glob(\"asset-inventory.csv\"))\n with open(filename) as f:\n content = f.read()\n assert \"www.bbottest.notreal,,,127.0.0.1\" in content\n filename = next(module_test.scan.home.glob(\"asset-inventory-ip-addresses-table*.txt\"))\n with open(filename) as f:\n assert \"127.0.0.0/16\" in f.read()\n filename = next(module_test.scan.home.glob(\"asset-inventory-domains-table*.txt\"))\n with open(filename) as f:\n content = f.read()\n assert \"bbottest.notreal\" in content\n\n\nclass TestAsset_InventoryEmitPrevious(TestAsset_Inventory):\n config_overrides = {\"dns\": {\"minimal\": False}, \"modules\": {\"asset_inventory\": {\"use_previous\": True}}}\n modules_overrides = [\"asset_inventory\"]\n\n def check(self, module_test, events):\n assert any(e.data == \"www.bbottest.notreal:9999\" for e in events), \"No open port found\"\n assert any(e.data == \"www.bbottest.notreal\" for e in events), \"No DNS name found\"\n filename = next(module_test.scan.home.glob(\"asset-inventory.csv\"))\n with open(filename) as f:\n content = f.read()\n assert \"www.bbottest.notreal,,,127.0.0.1\" in content\n filename = next(module_test.scan.home.glob(\"asset-inventory-ip-addresses-table*.txt\"))\n with open(filename) as f:\n assert \"127.0.0.0/16\" in f.read()\n filename = next(module_test.scan.home.glob(\"asset-inventory-domains-table*.txt\"))\n with open(filename) as f:\n content = f.read()\n assert \"bbottest.notreal\" in content\n\n\nclass TestAsset_InventoryRecheck(TestAsset_Inventory):\n config_overrides = {\n \"dns\": {\"minimal\": False},\n \"modules\": {\"asset_inventory\": {\"use_previous\": True, \"recheck\": True}},\n }\n modules_overrides = [\"asset_inventory\"]\n\n def check(self, module_test, events):\n assert not any(e.type == \"OPEN_TCP_PORT\" for e in events), \"Open port was emitted\"\n assert any(e.data == \"www.bbottest.notreal\" for e in events), \"No DNS name found\"\n filename = next(module_test.scan.home.glob(\"asset-inventory.csv\"))\n with open(filename) as f:\n content = f.read()\n assert \"www.bbottest.notreal,,,127.0.0.1\" in content\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_azure_realm.py", "content": "from .base import ModuleTestBase\n\n\nclass TestAzure_Realm(ModuleTestBase):\n targets = [\"evilcorp.com\"]\n config_overrides = {\"scope\": {\"report_distance\": 1}}\n\n response_json = {\n \"State\": 3,\n \"UserState\": 2,\n \"Login\": \"test@evilcorp.com\",\n \"NameSpaceType\": \"Federated\",\n \"DomainName\": \"evilcorp.com\",\n \"FederationGlobalVersion\": -1,\n \"AuthURL\": \"https://evilcorp.okta.com/app/office365/deadbeef/sso/wsfed/passive?username=test%40evilcorp.com&wa=wsignin1.0&wtrevilcorplm=urn%3afederation%3aMicrosoftOnline&wctx=\",\n \"FederationBrandName\": \"EvilCorp\",\n \"AuthNForwardType\": 1,\n \"CloudInstanceName\": \"microsoftonline.com\",\n \"CloudInstanceIssuerUri\": \"urn:federation:MicrosoftOnline\",\n }\n\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns({\"evilcorp.com\": {\"A\": [\"127.0.0.5\"]}})\n module_test.httpx_mock.add_response(\n url=\"https://login.microsoftonline.com/getuserrealm.srf?login=test@evilcorp.com\",\n json=self.response_json,\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"https://evilcorp.okta.com/app/office365/deadbeef/sso/wsfed/passive\" for e in events), (\n \"Failed to detect URL\"\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_azure_tenant.py", "content": "from .base import ModuleTestBase\n\n\nclass TestAzure_Tenant(ModuleTestBase):\n tenant_response = {\n \"tenant_id\": \"cc74fc12-4142-400e-a653-f98bdeadbeef\",\n \"tenant_name\": \"blacklanternsecurity\",\n \"domain\": \"blacklanternsecurity.com\",\n \"email_domains\": [\n \"blacklanternsecurity.com\",\n \"blacklanternsecurity.onmicrosoft.com\",\n \"blsgvt.com\",\n \"o365.blacklanternsecurity.com\",\n ],\n }\n\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://azmap.dev/api/tenant?domain=blacklanternsecurity.com&extract=true\",\n json=self.tenant_response,\n )\n\n def check(self, module_test, events):\n assert any(\n e.type.startswith(\"DNS_NAME\")\n and e.data == \"blacklanternsecurity.onmicrosoft.com\"\n and \"affiliate\" in e.tags\n for e in events\n )\n assert any(\n e.type == \"AZURE_TENANT\"\n and e.data[\"tenant-id\"] == \"cc74fc12-4142-400e-a653-f98bdeadbeef\"\n and \"blacklanternsecurity.onmicrosoft.com\" in e.data[\"domains\"]\n and \"blacklanternsecurity\" in e.data[\"tenant-names\"]\n for e in events\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_baddns.py", "content": "from .base import ModuleTestBase\n\n\nclass BaseTestBaddns(ModuleTestBase):\n modules_overrides = [\"baddns\"]\n targets = [\"bad.dns\"]\n config_overrides = {\"dns\": {\"minimal\": False}}\n\n async def dispatchWHOIS(x):\n return None\n\n def select_modules(self):\n from baddns.base import get_all_modules\n\n selected_modules = []\n for m in get_all_modules():\n if m.name in [\"CNAME\"]:\n selected_modules.append(m)\n return selected_modules\n\n\nclass TestBaddns_cname_nxdomain(BaseTestBaddns):\n async def setup_after_prep(self, module_test):\n from bbot.modules import baddns as baddns_module\n from baddns.lib.whoismanager import WhoisManager\n\n await module_test.mock_dns(\n {\"bad.dns\": {\"CNAME\": [\"baddns.azurewebsites.net.\"]}, \"_NXDOMAIN\": [\"baddns.azurewebsites.net\"]}\n )\n module_test.monkeypatch.setattr(baddns_module.baddns, \"select_modules\", self.select_modules)\n module_test.monkeypatch.setattr(WhoisManager, \"dispatchWHOIS\", self.dispatchWHOIS)\n\n def check(self, module_test, events):\n assert any(e.data == \"baddns.azurewebsites.net\" for e in events), \"CNAME detection failed\"\n assert any(e.type == \"VULNERABILITY\" for e in events), \"Failed to emit VULNERABILITY\"\n assert any(\"baddns-cname\" in e.tags for e in events), \"Failed to add baddns tag\"\n\n\nclass TestBaddns_cname_signature(BaseTestBaddns):\n targets = [\"bad.dns:8888\"]\n modules_overrides = [\"baddns\", \"speculate\"]\n\n async def setup_after_prep(self, module_test):\n from bbot.modules import baddns as baddns_module\n from baddns.base import BadDNS_base\n from baddns.lib.whoismanager import WhoisManager\n\n def set_target(self, target):\n return \"127.0.0.1:8888\"\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": \"

Oops! We couldn’t find that page.

\", \"status\": 200}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n await module_test.mock_dns(\n {\"bad.dns\": {\"CNAME\": [\"baddns.bigcartel.com.\"]}, \"baddns.bigcartel.com\": {\"A\": [\"127.0.0.1\"]}}\n )\n module_test.monkeypatch.setattr(baddns_module.baddns, \"select_modules\", self.select_modules)\n module_test.monkeypatch.setattr(BadDNS_base, \"set_target\", set_target)\n module_test.monkeypatch.setattr(WhoisManager, \"dispatchWHOIS\", self.dispatchWHOIS)\n\n def check(self, module_test, events):\n assert any(e for e in events)\n assert any(e.type == \"VULNERABILITY\" and \"bigcartel.com\" in e.data[\"description\"] for e in events), (\n \"Failed to emit VULNERABILITY\"\n )\n assert any(\"baddns-cname\" in e.tags for e in events), \"Failed to add baddns tag\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_baddns_direct.py", "content": "from .base import ModuleTestBase\nfrom bbot.modules.base import BaseModule\n\n\nclass BaseTestBaddns(ModuleTestBase):\n modules_overrides = [\"baddns_direct\"]\n targets = [\"bad.dns\"]\n config_overrides = {\"dns\": {\"minimal\": False}, \"cloudcheck\": True}\n\n\nclass TestBaddns_direct_cloudflare(BaseTestBaddns):\n targets = [\"bad.dns:8888\"]\n modules_overrides = [\"baddns_direct\"]\n\n async def dispatchWHOIS(self):\n return None\n\n class DummyModule(BaseModule):\n watched_events = [\"DNS_NAME\"]\n _name = \"dummy_module\"\n events_seen = []\n\n async def handle_event(self, event):\n if event.data == \"bad.dns\":\n await self.helpers.sleep(0.5)\n self.events_seen.append(event.data)\n url = \"http://bad.dns:8888/\"\n url_event = self.scan.make_event(\n url, \"URL\", parent=self.scan.root_event, tags=[\"cdn-cloudflare\", \"in-scope\", \"status-401\"]\n )\n if url_event is not None:\n await self.emit_event(url_event)\n\n async def setup_after_prep(self, module_test):\n from baddns.base import BadDNS_base\n from baddns.lib.whoismanager import WhoisManager\n\n def set_target(self, target):\n return \"127.0.0.1:8888\"\n\n self.module_test = module_test\n\n self.dummy_module = self.DummyModule(module_test.scan)\n module_test.scan.modules[\"dummy_module\"] = self.dummy_module\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": \"The specified bucket does not exist\", \"status\": 401}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n await module_test.mock_dns({\"bad.dns\": {\"A\": [\"127.0.0.1\"]}})\n\n module_test.monkeypatch.setattr(BadDNS_base, \"set_target\", set_target)\n module_test.monkeypatch.setattr(WhoisManager, \"dispatchWHOIS\", self.dispatchWHOIS)\n\n def check(self, module_test, events):\n assert any(\n e.type == \"FINDING\"\n and \"Possible [AWS Bucket Takeover Detection] via direct BadDNS analysis. Indicator: [[Words: The specified bucket does not exist | Condition: and | Part: body] Matchers-Condition: and] Trigger: [self] baddns Module: [CNAME]\"\n in e.data[\"description\"]\n for e in events\n ), \"Failed to emit FINDING\"\n assert any(\"baddns-cname\" in e.tags for e in events), \"Failed to add baddns tag\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_baddns_zone.py", "content": "import dns\nfrom .base import ModuleTestBase\n\n\nclass BaseTestBaddns_zone(ModuleTestBase):\n modules_overrides = [\"baddns_zone\"]\n targets = [\"bad.dns\"]\n config_overrides = {\"dns\": {\"minimal\": False}}\n\n async def dispatchWHOIS(x):\n return None\n\n\nclass TestBaddns_zone_zonetransfer(BaseTestBaddns_zone):\n async def setup_after_prep(self, module_test):\n from baddns.lib.whoismanager import WhoisManager\n\n def from_xfr(*args, **kwargs):\n zone_text = \"\"\"\n@ 600 IN SOA ns.bad.dns. admin.bad.dns. (\n 1 ; Serial\n 3600 ; Refresh\n 900 ; Retry\n 604800 ; Expire\n 86400 ) ; Minimum TTL\n@ 600 IN NS ns.bad.dns.\n@ 600 IN A 127.0.0.1\nasdf 600 IN A 127.0.0.1\nzzzz 600 IN AAAA dead::beef\n\"\"\"\n zone = dns.zone.from_text(zone_text, origin=\"bad.dns.\")\n return zone\n\n await module_test.mock_dns({\"bad.dns\": {\"NS\": [\"ns1.bad.dns.\"]}, \"ns1.bad.dns\": {\"A\": [\"127.0.0.1\"]}})\n module_test.monkeypatch.setattr(\"dns.zone.from_xfr\", from_xfr)\n module_test.monkeypatch.setattr(WhoisManager, \"dispatchWHOIS\", self.dispatchWHOIS)\n\n def check(self, module_test, events):\n assert any(e.data == \"zzzz.bad.dns\" for e in events), \"Zone transfer failed (1)\"\n assert any(e.data == \"asdf.bad.dns\" for e in events), \"Zone transfer failed (2)\"\n assert any(e.type == \"VULNERABILITY\" for e in events), \"Failed to emit VULNERABILITY\"\n assert any(\"baddns-zonetransfer\" in e.tags for e in events), \"Failed to add baddns tag\"\n\n\nclass TestBaddns_zone_nsec(BaseTestBaddns_zone):\n async def setup_after_prep(self, module_test):\n from baddns.lib.whoismanager import WhoisManager\n\n await module_test.mock_dns(\n {\n \"bad.dns\": {\"A\": [\"127.0.0.5\"], \"NSEC\": [\"asdf.bad.dns\"]},\n \"asdf.bad.dns\": {\"NSEC\": [\"zzzz.bad.dns\"]},\n \"zzzz.bad.dns\": {\"NSEC\": [\"xyz.bad.dns\"]},\n }\n )\n module_test.monkeypatch.setattr(WhoisManager, \"dispatchWHOIS\", self.dispatchWHOIS)\n\n def check(self, module_test, events):\n assert any(e.data == \"zzzz.bad.dns\" for e in events), \"NSEC Walk Failed (1)\"\n assert any(e.data == \"xyz.bad.dns\" for e in events), \"NSEC Walk Failed (2)\"\n assert any(e.type == \"VULNERABILITY\" for e in events), \"Failed to emit VULNERABILITY\"\n assert any(\"baddns-nsec\" in e.tags for e in events), \"Failed to add baddns tag\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_badsecrets.py", "content": "from .base import ModuleTestBase, tempwordlist\n\n\nclass TestBadSecrets(ModuleTestBase):\n targets = [\n \"http://127.0.0.1:8888/\",\n \"http://127.0.0.1:8888/test.aspx\",\n \"http://127.0.0.1:8888/cookie.aspx\",\n \"http://127.0.0.1:8888/cookie2.aspx\",\n \"http://127.0.0.1:8888/cookie3.aspx\",\n ]\n\n sample_viewstate = \"\"\"\n
\n
\n\n
\n\n
\n\n \n \n
\n
\n\n\n\"\"\"\n\n sample_jsf_notvuln = \"\"\"\n

\n\"\"\"\n\n modules_overrides = [\"badsecrets\", \"httpx\"]\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"uri\": \"/test.aspx\"}\n respond_args = {\"response_data\": self.sample_viewstate}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n respond_args = {\"response_data\": self.sample_jsf_notvuln}\n module_test.set_expect_requests(respond_args=respond_args)\n\n expect_args = {\"uri\": \"/cookie.aspx\"}\n respond_args = {\n \"response_data\": \"

JWT Cookie Test

\",\n \"headers\": {\n \"set-cookie\": \"vulnjwt=eyJhbGciOiJIUzI1NiJ9.eyJJc3N1ZXIiOiJJc3N1ZXIiLCJVc2VybmFtZSI6IkJhZFNlY3JldHMiLCJleHAiOjE1OTMxMzM0ODMsImlhdCI6MTQ2NjkwMzA4M30.ovqRikAo_0kKJ0GVrAwQlezymxrLGjcEiW_s3UJMMCo; secure\"\n },\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"uri\": \"/cookie2.aspx\"}\n respond_args = {\n \"response_data\": \"

Express Cookie Test (ES)

\",\n \"headers\": {\n \"set-cookie\": \"connect.sid=s%3A8FnPwdeM9kdGTZlWvdaVtQ0S1BCOhY5G.qys7H2oGSLLdRsEq7sqh7btOohHsaRKqyjV4LiVnBvc; Path=/; Expires=Wed, 05 Apr 2023 04:47:29 GMT; HttpOnly\"\n },\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"uri\": \"/cookie3.aspx\"}\n respond_args = {\n \"response_data\": \"

Express Cookie Test (CS)

\",\n \"headers\": {\n \"set-cookie\": [\n \"foo=eyJ1c2VybmFtZSI6IkJib3RJc0xpZmUifQ==; path=/; HttpOnly\",\n \"foo.sig=zOQU7v7aTe_3zu7tnVuHi1MJ2DU; path=/; HttpOnly\",\n ],\n },\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n SecretFound = False\n IdentifyOnly = False\n CookieBasedDetection = False\n CookieBasedDetection_2 = False\n CookieBasedDetection_3 = False\n\n for e in events:\n if (\n e.type == \"VULNERABILITY\"\n and \"Known Secret Found.\" in e.data[\"description\"]\n and \"validationKey: 0F97BAE23F6F36801ABDB5F145124E00A6F795A97093D778EE5CD24F35B78B6FC4C0D0D4420657689C4F321F8596B59E83F02E296E970C4DEAD2DFE226294979 validationAlgo: SHA1 encryptionKey: 8CCFBC5B7589DD37DC3B4A885376D7480A69645DAEEC74F418B4877BEC008156 encryptionAlgo: AES\"\n in e.data[\"description\"]\n ):\n SecretFound = True\n\n if (\n e.type == \"FINDING\"\n and \"AHo0wmLu5ceItIi+I7XkEi1GAb4h12WZ894pA+Z4OH7bco2jXEy1RSCWwjtJcZNbWPcvPqL5zzfl03DoeMZfGGX7a9PSv+fUT8MAeKNouAGj1dZuO8srXt8xZIGg+wPCWWCzcX6IhWOtgWUwiXeSojCDTKXklsYt+kAAAAk5wOsXvb2lTJoO0Q==\"\n in e.data[\"description\"]\n ):\n IdentifyOnly = True\n\n if (\n e.type == \"VULNERABILITY\"\n and \"1234\" in e.data[\"description\"]\n and \"eyJhbGciOiJIUzI1NiJ9.eyJJc3N1ZXIiOiJJc3N1ZXIiLCJVc2VybmFtZSI6IkJhZFNlY3JldHMiLCJleHAiOjE1OTMxMzM0ODMsImlhdCI6MTQ2NjkwMzA4M30.ovqRikAo_0kKJ0GVrAwQlezymxrLGjcEiW_s3UJMMCo\"\n in e.data[\"description\"]\n ):\n CookieBasedDetection = True\n\n if (\n e.type == \"VULNERABILITY\"\n and \"keyboard cat\" in e.data[\"description\"]\n and \"s%3A8FnPwdeM9kdGTZlWvdaVtQ0S1BCOhY5G.qys7H2oGSLLdRsEq7sqh7btOohHsaRKqyjV4LiVnBvc\"\n in e.data[\"description\"]\n ):\n CookieBasedDetection_2 = True\n\n if (\n e.type == \"VULNERABILITY\"\n and \"Express.js Secret (cookie-session)\" in e.data[\"description\"]\n and \"zOQU7v7aTe_3zu7tnVuHi1MJ2DU\" in e.data[\"description\"]\n ):\n CookieBasedDetection_3 = True\n\n assert SecretFound, \"No secret found\"\n assert IdentifyOnly, \"No crypto product identified\"\n assert CookieBasedDetection, \"No JWT cookie vuln detected\"\n assert CookieBasedDetection_2, \"No Express.js cookie vuln detected\"\n assert CookieBasedDetection_3, \"No Express.js (cs dual cookies) vuln detected\"\n\n\nclass TestBadSecrets_customsecrets(TestBadSecrets):\n config_overrides = {\n \"modules\": {\n \"badsecrets\": {\n \"custom_secrets\": tempwordlist(\n [\n \"DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF,DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF\"\n ]\n )\n }\n }\n }\n\n sample_viewstate = \"\"\"\n
\n
\n\n
\n\n
\n\n \n \n
\n
\n\n\n\"\"\"\n\n def check(self, module_test, events):\n SecretFound = False\n for e in events:\n if (\n e.type == \"VULNERABILITY\"\n and \"Known Secret Found.\" in e.data[\"description\"]\n and \"DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF\" in e.data[\"description\"]\n ):\n SecretFound = True\n assert SecretFound, \"No secret found\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_bevigil.py", "content": "import random\n\nfrom .base import ModuleTestBase\n\n\nclass TestBeVigil(ModuleTestBase):\n module_name = \"bevigil\"\n config_overrides = {\"modules\": {\"bevigil\": {\"api_key\": \"asdf\", \"urls\": True}}}\n\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://osint.bevigil.com/api/blacklanternsecurity.com/subdomains/\",\n match_headers={\"X-Access-Token\": \"asdf\"},\n json={\n \"domain\": \"blacklanternsecurity.com\",\n \"subdomains\": [\n \"asdf.blacklanternsecurity.com\",\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://osint.bevigil.com/api/blacklanternsecurity.com/urls/\",\n json={\"domain\": \"blacklanternsecurity.com\", \"urls\": [\"https://asdf.blacklanternsecurity.com\"]},\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n assert any(e.data == \"https://asdf.blacklanternsecurity.com/\" for e in events), \"Failed to detect url\"\n\n\nclass TestBeVigilMultiKey(TestBeVigil):\n api_keys = [\"1234\", \"4321\", \"asdf\", \"fdsa\"]\n random.shuffle(api_keys)\n config_overrides = {\"modules\": {\"bevigil\": {\"api_key\": api_keys, \"urls\": True}}}\n\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://osint.bevigil.com/api/blacklanternsecurity.com/subdomains/\",\n match_headers={\"X-Access-Token\": \"fdsa\"},\n json={\n \"domain\": \"blacklanternsecurity.com\",\n \"subdomains\": [\n \"asdf.blacklanternsecurity.com\",\n ],\n },\n )\n module_test.httpx_mock.add_response(\n match_headers={\"X-Access-Token\": \"asdf\"},\n url=\"https://osint.bevigil.com/api/blacklanternsecurity.com/urls/\",\n json={\"domain\": \"blacklanternsecurity.com\", \"urls\": [\"https://asdf.blacklanternsecurity.com\"]},\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_bucket_amazon.py", "content": "import re\n\nfrom .base import ModuleTestBase\nfrom bbot.core.helpers.misc import rand_string\n\n__all__ = [\"random_bucket_name_1\", \"random_bucket_name_2\", \"random_bucket_name_3\", \"Bucket_Amazon_Base\"]\n\n# first one is a normal bucket\nrandom_bucket_name_1 = rand_string(15, digits=False)\n# second one is open/vulnerable\nrandom_bucket_name_2 = rand_string(15, digits=False)\n# third one is a mutation\nrandom_bucket_name_3 = f\"{random_bucket_name_2}-dev\"\n\n\nclass Bucket_Amazon_Base(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n provider = \"amazon\"\n\n random_bucket_1 = f\"{random_bucket_name_1}.s3.amazonaws.com\"\n random_bucket_2 = f\"{random_bucket_name_2}.s3-ap-southeast-2.amazonaws.com\"\n random_bucket_3 = f\"{random_bucket_name_3}.s3.amazonaws.com\"\n\n nonexistent_is_404 = True\n\n open_bucket_body = \"\"\"\n vpn-static1000falsestyle.css2017-03-18T06:41:59.000Z"bf9e72bdab09b785f05ff0395023cc35"429STANDARD\"\"\"\n\n @property\n def config_overrides(self):\n return {\"modules\": {self.module_name: {\"permutations\": True}}}\n\n @property\n def module_name(self):\n return self.__class__.__name__.lower().split(\"test\")[-1]\n\n @property\n def modules_overrides(self):\n return [\"excavate\", \"speculate\", \"httpx\", self.module_name, \"cloudcheck\"]\n\n def url_setup(self):\n self.url_1 = f\"https://{self.random_bucket_1}/\"\n self.url_2 = f\"https://{self.random_bucket_2}/\"\n self.url_3 = f\"https://{self.random_bucket_3}/\"\n\n def bucket_setup(self):\n self.url_setup()\n self.website_body = f\"\"\"\n \n \n \"\"\"\n\n async def setup_after_prep(self, module_test):\n self.bucket_setup()\n # patch mutations\n module_test.scan.helpers.word_cloud.mutations = lambda b, cloud=False: [\n (b, \"dev\"),\n ]\n module_test.set_expect_requests(\n expect_args={\"method\": \"GET\", \"uri\": \"/\"}, respond_args={\"response_data\": self.website_body}\n )\n if module_test.module.supports_open_check:\n module_test.httpx_mock.add_response(\n url=self.url_2,\n text=self.open_bucket_body,\n )\n module_test.httpx_mock.add_response(\n url=self.url_3,\n text=\"\",\n )\n if self.nonexistent_is_404:\n module_test.httpx_mock.add_response(url=re.compile(\".*\"), text=\"\", status_code=404)\n\n def check(self, module_test, events):\n storage_buckets = [e for e in events if e.type == \"STORAGE_BUCKET\"]\n assert len(storage_buckets) == 3\n assert 1 == len(\n [\n e\n for e in storage_buckets\n if e.data[\"name\"] == random_bucket_name_1\n and str(e.module) == \"cloudcheck\"\n and f\"cloud-{self.provider}\" in e.tags\n and f\"{self.provider}-domain\" in e.tags\n ]\n )\n assert 1 == len(\n [\n e\n for e in storage_buckets\n if e.data[\"name\"] == random_bucket_name_2\n and str(e.module) == \"cloudcheck\"\n and f\"cloud-{self.provider}\" in e.tags\n and f\"{self.provider}-domain\" in e.tags\n ]\n )\n assert 1 == len(\n [\n e\n for e in storage_buckets\n if e.data[\"name\"] == random_bucket_name_3\n and str(e.module) == str(self.module_name)\n and f\"cloud-{module_test.module.cloudcheck_provider_name.lower()}\" in e.tags\n and f\"{module_test.module.cloudcheck_provider_name.lower()}-domain\" in e.tags\n ]\n )\n # make sure open buckets were found\n if module_test.module.supports_open_check:\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"FINDING\"\n and str(e.module) == self.module_name\n and e.data.get(\"url\") == f\"https://{self.random_bucket_2}/\"\n ]\n ), f'open bucket not found for module \"{self.module_name}\"'\n # make sure bucket mutations were found\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"STORAGE_BUCKET\"\n and str(e.module) == self.module_name\n and f\"{random_bucket_name_3}\" in e.data[\"url\"]\n ]\n ), f'bucket (dev mutation: {self.random_bucket_3}) not found for module \"{self.module_name}\"'\n\n\nclass TestBucket_Amazon(Bucket_Amazon_Base):\n pass\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_bucket_digitalocean.py", "content": "from .test_module_bucket_amazon import *\n\n\nclass TestBucket_DigitalOcean(Bucket_Amazon_Base):\n provider = \"digitalocean\"\n random_bucket_1 = f\"{random_bucket_name_1}.fra1.digitaloceanspaces.com\"\n random_bucket_2 = f\"{random_bucket_name_2}.fra1.digitaloceanspaces.com\"\n random_bucket_3 = f\"{random_bucket_name_3}.fra1.digitaloceanspaces.com\"\n\n open_bucket_body = \"\"\"cloud011000falsetest.doc2020-10-14T15:23:37.545Z"4d25c8699f7347acc9f41e57148c62c0"13362425STANDARD19578831957883Normal\"\"\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_bucket_file_enum.py", "content": "from .base import ModuleTestBase\nfrom bbot.test.bbot_fixtures import bbot_test_dir\n\n\nclass TestBucket_File_Enum(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"bucket_file_enum\", \"filedownload\", \"httpx\", \"excavate\", \"cloudcheck\"]\n\n download_dir = bbot_test_dir / \"test_bucket_file_enum\"\n config_overrides = {\n \"scope\": {\"report_distance\": 5},\n \"modules\": {\"filedownload\": {\"output_folder\": str(download_dir)}},\n }\n\n open_bucket_url = \"https://testbucket.s3.amazonaws.com/\"\n open_bucket_body = \"\"\"testbucket1000falseindex.html2023-05-22T23:04:38.000Z"4a2d2d114f3abf90f8bd127c1f25095a"5STANDARDtest.pdf2022-04-30T21:13:40.000Z"723b0018c2f5a7ef06a34f84f6fa97e4"388901STANDARD\"\"\"\n\n pdf_data = \"\"\"%PDF-1.\n1 0 obj<>endobj\n2 0 obj<>endobj\n3 0 obj<>endobj\ntrailer <>\"\"\"\n\n async def setup_before_prep(self, module_test):\n module_test.httpserver.expect_request(\"/\").respond_with_data(f'')\n module_test.httpx_mock.add_response(\n url=self.open_bucket_url,\n text=self.open_bucket_body,\n )\n module_test.httpx_mock.add_response(\n url=f\"{self.open_bucket_url}test.pdf\",\n text=self.pdf_data,\n headers={\"Content-Type\": \"application/pdf\"},\n )\n module_test.httpx_mock.add_response(\n url=f\"{self.open_bucket_url}test.css\",\n text=\"\",\n )\n\n def check(self, module_test, events):\n files = list((self.download_dir / \"filedownload\").glob(\"*.pdf\"))\n assert any(e.type == \"URL_UNVERIFIED\" and e.data.endswith(\"test.pdf\") for e in events)\n assert not any(e.type == \"URL_UNVERIFIED\" and e.data.endswith(\"test.css\") for e in events)\n assert any(f.name.endswith(\"test.pdf\") for f in files), \"Failed to download PDF file from open bucket\"\n assert not any(f.name.endswith(\"test.css\") for f in files), \"Unwanted CSS file was downloaded\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_bucket_firebase.py", "content": "from .test_module_bucket_amazon import *\n\n\nclass TestBucket_Firebase(Bucket_Amazon_Base):\n provider = \"google\"\n random_bucket_1 = f\"{random_bucket_name_1}.firebaseio.com\"\n random_bucket_2 = f\"{random_bucket_name_2}.firebaseio.com\"\n random_bucket_3 = f\"{random_bucket_name_3}.firebaseio.com\"\n\n def url_setup(self):\n self.url_1 = f\"https://{self.random_bucket_1}\"\n self.url_2 = f\"https://{self.random_bucket_2}/.json\"\n self.url_3 = f\"https://{self.random_bucket_3}/.json\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_bucket_google.py", "content": "from .test_module_bucket_amazon import *\n\n\nclass TestBucket_Google(Bucket_Amazon_Base):\n provider = \"google\"\n random_bucket_1 = f\"{random_bucket_name_1}.storage.googleapis.com\"\n random_bucket_2 = f\"{random_bucket_name_2}.storage.googleapis.com\"\n random_bucket_3 = f\"{random_bucket_name_3}.storage.googleapis.com\"\n open_bucket_body = \"\"\"{\n \"kind\": \"storage#testIamPermissionsResponse\",\n \"permissions\": [\n \"storage.objects.create\",\n \"storage.objects.list\"\n ]\n}\"\"\"\n\n def bucket_setup(self):\n self.url_setup()\n self.website_body = f\"\"\"\n \n \n \"\"\"\n\n def url_setup(self):\n self.url_1 = f\"https://{random_bucket_name_1}.storage.googleapis.com\"\n self.url_2 = f\"https://www.googleapis.com/storage/v1/b/{random_bucket_name_2}/iam/testPermissions?&permissions=storage.buckets.get&permissions=storage.buckets.list&permissions=storage.buckets.create&permissions=storage.buckets.delete&permissions=storage.buckets.setIamPolicy&permissions=storage.objects.get&permissions=storage.objects.list&permissions=storage.objects.create&permissions=storage.objects.delete&permissions=storage.objects.setIamPolicy\"\n self.url_3 = f\"https://www.googleapis.com/storage/v1/b/{random_bucket_name_3}\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_bucket_microsoft.py", "content": "from .test_module_bucket_amazon import *\nfrom .base import ModuleTestBase\n\n\nclass TestBucket_Microsoft(Bucket_Amazon_Base):\n provider = \"microsoft\"\n random_bucket_1 = f\"{random_bucket_name_1}.blob.core.windows.net\"\n random_bucket_2 = f\"{random_bucket_name_2}.blob.core.windows.net\"\n random_bucket_3 = f\"{random_bucket_name_3}.blob.core.windows.net\"\n\n nonexistent_is_404 = False\n\n def url_setup(self):\n self.url_1 = f\"https://{self.random_bucket_1}\"\n self.url_2 = f\"https://{self.random_bucket_2}\"\n self.url_3 = f\"https://{self.random_bucket_3}/{random_bucket_name_3}?restype=container\"\n\n\nclass TestBucket_Microsoft_NoDup(ModuleTestBase):\n targets = [\"tesla.com\"]\n module_name = \"bucket_microsoft\"\n config_overrides = {\"cloudcheck\": True}\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://tesla.blob.core.windows.net/tesla?restype=container\",\n text=\"\",\n )\n await module_test.mock_dns(\n {\n \"tesla.com\": {\"A\": [\"1.2.3.4\"]},\n \"tesla.blob.core.windows.net\": {\"A\": [\"1.2.3.4\"]},\n }\n )\n\n def check(self, module_test, events):\n assert 1 == len([e for e in events if e.type == \"STORAGE_BUCKET\"])\n bucket_event = [e for e in events if e.type == \"STORAGE_BUCKET\"][0]\n assert bucket_event.data[\"name\"] == \"tesla\"\n assert bucket_event.data[\"url\"] == \"https://tesla.blob.core.windows.net/\"\n assert (\n bucket_event.discovery_context\n == f\"bucket_azure tried bucket variations of {event.data} and found {{event.type}} at {url}\"\n )\n\n\nclass TestBucket_Microsoft_NoDup(TestBucket_Microsoft_NoDup):\n \"\"\"\n This tests _suppress_chain_dupes functionality to make sure it works as expected\n \"\"\"\n\n async def setup_after_prep(self, module_test):\n from bbot.core.event.base import STORAGE_BUCKET\n\n module_test.monkeypatch.setattr(STORAGE_BUCKET, \"_suppress_chain_dupes\", False)\n\n def check(self, module_test, events):\n assert 2 == len([e for e in events if e.type == \"STORAGE_BUCKET\"])\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_bufferoverrun.py", "content": "from .base import ModuleTestBase\n\n\nclass TestBufferOverrun(ModuleTestBase):\n config_overrides = {\"modules\": {\"bufferoverrun\": {\"api_key\": \"asdf\", \"commercial\": False}}}\n\n async def setup_before_prep(self, module_test):\n # Mock response for non-commercial API\n module_test.httpx_mock.add_response(\n url=\"https://tls.bufferover.run/dns?q=.blacklanternsecurity.com\",\n match_headers={\"x-api-key\": \"asdf\"},\n json={\"Results\": [\"1.2.3.4,example.com,*,*,sub.blacklanternsecurity.com\"]},\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"sub.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain for free API\"\n\n\nclass TestBufferOverrunCommercial(ModuleTestBase):\n modules_overrides = [\"bufferoverrun\"]\n module_name = \"bufferoverrun\"\n config_overrides = {\"modules\": {\"bufferoverrun\": {\"api_key\": \"asdf\", \"commercial\": True}}}\n\n async def setup_before_prep(self, module_test):\n # Mock response for commercial API\n module_test.httpx_mock.add_response(\n url=\"https://bufferover-run-tls.p.rapidapi.com/ipv4/dns?q=.blacklanternsecurity.com\",\n match_headers={\"x-rapidapi-host\": \"bufferover-run-tls.p.rapidapi.com\", \"x-rapidapi-key\": \"asdf\"},\n json={\"Results\": [\"5.6.7.8,blacklanternsecurity.com,*,*,sub.blacklanternsecurity.com\"]},\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"sub.blacklanternsecurity.com\" for e in events), (\n \"Failed to detect subdomain for commercial API\"\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_builtwith.py", "content": "from .base import ModuleTestBase\n\n\nclass TestBuiltWith(ModuleTestBase):\n config_overrides = {\"modules\": {\"builtwith\": {\"api_key\": \"asdf\"}}}\n\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.builtwith.com/v20/api.json?KEY=asdf&LOOKUP=blacklanternsecurity.com&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes\",\n json={\n \"Results\": [\n {\n \"Result\": {\n \"IsDB\": \"True\",\n \"Spend\": 734,\n \"Paths\": [\n {\n \"Technologies\": [\n {\n \"Name\": \"nginx\",\n \"Tag\": \"Web Server\",\n \"FirstDetected\": 1533510000000,\n \"LastDetected\": 1559516400000,\n \"IsPremium\": \"no\",\n },\n {\n \"Parent\": \"nginx\",\n \"Name\": \"Nginx 1.14\",\n \"Tag\": \"Web Server\",\n \"FirstDetected\": 1555542000000,\n \"LastDetected\": 1559516400000,\n \"IsPremium\": \"no\",\n },\n {\n \"Name\": \"Domain Not Resolving\",\n \"Tag\": \"hosting\",\n \"FirstDetected\": 1613894400000,\n \"LastDetected\": 1633244400000,\n \"IsPremium\": \"no\",\n },\n ],\n \"FirstIndexed\": 1533510000000,\n \"LastIndexed\": 1633244400000,\n \"Domain\": \"blacklanternsecurity.com\",\n \"Url\": \"\",\n \"SubDomain\": \"asdf\",\n }\n ],\n },\n \"Meta\": {\n \"Majestic\": 0,\n \"Umbrella\": 0,\n \"Vertical\": \"\",\n \"Social\": None,\n \"CompanyName\": None,\n \"Telephones\": None,\n \"Emails\": [],\n \"City\": None,\n \"State\": None,\n \"Postcode\": None,\n \"Country\": \"US\",\n \"Names\": None,\n \"ARank\": 6249242,\n \"QRank\": -1,\n },\n \"Attributes\": {\n \"Employees\": 0,\n \"MJRank\": 0,\n \"MJTLDRank\": 0,\n \"RefSN\": 0,\n \"RefIP\": 0,\n \"Followers\": 0,\n \"Sitemap\": 0,\n \"GTMTags\": 0,\n \"QubitTags\": 0,\n \"TealiumTags\": 0,\n \"AdobeTags\": 0,\n \"CDimensions\": 0,\n \"CGoals\": 0,\n \"CMetrics\": 0,\n \"ProductCount\": 0,\n },\n \"FirstIndexed\": 1389481200000,\n \"LastIndexed\": 1684220400000,\n \"Lookup\": \"blacklanternsecurity.com\",\n \"SalesRevenue\": 0,\n }\n ],\n \"Errors\": [],\n \"Trust\": None,\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.builtwith.com/redirect1/api.json?KEY=asdf&LOOKUP=blacklanternsecurity.com\",\n json={\n \"Lookup\": \"blacklanternsecurity.com\",\n \"Inbound\": [\n {\n \"Domain\": \"blacklanternsecurity.github.io\",\n \"FirstDetected\": 1564354800000,\n \"LastDetected\": 1683783431121,\n }\n ],\n \"Outbound\": None,\n },\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n assert any(e.data == \"blacklanternsecurity.github.io\" for e in events), \"Failed to detect redirect\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_bypass403.py", "content": "import re\nfrom .base import ModuleTestBase\n\n\nclass TestBypass403(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/test\"]\n modules_overrides = [\"bypass403\", \"httpx\"]\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/test..;/\"}\n respond_args = {\"response_data\": \"alive\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n module_test.httpserver.no_handler_status_code = 403\n\n def check(self, module_test, events):\n findings = [e for e in events if e.type == \"FINDING\"]\n assert len(findings) == 1\n finding = findings[0]\n assert \"http://127.0.0.1:8888/test..;/\" in finding.data[\"description\"]\n\n\nclass TestBypass403_collapsethreshold(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/test\"]\n modules_overrides = [\"bypass403\", \"httpx\"]\n\n async def setup_after_prep(self, module_test):\n respond_args = {\"response_data\": \"alive\"}\n\n # some of these won't work outside of the module because of the complex logic. This doesn't matter, we just need to get more alerts than the threshold.\n\n query_payloads = [\n \"%09\",\n \"%20\",\n \"%23\",\n \"%2e\",\n \"%2f\",\n \".\",\n \"?\",\n \";\",\n \"..;\",\n \";%09\",\n \";%09..\",\n \";%09..;\",\n \";%2f..\",\n \"*\",\n \"/*\",\n \"..;/\",\n \";/\",\n \"/..;/\",\n \"/;/\",\n \"/./\",\n \"//\",\n \"/.\",\n \"/?anything\",\n \".php\",\n \".json\",\n \".html\",\n ]\n\n for qp in query_payloads:\n expect_args = {\"method\": \"GET\", \"uri\": f\"/test{qp}\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n module_test.httpserver.no_handler_status_code = 403\n\n def check(self, module_test, events):\n findings = [e for e in events if e.type == \"FINDING\"]\n assert len(findings) == 1\n finding = findings[0]\n assert \"403 Bypass MULTIPLE SIGNATURES (exceeded threshold\" in finding.data[\"description\"]\n\n\nclass TestBypass403_aspnetcookieless(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/admin.aspx\"]\n modules_overrides = [\"bypass403\", \"httpx\"]\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": re.compile(r\"\\/\\([sS]\\(\\w+\\)\\)\\/.+\\.aspx\")}\n respond_args = {\"response_data\": \"alive\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n module_test.httpserver.no_handler_status_code = 403\n\n def check(self, module_test, events):\n findings = [e for e in events if e.type == \"FINDING\"]\n assert len(findings) == 2\n assert all(\"(S(X))/admin.aspx\" in e.data[\"description\"] for e in findings)\n\n\nclass TestBypass403_waf(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/test\"]\n modules_overrides = [\"bypass403\", \"httpx\"]\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/test..;/\"}\n respond_args = {\"response_data\": \"The requested URL was rejected\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n module_test.httpserver.no_handler_status_code = 403\n\n def check(self, module_test, events):\n findings = [e for e in events if e.type == \"FINDING\"]\n assert not any(findings)\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_c99.py", "content": "import httpx\n\nfrom .base import ModuleTestBase\n\n\nclass TestC99(ModuleTestBase):\n module_name = \"c99\"\n config_overrides = {\"modules\": {\"c99\": {\"api_key\": \"asdf\"}}}\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.c99.nl/randomnumber?key=asdf&between=1,100&json\",\n json={\"success\": True, \"output\": 65},\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.c99.nl/subdomainfinder?key=asdf&domain=blacklanternsecurity.com&json\",\n json={\n \"success\": True,\n \"subdomains\": [\n {\"subdomain\": \"asdf.blacklanternsecurity.com\", \"ip\": \"1.2.3.4\", \"cloudflare\": True},\n ],\n \"cached\": True,\n \"cache_time\": \"2023-05-19 03:13:05\",\n },\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n\n\nclass TestC99AbortThreshold1(TestC99):\n config_overrides = {\"modules\": {\"c99\": {\"api_key\": [\"6789\", \"fdsa\", \"1234\", \"4321\"]}}}\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.c99.nl/randomnumber?key=fdsa&between=1,100&json\",\n json={\"success\": True, \"output\": 65},\n )\n\n self.url_count = {}\n\n async def custom_callback(request):\n url = str(request.url)\n try:\n self.url_count[url] += 1\n except KeyError:\n self.url_count[url] = 1\n raise httpx.TimeoutException(\"timeout\")\n\n module_test.httpx_mock.add_callback(custom_callback)\n\n def check(self, module_test, events):\n assert module_test.module.api_failure_abort_threshold == 13\n assert module_test.module.errored is False\n # assert module_test.module._api_request_failures == 4\n assert module_test.module.api_retries == 4\n assert {e.data for e in events if e.type == \"DNS_NAME\"} == {\"blacklanternsecurity.com\"}\n assert self.url_count == {\n \"https://api.c99.nl/randomnumber?key=6789&between=1,100&json\": 1,\n \"https://api.c99.nl/randomnumber?key=4321&between=1,100&json\": 1,\n \"https://api.c99.nl/randomnumber?key=1234&between=1,100&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=fdsa&domain=blacklanternsecurity.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=6789&domain=blacklanternsecurity.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=4321&domain=blacklanternsecurity.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=1234&domain=blacklanternsecurity.com&json\": 1,\n }\n\n\nclass TestC99AbortThreshold2(TestC99AbortThreshold1):\n targets = [\"blacklanternsecurity.com\", \"evilcorp.com\"]\n\n async def setup_before_prep(self, module_test):\n await super().setup_before_prep(module_test)\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.com\": {\"A\": [\"127.0.0.88\"]},\n \"evilcorp.com\": {\"A\": [\"127.0.0.11\"]},\n \"evilcorp.net\": {\"A\": [\"127.0.0.22\"]},\n \"evilcorp.co.uk\": {\"A\": [\"127.0.0.33\"]},\n }\n )\n\n def check(self, module_test, events):\n assert module_test.module.api_failure_abort_threshold == 13\n assert module_test.module.errored is False\n assert module_test.module._api_request_failures == 8\n assert module_test.module.api_retries == 4\n assert {e.data for e in events if e.type == \"DNS_NAME\"} == {\"blacklanternsecurity.com\", \"evilcorp.com\"}\n assert self.url_count == {\n \"https://api.c99.nl/randomnumber?key=6789&between=1,100&json\": 1,\n \"https://api.c99.nl/randomnumber?key=4321&between=1,100&json\": 1,\n \"https://api.c99.nl/randomnumber?key=1234&between=1,100&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=fdsa&domain=blacklanternsecurity.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=6789&domain=blacklanternsecurity.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=4321&domain=blacklanternsecurity.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=1234&domain=blacklanternsecurity.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=fdsa&domain=evilcorp.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=6789&domain=evilcorp.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=4321&domain=evilcorp.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=1234&domain=evilcorp.com&json\": 1,\n }\n\n\nclass TestC99AbortThreshold3(TestC99AbortThreshold2):\n targets = [\"blacklanternsecurity.com\", \"evilcorp.com\", \"evilcorp.net\"]\n\n def check(self, module_test, events):\n assert module_test.module.api_failure_abort_threshold == 13\n assert module_test.module.errored is False\n assert module_test.module._api_request_failures == 12\n assert module_test.module.api_retries == 4\n assert {e.data for e in events if e.type == \"DNS_NAME\"} == {\n \"blacklanternsecurity.com\",\n \"evilcorp.com\",\n \"evilcorp.net\",\n }\n assert self.url_count == {\n \"https://api.c99.nl/randomnumber?key=6789&between=1,100&json\": 1,\n \"https://api.c99.nl/randomnumber?key=4321&between=1,100&json\": 1,\n \"https://api.c99.nl/randomnumber?key=1234&between=1,100&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=fdsa&domain=blacklanternsecurity.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=6789&domain=blacklanternsecurity.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=4321&domain=blacklanternsecurity.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=1234&domain=blacklanternsecurity.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=fdsa&domain=evilcorp.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=6789&domain=evilcorp.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=4321&domain=evilcorp.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=1234&domain=evilcorp.com&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=fdsa&domain=evilcorp.net&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=6789&domain=evilcorp.net&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=4321&domain=evilcorp.net&json\": 1,\n \"https://api.c99.nl/subdomainfinder?key=1234&domain=evilcorp.net&json\": 1,\n }\n\n\nclass TestC99AbortThreshold4(TestC99AbortThreshold3):\n targets = [\"blacklanternsecurity.com\", \"evilcorp.com\", \"evilcorp.net\", \"evilcorp.co.uk\"]\n\n def check(self, module_test, events):\n assert module_test.module.api_failure_abort_threshold == 13\n assert module_test.module.errored is True\n assert module_test.module._api_request_failures == 13\n assert module_test.module.api_retries == 4\n assert {e.data for e in events if e.type == \"DNS_NAME\"} == {\n \"blacklanternsecurity.com\",\n \"evilcorp.com\",\n \"evilcorp.net\",\n \"evilcorp.co.uk\",\n }\n assert len(self.url_count) == 16\n assert all(v == 1 for v in self.url_count.values())\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_censys_dns.py", "content": "from .base import ModuleTestBase\n\n\nclass TestCensys_DNS(ModuleTestBase):\n config_overrides = {\"modules\": {\"censys_dns\": {\"api_key\": \"api_id:api_secret\"}}}\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://search.censys.io/api/v1/account\",\n match_headers={\"Authorization\": \"Basic YXBpX2lkOmFwaV9zZWNyZXQ=\"},\n json={\n \"email\": \"info@blacklanternsecurity.com\",\n \"login\": \"nope\",\n \"first_login\": \"1917-08-03 20:03:55\",\n \"last_login\": \"1918-05-19 01:15:22\",\n \"quota\": {\"used\": 26, \"allowance\": 250, \"resets_at\": \"1919-06-03 16:30:32\"},\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://search.censys.io/api/v2/certificates/search\",\n match_headers={\"Authorization\": \"Basic YXBpX2lkOmFwaV9zZWNyZXQ=\"},\n method=\"POST\",\n match_json={\"q\": \"names: blacklanternsecurity.com\", \"per_page\": 100},\n json={\n \"code\": 200,\n \"status\": \"OK\",\n \"result\": {\n \"query\": \"names: blacklanternsecurity.com\",\n \"total\": 196,\n \"duration_ms\": 1046,\n \"hits\": [\n {\n \"parsed\": {\n \"validity_period\": {\n \"not_before\": \"2021-11-18T00:09:46Z\",\n \"not_after\": \"2022-11-18T00:09:46Z\",\n },\n \"issuer_dn\": \"C=US, ST=Arizona, L=Scottsdale, O=GoDaddy.com\\\\, Inc., OU=http://certs.godaddy.com/repository/, CN=Go Daddy Secure Certificate Authority - G2\",\n \"subject_dn\": \"CN=asdf.blacklanternsecurity.com\",\n },\n \"fingerprint_sha256\": \"590ad51b8db62925f0fd3f300264c6a36692e20ceec2b5a22e7e4b41c1575cdc\",\n \"names\": [\"asdf.blacklanternsecurity.com\", \"asdf2.blacklanternsecurity.com\"],\n },\n ],\n \"links\": {\"next\": \"NextToken\", \"prev\": \"\"},\n },\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://search.censys.io/api/v2/certificates/search\",\n match_headers={\"Authorization\": \"Basic YXBpX2lkOmFwaV9zZWNyZXQ=\"},\n method=\"POST\",\n match_json={\"q\": \"names: blacklanternsecurity.com\", \"per_page\": 100, \"cursor\": \"NextToken\"},\n json={\n \"code\": 200,\n \"status\": \"OK\",\n \"result\": {\n \"query\": \"names: blacklanternsecurity.com\",\n \"total\": 196,\n \"duration_ms\": 1046,\n \"hits\": [\n {\n \"parsed\": {\n \"validity_period\": {\n \"not_before\": \"2021-11-18T00:09:46Z\",\n \"not_after\": \"2022-11-18T00:09:46Z\",\n },\n \"issuer_dn\": \"C=US, ST=Arizona, L=Scottsdale, O=GoDaddy.com\\\\, Inc., OU=http://certs.godaddy.com/repository/, CN=Go Daddy Secure Certificate Authority - G2\",\n \"subject_dn\": \"CN=zzzz.blacklanternsecurity.com\",\n },\n \"fingerprint_sha256\": \"590ad51b8db62925f0fd3f300264c6a36692e20ceec2b5a22e7e4b41c1575cdc\",\n \"names\": [\"zzzz.blacklanternsecurity.com\"],\n },\n ],\n \"links\": {\"next\": \"\", \"prev\": \"\"},\n },\n },\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect asdf subdomain\"\n assert any(e.data == \"asdf2.blacklanternsecurity.com\" for e in events), \"Failed to detect asdf2 subdomain\"\n assert any(e.data == \"zzzz.blacklanternsecurity.com\" for e in events), \"Failed to detect zzzz subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_censys_ip.py", "content": "from .base import ModuleTestBase\n\n\nclass TestCensys_IP(ModuleTestBase):\n targets = [\"1.2.3.4\"]\n config_overrides = {\"modules\": {\"censys_ip\": {\"api_key\": \"api_id:api_secret\"}}}\n\n async def setup_before_prep(self, module_test):\n await module_test.mock_dns(\n {\n \"wildcard.evilcorp.com\": {\n \"A\": [\"1.2.3.4\"],\n },\n \"certname.evilcorp.com\": {\n \"A\": [\"1.2.3.4\"],\n },\n \"certsubject.evilcorp.com\": {\n \"A\": [\"1.2.3.4\"],\n },\n \"reversedns.evilcorp.com\": {\n \"A\": [\"1.2.3.4\"],\n },\n \"ptr.evilcorp.com\": {\n \"A\": [\"1.2.3.4\"],\n },\n }\n )\n module_test.httpx_mock.add_response(\n url=\"https://search.censys.io/api/v1/account\",\n match_headers={\"Authorization\": \"Basic YXBpX2lkOmFwaV9zZWNyZXQ=\"},\n json={\n \"email\": \"info@blacklanternsecurity.com\",\n \"login\": \"nope\",\n \"first_login\": \"1917-08-03 20:03:55\",\n \"last_login\": \"1918-05-19 01:15:22\",\n \"quota\": {\"used\": 26, \"allowance\": 250, \"resets_at\": \"1919-06-03 16:30:32\"},\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://search.censys.io/api/v2/hosts/1.2.3.4\",\n match_headers={\"Authorization\": \"Basic YXBpX2lkOmFwaV9zZWNyZXQ=\"},\n json={\n \"code\": 200,\n \"status\": \"OK\",\n \"result\": {\n \"ip\": \"1.2.3.4\",\n \"services\": [\n {\n \"port\": 53,\n \"service_name\": \"DNS\",\n \"transport_protocol\": \"UDP\",\n },\n {\n \"port\": 80,\n \"service_name\": \"HTTP\",\n \"extended_service_name\": \"HTTP\",\n \"transport_protocol\": \"TCP\",\n \"http\": {\n \"request\": {\n \"method\": \"GET\",\n \"uri\": \"http://1.2.3.4/\",\n },\n },\n },\n {\n \"port\": 443,\n # Real API returns service_name: \"HTTP\" for HTTPS\n \"service_name\": \"HTTP\",\n \"extended_service_name\": \"HTTPS\",\n \"transport_protocol\": \"TCP\",\n \"http\": {\n \"request\": {\n \"method\": \"GET\",\n \"uri\": \"https://1.2.3.4/\",\n },\n },\n \"tls\": {\n \"certificates\": {\n \"leaf_data\": {\n \"names\": [\n \"*.wildcard.evilcorp.com\",\n \"certname.evilcorp.com\",\n ],\n \"subject\": {\n \"common_name\": [\"certsubject.evilcorp.com\"],\n },\n },\n },\n },\n },\n {\n \"port\": 8443,\n # Real API returns service_name: \"HTTP\" for HTTPS\n \"service_name\": \"HTTP\",\n \"extended_service_name\": \"HTTPS\",\n \"transport_protocol\": \"TCP\",\n \"http\": {\n \"request\": {\n \"method\": \"GET\",\n \"uri\": \"https://1.2.3.4:8443/admin\",\n },\n },\n \"software\": [\n {\n \"uniform_resource_identifier\": \"cpe:2.3:a:apache:tomcat:9.0.50:*:*:*:*:*:*:*\",\n \"product\": \"Apache Tomcat\",\n \"vendor\": \"Apache\",\n },\n {\n \"product\": \"Java\",\n },\n ],\n },\n {\n \"port\": 22,\n \"service_name\": \"SSH\",\n \"extended_service_name\": \"SSH\",\n \"transport_protocol\": \"TCP\",\n },\n {\n \"port\": 443,\n # Real API returns service_name: \"UNKNOWN\" and transport_protocol: \"QUIC\"\n \"service_name\": \"UNKNOWN\",\n \"extended_service_name\": \"UNKNOWN\",\n \"transport_protocol\": \"QUIC\",\n },\n ],\n \"dns\": {\n \"names\": [\n \"reversedns.evilcorp.com\",\n \"ptr.evilcorp.com\",\n ],\n },\n },\n },\n )\n\n def check(self, module_test, events):\n # Check OPEN_UDP_PORT event for DNS\n assert any(e.type == \"OPEN_UDP_PORT\" and e.data == \"1.2.3.4:53\" for e in events), (\n \"Failed to detect UDP port 53\"\n )\n\n # Check OPEN_TCP_PORT events\n assert any(e.type == \"OPEN_TCP_PORT\" and e.data == \"1.2.3.4:22\" for e in events), (\n \"Failed to detect TCP port 22 (SSH)\"\n )\n assert any(e.type == \"OPEN_TCP_PORT\" and e.data == \"1.2.3.4:80\" for e in events), (\n \"Failed to detect TCP port 80\"\n )\n assert any(e.type == \"OPEN_TCP_PORT\" and e.data == \"1.2.3.4:443\" for e in events), (\n \"Failed to detect TCP port 443\"\n )\n assert any(e.type == \"OPEN_TCP_PORT\" and e.data == \"1.2.3.4:8443\" for e in events), (\n \"Failed to detect TCP port 8443\"\n )\n\n # Check OPEN_UDP_PORT for QUIC\n assert any(e.type == \"OPEN_UDP_PORT\" and e.data == \"1.2.3.4:443\" for e in events), (\n \"Failed to detect UDP port 443 (QUIC)\"\n )\n\n # Check URL_UNVERIFIED events\n assert any(e.type == \"URL_UNVERIFIED\" and e.data == \"http://1.2.3.4/\" for e in events), (\n \"Failed to detect HTTP URL\"\n )\n assert any(e.type == \"URL_UNVERIFIED\" and e.data == \"https://1.2.3.4/\" for e in events), (\n \"Failed to detect HTTPS URL\"\n )\n assert any(e.type == \"URL_UNVERIFIED\" and e.data == \"https://1.2.3.4:8443/admin\" for e in events), (\n \"Failed to detect HTTPS URL on port 8443\"\n )\n\n # Check DNS_NAME events from TLS certificate names\n assert any(e.type == \"DNS_NAME\" and e.data == \"wildcard.evilcorp.com\" for e in events), (\n \"Failed to detect wildcard.evilcorp.com from TLS cert names (wildcard stripped)\"\n )\n assert any(e.type == \"DNS_NAME\" and e.data == \"certname.evilcorp.com\" for e in events), (\n \"Failed to detect certname.evilcorp.com from TLS cert names\"\n )\n\n # Check DNS_NAME events from TLS certificate subject common_name\n assert any(\n e.type == \"DNS_NAME\" and e.data == \"certsubject.evilcorp.com\" and e.scope_distance == 0 for e in events\n ), \"Failed to detect certsubject.evilcorp.com from TLS cert subject\"\n\n # Check DNS_NAME events from dns.names (reverse DNS)\n assert any(e.type == \"DNS_NAME\" and e.data == \"reversedns.evilcorp.com\" for e in events), (\n \"Failed to detect reversedns.evilcorp.com from reverse DNS\"\n )\n assert any(e.type == \"DNS_NAME\" and e.data == \"ptr.evilcorp.com\" for e in events), (\n \"Failed to detect ptr.evilcorp.com from reverse DNS\"\n )\n\n # Check TECHNOLOGY events from software\n assert any(\n e.type == \"TECHNOLOGY\" and e.data[\"technology\"] == \"cpe:2.3:a:apache:tomcat:9.0.50:*:*:*:*:*:*:*\"\n for e in events\n ), \"Failed to detect Apache Tomcat technology with CPE\"\n assert any(e.type == \"TECHNOLOGY\" and e.data[\"technology\"] == \"Java\" for e in events), (\n \"Failed to detect Java technology without CPE\"\n )\n\n # Check PROTOCOL events (non-HTTP/DNS services)\n assert any(\n e.type == \"PROTOCOL\" and e.data[\"protocol\"] == \"SSH\" and e.data.get(\"port\") == 22 for e in events\n ), \"Failed to detect SSH protocol\"\n assert any(\n e.type == \"PROTOCOL\" and e.data[\"protocol\"] == \"QUIC\" and e.data.get(\"port\") == 443 for e in events\n ), \"Failed to detect QUIC protocol\"\n\n # Ensure HTTP/HTTPS services don't emit PROTOCOL events (but DNS does)\n assert not any(e.type == \"PROTOCOL\" and e.data[\"protocol\"] in (\"HTTP\", \"HTTPS\") for e in events), (\n \"Should not emit PROTOCOL for HTTP/HTTPS services\"\n )\n assert any(e.type == \"PROTOCOL\" and e.data[\"protocol\"] == \"DNS\" for e in events), (\n \"Should emit PROTOCOL for DNS services\"\n )\n\n\nclass TestCensys_IP_InScopeOnly(ModuleTestBase):\n \"\"\"Test that in_scope_only=True (default) does NOT query out-of-scope IPs.\"\"\"\n\n targets = [\"evilcorp.com\"]\n module_name = \"censys_ip\"\n config_overrides = {\"modules\": {\"censys_ip\": {\"api_key\": \"api_id:api_secret\", \"in_scope_only\": True}}}\n\n async def setup_before_prep(self, module_test):\n await module_test.mock_dns({\"evilcorp.com\": {\"A\": [\"1.1.1.1\"]}})\n module_test.httpx_mock.add_response(\n url=\"https://search.censys.io/api/v1/account\",\n match_headers={\"Authorization\": \"Basic YXBpX2lkOmFwaV9zZWNyZXQ=\"},\n json={\n \"quota\": {\"used\": 26, \"allowance\": 250, \"resets_at\": \"1919-06-03 16:30:32\"},\n },\n )\n # This should NOT be called because in_scope_only=True\n module_test.httpx_mock.add_response(\n url=\"https://search.censys.io/api/v2/hosts/1.1.1.1\",\n match_headers={\"Authorization\": \"Basic YXBpX2lkOmFwaV9zZWNyZXQ=\"},\n json={\n \"code\": 200,\n \"status\": \"OK\",\n \"result\": {\n \"ip\": \"1.1.1.1\",\n \"services\": [{\"port\": 80, \"service_name\": \"HTTP\", \"transport_protocol\": \"TCP\"}],\n },\n },\n )\n\n def check(self, module_test, events):\n # Should NOT have queried the IP since it's out of scope\n assert not any(e.type == \"OPEN_TCP_PORT\" and \"1.1.1.1\" in e.data for e in events), (\n \"Should not have queried out-of-scope IP with in_scope_only=True\"\n )\n\n\nclass TestCensys_IP_OutOfScope(ModuleTestBase):\n \"\"\"Test that in_scope_only=False DOES query out-of-scope IPs (up to distance 1).\"\"\"\n\n targets = [\"evilcorp.com\"]\n module_name = \"censys_ip\"\n config_overrides = {\n \"modules\": {\"censys_ip\": {\"api_key\": \"api_id:api_secret\", \"in_scope_only\": False}},\n \"dns\": {\"minimal\": False},\n \"scope\": {\"report_distance\": 1},\n }\n\n async def setup_before_prep(self, module_test):\n await module_test.mock_dns({\"evilcorp.com\": {\"A\": [\"1.1.1.1\"]}})\n module_test.httpx_mock.add_response(\n url=\"https://search.censys.io/api/v1/account\",\n match_headers={\"Authorization\": \"Basic YXBpX2lkOmFwaV9zZWNyZXQ=\"},\n json={\n \"quota\": {\"used\": 26, \"allowance\": 250, \"resets_at\": \"1919-06-03 16:30:32\"},\n },\n )\n # This SHOULD be called because in_scope_only=False\n module_test.httpx_mock.add_response(\n url=\"https://search.censys.io/api/v2/hosts/1.1.1.1\",\n match_headers={\"Authorization\": \"Basic YXBpX2lkOmFwaV9zZWNyZXQ=\"},\n json={\n \"code\": 200,\n \"status\": \"OK\",\n \"result\": {\n \"ip\": \"1.1.1.1\",\n \"services\": [{\"port\": 80, \"service_name\": \"HTTP\", \"transport_protocol\": \"TCP\"}],\n },\n },\n )\n\n def check(self, module_test, events):\n # Should have queried the IP since in_scope_only=False\n assert any(e.type == \"OPEN_TCP_PORT\" and e.data == \"1.1.1.1:80\" for e in events), (\n \"Should have queried out-of-scope IP with in_scope_only=False\"\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_certspotter.py", "content": "from .base import ModuleTestBase\n\n\nclass TestCertspotter(ModuleTestBase):\n async def setup_after_prep(self, module_test):\n module_test.module.abort_if = lambda e: False\n for t in self.targets:\n module_test.httpx_mock.add_response(\n url=\"https://api.certspotter.com/v1/issuances?domain=blacklanternsecurity.com&include_subdomains=true&expand=dns_names\",\n json=[{\"dns_names\": [\"*.asdf.blacklanternsecurity.com\"]}],\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_chaos.py", "content": "from .base import ModuleTestBase\n\n\nclass TestChaos(ModuleTestBase):\n config_overrides = {\"modules\": {\"chaos\": {\"api_key\": \"asdf\"}}}\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://dns.projectdiscovery.io/dns/example.com\",\n match_headers={\"Authorization\": \"asdf\"},\n json={\"domain\": \"example.com\", \"subdomains\": 65},\n )\n module_test.httpx_mock.add_response(\n url=\"https://dns.projectdiscovery.io/dns/blacklanternsecurity.com/subdomains\",\n match_headers={\"Authorization\": \"asdf\"},\n json={\n \"domain\": \"blacklanternsecurity.com\",\n \"subdomains\": [\n \"*.asdf.cloud\",\n ],\n },\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.cloud.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_cloudcheck.py", "content": "from .base import ModuleTestBase\n\nfrom bbot.scanner import Scanner\n\n\nclass TestCloudCheck(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\", \"asdf2.storage.googleapis.com\"]\n modules_overrides = [\"httpx\", \"excavate\", \"cloudcheck\"]\n\n async def setup_after_prep(self, module_test):\n module_test.set_expect_requests({\"uri\": \"/\"}, {\"response_data\": \"\"})\n\n scan = Scanner(config={\"cloudcheck\": True})\n await scan._prep()\n module = scan.modules[\"cloudcheck\"]\n from cloudcheck import providers\n\n # make sure we have at least one provider\n assert providers.Amazon.name == \"Amazon\"\n\n ip_event = scan.make_event(\"8.8.8.8\", parent=scan.root_event)\n aws_event1 = scan.make_event(\"amazonaws.com\", parent=scan.root_event)\n aws_event2 = scan.make_event(\"asdf.amazonaws.com\", parent=scan.root_event)\n aws_event3 = scan.make_event(\"asdfamazonaws.com\", parent=scan.root_event)\n aws_event4 = scan.make_event(\"test.asdf.aws\", parent=scan.root_event)\n\n other_event1 = scan.make_event(\"cname.evilcorp.com\", parent=scan.root_event)\n other_event2 = scan.make_event(\"cname2.evilcorp.com\", parent=scan.root_event)\n other_event3 = scan.make_event(\"cname3.evilcorp.com\", parent=scan.root_event)\n other_event2._resolved_hosts = {\"8.8.8.8\"}\n other_event3._resolved_hosts = {\"asdf.amazonaws.com\"}\n\n for event in (ip_event, other_event2):\n await module.handle_event(ip_event)\n assert \"cloud-google\" in ip_event.tags\n assert \"google-ip\" in ip_event.tags\n\n for event in (aws_event1, aws_event2, aws_event4, other_event3):\n await module.handle_event(event)\n assert \"cloud-amazon\" in event.tags, f\"{event} was not properly cloud-tagged\"\n\n assert \"amazon-domain\" in aws_event1.tags\n assert \"amazon-cname\" in other_event3.tags\n\n for event in (aws_event3, other_event1):\n await module.handle_event(event)\n assert \"cloud-amazon\" not in event.tags, f\"{event} was improperly cloud-tagged\"\n assert not any(t for t in event.tags if t.startswith(\"cloud-\") or t.startswith(\"cdn-\")), (\n f\"{event} was improperly cloud-tagged\"\n )\n\n google_event1 = scan.make_event(\"asdf.googleapis.com\", parent=scan.root_event)\n google_event2 = scan.make_event(\"asdf.google\", parent=scan.root_event)\n google_event3 = scan.make_event(\"asdf.evilcorp.com\", parent=scan.root_event)\n google_event3._resolved_hosts = {\"asdf.storage.googleapis.com\"}\n\n for event in (google_event1, google_event2, google_event3):\n await module.handle_event(event)\n assert \"cloud-google\" in event.tags, f\"{event} was not properly cloud-tagged\"\n\n await scan._cleanup()\n\n def check(self, module_test, events):\n assert 2 == len([e for e in events if e.type == \"STORAGE_BUCKET\"])\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"STORAGE_BUCKET\"\n and e.data[\"name\"] == \"asdf\"\n and str(e.module) == \"cloudcheck\"\n and \"cloud-amazon\" in e.tags\n and \"amazon-domain\" in e.tags\n and e.scope_distance == 1\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"STORAGE_BUCKET\"\n and e.data[\"name\"] == \"asdf2\"\n and str(e.module) == \"cloudcheck\"\n and \"cloud-google\" in e.tags\n and \"google-domain\" in e.tags\n and e.scope_distance == 0\n ]\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_code_repository.py", "content": "from .base import ModuleTestBase\n\n\nclass TestCodeRepository(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"excavate\", \"code_repository\"]\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\n \"response_data\": \"\"\"\n \n \n \n \n \n \n \n \"\"\"\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n assert 5 == len([e for e in events if e.type == \"CODE_REPOSITORY\"])\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"CODE_REPOSITORY\"\n and \"git\" in e.tags\n and e.data[\"url\"] == \"https://github.com/blacklanternsecurity/bbot\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"CODE_REPOSITORY\"\n and \"git\" in e.tags\n and e.data[\"url\"] == \"https://gitlab.com/blacklanternsecurity/bbot\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"CODE_REPOSITORY\"\n and \"git\" in e.tags\n and e.data[\"url\"] == \"https://gitlab.org/blacklanternsecurity/bbot\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"CODE_REPOSITORY\"\n and \"docker\" in e.tags\n and e.data[\"url\"] == \"https://hub.docker.com/r/blacklanternsecurity/bbot\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"CODE_REPOSITORY\"\n and \"postman\" in e.tags\n and e.data[\"url\"] == \"https://www.postman.com/blacklanternsecurity/bbot\"\n ]\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_credshed.py", "content": "from .base import ModuleTestBase\n\n\ncredshed_auth_response = {\n \"access_token\": \"big_access_token\",\n \"login\": True,\n}\n\n\ncredshed_response = {\n \"accounts\": [\n {\n \"e\": \"bob@blacklanternsecurity.com\",\n \"h\": [],\n \"m\": \"hello my name is bob\",\n \"p\": \"\",\n \"s\": [121562],\n \"u\": \"\",\n },\n {\n \"e\": \"judy@blacklanternsecurity.com\",\n \"h\": [\n \"539FE8942DEADBEEFBC49E6EB2F175AC\",\n \"D2D8F0E9A4A2DEADBEEF1AC80F36D61F\",\n \"$2a$12$SHIC49jLIwsobdeadbeefuWb2BKWHUOk2yhpD77A0itiZI1vJqXHm\",\n ],\n \"m\": \"hello my name is judy\",\n \"p\": \"\",\n \"s\": [80437],\n \"u\": \"\",\n },\n {\n \"e\": \"tim@blacklanternsecurity.com\",\n \"h\": [],\n \"m\": \"hello my name is tim\",\n \"p\": \"TimTamSlam69\",\n \"s\": [80437],\n \"u\": \"tim\",\n },\n ],\n \"stats\": {\n \"accounts_searched\": 9820758365,\n \"elapsed\": \"0.00\",\n \"limit\": 1000,\n \"query\": \"blacklanternsecurity.com\",\n \"query_type\": \"domain\",\n \"sources_searched\": 129957,\n \"total_count\": 3,\n \"unique_count\": 3,\n },\n}\n\n\nclass TestCredshed(ModuleTestBase):\n config_overrides = {\n \"modules\": {\"credshed\": {\"username\": \"admin\", \"password\": \"password\", \"credshed_url\": \"https://credshed.com\"}}\n }\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://credshed.com/api/auth\",\n json=credshed_auth_response,\n method=\"POST\",\n )\n module_test.httpx_mock.add_response(\n url=\"https://credshed.com/api/search\",\n json=credshed_response,\n method=\"POST\",\n )\n\n def check(self, module_test, events):\n assert len(events) == 11\n assert 1 == len([e for e in events if e.type == \"EMAIL_ADDRESS\" and e.data == \"bob@blacklanternsecurity.com\"])\n assert 1 == len([e for e in events if e.type == \"EMAIL_ADDRESS\" and e.data == \"judy@blacklanternsecurity.com\"])\n assert 1 == len([e for e in events if e.type == \"EMAIL_ADDRESS\" and e.data == \"tim@blacklanternsecurity.com\"])\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"HASHED_PASSWORD\"\n and e.data == \"judy@blacklanternsecurity.com:539FE8942DEADBEEFBC49E6EB2F175AC\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"HASHED_PASSWORD\"\n and e.data == \"judy@blacklanternsecurity.com:D2D8F0E9A4A2DEADBEEF1AC80F36D61F\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"HASHED_PASSWORD\"\n and e.data\n == \"judy@blacklanternsecurity.com:$2a$12$SHIC49jLIwsobdeadbeefuWb2BKWHUOk2yhpD77A0itiZI1vJqXHm\"\n ]\n )\n assert 1 == len(\n [e for e in events if e.type == \"PASSWORD\" and e.data == \"tim@blacklanternsecurity.com:TimTamSlam69\"]\n )\n assert 1 == len([e for e in events if e.type == \"USERNAME\" and e.data == \"tim@blacklanternsecurity.com:tim\"])\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_crt.py", "content": "from .base import ModuleTestBase\n\n\nclass TestCRT(ModuleTestBase):\n async def setup_after_prep(self, module_test):\n module_test.module.abort_if = lambda e: False\n for t in self.targets:\n module_test.httpx_mock.add_response(\n url=\"https://crt.sh?q=%25.blacklanternsecurity.com&output=json\",\n json=[{\"id\": 1, \"name_value\": \"asdf.blacklanternsecurity.com\\nzzzz.blacklanternsecurity.com\"}],\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n assert any(e.data == \"zzzz.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_crt_db.py", "content": "from .base import ModuleTestBase\n\n\nclass TestCRT_DB(ModuleTestBase):\n async def setup_after_prep(self, module_test):\n class AsyncMock:\n async def fetch(self, *args, **kwargs):\n return [\n {\"name_value\": \"asdf.blacklanternsecurity.com\"},\n {\"name_value\": \"zzzz.blacklanternsecurity.com\"},\n ]\n\n async def close(self):\n pass\n\n async def mock_connect(*args, **kwargs):\n return AsyncMock()\n\n module_test.monkeypatch.setattr(\"asyncpg.connect\", mock_connect)\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n assert any(e.data == \"zzzz.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_csv.py", "content": "from .base import ModuleTestBase\n\n\nclass TestCSV(ModuleTestBase):\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns({\"blacklanternsecurity.com\": {\"A\": [\"127.0.0.5\"]}})\n\n def check(self, module_test, events):\n csv_file = module_test.scan.home / \"output.csv\"\n context_data = f\"Scan {module_test.scan.name} seeded with DNS_NAME: blacklanternsecurity.com\"\n\n with open(csv_file) as f:\n data = f.read()\n assert \"blacklanternsecurity.com,127.0.0.5,TARGET\" in data\n assert context_data in data\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_dehashed.py", "content": "from .base import ModuleTestBase\n\n\nclass TestDehashed(ModuleTestBase):\n modules_overrides = [\"dehashed\", \"speculate\"]\n config_overrides = {\n \"scope\": {\"report_distance\": 2},\n \"modules\": {\"dehashed\": {\"api_key\": \"deadbeef\"}},\n }\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.dehashed.com/v2/search\",\n method=\"POST\",\n json={\n \"balance\": 10000,\n \"entries\": [\n {\n \"id\": \"4363462346\",\n \"email\": [\"bob@blacklanternsecurity.com\"],\n \"ip_address\": [\"127.0.0.9\"],\n \"username\": [\"bob@bob.com\"],\n \"hashed_password\": [\"$2a$12$pVmwJ7pXEr3mE.DmCCE4fOUDdeadbeefd2KuCy/tq1ZUFyEOH2bve\"],\n \"name\": [\"Bob Smith\"],\n \"phone\": [\"+91283423839\"],\n \"database_name\": \"eatstreet\",\n \"raw_record\": {\"le_only\": True, \"unstructured\": True},\n },\n {\n \"id\": \"234623453454\",\n \"email\": [\"tim@blacklanternsecurity.com\"],\n \"username\": [\"timmy\"],\n \"password\": [\"TimTamSlam69\"],\n \"name\": \"Tim Tam\",\n \"phone\": [\"+123455667\"],\n \"database_name\": \"eatstreet\",\n },\n ],\n \"took\": \"61ms\",\n \"total\": 2,\n },\n )\n await module_test.mock_dns(\n {\n \"bob.com\": {\"A\": [\"127.0.0.1\"]},\n \"blacklanternsecurity.com\": {\"A\": [\"127.0.0.1\"]},\n }\n )\n\n def check(self, module_test, events):\n assert len(events) == 12\n assert 1 == len([e for e in events if e.type == \"DNS_NAME\" and e.data == \"blacklanternsecurity.com\"])\n assert 1 == len([e for e in events if e.type == \"ORG_STUB\" and e.data == \"blacklanternsecurity\"])\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"EMAIL_ADDRESS\"\n and e.data == \"bob@bob.com\"\n and e.scope_distance == 1\n and \"affiliate\" in e.tags\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\" and e.data == \"bob.com\" and e.scope_distance == 1 and \"affiliate\" in e.tags\n ]\n )\n assert 1 == len([e for e in events if e.type == \"EMAIL_ADDRESS\" and e.data == \"bob@blacklanternsecurity.com\"])\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"USERNAME\"\n and e.data == \"bob@blacklanternsecurity.com:bob@bob.com\"\n and e.parent.data == \"bob@blacklanternsecurity.com\"\n ]\n )\n assert 1 == len([e for e in events if e.type == \"EMAIL_ADDRESS\" and e.data == \"tim@blacklanternsecurity.com\"])\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"HASHED_PASSWORD\"\n and e.data\n == \"bob@blacklanternsecurity.com:$2a$12$pVmwJ7pXEr3mE.DmCCE4fOUDdeadbeefd2KuCy/tq1ZUFyEOH2bve\"\n ]\n )\n assert 1 == len(\n [e for e in events if e.type == \"PASSWORD\" and e.data == \"tim@blacklanternsecurity.com:TimTamSlam69\"]\n )\n assert 1 == len([e for e in events if e.type == \"USERNAME\" and e.data == \"tim@blacklanternsecurity.com:timmy\"])\n\n\nclass TestDehashedBadEmail(TestDehashed):\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.dehashed.com/v2/search\",\n method=\"POST\",\n json={\n \"balance\": 10000,\n \"entries\": [\n {\n \"id\": \"EZxg4Lz-INLUt6uRXZaV\",\n \"email\": [\"foo.example.com\"],\n \"database_name\": \"Collections\",\n },\n ],\n \"took\": \"41ms\",\n \"total\": 1,\n },\n )\n\n def check(self, module_test, events):\n debug_log_content = open(module_test.scan.home / \"debug.log\").read()\n assert \"Invalid email from dehashed.com: foo.example.com\" in debug_log_content\n\n\nclass TestDehashedHTTPError(TestDehashed):\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.dehashed.com/v2/search\",\n method=\"POST\",\n json={\"error\": \"issue with request body\"},\n status_code=400,\n )\n\n def check(self, module_test, events):\n scan_log_content = open(module_test.scan.home / \"scan.log\").read()\n assert (\n 'Error retrieving results from dehashed.com (status code 400): {\"error\":\"issue with request body\"}'\n in scan_log_content\n )\n\n\nclass TestDehashedTooManyResults(TestDehashed):\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.dehashed.com/v2/search\",\n method=\"POST\",\n json={\n \"balance\": 10000,\n \"entries\": [\n {\n \"id\": \"VXhNxj46SGsW4Lworh-G\",\n \"email\": [\"bob@bob.com\"],\n \"database_name\": \"Collections\",\n },\n ],\n \"took\": \"40ms\",\n \"total\": 10001,\n },\n )\n\n def check(self, module_test, events):\n scan_log_content = open(module_test.scan.home / \"scan.log\").read()\n assert \"has 10,001 results in Dehashed. The API can only process the first 10,000 results.\" in scan_log_content\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_digitorus.py", "content": "from .base import ModuleTestBase\n\n\nclass TestDigitorus(ModuleTestBase):\n web_response = \"\"\"www.blacklanternsecurity.com
\nchat.blacklanternsecurity.com
\nwww.blacklanternsecurity.com
\ntasks.blacklanternsecurity.com
\ngitlab.blacklanternsecurity.com
\n\"\"\"\n\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://certificatedetails.com/blacklanternsecurity.com\",\n text=self.web_response,\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n assert any(e.data == \"zzzz.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_discord.py", "content": "import httpx\n\nfrom .base import ModuleTestBase\n\n\nclass TestDiscord(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/cookie.aspx\", \"http://127.0.0.1:8888/cookie2.aspx\", \"foo.bar\"]\n modules_overrides = [\"discord\", \"excavate\", \"badsecrets\", \"httpx\"]\n\n webhook_url = \"https://discord.com/api/webhooks/1234/deadbeef-P-uF-asdf\"\n config_overrides = {\"modules\": {\"discord\": {\"webhook_url\": webhook_url}}}\n\n def custom_setup(self, module_test):\n respond_args = {\n \"response_data\": '

Express Cookie Test

',\n \"headers\": {\n \"set-cookie\": \"connect.sid=s%3A8FnPwdeM9kdGTZlWvdaVtQ0S1BCOhY5G.qys7H2oGSLLdRsEq7sqh7btOohHsaRKqyjV4LiVnBvc; Path=/; Expires=Wed, 05 Apr 2023 04:47:29 GMT; HttpOnly\"\n },\n }\n module_test.set_expect_requests(expect_args={\"uri\": \"/cookie.aspx\"}, respond_args=respond_args)\n module_test.set_expect_requests(expect_args={\"uri\": \"/cookie2.aspx\"}, respond_args=respond_args)\n module_test.request_count = 0\n\n async def setup_after_prep(self, module_test):\n self.custom_setup(module_test)\n\n def custom_response(request: httpx.Request):\n module_test.request_count += 1\n if module_test.request_count == 2:\n return httpx.Response(status_code=429, json={\"retry_after\": 0.01})\n else:\n return httpx.Response(status_code=200)\n\n module_test.httpx_mock.add_callback(custom_response, url=self.webhook_url)\n\n def check(self, module_test, events):\n vulns = [e for e in events if e.type == \"VULNERABILITY\"]\n findings = [e for e in events if e.type == \"FINDING\"]\n assert len(findings) == 1\n assert len(vulns) == 2\n assert module_test.request_count == 4\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_dnsbimi.py", "content": "from .base import ModuleTestBase\n\nraw_bimi_txt_default = (\n '\"v=BIMI1;l=https://bimi.test.localdomain/logo.svg; a=https://bimi.test.localdomain/certificate.pem\"'\n)\nraw_bimi_txt_nondefault = '\"v=BIMI1; l=https://nondefault.thirdparty.tld/brand/logo.svg;a=https://nondefault.thirdparty.tld/brand/certificate.pem;\"'\n\n\nclass TestDnsbimi(ModuleTestBase):\n targets = [\"test.localdomain\"]\n modules_overrides = [\"dnsbimi\", \"speculate\"]\n config_overrides = {\n \"modules\": {\"dnsbimi\": {\"emit_raw_dns_records\": True, \"selectors\": \"default,nondefault\"}},\n \"omit_event_types\": [\"HTTP_RESPONSE\", \"RAW_TEXT\", \"DNS_NAME_UNRESOLVED\", \"FILESYSTEM\", \"WEB_PARAMETER\"],\n }\n\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns(\n {\n \"test.localdomain\": {\n \"A\": [\"127.0.0.11\"],\n },\n \"bimi.test.localdomain\": {\n \"A\": [\"127.0.0.22\"],\n },\n \"_bimi.test.localdomain\": {\n \"A\": [\"127.0.0.33\"],\n },\n \"default._bimi.test.localdomain\": {\n \"A\": [\"127.0.0.44\"],\n \"TXT\": [raw_bimi_txt_default],\n },\n \"nondefault._bimi.test.localdomain\": {\n \"A\": [\"127.0.0.44\"],\n \"TXT\": [raw_bimi_txt_nondefault],\n },\n \"_bimi.default._bimi.test.localdomain\": {\n \"A\": [\"127.0.0.44\"],\n \"TXT\": [raw_bimi_txt_default],\n },\n \"_bimi.nondefault._bimi.test.localdomain\": {\n \"A\": [\"127.0.0.44\"],\n \"TXT\": [raw_bimi_txt_default],\n },\n \"default._bimi.default._bimi.test.localdomain\": {\n \"A\": [\"127.0.0.44\"],\n \"TXT\": [raw_bimi_txt_default],\n },\n \"nondefault._bimi.nondefault._bimi.test.localdomain\": {\n \"A\": [\"127.0.0.44\"],\n \"TXT\": [raw_bimi_txt_nondefault],\n },\n }\n )\n\n def check(self, module_test, events):\n assert any(\n e.type == \"RAW_DNS_RECORD\"\n and e.data[\"host\"] == \"default._bimi.test.localdomain\"\n and e.data[\"type\"] == \"TXT\"\n and e.data[\"answer\"] == raw_bimi_txt_default\n for e in events\n ), \"Failed to emit RAW_DNS_RECORD\"\n assert any(\n e.type == \"RAW_DNS_RECORD\"\n and e.data[\"host\"] == \"nondefault._bimi.test.localdomain\"\n and e.data[\"type\"] == \"TXT\"\n and e.data[\"answer\"] == raw_bimi_txt_nondefault\n for e in events\n ), \"Failed to emit RAW_DNS_RECORD\"\n\n assert any(e.type == \"DNS_NAME\" and e.data == \"bimi.test.localdomain\" for e in events), (\n \"Failed to emit DNS_NAME\"\n )\n\n # This should be filtered by a default BBOT configuration\n assert not any(str(e.data) == \"https://nondefault.thirdparty.tld/brand/logo.svg\" for e in events)\n\n # This should not be filtered by a default BBOT configuration\n assert any(\n e.type == \"URL_UNVERIFIED\" and e.data == \"https://bimi.test.localdomain/certificate.pem\" for e in events\n ), \"Failed to emit URL_UNVERIFIED\"\n\n # These should be filtered simply due to distance\n assert not any(str(e.data) == \"https://nondefault.thirdparty.tld/brand/logo.svg\" for e in events)\n assert not any(str(e.data) == \"https://nondefault.thirdparty.tld/certificate.pem\" for e in events)\n\n # These should have been filtered via filter_event()\n assert not any(\n e.type == \"RAW_DNS_RECORD\" and e.data[\"host\"] == \"default._bimi.default._bimi.test.localdomain\"\n for e in events\n ), \"Unwanted recursion occurring\"\n assert not any(\n e.type == \"RAW_DNS_RECORD\" and e.data[\"host\"] == \"nondefault._bimi.nondefault._bimi.test.localdomain\"\n for e in events\n ), \"Unwanted recursion occurring\"\n assert not any(\n e.type == \"RAW_DNS_RECORD\" and e.data[\"host\"] == \"nondefault._bimi.default._bimi.test.localdomain\"\n for e in events\n ), \"Unwanted recursion occurring\"\n assert not any(\n e.type == \"RAW_DNS_RECORD\" and e.data[\"host\"] == \"default._bimi.nondefault._bimi.test.localdomain\"\n for e in events\n ), \"Unwanted recursion occurring\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_dnsbrute.py", "content": "from .base import ModuleTestBase, tempwordlist\n\n\nclass TestDnsbrute(ModuleTestBase):\n subdomain_wordlist = tempwordlist([\"www\", \"asdf\"])\n blacklist = [\"api.asdf.blacklanternsecurity.com\"]\n config_overrides = {\"modules\": {\"dnsbrute\": {\"wordlist\": str(subdomain_wordlist), \"max_depth\": 3}}}\n\n async def setup_after_prep(self, module_test):\n old_run_live = module_test.scan.helpers.run_live\n\n async def new_run_live(*command, check=False, text=True, **kwargs):\n if \"massdns\" in command[:2]:\n _input = [l async for l in kwargs[\"input\"]]\n if \"asdf.blacklanternsecurity.com\" in _input:\n yield \"\"\"{\"name\": \"asdf.blacklanternsecurity.com.\", \"type\": \"A\", \"class\": \"IN\", \"status\": \"NOERROR\", \"rx_ts\": 1713974911725326170, \"data\": {\"answers\": [{\"ttl\": 86400, \"type\": \"A\", \"class\": \"IN\", \"name\": \"asdf.blacklanternsecurity.com.\", \"data\": \"1.2.3.4.\"}]}, \"flags\": [\"rd\", \"ra\"], \"resolver\": \"195.226.187.130:53\", \"proto\": \"UDP\"}\"\"\"\n else:\n async for _ in old_run_live(*command, check=False, text=True, **kwargs):\n yield _\n\n module_test.monkeypatch.setattr(module_test.scan.helpers, \"run_live\", new_run_live)\n\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.com\": {\"A\": [\"4.3.2.1\"]},\n \"asdf.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n }\n )\n\n module = module_test.module\n scan = module_test.scan\n\n # test query logic\n event = scan.make_event(\"blacklanternsecurity.com\", \"DNS_NAME\", dummy=True)\n assert module.make_query(event) == \"blacklanternsecurity.com\"\n event = scan.make_event(\"asdf.blacklanternsecurity.com\", \"DNS_NAME\", dummy=True)\n assert module.make_query(event) == \"blacklanternsecurity.com\"\n event = scan.make_event(\"api.asdf.blacklanternsecurity.com\", \"DNS_NAME\", dummy=True)\n assert module.make_query(event) == \"asdf.blacklanternsecurity.com\"\n event = scan.make_event(\"test.api.asdf.blacklanternsecurity.com\", \"DNS_NAME\", dummy=True)\n assert module.make_query(event) == \"asdf.blacklanternsecurity.com\"\n\n assert module.dedup_strategy == \"lowest_parent\"\n module.dedup_strategy = \"highest_parent\"\n event = scan.make_event(\"blacklanternsecurity.com\", \"DNS_NAME\", dummy=True)\n assert module.make_query(event) == \"blacklanternsecurity.com\"\n event = scan.make_event(\"asdf.blacklanternsecurity.com\", \"DNS_NAME\", dummy=True)\n assert module.make_query(event) == \"blacklanternsecurity.com\"\n event = scan.make_event(\"api.asdf.blacklanternsecurity.com\", \"DNS_NAME\", dummy=True)\n assert module.make_query(event) == \"blacklanternsecurity.com\"\n event = scan.make_event(\"test.api.asdf.blacklanternsecurity.com\", \"DNS_NAME\", dummy=True)\n assert module.make_query(event) == \"blacklanternsecurity.com\"\n module.dedup_strategy = \"lowest_parent\"\n\n # test recursive brute-force event filtering\n event = module_test.scan.make_event(\"blacklanternsecurity.com\", \"DNS_NAME\", parent=module_test.scan.root_event)\n event.scope_distance = 0\n result, reason = await module_test.module.filter_event(event)\n assert result is True\n event = module_test.scan.make_event(\n \"www.blacklanternsecurity.com\", \"DNS_NAME\", parent=module_test.scan.root_event\n )\n event.scope_distance = 0\n result, reason = await module_test.module.filter_event(event)\n assert result is True\n event = module_test.scan.make_event(\n \"test.www.blacklanternsecurity.com\", \"DNS_NAME\", parent=module_test.scan.root_event\n )\n event.scope_distance = 0\n result, reason = await module_test.module.filter_event(event)\n assert result is True\n event = module_test.scan.make_event(\n \"asdf.test.www.blacklanternsecurity.com\", \"DNS_NAME\", parent=module_test.scan.root_event\n )\n event.scope_distance = 0\n result, reason = await module_test.module.filter_event(event)\n assert result is True\n event = module_test.scan.make_event(\n \"wat.asdf.test.www.blacklanternsecurity.com\", \"DNS_NAME\", parent=module_test.scan.root_event\n )\n event.scope_distance = 0\n result, reason = await module_test.module.filter_event(event)\n assert result is False\n assert reason == \"subdomain depth of *.asdf.test.www.blacklanternsecurity.com (4) > max_depth (3)\"\n event = module_test.scan.make_event(\n \"hmmm.ptr1234.blacklanternsecurity.com\", \"DNS_NAME\", parent=module_test.scan.root_event\n )\n event.scope_distance = 0\n result, reason = await module_test.module.filter_event(event)\n assert result is False\n assert reason == '\"ptr1234.blacklanternsecurity.com\" looks like an autogenerated PTR'\n\n def check(self, module_test, events):\n assert len(events) == 4\n assert 1 == len(\n [e for e in events if e.data == \"asdf.blacklanternsecurity.com\" and str(e.module) == \"dnsbrute\"]\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py", "content": "from .base import ModuleTestBase\n\n\nclass TestDnsbrute_mutations(ModuleTestBase):\n targets = [\n \"blacklanternsecurity.com\",\n \"rrrr.blacklanternsecurity.com\",\n \"asdff-ffdsa.blacklanternsecurity.com\",\n \"hmmmm.test1.blacklanternsecurity.com\",\n \"uuuuu.test2.blacklanternsecurity.com\",\n ]\n\n async def setup_after_prep(self, module_test):\n old_run_live = module_test.scan.helpers.run_live\n\n async def new_run_live(*command, check=False, text=True, **kwargs):\n if \"massdns\" in command[:2]:\n _input = [l async for l in kwargs[\"input\"]]\n if \"rrrr-test.blacklanternsecurity.com\" in _input:\n yield \"\"\"{\"name\": \"rrrr-test.blacklanternsecurity.com.\", \"type\": \"A\", \"class\": \"IN\", \"status\": \"NOERROR\", \"rx_ts\": 1713974911725326170, \"data\": {\"answers\": [{\"ttl\": 86400, \"type\": \"A\", \"class\": \"IN\", \"name\": \"rrrr-test.blacklanternsecurity.com.\", \"data\": \"1.2.3.4.\"}]}, \"flags\": [\"rd\", \"ra\"], \"resolver\": \"195.226.187.130:53\", \"proto\": \"UDP\"}\"\"\"\n if \"rrrr-ffdsa.blacklanternsecurity.com\" in _input:\n yield \"\"\"{\"name\": \"rrrr-ffdsa.blacklanternsecurity.com.\", \"type\": \"A\", \"class\": \"IN\", \"status\": \"NOERROR\", \"rx_ts\": 1713974911725326170, \"data\": {\"answers\": [{\"ttl\": 86400, \"type\": \"A\", \"class\": \"IN\", \"name\": \"rrrr-ffdsa.blacklanternsecurity.com.\", \"data\": \"1.2.3.4.\"}]}, \"flags\": [\"rd\", \"ra\"], \"resolver\": \"195.226.187.130:53\", \"proto\": \"UDP\"}\"\"\"\n if \"hmmmm.test2.blacklanternsecurity.com\" in _input:\n yield \"\"\"{\"name\": \"hmmmm.test2.blacklanternsecurity.com.\", \"type\": \"A\", \"class\": \"IN\", \"status\": \"NOERROR\", \"rx_ts\": 1713974911725326170, \"data\": {\"answers\": [{\"ttl\": 86400, \"type\": \"A\", \"class\": \"IN\", \"name\": \"hmmmm.test2.blacklanternsecurity.com.\", \"data\": \"1.2.3.4.\"}]}, \"flags\": [\"rd\", \"ra\"], \"resolver\": \"195.226.187.130:53\", \"proto\": \"UDP\"}\"\"\"\n else:\n async for _ in old_run_live(*command, check=False, text=True, **kwargs):\n yield _\n\n module_test.monkeypatch.setattr(module_test.scan.helpers, \"run_live\", new_run_live)\n\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n # targets\n \"rrrr.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n \"asdff-ffdsa.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n \"hmmmm.test1.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n \"uuuuu.test2.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n # devops mutation\n \"rrrr-test.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n # target-specific dns mutation\n \"rrrr-ffdsa.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n # subdomain from one subdomain on a different subdomain\n \"hmmmm.test2.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n }\n )\n\n def check(self, module_test, events):\n assert len(events) == 10\n assert 1 == len(\n [\n e\n for e in events\n if e.data == \"rrrr-test.blacklanternsecurity.com\" and str(e.module) == \"dnsbrute_mutations\"\n ]\n ), \"Failed to find devops mutation (word_cloud)\"\n assert 1 == len(\n [\n e\n for e in events\n if e.data == \"rrrr-ffdsa.blacklanternsecurity.com\" and str(e.module) == \"dnsbrute_mutations\"\n ]\n ), \"Failed to find target-specific mutation (word_cloud.dns_mutator)\"\n assert 1 == len(\n [\n e\n for e in events\n if e.data == \"hmmmm.test2.blacklanternsecurity.com\" and str(e.module) == \"dnsbrute_mutations\"\n ]\n ), \"Failed to find subdomain taken from another subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_dnscaa.py", "content": "from .base import ModuleTestBase\n\n\nclass TestDNSCAA(ModuleTestBase):\n targets = [\"blacklanternsecurity.notreal\"]\n modules_overrides = [\"dnscaa\", \"speculate\"]\n config_overrides = {\n \"scope\": {\n \"report_distance\": 1,\n }\n }\n\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.notreal\": {\n \"A\": [\"127.0.0.11\"],\n \"CAA\": [\n '0 iodef \"https://caa.blacklanternsecurity.notreal\"',\n '128 iodef \"mailto:caa@blacklanternsecurity.notreal\"',\n '0 issue \"comodoca.com\"',\n '1 issue \"digicert.com; cansignhttpexchanges=yes\"',\n '0 issuewild \"letsencrypt.org\"',\n '128 issuewild \"pki.goog; cansignhttpexchanges=yes\"',\n ],\n },\n \"caa.blacklanternsecurity.notreal\": {\"A\": [\"127.0.0.22\"]},\n \"comodoca.com\": {\n \"A\": [\"127.0.0.33\"],\n \"CAA\": [\n '0 iodef \"https://caa.comodoca.com\"',\n ],\n },\n \"caa.comodoca.com\": {\"A\": [\"127.0.0.33\"]},\n \"digicert.com\": {\"A\": [\"127.0.0.44\"]},\n \"letsencrypt.org\": {\"A\": [\"127.0.0.55\"]},\n \"pki.goog\": {\"A\": [\"127.0.0.66\"]},\n }\n )\n\n def check(self, module_test, events):\n assert any(e.type == \"DNS_NAME\" and e.data == \"comodoca.com\" for e in events), \"Failed to detect CA DNS name\"\n assert any(e.type == \"DNS_NAME\" and e.data == \"digicert.com\" for e in events), \"Failed to detect CA DNS name\"\n assert any(e.type == \"DNS_NAME\" and e.data == \"letsencrypt.org\" for e in events), (\n \"Failed to detect CA DNS name\"\n )\n assert any(e.type == \"DNS_NAME\" and e.data == \"pki.goog\" for e in events), \"Failed to detect CA DNS name\"\n assert any(\n e.type == \"URL_UNVERIFIED\" and e.data == \"https://caa.blacklanternsecurity.notreal/\" for e in events\n ), \"Failed to detect URL\"\n assert any(e.type == \"EMAIL_ADDRESS\" and e.data == \"caa@blacklanternsecurity.notreal\" for e in events), (\n \"Failed to detect email address\"\n )\n # make sure we're not checking CAA records for out-of-scope hosts\n assert not any(str(e.host) == \"caa.comodoca.com\" for e in events)\n\n\nclass TestDNSCAAInScopeFalse(TestDNSCAA):\n config_overrides = {\"scope\": {\"report_distance\": 3}, \"modules\": {\"dnscaa\": {\"in_scope_only\": False}}}\n\n def check(self, module_test, events):\n assert any(str(e.host) == \"caa.comodoca.com\" for e in events)\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py", "content": "from .base import ModuleTestBase\n\n\nclass TestDNSCommonSRV(ModuleTestBase):\n targets = [\"media.www.test.api.blacklanternsecurity.com\"]\n whitelist = [\"blacklanternsecurity.com\"]\n modules_overrides = [\"dnscommonsrv\", \"speculate\"]\n config_overrides = {\"dns\": {\"minimal\": False}}\n\n async def setup_after_prep(self, module_test):\n old_run_live = module_test.scan.helpers.run_live\n\n async def new_run_live(*command, check=False, text=True, **kwargs):\n if \"massdns\" in command[:2]:\n _input = [l async for l in kwargs[\"input\"]]\n if \"_ldap._tcp.gc._msdcs.blacklanternsecurity.com\" in _input:\n yield \"\"\"{\"name\":\"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.\",\"type\":\"SRV\",\"class\":\"IN\",\"status\":\"NOERROR\",\"rx_ts\":1713974911725326170,\"data\":{\"answers\":[{\"ttl\":86400,\"type\":\"SRV\",\"class\":\"IN\",\"name\":\"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.\",\"data\":\"10 10 1720 asdf.blacklanternsecurity.com.\"},{\"ttl\":86400,\"type\":\"SRV\",\"class\":\"IN\",\"name\":\"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.\",\"data\":\"10 10 1720 asdf.blacklanternsecurity.com.\"}]},\"flags\":[\"rd\",\"ra\"],\"resolver\":\"195.226.187.130:53\",\"proto\":\"UDP\"}\"\"\"\n if \"_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com\" in _input:\n yield \"\"\"{\"name\":\"_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com.\",\"type\":\"SRV\",\"class\":\"IN\",\"status\":\"NOERROR\",\"rx_ts\":1713974911725326170,\"data\":{\"answers\":[{\"ttl\":86400,\"type\":\"SRV\",\"class\":\"IN\",\"name\":\"_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com.\",\"data\":\"10 10 1720 asdf.blacklanternsecurity.com.\"},{\"ttl\":86400,\"type\":\"SRV\",\"class\":\"IN\",\"name\":\"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.\",\"data\":\"10 10 1720 asdf.blacklanternsecurity.com.\"}]},\"flags\":[\"rd\",\"ra\"],\"resolver\":\"195.226.187.130:53\",\"proto\":\"UDP\"}\"\"\"\n if \"_ldap._tcp.gc._msdcs.test.api.blacklanternsecurity.com\" in _input:\n yield \"\"\"{\"name\":\"_ldap._tcp.gc._msdcs.test.api.blacklanternsecurity.com.\",\"type\":\"SRV\",\"class\":\"IN\",\"status\":\"NOERROR\",\"rx_ts\":1713974911725326170,\"data\":{\"answers\":[{\"ttl\":86400,\"type\":\"SRV\",\"class\":\"IN\",\"name\":\"_ldap._tcp.gc._msdcs.test.api.blacklanternsecurity.com.\",\"data\":\"10 10 1720 asdf.blacklanternsecurity.com.\"},{\"ttl\":86400,\"type\":\"SRV\",\"class\":\"IN\",\"name\":\"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.\",\"data\":\"10 10 1720 asdf.blacklanternsecurity.com.\"}]},\"flags\":[\"rd\",\"ra\"],\"resolver\":\"195.226.187.130:53\",\"proto\":\"UDP\"}\"\"\"\n if \"_ldap._tcp.gc._msdcs.www.test.api.blacklanternsecurity.com\" in _input:\n yield \"\"\"{\"name\":\"_ldap._tcp.gc._msdcs.www.test.api.blacklanternsecurity.com.\",\"type\":\"SRV\",\"class\":\"IN\",\"status\":\"NOERROR\",\"rx_ts\":1713974911725326170,\"data\":{\"answers\":[{\"ttl\":86400,\"type\":\"SRV\",\"class\":\"IN\",\"name\":\"_ldap._tcp.gc._msdcs.www.test.api.blacklanternsecurity.com.\",\"data\":\"10 10 1720 asdf.blacklanternsecurity.com.\"},{\"ttl\":86400,\"type\":\"SRV\",\"class\":\"IN\",\"name\":\"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.\",\"data\":\"10 10 1720 asdf.blacklanternsecurity.com.\"}]},\"flags\":[\"rd\",\"ra\"],\"resolver\":\"195.226.187.130:53\",\"proto\":\"UDP\"}\"\"\"\n if \"_ldap._tcp.gc._msdcs.media.www.test.api.blacklanternsecurity.com\" in _input:\n yield \"\"\"{\"name\":\"_ldap._tcp.gc._msdcs.www.test.api.blacklanternsecurity.com.\",\"type\":\"SRV\",\"class\":\"IN\",\"status\":\"NOERROR\",\"rx_ts\":1713974911725326170,\"data\":{\"answers\":[{\"ttl\":86400,\"type\":\"SRV\",\"class\":\"IN\",\"name\":\"_ldap._tcp.gc._msdcs.media.www.test.api.blacklanternsecurity.com.\",\"data\":\"10 10 1720 asdf.blacklanternsecurity.com.\"},{\"ttl\":86400,\"type\":\"SRV\",\"class\":\"IN\",\"name\":\"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.\",\"data\":\"10 10 1720 asdf.blacklanternsecurity.com.\"}]},\"flags\":[\"rd\",\"ra\"],\"resolver\":\"195.226.187.130:53\",\"proto\":\"UDP\"}\"\"\"\n else:\n async for _ in old_run_live(*command, check=False, text=True, **kwargs):\n yield _\n\n module_test.monkeypatch.setattr(module_test.scan.helpers, \"run_live\", new_run_live)\n\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n \"api.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n \"test.api.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n \"www.test.api.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n \"media.www.test.api.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n \"_ldap._tcp.gc._msdcs.blacklanternsecurity.com\": {\"SRV\": [\"0 100 3268 asdf.blacklanternsecurity.com\"]},\n \"_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com\": {\n \"SRV\": [\"0 100 3268 asdf.blacklanternsecurity.com\"]\n },\n \"_ldap._tcp.gc._msdcs.test.api.blacklanternsecurity.com\": {\n \"SRV\": [\"0 100 3268 asdf.blacklanternsecurity.com\"]\n },\n \"_ldap._tcp.gc._msdcs.www.test.api.blacklanternsecurity.com\": {\n \"SRV\": [\"0 100 3268 asdf.blacklanternsecurity.com\"]\n },\n \"_ldap._tcp.gc._msdcs.media.www.test.api.blacklanternsecurity.com\": {\n \"SRV\": [\"0 100 3268 asdf.blacklanternsecurity.com\"]\n },\n \"asdf.blacklanternsecurity.com\": {\"A\": [\"1.2.3.5\"]},\n \"_msdcs.api.blacklanternsecurity.com\": {\"A\": [\"1.2.3.5\"]},\n }\n )\n\n def check(self, module_test, events):\n assert len(events) == 20\n assert 1 == len([e for e in events if e.type == \"DNS_NAME\" and e.data == \"blacklanternsecurity.com\"])\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"_ldap._tcp.gc._msdcs.blacklanternsecurity.com\"\n and str(e.module) == \"dnscommonsrv\"\n ]\n ), \"Failed to detect subdomain 1\"\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com\"\n and str(e.module) == \"dnscommonsrv\"\n ]\n ), \"Failed to detect subdomain 2\"\n assert 2 == len([e for e in events if e.type == \"DNS_NAME\" and e.data == \"asdf.blacklanternsecurity.com\"]), (\n \"Failed to detect subdomain 3\"\n )\n assert 1 == len([e for e in events if e.type == \"DNS_NAME\" and e.data == \"api.blacklanternsecurity.com\"]), (\n \"Failed to detect subdomain 4\"\n )\n assert 1 == len(\n [e for e in events if e.type == \"DNS_NAME\" and e.data == \"test.api.blacklanternsecurity.com\"]\n ), \"Failed to detect subdomain 5\"\n assert 1 == len(\n [e for e in events if e.type == \"DNS_NAME\" and e.data == \"_msdcs.api.blacklanternsecurity.com\"]\n ), \"Failed to detect subdomain 5\"\n assert 1 == len([e for e in events if e.type == \"DNS_NAME\" and e.data == \"blacklanternsecurity.com\"]), (\n \"Failed to detect main domain\"\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"RAW_DNS_RECORD\"\n and e.data[\"host\"] == \"_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com\"\n and e.data[\"answer\"] == \"0 100 3268 asdf.blacklanternsecurity.com\"\n ]\n ), \"Failed to emit RAW_DNS_RECORD for _ldap._tcp.gc._msdcs.api.blacklanternsecurity.com\"\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"RAW_DNS_RECORD\"\n and e.data[\"host\"] == \"_ldap._tcp.gc._msdcs.blacklanternsecurity.com\"\n and e.data[\"answer\"] == \"0 100 3268 asdf.blacklanternsecurity.com\"\n ]\n ), \"Failed to emit RAW_DNS_RECORD for _ldap._tcp.gc._msdcs.blacklanternsecurity.com\"\n assert 2 == len([e for e in events if e.type == \"RAW_DNS_RECORD\"])\n assert 10 == len([e for e in events if e.type == \"DNS_NAME\"])\n assert 5 == len([e for e in events if e.type == \"DNS_NAME_UNRESOLVED\"])\n assert 5 == len([e for e in events if e.type == \"DNS_NAME_UNRESOLVED\" and str(e.module) == \"speculate\"])\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_dnsdumpster.py", "content": "from .base import ModuleTestBase\n\n\nclass TestDNSDumpster(ModuleTestBase):\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://dnsdumpster.com\",\n content=b\"\"\"
\"\"\",\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.dnsdumpster.com/htmld/\",\n content=b\"asdf.blacklanternsecurity.com\",\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_dnsresolve.py", "content": "from .base import ModuleTestBase\n\n\nclass TestDNSREsolve(ModuleTestBase):\n config_overrides = {\"dns\": {\"minimal\": False}, \"scope\": {\"report_distance\": 1}}\n\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.com\": {\n \"A\": [\"192.168.0.7\"],\n \"AAAA\": [\"::1\"],\n \"CNAME\": [\"www.blacklanternsecurity.com\"],\n },\n \"www.blacklanternsecurity.com\": {\"A\": [\"192.168.0.8\"]},\n }\n )\n\n def check(self, module_test, events):\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"blacklanternsecurity.com\"\n and \"a-record\" in e.tags\n and \"aaaa-record\" in e.tags\n and \"cname-record\" in e.tags\n and \"private-ip\" in e.tags\n and e.scope_distance == 0\n and \"192.168.0.7\" in e.resolved_hosts\n and \"::1\" in e.resolved_hosts\n and \"www.blacklanternsecurity.com\" in e.resolved_hosts\n and e.dns_children\n == {\"A\": {\"192.168.0.7\"}, \"AAAA\": {\"::1\"}, \"CNAME\": {\"www.blacklanternsecurity.com\"}}\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"www.blacklanternsecurity.com\"\n and \"a-record\" in e.tags\n and \"private-ip\" in e.tags\n and e.scope_distance == 0\n and \"192.168.0.8\" in e.resolved_hosts\n and e.dns_children == {\"A\": {\"192.168.0.8\"}}\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"IP_ADDRESS\"\n and e.data == \"192.168.0.7\"\n and \"private-ip\" in e.tags\n and e.scope_distance == 1\n ]\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_dnstlsrpt.py", "content": "from .base import ModuleTestBase\n\nraw_smtp_tls_txt = '\"v=TLSRPTv1; rua=mailto:tlsrpt@sub.blacklanternsecurity.notreal,mailto:test@on.thirdparty.com, https://tlspost.example.com;\"'\n\n\nclass TestDNSTLSRPT(ModuleTestBase):\n targets = [\"blacklanternsecurity.notreal\"]\n modules_overrides = [\"dnstlsrpt\", \"speculate\"]\n config_overrides = {\"modules\": {\"dnstlsrpt\": {\"emit_raw_dns_records\": True}}, \"scope\": {\"report_distance\": 1}}\n\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.notreal\": {\n \"A\": [\"127.0.0.11\"],\n },\n \"_tls.blacklanternsecurity.notreal\": {\n \"A\": [\"127.0.0.22\"],\n },\n \"_smtp._tls.blacklanternsecurity.notreal\": {\n \"A\": [\"127.0.0.33\"],\n \"TXT\": [raw_smtp_tls_txt],\n },\n \"_tls._smtp._tls.blacklanternsecurity.notreal\": {\n \"A\": [\"127.0.0.44\"],\n },\n \"_smtp._tls._smtp._tls.blacklanternsecurity.notreal\": {\n \"TXT\": [raw_smtp_tls_txt],\n },\n \"sub.blacklanternsecurity.notreal\": {\n \"A\": [\"127.0.0.55\"],\n },\n }\n )\n\n def check(self, module_test, events):\n assert any(e.type == \"RAW_DNS_RECORD\" and e.data[\"answer\"] == raw_smtp_tls_txt for e in events), (\n \"Failed to emit RAW_DNS_RECORD\"\n )\n assert any(e.type == \"DNS_NAME\" and e.data == \"sub.blacklanternsecurity.notreal\" for e in events), (\n \"Failed to detect sub-domain\"\n )\n assert any(\n e.type == \"EMAIL_ADDRESS\" and e.data == \"tlsrpt@sub.blacklanternsecurity.notreal\" for e in events\n ), \"Failed to detect email address\"\n assert any(e.type == \"EMAIL_ADDRESS\" and e.data == \"test@on.thirdparty.com\" for e in events), (\n \"Failed to detect third party email address\"\n )\n assert any(e.type == \"URL_UNVERIFIED\" and e.data == \"https://tlspost.example.com/\" for e in events), (\n \"Failed to detect third party URL\"\n )\n\n\nclass TestDNSTLSRPTRecursiveRecursion(TestDNSTLSRPT):\n config_overrides = {\n \"scope\": {\"report_distance\": 1},\n \"modules\": {\"dnstlsrpt\": {\"emit_raw_dns_records\": True}},\n }\n\n def check(self, module_test, events):\n assert not any(\n e.type == \"RAW_DNS_RECORD\" and e.data[\"host\"] == \"_mta-sts._mta-sts.blacklanternsecurity.notreal\"\n for e in events\n ), \"Unwanted recursion occurring\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_docker_pull.py", "content": "import io\nimport tarfile\nfrom pathlib import Path\n\nfrom .base import ModuleTestBase\nfrom bbot.test.bbot_fixtures import bbot_test_dir\n\n\nclass TestDockerPull(ModuleTestBase):\n modules_overrides = [\"speculate\", \"dockerhub\", \"docker_pull\"]\n config_overrides = {\"modules\": {\"docker_pull\": {\"output_folder\": str(bbot_test_dir / \"test_docker_files\")}}}\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://hub.docker.com/v2/users/blacklanternsecurity\",\n json={\n \"id\": \"f90895d9cf484d9182c6dbbef2632329\",\n \"uuid\": \"f90895d9-cf48-4d91-82c6-dbbef2632329\",\n \"username\": \"blacklanternsecurity\",\n \"full_name\": \"\",\n \"location\": \"\",\n \"company\": \"Black Lantern Security\",\n \"profile_url\": \"https://github.com/blacklanternsecurity\",\n \"date_joined\": \"2022-08-29T15:27:10.227081Z\",\n \"gravatar_url\": \"\",\n \"gravatar_email\": \"\",\n \"type\": \"User\",\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://hub.docker.com/v2/repositories/blacklanternsecurity?page_size=25&page=1\",\n json={\n \"count\": 2,\n \"next\": None,\n \"previous\": None,\n \"results\": [\n {\n \"name\": \"helloworld\",\n \"namespace\": \"blacklanternsecurity\",\n \"repository_type\": \"image\",\n \"status\": 1,\n \"status_description\": \"active\",\n \"description\": \"\",\n \"is_private\": False,\n \"star_count\": 0,\n \"pull_count\": 1,\n \"last_updated\": \"2021-12-20T17:19:58.88296Z\",\n \"date_registered\": \"2021-12-20T17:19:58.507614Z\",\n \"affiliation\": \"\",\n \"media_types\": [\"application/vnd.docker.container.image.v1+json\"],\n \"content_types\": [\"image\"],\n \"categories\": [],\n },\n {\n \"name\": \"testimage\",\n \"namespace\": \"blacklanternsecurity\",\n \"repository_type\": \"image\",\n \"status\": 1,\n \"status_description\": \"active\",\n \"description\": \"\",\n \"is_private\": False,\n \"star_count\": 0,\n \"pull_count\": 1,\n \"last_updated\": \"2022-01-10T20:16:46.170738Z\",\n \"date_registered\": \"2022-01-07T13:28:59.756641Z\",\n \"affiliation\": \"\",\n \"media_types\": [\"application/vnd.docker.container.image.v1+json\"],\n \"content_types\": [\"image\"],\n \"categories\": [],\n },\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/helloworld/tags/list\",\n json={\n \"errors\": [\n {\n \"code\": \"UNAUTHORIZED\",\n \"message\": \"authentication required\",\n \"detail\": [\n {\n \"Type\": \"repository\",\n \"Class\": \"\",\n \"Name\": \"blacklanternsecurity/helloworld\",\n \"Action\": \"pull\",\n }\n ],\n }\n ]\n },\n headers={\n \"www-authenticate\": 'Bearer realm=\"https://auth.docker.io/token\",service=\"registry.docker.io\",scope=\"blacklanternsecurity/helloworld:pull\"'\n },\n status_code=401,\n )\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/testimage/tags/list\",\n json={\n \"errors\": [\n {\n \"code\": \"UNAUTHORIZED\",\n \"message\": \"authentication required\",\n \"detail\": [\n {\n \"Type\": \"repository\",\n \"Class\": \"\",\n \"Name\": \"blacklanternsecurity/testimage\",\n \"Action\": \"pull\",\n }\n ],\n }\n ]\n },\n headers={\n \"www-authenticate\": 'Bearer realm=\"https://auth.docker.io/token\",service=\"registry.docker.io\",scope=\"blacklanternsecurity/testimage:pull\"'\n },\n status_code=401,\n )\n module_test.httpx_mock.add_response(\n url=\"https://auth.docker.io/token?service=registry.docker.io&scope=blacklanternsecurity/helloworld:pull\",\n json={\n \"token\": \"QWERTYUIOPASDFGHJKLZXCBNM\",\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://auth.docker.io/token?service=registry.docker.io&scope=blacklanternsecurity/testimage:pull\",\n json={\n \"token\": \"QWERTYUIOPASDFGHJKLZXCBNM\",\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/helloworld/tags/list\",\n json={\n \"name\": \"blacklanternsecurity/helloworld\",\n \"tags\": [\n \"dev\",\n \"latest\",\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/testimage/tags/list\",\n json={\n \"name\": \"blacklanternsecurity/testimage\",\n \"tags\": [\n \"dev\",\n \"latest\",\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/helloworld/manifests/latest\",\n json={\n \"schemaVersion\": 2,\n \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n \"config\": {\n \"mediaType\": \"application/vnd.docker.container.image.v1+json\",\n \"size\": 8614,\n \"digest\": \"sha256:a9910947b74a4f0606cfc8669ae8808d2c328beaee9e79f489dc17df14cd50b1\",\n },\n \"layers\": [\n {\n \"mediaType\": \"application/vnd.docker.image.rootfs.diff.tar.gzip\",\n \"size\": 29124181,\n \"digest\": \"sha256:8a1e25ce7c4f75e372e9884f8f7b1bedcfe4a7a7d452eb4b0a1c7477c9a90345\",\n },\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/testimage/manifests/latest\",\n json={\n \"mediaType\": \"application/vnd.docker.distribution.manifest.list.v2+json\",\n \"schemaVersion\": 2,\n \"manifests\": [\n {\n \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n \"platform\": {\"os\": \"linux\", \"architecture\": \"s390x\"},\n \"digest\": \"sha256:3e8a8b63afab946f4a64c1dc63563d91b2cb1e5eadadac1eff20231695c53d24\",\n \"size\": 1953,\n },\n {\n \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n \"platform\": {\"os\": \"linux\", \"architecture\": \"amd64\"},\n \"digest\": \"sha256:7c75331408141f1e3ef37eac7c45938fbfb0d421a86201ad45d2ab8b70ddd527\",\n \"size\": 1953,\n },\n {\n \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n \"platform\": {\"os\": \"linux\", \"architecture\": \"ppc64le\"},\n \"digest\": \"sha256:33d30a60996db4bc8158151ce516a8503cc56ce8d146e450e117a57ca5bf06e7\",\n \"size\": 1953,\n },\n {\n \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n \"platform\": {\"os\": \"linux\", \"architecture\": \"arm64\", \"variant\": \"v8\"},\n \"digest\": \"sha256:d0eacd0089db7309a5ce40ec3334fcdd4ce7d67324f1ccc4433dd4fae4a771a4\",\n \"size\": 1953,\n },\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/helloworld/blobs/sha256:a9910947b74a4f0606cfc8669ae8808d2c328beaee9e79f489dc17df14cd50b1\",\n json={\n \"architecture\": \"amd64\",\n \"config\": {\n \"Env\": [\n \"PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\n \"LANG=C.UTF-8\",\n \"GPG_KEY=QWERTYUIOPASDFGHJKLZXCBNM\",\n \"PYTHON_VERSION=3.10.14\",\n \"PYTHON_PIP_VERSION=23.0.1\",\n \"PYTHON_SETUPTOOLS_VERSION=65.5.1\",\n \"PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py\",\n \"PYTHON_GET_PIP_SHA256=dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9\",\n \"LC_ALL=C.UTF-8\",\n \"PIP_NO_CACHE_DIR=off\",\n ],\n \"Entrypoint\": [\"helloworld\"],\n \"WorkingDir\": \"/root\",\n \"ArgsEscaped\": True,\n \"OnBuild\": None,\n },\n \"created\": \"2024-03-24T03:46:29.788993495Z\",\n \"history\": [\n {\n \"created\": \"2024-03-12T01:21:01.529814652Z\",\n \"created_by\": \"/bin/sh -c #(nop) ADD file:b86ae1c7ca3586d8feedcd9ff1b2b1e8ab872caf6587618f1da689045a5d7ae4 in / \",\n },\n {\n \"created\": \"2024-03-12T01:21:01.866693306Z\",\n \"created_by\": '/bin/sh -c #(nop) CMD [\"bash\"]',\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV LANG=C.UTF-8\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"RUN /bin/sh -c set -eux; \\tapt-get update; \\tapt-get install -y --no-install-recommends \\t\\tca-certificates \\t\\tnetbase \\t\\ttzdata \\t; \\trm -rf /var/lib/apt/lists/* # buildkit\",\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV GPG_KEY=QWERTYUIOPASDFGHJKLZXCBNM\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV PYTHON_VERSION=3.10.14\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": 'RUN /bin/sh -c set -eux; \\t\\tsavedAptMark=\"$(apt-mark showmanual)\"; \\tapt-get update; \\tapt-get install -y --no-install-recommends \\t\\tdpkg-dev \\t\\tgcc \\t\\tgnupg \\t\\tlibbluetooth-dev \\t\\tlibbz2-dev \\t\\tlibc6-dev \\t\\tlibdb-dev \\t\\tlibexpat1-dev \\t\\tlibffi-dev \\t\\tlibgdbm-dev \\t\\tliblzma-dev \\t\\tlibncursesw5-dev \\t\\tlibreadline-dev \\t\\tlibsqlite3-dev \\t\\tlibssl-dev \\t\\tmake \\t\\ttk-dev \\t\\tuuid-dev \\t\\twget \\t\\txz-utils \\t\\tzlib1g-dev \\t; \\t\\twget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\"; \\twget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\"; \\tGNUPGHOME=\"$(mktemp -d)\"; export GNUPGHOME; \\tgpg --batch --keyserver hkps://keys.openpgp.org --recv-keys \"$GPG_KEY\"; \\tgpg --batch --verify python.tar.xz.asc python.tar.xz; \\tgpgconf --kill all; \\trm -rf \"$GNUPGHOME\" python.tar.xz.asc; \\tmkdir -p /usr/src/python; \\ttar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \\trm python.tar.xz; \\t\\tcd /usr/src/python; \\tgnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\"; \\t./configure \\t\\t--build=\"$gnuArch\" \\t\\t--enable-loadable-sqlite-extensions \\t\\t--enable-optimizations \\t\\t--enable-option-checking=fatal \\t\\t--enable-shared \\t\\t--with-lto \\t\\t--with-system-expat \\t\\t--without-ensurepip \\t; \\tnproc=\"$(nproc)\"; \\tEXTRA_CFLAGS=\"$(dpkg-buildflags --get CFLAGS)\"; \\tLDFLAGS=\"$(dpkg-buildflags --get LDFLAGS)\"; \\tLDFLAGS=\"${LDFLAGS:--Wl},--strip-all\"; \\tmake -j \"$nproc\" \\t\\t\"EXTRA_CFLAGS=${EXTRA_CFLAGS:-}\" \\t\\t\"LDFLAGS=${LDFLAGS:-}\" \\t\\t\"PROFILE_TASK=${PROFILE_TASK:-}\" \\t; \\trm python; \\tmake -j \"$nproc\" \\t\\t\"EXTRA_CFLAGS=${EXTRA_CFLAGS:-}\" \\t\\t\"LDFLAGS=${LDFLAGS:--Wl},-rpath=\\'\\\\$\\\\$ORIGIN/../lib\\'\" \\t\\t\"PROFILE_TASK=${PROFILE_TASK:-}\" \\t\\tpython \\t; \\tmake install; \\t\\tcd /; \\trm -rf /usr/src/python; \\t\\tfind /usr/local -depth \\t\\t\\\\( \\t\\t\\t\\\\( -type d -a \\\\( -name test -o -name tests -o -name idle_test \\\\) \\\\) \\t\\t\\t-o \\\\( -type f -a \\\\( -name \\'*.pyc\\' -o -name \\'*.pyo\\' -o -name \\'libpython*.a\\' \\\\) \\\\) \\t\\t\\\\) -exec rm -rf \\'{}\\' + \\t; \\t\\tldconfig; \\t\\tapt-mark auto \\'.*\\' > /dev/null; \\tapt-mark manual $savedAptMark; \\tfind /usr/local -type f -executable -not \\\\( -name \\'*tkinter*\\' \\\\) -exec ldd \\'{}\\' \\';\\' \\t\\t| awk \\'/=>/ { so = $(NF-1); if (index(so, \"/usr/local/\") == 1) { next }; gsub(\"^/(usr/)?\", \"\", so); printf \"*%s\\\\n\", so }\\' \\t\\t| sort -u \\t\\t| xargs -r dpkg-query --search \\t\\t| cut -d: -f1 \\t\\t| sort -u \\t\\t| xargs -r apt-mark manual \\t; \\tapt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \\trm -rf /var/lib/apt/lists/*; \\t\\tpython3 --version # buildkit',\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": 'RUN /bin/sh -c set -eux; \\tfor src in idle3 pydoc3 python3 python3-config; do \\t\\tdst=\"$(echo \"$src\" | tr -d 3)\"; \\t\\t[ -s \"/usr/local/bin/$src\" ]; \\t\\t[ ! -e \"/usr/local/bin/$dst\" ]; \\t\\tln -svT \"$src\" \"/usr/local/bin/$dst\"; \\tdone # buildkit',\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV PYTHON_PIP_VERSION=23.0.1\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV PYTHON_SETUPTOOLS_VERSION=65.5.1\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV PYTHON_GET_PIP_SHA256=dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": 'RUN /bin/sh -c set -eux; \\t\\tsavedAptMark=\"$(apt-mark showmanual)\"; \\tapt-get update; \\tapt-get install -y --no-install-recommends wget; \\t\\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \\techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum -c -; \\t\\tapt-mark auto \\'.*\\' > /dev/null; \\t[ -z \"$savedAptMark\" ] || apt-mark manual $savedAptMark > /dev/null; \\tapt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \\trm -rf /var/lib/apt/lists/*; \\t\\texport PYTHONDONTWRITEBYTECODE=1; \\t\\tpython get-pip.py \\t\\t--disable-pip-version-check \\t\\t--no-cache-dir \\t\\t--no-compile \\t\\t\"pip==$PYTHON_PIP_VERSION\" \\t\\t\"setuptools==$PYTHON_SETUPTOOLS_VERSION\" \\t; \\trm -f get-pip.py; \\t\\tpip --version # buildkit',\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": 'CMD [\"python3\"]',\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-24T03:45:39.322168741Z\",\n \"created_by\": \"ENV LANG=C.UTF-8\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-24T03:45:39.322168741Z\",\n \"created_by\": \"ENV LC_ALL=C.UTF-8\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-24T03:45:39.322168741Z\",\n \"created_by\": \"ENV PIP_NO_CACHE_DIR=off\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-24T03:45:39.322168741Z\",\n \"created_by\": \"WORKDIR /usr/src/helloworld\",\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-24T03:45:52.226201188Z\",\n \"created_by\": \"RUN /bin/sh -c apt-get update && apt-get install -y openssl gcc git make unzip curl wget vim nano sudo # buildkit\",\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-24T03:45:52.391597947Z\",\n \"created_by\": \"COPY . . # buildkit\",\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-24T03:46:29.76589069Z\",\n \"created_by\": \"RUN /bin/sh -c pip install . # buildkit\",\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-24T03:46:29.788993495Z\",\n \"created_by\": \"WORKDIR /root\",\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-24T03:46:29.788993495Z\",\n \"created_by\": 'ENTRYPOINT [\"helloworld\"]',\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n ],\n \"os\": \"linux\",\n \"rootfs\": {\n \"type\": \"layers\",\n \"diff_ids\": [\n \"sha256:a483da8ab3e941547542718cacd3258c6c705a63e94183c837c9bc44eb608999\",\n \"sha256:c8f253aef5606f6716778771171c3fdf6aa135b76a5fa8bf66ba45c12c15b540\",\n \"sha256:b4a9dcc697d250c7be53887bb8e155c8f7a06f9c63a3aa627c647bb4a426d3f0\",\n \"sha256:120fda24c420b4e5d52f1c288b35c75b07969057bce41ec34cfb05606b2d7c11\",\n \"sha256:c2287f03e33f4896b2720f0cb64e6b6050759a3eb5914e531e98fc3499b4e687\",\n \"sha256:afe6e55a5cf240c050a4d2b72ec7b7d009a131cba8fe2753e453a8e62ef7e45c\",\n \"sha256:ae6df275ba2e8f40c598e30588afe43f6bfa92e4915e8450b77cb5db5c89dfd5\",\n \"sha256:621ab22fb386a9e663178637755b651beddc0eb4762804e74d8996cce0ddd441\",\n \"sha256:4c534ad16bd2df668c0b8f637616517746ede530ba8546d85f28772bc748e06f\",\n \"sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef\",\n ],\n },\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/testimage/manifests/sha256:7c75331408141f1e3ef37eac7c45938fbfb0d421a86201ad45d2ab8b70ddd527\",\n json={\n \"name\": \"testimage\",\n \"tag\": \"latest\",\n \"architecture\": \"amd64\",\n \"fsLayers\": [\n {\"blobSum\": \"sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef\"},\n ],\n \"history\": [\n {\n \"v1Compatibility\": '{\"id\":\"e45a5af57b00862e5ef5782a9925979a02ba2b12dff832fd0991335f4a11e5c5\",\"parent\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"created\":\"2014-12-31T22:57:59.178729048Z\",\"container\":\"27b45f8fb11795b52e9605b686159729b0d9ca92f76d40fb4f05a62e19c46b4f\",\"container_config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [/hello]\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"docker_version\":\"1.4.1\",\"config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/hello\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\\n'\n },\n {\n \"v1Compatibility\": '{\"id\":\"e45a5af57b00862e5ef5782a9925979a02ba2b12dff832fd0991335f4a11e5c5\",\"parent\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"created\":\"2014-12-31T22:57:59.178729048Z\",\"container\":\"27b45f8fb11795b52e9605b686159729b0d9ca92f76d40fb4f05a62e19c46b4f\",\"container_config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) CMD [/hello]\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"docker_version\":\"1.4.1\",\"config\":{\"Hostname\":\"8ce6509d66e2\",\"Domainname\":\"\",\"User\":\"\",\"Memory\":0,\"MemorySwap\":0,\"CpuShares\":0,\"Cpuset\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"PortSpecs\":null,\"ExposedPorts\":null,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/hello\"],\"Image\":\"31cbccb51277105ba3ae35ce33c22b69c9e3f1002e76e4c736a2e8ebff9d7b5d\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"NetworkDisabled\":false,\"MacAddress\":\"\",\"OnBuild\":[],\"SecurityOpt\":null,\"Labels\":null},\"architecture\":\"amd64\",\"os\":\"linux\",\"Size\":0}\\n'\n },\n ],\n \"schemaVersion\": 1,\n \"signatures\": [\n {\n \"header\": {\n \"jwk\": {\n \"crv\": \"P-256\",\n \"kid\": \"OD6I:6DRK:JXEJ:KBM4:255X:NSAA:MUSF:E4VM:ZI6W:CUN2:L4Z6:LSF4\",\n \"kty\": \"EC\",\n \"x\": \"3gAwX48IQ5oaYQAYSxor6rYYc_6yjuLCjtQ9LUakg4A\",\n \"y\": \"t72ge6kIA1XOjqjVoEOiPPAURltJFBMGDSQvEGVB010\",\n },\n \"alg\": \"ES256\",\n },\n \"signature\": \"XREm0L8WNn27Ga_iE_vRnTxVMhhYY0Zst_FfkKopg6gWSoTOZTuW4rK0fg_IqnKkEKlbD83tD46LKEGi5aIVFg\",\n \"protected\": \"eyJmb3JtYXRMZW5ndGgiOjY2MjgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS0wNC0wOFQxODo1Mjo1OVoifQ\",\n }\n ],\n },\n )\n temp_path = Path(\"/tmp/.bbot_test\")\n tar_path = temp_path / \"docker_pull_test.tar.gz\"\n with tarfile.open(tar_path, \"w:gz\") as tar:\n file_io = io.BytesIO(\"This is a test file\".encode())\n file_info = tarfile.TarInfo(name=\"file.txt\")\n file_info.size = len(file_io.getvalue())\n file_io.seek(0)\n tar.addfile(file_info, file_io)\n with open(tar_path, \"rb\") as file:\n layer_file = file.read()\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/helloworld/blobs/sha256:8a1e25ce7c4f75e372e9884f8f7b1bedcfe4a7a7d452eb4b0a1c7477c9a90345\",\n content=layer_file,\n )\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/testimage/blobs/sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef\",\n content=layer_file,\n )\n\n def check(self, module_test, events):\n filesystem_events = [\n e\n for e in events\n if e.type == \"FILESYSTEM\"\n and (\n \"blacklanternsecurity_helloworld_latest.tar\" in e.data[\"path\"]\n or \"blacklanternsecurity_testimage_latest.tar\" in e.data[\"path\"]\n )\n and \"docker\" in e.tags\n and e.scope_distance == 1\n ]\n assert 2 == len(filesystem_events), \"Failed to download docker images\"\n filesystem_event = filesystem_events[0]\n folder = Path(filesystem_event.data[\"path\"])\n assert folder.is_file(), \"Destination tar doesn't exist\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_dockerhub.py", "content": "from .base import ModuleTestBase\n\n\nclass TestDockerhub(ModuleTestBase):\n modules_overrides = [\"dockerhub\", \"speculate\"]\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://hub.docker.com/v2/users/blacklanternsecurity\",\n json={\n \"id\": \"f90895d9cf484d9182c6dbbef2632329\",\n \"uuid\": \"f90895d9-cf48-4d91-82c6-dbbef2632329\",\n \"username\": \"blacklanternsecurity\",\n \"full_name\": \"\",\n \"location\": \"\",\n \"company\": \"Black Lantern Security\",\n \"profile_url\": \"https://github.com/blacklanternsecurity\",\n \"date_joined\": \"2022-08-29T15:27:10.227081Z\",\n \"gravatar_url\": \"\",\n \"gravatar_email\": \"\",\n \"type\": \"User\",\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://hub.docker.com/v2/repositories/blacklanternsecurity?page_size=25&page=1\",\n json={\n \"count\": 2,\n \"next\": None,\n \"previous\": None,\n \"results\": [\n {\n \"name\": \"helloworld\",\n \"namespace\": \"blacklanternsecurity\",\n \"repository_type\": \"image\",\n \"status\": 1,\n \"status_description\": \"active\",\n \"description\": \"\",\n \"is_private\": False,\n \"star_count\": 0,\n \"pull_count\": 1,\n \"last_updated\": \"2021-12-20T17:19:58.88296Z\",\n \"date_registered\": \"2021-12-20T17:19:58.507614Z\",\n \"affiliation\": \"\",\n \"media_types\": [\"application/vnd.docker.container.image.v1+json\"],\n \"content_types\": [\"image\"],\n \"categories\": [],\n },\n {\n \"name\": \"testimage\",\n \"namespace\": \"blacklanternsecurity\",\n \"repository_type\": \"image\",\n \"status\": 1,\n \"status_description\": \"active\",\n \"description\": \"\",\n \"is_private\": False,\n \"star_count\": 0,\n \"pull_count\": 1,\n \"last_updated\": \"2022-01-10T20:16:46.170738Z\",\n \"date_registered\": \"2022-01-07T13:28:59.756641Z\",\n \"affiliation\": \"\",\n \"media_types\": [\"application/vnd.docker.container.image.v1+json\"],\n \"content_types\": [\"image\"],\n \"categories\": [],\n },\n ],\n },\n )\n\n def check(self, module_test, events):\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"SOCIAL\"\n and e.data[\"platform\"] == \"docker\"\n and e.data[\"profile_name\"] == \"blacklanternsecurity\"\n ]\n ), \"Failed to find blacklanternsecurity docker\"\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"CODE_REPOSITORY\"\n and e.data[\"url\"] == \"https://hub.docker.com/r/blacklanternsecurity/helloworld\"\n and \"docker\" in e.tags\n ]\n ), \"Failed to find helloworld docker repo\"\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"CODE_REPOSITORY\"\n and e.data[\"url\"] == \"https://hub.docker.com/r/blacklanternsecurity/testimage\"\n and \"docker\" in e.tags\n ]\n ), \"Failed to find testimage docker repo\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py", "content": "import re\nfrom .base import ModuleTestBase\nfrom werkzeug.wrappers import Response\n\n\ndotnetnuke_http_response = \"\"\"\n \n\n\n\n\n\n\n\n\n\"\"\"\n\n\nclass TestDotnetnuke(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"dotnetnuke\"]\n config_overrides = {\"interactsh_disable\": \"True\"}\n\n exploit_probe = {\n \"Cookie\": r'DNNPersonalization=<profile><item key=\"name1: key1\" type=\"System.Data.Services.Internal.ExpandedWrapper`2[[DotNetNuke.Common.Utilities.FileSystemUtils],[System.Windows.Data.ObjectDataProvider, PresentationFramework, Version=4.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35]], System.Data.Services, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089\"><ExpandedWrapperOfFileSystemUtilsObjectDataProvider xmlns:xsd=\"http://www.w3.org/2001/XMLSchema\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"><ExpandedElement/><ProjectedProperty0><MethodName>WriteFile</MethodName><MethodParameters><anyType xsi:type=\"xsd:string\">C:\\Windows\\win.ini</anyType></MethodParameters><ObjectInstance xsi:type=\"FileSystemUtils\"></ObjectInstance></ProjectedProperty0></ExpandedWrapperOfFileSystemUtilsObjectDataProvider></item></profile>'\n }\n\n exploit_response = \"\"\"\n ; for 16-bit app support\n[fonts]\n[extensions]\n[mci extensions]\n[files]\n[Mail]\nMAPI=1\n\"\"\"\n\n webconfig_response = \"\"\"\n <?xml version=\"1.0\" encoding=\"utf-8\"?>\n<configuration>\n <!-- register local configuration handlers -->\n <configSections>\n <sectionGroup name=\"dotnetnuke\">\n </sectionGroup>\n </configSections>\n</configuration>\n \"\"\"\n\n async def setup_before_prep(self, module_test):\n # Simulate DotNetNuke Instance\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": dotnetnuke_http_response}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # DNNPersonalization Deserialization Detection\n expect_args = {\"method\": \"GET\", \"uri\": \"/__\", \"headers\": self.exploit_probe}\n respond_args = {\"response_data\": self.exploit_response}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # NewsArticlesSlider ImageHandler.ashx File Read\n expect_args = {\n \"method\": \"GET\",\n \"uri\": \"/DesktopModules/dnnUI_NewsArticlesSlider/ImageHandler.ashx\",\n \"query_string\": b\"img=~/web.config\",\n }\n respond_args = {\"response_data\": self.webconfig_response}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # DNNArticle GetCSS.ashx File Read\n expect_args = {\n \"method\": \"GET\",\n \"uri\": \"/DesktopModules/DNNArticle/getcss.ashx\",\n \"query_string\": b\"CP=%2fweb.config&smid=512&portalid=3\",\n }\n respond_args = {\"response_data\": self.webconfig_response}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # InstallWizard SuperUser Privilege Escalation\n expect_args = {\"method\": \"GET\", \"uri\": \"/Install/InstallWizard.aspx\", \"query_string\": b\"__viewstate=1\"}\n respond_args = {\"status\": 500}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/Install/InstallWizard.aspx\"}\n respond_args = {\"status\": 200}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n dnn_technology_detection = False\n dnn_personalization_deserialization_detection = False\n dnn_getcss_fileread_detection = False\n dnn_imagehandler_fileread_detection = False\n dnn_installwizard_privesc_detection = False\n\n for e in events:\n if e.type == \"TECHNOLOGY\" and \"DotNetNuke\" in e.data[\"technology\"]:\n dnn_technology_detection = True\n\n if (\n e.type == \"VULNERABILITY\"\n and \"DotNetNuke Personalization Cookie Deserialization\" in e.data[\"description\"]\n ):\n dnn_personalization_deserialization_detection = True\n\n if (\n e.type == \"VULNERABILITY\"\n and \"DotNetNuke DNNArticle Module GetCSS.ashx Arbitrary File Read\" in e.data[\"description\"]\n ):\n dnn_getcss_fileread_detection = True\n\n if (\n e.type == \"VULNERABILITY\"\n and \"DotNetNuke dnnUI_NewsArticlesSlider Module Arbitrary File Read\" in e.data[\"description\"]\n ):\n dnn_imagehandler_fileread_detection = True\n\n if (\n e.type == \"VULNERABILITY\"\n and \"DotNetNuke InstallWizard SuperUser Privilege Escalation\" in e.data[\"description\"]\n ):\n dnn_installwizard_privesc_detection = True\n\n assert dnn_technology_detection, \"DNN Technology Detection Failed\"\n assert dnn_personalization_deserialization_detection, \"DNN Personalization Deserialization Detection Failed\"\n assert dnn_getcss_fileread_detection, \"getcss.ashx File Read Detection Failed\"\n assert dnn_imagehandler_fileread_detection, \"imagehandler.ashx File Read Detection Failed\"\n assert dnn_installwizard_privesc_detection, \"InstallWizard privesc Detection Failed\"\n\n\ndef extract_subdomain_tag(data):\n pattern = r\"([a-z0-9]{4})\\.fakedomain\\.fakeinteractsh\\.com\"\n match = re.search(pattern, data)\n if match:\n return match.group(1)\n\n\nclass TestDotnetnuke_blindssrf(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n module_name = \"dotnetnuke\"\n modules_overrides = [\"httpx\", \"dotnetnuke\"]\n\n def request_handler(self, request):\n subdomain_tag = None\n subdomain_tag = extract_subdomain_tag(request.full_path)\n if subdomain_tag:\n self.interactsh_mock_instance.mock_interaction(subdomain_tag)\n return Response(\"alive\", status=200)\n\n async def setup_before_prep(self, module_test):\n self.interactsh_mock_instance = module_test.mock_interactsh(\"dotnetnuke_blindssrf\")\n module_test.monkeypatch.setattr(\n module_test.scan.helpers, \"interactsh\", lambda *args, **kwargs: self.interactsh_mock_instance\n )\n\n async def setup_after_prep(self, module_test):\n # Simulate DotNetNuke Instance\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": dotnetnuke_http_response}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n dnn_technology_detection = False\n dnn_dnnimagehandler_blindssrf = False\n\n for e in events:\n if e.type == \"TECHNOLOGY\" and \"DotNetNuke\" in e.data[\"technology\"]:\n dnn_technology_detection = True\n\n if e.type == \"VULNERABILITY\" and \"DotNetNuke Blind-SSRF (CVE 2017-0929)\" in e.data[\"description\"]:\n dnn_dnnimagehandler_blindssrf = True\n\n assert dnn_technology_detection, \"DNN Technology Detection Failed\"\n assert dnn_dnnimagehandler_blindssrf, \"dnnimagehandler.ashx Blind SSRF Detection Failed\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_emailformat.py", "content": "from .base import ModuleTestBase\n\n\nclass TestEmailFormat(ModuleTestBase):\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://www.email-format.com/d/blacklanternsecurity.com/\",\n text=\"\"\"<a href=\"/cdn-cgi/l/email-protection\" class=\"__cf_email__\" data-cfemail=\"0a63646c654a68666b6961666b647e6f7864796f697f78637e7324696567\">[email protected]</a>\"\"\",\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"info@blacklanternsecurity.com\" for e in events), \"Failed to detect email\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_emails.py", "content": "from .base import ModuleTestBase\n\n\nclass TestEmails(ModuleTestBase):\n modules_overrides = [\"emails\", \"emailformat\", \"skymem\"]\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://www.email-format.com/d/blacklanternsecurity.com/\",\n text=\"\"\"<a href=\"/cdn-cgi/l/email-protection\" class=\"__cf_email__\" data-cfemail=\"0a63646c654a68666b6961666b647e6f7864796f697f78637e7324696567\">[email protected]</a>\"\"\",\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.skymem.info/srch?q=blacklanternsecurity.com\",\n text=\"<p>info@blacklanternsecurity.com</p>\",\n )\n\n def check(self, module_test, events):\n assert 2 == len([e for e in events if e.data == \"info@blacklanternsecurity.com\"])\n email_file = module_test.scan.home / \"emails.txt\"\n emails = open(email_file).read().splitlines()\n # make sure deduping works as intended\n assert len(emails) == 1\n assert set(emails) == {\"info@blacklanternsecurity.com\"}\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_excavate.py", "content": "from ...bbot_fixtures import *\nfrom bbot.modules.base import BaseModule\nfrom .base import ModuleTestBase, tempwordlist\n\nfrom bbot.modules.internal.excavate import ExcavateRule\n\nfrom pathlib import Path\nimport yara\n\n\nclass TestExcavate(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/\", \"test.notreal\", \"http://127.0.0.1:8888/subdir/links.html\"]\n modules_overrides = [\"excavate\", \"httpx\"]\n config_overrides = {\"web\": {\"spider_distance\": 1, \"spider_depth\": 1}}\n\n async def setup_before_prep(self, module_test):\n response_data = \"\"\"\n ftp://ftp.test.notreal\n \\\\nhttps://www1.test.notreal\n \\\\x3dhttps://www2.test.notreal\n %0ahttps://www3.test.notreal\n \\\\u000ahttps://www4.test.notreal:\n \\nwww5.test.notreal\n \\\\x3dwww6.test.notreal\n %0awww7.test.notreal\n \\\\u000awww8.test.notreal\n <a href=\"/a_relative.txt\">\n <link href=\"/link_relative.txt\">\n <a href=\"mailto:bob@evilcorp.org?subject=help\">Help</a>\n <li class=\"toctree-l3\"><a class=\"reference internal\" href=\"miscellaneous.html#x50-uart-driver\">16x50 UART Driver</a></li>\n # these ones should get emitted as URL_UNVERIFIED events (processed by httpx which has accept_js_url=True)\n <a href=\"/a_relative.js\">\n <link href=\"/link_relative.js\">\n \"\"\"\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": response_data}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # verify relatives path a-tag parsing is working correctly\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/subdir/links.html\"}\n respond_args = {\"response_data\": \"<a href='../relative.html'/><a href='/2/depth2.html'/>\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/relative.html\"}\n respond_args = {\"response_data\": \"<a href='/distance2.html'/>\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n module_test.httpserver.no_handler_status_code = 404\n\n async def setup_after_prep(self, module_test):\n # here we create a dummy module to consume all events including internal ones\n\n class DummyModule(BaseModule):\n watched_events = [\"*\"]\n _name = \"dummy_module\"\n accept_dupes = True\n accept_url_special = True\n events_seen = []\n\n async def handle_event(self, event):\n self.events_seen.append(event)\n\n module_test.scan.modules[\"dummy_module\"] = DummyModule(module_test.scan)\n\n def check(self, module_test, events):\n event_data = [e.data for e in events]\n assert \"https://www1.test.notreal/\" in event_data\n assert \"https://www2.test.notreal/\" in event_data\n assert \"https://www3.test.notreal/\" in event_data\n assert \"https://www4.test.notreal/\" in event_data\n assert \"www1.test.notreal\" in event_data\n assert \"www2.test.notreal\" in event_data\n assert \"www3.test.notreal\" in event_data\n assert \"www4.test.notreal\" in event_data\n assert \"www5.test.notreal\" in event_data\n assert \"www6.test.notreal\" in event_data\n assert \"www7.test.notreal\" in event_data\n assert \"www8.test.notreal\" in event_data\n # .js files should be emitted as URL_UNVERIFIED events (they are processed by httpx which has accept_js_url=True)\n # they are seen by internal modules but not by output modules\n assert \"http://127.0.0.1:8888/a_relative.js\" not in event_data\n assert \"http://127.0.0.1:8888/link_relative.js\" not in event_data\n assert \"http://127.0.0.1:8888/a_relative.txt\" in event_data\n assert \"http://127.0.0.1:8888/link_relative.txt\" in event_data\n dummy_module_event_data = [e.data for e in module_test.scan.modules[\"dummy_module\"].events_seen]\n assert \"http://127.0.0.1:8888/a_relative.js\" in dummy_module_event_data\n assert \"http://127.0.0.1:8888/link_relative.js\" in dummy_module_event_data\n assert \"http://127.0.0.1:8888/a_relative.txt\" in dummy_module_event_data\n assert \"http://127.0.0.1:8888/link_relative.txt\" in dummy_module_event_data\n\n assert \"nhttps://www1.test.notreal/\" not in event_data\n assert \"x3dhttps://www2.test.notreal/\" not in event_data\n assert \"a2https://www3.test.notreal/\" not in event_data\n assert \"uac20https://www4.test.notreal/\" not in event_data\n\n assert any(\n e.type == \"FINDING\" and e.data.get(\"description\", \"\") == \"Non-HTTP URI: ftp://ftp.test.notreal\"\n for e in events\n )\n assert any(\n e.type == \"PROTOCOL\"\n and e.data.get(\"protocol\", \"\") == \"FTP\"\n and e.data.get(\"host\", \"\") == \"ftp.test.notreal\"\n for e in events\n )\n\n assert any(\n e.type == \"URL_UNVERIFIED\"\n and e.data == \"http://127.0.0.1:8888/relative.html\"\n and \"spider-max\" not in e.tags\n and \"endpoint\" in e.tags\n and \"extension-html\" in e.tags\n and \"in-scope\" in e.tags\n and e.scope_distance == 0\n for e in events\n )\n\n assert any(\n e.type == \"URL_UNVERIFIED\" and e.data == \"http://127.0.0.1:8888/2/depth2.html\" and \"spider-max\" in e.tags\n for e in events\n )\n\n assert any(\n e.type == \"URL_UNVERIFIED\" and e.data == \"http://127.0.0.1:8888/distance2.html\" and \"spider-max\" in e.tags\n for e in events\n )\n\n assert any(\n e.type == \"URL_UNVERIFIED\" and \"miscellaneous.html\" in e.data and \"x50-uart-driver\" not in e.data\n for e in events\n )\n\n\nclass TestExcavate2(TestExcavate):\n targets = [\"http://127.0.0.1:8888/\", \"test.notreal\", \"http://127.0.0.1:8888/subdir/\"]\n\n async def setup_before_prep(self, module_test):\n # root relative\n expect_args = {\"method\": \"GET\", \"uri\": \"/rootrelative.html\"}\n respond_args = {\"response_data\": \"alive\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # page relative\n expect_args = {\"method\": \"GET\", \"uri\": \"/subdir/pagerelative.html\"}\n respond_args = {\"response_data\": \"alive\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/subdir/\"}\n respond_args = {\n \"response_data\": \"\"\"\n <a href='/rootrelative.html'>root relative</a>\n <a href='pagerelative1.html'>page relative 1</a>\n <a href='./pagerelative2.html'>page relative 2</a>\n \"\"\"\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n module_test.httpserver.no_handler_status_code = 404\n\n def check(self, module_test, events):\n root_relative_detection = False\n page_relative_detection_1 = False\n page_relative_detection_2 = False\n root_page_confusion_1 = False\n root_page_confusion_2 = False\n\n for e in events:\n if e.type == \"URL_UNVERIFIED\":\n # these cases represent the desired behavior for parsing relative links\n if e.data == \"http://127.0.0.1:8888/rootrelative.html\":\n root_relative_detection = True\n if e.data == \"http://127.0.0.1:8888/subdir/pagerelative1.html\":\n page_relative_detection_1 = True\n if e.data == \"http://127.0.0.1:8888/subdir/pagerelative2.html\":\n page_relative_detection_2 = True\n\n # these cases indicates that excavate parsed the relative links incorrectly\n if e.data == \"http://127.0.0.1:8888/pagerelative.html\":\n root_page_confusion_1 = True\n if e.data == \"http://127.0.0.1:8888/subdir/rootrelative.html\":\n root_page_confusion_2 = True\n\n assert root_relative_detection, \"Failed to properly excavate root-relative URL\"\n assert page_relative_detection_1, \"Failed to properly excavate page-relative URL\"\n assert page_relative_detection_2, \"Failed to properly excavate page-relative URL\"\n assert not root_page_confusion_1, \"Incorrectly detected page-relative URL\"\n assert not root_page_confusion_2, \"Incorrectly detected root-relative URL\"\n\n\nclass TestExcavateInScopeJavascript(TestExcavate):\n targets = [\"http://127.0.0.1:8888/\"]\n modules_overrides = [\"excavate\", \"httpx\", \"badsecrets\"]\n\n async def setup_before_prep(self, module_test):\n module_test.httpserver.expect_request(\"/\").respond_with_data(\n \"<script>window.location.href = 'http://127.0.0.1:8888/script.js';</script>\"\n )\n module_test.httpserver.expect_request(\"/script.js\").respond_with_data(\n \"var = 'eyJhbGciOiJIUzI1NiJ9.eyJJc3N1ZXIiOiJJc3N1ZXIiLCJVc2VybmFtZSI6IkJhZFNlY3JldHMiLCJleHAiOjE1OTMxMzM0ODMsImlhdCI6MTQ2NjkwMzA4M30.ovqRikAo_0kKJ0GVrAwQlezymxrLGjcEiW_s3UJMMCo';\"\n )\n\n def check(self, module_test, events):\n found_js_url_event = bool(\n [e for e in events if e.type == \"URL\" and e.data == \"http://127.0.0.1:8888/script.js\"]\n )\n found_excavate_jwt_finding = bool(\n [\n e\n for e in events\n if e.type == \"FINDING\" and \"JWT\" in e.data[\"description\"] and str(e.module) == \"excavate\"\n ]\n )\n found_badsecrets_vulnerability = bool(\n [e for e in events if e.type == \"VULNERABILITY\" and str(e.module) == \"badsecrets\"]\n )\n\n assert found_js_url_event, \"Failed to find URL event for script.js\"\n assert found_badsecrets_vulnerability, \"Failed to find BADSECRETs event from script.js\"\n assert found_excavate_jwt_finding, \"Failed to find JWT finding from script.js\"\n\n\nclass TestExcavateRedirect(TestExcavate):\n targets = [\"http://127.0.0.1:8888/\", \"http://127.0.0.1:8888/relative/\", \"http://127.0.0.1:8888/nonhttpredirect/\"]\n config_overrides = {\"scope\": {\"report_distance\": 1}}\n\n async def setup_before_prep(self, module_test):\n # absolute redirect\n module_test.httpserver.expect_request(\"/\").respond_with_data(\n \"\", status=302, headers={\"Location\": \"https://www.test.notreal/yep\"}\n )\n module_test.httpserver.expect_request(\"/relative/\").respond_with_data(\n \"\", status=302, headers={\"Location\": \"./owa/\"}\n )\n module_test.httpserver.expect_request(\"/relative/owa/\").respond_with_data(\n \"ftp://127.0.0.1:2121\\nsmb://127.0.0.1\\nssh://127.0.0.2\"\n )\n module_test.httpserver.expect_request(\"/nonhttpredirect/\").respond_with_data(\n \"\", status=302, headers={\"Location\": \"awb://127.0.0.1:7777\"}\n )\n module_test.httpserver.no_handler_status_code = 404\n\n def check(self, module_test, events):\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"URL_UNVERIFIED\" and e.data == \"https://www.test.notreal/yep\" and e.scope_distance == 1\n ]\n )\n assert 1 == len([e for e in events if e.type == \"URL\" and e.data == \"http://127.0.0.1:8888/relative/owa/\"])\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"FINDING\" and e.data[\"description\"] == \"Non-HTTP URI: awb://127.0.0.1:7777\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"PROTOCOL\" and e.data[\"protocol\"] == \"AWB\" and e.data.get(\"port\", 0) == 7777\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"FINDING\" and e.data[\"description\"] == \"Non-HTTP URI: ftp://127.0.0.1:2121\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"PROTOCOL\" and e.data[\"protocol\"] == \"FTP\" and e.data.get(\"port\", 0) == 2121\n ]\n )\n assert 1 == len(\n [e for e in events if e.type == \"FINDING\" and e.data[\"description\"] == \"Non-HTTP URI: smb://127.0.0.1\"]\n )\n assert 1 == len(\n [e for e in events if e.type == \"PROTOCOL\" and e.data[\"protocol\"] == \"SMB\" and \"port\" not in e.data]\n )\n assert 0 == len([e for e in events if e.type == \"FINDING\" and \"ssh://127.0.0.1\" in e.data[\"description\"]])\n assert 0 == len([e for e in events if e.type == \"PROTOCOL\" and e.data[\"protocol\"] == \"SSH\"])\n\n\nclass TestExcavateQuerystringRemoveTrue(TestExcavate):\n targets = [\"http://127.0.0.1:8888/\"]\n config_overrides = {\"url_querystring_remove\": True, \"url_querystring_collapse\": True}\n lots_of_params = \"\"\"\n <a href=\"http://127.0.0.1:8888/endpoint?foo=1\"/>\n <a href=\"http://127.0.0.1:8888/endpoint?foo=2\"/>\n <a href=\"http://127.0.0.1:8888/endpoint?foo=3\"/>\n <a href=\"http://127.0.0.1:8888/endpoint?foo=4\"/>\n <a href=\"http://127.0.0.1:8888/endpoint?foo=5\"/>\n <a href=\"http://127.0.0.1:8888/endpoint?foo=6\"/>\n <a href=\"http://127.0.0.1:8888/endpoint?foo=7\"/>\n <a href=\"http://127.0.0.1:8888/endpoint?foo=8\"/>\n <a href=\"http://127.0.0.1:8888/endpoint?foo=9\"/>\n <a href=\"http://127.0.0.1:8888/endpoint?foo=10\"/>\n \"\"\"\n\n async def setup_before_prep(self, module_test):\n module_test.httpserver.expect_request(\"/\").respond_with_data(self.lots_of_params)\n\n def check(self, module_test, events):\n assert len([e for e in events if e.type == \"URL_UNVERIFIED\"]) == 2\n assert (\n len([e for e in events if e.type == \"URL_UNVERIFIED\" and e.data == \"http://127.0.0.1:8888/endpoint\"]) == 1\n )\n\n\nclass TestExcavateQuerystringRemoveFalse(TestExcavateQuerystringRemoveTrue):\n config_overrides = {\"url_querystring_remove\": False, \"url_querystring_collapse\": True}\n\n def check(self, module_test, events):\n assert (\n len(\n [\n e\n for e in events\n if e.type == \"URL_UNVERIFIED\" and e.data.startswith(\"http://127.0.0.1:8888/endpoint?\")\n ]\n )\n == 1\n )\n\n\nclass TestExcavateQuerystringCollapseFalse(TestExcavateQuerystringRemoveTrue):\n config_overrides = {\"url_querystring_remove\": False, \"url_querystring_collapse\": False}\n\n def check(self, module_test, events):\n assert (\n len(\n [\n e\n for e in events\n if e.type == \"URL_UNVERIFIED\" and e.data.startswith(\"http://127.0.0.1:8888/endpoint?\")\n ]\n )\n == 10\n )\n\n\nclass TestExcavateMaxLinksPerPage(TestExcavate):\n targets = [\"http://127.0.0.1:8888/\"]\n config_overrides = {\"web\": {\"spider_links_per_page\": 10, \"spider_distance\": 1}}\n\n lots_of_links = \"\"\"\n <a href=\"http://127.0.0.1:8888/1\"/>\n <a href=\"http://127.0.0.1:8888/2\"/>\n <a href=\"http://127.0.0.1:8888/3\"/>\n <a href=\"http://127.0.0.1:8888/4\"/>\n <a href=\"http://127.0.0.1:8888/5\"/>\n <a href=\"http://127.0.0.1:8888/6\"/>\n <a href=\"http://127.0.0.1:8888/7\"/>\n <a href=\"http://127.0.0.1:8888/8\"/>\n <a href=\"http://127.0.0.1:8888/9\"/>\n <a href=\"http://127.0.0.1:8888/10\"/>\n <a href=\"http://127.0.0.1:8888/11\"/>\n <a href=\"http://127.0.0.1:8888/12\"/>\n <a href=\"http://127.0.0.1:8888/13\"/>\n <a href=\"http://127.0.0.1:8888/14\"/>\n <a href=\"http://127.0.0.1:8888/15\"/>\n <a href=\"http://127.0.0.1:8888/16\"/>\n <a href=\"http://127.0.0.1:8888/17\"/>\n <a href=\"http://127.0.0.1:8888/18\"/>\n <a href=\"http://127.0.0.1:8888/19\"/>\n <a href=\"http://127.0.0.1:8888/20\"/>\n <a href=\"http://127.0.0.1:8888/21\"/>\n <a href=\"http://127.0.0.1:8888/22\"/>\n <a href=\"http://127.0.0.1:8888/23\"/>\n <a href=\"http://127.0.0.1:8888/24\"/>\n <a href=\"http://127.0.0.1:8888/25\"/>\n \"\"\"\n\n async def setup_before_prep(self, module_test):\n module_test.httpserver.expect_request(\"/\").respond_with_data(self.lots_of_links)\n\n def check(self, module_test, events):\n url_unverified_events = [e for e in events if e.type == \"URL_UNVERIFIED\"]\n # base URL + 25 links = 26\n assert len(url_unverified_events) == 26\n url_data = [e.data for e in url_unverified_events if \"spider-max\" not in e.tags and \"spider-danger\" in e.tags]\n assert len(url_data) >= 10 and len(url_data) <= 12\n url_events = [e for e in events if e.type == \"URL\"]\n assert len(url_events) == 11\n\n\nclass TestExcavateCSP(TestExcavate):\n csp_test_header = \"default-src 'self'; script-src asdf.test.notreal; object-src 'none';\"\n\n async def setup_before_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"headers\": {\"Content-Security-Policy\": self.csp_test_header}}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.test.notreal\" for e in events)\n\n\nclass TestExcavateURL(TestExcavate):\n async def setup_before_prep(self, module_test):\n module_test.httpserver.expect_request(\"/\").respond_with_data(\n \"SomeSMooshedDATAhttps://asdffoo.test.notreal/some/path\"\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdffoo.test.notreal\" for e in events)\n assert any(e.data == \"https://asdffoo.test.notreal/some/path\" for e in events)\n\n\nclass TestExcavateURL_IP(TestExcavate):\n targets = [\"http://127.0.0.1:8888/\", \"127.0.0.2\"]\n\n async def setup_before_prep(self, module_test):\n module_test.httpserver.expect_request(\"/\").respond_with_data(\"SomeSMooshedDATAhttps://127.0.0.2/some/path\")\n\n def check(self, module_test, events):\n assert any(e.data == \"127.0.0.2\" for e in events)\n assert any(e.data == \"https://127.0.0.2/some/path\" for e in events)\n\n\nclass TestExcavateSerializationNegative(TestExcavate):\n async def setup_before_prep(self, module_test):\n module_test.httpserver.expect_request(\"/\").respond_with_data(\n \"<html><p>llsdtVVFlJxhcGGYTo2PMGTRNFVKZxeKTVbhyosM3Sm/5apoY1/yUmN6HVcn+Xt798SPzgXQlZMttsqp1U1iJFmFO2aCGL/v3tmm/fs7itYsoNnJCelWvm9P4ic1nlKTBOpMjT5B5NmriZwTAzZ5ASjCKcmN8Vh=</p></html>\"\n )\n\n def check(self, module_test, events):\n assert not any(e.type == \"FINDING\" for e in events), \"Found Results without word boundary\"\n\n\nclass TestExcavateSerializationPositive(TestExcavate):\n async def setup_before_prep(self, module_test):\n module_test.httpserver.expect_request(\"/\").respond_with_data(\n \"\"\"<html>\n<h1>.NET</h1>\n<p>AAEAAAD/////AQAAAAAAAAAMAgAAAFJTeXN0ZW0uQ29sbGVjdGlvbnMuR2VuZXJpYy5MaXN0YDFbW1N5c3RlbS5TdHJpbmddXSwgU3lzdGVtLCBWZXJzaW9uPTQuMC4wLjAsIEN1bHR1cmU9bmV1dHJhbCwgUHVibGljS2V5VG9rZW49YjAzZjVmN2YxMWQ1MGFlMwEAAAAIQ29tcGFyZXIQSXRlbUNvdW50AQMAAAAJAwAAAAlTeXN0ZW0uU3RyaW5nW10FAAAACQIAAAAJBAAAAAkFAAAACRcAAAAJCgAAAAkLAAAACQwAAAAJDQAAAAkOAAAACQ8AAAAJEAAAAAkRAAAACRIAAAAJEwAAAA==</p>\n<h1>Java</h1>\n<p>rO0ABXQADUhlbGxvLCB3b3JsZCE=</p>\n<h1>PHP (string)</h1>\n<p>czoyNDoiSGVsbG8sIHdvcmxkISBNb3JlIHRleHQuIjs=</p>\n<h1>PHP (array)</h1>\n<p>YTo0OntpOjA7aToxO2k6MTtzOjE0OiJzZWNvbmQgZWxlbWVudCI7aToyO2k6MztpOjM7czoxODoiTW9yZSB0ZXh0IGluIGFycmF5Ijt9</p>\n<h1>PHP (object)</h1>\n<p>TzoxMjoiU2FtcGxlT2JqZWN0IjoyOntzOjg6InByb3BlcnR5IjtzOjEzOiJJbml0aWFsIHZhbHVlIjtzOjE2OiJhZGRpdGlvbmFsU3RyaW5nIjtzOjIxOiJFeHRyYSB0ZXh0IGluIG9iamVjdC4iO30=</p>\n<h1>Compression</h1>\n<p>H4sIAAAAAAAA/yu2MjS2UvJIzcnJ11Eozy/KSVFUsgYAZN5upRUAAAA=</p>\n</html>\n\"\"\"\n )\n\n def check(self, module_test, events):\n for serialize_type in [\"Java\", \"DOTNET\", \"PHP_Array\", \"PHP_String\", \"PHP_Object\", \"Possible_Compressed\"]:\n assert any(e.type == \"FINDING\" and serialize_type in e.data[\"description\"] for e in events), (\n f\"Did not find {serialize_type} Serialized Object\"\n )\n\n\nclass TestExcavateNonHttpScheme(TestExcavate):\n targets = [\"http://127.0.0.1:8888/\", \"test.notreal\"]\n\n non_http_scheme_html = \"\"\"\n\n <html>\n <head>\n </head>\n <body>\n <p>hxxp://test.notreal</p>\n <p>ftp://test.notreal</p>\n <p>nonsense://test.notreal</p>\n </body>\n </html>\n \"\"\"\n\n async def setup_before_prep(self, module_test):\n module_test.httpserver.expect_request(\"/\").respond_with_data(self.non_http_scheme_html)\n\n def check(self, module_test, events):\n found_hxxp_url = False\n found_ftp_url = False\n found_nonsense_url = False\n\n for e in events:\n if e.type == \"FINDING\":\n if e.data[\"description\"] == \"Non-HTTP URI: hxxp://test.notreal\":\n found_hxxp_url = True\n if e.data[\"description\"] == \"Non-HTTP URI: ftp://test.notreal\":\n found_ftp_url = True\n if \"nonsense\" in e.data[\"description\"]:\n found_nonsense_url = True\n assert found_hxxp_url\n assert found_ftp_url\n assert not found_nonsense_url\n\n\nclass TestExcavateParameterExtraction(TestExcavate):\n # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER\n modules_overrides = [\"excavate\", \"httpx\", \"hunt\"]\n targets = [\"http://127.0.0.1:8888/\"]\n parameter_extraction_html = \"\"\"\n <html>\n <head>\n <title>Get extract\n \n \n \n

Simple GET Form

\n

Use the form below to submit a GET request:

\n \n \n

\n

\n \n \n

Simple POST Form

\n

Use the form below to submit a POST request:

\n
\n \n

\n

\n \n
\n

Simple Generic Form

\n

Use the form below to submit a request:

\n
\n \n

\n \n
\n

Links

\n href\n img\n
\n \n
\n \n \n \"\"\"\n\n async def setup_before_prep(self, module_test):\n module_test.httpserver.expect_request(\"/\").respond_with_data(self.parameter_extraction_html)\n\n def check(self, module_test, events):\n found_jquery_get = False\n found_jquery_post = False\n found_form_get = False\n found_form_post = False\n found_form_generic = False\n found_jquery_get_original_value = False\n found_jquery_post_original_value = False\n found_form_get_original_value = False\n found_form_post_original_value = False\n found_form_generic_original_value = False\n found_htmltags_a = False\n found_htmltags_img = False\n found_select_noquotes = False\n avoid_truncated_values = True\n found_form_input_with_spaces = False\n found_form_get_additional_params = False\n found_form_post_additional_params = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if e.data[\"description\"] == \"HTTP Extracted Parameter [jqueryget] (GET jquery Submodule)\":\n found_jquery_get = True\n if e.data[\"original_value\"] == \"value1\":\n found_jquery_get_original_value = True\n\n if e.data[\"description\"] == \"HTTP Extracted Parameter [jquerypost] (POST jquery Submodule)\":\n found_jquery_post = True\n if e.data[\"original_value\"] == \"value2\":\n found_jquery_post_original_value = True\n\n if e.data[\"description\"] == \"HTTP Extracted Parameter [q1] (GET Form Submodule)\":\n found_form_get = True\n if e.data[\"original_value\"] == \"flowers\":\n found_form_get_original_value = True\n if \"q4\" in e.data[\"additional_params\"].keys():\n found_form_get_additional_params = True\n\n if e.data[\"description\"] == \"HTTP Extracted Parameter [q2] (POST Form Submodule)\":\n found_form_post = True\n if e.data[\"original_value\"] == \"boats\":\n found_form_post_original_value = True\n if \"q5\" in e.data[\"additional_params\"].keys():\n found_form_post_additional_params = True\n\n if e.data[\"description\"] == \"HTTP Extracted Parameter [q3] (Generic Form Submodule)\":\n found_form_generic = True\n if e.data[\"original_value\"] == \"candles\":\n found_form_generic_original_value = True\n\n if e.data[\"description\"] == \"HTTP Extracted Parameter [age] (HTML Tags Submodule)\":\n if e.data[\"original_value\"] == \"456\":\n if \"id\" in e.data[\"additional_params\"].keys():\n found_htmltags_a = True\n\n if e.data[\"description\"] == \"HTTP Extracted Parameter [size] (HTML Tags Submodule)\":\n if e.data[\"original_value\"] == \"m\":\n if \"fit\" in e.data[\"additional_params\"].keys():\n found_htmltags_img = True\n\n if (\n e.data[\"description\"]\n == \"HTTP Extracted Parameter [blog-post-author-display] (POST Form Submodule)\"\n ):\n if e.data[\"original_value\"] == \"user.name\":\n if \"csrf\" in e.data[\"additional_params\"].keys():\n found_select_noquotes = True\n\n if e.data[\"description\"] == \"HTTP Extracted Parameter [q4] (GET Form Submodule)\":\n if e.data[\"original_value\"] == \"trees and forests\":\n found_form_input_with_spaces = True\n if e.data[\"original_value\"] == \"trees\":\n avoid_truncated_values = False\n\n assert found_jquery_get, \"Did not extract Jquery GET parameters\"\n assert found_jquery_post, \"Did not extract Jquery POST parameters\"\n assert found_form_get, \"Did not extract Form GET parameters\"\n assert found_form_post, \"Did not extract Form POST parameters\"\n assert found_form_generic, \"Did not extract Form (Generic) parameters\"\n assert found_form_input_with_spaces, \"Did not extract Form input with spaces\"\n assert avoid_truncated_values, \"Emitted a parameter with spaces without the entire value\"\n assert found_jquery_get_original_value, \"Did not extract Jquery GET parameter original_value\"\n assert found_jquery_post_original_value, \"Did not extract Jquery POST parameter original_value\"\n assert found_form_get_original_value, \"Did not extract Form GET parameter original_value\"\n assert found_form_post_original_value, \"Did not extract Form POST parameter original_value\"\n assert found_form_generic_original_value, \"Did not extract Form (Generic) parameter original_value\"\n assert found_htmltags_a, \"Did not extract parameter(s) from a-tag\"\n assert found_htmltags_img, \"Did not extract parameter(s) from img-tag\"\n assert found_select_noquotes, \"Did not extract parameter(s) from select-tag\"\n assert found_form_get_additional_params, \"Did not extract additional parameters from GET form\"\n assert found_form_post_additional_params, \"Did not extract additional parameters from POST form\"\n\n\nclass TestExcavateParameterExtraction_postform_noaction(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/\"]\n\n # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER\n modules_overrides = [\"httpx\", \"excavate\", \"hunt\"]\n postform_extract_html = \"\"\"\n\n

Post for without action

\n
\n \n \n

\n \n
\n\n \"\"\"\n\n async def setup_after_prep(self, module_test):\n respond_args = {\"response_data\": self.postform_extract_html, \"headers\": {\"Content-Type\": \"text/html\"}}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n excavate_formnoaction_extraction = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [state] (POST Form (no action) Submodule)\" in e.data[\"description\"]:\n excavate_formnoaction_extraction = True\n assert excavate_formnoaction_extraction, \"Excavate failed to extract web parameter\"\n\n\nclass TestExcavateParameterExtraction_postform_htmlencodedaction(TestExcavateParameterExtraction_postform_noaction):\n postform_extract_html = \"\"\"\n\n \n \n
\n
\n \n
\n \n
\n \n\n \"\"\"\n\n def check(self, module_test, events):\n excavate_handle_htmlencoded_action = True\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if (\n \"HTTP Extracted Parameter [value] (POST Form Submodule)\" in e.data[\"description\"]\n and e.data[\"url\"] == \"https://127.0.0.1:8080/sso-web/singleSignOn.action\"\n ):\n excavate_handle_htmlencoded_action = True\n assert excavate_handle_htmlencoded_action, \"Excavate failed to extract web parameter\"\n\n\nclass TestExcavateParameterExtraction_additionalparams(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/\"]\n\n # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER\n modules_overrides = [\"httpx\", \"excavate\", \"hunt\"]\n postformnoaction_extract_multiparams_html = \"\"\"\n\n

Post for without action

\n
\n \n \n \n \n
\n\n \"\"\"\n\n async def setup_after_prep(self, module_test):\n respond_args = {\n \"response_data\": self.postformnoaction_extract_multiparams_html,\n \"headers\": {\"Content-Type\": \"text/html\"},\n }\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n excavate_additionalparam_extraction_param1 = False\n excavate_additionalparam_extraction_param2 = False\n excavate_additionalparam_extraction_param3 = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if (\n e.data[\"name\"] == \"template-action\"\n and \"csrf\" in e.data[\"additional_params\"].keys()\n and \"template\" in e.data[\"additional_params\"].keys()\n ):\n excavate_additionalparam_extraction_param1 = True\n if (\n e.data[\"name\"] == \"template\"\n and \"csrf\" in e.data[\"additional_params\"].keys()\n and \"template-action\" in e.data[\"additional_params\"].keys()\n ):\n excavate_additionalparam_extraction_param2 = True\n if (\n e.data[\"name\"] == \"csrf\"\n and \"template\" in e.data[\"additional_params\"].keys()\n and \"template-action\" in e.data[\"additional_params\"].keys()\n ):\n excavate_additionalparam_extraction_param3 = True\n assert excavate_additionalparam_extraction_param1, (\n \"Excavate failed to extract web parameter with correct additional data (param 1)\"\n )\n assert excavate_additionalparam_extraction_param2, (\n \"Excavate failed to extract web parameter with correct additional data (param 2)\"\n )\n assert excavate_additionalparam_extraction_param3, (\n \"Excavate failed to extract web parameter with correct additional data (param 3)\"\n )\n\n\nclass TestExcavateParameterExtraction_getparam(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/\"]\n\n # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER\n modules_overrides = [\"httpx\", \"excavate\", \"hunt\"]\n getparam_extract_html = \"\"\"\nping\n \"\"\"\n\n async def setup_after_prep(self, module_test):\n respond_args = {\"response_data\": self.getparam_extract_html, \"headers\": {\"Content-Type\": \"text/html\"}}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n excavate_getparam_extraction = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [hack] (HTML Tags Submodule)\" in e.data[\"description\"]:\n excavate_getparam_extraction = True\n assert excavate_getparam_extraction, \"Excavate failed to extract web parameter\"\n\n\nclass TestExcavateParameterExtraction_relativeurl(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/\"]\n\n # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER\n modules_overrides = [\"httpx\", \"excavate\", \"hunt\"]\n config_overrides = {\"web\": {\"spider_distance\": 2, \"spider_depth\": 3}}\n\n # Secondary page that has a relative link to a traversal URL\n secondary_page_html = \"\"\"\n \n Go to root\n \n \"\"\"\n\n # Primary page that leads to the secondary page\n primary_page_html = \"\"\"\n \n Go to secondary page\n \n \"\"\"\n\n # Root page content\n root_page_html = \"Root page\"\n\n async def setup_after_prep(self, module_test):\n module_test.httpserver.expect_request(\"/\").respond_with_data(self.primary_page_html)\n module_test.httpserver.expect_request(\"/secondary\").respond_with_data(self.secondary_page_html)\n module_test.httpserver.expect_request(\"/root.html\").respond_with_data(self.root_page_html)\n\n def check(self, module_test, events):\n # Validate that the traversal was successful and WEB_PARAMETER was extracted\n traversed_to_root = False\n parameter_extraction_found = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter\" in e.data[\"description\"]:\n parameter_extraction_found = True\n\n if e.type == \"URL\":\n if \"root.html\" in e.parsed_url.path:\n traversed_to_root = True\n\n assert traversed_to_root, \"Failed to follow the relative traversal to /root.html\"\n assert parameter_extraction_found, \"Excavate failed to extract parameter after traversal\"\n\n\nclass TestExcavateParameterExtraction_getparam_novalue(TestExcavateParameterExtraction_getparam):\n getparam_extract_html = \"\"\"\n
\n
\n \n \n \n \n
\n
\n \"\"\"\n\n def check(self, module_test, events):\n excavate_getparam_extraction = False\n found_no_value_additional_params = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [searchTerm] (GET Form Submodule)\" in e.data[\"description\"]:\n excavate_getparam_extraction = True\n if \"searchTerm2\" in e.data[\"additional_params\"].keys():\n found_no_value_additional_params = True\n assert excavate_getparam_extraction, \"Excavate failed to extract web parameter\"\n assert found_no_value_additional_params, (\n \"Excavate failed to extract additional parameters for input tag with no value\"\n )\n\n\nclass TestExcavateParameterExtraction_json(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/\"]\n modules_overrides = [\"httpx\", \"excavate\", \"paramminer_getparams\"]\n config_overrides = {\n \"modules\": {\n \"excavate\": {\"speculate_params\": True},\n \"paramminer_getparams\": {\"wordlist\": tempwordlist([]), \"recycle_words\": True},\n }\n }\n getparam_extract_json = \"\"\"\n {\n \"obscureParameter\": 1,\n \"common\": 1\n}\n \"\"\"\n\n async def setup_after_prep(self, module_test):\n respond_args = {\"response_data\": self.getparam_extract_json, \"headers\": {\"Content-Type\": \"application/json\"}}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n excavate_json_extraction = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if (\n \"HTTP Extracted Parameter (speculative from json content) [obscureParameter]\"\n in e.data[\"description\"]\n ):\n excavate_json_extraction = True\n assert excavate_json_extraction, \"Excavate failed to extract json parameter\"\n\n\nclass TestExcavateParameterExtraction_xml(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/\"]\n modules_overrides = [\"httpx\", \"excavate\", \"paramminer_getparams\"]\n config_overrides = {\n \"modules\": {\n \"excavate\": {\"speculate_params\": True},\n \"paramminer_getparams\": {\"wordlist\": tempwordlist([]), \"recycle_words\": True},\n }\n }\n getparam_extract_xml = \"\"\"\n \n 1\n 1\n \n \"\"\"\n\n async def setup_after_prep(self, module_test):\n respond_args = {\"response_data\": self.getparam_extract_xml, \"headers\": {\"Content-Type\": \"application/xml\"}}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n excavate_xml_extraction = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if (\n \"HTTP Extracted Parameter (speculative from xml content) [obscureParameter]\"\n in e.data[\"description\"]\n ):\n excavate_xml_extraction = True\n assert excavate_xml_extraction, \"Excavate failed to extract xml parameter\"\n\n\nclass TestExcavateParameterExtraction_xml_invalid(TestExcavateParameterExtraction_xml):\n getparam_extract_xml = \"\"\"\n \n 1\n invalid\\nwith\\nnewlines\n \n \"\"\"\n\n async def setup_after_prep(self, module_test):\n respond_args = {\"response_data\": self.getparam_extract_xml, \"headers\": {\"Content-Type\": \"application/xml\"}}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n excavate_xml_extraction = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if (\n \"HTTP Extracted Parameter (speculative from xml content) [newlines]\" in e.data[\"description\"]\n and \"\\n\" not in e.data[\"original_value\"]\n ):\n excavate_xml_extraction = True\n assert excavate_xml_extraction, \"Excavate failed to extract xml parameter\"\n\n\nclass TestExcavateParameterExtraction_inputtagnovalue(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/\"]\n\n # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER\n modules_overrides = [\"httpx\", \"excavate\", \"hunt\"]\n getparam_extract_html = \"\"\"\n
\n \"\"\"\n\n async def setup_after_prep(self, module_test):\n respond_args = {\"response_data\": self.getparam_extract_html, \"headers\": {\"Content-Type\": \"text/html\"}}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n excavate_getparam_extraction = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [novalue] (GET Form Submodule)\" in e.data[\"description\"]:\n excavate_getparam_extraction = True\n assert excavate_getparam_extraction, \"Excavate failed to extract web parameter\"\n\n\nclass TestExcavateParameterExtraction_jqueryjsonajax(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/\"]\n modules_overrides = [\"httpx\", \"excavate\", \"hunt\"]\n jsonajax_extract_html = \"\"\"\n \n \n\n\n

test

\n\n\n \"\"\"\n return Response(xss_block, status=200)\n return Response(self.parameter_block, status=200)\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"lightfuzz\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n expect_args = re.compile(\"/otherpage.php\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n original_value_captured = False\n xss_finding_emitted = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [language]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n if e.data[\"original_value\"] == \"en\":\n original_value_captured = True\n\n if e.type == \"FINDING\":\n if \"Possible Reflected XSS. Parameter: [language] Context: [In Javascript]\" in e.data[\"description\"]:\n xss_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER was not emitted\"\n assert original_value_captured, \"original_value not captured\"\n assert xss_finding_emitted, \"In Javascript XSS FINDING not emitted\"\n\n\n# XSS Parameter Needing URL-Encoding\nclass Test_Lightfuzz_urlencoding(Test_Lightfuzz_xss_injs):\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"cmdi\", \"crypto\", \"path\", \"serial\", \"sqli\", \"ssti\", \"xss\", \"esi\"],\n }\n },\n }\n\n parameter_block = \"\"\"\n \n Link\n \n \"\"\"\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n original_value_captured = False\n xss_finding_emitted = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [language]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n if e.data[\"original_value\"] is not None and e.data[\"original_value\"] == \"parameter with spaces\":\n original_value_captured = True\n\n if e.type == \"FINDING\":\n if \"Possible Reflected XSS. Parameter: [language] Context: [In Javascript]\" in e.data[\"description\"]:\n xss_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER was not emitted\"\n assert original_value_captured, \"original_value not captured\"\n assert xss_finding_emitted, \"In Javascript XSS FINDING not emitted\"\n\n\n# SQLI Single Quote/Two Single Quote (getparam)\nclass Test_Lightfuzz_sqli(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"lightfuzz\", \"excavate\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"sqli\"],\n }\n },\n }\n\n def request_handler(self, request):\n qs = str(request.query_string.decode())\n parameter_block = \"\"\"\n
\n
\n \n \n
\n
\n \"\"\"\n if \"search=\" in qs:\n value = qs.split(\"=\")[1]\n\n if \"&\" in value:\n value = value.split(\"&\")[0]\n\n sql_block_normal = f\"\"\"\n
\n

0 search results for '{unquote(value)}'

\n
\n
\n \"\"\"\n\n sql_block_error = \"\"\"\n
\n

Found error in SQL query

\n
\n
\n \"\"\"\n if value.endswith(\"'\"):\n if value.endswith(\"''\"):\n return Response(sql_block_normal, status=200)\n return Response(sql_block_error, status=500)\n return Response(parameter_block, status=200)\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"lightfuzz\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n sqli_finding_emitted = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [search]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n if e.type == \"FINDING\":\n if (\n \"Possible SQL Injection. Parameter: [search] Parameter Type: [GETPARAM] Detection Method: [Single Quote/Two Single Quote, Code Change (200->500->200)]\"\n in e.data[\"description\"]\n ):\n sqli_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER was not emitted\"\n assert sqli_finding_emitted, \"SQLi Single/Double Quote getparam FINDING not emitted\"\n\n\n# SQLI Single Quote/Two Single Quote (postparam)\nclass Test_Lightfuzz_sqli_post(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"lightfuzz\", \"excavate\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"sqli\"],\n }\n },\n }\n\n def request_handler(self, request):\n parameter_block = \"\"\"\n
\n
\n \n \n
\n
\n \"\"\"\n\n if \"search\" in request.form.keys():\n value = request.form[\"search\"]\n\n sql_block_normal = f\"\"\"\n
\n

0 search results for '{unquote(value)}'

\n
\n
\n \"\"\"\n\n sql_block_error = \"\"\"\n
\n

Found error in SQL query

\n
\n
\n \"\"\"\n if value.endswith(\"'\"):\n if value.endswith(\"''\"):\n return Response(sql_block_normal, status=200)\n return Response(sql_block_error, status=500)\n return Response(parameter_block, status=200)\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"lightfuzz\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n sqli_finding_emitted = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [search]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n\n if e.type == \"FINDING\":\n if (\n \"Possible SQL Injection. Parameter: [search] Parameter Type: [POSTPARAM] Detection Method: [Single Quote/Two Single Quote, Code Change (200->500->200)]\"\n in e.data[\"description\"]\n ):\n sqli_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER was not emitted\"\n assert sqli_finding_emitted, \"SQLi Single/Double Quote postparam FINDING not emitted\"\n\n\n# disable_post test\nclass Test_Lightfuzz_disable_post(Test_Lightfuzz_sqli_post):\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"sqli\"],\n \"disable_post\": True,\n }\n },\n }\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n sqli_finding_emitted = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [search]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n\n if e.type == \"FINDING\":\n if (\n \"Possible SQL Injection. Parameter: [search] Parameter Type: [POSTPARAM] Detection Method: [Single Quote/Two Single Quote, Code Change (200->500->200)]\"\n in e.data[\"description\"]\n ):\n sqli_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER was not emitted\"\n assert not sqli_finding_emitted, \"post-based SQLI emitted despite post-parameters being disabled\"\n\n\n# SQLI Single Quote/Two Single Quote (headers)\nclass Test_Lightfuzz_sqli_headers(Test_Lightfuzz_sqli):\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"lightfuzz\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n seed_events = []\n parent_event = module_test.scan.make_event(\n \"http://127.0.0.1:8888/\",\n \"URL\",\n module_test.scan.root_event,\n module=\"httpx\",\n tags=[\"status-200\", \"distance-0\"],\n )\n\n data = {\n \"host\": \"127.0.0.1\",\n \"type\": \"HEADER\",\n \"name\": \"testheader\",\n \"original_value\": None,\n \"url\": \"http://127.0.0.1:8888\",\n \"description\": \"Test Dummy Header\",\n }\n seed_event = module_test.scan.make_event(data, \"WEB_PARAMETER\", parent_event, tags=[\"distance-0\"])\n seed_events.append(seed_event)\n for event in seed_events:\n await module_test.scan.ingress_module.incoming_event_queue.put(event)\n\n def request_handler(self, request):\n placeholder_block = \"\"\"\n \n

placeholder

\n \n \"\"\"\n\n if request.headers.get(\"testheader\") is not None:\n header_value = request.headers.get(\"testheader\")\n\n header_block_normal = f\"\"\"\n \n

placeholder

\n

test: {header_value}

\n \n \"\"\"\n header_block_error = \"\"\"\n \n

placeholder

\n

Error!

\n \n \"\"\"\n if header_value.endswith(\"'\") and not header_value.endswith(\"''\"):\n return Response(header_block_error, status=500)\n return Response(header_block_normal, status=200)\n return Response(placeholder_block, status=200)\n\n def check(self, module_test, events):\n sqli_finding_emitted = False\n for e in events:\n if e.type == \"FINDING\":\n if (\n \"Possible SQL Injection. Parameter: [testheader] Parameter Type: [HEADER] Detection Method: [Single Quote/Two Single Quote, Code Change (200->500->200)]\"\n in e.data[\"description\"]\n ):\n sqli_finding_emitted = True\n assert sqli_finding_emitted, \"SQLi Single/Double Quote headers FINDING not emitted\"\n\n\n# SQLI Single Quote/Two Single Quote (cookies)\nclass Test_Lightfuzz_sqli_cookies(Test_Lightfuzz_sqli):\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"lightfuzz\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n seed_events = []\n parent_event = module_test.scan.make_event(\n \"http://127.0.0.1:8888/\",\n \"URL\",\n module_test.scan.root_event,\n module=\"httpx\",\n tags=[\"status-200\", \"distance-0\"],\n )\n\n data = {\n \"host\": \"127.0.0.1\",\n \"type\": \"COOKIE\",\n \"name\": \"test\",\n \"original_value\": None,\n \"url\": \"http://127.0.0.1:8888\",\n \"description\": \"Test Dummy Cookie\",\n }\n seed_event = module_test.scan.make_event(data, \"WEB_PARAMETER\", parent_event, tags=[\"distance-0\"])\n seed_events.append(seed_event)\n for event in seed_events:\n await module_test.scan.ingress_module.incoming_event_queue.put(event)\n\n def request_handler(self, request):\n placeholder_block = \"\"\"\n \n

placeholder

\n \n \"\"\"\n\n if request.cookies.get(\"test\") is not None:\n header_value = request.cookies.get(\"test\")\n\n header_block_normal = f\"\"\"\n \n

placeholder

\n

test: {header_value}

\n \n \"\"\"\n\n header_block_error = \"\"\"\n \n

placeholder

\n

Error!

\n \n \"\"\"\n if header_value.endswith(\"'\") and not header_value.endswith(\"''\"):\n return Response(header_block_error, status=500)\n return Response(header_block_normal, status=200)\n return Response(placeholder_block, status=200)\n\n def check(self, module_test, events):\n sqli_finding_emitted = False\n for e in events:\n if e.type == \"FINDING\":\n if (\n \"Possible SQL Injection. Parameter: [test] Parameter Type: [COOKIE] Detection Method: [Single Quote/Two Single Quote, Code Change (200->500->200)]\"\n in e.data[\"description\"]\n ):\n sqli_finding_emitted = True\n assert sqli_finding_emitted, \"SQLi Single/Double Quote cookies FINDING not emitted\"\n\n\n# SQLi Delay Probe\nclass Test_Lightfuzz_sqli_delay(Test_Lightfuzz_sqli):\n def request_handler(self, request):\n from time import sleep\n\n qs = str(request.query_string.decode())\n\n parameter_block = \"\"\"\n
\n
\n \n \n
\n
\n\n \"\"\"\n if \"search=\" in qs:\n value = qs.split(\"=\")[1]\n\n if \"&\" in value:\n value = value.split(\"&\")[0]\n\n sql_block = \"\"\"\n
\n

0 search results found

\n
\n
\n \"\"\"\n if \"' AND (SLEEP(5)) AND '\" in unquote(value):\n sleep(5)\n return Response(sql_block, status=200)\n return Response(parameter_block, status=200)\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n sqldelay_finding_emitted = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [search]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n\n if e.type == \"FINDING\":\n if (\n \"Possible Blind SQL Injection. Parameter: [search] Parameter Type: [GETPARAM] Detection Method: [Delay Probe (1' AND (SLEEP(5)) AND ')]\"\n in e.data[\"description\"]\n ):\n sqldelay_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER was not emitted\"\n assert sqldelay_finding_emitted, \"SQLi Delay FINDING not emitted\"\n\n\n# Serialization Module (Error Resolution)\nclass Test_Lightfuzz_serial_errorresolution(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"lightfuzz\", \"excavate\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"serial\"],\n }\n },\n }\n\n dotnet_serial_error = \"\"\"\n \n Description: An unhandled exception occurred during the execution of the current web request. Please review the stack trace for more information about the error and where it originated in the code.\n\n

\n\n Exception Details: System.Runtime.Serialization.SerializationException: End of Stream encountered before parsing was completed.

\n \n \"\"\"\n\n dotnet_serial_html = \"\"\"\n \n \n \n Deserialization RCE Example\n \n \n
\n
\n \n
\n\n
\n\n \n \n
\n
\n

Deserialization Test

\n Enter serialized data:
\n

\n

\n
\n
\n\n \n \n \n \"\"\"\n\n async def setup_after_prep(self, module_test):\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def request_handler(self, request):\n dotnet_serial_error_resolved = (\n \"Deserialization successful! Object type: System.String\"\n )\n post_params = request.form\n\n if \"TextBox1\" not in post_params.keys():\n return Response(self.dotnet_serial_html, status=200)\n\n else:\n if post_params[\"__VIEWSTATE\"] != \"/wEPDwULLTE5MTI4MzkxNjVkZNt7ICM+GixNryV6ucx+srzhXlwP\":\n return Response(self.dotnet_serial_error, status=500)\n if post_params[\"TextBox1\"] == \"AAEAAAD/////AQAAAAAAAAAGAQAAAAdndXN0YXZvCw==\":\n return Response(dotnet_serial_error_resolved, status=200)\n else:\n return Response(self.dotnet_serial_error, status=500)\n\n def check(self, module_test, events):\n excavate_extracted_form_parameter = False\n excavate_extracted_form_parameter_details = False\n lightfuzz_serial_detect_errorresolution = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if e.data[\"name\"] == \"TextBox1\":\n excavate_extracted_form_parameter = True\n if (\n e.data[\"url\"] == \"http://127.0.0.1:8888/deser.aspx\"\n and e.data[\"host\"] == \"127.0.0.1\"\n and e.data[\"additional_params\"]\n == {\n \"__VIEWSTATE\": \"/wEPDwULLTE5MTI4MzkxNjVkZNt7ICM+GixNryV6ucx+srzhXlwP\",\n \"__VIEWSTATEGENERATOR\": \"AD6F025C\",\n \"__EVENTVALIDATION\": \"/wEdAANdCjkiIFhjCB8ta8aO/EhuESCFkFW/RuhzY1oLb/NUVM34O/GfAV4V4n0wgFZHr3czZjft8VgObR/WUivai7w4kfR1wg==\",\n \"Button1\": \"Submit\",\n }\n ):\n excavate_extracted_form_parameter_details = True\n if e.type == \"FINDING\":\n if (\n e.data[\"description\"]\n == \"POSSIBLE Unsafe Deserialization. Parameter: [TextBox1] Parameter Type: [POSTPARAM] Technique: [Error Resolution (Baseline: [500] -> Probe: [200] )] Serialization Payload: [dotnet_base64]\"\n ):\n lightfuzz_serial_detect_errorresolution = True\n\n assert excavate_extracted_form_parameter, \"WEB_PARAMETER for POST form was not emitted\"\n assert excavate_extracted_form_parameter_details, \"WEB_PARAMETER for POST form did not have correct data\"\n assert lightfuzz_serial_detect_errorresolution, (\n \"Lightfuzz Serial module failed to detect ASP.NET error resolution based deserialization\"\n )\n\n\n# Serialization Module (Error Resolution False Positive)\nclass Test_Lightfuzz_serial_errorresolution_falsepositive(Test_Lightfuzz_serial_errorresolution):\n def request_handler(self, request):\n dotnet_serial_error_resolved_with_general_error = (\n \"Internal Server Error (invalid characters!)\"\n )\n post_params = request.form\n\n if \"TextBox1\" not in post_params.keys():\n return Response(self.dotnet_serial_html, status=200)\n\n else:\n if post_params[\"__VIEWSTATE\"] != \"/wEPDwULLTE5MTI4MzkxNjVkZNt7ICM+GixNryV6ucx+srzhXlwP\":\n return Response(self.dotnet_serial_error, status=500)\n if post_params[\"TextBox1\"] == \"AAEAAAD/////AQAAAAAAAAAGAQAAAAdndXN0YXZvCw==\":\n return Response(dotnet_serial_error_resolved_with_general_error, status=200)\n else:\n return Response(self.dotnet_serial_error, status=500)\n\n def check(self, module_test, events):\n no_finding_emitted = True\n\n for e in events:\n if e.type == \"FINDING\":\n no_finding_emitted = False\n\n assert no_finding_emitted, \"False positive finding was emitted\"\n\n\nclass Test_Lightfuzz_serial_errorresolution_existingvalue_valid(Test_Lightfuzz_serial_errorresolution):\n dotnet_serial_html = \"\"\"\n \n \n \n Deserialization RCE Example\n \n \n
\n
\n \n
\n\n
\n\n \n \n
\n
\n

Deserialization Test

\n Enter serialized data:
\n

\n

\n
\n
\n\n \n \n \n \"\"\"\n\n def check(self, module_test, events):\n excavate_extracted_form_parameter = False\n excavate_extracted_form_parameter_details = False\n excavate_detect_serialization_value = False\n lightfuzz_serial_detect_errorresolution = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if e.data[\"name\"] == \"TextBox1\":\n excavate_extracted_form_parameter = True\n if (\n e.data[\"url\"] == \"http://127.0.0.1:8888/deser.aspx\"\n and e.data[\"host\"] == \"127.0.0.1\"\n and e.data[\"original_value\"] == \"AAEAAAD/////AQAAAAAAAAAGAQAAAAdndXN0YXZvCw==\"\n and e.data[\"additional_params\"]\n == {\n \"__VIEWSTATE\": \"/wEPDwULLTE5MTI4MzkxNjVkZNt7ICM+GixNryV6ucx+srzhXlwP\",\n \"__VIEWSTATEGENERATOR\": \"AD6F025C\",\n \"__EVENTVALIDATION\": \"/wEdAANdCjkiIFhjCB8ta8aO/EhuESCFkFW/RuhzY1oLb/NUVM34O/GfAV4V4n0wgFZHr3czZjft8VgObR/WUivai7w4kfR1wg==\",\n \"Button1\": \"Submit\",\n }\n ):\n excavate_extracted_form_parameter_details = True\n if e.type == \"FINDING\":\n if e.data[\"description\"] == \"HTTP response (body) contains a possible serialized object (DOTNET)\":\n excavate_detect_serialization_value = True\n if (\n e.data[\"description\"]\n == \"POSSIBLE Unsafe Deserialization. Parameter: [TextBox1] Parameter Type: [POSTPARAM] Original Value: [AAEAAAD/////AQAAAAAAAAAGAQAAAAdndXN0YXZvCw==] Technique: [Error Resolution (Baseline: [500] -> Probe: [200] )] Serialization Payload: [dotnet_base64]\"\n ):\n lightfuzz_serial_detect_errorresolution = True\n\n assert excavate_extracted_form_parameter, \"WEB_PARAMETER for POST form was not emitted\"\n assert excavate_extracted_form_parameter_details, \"WEB_PARAMETER for POST form did not have correct data\"\n assert excavate_detect_serialization_value, \"WEB_PARAMETER for POST form did not have correct data\"\n assert lightfuzz_serial_detect_errorresolution, (\n \"Lightfuzz Serial module failed to detect ASP.NET error resolution based deserialization\"\n )\n\n\nclass Test_Lightfuzz_serial_errorresolution_existingvalue_invalid(Test_Lightfuzz_serial_errorresolution_falsepositive):\n dotnet_serial_html = \"\"\"\n \n \n \n Deserialization RCE Example\n \n \n
\n
\n \n
\n\n
\n\n \n \n
\n
\n

Deserialization Test

\n Enter serialized data:
\n

\n

\n
\n
\n\n \n \n \n \"\"\"\n\n\n# Serialization Module (Error Differential)\nclass Test_Lightfuzz_serial_errordifferential(Test_Lightfuzz_serial_errorresolution):\n def request_handler(self, request):\n java_serial_error = \"\"\"\n \n

Internal Server Error

\n

java.io.StreamCorruptedException: invalid stream header: 0C400304

\n \n \"\"\"\n\n java_serial_error_keyword = \"\"\"\n \n

Internal Server Error

\n

java.lang.ClassCastException: Cannot cast java.lang.String to lab.actions.common.serializable.AccessTokenUser

\n \n \"\"\"\n\n java_serial_html = \"\"\"\n \n \n \n Deserialization RCE Example\n \n \n Please log in to continue.\n \n \n \"\"\"\n\n cookies = request.cookies\n\n if \"session\" not in cookies.keys():\n response = Response(java_serial_html, status=200)\n response.set_cookie(\"session\", value=\"\", max_age=3600, httponly=True)\n return response\n\n else:\n if unquote(cookies[\"session\"]) == \"rO0ABXQABHRlc3Q=\":\n return Response(java_serial_error_keyword, status=500)\n else:\n return Response(java_serial_error, status=500)\n\n def check(self, module_test, events):\n excavate_extracted_cookie_parameter = False\n lightfuzz_serial_detect_errordifferential = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if e.data[\"description\"] == \"Set-Cookie Assigned Cookie [session]\" and e.data[\"type\"] == \"COOKIE\":\n excavate_extracted_cookie_parameter = True\n\n if e.type == \"FINDING\":\n if (\n e.data[\"description\"]\n == \"POSSIBLE Unsafe Deserialization. Parameter: [session] Parameter Type: [COOKIE] Technique: [Differential Error Analysis] Error-String: [cannot cast java.lang.string] Payload: [java_base64_string_error]\"\n ):\n lightfuzz_serial_detect_errordifferential = True\n\n assert excavate_extracted_cookie_parameter, \"WEB_PARAMETER for cookie was not emitted\"\n assert lightfuzz_serial_detect_errordifferential, (\n \"Lightfuzz Serial module failed to detect Java error differential based deserialization\"\n )\n\n\n# Serialization Modules (Error Differential - False positive check)\nclass Test_Lightfuzz_serial_errordifferential_falsepositive(Test_Lightfuzz_serial_errorresolution):\n def request_handler(self, request):\n post_params = request.form\n if \"TextBox1\" not in post_params.keys():\n return Response(self.dotnet_serial_html, status=200)\n\n else:\n dotnet_serial_reflection = (\n f\"

invalid user

reflected input: {post_params['TextBox1']}\"\n )\n return Response(dotnet_serial_reflection, status=500)\n\n def check(self, module_test, events):\n finding_count = 0\n for e in events:\n if e.type == \"FINDING\":\n finding_count += 1\n assert finding_count == 0, \"Unexpected FINDING events reported\"\n\n\n# CMDi echo canary\nclass Test_Lightfuzz_cmdi(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"lightfuzz\", \"excavate\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"cmdi\"],\n }\n },\n }\n\n def request_handler(self, request):\n qs = str(request.query_string.decode())\n\n parameter_block = \"\"\"\n

\n
\n \n \n
\n
\n \"\"\"\n if \"search=\" in qs:\n value = qs.split(\"=\")[1]\n if \"&\" in value:\n value = value.split(\"&\")[0]\n if \"&& echo \" in unquote(value):\n cmdi_value = unquote(value).split(\"&& echo \")[1].split(\" \")[0]\n else:\n cmdi_value = value\n cmdi_block = f\"\"\"\n
\n

0 search results for '{unquote(cmdi_value)}'

\n
\n
\n \"\"\"\n return Response(cmdi_block, status=200)\n\n return Response(parameter_block, status=200)\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"lightfuzz\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n cmdi_echocanary_finding_emitted = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [search]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n\n if e.type == \"FINDING\":\n if (\n \"POSSIBLE OS Command Injection. Parameter: [search] Parameter Type: [GETPARAM] Detection Method: [echo canary] CMD Probe Delimeters: [&&]\"\n in e.data[\"description\"]\n ):\n cmdi_echocanary_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER was not emitted\"\n assert cmdi_echocanary_finding_emitted, \"echo canary CMDi FINDING not emitted\"\n\n\n# CMDi interactsh\nclass Test_Lightfuzz_cmdi_interactsh(Test_Lightfuzz_cmdi):\n @staticmethod\n def extract_subdomain_tag(data):\n pattern = r\"search=.+%26%26%20nslookup%20(.+)\\.fakedomain\\.fakeinteractsh.com%20%26%26\"\n match = re.search(pattern, data)\n if match:\n return match.group(1)\n\n config_overrides = {\n \"interactsh_disable\": False,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"cmdi\"],\n }\n },\n }\n\n def request_handler(self, request):\n qs = str(request.query_string.decode())\n\n parameter_block = \"\"\"\n
\n
\n \n \n
\n
\n \"\"\"\n\n if \"search=\" in qs:\n subdomain_tag = None\n subdomain_tag = self.extract_subdomain_tag(request.full_path)\n\n if subdomain_tag:\n self.interactsh_mock_instance.mock_interaction(subdomain_tag)\n return Response(parameter_block, status=200)\n\n async def setup_before_prep(self, module_test):\n self.interactsh_mock_instance = module_test.mock_interactsh(\"lightfuzz\")\n\n module_test.monkeypatch.setattr(\n module_test.scan.helpers, \"interactsh\", lambda *args, **kwargs: self.interactsh_mock_instance\n )\n\n async def setup_after_prep(self, module_test):\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n cmdi_interacttsh_finding_emitted = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [search]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n\n if e.type == \"VULNERABILITY\":\n if (\n \"OS Command Injection (OOB Interaction) Type: [GETPARAM] Parameter Name: [search] Probe: [&&]\"\n in e.data[\"description\"]\n ):\n cmdi_interacttsh_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER was not emitted\"\n assert cmdi_interacttsh_finding_emitted, \"interactsh CMDi FINDING not emitted\"\n\n\nclass Test_Lightfuzz_speculative(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/\"]\n modules_overrides = [\"httpx\", \"excavate\", \"paramminer_getparams\", \"lightfuzz\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\"enabled_submodules\": [\"xss\"]},\n \"paramminer_getparams\": {\"wordlist\": tempwordlist([]), \"recycle_words\": True},\n \"excavate\": {\"speculate_params\": True},\n },\n }\n\n def request_handler(self, request):\n qs = str(request.query_string.decode())\n parameter_block = \"\"\"\n {\n \"search\": 1,\n \"common\": 1\n }\n \"\"\"\n if \"search=\" in qs:\n value = qs.split(\"=\")[1]\n if \"&\" in value:\n value = value.split(\"&\")[0]\n xss_block = f\"\"\"\n
\n

0 search results for '{unquote(value)}'

\n
\n
\n \"\"\"\n return Response(xss_block, status=200)\n return Response(parameter_block, status=200, headers={\"Content-Type\": \"application/json\"})\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"lightfuzz\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n excavate_json_extraction = False\n xss_finding_emitted = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter (speculative from json content) [search]\" in e.data[\"description\"]:\n excavate_json_extraction = True\n\n if e.type == \"FINDING\":\n if \"Possible Reflected XSS. Parameter: [search] Context: [Between Tags\" in e.data[\"description\"]:\n xss_finding_emitted = True\n\n assert excavate_json_extraction, \"Excavate failed to extract json parameter\"\n assert xss_finding_emitted, \"Between Tags XSS FINDING not emitted\"\n\n\nclass Test_Lightfuzz_crypto_error(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/\"]\n modules_overrides = [\"httpx\", \"excavate\", \"lightfuzz\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\"enabled_submodules\": [\"crypto\"]},\n },\n }\n\n def request_handler(self, request):\n qs = str(request.query_string.decode())\n\n parameter_block = \"\"\"\n
\n
\n \n \n
\n
\n \"\"\"\n crypto_block = \"\"\"\n
\n

Access Denied!

\n
\n
\n \"\"\"\n if \"secret=\" in qs:\n value = qs.split(\"=\")[1]\n if value:\n return Response(crypto_block, status=200)\n\n return Response(parameter_block, status=200)\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"lightfuzz\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n cryptoerror_parameter_extracted = False\n cryptoerror_finding_emitted = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [secret] (GET Form Submodule)\" in e.data[\"description\"]:\n cryptoerror_parameter_extracted = True\n if e.type == \"FINDING\":\n if (\n \"Possible Cryptographic Error. Parameter: [secret] Parameter Type: [GETPARAM] Original Value: [08a5a2cea9c5a5576e6e5314edcba581d21c7111c9c0c06990327b9127058d67]\"\n in e.data[\"description\"]\n ):\n cryptoerror_finding_emitted = True\n assert cryptoerror_parameter_extracted, \"Parameter not extracted\"\n assert cryptoerror_finding_emitted, \"Crypto Error Message FINDING not emitted\"\n\n\nclass Test_Lightfuzz_crypto_error_falsepositive(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/\"]\n modules_overrides = [\"httpx\", \"excavate\", \"lightfuzz\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\"enabled_submodules\": [\"crypto\"]},\n },\n }\n\n def request_handler(self, request):\n fp_block = \"\"\"\n
\n
\n \n \n
\n

Access Denied!

\n
\n \"\"\"\n return Response(fp_block, status=200)\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"lightfuzz\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n cryptoerror_parameter_extracted = False\n cryptoerror_finding_emitted = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [secret] (GET Form Submodule)\" in e.data[\"description\"]:\n cryptoerror_parameter_extracted = True\n if e.type == \"FINDING\":\n if \"Possible Cryptographic Error\" in e.data[\"description\"]:\n cryptoerror_finding_emitted = True\n assert cryptoerror_parameter_extracted, \"Parameter not extracted\"\n assert not cryptoerror_finding_emitted, (\n \"Crypto Error Message FINDING was emitted (it is an intentional false positive)\"\n )\n\n\nclass Test_Lightfuzz_PaddingOracleDetection(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"excavate\", \"lightfuzz\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"crypto\"],\n }\n },\n }\n\n def request_handler(self, request):\n encrypted_value = quote(\n \"dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q+4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg==\"\n )\n default_html_response = f\"\"\"\n \n \n
\n \n \n
\n \n \n \"\"\"\n\n if \"/decrypt\" in request.url and request.method == \"POST\":\n if request.form and request.form[\"encrypted_data\"]:\n encrypted_data = request.form[\"encrypted_data\"]\n if \"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALwAgLKWJi2nWKbh9ag5rnhm\" in encrypted_data:\n response_content = \"Padding error detected\"\n elif \"4GXVGZbo0DTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg\" in encrypted_data:\n response_content = \"DIFFERENT CRYPTOGRAPHIC ERROR\"\n elif \"AAAAAAA\" in encrypted_data:\n response_content = \"YET DIFFERENT CRYPTOGRAPHIC ERROR\"\n else:\n response_content = \"Decryption failed\"\n\n return Response(response_content, status=200)\n else:\n return Response(default_html_response, status=200)\n\n async def setup_after_prep(self, module_test):\n module_test.set_expect_requests_handler(expect_args=re.compile(\".*\"), request_handler=self.request_handler)\n\n def check(self, module_test, events):\n web_parameter_extracted = False\n cryptographic_parameter_finding = False\n padding_oracle_detected = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [encrypted_data] (POST Form\" in e.data[\"description\"]:\n web_parameter_extracted = True\n if e.type == \"FINDING\":\n if (\n e.data[\"description\"]\n == \"Probable Cryptographic Parameter. Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Detection Technique(s): [Single-byte Mutation] Envelopes: [URL-Encoded]\"\n ):\n cryptographic_parameter_finding = True\n\n if e.type == \"VULNERABILITY\":\n if (\n e.data[\"description\"]\n == \"Padding Oracle Vulnerability. Block size: [16] Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Envelopes: [URL-Encoded]\"\n ):\n padding_oracle_detected = True\n\n assert web_parameter_extracted, \"Web parameter was not extracted\"\n assert cryptographic_parameter_finding, \"Cryptographic parameter not detected\"\n assert padding_oracle_detected, \"Padding oracle vulnerability was not detected\"\n\n\nclass Test_Lightfuzz_PaddingOracleDetection_Reflecting(Test_Lightfuzz_PaddingOracleDetection):\n \"\"\"Padding oracle test where the server reflects the submitted value in the response body.\n Without reflection-stripping logic, every probe body differs and detection always fails.\"\"\"\n\n def request_handler(self, request):\n encrypted_value = quote(\n \"dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q+4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg==\"\n )\n default_html_response = f\"\"\"\n \n \n
\n \n \n
\n \n \n \"\"\"\n\n if \"/decrypt\" in request.url and request.method == \"POST\":\n if request.form and request.form[\"encrypted_data\"]:\n encrypted_data = request.form[\"encrypted_data\"]\n if \"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALwAgLKWJi2nWKbh9ag5rnhm\" in encrypted_data:\n response_content = f\"Padding error detected. Input: {encrypted_data}\"\n elif \"4GXVGZbo0DTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg\" in encrypted_data:\n response_content = f\"DIFFERENT CRYPTOGRAPHIC ERROR. Input: {encrypted_data}\"\n elif \"AAAAAAA\" in encrypted_data:\n response_content = f\"YET DIFFERENT CRYPTOGRAPHIC ERROR. Input: {encrypted_data}\"\n else:\n response_content = f\"Decryption failed. Input: {encrypted_data}\"\n\n return Response(response_content, status=200)\n else:\n return Response(default_html_response, status=200)\n\n def check(self, module_test, events):\n web_parameter_extracted = False\n cryptographic_parameter_finding = False\n padding_oracle_detected = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [encrypted_data] (POST Form\" in e.data[\"description\"]:\n web_parameter_extracted = True\n if e.type == \"FINDING\":\n if (\n \"Probable Cryptographic Parameter.\" in e.data[\"description\"]\n and \"encrypted_data\" in e.data[\"description\"]\n ):\n cryptographic_parameter_finding = True\n\n if e.type == \"VULNERABILITY\":\n if (\n \"Padding Oracle Vulnerability. Block size: [16]\" in e.data[\"description\"]\n and \"encrypted_data\" in e.data[\"description\"]\n ):\n padding_oracle_detected = True\n\n assert web_parameter_extracted, \"Web parameter was not extracted\"\n assert cryptographic_parameter_finding, \"Cryptographic parameter not detected\"\n assert padding_oracle_detected, \"Padding oracle vulnerability was not detected\"\n\n\nclass Test_Lightfuzz_PaddingOracleDetection_Noisy(Test_Lightfuzz_PaddingOracleDetection):\n \"\"\"Padding oracle negative test: the server returns different responses for ~30 byte values,\n which exceeds any valid block size. This should NOT produce a VULNERABILITY.\"\"\"\n\n def request_handler(self, request):\n encrypted_value = quote(\n \"dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q+4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg==\"\n )\n default_html_response = f\"\"\"\n \n \n
\n \n \n
\n \n \n \"\"\"\n\n if \"/decrypt\" in request.url and request.method == \"POST\":\n if request.form and request.form[\"encrypted_data\"]:\n encrypted_data = request.form[\"encrypted_data\"]\n # Check for the data block from the original ciphertext (mutate/truncate probes)\n if \"4GXVGZbo0DTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg\" in encrypted_data:\n response_content = \"DIFFERENT CRYPTOGRAPHIC ERROR\"\n # Padding oracle probes: null IV + padding blocks produce long runs of A's in base64\n elif encrypted_data.startswith(\"AAAAAAAAAAAAAAAA\"):\n try:\n decoded = base64.b64decode(encrypted_data)\n if len(decoded) >= 32:\n varying_byte = decoded[31]\n # 30 byte values produce a different response - way over any block size\n if 100 <= varying_byte <= 129:\n response_content = \"Noisy error type A\"\n else:\n response_content = \"Decryption failed\"\n else:\n response_content = \"Decryption failed\"\n except Exception:\n response_content = \"Decryption failed\"\n # Arbitrary probe\n elif \"AAAAAAA\" in encrypted_data:\n response_content = \"YET DIFFERENT CRYPTOGRAPHIC ERROR\"\n else:\n response_content = \"Decryption failed\"\n\n return Response(response_content, status=200)\n else:\n return Response(default_html_response, status=200)\n\n def check(self, module_test, events):\n web_parameter_extracted = False\n cryptographic_parameter_finding = False\n padding_oracle_detected = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [encrypted_data] (POST Form\" in e.data[\"description\"]:\n web_parameter_extracted = True\n if e.type == \"FINDING\":\n if (\n \"Probable Cryptographic Parameter.\" in e.data[\"description\"]\n and \"encrypted_data\" in e.data[\"description\"]\n ):\n cryptographic_parameter_finding = True\n if e.type == \"VULNERABILITY\":\n if \"Padding Oracle\" in e.data[\"description\"]:\n padding_oracle_detected = True\n\n assert web_parameter_extracted, \"Web parameter was not extracted\"\n assert cryptographic_parameter_finding, \"Cryptographic parameter not detected\"\n assert not padding_oracle_detected, (\n \"Padding oracle should NOT be detected when 30 probes differ (exceeds block size)\"\n )\n\n\nclass Test_Lightfuzz_XSS_jsquotecontext(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"lightfuzz\", \"excavate\", \"paramminer_getparams\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\"enabled_submodules\": [\"xss\"]},\n \"paramminer_getparams\": {\"wordlist\": tempwordlist([\"junk\", \"input\"]), \"recycle_words\": True},\n },\n }\n\n def request_handler(self, request):\n # Decode the query string\n qs = str(request.query_string.decode())\n default_output = \"\"\"\n \n
\n \n \n
\n \n \"\"\"\n\n if \"input=\" in qs:\n # Split the query string to isolate the 'input' parameter\n params = qs.split(\"&\")\n input_value = None\n for param in params:\n if param.startswith(\"input=\"):\n input_value = param.split(\"=\")[1]\n break\n\n if input_value is not None:\n # Simulate flawed escaping\n sanitized_input = input_value.replace('\"', '\\\\\"').replace(\"'\", \"\\\\'\")\n sanitized_input = sanitized_input.replace(\"<\", \"%3C\").replace(\">\", \"%3E\")\n\n # Construct the reflected block with the sanitized input\n reflected_block = f\"\"\"\n \n \n \n \"\"\"\n return Response(reflected_block, status=200)\n\n return Response(default_output, status=200)\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"paramminer_getparams\"].rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n module_test.monkeypatch.setattr(\n helper.HttpCompare, \"gen_cache_buster\", lambda *args, **kwargs: {\"AAAAAA\": \"1\"}\n )\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n xss_finding_emitted = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"[Paramminer] Getparam: [input] Reasons: [body] Reflection: [True]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n\n if e.type == \"FINDING\":\n if \"Possible Reflected XSS. Parameter: [input] Context: [In Javascript (escaping the escape character, single quote)] Parameter Type: [GETPARAM]\":\n xss_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER for was not emitted\"\n assert xss_finding_emitted, \"XSS FINDING not emitted\"\n\n\nclass Test_Lightfuzz_XSS_jsquotecontext_doublequote(Test_Lightfuzz_XSS_jsquotecontext):\n def request_handler(self, request):\n qs = str(request.query_string.decode())\n default_output = \"\"\"\n \n
\n \n \n
\n \n \"\"\"\n\n if \"input=\" in qs:\n params = qs.split(\"&\")\n input_value = None\n for param in params:\n if param.startswith(\"input=\"):\n input_value = param.split(\"=\")[1]\n break\n\n if input_value is not None:\n # Simulate flawed escaping with opposite quotes\n sanitized_input = input_value.replace(\"'\", \"\\\\\").replace(\"%22\", '\\\\\"')\n sanitized_input = sanitized_input.replace(\"<\", \"%3C\").replace(\">\", \"%3E\")\n\n reflected_block = f\"\"\"\n \n \n \n \"\"\"\n return Response(reflected_block, status=200)\n\n return Response(default_output, status=200)\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n xss_finding_emitted = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"[Paramminer] Getparam: [input] Reasons: [body] Reflection: [True]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n\n if e.type == \"FINDING\":\n if \"Possible Reflected XSS. Parameter: [input] Context: [In Javascript (escaping the escape character, double quote)] Parameter Type: [GETPARAM]\":\n xss_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER for was not emitted\"\n assert xss_finding_emitted, \"XSS FINDING not emitted\"\n\n\nclass Test_Lightfuzz_esi(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"lightfuzz\", \"excavate\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"esi\"],\n }\n },\n }\n\n def request_handler(self, request):\n qs = str(request.query_string.decode())\n\n parameter_block = \"\"\"\n
\n
\n \n \n
\n
\n \"\"\"\n if \"search=\" in qs:\n value = qs.split(\"=\")[1]\n if \"&\" in value:\n value = value.split(\"&\")[0]\n # Decode the URL-encoded value\n decoded_value = unquote(value)\n # Simulate ESI processing: if the payload contains , remove it\n if \"\" in decoded_value:\n # ESI processor removes tag, leaving the rest\n processed_value = decoded_value.replace(\"\", \"\")\n else:\n # For non-ESI payloads, just reflect the value as-is\n processed_value = decoded_value\n\n esi_block = f\"\"\"\n
\n

Search results for '{processed_value}'

\n
\n
\n \"\"\"\n return Response(esi_block, status=200)\n\n return Response(parameter_block, status=200)\n\n async def setup_after_prep(self, module_test):\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n esi_finding_emitted = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [search]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n\n if e.type == \"FINDING\":\n if \"Edge Side Include. Parameter: [search] Parameter Type: [GETPARAM]\" in e.data[\"description\"]:\n esi_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER was not emitted\"\n assert esi_finding_emitted, \"ESI FINDING not emitted\"\n\n\n# Envelope state isolation: crypto error detection with all submodules enabled.\n# Crypto runs after sqli/cmdi/xss/path/ssti. Each prior submodule calls outgoing_probe_value()\n# which must not corrupt the envelope state that crypto reads via incoming_probe_value().\nclass Test_Lightfuzz_envelope_isolation_crypto(Test_Lightfuzz_crypto_error):\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"sqli\", \"cmdi\", \"xss\", \"path\", \"ssti\", \"crypto\", \"serial\", \"esi\"],\n }\n },\n }\n\n\n# Envelope state isolation: padding oracle detection with all submodules enabled.\nclass Test_Lightfuzz_envelope_isolation_paddingoracle(Test_Lightfuzz_PaddingOracleDetection):\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"sqli\", \"cmdi\", \"xss\", \"path\", \"ssti\", \"crypto\", \"serial\", \"esi\"],\n }\n },\n }\n\n\n# Envelope state isolation: reflecting padding oracle detection with all submodules enabled.\nclass Test_Lightfuzz_envelope_isolation_paddingoracle_reflecting(Test_Lightfuzz_PaddingOracleDetection_Reflecting):\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"sqli\", \"cmdi\", \"xss\", \"path\", \"ssti\", \"crypto\", \"serial\", \"esi\"],\n }\n },\n }\n\n\n# Test filter_event method with WAF tags\nclass Test_Lightfuzz_filter_event(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"lightfuzz\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"xss\"],\n \"avoid_wafs\": True,\n }\n },\n }\n\n async def setup_after_prep(self, module_test):\n # Create test events with WAF tags\n self.url_event_with_waf = module_test.scan.make_event(\n \"http://127.0.0.1:8888/\",\n \"URL\",\n module_test.scan.root_event,\n module=\"httpx\",\n tags=[\"status-200\", \"distance-0\", \"waf\"],\n )\n\n self.web_param_event_with_waf = module_test.scan.make_event(\n {\n \"host\": \"127.0.0.1\",\n \"type\": \"GETPARAM\",\n \"name\": \"test\",\n \"original_value\": \"value\",\n \"url\": \"http://127.0.0.1:8888/\",\n \"description\": \"Test parameter\",\n },\n \"WEB_PARAMETER\",\n module_test.scan.root_event,\n module=\"excavate\",\n tags=[\"distance-0\", \"waf\"],\n )\n\n self.url_event_without_waf = module_test.scan.make_event(\n \"http://127.0.0.1:8888/\",\n \"URL\",\n module_test.scan.root_event,\n module=\"httpx\",\n tags=[\"status-200\", \"distance-0\"],\n )\n\n self.web_param_event_without_waf = module_test.scan.make_event(\n {\n \"host\": \"127.0.0.1\",\n \"type\": \"GETPARAM\",\n \"name\": \"test\",\n \"original_value\": \"value\",\n \"url\": \"http://127.0.0.1:8888/\",\n \"description\": \"Test parameter\",\n },\n \"WEB_PARAMETER\",\n module_test.scan.root_event,\n module=\"excavate\",\n tags=[\"distance-0\"],\n )\n\n async def test_filter_event(self, module_test):\n lightfuzz_module = module_test.scan.modules[\"lightfuzz\"]\n\n # Test URL event with WAF tag - should be filtered out\n result = await lightfuzz_module.filter_event(self.url_event_with_waf)\n assert result is False, \"URL event with waf tag should be filtered out\"\n\n # Test WEB_PARAMETER event with WAF tag - should be filtered out\n result = await lightfuzz_module.filter_event(self.web_param_event_with_waf)\n assert result is False, \"WEB_PARAMETER event with waf tag should be filtered out\"\n\n # Test URL event without WAF tag - should not be filtered\n result = await lightfuzz_module.filter_event(self.url_event_without_waf)\n assert result is True, \"URL event without WAF tag should not be filtered\"\n\n # Test WEB_PARAMETER event without WAF tag - should not be filtered\n result = await lightfuzz_module.filter_event(self.web_param_event_without_waf)\n assert result is True, \"WEB_PARAMETER event without WAF tag should not be filtered\"\n\n def check(self, module_test, events):\n # This test doesn't need to check events since it's testing the filter method directly\n pass\n\n\n# try_post_as_get: fuzz POST parameters as GET parameters\nclass Test_Lightfuzz_try_post_as_get(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"lightfuzz\", \"excavate\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"sqli\"],\n \"disable_post\": True,\n \"try_post_as_get\": True,\n }\n },\n }\n\n def request_handler(self, request):\n qs = str(request.query_string.decode())\n\n parameter_block = \"\"\"\n
\n
\n \n \n
\n
\n \"\"\"\n\n if \"search=\" in qs:\n value = qs.split(\"=\")[1]\n if \"&\" in value:\n value = value.split(\"&\")[0]\n\n sql_block_normal = f\"\"\"\n
\n

0 search results for '{unquote(value)}'

\n
\n
\n \"\"\"\n\n sql_block_error = \"\"\"\n
\n

Found error in SQL query

\n
\n
\n \"\"\"\n if value.endswith(\"'\"):\n if value.endswith(\"''\"):\n return Response(sql_block_normal, status=200)\n return Response(sql_block_error, status=500)\n return Response(parameter_block, status=200)\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"lightfuzz\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n sqli_getparam_finding_emitted = False\n sqli_postparam_finding_emitted = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [search]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n\n if e.type == \"FINDING\":\n if (\n \"Possible SQL Injection. Parameter: [search] Parameter Type: [GETPARAM] (converted from POSTPARAM) Detection Method: [Single Quote/Two Single Quote, Code Change (200->500->200)]\"\n in e.data[\"description\"]\n ):\n sqli_getparam_finding_emitted = True\n if \"Possible SQL Injection. Parameter: [search] Parameter Type: [POSTPARAM]\" in e.data[\"description\"]:\n sqli_postparam_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER was not emitted\"\n assert sqli_getparam_finding_emitted, (\n \"SQLi GETPARAM (converted from POSTPARAM) FINDING not emitted (try_post_as_get failed)\"\n )\n assert not sqli_postparam_finding_emitted, \"POSTPARAM FINDING emitted despite disable_post=True\"\n\n\n# try_get_as_post: fuzz GET parameters as POST parameters\nclass Test_Lightfuzz_try_get_as_post(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"lightfuzz\", \"excavate\"]\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\n \"lightfuzz\": {\n \"enabled_submodules\": [\"sqli\"],\n \"try_get_as_post\": True,\n }\n },\n }\n\n def request_handler(self, request):\n parameter_block = \"\"\"\n
\n
\n \n \n
\n
\n \"\"\"\n\n if request.method == \"POST\" and \"search\" in request.form.keys():\n value = request.form[\"search\"]\n\n sql_block_normal = f\"\"\"\n
\n

0 search results for '{unquote(value)}'

\n
\n
\n \"\"\"\n\n sql_block_error = \"\"\"\n
\n

Found error in SQL query

\n
\n
\n \"\"\"\n if value.endswith(\"'\"):\n if value.endswith(\"''\"):\n return Response(sql_block_normal, status=200)\n return Response(sql_block_error, status=500)\n return Response(parameter_block, status=200)\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"lightfuzz\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n web_parameter_emitted = False\n sqli_postparam_converted_finding_emitted = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [search]\" in e.data[\"description\"]:\n web_parameter_emitted = True\n\n if e.type == \"FINDING\":\n if (\n \"Possible SQL Injection. Parameter: [search] Parameter Type: [POSTPARAM] (converted from GETPARAM) Detection Method: [Single Quote/Two Single Quote, Code Change (200->500->200)]\"\n in e.data[\"description\"]\n ):\n sqli_postparam_converted_finding_emitted = True\n\n assert web_parameter_emitted, \"WEB_PARAMETER was not emitted\"\n assert sqli_postparam_converted_finding_emitted, (\n \"SQLi POSTPARAM (converted from GETPARAM) FINDING not emitted (try_get_as_post failed)\"\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_medusa.py", "content": "from .base import ModuleTestBase, tempwordlist\nimport pytest\n\n\n@pytest.fixture\ndef mock_medusa_run_process(monkeypatch):\n async def fake_run_process(self, cmd):\n class FakeResult:\n stdout = \"ACCOUNT FOUND: [snmp] Host: 127.0.0.1 User: (null) Password: public [ERROR]\\n\"\n stderr = (\n \"ERROR: [snmp.mod] Error processing SNMP response (1).\\n\"\n \"ERROR: [snmp.mod] Community string appears to have only READ access.\\n\"\n )\n\n return FakeResult()\n\n from bbot.modules.base import BaseModule\n\n monkeypatch.setattr(BaseModule, \"run_process\", fake_run_process)\n\n\n@pytest.mark.usefixtures(\"mock_medusa_run_process\")\nclass TestMedusa(ModuleTestBase):\n targets = [\"127.0.0.1\"]\n temp_snmp_wordlist = tempwordlist([\"public\", \"private, admin\"])\n config_overrides = {\n \"modules\": {\n \"medusa\": {\n \"snmp_versions\": [\"2C\"],\n \"timeout_s\": 1,\n \"snmp_wordlist\": str(temp_snmp_wordlist),\n }\n }\n }\n\n async def setup_after_prep(self, module_test):\n protocol_data = {\"host\": str(self.targets[0]), \"protocol\": \"snmp\", \"port\": 161}\n\n protocol_event = module_test.scan.make_event(\n protocol_data,\n \"PROTOCOL\",\n parent=module_test.scan.root_event,\n )\n await module_test.module.emit_event(protocol_event)\n\n def check(self, module_test, events):\n vuln_events = [e for e in events if e.type == \"VULNERABILITY\"]\n\n assert len(vuln_events) == 1\n assert \"VALID [SNMPV2C] CREDENTIALS FOUND: public [READ]\" in vuln_events[0].data[\"description\"]\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_mysql.py", "content": "import asyncio\nimport time\n\nfrom .base import ModuleTestBase\n\n\nclass TestMySQL(ModuleTestBase):\n targets = [\"evilcorp.com\"]\n skip_distro_tests = True\n\n async def setup_before_prep(self, module_test):\n process = await asyncio.create_subprocess_exec(\n \"docker\",\n \"run\",\n \"--name\",\n \"bbot-test-mysql\",\n \"--rm\",\n \"-e\",\n \"MYSQL_ROOT_PASSWORD=bbotislife\",\n \"-e\",\n \"MYSQL_DATABASE=bbot\",\n \"-p\",\n \"3306:3306\",\n \"-d\",\n \"mysql\",\n stdout=asyncio.subprocess.PIPE,\n stderr=asyncio.subprocess.PIPE,\n )\n stdout, stderr = await process.communicate()\n\n import aiomysql\n\n # wait for the container to start\n start_time = time.time()\n while True:\n try:\n conn = await aiomysql.connect(user=\"root\", password=\"bbotislife\", db=\"bbot\", host=\"localhost\")\n conn.close()\n break\n except Exception as e:\n if time.time() - start_time > 60: # timeout after 60 seconds\n self.log.error(\"MySQL server did not start in time.\")\n raise e\n await asyncio.sleep(1)\n\n if process.returncode != 0:\n self.log.error(f\"Failed to start MySQL server: {stderr.decode()}\")\n\n async def check(self, module_test, events):\n import aiomysql\n\n # Connect to the MySQL database\n conn = await aiomysql.connect(user=\"root\", password=\"bbotislife\", db=\"bbot\", host=\"localhost\")\n\n try:\n async with conn.cursor() as cur:\n await cur.execute(\"SELECT * FROM event\")\n events = await cur.fetchall()\n assert len(events) == 3, \"No events found in MySQL database\"\n\n await cur.execute(\"SELECT * FROM scan\")\n scans = await cur.fetchall()\n assert len(scans) == 1, \"No scans found in MySQL database\"\n\n await cur.execute(\"SELECT * FROM target\")\n targets = await cur.fetchall()\n assert len(targets) == 1, \"No targets found in MySQL database\"\n finally:\n conn.close()\n process = await asyncio.create_subprocess_exec(\n \"docker\", \"stop\", \"bbot-test-mysql\", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE\n )\n stdout, stderr = await process.communicate()\n\n if process.returncode != 0:\n raise Exception(f\"Failed to stop MySQL server: {stderr.decode()}\")\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_myssl.py", "content": "from .base import ModuleTestBase\n\n\nclass TestMySSL(ModuleTestBase):\n async def setup_after_prep(self, module_test):\n module_test.module.abort_if = lambda e: False\n module_test.httpx_mock.add_response(\n url=\"https://myssl.com/api/v1/discover_sub_domain?domain=blacklanternsecurity.com\",\n json={\n \"code\": 0,\n \"data\": [\n {\n \"ip\": \"1.2.3.4\",\n \"port\": \"443\",\n \"tips\": [],\n \"level\": 2,\n \"title\": \"\",\n \"domain\": \"asdf.blacklanternsecurity.com\",\n \"is_ats\": True,\n \"is_pci\": False,\n \"server\": \"\",\n \"is_tlcp\": False,\n \"duration\": 46,\n \"icon_url\": \"\",\n \"is_sslvpn\": False,\n \"level_str\": \"A\",\n \"ip_location\": \"įžŽå›Ŋ\",\n \"is_enable_gm\": False,\n \"evaluate_date\": \"2022-03-13T02:38:08Z\",\n \"demotion_reason\": [],\n \"ignore_trust_level\": \"A\",\n \"meet_gm_double_cert_statndard\": False,\n }\n ],\n },\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_neo4j.py", "content": "from .base import ModuleTestBase\n\n\nclass TestNeo4j(ModuleTestBase):\n config_overrides = {\"modules\": {\"neo4j\": {\"uri\": \"bolt://127.0.0.1:11111\"}}}\n\n async def setup_before_prep(self, module_test):\n # install neo4j\n deps_pip = module_test.preloaded[\"neo4j\"][\"deps\"][\"pip\"]\n await module_test.scan.helpers.depsinstaller.pip_install(deps_pip)\n\n self.neo4j_used = False\n\n class MockResult:\n async def data(s):\n self.neo4j_used = True\n return [\n {\n \"neo4j_id\": \"4:ee79a477-5f5b-445a-9def-7c051b2a533c:115\",\n \"event_id\": \"DNS_NAME:c8fab50640cb87f8712d1998ecc78caf92b90f71\",\n }\n ]\n\n class MockSession:\n async def run(s, *args, **kwargs):\n return MockResult()\n\n async def close(self):\n pass\n\n class MockDriver:\n def __init__(self, *args, **kwargs):\n pass\n\n def session(self, *args, **kwargs):\n return MockSession()\n\n async def close(self):\n pass\n\n module_test.monkeypatch.setattr(\"neo4j.AsyncGraphDatabase.driver\", MockDriver)\n\n def check(self, module_test, events):\n assert self.neo4j_used is True\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_newsletters.py", "content": "from .base import ModuleTestBase\n\n# import logging\n\n\nclass TestNewsletters(ModuleTestBase):\n found_tgt = \"http://127.0.0.1:8888/found\"\n missing_tgt = \"http://127.0.0.1:8888/missing\"\n targets = [found_tgt, missing_tgt]\n modules_overrides = [\"speculate\", \"httpx\", \"newsletters\"]\n\n html_with_newsletter = \"\"\"\n \n \"\"\"\n\n html_without_newsletter = \"\"\"\n
\n

Example Domain

\n

This domain is for use in illustrative examples in documents. You may use this\n domain in literature without prior coordination or asking for permission.

\n

More information...

\n
\n \"\"\"\n\n async def setup_after_prep(self, module_test):\n request_args = {\"uri\": \"/found\", \"headers\": {\"test\": \"header\"}}\n respond_args = {\"response_data\": self.html_with_newsletter}\n module_test.set_expect_requests(request_args, respond_args)\n request_args = {\"uri\": \"/missing\", \"headers\": {\"test\": \"header\"}}\n respond_args = {\"response_data\": self.html_without_newsletter}\n module_test.set_expect_requests(request_args, respond_args)\n\n def check(self, module_test, events):\n found = False\n missing = True\n for event in events:\n # self.log.info(f\"event type: {event.type}\")\n if event.type == \"FINDING\":\n # self.log.info(f\"event data: {event.data}\")\n # Verify Positive Result\n if event.data[\"url\"] == self.found_tgt:\n found = True\n # Verify Negative Result (should skip this statement if correct)\n elif event.data[\"url\"] == self.missing_tgt:\n missing = False\n assert found, \"NEWSLETTER 'Found' Error - Expect status of True but got False\"\n assert missing, \"NEWSLETTER 'Missing' Error - Expect status of True but got False\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_nmap_xml.py", "content": "import xml.etree.ElementTree as ET\n\nfrom bbot.modules.base import BaseModule\nfrom .base import ModuleTestBase\n\n\nclass TestNmap_XML(ModuleTestBase):\n modules_overrides = [\"nmap_xml\", \"speculate\"]\n targets = [\"blacklanternsecurity.com\", \"127.0.0.3\"]\n config_overrides = {\"dns\": {\"minimal\": False}}\n\n class DummyModule(BaseModule):\n watched_events = [\"OPEN_TCP_PORT\"]\n _name = \"dummy_module\"\n\n async def handle_event(self, event):\n if event.port == 80:\n await self.emit_event(\n {\"host\": str(event.host), \"port\": event.port, \"protocol\": \"http\", \"banner\": \"Apache\"},\n \"PROTOCOL\",\n parent=event,\n )\n elif event.port == 443:\n await self.emit_event(\n {\"host\": str(event.host), \"port\": event.port, \"protocol\": \"https\"}, \"PROTOCOL\", parent=event\n )\n\n async def setup_before_prep(self, module_test):\n self.dummy_module = self.DummyModule(module_test.scan)\n module_test.scan.modules[\"dummy_module\"] = self.dummy_module\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.com\": {\"A\": [\"127.0.0.1\", \"127.0.0.2\"]},\n \"3.0.0.127.in-addr.arpa\": {\"PTR\": [\"www.blacklanternsecurity.com\"]},\n \"www.blacklanternsecurity.com\": {\"A\": [\"127.0.0.1\"]},\n }\n )\n\n def check(self, module_test, events):\n nmap_xml_file = module_test.scan.modules[\"nmap_xml\"].output_file\n nmap_xml = open(nmap_xml_file).read()\n\n # Parse the XML\n root = ET.fromstring(nmap_xml)\n\n # Expected IP addresses\n expected_ips = {\"127.0.0.1\", \"127.0.0.2\", \"127.0.0.3\"}\n found_ips = set()\n\n # Iterate over each host in the XML\n for host in root.findall(\"host\"):\n # Get the IP address\n address = host.find(\"address\").get(\"addr\")\n found_ips.add(address)\n\n # Get hostnames if available\n hostnames = sorted([hostname.get(\"name\") for hostname in host.findall(\".//hostname\")])\n\n # Get open ports and services\n ports = []\n for port in host.findall(\".//port\"):\n port_id = port.get(\"portid\")\n state = port.find(\"state\").get(\"state\")\n if state == \"open\":\n service_name = port.find(\"service\").get(\"name\")\n service_product = port.find(\"service\").get(\"product\", \"\")\n service_extrainfo = port.find(\"service\").get(\"extrainfo\", \"\")\n ports.append((port_id, service_name, service_product, service_extrainfo))\n\n # Sort ports for consistency\n ports.sort()\n\n # Assertions\n if address == \"127.0.0.1\":\n assert hostnames == [\"blacklanternsecurity.com\", \"www.blacklanternsecurity.com\"]\n assert ports == sorted([(\"80\", \"http\", \"Apache\", \"Apache\"), (\"443\", \"https\", \"\", \"\")])\n elif address == \"127.0.0.2\":\n assert hostnames == sorted([\"blacklanternsecurity.com\"])\n assert ports == sorted([(\"80\", \"http\", \"Apache\", \"Apache\"), (\"443\", \"https\", \"\", \"\")])\n elif address == \"127.0.0.3\":\n assert hostnames == [] # No hostnames for this IP\n assert ports == sorted([(\"80\", \"http\", \"Apache\", \"Apache\"), (\"443\", \"https\", \"\", \"\")])\n\n # Assert that all expected IPs were found\n assert found_ips == expected_ips\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_ntlm.py", "content": "from .base import ModuleTestBase\n\n\nclass TestNTLM(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"ntlm\"]\n config_overrides = {\"modules\": {\"ntlm\": {\"try_all\": True}}}\n\n async def setup_after_prep(self, module_test):\n request_args = {\"uri\": \"/\", \"headers\": {\"test\": \"header\"}}\n module_test.set_expect_requests(request_args, {})\n request_args = {\n \"uri\": \"/oab/\",\n \"headers\": {\"Authorization\": \"NTLM TlRMTVNTUAABAAAAl4II4gAAAAAAAAAAAAAAAAAAAAAKAGFKAAAADw==\"},\n }\n respond_args = {\n \"headers\": {\n \"WWW-Authenticate\": \"NTLM TlRMTVNTUAACAAAABgAGADgAAAAVgoni89aZT4Q0mH0AAAAAAAAAAHYAdgA+AAAABgGxHQAAAA9WAE4ATwACAAYAVgBOAE8AAQAKAEUAWABDADAAMQAEABIAdgBuAG8ALgBsAG8AYwBhAGwAAwAeAEUAWABDADAAMQAuAHYAbgBvAC4AbABvAGMAYQBsAAUAEgB2AG4AbwAuAGwAbwBjAGEAbAAHAAgAXxo0p/6L2QEAAAAA\"\n }\n }\n module_test.set_expect_requests(request_args, respond_args)\n\n def check(self, module_test, events):\n assert any(e.type == \"FINDING\" and \"EXC01.vno.local\" in e.data[\"description\"] for e in events)\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_nuclei.py", "content": "from ...bbot_fixtures import *\nfrom .base import ModuleTestBase\n\n\nclass TestNucleiManual(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"excavate\", \"nuclei\"]\n config_overrides = {\n \"web\": {\n \"spider_distance\": 1,\n \"spider_depth\": 1,\n },\n \"modules\": {\n \"nuclei\": {\n \"mode\": \"manual\",\n \"concurrency\": 2,\n \"ratelimit\": 10,\n \"templates\": \"/tmp/.bbot_test/tools/nuclei-templates/http/miscellaneous/\",\n \"interactsh_disable\": True,\n \"directory_only\": False,\n }\n },\n }\n\n test_html = \"\"\"\n html>\n \n Index of /test\n \n \n

Index of /test

\n \n \n \n\n
NameLast modifiedSize
Parent Directory  -
\n
Apache/2.4.38 (Debian) Server at http://127.0.0.1:8888/testmultipleruns.html
\n\n\"\"\"\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": self.test_html}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n expect_args = {\"method\": \"GET\", \"uri\": \"/testmultipleruns.html\"}\n respond_args = {\"response_data\": \"Copyright 1984\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n first_run_detect = False\n second_run_detect = False\n for e in events:\n if e.type == \"FINDING\":\n if \"Directory listing enabled\" in e.data[\"description\"]:\n first_run_detect = True\n elif \"Copyright\" in e.data[\"description\"]:\n second_run_detect = True\n assert first_run_detect\n assert second_run_detect\n\n\nclass TestNucleiSevere(TestNucleiManual):\n modules_overrides = [\"httpx\", \"nuclei\"]\n config_overrides = {\n \"modules\": {\n \"nuclei\": {\n \"mode\": \"severe\",\n \"concurrency\": 1,\n \"templates\": \"/tmp/.bbot_test/tools/nuclei-templates/http/vulnerabilities/generic/generic-env.yaml\",\n }\n },\n \"interactsh_disable\": True,\n }\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/.env\"}\n respond_args = {\"response_data\": \"AAAKEYBBB=\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": \"alive\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n assert any(\n e.type == \"VULNERABILITY\" and \"Generic Env File Disclosure\" in e.data[\"description\"] for e in events\n )\n\n\nclass TestNucleiTechnology(TestNucleiManual):\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\"nuclei\": {\"mode\": \"technology\", \"concurrency\": 2, \"tags\": \"apache\"}},\n }\n\n async def setup_before_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\n \"response_data\": \"\",\n \"headers\": {\"Server\": \"Apache/2.4.52 (Ubuntu)\"},\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n assert any(e.type == \"TECHNOLOGY\" and \"apache\" in e.data[\"technology\"].lower() for e in events)\n assert \"Using Interactsh Server\" not in open(module_test.scan.home / \"debug.log\").read()\n\n\nclass TestNucleiBudget(TestNucleiManual):\n config_overrides = {\n \"modules\": {\n \"nuclei\": {\n \"mode\": \"budget\",\n \"concurrency\": 1,\n \"tags\": \"spiderfoot\",\n \"templates\": \"/tmp/.bbot_test/tools/nuclei-templates/exposed-panels/spiderfoot.yaml\",\n \"interactsh_disable\": True,\n }\n }\n }\n\n async def setup_before_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": \"SpiderFoot

support@spiderfoot.net

\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n assert any(e.type == \"TECHNOLOGY\" and \"spider\" in e.data[\"technology\"] for e in events)\n\n\nclass TestNucleiRetries(TestNucleiManual):\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\"nuclei\": {\"tags\": \"musictraveler\"}},\n }\n\n async def setup_before_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\n \"response_data\": \"content\",\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n assert \"-retries 0\" in open(module_test.scan.home / \"debug.log\").read()\n\n\nclass TestNucleiRetriesCustom(TestNucleiRetries):\n config_overrides = {\n \"interactsh_disable\": True,\n \"modules\": {\"nuclei\": {\"tags\": \"musictraveler\", \"retries\": 1}},\n }\n\n def check(self, module_test, events):\n assert \"-retries 1\" in open(module_test.scan.home / \"debug.log\").read()\n\n\nclass TestNucleiCustomHeaders(TestNucleiManual):\n custom_headers = {\"testheader1\": \"test1\", \"testheader2\": \"test2\"}\n config_overrides = TestNucleiManual.config_overrides\n config_overrides[\"web\"][\"http_headers\"] = custom_headers\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\", \"headers\": self.custom_headers}\n respond_args = {\"response_data\": self.test_html}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n expect_args = {\"method\": \"GET\", \"uri\": \"/testmultipleruns.html\", \"headers\": {\"nonexistent\": \"nope\"}}\n respond_args = {\"response_data\": \"Copyright 1984\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n first_run_detect = False\n second_run_detect = False\n for e in events:\n if e.type == \"FINDING\":\n if \"Directory listing enabled\" in e.data[\"description\"]:\n first_run_detect = True\n elif \"Copyright\" in e.data[\"description\"]:\n second_run_detect = True\n # we should find the first one because it requires our custom headers\n assert first_run_detect\n # the second one requires different headers, so we shouldn't find it\n assert not second_run_detect\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_oauth.py", "content": "from .base import ModuleTestBase\n\nfrom .test_module_azure_realm import TestAzure_Realm as Azure_Realm\n\n\nclass TestOAUTH(ModuleTestBase):\n targets = [\"evilcorp.com\"]\n config_overrides = {\"scope\": {\"report_distance\": 1}, \"omit_event_types\": []}\n modules_overrides = [\"azure_realm\", \"oauth\"]\n openid_config_azure = {\n \"token_endpoint\": \"https://login.windows.net/cc74fc12-4142-400e-a653-f98bdeadbeef/oauth2/token\",\n \"token_endpoint_auth_methods_supported\": [\"client_secret_post\", \"private_key_jwt\", \"client_secret_basic\"],\n \"jwks_uri\": \"https://login.windows.net/common/discovery/keys\",\n \"response_modes_supported\": [\"query\", \"fragment\", \"form_post\"],\n \"subject_types_supported\": [\"pairwise\"],\n \"id_token_signing_alg_values_supported\": [\"RS256\"],\n \"response_types_supported\": [\"code\", \"id_token\", \"code id_token\", \"token id_token\", \"token\"],\n \"scopes_supported\": [\"openid\"],\n \"issuer\": \"https://sts.windows.net/cc74fc12-4142-400e-a653-f98bdeadbeef/\",\n \"microsoft_multi_refresh_token\": True,\n \"authorization_endpoint\": \"https://login.windows.net/cc74fc12-4142-400e-a653-f98bdeadbeef/oauth2/authorize\",\n \"device_authorization_endpoint\": \"https://login.windows.net/cc74fc12-4142-400e-a653-f98bdeadbeef/oauth2/devicecode\",\n \"http_logout_supported\": True,\n \"frontchannel_logout_supported\": True,\n \"end_session_endpoint\": \"https://login.windows.net/cc74fc12-4142-400e-a653-f98bdeadbeef/oauth2/logout\",\n \"claims_supported\": [\n \"sub\",\n \"iss\",\n \"cloud_instance_name\",\n \"cloud_instance_host_name\",\n \"cloud_graph_host_name\",\n \"msgraph_host\",\n \"aud\",\n \"exp\",\n \"iat\",\n \"auth_time\",\n \"acr\",\n \"amr\",\n \"nonce\",\n \"email\",\n \"given_name\",\n \"family_name\",\n \"nickname\",\n ],\n \"check_session_iframe\": \"https://login.windows.net/cc74fc12-4142-400e-a653-f98bdeadbeef/oauth2/checksession\",\n \"userinfo_endpoint\": \"https://login.windows.net/cc74fc12-4142-400e-a653-f98bdeadbeef/openid/userinfo\",\n \"kerberos_endpoint\": \"https://login.windows.net/cc74fc12-4142-400e-a653-f98bdeadbeef/kerberos\",\n \"tenant_region_scope\": \"NA\",\n \"cloud_instance_name\": \"microsoftonline.com\",\n \"cloud_graph_host_name\": \"graph.windows.net\",\n \"msgraph_host\": \"graph.microsoft.com\",\n \"rbac_url\": \"https://pas.windows.net\",\n }\n openid_config_okta = {\n \"issuer\": \"https://evilcorp.okta.com\",\n \"authorization_endpoint\": \"https://evilcorp.okta.com/oauth2/v1/authorize\",\n \"token_endpoint\": \"https://evilcorp.okta.com/oauth2/v1/token\",\n \"userinfo_endpoint\": \"https://evilcorp.okta.com/oauth2/v1/userinfo\",\n \"registration_endpoint\": \"https://evilcorp.okta.com/oauth2/v1/clients\",\n \"jwks_uri\": \"https://evilcorp.okta.com/oauth2/v1/keys\",\n \"response_types_supported\": [\n \"code\",\n \"id_token\",\n \"code id_token\",\n \"code token\",\n \"id_token token\",\n \"code id_token token\",\n ],\n \"response_modes_supported\": [\"query\", \"fragment\", \"form_post\", \"okta_post_message\"],\n \"grant_types_supported\": [\n \"authorization_code\",\n \"implicit\",\n \"refresh_token\",\n \"password\",\n \"urn:ietf:params:oauth:grant-type:device_code\",\n \"urn:openid:params:grant-type:ciba\",\n ],\n \"subject_types_supported\": [\"public\"],\n \"id_token_signing_alg_values_supported\": [\"RS256\"],\n \"scopes_supported\": [\"openid\", \"email\", \"profile\", \"address\", \"phone\", \"offline_access\", \"groups\"],\n \"token_endpoint_auth_methods_supported\": [\n \"client_secret_basic\",\n \"client_secret_post\",\n \"client_secret_jwt\",\n \"private_key_jwt\",\n \"none\",\n ],\n \"claims_supported\": [\n \"iss\",\n \"ver\",\n \"sub\",\n \"aud\",\n \"iat\",\n \"exp\",\n \"jti\",\n \"auth_time\",\n \"amr\",\n \"idp\",\n \"nonce\",\n \"name\",\n \"nickname\",\n \"preferred_username\",\n \"given_name\",\n \"middle_name\",\n \"family_name\",\n \"email\",\n \"email_verified\",\n \"profile\",\n \"zoneinfo\",\n \"locale\",\n \"address\",\n \"phone_number\",\n \"picture\",\n \"website\",\n \"gender\",\n \"birthdate\",\n \"updated_at\",\n \"at_hash\",\n \"c_hash\",\n ],\n \"code_challenge_methods_supported\": [\"S256\"],\n \"introspection_endpoint\": \"https://evilcorp.okta.com/oauth2/v1/introspect\",\n \"introspection_endpoint_auth_methods_supported\": [\n \"client_secret_basic\",\n \"client_secret_post\",\n \"client_secret_jwt\",\n \"private_key_jwt\",\n \"none\",\n ],\n \"revocation_endpoint\": \"https://evilcorp.okta.com/oauth2/v1/revoke\",\n \"revocation_endpoint_auth_methods_supported\": [\n \"client_secret_basic\",\n \"client_secret_post\",\n \"client_secret_jwt\",\n \"private_key_jwt\",\n \"none\",\n ],\n \"end_session_endpoint\": \"https://evilcorp.okta.com/oauth2/v1/logout\",\n \"request_parameter_supported\": True,\n \"request_object_signing_alg_values_supported\": [\n \"HS256\",\n \"HS384\",\n \"HS512\",\n \"RS256\",\n \"RS384\",\n \"RS512\",\n \"ES256\",\n \"ES384\",\n \"ES512\",\n ],\n \"device_authorization_endpoint\": \"https://evilcorp.okta.com/oauth2/v1/device/authorize\",\n \"pushed_authorization_request_endpoint\": \"https://evilcorp.okta.com/oauth2/v1/par\",\n \"backchannel_token_delivery_modes_supported\": [\"poll\"],\n \"backchannel_authentication_request_signing_alg_values_supported\": [\n \"HS256\",\n \"HS384\",\n \"HS512\",\n \"RS256\",\n \"RS384\",\n \"RS512\",\n \"ES256\",\n \"ES384\",\n \"ES512\",\n ],\n }\n\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns({\"evilcorp.com\": {\"A\": [\"127.0.0.1\"]}})\n module_test.httpx_mock.add_response(\n url=\"https://login.microsoftonline.com/getuserrealm.srf?login=test@evilcorp.com\",\n json=Azure_Realm.response_json,\n )\n module_test.httpx_mock.add_response(\n url=\"https://login.windows.net/evilcorp.com/.well-known/openid-configuration\",\n json=self.openid_config_azure,\n )\n module_test.httpx_mock.add_response(\n url=\"https://evilcorp.okta.com/.well-known/openid-configuration\",\n json=self.openid_config_okta,\n )\n module_test.httpx_mock.add_response(\n url=\"https://login.windows.net/cc74fc12-4142-400e-a653-f98bdeadbeef/oauth2/token\",\n json={\n \"error\": \"invalid_grant\",\n \"error_description\": \"AADSTS9002313: Invalid request. Request is malformed or invalid.\\r\\nTrace ID: a3618b0d-d3b2-4669-96bc-ce414e202300\\r\\nCorrelation ID: fc54afc5-6f9d-4488-90ba-d8213515b847\\r\\nTimestamp: 2023-07-12 20:39:45Z\",\n \"error_codes\": [9002313],\n \"timestamp\": \"2023-07-12 20:39:45Z\",\n \"trace_id\": \"a3618b0d-d3b2-4669-96bc-ce414e202300\",\n \"correlation_id\": \"fc54afc5-6f9d-4488-90ba-d8213515b847\",\n \"error_uri\": \"https://login.windows.net/error?code=9002313\",\n },\n status_code=400,\n )\n module_test.httpx_mock.add_response(\n url=\"https://evilcorp.okta.com/oauth2/v1/token\",\n json={\n \"errorCode\": \"invalid_client\",\n \"errorSummary\": \"Invalid value for 'client_id' parameter.\",\n \"errorLink\": \"invalid_client\",\n \"errorId\": \"oae06YVQDq4Qz-WEuP3dU14XQ\",\n \"errorCauses\": [],\n },\n status_code=400,\n )\n\n def check(self, module_test, events):\n assert any(\n e.type == \"FINDING\"\n and e.data[\"description\"]\n == \"OpenID Connect Endpoint (domain: evilcorp.com) found at https://login.windows.net/evilcorp.com/.well-known/openid-configuration\"\n for e in events\n )\n assert any(\n e.type == \"FINDING\"\n and e.data[\"description\"]\n == \"OpenID Connect Endpoint (domain: evilcorp.com) found at https://evilcorp.okta.com/.well-known/openid-configuration\"\n for e in events\n )\n assert any(\n e.type == \"FINDING\"\n and e.data[\"description\"]\n == \"Potentially Sprayable OAUTH Endpoint (domain: evilcorp.com) at https://login.windows.net/cc74fc12-4142-400e-a653-f98bdeadbeef/oauth2/token\"\n for e in events\n )\n assert any(\n e.type == \"FINDING\"\n and e.data[\"description\"]\n == \"Potentially Sprayable OAUTH Endpoint (domain: evilcorp.com) at https://evilcorp.okta.com/oauth2/v1/token\"\n for e in events\n )\n assert any(e.data == \"https://sts.windows.net/cc74fc12-4142-400e-a653-f98bdeadbeef/\" for e in events)\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_otx.py", "content": "from .base import ModuleTestBase\n\n\nclass TestOTX(ModuleTestBase):\n config_overrides = {\"modules\": {\"otx\": {\"api_key\": \"test\"}}}\n\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://otx.alienvault.com/api/v1/indicators/domain/blacklanternsecurity.com/passive_dns\",\n json={\n \"passive_dns\": [\n {\n \"address\": \"2606:50c0:8000::153\",\n \"first\": \"2021-10-28T20:23:08\",\n \"last\": \"2022-08-24T18:29:49\",\n \"hostname\": \"asdf.blacklanternsecurity.com\",\n \"record_type\": \"AAAA\",\n \"indicator_link\": \"/indicator/hostname/www.blacklanternsecurity.com\",\n \"flag_url\": \"assets/images/flags/us.png\",\n \"flag_title\": \"United States\",\n \"asset_type\": \"hostname\",\n \"asn\": \"AS54113 fastly\",\n }\n ]\n },\n headers={\"X-OTX-API-KEY\": \"test\"},\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_paramminer_cookies.py", "content": "from .test_module_paramminer_headers import Paramminer_Headers, tempwordlist, helper\n\n\nclass TestParamminer_Cookies(Paramminer_Headers):\n modules_overrides = [\"httpx\", \"paramminer_cookies\"]\n config_overrides = {\"modules\": {\"paramminer_cookies\": {\"wordlist\": tempwordlist([\"junkcookie\", \"admincookie\"])}}}\n\n cookies_body = \"\"\"\n \n the title\n \n

Hello null!

';\n \n \n \"\"\"\n\n cookies_body_match = \"\"\"\n \n the title\n \n

Hello AAAAAAAAAAAAAA!

';\n \n \n \"\"\"\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"paramminer_cookies\"].rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n module_test.monkeypatch.setattr(\n helper.HttpCompare, \"gen_cache_buster\", lambda *args, **kwargs: {\"AAAAAA\": \"1\"}\n )\n expect_args = {\"headers\": {\"Cookie\": \"admincookie=AAAAAAAAAAAAAA\"}}\n respond_args = {\"response_data\": self.cookies_body_match}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n respond_args = {\"response_data\": self.cookies_body}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n found_reflected_cookie = False\n false_positive_match = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"[Paramminer] Cookie: [admincookie] Reasons: [body] Reflection: [True]\" in e.data[\"description\"]:\n found_reflected_cookie = True\n\n if \"junkcookie\" in e.data[\"description\"]:\n false_positive_match = True\n\n assert found_reflected_cookie, \"Failed to find hidden reflected cookie parameter\"\n assert not false_positive_match, \"Found word which was in wordlist but not a real match\"\n\n\nclass TestParamminer_Cookies_noreflection(TestParamminer_Cookies):\n cookies_body_match = \"\"\"\n \n the title\n \n

Hello ADMINISTRATOR!

';\n \n \n \"\"\"\n\n def check(self, module_test, events):\n assert any(\n e.type == \"WEB_PARAMETER\"\n and \"[Paramminer] Cookie: [admincookie] Reasons: [body] Reflection: [False]\" in e.data[\"description\"]\n for e in events\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_paramminer_getparams.py", "content": "from .test_module_paramminer_headers import Paramminer_Headers, tempwordlist, helper\n\n\nclass TestParamminer_Getparams(Paramminer_Headers):\n modules_overrides = [\"httpx\", \"paramminer_getparams\"]\n config_overrides = {\"modules\": {\"paramminer_getparams\": {\"wordlist\": tempwordlist([\"canary\", \"id\"])}}}\n\n getparam_body = \"\"\"\n \n the title\n \n

Hello null!

';\n \n \n \"\"\"\n\n getparam_body_match = \"\"\"\n \n the title\n \n

Hello AAAAAAAAAAAAAA!

';\n \n \n \"\"\"\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"paramminer_getparams\"].rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n module_test.monkeypatch.setattr(\n helper.HttpCompare, \"gen_cache_buster\", lambda *args, **kwargs: {\"AAAAAA\": \"1\"}\n )\n expect_args = {\"query_string\": b\"id=AAAAAAAAAAAAAA&AAAAAA=1\"}\n respond_args = {\"response_data\": self.getparam_body_match}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n respond_args = {\"response_data\": self.getparam_body}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n assert any(\n e.type == \"WEB_PARAMETER\"\n and \"[Paramminer] Getparam: [id] Reasons: [body] Reflection: [True]\" in e.data[\"description\"]\n for e in events\n )\n assert not any(\n e.type == \"WEB_PARAMETER\" and \"[Paramminer] Getparam: [canary] Reasons: [body]\" in e.data[\"description\"]\n for e in events\n )\n\n\nclass TestParamminer_Getparams_noreflection(TestParamminer_Getparams):\n getparam_body_match = \"\"\"\n \n the title\n \n

Hello ADMINISTRATOR!

';\n \n \n \"\"\"\n\n def check(self, module_test, events):\n assert any(\n e.type == \"WEB_PARAMETER\"\n and \"[Paramminer] Getparam: [id] Reasons: [body] Reflection: [False]\" in e.data[\"description\"]\n for e in events\n )\n\n\nclass TestParamminer_Getparams_singlewordlist(TestParamminer_Getparams):\n config_overrides = {\"modules\": {\"paramminer_getparams\": {\"wordlist\": tempwordlist([\"id\"])}}}\n\n\nclass TestParamminer_Getparams_boring_off(TestParamminer_Getparams):\n config_overrides = {\n \"modules\": {\n \"paramminer_getparams\": {\"skip_boring_words\": False, \"wordlist\": tempwordlist([\"canary\", \"utm_term\"])}\n }\n }\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"paramminer_getparams\"].rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n module_test.monkeypatch.setattr(\n helper.HttpCompare, \"gen_cache_buster\", lambda *args, **kwargs: {\"AAAAAA\": \"1\"}\n )\n expect_args = {\"query_string\": b\"utm_term=AAAAAAAAAAAAAA&AAAAAA=1\"}\n respond_args = {\"response_data\": self.getparam_body_match}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n respond_args = {\"response_data\": self.getparam_body}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n emitted_boring_parameter = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"utm_term\" in e.data[\"description\"]:\n emitted_boring_parameter = True\n assert emitted_boring_parameter, \"failed to emit boring parameter with skip_boring_words disabled\"\n\n\nclass TestParamminer_Getparams_boring_on(TestParamminer_Getparams_boring_off):\n config_overrides = {\n \"modules\": {\n \"paramminer_getparams\": {\"skip_boring_words\": True, \"wordlist\": tempwordlist([\"canary\", \"boring\"])}\n }\n }\n\n def check(self, module_test, events):\n emitted_boring_parameter = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"boring\" in e.data[\"description\"]:\n emitted_boring_parameter = True\n\n assert not emitted_boring_parameter, \"emitted boring parameter with skip_boring_words enabled\"\n\n\nclass TestParamminer_Getparams_finish(Paramminer_Headers):\n modules_overrides = [\"httpx\", \"excavate\", \"paramminer_getparams\"]\n config_overrides = {\n \"modules\": {\"paramminer_getparams\": {\"wordlist\": tempwordlist([\"canary\", \"canary2\"]), \"recycle_words\": True}}\n }\n\n targets = [\"http://127.0.0.1:8888/test1.php\", \"http://127.0.0.1:8888/test2.php\"]\n\n test_1_html = \"\"\"\nparamstest2\n \"\"\"\n\n test_2_html = \"\"\"\n

Hello

\n \"\"\"\n\n test_2_html_match = \"\"\"\n

HackThePlanet!

\n \"\"\"\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"paramminer_getparams\"].rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n module_test.monkeypatch.setattr(\n helper.HttpCompare, \"gen_cache_buster\", lambda *args, **kwargs: {\"AAAAAA\": \"1\"}\n )\n\n expect_args = {\"uri\": \"/test2.php\", \"query_string\": b\"abcd1234=AAAAAAAAAAAAAA&AAAAAA=1\"}\n respond_args = {\"response_data\": self.test_2_html_match}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"uri\": \"/test2.php\"}\n respond_args = {\"response_data\": self.test_2_html}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"uri\": \"/test1.php\", \"query_string\": b\"abcd1234=AAAAAAAAAAAAAA&AAAAAA=1\"}\n respond_args = {\"response_data\": self.test_2_html_match}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"uri\": \"/test1.php\"}\n respond_args = {\"response_data\": self.test_1_html}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n excavate_extracted_web_parameter = False\n found_hidden_getparam_recycled = False\n emitted_excavate_paramminer_duplicate = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if (\n \"http://127.0.0.1:8888/test2.php\" in e.data[\"url\"]\n and \"HTTP Extracted Parameter [abcd1234] (HTML Tags Submodule)\" in e.data[\"description\"]\n ):\n excavate_extracted_web_parameter = True\n\n if (\n \"http://127.0.0.1:8888/test1.php\" in e.data[\"url\"]\n and \"[Paramminer] Getparam: [abcd1234] Reasons: [body] Reflection: [False]\"\n in e.data[\"description\"]\n ):\n found_hidden_getparam_recycled = True\n\n if (\n \"http://127.0.0.1:8888/test2.php\" in e.data[\"url\"]\n and \"[Paramminer] Getparam: [abcd1234] Reasons: [body] Reflection: [False]\"\n in e.data[\"description\"]\n ):\n emitted_excavate_paramminer_duplicate = True\n\n assert excavate_extracted_web_parameter, \"Excavate failed to extract GET parameter\"\n assert found_hidden_getparam_recycled, \"Failed to find hidden GET parameter\"\n # the fact that it is a duplicate is OK, because it still won't be consumed mutltiple times. But we do want to make sure both modules try to emit it\n assert emitted_excavate_paramminer_duplicate, \"Paramminer emitted duplicate already found by excavate\"\n\n\nclass TestParamminer_Getparams_xmlspeculative(Paramminer_Headers):\n targets = [\"http://127.0.0.1:8888/\"]\n modules_overrides = [\"httpx\", \"excavate\", \"paramminer_getparams\"]\n config_overrides = {\n \"modules\": {\n \"excavate\": {\"speculate_params\": True},\n \"paramminer_getparams\": {\"wordlist\": tempwordlist([\"data\", \"common\"]), \"recycle_words\": False},\n }\n }\n getparam_extract_xml = \"\"\"\n \n 1\n 1\n 1\n \n \"\"\"\n\n getparam_speculative_used = \"\"\"\n \n

secret parameter used

\n \n \"\"\"\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"paramminer_getparams\"].rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n module_test.monkeypatch.setattr(\n helper.HttpCompare, \"gen_cache_buster\", lambda *args, **kwargs: {\"AAAAAA\": \"1\"}\n )\n expect_args = {\"query_string\": b\"obscureParameter=AAAAAAAAAAAAAA&AAAAAA=1\"}\n respond_args = {\"response_data\": self.getparam_speculative_used}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"query_string\": b\"data=AAAAAAAAAAAAAA&obscureParameter=AAAAAAAAAAAAAA&AAAAAA=1\"}\n respond_args = {\"response_data\": self.getparam_speculative_used}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n respond_args = {\"response_data\": self.getparam_extract_xml, \"headers\": {\"Content-Type\": \"application/xml\"}}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n excavate_discovered_speculative = False\n paramminer_used_speculative = False\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if (\n \"HTTP Extracted Parameter (speculative from xml content) [obscureParameter]\"\n in e.data[\"description\"]\n ):\n excavate_discovered_speculative = True\n\n if (\n \"[Paramminer] Getparam: [obscureParameter] Reasons: [header,body] Reflection: [False]\"\n in e.data[\"description\"]\n ):\n paramminer_used_speculative = True\n\n assert excavate_discovered_speculative, \"Excavate failed to discover speculative xml parameter\"\n assert paramminer_used_speculative, \"Paramminer failed to confirm speculative GET parameter\"\n\n\nclass TestParamminer_Getparams_filter_static(TestParamminer_Getparams_finish):\n targets = [\"http://127.0.0.1:8888/test1.php\", \"http://127.0.0.1:8888/test2.pdf\"]\n\n test_1_html = \"\"\"\n paramstest2\n \"\"\"\n\n def check(self, module_test, events):\n found_hidden_getparam_recycled = False\n emitted_excavate_paramminer_duplicate = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if (\n \"http://127.0.0.1:8888/test1.php\" in e.data[\"url\"]\n and \"[Paramminer] Getparam: [abcd1234] Reasons: [body] Reflection: [False]\"\n in e.data[\"description\"]\n ):\n found_hidden_getparam_recycled = True\n\n if (\n \"http://127.0.0.1:8888/test2.pdf\" in e.data[\"url\"]\n and \"[Paramminer] Getparam: [abcd1234] Reasons: [body] Reflection: [False]\"\n in e.data[\"description\"]\n ):\n emitted_excavate_paramminer_duplicate = True\n\n assert found_hidden_getparam_recycled, \"Failed to find hidden GET parameter\"\n assert not emitted_excavate_paramminer_duplicate, \"Paramminer emitted parameter for static URL\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py", "content": "from bbot.core.helpers import helper\n\nfrom .base import ModuleTestBase, tempwordlist\n\n\nclass Paramminer_Headers(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"paramminer_headers\"]\n config_overrides = {\"modules\": {\"paramminer_headers\": {\"wordlist\": tempwordlist([\"junkword1\", \"tracestate\"])}}}\n\n headers_body = \"\"\"\n \n the title\n \n

Hello null!

';\n \n \n \"\"\"\n\n headers_body_match = \"\"\"\n \n the title\n \n

Hello AAAAAAAAAAAAAA!

';\n \n \n \"\"\"\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"paramminer_headers\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n module_test.monkeypatch.setattr(\n helper.HttpCompare, \"gen_cache_buster\", lambda *args, **kwargs: {\"AAAAAA\": \"1\"}\n )\n expect_args = {\"headers\": {\"tracestate\": \"AAAAAAAAAAAAAA\"}}\n respond_args = {\"response_data\": self.headers_body_match}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n respond_args = {\"response_data\": self.headers_body}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n found_reflected_header = False\n false_positive_match = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"[Paramminer] Header: [tracestate] Reasons: [body] Reflection: [True]\" in e.data[\"description\"]:\n found_reflected_header = True\n\n if \"junkword1\" in e.data[\"description\"]:\n false_positive_match = True\n\n assert found_reflected_header, \"Failed to find hidden reflected header parameter\"\n assert not false_positive_match, \"Found word which was in wordlist but not a real match\"\n\n\nclass TestParamminer_Headers(Paramminer_Headers):\n pass\n\n\nclass TestParamminer_Headers_noreflection(Paramminer_Headers):\n found_nonreflected_header = False\n\n headers_body_match = \"\"\"\n \n the title\n \n

Hello Administrator!

';\n \n \n \"\"\"\n\n def check(self, module_test, events):\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"[Paramminer] Header: [tracestate] Reasons: [body] Reflection: [False]\" in e.data[\"description\"]:\n found_nonreflected_header = True\n\n assert found_nonreflected_header, \"Failed to find hidden non-reflected header parameter\"\n\n\nclass TestParamminer_Headers_extract(Paramminer_Headers):\n modules_overrides = [\"httpx\", \"paramminer_headers\", \"excavate\"]\n config_overrides = {\n \"modules\": {\n \"paramminer_headers\": {\"wordlist\": tempwordlist([\"junkword1\", \"tracestate\"]), \"recycle_words\": True}\n }\n }\n\n headers_body = \"\"\"\n \n the title\n \n Click Me\n \n \n \"\"\"\n\n headers_body_match = \"\"\"\n \n the title\n \n Click Me\n Click Me\n

Secret param \"foo\" found with value: AAAAAAAAAAAAAA

\n \n \n \"\"\"\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"paramminer_headers\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n module_test.monkeypatch.setattr(\n helper.HttpCompare, \"gen_cache_buster\", lambda *args, **kwargs: {\"AAAAAA\": \"1\"}\n )\n expect_args = {\"headers\": {\"foo\": \"AAAAAAAAAAAAAA\"}}\n respond_args = {\"response_data\": self.headers_body_match}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n respond_args = {\"response_data\": self.headers_body}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n excavate_extracted_web_parameter = False\n used_recycled_parameter = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [foo] (HTML Tags Submodule)\" in e.data[\"description\"]:\n excavate_extracted_web_parameter = True\n if \"[Paramminer] Header: [foo] Reasons: [body] Reflection: [True]\" in e.data[\"description\"]:\n used_recycled_parameter = True\n\n assert excavate_extracted_web_parameter, \"Excavate failed to extract WEB_PARAMETER\"\n assert used_recycled_parameter, \"Failed to find header with recycled parameter\"\n\n\nclass TestParamminer_Headers_extract_norecycle(TestParamminer_Headers_extract):\n modules_overrides = [\"httpx\", \"excavate\"]\n config_overrides = {}\n\n async def setup_after_prep(self, module_test):\n respond_args = {\"response_data\": self.headers_body}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n excavate_extracted_web_parameter = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"HTTP Extracted Parameter [foo] (HTML Tags Submodule)\" in e.data[\"description\"]:\n excavate_extracted_web_parameter = True\n\n assert not excavate_extracted_web_parameter, (\n \"Excavate extract WEB_PARAMETER despite disabling parameter extraction\"\n )\n\n\nclass TestParamminer_Headers_NoCookieRetention(Paramminer_Headers):\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"paramminer_headers\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n module_test.monkeypatch.setattr(\n helper.HttpCompare, \"gen_cache_buster\", lambda *args, **kwargs: {\"AAAAAA\": \"1\"}\n )\n\n expect_args = {\"headers\": {\"tracestate\": \"AAAAAAAAAAAAAA\"}}\n respond_args = {\"response_data\": self.headers_body_match}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n headers_body_with_cookie = \"\"\"\n \n the title\n \n

Hello with cookie!

';\n \n \n \"\"\"\n expect_args = {\"headers\": {\"Cookie\": \"test_cookie=cookie_value; AAAAAAAAAAAAAA=AAAAAAAAAAAAAA\"}}\n respond_args_with_cookie_body_change = {\"response_data\": headers_body_with_cookie}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args_with_cookie_body_change)\n\n respond_args_default = {\n \"response_data\": self.headers_body,\n \"headers\": {\"set-cookie\": \"test_cookie=cookie_value\"},\n }\n module_test.set_expect_requests(respond_args=respond_args_default)\n\n def check(self, module_test, events):\n found_web_parameter = False\n found_web_parameter_false_positive = False\n\n for e in events:\n if e.type == \"WEB_PARAMETER\":\n if \"[Paramminer] Header: [tracestate]\" in e.data[\"description\"]:\n found_web_parameter = True\n if \"junkword1\" in e.data[\"description\"]:\n found_web_parameter_false_positive = True\n\n assert found_web_parameter, \"WEB_PARAMETER event was not emitted\"\n assert not found_web_parameter_false_positive, \"WEB_PARAMETER event was emitted with false positive\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_passivetotal.py", "content": "from .base import ModuleTestBase\n\n\nclass TestPassiveTotal(ModuleTestBase):\n config_overrides = {\"modules\": {\"passivetotal\": {\"api_key\": \"jon@bls.fakedomain:asdf\"}}}\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.passivetotal.org/v2/account/quota\",\n match_headers={\"Authorization\": \"Basic am9uQGJscy5mYWtlZG9tYWluOmFzZGY=\"},\n json={\"user\": {\"counts\": {\"search_api\": 10}, \"limits\": {\"search_api\": 20}}},\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.passivetotal.org/v2/enrichment/subdomains?query=blacklanternsecurity.com\",\n match_headers={\"Authorization\": \"Basic am9uQGJscy5mYWtlZG9tYWluOmFzZGY=\"},\n json={\"subdomains\": [\"asdf\"]},\n )\n\n async def setup_after_prep(self, module_test):\n module_test.monkeypatch.setattr(module_test.scan.modules[\"passivetotal\"], \"abort_if\", lambda e: False)\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_pgp.py", "content": "from .base import ModuleTestBase\n\n\nclass TestPGP(ModuleTestBase):\n web_body = \"\"\"\n\n\nSearch results for 'blacklanternsecurity.com'\n\n\n

Search results for 'blacklanternsecurity.com'

Type bits/keyID            cr. time   exp time   key expir\n
\n\n\n
pub eddsa263/0xd4e98af823deadbeef 2022-09-14T15:11:31Z\n\nuid Asdf <asdf@blacklanternsecurity.com>\nsig  sig  0xd4e98af823deadbeef 2022-09-14T15:11:31Z 2024-09-14T17:00:00Z ____________________ [selfsig]\n\n
\n\"\"\"\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=blacklanternsecurity.com\",\n text=self.web_body,\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf@blacklanternsecurity.com\" for e in events), \"Failed to detect email\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_portfilter.py", "content": "from .base import ModuleTestBase\n\n\nclass TestPortfilter_disabled(ModuleTestBase):\n modules_overrides = []\n\n async def setup_before_prep(self, module_test):\n from bbot.modules.base import BaseModule\n\n class DummyModule(BaseModule):\n _name = \"dummy_module\"\n watched_events = [\"DNS_NAME\"]\n\n async def handle_event(self, event):\n if event.type == \"DNS_NAME\" and event.data == \"blacklanternsecurity.com\":\n await self.emit_event(\n \"www.blacklanternsecurity.com:443\",\n \"OPEN_TCP_PORT\",\n parent=event,\n tags=[\"cdn-ip\", \"cdn-amazon\"],\n )\n # when portfilter is enabled, this should be filtered out\n await self.emit_event(\n \"www.blacklanternsecurity.com:8080\",\n \"OPEN_TCP_PORT\",\n parent=event,\n tags=[\"cdn-ip\", \"cdn-amazon\"],\n )\n await self.emit_event(\"www.blacklanternsecurity.com:21\", \"OPEN_TCP_PORT\", parent=event)\n\n module_test.scan.modules[\"dummy_module\"] = DummyModule(module_test.scan)\n\n def check(self, module_test, events):\n open_ports = {event.data for event in events if event.type == \"OPEN_TCP_PORT\"}\n assert open_ports == {\n \"www.blacklanternsecurity.com:443\",\n \"www.blacklanternsecurity.com:8080\",\n \"www.blacklanternsecurity.com:21\",\n }\n\n\nclass TestPortfilter_enabled(TestPortfilter_disabled):\n modules_overrides = [\"portfilter\"]\n\n def check(self, module_test, events):\n # even though portfilter listens for URLs, enabling it should not automatically enable httpx\n assert \"httpx\" not in module_test.scan.modules\n open_ports = {event.data for event in events if event.type == \"OPEN_TCP_PORT\"}\n # we should be missing the 8080 port because it's a CDN and not in portfilter's allowed list of open ports\n assert open_ports == {\"www.blacklanternsecurity.com:443\", \"www.blacklanternsecurity.com:21\"}\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_portscan.py", "content": "from .base import ModuleTestBase\n\n\nclass TestPortscan(ModuleTestBase):\n targets = [\n \"www.evilcorp.com\",\n \"evilcorp.com\",\n \"8.8.8.8/32\",\n \"8.8.8.8/24\",\n \"8.8.4.4\",\n \"asdf.evilcorp.net\",\n \"8.8.4.4/24\",\n ]\n scan_name = \"test_portscan\"\n config_overrides = {\"modules\": {\"portscan\": {\"ports\": \"443\", \"wait\": 1}}, \"dns\": {\"minimal\": False}}\n\n masscan_output_1 = \"\"\"{ \"ip\": \"8.8.8.8\", \"timestamp\": \"1680197558\", \"ports\": [ {\"port\": 443, \"proto\": \"tcp\", \"status\": \"open\", \"reason\": \"syn-ack\", \"ttl\": 54} ] }\"\"\"\n masscan_output_2 = \"\"\"{ \"ip\": \"8.8.4.5\", \"timestamp\": \"1680197558\", \"ports\": [ {\"port\": 80, \"proto\": \"tcp\", \"status\": \"open\", \"reason\": \"syn-ack\", \"ttl\": 54} ] }\"\"\"\n masscan_output_3 = \"\"\"{ \"ip\": \"8.8.4.6\", \"timestamp\": \"1680197558\", \"ports\": [ {\"port\": 631, \"proto\": \"tcp\", \"status\": \"open\", \"reason\": \"syn-ack\", \"ttl\": 54} ] }\"\"\"\n\n masscan_output_ping = \"\"\"{ \"ip\": \"8.8.8.8\", \"timestamp\": \"1719862594\", \"ports\": [ {\"port\": 0, \"proto\": \"icmp\", \"status\": \"open\", \"reason\": \"none\", \"ttl\": 54} ] }\"\"\"\n\n async def setup_after_prep(self, module_test):\n from bbot.modules.base import BaseModule\n\n class DummyModule(BaseModule):\n _name = \"dummy_module\"\n watched_events = [\"*\"]\n\n async def handle_event(self, event):\n if event.type == \"DNS_NAME\":\n if \"dummy\" not in event.host:\n await self.emit_event(f\"dummy.{event.data}\", \"DNS_NAME\", parent=event)\n\n module_test.scan.modules[\"dummy_module\"] = DummyModule(module_test.scan)\n\n await module_test.mock_dns(\n {\n \"www.evilcorp.com\": {\"A\": [\"8.8.8.8\"]},\n \"evilcorp.com\": {\"A\": [\"8.8.8.8\"]},\n \"asdf.evilcorp.net\": {\"A\": [\"8.8.4.5\"]},\n \"dummy.asdf.evilcorp.net\": {\"A\": [\"8.8.4.5\"]},\n \"dummy.evilcorp.com\": {\"A\": [\"8.8.4.6\"]},\n \"dummy.www.evilcorp.com\": {\"A\": [\"8.8.4.4\"]},\n }\n )\n\n self.syn_scanned = []\n self.ping_scanned = []\n self.syn_runs = 0\n self.ping_runs = 0\n\n async def run_masscan(command, *args, **kwargs):\n if \"masscan\" in command[:2]:\n targets = open(command[11]).read().splitlines()\n yield \"[\"\n if \"--ping\" in command:\n self.ping_runs += 1\n self.ping_scanned += targets\n yield self.masscan_output_ping\n else:\n self.syn_runs += 1\n self.syn_scanned += targets\n if \"8.8.8.0/24\" in targets or \"8.8.8.8/32\" in targets:\n yield self.masscan_output_1\n if \"8.8.4.0/24\" in targets:\n yield self.masscan_output_2\n yield self.masscan_output_3\n yield \"]\"\n else:\n async for l in module_test.scan.helpers.run_live(command, *args, **kwargs):\n yield l\n\n module_test.monkeypatch.setattr(module_test.scan.helpers, \"run_live\", run_masscan)\n\n def check(self, module_test, events):\n assert set(self.syn_scanned) == {\"8.8.8.0/24\", \"8.8.4.0/24\"}\n assert set(self.ping_scanned) == set()\n assert self.syn_runs >= 1\n assert self.ping_runs == 0\n assert 1 == len(\n [e for e in events if e.type == \"DNS_NAME\" and e.data == \"evilcorp.com\" and str(e.module) == \"TARGET\"]\n )\n assert 1 == len(\n [e for e in events if e.type == \"DNS_NAME\" and e.data == \"www.evilcorp.com\" and str(e.module) == \"TARGET\"]\n )\n assert 1 == len(\n [e for e in events if e.type == \"DNS_NAME\" and e.data == \"asdf.evilcorp.net\" and str(e.module) == \"TARGET\"]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\" and e.data == \"dummy.evilcorp.com\" and str(e.module) == \"dummy_module\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\" and e.data == \"dummy.www.evilcorp.com\" and str(e.module) == \"dummy_module\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\" and e.data == \"dummy.asdf.evilcorp.net\" and str(e.module) == \"dummy_module\"\n ]\n )\n # the reason these numbers aren't exactly predictable is because we can't predict which one arrives first\n # to the portscan module. Sometimes, one that would normally be deduped is force-emitted because it led to a new open port.\n assert 2 <= len([e for e in events if e.type == \"IP_ADDRESS\" and e.data == \"8.8.8.8\"]) <= 4\n assert 2 <= len([e for e in events if e.type == \"IP_ADDRESS\" and e.data == \"8.8.4.4\"]) <= 4\n assert 2 <= len([e for e in events if e.type == \"IP_ADDRESS\" and e.data == \"8.8.4.5\"]) <= 4\n assert 2 <= len([e for e in events if e.type == \"IP_ADDRESS\" and e.data == \"8.8.4.6\"]) <= 4\n assert 1 == len([e for e in events if e.type == \"OPEN_TCP_PORT\" and e.data == \"8.8.8.8:443\"])\n assert 1 == len([e for e in events if e.type == \"OPEN_TCP_PORT\" and e.data == \"8.8.4.5:80\"])\n assert 1 == len([e for e in events if e.type == \"OPEN_TCP_PORT\" and e.data == \"8.8.4.6:631\"])\n assert 1 == len([e for e in events if e.type == \"OPEN_TCP_PORT\" and e.data == \"evilcorp.com:443\"])\n assert 1 == len([e for e in events if e.type == \"OPEN_TCP_PORT\" and e.data == \"www.evilcorp.com:443\"])\n assert 1 == len([e for e in events if e.type == \"OPEN_TCP_PORT\" and e.data == \"asdf.evilcorp.net:80\"])\n assert 1 == len([e for e in events if e.type == \"OPEN_TCP_PORT\" and e.data == \"dummy.asdf.evilcorp.net:80\"])\n assert 1 == len([e for e in events if e.type == \"OPEN_TCP_PORT\" and e.data == \"dummy.evilcorp.com:631\"])\n assert not any(e for e in events if e.type == \"OPEN_TCP_PORT\" and e.host == \"dummy.www.evilcorp.com\")\n\n\nclass TestPortscanPingFirst(TestPortscan):\n modules_overrides = {\"portscan\"}\n config_overrides = {\"modules\": {\"portscan\": {\"ports\": \"443\", \"wait\": 1, \"ping_first\": True}}}\n\n def check(self, module_test, events):\n assert set(self.syn_scanned) == {\"8.8.8.8/32\"}\n assert set(self.ping_scanned) == {\"8.8.8.0/24\", \"8.8.4.0/24\"}\n assert self.syn_runs == 1\n assert self.ping_runs >= 1\n open_port_events = [e for e in events if e.type == \"OPEN_TCP_PORT\"]\n assert len(open_port_events) == 3\n assert {e.data for e in open_port_events} == {\"8.8.8.8:443\", \"evilcorp.com:443\", \"www.evilcorp.com:443\"}\n\n\nclass TestPortscanPingOnly(TestPortscan):\n modules_overrides = {\"portscan\"}\n config_overrides = {\"modules\": {\"portscan\": {\"ports\": \"443\", \"wait\": 1, \"ping_only\": True}}}\n\n targets = [\"8.8.8.8/24\", \"8.8.4.4/24\"]\n\n def check(self, module_test, events):\n assert set(self.syn_scanned) == set()\n assert set(self.ping_scanned) == {\"8.8.8.0/24\", \"8.8.4.0/24\"}\n assert self.syn_runs == 0\n assert self.ping_runs >= 1\n open_port_events = [e for e in events if e.type == \"OPEN_TCP_PORT\"]\n assert len(open_port_events) == 0\n ip_events = [e for e in events if e.type == \"IP_ADDRESS\"]\n assert len(ip_events) == 1\n assert {e.data for e in ip_events} == {\"8.8.8.8\"}\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_postgres.py", "content": "import time\nimport asyncio\n\nfrom .base import ModuleTestBase\n\n\nclass TestPostgres(ModuleTestBase):\n targets = [\"evilcorp.com\"]\n skip_distro_tests = True\n\n async def setup_before_prep(self, module_test):\n process = await asyncio.create_subprocess_exec(\n \"docker\",\n \"run\",\n \"--name\",\n \"bbot-test-postgres\",\n \"--rm\",\n \"-e\",\n \"POSTGRES_PASSWORD=bbotislife\",\n \"-e\",\n \"POSTGRES_USER=postgres\",\n \"-p\",\n \"5432:5432\",\n \"-d\",\n \"postgres\",\n )\n\n import asyncpg\n\n # wait for the container to start\n start_time = time.time()\n while True:\n try:\n # Connect to the default 'postgres' database to create 'bbot'\n conn = await asyncpg.connect(\n user=\"postgres\", password=\"bbotislife\", database=\"postgres\", host=\"127.0.0.1\"\n )\n await conn.execute(\"CREATE DATABASE bbot\")\n await conn.close()\n break\n except asyncpg.exceptions.DuplicateDatabaseError:\n # If the database already exists, break the loop\n break\n except Exception as e:\n if time.time() - start_time > 60: # timeout after 60 seconds\n self.log.error(\"PostgreSQL server did not start in time.\")\n raise e\n await asyncio.sleep(1)\n\n if process.returncode != 0:\n self.log.error(\"Failed to start PostgreSQL server\")\n\n async def check(self, module_test, events):\n import asyncpg\n\n # Connect to the PostgreSQL database\n conn = await asyncpg.connect(user=\"postgres\", password=\"bbotislife\", database=\"bbot\", host=\"127.0.0.1\")\n\n try:\n events = await conn.fetch(\"SELECT * FROM event\")\n assert len(events) == 3, \"No events found in PostgreSQL database\"\n scans = await conn.fetch(\"SELECT * FROM scan\")\n assert len(scans) == 1, \"No scans found in PostgreSQL database\"\n targets = await conn.fetch(\"SELECT * FROM target\")\n assert len(targets) == 1, \"No targets found in PostgreSQL database\"\n finally:\n await conn.close()\n process = await asyncio.create_subprocess_exec(\n \"docker\", \"stop\", \"bbot-test-postgres\", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE\n )\n stdout, stderr = await process.communicate()\n\n if process.returncode != 0:\n raise Exception(f\"Failed to stop PostgreSQL server: {stderr.decode()}\")\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_postman.py", "content": "from .base import ModuleTestBase\n\n\nclass TestPostman(ModuleTestBase):\n config_overrides = {\"modules\": {\"postman\": {\"api_key\": \"asdf\"}}}\n modules_overrides = [\"postman\", \"speculate\"]\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/me\",\n match_headers={\"X-Api-Key\": \"asdf\"},\n json={\n \"user\": {\n \"id\": 000000,\n \"username\": \"test_key\",\n \"email\": \"blacklanternsecurity@test.com\",\n \"fullName\": \"Test Key\",\n \"avatar\": \"\",\n \"isPublic\": True,\n \"teamId\": 0,\n \"teamDomain\": \"\",\n \"roles\": [\"user\"],\n },\n \"operations\": [\n {\"name\": \"api_object_usage\", \"limit\": 3, \"usage\": 0, \"overage\": 0},\n {\"name\": \"collection_run_limit\", \"limit\": 25, \"usage\": 0, \"overage\": 0},\n {\"name\": \"file_storage_limit\", \"limit\": 20, \"usage\": 0, \"overage\": 0},\n {\"name\": \"flow_count\", \"limit\": 5, \"usage\": 0, \"overage\": 0},\n {\"name\": \"flow_requests\", \"limit\": 5000, \"usage\": 0, \"overage\": 0},\n {\"name\": \"performance_test_limit\", \"limit\": 25, \"usage\": 0, \"overage\": 0},\n {\"name\": \"postbot_calls\", \"limit\": 50, \"usage\": 0, \"overage\": 0},\n {\"name\": \"reusable_packages\", \"limit\": 3, \"usage\": 0, \"overage\": 0},\n {\"name\": \"test_data_retrieval\", \"limit\": 1000, \"usage\": 0, \"overage\": 0},\n {\"name\": \"test_data_storage\", \"limit\": 10, \"usage\": 0, \"overage\": 0},\n {\"name\": \"mock_usage\", \"limit\": 1000, \"usage\": 0, \"overage\": 0},\n {\"name\": \"monitor_request_runs\", \"limit\": 1000, \"usage\": 0, \"overage\": 0},\n {\"name\": \"api_usage\", \"limit\": 1000, \"usage\": 0, \"overage\": 0},\n ],\n },\n )\n\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns(\n {\"blacklanternsecurity.com\": {\"A\": [\"127.0.0.99\"]}, \"github.com\": {\"A\": [\"127.0.0.99\"]}}\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.postman.com/_api/ws/proxy\",\n match_json={\n \"service\": \"search\",\n \"method\": \"POST\",\n \"path\": \"/search-all\",\n \"body\": {\n \"queryIndices\": [\"collaboration.workspace\"],\n \"queryText\": \"blacklanternsecurity\",\n \"size\": 25,\n \"from\": 0,\n \"clientTraceId\": \"\",\n \"requestOrigin\": \"srp\",\n \"mergeEntities\": \"true\",\n \"nonNestedRequests\": \"true\",\n \"domain\": \"public\",\n },\n },\n json={\n \"data\": [\n {\n \"score\": 611.41156,\n \"normalizedScore\": 23,\n \"document\": {\n \"watcherCount\": 6,\n \"apiCount\": 0,\n \"forkCount\": 0,\n \"isblacklisted\": \"false\",\n \"createdAt\": \"2021-06-15T14:03:51\",\n \"publishertype\": \"team\",\n \"publisherHandle\": \"blacklanternsecurity\",\n \"id\": \"11498add-357d-4bc5-a008-0a2d44fb8829\",\n \"slug\": \"bbot-public\",\n \"updatedAt\": \"2024-07-30T11:00:35\",\n \"entityType\": \"workspace\",\n \"visibilityStatus\": \"public\",\n \"forkcount\": \"0\",\n \"tags\": [],\n \"createdat\": \"2021-06-15T14:03:51\",\n \"forkLabel\": \"\",\n \"publisherName\": \"blacklanternsecurity\",\n \"name\": \"BlackLanternSecurity BBOT [Public]\",\n \"dependencyCount\": 7,\n \"collectionCount\": 6,\n \"warehouse__updated_at\": \"2024-07-30 11:00:00\",\n \"privateNetworkFolders\": [],\n \"isPublisherVerified\": False,\n \"publisherType\": \"team\",\n \"curatedInList\": [],\n \"creatorId\": \"6900157\",\n \"description\": \"\",\n \"forklabel\": \"\",\n \"publisherId\": \"299401\",\n \"publisherLogo\": \"\",\n \"popularity\": 5,\n \"isPublic\": True,\n \"categories\": [],\n \"universaltags\": \"\",\n \"views\": 5788,\n \"summary\": \"BLS public workspaces.\",\n \"memberCount\": 2,\n \"isBlacklisted\": False,\n \"publisherid\": \"299401\",\n \"isPrivateNetworkEntity\": False,\n \"isDomainNonTrivial\": True,\n \"privateNetworkMeta\": \"\",\n \"updatedat\": \"2021-10-20T16:19:29\",\n \"documentType\": \"workspace\",\n },\n \"highlight\": {\"summary\": \"BLS BBOT api test.\"},\n },\n {\n \"score\": 611.41156,\n \"normalizedScore\": 23,\n \"document\": {\n \"watcherCount\": 6,\n \"apiCount\": 0,\n \"forkCount\": 0,\n \"isblacklisted\": \"false\",\n \"createdAt\": \"2021-06-15T14:03:51\",\n \"publishertype\": \"team\",\n \"publisherHandle\": \"testteam\",\n \"id\": \"11498add-357d-4bc5-a008-0a2d44fb8829\",\n \"slug\": \"testing-bbot-api\",\n \"updatedAt\": \"2024-07-30T11:00:35\",\n \"entityType\": \"workspace\",\n \"visibilityStatus\": \"public\",\n \"forkcount\": \"0\",\n \"tags\": [],\n \"createdat\": \"2021-06-15T14:03:51\",\n \"forkLabel\": \"\",\n \"publisherName\": \"testteam\",\n \"name\": \"Test BlackLanternSecurity API Team Workspace\",\n \"dependencyCount\": 7,\n \"collectionCount\": 6,\n \"warehouse__updated_at\": \"2024-07-30 11:00:00\",\n \"privateNetworkFolders\": [],\n \"isPublisherVerified\": False,\n \"publisherType\": \"team\",\n \"curatedInList\": [],\n \"creatorId\": \"6900157\",\n \"description\": \"\",\n \"forklabel\": \"\",\n \"publisherId\": \"299401\",\n \"publisherLogo\": \"\",\n \"popularity\": 5,\n \"isPublic\": True,\n \"categories\": [],\n \"universaltags\": \"\",\n \"views\": 5788,\n \"summary\": \"Private test of BBOTs public API\",\n \"memberCount\": 2,\n \"isBlacklisted\": False,\n \"publisherid\": \"299401\",\n \"isPrivateNetworkEntity\": False,\n \"isDomainNonTrivial\": True,\n \"privateNetworkMeta\": \"\",\n \"updatedat\": \"2021-10-20T16:19:29\",\n \"documentType\": \"workspace\",\n },\n \"highlight\": {\"summary\": \"Private test of BBOTs Public API\"},\n },\n ],\n \"meta\": {\n \"queryText\": \"blacklanternsecurity\",\n \"total\": {\n \"collection\": 0,\n \"request\": 0,\n \"workspace\": 2,\n \"api\": 0,\n \"team\": 0,\n \"user\": 0,\n \"flow\": 0,\n \"apiDefinition\": 0,\n \"privateNetworkFolder\": 0,\n },\n \"state\": \"AQ4\",\n \"spellCorrection\": {\"count\": {\"all\": 2, \"workspace\": 2}, \"correctedQueryText\": None},\n \"featureFlags\": {\n \"enabledPublicResultCuration\": True,\n \"boostByPopularity\": True,\n \"reRankPostNormalization\": True,\n \"enableUrlBarHostNameSearch\": True,\n },\n },\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.postman.com/_api/ws/proxy\",\n match_json={\n \"service\": \"workspaces\",\n \"method\": \"GET\",\n \"path\": \"/workspaces?handle=blacklanternsecurity&slug=bbot-public\",\n },\n json={\n \"meta\": {\"model\": \"workspace\", \"action\": \"find\", \"nextCursor\": \"\"},\n \"data\": [\n {\n \"id\": \"3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b\",\n \"name\": \"BlackLanternSecurity BBOT [Public]\",\n \"description\": None,\n \"summary\": \"BLS public workspaces.\",\n \"createdBy\": \"299401\",\n \"updatedBy\": \"299401\",\n \"team\": None,\n \"createdAt\": \"2021-10-20T16:19:29\",\n \"updatedAt\": \"2021-10-20T16:19:29\",\n \"visibilityStatus\": \"public\",\n \"profileInfo\": {\n \"slug\": \"bbot-public\",\n \"profileType\": \"team\",\n \"profileId\": \"000000\",\n \"publicHandle\": \"https://www.postman.com/blacklanternsecurity\",\n \"publicImageURL\": \"\",\n \"publicName\": \"BlackLanternSecurity\",\n \"isVerified\": False,\n },\n }\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.postman.com/_api/ws/proxy\",\n match_json={\n \"service\": \"workspaces\",\n \"method\": \"GET\",\n \"path\": \"/workspaces?handle=testteam&slug=testing-bbot-api\",\n },\n json={\n \"meta\": {\"model\": \"workspace\", \"action\": \"find\", \"nextCursor\": \"\"},\n \"data\": [\n {\n \"id\": \"a4dfe981-2593-4f0b-b4c3-5145e8640f7d\",\n \"name\": \"Test BlackLanternSecurity API Team Workspace\",\n \"description\": None,\n \"summary\": \"Private test of BBOTs public API\",\n \"createdBy\": \"299401\",\n \"updatedBy\": \"299401\",\n \"team\": None,\n \"createdAt\": \"2021-10-20T16:19:29\",\n \"updatedAt\": \"2021-10-20T16:19:29\",\n \"visibilityStatus\": \"public\",\n \"profileInfo\": {\n \"slug\": \"bbot-public\",\n \"profileType\": \"team\",\n \"profileId\": \"000000\",\n \"publicHandle\": \"https://www.postman.com/testteam\",\n \"publicImageURL\": \"\",\n \"publicName\": \"testteam\",\n \"isVerified\": False,\n },\n }\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/workspaces/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b\",\n match_headers={\"X-Api-Key\": \"asdf\"},\n json={\n \"workspace\": {\n \"id\": \"3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b\",\n \"name\": \"BlackLanternSecurity BBOT [Public]\",\n \"type\": \"personal\",\n \"description\": None,\n \"visibility\": \"public\",\n \"createdBy\": \"00000000\",\n \"updatedBy\": \"00000000\",\n \"createdAt\": \"2021-11-17T06:09:01.000Z\",\n \"updatedAt\": \"2021-11-17T08:57:16.000Z\",\n \"collections\": [\n {\n \"id\": \"2aab9fd0-3715-4abe-8bb0-8cb0264d023f\",\n \"name\": \"BBOT Public\",\n \"uid\": \"10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f\",\n },\n ],\n \"environments\": [\n {\n \"id\": \"f770f816-9c6a-40f7-bde3-c0855d2a1089\",\n \"name\": \"BBOT Test\",\n \"uid\": \"10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089\",\n }\n ],\n \"apis\": [],\n }\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/workspaces/a4dfe981-2593-4f0b-b4c3-5145e8640f7d\",\n json={\n \"workspace\": {\n \"id\": \"a4dfe981-2593-4f0b-b4c3-5145e8640f7d\",\n \"name\": \"Test BlackLanternSecurity API Team Workspace\",\n \"type\": \"personal\",\n \"description\": None,\n \"visibility\": \"public\",\n \"createdBy\": \"00000000\",\n \"updatedBy\": \"00000000\",\n \"createdAt\": \"2021-11-17T06:09:01.000Z\",\n \"updatedAt\": \"2021-11-17T08:57:16.000Z\",\n \"collections\": [\n {\n \"id\": \"f46bebfd-420a-4adf-97d1-6fb5a02cf7fc\",\n \"name\": \"BBOT Public\",\n \"uid\": \"10197090-f46bebfd-420a-4adf-97d1-6fb5a02cf7fc\",\n },\n ],\n \"environments\": [],\n \"apis\": [],\n }\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.postman.com/_api/workspace/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b/globals\",\n json={\n \"model_id\": \"8be7574b-219f-49e0-8d25-da447a882e4e\",\n \"meta\": {\"model\": \"globals\", \"action\": \"find\"},\n \"data\": {\n \"workspace\": \"3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b\",\n \"lastUpdatedBy\": \"00000000\",\n \"lastRevision\": 1637239113000,\n \"id\": \"8be7574b-219f-49e0-8d25-da447a882e4e\",\n \"values\": [\n {\n \"key\": \"endpoint_url\",\n \"value\": \"https://api.blacklanternsecurity.com/\",\n \"enabled\": True,\n },\n ],\n \"createdAt\": \"2021-11-17T06:09:01.000Z\",\n \"updatedAt\": \"2021-11-18T12:38:33.000Z\",\n },\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.postman.com/_api/workspace/a4dfe981-2593-4f0b-b4c3-5145e8640f7d/globals\",\n json={\n \"model_id\": \"8be7574b-219f-49e0-8d25-da447a882e4e\",\n \"meta\": {\"model\": \"globals\", \"action\": \"find\"},\n \"data\": {\n \"workspace\": \"a4dfe981-2593-4f0b-b4c3-5145e8640f7d\",\n \"lastUpdatedBy\": \"00000000\",\n \"lastRevision\": 1637239113000,\n \"id\": \"8be7574b-219f-49e0-8d25-da447a882e4e\",\n \"values\": [],\n \"createdAt\": \"2021-11-17T06:09:01.000Z\",\n \"updatedAt\": \"2021-11-18T12:38:33.000Z\",\n },\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/environments/10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089\",\n match_headers={\"X-Api-Key\": \"asdf\"},\n json={\n \"environment\": {\n \"id\": \"f770f816-9c6a-40f7-bde3-c0855d2a1089\",\n \"name\": \"BBOT Test\",\n \"owner\": \"00000000\",\n \"createdAt\": \"2021-11-17T06:29:54.000Z\",\n \"updatedAt\": \"2021-11-23T07:06:53.000Z\",\n \"values\": [\n {\n \"key\": \"temp_session_endpoint\",\n \"value\": \"https://api.blacklanternsecurity.com/\",\n \"enabled\": True,\n },\n ],\n \"isPublic\": True,\n }\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/collections/10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f\",\n match_headers={\"X-Api-Key\": \"asdf\"},\n json={\n \"collection\": {\n \"info\": {\n \"_postman_id\": \"62b91565-d2e2-4bcd-8248-4dba2e3452f0\",\n \"name\": \"BBOT Public\",\n \"schema\": \"https://schema.getpostman.com/json/collection/v2.1.0/collection.json\",\n \"updatedAt\": \"2021-11-17T07:13:16.000Z\",\n \"createdAt\": \"2021-11-17T07:13:15.000Z\",\n \"lastUpdatedBy\": \"00000000\",\n \"uid\": \"00000000-62b91565-d2e2-4bcd-8248-4dba2e3452f0\",\n },\n \"item\": [\n {\n \"name\": \"Generate API Session\",\n \"id\": \"c1bac38c-dfc9-4cc0-9c19-828cbc8543b1\",\n \"protocolProfileBehavior\": {\"disableBodyPruning\": True},\n \"request\": {\n \"method\": \"POST\",\n \"header\": [{\"key\": \"Content-Type\", \"value\": \"application/json\"}],\n \"body\": {\n \"mode\": \"raw\",\n \"raw\": '{\"username\": \"test\", \"password\": \"Test\"}',\n },\n \"url\": {\"raw\": \"{{endpoint_url}}\", \"host\": [\"{{endpoint_url}}\"]},\n \"description\": \"\",\n },\n \"response\": [],\n \"uid\": \"10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1\",\n },\n ],\n }\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/collections/10197090-f46bebfd-420a-4adf-97d1-6fb5a02cf7fc\",\n json={\n \"collection\": {\n \"info\": {\n \"_postman_id\": \"f46bebfd-420a-4adf-97d1-6fb5a02cf7fc\",\n \"name\": \"BBOT Public\",\n \"schema\": \"https://schema.getpostman.com/json/collection/v2.1.0/collection.json\",\n \"updatedAt\": \"2021-11-17T07:13:16.000Z\",\n \"createdAt\": \"2021-11-17T07:13:15.000Z\",\n \"lastUpdatedBy\": \"00000000\",\n \"uid\": \"00000000-f46bebfd-420a-4adf-97d1-6fb5a02cf7fc\",\n },\n \"item\": [\n {\n \"name\": \"Out of Scope API request\",\n \"id\": \"c1bac38c-dfc9-4cc0-9c19-828cbc8543b1\",\n \"protocolProfileBehavior\": {\"disableBodyPruning\": True},\n \"request\": {\n \"method\": \"POST\",\n \"header\": [{\"key\": \"Content-Type\", \"value\": \"application/json\"}],\n \"body\": {\n \"mode\": \"raw\",\n \"raw\": '{\"username\": \"test\", \"password\": \"Test\"}',\n },\n \"url\": {\"raw\": \"https://www.outofscope.com\", \"host\": [\"www.outofscope.com\"]},\n \"description\": \"\",\n },\n \"response\": [],\n \"uid\": \"10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1\",\n },\n ],\n }\n },\n )\n\n def check(self, module_test, events):\n assert len(events) == 5\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\" and e.data == \"blacklanternsecurity.com\" and e.scope_distance == 0\n ]\n ), \"Failed to emit target DNS_NAME\"\n assert 1 == len(\n [e for e in events if e.type == \"ORG_STUB\" and e.data == \"blacklanternsecurity\" and e.scope_distance == 0]\n ), \"Failed to find ORG_STUB\"\n # Find only 1 in-scope workspace the other will be out of scope\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"CODE_REPOSITORY\"\n and \"postman\" in e.tags\n and e.data[\"url\"] == \"https://www.postman.com/blacklanternsecurity/bbot-public\"\n and e.scope_distance == 1\n ]\n ), \"Failed to find blacklanternsecurity postman workspace\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_postman_download.py", "content": "from .base import ModuleTestBase\nfrom bbot.test.bbot_fixtures import bbot_test_dir\n\n\nclass TestPostman_Download(ModuleTestBase):\n config_overrides = {\n \"modules\": {\n \"postman_download\": {\"api_key\": \"asdf\", \"output_folder\": str(bbot_test_dir / \"test_postman_files\")}\n }\n }\n modules_overrides = [\"postman\", \"postman_download\", \"speculate\"]\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/me\",\n match_headers={\"X-Api-Key\": \"asdf\"},\n json={\n \"user\": {\n \"id\": 000000,\n \"username\": \"test_key\",\n \"email\": \"blacklanternsecurity@test.com\",\n \"fullName\": \"Test Key\",\n \"avatar\": \"\",\n \"isPublic\": True,\n \"teamId\": 0,\n \"teamDomain\": \"\",\n \"roles\": [\"user\"],\n },\n \"operations\": [\n {\"name\": \"api_object_usage\", \"limit\": 3, \"usage\": 0, \"overage\": 0},\n {\"name\": \"collection_run_limit\", \"limit\": 25, \"usage\": 0, \"overage\": 0},\n {\"name\": \"file_storage_limit\", \"limit\": 20, \"usage\": 0, \"overage\": 0},\n {\"name\": \"flow_count\", \"limit\": 5, \"usage\": 0, \"overage\": 0},\n {\"name\": \"flow_requests\", \"limit\": 5000, \"usage\": 0, \"overage\": 0},\n {\"name\": \"performance_test_limit\", \"limit\": 25, \"usage\": 0, \"overage\": 0},\n {\"name\": \"postbot_calls\", \"limit\": 50, \"usage\": 0, \"overage\": 0},\n {\"name\": \"reusable_packages\", \"limit\": 3, \"usage\": 0, \"overage\": 0},\n {\"name\": \"test_data_retrieval\", \"limit\": 1000, \"usage\": 0, \"overage\": 0},\n {\"name\": \"test_data_storage\", \"limit\": 10, \"usage\": 0, \"overage\": 0},\n {\"name\": \"mock_usage\", \"limit\": 1000, \"usage\": 0, \"overage\": 0},\n {\"name\": \"monitor_request_runs\", \"limit\": 1000, \"usage\": 0, \"overage\": 0},\n {\"name\": \"api_usage\", \"limit\": 1000, \"usage\": 0, \"overage\": 0},\n ],\n },\n )\n\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns(\n {\"blacklanternsecurity.com\": {\"A\": [\"127.0.0.99\"]}, \"github.com\": {\"A\": [\"127.0.0.99\"]}}\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.postman.com/_api/ws/proxy\",\n match_json={\n \"service\": \"search\",\n \"method\": \"POST\",\n \"path\": \"/search-all\",\n \"body\": {\n \"queryIndices\": [\"collaboration.workspace\"],\n \"queryText\": \"blacklanternsecurity\",\n \"size\": 25,\n \"from\": 0,\n \"clientTraceId\": \"\",\n \"requestOrigin\": \"srp\",\n \"mergeEntities\": \"true\",\n \"nonNestedRequests\": \"true\",\n \"domain\": \"public\",\n },\n },\n json={\n \"data\": [\n {\n \"score\": 611.41156,\n \"normalizedScore\": 23,\n \"document\": {\n \"watcherCount\": 6,\n \"apiCount\": 0,\n \"forkCount\": 0,\n \"isblacklisted\": \"false\",\n \"createdAt\": \"2021-06-15T14:03:51\",\n \"publishertype\": \"team\",\n \"publisherHandle\": \"blacklanternsecurity\",\n \"id\": \"11498add-357d-4bc5-a008-0a2d44fb8829\",\n \"slug\": \"bbot-public\",\n \"updatedAt\": \"2024-07-30T11:00:35\",\n \"entityType\": \"workspace\",\n \"visibilityStatus\": \"public\",\n \"forkcount\": \"0\",\n \"tags\": [],\n \"createdat\": \"2021-06-15T14:03:51\",\n \"forkLabel\": \"\",\n \"publisherName\": \"blacklanternsecurity\",\n \"name\": \"BlackLanternSecurity BBOT [Public]\",\n \"dependencyCount\": 7,\n \"collectionCount\": 6,\n \"warehouse__updated_at\": \"2024-07-30 11:00:00\",\n \"privateNetworkFolders\": [],\n \"isPublisherVerified\": False,\n \"publisherType\": \"team\",\n \"curatedInList\": [],\n \"creatorId\": \"6900157\",\n \"description\": \"\",\n \"forklabel\": \"\",\n \"publisherId\": \"299401\",\n \"publisherLogo\": \"\",\n \"popularity\": 5,\n \"isPublic\": True,\n \"categories\": [],\n \"universaltags\": \"\",\n \"views\": 5788,\n \"summary\": \"BLS public workspaces.\",\n \"memberCount\": 2,\n \"isBlacklisted\": False,\n \"publisherid\": \"299401\",\n \"isPrivateNetworkEntity\": False,\n \"isDomainNonTrivial\": True,\n \"privateNetworkMeta\": \"\",\n \"updatedat\": \"2021-10-20T16:19:29\",\n \"documentType\": \"workspace\",\n },\n \"highlight\": {\"summary\": \"BLS BBOT api test.\"},\n },\n ],\n \"meta\": {\n \"queryText\": \"blacklanternsecurity\",\n \"total\": {\n \"collection\": 0,\n \"request\": 0,\n \"workspace\": 1,\n \"api\": 0,\n \"team\": 0,\n \"user\": 0,\n \"flow\": 0,\n \"apiDefinition\": 0,\n \"privateNetworkFolder\": 0,\n },\n \"state\": \"AQ4\",\n \"spellCorrection\": {\"count\": {\"all\": 1, \"workspace\": 1}, \"correctedQueryText\": None},\n \"featureFlags\": {\n \"enabledPublicResultCuration\": True,\n \"boostByPopularity\": True,\n \"reRankPostNormalization\": True,\n \"enableUrlBarHostNameSearch\": True,\n },\n },\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.postman.com/_api/ws/proxy\",\n match_json={\n \"service\": \"workspaces\",\n \"method\": \"GET\",\n \"path\": \"/workspaces?handle=blacklanternsecurity&slug=bbot-public\",\n },\n json={\n \"meta\": {\"model\": \"workspace\", \"action\": \"find\", \"nextCursor\": \"\"},\n \"data\": [\n {\n \"id\": \"3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b\",\n \"name\": \"BlackLanternSecurity BBOT [Public]\",\n \"description\": None,\n \"summary\": \"BLS public workspaces.\",\n \"createdBy\": \"299401\",\n \"updatedBy\": \"299401\",\n \"team\": None,\n \"createdAt\": \"2021-10-20T16:19:29\",\n \"updatedAt\": \"2021-10-20T16:19:29\",\n \"visibilityStatus\": \"public\",\n \"profileInfo\": {\n \"slug\": \"bbot-public\",\n \"profileType\": \"team\",\n \"profileId\": \"000000\",\n \"publicHandle\": \"https://www.postman.com/blacklanternsecurity\",\n \"publicImageURL\": \"\",\n \"publicName\": \"BlackLanternSecurity\",\n \"isVerified\": False,\n },\n }\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/workspaces/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b\",\n match_headers={\"X-Api-Key\": \"asdf\"},\n json={\n \"workspace\": {\n \"id\": \"3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b\",\n \"name\": \"BlackLanternSecurity BBOT [Public]\",\n \"type\": \"personal\",\n \"description\": None,\n \"visibility\": \"public\",\n \"createdBy\": \"00000000\",\n \"updatedBy\": \"00000000\",\n \"createdAt\": \"2021-11-17T06:09:01.000Z\",\n \"updatedAt\": \"2021-11-17T08:57:16.000Z\",\n \"collections\": [\n {\n \"id\": \"2aab9fd0-3715-4abe-8bb0-8cb0264d023f\",\n \"name\": \"BBOT Public\",\n \"uid\": \"10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f\",\n },\n ],\n \"environments\": [\n {\n \"id\": \"f770f816-9c6a-40f7-bde3-c0855d2a1089\",\n \"name\": \"BBOT Test\",\n \"uid\": \"10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089\",\n }\n ],\n \"apis\": [],\n }\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.postman.com/_api/workspace/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b/globals\",\n json={\n \"model_id\": \"8be7574b-219f-49e0-8d25-da447a882e4e\",\n \"meta\": {\"model\": \"globals\", \"action\": \"find\"},\n \"data\": {\n \"workspace\": \"3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b\",\n \"lastUpdatedBy\": \"00000000\",\n \"lastRevision\": 1637239113000,\n \"id\": \"8be7574b-219f-49e0-8d25-da447a882e4e\",\n \"values\": [\n {\n \"key\": \"endpoint_url\",\n \"value\": \"https://api.blacklanternsecurity.com/\",\n \"enabled\": True,\n },\n ],\n \"createdAt\": \"2021-11-17T06:09:01.000Z\",\n \"updatedAt\": \"2021-11-18T12:38:33.000Z\",\n },\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/environments/10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089\",\n match_headers={\"X-Api-Key\": \"asdf\"},\n json={\n \"environment\": {\n \"id\": \"f770f816-9c6a-40f7-bde3-c0855d2a1089\",\n \"name\": \"BBOT Test\",\n \"owner\": \"00000000\",\n \"createdAt\": \"2021-11-17T06:29:54.000Z\",\n \"updatedAt\": \"2021-11-23T07:06:53.000Z\",\n \"values\": [\n {\n \"key\": \"temp_session_endpoint\",\n \"value\": \"https://api.blacklanternsecurity.com/\",\n \"enabled\": True,\n },\n ],\n \"isPublic\": True,\n }\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/collections/10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f\",\n match_headers={\"X-Api-Key\": \"asdf\"},\n json={\n \"collection\": {\n \"info\": {\n \"_postman_id\": \"62b91565-d2e2-4bcd-8248-4dba2e3452f0\",\n \"name\": \"BBOT Public\",\n \"schema\": \"https://schema.getpostman.com/json/collection/v2.1.0/collection.json\",\n \"updatedAt\": \"2021-11-17T07:13:16.000Z\",\n \"createdAt\": \"2021-11-17T07:13:15.000Z\",\n \"lastUpdatedBy\": \"00000000\",\n \"uid\": \"00000000-62b91565-d2e2-4bcd-8248-4dba2e3452f0\",\n },\n \"item\": [\n {\n \"name\": \"Generate API Session\",\n \"id\": \"c1bac38c-dfc9-4cc0-9c19-828cbc8543b1\",\n \"protocolProfileBehavior\": {\"disableBodyPruning\": True},\n \"request\": {\n \"method\": \"POST\",\n \"header\": [{\"key\": \"Content-Type\", \"value\": \"application/json\"}],\n \"body\": {\n \"mode\": \"raw\",\n \"raw\": '{\"username\": \"test\", \"password\": \"Test\"}',\n },\n \"url\": {\"raw\": \"{{endpoint_url}}\", \"host\": [\"{{endpoint_url}}\"]},\n \"description\": \"\",\n },\n \"response\": [],\n \"uid\": \"10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1\",\n },\n ],\n }\n },\n )\n\n def check(self, module_test, events):\n assert 1 == len(\n [e for e in events if e.type == \"CODE_REPOSITORY\" and \"postman\" in e.tags and e.scope_distance == 1]\n ), \"Failed to find blacklanternsecurity postman workspace\"\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"FILESYSTEM\"\n and \"postman_workspaces/BlackLanternSecurity BBOT [Public]\" in e.data[\"path\"]\n and \"postman\" in e.tags\n and e.scope_distance == 1\n ]\n ), \"Failed to find blacklanternsecurity postman workspace\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_python.py", "content": "from .base import ModuleTestBase\n\n\nclass TestPython(ModuleTestBase):\n def check(self, module_test, events):\n assert any(e.data == \"blacklanternsecurity.com\" for e in events)\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_rapiddns.py", "content": "import httpx\n\nfrom .base import ModuleTestBase\n\n\nclass TestRapidDNS(ModuleTestBase):\n web_body = \"\"\"12\nasdf.blacklanternsecurity.com\nasdf.blacklanternsecurity.com.\"\"\"\n\n async def setup_after_prep(self, module_test):\n module_test.module.abort_if = lambda e: False\n module_test.httpx_mock.add_response(\n url=\"https://rapiddns.io/subdomain/blacklanternsecurity.com?full=1#result\", text=self.web_body\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n\n\nclass TestRapidDNSAbortThreshold1(TestRapidDNS):\n module_name = \"rapiddns\"\n\n async def setup_after_prep(self, module_test):\n self.url_count = {}\n\n async def custom_callback(request):\n url = str(request.url)\n try:\n self.url_count[url] += 1\n except KeyError:\n self.url_count[url] = 1\n raise httpx.TimeoutException(\"timeout\")\n\n module_test.httpx_mock.add_callback(custom_callback)\n\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.com\": {\"A\": [\"127.0.0.88\"]},\n \"evilcorp.com\": {\"A\": [\"127.0.0.11\"]},\n \"evilcorp.net\": {\"A\": [\"127.0.0.22\"]},\n \"evilcorp.co.uk\": {\"A\": [\"127.0.0.33\"]},\n }\n )\n\n def check(self, module_test, events):\n assert module_test.module.api_failure_abort_threshold == 10\n assert module_test.module.errored is False\n assert module_test.module._api_request_failures == 3\n assert module_test.module.api_retries == 3\n assert {e.data for e in events if e.type == \"DNS_NAME\"} == {\"blacklanternsecurity.com\"}\n assert self.url_count == {\n \"https://rapiddns.io/subdomain/blacklanternsecurity.com?full=1#result\": 3,\n }\n\n\nclass TestRapidDNSAbortThreshold2(TestRapidDNSAbortThreshold1):\n targets = [\"blacklanternsecurity.com\", \"evilcorp.com\"]\n\n def check(self, module_test, events):\n assert module_test.module.api_failure_abort_threshold == 10\n assert module_test.module.errored is False\n assert module_test.module._api_request_failures == 6\n assert module_test.module.api_retries == 3\n assert {e.data for e in events if e.type == \"DNS_NAME\"} == {\"blacklanternsecurity.com\", \"evilcorp.com\"}\n assert self.url_count == {\n \"https://rapiddns.io/subdomain/blacklanternsecurity.com?full=1#result\": 3,\n \"https://rapiddns.io/subdomain/evilcorp.com?full=1#result\": 3,\n }\n\n\nclass TestRapidDNSAbortThreshold3(TestRapidDNSAbortThreshold1):\n targets = [\"blacklanternsecurity.com\", \"evilcorp.com\", \"evilcorp.net\"]\n\n def check(self, module_test, events):\n assert module_test.module.api_failure_abort_threshold == 10\n assert module_test.module.errored is False\n assert module_test.module._api_request_failures == 9\n assert module_test.module.api_retries == 3\n assert {e.data for e in events if e.type == \"DNS_NAME\"} == {\n \"blacklanternsecurity.com\",\n \"evilcorp.com\",\n \"evilcorp.net\",\n }\n assert self.url_count == {\n \"https://rapiddns.io/subdomain/blacklanternsecurity.com?full=1#result\": 3,\n \"https://rapiddns.io/subdomain/evilcorp.com?full=1#result\": 3,\n \"https://rapiddns.io/subdomain/evilcorp.net?full=1#result\": 3,\n }\n\n\nclass TestRapidDNSAbortThreshold4(TestRapidDNSAbortThreshold1):\n targets = [\"blacklanternsecurity.com\", \"evilcorp.com\", \"evilcorp.net\", \"evilcorp.co.uk\"]\n\n def check(self, module_test, events):\n assert module_test.module.api_failure_abort_threshold == 10\n assert module_test.module.errored is True\n assert module_test.module._api_request_failures == 10\n assert module_test.module.api_retries == 3\n assert {e.data for e in events if e.type == \"DNS_NAME\"} == {\n \"blacklanternsecurity.com\",\n \"evilcorp.com\",\n \"evilcorp.net\",\n \"evilcorp.co.uk\",\n }\n assert len(self.url_count) == 4\n assert list(self.url_count.values()).count(3) == 3\n assert list(self.url_count.values()).count(1) == 1\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_reflected_parameters.py", "content": "from .base import ModuleTestBase, tempwordlist\nfrom werkzeug.wrappers import Response\nimport re\n\nfrom .test_module_paramminer_getparams import TestParamminer_Getparams\nfrom .test_module_paramminer_headers import helper\n\n\nclass TestReflected_parameters_fromexcavate(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"reflected_parameters\", \"excavate\"]\n\n def request_handler(self, request):\n normal_block = 'foo'\n qs = str(request.query_string.decode())\n if \"reflected=\" in qs:\n value = qs.split(\"=\")[1]\n if \"&\" in value:\n value = value.split(\"&\")[0]\n reflected_block = f''\n return Response(reflected_block, status=200)\n else:\n return Response(normal_block, status=200)\n\n async def setup_after_prep(self, module_test):\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n assert any(\n e.type == \"FINDING\"\n and e.data[\"description\"]\n == \"[GETPARAM] Parameter value reflected in response body. Name: [reflected] Source Module: [excavate] Original Value: [foo]\"\n for e in events\n )\n\n\nclass TestReflected_parameters_headers(TestReflected_parameters_fromexcavate):\n modules_overrides = [\"httpx\", \"reflected_parameters\", \"excavate\", \"paramminer_headers\"]\n config_overrides = {\n \"modules\": {\n \"paramminer_headers\": {\"wordlist\": tempwordlist([\"junkword1\", \"tracestate\"]), \"recycle_words\": True}\n }\n }\n\n def request_handler(self, request):\n headers = {k.lower(): v for k, v in request.headers.items()}\n if \"tracestate\" in headers:\n reflected_value = headers[\"tracestate\"]\n reflected_block = f\"
{reflected_value}
\"\n return Response(reflected_block, status=200)\n else:\n return Response(\"
\", status=200)\n\n def check(self, module_test, events):\n assert any(\n e.type == \"FINDING\"\n and e.data[\"description\"]\n == \"[HEADER] Parameter value reflected in response body. Name: [tracestate] Source Module: [paramminer_headers]\"\n for e in events\n )\n\n\nclass TestReflected_parameters_fromparamminer(TestParamminer_Getparams):\n modules_overrides = [\"httpx\", \"paramminer_getparams\", \"reflected_parameters\"]\n\n def request_handler(self, request):\n normal_block = \"\"\n qs = str(request.query_string.decode())\n if \"id=\" in qs:\n value = qs.split(\"=\")[1]\n if \"&\" in value:\n value = value.split(\"&\")[0]\n reflected_block = f''\n return Response(reflected_block, status=200)\n else:\n return Response(normal_block, status=200)\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"paramminer_getparams\"].rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n module_test.monkeypatch.setattr(\n helper.HttpCompare, \"gen_cache_buster\", lambda *args, **kwargs: {\"AAAAAA\": \"1\"}\n )\n\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n assert any(\n e.type == \"FINDING\"\n and \"[GETPARAM] Parameter value reflected in response body. Name: [id] Source Module: [paramminer_getparams]\"\n in e.data[\"description\"]\n for e in events\n )\n\n\nclass TestReflected_parameters_with_canary(TestReflected_parameters_fromexcavate):\n def request_handler(self, request):\n normal_block = 'foo'\n qs = str(request.query_string.decode())\n if qs:\n # Split the query string into key-value pairs\n params = qs.split(\"&\")\n # Construct the reflected block with all parameters\n reflected_block = '
{reflected_value}
\"\n return Response(reflected_block, status=200)\n else:\n return Response(\"
\", status=200)\n\n def check(self, module_test, events):\n assert any(\n e.type == \"FINDING\"\n and e.data[\"description\"]\n == \"[COOKIE] Parameter value reflected in response body. Name: [testcookie] Source Module: [paramminer_cookies]\"\n for e in events\n )\n\n\nclass TestReflected_parameters_postparams(TestReflected_parameters_fromexcavate):\n modules_overrides = [\"httpx\", \"reflected_parameters\", \"excavate\"]\n\n def request_handler(self, request):\n form_data = request.form\n if \"testparam\" in form_data:\n reflected_value = form_data[\"testparam\"]\n reflected_block = f\"
{reflected_value}
\"\n return Response(reflected_block, status=200)\n else:\n form_html = \"\"\"\n \n \n
\n \n \n
\n \n \n \"\"\"\n return Response(form_html, status=200)\n\n def check(self, module_test, events):\n assert any(\n e.type == \"FINDING\"\n and e.data[\"description\"]\n == \"[POSTPARAM] Parameter value reflected in response body. Name: [testparam] Source Module: [excavate] Original Value: [default_value]\"\n for e in events\n )\n\n\nclass TestReflected_parameters_bodyjson(TestReflected_parameters_fromexcavate):\n modules_overrides = [\"httpx\", \"reflected_parameters\", \"excavate\"]\n\n def request_handler(self, request):\n # Ensure the request is expecting JSON data\n if request.content_type == \"application/json\":\n json_data = request.json\n if \"username\" in json_data:\n reflected_value = json_data[\"username\"]\n reflected_block = f\"
{reflected_value}
\"\n return Response(reflected_block, status=200)\n # Provide an HTML page with a jQuery AJAX call\n jsonajax_extract_html = \"\"\"\n \n \n
\n \n \"\"\"\n return Response(jsonajax_extract_html, status=200)\n\n async def setup_after_prep(self, module_test):\n expect_args = re.compile(\"/\")\n module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)\n\n def check(self, module_test, events):\n assert any(\n e.type == \"FINDING\"\n and e.data[\"description\"]\n == \"[BODYJSON] Parameter value reflected in response body. Name: [username] Source Module: [excavate]\"\n for e in events\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_retirejs.py", "content": "from .base import ModuleTestBase\n\n\nclass TestRetireJS(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"excavate\", \"retirejs\"]\n\n # HTML page with vulnerable JavaScript libraries\n vulnerable_html = \"\"\"\n\n\n \n retire.js test page\n\n\n

retire.js test page

\n

This page includes JavaScript libraries for testing.

\n\n \n \n\n \n \n\n \n \n\n \n\n\"\"\"\n\n # Sample jQuery 3.4.1 content\n jquery_content = \"\"\"/*!\n * jQuery JavaScript Library v3.4.1\n * https://jquery.com/\n */\n(function( global, factory ) {\n \"use strict\";\n factory( global );\n})(typeof window !== \"undefined\" ? window : this, function( window, noGlobal ) {\n var jQuery = function( selector, context ) {\n return new jQuery.fn.init( selector, context );\n };\n jQuery.fn = jQuery.prototype = {};\n jQuery.fn.jquery = \"3.4.1\";\n if ( typeof noGlobal === \"undefined\" ) {\n window.jQuery = window.$ = jQuery;\n }\n return jQuery;\n});\"\"\"\n\n # Sample Lodash 4.17.11 content\n lodash_content = \"\"\"/**\n * @license\n * Lodash lodash.com/license | Underscore.js 1.8.3 underscorejs.org/LICENSE\n */\n;(function(){\nvar i=\"4.17.11\";\nvar Mn={VERSION:i};\nif(typeof define==\"function\"&&define.amd)define(function(){return Mn});else if(typeof module==\"object\"&&module.exports)module.exports=Mn;else this._=Mn}());\"\"\"\n\n # Sample Handlebars 4.0.5 content\n handlebars_content = \"\"\"/*!\n handlebars v4.0.5\n*/\n!function(a,b){\"object\"==typeof exports&&\"object\"==typeof module?module.exports=b():\"function\"==typeof define&&define.amd?define([],b):\"object\"==typeof exports?exports.Handlebars=b():a.Handlebars=b()}(this,function(){\nvar Handlebars={};\nHandlebars.VERSION=\"4.0.5\";\nreturn Handlebars;\n});\"\"\"\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"uri\": \"/\"}\n respond_args = {\"response_data\": self.vulnerable_html}\n module_test.set_expect_requests(expect_args, respond_args)\n\n expect_args = {\"uri\": \"/jquery-3.4.1.min.js\"}\n respond_args = {\"response_data\": self.jquery_content}\n module_test.set_expect_requests(expect_args, respond_args)\n\n expect_args = {\"uri\": \"/lodash.min.js\"}\n respond_args = {\"response_data\": self.lodash_content}\n module_test.set_expect_requests(expect_args, respond_args)\n\n expect_args = {\"uri\": \"/handlebars.min.js\"}\n respond_args = {\"response_data\": self.handlebars_content}\n module_test.set_expect_requests(expect_args, respond_args)\n\n def check(self, module_test, events):\n # Check that excavate found the JavaScript URLs\n url_unverified_events = [e for e in events if e.type == \"URL_UNVERIFIED\"]\n js_url_events = [e for e in url_unverified_events if \"extension-js\" in e.tags]\n\n # Two out of the three URLs should be in the output\n # The third, non-vulnerable URL (lodash.min.js) is not output because it's a \"special URL\", and\n # nothing interesting has been discovered from it.\n vuln_urls = {\"http://127.0.0.1:8888/handlebars.min.js\", \"http://127.0.0.1:8888/jquery-3.4.1.min.js\"}\n assert {e.data for e in js_url_events} == vuln_urls, \"Expected to find the vulnerable URLs in the output\"\n\n # Check for FINDING events generated by retirejs\n finding_events = [e for e in events if e.type == \"FINDING\"]\n retirejs_findings = [\n e\n for e in finding_events\n if \"vulnerable javascript library detected:\" in e.data.get(\"description\", \"\").lower()\n ]\n\n # We should have at least some findings from our vulnerable libraries\n assert len(retirejs_findings) > 0, (\n f\"Expected retirejs to find vulnerabilities, but got {len(retirejs_findings)} findings\"\n )\n\n # Check for specific expected vulnerability descriptions\n descriptions = [finding.data.get(\"description\", \"\") for finding in retirejs_findings]\n all_descriptions = \"\\n\".join(descriptions)\n\n # Look for specific vulnerabilities we expect to find\n expected_handlebars_vuln = \"Vulnerable JavaScript library detected: handlebars v4.0.5 Severity: HIGH Summary: Regular Expression Denial of Service in Handlebars JavaScript URL: http://127.0.0.1:8888/handlebars.min.js CVE(s): CVE-2019-20922 Affected versions: [4.0.0 to 4.4.5)\"\n expected_jquery_vuln = \"Vulnerable JavaScript library detected: jquery v3.4.1 Severity: MEDIUM Summary: Regex in its jQuery.htmlPrefilter sometimes may introduce XSS JavaScript URL: http://127.0.0.1:8888/jquery-3.4.1.min.js CVE(s): CVE-2020-11022 Affected versions: [1.2.0 to 3.5.0)\"\n\n # Verify at least one of the expected vulnerabilities is found\n handlebars_found = expected_handlebars_vuln in all_descriptions\n jquery_found = expected_jquery_vuln in all_descriptions\n\n assert handlebars_found and jquery_found, (\n f\"Expected to find specific vulnerabilities but didn't find them. Found descriptions:\\n{all_descriptions}\"\n )\n\n # Basic validation of findings structure\n for finding in retirejs_findings:\n assert \"description\" in finding.data, \"Finding should have description\"\n assert \"url\" in finding.data, \"Finding should have url\"\n assert finding.parent.type == \"URL_UNVERIFIED\", \"Parent should be URL_UNVERIFIED\"\n\n\nclass TestRetireJSNoExcavate(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"retirejs\"]\n force_start = True # Allow scan to continue even if modules fail setup\n config_overrides = {\n \"excavate\": False,\n }\n\n def check(self, module_test, events):\n # When excavate is disabled, retirejs should fail setup but scan should still run\n retirejs_module = module_test.scan.modules.get(\"retirejs\")\n\n if retirejs_module:\n # Check that the module exists but setup failed\n setup_status = getattr(retirejs_module, \"_setup_status\", None)\n if setup_status is not None:\n success, error_msg = setup_status\n assert success is False, \"retirejs setup should have failed without excavate\"\n expected_error = \"retirejs will not function without excavate enabled\"\n assert error_msg == expected_error, f\"Expected error message '{expected_error}', but got '{error_msg}'\"\n\n # No retirejs findings should be generated since setup failed\n retirejs_findings = [e for e in events if e.type == \"FINDING\" and getattr(e, \"module\", None) == \"retirejs\"]\n assert len(retirejs_findings) == 0, \"retirejs should not generate findings when setup fails\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_robots.py", "content": "import re\nfrom .base import ModuleTestBase\n\n\nclass TestRobots(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"robots\"]\n config_overrides = {\"modules\": {\"robots\": {\"include_sitemap\": True}}}\n\n async def setup_after_prep(self, module_test):\n sample_robots = f\"Allow: /allow/\\nDisallow: /disallow/\\nJunk: test.com\\nDisallow: /*/wildcard.txt\\nSitemap: {self.targets[0]}/sitemap.txt\"\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/robots.txt\"}\n respond_args = {\"response_data\": sample_robots}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n allow_bool = False\n disallow_bool = False\n sitemap_bool = False\n wildcard_bool = False\n\n for e in events:\n if e.type == \"URL_UNVERIFIED\":\n if str(e.module) != \"TARGET\":\n assert \"spider-danger\" in e.tags, f\"{e} doesn't have spider-danger tag\"\n if e.data == \"http://127.0.0.1:8888/allow/\":\n allow_bool = True\n\n if e.data == \"http://127.0.0.1:8888/disallow/\":\n disallow_bool = True\n\n if e.data == \"http://127.0.0.1:8888/sitemap.txt\":\n sitemap_bool = True\n\n if re.match(r\"http://127\\.0\\.0\\.1:8888/\\w+/wildcard\\.txt\", e.data):\n wildcard_bool = True\n\n assert allow_bool\n assert disallow_bool\n assert sitemap_bool\n assert wildcard_bool\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_securitytrails.py", "content": "from .base import ModuleTestBase\n\n\nclass TestSecurityTrails(ModuleTestBase):\n config_overrides = {\"modules\": {\"securitytrails\": {\"api_key\": \"asdf\"}}}\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.securitytrails.com/v1/ping?apikey=asdf\",\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.securitytrails.com/v1/domain/blacklanternsecurity.com/subdomains?apikey=asdf\",\n json={\n \"subdomains\": [\n \"asdf\",\n ],\n },\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_securitytxt.py", "content": "from .base import ModuleTestBase\n\n\nclass TestSecurityTxt(ModuleTestBase):\n targets = [\"blacklanternsecurity.notreal\"]\n modules_overrides = [\"securitytxt\", \"speculate\"]\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://blacklanternsecurity.notreal/.well-known/security.txt\",\n text=\"-----BEGIN PGP SIGNED MESSAGE-----\\nHash: SHA512\\n\\nContact: mailto:joe.smith@blacklanternsecurity.notreal\\nContact: mailto:vdp@example.com\\nContact: https://vdp.example.com\\nExpires: 2025-01-01T00:00:00.000Z\\nPreferred-Languages: fr, en\\nCanonical: https://blacklanternsecurity.notreal/.well-known/security.txt\\nPolicy: https://example.com/cert\\nHiring: https://www.careers.example.com\\n-----BEGIN PGP SIGNATURE-----\\n\\nSIGNATURE\\n\\n-----END PGP SIGNATURE-----\",\n )\n\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.notreal\": {\n \"A\": [\"127.0.0.11\"],\n },\n }\n )\n\n def check(self, module_test, events):\n assert any(e.type == \"EMAIL_ADDRESS\" and e.data == \"joe.smith@blacklanternsecurity.notreal\" for e in events), (\n \"Failed to detect email address\"\n )\n assert not any(\n e.type == \"URL_UNVERIFIED\" and e.data == \"https://blacklanternsecurity.notreal/.well-known/security.txt\"\n for e in events\n ), \"Failed to filter Canonical URL to self\"\n assert not any(str(e.data) == \"vdp@example.com\" for e in events)\n\n\nclass TestSecurityTxtEmailsFalse(TestSecurityTxt):\n config_overrides = {\n \"scope\": {\"report_distance\": 1},\n \"modules\": {\"securitytxt\": {\"emails\": False}},\n }\n\n def check(self, module_test, events):\n assert not any(e.type == \"EMAIL_ADDRESS\" for e in events), \"Detected email address when emails=False\"\n assert any(e.type == \"URL_UNVERIFIED\" and e.data == \"https://vdp.example.com/\" for e in events), (\n \"Failed to detect URL\"\n )\n assert any(e.type == \"URL_UNVERIFIED\" and e.data == \"https://example.com/cert\" for e in events), (\n \"Failed to detect URL\"\n )\n assert any(e.type == \"URL_UNVERIFIED\" and e.data == \"https://www.careers.example.com/\" for e in events), (\n \"Failed to detect URL\"\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_shodan_dns.py", "content": "from .base import ModuleTestBase\n\n\nclass TestShodan_DNS(ModuleTestBase):\n config_overrides = {\"modules\": {\"shodan\": {\"api_key\": \"asdf\"}}}\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.shodan.io/api-info?key=asdf\",\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.shodan.io/dns/domain/blacklanternsecurity.com?key=asdf&page=1\",\n json={\n \"subdomains\": [\n \"asdf\",\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.shodan.io/dns/domain/blacklanternsecurity.com?key=asdf&page=2\",\n json={\n \"subdomains\": [\n \"www\",\n ],\n },\n )\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.com\": {\n \"A\": [\"127.0.0.11\"],\n },\n \"www.blacklanternsecurity.com\": {\"A\": [\"127.0.0.22\"]},\n \"asdf.blacklanternsecurity.com\": {\"A\": [\"127.0.0.33\"]},\n }\n )\n\n def check(self, module_test, events):\n assert len([e for e in events if e.type == \"DNS_NAME\"]) == 3, \"Failed to detect both subdomains\"\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n assert any(e.data == \"www.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_shodan_idb.py", "content": "from .base import ModuleTestBase\n\n\nclass TestShodan_IDB(ModuleTestBase):\n config_overrides = {\"dns\": {\"minimal\": False}}\n\n async def setup_before_prep(self, module_test):\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n \"autodiscover.blacklanternsecurity.com\": {\"A\": [\"2.3.4.5\"]},\n \"mail.blacklanternsecurity.com\": {\"A\": [\"3.4.5.6\"]},\n }\n )\n\n module_test.httpx_mock.add_response(\n url=\"https://internetdb.shodan.io/1.2.3.4\",\n json={\n \"cpes\": [\n \"cpe:/a:microsoft:internet_information_services\",\n \"cpe:/a:microsoft:outlook_web_access:15.0.1367\",\n ],\n \"hostnames\": [\n \"autodiscover.blacklanternsecurity.com\",\n \"mail.blacklanternsecurity.com\",\n ],\n \"ip\": \"1.2.3.4\",\n \"ports\": [\n 25,\n 80,\n 443,\n ],\n \"tags\": [\"starttls\", \"self-signed\", \"eol-os\"],\n \"vulns\": [\"CVE-2021-26857\", \"CVE-2021-26855\"],\n },\n )\n\n def check(self, module_test, events):\n assert 8 == len([e for e in events if str(e.module) == \"shodan_idb\"])\n assert 1 == len(\n [e for e in events if e.type == \"DNS_NAME\" and e.data == \"autodiscover.blacklanternsecurity.com\"]\n )\n assert 1 == len([e for e in events if e.type == \"DNS_NAME\" and e.data == \"mail.blacklanternsecurity.com\"])\n assert 3 == len(\n [\n e\n for e in events\n if e.type == \"OPEN_TCP_PORT\" and e.host == \"blacklanternsecurity.com\" and str(e.module) == \"shodan_idb\"\n ]\n )\n assert 1 == len([e for e in events if e.type == \"FINDING\" and str(e.module) == \"shodan_idb\"])\n assert 1 == len([e for e in events if e.type == \"FINDING\" and \"CVE-2021-26857\" in e.data[\"description\"]])\n assert 2 == len([e for e in events if e.type == \"TECHNOLOGY\" and str(e.module) == \"shodan_idb\"])\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"TECHNOLOGY\" and e.data[\"technology\"] == \"cpe:/a:microsoft:outlook_web_access:15.0.1367\"\n ]\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_sitedossier.py", "content": "from .base import ModuleTestBase\n\npage1 = \"\"\"\n\n\n\n\nParent domain: evilcorp.com\n\n\n\n\n
\n
\n\"logo\"\n
\n
\"\"
\n
\"\"
\n
\"\"
\n
\"\"
\n
\n
\n\n\n\n\n\n\n
\"nw\"\"ne\"
\n

Parent domain: evilcorp.com

\n
\n\n
Displaying items 101 to 200, out of a total of 685\n
\n
    \n
  1.   http://asdf.evilcorp.com/
    \n
  2.   http://zzzz.evilcorp.com/
    \n
\nShow next 100 items
\n\n
\"sw\"\"se\"
\n
\n
\n
\n\n\n\"\"\"\n\npage2 = \"\"\"\n\n\n\n\nParent domain: evilcorp.com\n\n\n\n\n
\n
\n\"logo\"\n
\n
\"\"
\n
\"\"
\n
\"\"
\n
\"\"
\n
\n
\n\n\n\n\n\n\n
\"nw\"\"ne\"
\n

Parent domain: evilcorp.com

\n
\n\n
Displaying items 101 to 200, out of a total of 685\n
\n
    \n
  1.   http://xxxx.evilcorp.com/
    \n
  2.   http://ffff.evilcorp.com/
    \n
\n\n
\"sw\"\"se\"
\n
\n
\n
\n\n\n\"\"\"\n\n\nclass TestSitedossier(ModuleTestBase):\n targets = [\"evilcorp.com\"]\n\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns(\n {\n \"evilcorp.com\": {\"A\": [\"127.0.0.1\"]},\n \"asdf.evilcorp.com\": {\"A\": [\"127.0.0.1\"]},\n \"zzzz.evilcorp.com\": {\"A\": [\"127.0.0.1\"]},\n \"xxxx.evilcorp.com\": {\"A\": [\"127.0.0.1\"]},\n \"ffff.evilcorp.com\": {\"A\": [\"127.0.0.1\"]},\n }\n )\n module_test.httpx_mock.add_response(\n url=\"http://www.sitedossier.com/parentdomain/evilcorp.com\",\n text=page1,\n )\n module_test.httpx_mock.add_response(\n url=\"http://www.sitedossier.com/parentdomain/evilcorp.com/101\",\n text=page2,\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.evilcorp.com\" for e in events), \"Failed to detect subdomain\"\n assert any(e.data == \"zzzz.evilcorp.com\" for e in events), \"Failed to detect subdomain\"\n assert any(e.data == \"xxxx.evilcorp.com\" for e in events), \"Failed to detect subdomain\"\n assert any(e.data == \"ffff.evilcorp.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_skymem.py", "content": "from .base import ModuleTestBase\n\n\nclass TestSkymem(ModuleTestBase):\n targets = [\"blacklanternsecurity.com\"]\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://www.skymem.info/srch?q=blacklanternsecurity.com\",\n text=page_1_body,\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.skymem.info/domain/5679236812ad5b3f748a413d?p=2\",\n text=page_2_body,\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.skymem.info/domain/5679236812ad5b3f748a413d?p=3\",\n text=page_3_body,\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"page1email@blacklanternsecurity.com\" for e in events), \"Failed to detect first email\"\n assert any(e.data == \"page2email@blacklanternsecurity.com\" for e in events), \"Failed to detect second email\"\n assert any(e.data == \"page3email@blacklanternsecurity.com\" for e in events), \"Failed to detect third email\"\n\n\npage_1_body = \"\"\"\npage1email@blacklanternsecurity.com\n More emails for blacklanternsecurity.com ...\n More emails for blacklanternsecurity.com ...\n\"\"\"\n\npage_2_body = \"\"\"\npage2email@blacklanternsecurity.com\n More emails for blacklanternsecurity.com ...\n More emails for blacklanternsecurity.com ...\n\"\"\"\n\npage_3_body = \"\"\"\npage3email@blacklanternsecurity.com\n More emails for blacklanternsecurity.com ...\n More emails for blacklanternsecurity.com ...\n\"\"\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_slack.py", "content": "from .test_module_discord import TestDiscord as DiscordBase\n\n\nclass TestSlack(DiscordBase):\n modules_overrides = [\"slack\", \"excavate\", \"badsecrets\", \"httpx\"]\n webhook_url = \"https://hooks.slack.com/services/deadbeef/deadbeef/deadbeef\"\n config_overrides = {\"modules\": {\"slack\": {\"webhook_url\": webhook_url}}}\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_smuggler.py", "content": "from .base import ModuleTestBase\n\nsmuggler_text = r\"\"\"\n ______ _\n / _____) | |\n ( (____ ____ _ _ ____ ____| | _____ ____\n \\____ \\| \\| | | |/ _ |/ _ | || ___ |/ ___)\n _____) ) | | | |_| ( (_| ( (_| | || ____| |\n (______/|_|_|_|____/ \\___ |\\___ |\\_)_____)_|\n (_____(_____|\n\n @defparam v1.1\n\n [+] URL : http://127.0.0.1:8888\n [+] Method : POST\n [+] Endpoint : /\n [+] Configfile : default.py\n [+] Timeout : 5.0 seconds\n [+] Cookies : 1 (Appending to the attack)\n [nameprefix1] : Checking TECL...\n [nameprefix1] : Checking CLTE...\n [nameprefix1] : OK (TECL: 0.61 - 405) (CLTE: 0.62 - 405)\n [tabprefix1] : Checking TECL...git\n [tabprefix1] : Checking CLTE...\n [tabprefix1] : Checking TECL...\n [tabprefix1] : Checking CLTE...\n [tabprefix1] : Checking TECL...\n [tabprefix1] : Checking CLTE...\n [tabprefix1] : Potential CLTE Issue Found - POST @ http://127.0.0.1:8888 - default.py\n [CRITICAL] : CLTE Payload: /home/user/.bbot/tools/smuggler/payloads/http_127.0.0.1_net_CLTE_tabprefix1.txt URL: http://127.0.0.1:8888/\n \"\"\"\n\n\nclass TestSmuggler(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"smuggler\"]\n\n async def setup_after_prep(self, module_test):\n old_run_live = module_test.scan.helpers.run_live\n\n async def smuggler_mock_run_live(*command, **kwargs):\n if \"smuggler\" not in command[0][1]:\n async for l in old_run_live(*command, **kwargs):\n yield l\n else:\n for line in smuggler_text.splitlines():\n yield line\n\n module_test.monkeypatch.setattr(module_test.scan.helpers, \"run_live\", smuggler_mock_run_live)\n\n request_args = {\"uri\": \"/\"}\n respond_args = {\"response_data\": \"alive\"}\n module_test.set_expect_requests(request_args, respond_args)\n\n def check(self, module_test, events):\n assert any(\n e.type == \"FINDING\"\n and \"[HTTP SMUGGLER] [Potential CLTE Issue Found] Technique: [tabprefix1]\" in e.data[\"description\"]\n for e in events\n ), \"Failed to parse mocked command output\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_social.py", "content": "from .base import ModuleTestBase\n\n\nclass TestSocial(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"excavate\", \"social\"]\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\n \"response_data\": \"\"\"\n \n \n \n \n \n \n \n \"\"\"\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n assert 4 == len([e for e in events if e.type == \"SOCIAL\"])\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"SOCIAL\" and e.data[\"platform\"] == \"discord\" and e.data[\"profile_name\"] == \"asdf\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"SOCIAL\"\n and e.data[\"platform\"] == \"docker\"\n and e.data[\"profile_name\"] == \"blacklanternsecurity\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"SOCIAL\"\n and e.data[\"platform\"] == \"github\"\n and e.data[\"profile_name\"] == \"blacklanternsecurity\"\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"SOCIAL\"\n and e.data[\"platform\"] == \"postman\"\n and e.data[\"profile_name\"] == \"blacklanternsecurity\"\n ]\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_speculate.py", "content": "from .base import ModuleTestBase\n\n\nclass TestSpeculate_Subdirectories(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/subdir1/subdir2/\"]\n modules_overrides = [\"httpx\", \"speculate\"]\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": \"alive\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/subdir1/\"}\n respond_args = {\"response_data\": \"alive\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/subdir1/subdir2/\"}\n respond_args = {\"response_data\": \"alive\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n assert any(e.type == \"URL_UNVERIFIED\" and e.data == \"http://127.0.0.1:8888/subdir1/\" for e in events)\n\n\nclass TestSpeculate_OpenPorts(ModuleTestBase):\n targets = [\"evilcorp.com\"]\n modules_overrides = [\"speculate\", \"certspotter\", \"shodan_idb\"]\n config_overrides = {\"speculate\": True}\n\n async def setup_before_prep(self, module_test):\n await module_test.mock_dns(\n {\n \"evilcorp.com\": {\"A\": [\"127.0.254.1\"]},\n \"asdf.evilcorp.com\": {\"A\": [\"127.0.254.2\"]},\n }\n )\n\n module_test.httpx_mock.add_response(\n url=\"https://api.certspotter.com/v1/issuances?domain=evilcorp.com&include_subdomains=true&expand=dns_names\",\n json=[{\"dns_names\": [\"*.asdf.evilcorp.com\"]}],\n )\n\n from bbot.modules.base import BaseModule\n\n class DummyModule(BaseModule):\n _name = \"dummy\"\n watched_events = [\"OPEN_TCP_PORT\"]\n scope_distance_modifier = 10\n accept_dupes = True\n\n async def setup(self):\n self.events = []\n return True\n\n async def handle_event(self, event):\n self.events.append(event)\n\n module_test.scan.modules[\"dummy\"] = DummyModule(module_test.scan)\n\n def check(self, module_test, events):\n events_data = set()\n for e in module_test.scan.modules[\"dummy\"].events:\n events_data.add(e.data)\n assert all(\n x in events_data\n for x in (\"evilcorp.com:80\", \"evilcorp.com:443\", \"asdf.evilcorp.com:80\", \"asdf.evilcorp.com:443\")\n )\n\n\nclass TestSpeculate_OpenPorts_Portscanner(TestSpeculate_OpenPorts):\n targets = [\"evilcorp.com\"]\n modules_overrides = [\"speculate\", \"certspotter\", \"portscan\"]\n config_overrides = {\"speculate\": True}\n\n def check(self, module_test, events):\n events_data = set()\n for e in module_test.scan.modules[\"dummy\"].events:\n events_data.add(e.data)\n assert not any(\n x in events_data\n for x in (\"evilcorp.com:80\", \"evilcorp.com:443\", \"asdf.evilcorp.com:80\", \"asdf.evilcorp.com:443\")\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_splunk.py", "content": "import json\nimport httpx\n\nfrom .base import ModuleTestBase\n\n\nclass TestSplunk(ModuleTestBase):\n downstream_url = \"https://splunk.blacklanternsecurity.fakedomain:1234/services/collector\"\n config_overrides = {\n \"modules\": {\n \"splunk\": {\n \"url\": downstream_url,\n \"hectoken\": \"HECTOKEN\",\n \"index\": \"bbot_index\",\n \"source\": \"bbot_source\",\n }\n }\n }\n\n def verify_data(self, j):\n if not j[\"source\"] == \"bbot_source\":\n return False\n if not j[\"index\"] == \"bbot_index\":\n return False\n data = j[\"event\"]\n if not data[\"data\"] == \"blacklanternsecurity.com\" and data[\"type\"] == \"DNS_NAME\":\n return False\n return True\n\n async def setup_after_prep(self, module_test):\n self.url_correct = False\n self.method_correct = False\n self.got_event = False\n self.headers_correct = False\n\n async def custom_callback(request):\n j = json.loads(request.content)\n if request.url == self.downstream_url:\n self.url_correct = True\n if request.method == \"POST\":\n self.method_correct = True\n if \"Authorization\" in request.headers:\n self.headers_correct = True\n if self.verify_data(j):\n self.got_event = True\n return httpx.Response(\n status_code=200,\n )\n\n module_test.httpx_mock.add_callback(custom_callback)\n module_test.httpx_mock.add_callback(custom_callback)\n module_test.httpx_mock.add_response()\n\n def check(self, module_test, events):\n assert self.got_event is True\n assert self.headers_correct is True\n assert self.method_correct is True\n assert self.url_correct is True\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_sqlite.py", "content": "import sqlite3\nfrom .base import ModuleTestBase\n\n\nclass TestSQLite(ModuleTestBase):\n targets = [\"evilcorp.com\"]\n\n def check(self, module_test, events):\n sqlite_output_file = module_test.scan.home / \"output.sqlite\"\n assert sqlite_output_file.exists(), \"SQLite output file not found\"\n with sqlite3.connect(sqlite_output_file) as db:\n cursor = db.cursor()\n results = cursor.execute(\"SELECT * FROM event\").fetchall()\n assert len(results) == 3, \"No events found in SQLite database\"\n results = cursor.execute(\"SELECT * FROM scan\").fetchall()\n assert len(results) == 1, \"No scans found in SQLite database\"\n results = cursor.execute(\"SELECT * FROM target\").fetchall()\n assert len(results) == 1, \"No targets found in SQLite database\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_sslcert.py", "content": "from .base import ModuleTestBase\n\n\nclass TestSSLCert(ModuleTestBase):\n targets = [\"127.0.0.1:9999\", \"bbottest.notreal\"]\n config_overrides = {\"scope\": {\"report_distance\": 1}}\n\n def check(self, module_test, events):\n assert len(events) == 7\n assert 1 == len(\n [\n e\n for e in events\n if e.data == \"www.bbottest.notreal\" and str(e.module) == \"sslcert\" and e.scope_distance == 0\n ]\n ), \"Failed to detect subject alternate name (SAN)\"\n assert 1 == len(\n [e for e in events if e.data == \"test.notreal\" and str(e.module) == \"sslcert\" and e.scope_distance == 1]\n ), \"Failed to detect main subject\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_stdout.py", "content": "import json\n\nfrom .base import ModuleTestBase\n\n\nclass TestStdout(ModuleTestBase):\n modules_overrides = [\"stdout\"]\n\n def check(self, module_test, events):\n out, err = module_test.capsys.readouterr()\n assert out.startswith(\"[SCAN] \\tteststdout\")\n assert \"[DNS_NAME] \\tblacklanternsecurity.com\\tTARGET\" in out\n\n\nclass TestStdoutEventTypes(TestStdout):\n config_overrides = {\"modules\": {\"stdout\": {\"event_types\": [\"DNS_NAME\"]}}}\n\n def check(self, module_test, events):\n out, err = module_test.capsys.readouterr()\n assert len(out.splitlines()) == 1\n assert out.startswith(\"[DNS_NAME] \\tblacklanternsecurity.com\\tTARGET\")\n\n\nclass TestStdoutEventFields(TestStdout):\n config_overrides = {\"modules\": {\"stdout\": {\"event_types\": [\"DNS_NAME\"], \"event_fields\": [\"data\"]}}}\n\n def check(self, module_test, events):\n out, err = module_test.capsys.readouterr()\n assert out == \"blacklanternsecurity.com\\n\"\n\n\nclass TestStdoutJSON(TestStdout):\n config_overrides = {\n \"modules\": {\n \"stdout\": {\n \"format\": \"json\",\n }\n }\n }\n\n def check(self, module_test, events):\n out, err = module_test.capsys.readouterr()\n lines = out.splitlines()\n assert len(lines) == 3\n for i, line in enumerate(lines):\n event = json.loads(line)\n if i == 0:\n assert event[\"type\"] == \"SCAN\"\n elif i == 1:\n assert event[\"type\"] == \"DNS_NAME\" and event[\"data\"] == \"blacklanternsecurity.com\"\n if i == 2:\n assert event[\"type\"] == \"SCAN\"\n\n\nclass TestStdoutJSONFields(TestStdout):\n config_overrides = {\"modules\": {\"stdout\": {\"format\": \"json\", \"event_fields\": [\"data\", \"module_sequence\"]}}}\n\n def check(self, module_test, events):\n out, err = module_test.capsys.readouterr()\n lines = out.splitlines()\n assert len(lines) == 3\n for line in lines:\n event = json.loads(line)\n assert set(event) == {\"data\", \"module_sequence\"}\n\n\nclass TestStdoutDupes(TestStdout):\n targets = [\"blacklanternsecurity.com\", \"127.0.0.2\"]\n config_overrides = {\n \"dns\": {\"minimal\": False},\n \"modules\": {\n \"stdout\": {\n \"event_types\": [\"DNS_NAME\", \"IP_ADDRESS\"],\n }\n },\n }\n\n async def setup_after_prep(self, module_test):\n await module_test.mock_dns({\"blacklanternsecurity.com\": {\"A\": [\"127.0.0.2\"]}})\n\n def check(self, module_test, events):\n out, err = module_test.capsys.readouterr()\n lines = out.splitlines()\n assert len(lines) == 3\n assert out.count(\"[IP_ADDRESS] \\t127.0.0.2\") == 2\n\n\nclass TestStdoutNoDupes(TestStdoutDupes):\n config_overrides = {\n \"dns\": {\"minimal\": False},\n \"modules\": {\n \"stdout\": {\n \"event_types\": [\"DNS_NAME\", \"IP_ADDRESS\"],\n \"accept_dupes\": False,\n }\n },\n }\n\n def check(self, module_test, events):\n out, err = module_test.capsys.readouterr()\n lines = out.splitlines()\n assert len(lines) == 2\n assert out.count(\"[IP_ADDRESS] \\t127.0.0.2\") == 1\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_subdomaincenter.py", "content": "from .base import ModuleTestBase\n\n\nclass TestSubdomainCenter(ModuleTestBase):\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.subdomain.center/?domain=blacklanternsecurity.com\",\n json=[\"asdf.blacklanternsecurity.com\", \"zzzz.blacklanternsecurity.com\"],\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n assert any(e.data == \"zzzz.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_subdomainradar.py", "content": "from .base import ModuleTestBase\n\n\nclass TestSubDomainRadar(ModuleTestBase):\n config_overrides = {\"modules\": {\"subdomainradar\": {\"api_key\": \"asdf\"}}}\n\n async def setup_before_prep(self, module_test):\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.com\": {\"A\": [\"127.0.0.88\"]},\n \"www.blacklanternsecurity.com\": {\"A\": [\"127.0.0.88\"]},\n \"asdf.blacklanternsecurity.com\": {\"A\": [\"127.0.0.88\"]},\n }\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.subdomainradar.io/profile\",\n match_headers={\"Authorization\": \"Bearer asdf\"},\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.subdomainradar.io/enumerate\",\n method=\"POST\",\n json={\n \"tasks\": {\"blacklanternsecurity.com\": \"86de4531-0a67-41fe-b5e4-8ce8207d6245\"},\n \"message\": \"Tasks initiated\",\n },\n match_headers={\"Authorization\": \"Bearer asdf\"},\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.subdomainradar.io/tasks/86de4531-0a67-41fe-b5e4-8ce8207d6245\",\n match_headers={\"Authorization\": \"Bearer asdf\"},\n json={\n \"task_id\": \"86de4531-0a67-41fe-b5e4-8ce8207d6245\",\n \"status\": \"completed\",\n \"domain\": \"blacklanternsecurity.com\",\n \"subdomains\": [\n {\n \"subdomain\": \"www.blacklanternsecurity.com\",\n \"ip\": None,\n \"reverse_dns\": [],\n \"country\": None,\n \"timestamp\": None,\n },\n {\n \"subdomain\": \"asdf.blacklanternsecurity.com\",\n \"ip\": None,\n \"reverse_dns\": [],\n \"country\": None,\n \"timestamp\": None,\n },\n ],\n \"total_subdomains\": 2,\n \"rank\": None,\n \"whois\": {\n \"domain_name\": [\"BLACKLANTERNSECURITY.COM\", \"blacklanternsecurity.com\"],\n \"registrar\": \"MarkMonitor, Inc.\",\n \"creation_date\": [\"1992-11-04T05:00:00\", \"1992-11-04T05:00:00+00:00\"],\n \"expiration_date\": [\"2026-11-03T05:00:00\", \"2026-11-03T00:00:00+00:00\"],\n \"last_updated\": [\"2024-10-02T10:15:20\", \"2024-10-02T10:15:20+00:00\"],\n \"status\": [\n \"clientDeleteProhibited https://icann.org/epp#clientDeleteProhibited\",\n \"clientTransferProhibited https://icann.org/epp#clientTransferProhibited\",\n \"clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited\",\n \"serverDeleteProhibited https://icann.org/epp#serverDeleteProhibited\",\n \"serverTransferProhibited https://icann.org/epp#serverTransferProhibited\",\n \"serverUpdateProhibited https://icann.org/epp#serverUpdateProhibited\",\n \"clientUpdateProhibited (https://www.icann.org/epp#clientUpdateProhibited)\",\n \"clientTransferProhibited (https://www.icann.org/epp#clientTransferProhibited)\",\n \"clientDeleteProhibited (https://www.icann.org/epp#clientDeleteProhibited)\",\n \"serverUpdateProhibited (https://www.icann.org/epp#serverUpdateProhibited)\",\n \"serverTransferProhibited (https://www.icann.org/epp#serverTransferProhibited)\",\n \"serverDeleteProhibited (https://www.icann.org/epp#serverDeleteProhibited)\",\n ],\n \"nameservers\": [\n \"A1-12.AKAM.NET\",\n \"A10-67.AKAM.NET\",\n \"A12-64.AKAM.NET\",\n \"A28-65.AKAM.NET\",\n \"A7-66.AKAM.NET\",\n \"A9-67.AKAM.NET\",\n \"EDNS69.ULTRADNS.BIZ\",\n \"EDNS69.ULTRADNS.COM\",\n \"EDNS69.ULTRADNS.NET\",\n \"EDNS69.ULTRADNS.ORG\",\n \"edns69.ultradns.biz\",\n \"a12-64.akam.net\",\n \"edns69.ultradns.net\",\n \"edns69.ultradns.org\",\n \"a10-67.akam.net\",\n \"a28-65.akam.net\",\n \"a9-67.akam.net\",\n \"a1-12.akam.net\",\n \"a7-66.akam.net\",\n \"edns69.ultradns.com\",\n ],\n \"emails\": [\n \"abusecomplaints@markmonitor.com\",\n \"admin@dnstinations.com\",\n \"whoisrequest@markmonitor.com\",\n ],\n \"dnssec\": \"unsigned\",\n \"org\": \"DNStination Inc.\",\n \"address\": \"3450 Sacramento Street, Suite 405\",\n \"city\": \"San Francisco\",\n \"state\": \"CA\",\n \"zipcode\": None,\n \"country\": \"US\",\n },\n \"enumerators\": [\"Aquarius Enumerator\", \"Beta Enumerator\", \"Chi Enumerator\", \"Eta Enumerator\"],\n \"timestamp\": \"2024-10-06T02:48:10.075636\",\n \"error\": None,\n \"is_notification\": False,\n \"notification_domain_id\": None,\n \"demo\": False,\n \"user_id\": 49,\n \"time_to_finish\": 41,\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.subdomainradar.io/enumerators/groups\",\n match_headers={\"Authorization\": \"Bearer asdf\"},\n json=[\n {\n \"id\": \"1\",\n \"name\": \"Fast\",\n \"description\": \"Enumerators optimized for high-speed scanning and rapid data collection\",\n \"enumerators\": [\n {\"display_name\": \"Beta Enumerator\"},\n {\"display_name\": \"Chi Enumerator\"},\n {\"display_name\": \"Aquarius Enumerator\"},\n {\"display_name\": \"Eta Enumerator\"},\n ],\n },\n {\n \"id\": \"2\",\n \"name\": \"Medium\",\n \"description\": \"Enumerators balanced for moderate speed with a focus on thoroughness\",\n \"enumerators\": [\n {\"display_name\": \"Kappa Enumerator\"},\n {\"display_name\": \"Lambda Enumerator\"},\n {\"display_name\": \"Mu Enumerator\"},\n {\"display_name\": \"Pi Enumerator\"},\n {\"display_name\": \"Tau Enumerator\"},\n {\"display_name\": \"Beta Enumerator\"},\n {\"display_name\": \"Chi Enumerator\"},\n {\"display_name\": \"Psi Enumerator\"},\n {\"display_name\": \"Aquarius Enumerator\"},\n {\"display_name\": \"Zeta Enumerator\"},\n {\"display_name\": \"Eta Enumerator\"},\n ],\n },\n {\n \"id\": \"3\",\n \"name\": \"Deep\",\n \"description\": \"Enumerators designed for exhaustive searches and in-depth data analysis\",\n \"enumerators\": [\n {\"display_name\": \"Alpha Enumerator\"},\n {\"display_name\": \"Kappa Enumerator\"},\n {\"display_name\": \"Lambda Enumerator\"},\n {\"display_name\": \"Mu Enumerator\"},\n {\"display_name\": \"Nu Enumerator\"},\n {\"display_name\": \"Xi Enumerator\"},\n {\"display_name\": \"Pi Enumerator\"},\n {\"display_name\": \"Rho Enumerator\"},\n {\"display_name\": \"Sigma Enumerator\"},\n {\"display_name\": \"Tau Enumerator\"},\n {\"display_name\": \"Beta Enumerator\"},\n {\"display_name\": \"Chi Enumerator\"},\n {\"display_name\": \"Omega Enumerator\"},\n {\"display_name\": \"Psi Enumerator\"},\n {\"display_name\": \"Phi Enumerator\"},\n {\"display_name\": \"Axon Enumerator\"},\n {\"display_name\": \"Aquarius Enumerator\"},\n {\"display_name\": \"Pegasus Enumerator\"},\n {\"display_name\": \"Petra Enumerator\"},\n {\"display_name\": \"Oasis Enumerator\"},\n {\"display_name\": \"Mike Enumerator\"},\n {\"display_name\": \"Cat Enumerator\"},\n {\"display_name\": \"Brutus Enumerator\"},\n {\"display_name\": \"Dee Enumerator\"},\n {\"display_name\": \"Jul Enumerator\"},\n {\"display_name\": \"Eve Enumerator\"},\n {\"display_name\": \"Frank Enumerator\"},\n {\"display_name\": \"Gus Enumerator\"},\n {\"display_name\": \"Hank Enumerator\"},\n {\"display_name\": \"Delta Enumerator\"},\n {\"display_name\": \"Ivy Enumerator\"},\n {\"display_name\": \"Jack Enumerator\"},\n {\"display_name\": \"Karl Enumerator\"},\n {\"display_name\": \"Liam Enumerator\"},\n {\"display_name\": \"Nora Enumerator\"},\n {\"display_name\": \"Mars Enumerator\"},\n {\"display_name\": \"Neptune Enumerator\"},\n {\"display_name\": \"Orion Enumerator\"},\n {\"display_name\": \"Oedipus Enumerator\"},\n {\"display_name\": \"Pandora Enumerator\"},\n {\"display_name\": \"Epsilon Enumerator\"},\n {\"display_name\": \"Zeta Enumerator\"},\n {\"display_name\": \"Eta Enumerator\"},\n {\"display_name\": \"Theta Enumerator\"},\n {\"display_name\": \"Iota Enumerator\"},\n ],\n },\n ],\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"www.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain #1\"\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain #2\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_subdomains.py", "content": "from .base import ModuleTestBase\n\n\nclass TestSubdomains(ModuleTestBase):\n modules_overrides = [\"subdomains\", \"subdomaincenter\"]\n\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.subdomain.center/?domain=blacklanternsecurity.com\",\n json=[\"asdfasdf.blacklanternsecurity.com\", \"zzzzzzzz.blacklanternsecurity.com\"],\n )\n\n def check(self, module_test, events):\n sub_file = module_test.scan.home / \"subdomains.txt\"\n subdomains = set(open(sub_file).read().splitlines())\n assert subdomains == {\"blacklanternsecurity.com\"}\n\n\nclass TestSubdomainsUnresolved(TestSubdomains):\n config_overrides = {\"modules\": {\"subdomains\": {\"include_unresolved\": True}}}\n\n def check(self, module_test, events):\n sub_file = module_test.scan.home / \"subdomains.txt\"\n subdomains = set(open(sub_file).read().splitlines())\n assert subdomains == {\n \"blacklanternsecurity.com\",\n \"asdfasdf.blacklanternsecurity.com\",\n \"zzzzzzzz.blacklanternsecurity.com\",\n }\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_teams.py", "content": "import httpx\n\nfrom .test_module_discord import TestDiscord as DiscordBase\n\n\nclass TestTeams(DiscordBase):\n modules_overrides = [\"teams\", \"excavate\", \"badsecrets\", \"httpx\"]\n\n webhook_url = \"https://evilcorp.webhook.office.com/webhookb2/deadbeef@deadbeef/IncomingWebhook/deadbeef/deadbeef\"\n config_overrides = {\"modules\": {\"teams\": {\"webhook_url\": webhook_url, \"retries\": 5}}}\n\n async def setup_after_prep(self, module_test):\n self.custom_setup(module_test)\n\n def custom_response(request: httpx.Request):\n module_test.request_count += 1\n if module_test.request_count == 2:\n return httpx.Response(status_code=429, headers={\"Retry-After\": \"0.01\"})\n elif module_test.request_count == 3:\n return httpx.Response(\n status_code=400,\n json={\n \"error\": {\n \"code\": \"WorkflowTriggerIsNotEnabled\",\n \"message\": \"Could not execute workflow 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx' trigger 'manual' with state 'Disabled': trigger is not enabled.\",\n }\n },\n )\n else:\n return httpx.Response(status_code=200)\n\n module_test.httpx_mock.add_callback(custom_response, url=self.webhook_url)\n\n def check(self, module_test, events):\n vulns = [e for e in events if e.type == \"VULNERABILITY\"]\n findings = [e for e in events if e.type == \"FINDING\"]\n assert len(findings) == 1\n assert len(vulns) == 2\n assert module_test.request_count == 5\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_telerik.py", "content": "import re\nfrom .base import ModuleTestBase\n\n\nclass TestTelerik(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\", \"http://127.0.0.1:8888/telerik.aspx\"]\n modules_overrides = [\"httpx\", \"telerik\"]\n config_overrides = {\"modules\": {\"telerik\": {\"exploit_RAU_crypto\": True}}}\n\n async def setup_before_prep(self, module_test):\n # Simulate Telerik.Web.UI.WebResource.axd?type=rau detection\n expect_args = {\"method\": \"GET\", \"uri\": \"/Telerik.Web.UI.WebResource.axd\", \"query_string\": \"type=rau\"}\n respond_args = {\n \"response_data\": '{ \"message\" : \"RadAsyncUpload handler is registered succesfully, however, it may not be accessed directly.\" }'\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Simulate Vulnerable Telerik.Web.UI.WebResource.axd\n vuln_data = \"ATTu5i4R+ViNFYO6kst0jC11wM/1iqH+W/isjhaDjNuCI7eJ/BY5d1E9eqZK27CJCMuon9u8/hgRIM/cTlgLlv4qOYjPBjs81Y3dAZAdtIr3TXiCmZi9M09a1BYMxjvGKfVky3b7PoOppeWS/3rglTwL1e8oyqLGx2NKUH5y8Cd+kLKV2f31J1sV4I5HTDKgDmvziJp3zlDrCb0Fi9ilKH+O1cbVx6SdBop/U30FxLaB/QIbt2N1rQHREJ5Skpgo7dilPxzBaTObdBhCVyB/FiJhenS/0u3h0Mpi6+A40SylICcyyxQha7+Uh7lEJ8Ne+2eTs4WqcaaQbvIhy7oHc+D0soxRKMZRjo7Up+UWHQJJh6KtWSCxUESNSdNcxjPQZE9HqsPlldVlkeC+ehSGce5bR0Ylots6Iz1OoCgMEWwxByeG3VzgxF6XpitL61A1hFcNo9euSTnCfOWh0vrQHON7DN5LpM9xr7SoD0Dnu01hZ9NS1PHhPLyN5WS87u5qdZp/z3Sxwc3wawIdo62RNf4Iz2gAKJZnPfxrE1mRn5kBe7f6O44rcuv6lcdao/DGlwbERKwRI6/n+FxGmc7H5iEKyihIwS2XUoOgsYTx5CWCDM8CuOXTk+H5fPYp9APRPbkD1IS9I/vRmvNPwWsgv8/7DzttqdBsGxiZJfCw1uZ7KSVmbItgXPAcscNxGEMaHXyJzkAl/mlM5/t/YSejwYoSW6jFfQcLdaVx2dpIpl5UmmQjFedzKeiNqpZDCk4yzXFHX24XUODYMJDtIJK2Hz1KTZmFG+LAOJjB9QOI58hFAnytcKay+JWFrzah/IvoNZxJUtlYdxw0YEyKs/ExET7AXgYQN0S+8j2PfaMMpzDSctTqpp5XBFV4Mt718GiqVnQJtWQv2p9Xl8XXOerBthbzzAciVcB8AV2WfZ51W3e4aX4kcyT/sCJhm7NR5WrNG5mX/ns0TTnGnzlPYhJcbu8uMFjMGDpXuhVyroJ7wmZucaIvesg0h5Y9cMEFviqsdy15vjMzFh+v9uO9Vicf6n9Z9JGSpWKE8wer2JU5b53Zw0cTfulAAffLWXnzOnfu&6R/cGaqQeHVAzdJ9wTFOyCsrMSTtqcjLe8AHwiPckPDUwecnJyNlkDYwDQpxGYQ9hs6YxhupK310sbCbtXB4H6Dz5rGNL40nkkyo4j2clmRr08jtFsPQ0RpE5BGsulPT3l0MxyAvPFMs8bMybUyAP+9RB9LoHE3Xo8BqDadX3HQakpPfGtiDMp+wxkWRgaNpCnXeY1QewWTF6z/duLzbu6CT6s+H4HgBHrOLTpemC2PvP2bDm0ySPHLdpapLYxU8nIYjLKIyYJgwv9S9jNckIVpcGVTWVul7CauCKxAB2mMnM9jJi8zfFwKajT5d2d9XfpkiVMrdlmikSB/ehyX1wQ==\"\n expect_args = {\n \"method\": \"POST\",\n \"uri\": \"/Telerik.Web.UI.WebResource.axd\",\n \"query_string\": \"type=rau\",\n \"data\": vuln_data,\n }\n respond_args = {\n \"response_data\": '{\"fileInfo\":{\"FileName\":\"RAU_crypto.bypass\",\"ContentType\":\"text/html\",\"ContentLength\":5,\"DateJson\":\"2019-01-02T03:04:05.067Z\",\"Index\":0}, \"metaData\":\"CS8S/Z0J/b2982DRxDin0BBslA7fI0cWMuWlPu4W3FkE4tKaVoIEiAOtVlJ6D+0RQsfu8ox6gvMYxceQ0LtWyTkQBaIUa8LgLQg05DMaQuufHNx0YQ2ACi5neqDBvduj2MGiSGC0hNKzSWsHystZGUfFPLTZuJXYnff+WXurecuRzSI7d4Q1aj0bcTKKvfyQtH+fsTEafWRRZ99X/xgi4ON2OsRZ738uQHw7pQT2e1v7AtN46mxO/BmhEuZQr6m6HEvxK0pJRNkBhFUiQ+poeu8j3JzicOjvPDwFE4Rjqf3RVILt83XZrju2VpRIJqAEtf//znhH8BhT5BWvhnRo+J3ML5qoZLa2joE/QK8Ctf3UPvAFkHIUMdOH2mLNgZ+U87tdVE6fYfzvphZsLxmJRG45H8ZTZuYhJbOfei2LQ4fqHmr7p8KpJNVqoz/ev1dnBclAf5ayb40qJKEVsGXIbWEbIZwg7TTsLFc29aP7DPg=\" }'\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Simulate SpellCheckHandler detection\n expect_args = {\"method\": \"GET\", \"uri\": \"/Telerik.Web.UI.SpellCheckHandler.axd\"}\n respond_args = {\"status\": 500}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Simulate SpellCheckHandler false positive detection\n expect_args = {\"method\": \"GET\", \"uri\": \"/AAAAAAAAAAAAAA.axd\"}\n respond_args = {\"status\": 200}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Simulate DialogHandler detection\n expect_args = {\"method\": \"GET\", \"uri\": \"/App_Master/Telerik.Web.UI.DialogHandler.aspx\"}\n respond_args = {\n \"response_data\": '
Cannot deserialize dialog parameters. Please refresh the editor page.
Error Message:Invalid length for a Base-64 char array or string.
'\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Simulate ChartImage.axd Detection\n expect_args = {\n \"method\": \"GET\",\n \"uri\": \"/ChartImage.axd\",\n \"query_string\": \"ImageName=bqYXJAqm315eEd6b%2bY4%2bGqZpe7a1kY0e89gfXli%2bjFw%3d\",\n }\n respond_args = {\"status\": 200}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/ChartImage.axd\", \"query_string\": \"ImageName=\"}\n respond_args = {\"status\": 500}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Simulate Dialog Parameters in URL\n expect_args = {\"method\": \"GET\", \"uri\": \"/telerik.aspx\"}\n respond_args = {\"response_data\": '{\"ImageManager\":{\"SerializedParameters\":\"MBwZB\"}'}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Fallback\n expect_args = {\"uri\": re.compile(r\"^/\\w{10}$\")}\n respond_args = {\"status\": 200}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"telerik\"].helpers.rand_string = lambda *args, **kwargs: \"AAAAAAAAAAAAAA\"\n module_test.scan.modules[\"telerik\"].telerikVersions = [\"2014.2.724\", \"2014.3.1024\", \"2015.1.204\"]\n module_test.scan.modules[\"telerik\"].DialogHandlerUrls = [\n \"Admin/ServerSide/Telerik.Web.UI.DialogHandler.aspx\",\n \"App_Master/Telerik.Web.UI.DialogHandler.aspx\",\n \"AsiCommon/Controls/ContentManagement/ContentDesigner/Telerik.Web.UI.DialogHandler.aspx\",\n ]\n\n def check(self, module_test, events):\n telerik_axd_detection = False\n telerik_axd_vulnerable = False\n telerik_spellcheck_detection = False\n telerik_dialoghandler_detection = False\n telerik_chartimage_detection = False\n telerik_http_response_parameters_detection = False\n\n for e in events:\n if e.type == \"FINDING\" and \"Telerik RAU AXD Handler detected\" in e.data[\"description\"]:\n e.data[\"description\"]\n telerik_axd_detection = True\n continue\n\n if e.type == \"VULNERABILITY\" and \"Confirmed Vulnerable Telerik (version: 2014.3.1024)\":\n telerik_axd_vulnerable = True\n continue\n\n if e.type == \"FINDING\" and \"Telerik DialogHandler detected\" in e.data[\"description\"]:\n telerik_dialoghandler_detection = True\n continue\n\n if e.type == \"FINDING\" and \"Telerik SpellCheckHandler detected\" in e.data[\"description\"]:\n telerik_spellcheck_detection = True\n continue\n\n if e.type == \"FINDING\" and \"Telerik ChartImage AXD Handler Detected\" in e.data[\"description\"]:\n telerik_chartimage_detection = True\n continue\n\n if (\n e.type == \"FINDING\"\n and \"Telerik DialogHandler [SerializedParameters] Detected in HTTP Response\" in e.data[\"description\"]\n ):\n telerik_http_response_parameters_detection = True\n continue\n\n assert telerik_axd_detection, \"Telerik AXD detection failed\"\n assert telerik_axd_vulnerable, \"Telerik vulnerable AXD detection failed\"\n assert telerik_spellcheck_detection, \"Telerik spellcheck detection failed\"\n assert telerik_dialoghandler_detection, \"Telerik dialoghandler detection failed\"\n assert telerik_chartimage_detection, \"Telerik chartimage detection failed\"\n assert telerik_http_response_parameters_detection, \"Telerik SerializedParameters detection failed\"\n\n\nclass TestTelerikDialogHandler_includesubdirs(TestTelerik):\n targets = [\"http://127.0.0.1:8888/\", \"http://127.0.0.1:8888/temp/\"]\n config_overrides = {\n \"modules\": {\n \"telerik\": {\n \"include_subdirs\": True,\n },\n }\n }\n modules_overrides = [\"httpx\", \"telerik\"]\n\n async def setup_before_prep(self, module_test):\n # Simulate NO SpellCheckHandler detection (not testing for that with this test)\n expect_args = {\"method\": \"GET\", \"uri\": \"/Telerik.Web.UI.SpellCheckHandler.axd\"}\n respond_args = {\"status\": 404}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Simulate DialogHandler detection\n expect_args = {\"method\": \"GET\", \"uri\": \"/App_Master/Telerik.Web.UI.DialogHandler.aspx\"}\n respond_args = {\n \"response_data\": '
Cannot deserialize dialog parameters. Please refresh the editor page.
Error Message:Invalid length for a Base-64 char array or string.
'\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Simulate DialogHandler detection (in /temp)\n expect_args = {\"method\": \"GET\", \"uri\": \"/temp/App_Master/Telerik.Web.UI.DialogHandler.aspx\"}\n respond_args = {\n \"response_data\": '
Cannot deserialize dialog parameters. Please refresh the editor page.
Error Message:Invalid length for a Base-64 char array or string.
'\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Simulate /temp directory detection\n expect_args = {\"method\": \"GET\", \"uri\": \"/temp/\"}\n respond_args = {\"response_data\": \"Temporary directory found\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n # Fallback\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": \"alive\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n async def setup_after_prep(self, module_test):\n module_test.scan.modules[\"telerik\"].telerikVersions = [\"2014.2.724\", \"2014.3.1024\", \"2015.1.204\"]\n module_test.scan.modules[\"telerik\"].DialogHandlerUrls = [\n \"App_Master/Telerik.Web.UI.DialogHandler.aspx\",\n ]\n\n def check(self, module_test, events):\n # Check if the expected requests were made\n finding_count = sum(\n 1 for e in events if e.type == \"FINDING\" and \"Telerik DialogHandler detected\" in e.data[\"description\"]\n )\n assert finding_count == 2, \"Expected 2 FINDING events (root and /temp), got {finding_count}\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_trickest.py", "content": "from .base import ModuleTestBase\n\n\nclass TestTrickest(ModuleTestBase):\n config_overrides = {\"modules\": {\"trickest\": {\"api_key\": \"deadbeef\"}}}\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be/dataset\",\n match_headers={\"Authorization\": \"Token deadbeef\"},\n json={},\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be/view?q=hostname%20~%20%22.blacklanternsecurity.com%22&dataset_id=a0a49ca9-03bb-45e0-aa9a-ad59082ebdfc&limit=50&offset=0&select=hostname&orderby=hostname\",\n match_headers={\"Authorization\": \"Token deadbeef\"},\n json={\"results\": [{\"hostname\": \"asdf.blacklanternsecurity.com\"}]},\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be/view?q=hostname%20~%20%22.blacklanternsecurity.com%22&dataset_id=a0a49ca9-03bb-45e0-aa9a-ad59082ebdfc&limit=50&offset=50&select=hostname&orderby=hostname\",\n match_headers={\"Authorization\": \"Token deadbeef\"},\n json={\"results\": [{\"hostname\": \"www.blacklanternsecurity.com\"}]},\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n assert any(e.data == \"www.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_trufflehog.py", "content": "import io\nimport shutil\nimport zipfile\nimport tarfile\nimport subprocess\nfrom copy import copy\nfrom pathlib import Path\n\nfrom .base import ModuleTestBase\nfrom bbot.test.bbot_fixtures import bbot_test_dir\n\n\nclass TestTrufflehog(ModuleTestBase):\n download_dir = bbot_test_dir / \"test_trufflehog\"\n config_overrides = {\n \"modules\": {\n \"postman_download\": {\"api_key\": \"asdf\", \"output_folder\": str(download_dir)},\n \"docker_pull\": {\"output_folder\": str(download_dir)},\n \"github_org\": {\"api_key\": \"asdf\"},\n \"git_clone\": {\"output_folder\": str(download_dir)},\n }\n }\n modules_overrides = [\n \"github_org\",\n \"speculate\",\n \"git_clone\",\n \"github_workflows\",\n \"dockerhub\",\n \"docker_pull\",\n \"postman\",\n \"postman_download\",\n \"trufflehog\",\n ]\n\n file_content = \"Verifiable Secret:\\nhttps://admin:admin@the-internet.herokuapp.com/basic_auth\\n\\nUnverifiable Secret:\\nhttps://admin:admin@internal.host.com\"\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://api.github.com/zen\", match_headers={\"Authorization\": \"token asdf\"}\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/me\",\n json={\n \"user\": {\n \"id\": 000000,\n \"username\": \"test_key\",\n \"email\": \"blacklanternsecurity@test.com\",\n \"fullName\": \"Test Key\",\n \"avatar\": \"\",\n \"isPublic\": True,\n \"teamId\": 0,\n \"teamDomain\": \"\",\n \"roles\": [\"user\"],\n },\n \"operations\": [\n {\"name\": \"api_object_usage\", \"limit\": 3, \"usage\": 0, \"overage\": 0},\n {\"name\": \"collection_run_limit\", \"limit\": 25, \"usage\": 0, \"overage\": 0},\n {\"name\": \"file_storage_limit\", \"limit\": 20, \"usage\": 0, \"overage\": 0},\n {\"name\": \"flow_count\", \"limit\": 5, \"usage\": 0, \"overage\": 0},\n {\"name\": \"flow_requests\", \"limit\": 5000, \"usage\": 0, \"overage\": 0},\n {\"name\": \"performance_test_limit\", \"limit\": 25, \"usage\": 0, \"overage\": 0},\n {\"name\": \"postbot_calls\", \"limit\": 50, \"usage\": 0, \"overage\": 0},\n {\"name\": \"reusable_packages\", \"limit\": 3, \"usage\": 0, \"overage\": 0},\n {\"name\": \"test_data_retrieval\", \"limit\": 1000, \"usage\": 0, \"overage\": 0},\n {\"name\": \"test_data_storage\", \"limit\": 10, \"usage\": 0, \"overage\": 0},\n {\"name\": \"mock_usage\", \"limit\": 1000, \"usage\": 0, \"overage\": 0},\n {\"name\": \"monitor_request_runs\", \"limit\": 1000, \"usage\": 0, \"overage\": 0},\n {\"name\": \"api_usage\", \"limit\": 1000, \"usage\": 0, \"overage\": 0},\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.github.com/orgs/blacklanternsecurity\",\n match_headers={\"Authorization\": \"token asdf\"},\n json={\n \"login\": \"blacklanternsecurity\",\n \"id\": 25311592,\n \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjI1MzExNTky\",\n \"url\": \"https://api.github.com/orgs/blacklanternsecurity\",\n \"repos_url\": \"https://api.github.com/orgs/blacklanternsecurity/repos\",\n \"events_url\": \"https://api.github.com/orgs/blacklanternsecurity/events\",\n \"hooks_url\": \"https://api.github.com/orgs/blacklanternsecurity/hooks\",\n \"issues_url\": \"https://api.github.com/orgs/blacklanternsecurity/issues\",\n \"members_url\": \"https://api.github.com/orgs/blacklanternsecurity/members{/member}\",\n \"public_members_url\": \"https://api.github.com/orgs/blacklanternsecurity/public_members{/member}\",\n \"avatar_url\": \"https://avatars.githubusercontent.com/u/25311592?v=4\",\n \"description\": \"Security Organization\",\n \"name\": \"Black Lantern Security\",\n \"company\": None,\n \"blog\": \"www.blacklanternsecurity.com\",\n \"location\": \"Charleston, SC\",\n \"email\": None,\n \"twitter_username\": None,\n \"is_verified\": False,\n \"has_organization_projects\": True,\n \"has_repository_projects\": True,\n \"public_repos\": 70,\n \"public_gists\": 0,\n \"followers\": 415,\n \"following\": 0,\n \"html_url\": \"https://github.com/blacklanternsecurity\",\n \"created_at\": \"2017-01-24T00:14:46Z\",\n \"updated_at\": \"2022-03-28T11:39:03Z\",\n \"archived_at\": None,\n \"type\": \"Organization\",\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.github.com/orgs/blacklanternsecurity/repos?per_page=100&page=1\",\n match_headers={\"Authorization\": \"token asdf\"},\n json=[\n {\n \"id\": 459780477,\n \"node_id\": \"R_kgDOG2exfQ\",\n \"name\": \"test_keys\",\n \"full_name\": \"blacklanternsecurity/test_keys\",\n \"private\": False,\n \"owner\": {\n \"login\": \"blacklanternsecurity\",\n \"id\": 79229934,\n \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjc5MjI5OTM0\",\n \"avatar_url\": \"https://avatars.githubusercontent.com/u/79229934?v=4\",\n \"gravatar_id\": \"\",\n \"url\": \"https://api.github.com/users/blacklanternsecurity\",\n \"html_url\": \"https://github.com/blacklanternsecurity\",\n \"followers_url\": \"https://api.github.com/users/blacklanternsecurity/followers\",\n \"following_url\": \"https://api.github.com/users/blacklanternsecurity/following{/other_user}\",\n \"gists_url\": \"https://api.github.com/users/blacklanternsecurity/gists{/gist_id}\",\n \"starred_url\": \"https://api.github.com/users/blacklanternsecurity/starred{/owner}{/repo}\",\n \"subscriptions_url\": \"https://api.github.com/users/blacklanternsecurity/subscriptions\",\n \"organizations_url\": \"https://api.github.com/users/blacklanternsecurity/orgs\",\n \"repos_url\": \"https://api.github.com/users/blacklanternsecurity/repos\",\n \"events_url\": \"https://api.github.com/users/blacklanternsecurity/events{/privacy}\",\n \"received_events_url\": \"https://api.github.com/users/blacklanternsecurity/received_events\",\n \"type\": \"Organization\",\n \"site_admin\": False,\n },\n \"html_url\": \"https://github.com/blacklanternsecurity/test_keys\",\n \"description\": None,\n \"fork\": False,\n \"url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys\",\n \"forks_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/forks\",\n \"keys_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/keys{/key_id}\",\n \"collaborators_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/collaborators{/collaborator}\",\n \"teams_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/teams\",\n \"hooks_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/hooks\",\n \"issue_events_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/issues/events{/number}\",\n \"events_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/events\",\n \"assignees_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/assignees{/user}\",\n \"branches_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/branches{/branch}\",\n \"tags_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/tags\",\n \"blobs_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/git/blobs{/sha}\",\n \"git_tags_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/git/tags{/sha}\",\n \"git_refs_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/git/refs{/sha}\",\n \"trees_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/git/trees{/sha}\",\n \"statuses_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/statuses/{sha}\",\n \"languages_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/languages\",\n \"stargazers_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/stargazers\",\n \"contributors_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/contributors\",\n \"subscribers_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/subscribers\",\n \"subscription_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/subscription\",\n \"commits_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/commits{/sha}\",\n \"git_commits_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/git/commits{/sha}\",\n \"comments_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/comments{/number}\",\n \"issue_comment_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/issues/comments{/number}\",\n \"contents_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/contents/{+path}\",\n \"compare_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/compare/{base}...{head}\",\n \"merges_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/merges\",\n \"archive_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/{archive_format}{/ref}\",\n \"downloads_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/downloads\",\n \"issues_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/issues{/number}\",\n \"pulls_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/pulls{/number}\",\n \"milestones_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/milestones{/number}\",\n \"notifications_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/notifications{?since,all,participating}\",\n \"labels_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/labels{/name}\",\n \"releases_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/releases{/id}\",\n \"deployments_url\": \"https://api.github.com/repos/blacklanternsecurity/test_keys/deployments\",\n \"created_at\": \"2022-02-15T23:10:51Z\",\n \"updated_at\": \"2023-09-02T12:20:13Z\",\n \"pushed_at\": \"2023-10-19T02:56:46Z\",\n \"git_url\": \"git://github.com/blacklanternsecurity/test_keys.git\",\n \"ssh_url\": \"git@github.com:blacklanternsecurity/test_keys.git\",\n \"clone_url\": \"https://github.com/blacklanternsecurity/test_keys.git\",\n \"svn_url\": \"https://github.com/blacklanternsecurity/test_keys\",\n \"homepage\": None,\n \"size\": 2,\n \"stargazers_count\": 2,\n \"watchers_count\": 2,\n \"language\": None,\n \"has_issues\": True,\n \"has_projects\": True,\n \"has_downloads\": True,\n \"has_wiki\": True,\n \"has_pages\": False,\n \"has_discussions\": False,\n \"forks_count\": 32,\n \"mirror_url\": None,\n \"archived\": False,\n \"disabled\": False,\n \"open_issues_count\": 2,\n \"license\": None,\n \"allow_forking\": True,\n \"is_template\": False,\n \"web_commit_signoff_required\": False,\n \"topics\": [],\n \"visibility\": \"public\",\n \"forks\": 32,\n \"open_issues\": 2,\n \"watchers\": 2,\n \"default_branch\": \"main\",\n \"permissions\": {\"admin\": False, \"maintain\": False, \"push\": False, \"triage\": False, \"pull\": True},\n },\n {\n \"id\": 459780477,\n \"node_id\": \"R_kgDOG2exfQ\",\n \"name\": \"bbot\",\n \"full_name\": \"blacklanternsecurity/bbot\",\n \"private\": False,\n \"owner\": {\n \"login\": \"blacklanternsecurity\",\n \"id\": 79229934,\n \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjc5MjI5OTM0\",\n \"avatar_url\": \"https://avatars.githubusercontent.com/u/79229934?v=4\",\n \"gravatar_id\": \"\",\n \"url\": \"https://api.github.com/users/blacklanternsecurity\",\n \"html_url\": \"https://github.com/blacklanternsecurity\",\n \"followers_url\": \"https://api.github.com/users/blacklanternsecurity/followers\",\n \"following_url\": \"https://api.github.com/users/blacklanternsecurity/following{/other_user}\",\n \"gists_url\": \"https://api.github.com/users/blacklanternsecurity/gists{/gist_id}\",\n \"starred_url\": \"https://api.github.com/users/blacklanternsecurity/starred{/owner}{/repo}\",\n \"subscriptions_url\": \"https://api.github.com/users/blacklanternsecurity/subscriptions\",\n \"organizations_url\": \"https://api.github.com/users/blacklanternsecurity/orgs\",\n \"repos_url\": \"https://api.github.com/users/blacklanternsecurity/repos\",\n \"events_url\": \"https://api.github.com/users/blacklanternsecurity/events{/privacy}\",\n \"received_events_url\": \"https://api.github.com/users/blacklanternsecurity/received_events\",\n \"type\": \"Organization\",\n \"site_admin\": False,\n },\n \"html_url\": \"https://github.com/blacklanternsecurity/bbot\",\n \"description\": None,\n \"fork\": False,\n \"url\": \"https://api.github.com/repos/blacklanternsecurity/bbot\",\n \"forks_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/forks\",\n \"keys_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/keys{/key_id}\",\n \"collaborators_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/collaborators{/collaborator}\",\n \"teams_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/teams\",\n \"hooks_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/hooks\",\n \"issue_events_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/issues/events{/number}\",\n \"events_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/events\",\n \"assignees_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/assignees{/user}\",\n \"branches_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/branches{/branch}\",\n \"tags_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/tags\",\n \"blobs_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/blobs{/sha}\",\n \"git_tags_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/tags{/sha}\",\n \"git_refs_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/refs{/sha}\",\n \"trees_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/trees{/sha}\",\n \"statuses_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/statuses/{sha}\",\n \"languages_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/languages\",\n \"stargazers_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/stargazers\",\n \"contributors_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/contributors\",\n \"subscribers_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/subscribers\",\n \"subscription_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/subscription\",\n \"commits_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/commits{/sha}\",\n \"git_commits_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/commits{/sha}\",\n \"comments_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/comments{/number}\",\n \"issue_comment_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/issues/comments{/number}\",\n \"contents_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/contents/{+path}\",\n \"compare_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/compare/{base}...{head}\",\n \"merges_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/merges\",\n \"archive_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/{archive_format}{/ref}\",\n \"downloads_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/downloads\",\n \"issues_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/issues{/number}\",\n \"pulls_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/pulls{/number}\",\n \"milestones_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/milestones{/number}\",\n \"notifications_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/notifications{?since,all,participating}\",\n \"labels_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/labels{/name}\",\n \"releases_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/releases{/id}\",\n \"deployments_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/deployments\",\n \"created_at\": \"2022-02-15T23:10:51Z\",\n \"updated_at\": \"2023-09-02T12:20:13Z\",\n \"pushed_at\": \"2023-10-19T02:56:46Z\",\n \"git_url\": \"git://github.com/blacklanternsecurity/bbot.git\",\n \"ssh_url\": \"git@github.com:blacklanternsecurity/bbot.git\",\n \"clone_url\": \"https://github.com/blacklanternsecurity/bbot.git\",\n \"svn_url\": \"https://github.com/blacklanternsecurity/bbot\",\n \"homepage\": None,\n \"size\": 2,\n \"stargazers_count\": 2,\n \"watchers_count\": 2,\n \"language\": None,\n \"has_issues\": True,\n \"has_projects\": True,\n \"has_downloads\": True,\n \"has_wiki\": True,\n \"has_pages\": False,\n \"has_discussions\": False,\n \"forks_count\": 32,\n \"mirror_url\": None,\n \"archived\": False,\n \"disabled\": False,\n \"open_issues_count\": 2,\n \"license\": None,\n \"allow_forking\": True,\n \"is_template\": False,\n \"web_commit_signoff_required\": False,\n \"topics\": [],\n \"visibility\": \"public\",\n \"forks\": 32,\n \"open_issues\": 2,\n \"watchers\": 2,\n \"default_branch\": \"main\",\n \"permissions\": {\"admin\": False, \"maintain\": False, \"push\": False, \"triage\": False, \"pull\": True},\n },\n ],\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows?per_page=100&page=1\",\n match_headers={\"Authorization\": \"token asdf\"},\n json={\n \"total_count\": 3,\n \"workflows\": [\n {\n \"id\": 22452226,\n \"node_id\": \"W_kwDOG_O3ns4BVpgC\",\n \"name\": \"tests\",\n \"path\": \".github/workflows/tests.yml\",\n \"state\": \"active\",\n \"created_at\": \"2022-03-23T15:09:22.000Z\",\n \"updated_at\": \"2022-09-27T17:49:34.000Z\",\n \"url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226\",\n \"html_url\": \"https://github.com/blacklanternsecurity/bbot/blob/stable/.github/workflows/tests.yml\",\n \"badge_url\": \"https://github.com/blacklanternsecurity/bbot/workflows/tests/badge.svg\",\n },\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226/runs?status=success&per_page=1\",\n match_headers={\"Authorization\": \"token asdf\"},\n json={\n \"total_count\": 2993,\n \"workflow_runs\": [\n {\n \"id\": 8839360698,\n \"name\": \"tests\",\n \"node_id\": \"WFR_kwLOG_O3ns8AAAACDt3wug\",\n \"head_branch\": \"dnsbrute-helperify\",\n \"head_sha\": \"c5de1360e8e5ccba04b23035f675a529282b7dc2\",\n \"path\": \".github/workflows/tests.yml\",\n \"display_title\": \"Helperify Massdns\",\n \"run_number\": 4520,\n \"event\": \"pull_request\",\n \"status\": \"completed\",\n \"conclusion\": \"success\",\n \"workflow_id\": 22452226,\n \"check_suite_id\": 23162098295,\n \"check_suite_node_id\": \"CS_kwDOG_O3ns8AAAAFZJGSdw\",\n \"url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698\",\n \"html_url\": \"https://github.com/blacklanternsecurity/bbot/actions/runs/8839360698\",\n \"pull_requests\": [\n {\n \"url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/pulls/1303\",\n \"id\": 1839332952,\n \"number\": 1303,\n \"head\": {\n \"ref\": \"dnsbrute-helperify\",\n \"sha\": \"c5de1360e8e5ccba04b23035f675a529282b7dc2\",\n \"repo\": {\n \"id\": 468957086,\n \"url\": \"https://api.github.com/repos/blacklanternsecurity/bbot\",\n \"name\": \"bbot\",\n },\n },\n \"base\": {\n \"ref\": \"faster-regexes\",\n \"sha\": \"7baf219c7f3a4ba165639c5ddb62322453a8aea8\",\n \"repo\": {\n \"id\": 468957086,\n \"url\": \"https://api.github.com/repos/blacklanternsecurity/bbot\",\n \"name\": \"bbot\",\n },\n },\n }\n ],\n \"created_at\": \"2024-04-25T21:04:32Z\",\n \"updated_at\": \"2024-04-25T21:19:43Z\",\n \"actor\": {\n \"login\": \"TheTechromancer\",\n \"id\": 20261699,\n \"node_id\": \"MDQ6VXNlcjIwMjYxNjk5\",\n \"avatar_url\": \"https://avatars.githubusercontent.com/u/20261699?v=4\",\n \"gravatar_id\": \"\",\n \"url\": \"https://api.github.com/users/TheTechromancer\",\n \"html_url\": \"https://github.com/TheTechromancer\",\n \"followers_url\": \"https://api.github.com/users/TheTechromancer/followers\",\n \"following_url\": \"https://api.github.com/users/TheTechromancer/following{/other_user}\",\n \"gists_url\": \"https://api.github.com/users/TheTechromancer/gists{/gist_id}\",\n \"starred_url\": \"https://api.github.com/users/TheTechromancer/starred{/owner}{/repo}\",\n \"subscriptions_url\": \"https://api.github.com/users/TheTechromancer/subscriptions\",\n \"organizations_url\": \"https://api.github.com/users/TheTechromancer/orgs\",\n \"repos_url\": \"https://api.github.com/users/TheTechromancer/repos\",\n \"events_url\": \"https://api.github.com/users/TheTechromancer/events{/privacy}\",\n \"received_events_url\": \"https://api.github.com/users/TheTechromancer/received_events\",\n \"type\": \"User\",\n \"site_admin\": False,\n },\n \"run_attempt\": 1,\n \"referenced_workflows\": [],\n \"run_started_at\": \"2024-04-25T21:04:32Z\",\n \"triggering_actor\": {\n \"login\": \"TheTechromancer\",\n \"id\": 20261699,\n \"node_id\": \"MDQ6VXNlcjIwMjYxNjk5\",\n \"avatar_url\": \"https://avatars.githubusercontent.com/u/20261699?v=4\",\n \"gravatar_id\": \"\",\n \"url\": \"https://api.github.com/users/TheTechromancer\",\n \"html_url\": \"https://github.com/TheTechromancer\",\n \"followers_url\": \"https://api.github.com/users/TheTechromancer/followers\",\n \"following_url\": \"https://api.github.com/users/TheTechromancer/following{/other_user}\",\n \"gists_url\": \"https://api.github.com/users/TheTechromancer/gists{/gist_id}\",\n \"starred_url\": \"https://api.github.com/users/TheTechromancer/starred{/owner}{/repo}\",\n \"subscriptions_url\": \"https://api.github.com/users/TheTechromancer/subscriptions\",\n \"organizations_url\": \"https://api.github.com/users/TheTechromancer/orgs\",\n \"repos_url\": \"https://api.github.com/users/TheTechromancer/repos\",\n \"events_url\": \"https://api.github.com/users/TheTechromancer/events{/privacy}\",\n \"received_events_url\": \"https://api.github.com/users/TheTechromancer/received_events\",\n \"type\": \"User\",\n \"site_admin\": False,\n },\n \"jobs_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/jobs\",\n \"logs_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/logs\",\n \"check_suite_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/check-suites/23162098295\",\n \"artifacts_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/artifacts\",\n \"cancel_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/cancel\",\n \"rerun_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/rerun\",\n \"previous_attempt_url\": None,\n \"workflow_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226\",\n \"head_commit\": {\n \"id\": \"c5de1360e8e5ccba04b23035f675a529282b7dc2\",\n \"tree_id\": \"fe9b345c0745a5bbacb806225e92e1c48fccf35c\",\n \"message\": \"remove debug message\",\n \"timestamp\": \"2024-04-25T21:02:37Z\",\n \"author\": {\"name\": \"TheTechromancer\", \"email\": \"thetechromancer@protonmail.com\"},\n \"committer\": {\"name\": \"TheTechromancer\", \"email\": \"thetechromancer@protonmail.com\"},\n },\n \"repository\": {\n \"id\": 468957086,\n \"node_id\": \"R_kgDOG_O3ng\",\n \"name\": \"bbot\",\n \"full_name\": \"blacklanternsecurity/bbot\",\n \"private\": False,\n \"owner\": {\n \"login\": \"blacklanternsecurity\",\n \"id\": 25311592,\n \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjI1MzExNTky\",\n \"avatar_url\": \"https://avatars.githubusercontent.com/u/25311592?v=4\",\n \"gravatar_id\": \"\",\n \"url\": \"https://api.github.com/users/blacklanternsecurity\",\n \"html_url\": \"https://github.com/blacklanternsecurity\",\n \"followers_url\": \"https://api.github.com/users/blacklanternsecurity/followers\",\n \"following_url\": \"https://api.github.com/users/blacklanternsecurity/following{/other_user}\",\n \"gists_url\": \"https://api.github.com/users/blacklanternsecurity/gists{/gist_id}\",\n \"starred_url\": \"https://api.github.com/users/blacklanternsecurity/starred{/owner}{/repo}\",\n \"subscriptions_url\": \"https://api.github.com/users/blacklanternsecurity/subscriptions\",\n \"organizations_url\": \"https://api.github.com/users/blacklanternsecurity/orgs\",\n \"repos_url\": \"https://api.github.com/users/blacklanternsecurity/repos\",\n \"events_url\": \"https://api.github.com/users/blacklanternsecurity/events{/privacy}\",\n \"received_events_url\": \"https://api.github.com/users/blacklanternsecurity/received_events\",\n \"type\": \"Organization\",\n \"site_admin\": False,\n },\n \"html_url\": \"https://github.com/blacklanternsecurity/bbot\",\n \"description\": \"A recursive internet scanner for hackers.\",\n \"fork\": False,\n \"url\": \"https://api.github.com/repos/blacklanternsecurity/bbot\",\n \"forks_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/forks\",\n \"keys_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/keys{/key_id}\",\n \"collaborators_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/collaborators{/collaborator}\",\n \"teams_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/teams\",\n \"hooks_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/hooks\",\n \"issue_events_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/issues/events{/number}\",\n \"events_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/events\",\n \"assignees_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/assignees{/user}\",\n \"branches_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/branches{/branch}\",\n \"tags_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/tags\",\n \"blobs_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/blobs{/sha}\",\n \"git_tags_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/tags{/sha}\",\n \"git_refs_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/refs{/sha}\",\n \"trees_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/trees{/sha}\",\n \"statuses_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/statuses/{sha}\",\n \"languages_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/languages\",\n \"stargazers_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/stargazers\",\n \"contributors_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/contributors\",\n \"subscribers_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/subscribers\",\n \"subscription_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/subscription\",\n \"commits_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/commits{/sha}\",\n \"git_commits_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/commits{/sha}\",\n \"comments_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/comments{/number}\",\n \"issue_comment_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/issues/comments{/number}\",\n \"contents_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/contents/{+path}\",\n \"compare_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/compare/{base}...{head}\",\n \"merges_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/merges\",\n \"archive_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/{archive_format}{/ref}\",\n \"downloads_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/downloads\",\n \"issues_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/issues{/number}\",\n \"pulls_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/pulls{/number}\",\n \"milestones_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/milestones{/number}\",\n \"notifications_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/notifications{?since,all,participating}\",\n \"labels_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/labels{/name}\",\n \"releases_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/releases{/id}\",\n \"deployments_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/deployments\",\n },\n \"head_repository\": {\n \"id\": 468957086,\n \"node_id\": \"R_kgDOG_O3ng\",\n \"name\": \"bbot\",\n \"full_name\": \"blacklanternsecurity/bbot\",\n \"private\": False,\n \"owner\": {\n \"login\": \"blacklanternsecurity\",\n \"id\": 25311592,\n \"node_id\": \"MDEyOk9yZ2FuaXphdGlvbjI1MzExNTky\",\n \"avatar_url\": \"https://avatars.githubusercontent.com/u/25311592?v=4\",\n \"gravatar_id\": \"\",\n \"url\": \"https://api.github.com/users/blacklanternsecurity\",\n \"html_url\": \"https://github.com/blacklanternsecurity\",\n \"followers_url\": \"https://api.github.com/users/blacklanternsecurity/followers\",\n \"following_url\": \"https://api.github.com/users/blacklanternsecurity/following{/other_user}\",\n \"gists_url\": \"https://api.github.com/users/blacklanternsecurity/gists{/gist_id}\",\n \"starred_url\": \"https://api.github.com/users/blacklanternsecurity/starred{/owner}{/repo}\",\n \"subscriptions_url\": \"https://api.github.com/users/blacklanternsecurity/subscriptions\",\n \"organizations_url\": \"https://api.github.com/users/blacklanternsecurity/orgs\",\n \"repos_url\": \"https://api.github.com/users/blacklanternsecurity/repos\",\n \"events_url\": \"https://api.github.com/users/blacklanternsecurity/events{/privacy}\",\n \"received_events_url\": \"https://api.github.com/users/blacklanternsecurity/received_events\",\n \"type\": \"Organization\",\n \"site_admin\": False,\n },\n \"html_url\": \"https://github.com/blacklanternsecurity/bbot\",\n \"description\": \"A recursive internet scanner for hackers.\",\n \"fork\": False,\n \"url\": \"https://api.github.com/repos/blacklanternsecurity/bbot\",\n \"forks_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/forks\",\n \"keys_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/keys{/key_id}\",\n \"collaborators_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/collaborators{/collaborator}\",\n \"teams_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/teams\",\n \"hooks_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/hooks\",\n \"issue_events_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/issues/events{/number}\",\n \"events_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/events\",\n \"assignees_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/assignees{/user}\",\n \"branches_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/branches{/branch}\",\n \"tags_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/tags\",\n \"blobs_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/blobs{/sha}\",\n \"git_tags_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/tags{/sha}\",\n \"git_refs_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/refs{/sha}\",\n \"trees_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/trees{/sha}\",\n \"statuses_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/statuses/{sha}\",\n \"languages_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/languages\",\n \"stargazers_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/stargazers\",\n \"contributors_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/contributors\",\n \"subscribers_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/subscribers\",\n \"subscription_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/subscription\",\n \"commits_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/commits{/sha}\",\n \"git_commits_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/git/commits{/sha}\",\n \"comments_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/comments{/number}\",\n \"issue_comment_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/issues/comments{/number}\",\n \"contents_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/contents/{+path}\",\n \"compare_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/compare/{base}...{head}\",\n \"merges_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/merges\",\n \"archive_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/{archive_format}{/ref}\",\n \"downloads_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/downloads\",\n \"issues_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/issues{/number}\",\n \"pulls_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/pulls{/number}\",\n \"milestones_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/milestones{/number}\",\n \"notifications_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/notifications{?since,all,participating}\",\n \"labels_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/labels{/name}\",\n \"releases_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/releases{/id}\",\n \"deployments_url\": \"https://api.github.com/repos/blacklanternsecurity/bbot/deployments\",\n },\n },\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/logs\",\n match_headers={\"Authorization\": \"token asdf\"},\n headers={\n \"location\": \"https://productionresultssa10.blob.core.windows.net/actions-results/7beb304e-f42c-4830-a027-4f5dec53107d/workflow-job-run-3a559e2a-952e-58d2-b8db-2e604a9266d7/logs/steps/step-logs-0e34a19a-18b0-4208-b27a-f8c031db2d17.txt?rsct=text%2Fplain&se=2024-04-26T16%3A25%3A39Z&sig=a%2FiN8dOw0e3tiBQZAfr80veI8OYChb9edJ1eFY136B4%3D&sp=r&spr=https&sr=b&st=2024-04-26T16%3A15%3A34Z&sv=2021-12-02\"\n },\n status_code=302,\n )\n data = io.BytesIO()\n with zipfile.ZipFile(data, mode=\"w\", compression=zipfile.ZIP_DEFLATED) as z:\n z.writestr(\"test.txt\", self.file_content)\n z.writestr(\"folder/test2.txt\", self.file_content)\n data.seek(0)\n zip_content = data.getvalue()\n module_test.httpx_mock.add_response(\n url=\"https://productionresultssa10.blob.core.windows.net/actions-results/7beb304e-f42c-4830-a027-4f5dec53107d/workflow-job-run-3a559e2a-952e-58d2-b8db-2e604a9266d7/logs/steps/step-logs-0e34a19a-18b0-4208-b27a-f8c031db2d17.txt?rsct=text%2Fplain&se=2024-04-26T16%3A25%3A39Z&sig=a%2FiN8dOw0e3tiBQZAfr80veI8OYChb9edJ1eFY136B4%3D&sp=r&spr=https&sr=b&st=2024-04-26T16%3A15%3A34Z&sv=2021-12-02\",\n content=zip_content,\n )\n module_test.httpx_mock.add_response(\n url=\"https://hub.docker.com/v2/users/blacklanternsecurity\",\n json={\n \"id\": \"f90895d9cf484d9182c6dbbef2632329\",\n \"uuid\": \"f90895d9-cf48-4d91-82c6-dbbef2632329\",\n \"username\": \"blacklanternsecurity\",\n \"full_name\": \"\",\n \"location\": \"\",\n \"company\": \"Black Lantern Security\",\n \"profile_url\": \"https://github.com/blacklanternsecurity\",\n \"date_joined\": \"2022-08-29T15:27:10.227081Z\",\n \"gravatar_url\": \"\",\n \"gravatar_email\": \"\",\n \"type\": \"User\",\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://hub.docker.com/v2/repositories/blacklanternsecurity?page_size=25&page=1\",\n json={\n \"count\": 2,\n \"next\": None,\n \"previous\": None,\n \"results\": [\n {\n \"name\": \"helloworld\",\n \"namespace\": \"blacklanternsecurity\",\n \"repository_type\": \"image\",\n \"status\": 1,\n \"status_description\": \"active\",\n \"description\": \"\",\n \"is_private\": False,\n \"star_count\": 0,\n \"pull_count\": 1,\n \"last_updated\": \"2021-12-20T17:19:58.88296Z\",\n \"date_registered\": \"2021-12-20T17:19:58.507614Z\",\n \"affiliation\": \"\",\n \"media_types\": [\"application/vnd.docker.container.image.v1+json\"],\n \"content_types\": [\"image\"],\n \"categories\": [],\n },\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/helloworld/tags/list\",\n json={\n \"name\": \"blacklanternsecurity/helloworld\",\n \"tags\": [\n \"dev\",\n \"latest\",\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/helloworld/manifests/latest\",\n json={\n \"schemaVersion\": 2,\n \"mediaType\": \"application/vnd.docker.distribution.manifest.v2+json\",\n \"config\": {\n \"mediaType\": \"application/vnd.docker.container.image.v1+json\",\n \"size\": 8614,\n \"digest\": \"sha256:a9910947b74a4f0606cfc8669ae8808d2c328beaee9e79f489dc17df14cd50b1\",\n },\n \"layers\": [\n {\n \"mediaType\": \"application/vnd.docker.image.rootfs.diff.tar.gzip\",\n \"size\": 29124181,\n \"digest\": \"sha256:8a1e25ce7c4f75e372e9884f8f7b1bedcfe4a7a7d452eb4b0a1c7477c9a90345\",\n },\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/helloworld/blobs/sha256:a9910947b74a4f0606cfc8669ae8808d2c328beaee9e79f489dc17df14cd50b1\",\n json={\n \"architecture\": \"amd64\",\n \"config\": {\n \"Env\": [\n \"PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\n \"LANG=C.UTF-8\",\n \"GPG_KEY=QWERTYUIOPASDFGHJKLZXCBNM\",\n \"PYTHON_VERSION=3.10.14\",\n \"PYTHON_PIP_VERSION=23.0.1\",\n \"PYTHON_SETUPTOOLS_VERSION=65.5.1\",\n \"PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py\",\n \"PYTHON_GET_PIP_SHA256=dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9\",\n \"LC_ALL=C.UTF-8\",\n \"PIP_NO_CACHE_DIR=off\",\n ],\n \"Entrypoint\": [\"helloworld\"],\n \"WorkingDir\": \"/root\",\n \"ArgsEscaped\": True,\n \"OnBuild\": None,\n },\n \"created\": \"2024-03-24T03:46:29.788993495Z\",\n \"history\": [\n {\n \"created\": \"2024-03-12T01:21:01.529814652Z\",\n \"created_by\": \"/bin/sh -c #(nop) ADD file:b86ae1c7ca3586d8feedcd9ff1b2b1e8ab872caf6587618f1da689045a5d7ae4 in / \",\n },\n {\n \"created\": \"2024-03-12T01:21:01.866693306Z\",\n \"created_by\": '/bin/sh -c #(nop) CMD [\"bash\"]',\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV LANG=C.UTF-8\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"RUN /bin/sh -c set -eux; \\tapt-get update; \\tapt-get install -y --no-install-recommends \\t\\tca-certificates \\t\\tnetbase \\t\\ttzdata \\t; \\trm -rf /var/lib/apt/lists/* # buildkit\",\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV GPG_KEY=QWERTYUIOPASDFGHJKLZXCBNM\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV PYTHON_VERSION=3.10.14\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": 'RUN /bin/sh -c set -eux; \\t\\tsavedAptMark=\"$(apt-mark showmanual)\"; \\tapt-get update; \\tapt-get install -y --no-install-recommends \\t\\tdpkg-dev \\t\\tgcc \\t\\tgnupg \\t\\tlibbluetooth-dev \\t\\tlibbz2-dev \\t\\tlibc6-dev \\t\\tlibdb-dev \\t\\tlibexpat1-dev \\t\\tlibffi-dev \\t\\tlibgdbm-dev \\t\\tliblzma-dev \\t\\tlibncursesw5-dev \\t\\tlibreadline-dev \\t\\tlibsqlite3-dev \\t\\tlibssl-dev \\t\\tmake \\t\\ttk-dev \\t\\tuuid-dev \\t\\twget \\t\\txz-utils \\t\\tzlib1g-dev \\t; \\t\\twget -O python.tar.xz \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz\"; \\twget -O python.tar.xz.asc \"https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc\"; \\tGNUPGHOME=\"$(mktemp -d)\"; export GNUPGHOME; \\tgpg --batch --keyserver hkps://keys.openpgp.org --recv-keys \"$GPG_KEY\"; \\tgpg --batch --verify python.tar.xz.asc python.tar.xz; \\tgpgconf --kill all; \\trm -rf \"$GNUPGHOME\" python.tar.xz.asc; \\tmkdir -p /usr/src/python; \\ttar --extract --directory /usr/src/python --strip-components=1 --file python.tar.xz; \\trm python.tar.xz; \\t\\tcd /usr/src/python; \\tgnuArch=\"$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)\"; \\t./configure \\t\\t--build=\"$gnuArch\" \\t\\t--enable-loadable-sqlite-extensions \\t\\t--enable-optimizations \\t\\t--enable-option-checking=fatal \\t\\t--enable-shared \\t\\t--with-lto \\t\\t--with-system-expat \\t\\t--without-ensurepip \\t; \\tnproc=\"$(nproc)\"; \\tEXTRA_CFLAGS=\"$(dpkg-buildflags --get CFLAGS)\"; \\tLDFLAGS=\"$(dpkg-buildflags --get LDFLAGS)\"; \\tLDFLAGS=\"${LDFLAGS:--Wl},--strip-all\"; \\tmake -j \"$nproc\" \\t\\t\"EXTRA_CFLAGS=${EXTRA_CFLAGS:-}\" \\t\\t\"LDFLAGS=${LDFLAGS:-}\" \\t\\t\"PROFILE_TASK=${PROFILE_TASK:-}\" \\t; \\trm python; \\tmake -j \"$nproc\" \\t\\t\"EXTRA_CFLAGS=${EXTRA_CFLAGS:-}\" \\t\\t\"LDFLAGS=${LDFLAGS:--Wl},-rpath=\\'\\\\$\\\\$ORIGIN/../lib\\'\" \\t\\t\"PROFILE_TASK=${PROFILE_TASK:-}\" \\t\\tpython \\t; \\tmake install; \\t\\tcd /; \\trm -rf /usr/src/python; \\t\\tfind /usr/local -depth \\t\\t\\\\( \\t\\t\\t\\\\( -type d -a \\\\( -name test -o -name tests -o -name idle_test \\\\) \\\\) \\t\\t\\t-o \\\\( -type f -a \\\\( -name \\'*.pyc\\' -o -name \\'*.pyo\\' -o -name \\'libpython*.a\\' \\\\) \\\\) \\t\\t\\\\) -exec rm -rf \\'{}\\' + \\t; \\t\\tldconfig; \\t\\tapt-mark auto \\'.*\\' > /dev/null; \\tapt-mark manual $savedAptMark; \\tfind /usr/local -type f -executable -not \\\\( -name \\'*tkinter*\\' \\\\) -exec ldd \\'{}\\' \\';\\' \\t\\t| awk \\'/=>/ { so = $(NF-1); if (index(so, \"/usr/local/\") == 1) { next }; gsub(\"^/(usr/)?\", \"\", so); printf \"*%s\\\\n\", so }\\' \\t\\t| sort -u \\t\\t| xargs -r dpkg-query --search \\t\\t| cut -d: -f1 \\t\\t| sort -u \\t\\t| xargs -r apt-mark manual \\t; \\tapt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \\trm -rf /var/lib/apt/lists/*; \\t\\tpython3 --version # buildkit',\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": 'RUN /bin/sh -c set -eux; \\tfor src in idle3 pydoc3 python3 python3-config; do \\t\\tdst=\"$(echo \"$src\" | tr -d 3)\"; \\t\\t[ -s \"/usr/local/bin/$src\" ]; \\t\\t[ ! -e \"/usr/local/bin/$dst\" ]; \\t\\tln -svT \"$src\" \"/usr/local/bin/$dst\"; \\tdone # buildkit',\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV PYTHON_PIP_VERSION=23.0.1\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV PYTHON_SETUPTOOLS_VERSION=65.5.1\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": \"ENV PYTHON_GET_PIP_SHA256=dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": 'RUN /bin/sh -c set -eux; \\t\\tsavedAptMark=\"$(apt-mark showmanual)\"; \\tapt-get update; \\tapt-get install -y --no-install-recommends wget; \\t\\twget -O get-pip.py \"$PYTHON_GET_PIP_URL\"; \\techo \"$PYTHON_GET_PIP_SHA256 *get-pip.py\" | sha256sum -c -; \\t\\tapt-mark auto \\'.*\\' > /dev/null; \\t[ -z \"$savedAptMark\" ] || apt-mark manual $savedAptMark > /dev/null; \\tapt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \\trm -rf /var/lib/apt/lists/*; \\t\\texport PYTHONDONTWRITEBYTECODE=1; \\t\\tpython get-pip.py \\t\\t--disable-pip-version-check \\t\\t--no-cache-dir \\t\\t--no-compile \\t\\t\"pip==$PYTHON_PIP_VERSION\" \\t\\t\"setuptools==$PYTHON_SETUPTOOLS_VERSION\" \\t; \\trm -f get-pip.py; \\t\\tpip --version # buildkit',\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-20T18:33:29Z\",\n \"created_by\": 'CMD [\"python3\"]',\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-24T03:45:39.322168741Z\",\n \"created_by\": \"ENV LANG=C.UTF-8\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-24T03:45:39.322168741Z\",\n \"created_by\": \"ENV LC_ALL=C.UTF-8\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-24T03:45:39.322168741Z\",\n \"created_by\": \"ENV PIP_NO_CACHE_DIR=off\",\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n {\n \"created\": \"2024-03-24T03:45:39.322168741Z\",\n \"created_by\": \"WORKDIR /usr/src/helloworld\",\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-24T03:45:52.226201188Z\",\n \"created_by\": \"RUN /bin/sh -c apt-get update && apt-get install -y openssl gcc git make unzip curl wget vim nano sudo # buildkit\",\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-24T03:45:52.391597947Z\",\n \"created_by\": \"COPY . . # buildkit\",\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-24T03:46:29.76589069Z\",\n \"created_by\": \"RUN /bin/sh -c pip install . # buildkit\",\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-24T03:46:29.788993495Z\",\n \"created_by\": \"WORKDIR /root\",\n \"comment\": \"buildkit.dockerfile.v0\",\n },\n {\n \"created\": \"2024-03-24T03:46:29.788993495Z\",\n \"created_by\": 'ENTRYPOINT [\"helloworld\"]',\n \"comment\": \"buildkit.dockerfile.v0\",\n \"empty_layer\": True,\n },\n ],\n \"os\": \"linux\",\n \"rootfs\": {\n \"type\": \"layers\",\n \"diff_ids\": [\n \"sha256:a483da8ab3e941547542718cacd3258c6c705a63e94183c837c9bc44eb608999\",\n \"sha256:c8f253aef5606f6716778771171c3fdf6aa135b76a5fa8bf66ba45c12c15b540\",\n \"sha256:b4a9dcc697d250c7be53887bb8e155c8f7a06f9c63a3aa627c647bb4a426d3f0\",\n \"sha256:120fda24c420b4e5d52f1c288b35c75b07969057bce41ec34cfb05606b2d7c11\",\n \"sha256:c2287f03e33f4896b2720f0cb64e6b6050759a3eb5914e531e98fc3499b4e687\",\n \"sha256:afe6e55a5cf240c050a4d2b72ec7b7d009a131cba8fe2753e453a8e62ef7e45c\",\n \"sha256:ae6df275ba2e8f40c598e30588afe43f6bfa92e4915e8450b77cb5db5c89dfd5\",\n \"sha256:621ab22fb386a9e663178637755b651beddc0eb4762804e74d8996cce0ddd441\",\n \"sha256:4c534ad16bd2df668c0b8f637616517746ede530ba8546d85f28772bc748e06f\",\n \"sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef\",\n ],\n },\n },\n )\n temp_path = Path(\"/tmp/.bbot_test\")\n tar_path = temp_path / \"docker_pull_test.tar.gz\"\n shutil.rmtree(tar_path, ignore_errors=True)\n with tarfile.open(tar_path, \"w:gz\") as tar:\n file_io = io.BytesIO(self.file_content.encode())\n file_info = tarfile.TarInfo(name=\"file.txt\")\n file_info.size = len(file_io.getvalue())\n file_io.seek(0)\n tar.addfile(file_info, file_io)\n with open(tar_path, \"rb\") as file:\n layer_file = file.read()\n module_test.httpx_mock.add_response(\n url=\"https://registry-1.docker.io/v2/blacklanternsecurity/helloworld/blobs/sha256:8a1e25ce7c4f75e372e9884f8f7b1bedcfe4a7a7d452eb4b0a1c7477c9a90345\",\n content=layer_file,\n )\n\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://www.postman.com/_api/ws/proxy\",\n match_json={\n \"service\": \"search\",\n \"method\": \"POST\",\n \"path\": \"/search-all\",\n \"body\": {\n \"queryIndices\": [\"collaboration.workspace\"],\n \"queryText\": \"blacklanternsecurity\",\n \"size\": 25,\n \"from\": 0,\n \"clientTraceId\": \"\",\n \"requestOrigin\": \"srp\",\n \"mergeEntities\": \"true\",\n \"nonNestedRequests\": \"true\",\n \"domain\": \"public\",\n },\n },\n json={\n \"data\": [\n {\n \"score\": 611.41156,\n \"normalizedScore\": 23,\n \"document\": {\n \"watcherCount\": 6,\n \"apiCount\": 0,\n \"forkCount\": 0,\n \"isblacklisted\": \"false\",\n \"createdAt\": \"2021-06-15T14:03:51\",\n \"publishertype\": \"team\",\n \"publisherHandle\": \"blacklanternsecurity\",\n \"id\": \"11498add-357d-4bc5-a008-0a2d44fb8829\",\n \"slug\": \"bbot-public\",\n \"updatedAt\": \"2024-07-30T11:00:35\",\n \"entityType\": \"workspace\",\n \"visibilityStatus\": \"public\",\n \"forkcount\": \"0\",\n \"tags\": [],\n \"createdat\": \"2021-06-15T14:03:51\",\n \"forkLabel\": \"\",\n \"publisherName\": \"blacklanternsecurity\",\n \"name\": \"BlackLanternSecurity BBOT [Public]\",\n \"dependencyCount\": 7,\n \"collectionCount\": 6,\n \"warehouse__updated_at\": \"2024-07-30 11:00:00\",\n \"privateNetworkFolders\": [],\n \"isPublisherVerified\": False,\n \"publisherType\": \"team\",\n \"curatedInList\": [],\n \"creatorId\": \"6900157\",\n \"description\": \"\",\n \"forklabel\": \"\",\n \"publisherId\": \"299401\",\n \"publisherLogo\": \"\",\n \"popularity\": 5,\n \"isPublic\": True,\n \"categories\": [],\n \"universaltags\": \"\",\n \"views\": 5788,\n \"summary\": \"BLS public workspaces.\",\n \"memberCount\": 2,\n \"isBlacklisted\": False,\n \"publisherid\": \"299401\",\n \"isPrivateNetworkEntity\": False,\n \"isDomainNonTrivial\": True,\n \"privateNetworkMeta\": \"\",\n \"updatedat\": \"2021-10-20T16:19:29\",\n \"documentType\": \"workspace\",\n },\n \"highlight\": {\"summary\": \"BLS BBOT api test.\"},\n },\n ],\n \"meta\": {\n \"queryText\": \"blacklanternsecurity\",\n \"total\": {\n \"collection\": 0,\n \"request\": 0,\n \"workspace\": 1,\n \"api\": 0,\n \"team\": 0,\n \"user\": 0,\n \"flow\": 0,\n \"apiDefinition\": 0,\n \"privateNetworkFolder\": 0,\n },\n \"state\": \"AQ4\",\n \"spellCorrection\": {\"count\": {\"all\": 1, \"workspace\": 1}, \"correctedQueryText\": None},\n \"featureFlags\": {\n \"enabledPublicResultCuration\": True,\n \"boostByPopularity\": True,\n \"reRankPostNormalization\": True,\n \"enableUrlBarHostNameSearch\": True,\n },\n },\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.postman.com/_api/ws/proxy\",\n match_json={\n \"service\": \"workspaces\",\n \"method\": \"GET\",\n \"path\": \"/workspaces?handle=blacklanternsecurity&slug=bbot-public\",\n },\n json={\n \"meta\": {\"model\": \"workspace\", \"action\": \"find\", \"nextCursor\": \"\"},\n \"data\": [\n {\n \"id\": \"3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b\",\n \"name\": \"BlackLanternSecurity BBOT [Public]\",\n \"description\": None,\n \"summary\": \"BLS public workspaces.\",\n \"createdBy\": \"299401\",\n \"updatedBy\": \"299401\",\n \"team\": None,\n \"createdAt\": \"2021-10-20T16:19:29\",\n \"updatedAt\": \"2021-10-20T16:19:29\",\n \"visibilityStatus\": \"public\",\n \"profileInfo\": {\n \"slug\": \"bbot-public\",\n \"profileType\": \"team\",\n \"profileId\": \"000000\",\n \"publicHandle\": \"https://www.postman.com/blacklanternsecurity\",\n \"publicImageURL\": \"\",\n \"publicName\": \"BlackLanternSecurity\",\n \"isVerified\": False,\n },\n }\n ],\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/workspaces/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b\",\n json={\n \"workspace\": {\n \"id\": \"3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b\",\n \"name\": \"BlackLanternSecurity BBOT [Public]\",\n \"type\": \"personal\",\n \"description\": None,\n \"visibility\": \"public\",\n \"createdBy\": \"00000000\",\n \"updatedBy\": \"00000000\",\n \"createdAt\": \"2021-11-17T06:09:01.000Z\",\n \"updatedAt\": \"2021-11-17T08:57:16.000Z\",\n \"collections\": [\n {\n \"id\": \"2aab9fd0-3715-4abe-8bb0-8cb0264d023f\",\n \"name\": \"BBOT Public\",\n \"uid\": \"10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f\",\n },\n ],\n \"environments\": [\n {\n \"id\": \"f770f816-9c6a-40f7-bde3-c0855d2a1089\",\n \"name\": \"BBOT Test\",\n \"uid\": \"10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089\",\n }\n ],\n \"apis\": [],\n }\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://www.postman.com/_api/workspace/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b/globals\",\n json={\n \"model_id\": \"8be7574b-219f-49e0-8d25-da447a882e4e\",\n \"meta\": {\"model\": \"globals\", \"action\": \"find\"},\n \"data\": {\n \"workspace\": \"3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b\",\n \"lastUpdatedBy\": \"00000000\",\n \"lastRevision\": 1637239113000,\n \"id\": \"8be7574b-219f-49e0-8d25-da447a882e4e\",\n \"values\": [\n {\n \"key\": \"endpoint_url\",\n \"value\": \"https://api.blacklanternsecurity.com/\",\n \"enabled\": True,\n },\n ],\n \"createdAt\": \"2021-11-17T06:09:01.000Z\",\n \"updatedAt\": \"2021-11-18T12:38:33.000Z\",\n },\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/environments/10197090-f770f816-9c6a-40f7-bde3-c0855d2a1089\",\n json={\n \"environment\": {\n \"id\": \"f770f816-9c6a-40f7-bde3-c0855d2a1089\",\n \"name\": \"BBOT Test\",\n \"owner\": \"00000000\",\n \"createdAt\": \"2021-11-17T06:29:54.000Z\",\n \"updatedAt\": \"2021-11-23T07:06:53.000Z\",\n \"values\": [\n {\n \"key\": \"temp_session_endpoint\",\n \"value\": \"https://api.blacklanternsecurity.com/\",\n \"enabled\": True,\n },\n ],\n \"isPublic\": True,\n }\n },\n )\n module_test.httpx_mock.add_response(\n url=\"https://api.getpostman.com/collections/10197090-2aab9fd0-3715-4abe-8bb0-8cb0264d023f\",\n json={\n \"collection\": {\n \"info\": {\n \"_postman_id\": \"62b91565-d2e2-4bcd-8248-4dba2e3452f0\",\n \"name\": \"BBOT Public\",\n \"schema\": \"https://schema.getpostman.com/json/collection/v2.1.0/collection.json\",\n \"updatedAt\": \"2021-11-17T07:13:16.000Z\",\n \"createdAt\": \"2021-11-17T07:13:15.000Z\",\n \"lastUpdatedBy\": \"00000000\",\n \"uid\": \"172983-62b91565-d2e2-4bcd-8248-4dba2e3452f0\",\n },\n \"item\": [\n {\n \"name\": \"Generate API Session\",\n \"id\": \"c1bac38c-dfc9-4cc0-9c19-828cbc8543b1\",\n \"protocolProfileBehavior\": {\"disableBodyPruning\": True},\n \"request\": {\n \"method\": \"POST\",\n \"header\": [{\"key\": \"Content-Type\", \"value\": \"application/json\"}],\n \"body\": {\n \"mode\": \"raw\",\n \"raw\": '{\"username\": \"test\", \"password\": \"Test\"}',\n },\n \"url\": {\n \"raw\": \"https://admin:admin@the-internet.herokuapp.com/basic_auth\",\n \"host\": [\"https://admin:admin@the-internet.herokuapp.com/basic_auth\"],\n },\n \"description\": \"\",\n },\n \"response\": [],\n \"uid\": \"10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1\",\n },\n {\n \"name\": \"Generate API Session\",\n \"id\": \"c1bac38c-dfc9-4cc0-9c19-828cbc8543b1\",\n \"protocolProfileBehavior\": {\"disableBodyPruning\": True},\n \"request\": {\n \"method\": \"POST\",\n \"header\": [{\"key\": \"Content-Type\", \"value\": \"application/json\"}],\n \"body\": {\n \"mode\": \"raw\",\n \"raw\": '{\"username\": \"test\", \"password\": \"Test\"}',\n },\n \"url\": {\n \"raw\": \"https://admin:admin@internal.host.com\",\n \"host\": [\"https://admin:admin@internal.host.com\"],\n },\n \"description\": \"\",\n },\n \"response\": [],\n \"uid\": \"10197090-c1bac38c-dfc9-4cc0-9c19-828cbc8543b1\",\n },\n ],\n }\n },\n )\n temp_path = Path(\"/tmp/.bbot_test\")\n temp_repo_path = temp_path / \"test_keys\"\n shutil.rmtree(temp_repo_path, ignore_errors=True)\n subprocess.run([\"git\", \"init\", \"test_keys\"], cwd=temp_path)\n with open(temp_repo_path / \"keys.txt\", \"w\") as f:\n f.write(self.file_content)\n subprocess.run([\"git\", \"add\", \".\"], cwd=temp_repo_path)\n subprocess.run(\n [\n \"git\",\n \"-c\",\n \"user.name='BBOT Test'\",\n \"-c\",\n \"user.email='bbot@blacklanternsecurity.com'\",\n \"commit\",\n \"-m\",\n \"Initial commit\",\n ],\n check=True,\n cwd=temp_repo_path,\n )\n\n # we need this test to work offline, so we patch git_clone to pull from a local file:// path\n old_handle_event = module_test.scan.modules[\"git_clone\"].handle_event\n\n async def new_handle_event(event):\n if event.type == \"CODE_REPOSITORY\":\n event = copy(event)\n data = dict(event.data)\n data[\"url\"] = event.data[\"url\"].replace(\n \"https://github.com/blacklanternsecurity\", f\"file://{temp_path}\"\n )\n event.data = data\n return await old_handle_event(event)\n\n module_test.monkeypatch.setattr(module_test.scan.modules[\"git_clone\"], \"handle_event\", new_handle_event)\n\n def check(self, module_test, events):\n vuln_events = [\n e\n for e in events\n if e.type == \"VULNERABILITY\"\n and (\n e.data[\"host\"] == \"hub.docker.com\"\n or e.data[\"host\"] == \"github.com\"\n or e.data[\"host\"] == \"www.postman.com\"\n )\n and \"Verified Secret Found.\" in e.data[\"description\"]\n and \"Raw result: [https://admin:admin@the-internet.herokuapp.com]\" in e.data[\"description\"]\n and \"RawV2 result: [https://admin:admin@the-internet.herokuapp.com/basic_auth]\" in e.data[\"description\"]\n ]\n\n # Trufflehog should find 4 verifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman.\n assert 4 == len(vuln_events), \"Failed to find secret in events\"\n github_repo_event = [e for e in vuln_events if \"test_keys\" in e.data[\"description\"]][0].parent\n folder = Path(github_repo_event.data[\"path\"])\n assert folder.is_dir(), \"Destination folder doesn't exist\"\n with open(folder / \"keys.txt\") as f:\n content = f.read()\n assert content == self.file_content, \"File content doesn't match\"\n filesystem_events = [e.parent for e in vuln_events]\n assert len(filesystem_events) == 4\n assert all(e.type == \"FILESYSTEM\" for e in filesystem_events)\n assert 1 == len(\n [\n e\n for e in filesystem_events\n if e.data[\"path\"].endswith(\"/git_repos/.bbot_test/test_keys\") and Path(e.data[\"path\"]).is_dir()\n ]\n ), \"Test keys repo dir does not exist\"\n assert 1 == len(\n [\n e\n for e in filesystem_events\n if e.data[\"path\"].endswith(\"/workflow_logs/blacklanternsecurity/bbot/test.txt\")\n and Path(e.data[\"path\"]).is_file()\n ]\n ), \"Workflow log file does not exist\"\n assert 1 == len(\n [\n e\n for e in filesystem_events\n if e.data[\"path\"].endswith(\"/docker_images/blacklanternsecurity_helloworld_latest.tar\")\n and Path(e.data[\"path\"]).is_file()\n ]\n ), \"Docker image file does not exist\"\n assert 1 == len(\n [\n e\n for e in filesystem_events\n if e.data[\"path\"].endswith(\n \"/postman_workspaces/BlackLanternSecurity BBOT [Public]/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b.zip\"\n )\n and Path(e.data[\"path\"]).is_file()\n ]\n ), \"Failed to find blacklanternsecurity postman workspace\"\n\n\nclass TestTrufflehog_NonVerified(TestTrufflehog):\n download_dir = bbot_test_dir / \"test_trufflehog_nonverified\"\n config_overrides = {\n \"modules\": {\n \"trufflehog\": {\"only_verified\": False},\n \"docker_pull\": {\"output_folder\": str(download_dir)},\n \"postman_download\": {\"api_key\": \"asdf\", \"output_folder\": str(download_dir)},\n \"github_org\": {\"api_key\": \"asdf\"},\n \"git_clone\": {\"output_folder\": str(download_dir)},\n }\n }\n\n def check(self, module_test, events):\n finding_events = [\n e\n for e in events\n if e.type == e.type == \"FINDING\"\n and (\n e.data[\"host\"] == \"hub.docker.com\"\n or e.data[\"host\"] == \"github.com\"\n or e.data[\"host\"] == \"www.postman.com\"\n )\n and \"Possible Secret Found.\" in e.data[\"description\"]\n and \"Raw result: [https://admin:admin@internal.host.com]\" in e.data[\"description\"]\n ]\n # Trufflehog should find 4 unverifiable secrets, 1 from the github, 1 from the workflow log, 1 from the docker image and 1 from the postman.\n assert 4 == len(finding_events), \"Failed to find secret in events\"\n github_repo_event = [e for e in finding_events if \"test_keys\" in e.data[\"description\"]][0].parent\n folder = Path(github_repo_event.data[\"path\"])\n assert folder.is_dir(), \"Destination folder doesn't exist\"\n with open(folder / \"keys.txt\") as f:\n content = f.read()\n assert content == self.file_content, \"File content doesn't match\"\n filesystem_events = [e.parent for e in finding_events]\n assert len(filesystem_events) == 4\n assert all(e.type == \"FILESYSTEM\" for e in filesystem_events)\n assert 1 == len(\n [\n e\n for e in filesystem_events\n if e.data[\"path\"].endswith(\"/git_repos/.bbot_test/test_keys\") and Path(e.data[\"path\"]).is_dir()\n ]\n ), \"Test keys repo dir does not exist\"\n assert 1 == len(\n [\n e\n for e in filesystem_events\n if e.data[\"path\"].endswith(\"/workflow_logs/blacklanternsecurity/bbot/test.txt\")\n and Path(e.data[\"path\"]).is_file()\n ]\n ), \"Workflow log file does not exist\"\n assert 1 == len(\n [\n e\n for e in filesystem_events\n if e.data[\"path\"].endswith(\"/docker_images/blacklanternsecurity_helloworld_latest.tar\")\n and Path(e.data[\"path\"]).is_file()\n ]\n ), \"Docker image file does not exist\"\n assert 1 == len(\n [\n e\n for e in filesystem_events\n if e.data[\"path\"].endswith(\n \"/postman_workspaces/BlackLanternSecurity BBOT [Public]/3a7e4bdc-7ff7-4dd4-8eaa-61ddce1c3d1b.zip\"\n )\n and Path(e.data[\"path\"]).is_file()\n ]\n ), \"Failed to find blacklanternsecurity postman workspace\"\n\n\nclass TestTrufflehog_HTTPResponse(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"trufflehog\"]\n config_overrides = {\"modules\": {\"trufflehog\": {\"only_verified\": False}}}\n\n async def setup_before_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": \"https://admin:admin@internal.host.com\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n assert any(e.type == \"FINDING\" for e in events)\n\n\nclass TestTrufflehog_RAWText(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888/test.pdf\"]\n modules_overrides = [\"httpx\", \"trufflehog\", \"filedownload\", \"extractous\"]\n\n download_dir = bbot_test_dir / \"test_trufflehog_rawtext\"\n config_overrides = {\n \"modules\": {\"trufflehog\": {\"only_verified\": False}, \"filedownload\": {\"output_folder\": str(download_dir)}}\n }\n\n async def setup_before_prep(self, module_test):\n expect_args = {\n \"method\": \"GET\",\n \"uri\": \"/test.pdf\",\n }\n respond_args = {\n \"response_data\": b\"%PDF-1.4\\n%\\xc7\\xec\\x8f\\xa2\\n%%Invocation: path/gs -P- -dSAFER -dCompatibilityLevel=1.4 -dWriteXRefStm=false -dWriteObjStms=false -q -P- -dNOPAUSE -dBATCH -sDEVICE=pdfwrite -sstdout=? -sOutputFile=? -P- -dSAFER -dCompatibilityLevel=1.4 -dWriteXRefStm=false -dWriteObjStms=false -\\n5 0 obj\\n<>\\nstream\\nx\\x9c-\\x8c\\xb1\\x0e\\x82@\\x10D\\xfb\\xfd\\x8a-\\xa1\\xe0\\xd8\\xe5@\\xe1*c\\xb4\\xb1\\xd3lba,\\xc8\\x81\\x82\\xf1@\\xe4\\xfe?\\x02\\x92If\\x92\\x97\\x99\\x19\\x90\\x14#\\xcdZ\\xd3: |\\xc2\\x00\\xbcP\\\\\\xc3:\\xdc\\x0b\\xc4\\x97\\xed\\x0c\\xe4\\x01\\xff2\\xe36\\xc5\\x9c6Jk\\x8d\\xe2\\xe0\\x16\\\\\\xeb\\n\\x0f\\xb5E\\xce\\x913\\x93\\x15F3&\\x94\\xa4a\\x94fD\\x01\\x87w9M7\\xc5z3Q\\x8cx\\xd9'(\\x15\\x04\\x8d\\xf7\\x9f\\xd1\\xc4qY\\xb9\\xb63\\x8b\\xef\\xda\\xce\\xd7\\xdf\\xae|\\xab\\xa6\\x1f\\xbd\\xb2\\xbd\\x0b\\xe5\\x05G\\x81\\xf3\\xa4\\x1f~q-\\xc7endstream\\nendobj\\n6 0 obj\\n155\\nendobj\\n4 0 obj\\n<>\\n/Contents 5 0 R\\n>>\\nendobj\\n3 0 obj\\n<< /Type /Pages /Kids [\\n4 0 R\\n] /Count 1\\n>>\\nendobj\\n1 0 obj\\n<>\\nendobj\\n11 0 obj\\n<>\\nendobj\\n9 0 obj\\n<>\\nendobj\\n7 0 obj\\n<>\\nendobj\\n10 0 obj\\n<>\\nendobj\\n12 0 obj\\n<>stream\\nx\\x9c\\x9dT{TS\\xf7\\x1d\\xbf\\x91ps\\x8f\\xa0\\xb2\\xdc\\x06\\x1f\\xe8\\xbdX[|\\xa0\\x85\\xaa\\xad\\xa7\\xf4\\x14P\\x1eG9\\x05\\x9c\\xa2\\x08\\xb4\\xee@\\x88\\xc83\\x08\\x04\\x84\\x80\\x84@B\\xd3\\x1f84!@\\x12\\x08\\xe0\\x8b\\x97S\\xe9\\xc4U\\xf4\\x06\\xb5\\x15\\xdd:5\\xc8&j=\\xb2\\xad:'T9\\xeb\\xce\\xbe\\xb7\\xe7\\xban\\xbf\\x80\\x16\\xdb\\xd3\\xed\\x8f\\x9d\\x93?n\\xee\\xe3\\xf3\\xfb~\\x1e\\xdf\\x8f\\x88\\x10\\xcf D\\\"\\x11\\x15\\xa6T\\xe5\\xa5+\\xf2\\\\\\xd7\\xabx\\x1f\\x11\\xbfp\\x06\\xbf\\xc8\\r\\tQ\\xfc\\xd8\\xb7\\xab\\xdcy\\xc6\\x93\\xa8\\xf1\\x14!O7\\xe4)n_H\\x19\\xa4\\xd0\\xfb3\\xa8\\x9d\\x03\\xc5^\\x84X$Z\\x17\\x9dd]\\xb6mK\\xfcr\\x7f\\xff\\x95a\\xca\\xdc\\xe2\\xbc\\xf4\\xb4\\xdd\\x05\\xbe\\xab\\x03\\xdf\\\\\\xeb\\x9bR\\xec\\xfb\\xfc\\x89o\\xb8\\\"?=-\\xc7\\xd7\\x0f_\\x14*\\xb2\\x94\\xb9\\xd9\\x8a\\x9c\\x82\\x98\\xf4\\xec\\x14U\\xbeo\\xb42G\\xe9\\xbby\\xab\\xef\\x16E\\x9a*+9\\xef\\x87w\\xa7\\x11\\xff\\xbf3\\x08\\x82\\x90\\xe6(s\\xf3\\xf2\\x0b\\x92\\xe5\\xa9\\x8a\\xdd\\xe9Y\\xd9o\\x04\\x04\\x85\\x12D,\\xb1\\x99\\xf89\\xb1\\x95\\x88#\\xb6\\x11\\x1b\\x88p\\\"\\x82\\x88$6\\x11QD4\\x11C\\xcc!\\xbc\\x08\\x1fb1Acq\\x081\\xa1'\\x06E\\x1bE}3>\\x9cq\\xc1m\\x93[\\x9fx\\x89\\xb8P\\x0c\\xee\\x91\\xee\\x95\\xe4\\xab\\xe4zRIvJ\\xd6\\xf3\\xe3\\xb3\\xf9q\\xc4\\xc1}N:\\x08\\xee\\xf1\\x0eht\\xcc\\xa5Ga=\\xbfN\\x16D\\xaa**KJ\\xcc\\xdaV\\x96\\x1e\\xe9\\x10\\x9crR\\xa5\\xd1\\xaaK\\x1a\\xf0\\x7f\\x98G\\xb6\\x9aM6\\xab\\xc6T\\xc8\\xcaAG^\\xf9\\xe3a\\xcb\\x15t\\x02\\xb5\\xe8\\xda\\x8a\\x0f\\x155\\x14\\xa0\\\\J\\xa8PJ\\xa6\\xdf\\x17\\x91\\xf6\\x86\\xe7\\xef\\xe7\\xc0G\\xe4\\xed\\x88\\xc1\\x00\\x86\\x1e\\x8dAi\\xc5\\xdb\\xb7Rx\\x025\\x07O9\\xd15\\x07\\xfc\\xdb\\xe1\\x06\\x9f\\xf1\\x112a\\xc1k\\xcb\\x05Z\\xf0\\xfaf)x\\x83\\xf7\\xdf\\x9f\\x80\\x14\\xe6\\xbc6!\\xd0\\xacn\\x87\\xec\\x9b\\xbb\\xa1\\xcb\\xfc\\xdf\\r\\xf6\\xf3\\x0b\\x1a\\x19\\x7f|\\xf7\\xf6\\x13\\x16\\x03\\x08Q\\x1c,\\xe6`\\x90\\xdb\\xc5Im0\\x1f\\x13\\xf9\\x1a\\x13y\\x04+0\\x11\\xbf\\x97\\x88|u\\xeeYu\\\"I?*t\\x8d\\xe6\\xba\\x03\\xdb\\xc8\\xb6)**\\x96~\\x18\\x00\\x05\\xe4\\xa7[.\\xee\\x19F\\x14H\\xc7\\x1f\\x81\\x07K/\\x00O\\xff\\x87\\xc2+\\xeb\\x93\\xf2cv0t\\\"\\x04\\x1f\\x97=\\xb9\\x15\\x11\\xb8:$\\xdc\\x7fE\\xc8\\xd0\\x83\\xbf\\xdc\\xba\\xf97vJC'\\x97\\xc2I\\xe1\\x17\\xf8\\xdc\\x1b`\\xc4\\xe7\\n\\xb3\\xc8\\xc2r\\xadZ\\xddP\\xd1\\xca\\xde\\x10\\x9c\\x81\\xf8_E\\xe9\\x94\\x1e\\xceI=,\\xe5\\xf5E\\xac\\xb0\\x01RI:p\\x1c\\x88\\x9e\\xb6>\\x1f;j\\xd6\\x1e\\xca7V\\xed7\\x98\\x10e1\\x9b\\xad\\xf5:\\xd3^\\x0b\\x9b\\xdb\\xae2e\\xa1x\\xf4\\xc1\\x9e5\\xefM\\xe9\\xb5\\xdb\\x0e\\xdfq\\xe9v)x\\\\\\x82\\xc3\\x97\\xe6\\xd2\\xef\\xc3\\n\\x98)\\xb3j\\xcc\\xa5%ZM!\\x13$)4ilV\\x93\\xd9\\xce\\xd0=Y\\xa7\\x06\\xd4W|`\\xe6\\xfdKwN\\x14\\xfd*\\xb3\\x95\\xcdh\\xdbe\\x8e>\\xb0\\xa6^_\\xa3j,6k,\\xa8\\x89\\xea\\x1d\\xe8\\xb89|>7\\xa5\\x8e\\xa9-6j-\\x88\\xb2\\x99\\xcc\\xad\\xecu\\t\\xbd\\xb0UkV\\x97UT\\x94\\x1a0\\xd2\\x91\\xf4\\x9d\\x8d\\xdb|\\xfcB\\x137f4gu\\x16\\xb3\\x1d\\xc5\\x1dU\\x7f\\xa8\\xba\\xa8;\\xa2;Rzx\\x9fU\\x85\\n\\xa9\\xc4\\xf7\\xd3\\xde~g\\xe3\\xf1\\xd3\\xcc\\x94\\xad\\x7f\\xe2D\\xe0\\x8bM\\x8d\\xc3\\x82\\x80X\\xd2\\xaa\\xad/\\xc1\\x03\\x161\\x828\\x12\\xe7c\\xd2\\x966\\xac\\x8e\\x99\\x0c\\xf9m\\xc2\\xd7g/\\x99\\x9b\\xfb\\x99\\x93M\\xd6Fd\\xa1\\x9a4\\xe62}\\xf5\\xc7:-\\x93\\xaa\\x8aT\\xc7!jSJ\\xe7Y\\x16L\\x90!q9f\\xd3\\x18U\\xec\\x94\\x14\\x1c\\xbc\\xc5\\x81\\x07'\\xc5\\xf9\\xe9w\\xc4\\xc3\\xfc\\xb9t\\x1e\\xbf\\xda{b:\\xa3ti\\\"\\x98\\xc8\\xe1\\xf0\\x01\\x7fE\\xd4\\xbe\\xbdqL\\x99\\xbe\\xaa\\x12\\x95SefMc\\xdd\\xfe\\x9a_62\\x9f5\\x9f6v#\\xca\\xd9\\x9f\\xbd\\x93\\x8d\\x96\\xc4Z\\xf2\\xf6\\xefD\\x94\\xe0\\xbd6v5Kk\\x83\\xbf\\xd8>v\\xe3b\\xdb\\xc0U,\\xc0eqTl|A$\\xa26&w\\xf5\\x7f\\xee\\xfc\\xe4\\xe9\\x99~}e\\x0f\\xfb\\\"\\xc2\\xd8\\x90;.\\xff\\xf9]\\xbcL&\\xef\\xdan\\xdb\\x8ca\\x16-_)\\xcc\\x17dc\\x01\\xe0s\\xed\\xf7-'\\x06\\xd8N\\xbb\\xa5\\x19K\\xde\\xa81\\xef\\xab\\xd4\\x1b\\xb4Z&\\xe1\\xc3\\x98\\x820D-\\x0euN\\xfccx\\xe8\\x9f\\xf7\\xae)\\x12\\x0e\\xb0\\xb5E\\xc6\\xca)\\x1f\\xec\\xec\\x03\\t\\x1d\\x88}()\\xa9\\xc4\\xde\\xbe }\\x7f\\x92\\xf4\\xe7\\x0ehvQ>\\xc7\\xd7\\xf1Oq\\xd6\\xbfO\\xf69a\\x17\\xb9s0\\xb6+\\x1c\\x8f0g\\xd9R\\xc1K\\xf0z\\xe2\\x07\\xb3\\x87\\xaev_>\\x83\\x15\\t\\x9d\\x90|\\xafO\\\")\\x14\\xc1}\\x9c\\xeb\\xd0e,\\xdd\\xe3\\x1f\\x1c\\x8c\\xa3=2>vk\\xe4\\xf1s\\x17\\xd7r\\xb0\\x90\\x13\\xf1\\xed\\x10/3J\\x0eJ\\xe0\\x95\\xa5\\x8f\\x85\\x05\\xc2\\xbc\\xd7W\\t\\xb3\\x84y z\\x1d\\xd8q\\xf0\\xe8?\\xe5\\xb2LWm\\xd0U2\\xf2\\xec0U,Z\\x82\\xde\\xfb]\\xd9\\x18\\xc5\\x89m\\xf7n^\\xf8+z\\x88\\x86\\xe3\\xacA\\xd4\\x8b\\xc6\\xc1\\xd3\\x8b\\xc0\\xc3\\x01M8\\x1e!?\\x9a\\xfd\\x99\\xe1Gu\\xd3\\xf0|G\\xe5PM\\x1e\\xed\\xb4\\xb5\\x1c\\xa8\\xeb8t\\xb4\\xfe\\x14\\xeaEvW\\xe9\\xec\\xc5\\xa5\\xa3\\xc4\\xa5#\\x97Lo\\xf6\\x0f\\xbe\\xaa\\\"\\xefE\\x0e\\xae\\x8cM)\\xda\\x9e\\xc4\\xbcX\\xd7\\x07\\xe0.\\x85\\x83\\xce\\x84\\xc9\\xa6\\xb8\\xe3\\xda\\xd8w\\xa6\\xab\\x02\\xdc\\x05\\xa7\\x100=\\x12|7\\r\\x87\\xef\\xd3\\x13\\x06\\xfe\\xba,Bpw\\x92\\x93p\\xbc\\x01\\x939\\x8a\\x99\\xdc\\xc1L\\x84uS\\xc3\\xbb\\xb2\\rn\\xcf\\x0c\\xff\\x03\\xc7\\xf5\\xb1k\\x95\\xa5\\x07@\\xbc\\x83\\x835\\xae\\x9f\\xab\\x81g\\xe2q\\xde}\\xa9\\xb8n\\xe0\\x06\\xce!\\xe9Q\\x17\\x0en\\x94\\x16W\\xa7b\\x1c\\xabm\\xb2\\xb8\\xbeT\\x82\\x91<1\\xd0\\xd9~\\x1cQ]\\xc72w\\xb3\\xc2\\xf5\\xbb\\xd3\\xf6\\xe6L>\\xech\\xefAT\\xcf\\xb1\\xectV\\x18\\xba+y\\xa9\\x8f\\x0f\\x91W\\x12\\xce\\xc7\\xa4d\\x97$\\xc9\\x99\\xfc3\\x99\\xad\\xc9\\x88\\xa2G\\xe5(G\\x9d\\xa5pyUj\\x17A?x\\xc9\\x923\\xb3SS\\xbb\\xb3N\\xb3f\\xf2tw\\xe7'\\xbd\\x99\\x9d\\xc9\\xae\\xdc\\xf3\\xeao\\xc5\\xb2\\xba\\xfa\\x9aZTG5\\x96\\x9b\\xcb\\xca\\xab\\xf4\\xa5U\\x8c\\xf0\\xe5\\xbfB\\xaa+?\\xaeF\\xfa\\xf9\\xfb\\x1a4M\\r\\x07\\xeb,\\x07\\x99I0~\\xd1O\\xe1u\\xf5N\\xe2i\\xe0\\xec\\x7f;'\\xe6<\\x04p\\xbc''z\\xea\\x18u\\x80\\x97\\xc3\\x8d\\x7f\\x13^\\x95\\xf5\\xe2%767T\\x99\\xca\\xf7\\xb3`\\x97<\\nw\\xbe!Po\\x0bn\\xc2JFX#Aa-\\xd1'w\\x9c\\x8c\\xffM\\xfeUD\\xdd\\x1e\\xe99\\x8eW\\xaeT\\xa77T\\xeb\\xd9=\\xf9\\x19\\x9aD\\x94\\x842l{Nf\\xf7\\xa9/\\xa2\\xcb\\x14\\x04J@z\\xf5\\xab?\\x7fq\\xf6\\x83(F.Y\\xf2QX,ZGm\\x18\\x8c\\xbbg6\\xd5\\xd461\\xe7\\xc5j\\x83\\x1eU *N\\xd1\\xfd\\xe9\\x85\\x81_\\x0f\\xd5\\xb0\\xb3\\xd5V\\xfe-+x7\\x1ck$\\x1d39\\x8f>\\x93\\xa7g\\x9f\\xd1s\\x16A\\xfc\\x07\\xbe\\x9e\\x12\\xf0\\nendstream\\nendobj\\n8 0 obj\\n<>\\nendobj\\n13 0 obj\\n<>stream\\nx\\x9c\\x9d\\x93{PSg\\x1a\\xc6O\\x80\\x9c\\x9c\\xad\\xb4\\\"\\xd9S\\xd4\\xb6Iv\\xba\\xabh\\x91\\x11\\xa4\\xad\\xbbu\\xb7\\xd3B\\xcb\\xb6\\x16G\\xc1\\x16P\\xa0\\x18\\x03$\\x84\\\\ AHBX\\x92p1\\xbc\\x04\\xb9$\\xe1\\x12 @@B@.\\xca\\x1dA\\xb7\\x8a\\x80\\x8e\\x8b\\xbb\\x9d\\xae\\xb3\\xf62\\xbb\\xba[;[hw\\xc3\\xd4\\xef\\x8cGg\\xf6$\\xe8t\\xf7\\xdf\\xfd\\xeb\\x9cy\\xbfs\\xde\\xf7\\xf9~\\xcf\\xf3\\xb2\\xb0\\xa0\\x00\\x8c\\xc5b=\\x1b\\xab(,\\x90d\\x15\\xecy[\\x91'\\xf2\\x15\\\"\\xa8\\x17X\\xd4\\x8b\\x01\\xd4K\\x81\\xfa\\x12\\xea1\\xf5\\x98M\\xf1\\x82\\xb1\\x9a`\\x16\\x04\\x07BpP\\xc7\\x8b\\x9c\\x0b\\xa1\\xc8\\xb3\\x05\\xc1f\\xa4\\r\\xc1\\x82X\\xac\\xd7\\xdfOi\\x0e\\xff01y\\xd7+\\xafD\\xc4*\\x94\\x9a\\x02I\\x8eX-\\x88\\xde\\x1b\\x15#\\x10j\\x04ON\\x04qY*I\\x8e\\\\\\xb0\\x83y9\\x95\\x95\\xa7P\\xca\\xb2\\xe4\\xeaC\\x12\\x99\\xb0P%HP\\xc8\\x15\\x82\\xc3I\\x02\\x9f\\x80\\xff-\\xfd\\xd8\\xee\\xff\\x1b\\x80a\\xd8\\xe6\\xb8\\x93\\xa2\\xac\\xe4\\xbdQ\\xd1\\xfbb^\\x15\\xec\\xff\\xe5\\xaf0\\xec\\x17X\\x1c\\xf6\\x0e\\xf6.\\xb6\\x1f\\xdb\\x82\\x85b\\\\\\xec\\xa7\\x18\\x89=\\x8f\\xb1\\xb0m\\xd8v\\xec\\x05,\\x84\\x81\\x82\\x05aE\\x18\\xc5r\\x07\\x04\\x04X\\x03\\x1e\\x04&\\x05^\\tJ\\x0bZ`\\xc7\\xb3\\xdfg/\\xe1\\xb1\\xb8\\x86Z}\\x8eZ\\x05/z\\xe8eQ\\x89\\x08\\x0b\\xfc\\xa3\\x97\\xcc\\xaaV\\x17C\\x1eh\\xad\\xbaf\\xa3\\xad\\xbc\\xf5\\xb4\\x0b\\x08\\x94\\x89\\xa3\\xe8*\\x14\\xf8\\xef\\x1a\\x14ALr\\x00\\xed\\xa19h\\x13\\xbd\\xd3L\\xd0b\\\\\\t\\xa6jC\\x85\\xce`\\xd0\\x82\\xd6\\xf7W\\x8b\\xd1Z\\xde`\\xee\\xaa&\\x10F?$\\xd1\\xc3\\x1f8\\xf7\\xcf\\xac\\xbck\\t'28\\x10\\x91p$\\xfc\\x0c\\xc1\\x8c,\\xf1\\xa2j/k\\x8e\\x99H\\x8dQ89\\xad\\xeb\\xcc),3\\x15\\x97\\xf3\\xb2\\xda\\x8fY\\x8f\\x02A\\xef\\x11\\xec\\xa6\\xf9\\x87;S\\xc6D\\xfc\\xb9\\xb4\\xebEk\\xf0\\x19\\xdc\\xb0\\x8f9';\\xbb{\\xe1,\\xd1\\xa7r\\xc9J\\rU&\\x03\\xefd\\xae\\xd4\\xf8\\x06\\xf3='q\\xf4\\xcf_,^\\xfafb\\xc8\\xa4\\xeb\\xe17\\x95\\xd7\\x9bjuu\\x85\\xb5\\x15\\x8d\\xe5V\\x93\\xa3\\xa2\\x05\\xda\\xc0\\xd1hon\\xb4Yl\\xd0\\xeb\\x13P\\xea\\x8dr\\xa2\\x15o\\xa8\\x1bah\\x02aa\\xdc)j\\x80\\xfa\\x9e\\xa4\\x83\\xf1\\xfc\\xa7\\xf7\\xd1\\x81\\x06\\xb4\\x8d%-\\x06{\\xb9\\xed\\xf4Y \\x9a~\\x86\\x8b\\xdc\\xa9\\xad\\x89\\xf0\\x1bH,J\\xcbL\\xcbT%\\xc1\\x07p\\xd0\\x954\\x939\\x93y\\xb5\\xe86,\\xc0\\x85\\xa6\\x8b\\x1e\\x82[,C\\xc1\\x1c\\x17\\xd8-\\xd6:\\x87\\xcd\\xd6\\x06\\xed\\xe009\\xf4\\xb6\\xb2\\x06\\xa3E\\x01\\xc4\\xefp\\xba\\x1e\\x95\\x90\\xb3\\xe0)\\xeb\\xcbw\\x15\\xb6HAFp\\xa7\\xde:\\x9c\\x1a\\x93\\x9e\\xdb\\xd4\\xa3\\xe4\\xa9\\xba\\xf5\\x1e\\x18\\x00O\\x8b\\xc7\\xd5}\\xb6w\\xc0>\\x0b\\x1b\\xc0n\\xdf\\xff\\x0bc\\xd2<\\xdaO\\x8eq\\xd0v:p\\x8d\\x8e\\xa0w\\xd1\\xecp\\x9a\\xa4\\xc3P@$\\x8a\\xfe\\xd4\\xdb\\xe6\\x9c\\xe2\\xf5\\xd8\\x9aZ\\xa1\\x93p\\x17v\\xcb\\xcb\\xca\\xcc\\xa7KyQ\\xea\\xfc\\xaat\\xd8\\x0f\\xa9\\xae\\x82K\\x84\\xe5>\\xe9\\x98^\\x18X\\x81\\x15\\xb8*mK\\xf7u\\x06'\\x95\\xe0e\\xa1\\xcb\\xc8F~M\\xdb\\xd8\\x88\\xc0\\x17)a\\x7f][\\x07\\x9c\\xdd\\xc6\\x08o\\xd5\\xdb\\x9f\\x08\\xa7\\xc3\\x9e\\xb21\\x1a4>\\xaf\\x1b\\x19\\xaf\\xed&\\xbb\\xb9\\x17\\x88\\x8bx.m\\x8cE\\x1f\\xb3i\\x0c\\x8f\\xa5?\\xceEF\\xf6\\x04\\xeeC`\\xfb\\x11A+\\x83\\xa0\\xd1\\xf0\\xa4\\x93\\x12\\xca\\x99NZ\\x83Q\\x07E\\xa0ph\\xfb\\xab\\x96\\x1f\\t\\xb7\\xa2gpF\\x91\\xdeK\\xfd\\xda\\xcb\\xba\\xc38s\\xca\\x17\\x90v\\xf4\\x1d\\t\\xf7\\xe4wR\\xe7s\\x86\\x8e\\xb7\\x1f\\x81#p\\\\\\x93#NM\\x91\\x1f\\x80}D\\x14\\x07b\\xdco\\xcc\\xa5\\x0e\\x8bg5\\x0b\\x8c\\x03\\xb3\\xed\\xc3Css\\xee\\xcf\\xe1.A\\xdf]%\\xd7&\\xaf\\xdf\\xba5\\xf9\\xc1.\\xde\\xcf9\\xbb3\\x0e\\xc6\\xc7g\\xdcX\\xe5m$\\xfe\\xae\\x93\\x85\\xaa\\x99\\xf6\\xe8\\x01\\xf5\\x98\\xa4e\\x1f\\x9d0\\xe8\\xf5 \\xdf&\\xebR\\xf5\\xd9jk\\xea\\x9c\\xbc/;\\xd9\\x8f\\xb6\\xec\\xe6\\xe4\\xffw\\xbcuV\\xed\\xc6Rt3K\\xf1\\t>\\xedj?\\xe7\\xbf\\x17\\xdfw1%\\x10\\xbb}\\xf2a\\x9d\\x8ad\\x9cz\\xd9\\xd7\\\\\\xbeN\\xa2f\\x94\\xe5\\x1e\\x84\\xaf\\x88\\x07\\x91_\\xd0!\\x87\\x92\\x8a\\xc4B\\x9eX\\xa6L\\x03)\\xa1\\xecQ\\xbb\\xbb\\x9dM\\xed\\xf5<\\xbb\\xa7\\xc6b\\xb5u\\xb9\\x06[\\xce\\x03q}V\\x9c\\x96\\xa7+\\xde\\x19\\xc3\\x17\\xe6\\xbc\\x93H\\x13Q\\x15\\x95[\\x05\\x94\\xf0\\x1e\\x07\\\\fk\\x85\\xcd\\xd0\\xaa\\xb5\\x16\\x83\\x14\\xb4\\xba*1\\xe1\\xc7\\x85\\xbes^\\xf3\\x86R;\\x11\\xf6\\xaa/\\xca\\xdf 7\\xf5\\x13R\\xaa*\\x94\\xcb\\x9d\\xda!3\\x7f\\xcal7;M\\xd3\\x9a>)H\\xe0T\\x99ZW\\x9a\\xaf\\xce1\\xc6\\xc3A\\x90\\xd7\\xa9\\x1cZ[\\xa5\\xa5\\x14\\x88<\\xb5Z\\x9e\\xf2U.\\n\\xbdw\\xb9yp\\x8a?s\\xce\\xfd\\t\\\\\\x85\\xc5\\xec\\xb9\\xb8s\\x04\\xf7_\\x8bC\\xbd\\xa3\\xf3\\xdba\\xbcx\\\\\\xea\\x11\\x8d$w\\xc43&\\x06\\x86'\\x1f\\x91\\xbb\\xd4\\xee\\xd6\\x96z\\x9b\\x95?0\\xd8k\\xfb=\\x10\\x7f\\x18\\xcf?!:)I\\xe3\\xfb)\\xbb}\\xd2X\\xe8[\\x9f\\x8d\\xc9\\xd4\\x1aI\\xbf\\x84\\xd3U\\x8fH\\xf6\\xeb\\xa8G.\\xe1\\x14\\x80\\xd1l\\xa8\\xdc@KH\\\\\\x9ai\\x1e\\xda\\x8a\\xcf\\xf8\\x99:\\xf4V\\xbe\\xa1\\xa1\\xdcRXC\\xb89\\xe7k\\xba:\\x98\\x8d\\xf0/\\x91\\xa1\\xde_\\xa4\\xb1\\xe7i\\x1e\\x8ex(\\x97\\xbdA \\xdf\\xfbW&\\xc4\\x1c&3\\x19>\\xee*\\xaa\\x92D\\xc7\\xf0.h\\xb14>M`\\x9b?\\x81\\r~\\xa3\\xe8kt\\x1f\\x9e\\xdb\\xad\\xf2\\xd8\\xcf\\xd44\\xb4\\xf0\\xc6\\x9c\\xd3\\xcd\\x1e nNd\\xc4\\xbf\\x95.\\xd9\\xf1\\x9e\\xa2\\xa1[\\xc6/i6\\xd5\\x96\\x00!/P+\\x92\\xee\\x9f@!\\xdf.t\\xccL\\xf1\\x87G\\x9d\\xf3p\\x85@[\\xf6~M\\x87\\xc8\\xf3*\\rb_\\xa06D\\xbc\\xb6\\x8e\\xf6yC\\x99\\xe0\\x863:D\\xfeG\\x18w\\x95z\\x13-\\x91W\\x86\\xddSp\\x91\\xf8>\\xf2\\x0e\\xbd\\x89\\xde\\x14y`g\\xaa;\\xf3J6\\x8f\\xebM\\xc8\\x96\\xa6\\x1c\\xde\\xfe\\xf2\\xdf\\xe3P\\x18\\xda\\xfa\\x8f?\\xad_\\x93\\xce'\\x8c\\xf0\\xb8\\xab4\\x17\\t\\xc9\\xa5\\ti\\xfa\\xb1\\x13\\xd2\\x84C\\x99\\x8333\\xe3\\x03\\xcb|\\xae\\x97v\\x04-\\xcf\\xe7d\\x1cO\\xcf\\xfd\\xed{i\\x833\\xd3\\xf3\\xc3\\xcb>\\xd6\\xfa\\x1fP\\xe8::\\xeae=\\xf0\\xb1\\x8eC\\xfd\\xa4\\x92f\\xed{s\\x07\\x18\\xe1t\\x8d\\xa1V[o\\xb0\\x18\\x80\\x90\\x15\\xa8e\\xa2\\xd9\\xfcO\\xff\\xf9\\xe5\\x85\\xcfW\\xf8\\x97\\x96z?\\x83\\xbf\\xc1-\\xcdm\\xe5\\xb4\\xe8\\xe6\\xa1\\xc1\\xd7 \\x1eR\\x8b\\xb3E\\x92\\x9c\\xe2T8\\xca\\x18|7\\x1aa\\xb3\\xa3m\\xe3\\x93<\\x13\\xdaL\\xe6g\\x1c\\xcb\\x15\\x02\\x91,\\x1c\\xbf\\xbc4<\\xbcx\\xe3\\x9c\\xf8@\\xab\\x7f4\\xe3\\xf0\\xb2\\x9e<\\xefq\\x8f\\x8e\\xe4\\xf5\\x8b\\xf8\\x1a>stream\\n\\n\\n\\n\\n\\n2024-12-18T15:59:31-05:00\\n2024-12-18T15:59:31-05:00\\nGNU Enscript 1.6.6\\n\\nEnscript Output\\n\\n\\n \\n \\n\\nendstream\\nendobj\\n2 0 obj\\n<>endobj\\nxref\\n0 15\\n0000000000 65535 f \\n0000000711 00000 n \\n0000007145 00000 n \\n0000000652 00000 n \\n0000000510 00000 n \\n0000000266 00000 n \\n0000000491 00000 n \\n0000001145 00000 n \\n0000003652 00000 n \\n0000000815 00000 n \\n0000001471 00000 n \\n0000000776 00000 n \\n0000001773 00000 n \\n0000003974 00000 n \\n0000005817 00000 n \\ntrailer\\n<< /Size 15 /Root 1 0 R /Info 2 0 R\\n/ID [<9BB34E42BF7AF21FE61720F4EBDFCCF8><9BB34E42BF7AF21FE61720F4EBDFCCF8>]\\n>>\\nstartxref\\n7334\\n%%EOF\\n\"\n }\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n finding_events = [e for e in events if e.type == \"FINDING\"]\n assert len(finding_events) == 1\n assert \"Possible Secret Found\" in finding_events[0].data[\"description\"]\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_txt.py", "content": "from .base import ModuleTestBase\n\n\nclass TestTXT(ModuleTestBase):\n def check(self, module_test, events):\n txt_file = module_test.scan.home / \"output.txt\"\n with open(txt_file) as f:\n assert f.read().startswith(\"[SCAN]\")\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_unarchive.py", "content": "import asyncio\n\nfrom pathlib import Path\nfrom .base import ModuleTestBase\n\nfrom ...bbot_fixtures import *\n\n\nclass TestUnarchive(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"filedownload\", \"httpx\", \"excavate\", \"speculate\", \"unarchive\"]\n config_overrides = {\n \"modules\": {\n \"filedownload\": {\n \"output_folder\": bbot_test_dir / \"filedownload\",\n },\n }\n }\n\n async def setup_after_prep(self, module_test):\n temp_path = Path(\"/tmp/.bbot_test\")\n\n # Create a text file to compress\n text_file = temp_path / \"test.txt\"\n with open(text_file, \"w\") as f:\n f.write(\"This is a test file\")\n zip_file = temp_path / \"test.zip\"\n zip_zip_file = temp_path / \"test_zip.zip\"\n bz2_file = temp_path / \"test.bz2\"\n xz_file = temp_path / \"test.xz\"\n zip7_file = temp_path / \"test.7z\"\n # lzma_file = temp_path / \"test.lzma\"\n tar_file = temp_path / \"test.tar\"\n tgz_file = temp_path / \"test.tgz\"\n commands = [\n (\"7z\", \"a\", \"-aoa\", f\"{zip_file}\", f\"{text_file}\"),\n (\"7z\", \"a\", \"-aoa\", f\"{zip_zip_file}\", f\"{zip_file}\"),\n (\"tar\", \"-C\", f\"{temp_path}\", \"-cvjf\", f\"{bz2_file}\", f\"{text_file.name}\"),\n (\"tar\", \"-C\", f\"{temp_path}\", \"-cvJf\", f\"{xz_file}\", f\"{text_file.name}\"),\n (\"7z\", \"a\", \"-aoa\", f\"{zip7_file}\", f\"{text_file}\"),\n # (\"tar\", \"-C\", f\"{temp_path}\", \"--lzma\", \"-cvf\", f\"{lzma_file}\", f\"{text_file.name}\"),\n (\"tar\", \"-C\", f\"{temp_path}\", \"-cvf\", f\"{tar_file}\", f\"{text_file.name}\"),\n (\"tar\", \"-C\", f\"{temp_path}\", \"-cvzf\", f\"{tgz_file}\", f\"{text_file.name}\"),\n ]\n\n for command in commands:\n process = await asyncio.create_subprocess_exec(\n *command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE\n )\n stdout, stderr = await process.communicate()\n assert process.returncode == 0, f\"Command {command} failed with error: {stderr.decode()}\"\n\n module_test.set_expect_requests(\n dict(uri=\"/\"),\n dict(\n response_data=\"\"\"\n \n \n \n \n \n \"\"\",\n ),\n )\n (\n module_test.set_expect_requests(\n dict(uri=\"/test.zip\"),\n dict(\n response_data=zip_file.read_bytes(),\n headers={\"Content-Type\": \"application/zip\"},\n ),\n ),\n )\n (\n module_test.set_expect_requests(\n dict(uri=\"/test-zip.zip\"),\n dict(\n response_data=zip_zip_file.read_bytes(),\n headers={\"Content-Type\": \"application/zip\"},\n ),\n ),\n )\n (\n module_test.set_expect_requests(\n dict(uri=\"/test.bz2\"),\n dict(\n response_data=bz2_file.read_bytes(),\n headers={\"Content-Type\": \"application/x-bzip2\"},\n ),\n ),\n )\n (\n module_test.set_expect_requests(\n dict(uri=\"/test.xz\"),\n dict(\n response_data=xz_file.read_bytes(),\n headers={\"Content-Type\": \"application/x-xz\"},\n ),\n ),\n )\n (\n module_test.set_expect_requests(\n dict(uri=\"/test.7z\"),\n dict(\n response_data=zip7_file.read_bytes(),\n headers={\"Content-Type\": \"application/x-7z-compressed\"},\n ),\n ),\n )\n # (\n # module_test.set_expect_requests(\n # dict(uri=\"/test.rar\"),\n # dict(\n # response_data=b\"Rar!\\x1a\\x07\\x01\\x003\\x92\\xb5\\xe5\\n\\x01\\x05\\x06\\x00\\x05\\x01\\x01\\x80\\x80\\x00\\xa2N\\x8ec&\\x02\\x03\\x0b\\x93\\x00\\x04\\x93\\x00\\xa4\\x83\\x02\\xc9\\x11f\\x06\\x80\\x00\\x01\\x08test.txt\\n\\x03\\x13S\\x96ug\\x96\\xf3\\x1b\\x06This is a test file\\x1dwVQ\\x03\\x05\\x04\\x00\",\n # headers={\"Content-Type\": \"application/vnd.rar\"},\n # ),\n # ),\n # )\n # (\n # module_test.set_expect_requests(\n # dict(uri=\"/test.lzma\"),\n # dict(\n # response_data=lzma_file.read_bytes(),\n # headers={\"Content-Type\": \"application/x-lzma\"},\n # ),\n # ),\n # )\n (\n module_test.set_expect_requests(\n dict(uri=\"/test.tar\"),\n dict(\n response_data=tar_file.read_bytes(),\n headers={\"Content-Type\": \"application/x-tar\"},\n ),\n ),\n )\n (\n module_test.set_expect_requests(\n dict(uri=\"/test.tgz\"),\n dict(\n response_data=tgz_file.read_bytes(),\n headers={\"Content-Type\": \"application/x-tgz\"},\n ),\n ),\n )\n\n def check(self, module_test, events):\n filesystem_events = [e for e in events if e.type == \"FILESYSTEM\"]\n\n # ZIP\n zip_file_event = [e for e in filesystem_events if \"test.zip\" in e.data[\"path\"]]\n assert 1 == len(zip_file_event), \"No zip file found\"\n file = Path(zip_file_event[0].data[\"path\"])\n assert file.is_file(), f\"File not found at {file}\"\n extract_event = [e for e in filesystem_events if \"test_zip\" in e.data[\"path\"] and \"folder\" in e.tags]\n assert 1 == len(extract_event), \"Failed to extract zip\"\n extract_path = Path(extract_event[0].data[\"path\"]) / \"test.txt\"\n assert extract_path.is_file(), \"Failed to extract the test file\"\n\n # Recursive ZIP\n zip_zip_file_event = [e for e in filesystem_events if \"test-zip.zip\" in e.data[\"path\"]]\n assert 1 == len(zip_zip_file_event), \"No recursive file found\"\n file = Path(zip_zip_file_event[0].data[\"path\"])\n assert file.is_file(), f\"File not found at {file}\"\n extract_event = [e for e in filesystem_events if \"test-zip_zip\" in e.data[\"path\"] and \"folder\" in e.tags]\n assert 1 == len(extract_event), \"Failed to extract zip\"\n extract_path = Path(extract_event[0].data[\"path\"]) / \"test\" / \"test.txt\"\n assert extract_path.is_file(), \"Failed to extract the test file\"\n\n # BZ2\n bz2_file_event = [e for e in filesystem_events if \"test.bz2\" in e.data[\"path\"]]\n assert 1 == len(bz2_file_event), \"No bz2 file found\"\n file = Path(bz2_file_event[0].data[\"path\"])\n assert file.is_file(), f\"File not found at {file}\"\n extract_event = [e for e in filesystem_events if \"test_bz2\" in e.data[\"path\"] and \"folder\" in e.tags]\n assert 1 == len(extract_event), \"Failed to extract bz2\"\n extract_path = Path(extract_event[0].data[\"path\"]) / \"test.txt\"\n assert extract_path.is_file(), \"Failed to extract the test file\"\n\n # XZ\n xz_file_event = [e for e in filesystem_events if \"test.xz\" in e.data[\"path\"]]\n assert 1 == len(xz_file_event), \"No xz file found\"\n file = Path(xz_file_event[0].data[\"path\"])\n assert file.is_file(), f\"File not found at {file}\"\n extract_event = [e for e in filesystem_events if \"test_xz\" in e.data[\"path\"] and \"folder\" in e.tags]\n assert 1 == len(extract_event), \"Failed to extract xz\"\n extract_path = Path(extract_event[0].data[\"path\"]) / \"test.txt\"\n assert extract_path.is_file(), \"Failed to extract the test file\"\n\n # 7z\n zip7_file_event = [e for e in filesystem_events if \"test.7z\" in e.data[\"path\"]]\n assert 1 == len(zip7_file_event), \"No 7z file found\"\n file = Path(zip7_file_event[0].data[\"path\"])\n assert file.is_file(), f\"File not found at {file}\"\n extract_event = [e for e in filesystem_events if \"test_7z\" in e.data[\"path\"] and \"folder\" in e.tags]\n assert 1 == len(extract_event), \"Failed to extract 7z\"\n extract_path = Path(extract_event[0].data[\"path\"]) / \"test.txt\"\n assert extract_path.is_file(), \"Failed to extract the test file\"\n\n # RAR\n # rar_file_event = [e for e in filesystem_events if \"test.rar\" in e.data[\"path\"]]\n # assert 1 == len(rar_file_event), \"No rar file found\"\n # file = Path(rar_file_event[0].data[\"path\"])\n # assert file.is_file(), f\"File not found at {file}\"\n # extract_event = [e for e in filesystem_events if \"test_rar\" in e.data[\"path\"] and \"folder\" in e.tags]\n # assert 1 == len(extract_event), \"Failed to extract rar\"\n # extract_path = Path(extract_event[0].data[\"path\"]) / \"test.txt\"\n # assert extract_path.is_file(), list(extract_path.parent.iterdir())\n\n # LZMA\n # lzma_file_event = [e for e in filesystem_events if \"test.lzma\" in e.data[\"path\"]]\n # assert 1 == len(lzma_file_event), \"No lzma file found\"\n # file = Path(lzma_file_event[0].data[\"path\"])\n # assert file.is_file(), f\"File not found at {file}\"\n # extract_event = [e for e in filesystem_events if \"test_lzma\" in e.data[\"path\"] and \"folder\" in e.tags]\n # assert 1 == len(extract_event), \"Failed to extract lzma\"\n # extract_path = Path(extract_event[0].data[\"path\"]) / \"test.txt\"\n # assert extract_path.is_file(), \"Failed to extract the test file\"\n\n # TAR\n tar_file_event = [e for e in filesystem_events if \"test.tar\" in e.data[\"path\"]]\n assert 1 == len(tar_file_event), \"No tar file found\"\n file = Path(tar_file_event[0].data[\"path\"])\n assert file.is_file(), f\"File not found at {file}\"\n extract_event = [e for e in filesystem_events if \"test_tar\" in e.data[\"path\"] and \"folder\" in e.tags]\n assert 1 == len(extract_event), \"Failed to extract tar\"\n extract_path = Path(extract_event[0].data[\"path\"]) / \"test.txt\"\n assert extract_path.is_file(), \"Failed to extract the test file\"\n\n # TGZ\n tgz_file_event = [e for e in filesystem_events if \"test.tgz\" in e.data[\"path\"]]\n assert 1 == len(tgz_file_event), \"No tgz file found\"\n file = Path(tgz_file_event[0].data[\"path\"])\n assert file.is_file(), f\"File not found at {file}\"\n extract_event = [e for e in filesystem_events if \"test_tgz\" in e.data[\"path\"] and \"folder\" in e.tags]\n assert 1 == len(extract_event), \"Failed to extract tgz\"\n extract_path = Path(extract_event[0].data[\"path\"]) / \"test.txt\"\n assert extract_path.is_file(), \"Failed to extract the test file\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_url_manipulation.py", "content": "from .base import ModuleTestBase\n\n\nclass TestUrl_Manipulation(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"url_manipulation\"]\n body = \"\"\"\n \n the title\n \n

Hello null!

';\n \n \n \"\"\"\n\n body_match = \"\"\"\n \n the title\n \n

Hello AAAAAAAAAAAAAA!

';\n \n \n \"\"\"\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"query_string\": f\"{module_test.module.rand_string}=.xml\".encode()}\n respond_args = {\"response_data\": self.body_match}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n respond_args = {\"response_data\": self.body}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n assert any(\n e.type == \"FINDING\"\n and e.data[\"description\"]\n == f\"Url Manipulation: [body] Sig: [Modified URL: http://127.0.0.1:8888/?{module_test.module.rand_string}=.xml]\"\n for e in events\n )\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_urlscan.py", "content": "from .base import ModuleTestBase\n\n\nclass TestUrlScan(ModuleTestBase):\n config_overrides = {\"modules\": {\"urlscan\": {\"urls\": True}}}\n\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://urlscan.io/api/v1/search/?q=blacklanternsecurity.com\",\n json={\n \"results\": [\n {\n \"task\": {\n \"visibility\": \"public\",\n \"method\": \"api\",\n \"domain\": \"asdf.blacklanternsecurity.com\",\n \"apexDomain\": \"blacklanternsecurity.com\",\n \"time\": \"2023-05-17T01:45:11.391Z\",\n \"uuid\": \"c558b3b3-b274-4339-99ef-301eb043741f\",\n \"url\": \"https://asdf.blacklanternsecurity.com/cna.html\",\n },\n \"stats\": {\n \"uniqIPs\": 6,\n \"uniqCountries\": 3,\n \"dataLength\": 926713,\n \"encodedDataLength\": 332213,\n \"requests\": 22,\n },\n \"page\": {\n \"country\": \"US\",\n \"server\": \"GitHub.com\",\n \"ip\": \"2606:50c0:8002::153\",\n \"mimeType\": \"text/html\",\n \"title\": \"Vulnerability Program | Black Lantern Security\",\n \"url\": \"https://asdf.blacklanternsecurity.com/cna.html\",\n \"tlsValidDays\": 89,\n \"tlsAgeDays\": 25,\n \"tlsValidFrom\": \"2023-04-21T19:16:58.000Z\",\n \"domain\": \"asdf.blacklanternsecurity.com\",\n \"apexDomain\": \"blacklanternsecurity.com\",\n \"asnname\": \"FASTLY, US\",\n \"asn\": \"AS54113\",\n \"tlsIssuer\": \"R3\",\n \"status\": \"200\",\n },\n \"_id\": \"c558b3b3-b274-4339-99ef-301eb043741f\",\n \"_score\": None,\n \"sort\": [1684287911391, \"c558b3b3-b274-4339-99ef-301eb043741f\"],\n \"result\": \"https://urlscan.io/api/v1/result/c558b3b3-b274-4339-99ef-301eb043741f/\",\n \"screenshot\": \"https://urlscan.io/screenshots/c558b3b3-b274-4339-99ef-301eb043741f.png\",\n }\n ]\n },\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n assert any(e.data == \"https://asdf.blacklanternsecurity.com/cna.html\" for e in events), \"Failed to detect URL\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_vhost.py", "content": "from .base import ModuleTestBase, tempwordlist\n\n\nclass TestVhost(ModuleTestBase):\n targets = [\"http://localhost:8888\", \"secret.localhost\"]\n modules_overrides = [\"httpx\", \"vhost\"]\n test_wordlist = [\"11111111\", \"admin\", \"cloud\", \"junkword1\", \"zzzjunkword2\"]\n config_overrides = {\n \"modules\": {\n \"vhost\": {\n \"wordlist\": tempwordlist(test_wordlist),\n }\n }\n }\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\", \"headers\": {\"Host\": \"admin.localhost:8888\"}}\n respond_args = {\"response_data\": \"Alive vhost admin\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/\", \"headers\": {\"Host\": \"cloud.localhost:8888\"}}\n respond_args = {\"response_data\": \"Alive vhost cloud\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/\", \"headers\": {\"Host\": \"q-cloud.localhost:8888\"}}\n respond_args = {\"response_data\": \"Alive vhost q-cloud\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/\", \"headers\": {\"Host\": \"secret.localhost:8888\"}}\n respond_args = {\"response_data\": \"Alive vhost secret\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/\", \"headers\": {\"Host\": \"host.docker.internal\"}}\n respond_args = {\"response_data\": \"Alive vhost host.docker.internal\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": \"alive\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n basic_detection = False\n mutaton_of_detected = False\n basehost_mutation = False\n special_vhost_list = False\n wordcloud_detection = False\n\n for e in events:\n if e.type == \"VHOST\":\n if e.data[\"vhost\"] == \"admin\":\n basic_detection = True\n if e.data[\"vhost\"] == \"cloud\":\n mutaton_of_detected = True\n if e.data[\"vhost\"] == \"q-cloud\":\n basehost_mutation = True\n if e.data[\"vhost\"] == \"host.docker.internal\":\n special_vhost_list = True\n if e.data[\"vhost\"] == \"secret\":\n wordcloud_detection = True\n\n assert basic_detection\n assert mutaton_of_detected\n assert basehost_mutation\n assert special_vhost_list\n assert wordcloud_detection\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_viewdns.py", "content": "from .base import ModuleTestBase\n\n\nclass TestViewDNS(ModuleTestBase):\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://viewdns.info/reversewhois/?q=blacklanternsecurity.com\",\n text=web_body,\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"hyperloop.com\" and \"affiliate\" in e.tags for e in events), (\n \"Failed to detect affiliate domain\"\n )\n\n\nweb_body = \"\"\"\n \n \n Reverse Whois Lookup - ViewDNS.info\n \n \n \n \n \n \n \n \n \n \n \n
\"ViewDNS.info\n \n \n \n \n
\n \n \n
\n \n \n \n \n \n \n
\n \n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n ViewDNS.info > Tools >\n

Reverse Whois Lookup

\n

This free tool will allow you to find domain names owned by an individual person or company. Simply enter the email address or name of the person or company to find other domains registered using those same details. FAQ.

\n
Frequently Asked Questions
Q. Will this tool return results for all domains including ccTLD's?
A. Unfortunately no. Whilst we do our best to ensure our data is as complete as possible, we are not able to return results for all ccTLD's. Due to a number of technical limitations with whois data, the results from any Reverse Whois tool should not be considered as exhaustive.

Q. Is your data live?
A. Our data is not live. We do our best to update the data as often as possible with daily updates for selected TLD's and quarterly updates for others.

Q. How do I see all records for a specific person/company rather than the limited number you show on your site?
A. Please email us with your request and we'll see what we can do for you.

\n
Registrant Name or Email Address:
\n \n
\n
\n Reverse Whois results for blacklanternsecurity.com
==============

There are 20 domains that matched this search query.
These are listed below:

\n \n \n \n \n \n \n
hyperloop.com2003-12-04NETWORK SOLUTIONS, LLC.
\n
\n
\n \n \n \n \n \n
\n \n \n \n \n

\n \n
\n \n \n Follow @viewdns\n \n
\n
\n All content © 2023 ViewDNS.info
Feedback / Suggestions / Contact Us - Privacy Policy\n
\n
\n \n \n \n \n
\n \n \n \n
\n
\n
\n
\n
\n \n \n\"\"\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_virustotal.py", "content": "from .base import ModuleTestBase\n\n\nclass TestVirusTotal(ModuleTestBase):\n config_overrides = {\"modules\": {\"virustotal\": {\"api_key\": \"asdf\"}}}\n\n async def setup_before_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"https://www.virustotal.com/api/v3/domains/blacklanternsecurity.com/subdomains\",\n json={\n \"meta\": {\"count\": 25, \"cursor\": \"eyJsaW1pdCI6IDEwLCAib2Zmc2V0IjogMTB9\"},\n \"data\": [\n {\n \"attributes\": {\n \"last_dns_records\": [{\"type\": \"A\", \"value\": \"168.62.180.225\", \"ttl\": 3600}],\n \"whois\": \"Creation Date: 2013-07-30T20:14:50Z\\nDNSSEC: unsigned\\nDomain Name: BLACKLANTERNSECURITY.COM\\nDomain Status: clientDeleteProhibited https://icann.org/epp#clientDeleteProhibited\\nDomain Status: clientRenewProhibited https://icann.org/epp#clientRenewProhibited\\nDomain Status: clientTransferProhibited https://icann.org/epp#clientTransferProhibited\\nDomain Status: clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited\\nName Server: NS01.DOMAINCONTROL.COM\\nName Server: NS02.DOMAINCONTROL.COM\\nRegistrar Abuse Contact Email: abuse@godaddy.com\\nRegistrar Abuse Contact Phone: 480-624-2505\\nRegistrar IANA ID: 146\\nRegistrar URL: http://www.godaddy.com\\nRegistrar WHOIS Server: whois.godaddy.com\\nRegistrar: GoDaddy.com, LLC\\nRegistry Domain ID: 1818679075_DOMAIN_COM-VRSN\\nRegistry Expiry Date: 2023-07-30T20:14:50Z\\nUpdated Date: 2022-09-14T16:28:14Z\",\n \"tags\": [],\n \"popularity_ranks\": {},\n \"last_dns_records_date\": 1657734301,\n \"last_analysis_stats\": {\n \"harmless\": 0,\n \"malicious\": 0,\n \"suspicious\": 0,\n \"undetected\": 86,\n \"timeout\": 0,\n },\n \"creation_date\": 1375215290,\n \"reputation\": 0,\n \"registrar\": \"GoDaddy.com, LLC\",\n \"last_analysis_results\": {},\n \"last_update_date\": 1663172894,\n \"last_modification_date\": 1657734301,\n \"tld\": \"com\",\n \"categories\": {},\n \"total_votes\": {\"harmless\": 0, \"malicious\": 0},\n },\n \"type\": \"domain\",\n \"id\": \"asdf.blacklanternsecurity.com\",\n \"links\": {\"self\": \"https://www.virustotal.com/api/v3/domains/asdf.blacklanternsecurity.com\"},\n \"context_attributes\": {\"timestamp\": 1657734301},\n }\n ],\n \"links\": {\n \"self\": \"https://www.virustotal.com/api/v3/domains/blacklanternsecurity.com/subdomains?limit=10\",\n \"next\": \"https://www.virustotal.com/api/v3/domains/blacklanternsecurity.com/subdomains?cursor=eyJsaW1pdCI6IDEwLCAib2Zmc2V0IjogMTB9&limit=10\",\n },\n },\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_wafw00f.py", "content": "from .base import ModuleTestBase\n\nfrom werkzeug.wrappers import Response\n\n\nclass TestWafw00f(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"wafw00f\"]\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"response_data\": \"Proudly powered by litespeed web server\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n assert any(e.type == \"WAF\" and \"LiteSpeed\" in e.data[\"waf\"] for e in events)\n\n\nclass TestWafw00f_noredirect(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"wafw00f\"]\n\n async def setup_after_prep(self, module_test):\n expect_args = {\"method\": \"GET\", \"uri\": \"/\"}\n respond_args = {\"status\": 301, \"headers\": {\"Location\": \"/redirect\"}}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n expect_args = {\"method\": \"GET\", \"uri\": \"/redirect\"}\n respond_args = {\"response_data\": \"Proudly powered by litespeed web server\"}\n module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)\n\n def check(self, module_test, events):\n assert not any(e.type == \"WAF\" for e in events)\n\n\nclass TestWafw00f_genericdetection(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"wafw00f\"]\n\n async def setup_after_prep(self, module_test):\n def handler(request):\n if \"SLEEP\" in request.url:\n return Response(\"nope\", status=403)\n return Response(\"yep\")\n\n module_test.httpserver.expect_request(\"/\").respond_with_handler(handler)\n\n def check(self, module_test, events):\n waf_events = [e for e in events if e.type == \"WAF\"]\n assert len(waf_events) == 1\n assert waf_events[0].data[\"waf\"] == \"generic detection\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_wayback.py", "content": "from .base import ModuleTestBase\n\n\nclass TestWayback(ModuleTestBase):\n async def setup_after_prep(self, module_test):\n module_test.httpx_mock.add_response(\n url=\"http://web.archive.org/cdx/search/cdx?url=blacklanternsecurity.com&matchType=domain&output=json&fl=original&collapse=original\",\n json=[[\"original\"], [\"http://asdf.blacklanternsecurity.com\"]],\n )\n\n def check(self, module_test, events):\n assert any(e.data == \"asdf.blacklanternsecurity.com\" for e in events), \"Failed to detect subdomain\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_web_parameters.py", "content": "from .test_module_excavate import TestExcavateParameterExtraction\n\n\nclass TestWebParameters(TestExcavateParameterExtraction):\n modules_overrides = [\"excavate\", \"httpx\", \"web_parameters\"]\n\n def check(self, module_test, events):\n parameters_file = module_test.scan.home / \"web_parameters.txt\"\n with open(parameters_file) as f:\n data = f.read()\n\n assert \"age\" in data\n assert \"fit\" in data\n assert \"id\" in data\n assert \"jqueryget\" in data\n assert \"jquerypost\" in data\n assert \"size\" in data\n\n # after lightfuzz is merged uncomment these additional parameters\n # assert \"blog-post-author-display\" in data\n # assert \"csrf\" in data\n # assert \"q1\" in data\n # assert \"q2\" in data\n # assert \"q3\" in data\n # assert \"test\" in data\n\n\nclass TestWebParameters_include_count(TestWebParameters):\n config_overrides = {\n \"web\": {\"spider_distance\": 1, \"spider_depth\": 1},\n \"modules\": {\"web_parameters\": {\"include_count\": True}},\n }\n\n def check(self, module_test, events):\n parameters_file = module_test.scan.home / \"web_parameters.txt\"\n with open(parameters_file) as f:\n data = f.read()\n assert \"2\\tq\" in data\n assert \"1\\tage\" in data\n assert \"1\\tfit\" in data\n assert \"1\\tid\" in data\n assert \"1\\tjqueryget\" in data\n assert \"1\\tjquerypost\" in data\n assert \"1\\tsize\" in data\n\n # after lightfuzz is merged, these will be the correct parameters to check\n\n # assert \"3\\ttest\" in data\n # assert \"2\\tblog-post-author-display\" in data\n # assert \"2\\tcsrf\" in data\n # assert \"2\\tq2\" in data\n # assert \"1\\tage\" in data\n # assert \"1\\tfit\" in data\n # assert \"1\\tid\" in data\n # assert \"1\\tjqueryget\" in data\n # assert \"1\\tjquerypost\" in data\n # assert \"1\\tq1\" in data\n # assert \"1\\tq3\" in data\n # assert \"1\\tsize\" in data\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_web_report.py", "content": "from .base import ModuleTestBase\n\n\nclass TestWebReport(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"dotnetnuke\", \"badsecrets\", \"web_report\", \"trufflehog\"]\n config_overrides = {\"modules\": {\"trufflehog\": {\"only_verified\": False}}}\n\n async def setup_before_prep(self, module_test):\n # trufflehog --> FINDING\n # dotnetnuke --> TECHNOLOGY\n # badsecrets --> VULNERABILITY\n respond_args = {\"response_data\": web_body}\n module_test.set_expect_requests(respond_args=respond_args)\n\n def check(self, module_test, events):\n report_file = module_test.scan.home / \"web_report.html\"\n with open(report_file) as f:\n report_content = f.read()\n assert \"
  • [CRITICAL] Known Secret Found\" in report_content\n assert (\n \"\"\"

    URL

    \n
      \n
    • http://127.0.0.1:8888/\"\"\"\n in report_content\n )\n assert \"\"\"Possible Secret Found. Detector Type: [PrivateKey]\"\"\" in report_content\n assert \"

      TECHNOLOGY

      \" in report_content\n assert \"
    • DotNetNuke
      • \" in report_content\n\n\nweb_body = \"\"\"\n\n\n\n\n\n\n
      \n
      \n\n
      \n\n
      \n\n \n \n
      \n
      \n

      -----BEGIN PRIVATE KEY-----\nMIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAOBY2pd9PSQvuxqu\nWXFNVgILTWuUc721Wc2sFNvp4beowhUe1lfxaq5ZfCJcz7z4QsqFhOeks69O9UIb\noiOTDocPDog9PHO8yZXopHm0StFZvSjjKSNuFvy/WopPTGpxUZ5boCaF1CXumY7W\nFL+jIap5faimLL9prIwaQKBwv80lAgMBAAECgYEAxvpHtgCgD849tqZYMgOTevCn\nU/kwxltoMOClB39icNA+gxj8prc6FTTMwnVq0oGmS5UskX8k1yHCqUV1AvRU9o+q\nI8L8a3F3TQKQieI/YjiUNK8A87bKkaiN65ooOnhT+I3ZjZMPR5YEyycimMp22jsv\nLyX/35J/wf1rNiBs/YECQQDvtxgmMhE+PeajXqw1w2C3Jds27hI3RPDnamEyWr/L\nKkSplbKTF6FuFDYOFdJNPrfxm1tx2MZ2cBfs+h/GnCJVAkEA75Z9w7q8obbqGBHW\n9bpuFvLjW7bbqO7HBuXYX9zQcZL6GSArFP0ba5lhgH1qsVQfxVWVyiV9/chme7xc\nljfvkQJBAJ7MpSPQcRnRefNp6R0ok+5gFqt55PlWI1y6XS81bO7Szm+laooE0n0Q\nyIpmLE3dqY9VgquVlkupkD/9poU0s40CQD118ZVAVht1/N9n1Cj9RjiE3mYspnTT\nrCLM25Db6Gz6M0Y2xlaAB4S2uBhqE/Chj/TjW6WbsJJl0kRzsZynhMECQFYKiM1C\nT4LB26ynW00VE8z4tEWSoYt4/Vn/5wFhalVjzoSJ8Hm2qZiObRYLQ1m0X4KnkShk\nGnl54dJHT+EhlfY=\n-----END PRIVATE KEY-----

      \n\n\n\"\"\"\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_websocket.py", "content": "import json\nimport asyncio\nimport logging\nfrom websockets.asyncio.server import serve\n\nfrom .base import ModuleTestBase\n\nlog = logging.getLogger(\"bbot.testing\")\n\nresults = {\"events\": []}\n\n\nasync def websocket_handler(websocket):\n results[\"path\"] = websocket.request.path\n async for message in websocket:\n results[\"events\"].append(message)\n\n\n# Define a coroutine for the server\nasync def server_coroutine():\n async with serve(websocket_handler, \"127.0.0.1\", 8765) as server:\n await server.serve_forever()\n\n\nclass TestWebsocket(ModuleTestBase):\n config_overrides = {\"modules\": {\"websocket\": {\"url\": \"ws://127.0.0.1:8765/testing\"}}}\n\n async def setup_before_prep(self, module_test):\n self.server_task = asyncio.create_task(server_coroutine())\n\n def check(self, module_test, events):\n assert results[\"path\"] == \"/testing\"\n decoded_events = [json.loads(e) for e in results[\"events\"]]\n assert any(e[\"type\"] == \"SCAN\" for e in decoded_events)\n self.server_task.cancel()\n" }, { "path": "bbot/test/test_step_2/module_tests/test_module_wpscan.py", "content": "from subprocess import CompletedProcess\nfrom .base import ModuleTestBase\n\n\nclass Testwpscan(ModuleTestBase):\n targets = [\"http://127.0.0.1:8888\"]\n modules_overrides = [\"httpx\", \"wpscan\"]\n\n wpscan_output_json = \"\"\"{\n \"banner\": {\n \"description\": \"WordPress Security Scanner by the WPScan Team\",\n \"version\": \"3.8.25\",\n \"authors\": [\n \"@_WPScan_\",\n \"@ethicalhack3r\",\n \"@erwan_lr\",\n \"@firefart\"\n ],\n \"sponsor\": \"Sponsored by Automattic - https://automattic.com/\"\n },\n \"start_time\": 1717183319,\n \"start_memory\": 49950720,\n \"target_url\": \"http://127.0.0.1:8888/\",\n \"target_ip\": \"172.29.64.1\",\n \"effective_url\": \"http://127.0.0.1:8888/\",\n \"interesting_findings\": [\n {\n \"url\": \"http://127.0.0.1:8888/\",\n \"to_s\": \"Headers\",\n \"type\": \"headers\",\n \"found_by\": \"Headers (Passive Detection)\",\n \"confidence\": 100,\n \"confirmed_by\": {\n\n },\n \"references\": {\n\n },\n \"interesting_entries\": [\n \"Server: Apache/2.4.38 (Debian)\",\n \"X-Powered-By: PHP/7.1.33\"\n ]\n },\n {\n \"url\": \"http://127.0.0.1:8888/xmlrpc.php\",\n \"to_s\": \"XML-RPC seems to be enabled: http://127.0.0.1:8888/xmlrpc.php\",\n \"type\": \"xmlrpc\",\n \"found_by\": \"Direct Access (Aggressive Detection)\",\n \"confidence\": 100,\n \"confirmed_by\": {\n\n },\n \"references\": {\n \"url\": [\n \"http://codex.wordpress.org/XML-RPC_Pingback_API\"\n ],\n \"metasploit\": [\n \"auxiliary/scanner/http/wordpress_ghost_scanner\",\n \"auxiliary/dos/http/wordpress_xmlrpc_dos\",\n \"auxiliary/scanner/http/wordpress_xmlrpc_login\",\n \"auxiliary/scanner/http/wordpress_pingback_access\"\n ]\n },\n \"interesting_entries\": [\n\n ]\n },\n {\n \"url\": \"http://127.0.0.1:8888/readme.html\",\n \"to_s\": \"WordPress readme found: http://127.0.0.1:8888/readme.html\",\n \"type\": \"readme\",\n \"found_by\": \"Direct Access (Aggressive Detection)\",\n \"confidence\": 100,\n \"confirmed_by\": {\n\n },\n \"references\": {\n\n },\n \"interesting_entries\": [\n \"/wp-admin/\",\n \"/wp-admin/admin-ajax.php\",\n \" \"\n ]\n },\n {\n \"url\": \"http://127.0.0.1:8888/wp-cron.php\",\n \"to_s\": \"The external WP-Cron seems to be enabled: http://127.0.0.1:8888/wp-cron.php\",\n \"type\": \"wp_cron\",\n \"found_by\": \"Direct Access (Aggressive Detection)\",\n \"confidence\": 60,\n \"confirmed_by\": {\n\n },\n \"references\": {\n \"url\": [\n \"https://www.iplocation.net/defend-wordpress-from-ddos\",\n \"https://github.com/wpscanteam/wpscan/issues/1299\"\n ]\n },\n \"interesting_entries\": [\n\n ]\n }\n ],\n \"version\": {\n \"number\": \"5.3\",\n \"release_date\": \"2019-11-12\",\n \"status\": \"insecure\",\n \"found_by\": \"Emoji Settings (Passive Detection)\",\n \"confidence\": 100,\n \"interesting_entries\": [\n \"http://127.0.0.1:8888/, Match: 'wp-includes\\\\/js\\\\/wp-emoji-release.min.js?ver=5.3'\"\n ],\n \"confirmed_by\": {\n \"Meta Generator (Passive Detection)\": {\n \"confidence\": 60,\n \"interesting_entries\": [\n \"http://127.0.0.1:8888/, Match: 'WordPress 5.3'\"\n ]\n }\n },\n \"vulnerabilities\": [\n {\n \"title\": \"WordPress <= 5.3 - Authenticated Improper Access Controls in REST API\",\n \"fixed_in\": \"5.3.1\",\n \"references\": {\n \"cve\": [\n \"2019-20043\",\n \"2019-16788\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2019/12/wordpress-5-3-1-security-and-maintenance-release/\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-g7rg-hchx-c2gw\"\n ],\n \"wpvulndb\": [\n \"4a6de154-5fbd-4c80-acd3-8902ee431bd8\"\n ]\n }\n },\n {\n \"title\": \"WordPress <= 5.3 - Authenticated Stored XSS via Crafted Links\",\n \"fixed_in\": \"5.3.1\",\n \"references\": {\n \"cve\": [\n \"2019-20042\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2019/12/wordpress-5-3-1-security-and-maintenance-release/\",\n \"https://hackerone.com/reports/509930\",\n \"https://github.com/WordPress/wordpress-develop/commit/1f7f3f1f59567e2504f0fbebd51ccf004b3ccb1d\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-xvg2-m2f4-83m7\"\n ],\n \"wpvulndb\": [\n \"23553517-34e3-40a9-a406-f3ffbe9dd265\"\n ]\n }\n },\n {\n \"title\": \"WordPress <= 5.3 - Authenticated Stored XSS via Block Editor Content\",\n \"fixed_in\": \"5.3.1\",\n \"references\": {\n \"cve\": [\n \"2019-16781\",\n \"2019-16780\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2019/12/wordpress-5-3-1-security-and-maintenance-release/\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-pg4x-64rh-3c9v\"\n ],\n \"wpvulndb\": [\n \"be794159-4486-4ae1-a5cc-5c190e5ddf5f\"\n ]\n }\n },\n {\n \"title\": \"WordPress <= 5.3 - wp_kses_bad_protocol() Colon Bypass\",\n \"fixed_in\": \"5.3.1\",\n \"references\": {\n \"cve\": [\n \"2019-20041\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2019/12/wordpress-5-3-1-security-and-maintenance-release/\",\n \"https://github.com/WordPress/wordpress-develop/commit/b1975463dd995da19bb40d3fa0786498717e3c53\"\n ],\n \"wpvulndb\": [\n \"8fac612b-95d2-477a-a7d6-e5ec0bb9ca52\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.4.1 - Password Reset Tokens Failed to Be Properly Invalidated\",\n \"fixed_in\": \"5.3.3\",\n \"references\": {\n \"cve\": [\n \"2020-11027\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2020/04/wordpress-5-4-1/\",\n \"https://core.trac.wordpress.org/changeset/47634/\",\n \"https://www.wordfence.com/blog/2020/04/unpacking-the-7-vulnerabilities-fixed-in-todays-wordpress-5-4-1-security-update/\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-ww7v-jg8c-q6jw\"\n ],\n \"wpvulndb\": [\n \"7db191c0-d112-4f08-a419-a1cd81928c4e\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.4.1 - Unauthenticated Users View Private Posts\",\n \"fixed_in\": \"5.3.3\",\n \"references\": {\n \"cve\": [\n \"2020-11028\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2020/04/wordpress-5-4-1/\",\n \"https://core.trac.wordpress.org/changeset/47635/\",\n \"https://www.wordfence.com/blog/2020/04/unpacking-the-7-vulnerabilities-fixed-in-todays-wordpress-5-4-1-security-update/\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-xhx9-759f-6p2w\"\n ],\n \"wpvulndb\": [\n \"d1e1ba25-98c9-4ae7-8027-9632fb825a56\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.4.1 - Authenticated Cross-Site Scripting (XSS) in Customizer\",\n \"fixed_in\": \"5.3.3\",\n \"references\": {\n \"cve\": [\n \"2020-11025\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2020/04/wordpress-5-4-1/\",\n \"https://core.trac.wordpress.org/changeset/47633/\",\n \"https://www.wordfence.com/blog/2020/04/unpacking-the-7-vulnerabilities-fixed-in-todays-wordpress-5-4-1-security-update/\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-4mhg-j6fx-5g3c\"\n ],\n \"wpvulndb\": [\n \"4eee26bd-a27e-4509-a3a5-8019dd48e429\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.4.1 - Authenticated Cross-Site Scripting (XSS) in Search Block\",\n \"fixed_in\": \"5.3.3\",\n \"references\": {\n \"cve\": [\n \"2020-11030\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2020/04/wordpress-5-4-1/\",\n \"https://core.trac.wordpress.org/changeset/47636/\",\n \"https://www.wordfence.com/blog/2020/04/unpacking-the-7-vulnerabilities-fixed-in-todays-wordpress-5-4-1-security-update/\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-vccm-6gmc-qhjh\"\n ],\n \"wpvulndb\": [\n \"e4bda91b-067d-45e4-a8be-672ccf8b1a06\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.4.1 - Cross-Site Scripting (XSS) in wp-object-cache\",\n \"fixed_in\": \"5.3.3\",\n \"references\": {\n \"cve\": [\n \"2020-11029\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2020/04/wordpress-5-4-1/\",\n \"https://core.trac.wordpress.org/changeset/47637/\",\n \"https://www.wordfence.com/blog/2020/04/unpacking-the-7-vulnerabilities-fixed-in-todays-wordpress-5-4-1-security-update/\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-568w-8m88-8g2c\"\n ],\n \"wpvulndb\": [\n \"e721d8b9-a38f-44ac-8520-b4a9ed6a5157\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.4.1 - Authenticated Cross-Site Scripting (XSS) in File Uploads\",\n \"fixed_in\": \"5.3.3\",\n \"references\": {\n \"cve\": [\n \"2020-11026\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2020/04/wordpress-5-4-1/\",\n \"https://core.trac.wordpress.org/changeset/47638/\",\n \"https://www.wordfence.com/blog/2020/04/unpacking-the-7-vulnerabilities-fixed-in-todays-wordpress-5-4-1-security-update/\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-3gw2-4656-pfr2\",\n \"https://hackerone.com/reports/179695\"\n ],\n \"wpvulndb\": [\n \"55438b63-5fc9-4812-afc4-2f1eff800d5f\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.4.2 - Authenticated XSS in Block Editor\",\n \"fixed_in\": \"5.3.4\",\n \"references\": {\n \"cve\": [\n \"2020-4046\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2020/06/wordpress-5-4-2-security-and-maintenance-release/\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-rpwf-hrh2-39jf\",\n \"https://pentest.co.uk/labs/research/subtle-stored-xss-wordpress-core/\"\n ],\n \"youtube\": [\n \"https://www.youtube.com/watch?v=tCh7Y8z8fb4\"\n ],\n \"wpvulndb\": [\n \"831e4a94-239c-4061-b66e-f5ca0dbb84fa\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.4.2 - Authenticated XSS via Media Files\",\n \"fixed_in\": \"5.3.4\",\n \"references\": {\n \"cve\": [\n \"2020-4047\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2020/06/wordpress-5-4-2-security-and-maintenance-release/\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-8q2w-5m27-wm27\"\n ],\n \"wpvulndb\": [\n \"741d07d1-2476-430a-b82f-e1228a9343a4\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.4.2 - Open Redirection\",\n \"fixed_in\": \"5.3.4\",\n \"references\": {\n \"cve\": [\n \"2020-4048\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2020/06/wordpress-5-4-2-security-and-maintenance-release/\",\n \"https://github.com/WordPress/WordPress/commit/10e2a50c523cf0b9785555a688d7d36a40fbeccf\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-q6pw-gvf4-5fj5\"\n ],\n \"wpvulndb\": [\n \"12855f02-432e-4484-af09-7d0fbf596909\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.4.2 - Authenticated Stored XSS via Theme Upload\",\n \"fixed_in\": \"5.3.4\",\n \"references\": {\n \"cve\": [\n \"2020-4049\"\n ],\n \"exploitdb\": [\n \"48770\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2020/06/wordpress-5-4-2-security-and-maintenance-release/\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-87h4-phjv-rm6p\",\n \"https://hackerone.com/reports/406289\"\n ],\n \"wpvulndb\": [\n \"d8addb42-e70b-4439-b828-fd0697e5d9d4\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.4.2 - Misuse of set-screen-option Leading to Privilege Escalation\",\n \"fixed_in\": \"5.3.4\",\n \"references\": {\n \"cve\": [\n \"2020-4050\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2020/06/wordpress-5-4-2-security-and-maintenance-release/\",\n \"https://github.com/WordPress/WordPress/commit/dda0ccdd18f6532481406cabede19ae2ed1f575d\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-4vpv-fgg2-gcqc\"\n ],\n \"wpvulndb\": [\n \"b6f69ff1-4c11-48d2-b512-c65168988c45\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.4.2 - Disclosure of Password-Protected Page/Post Comments\",\n \"fixed_in\": \"5.3.4\",\n \"references\": {\n \"cve\": [\n \"2020-25286\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2020/06/wordpress-5-4-2-security-and-maintenance-release/\",\n \"https://github.com/WordPress/WordPress/commit/c075eec24f2f3214ab0d0fb0120a23082e6b1122\"\n ],\n \"wpvulndb\": [\n \"eea6dbf5-e298-44a7-9b0d-f078ad4741f9\"\n ]\n }\n },\n {\n \"title\": \"WordPress 4.7-5.7 - Authenticated Password Protected Pages Exposure\",\n \"fixed_in\": \"5.3.7\",\n \"references\": {\n \"cve\": [\n \"2021-29450\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2021/04/wordpress-5-7-1-security-and-maintenance-release/\",\n \"https://blog.wpscan.com/2021/04/15/wordpress-571-security-vulnerability-release.html\",\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-pmmh-2f36-wvhq\",\n \"https://core.trac.wordpress.org/changeset/50717/\"\n ],\n \"youtube\": [\n \"https://www.youtube.com/watch?v=J2GXmxAdNWs\"\n ],\n \"wpvulndb\": [\n \"6a3ec618-c79e-4b9c-9020-86b157458ac5\"\n ]\n }\n },\n {\n \"title\": \"WordPress 3.7 to 5.7.1 - Object Injection in PHPMailer\",\n \"fixed_in\": \"5.3.8\",\n \"references\": {\n \"cve\": [\n \"2020-36326\",\n \"2018-19296\"\n ],\n \"url\": [\n \"https://github.com/WordPress/WordPress/commit/267061c9595fedd321582d14c21ec9e7da2dcf62\",\n \"https://wordpress.org/news/2021/05/wordpress-5-7-2-security-release/\",\n \"https://github.com/PHPMailer/PHPMailer/commit/e2e07a355ee8ff36aba21d0242c5950c56e4c6f9\",\n \"https://www.wordfence.com/blog/2021/05/wordpress-5-7-2-security-release-what-you-need-to-know/\"\n ],\n \"youtube\": [\n \"https://www.youtube.com/watch?v=HaW15aMzBUM\"\n ],\n \"wpvulndb\": [\n \"4cd46653-4470-40ff-8aac-318bee2f998d\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.8.2 - Expired DST Root CA X3 Certificate\",\n \"fixed_in\": \"5.3.10\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2021/11/wordpress-5-8-2-security-and-maintenance-release/\",\n \"https://core.trac.wordpress.org/ticket/54207\"\n ],\n \"wpvulndb\": [\n \"cc23344a-5c91-414a-91e3-c46db614da8d\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.8 - Plugin Confusion\",\n \"fixed_in\": \"5.8\",\n \"references\": {\n \"cve\": [\n \"2021-44223\"\n ],\n \"url\": [\n \"https://vavkamil.cz/2021/11/25/wordpress-plugin-confusion-update-can-get-you-pwned/\"\n ],\n \"wpvulndb\": [\n \"95e01006-84e4-4e95-b5d7-68ea7b5aa1a8\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.8.3 - SQL Injection via WP_Query\",\n \"fixed_in\": \"5.3.11\",\n \"references\": {\n \"cve\": [\n \"2022-21661\"\n ],\n \"url\": [\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-6676-cqfm-gw84\",\n \"https://hackerone.com/reports/1378209\"\n ],\n \"wpvulndb\": [\n \"7f768bcf-ed33-4b22-b432-d1e7f95c1317\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.8.3 - Author+ Stored XSS via Post Slugs\",\n \"fixed_in\": \"5.3.11\",\n \"references\": {\n \"cve\": [\n \"2022-21662\"\n ],\n \"url\": [\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-699q-3hj9-889w\",\n \"https://hackerone.com/reports/425342\",\n \"https://blog.sonarsource.com/wordpress-stored-xss-vulnerability\"\n ],\n \"wpvulndb\": [\n \"dc6f04c2-7bf2-4a07-92b5-dd197e4d94c8\"\n ]\n }\n },\n {\n \"title\": \"WordPress 4.1-5.8.2 - SQL Injection via WP_Meta_Query\",\n \"fixed_in\": \"5.3.11\",\n \"references\": {\n \"cve\": [\n \"2022-21664\"\n ],\n \"url\": [\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-jp3p-gw8h-6x86\"\n ],\n \"wpvulndb\": [\n \"24462ac4-7959-4575-97aa-a6dcceeae722\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.8.3 - Super Admin Object Injection in Multisites\",\n \"fixed_in\": \"5.3.11\",\n \"references\": {\n \"cve\": [\n \"2022-21663\"\n ],\n \"url\": [\n \"https://github.com/WordPress/wordpress-develop/security/advisories/GHSA-jmmq-m8p8-332h\",\n \"https://hackerone.com/reports/541469\"\n ],\n \"wpvulndb\": [\n \"008c21ab-3d7e-4d97-b6c3-db9d83f390a7\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 5.9.2 - Prototype Pollution in jQuery\",\n \"fixed_in\": \"5.3.12\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/03/wordpress-5-9-2-security-maintenance-release/\"\n ],\n \"wpvulndb\": [\n \"1ac912c1-5e29-41ac-8f76-a062de254c09\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.2 - Reflected Cross-Site Scripting\",\n \"fixed_in\": \"5.3.13\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/08/wordpress-6-0-2-security-and-maintenance-release/\"\n ],\n \"wpvulndb\": [\n \"622893b0-c2c4-4ee7-9fa1-4cecef6e36be\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.2 - Authenticated Stored Cross-Site Scripting\",\n \"fixed_in\": \"5.3.13\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/08/wordpress-6-0-2-security-and-maintenance-release/\"\n ],\n \"wpvulndb\": [\n \"3b1573d4-06b4-442b-bad5-872753118ee0\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.2 - SQLi via Link API\",\n \"fixed_in\": \"5.3.13\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/08/wordpress-6-0-2-security-and-maintenance-release/\"\n ],\n \"wpvulndb\": [\n \"601b0bf9-fed2-4675-aec7-fed3156a022f\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.3 - Stored XSS via wp-mail.php\",\n \"fixed_in\": \"5.3.14\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/10/wordpress-6-0-3-security-release/\",\n \"https://github.com/WordPress/wordpress-develop/commit/abf236fdaf94455e7bc6e30980cf70401003e283\"\n ],\n \"wpvulndb\": [\n \"713bdc8b-ab7c-46d7-9847-305344a579c4\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.3 - Open Redirect via wp_nonce_ays\",\n \"fixed_in\": \"5.3.14\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/10/wordpress-6-0-3-security-release/\",\n \"https://github.com/WordPress/wordpress-develop/commit/506eee125953deb658307bb3005417cb83f32095\"\n ],\n \"wpvulndb\": [\n \"926cd097-b36f-4d26-9c51-0dfab11c301b\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.3 - Email Address Disclosure via wp-mail.php\",\n \"fixed_in\": \"5.3.14\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/10/wordpress-6-0-3-security-release/\",\n \"https://github.com/WordPress/wordpress-develop/commit/5fcdee1b4d72f1150b7b762ef5fb39ab288c8d44\"\n ],\n \"wpvulndb\": [\n \"c5675b59-4b1d-4f64-9876-068e05145431\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.3 - Reflected XSS via SQLi in Media Library\",\n \"fixed_in\": \"5.3.14\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/10/wordpress-6-0-3-security-release/\",\n \"https://github.com/WordPress/wordpress-develop/commit/8836d4682264e8030067e07f2f953a0f66cb76cc\"\n ],\n \"wpvulndb\": [\n \"cfd8b50d-16aa-4319-9c2d-b227365c2156\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.3 - CSRF in wp-trackback.php\",\n \"fixed_in\": \"5.3.14\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/10/wordpress-6-0-3-security-release/\",\n \"https://github.com/WordPress/wordpress-develop/commit/a4f9ca17fae0b7d97ff807a3c234cf219810fae0\"\n ],\n \"wpvulndb\": [\n \"b60a6557-ae78-465c-95bc-a78cf74a6dd0\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.3 - Stored XSS via the Customizer\",\n \"fixed_in\": \"5.3.14\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/10/wordpress-6-0-3-security-release/\",\n \"https://github.com/WordPress/wordpress-develop/commit/2ca28e49fc489a9bb3c9c9c0d8907a033fe056ef\"\n ],\n \"wpvulndb\": [\n \"2787684c-aaef-4171-95b4-ee5048c74218\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.3 - Stored XSS via Comment Editing\",\n \"fixed_in\": \"5.3.14\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/10/wordpress-6-0-3-security-release/\",\n \"https://github.com/WordPress/wordpress-develop/commit/89c8f7919460c31c0f259453b4ffb63fde9fa955\"\n ],\n \"wpvulndb\": [\n \"02d76d8e-9558-41a5-bdb6-3957dc31563b\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.3 - Content from Multipart Emails Leaked\",\n \"fixed_in\": \"5.3.14\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/10/wordpress-6-0-3-security-release/\",\n \"https://github.com/WordPress/wordpress-develop/commit/3765886b4903b319764490d4ad5905bc5c310ef8\"\n ],\n \"wpvulndb\": [\n \"3f707e05-25f0-4566-88ed-d8d0aff3a872\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.3 - SQLi in WP_Date_Query\",\n \"fixed_in\": \"5.3.14\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/10/wordpress-6-0-3-security-release/\",\n \"https://github.com/WordPress/wordpress-develop/commit/d815d2e8b2a7c2be6694b49276ba3eee5166c21f\"\n ],\n \"wpvulndb\": [\n \"1da03338-557f-4cb6-9a65-3379df4cce47\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.3 - Stored XSS via RSS Widget\",\n \"fixed_in\": \"5.3.14\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/10/wordpress-6-0-3-security-release/\",\n \"https://github.com/WordPress/wordpress-develop/commit/929cf3cb9580636f1ae3fe944b8faf8cca420492\"\n ],\n \"wpvulndb\": [\n \"58d131f5-f376-4679-b604-2b888de71c5b\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.3 - Data Exposure via REST Terms/Tags Endpoint\",\n \"fixed_in\": \"5.3.14\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/10/wordpress-6-0-3-security-release/\",\n \"https://github.com/WordPress/wordpress-develop/commit/ebaac57a9ac0174485c65de3d32ea56de2330d8e\"\n ],\n \"wpvulndb\": [\n \"b27a8711-a0c0-4996-bd6a-01734702913e\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.0.3 - Multiple Stored XSS via Gutenberg\",\n \"fixed_in\": \"5.3.14\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2022/10/wordpress-6-0-3-security-release/\",\n \"https://github.com/WordPress/gutenberg/pull/45045/files\"\n ],\n \"wpvulndb\": [\n \"f513c8f6-2e1c-45ae-8a58-36b6518e2aa9\"\n ]\n }\n },\n {\n \"title\": \"WP <= 6.2 - Unauthenticated Blind SSRF via DNS Rebinding\",\n \"fixed_in\": null,\n \"references\": {\n \"cve\": [\n \"2022-3590\"\n ],\n \"url\": [\n \"https://blog.sonarsource.com/wordpress-core-unauthenticated-blind-ssrf/\"\n ],\n \"wpvulndb\": [\n \"c8814e6e-78b3-4f63-a1d3-6906a84c1f11\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.2.1 - Directory Traversal via Translation Files\",\n \"fixed_in\": \"5.3.15\",\n \"references\": {\n \"cve\": [\n \"2023-2745\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2023/05/wordpress-6-2-1-maintenance-security-release/\"\n ],\n \"wpvulndb\": [\n \"2999613a-b8c8-4ec0-9164-5dfe63adf6e6\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.2.1 - Thumbnail Image Update via CSRF\",\n \"fixed_in\": \"5.3.15\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2023/05/wordpress-6-2-1-maintenance-security-release/\"\n ],\n \"wpvulndb\": [\n \"a03d744a-9839-4167-a356-3e7da0f1d532\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.2.1 - Contributor+ Stored XSS via Open Embed Auto Discovery\",\n \"fixed_in\": \"5.3.15\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2023/05/wordpress-6-2-1-maintenance-security-release/\"\n ],\n \"wpvulndb\": [\n \"3b574451-2852-4789-bc19-d5cc39948db5\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.2.2 - Shortcode Execution in User Generated Data\",\n \"fixed_in\": \"5.3.15\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2023/05/wordpress-6-2-1-maintenance-security-release/\",\n \"https://wordpress.org/news/2023/05/wordpress-6-2-2-security-release/\"\n ],\n \"wpvulndb\": [\n \"ef289d46-ea83-4fa5-b003-0352c690fd89\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.2.1 - Contributor+ Content Injection\",\n \"fixed_in\": \"5.3.15\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2023/05/wordpress-6-2-1-maintenance-security-release/\"\n ],\n \"wpvulndb\": [\n \"1527ebdb-18bc-4f9d-9c20-8d729a628670\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.3.2 - Denial of Service via Cache Poisoning\",\n \"fixed_in\": \"5.3.16\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2023/10/wordpress-6-3-2-maintenance-and-security-release/\"\n ],\n \"wpvulndb\": [\n \"6d80e09d-34d5-4fda-81cb-e703d0e56e4f\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.3.2 - Subscriber+ Arbitrary Shortcode Execution\",\n \"fixed_in\": \"5.3.16\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2023/10/wordpress-6-3-2-maintenance-and-security-release/\"\n ],\n \"wpvulndb\": [\n \"3615aea0-90aa-4f9a-9792-078a90af7f59\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.3.2 - Contributor+ Comment Disclosure\",\n \"fixed_in\": \"5.3.16\",\n \"references\": {\n \"cve\": [\n \"2023-39999\"\n ],\n \"url\": [\n \"https://wordpress.org/news/2023/10/wordpress-6-3-2-maintenance-and-security-release/\"\n ],\n \"wpvulndb\": [\n \"d35b2a3d-9b41-4b4f-8e87-1b8ccb370b9f\"\n ]\n }\n },\n {\n \"title\": \"WP < 6.3.2 - Unauthenticated Post Author Email Disclosure\",\n \"fixed_in\": \"5.3.16\",\n \"references\": {\n \"cve\": [\n \"2023-5561\"\n ],\n \"url\": [\n \"https://wpscan.com/blog/email-leak-oracle-vulnerability-addressed-in-wordpress-6-3-2/\",\n \"https://wordpress.org/news/2023/10/wordpress-6-3-2-maintenance-and-security-release/\"\n ],\n \"wpvulndb\": [\n \"19380917-4c27-4095-abf1-eba6f913b441\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 6.4.3 - Deserialization of Untrusted Data\",\n \"fixed_in\": \"5.3.17\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2024/01/wordpress-6-4-3-maintenance-and-security-release/\"\n ],\n \"wpvulndb\": [\n \"5e9804e5-bbd4-4836-a5f0-b4388cc39225\"\n ]\n }\n },\n {\n \"title\": \"WordPress < 6.4.3 - Admin+ PHP File Upload\",\n \"fixed_in\": \"5.3.17\",\n \"references\": {\n \"url\": [\n \"https://wordpress.org/news/2024/01/wordpress-6-4-3-maintenance-and-security-release/\"\n ],\n \"wpvulndb\": [\n \"a8e12fbe-c70b-4078-9015-cf57a05bdd4a\"\n ]\n }\n }\n ]\n },\n \"main_theme\": null,\n \"plugins\": {\n \"social-warfare\": {\n \"slug\": \"social-warfare\",\n \"location\": \"http://127.0.0.1:8888/wp-content/plugins/social-warfare/\",\n \"latest_version\": \"4.4.6.3\",\n \"last_updated\": \"2024-04-07T19:32:00.000Z\",\n \"outdated\": true,\n \"readme_url\": null,\n \"directory_listing\": null,\n \"error_log_url\": null,\n \"found_by\": \"Comment (Passive Detection)\",\n \"confidence\": 30,\n \"interesting_entries\": [\n\n ],\n \"confirmed_by\": {\n\n },\n \"vulnerabilities\": [\n {\n \"title\": \"Social Warfare <= 3.5.2 - Unauthenticated Arbitrary Settings Update\",\n \"fixed_in\": \"3.5.3\",\n \"references\": {\n \"cve\": [\n \"2019-9978\"\n ],\n \"url\": [\n \"https://wordpress.org/support/topic/malware-into-new-update/\",\n \"https://www.wordfence.com/blog/2019/03/unpatched-zero-day-vulnerability-in-social-warfare-plugin-exploited-in-the-wild/\",\n \"https://threatpost.com/wordpress-plugin-removed-after-zero-day-discovered/143051/\",\n \"https://twitter.com/warfareplugins/status/1108826025188909057\",\n \"https://www.wordfence.com/blog/2019/03/recent-social-warfare-vulnerability-allowed-remote-code-execution/\"\n ],\n \"wpvulndb\": [\n \"32085d2d-1235-42b4-baeb-bc43172a4972\"\n ]\n }\n },\n {\n \"title\": \"Social Warfare <= 3.5.2 - Unauthenticated Remote Code Execution (RCE)\",\n \"fixed_in\": \"3.5.3\",\n \"references\": {\n \"url\": [\n \"https://www.webarxsecurity.com/social-warfare-vulnerability/\"\n ],\n \"wpvulndb\": [\n \"7b412469-cc03-4899-b397-38580ced5618\"\n ]\n }\n },\n {\n \"title\": \"Social Warfare < 4.3.1 - Subscriber+ Post Meta Deletion\",\n \"fixed_in\": \"4.3.1\",\n \"references\": {\n \"cve\": [\n \"2023-0402\"\n ],\n \"wpvulndb\": [\n \"5116068f-4b84-42ad-a88d-03e46096b41c\"\n ]\n }\n },\n {\n \"title\": \"Social Warfare < 4.4.0 - Post Meta Deletion via CSRF\",\n \"fixed_in\": \"4.4.0\",\n \"references\": {\n \"cve\": [\n \"2023-0403\"\n ],\n \"wpvulndb\": [\n \"7140abf5-5966-4361-bd51-ee29d3071a30\"\n ]\n }\n },\n {\n \"title\": \"Social Sharing Plugin - Social Warfare < 4.4.4 - Authenticated (Contributor+) Stored Cross-Site Scripting via Shortcode\",\n \"fixed_in\": \"4.4.4\",\n \"references\": {\n \"cve\": [\n \"2023-4842\"\n ],\n \"url\": [\n \"https://www.wordfence.com/threat-intel/vulnerabilities/id/8f5b9aff-0833-4887-ae59-df5bc88c7f91\"\n ],\n \"wpvulndb\": [\n \"ab221b58-369e-4010-ae36-be099b2f4c9b\"\n ]\n }\n },\n {\n \"title\": \"Social Sharing Plugin – Social Warfare < 4.4.6.2 - Authenticated(Contributor+) Stored Cross-Site Scripting via Shortcode\",\n \"fixed_in\": \"4.4.6.2\",\n \"references\": {\n \"cve\": [\n \"2024-1959\"\n ],\n \"url\": [\n \"https://www.wordfence.com/threat-intel/vulnerabilities/id/1016f16c-0ab2-4cac-a7a5-8d93a37e7894\"\n ],\n \"wpvulndb\": [\n \"26ad138e-990a-4401-84e4-ea694ccf6e7f\"\n ]\n }\n },\n {\n \"title\": \"Social Sharing Plugin – Social Warfare < 4.4.6 - Cross-Site Request Forgery\",\n \"fixed_in\": \"4.4.6\",\n \"references\": {\n \"cve\": [\n \"2024-34825\"\n ],\n \"url\": [\n \"https://www.wordfence.com/threat-intel/vulnerabilities/id/f105bee6-21b2-4014-bb0a-9e53c49e29b0\"\n ],\n \"wpvulndb\": [\n \"acb8b33c-6b74-4d65-a3a5-5cad0c1ea8b0\"\n ]\n }\n }\n ],\n \"version\": {\n \"number\": \"3.5.2\",\n \"confidence\": 100,\n \"found_by\": \"Comment (Passive Detection)\",\n \"interesting_entries\": [\n \"http://127.0.0.1:8888/, Match: 'Social Warfare v3.5.2'\"\n ],\n \"confirmed_by\": {\n \"Readme - Stable Tag (Aggressive Detection)\": {\n \"confidence\": 80,\n \"interesting_entries\": [\n \"http://127.0.0.1:8888/wp-content/plugins/social-warfare/readme.txt\"\n ]\n },\n \"Readme - ChangeLog Section (Aggressive Detection)\": {\n \"confidence\": 50,\n \"interesting_entries\": [\n \"http://127.0.0.1:8888/wp-content/plugins/social-warfare/readme.txt\"\n ]\n }\n }\n }\n }\n },\n \"config_backups\": {\n\n },\n \"vuln_api\": {\n \"plan\": \"free\",\n \"requests_done_during_scan\": 0,\n \"requests_remaining\": 15\n },\n \"stop_time\": 1717183322,\n \"elapsed\": 3,\n \"requests_done\": 169,\n \"cached_requests\": 6,\n \"data_sent\": 59178,\n \"data_sent_humanised\": \"57.791 KB\",\n \"data_received\": 313184,\n \"data_received_humanised\": \"305.844 KB\",\n \"used_memory\": 225398784,\n \"used_memory_humanised\": \"214.957 MB\"\n}\"\"\"\n\n async def setup_after_prep(self, module_test):\n async def wpscan_mock_run(*command, **kwargs):\n return CompletedProcess(command, 0, self.wpscan_output_json, \"\")\n\n module_test.monkeypatch.setattr(module_test.scan.helpers, \"run\", wpscan_mock_run)\n\n def check(self, module_test, events):\n findings = [e for e in events if e.type == \"FINDING\"]\n vulnerabilities = [e for e in events if e.type == \"VULNERABILITY\"]\n technologies = [e for e in events if e.type == \"TECHNOLOGY\"]\n assert len(findings) == 1\n assert len(vulnerabilities) == 59\n assert len(technologies) == 4\n" }, { "path": "bbot/test/test_step_2/template_tests/__init__.py", "content": "" }, { "path": "bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py", "content": "from ..module_tests.base import ModuleTestBase\n\n\nclass TestSubdomainEnum(ModuleTestBase):\n targets = [\"blacklanternsecurity.com\"]\n modules_overrides = []\n config_overrides = {\"dns\": {\"minimal\": False}, \"scope\": {\"report_distance\": 10}}\n dedup_strategy = \"highest_parent\"\n\n txt = [\n \"www.blacklanternsecurity.com\",\n \"asdf.www.blacklanternsecurity.com\",\n \"test.asdf.www.blacklanternsecurity.com\",\n \"api.test.asdf.www.blacklanternsecurity.com\",\n ]\n\n async def setup_after_prep(self, module_test):\n dns_mock = {\n \"evilcorp.com\": {\"A\": [\"127.0.0.6\"]},\n \"blacklanternsecurity.com\": {\"A\": [\"127.0.0.5\"]},\n \"www.blacklanternsecurity.com\": {\"A\": [\"127.0.0.5\"]},\n \"asdf.www.blacklanternsecurity.com\": {\"A\": [\"127.0.0.5\"]},\n \"test.asdf.www.blacklanternsecurity.com\": {\"A\": [\"127.0.0.5\"]},\n \"api.test.asdf.www.blacklanternsecurity.com\": {\"A\": [\"127.0.0.5\"]},\n }\n if self.txt:\n dns_mock[\"blacklanternsecurity.com\"][\"TXT\"] = self.txt\n await module_test.mock_dns(dns_mock)\n\n # load subdomain enum template as module\n from bbot.modules.templates.subdomain_enum import subdomain_enum\n\n subdomain_enum_module = subdomain_enum(module_test.scan)\n\n self.queries = []\n\n async def mock_query(query):\n self.queries.append(query)\n\n subdomain_enum_module.query = mock_query\n subdomain_enum_module.dedup_strategy = self.dedup_strategy\n module_test.scan.modules[\"subdomain_enum\"] = subdomain_enum_module\n\n def check(self, module_test, events):\n in_scope_dns_names = [e for e in events if e.type == \"DNS_NAME\" and e.scope_distance == 0]\n assert len(in_scope_dns_names) == 5\n assert 1 == len([e for e in in_scope_dns_names if e.data == \"blacklanternsecurity.com\"])\n assert 1 == len([e for e in in_scope_dns_names if e.data == \"www.blacklanternsecurity.com\"])\n assert 1 == len([e for e in in_scope_dns_names if e.data == \"asdf.www.blacklanternsecurity.com\"])\n assert 1 == len([e for e in in_scope_dns_names if e.data == \"test.asdf.www.blacklanternsecurity.com\"])\n assert 1 == len([e for e in in_scope_dns_names if e.data == \"api.test.asdf.www.blacklanternsecurity.com\"])\n assert len(self.queries) == 1\n assert self.queries[0] == \"blacklanternsecurity.com\"\n\n\nclass TestSubdomainEnumHighestParent(TestSubdomainEnum):\n targets = [\"api.test.asdf.www.blacklanternsecurity.com\", \"evilcorp.com\"]\n whitelist = [\"www.blacklanternsecurity.com\"]\n modules_overrides = [\"speculate\"]\n dedup_strategy = \"highest_parent\"\n txt = None\n\n def check(self, module_test, events):\n in_scope_dns_names = [e for e in events if e.type == \"DNS_NAME\" and e.scope_distance == 0]\n distance_1_dns_names = [e for e in events if e.type == \"DNS_NAME\" and e.scope_distance == 1]\n assert len(in_scope_dns_names) == 4\n assert 1 == len([e for e in in_scope_dns_names if e.data == \"www.blacklanternsecurity.com\"])\n assert 1 == len([e for e in in_scope_dns_names if e.data == \"asdf.www.blacklanternsecurity.com\"])\n assert 1 == len([e for e in in_scope_dns_names if e.data == \"test.asdf.www.blacklanternsecurity.com\"])\n assert 1 == len([e for e in in_scope_dns_names if e.data == \"api.test.asdf.www.blacklanternsecurity.com\"])\n assert len(distance_1_dns_names) == 2\n assert 1 == len([e for e in distance_1_dns_names if e.data == \"evilcorp.com\"])\n assert 1 == len([e for e in distance_1_dns_names if e.data == \"blacklanternsecurity.com\"])\n assert len(self.queries) == 1\n assert self.queries[0] == \"www.blacklanternsecurity.com\"\n\n\nclass TestSubdomainEnumLowestParent(TestSubdomainEnumHighestParent):\n dedup_strategy = \"lowest_parent\"\n\n def check(self, module_test, events):\n assert set(self.queries) == {\n \"test.asdf.www.blacklanternsecurity.com\",\n \"asdf.www.blacklanternsecurity.com\",\n \"www.blacklanternsecurity.com\",\n }\n\n\nclass TestSubdomainEnumWildcardBaseline(ModuleTestBase):\n # oh walmart.cn why are you like this\n targets = [\"www.walmart.cn\"]\n whitelist = [\"walmart.cn\"]\n modules_overrides = []\n config_overrides = {\"dns\": {\"minimal\": False}, \"scope\": {\"report_distance\": 10}, \"omit_event_types\": []}\n dedup_strategy = \"highest_parent\"\n\n dns_mock_data = {\n \"walmart.cn\": {\"A\": [\"127.0.0.1\"]},\n \"www.walmart.cn\": {\"A\": [\"127.0.0.1\"]},\n \"test.walmart.cn\": {\"A\": [\"127.0.0.1\"]},\n }\n\n async def setup_before_prep(self, module_test):\n await module_test.mock_dns(self.dns_mock_data)\n self.queries = []\n\n async def mock_query(query):\n self.queries.append(query)\n return [\"walmart.cn\", \"www.walmart.cn\", \"test.walmart.cn\", \"asdf.walmart.cn\"]\n\n # load subdomain enum template as module\n from bbot.modules.templates.subdomain_enum import subdomain_enum\n\n subdomain_enum_module = subdomain_enum(module_test.scan)\n\n subdomain_enum_module.query = mock_query\n subdomain_enum_module._name = \"subdomain_enum\"\n subdomain_enum_module.dedup_strategy = self.dedup_strategy\n module_test.scan.modules[\"subdomain_enum\"] = subdomain_enum_module\n\n def check(self, module_test, events):\n assert self.queries == [\"walmart.cn\"]\n assert len(events) == 7\n assert 2 == len(\n [\n e\n for e in events\n if e.type == \"IP_ADDRESS\" and e.data == \"127.0.0.1\" and str(e.module) == \"A\" and e.scope_distance == 1\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"www.walmart.cn\"\n and str(e.module) == \"TARGET\"\n and e.scope_distance == 0\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"test.walmart.cn\"\n and str(e.module) == \"subdomain_enum\"\n and e.scope_distance == 0\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME_UNRESOLVED\"\n and e.data == \"asdf.walmart.cn\"\n and str(e.module) == \"subdomain_enum\"\n and e.scope_distance == 0\n ]\n )\n\n\nclass TestSubdomainEnumWildcardDefense(TestSubdomainEnumWildcardBaseline):\n # oh walmart.cn why are you like this\n targets = [\"walmart.cn\"]\n modules_overrides = []\n config_overrides = {\"dns\": {\"minimal\": False}, \"scope\": {\"report_distance\": 10}}\n dedup_strategy = \"highest_parent\"\n\n dns_mock_data = {\n \"walmart.cn\": {\"A\": [\"127.0.0.2\"], \"TXT\": [\"asdf.walmart.cn\"]},\n }\n\n async def setup_after_prep(self, module_test):\n # simulate wildcard\n custom_lookup = \"\"\"\ndef custom_lookup(query, rdtype):\n import random\n if rdtype == \"A\" and query.endswith(\".walmart.cn\"):\n ip = \".\".join([str(random.randint(0,256)) for _ in range(4)])\n return {ip}\n\"\"\"\n await module_test.mock_dns(self.dns_mock_data, custom_lookup_fn=custom_lookup)\n\n def check(self, module_test, events):\n # no subdomain enum should happen on this domain!\n assert self.queries == []\n assert len(events) == 7\n assert 2 == len(\n [e for e in events if e.type == \"IP_ADDRESS\" and str(e.module) == \"A\" and e.scope_distance == 1]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"walmart.cn\"\n and str(e.module) == \"TARGET\"\n and e.scope_distance == 0\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"DNS_NAME\"\n and e.data == \"asdf.walmart.cn\"\n and str(e.module) == \"TXT\"\n and e.scope_distance == 0\n and \"wildcard-possible\" in e.tags\n and \"a-wildcard-possible\" in e.tags\n ]\n )\n assert 1 == len(\n [\n e\n for e in events\n if e.type == \"RAW_DNS_RECORD\"\n and e.data == {\"host\": \"walmart.cn\", \"type\": \"TXT\", \"answer\": '\"asdf.walmart.cn\"'}\n ]\n )\n" }, { "path": "bbot/test/testsslcert.pem", "content": "-----BEGIN CERTIFICATE-----\nMIIDADCCAeigAwIBAgIUJnHoP2WYqS692n3bHQxkGlYlX1MwDQYJKoZIhvcNAQEL\nBQAwFzEVMBMGA1UEAwwMdGVzdC5ub3RyZWFsMCAXDTIzMTAxMzE3NTM0NFoYDzIw\nNTEwMjI3MTc1MzQ0WjAXMRUwEwYDVQQDDAx0ZXN0Lm5vdHJlYWwwggEiMA0GCSqG\nSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYDFf5yrTe23FF2zv2dQxQs+VdwxF7lCS/\nF6Tycuh/7+4aDLG9+3IQMeqFE7VlnaQb/M2QHsjMCeFlHUnd1jxXbmt+dWQ5Pxtz\nA8Vi0ypDDM6flHoT/f4CTVdDd1sc99ExBHApDAvRi6yyEnu0DxaZqzNTIRP2ijQq\neHDTO4Hx+K/K/NvSCF05FnASS5EnOCx745lURtETatdAwa7HZADZ8NDgG9Dj8fa/\nuRq3FclBbbbmq9LWKTw3cAEXTz8+5N9F2/xSGk7NZpvIv5u15gtfbMfZcVADLSVe\nHR6NCfzgd/ZiHAx8CJf/ZStlMYksxZDSkb7wpdm9KeWNUpTjVknhAgMBAAGjQjBA\nMB8GA1UdEQQYMBaCFHd3dy5iYm90dGVzdC5ub3RyZWFsMB0GA1UdDgQWBBQC20kP\nJq3PPZoWef0lV+c/ckbocjANBgkqhkiG9w0BAQsFAAOCAQEAzTLHR72bt2Bxc0bF\naUQtumrX1rtuO3Cb2AiqKLPgb3nwnP5q+RZq991U1vMUFTiXUjplh86/Bh5IRJ8X\n1HUnMwTo6Co/77Ezx3Na2L62ajg2TpLo5YDOkIgMlOI63cGuk0ahelyxcsFVYdgA\n2/Jrh/xsybdKA5l1VG5jxzZ3s9d0Gd1wXpNe+bpwFR7gby52TkibPPviZ/CKF7NB\n7UdVj+SREXuSWH5NIicNQ71MJNE4CNNCOwy+yVoGY2E7WzqZNE+KZW5K5Sxp4Pnb\nZ9ZnCPq5m0RL7wBd+BhB2WxLVuvt0XdVS3H21cGuD/NR7r4OAsUNrf1nUwARNKPu\nBgZQhw==\n-----END CERTIFICATE-----\n" }, { "path": "bbot/test/testsslkey.pem", "content": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDYDFf5yrTe23FF\n2zv2dQxQs+VdwxF7lCS/F6Tycuh/7+4aDLG9+3IQMeqFE7VlnaQb/M2QHsjMCeFl\nHUnd1jxXbmt+dWQ5PxtzA8Vi0ypDDM6flHoT/f4CTVdDd1sc99ExBHApDAvRi6yy\nEnu0DxaZqzNTIRP2ijQqeHDTO4Hx+K/K/NvSCF05FnASS5EnOCx745lURtETatdA\nwa7HZADZ8NDgG9Dj8fa/uRq3FclBbbbmq9LWKTw3cAEXTz8+5N9F2/xSGk7NZpvI\nv5u15gtfbMfZcVADLSVeHR6NCfzgd/ZiHAx8CJf/ZStlMYksxZDSkb7wpdm9KeWN\nUpTjVknhAgMBAAECggEAKsJYqB7LKN9YHhoLllXoo9FS+DlrDKEPm8V3dyewZd/L\n6VpxVDc/hj6G2qNBr9ShHgvs+FTra1yaQDupeq8Tvr8jJcJgnWbkzSDmME64StBu\nVY2akrnei8CYYIkvHn7ap3+oHiuc7DJfcdfwJT0mPTAxxoZhr9X/CJfRRrE8oPG4\n6w9WNS0CuyoDZ++xYwbWkNsF4XXtoOfkVgyXgtZDlIEAyRLvzVymDE05JjkmLRWt\nmmk4dmxJrYh/vd0DNAK3w1qmV3iaACs+1KG/TSNKeTrDipl+WyZE5KpJe/wPiOSV\nKG5hm4pXHRkN250k/5xUWWv+zQEC+fLd/JJdBev6JQKBgQDs8a28ltKstxYyTX2j\nW2L/C+jdQi1COCN7u3M0rFKw+vxFoTnvlCj1X9FLQ9lbgZr96vE2cpSMmskkmDDb\nKVneR6xBNLdqa/S0of/Ax9ZXwxR4k5EPUYMh3yfxuISbQyGe7qqLCeTbNZldqSYO\nigGpQ0nhxFwEJ7d9VzZrBTC7JQKBgQDpbHN8jlxzBQIox56wB+CS2ISj6c0dsIOI\n76J0nEdJ4qoDK1k185xaUvUyn43LmQOF2UHfwKOwVz09YwX+vFRkVNFKs7KCtUqg\ne0Z7C9oiSzeqKwxUope4yKz8MYtRFgUhAwrBa0WyRRLAyfRJQ6mPzie0GdVWt4pM\ntZ889lvVDQKBgQCCNo70BS7iG/vmyQ8ypxZQc4sVjTiyG4fkh69YUxteh4/79A6S\nyyl3L6Ela7QXxbIXuPW2pmFco/PGWJ0A1Ei/D0Rq0T27Dnj8i8qxdyEkOeEWIoKl\nmHYoNysMfArkCJCBd0fiAR30GhCemEaB1vXyvzfrCq5G2kzMZRFS3xdYwQKBgFl0\nsp2dgVijJryyI+KaYjpkuBCJXY5vQzmLfNrruXZbY4RrbHj8r4L+H/ISq6jHL05w\ngIpbrV+7T0DjXjzNuBnrV3ole9gT2lG+bLhjRmm2IdMZRFR7K2IppgHQiu+8XKLW\nI50Um1VCm3k+7FvXjngKLbUb4WKmXF4hjLE0SOVRAoGAZIXumUjy26y1dLL0E9F2\nHC5YEQfokVylQCWV+Ws5yjAZnAij8i0DWPHf2zvLAJ2BbmeMQAuYj5bUN1AUIFpG\n/ve/yLM0635dKgoH1Zlk83iQMrjXAuXkKc4gwfocPUnGFJ/LRXufodIKI2SP7Nff\niVrq/w6VxVfc7EK0a7bfzxo=\n-----END PRIVATE KEY-----\n" }, { "path": "bbot/wordlists/devops_mutations.txt", "content": "www\nsub\ncom\ntest\ndev\nen\napi\nde\nm\ne\nstaging\nmetric\na\ncsg\ncdn\nhost\nweb\nnet\nus\nthe\njp\nblog\neu\nprod\nqa\nssl\nmail\nonline\ndemo\nfr\napp\ninit\ns\nfna\nbr\nsecure\ncareers\nppls\nit\np\nstatic\nru\nhealth\nfree\nsn\nfamily\nstage\nadmin\nd\ninfo\nmvm\nt\nam\nc\nin\nlive\ni\nmobile\nuat\nstg\nca\nnew\nri\naccount\nof\nsupport\npc\nshop\ndr\nservice\nsasg\nwest\nes\nat\nsrv\nmedia\nh\nportal\nar\nmy\nid\nsas\nr\nhss\nvideo\norg\nserver\nlogin\nla\ncms\nbeta\nupdate\nvpn\nservices\nint\nb\nassets\npro\neast\nuk\ncloud\nip\nmx\nstore\nhome\nproduction\nauto\ncn\n" }, { "path": "bbot/wordlists/ms_on_prem_subdomains.txt", "content": "adfs\nadfs01\nadfs02\nadfs1\nadfs2\nadfs3\nadfsproxy\nadfstest\nauth\nfed\nfederate\nfederated\nfederation\nfederationfs\nfs\nfs1\nfs2\nfs3\nfs4\ngateway\nlogin\nportal\nsaml\nsso\nsts\nwap\nwebmail\nowa\nhybrid\nhybrid-cloud\nemail\noutlook\nexchange\nmail2\nwebmail2\nmail1\nmailbox\nmail01\nmailman\nmailgate\nmailbackup\nmail3\nwebmail1\nwebmail3\nmailing\nmailserver\nmailhost\nmailer\nmailadmin\nimap\npop3\npost\npost1\npost2\nmail\nremote\ndesktop\ndesktop1\ndesktop2\ndesktops\nextranet\nmydesktop\nra\nrdesktop\nrdgate\nrdp\nrdpweb\nrds\nrdsh\nrdweb\nremote01\nremote02\nremote1\nremote2\nremote3\nremote4\nremoteapp\nremoteapps\nremotedesktop\nremotegateway\ntsweb\nvdesktop\nvdi\ndialin\nmeet\nlync\nlyncweb\nsip\nskype\nsfbweb\nscheduler\nlyncext\nlyncdiscoverinternal\naccess\nlyncaccess01\nlyncaccess\nlync10\nwac\n_sipinternaltls\nuc\nlyncdiscover\n" }, { "path": "bbot/wordlists/nameservers.txt", "content": "# validated DNS servers pulled from public-dns.info on 2022/09/01\n198.153.194.50\n172.64.37.44\n69.67.97.18\n184.155.36.194\n45.225.123.238\n103.3.252.5\n212.187.140.53\n103.47.134.195\n204.199.98.173\n8.14.62.67\n103.121.228.5\n8.28.109.125\n45.90.28.13\n45.90.28.10\n64.158.240.81\n172.64.37.9\n4.15.141.202\n155.254.21.250\n172.64.37.76\n190.217.113.18\n195.186.4.110\n128.127.104.108\n50.204.42.225\n82.113.224.113\n195.74.68.2\n190.216.19.27\n190.216.125.220\n8.243.126.14\n217.160.70.42\n45.90.30.21\n162.159.36.125\n206.169.117.104\n209.12.133.148\n45.90.28.15\n103.196.38.38\n185.108.141.114\n172.64.37.131\n190.216.251.19\n146.190.6.13\n110.145.125.13\n162.159.36.132\n172.64.37.127\n173.163.85.121\n216.254.141.2\n77.88.8.3\n172.64.36.47\n211.115.194.5\n68.87.72.134\n172.64.37.168\n164.124.107.9\n190.216.250.222\n1.0.202.161\n103.85.107.99\n193.42.159.2\n74.203.74.105\n192.76.144.66\n68.105.46.149\n190.216.69.13\n172.64.46.9\n195.186.1.109\n12.204.162.62\n109.248.149.133\n172.64.46.22\n8.35.35.35\n4.4.53.164\n202.43.108.1\n172.64.36.104\n110.35.78.66\n162.159.36.46\n190.216.69.9\n209.234.212.26\n64.212.76.178\n194.225.73.141\n162.159.56.128\n194.98.65.165\n1.0.170.31\n162.159.51.23\n73.7.178.166\n204.199.106.78\n162.159.50.61\n4.7.75.194\n172.64.37.38\n172.64.36.235\n8.243.96.154\n12.71.143.33\n45.225.123.207\n172.64.36.79\n195.186.4.111\n173.244.51.54\n194.102.42.3\n172.64.37.75\n162.159.46.51\n66.162.169.190\n45.90.28.26\n8.243.96.156\n172.64.37.21\n8.28.109.70\n1.0.215.118\n172.64.36.182\n8.243.126.27\n113.161.116.150\n184.177.84.201\n64.132.21.189\n172.64.47.174\n172.64.36.44\n5.164.26.4\n116.118.119.167\n172.64.46.229\n209.200.100.151\n172.64.37.98\n73.128.218.47\n172.64.37.190\n8.243.104.162\n190.216.253.172\n193.230.161.3\n162.159.46.18\n103.160.248.44\n172.64.36.80\n200.221.11.101\n9.9.9.11\n162.159.56.43\n8.243.126.28\n109.228.22.126\n122.129.122.99\n222.255.167.61\n172.64.37.1\n64.105.199.74\n146.70.31.43\n162.159.36.181\n67.28.70.130\n67.73.141.62\n203.113.135.28\n88.198.92.222\n24.99.149.148\n172.64.36.164\n172.64.36.78\n101.102.103.104\n198.153.194.40\n149.112.122.20\n172.64.37.112\n174.69.40.212\n172.64.36.151\n63.209.155.118\n8.41.17.84\n45.90.30.19\n216.202.247.10\n1.0.209.99\n45.11.45.11\n162.159.50.152\n50.217.25.225\n1.0.169.175\n172.64.37.149\n66.193.38.100\n8.28.109.82\n162.159.56.84\n162.159.51.117\n172.64.37.182\n203.54.212.126\n156.154.70.16\n172.64.37.184\n172.64.37.234\n162.159.56.253\n204.199.122.5\n172.64.37.45\n216.229.0.25\n172.64.46.217\n45.125.208.8\n203.38.225.3\n172.64.37.20\n200.16.208.187\n8.28.109.252\n80.67.188.188\n172.64.37.83\n103.86.96.100\n172.64.36.45\n12.165.204.94\n1.0.133.90\n77.88.8.2\n200.41.78.209\n1.0.138.176\n190.216.67.53\n172.64.36.200\n1.0.170.113\n45.225.123.249\n201.184.230.34\n209.12.244.162\n75.103.115.94\n172.64.37.69\n190.216.203.224\n91.144.22.198\n1.0.170.39\n172.64.36.16\n174.48.45.128\n193.135.143.23\n159.69.114.157\n204.199.130.91\n4.2.167.65\n45.225.123.214\n71.58.100.49\n116.193.64.22\n172.64.36.96\n50.217.25.200\n204.199.157.70\n24.125.55.22\n109.224.233.174\n172.64.37.185\n198.153.192.40\n172.64.36.215\n172.64.36.166\n204.70.127.127\n67.187.17.182\n4.15.208.86\n45.225.123.239\n156.154.71.1\n195.186.1.111\n70.171.58.112\n172.64.37.253\n200.87.100.10\n41.225.236.101\n8.28.109.109\n4.4.26.135\n8.25.184.252\n172.64.37.245\n146.70.82.3\n172.64.36.152\n45.90.28.28\n159.203.187.29\n172.64.37.107\n172.64.36.30\n172.64.37.130\n8.20.247.7\n45.19.183.181\n172.64.36.223\n45.90.30.27\n162.159.57.6\n84.236.142.130\n103.23.150.89\n4.79.123.69\n172.64.36.203\n172.64.36.226\n98.232.103.71\n98.179.205.194\n149.112.112.10\n172.64.37.207\n195.77.235.10\n66.93.87.2\n172.64.36.155\n117.103.228.101\n97.65.124.6\n212.72.130.20\n92.255.164.166\n50.200.245.136\n1.0.209.242\n8.18.4.19\n172.64.36.9\n65.91.52.25\n50.223.22.178\n165.87.201.244\n172.64.36.251\n8.33.239.234\n172.64.36.187\n12.231.169.28\n8.26.56.11\n172.64.36.121\n172.64.37.218\n193.135.143.21\n98.38.222.66\n8.20.45.6\n96.102.121.126\n172.64.37.77\n190.0.15.18\n46.147.195.82\n8.26.56.17\n172.64.36.2\n185.42.192.114\n45.225.123.178\n4.14.199.129\n99.99.99.193\n66.192.104.68\n210.23.129.34\n172.64.46.192\n23.226.134.242\n172.64.37.106\n162.159.46.166\n172.64.36.176\n162.159.56.66\n172.64.36.28\n113.161.182.253\n98.39.154.157\n162.159.50.85\n8.243.126.2\n172.64.37.251\n194.7.1.4\n77.235.219.211\n216.36.31.135\n8.28.109.110\n172.64.37.225\n209.164.189.56\n162.159.56.228\n119.17.138.116\n172.64.46.111\n8.9.163.237\n162.159.57.183\n103.239.32.81\n193.135.143.35\n204.194.234.200\n158.43.128.72\n201.234.44.129\n24.56.77.138\n172.64.36.146\n4.28.150.154\n204.199.116.45\n8.46.206.93\n172.64.36.83\n188.225.225.25\n8.242.215.91\n172.64.37.163\n211.115.194.2\n112.197.12.40\n172.64.47.91\n69.44.110.204\n37.120.232.43\n172.64.37.179\n190.93.189.30\n12.127.17.72\n12.121.118.9\n172.64.36.159\n77.88.8.88\n172.64.36.27\n162.159.46.23\n4.1.67.166\n172.64.36.22\n76.104.155.196\n50.234.132.241\n195.186.1.107\n213.211.50.2\n210.220.163.82\n216.194.28.33\n162.159.36.6\n204.152.204.100\n216.106.1.254\n172.64.37.2\n168.95.192.1\n172.64.47.200\n162.159.36.227\n75.103.95.14\n190.216.237.18\n8.242.172.200\n8.36.139.1\n118.68.218.173\n8.243.220.194\n205.151.222.251\n172.64.36.53\n8.243.126.118\n14.225.232.19\n122.2.65.202\n172.64.37.144\n190.216.69.0\n4.59.232.194\n45.90.30.29\n172.64.37.28\n164.163.1.90\n172.64.37.51\n8.26.56.16\n8.243.126.18\n162.159.46.177\n172.64.36.212\n172.64.36.64\n8.29.3.37\n142.103.1.1\n95.80.104.128\n8.20.247.3\n190.217.110.10\n209.200.100.150\n172.64.37.70\n91.121.157.83\n72.237.206.37\n8.20.247.16\n8.242.49.142\n50.221.57.204\n94.28.20.249\n204.199.194.28\n208.67.220.2\n50.217.25.205\n162.159.36.104\n8.243.126.135\n64.157.242.118\n172.64.36.239\n172.64.36.229\n172.64.37.154\n172.64.36.14\n172.64.36.249\n172.64.37.192\n210.87.253.60\n8.242.214.61\n208.67.222.220\n8.30.101.114\n195.168.91.238\n193.78.240.12\n195.76.233.2\n172.64.37.204\n156.154.70.11\n190.217.25.34\n162.159.46.120\n172.64.37.169\n202.248.20.133\n162.159.36.139\n81.163.3.1\n4.79.244.118\n172.64.36.15\n162.159.56.16\n172.64.46.29\n217.138.220.243\n14.225.24.83\n172.64.36.29\n210.87.253.35\n172.64.36.218\n208.91.112.220\n221.163.74.11\n1.0.215.158\n125.234.104.230\n216.175.203.51\n172.64.37.177\n107.0.74.232\n95.158.129.2\n181.224.160.11\n83.143.8.249\n162.159.51.205\n172.64.46.159\n172.64.36.169\n172.64.46.28\n172.64.37.139\n144.91.64.224\n50.235.228.46\n8.243.126.105\n172.64.37.71\n8.30.83.132\n109.195.187.172\n172.64.37.196\n172.64.36.177\n165.22.241.78\n1.0.0.2\n172.64.37.99\n208.91.112.52\n172.64.47.216\n8.28.109.115\n8.29.3.132\n172.64.36.186\n172.64.46.34\n173.184.62.167\n8.30.101.125\n109.228.24.15\n8.20.247.17\n199.44.194.2\n172.64.37.43\n8.29.2.132\n5.11.11.5\n172.64.36.248\n4.15.23.203\n195.129.111.49\n64.76.25.120\n209.234.212.184\n8.243.126.112\n12.97.174.103\n216.54.240.147\n70.171.60.6\n1.0.221.165\n207.138.37.4\n85.21.144.55\n190.216.19.16\n8.26.21.127\n151.80.145.143\n204.199.97.162\n172.64.37.49\n190.216.247.150\n23.19.67.116\n64.64.110.3\n216.84.166.166\n8.29.103.224\n172.64.37.3\n67.73.188.138\n172.64.37.114\n8.242.215.226\n203.54.152.226\n172.64.37.92\n162.159.46.1\n172.64.36.253\n82.197.214.133\n193.135.143.13\n167.250.65.246\n94.28.26.138\n8.243.126.71\n190.216.65.166\n172.64.37.133\n162.159.57.19\n45.90.28.189\n172.64.36.69\n1.0.218.23\n172.64.36.85\n212.187.166.54\n62.149.132.2\n37.120.207.131\n98.180.23.77\n195.46.39.39\n8.242.215.202\n172.64.36.179\n162.159.46.28\n198.82.247.34\n172.64.37.121\n114.130.5.6\n162.159.57.78\n8.38.89.46\n149.156.12.250\n202.136.162.12\n172.64.36.99\n45.90.28.17\n45.90.28.23\n162.159.57.1\n8.242.173.2\n213.149.113.211\n172.64.37.59\n172.64.36.94\n45.90.30.10\n162.159.46.90\n12.97.174.104\n4.34.133.226\n45.90.28.22\n189.125.136.8\n108.175.22.60\n172.64.36.26\n96.102.76.175\n24.99.149.127\n209.51.161.14\n172.64.36.135\n172.64.47.195\n36.37.160.242\n209.136.31.102\n37.120.152.235\n109.194.17.191\n162.159.51.239\n172.64.36.174\n162.159.51.224\n172.64.36.76\n50.231.115.22\n216.244.192.3\n221.139.13.130\n172.64.36.67\n204.199.128.123\n208.51.60.81\n193.47.83.251\n172.64.36.12\n1.0.156.34\n45.90.28.193\n201.234.130.31\n172.64.36.72\n172.64.36.213\n172.64.36.18\n1.0.160.109\n203.21.196.20\n8.29.3.133\n139.134.2.190\n172.64.37.6\n45.90.30.17\n172.64.36.114\n192.71.166.92\n9.9.9.9\n172.64.37.187\n50.223.23.54\n199.227.106.122\n45.90.28.21\n73.31.121.3\n212.73.198.88\n67.99.197.123\n4.14.162.237\n190.216.19.21\n1.0.214.3\n172.64.37.25\n172.64.37.102\n172.64.36.175\n8.17.30.61\n172.64.37.46\n45.117.80.200\n172.64.37.157\n63.232.89.67\n220.239.164.49\n49.156.53.166\n189.126.192.4\n190.216.64.230\n4.15.7.161\n72.207.237.152\n172.64.36.170\n8.9.113.35\n216.146.35.35\n189.125.19.198\n212.187.156.31\n208.72.160.67\n89.163.221.181\n172.64.37.195\n190.216.241.71\n8.14.62.70\n72.237.212.20\n67.99.200.1\n172.64.37.242\n176.212.194.184\n85.214.91.66\n14.238.93.131\n209.244.104.184\n8.23.82.186\n4.79.140.163\n172.64.37.215\n194.69.194.3\n4.7.194.66\n45.90.30.28\n67.73.245.181\n209.244.104.187\n194.2.0.50\n63.209.154.102\n24.4.172.85\n222.255.167.73\n98.34.183.199\n172.64.36.74\n172.64.37.53\n209.0.191.6\n172.64.36.202\n162.159.46.202\n75.103.115.95\n172.64.46.72\n8.35.114.228\n172.64.36.52\n190.216.229.111\n8.243.120.54\n204.199.116.210\n113.161.86.104\n190.2.210.115\n64.120.5.251\n94.141.24.92\n8.20.247.4\n172.64.47.204\n172.64.37.123\n172.64.37.37\n24.250.147.79\n172.64.37.68\n172.64.46.103\n181.224.160.10\n172.64.37.136\n103.150.209.246\n8.28.113.202\n103.31.228.150\n8.28.109.247\n172.64.37.78\n176.103.130.130\n8.242.178.122\n162.159.57.251\n1.0.205.75\n172.64.37.105\n204.199.102.115\n162.159.57.139\n172.64.37.226\n24.116.92.101\n8.25.184.107\n8.242.48.97\n45.225.123.233\n1.0.170.69\n23.19.245.84\n50.216.25.75\n190.217.14.65\n45.90.28.169\n81.3.27.54\n172.64.36.6\n27.71.233.116\n66.192.104.191\n45.90.30.14\n8.29.3.211\n172.64.37.103\n51.15.69.236\n8.28.109.13\n205.214.45.10\n195.208.5.1\n162.159.57.36\n165.158.1.2\n216.84.166.42\n51.158.105.245\n172.64.36.158\n5.1.66.255\n50.238.53.122\n91.225.226.39\n209.244.104.189\n172.64.37.143\n172.64.46.198\n162.159.57.204\n201.234.130.26\n162.159.57.85\n64.129.104.46\n172.64.36.241\n172.64.36.66\n172.64.47.104\n58.186.80.18\n172.64.36.54\n45.225.123.101\n212.230.255.1\n8.243.113.190\n172.64.37.221\n1.0.150.32\n172.64.36.61\n8.0.7.0\n206.169.200.135\n4.49.73.138\n193.135.143.33\n172.64.36.144\n1.0.194.216\n172.64.37.148\n181.224.163.11\n210.181.1.24\n8.243.126.133\n168.205.99.11\n78.47.243.3\n172.64.37.62\n80.78.132.79\n162.159.51.17\n199.77.135.211\n172.64.37.115\n1.0.169.119\n190.216.69.8\n182.52.51.181\n172.64.37.167\n67.100.88.27\n180.211.158.90\n27.76.137.77\n103.196.16.2\n200.32.110.90\n8.30.101.115\n8.34.34.11\n201.132.162.254\n208.50.252.1\n77.37.232.237\n96.53.102.66\n204.199.99.99\n189.125.96.247\n172.64.46.177\n64.76.25.125\n109.228.18.5\n161.200.96.9\n198.181.254.34\n1.0.130.68\n98.38.222.6\n178.161.150.190\n8.28.109.101\n190.216.69.14\n165.16.22.130\n91.205.230.224\n172.64.47.171\n172.64.37.12\n172.64.47.9\n212.12.28.126\n172.64.36.86\n8.34.34.101\n204.199.172.132\n63.209.154.99\n209.247.118.9\n204.199.121.162\n68.87.72.130\n172.64.36.130\n193.240.108.125\n204.199.73.4\n213.202.216.236\n195.129.12.114\n190.217.10.230\n8.243.126.129\n172.64.36.62\n172.64.37.104\n81.201.58.99\n166.102.165.32\n1.0.225.244\n8.28.109.58\n134.75.122.2\n172.64.36.198\n172.64.36.247\n67.30.143.54\n121.254.134.99\n8.242.159.66\n194.187.251.67\n72.207.238.183\n45.90.30.16\n45.225.123.199\n8.242.213.37\n4.1.226.201\n172.64.37.249\n172.64.37.243\n8.21.123.101\n64.76.25.127\n172.64.37.201\n63.211.67.251\n213.55.96.166\n113.53.29.228\n162.159.57.114\n174.64.35.164\n172.64.37.173\n172.64.37.84\n172.64.47.147\n1.0.203.107\n8.242.187.227\n1.0.166.233\n216.55.100.220\n50.58.111.74\n193.135.143.39\n51.15.78.17\n8.33.239.149\n54.37.242.17\n12.127.16.67\n162.159.24.69\n1.0.233.221\n66.193.240.4\n45.90.30.25\n209.163.152.186\n1.1.136.105\n172.64.46.161\n8.29.3.226\n172.64.36.98\n172.64.36.58\n202.136.162.11\n8.242.205.35\n172.64.36.59\n162.159.50.157\n172.64.37.219\n172.64.37.52\n8.28.109.246\n8.28.109.253\n204.199.129.38\n172.64.37.101\n172.64.36.128\n62.176.12.111\n172.64.37.124\n162.159.56.255\n45.90.28.19\n8.242.184.54\n1.0.154.199\n185.233.106.232\n172.64.36.122\n193.227.50.3\n172.64.37.181\n8.35.35.10\n204.199.98.172\n200.76.5.147\n172.64.37.22\n172.64.37.31\n185.74.5.1\n172.64.47.170\n172.64.36.55\n172.64.36.42\n208.48.253.142\n89.107.129.15\n172.64.37.246\n50.59.195.149\n45.225.123.234\n172.64.37.145\n172.64.37.189\n202.138.120.86\n38.242.202.141\n172.64.37.134\n51.15.88.152\n172.64.36.77\n213.202.216.12\n172.64.37.210\n203.129.31.67\n63.208.141.14\n172.64.37.24\n45.90.28.14\n98.244.8.17\n198.153.192.50\n70.191.189.96\n172.64.37.198\n4.1.67.145\n172.64.37.146\n8.29.2.130\n172.64.36.65\n162.159.50.124\n67.166.30.234\n222.255.206.232\n8.243.126.117\n201.238.224.203\n8.242.184.52\n207.17.190.5\n202.29.218.138\n172.64.37.214\n46.245.253.5\n198.54.117.11\n172.64.37.40\n162.159.56.149\n162.159.50.83\n64.42.181.227\n172.64.37.110\n24.136.58.6\n1.0.138.245\n172.64.37.176\n172.64.36.199\n8.243.126.66\n45.225.123.206\n88.208.209.92\n195.158.0.3\n162.159.46.144\n172.64.37.162\n172.64.36.25\n1.0.170.44\n24.99.148.85\n199.77.206.122\n172.64.36.219\n94.140.14.141\n172.64.46.213\n64.105.199.76\n172.64.36.68\n64.76.25.117\n1.0.170.83\n45.90.28.12\n149.112.122.10\n1.0.169.69\n172.64.47.12\n190.217.80.59\n185.183.106.83\n45.90.28.18\n162.159.56.75\n50.59.58.165\n41.207.186.166\n1.232.188.2\n194.25.0.60\n172.64.36.41\n172.64.46.36\n41.65.236.37\n8.242.26.195\n172.64.37.238\n202.78.97.41\n45.90.30.30\n172.64.37.188\n37.120.211.91\n8.34.34.34\n98.208.56.152\n172.64.47.93\n212.72.130.21\n162.159.56.111\n103.136.202.93\n62.113.113.34\n213.157.50.130\n45.90.30.226\n24.98.20.141\n64.105.97.90\n76.76.10.0\n67.97.247.52\n207.138.37.173\n172.64.37.33\n194.7.15.70\n80.83.162.11\n195.10.195.195\n4.79.241.45\n172.64.36.123\n204.199.165.43\n8.42.68.81\n172.64.36.246\n1.0.248.135\n172.64.36.191\n70.171.58.83\n66.163.0.161\n103.239.32.36\n1.0.224.105\n172.64.37.150\n203.119.8.106\n198.153.192.1\n118.69.174.70\n24.104.140.229\n172.64.46.137\n172.64.36.33\n202.53.95.14\n46.224.1.42\n162.159.56.173\n68.1.40.86\n190.216.68.130\n8.243.126.30\n190.216.19.18\n172.64.37.108\n195.99.66.220\n172.64.37.85\n110.78.164.234\n176.9.1.117\n202.87.214.253\n203.38.225.13\n208.51.24.44\n8.242.6.242\n172.64.36.129\n172.64.36.254\n116.193.64.16\n203.38.225.45\n172.64.36.216\n162.159.56.1\n203.198.7.66\n73.206.234.153\n172.64.36.71\n8.242.172.206\n204.57.109.171\n190.216.69.5\n197.155.92.20\n162.159.57.109\n1.0.213.67\n172.64.37.194\n172.64.37.155\n123.30.108.151\n189.125.94.220\n40.70.57.226\n201.234.130.25\n190.216.69.12\n172.64.37.27\n162.159.56.132\n101.255.118.1\n73.137.96.252\n162.159.57.56\n172.64.36.225\n172.64.37.7\n8.29.87.186\n185.228.168.9\n45.90.30.15\n1.1.249.206\n195.153.19.5\n8.33.239.235\n1.0.240.7\n187.157.46.210\n172.64.37.90\n130.225.244.166\n172.64.46.203\n172.64.37.80\n209.87.64.70\n190.217.65.139\n1.0.248.182\n42.116.255.180\n8.30.97.161\n141.1.1.1\n45.90.30.23\n216.254.95.2\n50.58.155.130\n200.32.111.203\n162.159.50.109\n8.35.35.101\n172.64.37.128\n172.64.36.73\n190.216.125.81\n190.216.19.25\n50.219.55.167\n98.38.222.125\n216.21.128.22\n162.159.51.143\n162.159.46.48\n76.76.2.0\n193.202.121.50\n103.197.251.202\n200.41.102.254\n172.64.37.119\n37.209.219.30\n206.57.41.222\n162.159.50.116\n172.64.46.84\n162.159.51.100\n8.243.96.157\n8.19.132.50\n193.194.79.194\n8.29.2.42\n162.159.50.115\n80.248.48.14\n8.14.63.91\n193.135.143.15\n63.209.100.10\n64.192.52.197\n45.90.30.20\n103.139.14.2\n172.64.37.203\n8.243.126.109\n172.64.37.117\n165.87.13.129\n190.217.8.254\n190.216.69.6\n162.159.57.12\n1.0.163.94\n162.159.57.49\n5.152.215.29\n24.98.20.247\n8.243.126.9\n195.27.1.1\n76.120.201.96\n172.64.37.125\n8.29.3.67\n200.0.194.78\n172.64.36.110\n162.159.57.193\n8.243.217.24\n172.64.36.147\n8.43.56.38\n172.64.37.183\n8.36.160.49\n172.64.37.216\n178.136.2.208\n77.88.8.8\n193.240.207.200\n162.159.27.90\n1.0.152.224\n172.64.36.148\n8.243.126.11\n172.64.37.57\n8.26.56.26\n187.1.57.206\n45.90.30.18\n172.64.37.61\n64.192.69.159\n172.64.37.170\n172.64.37.88\n193.238.77.62\n92.60.50.40\n172.64.37.63\n194.177.199.1\n205.171.202.166\n8.28.109.122\n201.234.130.28\n172.64.36.209\n172.64.36.49\n118.69.109.45\n162.159.56.217\n172.64.36.92\n1.0.169.189\n172.64.37.32\n1.0.218.50\n1.0.170.33\n45.90.28.126\n162.159.46.218\n172.64.37.72\n210.4.2.61\n1.0.202.19\n103.196.16.3\n172.64.36.196\n162.159.56.156\n45.5.92.94\n172.64.36.167\n200.195.132.210\n54.37.138.118\n200.41.77.85\n172.64.37.4\n176.103.130.137\n208.67.222.2\n177.135.239.132\n172.64.37.109\n172.64.37.193\n103.147.187.246\n78.129.140.65\n110.142.40.60\n45.90.28.250\n162.159.46.47\n1.0.215.208\n172.64.46.176\n172.64.36.116\n8.242.48.100\n172.64.37.48\n172.64.37.197\n202.134.52.105\n45.90.28.1\n162.159.36.61\n86.106.74.219\n8.20.247.20\n172.64.36.211\n98.247.49.130\n1.0.203.136\n175.213.132.85\n204.152.204.10\n8.243.126.25\n193.135.143.3\n172.64.36.233\n189.125.148.1\n80.254.79.157\n94.140.14.140\n193.135.143.1\n194.108.42.253\n185.23.66.172\n185.184.222.222\n35.155.221.215\n190.90.86.81\n1.0.149.164\n172.64.36.160\n209.244.104.180\n177.54.145.131\n64.210.41.182\n8.243.126.19\n114.114.115.115\n103.106.112.18\n66.163.0.173\n190.216.69.2\n172.64.37.0\n8.9.160.30\n202.6.96.3\n8.243.126.108\n172.64.36.8\n172.64.46.45\n1.0.215.126\n172.64.37.55\n162.159.36.224\n213.211.50.1\n172.64.37.23\n211.25.11.15\n172.64.37.142\n45.191.130.26\n162.159.50.3\n172.64.37.247\n197.155.92.21\n118.69.197.57\n162.159.56.158\n172.64.36.193\n82.64.83.14\n149.112.122.30\n172.64.37.244\n50.59.195.147\n72.236.151.44\n172.64.36.190\n162.159.50.95\n8.8.8.8\n211.115.194.4\n172.64.37.222\n64.76.25.124\n210.87.250.59\n190.216.28.216\n162.159.46.53\n172.64.37.122\n8.28.109.84\n185.93.180.131\n204.117.214.10\n190.216.67.49\n65.220.42.38\n8.29.2.134\n172.64.37.229\n4.79.241.44\n172.64.46.42\n172.64.36.142\n68.183.235.124\n172.64.37.26\n200.169.88.1\n198.7.58.227\n89.163.140.67\n172.64.37.100\n113.161.116.121\n8.27.77.55\n172.64.36.183\n195.12.48.171\n172.64.47.227\n209.136.8.87\n64.157.63.18\n78.31.67.99\n172.64.47.158\n91.219.215.227\n172.64.36.181\n172.64.46.253\n85.9.129.38\n24.125.54.118\n172.64.36.208\n219.250.36.130\n172.64.37.205\n94.247.43.254\n190.216.204.79\n202.6.96.4\n85.114.138.119\n203.113.130.221\n190.217.63.34\n1.0.170.50\n113.161.230.19\n196.3.132.153\n208.91.112.53\n174.47.212.70\n5.164.28.186\n1.0.173.252\n4.53.133.131\n185.199.98.236\n211.115.194.3\n185.237.204.130\n212.89.130.180\n201.234.236.126\n198.71.117.66\n8.26.56.6\n213.176.123.5\n158.43.240.4\n184.187.143.241\n193.19.103.4\n1.0.168.238\n64.76.25.115\n162.159.50.144\n172.64.36.125\n193.135.143.9\n75.103.55.4\n125.235.11.66\n162.159.50.31\n192.133.129.2\n193.37.255.227\n203.133.1.8\n174.46.204.211\n209.244.104.188\n162.159.51.80\n8.28.109.114\n209.212.110.0\n178.212.65.61\n198.245.51.147\n8.35.35.103\n186.215.128.142\n172.64.37.10\n172.64.36.171\n200.41.50.6\n185.233.107.4\n172.64.36.139\n67.97.247.50\n172.64.36.48\n172.64.36.56\n76.76.10.1\n172.64.37.165\n8.243.126.130\n4.7.43.98\n198.60.22.2\n50.201.138.220\n162.159.57.229\n45.90.30.24\n64.192.91.38\n8.26.56.3\n8.243.126.69\n172.64.36.35\n193.67.79.39\n172.64.36.39\n180.182.54.14\n172.64.37.172\n172.64.37.137\n204.199.66.75\n203.162.125.129\n24.229.250.113\n139.130.4.4\n203.2.193.68\n91.192.196.226\n8.243.126.7\n51.89.88.77\n50.59.195.146\n172.64.36.161\n172.64.47.168\n8.29.3.174\n65.56.156.242\n213.230.90.106\n45.90.28.30\n4.34.37.186\n172.64.36.255\n203.119.36.106\n1.4.214.148\n89.218.58.122\n172.64.36.17\n172.64.37.208\n69.44.110.203\n8.20.247.5\n218.102.23.228\n209.58.147.36\n172.64.37.239\n45.90.30.0\n162.159.51.229\n69.44.4.100\n172.64.37.199\n172.64.46.202\n172.64.36.133\n4.7.175.6\n50.59.232.9\n172.64.36.111\n45.225.123.177\n125.234.254.184\n50.200.192.113\n1.0.216.155\n63.210.61.30\n1.0.202.216\n172.64.37.54\n198.153.194.1\n8.41.124.193\n190.216.253.143\n137.82.1.1\n172.64.37.16\n192.156.214.175\n172.64.36.236\n1.0.163.29\n201.234.210.179\n217.144.6.6\n189.125.208.155\n203.113.135.26\n212.12.14.122\n175.213.132.56\n172.64.36.163\n58.186.80.17\n8.29.64.122\n172.64.36.126\n24.98.164.7\n46.147.193.104\n203.253.64.1\n8.28.109.251\n173.89.30.48\n172.64.37.66\n4.15.10.86\n4.1.226.202\n63.209.154.98\n109.70.189.51\n172.64.37.73\n2.56.220.2\n1.0.170.234\n8.43.56.34\n66.28.0.61\n185.38.27.139\n149.211.153.51\n172.64.37.15\n195.129.12.83\n45.125.211.11\n212.73.198.95\n162.159.46.119\n199.85.126.20\n8.29.2.133\n173.8.207.139\n8.28.109.102\n76.76.2.4\n73.120.98.178\n172.64.46.109\n1.0.169.235\n103.15.241.241\n63.134.244.54\n116.100.88.123\n88.80.64.8\n172.64.37.64\n217.18.206.22\n94.28.49.131\n73.59.66.235\n103.74.122.91\n193.135.143.11\n206.51.143.55\n64.208.105.1\n209.164.189.54\n204.199.128.122\n206.80.23.5\n173.234.56.115\n184.55.4.145\n95.9.194.13\n24.99.148.175\n88.208.211.65\n205.171.3.66\n162.159.50.18\n210.94.0.7\n172.64.37.180\n203.248.252.2\n190.128.225.58\n37.120.236.11\n172.64.37.152\n172.64.37.232\n193.58.204.59\n45.90.28.16\n162.159.57.237\n45.90.28.24\n200.55.59.101\n172.64.37.11\n46.224.1.43\n8.21.123.119\n45.90.28.25\n201.234.239.155\n172.64.37.86\n173.10.78.68\n64.238.96.12\n162.159.56.64\n8.21.123.98\n200.55.59.102\n193.135.143.31\n193.135.143.7\n69.169.190.211\n50.216.244.79\n222.255.144.115\n204.199.84.229\n172.64.36.34\n45.225.123.161\n141.95.6.51\n172.64.36.118\n172.64.36.70\n197.210.211.1\n5.164.31.60\n1.0.167.125\n172.64.36.154\n37.120.235.187\n172.64.46.62\n172.64.37.224\n12.165.204.88\n8.242.155.74\n103.14.26.190\n193.135.143.19\n8.243.126.134\n8.28.109.124\n1.0.157.7\n207.191.5.59\n95.215.19.53\n189.125.73.13\n8.243.126.107\n176.103.130.136\n195.74.68.3\n172.64.36.136\n8.28.109.59\n172.64.37.42\n24.104.140.255\n172.64.37.60\n103.48.78.156\n12.51.21.245\n172.64.36.180\n172.64.36.143\n200.41.50.4\n172.64.36.238\n8.243.126.119\n14.161.2.38\n162.159.36.114\n199.85.126.10\n45.225.123.174\n208.51.60.215\n8.243.126.122\n189.125.136.9\n70.92.127.134\n76.76.10.2\n8.242.184.55\n8.243.126.10\n178.150.206.87\n210.87.253.61\n172.64.36.149\n172.64.36.150\n1.0.208.233\n180.182.54.12\n82.135.139.155\n50.222.131.40\n52.24.103.199\n8.243.217.66\n45.90.28.0\n172.64.36.221\n192.46.230.27\n180.182.54.11\n1.0.214.41\n31.7.37.37\n1.0.202.55\n162.159.56.223\n91.194.239.122\n45.90.28.20\n4.31.189.137\n193.135.143.29\n79.141.83.250\n209.203.82.54\n190.217.82.126\n172.64.36.252\n186.97.246.178\n97.65.32.114\n172.64.47.103\n172.64.37.160\n162.159.51.196\n200.222.51.208\n4.4.211.46\n80.80.218.218\n167.114.84.132\n81.16.18.228\n1.0.162.250\n84.54.64.35\n94.28.91.14\n50.201.178.63\n80.191.40.41\n50.201.178.29\n162.159.56.118\n50.59.65.117\n172.64.36.19\n1.0.212.99\n23.19.245.88\n64.210.72.157\n8.243.113.189\n162.159.36.243\n172.64.46.144\n172.64.37.132\n190.217.83.202\n200.41.12.163\n172.64.36.192\n172.64.36.210\n172.64.36.119\n8.27.215.70\n50.216.92.142\n190.216.66.178\n8.18.226.210\n172.64.46.179\n172.64.37.89\n80.254.77.39\n190.216.69.4\n172.64.36.107\n4.7.72.198\n162.159.51.90\n45.225.123.165\n64.76.25.119\n45.90.30.129\n204.199.80.58\n4.28.44.2\n172.64.36.250\n217.150.35.129\n73.176.118.83\n24.125.55.144\n118.69.172.151\n81.16.19.65\n213.154.80.203\n194.1.154.37\n210.245.87.16\n45.225.123.164\n73.69.70.235\n67.73.245.210\n172.64.37.81\n172.64.36.230\n50.229.154.179\n76.76.2.5\n162.159.51.140\n201.148.17.110\n190.216.111.247\n76.30.118.33\n4.2.237.6\n1.0.159.177\n190.216.230.18\n45.191.130.123\n95.158.128.2\n172.64.37.178\n156.154.71.25\n222.255.237.207\n172.64.37.126\n206.253.33.130\n189.125.108.13\n1.0.169.183\n8.243.126.21\n172.64.37.220\n204.199.86.147\n103.137.156.3\n172.64.37.164\n67.191.149.125\n8.28.109.42\n67.98.222.236\n8.26.56.12\n1.0.162.166\n172.107.199.19\n201.234.232.18\n8.14.63.82\n194.108.42.2\n210.80.58.66\n1.0.216.30\n62.76.62.76\n8.29.3.213\n190.216.73.163\n50.228.251.34\n98.212.26.255\n45.79.139.155\n194.25.0.52\n4.59.81.39\n45.231.223.250\n24.250.147.103\n8.40.240.36\n162.159.36.185\n162.159.50.119\n190.216.69.15\n201.234.159.214\n8.242.187.229\n194.61.59.25\n1.0.216.56\n172.64.46.13\n172.64.37.129\n130.226.161.34\n210.181.4.25\n8.28.109.77\n217.218.127.127\n204.199.194.24\n1.0.214.166\n204.199.85.178\n50.220.226.155\n172.64.36.127\n172.64.36.21\n4.26.50.74\n162.159.57.219\n172.64.36.217\n172.64.37.252\n172.64.36.197\n204.199.130.93\n62.94.0.42\n172.64.37.17\n109.224.233.190\n82.113.248.113\n73.32.84.89\n70.171.50.159\n8.243.126.77\n172.64.47.167\n172.64.37.241\n194.88.93.22\n162.159.36.134\n199.166.6.2\n8.243.126.72\n88.99.39.199\n172.64.47.133\n1.1.1.1\n8.28.109.227\n162.159.50.72\n204.57.114.215\n172.64.46.142\n203.27.182.202\n63.209.154.54\n8.29.3.214\n8.242.187.228\n113.161.230.20\n185.117.118.20\n4.79.132.219\n212.162.6.43\n172.64.46.230\n162.159.50.129\n162.159.46.117\n172.64.36.1\n82.99.242.155\n1.0.247.14\n8.26.56.13\n50.238.53.126\n45.90.28.129\n172.64.37.91\n103.7.172.7\n172.64.46.17\n162.159.46.38\n94.203.253.156\n190.216.229.78\n207.91.5.32\n172.64.36.101\n172.64.37.19\n172.64.36.7\n204.199.121.154\n96.9.69.164\n45.225.123.163\n95.85.95.85\n162.159.36.96\n194.225.62.80\n8.42.68.209\n66.243.10.180\n8.28.109.244\n172.64.46.255\n149.211.153.50\n172.64.37.138\n64.81.79.2\n180.250.252.218\n172.64.46.52\n172.64.37.156\n98.232.103.167\n50.58.112.194\n154.236.189.27\n120.150.56.245\n217.151.251.10\n212.3.255.178\n162.159.36.237\n172.64.36.222\n73.31.207.113\n201.148.17.116\n209.244.104.181\n172.64.36.157\n172.64.36.5\n103.164.113.26\n8.30.101.117\n172.64.36.227\n204.199.194.26\n64.132.102.138\n8.27.177.12\n209.244.104.186\n8.243.126.70\n203.54.189.178\n172.64.36.204\n162.159.57.128\n176.121.9.144\n172.64.37.56\n1.0.170.243\n8.21.123.86\n64.154.28.51\n4.16.3.86\n73.6.85.165\n149.112.121.30\n52.3.100.184\n216.248.55.141\n208.51.24.53\n1.0.138.14\n190.216.229.245\n81.27.162.100\n204.194.232.200\n4.14.161.87\n89.233.43.71\n172.64.37.217\n216.136.12.98\n51.68.190.250\n101.53.12.102\n172.64.36.206\n209.136.132.139\n69.75.150.3\n8.20.247.9\n65.18.114.254\n200.41.50.5\n8.18.4.20\n110.145.182.154\n8.34.34.103\n45.191.130.18\n172.64.36.91\n172.64.36.224\n172.64.37.174\n172.64.37.8\n24.170.197.68\n172.64.37.65\n71.66.130.90\n51.15.42.162\n202.78.224.134\n190.90.154.194\n162.159.56.78\n212.211.132.4\n172.64.47.254\n193.95.93.243\n190.216.19.22\n172.64.36.117\n8.243.126.29\n14.225.246.17\n62.149.128.2\n162.159.36.11\n104.255.175.2\n172.64.36.172\n216.27.175.2\n64.76.25.113\n178.175.129.16\n209.244.104.185\n146.190.6.140\n172.64.36.138\n172.64.37.235\n8.29.3.66\n172.64.37.228\n8.14.62.69\n121.52.154.225\n162.159.50.27\n162.159.36.158\n177.92.1.35\n172.64.36.43\n8.242.148.75\n1.0.170.108\n109.228.1.132\n156.154.70.7\n190.93.189.28\n199.2.252.10\n1.0.170.100\n41.222.4.34\n162.159.36.123\n162.159.56.8\n172.64.36.173\n162.159.50.239\n190.152.5.126\n4.53.7.42\n45.90.30.13\n217.138.219.219\n172.108.131.83\n80.211.55.138\n158.43.240.3\n200.55.63.133\n8.242.187.226\n181.209.105.154\n172.64.37.227\n162.159.51.183\n77.88.8.7\n1.0.247.85\n1.0.226.36\n162.159.36.36\n162.159.46.134\n172.64.37.14\n162.159.51.8\n94.142.242.40\n172.64.37.212\n45.225.123.237\n190.216.252.2\n64.233.217.2\n76.76.10.4\n172.64.37.236\n172.64.37.113\n209.216.160.131\n172.64.36.90\n64.81.45.2\n172.64.37.18\n8.242.24.58\n200.31.4.97\n45.90.30.26\n143.0.226.116\n172.64.36.23\n172.64.36.134\n172.64.36.184\n208.48.51.30\n172.64.37.116\n172.64.47.107\n212.78.94.40\n50.222.112.82\n172.64.47.106\n172.64.36.189\n194.36.144.87\n110.145.237.30\n172.64.37.94\n162.159.36.199\n172.64.36.103\n172.64.37.13\n210.94.0.73\n110.77.149.20\n200.12.130.66\n118.70.203.68\n172.64.36.37\n65.70.23.44\n212.73.221.107\n172.64.37.41\n4.16.64.169\n162.159.36.252\n195.186.4.109\n209.136.132.136\n118.69.174.71\n8.36.139.129\n8.243.126.120\n8.27.177.11\n68.87.74.166\n94.140.15.15\n190.216.19.17\n162.159.51.37\n8.28.109.10\n8.243.126.5\n172.64.47.45\n204.199.66.76\n172.64.37.39\n190.216.19.24\n8.243.126.20\n162.159.36.64\n190.181.21.50\n172.64.47.186\n128.199.128.150\n139.134.5.51\n200.125.168.132\n8.28.109.11\n172.64.36.36\n162.159.57.131\n1.0.218.58\n202.248.37.74\n75.73.8.108\n162.159.57.180\n171.251.51.138\n123.31.40.97\n205.151.222.250\n162.159.56.210\n1.0.215.132\n80.67.169.40\n172.64.36.112\n51.75.69.222\n83.69.179.25\n172.64.36.82\n172.65.25.255\n162.159.36.86\n35.167.25.37\n14.225.232.26\n38.132.106.139\n67.73.245.163\n1.0.216.220\n1.0.214.16\n8.21.123.80\n216.55.99.220\n172.64.37.161\n162.159.51.95\n41.65.236.61\n98.194.45.193\n8.21.123.117\n172.64.36.240\n190.216.254.200\n189.125.208.154\n9.9.9.12\n200.10.231.110\n49.231.140.120\n172.64.36.32\n144.76.83.104\n5.2.75.75\n123.30.184.141\n172.64.37.166\n172.64.36.237\n210.87.250.155\n201.234.119.52\n204.199.81.217\n193.111.144.145\n89.161.27.84\n8.28.109.60\n8.28.109.62\n45.225.123.235\n172.64.37.95\n200.55.19.247\n75.71.5.153\n172.64.36.168\n209.247.118.8\n76.76.2.3\n190.217.31.74\n172.64.36.100\n12.201.176.131\n162.159.50.42\n172.64.37.111\n172.64.36.207\n172.64.37.5\n8.12.246.73\n72.237.206.30\n64.156.223.254\n162.159.50.140\n172.64.36.102\n172.64.36.88\n172.64.37.87\n65.56.156.249\n45.225.123.253\n146.70.66.227\n5.164.31.108\n92.247.142.182\n202.136.163.11\n202.43.108.2\n24.125.55.212\n172.64.36.63\n172.64.36.124\n162.159.50.187\n27.76.137.76\n45.67.219.208\n172.64.37.35\n206.253.33.131\n174.69.40.233\n8.242.29.210\n162.159.51.116\n8.14.63.84\n172.64.37.211\n1.1.1.2\n172.64.37.200\n172.64.36.120\n172.64.36.0\n204.97.212.10\n172.64.37.30\n162.159.51.97\n172.64.36.140\n63.211.67.252\n64.195.220.221\n162.159.36.136\n185.222.222.222\n172.64.47.166\n172.64.36.51\n172.64.36.245\n45.90.30.169\n195.60.70.5\n1.0.218.57\n118.69.170.36\n81.116.141.156\n80.178.170.176\n5.11.11.11\n172.64.46.53\n200.41.50.3\n66.92.224.2\n203.146.237.237\n172.64.37.147\n8.224.34.74\n68.105.172.184\n172.64.37.29\n37.120.193.219\n45.90.30.12\n91.201.255.46\n1.0.0.19\n5.164.27.185\n66.162.13.184\n203.113.172.91\n72.237.212.21\n8.9.117.14\n172.64.46.127\n172.64.37.140\n110.35.78.65\n13.79.26.62\n162.159.51.248\n64.76.25.118\n50.233.102.227\n60.251.117.118\n172.64.36.93\n71.60.139.121\n190.216.19.30\n172.64.36.46\n162.159.50.138\n172.64.37.223\n8.243.126.3\n156.154.70.10\n204.199.35.164\n103.7.172.8\n216.146.36.36\n162.159.57.170\n4.0.0.53\n162.159.46.8\n172.64.36.81\n4.1.46.238\n88.208.244.225\n204.199.248.34\n202.78.224.129\n70.171.51.41\n162.159.46.214\n199.76.39.107\n4.7.43.100\n202.180.160.1\n172.64.47.44\n107.170.225.126\n82.221.128.44\n209.244.104.183\n98.195.112.169\n162.159.50.233\n85.132.85.85\n45.167.181.34\n1.0.214.195\n8.14.63.85\n45.225.123.162\n176.103.130.131\n8.26.56.15\n204.199.115.162\n198.77.228.151\n162.159.46.172\n209.247.111.148\n172.64.36.13\n172.64.46.27\n172.64.37.153\n172.64.36.165\n45.79.120.233\n109.228.0.226\n176.9.93.198\n193.135.143.37\n172.64.36.20\n209.234.196.12\n37.120.142.115\n50.220.47.51\n62.140.239.1\n85.9.129.36\n172.64.36.97\n156.154.70.5\n193.230.161.4\n172.64.36.105\n8.224.99.1\n204.199.122.6\n172.64.36.188\n185.51.92.108\n204.199.114.147\n204.199.97.171\n172.64.47.18\n172.64.36.162\n172.64.37.47\n190.216.73.28\n172.64.36.201\n24.113.32.30\n8.242.48.20\n8.29.3.219\n74.202.142.162\n70.171.61.101\n172.64.36.95\n64.105.172.26\n195.158.0.5\n165.231.253.163\n172.64.37.135\n45.90.30.11\n172.64.37.231\n185.5.17.19\n172.107.154.211\n162.159.51.11\n162.159.56.39\n172.64.36.178\n203.129.25.106\n45.225.123.179\n8.243.126.126\n8.29.3.222\n1.0.209.8\n193.238.77.61\n8.40.106.22\n172.64.37.50\n209.136.132.135\n172.64.36.153\n172.64.47.50\n8.30.101.118\n8.243.126.26\n1.0.247.215\n1.0.235.168\n8.24.104.109\n64.105.202.138\n172.64.47.224\n193.135.143.25\n204.199.130.156\n41.65.236.54\n8.243.126.73\n149.112.112.11\n172.64.37.175\n45.225.123.232\n172.64.37.58\n172.64.36.205\n72.198.188.68\n123.176.31.226\n172.64.36.185\n1.0.0.3\n172.64.37.97\n66.115.98.85\n103.48.78.157\n1.0.202.123\n149.156.132.100\n172.64.36.57\n103.160.248.45\n8.242.172.201\n172.64.37.209\n9.9.9.10\n162.159.50.221\n66.92.159.2\n162.159.50.133\n8.26.56.7\n172.64.36.156\n8.242.215.93\n172.64.37.191\n1.0.238.254\n8.243.126.116\n8.243.126.23\n176.214.35.182\n64.128.29.80\n45.90.30.22\n172.64.37.206\n14.238.96.162\n172.64.47.85\n123.30.27.24\n73.54.161.169\n162.159.56.21\n184.187.144.144\n172.64.37.120\n172.64.37.74\n75.150.197.154\n8.243.126.127\n8.28.109.44\n203.2.193.67\n172.64.36.50\n200.125.171.220\n64.76.23.53\n1.0.170.91\n172.64.37.213\n1.0.216.157\n164.52.192.24\n69.174.153.224\n49.156.53.165\n103.85.104.42\n204.199.33.244\n8.29.3.228\n185.74.5.5\n172.64.47.242\n162.159.56.242\n1.0.215.109\n8.28.109.117\n176.58.126.9\n162.159.36.110\n50.201.178.59\n172.64.37.93\n1.0.212.74\n204.199.6.86\n70.35.213.226\n172.64.36.87\n8.243.126.123\n8.28.109.106\n95.143.220.5\n202.87.213.253\n162.159.50.46\n66.162.142.38\n88.204.203.34\n223.6.6.6\n213.249.127.70\n8.29.3.220\n149.112.112.112\n162.159.36.247\n1.0.218.46\n202.78.224.130\n72.52.104.74\n196.27.105.130\n172.64.37.34\n162.159.46.197\n24.170.199.20\n172.64.37.82\n205.171.3.65\n190.216.241.5\n202.44.52.1\n76.76.2.2\n172.64.36.145\n63.209.154.100\n8.29.3.70\n162.159.56.33\n31.7.36.36\n98.249.57.2\n211.115.194.1\n4.1.131.250\n45.225.123.213\n85.204.79.2\n162.159.46.167\n172.64.36.244\n162.159.50.248\n172.64.37.79\n180.182.54.2\n66.251.199.51\n162.159.51.69\n172.64.36.89\n172.64.47.102\n172.64.36.38\n193.135.143.27\n172.64.36.60\n4.7.98.154\n8.28.109.99\n4.14.233.222\n64.119.80.100\n14.225.24.84\n66.162.85.79\n8.21.24.71\n162.159.57.81\n45.90.28.27\n172.64.37.159\n1.0.169.118\n172.64.36.214\n172.64.47.178\n201.234.235.90\n110.145.178.74\n103.196.38.39\n172.64.37.250\n129.250.35.250\n172.64.37.118\n172.64.37.141\n24.119.106.138\n190.216.67.52\n190.217.8.247\n185.43.135.1\n209.200.84.27\n45.225.123.172\n76.76.10.5\n94.198.41.235\n77.88.8.1\n50.58.191.11\n165.246.10.2\n172.64.36.84\n172.64.37.230\n181.224.160.14\n76.76.2.1\n198.54.117.10\n172.64.37.151\n195.129.111.50\n172.64.36.131\n172.64.36.75\n195.243.99.35\n41.65.236.53\n8.243.126.131\n162.159.57.86\n172.64.37.237\n162.159.36.216\n37.120.217.75\n190.216.251.5\n8.29.2.36\n1.0.162.56\n162.159.51.155\n147.0.63.59\n68.1.86.231\n24.99.148.61\n1.0.168.129\n64.76.25.123\n193.135.143.5\n172.64.37.248\n98.38.222.51\n45.90.28.29\n1.0.246.54\n204.199.85.179\n212.73.221.104\n1.0.216.26\n201.234.86.130\n156.200.116.73\n96.64.201.177\n172.64.37.240\n1.0.136.237\n1.1.220.28\n98.255.2.112\n45.90.30.126\n203.89.200.6\n8.38.117.156\n45.225.123.236\n45.90.28.11\n66.28.0.45\n204.199.81.94\n12.127.17.71\n162.159.46.147\n193.138.92.130\n8.25.185.131\n203.39.3.133\n118.69.187.252\n108.56.80.135\n" }, { "path": "bbot/wordlists/paramminer_headers.txt", "content": "accept\naccept-charset\naccept-encoding\naccept-language\naccept-ranges\naccess-control-allow-credentials\naccess-control-allow-headers\naccess-control-allow-methods\naccess-control-allow-origin\naccess-control-expose-headers\naccess-control-max-age\naccess-control-request-headers\naccess-control-request-method\nage\nallow\nauthorization\nauthenticate\ncache-control\nconnection\ncontact\ncontent-disposition\ncontent-encoding\ncontent-language\ncontent-length\ncontent-location\ncontent-range\ncontent-security-policy\ncontent-security-policy-report-only\ncontent-type\ncookie\ncookie2\ndnt\ndate\ndestination\netag\nexpect\nexpires\nforwarded\nfrom\nhost~%h:%s\nif-match\nif-modified-since\nif-none-match\nif-range\nif-unmodified-since\nkeep-alive\nlarge-allocation\nlast-modified\nlocation\norigin~https://%s.%h\npragma\nprofile\nproxy-authenticate\nproxy-authorization\npublic-key-pins\npublic-key-pins-report-only\nrange\nreferer~http://%s.%h/\nreferrer-policy\nreport-to\nretry-after\nserver\nset-cookie\nset-cookie2\nsourcemap\nstrict-transport-security\nte\ntiming-allow-origin\ntk\ntrailer\ntransfer-encoding\nupgrade-insecure-requests\nuser-agent\nvary\nvia\nwww-authenticate\nwarning\nx-content-type-options\nx-dns-prefetch-control\nx-forwarded-for\nx-forwarded-host~%s.%h\nx-forwarded-proto\nx-forwarded-port\nx-forwarded-prefix\nfront-end-https\nx-forwarded-protocol\nx-forwarded-ssl\nx-url-scheme\nx-cluster-client-ip\nx-forwarded-server~%s.%h\nproxy-host\nx-wap-profile\nx-original-url\nx-rewrite-url\nx-http-destinationurl\nproxy-connection\nx-uidh\ntrue-client-ip\nrequest-uri\norig_path_info\nclient-ip\nx-real-ip\nx-originating-ip\ncf-ipcountry\ncf-visitor\nremote-userhttps\nserver-software\nweb-server-api\nremote-addr\nremote-host\nremote-user\nrequest-method\nscript-name\npath-info\nunencoded-url\nx-arr-ssl\nx-arr-log-id\nsoapaction\nx-original-http-command\nx-server-name\nx-server-port\nquery-string\nauth-password\nauth-type\nauth-user\ncert-cookie\ncert-flags\ncert-issuer\ncert-keysize\ncert-secretkeysize\ncert-serialnumber\ncert-server-issuer\ncert-server-subject\ncert-subject\ncf-template-path\ncontext-path\ngateway-interface\nhttps-keysize\nhttps-secretkeysize\nhttps-server-issuer\nhttps-server-subject\nhttp-accept\nhttp-accept-encoding\nhttp-accept-language\nhttp-connection\nhttp-cookie\nhttp-host\nhttp-referer\nhttp-url\nhttp-user-agent\nlocal-addr\npath-translated\nserver-name\nserver-port\nserver-port-secure\nserver-protocol\ncloudfront-viewer-country\nx-scheme\nx-cascade\nx-http-method-override\nx-http-path-override\nx-http-host-override\nx-http-method\nx-method-override\nx-cf-url\nphp-auth-user\nphp-auth-pw\nerror\npost-vars\nraw-post-data\nproxy-request-fulluri\nrequest\nserver-varsabantecart\naccept-application\naccept-auth\naccept-encodxng\naccept-version\naction\nadmin\nakamai-origin-hop\napp\napp-key\napply-to-redirect-ref\natcept-language\nauth-digest-ie\nauth-key\nauth-realm\nbase-url\nbearer-indication\nbrowser-user-agent\ncase-files\ncategory\nch\nchallenge-response\ncharset\nclient-address\nclient-bad-request\nclient-conflict\nclient-error-connect\nclient-expectation-failed\nclient-forbidden\nclient-gone\nclient-length-required\nclient-method-not-allowed\nclient-not-acceptable\nclient-not-found\nclient-payment-required\nclient-precondition-failed\nclient-proxy-auth-required\nclient-quirk-mode\nclient-requested-range-not-possible\nclient-request-timeout\nclient-request-too-large\nclient-request-uri-too-large\nclient-unauthorized\nclient-unsupported-media-type\ncloudinary-name\ncloudinary-public-id\ncloudinaryurl\ncloudinary-version\ncompress\nconnection-type\ncontent\ncontent-type-xhtml\ncookies\ncore-base\ncredentials-filepath\ncurl\ncurl-multithreaded\ncustom-secret-header\ndataserviceversion\ndestroy\ndevblocksproxybase\ndevblocksproxyhost\ndevblocksproxyssl\ndigest\ndir\ndir-name\ndir-resource\ndisable-gzip\ndkim-signature\ndownload-bad-url\ndownload-cut-short\ndownload-mime-type\ndownload-no-server\ndownload-size\ndownload-status-not-found\ndownload-status-server-error\ndownload-status-unauthorized\ndownload-status-unknown\ndownload-url\nenv-silla-environment\nespo-authorization\nespo-cgi-auth\neve-charid\neve-charname\neve-solarsystemid\neve-solarsystemname\nex-copy-movie\next\nfake-header\nfastly-client-ip\nfb-appid\nfb-secret\nfilename\nfile-not-found\nfiles\nfiles-vars\nfoo-bar\nforce-language\nforce-local-xhprof\nforwarded-proto\nfromlink\ngivenname\nglobal-all\nglobal-cookie\nglobal-get\nglobal-post\ngoogle-code-project-hosting-hook-hmac\nh0st\nhome\nhost-liveserver\nhost-name\nhost-unavailable\nhttp-authorization\nif-modified-since-version\nif-posted-before\nif-unmodified-since-version\nimages\ninfo\nischedule-version\niv-groups\niv-user\njenkins\nkiss-rpc\nlast-event-id\nlocal-dir\nmail\nmax-conn\nmaxdataserviceversion\nmax-request-size\nmax-uri-length\nmessage\nmessage-b\nmode\nmod-env\nmod-security-message\nmodule-class\nmodule-class-path\nmodule-name\nms-asprotocolversion\nmsisdn\nmy-header\nmysqlport\nnative-sockets\nnonce\nnot-exists\nnotification-template\nonerror-return\norganizer\nparams-get-catid\nparams-get-currentday\nparams-get-disposition\nparams-get-downwards\nparams-get-givendate\nparams-get-lang\nparams-get-type\npasskey\npath-base\npath-themes\nphpthreads\nportsensor-auth\npost-error\npostredir-301\npostredir-302\npostredir-all\nprotocol\nprotocols\nproxy-agent\nproxy-http-1-0\nproxy-pwd\nproxy-socks4a\nproxy-socks5-hostname\nproxy-url\npull\nquerystring\nrealip\nreal-ip\nreal-method\nreason\nreason-phrase\nredirected-accept-language\nredirection-found\nredirection-multiple-choices\nredirection-not-modified\nredirection-permanent\nredirection-see-other\nredirection-temporary\nredirection-unused\nredirection-use-proxy\nredirect-problem-withoutwww\nredirect-problem-withwww\nref\nreferer\nrefresh\nremix-hash\nremote-host-wp\nrequest-method-\nresponse\nrest-key\nreturned-error\nrlnclientipaddr\nsafe-ports-list\nsafe-ports-ssl-list\nschedule-reply\nsec-websocket-accept\nsec-websocket-extensions\nsec-websocket-key1\nsec-websocket-key2\nsec-websocket-origin\nsec-websocket-protocol\nsec-websocket-version\nself\nsend-x-frame-options\nserver-bad-gateway\nserver-error\nserver-gateway-timeout\nserver-internal\nserver-not-implemented\nserver-service-unavailable\nserver-unsupported-version\nsession-id-tag\nshib-\nshib-identity-provider\nshib-logouturl\nshopilex\nsn\nsocketlog\nsomevar\nsp-client\nssl-offloaded\nsslsessionid\nssl-session-id\nstatus-\nstatus-403\nstatus-403-admin-del\nstatus-404\nstatus-code\nstatus-platform-403\nsuccess-accepted\nsuccess-created\nsuccess-no-content\nsuccess-non-authoritative\nsuccess-ok\nsuccess-partial-content\nsuccess-reset-content\ntest\ntest-config\ntest-server-path\ntest-something-anything\nticket\ntime-out\ntmp\ntranslate\nua-color\nua-resolution\nua-voice\nunit-test-mode\nupgrade\nuri\nurl-sanitize-path\nuse-gzip\nuseragent-via\nuser-email\nuser-id\nuser-photos\nutil\nverbose\nversioncode\nx-aastra-expmod1\nx-aastra-expmod2\nx-aastra-expmod3\nx-accel-mapping\nx-advertiser-id\nx-ajax-real-method\nx-alto-ajax-keyz\nx-api-signature\nx-api-timestamp\nx-apple-client-application\nx-apple-store-front\nx-authentication\nx-authentication-key\nx-auth-mode\nx-authorization\nx-auth-password\nx-auth-service-provider\nx-auth-token\nx-auth-userid\nx-auth-username\nx-avantgo-screensize\nx-azc-remote-addr\nx-bear-ajax-request\nx-bluecoat-via\nx-browser-height\nx-browser-width\nx-cache\nx-cept-encoding\nx-chrome-extension\nx-cisco-bbsm-clientip\nx-client-host\nx-client-id\nx-clientip\nx-client-key\nx-client-os\nx-client-os-ver\nx-collect-coverage\nx-credentials-request\nx-csrf-crumb\nx-cuid\nx-custom\nx-dagd-proxy\nx-davical-testcase\nx-debug-test\nx-dialog\nx-drestcg\nx-dsid\nx-enable-coverage\nx-environment-override\nx-experience-api-version\nx-fb-user-remote-addr\nx-file-id\nx-file-resume\nx-foo-bar\nx-forwarded-for-original\nx-forwarder-for\nx-forward-proto\nx-from\nx-gb-shared-secret\nx-geoip-country\nx-get-checksum\nx-helpscout-event\nx-hgarg-\nx-host\nx-https\nx-htx-agent\nx-if-unmodified-since\nx-imbo-test-config\nx-insight\nx-ip\nx-ip-trail\nx-iwproxy-nesting\nx-jphone-color\nx-jphone-geocode\nx-kaltura-remote-addr\nx-known-signature\nx-known-username\nx-litmus-second\nx-machine\nx-mandrill-signature\nx-mobile-ua\nx-mosso-dt\nx-msisdn\nx-ms-policykey\nx-myqee-system-debug\nx-myqee-system-hash\nx-myqee-system-isadmin\nx-myqee-system-isrest\nx-myqee-system-pathinfo\nx-myqee-system-project\nx-myqee-system-rstr\nx-myqee-system-time\nx-network-info\nx-nfsn-https\nx-ning-request-uri\nx-nokia-connection-mode\nx-nokia-msisdn\nx-nokia-wia-accept-original\nx-nokia-wtls\nx-nuget-apikey\nx-opera-info\nx-operamini-features\nx-orchestra-scheme\nx-orig-client\nx-original-host\nx-originally-forwarded-for\nx-originally-forwarded-proto\nx-original-remote-addr\nx-overlay\nx-pagelet-fragment\nx-password\nxpdb-debugger\nx-phabricator-csrf\nx-phpbb-using-plupload\nxproxy\nx-proxy-url\nx-pswd\nx-qafoo-profiler\nx-remote-protocol\nx-render-partial\nx-request\nx-request-id\nx-request-start\nx-response-format\nx-rest-cors\nx-sakura-forwarded-for\nx-scalr-auth-key\nx-scalr-auth-token\nx-scalr-env-id\nx-screen-height\nx-screen-width\nx-sendfile-type\nx-serialize\nx-serial-number\nx-server-id\nx-sina-proxyuser\nx-skyfire-screen\nx-ssl\nx-subdomain\nx-teamsite-preremap\nx-test-session-id\nx-tine20-jsonkey\nx-tine20-request-type\nx-tomboy-client\nx-tor\nx-twilio-signature\nx-uniquewcid\nx-up-calling-line-id\nx-up-devcap-screendepth\nx-upload-content-type\nx-upload-maxresolution\nx-upload-name\nx-upload-size\nx-upload-type\nx-user-agent\nx-username\nx-verify-credentials-authorization\nx-wap-client-sdu-size\nx-wap-gateway\nx-wap-network-client-ip\nx-wap-network-client-msisdn\nx-wap-proxy-cookie\nx-wap-session-id\nx-wap-tod\nx-wap-tod-coded\nx-wopi-override\nx-wikimedia-debug\nx-wp-pjax-prefetch\nx-ws-api-key\nx-xc-schema-version\nx-xhprof-debug\nx-xhr-referer\nx-xmlhttprequest\nx-xpid\nxxx-real-ip\nxxxxxxxxxxxxxxx\nx-zikula-ajax-token\nx-zotero-version\nx-ztgo-bearerinfo\ny\nzotero-api-version\nzotero-write-token\naccess-token\najax\napp-env\nbae-env-addr-bcms\nbae-env-addr-bus\nbae-env-addr-channel\nbae-logid\nbasic\ncatalog\nclientip\ndebug\ndelete\nenable-gzip\nenable-no-cache-headers\nerror-1\nerror-2\nerror-3\nerror-4\neve-trusted\nfire-breathing-dragon\nformat\ngzip-level\nhead\nhosti\nhtaccess\nimage\nincap-client-ip\nlocal-content-sha1\non-behalf-of\noptions\npassword\npink-pony\nproxy-password\nput\nrequest2-tests-base-url\nrequest2-tests-proxy-host\nrequest-timeout\nrest-sign\nroot\nsupport-events\ntoken\nuser\nuseragent\nuser-mail\nuser-name\nversion-none\nviad\nx\nx-access-token\nx-amz-date\nx-amz-server-side-encryption\nx-auth-key\nx-auth-user\nx-confirm-delete\nx-do-not-track\nx-elgg-nonce\nx-expected-entity-length\nx-filename\nx-flash-version\nx-flx-consumer-key\nx-flx-consumer-secret\nx-flx-redirect-url\nx-forwarded-scheme\nx-jphone-msname\nx-options\nx-os-prefs\nx-pjax-container\nx-request-timestamp\nx-rest-password\nx-rest-username\nx-te\nx-unique-id\nx-up-devcap-iscolor\naccesskey\nauth-any\nauth-basic\nauth-digest\nauth-gssneg\nauth-ntlm\ncode\ncookie-httponly\ncookie-parse-raw\ncookie-secure\ndeflate-level-def\ndeflate-level-max\ndeflate-level-min\ndeflate-strategy-def\ndeflate-strategy-filt\ndeflate-strategy-fixed\ndeflate-strategy-huff\ndeflate-strategy-rle\ndeflate-type-gzip\ndeflate-type-raw\ndeflate-type-zlib\ne-encoding\ne-header\ne-invalid-param\ne-malformed-headers\ne-message-type\nencoding-stream-flush-full\nencoding-stream-flush-none\nencoding-stream-flush-sync\ne-querystring\ne-request\ne-request-method\ne-request-pool\ne-response\ne-runtime\ne-socket\ne-url\nget\nheader\nhttp-phone-number\nipresolve-any\nipresolve-v4\nipresolve-v6\nlink\nmeth-acl\nmeth-baseline-control\nmeth-checkin\nmeth-checkout\nmeth-connect\nmeth-copy\nmeth-label\nmeth-lock\nmeth-merge\nmeth-mkactivity\nmeth-mkcol\nmeth-mkworkspace\nmeth-move\nmeth-options\nmeth-propfind\nmeth-proppatch\nmeth-report\nmeth-trace\nmeth-uncheckout\nmeth-unlock\nmeth-update\nmeth-version-control\nmsg-none\nmsg-request\nmsg-response\noc-chunked\nocs-apirequest\nparams-allow-comma\nparams-allow-failure\nparams-default\nparams-raise-error\npath\nphone-number\npragma-no-cache\nproxy-http\nproxy-socks4\nproxy-socks5\nquerystring-type-array\nquerystring-type-bool\nquerystring-type-float\nquerystring-type-int\nquerystring-type-object\nquerystring-type-string\nredirect\nredirect-found\nredirect-perm\nredirect-post\nredirect-proxy\nredirect-temp\nrefferer\nrequesttoken\nsec-ch-ua\nsec-ch-ua-arch\nsec-ch-ua-bitness\nsec-ch-ua-full-version-list\nsec-ch-ua-mobile\nsec-ch-ua-model\nsec-ch-ua-platform\nsec-ch-ua-platform-version\nsec-fetch-dest\nsec-fetch-mode\nsec-fetch-site\nsec-fetch-user\nsec-websocket-key\nsp-host\nssl\nssl-version-any\nstatus-bad-request\nstatus-forbidden\nsupport\nsupport-encodings\nsupport-magicmime\nsupport-requests\nsupport-sslrequests\nsurrogate-capability\nua\nupload-default-chmod\nurl\nurl-from-env\nverbose-throttle\nversion-1-0\nversion-1-1\nversion-any\nwebodf-member-id\nwebodf-session-id\nwebodf-session-revision\nwork-directory\nx-\nx-api-key\nx-apitoken\nx-csrftoken\nx-elgg-apikey\nx-elgg-hmac\nx-elgg-hmac-algo\nx-elgg-posthash\nx-elgg-posthash-algo\nx-elgg-time\nx-foo\nx-forwarded-by\nx-json\nx-litmus\nx-locking\nx-oc-mtime\nx-remote-addr\nx-request-signature\nx-ua-device\nx-update-range\nx-varnish\nx-wp-nonce\nauth\nbrief\nchunk-size\nclient\ndownload-attachment\ndownload-bz2\ndownload-e-headers-sent\ndownload-e-invalid-archive-type\ndownload-e-invalid-content-type\ndownload-e-invalid-file\ndownload-e-invalid-param\ndownload-e-invalid-request\ndownload-e-invalid-resource\ndownload-e-no-ext-mmagic\ndownload-e-no-ext-zlib\ndownload-inline\ndownload-tar\ndownload-tgz\ndownload-zip\nheader-lf\nheader-status-client-error\nheader-status-informational\nheader-status-redirect\nheader-status-server-error\nheader-status-successful\nhttps-from-lb\nmeth-delete\nmeth-head\nmeth-post\nmultipart-boundary\noriginator\nphp\nrecipient\nrequest-error\nrequest-vars\nsecretkey\nstatus-ok\nxauthorization\nx-codeception-codecoverage\nx-codeception-codecoverage-config\nx-codeception-codecoverage-debug\nx-codeception-codecoverage-suite\nx-csrf-token\nx-dokuwiki-do\nx-helpscout-signature\nx-nokia-bearer\nxonnection\nx-purpose\nxroxy-connection\nx-user\nbae-env-appid\ncatalog-server\ncookie-path\ncustom-header\nforwarded-for-ip\nmeth-get\nmeth-put\nopencart\nunless-modified-since\nwww-address\nx-content-type\nx-hub-signature\nx-signature\nbae-env-addr-sql-ip\nbae-env-addr-sql-port\ncache-info\nclient-error-cannot-access-local-file\nclient-error-cannot-connect\nclient-error-communication-failure\nclient-error-invalid-parameters\nclient-error-invalid-server-address\nclient-error-no-error\nclient-error-protocol-failure\nclient-error-unspecified-error\nerror-formatting-html\nlock-token\nonerror-continue\nonerror-die\noverwrite\nprefer\nshib-application-id\nx-fireloggerauth\ncookie-domain\nhttps\nmeth-\nmodauth\nport\npost\nread-state-begin\nread-state-body\nread-state-headers\nsocket-connection-err\nstr-match\ntransport-err\ncoming-from\nnl\nua-pixels\nx-coming-from\nx-jphone-display\nx-up-devcap-screenpixels\nx-whatever\nappname\nproxy-port\nversion\nx-forward-for\nproxy-user\nx-em-uid\nx-file-type\nbar\nproxy\ntimeout\nreferrer\nx-forwarded-ssl\nx-jphone-uid\nx-file-size\naccepted\nappcookie\nbad-gateway\nbae-env-addr-bcs\nconflict\ncontinue\ncreated\nexpectation-failed\nfailed-dependency\ngateway-time-out\ngone\ninsufficient-storage\ninternal-server-error\nlength-required\nlocked\nmethod-not-allowed\nmoved-permanently\nmoved-temporarily\nmultiple-choices\nmulti-status\nno-content\nnon-authoritative\nnot-acceptable\nnot-extended\nnot-implemented\nnot-modified\npartial-content\npayment-required\nprecondition-failed\nprocessing\nproxy-authentication-required\nrange-not-satisfiable\nrequest-entity-too-large\nrequest-time-out\nrequest-uri-too-large\nreset-content\nsee-other\nservice-unavailable\nswitching-protocols\ntemporary-redirect\nunprocessable-entity\nunsupported-media-type\nupgrade-required\nuse-proxy\nvariant-also-varies\nversion-not-supported\nx-operamini-phone\nbad-request\nforbidden\nunauthorized\nuser-agent-via\nappversion\nnot-found\nurl-strip-\nx-pjax\ncf-connecting-ip\nx-dcmguid\nfoo\ninfo-download-size\ninfo-download-time\ninfo-return-code\ninfo-total-request-stat\ninfo-total-response-stat\nx-firelogger\ncontent-md5\nx-up-subno\nbae-env-ak\nbae-env-sk\nif\nok\nurl-join-path\nurl-join-query\nurl-replace\nurl-strip-all\nurl-strip-auth\nurl-strip-fragment\nurl-strip-pass\nurl-strip-path\nurl-strip-port\nurl-strip-query\nurl-strip-user\ndepth\nx-file-name\nx-moz\nx-ucbrowser-device-ua\ndevice-stock-ua\nmod-rewrite\nx-nokia-ipaddress\nx-bolt-phone-ua\nx-original-user-agent\nx-skyfire-phone\ntitle\nssl-https\nrequest-error-file\nrequest-error-gzip-crc\nrequest-error-gzip-data\nrequest-error-gzip-method\nrequest-error-gzip-read\nrequest-error-proxy\nrequest-error-redirects\nrequest-error-response\nrequest-error-url\nslug\nx-att-deviceid\nauthentication\nx-firephp-version\nx-mobile-gateway\nrequest-mbstring\nx-device-user-agent\nx-huawei-userid\nx-orange-id\nx-vodafone-3gpdpcontext\nx-wap-clientid\nua-cpu\nwap-connection\nx-nokia-gateway-id\nua-os\nbody-maxlength\nbody-truncated\nmax-forwards\nmimetype\nverify-cert\nrequest-http-ver-1-0\nrequest-http-ver-1-1\nrequest-method-delete\nrequest-method-get\nrequest-method-head\nrequest-method-options\nrequest-method-post\nrequest-method-put\nrequest-method-trace\nx-operamini-phone-ua\nstatus\nx-update\nmethod\nforwarded-for\nx-forwarded\nscheme\nx-forwarded-server\norigin\nx-client-ip\nx-prototype-version\nclientaddress\nbase\npc-remote-addr\npost-files\nsession-vars\ncookie-vars\nenv-vars\nget-vars\nserver-vars\nx-forwarded-host\nx-requested-with\nreferer\nhost\nalt-used\nx-original-url~/%s\nx-rewrite-url~/%s\ncommand\n__requesturi\n__requestverb\nx-http-status-code-override\nx-amzn-remapped-host\nx-amz-website-redirect-location\nx-up-devcap-post-charset\nhttp_sm_authdirname\nhttp_sm_authdirnamespace\nhttp_sm_authdiroid\nhttp_sm_authdirserver\nhttp_sm_authreason\nhttp_sm_authtype\nhttp_sm_dominocn\nhttp_sm_realm\nhttp_sm_realmoid\nhttp_sm_sdomain\nhttp_sm_serveridentityspec\nhttp_sm_serversessionid\nhttp_sm_serversessionspec\nhttp_sm_sessiondrift\nhttp_sm_timetoexpire\nhttp_sm_transactionid\nhttp_sm_universalid\nhttp_sm_user\nhttp_sm_userdn\nhttp_sm_usermsg\nx-remote-ip\ntraceparent\ntracestate\n" }, { "path": "bbot/wordlists/paramminer_parameters.txt", "content": "id\nuser\naccount\nnumber\norder\nno\ndoc\nkey\nemail\ngroup\nprofile\nedit\nreport\ndaemon\nupload\ndir\nexecute\ndownload\nlog\nip\ncli\ncmd\nfile\ndocument\nfolder\nroot\npath\npg\nstyle\npdf\ntemplate\nphp_path\nselect\nrole\nupdate\nquery\nname\nsort\nwhere\nsearch\nparams\nprocess\nrow\nview\ntable\nfrom\nsel\nresults\nsleep\nfetch\nkeyword\ncolumn\nfield\ndelete\nstring\nfilter\ndest\nredirect\nuri\ncontinue\nurl\nwindow\nnext\ndata\nreference\nsite\nhtml\nval\nvalidate\ndomain\ncallback\nreturn\npage\nfeed\nhost\nport\nto\nout\nshow\nnavigation\nopen\npreview\nactivity\ncontent\naccess\nadmin\ndbg\ndebug\ngrant\ntest\nalter\nclone\ncreate\ndisable\nenable\nexec\nload\nmake\nmodify\nrename\nreset\nshell\ntoggle\nadm\ncfg\nconfig\naction\n_method\npassword\ntype\nusername\ntitle\ncode\nq\nsubmit\ntoken\nmessage\nt\nc\nmode\nlang\np\nstatus\nstart\ncharset\ndescription\ns\npost\nexcerpt\nlogin\ncomment\nstep\najax\nstate\nf\nerror\nsave\nformat\ntab\noffset\na\nlimit\ndo\nplugin\ntheme\ntext\nlanguage\nheight\nlogout\npass\nh\nvalue\nfilename\nyear\nversion\nsubject\nm\nu\nconfirm\nwidth\nw\nsize\ndate\nsource\nGLOBALS\nop\nmethod\nuid\ntag\ncategory\ntarget\nids\nterm\nnew\nlocale\nauthor\npaged\ncat\nmsg\nadd\nd\nday\nnonce\ncaptcha\noutput\nrevision\ni\nxml\ndb\ntime\nsection\nimage\nr\nfiles\ntags\nusers\nsend\nupdated\nskips\nn\ncheck\norderby\nnum\nimport\nprefix\nfields\npwd\npid\nmonth\nmodule\nparent\ncancel\nactivate\nchecked\nsuccess\ndesc\ncase\nremove\nposition\nlocation\nextra\ncount\nb\nrating\npass2\nhostname\nmove\nhash\ndry\ncid\nbody\nsrc\nlevel\ngenerate\ng\ndbname\noption\nuserid\nsql\noptions\naddress\nactivated\naction2\npassword2\npass1\nmeta\nID\ndeleted\nact\ne\ntaxonomy\nref\npublish\nsecret\napp\nrememberme\ncountry\nphone\nhidden\nforce\nexport\nsticky\nnickname\nv\nplugins\nlocked\ncommand\nreturnUrl\nitem\namount\ntimestamp\nserver\nsignature\npart\njson\ndel\ncomments\nvisible\nLoginForm\nkeywords\nenabled\nbase\nrefresh\nfoo\ny\nmedia\ninfo\nguid\ndt\nx\ntestdata\nlist\nvisibility\nUser\nthumb\nstage\nhistory\ntimezone\nupgrade\nmenu\nitems\nclass\nblog\nlink\nend\ndbhost\napproved\nstylesheet\nsid\nsettings\npostid\ndeactivate\nclosed\nposted\nnoheader\nContactForm\ntax\nss\ninline\ngid\nattachments\nadded\nreplytocom\ndismiss\nclear\ncity\nspam\nrequest\nall\nsidebar\ndbuser\ncheckbox\nshort\nactive\nsession\nregistration\nhh\nprice\nnsql\nmm\nloggedout\nlastname\nSMALLER\nsaved\nrsd\nps\nnewcontent\nmn\nlinkurl\njj\ninstall\nhidem\nfirstname\ndetached\ncolor\nclearsql\ncheckemail\nBIGGER\naa\nslug\nremember\nreferrer\nreason\no\nnote\nreferredby\nl\ndeletepost\ndbpass\nattached\ntid\ntestcookie\nnoredir\nnewcat\nmonthnum\nmetakeyinput\ninsertonlybutton\ninput\nform\nfailure\ndown\ndeletemeta\ndeletecomment\ncontext\nbackto\nundismiss\nsitename\nservice\nresetheader\nprint\nphperror\noitar\nmetavalue\nmetakeyselect\nmail\nliveupdate\nlinkcheck\ndeletebookmarks\nchangeit\nanswers\naddmeta\ntrashed\nfid\nback\nselection\nmod\nlabel\nimg\nfeatures\ndirection\nuname\nsidebars\nhide\nauth\nuntrashed\ntask\nsubmitted\ndatabase\naddnew\nSubmit\npurge\nnotes\neditwidget\nremovewidget\nnrows\ngroups\ndisabled\nzip\ntrash\nrepair\noverwrite\nreferer\nthemes\nmid\ndefaults\ncustom\nctype\nwidget\ntopic\nmain\njs\nblogname\nuntrash\nunspammed\nunspam\nspammed\nselectall\nquantity\nnewuser\nnetworkwide\ninvalid\nindex\nfunction\nscreen\nreply\nlat\ngender\nfind\ndisplay\ndirectory\nbatch\nalt\nset\nscrollto\nfwidth\nfheight\nsub\nsame\nrows\nreauth\nnotify\nconfirmdelete\nautosave\naid\nvote\nreview\nkeys\ndestination\nallusers\npasswd\nchange\napage\nallblogs\nprivate\nnoapi\ncharsout\ncatslist\ncategories\nup\nsubscribe\nscript\nremoveheader\npos\nperiod\nnocache\nkill\ncolumns\napi\nz\nsortby\nregister\nrecovered\npagenum\nlast\nevent\ncustomized\nattachment\nanswer\nwelcome\ntimeout\nscope\nrid\nresult\npublic\npayload\nns\nmobile\ncss\nalign\nwhat\nrank\nqqfile\nmax\ncreateuser\nbackground\navatar\nalias\ntotal\nquestion\npriority\ndays\ncache\nskin\nschema\norientation\ngroupid\ndone\nsummary\nskipped\nrange\ngo\ndump\nconfirmation\nCKEditorFuncNum\nchanges\nticket\npw\npointer\nparam\nfirst\nentry\ndrop\ndefault\nselected\npopup\nowner\nnolog\nnochange\nlength\ngoto\ncompany\nComment\nclose\nwebsite\nst\nskip\nrestart\npages\nnode\nlocalize\nfname\nexcept\nType\nrestore\nprofiler\npreviewed\npassword1\nNewFolderName\nlng\nleft\nlayout\nk\nfn\nflag\ndoaction2\ndetails\ncurrency\ncopy\ncompare\nbroken\nblock\npaper\nline\njax\nicon\nflush\nfileName\ndl\ncontroller\ncatid\nPayerID\nnewname\nflash\ndecomposition\nconfirmed\nchromeless\nbid\nyes\nweight\nverify\nvalues\nrun\nroute\nreplace\nread\nproject\nPost\nnid\nmd5\nmap\nlogopng\nlistInfo\nletter\nhour\nfullname\nexclude\ndbprefix\nauthors\nzoom\nuserId\ntrigger\nsetting\nrs\nprovider\npackage\noperation\nok\nobject\nmark\nlid\ninvoice\ninsertonly\nfull\nforum\nerr\ndoit\nbackup\nac\nsent\nphpThumbDebug\nphoto\ninterval\neditor\nechostr\nchannel\nargs\nagree\nWPLANG\nuserspage\nusersearch\ntriggers\ninsert\ninc\nhomepage\nhello\nfunc\nduration\ndid\ncookie\ncontact\nchunk\napply\nterms\ntables\nstartdate\nshortcode\nscale\nreverse\nrequired\norigin\nindexes\nidentifier\nhashed\nfontcolor\ndatabases\napprove\nadvanced\nwebfile\nurls\ntypes\ntoggledisplay\nsubaction\nsortorder\nsign\nsEcho\nsearchtype\nsaveasdraft\nrss\nrecipient\nprev\nnotice\nnjlowercolor\nnjform\nnjfontcolor\nmembers\nmember\nmd5s\ninit\nhs\nheaderimage\nheader\nfontdisplay\nfinish\nfax\nengine\ncurrent\nclient\ncc\ncallf\narticle\nver\nts\nroles\nregion\nraw\nqid\nold\nnick\nmodel\nlon\nlock\niDisplayLength\next\nexpire\nenddate\nempty\nchunks\nalbum\nuserselect\nuserName\ntelephone\nstats\nsaveauthors\nright\nrevert\nresponse\nnews\nlname\nimages\nhighlight\nfrob\nembed\ndenied\ndccharset\ncontents\ncompress\nCommand\narea\naim\naccept\nvid\nunit\nundeleted\nthread\ntextinputs\ntextcolor\nstore\nsqlite\nshowall\nrsargs\nreload\nrecord\nposts\npagenow\noverride\nopt\nopname\njob\nidx\nhelp\ngroupname\nfilters\nfileid\nexpand\nentity\ncp\nclean\ncaption\napikey\nverbose\nvar\ntpl\ntopics\ntop\ntablename\nsSearch\nsex\nseparator\nscripts\nrules\nrt\nrate\nproduct\nprepopulate\npgtIou\npgtId\npgsql\npermissions\noracle\noldpass\nmssql\nmodules\nlabels\nget\nfoldername\nfamily\ndelimiter\nCurrentFolder\nchoice\nbox\nautologin\nage\nagain\nactions\nwysiwyg\nword\nuserID\nunsort\nuninstall\nunfoldmenu\nsupport\nstartDate\nstandalone\nsince\nscore\nruntests\nregex\npublished\nproxy\npoints\nphrase\noldpassword\noid\nnoajax\nnewpassword\nnewName\nminute\nmac\nlangCode\niDisplayStart\ngenre\nFrom\nfont\nemails\neid\ndst\ndevice\ndemo\ndeletefile\ncropDetails\nconnection\ncollation\ncms\nattributes\nattribute\nas\nadduser\nzone\nzipcode\nwords\nviewtype\nusr\nTo\nssl\nsingle\nsendmail\nprotocol\nphpinfo\nperpage\nnewsletter\nnewsid\nnames\nName\nmin\nlogoutRequest\nlogo\ninterface\nfrequency\nfirstName\ndbName\ncriteria\nby\nbutton\nbreak\nbg\nban\nauthorize\nartist\nallow\nun\nstripeToken\nresize\nreplyto\nremote\nrandom\nproducts\npic\nperms\nparentid\noriginal\nopener\nnamespace\nmime\nloc\nlastName\njabber\nglobal\nforums\nfoo1\nFileName\nendpoint\nEmail\ndetail\ndescr\ndeny\ndelall\ncustomer\ncopyright\ncompression\ncollection\naddress2\nyim\nweek\nunsubscribe\ntruncate\ntableName\nspeed\nsortOrder\nsig\nshare\nservername\nsections\nroom\nresource\nreq\nqty\nperm\norderid\noperator\nnoconfirmation\nnewFileName\nmakedoc\nlicense\ngraph\nframe\nduplicate\ndiscount\ncreated\nclearcache\nCKEditor\nauto\nafter\nabout\nwsdl\nvideo\nuploaded\nunban\nthumbnail\nsubtitle\nstop\nstartIndex\nsorttype\nsnippet\nsilent\nsessionid\nsequence\nsender\nsearchTerm\nsd\nsc\nrule\nreg\nredir\nquote\nprune\nproductid\npopupurl\npopuptitle\npageid\noc\nnom\nnewpass\nmemo\nmaxResults\niSortingCols\ngateway\nfor\nfeedback\nfcksource\nextension\ndraft\ndev\ndeleteall\ncsv\nbusiness\nboard\naddress1\naddr\naddgroup\nwho\nunread\nttl\ntemp\ntagid\nsure\nsubpage\nstat\nshowThumbs\nsetup\nres\nqueryType\npostcode\npermission\npending\npattern\npasskey\nnr\nmatch\njsonp\nitemid\ninvites\ninvite\nfoo6\nfoo2\nfiletype\nfc\nencoding\nenc\nem\nelement\ndiscard\ndelay\ndef\ndbpassword\ncurrentFolder\ncourse\ncommit\ncols\nchallenge\ncall\nbranch\nblogid\nbanned\narray\narchive\nweb\nunlock\nuniqid\ntxt\ntwitter\ntodo\nthreadid\nteam\nsystem\nstorage\nSTATUS\nsites\nrollback\nresettext\nrepeat\nrem\nreceiver\nrebuild\nrebroadcast\nre\nquality\nqq\nProfile\nprivileges\nprimary\npoll\nPassword\nparameters\nos\norderbydate\nopauth\nmessages\nmaintenance\nlong\nlinks\nignore\nhandler\nforward\nfileext\nendDate\ndriver\ndocroot\ndeletepage\nd2\ncron\ncontrol\nconfigure\nconditions\nCollation\ncodepress\nchart\nbitrate\nbarcode\nAuthItemForm\nassign\nadminpass\nwrite\nwatch\nswitch\nsubtype\nstreet\nstr\nsiteurl\nshipping\nsalt\nrev\nreturnto\nrepo\nrel\nRegistrationForm\nr2\npre\nplayer\nplace\npk\nperson\npermalink\npc\npayment\npagename\nother\nopenid\nnotifications\nnojs\nnewPassword\nnewdir\nnetwork\nmulti\nmailbox\nlowercase\nlayer\njsoncallback\nitemName\nisbn\niid\ngrade\ngame\nexpires\nexpiration\nencode\nedited\ndropped\ndomains\ndept\ndbtype\nconf\ncol\ncname\nchar\nbrowse\nbio\nbanner\nbalance\nasc\nanonymous\nannouncement\nxmldump\nUserRecoveryForm\nUserLogin\nUserChangePassword\nUSER\nupdates\ntx\ntweet\ntrust\ntrack\ntopicid\ntool\ntimeformat\ntb\nstep2\nssid\nsendto\nseason\nSearch\nschedule\nscan\nsa\nrepassword\nreinstall\nrealname\nradius\npx\nproxyuser\nProfileField\npmid\npm\npicture\npaymentType\nparam2\nnopass\nnewfolder\nmysql\nmultiple\nMessage\nlongitude\nlogtype\nloader\nlatitude\nlanguages\njoin\nipaddress\ninstance\niframe\nid2\nhours\nhome\ngroupId\ngallery\nftp\nfriends\nfooter\nfld\nfieldtype\nfeature\nfail\nexplain\nepisode\nemail2\nEaseTemplateVer\ndistance\ndirname\ndepth\ndelfile\ndecode\ndbport\ncrop\ncost\nconnect\nconfirmpassword\ncom\nco\nchk\nchild\ncategoryid\nBody\nbirthdate\nbegin\nbefore\nBackURL\navatars\nautofocus\nauthenticate\nat\naname\nagreement\nadminname\nactivkey\nxajax\nviewonline\nunwatch\nui\ntypeid\nth\ntemplateid\ntargets\ntagged\nsw\nsuper\nsubname\nsubform\nsubdir\nstrings\nstrict\nstatistics\nstarttime\nspec\nsord\nsnapshot\nside\nsh\nserial\nsecond\nrewrite\nretry\nrealm\nrand\nprofiling\nprevious\npreset\nposter\npolicies\npn\nplatform\nplacement\npin\npID\nphp\nparentID\npagination\npagesize\np2\np1\noldPassword\nname2\nmsn\nmoved\nmonitor\nmigrate\nmerge\nmaxage\nmask\nmanufacturer\nls\nloginname\nld\nLang\nkid\ninclude\nidSelect\nhook\ngoback\nfs\nfrontpage\nfontsize\nfilepath\nFilename\nfilecontent\nfeatured\nfav\nfailed\nextend\neventId\neventid\nendtime\neditid\ndiv\ndelivery\ndbUser\ndbsize\ndbPassword\nDATA\ndashboard\ncursor\ncontainer\ncomponent\ncompact\ncolors\ncollapse\ncharacters\nch\ncats\ncart\ncalendar\nC\nbrowser\nbrand\nbirthday\nbcc\nattr\napps\nad\nzid\nxajaxargs\nwhich\nwarned\nvenue\nuuid\nusuario\nusesubform\nunique\nundelete\nuids\ntz\ntorrent\ntitles\ntemplates\ntemplatename\ntargetid\nTableList\nsyear\nsvg\nsuser\nsuffix\nsubtotal\nsubmitorderby\nsubmitoptions\nState\nstaff\nspecial\nsortBy\nsorder\nsname\nsm\nsitemap\nsiteid\nsimpledb\nsignin\nsidx\nsID\nShowFunctionFields\nshoutbox\nsec\nsample\nrevokeall\nresume\nresetpasskey\nregenerate\nrecursive\nrecover\nrecipients\nreceipt\nquota\nquiet\nqueue\npublisher\nprogress\nprogram\nproblem\npostsperpage\npostId\npollid\nplaylist\npaymentAmount\npassphrase\npagetitle\npageSize\npageno\npageID\npadding\notp\nonserver\nobfuscate\nnewvalue\nnewDir\nmongo\nmoderator\nmodal\nmimetype\nmID\nma\nlst\nloop\nlookup\nloggedin\nlastID\nissue\nintro\nin\nidp\nhead\nhandle\ngz\ngroupID\ngift\ngID\nfuncs\nfulltext\nfolderid\nflags\nfill\nfieldname\nfeedurl\nfeeds\nerrors\nentries\nelastic\ndontlimitchars\ndonor\ndob\ndisplayname\ndisp\ndes\ndepartment\ndelmarked\ndbusername\ndbstats\ndateformat\ncrypt\ncredit\ncreateview\ncpu\ncover\ncoppa\ncontentType\ncomplete\nComments\ncommentid\ncID\ncatorder\nbook\nauthkey\nattach\narticles\nappname\nappid\nappend\nand\nanalyze\nagreed\nagent\nadress\nadminmail\naddfolder\naddcomment\naccountid\ny2\nx2\nWriteTags\nwith\nwipe\nwhy\nwctx\nvp\nvideoType\nvcode\nvbrmethod\nuserrole\nuserpass\nUsername\nuseremail\nuserdata\nunsynchronizedtags\nunstick\nunsecuresubmit\nunbookmark\nua\ntyp\ntv\ntree\ntransfer\ntrackzero\nTracksTotal\ntracknoalbum\ntrackinalbum\nTrack\ntrace\ntot\ntorrentid\nToolbar\nTOKEN\ntodate\ntitlefeat\ntipo\nthumbs\ntel\ntc\ntagtypes\ntagname\nTagFormatsToWrite\nsynchronizetagsfrom\nsum\nsubdomain\nstype\nstub\nstruct\nstock\nstick\nstatic\nsrv\nsplit\nsp\nsn\nsmtp\nsku\nSkin\nsignout\nshowwysiwyg\nshowtagfiles\nShowMD5\nshowfiles\nshadow\nselector\nsecuresubmit\nsearchtext\nsearchKey\nsavemode\nsaveid\nsaveField\nSAMLResponse\nsamemix\nrpp\nrolename\nrights\nreturnURL\nreturnurl\nrestrict\nresolve\nrescanerrors\nreorder\nrenamefileto\nreminder\nrememberMe\nrelative\nrecent\nrealName\nradio\nquickmod\nqa\npw2\npsubmit\nproperties\nprojects\nproceed\nprivacy\npretty\npname\nphase\npersistent\npermanent\npercent\npay\nPASSWORD\npasswd2\npartial\npaid\norderId\noID\nnpassword\nnotmodrewrite\nnotapache\nnonemptycomments\nnoalert\nnewUser\nnewscan\nnewpw\nnewpass2\nnewpage\nnewfile\nmsgid\nmrpage\nmore\nmoney\nmoduleName\nmlpage\nmkdir\nmissingtrackvolume\nminutes\nminor\nmensaje\nmd5datadupes\nmanager\nm3utitle\nm3ufilename\nm3uartist\nm3u\nlongurl\nlogs\nLogin\nln\nlists\nlistid\nlistdirectory\nlinktype\nlines\nlike\nlib\nKEY\nitemType\nitemId\nisAjax\nint\ninitial\ngrp\ngroupName\nGenreOther\ngenredistribution\nGenre\nfullfolder\nframed\nformName\nformid\nformatdistribution\nfoldmenu\nflip\nfixid3v1padding\nfiletypelist\nfilesize\nfilenamepattern\nfilelist\nfileextensions\nfieldValue\nfieldName\nfieldid\nfID\nfeid\nextended\nextAction\nexisting\nex\nevents\neventName\nerrorswarnings\nencoderoptionsdistribution\nencodedbydistribution\nemptygenres\nemailAddress\nemailaddress\nedituser\ndp\ndisplayName\ndisallow\ndirs\ndictionary\ndeleteid\ndefaultValue\ndeadfilescheck\ndeactivated\ndd\ndbType\ndates\nctf\ncreatedb\nCountry\ncorrectcase\ncopied\ncookies\nconvert\ncontactname\nconfirmPassword\nconfiguration\ncondition\ncluster\nCKFinderFuncNum\nCKFinderCommand\nchmod\nchildren\nchat\ncep\ncd\ncb\ncatname\ncatID\nCardType\ncaching\nbookmark\nbodytext\nbgcolor\nbaseurl\nbar\nautofixforcesource\nautofixforcedest\nautofix\nauthtype\naudiobitrates\nassignment\nartisttitledupes\napplication\nAPICpictureType\nans\nannounce\nanchor\namt\nalways\nadv\naddusers\naccessType\ny1\nxrds\nx1\nwrap\nwork\nway\nwarning\nvotes\nvn\nviews\nvideoid\nverifypeer\nverifyhost\nvendor\nvarValue\nvarName\nvariant\nvariable\nutmr\nutmp\nutmdebug\nutmac\nuses\nuserEmail\nuse\nuporder\nupdatedb\nunbansubmit\nult\nul2\nul\nUA\nu2\nu1\ntype2\ntxtDescription\ntransaction\ntracker\ntos\ntorrentsperpage\ntopicsperpage\ntoboard\nTitle\ntimeframe\ntID\ntextarea\ntesting\ntestemail\ntbl\ntasks\ntaglist\nTag\ntableprefix\ntableId\nt2\nt1\nsurvey\nsurname\nsupportfor\nsubtab\nsubscription\nsubmit1\nsubj\nstyles\nstoryid\nstep1\nstay\nStatus\nstart2\nstandard\nspan\nso\nsmtpPort\nsmiley\nslogan\nslide\nsitetitle\nsignatures\nSID\nshowqueries\nshowpage\nshout\nsha1\nsf\nseverity\nsesskey\nsessidpass\nseries\nsectionid\nsearchText\nsearchid\nsearchField\nsdb\nsday\nscheme\nscene\nscenario\nsavesettings\nsavepms\nsavefile\nsaveData\nSave\nsandbox\nrotatefile\nrotate\nroleid\nrn\nrevoke\nreturnID\nresync\nrestock\nresolution\nresizetype\nresizefile\nresetkey\nresend\nrequestid\nreportid\nrenamefile\nrenameext\nremoveall\nrelease\nrelation\nrecurring\nRecordingUrl\nrecordid\nreasontype\nrace\nqs\npush\npub\nprovince\nprotection\nproperty\npref\npredefined\npp\nplay\nplan\npl\nping\npf\npermerror\npassw\nPASS\nPaRes\nparameter\norganization\norg\norderBy\nonline\noldusername\noldpwd\nolder\nobjects\nnowarn\nnotification\nnewpw2\nNEWPASS\nnewlang\nnav\nmyEditor\nmodname\nmodeextension\nmodcomment\nmetric\nmemberName\nmaxwidth\nmatchtype\nmapping\nmandatory\nls2\nlocal\nlightbox\nlevels\nlangID\nL\nkick\nkarma\nj\nItemid\nisDuplicate\niphone\nipexclude\ninvitecode\ninv\ninterests\ninterest\nins\ninputH\nindustry\nincldead\nimportance\nimgurl\nimgpath\nIMG\nimageid\nident\nid1\nId\nicq\nhref\nhostid\nhl\nhit\nheadline\nheading\nHeaderHexBytes\ngoodfiles\nGenerate\nft\nfragment\nforumid\nforeign\nfollowup\nfm\nfldr\nfileType\nfiletotal\nfileID\nfg\nfCancel\nfacebook\nextUpload\nextTID\nextMethod\nexpiry\nexample\nerrorCode\neol\nentityid\nencoded\nemphasis\nemailnotif\nelements\nedition\nediting\neditfile\neditaction\ndupfiles\ndonated\ndoinstall\ndocid\ndlt\ndl2\ndirect\ndip\nDigits\ndict\ndelid\ndeletepms\ndeleteImage\ndecoded\ndatetime\ndateStart\ndateEnd\ndate2\ndatatype\ncut\ncurrencyCodeType\nct\ncsrf\ncs\ncPath\ncourses\ncoupon\ncontrollers\ncontent1\ncontacts\ncontactid\nconn\ncommentId\ncod\ncm\nclientid\nclearLogs\nclassification\nchosen\nchannelmode\nchanid\nchan\nCategory\ncampaign\ncallerid\ncaller\ncached\nbulk\nbucket\nboards\nblogusers\nblogs\nbilling\nbID\nbib\nbbconfigloc\nbase64\nbansubmit\nbadfiles\nauthorID\nattempt\narguments\nanon\nangle\nalpha\nalert\nalbumid\nageverify\nagb\nafilter\nadminpassword\nadminid\nadminemail\nAddAuthItemForm\nactivation\nactionfile\nAction\nacceptpms\naccepted\nabstract\nabort\na2\nzoneid\nyoutube\nyourname\nwwname\nwmax\nwiki\nwidgets\nWidget\nwhitelist\nwait\nvoucher\nvol\nvl\nvisualizationSettings\nviewName\nviewname\nvia\nVersion\nvarname\nvariables\nvalidator\nvalid\nutype\nutf8\nusort\nUsers\nUSERNAME\nurl1\nURL\nuploadpos\nUpload\nUpdate\nupc\nuntil\nunset\nunselectall\nunpublished\nundo\nu9\nu8\nu7\nu6\nu50\nu5\nu49\nu48\nu47\nu46\nu45\nu44\nu43\nu42\nu41\nu40\nu4\nu39\nu38\nu37\nu36\nu35\nu34\nu33\nu32\nu31\nu30\nu3\nu29\nu28\nu27\nu26\nu25\nu24\nu23\nu22\nu21\nu20\nu19\nu18\nu17\nu16\nu15\nu14\nu13\nu12\nu11\nu10\ntxtEmail\ntrid\ntransactionID\ntrackusers\ntotalProductCount\ntopicID\ntokens\ntimes\ntimer\ntimelimit\nthumbnails\nthrottle\nthemename\ntestmethods\ntaskid\ntargetboard\ntac\ntableFields\ntabid\nsys\nsy\nsuspend\nsupplierID\nsubwdata\nsuburb\nsubstruc\nsubstep\nsubmit2\nsublogin\nsubjoin\nsubconst\nsubcat\nsubacc\nstudent\nSTRUCTURE\nstructure\nstrReferrer\nstrProfileData\nstrId\nstrFormId\nstream\nsteps\nstdDateFilterField\nstdDateFilter\nstation\nstartTime\nstartday\nsserver\nsquare\nsqlquery\nsq\nspass\nsound\nsortKey\nsortfield\nsortDir\nsort2\nsong\nsmonth\nskype\nsingleout\nsignup\nSignatureValue\nSignature\nshowtemplate\nshowSource\nShowFieldTypesInDataEditView\nshowAll\nshortname\nshop\nship\nsearchType\nsearchterm\nsearchbox\nsearchaction\nsearchable\nschool\nsaveToFile\nrunQuery\nruleid\nrp\nround\nRole\nrmFiles\nrm\nrID\nresponsecompression\nReset\nrequiredData\nrequestKey\nrequestcompression\nrepopulate\nremoveVariables\nremoveID\nremoveid\nremoveAll\nremark\nrelmodule\nRelayState\nregSubmit\nRegisterForm\nrefid\nreferral\nrecords\nrec\nreboot\nrc\nratio\nratings\nr1\nquick\nquest\nqueryPart\nqtype\nqr\npurpose\npto\nproxypwd\nproxyport\nproto\npromote\nprobe\nPRIVILEGES\nprintview\npreviewwrite\npressthis\nprenom\nposttext\npop\npoint\npms\npmnotif\nplus\npkg\nphpMyAdmin\nphonenumber\nphone2\nphone1\npfrom\npaypal\npaste\npasswrd\npasswordConfirm\npassword3\npartner\nparked\nparenttab\nParentID\nparam1\npanel\npageTitle\nPAGE\nPage\npack\np2ajax\nOutSum\nOUTPUTFILETEXT\nOUTPUT\norderNo\nor\noptimize\noldname\noffline\nocc\nnpw\nnp\nnowarned\nnombre\nnn\nnID\nnewuseremail\nnewtitle\nnewtext\nnewtag\nnewstatus\nnewpwd\nNEWPRIVILEGES\nnewpassword2\nnewPass2\nnewpass1\nnewPass\nNEWNAME\nNEWHOST\nnewdid\nNEWCHOICE\nnb\nname1\nNAME\nmytribe\nmtime\nmp\nmovie\nmovefile\nmood\nmonths\nmonitorconfig\nmodifier\nmodid\nmirror\nmhpw\nmetrics\nmethodpayload\nmembername\nmemberID\nmembergroups\nmediaid\nmaxtime\nmarkread\nmarkdown\nmailto\nmailSubject\nmailid\nlongtitle\nlogoff\nloginguest\nlogid\nlocations\nlocationName\nlistPrice\nlinkname\nlimitTypes\nlim\nlID\nlegend\nleap\nlead\nlcwidget\nlatest\nlanguageID\nlabelName\nkeystring\nkeepHTML\nkeep\nkeepalive\nItemId\nitemID\nitemCode\nipp\nIP\ninvoiceid\nInvId\nintTimestamp\nintDatabaseIndex\ninstitution\ninstallmode\ninst\nINSERTTYPE\ninitdb\nINDEXTYPE\nINDEXCOLUMNLIST\nimaptest\nIGNOREFIRST\nif\nidstring\nidlist\nhosts\nHOST\nhdnProductId\ngzip\ngrid\nGRANTOPTION\ngoogle\ngold\ngids\ngetInfos\nGenerateForm\ngenerated\nfullsite\nfrontend\nfromdate\nformSubmit\nFormbuilderTestModel\nFORMAT\nfollow\nfolders\nfolderID\nfoffset\nfocus\nfldName\nfiltertype\nfilterText\nfilterName\nfileFormat\nFields\nFIELDNAMES\nfield2\nfield1\nfee\nf2\nEXPORTTABLE\nexportImages\nEXPORTDB\nexception\nexact\neventID\neval\nendyear\nen\nemail1\nEMAIL\nelementId\neids\neducation\neditParts\nEdit\nec\ndtstart\ndtend\ndownloadpos\ndownloaded\ndname\ndm\ndlconfig\ndistinct\ndisplayVisualization\ndirector\ndirectmode\ndipl\ndifficulty\nDeviceId\ndesign\ndescending\ndesact\ndeluser\nDELIMITER\ndeleteUsers\ndeletefolder\ndeldir\ndecline\ndbms\nDBLIST\ndbase\ndayDelta\ndate1\ndataType\nDATABASE\nd1\ncvv\ncustomers\ncurrentid\ncurr\ncurfile\ncur\nctid\ncredits\ncreateclass\ncr\ncountryName\ncountryCode\ncounter\ncore\ncoords\ncontactName\nconnectt\nconflict\nconfigfile\ncompleted\ncomp\ncommenttext\ncolours\ncolName\nCollectionId\nCmd\nclientcookies\nclickedon\nclicked\ncleanup\nCHOICE\nchartSettings\nchars\ncharge\nchannelName\nchannelID\nchanged\ncf\ncert\ncdone\ncatId\ncard\ncanvas\ncampaignid\ncal\ncainfo\nbuild\nbtn\nbreakdown\nborder\nbool\nblocks\nblockid\nblacklist\nbirthDate\nbinary\nbi\nbbox\nbanreason\nbank\nbandwidth\nbackend\nautodeltime\nautodel\nautocomplete\nauthorName\nauthorized\nAuthItem\nAuthChildForm\natype\nAttachmentName\nAssignmentForm\nArtist\nArticle\naoe\nallrows\nalli2\nallDay\nakey\najxaction\najaxRequest\naggregate\nadminpwd\nadmid\naddon\nadditional\nADAPTER\nACTION\nACCESSLEVEL\na1\n3\n1\n\npng\nob\nmaxdays\naliases\nSHIPTOZIP\nSHIPTOSTATE\nSHIPTOCOUNTRY\nSHIPTOCITY\nDelete\nAddress\nzID\nyeniyer\nww\nwser\nwq\nwdir\nvpn\nvoting\nviewscount\nverified\nvPath\nux\nut\nusrid\nuserspec\nuserpicpersonal\nusefilename\nurldown\nuptime\nuploadloc\nupfile\nty\ntradercap\ntodoAction\ntoaddress\ntoAdd\ntmp\ntickets\ntemplateID\ntarfile\nsv\nsubmitcollation\nstep4\nstep3\nsrcport\nsqlf\nshortcut\nseqnum\nsearchlabel\nsearchip\nsearchClause2\nsearchClause\nscheduled\nsameall\nrw\nrto\nrmdir\nreveal\nresetVoteCount\nrenamefolder\nremoteserver\nregval\nregtype\nregname\nregistre\nredirection\nreadregname\nqaction\npu\nprog\nprepare\npreference\nprecmd\npower\npostgroup\npostRedirect\npool\npmsg\npipi\npids\nphpvarname\nphpexec\nphpev\npasswrd2\npasswrd1\npa\nox\novermodsecurity\norderdir\norderByColumn\nonserverover\noldpasswrd\noldemail\nobgz\nnewver\nnewdirectory\nnetmask\nnere\nmysqlpass\nmx\nmsgs\nmquery\nmoderators\nmkfile\nmissing\nmip\nminage\nmenuHashes\nmem\nmbname\nmaxPlotLimit\nmass\nlngfile\nldap\nkind\njump\nit\nispublic\nipaddr\ninside\nimmediate\nimagesize\niStart\niLength\niColumns\nhp\nhname\nguestname\ngf\ngetfile\ngeneralgroup\nfromname\nfixErrors\nfinished\nfilterCategory\nfilterAlert\nfileperm\nfileact\nfedit\nfdownload\nfdelete\nfchmod\nfallback\neventDate\nerorr\nephp\nep\nenv\nenquiry\nemailto\nemailActivate\neheight\nef\neditform\neditfilename\ned\ndup\ndstport\ndosyaa\ndontFormat\ndolma\ndoi\ndisplayAllColumns\ndirupload\ndif\ndelregname\ndelim\ndeleteuser\ndeleteAccount\ndc\ndbu\ndbsession\ndbp\ndbh\ndateFormat\ndataLabel\ncy\ncustomerid\ncustomWhereClause\ncurl\ncurdir\ncriteriaValues\ncriteriaTables\ncriteriaSort\ncriteriaShow\ncriteriaSearchType\ncriteriaSearchString\ncriteriaRowInsert\ncriteriaRowDelete\ncriteriaRowAdd\ncriteriaColumnTypes\ncriteriaColumnOperators\ncriteriaColumnNames\ncriteriaColumnName\ncriteriaColumnInsert\ncriteriaColumnDelete\ncriteriaColumnCount\ncriteriaColumnCollations\ncriteriaColumnAdd\ncriteriaColumn\ncriteriaAndOrRow\ncriteriaAndOrColumn\ncreatefolder\ncpy\ncoppaPost\ncoppaFax\ncoord\ncookiename\ncookielength\ncontactId\ncon\ncommunity\ncolumnsToDisplay\ncn\ncl\nchmod0\nchecksum\nchangeusername\ncertificate\ncensortext\ncensortest\ncensorWholeWord\ncensorIgnoreCase\ncalname\ncalid\nc99shcook\nbug\nbrd\nbport\nboardurl\nboardid\nboardaccess\nbgc\nbday2\nbackuptype\nbackconnectport\nbackcconnmsge\nbackcconnmsg\nappId\nanimate\nallday\nactionfolder\naclid\nabsolute\naPath\nTYPE\nSHIPTOSTREET\nProfileForm\nMohajer22\nMD\nM2\nF\nER\nDirection\nCURRENCYCODE\nA\nzrecord\nzpage\nzonetxt\nzonet\nzonesub\nyearend\nyPath\nxsrf\nwstype\nwoeid\nweekdays\nwebid\nwatermark\nvv\nvpassword\nviewed\nviewall\nviewUsers\nviewResults\nviewOption\nver2\nver1\nvariations\nusertype\nuserlength\nuserip\nusergroup\nuserGroup\nuserEnableRecovery\nusepost\nused\nupsql\nuploadfile\nuploadForm\nupdateRecordID\nupdateFileID\nupdateData\nupdateBiblioID\nupd\nupage\nunzip\nuntilDate\nunstable\nunhideNavItem\nuitype\nue\ntypE\ntxtCommand\ntxtAddComment\ntvid\ntt\ntransactionId\ntransStatus\ntransId\ntpp\ntp\ntotaltopics\ntopicseen\ntools\ntoolbar\ntok\ntimezonedetection\ntimeUnit\ntimeIncrement\nti\nthreshold\nthankyou\ntftp\ntfid\ntests\ntestmode\ntempLoanID\nte\ntaxid\ntagvalue\ntabs\nsync\nsymlinktarget\nsymlink\nsupplierPlace\nsupplierPhone\nsupplierName\nsupplierFax\nsupplierEmail\nsupplierContact\nsupplierAccount\nsubsection\nsubscribed\nsubs\nsubmitok\nsubjectType\nsubid\nsubfiles\nsubdom\nsubcategory\nsubact\nstrategy\nstrHtml\nstory\nstories\nstatusID\nstates\nstartval\nstarts\nstars\nstar\nstUpload\nssi\nsshport\nssearch\nsqluser\nsqlpass\nsqlhost\nspoiler\nspecialchars\nspecDetailInfo\nspage\nsmtpusername\nsmtpport\nsmtppassword\nsmodule\nsl\nskid\nsiteName\nshowsc\nshown\nshowh\nshowevent\nshowdupes\nshowUnhideDialog\nshowCheckbox\nshared\nshareWith\nshareType\nsetMetrics\nsetDefault\nsessionId\nsesc\nservices\nserverurl\nservertype\nservers\nserverid\nserveR\nseriesTitle\nserialID\nseqNumber\nseq\nseparate\nselectedDoc\nsecurity\nsect\nsearchin\nsearchby\nsearchString\nsearchName\nsearchId\nsea\nscid\nscdir\nscalingup\nsavemsg\nsaveandnext\nsaveZ\nsaveNclose\nsaveLogs\nsaveKardexes\nsalesrank\nsaction\nruncmd\nruletype\nruledefgroup\nruledef\nrssfeed\nrowspage\nrownumber\nrowid\nroutines\nroutes\nrmver\nrminstall\nreturnaction\nresultXML\nreshares\nresetpassword\nreserved\nreserveLimit\nreserveItemID\nreserveID\nreserveAlert\nresent\nrequireAgreement\nreqType\nreportsent\nreports\nreportView\nreportContentType\nreplies\nreplaceWith\nrepeatable\nren\nremovesess\nremoveFines\nremotefile\nremipp\nremail\nrelpathinfo\nreleasedate\nrelatedmodule\nregularity\nregexp\nregDate\nrefurl\nrecvDate\nrecsEachPage\nrecoveryPassword\nrecordSep\nrecordOffset\nrecordNum\nrecaptcha\nrecapBy\nreborrowLimit\nready\nrback\nrawfilter\nranking\nragename\nrage\nr4\nquirks\nquickReturnID\nquestionid\nquerY\nqt\nqindsub\nqcontent\nqact2\nqact\npublisherName\npublisherID\npublicUpload\nptype\nptID\npt\npruningOptions\nproxypass\nproxyhostmsg\nprotect\nprop\nprojectid\nprojectID\nprogresskey\nprofiles\nproducttype\nprocessed\npro\npriceCurrency\npr\npostto\npostgroups\npostfrom\npostal\nportalauth\npopuptype\npod\nplug\nplain\nplaceName\nplaceID\npipe\nphpini\nphpcode\npftext\npersonal\npd\npb\npaymentStatus\npause\npasswords\npasswd1\npasslength\npassWord\npasS\nparentId\npalette\npais\npageId\npackageName\noverrideID\noutbox\not\nordDate\noptimization\nopml\noperations\nopacHide\noldform\noldfilename\noff\noauth\nnzbpath\nnumbers\nnumExtended\nnull\nntp2\nntp1\nnoupdate\nnotsent\nnotificationType\nnotificationCode\nnoteid\nnotdeleted\nnotactivated\nnoredirect\nnoChangeGroup\nnfid\nnf\nnewowner\nnewgroupname\nnewf\nnewer\nnewemail\nnewdb\nnewWidth\nnewPassword2\nnewLoanDate\nnewHeight\nnewDueDate\nnewDirectory\nnentries\nmyip\nmsgfield\nms\nmovieview\nmountType\nmountPoint\nmodulename\nmoduleid\nmodulePath\nmoduleDesc\nmodifiedSince\nmisc\nminuteDelta\nminus\nmins\nminimum\nmini\nmicrohistory\nmethodsig\nmemory\nmemberTypeName\nmemberTypeID\nmemberPostal\nmemberPhone\nmemberPeriode\nmemberPIN\nmemberNotes\nmemberFax\nmemberEmail\nmemberAddress\nme\nmd5sum\nmd5sig\nmaxentries\nmaxUploadSize\nmatchword\nmatchuser\nmatchname\nmatchcase\nmassupload\nmarked\nmakenote\nmakedir\nmailtxt\nmailsub\nmailing\nmagic\nlogging\nlogfile\nlogdefaultblock\nlogMeIn\nlocationID\nloanStatus\nloanSessionID\nloanPeriode\nloanLimit\nloanID\nlistprice\nlistname\nlisting\nlistarea\nlistShow\nlink2\nlineid\nlifetime\nlibrary\nlen\nleave\nlayoutType\nlayers\nlasturl\nlastmodified\nlastid\nlastQueryStr\nlanguagePrefix\nlangName\nlabelDesc\nlabdef\nkw\nkstart\nkeyname\nkeydata\nkey2\nkey1\nkb\nk2\njupart\njufinal\njoindate\niv\nitemname\nitemStatusID\nitemStatus\nitemSourceName\nitemSource\nitemSite\nitemShares\nitemCollID\nitemAction\niso\nisdescending\nisPersonal\nisPending\ninvitepage\ninverse\ninventoryCode\ninvcDate\ninstallpath\ninstalled\ninstalldata\ninstallbind\ninstName\ninputSearchVal\ninheritperm\ninherit\nindxtxt\nindx\nincspeed\ninXML\ninUsername\ninPopUp\ninPassword\ninNewPass\nimdbid\nimdb\nie\nidtype\nidc\nhtaccess\nhot\nholiday\nholDesc\nholDateEnd\nholDate\nhideNavItem\nhex\nheaders\nharm\nharddiskstandby\ngx\nguest\ngtype\ngrouptype\ngroupreason\ngroupr\ngroupfilter\ngraphid\ngracePeriode\ngrabs\ngpack\ngoogleplus\ngmdName\ngmdID\ngmdCode\ngmd\ngiveout\ngetupdatestatus\ngetstatus\ngetprogress\ngetactivity\ngetDropdownValues\ngeoOption\ngeneric\ngen\ngameid\nfu\nftpuser\nfstype\nfront\nfromsearch\nfromemail\nfrequencyName\nfrequencyID\nfree\nfp\nforgot\nforeignTable\nforeignDb\nforceRefresh\nfolderpath\nflow\nfldname\nfldlength\nfldlabel\nflddecimal\nfldType\nfldPickList\nfldLength\nfldLabel\nfldDecimal\nfix\nfirstday\nfinishID\nfinesDesc\nfinesDate\nfineEachDay\nfindString\nfileurl\nfileto\nfileold\nfilenew\nfilename2\nfilefrom\nfileframe\nfilecontents\nfileURL\nfileTitle\nfileDir\nfileDesc\nfieldlabel\nfieldType\nfieldSep\nfieldId\nfieldEnc\nfh\nffile\nfavicon\nfam\nexternal\nextensions\nexponent\nexpirationyear\nexpirationmonth\nexpDateYear\nexpDateMonth\nexpDate\nexemplar\nexe\nexccat\nevtitle\neta\nerrorstr\nerrormsg\nerrormail\nerrmsg\nenroll\nends\nendday\nencryption\nencrypted\nencrypt\nenclose\nenableReserve\nemailcomplete\nemailId\neditf\neditable\neditUserGroupSubmit\neditUserGroup\neday\necotax\ndwld\ndue\ndto\ndos\ndocumentID\ndoaction\ndoSearch\ndoImport\ndoExport\ndnssec\ndns2\ndns1\ndn\ndmodule\ndisk\ndisablelocallogging\ndisabledBBC\ndis\ndirToken\ndim\ndigest\ndialog\ndhcp\ndfrom\ndf\ndepts\ndemolish\ndelsub\ndelrule\ndelrow\ndelgroup\ndeletesmiley\ndeleteip\ndeleteevent\ndeletecheck\ndeleteUserGroup\ndebet\ndbserver\ndbpw\ndbid\ndbPrefix\ndbPort\ndbHost\ndayname\ndatetype\ndateto\ndatefrom\ndateReceived\ndateExpected\ndataurl\ndataset\ndatadir\ndatabaseloginpassword\ndatabaseloginname\ndatabasehost\ndB\ncw\ncvv2Number\ncvmodule\ncustomfield\ncustid\ncust\ncurrentFolderPath\ncurpage\ncsid\ncrt\ncreditCardType\ncreditCardNumber\ncredentials\ncreatepages\ncreatemode\ncrdir\ncouponamount\ncounts\nconvertmode\nconversation\nconv\ncontest\ncontentTitle\ncontentPath\ncontentDesc\ncontbutt\ncontains\nconsumer\nconstraint\nconsoleview\nconfirmFinish\ncombine\ncolumnIndex\ncolor2\ncolltype\ncollTypeName\ncollTypeID\ncollType\ncodes\ncmspassword\ncmsadminemail\ncmsadmin\ncls\nclientId\ncleared\nclassOptions\nclaim\nchvalue\nchpage\nchkagree\ncheckprivstable\ncheckprivsdb\ncheckout\nchecking\ncheckboxes\ncheckShares\ncheckReshare\ncheck1\nchannels\nchangepassword\nchangecurrent\nchangeUserGroup\ncfgval\ncfgkey\ncategoryID\ncardtype\ncap\ncallbackPW\ncallNumber\ncalendarid\ncalcolor\nbzipcode\nbuddies\nbtnSubmit\nbstate\nbridge\nbreadcrumb\nbphone\nboxes\nbox3\nbox2\nbox1\nbootstrap\nbomb\nboardtheme\nboardseen\nboardprofile\nblocklabel\nblastname\nbits\nbirthyear\nbirthmonth\nbinding\nbill\nbiblioTitle\nbiblioID\nbfirstname\nbeta\nbemail\nbeginner\nbcountry\nbconfirmemail\nbcity\nbbc\nbaza\nbatchID\nbatchExtend\nbasedn\nbaddress2\nbaddress1\nbackupnow\nbackdrop\nbaba\nautoupdate\nautomatic\nauthorityType\nauthPin\nauthList\naudioFolder\nasin\narg\narch\napplicable\nappkey\nappeal\naop\nanimal\naltmethodpayload\nalterview\nalsoDeleteFile\nallsignups\nallflag\nallfiles\nallboards\naliasid\nalgorithm\nafterupload\naemail\nadopt\nadminuser\nadminpass2\nadminEnableRecovery\naddcategory\naddUserGroupSubmit\naddUserGroup\naddSpider\naddReply\naddMessage\naddList\nacttype\nactors\nactionName\nacl\nacct\naccountnumber\naccountname\nabc\naID\nWSDL\nUserChangePassForm\nUID\nTest\nTerm\nTab\nT\nSubmit1\nSettings\nSaveInSent\nSORT\nSHIPTOSTREET2\nReview\nReturnUrl\nRecordingDuration\nProject\nProduct\nPasswordResetForm\nPasswordForm\nOr\nMenuItem\nMenu\nMETHOD\nLanguage\nLOCALECODE\nIssue\nInstallForm\nGroup\nExpirationYear\nExpirationMonth\nERORR\nDialCallStatus\nDeviceType\nDATE\nD\nCondition\nCallSid\nCVV\nB\nAudioPlayerSubmit\nAudioPlayerReset\nAccountNumber\nzonefile\nzipName\nzhsd\nyy\nystart\nyellowtemp\nyellowstales\nyellowremfails\nyellowrejects\nyellowgetfails\nyellowgessper\nyellowfan\nyellowdiscards\nyellowavgmhper\nyears\nyahoo\nxxx\nxx\nxtype\nxnum\nxmode\nxmldata\nxjxmthd\nxjxfun\nxjxevt\nxjxcls\nxjxargs\nxjxGenerateStyle\nxjxGenerateJavascript\nxhrLocation\nxhprof\nxdebug\nwu\nwstoken\nwriteSchema\nwresult\nwrcont\nwpseo\nwpnonce\nwpas\nworkingdiR\nworkgroup\nworkflow\nworkerId\nwordlist\nwood\nwlk\nwli\nwithdraw\nwithCount\nwins2\nwins1\nwins\nwildcard\nwikitext\nwide\nwhw\nwhom\nwebsiteId\nwebserver\nwebpage\nwebguiproto\nwebguiport\nwbp\nwbcp\nwarn\nwant\nwakeall\nwa\nvuln\nvrt\nvpntype\nvouchersyncusername\nvouchersyncport\nvouchersyncpass\nvouchersyncdbip\nvouchers\nvolume\nvoid\nvnutr\nvlanprioset\nvlanprio\nvjcomp\nvillagename\nviewweek\nviewupgradelog\nviewscope\nviewMode\nviewBag\nvideos\nvideopress\nvideoTitle\nvideoTags\nvideoId\nvideoDescription\nvideoCategory\nvhostcontainer\nvhid\nvgrlf\nversions\nverse\nverifycode\nverification\nverboselog\nverb\nvecdo\nve\nvcheck\nvbxsite\nvbulletin\nvbss\nvbsq\nvat\nvars\nvariants\nvar2\nvar1\nvalor\nvalidation\nvalidateValue\nvalidateId\nustsub\nustools\nustname\nusrgroups\nusetoken\nusetcp\nuserrealname\nusernamefld\nusername2\nusermail\nuserlogin\nuserlevel\nuserinfo\nuserids\nuserf\nuseraction\nuserPassword\nuserEdit\nuserDialogResult\nuserAgent\nusepublicip\nuseicmp\nusecurl\nuseR\nuscmnds\nurlup\nurltype\nurlf\nurldd0\nurl2\nurL\nupports\nuploadurl\nuploading\nuploadhd\nuploadf\nuploader\nuploaddir\nuploadPath\nupl\nupip\nupin\nupff\nupf\nupdateurl\nupdatempd\nupdateme\nupdateid\nupdatefile\nupdateType\nupdateMsgCount\nupcont\nupcom\nupchange\nunverify\nunscheduled\nunreleased\nunpubdate\nunknown\nunits\nunitprice\nuniqueid\nuniqueID\nundodrag\nunbanreason\nulang\nuk\nuf\nucd\nuback\nuN\nuID\nu1p\ntypeofdata\ntypename\ntypefilter\ntype6\ntype1\ntxtwebemail\ntxtsupport\ntxtUsername\ntxtRecallBuffer\ntxtPHPCommand\ntxtCaptcha\ntxtAddress\ntxpower\ntxkey\ntxantenna\ntvname\ntuser\ntunable\ntribe\ntresc\ntrapstring\ntrapserverport\ntrapserver\ntrappercap\ntrapenable\ntransport\ntransient\ntraffic\ntracks\ntrackback\ntpshcook\ntplName\ntplID\ntown\ntouserid\ntouch\ntotalcount\ntotalTracks\ntotalItems\ntopsearch\ntoppool\ntooltip\ntomod\ntoid\ntoProcess\ntn\ntld\ntitulo\ntitre\ntint\ntimeupdateinterval\ntimeservers\ntimeoffset\ntimeint\ntimedescr\ntimedd0\ntimeFormat\ntile\ntids\nticketid\nticketbits\nthumbWidth\nthumbHeight\nthrowexception\nthreadID\nthisX\nthemeName\ntftpinterface\ntextonly\ntexto\ntextmail\ntextfield\ntextIn\ntext0Name\ntestvar\ntestdbpwd\ntestdb\ntestType\ntestMode\ntestID\ntemplatefile\ntempName\ntemat\nteamid\nteacher\ntdir\ntd\ntcpmssfix\ntcpidletimeout\ntcp\ntbname\ntbls\ntaxtype\ntaxrate\ntaskID\ntargetname\ntargetip\ntagcloudview\ntagId\ntablo\ntableList\ntabla\ntabAction\ntab1\nta\nt3\nsyslocation\nsysevents\nsysemail\nsyscontact\nsyscmd\nsyntax\nsynconupgrade\nsynchronize\nsyncfilter\nsymgo\nsymbol\nsvff\nsvdi\nsupprimer\nsuppr\nsunrise\nsubsubaction\nsubset\nsubscriptionId\nsubscribers\nsubqcmnds\nsubop\nsubnetv6\nsubnet\nsubmode\nsubmitv\nsubmitrobots\nsubmithtaccess\nsubmitf\nsubmitThemes\nsubmitReset\nsubmitFilter\nsubmitFilesAdminSettings\nsubmitEmail\nsubmitAdd\nsubmit4\nsubmit3\nsubmail\nsubjectid\nsubfolder\nsubdomains\nsubcanemaildomain\nsubId\nsubGenre\nstuid\nstuff\nstudents\nstudentidx\nsts\nstrukt\nstringtoh\nstrin\nstrictcn\nstrictbind\nstreamMode\nstp\nstoragegroup\nstoptime\nstoppool\nstoppga\nstopbtn\nstime\nstereo\nstepid\nstep5\nstdlib\nstderr\nstatut\nstatusid\nstatsgraph\nstaticarp\nstatetype\nstatetimeout\nstatetable\nstateid\nstateOrProvinceName\nstartyear\nstartpool\nstartpga\nstartnum\nstartmonth\nstartdisplayingat\nstartbtn\nstartMonth\nstarred\nstamp\nstaffId\nstack\nsshdkeyonly\nsrname\nsrm\nsrctype\nsrctrack\nsrctext\nsrcnot\nsrcmask\nsrch\nsrcfmt\nsrcendport\nsrcbeginposrt\nsrcbeginport\nsr\nsqtid\nsqsrv\nsqquery\nsqpwd\nsqprt\nsqlwxp\nsqluser4\nsqluser3\nsqluser2\nsqluser1\nsqltype\nsqlty\nsqlportb4\nsqlportb3\nsqlportb2\nsqlportb1\nsqlport4\nsqlport3\nsqlport2\nsqlport1\nsqlport\nsqlpass4\nsqlpass3\nsqlpass2\nsqlpass1\nsqlog\nsqlite2\nsqlhost4\nsqlhost3\nsqlhost2\nsqlhost1\nsqlfile\nsqldp\nsqldebug\nsqlcode\nsqlaction\nsqdbn\nsqconf\nspy\nspots\nspot\nsport\nspoofmac\nspellstring\nspelling\nspecs\nspecifiedpassword\nspecialsettings\nspeciallogfile\nspecialFiles\nspammer\nspamcheck\nsourcetracking\nsourceport\nsourceip\nsourcego\nsorttable\nsortname\nsorting\nsortdirection\nsortdir\nsortable\nsortField\nsongid\nsoname\nsomething\nsomestuff\nsome\nsolrsort\nsnn\nsnmpscanner\nsnatched\nsnaplen\nsmtptls\nsmtpssl\nsmtprelay\nsmtpnotifyemailaddress\nsmtpipaddress\nsmtpfromaddress\nsmtpPassword\nsms\nsmode\nsmile\nsmfdbu\nsmfdbp\nsmfdbn\nsmfdbh\nsmf\nsmartpagebreak\nsmartmonemail\nslot\nslid\nskiplang\nskipIOS\nskipANDROID\nskinname\nskinName\nsk\nsjid\nsizey\nsizes\nsitter2\nsitter1\nsitedown\nsiteId\nsimpin\nsilver\nshowthumbs\nshowtext\nshowslow\nshowmessage\nshowinfo\nshowinactive\nshowbd\nshowact\nshowIndex\nshowFooterMessage\nshorturl\nshortseq\nshopping\nshiptobilling\nsharing\nsharednet\nsh311\nsh3\nsfname\nsfldr\nsfilter\nsfilename\nsetype\nsetupid\nsettype\nsettags\nsetrw\nsetoption\nsetname\nsetlanguage\nsetlang\nsetdefault\nsetUserAgent\nsetPublic\nsessions\nsessid\nsess\nservicestatusfilter\nserviceName\nserversdisabled\nserverip\nservercn\nserverId\nserie\nserialspeed\nserialport\nserialize\nserdir\nser\nsentitems\nsenm\nsenha\nsendtime\nsendpassword\nsendmsg\nsendmethod\nsendit\nsendfile\nsenderEmail\nsendemail\nsendactivation\nsendTo\nselyear\nselmonth\nsellernick\nselectvalues\nselectop\nselectlist\nselectedmodule\nselectedTable\nselectcategory\nselectAmount\nsele\nselday\nselCountry\nsegment\nseed\nsedir\nsecurityscanner\nsecu\nsecs\nsecretKey\nsearchval\nsearchuser\nsearchstring\nsearchfield\nsearchadvsizeto\nsearchadvsizefrom\nsearchadvr\nsearchadvposter\nsearchadvgroups\nsearchadvcat\nsearchUsername\nsearchQuery\nsearchOper\nsearcc\nseC\nscrubrnid\nscrubnodf\nscores\nschooldatex\nschedule0\nsched\nscalepoints\nsca\nsbjct\nsavmode\nsavetest\nsaveoptions\nsavehostid\nsavegroup\nsavefolderurl\nsavefolder\nsavefilenameurl\nsavefilename\nsavedraft\nsaveconf\nsavePath\nsaveNedit\nsaveNcreate\nsaveNback\nsat\nsampledata\nsalutation\nsaleprice\nsafemodz\nsafefile\nsafecss\nsafe\nsabsetting\nsabapikeytype\nsYear\nsName\nsColumns\ns3key\ns3bucket\nrxantenna\nrwenable\nrwcommunity\nrvm\nrunsnippet\nrunid\nruner\nrunState\nrtl\nrstarget4\nrstarget3\nrstarget2\nrstarget1\nrsswidgettextlength\nrsswidgetheight\nrssurl\nrssmaxitems\nrrule\nrrdbackup\nrport\nrpassword\nrownum\nrowId\nrouteid\nrootpath\nrollbits\nrocommunity\nrobotsnew\nrname\nrmid\nrichtext\nrfiletxt\nrfile\nrfc959workaround\nrf\nreverseacct\nreturnsession\nreturnpage\nretries\nret\nresultmatch\nresultid\nresubmit\nrestorefile\nrestorearea\nrestartchk\nrespuesta\nresponsive\nresources\nresourcefile\nresidence\nresetwidgets\nresetpass\nresetlogs\nresetlog\nresetPassword\nrescanwifi\nrequests\nreqid\nreqFor\nreq128\nrepwd\nrepositoryurl\nreportname\nreportfun\nreportType\nreplayMode\nrepeatMonth\nrepass\nreopen\nrensub\nrenold\nrennew\nrenf\nrenderlinks\nrenderimages\nrenderforms\nrenderfields\nrender\nrempool\nremovep\nremovemp\nremovefields\nremoveOldVisits\nremoteserver3\nremoteserver2\nremotekey\nremoteip\nremot\nremmin\nremhrs\nremdays\nremarks\nreloadfilter\nrelevance\nrelayd\nrelay\nrelationships\nrelationship\nrelations\nrelated\nrela\nreglocation\nregistered\nreginput\nregid\nregdomain\nregdhcpstatic\nregdhcp\nregcountry\nrefuse\nrefund\nrefuid\nrefreshinterval\nreflectiontimeout\nrefkod\nreferid\nreferer2\nredirurl\nredirectto\nredirectUri\nredfi\nreddi\nrecurse\nrecurringtype\nrecurrence\nrecreate\nrecordsArray\nrecordcount\nrecordType\nrecordID\nreconstruct\nrecommend\nrecipientCurrency\nrecipientAmount\nrecherche\nreceipient\nrecache\nreauthenticateacct\nreauthenticate\nrealpath\nreadonly\nreadme\nreading\nrdata\nrawAuthMessage\nrasamednsasdhcp6\nrapriority\nrandkey\nramode\nrainterface\nradomainsearchlist\nradns2\nradns1\nradiusvendor\nradiusserverport\nradiusserveracctport\nradiusserver2port\nradiusserver2acctport\nradiusserver2\nradiusserver\nradiussecret2\nradiussecret\nradiussecenable\nradiusport4\nradiusport3\nradiusport2\nradiusport\nradiusnasid\nradiuskey4\nradiuskey3\nradiuskey2\nradiuskey\nradiusissueips\nradiusip4\nradiusip3\nradiusip2\nradiusip\nradiusenable\nradiusacctport\nradiobutton\nradPostPage\nrN\nrM\nr3\nr00t\nqx\nquoteid\nqunfatmpname\nquizid\nquitchk\nquietlogin\nquickmanagertv\nquickmanagerclose\nquickmanager\nquicklogin\nquestions\nquerytype\nquerysql\nqueryString\nquantityBackup\nqu\nqtranslateincompatiblemessage\nqsubject\nqqfafile\nqmrefresh\nqact3\nq3\nq2\npurgedb\npuremode\npurchaseorderid\npurchaseid\npublickey\npubkey\npubdate\npuT\nptpid\nptp\npsk\npsid\npseudo\nps2pdf\nprv\nproxyusername\nproxyurl\nproxypassword\nproxyhost\nprov\nprotocomp\nprotmode\npromiscuous\nprojectionxy\nprojection\nprofileId\nprof\nproductname\nproductlist\nproductcode\nproductDescription\nprocesslogin\nprocesslist\nprocessing\nprocedure\nprobability\nprj\nprivid\nprivatekey\npriv\npriority3\npriority2\npriority1\nprio\nprinter\nprincipal\nprimarymodule\nprimaryconsole\nprices\npri\nprevpage\npress\npresence\nprescription\nprereq\npreg\nprefork\nprefetchkey\nprefetch\npreauthurl\nppsstratum\nppsselect\nppsrefid\nppsport\nppsfudge1\nppsflag4\nppsflag3\nppsflag2\npppoeid\nppid\nppdebug\nppage\npotentialid\npotentalid\nposttype\npostedText\npostback\npostafterlogin\npostData\npost2\npost1\nportscanner\nportbw\nportbl\nportbc\nport1\nporder\npop3host\npools\npoolopts\npoolname\npollvote\npollport\npollQuestion\npollOptions\nplusminus\nplname\nplid\nplaylistTitle\nplaylistDescription\nplaintext\npkgs\npkgrepourl\npinned\npictitle\npics\npickfieldtable\npickfieldname\npickfieldlabel\npickfieldcolname\npick\npiasS\npi\nphpsettings\nphpsettingid\nphpenabled\nphpbbkat\nphpbbdbu\nphpbbdbp\nphpbbdbn\nphpbbdbh\nphpbb\nphotoid\nphoneNr\nphone3\npguser\npgsqlcon\npgport\npgdb\nperuserbw\npersonality\npersonId\npersistcommonwireless\npersist\nperms9\nperms8\nperms7\nperms6\nperms5\nperms4\nperms3\nperms2\nperms1\nperms0\npermStatus\nperiodo\nperiodidx\nperform\nperPage\npeerstats\npeace\npdouser\npdopass\npdodsn\npdocon\npdnpipe\npcid\npaypalListener\npayments\npaymentId\npaymentData\npaths\npathf\npath2news\npatch\npasswordnotifymethod\npasswordkey\npasswordgenmethod\npasswordfld2\npasswordfld1\npasswordfld\npasswordconfirm\npasswordc\npasswdList\npassthrumacaddusername\npassthrumacadd\npassgen\npassf\npassenger\npassd\npartition\nparseSchema\nparid\nparentqueue\nparentfieldid\nparanoia\npaporchap\npagestyle\npagestart\npageop\npagenumber\npageborder\npageType\npageOwner\npadID\npaID\npW\npUID\npPassConf\npPass\npPage\npName\npMail\npDesc\np3\np2p\np2index\np2entry\np1index\np1entry\noverwriteconfigxml\noverdue\nouT\nostlang\norionprofile\norigname\norganizationalUnitName\norganizationName\norgajax\norders\nordering\norderType\norauser\noraclecon\nopwd\noptin\noptimizer\noper\nopenings\nopened\nopenbasedir\noof\nonw\nonlyforuser\nonlyfind\nondemand\non\noldtime\nolddir\noldaction\noldPlaylistTitle\noldMountPoint\noldEmail\nodbcuser\nodbcpass\nodbcdsn\nodbccon\nodb\noccupation\nobjectIDs\nobj\nnurlen\nnurld\nnumwant\nnumlabel\nnumberposts\nnumail\nnuked\nnuf\nntporphan\nnslookup\nnrresults\nnpassworda\nnpage\nnoxml\nnowmodule\nnounce\nnotrap\nnotices\nnoti\nnot\nnosync\nnoserve\nnoreload\nnordr\nnoquery\nnopfsync\nnopeer\nnopackages\nnoofrows\nnone\nnonat\nnomodify\nnometool\nnome\nnomacfilter\nnolimit\nnolang\nnohttpsforwards\nnohttpreferercheck\nnohtml\nnogrants\nnoexpand\nnoedit\nnodraft\nnodnsrebindcheck\nnodeid\nnoconcurrentlogins\nnoantilockout\nnoaction\nnoRedirect\nnoOfBytes\nnmdf\nnfile\nnf4cs\nnf4c\nnf1\nnextserver\nnextid\nnextPage\nnewwin\nnewusername\nnewusergroup\nnewtype\nnewtime\nnewtheme\nnewtemplate\nnewrule\nnewprefix\nnewpref\nnewpath\nnewnick\nnewmessage\nnewids\nnewid\nnewgroup\nnewdocgroup\nnewcode\nnewcategory\nnewalbum\nnewaccount\nnewX10Monitor\nnewWindow\nnewVideoTitle\nnewVideoTags\nnewVideoDescription\nnewVideoCategory\nnewValue\nnewText\nnewSite\nnewProject\nnewPlaylistTitle\nnewPlaylistDescription\nnewPath\nnewMonitor\nnewGroup\nnewGame\nnewControl\nnetgraph\nnetboot\nnested\nneg\nncbase\nnc\nnatreflection\nnatport\nnameren\nnamelist\nnamefe\nname3\nnamE\nn1\nmyusername\nmysqls\nmysqlcon\nmypassword\nmyname\nmylogout\nmycode\nmybulletin\nmybbindex\nmybbdbu\nmybbdbp\nmybbdbn\nmybbdbh\nmw\nmve\nmvdi\nmute\nmusic\nmuser\nmultiplier\nmultifieldname\nmultifieldid\nmtype\nmtu\nmto\nmtext\nmsubj\nmssqlcon\nmss\nmsqur\nmsq1\nmsid\nmsi\nmsgtype\nmsgnoaccess\nmsgno\nmsgexpired\nmsgcachesize\nmsg1\nmru\nmpdconf\nmpath\nmpage\nmoveup\nmoveto\nmovedown\nmovd\nmount\nmotivo\nmotd\nmoodlewsrestformat\nmon\nmoduletype\nmoduleorder\nmoduleguid\nmoduleType\nmoduleId\nmodified\nmodfunc\nmodfile\nmoderate\nmodelId\nmodeid\nmodcat\nmodE\nmobj\nmobilephone\nmnam\nmmsg\nmmail\nmlist\nml\nmkF\nmkD\nmito\nminkills\nminViewability\nminJs\nminCss\nmimetypes\nmilw0\nmids\nmibii\nmhtc\nmhost\nmhash\nmh\nmg\nmfrom\nmfldr\nmffw\nmetadata\nmessagesubject\nmessageid\nmessagebody\nmessageMultiplier\nmess\nmeridiem\nmergefile\nmerchantReference\nmenutitle\nmenus\nmenuindex\nmenuid\nmemtype\nmemday944\nmemday942\nmemberPasswd2\nmemberPasswd\nmemberPassWord\nmediatype\nmediaopt\nmedalweek\nmedalid\nmdp\nmd5q\nmd5pass\nmd5hash\nmd5crack\nmd\nmcid\nmc\nmbox\nmbadmin\nmaxtry\nmaxtemp\nmaxstore\nmaxstales\nmaxremfails\nmaxrejects\nmaxprocperip\nmaxproc\nmaxmss\nmaxleasetime\nmaximumtableentries\nmaximumstates\nmaxgetfails\nmaxgessper\nmaxfan\nmaxdiscards\nmaxcrop\nmaxaddr\nmaxZipInputSize\nmaster\nmasssource\nmassedit\nmassdefaceurl\nmassdefacedir\nmassa\nmasdr\nmarker\nmarkdefault\nmanual\nmanagerlanguage\nmanage\nman\nmakeupdate\nmaintitle\nmaint\nmainmessage\nmainGenre\nmailsent\nmaillisttmpname\nmailcontent\nmailbodyid\nmailbody\nmailMethod\nmailAuth\nmagicfields\nmacname\nmV\nmSendm\nmKf\nmKd\nmD\nlucky\nlticket\nlp\nlosslow\nlosshigh\nloopstats\nlookfornewversion\nlonglastingsession\nlogsys\nlogprivatenets\nlogpeer\nlogoutid\nloglighttpd\nloglevel\nloginmessage\nloginemail\nloginautocomplete\nlogic\nloggedAt\nlogfilesize\nlogfilE\nlogf\nlogeraser\nlogdefaultpass\nlogbogons\nlogall\nlogable\nlogType\nlogFile\nlockid\nlocationid\nlocalized\nlocalityName\nlocalip\nlocalfile\nlocalf\nlocalbeginport\nloan\nlm\nlive\nlistorder\nlistmode\nliste2\nliste1\nliste\nlistSubmitted\nlistItem\nlistId\nlinkedin\nlink1\nlink0\nlimitpage\nlimitless\nlimite\nliked\nlfilename\nlemail\nlegendstyle\nlegendsize\nlegendfontsize\nlegendfontr\nlegendfontg\nlegendfontb\nlegendfont\nlecture\nleaptxt\nleadval\nleadsource\nlbg\nlbcp\nlatencylow\nlatencyhigh\nlastactive\nlastActive\nlangs\nlangname\nlanes\nlane\nlandscape\nlan\nlaggif\nl7container\nkr\nkod\nking\nkime\nkim\nkillfilter\nkil\nkeytype\nkeylen\nkeyid\nkeepslashes\nkeeppass\nkatid\njpeg\njoingroup\njoined\njid\njform\njenkins\njaxl\njahr\njCryption\nitemkey\nitemcount\nisverify\nissues\nispersis\nisocode\nisnano\nisim\nisenabled\nisemaildomain\niscustomreport\niscomment\niscatchall\nisbinddomain\nisactive\nisSwitch\nisDev\niron\nipv6allow\nipsecpsk\nipscanner\niprestricted\nipprotocol\nipproto\niplist\nipandport\nipaddrv6\ninvoiceId\ninviteesid\ninvited\ninvitation\ninvest\ninvalidate\nintroeditor\ninterfaces\ninstanceId\ninstallstep\ninstallGoingOn\ninputid\ninputSize\ninjector\ninitstr\ninitialtext\ninitialise\ninitdelay\ninifile\ninid\ninf3ct\nineligible\nindent\nincludenoncache\nincl\niname\ninajax\ninactive\ninViewWarnings\ninViewLogs\ninViewErrors\ninSessionSecuirty\ninRemember\ninNewUserName\ninForgotPassword\ninDownLoad\ninConfEmail\ninBindLog\nimportrobotsmeta\nimportonly\nimportmethod\nimportid\nimportfile\nimporter\nimportant\nimportaioseo\nimportType\nimportFile\nimpersonate\nimgtype\nimgid\nimdbID\nimagename\nimagefile\nimagedetails\nimageUrl\nimageThumbID\nimagE\nikesaid\nikeid\nignoresubjectmismatch\nignorephpver\nignorefatal\nignored\nignoreTV\nifnum\nifname\nieee8021x\nidname\nidletimeout\nidentity\nidentifiant\nidb\nidSite\nidL\nid9level\nid9gid\nid8level\nid8gid\nid7level\nid7gid\nid6level\nid6gid\nid5level\nid5gid\nid4level\nid4gid\nid40level\nid40gid\nid3level\nid3gid\nid39level\nid39gid\nid38level\nid38gid\nid37level\nid37gid\nid36level\nid36gid\nid35level\nid35gid\nid34level\nid34gid\nid33level\nid33gid\nid32level\nid32gid\nid31level\nid31gid\nid30level\nid30gid\nid2level\nid2gid\nid29level\nid29gid\nid28level\nid28gid\nid27level\nid27gid\nid26level\nid26gid\nid25level\nid25gid\nid24level\nid24gid\nid23level\nid23gid\nid22level\nid22gid\nid21level\nid21gid\nid20level\nid20gid\nid1level\nid1gid\nid19level\nid19gid\nid18level\nid18gid\nid17level\nid17gid\nid16level\nid16gid\nid15level\nid15gid\nid14level\nid14gid\nid13level\nid13gid\nid12level\nid12gid\nid11level\nid11gid\nid10level\nid10gid\nicp\nicode\nicmptype\nicerik\nical\nhwhy\nhtype\nhttpsverify\nhttpsname\nhttpscanner\nhttps\nhttpbanner\nhtmlemail\nhtml2xhtml\nhtcc\nhtc\nhtaccessnew\nhrs\nhowmuch\nhowmany\nhowlong\nhow\nhostres\nhostipformat\nhostapd\nhostName\nhosT\nhorario\nholdcnt\nhlp\nhldb\nhidrfile\nhidid\nhideversion\nhidemenu\nhideidentity\nhidFileID\nhid\nhellotime\nhealth\nhd\nhc\nhaving\nhashtoh\nhashkey\nhasAudio\nhardenglue\nham\ngtin\ngt\ngs\ngrupo\ngrps\ngrpage\ngrouped\ngroupdesc\ngroupdel\ngroupby\ngroupIDs\ngroupCounter\ngreif\ngraphtype\ngraphlot\ngranularity\ngrants\ngranted\ngr\ngpstype\ngpssubsec\ngpsstratum\ngpsspeed\ngpsselect\ngpsrefid\ngpsprefer\ngpsport\ngpsnmea\ngpsinitcmd\ngpsfudge2\ngpsfudge1\ngpsflag4\ngpsflag3\ngpsflag2\ngpsflag1\ngotod\ngoodsid\ngomkf\ngodb\ngodashboard\ngoal\ngn\ngithub\ngip\ngifif\nggid\ngfils\ngetpic\ngetm\ngetenv\ngetdyndnsstatus\ngetdb\ngetdate\ngetcfg\ngetThermalSensorsData\ngetOutputCompression\ngeneratekey\ngenerateKeypair\ngeneral\nged\ngeT\ngdork\ngd\ngc\ngbid\ngatewayv6\ngameID\ngadget\nga\nfyear\nfwdelay\nfw\nfvonly\nfuzz\nfunctionz\nfunctionp\nftype\nftpscanner\nftps\nftppass\nftphost\nfsOP\nfromAddress\nfrm\nfriendlyiface\nfriend\nfresh\nframes\nfqdn\nfq\nfpath\nfpassw\nforwarding\nforwarderid\nformname\nformfactor\nformdata\nformatup\nformats\nformatdown\nformage\nformId\nformAutosave\nforgotPassword\nforever\nforeground\nforceIcon\nforceFormat\nfontr\nfontg\nfonte\nfontb\nfontSize\nfollowing\nfolderId\nfmt\nflushcache\nflowtable\nfloor\nfloating\nfldMandatory\nflashtype\nflashpga\nfl\nfixmetadesc\nfirmwareurl\nfinds\nfindid\nfin\nfiltre\nfiltertext\nfilterlogentriesinterfaces\nfilterlogentries\nfilterdescriptions\nfilled\nfilew\nfiletosave\nfilesend\nfileoffset\nfilename64\nfilename32\nfilecreate\nfilecount\nfileOffset\nfileLength\nfileExistsAction\nfileEdit\nfileDataName\nfileContent\nfile2ch\nfilE\nfieldkey\nfieldCounter\nfid2\nfeedId\nfe\nfdo\nfdel\nfcsubmit\nfcopy\nfbclearall\nfavourite\nfavorites\nfavicons\nfast\nfamilyName\nfacility\nfacid\nfType\nezID\neyear\nextras\nextractDir\nextern\nextdisplay\nextdir\nexportVideo\nexportMisc\nexportFrames\nexportFormat\nexportFile\nexportDetail\nexploit\nexpirationDate\nexpid\nexpertise\nexpanded\nexpandAll\nexp\nexitsql\nexists\nexif\nexecuteForm\nexecmethod\nexecmassdeface\nexcludedRecords\nexchange\nexc\nexTime\nexT\newidth\neventname\neventTitle\nevap\nevalsource\nevalinfect\nevalcode\nevac\netag\net\neshopId\neshopAccount\nesId\nerror500path\nerror404path\nerror403path\nerne\nepot\nepoch\nentryid\nentryPoint\nentryId\nentryID\nentityID\nentire\nenhanced\nenforceHTTPS\nendport\nendmonth\nencoder\nencod\nenablestp\nenablesshd\nenableserial\nenablenatreflectionhelper\nenablebinatreflection\nemonth\neml\nembedded\nemailsubject\nemailfrom\nemailch\nemailToken\nemailList\nemailID\nemailBody\nelementType\nee\nedittxt\neditprofile\neditkey\neditgroup\neditedon\nedge\necraz\nealgo\ndynamic\ndxval\ndxsqlsearch\ndxportscan\ndxparam\ndxmode\ndxinstant\ndximg\ndxfile\ndxdirsimple\ndxdir\ndummy\ndumd\nduid\nduedate\ndsttype\ndstnot\ndstmask\ndstip\ndstendport\ndstbeginport\ndscp\ndryrun\ndrilldown\ndragtable\ndragdroporder\ndpgn\ndpath\ndownloadid\ndownloadbtn\ndownloadbackup\ndownloadIndex\ndownloaD\ndownf\ndownchange\ndosthisserver\ndosearch\ndopt\ndonotbackuprrd\ndomerge\ndomen\ndomainsearchlist\ndomainname\ndomaiN\ndoimage\ndocumentroot\ndocumentgroup\ndoctype\ndocs\ndocgroups\ndocgroup\ndoRegister\ndoDelete\ndnssrcip\ndnssecstripped\ndnsquery\ndnslocalhost\ndnsallowoverride\ndns4\ndns3\ndnpipe\ndlgzip\ndldone\ndlPath\ndkim\ndizin\ndivider\ndiversity\ndistribution\ndiskspace\ndiscipline\ndisapprove\ndisablevpnrules\ndisablesegmentationoffloading\ndisablescrub\ndisablereplyto\ndisablenegate\ndisablelargereceiveoffloading\ndisablehttpredirect\ndisablefilter\ndisableconsolemenu\ndisablechecksumoffloading\ndisablecheck\ndisablecarp\ndisablebeep\ndirr\ndirlisting\ndirfree\ndirectoryscanner\ndireccion\ndire\ndircreate\ndiract\ndirList\ndimensions\ndig\ndiff\ndhtc\ndhcpv6leaseinlocaltime\ndhcprejectfrom\ndhcpleaseinlocaltime\ndhcphostname\ndhcpfirst\ndhcpbackup\ndhcp6usev4iface\ndhcp6prefixonly\ndfilename\ndevid\ndeviceid\ndetail0\ndestslice\ndestino\ndestd\ndescripcion\ndesc2\ndesc1\ndeptid\ndeposit\ndepid\ndenyunknown\ndend\ndemoData\ndeltype\ndeltpl\ndelstring\ndelsel\ndelpref\ndelmac\ndeliveries\ndeliver\ndelimeter\ndelfriend\ndelfolder\ndelfl\ndelfbadmin\ndelf\ndeleteweek\ndeletesubmit\ndeleterule\ndeletegrp\ndeleteg\ndeletedir\ndeletedSpecs\ndeletecntlist\ndeleteUser\ndeletePrices\ndeleteList\ndeleteIndex\ndeleteImages\ndeleteCategory\ndeldat\ndeld\ndelName\ndegrees\ndeftime\ndefaulttemplate\ndefaultqueue\ndefaultleasetime\ndefaultgw\ndeduction\ndecrypt\ndebugmethods\ndebugfailover\ndebugbox\ndebug3\ndebug2\ndebit\ndeathplace\ndeathdate\ndeadline\ndeact\ndeS\ndeL\nddo\nddnsupdate\nddnsdomainprimary\nddnsdomainkeyname\nddnsdomainkey\nddnsdomain\ndbsocket\ndbn\ndbbase\ndbUsername\ndbTablePrefix\ndbPwd\ndbPass\ndbOP\ndatestamp\ndatechange\ndatasrt\ndataroot\ndataofs\ndatagapradius\ndatagapangle\ndataflt\ndatabasename\ndataangle\ndata2\ndarezz\ndare\ndID\ncx\ncustomernumber\ncustomcss\ncustomaddtplid\ncustomId\ncustomFieldId\ncurrentday\ncurrentPassword\ncurrentPage\ncurrencyid\ncurrencyCode\ncurpath\ncuenta\nctx\nctrl\nctag\ncsvIDs\ncsspreview\ncsrftoken\ncsr\ncs2\ncs1\ncrypo\ncrtty\ncrrt\ncrefile\ncreatestdsubdomain\ncreatelist\ncreatedon\ncreateaccount\ncre\ncrcf\ncrannycap\ncracK\ncpyto\ncpw\ncpath\ncpass\ncpage\ncoverage\ncourseId\ncouponcode\ncoupling\ncountryID\ncountonly\ncopyname\ncop\ncontenttype\ncontainerid\ncontactidlist\ncontactID\ncontactEmail\ncont\nconsumerSecret\nconsumerKey\nconst\nconsent\nconnsub\nconnport\nconnections\nconnectionType\nconnectback\nconfirmEmail\nconfirm3\nconfirm2\nconfigs\nconcepto\ncompr\ncompose\ncommunication\ncommonName\ncommits\ncommex\ncommentaire\ncommander\ncombo\ncolor1\ncollege\ncollectionto\ncollectionfrom\ncollectcolumn\ncoin\ncodetype\ncoded\ncodeblock\ncoauthors\ncoM\ncnpj\ncmspasswordconfirm\ncmode\ncmmd\ncmediafix\ncmdr\ncmdir\ncmdid\ncmdex\ncmde\nclosenotice\nclosedate\nclockstats\nclipboard\ncleartokens\nclearquery\nclearlogs\nclearSess\nclearLog\ncleancache\nclay\nclassname\ncktime\nckeditor\nck\ncipher\ncinterface\ncids\nchoix\nchoice2\nchmodnow\nchmodenum\nchm\nchkalldocs\nchfl\nchecksumbits\nchecknum\ncheckmetadesc\ncheckid\ncheckconnect\ncheckaliasesurlcert\nchdir\nchats\nchatmsg\nchartsize\ncharacterid\nchapter\nchapo\nchangestatus\nchangero\nchangeVisitAlpha\nchangePass\ncfy\ncfx\ncfilename\ncfile\ncfil\ncfed\ncertsubject\ncertref\ncertid\ncertdepth\ncds\ncdirname\ncdir\ncateid\ncategoryname\ncategoryName\ncatalogid\ncatalogName\ncasein\ncartId\ncaref\ncardno\ncapture\ncantidad\ncanpreview\ncanned\ncaneditphpsettings\ncaneditdomain\ncancelled\ncanceldelete\ncampo\ncambio\ncallop\ncallerId\ncaid\ncacheable\ncable\ncP\nc37url\nc2\nbyws\nbythis\nbysyml\nbypcu\nbypassstaticroutes\nbyoc\nbyfc9\nbyfc\nbyetc\nbye\nbycw\nbyapache\nbwdefaultup\nbwdefaultdn\nbv\nbuy\nbuttonval\nbuttons\nbusinessName\nbulletin\nbudget\nbs\nbroadcast\nbridgeif\nbreakpoints\nbreakpoint\nbps\nbpg\nbpage\nbounce\nbottom\nbots\nbootslice\nboolean\nbookings\nbonus\nbogonsinterval\nboardmod\nblogtitle\nblogtags\nblogbody\nblockpriv\nblockeduntil\nblockedmacsurl\nblockedafter\nblockbogons\nblatent\nbirthplace\nbirth\nbip\nbindpw\nbindip\nbinddn\nbgColor\nbenchmark\nbehaviour\nbe\nbduss\nbcip\nbaz\nbaslik\nbasket\nbasic\nbasemodule\nbantype\nbantime\nbannedUser\nbanip\nbanid\nbackurl\nbackupcount\nbackupbeforeupgrade\nbackuparea\nautorefresh\nautoredirect\nautoptp\nautoplay\nautogroup\nautoenable\nautoedge\nautoassign\nautoapprove\nautoadjust\nautoaddfields\nauthserver\nauthorship\nauthorizedkeys\nauthname\nauthmode\nauthlvl\nauthentication\nauthcn\nauthcfg\nauid\nattendance\nattachmentsid\nattachmentUploadDir\nattachmentId\nattachid\nattaches\nassigntype\nassignedTo\nassigned\nasset\nassertion\nasid\nasText\nartistid\narticleid\nargv\nargb\narg2\narchivo\narchivedate\narchiveDate\napple\napnum\napn\napinger\napiKey\naot\nanswerid\nannotation\nann\nanimetitle\nanidb\nandor\nampm\namountup\namountdown\nalturlenable\nalthostnames\nalternate\nallyid\nallss\nallqw\nallowopts\nallowinvalidsig\nallowed\nallowZipDownload\nallids\nallfields\nallergies\nallDepts\naliasimport\naliasesresolveinterval\naliA\nalgo\nalertEmail\nalbumname\nalI\nal\nak\najaxMode\najaxCalendar\najaxAction\naj\nairdate\nagentoption\naffw\naffiliate\naf\nadvskew\nadvbase\nadvancedview\nadsr\nadresse\nadmins\nadminlogin\nadminUser\nadminPass\nadminPWD\nadminEmail\nadlr\naddurl\naddtype\naddtxt\naddtag\naddsite\naddrule\naddressren\naddress0\naddpool\naddonkey\nadditionalData\naddfile\naddevent\naddcat\naddacc\naddUser\naddOption\naddComment\naddBase\nadaptivestart\nadaptiveend\nadapter\nad2syp\nad2syc\nactreject\nactpass\nactivityID\nactivationKey\nactionadd\nactionType\nactid\nactblock\nact3\nact2\nacpage\nackqueue\nack\nacfcomp\naccLimit\nacao\nabbr\nZipName\nYol\nY\nXL\nWIDsubject\nVerifyCode\nVPSSignature\nUserType\nUserSettingsForm\nUserName\nUserLoginForm\nUserForm\nUserCreateForm\nURI\nTxAuthNo\nTrYaG\nTouchm\nToucha\nTouch\nTaxonomy\nTask\nTarget\nTO\nTITL\nSysMessage\nSubsiteID\nSubmit2\nStoreCategory\nStepID\nSoups\nShareForm\nSettingsForm\nSetting\nService\nSecurityKey\nSearchForm\nSandwiches\nSalads\nSURN\nSUBMIT\nSPFX\nSAMLRequest\nResult\nResourceUploadForm\nResetRRD\nRegister\nReduxFrameworkPlugin\nROMN\nRESULT\nRESET\nREPO\nRECHECK\nRC\nQ\nPublic\nProjectUserForm\nPostCodeResult\nPostCode\nPlain\nPerson\nPerms\nPayerStatus\nParentPage\nParent\nPWD\nPUBL\nPHONE\nOwner\nOpt2\nOpt1\nOpenWith\nObject\nNetworkUserID\nNetworkScreenName\nNetworkPlatform\nNSFX\nNPFX\nNOTE\nNICK\nN3tshcook\nModuleVar\nLostPasswordForm\nLookup\nLegendMode\nLast4Digits\nLATEST\nKloutID\nJoomla\nImport\nIPv6\nHowMany\nHkrkoz\nHelp\nHeads\nHash\nHMACKey\nGood\nGiftAid\nGROUP\nGRAPHS\nGIVN\nGENDER\nForm\nFlag\nFilter\nFileIDs\nFile\nField\nFactoryName\nFactoryId\nFXuser\nFXpass\nFXimage\nFONE\nFILES\nFIELDS\nExport\nExpiryDate\nExample\nEvent\nEmailForm\nEVEN\nENCRYPTION\nE\nDownload\nDevForceUpdate\nDesserts\nDUMP\nDESC\nCustomer\nCustomPage\nCurrency\nCreate\nCoupon\nContentList\nContacts\nCity\nCancel\nCallStatus\nCalendar\nCV2Result\nCSalt\nCID\nCHIL\nCAVV\nCAPTCHA\nCALN\nBlog\nBlock\nBeverages\nAuthItemChild\nAttachment\nAlbania\nAdmin\nAddressStatus\nAddressResult\nAccounts\nAVSCV2\nAUTH\nAMOUNT\nALL\nABBR\n4\n3DSecureStatus\n23\n22\n21\n2\n17\n16\n15\n14\n13\n12\n11\n_escaped_fragment_\n__amp_source_origin\nhttp_host\napi-version\nx-method-override\nx-http-method-override\naccess_token\napplicationid\nassembly\nassemblyPath\nbloburl\nbuildid\ncheckin\ncheckno\nclassID\nclassid\nclassnames\ncodetext\nconnectionData\nconnectionId\nconnectionString\nconnectionToken\nculture\ncustomerNum\ndataid\necho\nFappId\nflinkid\nFLinkId\nidNoticia\nidUsuario\nUsuario\nNoticia\njobid\nlinkid\nLocationPath\nmethodid\nnextUrl\nnoscript\noauth_token\noauth_verifier\norderNumber\noriginalPath\nOriginalUrl\nOutputType\nOverridePath\npagepath\nPortalId\npromoId\nproxyRestUri\nReportPath\nreqclient\nrequestType\nresourceId\nreturnpath\nsearchword\nsecretcode\nServerPath\nsourcePage\nSourcePath\nstatuscode\nsuppress\ntabname\ntblname\ntestAction\nuploadType\nOAuthCookie\nshell_path\nuser_token\nadminCookie\nfullapp\nLandingUrl\n" }, { "path": "bbot/wordlists/raft-small-extensions-lowercase_CLEANED.txt", "content": "\n.0\n.0.0\n.0.1\n.0.2\n.0.3\n.0.4\n.0.5\n.0.8\n.0.html\n.0.pdf\n.00\n.00.8169\n.001\n.01\n.01.4511\n.025\n.03\n.04\n.06\n.07\n.075\n.077\n.08\n.083\n.09\n.1\n.1.0\n.1.1\n.1.2\n.1.3\n.1.5.swf\n.1.6\n.1.html\n.1.pdf\n.10\n.10.html\n.11\n.11.html\n.112\n.12\n.125\n.13\n.134\n.14\n.15\n.156\n.16\n.17\n.18\n.19\n.1a\n.1c\n.2\n.2.0\n.2.1\n.2.2\n.2.3\n.2.6\n.2.9\n.2.html\n.20\n.20.html\n.2007\n.2008\n.2011\n.206\n.21\n.211\n.22\n.23\n.24\n.246\n.25\n.25.html\n.26.13.391n35.50.38.816\n.26.24.165n35.50.24.134\n.26.56.247n35.52.03.605\n.26.html\n.27.02.940n35.49.56.075\n.27.15.919n35.52.04.300\n.27.29.262n35.47.15.083\n.2a\n.2ms2\n.3\n.3.0\n.3.1\n.3.2\n.3.2.min.js\n.3.3\n.3.4\n.3.5\n.3.html\n.30\n.30-i486\n.300\n.32\n.33\n.34\n.367\n.3gp\n.4\n.4.0\n.4.1\n.4.2\n.4.6\n.4.7\n.4.9.php\n.4.html\n.40.00.573n35.42.57.445\n.403\n.43.58.040n35.38.35.826\n.44.04.344n35.38.35.077\n.44.08.714n35.39.08.499\n.44.10.892n35.38.49.246\n.44.27.243n35.41.29.367\n.44.29.976n35.37.51.790\n.44.32.445n35.36.10.206\n.44.34.800n35.38.08.156\n.44.37.128n35.40.54.403\n.44.40.556n35.40.53.025\n.44.45.013n35.38.36.211\n.44.46.104n35.38.22.970\n.44.48.130n35.38.25.969\n.44.52.162n35.38.50.456\n.44.58.315n35.38.53.455\n.445\n.45\n.45.01.562n35.38.38.778\n.45.04.359n35.38.39.112\n.45.06.789n35.38.22.556\n.45.10.717n35.38.41.989\n.4511\n.455\n.456\n.499\n.5\n.5.0\n.5.1\n.5.3\n.5.4\n.5.6\n.5.html\n.5.php\n.50\n.556\n.6\n.6.0\n.6.1\n.6.12\n.6.19\n.6.2\n.6.3\n.6.5\n.6.9\n.6.edu\n.6.html\n.605\n.7\n.7.0\n.7.1\n.7.2\n.7.3\n.7.html\n.72\n.75.html\n.778\n.790\n.7z\n.8\n.8.1\n.8.2\n.8.3\n.816\n.8169\n.826\n.9\n.91\n.969\n.970\n.989\n.a\n.access.login\n.acgi\n.action\n.action2\n.adcode\n.add\n.admin\n.adp\n.ai\n.ajax\n.ajax.asp\n.ajax.php\n.alt\n.app\n.apsx\n.aquery\n.array-keys\n.array-merge\n.array-rand\n.as\n.asa\n.asax\n.asax.cs\n.asax.resx\n.asax.vb\n.asc\n.ascx\n.ascx.cs\n.ascx.vb\n.asd\n.asf\n.ashx\n.asm\n.asmx\n.asp\n.aspx\n.assets\n.asx\n.at\n.atom\n.au\n.avi\n.award\n.awm\n.axd\n.b\n.back\n.backup\n.bad\n.bak\n.bak2\n.bat\n.bck\n.bhtml\n.bin\n.bk\n.bkp\n.blog\n.bml\n.bmp\n.bok\n.browse\n.bsp\n.btr\n.bu\n.bz2\n.c\n.ca\n.cab\n.cache\n.calendar\n.call-user-func-array\n.captcha\n.captcha.aspx\n.cart\n.casino\n.cat\n.cc\n.cdr\n.cer\n.cfc\n.cfg\n.cfg.php\n.cfm\n.cfm.cfm\n.cfml\n.cgi\n.changelang.php\n.children\n.chm\n.class\n.class.php\n.cmd\n.cms\n.cn\n.cnf\n.co.uk\n.cocomore.txt\n.code\n.com\n.com-redirect\n.com.crt\n.com.html\n.com_backup_giornaliero\n.com_backup_settimanale\n.common.php\n.conf\n.config\n.config.php\n.content\n.contrib\n.controls\n.copy\n.core\n.count\n.cp\n.crt\n.cs\n.csi\n.csp\n.csproj\n.csproj.user\n.css\n.csv\n.cur\n.custom\n.cz\n.d\n.dat\n.data\n.db\n.dbf\n.dcr\n.de\n.de.html\n.de.txt\n.deb\n.default\n.delete\n.detail\n.details.php\n.dev\n.dhtml\n.dic\n.dict.php\n.diff\n.dir\n.disabled\n.dist.php\n.divx\n.djvu\n.dll\n.dmg\n.do\n.doc\n.docx\n.dot\n.ds\n.dta\n.dtd\n.dwf\n.dwg\n.dwt\n.e\n.ece\n.edit\n.edu\n.egov\n.email\n.eml\n.en\n.en.html\n.en.php\n.enfinity\n.enu\n.eot\n.ep\n.epc\n.epl\n.eps\n.epub\n.err\n.error\n.errors\n.es\n.eu\n.exclude\n.exe\n.extract\n.f4v\n.faces\n.fancybox\n.fcgi\n.feed\n.fil\n.file\n.file-get-contents\n.file-put-contents\n.filemtime\n.files\n.filesize\n.film\n.fla\n.flv\n.fopen\n.form\n.fpl\n.fr\n.fr.html\n.framework\n.fread\n.fsockopen\n.functions.php\n.g\n.geo\n.getimagesize\n.getmapimage\n.gif\n.gif.php\n.gif_var_de\n.git\n.go\n.googlebook\n.gpx\n.grp\n.gz\n.h\n.hml\n.hmtl\n.home\n.hotelname\n.hqx\n.ht\n.hta\n.htaccess\n.htc\n.htlm\n.htm\n.html\n.htmls\n.htx\n.i\n.ice\n.ico\n.ics\n.ida\n.idq\n.idx\n.ihtml\n.image\n.images\n.img\n.implode\n.in-array\n.inc\n.inc.asp\n.inc.html\n.inc.js\n.inc.php\n.include\n.include-once\n.includes\n.index\n.index.html\n.index.php\n.inf\n.info\n.ini\n.ini.php\n.ini.sample\n.iso\n.it\n.it.html\n.j\n.jad\n.jar\n.java\n.jbf\n.jhtml\n.jnlp\n.jp\n.jpe\n.jpeg\n.jpg\n.js\n.js2\n.jsf\n.json\n.jsp\n.jspa\n.jspf\n.jspx\n.kml\n.kmz\n.l\n.lang-en.php\n.lasso\n.layer\n.lbi\n.lck\n.letter\n.lib\n.lib.php\n.lic\n.licx\n.link\n.list\n.listevents\n.lnk\n.load\n.local\n.local.php\n.lock\n.log\n.log.0\n.login\n.login.php\n.lst\n.m\n.m3u\n.m4v\n.main\n.maninfo\n.map\n.master\n.master.cs\n.master.vb\n.mbox\n.mc_id\n.mdb\n.media\n.menu.php\n.mgi\n.mhtml\n.mi\n.mid\n.min.js\n.mkdir\n.mno\n.mod\n.mov\n.mp2\n.mp3\n.mp4\n.mpeg\n.mpg\n.mpl\n.msg\n.msi\n.mso\n.mspx\n.mv\n.mvc\n.mysql\n.mysql-connect\n.mysql-pconnect\n.mysql-query\n.mysql-result\n.mysql-select-db\n.net\n.net.html\n.new\n.new.html\n.new.php\n.news\n.nl\n.none\n.nsf\n.num\n.o\n.ocx\n.odt\n.off\n.ogg\n.old\n.old.php\n.old2\n.opendir\n.opml\n.org\n.orig\n.original\n.oui\n.out\n.outcontrol\n.p\n.p3p\n.p7b\n.pac\n.pad\n.page\n.pages\n.parse.errors\n.pd\n.pdb\n.pdf\n.pem\n.pfx\n.pgp\n.pgt\n.ph\n.php\n.php-dist\n.php1\n.php2\n.php3\n.php4\n.php5\n.php_files\n.phpp\n.phps\n.phtm\n.phtml\n.pl\n.plx\n.pm\n.png\n.pnp\n.po\n.pop_3d_viewer\n.pop_formata_viewer\n.popup.php\n.popup.pop_3d_viewer\n.popup.pop_formata_viewer\n.portal\n.pot\n.pps\n.ppt\n.pptx\n.preg-match\n.prep\n.prev_next\n.preview\n.preview-content.php\n.prg\n.price\n.print\n.print.html\n.print.php\n.printable\n.process\n.product_details\n.prt\n.ps\n.psd\n.psp\n.psql\n.pub\n.pvk\n.pwd\n.py\n.pyc\n.q\n.query\n.r\n.ra\n.ram\n.randomhouse\n.rar\n.raw\n.rb\n.rc\n.rdf\n.read\n.readme\n.readme_var_de\n.rec\n.red\n.reg\n.registration\n.require\n.require-once\n.results\n.resx\n.rhtml\n.rm\n.rpm\n.rss\n.rtf\n.ru\n.ru.html\n.run\n.run.adcode\n.s\n.s7\n.sample\n.sav\n.save\n.scc\n.scripts\n.sdb\n.se\n.sea\n.seam\n.search\n.search\n.sema\n.sendtoafriendform\n.ser\n.server\n.session\n.session-start\n.settings.php\n.setup\n.sh\n.shop\n.shtm\n.shtml\n.simplexml-load-file\n.sis\n.sit\n.site\n.sitemap\n.sitemap.xml\n.sitx\n.skins\n.sln\n.smi\n.smil\n.sponsors\n.sql\n.sql.gz\n.squery\n.src\n.srv\n.ssf\n.ssi\n.start\n.static\n.ste\n.stm\n.store\n.storefront\n.strpos\n.subscribe\n.suo\n.svc\n.svg\n.svn\n.swf\n.swi\n.swp\n.sxw\n.t\n.taf\n.tar\n.tar.bz2\n.tar.gz\n.tcl\n.tem\n.temp\n.template\n.template.php\n.templates\n.test\n.text\n.textsearch\n.tgz\n.thtml\n.tif\n.tiff\n.tmp\n.tmpl\n.top\n.torrent\n.tpl\n.trck\n.ttf\n.tv\n.txt\n.txt.gz\n.txt.php\n.types\n.ua\n.uguide\n.uk\n.unlink\n.unsubscribe\n.url\n.us\n.user\n.userloginpopup.php\n.v\n.vb\n.vbproj\n.vbproj.webinfo\n.vbs\n.vcf\n.vcs\n.view\n.visapopup.php\n.visapopupvalid.php\n.vm\n.vorteil\n.vspscc\n.vssscc\n.w\n.war\n.wav\n.wbp\n.wci\n.web\n.web.ui.webresource.axd\n.webinfo\n.wma\n.wmf\n.wml\n.wmv\n.woa\n.work\n.wpd\n.ws\n.wsdl\n.wvx\n.wws\n.x\n.x-affiliate\n.x-affiliate_var_de\n.x-aom\n.x-aom_var_de\n.x-fancycat\n.x-fancycat_var_de\n.x-fcomp\n.x-fcomp_var_de\n.x-giftreg\n.x-giftreg_var_de\n.x-magnifier\n.x-magnifier_var_de\n.x-offers\n.x-offers_var_de\n.x-pconf\n.x-pconf_var_de\n.x-rma\n.x-rma_var_de\n.x-survey\n.xhtm\n.xhtml\n.xls\n.xlsx\n.xml\n.xpi\n.xpml\n.xsd\n.xsl\n.xslt\n.xspf\n.y\n.z\n.zdat\n.zif\n.zip\n" }, { "path": "bbot/wordlists/top_open_ports_nmap.txt", "content": "80\n23\n443\n21\n22\n25\n3389\n110\n445\n139\n143\n53\n135\n3306\n8080\n1723\n111\n995\n993\n5900\n1025\n587\n8888\n199\n1720\n465\n548\n113\n81\n6001\n10000\n514\n5060\n179\n1026\n2000\n8443\n8000\n32768\n554\n26\n1433\n49152\n2001\n515\n8008\n49154\n1027\n5666\n646\n5000\n5631\n631\n49153\n8081\n2049\n88\n79\n5800\n106\n2121\n1110\n49155\n6000\n513\n990\n5357\n427\n49156\n543\n544\n5101\n144\n7\n389\n8009\n3128\n444\n9999\n5009\n7070\n5190\n3000\n5432\n1900\n3986\n13\n1029\n9\n5051\n6646\n49157\n1028\n873\n1755\n2717\n4899\n9100\n119\n37\n1000\n3001\n5001\n82\n10010\n1030\n9090\n2107\n1024\n2103\n6004\n1801\n5050\n19\n8031\n1041\n255\n1049\n1048\n2967\n1053\n3703\n1056\n1065\n1064\n1054\n17\n808\n3689\n1031\n1044\n1071\n5901\n100\n9102\n8010\n2869\n1039\n5120\n4001\n9000\n2105\n636\n1038\n2601\n1\n7000\n1066\n1069\n625\n311\n280\n254\n4000\n1761\n5003\n2002\n2005\n1998\n1032\n1050\n6112\n3690\n1521\n2161\n6002\n1080\n2401\n4045\n902\n7937\n787\n1058\n2383\n32771\n1033\n1040\n1059\n50000\n5555\n10001\n1494\n593\n2301\n3\n3268\n7938\n1234\n1022\n1074\n8002\n1036\n1035\n9001\n1037\n464\n497\n1935\n6666\n2003\n6543\n1352\n24\n3269\n1111\n407\n500\n20\n2006\n3260\n15000\n1218\n1034\n4444\n264\n2004\n33\n1042\n42510\n999\n3052\n1023\n1068\n222\n7100\n888\n563\n1717\n2008\n992\n32770\n32772\n7001\n8082\n2007\n5550\n2009\n5801\n1043\n512\n2701\n7019\n50001\n1700\n4662\n2065\n2010\n42\n9535\n2602\n3333\n161\n5100\n5002\n2604\n4002\n6059\n1047\n8192\n8193\n2702\n6789\n9595\n1051\n9594\n9593\n16993\n16992\n5226\n5225\n32769\n3283\n1052\n8194\n1055\n1062\n9415\n8701\n8652\n8651\n8089\n65389\n65000\n64680\n64623\n55600\n55555\n52869\n35500\n33354\n23502\n20828\n1311\n1060\n4443\n1067\n13782\n5902\n366\n9050\n1002\n85\n5500\n5431\n1864\n1863\n8085\n51103\n49999\n45100\n10243\n49\n6667\n90\n27000\n1503\n6881\n1500\n8021\n340\n5566\n8088\n2222\n9071\n8899\n6005\n9876\n1501\n5102\n32774\n32773\n9101\n5679\n163\n648\n146\n1666\n901\n83\n9207\n8001\n8083\n8084\n5004\n3476\n5214\n14238\n12345\n912\n30\n2605\n2030\n6\n541\n8007\n3005\n4\n1248\n2500\n880\n306\n4242\n1097\n9009\n2525\n1086\n1088\n8291\n52822\n6101\n900\n7200\n2809\n800\n32775\n12000\n1083\n211\n987\n705\n20005\n711\n13783\n6969\n3071\n5269\n5222\n1085\n1046\n5986\n5985\n5987\n5989\n5988\n2190\n3301\n11967\n8600\n3766\n7627\n8087\n30000\n9010\n7741\n14000\n3367\n1099\n1098\n3031\n2718\n6580\n15002\n4129\n6901\n3827\n3580\n2144\n8181\n3801\n1718\n2811\n9080\n2135\n1045\n2399\n3017\n10002\n1148\n9002\n8873\n2875\n9011\n5718\n8086\n20000\n3998\n2607\n11110\n4126\n9618\n2381\n1096\n3300\n3351\n1073\n8333\n3784\n5633\n15660\n6123\n3211\n1078\n5910\n5911\n3659\n3551\n2260\n2160\n2100\n16001\n3325\n3323\n1104\n9968\n9503\n9502\n9485\n9290\n9220\n8994\n8649\n8222\n7911\n7625\n7106\n65129\n63331\n6156\n6129\n60020\n5962\n5961\n5960\n5959\n5925\n5877\n5825\n5810\n58080\n57294\n50800\n50006\n50003\n49160\n49159\n49158\n48080\n40193\n34573\n34572\n34571\n3404\n33899\n32782\n32781\n31038\n30718\n28201\n27715\n25734\n24800\n22939\n21571\n20221\n20031\n19842\n19801\n19101\n17988\n1783\n16018\n16016\n15003\n14442\n13456\n10629\n10628\n10626\n10621\n10617\n10616\n10566\n10025\n10024\n10012\n1169\n5030\n5414\n1057\n6788\n1947\n1094\n1075\n1108\n4003\n1081\n1093\n4449\n1687\n1840\n1100\n1063\n1061\n9900\n1107\n1106\n9500\n20222\n7778\n1077\n1310\n2119\n2492\n1070\n8400\n1272\n6389\n7777\n1072\n1079\n1082\n8402\n89\n691\n1001\n32776\n1999\n212\n2020\n6003\n7002\n2998\n50002\n3372\n898\n5510\n32\n2033\n99\n749\n425\n5903\n43\n5405\n6106\n13722\n6502\n7007\n458\n9666\n8100\n3737\n5298\n1152\n8090\n2191\n3011\n1580\n9877\n5200\n3851\n3371\n3370\n3369\n7402\n5054\n3918\n3077\n7443\n3493\n3828\n1186\n2179\n1183\n19315\n19283\n3995\n5963\n1124\n8500\n1089\n10004\n2251\n1087\n5280\n3871\n3030\n62078\n5904\n9091\n4111\n1334\n3261\n2522\n5859\n1247\n9944\n9943\n9110\n8654\n8254\n8180\n8011\n7512\n7435\n7103\n61900\n61532\n5922\n5915\n5822\n56738\n55055\n51493\n50636\n50389\n49175\n49165\n49163\n3546\n32784\n27355\n27353\n27352\n24444\n19780\n18988\n16012\n15742\n10778\n4006\n2126\n4446\n3880\n1782\n1296\n9998\n9040\n32779\n1021\n32777\n2021\n32778\n616\n666\n700\n5802\n4321\n545\n1524\n1112\n49400\n84\n38292\n2040\n32780\n3006\n2111\n1084\n1600\n2048\n2638\n9111\n6699\n16080\n6547\n6007\n1533\n5560\n2106\n1443\n667\n720\n2034\n555\n801\n6025\n3221\n3826\n9200\n2608\n4279\n7025\n11111\n3527\n1151\n8200\n8300\n6689\n9878\n10009\n8800\n5730\n2394\n2393\n2725\n6566\n9081\n5678\n5906\n3800\n4550\n5080\n1201\n3168\n3814\n1862\n1114\n6510\n3905\n8383\n3914\n3971\n3809\n5033\n7676\n3517\n4900\n3869\n9418\n2909\n3878\n8042\n1091\n1090\n3920\n6567\n1138\n3945\n1175\n10003\n3390\n5907\n3889\n1131\n8292\n5087\n1119\n1117\n4848\n7800\n16000\n3324\n3322\n5221\n4445\n9917\n9575\n9099\n9003\n8290\n8099\n8093\n8045\n7921\n7920\n7496\n6839\n6792\n6779\n6692\n6565\n60443\n5952\n5950\n5862\n5850\n5815\n5811\n57797\n56737\n5544\n55056\n5440\n54328\n54045\n52848\n52673\n50500\n50300\n49176\n49167\n49161\n44501\n44176\n41511\n40911\n32785\n32783\n30951\n27356\n26214\n25735\n19350\n18101\n18040\n17877\n16113\n15004\n14441\n12265\n12174\n10215\n10180\n4567\n6100\n5061\n4004\n4005\n8022\n9898\n7999\n1271\n1199\n3003\n1122\n2323\n4224\n2022\n617\n777\n417\n714\n6346\n981\n722\n1009\n4998\n70\n1076\n5999\n10082\n765\n301\n524\n668\n2041\n6009\n1417\n1434\n259\n44443\n1984\n2068\n7004\n1007\n4343\n416\n2038\n6006\n109\n4125\n1461\n9103\n911\n726\n1010\n2046\n2035\n7201\n687\n2013\n481\n125\n6669\n6668\n903\n1455\n683\n1011\n2043\n2047\n256\n9929\n5998\n406\n31337\n44442\n783\n843\n2042\n2045\n4040\n6060\n6051\n1145\n3916\n9443\n9444\n1875\n7272\n4252\n4200\n7024\n1556\n13724\n1141\n1233\n8765\n1137\n3963\n5938\n9191\n3808\n8686\n3981\n2710\n3852\n3849\n3944\n3853\n9988\n1163\n4164\n3820\n6481\n3731\n5081\n40000\n8097\n4555\n3863\n1287\n4430\n7744\n7913\n1166\n1164\n1165\n8019\n10160\n4658\n7878\n3304\n3307\n1259\n1092\n7278\n3872\n10008\n7725\n3410\n1971\n3697\n3859\n3514\n4949\n4147\n7900\n5353\n3931\n8675\n1277\n3957\n1213\n2382\n6600\n3700\n3007\n4080\n1113\n3969\n1132\n1309\n3848\n7281\n3907\n3972\n3968\n1126\n5223\n1217\n3870\n3941\n8293\n1719\n1300\n2099\n6068\n3013\n3050\n1174\n3684\n2170\n3792\n1216\n5151\n7123\n7080\n22222\n4143\n5868\n8889\n12006\n1121\n3119\n8015\n10023\n3824\n1154\n20002\n3888\n4009\n5063\n3376\n1185\n1198\n1192\n1972\n1130\n1149\n4096\n6500\n8294\n3990\n3993\n8016\n5242\n3846\n3929\n1187\n5074\n5909\n8766\n5905\n1102\n2800\n9941\n9914\n9815\n9673\n9643\n9621\n9501\n9409\n9198\n9197\n9098\n8996\n8987\n8877\n8676\n8648\n8540\n8481\n8385\n8189\n8098\n8095\n8050\n7929\n7770\n7749\n7438\n7241\n7051\n7050\n6896\n6732\n6711\n65310\n6520\n6504\n6247\n6203\n61613\n60642\n60146\n60123\n5981\n5940\n59202\n59201\n59200\n5918\n5914\n59110\n5899\n58838\n5869\n58632\n58630\n5823\n5818\n5812\n5807\n58002\n58001\n57665\n55576\n55020\n53535\n5339\n53314\n53313\n53211\n52853\n52851\n52850\n52849\n52847\n5279\n52735\n52710\n52660\n5212\n51413\n51191\n5040\n50050\n49401\n49236\n49195\n49186\n49171\n49168\n49164\n4875\n47544\n46996\n46200\n44709\n41523\n41064\n40811\n3994\n39659\n39376\n39136\n38188\n38185\n37839\n35513\n33554\n33453\n32835\n32822\n32816\n32803\n32792\n32791\n30704\n30005\n29831\n29672\n28211\n27357\n26470\n23796\n23052\n2196\n21792\n19900\n18264\n18018\n17595\n16851\n16800\n16705\n15402\n15001\n12452\n12380\n12262\n12215\n12059\n12021\n10873\n10058\n10034\n10022\n10011\n2910\n1594\n1658\n1583\n3162\n2920\n1812\n26000\n2366\n4600\n1688\n1322\n2557\n1095\n1839\n2288\n1123\n5968\n9600\n1244\n1641\n2200\n1105\n6550\n5501\n1328\n2968\n1805\n1914\n1974\n31727\n3400\n1301\n1147\n1721\n1236\n2501\n2012\n6222\n1220\n1109\n1347\n502\n701\n2232\n2241\n4559\n710\n10005\n5680\n623\n913\n1103\n780\n930\n803\n725\n639\n540\n102\n5010\n1222\n953\n8118\n9992\n1270\n27\n123\n86\n447\n1158\n442\n18000\n419\n931\n874\n856\n250\n475\n2044\n441\n210\n6008\n7003\n5803\n1008\n556\n6103\n829\n3299\n55\n713\n1550\n709\n2628\n223\n3025\n87\n57\n10083\n5520\n980\n251\n1013\n9152\n1212\n2433\n1516\n333\n2011\n748\n1350\n1526\n7010\n1241\n127\n157\n220\n1351\n2067\n684\n77\n4333\n674\n943\n904\n840\n825\n792\n732\n1020\n1006\n657\n557\n610\n1547\n523\n996\n2025\n602\n3456\n862\n600\n2903\n257\n1522\n1353\n6662\n998\n660\n729\n730\n731\n782\n1357\n3632\n3399\n6050\n2201\n971\n969\n905\n846\n839\n823\n822\n795\n790\n778\n757\n659\n225\n1015\n1014\n1012\n655\n786\n6017\n6670\n690\n388\n44334\n754\n5011\n98\n411\n1525\n3999\n740\n12346\n802\n1337\n1127\n2112\n1414\n2600\n621\n606\n59\n928\n924\n922\n921\n918\n878\n864\n859\n806\n805\n728\n252\n1005\n1004\n641\n758\n669\n38037\n715\n1413\n2104\n1229\n3817\n6063\n6062\n6055\n6052\n6030\n6021\n6015\n6010\n3220\n6115\n3940\n2340\n8006\n4141\n3810\n1565\n3511\n33000\n2723\n9202\n4036\n4035\n2312\n3652\n3280\n4243\n4298\n4297\n4294\n4262\n4234\n4220\n4206\n22555\n9300\n7121\n1927\n4433\n5070\n2148\n1168\n9979\n7998\n4414\n1823\n3653\n1223\n8201\n4876\n3240\n2644\n4020\n2436\n3906\n4375\n4024\n5581\n5580\n9694\n6251\n7345\n7325\n7320\n7300\n3121\n5473\n5475\n3600\n3943\n4912\n2142\n1976\n1975\n5202\n5201\n4016\n5111\n9911\n10006\n3923\n3930\n1221\n2973\n3909\n5814\n3080\n4158\n3526\n1911\n5066\n2711\n2187\n3788\n3796\n3922\n2292\n16161\n4881\n3979\n3670\n4174\n3102\n3483\n2631\n1750\n3897\n7500\n5553\n5554\n9875\n4570\n3860\n3712\n8052\n2083\n8883\n2271\n4606\n1208\n3319\n3935\n3430\n1215\n3962\n3368\n3964\n1128\n5557\n4010\n9400\n1605\n3291\n7400\n5005\n1699\n1195\n5053\n3813\n1712\n3002\n3765\n3806\n43000\n2371\n3532\n3799\n3790\n3599\n3850\n4355\n4358\n4357\n4356\n5433\n3928\n4713\n4374\n3961\n9022\n3911\n3396\n7628\n3200\n1753\n3967\n2505\n5133\n3658\n8471\n1314\n2558\n6161\n4025\n3089\n9021\n30001\n8472\n5014\n9990\n1159\n1157\n1308\n5723\n3443\n4161\n1135\n9211\n9210\n4090\n7789\n6619\n9628\n12121\n4454\n3680\n3167\n3902\n3901\n3890\n3842\n16900\n4700\n4687\n8980\n1196\n4407\n3520\n3812\n5012\n10115\n1615\n2902\n4118\n2706\n2095\n2096\n3363\n5137\n3795\n8005\n10007\n3515\n8003\n3847\n3503\n5252\n27017\n2197\n4120\n1180\n5722\n1134\n1883\n1249\n3311\n27350\n3837\n2804\n4558\n4190\n2463\n1204\n4056\n1184\n19333\n9333\n3913\n3672\n4342\n4877\n3586\n8282\n1861\n1752\n9592\n1701\n6085\n2081\n4058\n2115\n8900\n4328\n2958\n2957\n7071\n3899\n2531\n2691\n5052\n1638\n3419\n2551\n5908\n4029\n3603\n1336\n2082\n1143\n3602\n1176\n4100\n3486\n6077\n4800\n2062\n1918\n12001\n12002\n9084\n7072\n1156\n2313\n3952\n4999\n5023\n2069\n28017\n27019\n27018\n3439\n6324\n1188\n1125\n3908\n7501\n8232\n1722\n2988\n10500\n1136\n1162\n10020\n22128\n1211\n3530\n12009\n9005\n3057\n3956\n4325\n1191\n3519\n5235\n1144\n4745\n1901\n1807\n2425\n3210\n32767\n5015\n5013\n3622\n4039\n10101\n5233\n5152\n3983\n3982\n9616\n4369\n3728\n3621\n2291\n5114\n7101\n1315\n2087\n5234\n1635\n3263\n4121\n4602\n2224\n3949\n9131\n3310\n3937\n2253\n3882\n3831\n2376\n2375\n3876\n3362\n3663\n3334\n47624\n1825\n4302\n5721\n1279\n2606\n1173\n22125\n17500\n12005\n6113\n1973\n3793\n3637\n8954\n3742\n9667\n41795\n41794\n4300\n8445\n12865\n3365\n4665\n3190\n3577\n3823\n2261\n2262\n2812\n1190\n22350\n3374\n4135\n2598\n2567\n1167\n8470\n10443\n8116\n3830\n8880\n2734\n3505\n3388\n3669\n1871\n8025\n1958\n3681\n3014\n8999\n4415\n3414\n4101\n6503\n9700\n3683\n1150\n18333\n4376\n3991\n3989\n3992\n2302\n3415\n1179\n3946\n2203\n4192\n4418\n2712\n25565\n4065\n5820\n3915\n2080\n3103\n2265\n8202\n2304\n8060\n4119\n4401\n1560\n3904\n4534\n1835\n1116\n8023\n8474\n3879\n4087\n4112\n6350\n9950\n3506\n3948\n3825\n2325\n1800\n1153\n6379\n3839\n4689\n47806\n5912\n3975\n3980\n4113\n2847\n2070\n3425\n6628\n3997\n3513\n3656\n2335\n1182\n1954\n3996\n4599\n2391\n3479\n5021\n5020\n1558\n1924\n4545\n2991\n6065\n1290\n1559\n1317\n5423\n1707\n5055\n9975\n9971\n9919\n9915\n9912\n9910\n9908\n9901\n9844\n9830\n9826\n9825\n9823\n9814\n9812\n9777\n9745\n9683\n9680\n9679\n9674\n9665\n9661\n9654\n9648\n9620\n9619\n9613\n9583\n9527\n9513\n9493\n9478\n9464\n9454\n9364\n9351\n9183\n9170\n9133\n9130\n9128\n9125\n9065\n9061\n9044\n9037\n9013\n9004\n8925\n8898\n8887\n8882\n8879\n8878\n8865\n8843\n8801\n8798\n8790\n8772\n8756\n8752\n8736\n8680\n8673\n8658\n8655\n8644\n8640\n8621\n8601\n8562\n8539\n8531\n8530\n8515\n8484\n8479\n8477\n8455\n8454\n8453\n8452\n8451\n8409\n8339\n8308\n8295\n8273\n8268\n8255\n8248\n8245\n8144\n8133\n8110\n8092\n8064\n8037\n8029\n8018\n8014\n7975\n7895\n7854\n7853\n7852\n7830\n7813\n7788\n7780\n7772\n7771\n7688\n7685\n7654\n7637\n7600\n7555\n7553\n7456\n7451\n7231\n7218\n7184\n7119\n7104\n7102\n7092\n7068\n7067\n7043\n7033\n6973\n6972\n6956\n6942\n6922\n6920\n6897\n6877\n6780\n6734\n6725\n6710\n6709\n6650\n6647\n6644\n6606\n65514\n65488\n6535\n65311\n65048\n64890\n64727\n64726\n64551\n64507\n64438\n64320\n6412\n64127\n64080\n63803\n63675\n6349\n63423\n6323\n63156\n6310\n63105\n6309\n62866\n6274\n6273\n62674\n6259\n62570\n62519\n6250\n62312\n62188\n62080\n62042\n62006\n61942\n61851\n61827\n61734\n61722\n61669\n61617\n61616\n61516\n61473\n61402\n6126\n6120\n61170\n61169\n61159\n60989\n6091\n6090\n60794\n60789\n60783\n60782\n60753\n60743\n60728\n60713\n6067\n60628\n60621\n60612\n60579\n60544\n60504\n60492\n60485\n60403\n60401\n60377\n60279\n60243\n60227\n60177\n60111\n60086\n60055\n60003\n60002\n60000\n59987\n59841\n59829\n59810\n59778\n5975\n5974\n5971\n59684\n5966\n5958\n59565\n5954\n5953\n59525\n59510\n59509\n59504\n5949\n59499\n5948\n5945\n5939\n5936\n5934\n59340\n5931\n5927\n5926\n5924\n5923\n59239\n5921\n5920\n59191\n5917\n59160\n59149\n59122\n59107\n59087\n58991\n58970\n58908\n5888\n5887\n5881\n5878\n5875\n5874\n58721\n5871\n58699\n58634\n58622\n58610\n5860\n5858\n58570\n58562\n5854\n5853\n5852\n5849\n58498\n5848\n58468\n5845\n58456\n58446\n58430\n5840\n5839\n5838\n58374\n5836\n5834\n5831\n58310\n58305\n5827\n5826\n58252\n5824\n5821\n5817\n58164\n58109\n58107\n5808\n58072\n5806\n5804\n57999\n57988\n57928\n57923\n57896\n57891\n57733\n57730\n57702\n57681\n57678\n57576\n57479\n57398\n57387\n5737\n57352\n57350\n5734\n57347\n57335\n5732\n57325\n57123\n5711\n57103\n57020\n56975\n56973\n56827\n56822\n56810\n56725\n56723\n56681\n5667\n56668\n5665\n56591\n56535\n56507\n56293\n56259\n5622\n5621\n5620\n5612\n5611\n56055\n56016\n55948\n55910\n55907\n55901\n55781\n55773\n55758\n55721\n55684\n55652\n55635\n55579\n55569\n55568\n55556\n5552\n55527\n55479\n55426\n55400\n55382\n55350\n55312\n55227\n55187\n55183\n55000\n54991\n54987\n54907\n54873\n54741\n54722\n54688\n54658\n54605\n5458\n5457\n54551\n54514\n5444\n5442\n5441\n54323\n54321\n54276\n54263\n54235\n54127\n54101\n54075\n53958\n53910\n53852\n53827\n53782\n5377\n53742\n5370\n53690\n53656\n53639\n53633\n53491\n5347\n53469\n53460\n53370\n53361\n53319\n53240\n53212\n53189\n53178\n53085\n52948\n5291\n52893\n52675\n52665\n5261\n5259\n52573\n52506\n52477\n52391\n52262\n52237\n52230\n52226\n52225\n5219\n52173\n52071\n52046\n52025\n52003\n52002\n52001\n52000\n51965\n51961\n51909\n51906\n51809\n51800\n51772\n51771\n51658\n51582\n51515\n51488\n51485\n51484\n5147\n51460\n51423\n51366\n51351\n51343\n51300\n5125\n51240\n51235\n51234\n51233\n5122\n5121\n51139\n51118\n51067\n51037\n51020\n51011\n50997\n5098\n5096\n5095\n50945\n5090\n50903\n5088\n50887\n50854\n50849\n50836\n50835\n50834\n50833\n50831\n50815\n50809\n50787\n50733\n50692\n50585\n50577\n50576\n50545\n50529\n50513\n50356\n50277\n50258\n50246\n50224\n50205\n50202\n50198\n50189\n5017\n5016\n50101\n50040\n50019\n50016\n49927\n49803\n49765\n49762\n49751\n49678\n49603\n49597\n49522\n49521\n49520\n49519\n49500\n49498\n49452\n49398\n49372\n49352\n4931\n49302\n49275\n49241\n49235\n49232\n49228\n49216\n49213\n49211\n49204\n49203\n49202\n49201\n49197\n49196\n49191\n49190\n49189\n49179\n49173\n49172\n49170\n49169\n49166\n49132\n49048\n4903\n49002\n48973\n48967\n48966\n48925\n48813\n48783\n48682\n48648\n48631\n4860\n4859\n48434\n48356\n4819\n48167\n48153\n48127\n48083\n48067\n48009\n47969\n47966\n4793\n47860\n47858\n47850\n4778\n47777\n4771\n4770\n47700\n4767\n47634\n4760\n47595\n47581\n47567\n47448\n47372\n47348\n47267\n47197\n4712\n47119\n47029\n47012\n46992\n46813\n46593\n4649\n4644\n46436\n46418\n46372\n46310\n46182\n46171\n46115\n4609\n46069\n46034\n45960\n45864\n45777\n45697\n45624\n45602\n45463\n45438\n45413\n4530\n45226\n45220\n4517\n4516\n45164\n45136\n45050\n45038\n44981\n44965\n4476\n4471\n44711\n44704\n4464\n44628\n44616\n44541\n44505\n44479\n44431\n44410\n44380\n44200\n44119\n44101\n44004\n4388\n43868\n4384\n43823\n43734\n43690\n43654\n43425\n43242\n43231\n43212\n43143\n43139\n43103\n43027\n43018\n43002\n42990\n42906\n42735\n42685\n42679\n42675\n42632\n42590\n42575\n42560\n42559\n42452\n42449\n42322\n42276\n42251\n42158\n42127\n42035\n42001\n41808\n41773\n41632\n41551\n41442\n41398\n41348\n41345\n41342\n41318\n41281\n41250\n41142\n41123\n40951\n40834\n40812\n40754\n40732\n40712\n40628\n40614\n40513\n40489\n40457\n40400\n40393\n40306\n40011\n40005\n40003\n40002\n40001\n39917\n39895\n39883\n39869\n39795\n39774\n39763\n39732\n39630\n39489\n39482\n39433\n39380\n39293\n39265\n39117\n39067\n38936\n38805\n38780\n38764\n38761\n38570\n38561\n38546\n38481\n38446\n38358\n38331\n38313\n38270\n38224\n38205\n38194\n38029\n37855\n37789\n37777\n37674\n37647\n37614\n37607\n37522\n37393\n37218\n37185\n37174\n37151\n37121\n36983\n36962\n36950\n36914\n36824\n36823\n36748\n36710\n36694\n36677\n36659\n36552\n36530\n36508\n36436\n36368\n36275\n36256\n36105\n36104\n36046\n35986\n35929\n35906\n35901\n35900\n35879\n35731\n35593\n35553\n35506\n35401\n35393\n35392\n35349\n35272\n35217\n35131\n35116\n35050\n35033\n34875\n34833\n34783\n34765\n34728\n34683\n34510\n34507\n34401\n34381\n34341\n34317\n34189\n34096\n34036\n34021\n33895\n33889\n33882\n33879\n33841\n33605\n33604\n33550\n33523\n33522\n33444\n33395\n33367\n33337\n33335\n33327\n33277\n33203\n33200\n33192\n33175\n33124\n33087\n33070\n33017\n33011\n32976\n32961\n32960\n32944\n32932\n32911\n32910\n32908\n32905\n32904\n32898\n32897\n32888\n32871\n32869\n32868\n32858\n32842\n32837\n32820\n32815\n32814\n32807\n32799\n32798\n32797\n32790\n32789\n32788\n32765\n32764\n32261\n32260\n32219\n32200\n32102\n32088\n32031\n32022\n32006\n31728\n31657\n31522\n31438\n31386\n31339\n31072\n31058\n31033\n30896\n30705\n30659\n30644\n30599\n30519\n30299\n30195\n30087\n29810\n29507\n29243\n29152\n29045\n28967\n28924\n28851\n28850\n28717\n28567\n28374\n28142\n28114\n27770\n27537\n27521\n27372\n27351\n27316\n27204\n27087\n27075\n27074\n27055\n27016\n27015\n26972\n26669\n26417\n26340\n26007\n26001\n25847\n25717\n25703\n25486\n25473\n25445\n25327\n25288\n25262\n25260\n25174\n24999\n24616\n24552\n24416\n24392\n24218\n23953\n23887\n23723\n23451\n23430\n23382\n23342\n23296\n23270\n23228\n23219\n23040\n23017\n22969\n22959\n22882\n22769\n22727\n22719\n22711\n22563\n22341\n22290\n22223\n22200\n22177\n22100\n22063\n22022\n21915\n21891\n21728\n21634\n21631\n21473\n21078\n21011\n20990\n20940\n20934\n20883\n20734\n20473\n20280\n20228\n20227\n20226\n20225\n20224\n20223\n20180\n20179\n20147\n20127\n20125\n20118\n20111\n20106\n20102\n20089\n20085\n20080\n20076\n20052\n20039\n20032\n20021\n20017\n20011\n19996\n19995\n19852\n19715\n19634\n19612\n19501\n19464\n19403\n19353\n19201\n19200\n19130\n19010\n18962\n18910\n18887\n18874\n18669\n18569\n18517\n18505\n18439\n18380\n18337\n18336\n18231\n18148\n18080\n18015\n18012\n17997\n17985\n17969\n17867\n17860\n17802\n17801\n17715\n17702\n17701\n17700\n17413\n17409\n17255\n17251\n17129\n17089\n17070\n17017\n17016\n16901\n16845\n16797\n16725\n16724\n16723\n16464\n16372\n16349\n16297\n16286\n16283\n16273\n16270\n16048\n15915\n15758\n15730\n15722\n15677\n15670\n15646\n15645\n15631\n15550\n15448\n15344\n15317\n15275\n15191\n15190\n15145\n15050\n15005\n14916\n14891\n14827\n14733\n14693\n14545\n14534\n14444\n14443\n14418\n14254\n14237\n14218\n14147\n13899\n13846\n13784\n13766\n13730\n13723\n13695\n13580\n13502\n13359\n13340\n13318\n13306\n13265\n13264\n13261\n13250\n13229\n13194\n13193\n13192\n13188\n13167\n13149\n13142\n13140\n13132\n13130\n13093\n13017\n12962\n12955\n12892\n12891\n12766\n12702\n12699\n12414\n12340\n12296\n12275\n12271\n12251\n12243\n12240\n12225\n12192\n12171\n12156\n12146\n12137\n12132\n12097\n12096\n12090\n12080\n12077\n12034\n12031\n12019\n11940\n11863\n11862\n11813\n11735\n11697\n11552\n11401\n11296\n11288\n11250\n11224\n11200\n11180\n11100\n11089\n11033\n11032\n11031\n11026\n11019\n11007\n11003\n10900\n10878\n10852\n10842\n10754\n10699\n10602\n10601\n10567\n10565\n10556\n10555\n10554\n10553\n10552\n10551\n10550\n10535\n10529\n10509\n10494\n10414\n10387\n10357\n10347\n10338\n10280\n10255\n10246\n10245\n10238\n10093\n10064\n10045\n10042\n10035\n10019\n10018\n1327\n2330\n2580\n2700\n1584\n9020\n3281\n2439\n1250\n14001\n1607\n1736\n1330\n2270\n2728\n2888\n3803\n5250\n1645\n1303\n3636\n1251\n1243\n1291\n1297\n1200\n1811\n4442\n1118\n8401\n2101\n2889\n1694\n1730\n1912\n29015\n28015\n1745\n2250\n1306\n2997\n2449\n1262\n4007\n1101\n1268\n1735\n1858\n1264\n1711\n3118\n4601\n1321\n1598\n1305\n1632\n9995\n1307\n1981\n2532\n1808\n2435\n1194\n1622\n1239\n1799\n2882\n1683\n3063\n3062\n1340\n4447\n1806\n6888\n2438\n1261\n5969\n9343\n2583\n2031\n3798\n2269\n20001\n2622\n11001\n1207\n2850\n21201\n2908\n3936\n3023\n2280\n2623\n7099\n2372\n1318\n1339\n1276\n11000\n48619\n3497\n1209\n1331\n1240\n3856\n2987\n2326\n25001\n25000\n1792\n3919\n1299\n2984\n1715\n1703\n1677\n2086\n1708\n1228\n3787\n5502\n1620\n1316\n1569\n1210\n1691\n1282\n2124\n1791\n2150\n9909\n4022\n3868\n1324\n2584\n2300\n9287\n2806\n1566\n1713\n1592\n3749\n1302\n1709\n3485\n2418\n2472\n24554\n3146\n2134\n2898\n9161\n9160\n2930\n1319\n5672\n3811\n2456\n2901\n6579\n2550\n8403\n31416\n22273\n7005\n66\n32786\n32787\n706\n914\n635\n6105\n400\n47\n830\n4008\n5977\n1989\n1444\n3985\n678\n27001\n591\n642\n446\n1441\n54320\n11\n769\n983\n979\n973\n967\n965\n961\n942\n935\n926\n925\n863\n858\n844\n834\n817\n815\n811\n809\n789\n779\n743\n1019\n1507\n1492\n509\n762\n5632\n578\n1495\n5308\n52\n219\n525\n1420\n665\n620\n3064\n3045\n653\n158\n716\n861\n9991\n3049\n1366\n1364\n833\n91\n1680\n3398\n750\n615\n603\n6110\n101\n989\n27010\n510\n810\n1139\n4199\n76\n847\n649\n707\n68\n449\n664\n75\n104\n629\n1652\n682\n577\n985\n984\n974\n958\n952\n949\n946\n923\n916\n899\n897\n894\n889\n835\n824\n814\n807\n804\n798\n733\n727\n237\n12\n10\n501\n122\n440\n771\n1663\n828\n860\n695\n634\n538\n1359\n1358\n1517\n1370\n3900\n492\n268\n27374\n605\n8076\n1651\n1178\n6401\n761\n5145\n50\n2018\n1349\n2014\n7597\n2120\n1445\n1402\n1465\n9104\n627\n4660\n7273\n950\n1384\n1388\n760\n92\n831\n5978\n4557\n45\n112\n456\n1214\n3086\n702\n6665\n1404\n651\n5300\n6347\n5400\n1389\n647\n448\n1356\n5232\n1484\n450\n1991\n1988\n1523\n1400\n1399\n221\n1385\n5191\n1346\n2024\n2430\n988\n962\n948\n945\n941\n938\n936\n929\n927\n919\n906\n883\n881\n875\n872\n870\n866\n855\n851\n850\n841\n836\n826\n820\n819\n816\n813\n791\n745\n736\n735\n724\n719\n343\n334\n300\n28\n249\n230\n16\n1018\n1016\n658\n1474\n696\n630\n663\n2307\n1552\n609\n741\n353\n638\n1551\n661\n491\n640\n507\n673\n632\n1354\n9105\n6143\n676\n214\n14141\n182\n69\n27665\n1475\n97\n633\n560\n799\n7009\n2015\n628\n751\n4480\n1403\n8123\n1527\n723\n1466\n1486\n1650\n991\n832\n137\n1348\n685\n1762\n6701\n994\n4500\n194\n180\n1539\n1379\n51\n886\n2064\n1405\n1435\n11371\n1401\n1369\n402\n103\n1372\n704\n854\n8892\n47557\n624\n1387\n3397\n1996\n1995\n1997\n18182\n18184\n3264\n3292\n13720\n9107\n9106\n201\n1381\n35\n6588\n5530\n3141\n670\n970\n968\n964\n963\n960\n959\n951\n947\n944\n939\n933\n909\n895\n891\n879\n869\n868\n867\n837\n821\n812\n797\n796\n794\n788\n756\n734\n721\n718\n708\n703\n60\n40\n253\n231\n14\n1017\n1003\n656\n975\n2026\n1497\n553\n511\n611\n689\n1668\n1664\n15\n561\n997\n505\n1496\n637\n213\n1412\n1515\n692\n694\n681\n680\n644\n675\n1467\n454\n622\n1476\n1373\n770\n262\n654\n1535\n58\n177\n26208\n677\n1519\n1398\n3457\n401\n412\n493\n13713\n94\n1498\n871\n1390\n6145\n133\n362\n118\n193\n115\n1549\n7008\n608\n1426\n1436\n915\n38\n74\n73\n71\n601\n136\n4144\n129\n16444\n1446\n4132\n308\n1528\n1365\n1393\n1394\n1493\n138\n5997\n397\n29\n31\n44\n2627\n6147\n1510\n568\n350\n2053\n6146\n6544\n1763\n3531\n399\n1537\n1992\n1355\n1454\n261\n887\n200\n1376\n1424\n6111\n1410\n1409\n686\n5301\n5302\n1513\n747\n9051\n1499\n7006\n1439\n1438\n8770\n853\n196\n93\n410\n462\n619\n1529\n1990\n1994\n1986\n1386\n18183\n18181\n6700\n1442\n95\n6400\n1432\n1548\n486\n1422\n114\n1397\n6142\n1827\n626\n422\n688\n206\n202\n204\n1483\n7634\n774\n699\n2023\n776\n672\n1545\n2431\n697\n982\n978\n972\n966\n957\n956\n934\n920\n908\n907\n892\n890\n885\n884\n882\n877\n876\n865\n857\n852\n849\n842\n838\n827\n818\n793\n785\n784\n755\n746\n738\n737\n717\n34\n336\n325\n303\n276\n273\n236\n235\n233\n181\n604\n1362\n712\n1437\n2027\n1368\n1531\n645\n65301\n260\n536\n764\n698\n607\n1667\n1662\n1661\n404\n224\n418\n176\n848\n315\n466\n403\n1456\n1479\n355\n763\n1472\n453\n759\n437\n2432\n120\n415\n1544\n1511\n1538\n346\n173\n54\n56\n265\n1462\n13701\n1518\n1457\n117\n1470\n13715\n13714\n267\n1419\n1418\n1407\n380\n518\n65\n391\n392\n413\n1391\n614\n1408\n162\n108\n4987\n1502\n598\n582\n487\n530\n1509\n72\n4672\n189\n209\n270\n7464\n408\n191\n1459\n5714\n5717\n5713\n564\n767\n583\n1395\n192\n1448\n428\n4133\n1416\n773\n1458\n526\n1363\n742\n1464\n1427\n1482\n569\n571\n6141\n351\n3984\n5490\n2\n13718\n373\n17300\n910\n148\n7326\n271\n423\n1451\n480\n1430\n1429\n781\n383\n2564\n613\n612\n652\n5303\n1383\n128\n19150\n1453\n190\n1505\n1371\n533\n27009\n27007\n27005\n27003\n27002\n744\n1423\n1374\n141\n1440\n1396\n352\n96\n48\n552\n570\n217\n528\n452\n451\n2766\n2108\n132\n1993\n1987\n130\n18187\n216\n3421\n142\n13721\n67\n15151\n364\n1411\n205\n6548\n124\n116\n5193\n258\n485\n599\n149\n1469\n775\n2019\n516\n986\n977\n976\n955\n954\n937\n932\n8\n896\n893\n845\n768\n766\n739\n337\n329\n326\n305\n295\n294\n293\n289\n288\n277\n238\n234\n229\n228\n226\n522\n2028\n150\n572\n596\n420\n460\n1543\n358\n361\n470\n360\n457\n643\n322\n168\n753\n369\n185\n43188\n1541\n1540\n752\n496\n662\n1449\n1480\n1473\n184\n1672\n1671\n1670\n435\n434\n1532\n1360\n174\n472\n1361\n17007\n414\n535\n432\n479\n473\n151\n1542\n438\n1488\n1508\n618\n316\n1367\n439\n284\n542\n370\n2016\n248\n1491\n44123\n41230\n7173\n5670\n18136\n3925\n7088\n1425\n17755\n17756\n4072\n5841\n2102\n4123\n2989\n10051\n10050\n31029\n3726\n5243\n9978\n9925\n6061\n6058\n6057\n6056\n6054\n6053\n6049\n6048\n6047\n6046\n6045\n6044\n6043\n6042\n6041\n6040\n6039\n6038\n6037\n6036\n6035\n6034\n6033\n6032\n6031\n6029\n6028\n6027\n6026\n6024\n6023\n6022\n6020\n6019\n6018\n6016\n6014\n6013\n6012\n6011\n36462\n5793\n3423\n3424\n4095\n3646\n3510\n3722\n2459\n3651\n14500\n3865\n15345\n3763\n38422\n3877\n9092\n5344\n3974\n2341\n6116\n2157\n165\n6936\n8041\n4888\n4889\n3074\n2165\n4389\n5770\n5769\n16619\n11876\n11877\n3741\n3633\n3840\n3717\n3716\n3590\n2805\n4537\n9762\n5007\n5006\n5358\n4879\n6114\n4185\n2784\n3724\n2596\n2595\n4417\n4845\n22321\n22289\n3219\n1338\n36411\n3861\n5166\n3674\n1785\n534\n6602\n47001\n5363\n8912\n2231\n5747\n5748\n11208\n7236\n4049\n4050\n22347\n63\n3233\n3359\n8908\n4177\n48050\n3111\n3427\n5321\n5320\n3702\n2907\n8991\n8990\n2054\n4847\n9802\n9800\n4368\n5990\n3563\n5744\n5743\n12321\n12322\n9206\n9204\n9205\n9201\n9203\n2949\n2948\n6626\n37472\n8199\n4145\n3482\n2216\n13708\n3786\n3375\n7566\n2539\n2387\n3317\n2410\n2255\n3883\n4299\n4296\n4295\n4293\n4292\n4291\n4290\n4289\n4288\n4287\n4286\n4285\n4284\n4283\n4282\n4281\n4280\n4278\n4277\n4276\n4275\n4274\n4273\n4272\n4271\n4270\n4269\n4268\n4267\n4266\n4265\n4264\n4263\n4261\n4260\n4259\n4258\n4257\n4256\n4255\n4254\n4253\n4251\n4250\n4249\n4248\n4247\n4246\n4245\n4244\n4241\n4240\n4239\n4238\n4237\n4236\n4235\n4233\n4232\n4231\n4230\n4229\n4228\n4227\n4226\n4225\n4223\n4222\n4221\n4219\n4218\n4217\n4216\n4215\n4214\n4213\n4212\n4211\n4210\n4209\n4208\n4207\n4205\n4204\n4203\n4202\n4201\n2530\n5164\n28200\n3845\n3541\n4052\n21590\n1796\n25793\n8699\n8182\n4991\n2474\n5780\n3676\n24249\n1631\n6672\n6673\n3601\n5046\n3509\n1852\n2386\n8473\n7802\n4789\n3555\n12013\n12012\n3752\n3245\n3231\n16666\n6678\n17184\n9086\n9598\n3073\n2074\n1956\n2610\n3738\n2994\n2993\n2802\n1885\n14149\n13786\n10100\n9284\n14150\n10107\n4032\n2821\n3207\n14154\n24323\n2771\n5646\n2426\n18668\n2554\n4188\n3654\n8034\n5675\n15118\n4031\n2529\n2248\n1142\n19194\n433\n3534\n3664\n2537\n519\n2655\n4184\n1506\n3098\n7887\n37654\n1979\n9629\n2357\n1889\n3314\n3313\n4867\n2696\n3217\n6306\n1189\n5281\n8953\n1910\n13894\n372\n3720\n1382\n2542\n3584\n4034\n145\n27999\n3791\n21800\n2670\n3492\n24678\n34249\n39681\n1846\n5197\n5462\n5463\n2862\n2977\n2978\n3468\n2675\n3474\n4422\n12753\n13709\n2573\n3012\n4307\n4725\n3346\n3686\n4070\n9555\n4711\n4323\n4322\n10200\n7727\n3608\n3959\n2405\n3858\n3857\n24322\n6118\n4176\n6442\n8937\n17224\n17225\n7234\n33434\n1906\n22351\n2158\n5153\n3885\n24465\n3040\n20167\n8066\n474\n2739\n3308\n590\n3309\n7902\n7901\n7903\n20046\n5582\n5583\n7872\n13716\n13717\n13705\n6252\n2915\n1965\n3459\n3160\n3754\n3243\n10261\n7932\n7933\n5450\n11971\n379\n7548\n1832\n28080\n3805\n16789\n8320\n8321\n4423\n2296\n7359\n7358\n7357\n7356\n7355\n7354\n7353\n7352\n7351\n7350\n7349\n7348\n7347\n7346\n7344\n7343\n7342\n7341\n7340\n7339\n7338\n7337\n7336\n7335\n7334\n7333\n7332\n7331\n7330\n7329\n7328\n7327\n7324\n7323\n7322\n7321\n7319\n7318\n7317\n7316\n7315\n7314\n7313\n7312\n7311\n7310\n7309\n7308\n7307\n7306\n7305\n7304\n7303\n7302\n7301\n8140\n5196\n5195\n6130\n5474\n5471\n5472\n5470\n4146\n3713\n5048\n31457\n7631\n3544\n41121\n11600\n3696\n3549\n1380\n22951\n22800\n3521\n2060\n6083\n9668\n3552\n1814\n1977\n2576\n2729\n24680\n13710\n13712\n25900\n2403\n2402\n2470\n5203\n3579\n2306\n1450\n7015\n7012\n7011\n22763\n2156\n2493\n4019\n4018\n4017\n4015\n2392\n3175\n32249\n1627\n10104\n2609\n5406\n3251\n4094\n3241\n6514\n6418\n3734\n2679\n4953\n5008\n2880\n8243\n8280\n26133\n8555\n5629\n3547\n5639\n5638\n5637\n5115\n3723\n4950\n3895\n3894\n3491\n3318\n6419\n3185\n243\n3212\n9536\n1925\n11171\n8404\n8405\n8989\n6787\n6483\n3867\n3866\n1860\n1870\n5306\n3816\n7588\n6786\n2084\n11165\n11161\n11163\n11162\n11164\n3708\n4850\n7677\n16959\n247\n3478\n5349\n3854\n5397\n7411\n9612\n11173\n9293\n5027\n5026\n5705\n8778\n527\n1312\n8808\n6144\n4157\n4156\n3249\n7471\n3615\n5777\n2154\n45966\n17235\n3018\n38800\n2737\n156\n3807\n2876\n1759\n7981\n3606\n3647\n3438\n4683\n9306\n9312\n7016\n33334\n3413\n3834\n3835\n2440\n6121\n8668\n2568\n17185\n7982\n2290\n2569\n2863\n1964\n4738\n2132\n17777\n16162\n6551\n3230\n4538\n3884\n9282\n9281\n4882\n5146\n580\n1967\n2659\n2409\n5416\n2657\n3380\n5417\n2658\n5161\n5162\n10162\n10161\n33656\n7560\n2599\n2704\n2703\n4170\n7734\n9522\n3158\n4426\n4786\n2721\n1608\n3516\n4988\n4408\n1847\n36423\n2826\n2827\n3556\n8111\n6456\n6455\n3874\n3611\n2629\n2630\n166\n5059\n3110\n1733\n40404\n2257\n2278\n4750\n4303\n3688\n4751\n5794\n4752\n7626\n16950\n3273\n3896\n3635\n1959\n4753\n2857\n4163\n1659\n2905\n2904\n2733\n4936\n5032\n3048\n29000\n28240\n2320\n4742\n22335\n22333\n5043\n4105\n1257\n3841\n43210\n4366\n5163\n11106\n5434\n6444\n6445\n5634\n5636\n5635\n6343\n4546\n3242\n5568\n4057\n24666\n21221\n6488\n6484\n6486\n6485\n6487\n6443\n6480\n6489\n7690\n2603\n4787\n2367\n9212\n9213\n5445\n45824\n8351\n13711\n4076\n5099\n2316\n3588\n5093\n9450\n8056\n8055\n8054\n8059\n8058\n8057\n8053\n3090\n3255\n2254\n2479\n2477\n2478\n4194\n3496\n3495\n2089\n38865\n9026\n9025\n9024\n9023\n3480\n1905\n3550\n7801\n2189\n5361\n32635\n3782\n3432\n3978\n6629\n3143\n7784\n2342\n2309\n2705\n2310\n2384\n6315\n5343\n9899\n5168\n5167\n3927\n266\n2577\n5307\n3838\n19007\n7708\n37475\n7701\n5435\n3499\n2719\n3352\n25576\n3942\n1644\n3755\n5574\n5573\n7542\n9310\n1129\n4079\n3038\n8768\n4033\n9401\n9402\n20012\n20013\n30832\n1606\n5410\n5422\n5409\n9801\n7743\n14034\n14033\n4952\n21801\n3452\n2760\n3153\n23272\n2578\n5156\n8554\n7401\n3771\n3138\n3137\n3500\n6900\n363\n3455\n1698\n13217\n2752\n3864\n10201\n6568\n2377\n3677\n520\n2258\n4124\n8051\n2223\n3194\n4041\n48653\n8270\n5693\n25471\n2416\n5994\n9208\n7810\n7870\n2249\n7473\n4664\n4590\n2777\n2776\n2057\n6148\n3296\n4410\n4684\n8230\n5842\n1431\n12109\n4756\n4336\n324\n323\n3019\n39\n2225\n4733\n30100\n2999\n3422\n107\n1232\n3418\n3537\n5\n8184\n3789\n5231\n4731\n4373\n45045\n12302\n2373\n6084\n16665\n16385\n18635\n18634\n10253\n7227\n3572\n3032\n5786\n2346\n2348\n2347\n2349\n45002\n3553\n43191\n5313\n3707\n3706\n3736\n32811\n1942\n44553\n35001\n35002\n35005\n35006\n35003\n35004\n532\n2214\n5569\n3142\n2332\n3768\n2774\n2773\n6099\n2167\n2714\n2713\n3533\n4037\n2457\n1953\n9345\n21553\n2408\n2736\n2188\n18104\n1813\n469\n1596\n3178\n5430\n5676\n2177\n4841\n5028\n7980\n3166\n3554\n3566\n3843\n5677\n7040\n2589\n8153\n10055\n5464\n2497\n4354\n9222\n5083\n5082\n45825\n2612\n6980\n5689\n6209\n2523\n2490\n2468\n3543\n5543\n7794\n4193\n4951\n3951\n4093\n7747\n7997\n8117\n6140\n2873\n4329\n320\n319\n597\n3453\n4457\n2303\n5360\n4487\n409\n344\n1460\n5716\n5715\n9640\n5798\n7663\n7798\n7797\n4352\n15999\n34962\n34963\n34964\n4749\n8032\n4182\n1283\n1778\n3248\n2722\n2039\n3650\n3133\n2618\n4168\n10631\n1392\n3910\n6716\n47809\n38638\n4690\n9280\n6163\n2315\n3607\n5630\n4455\n4456\n1587\n28001\n5134\n13224\n13223\n5507\n2443\n4150\n8432\n7172\n3710\n9889\n6464\n7787\n6771\n6770\n3055\n2487\n16310\n16311\n3540\n34379\n34378\n2972\n7633\n6355\n188\n2790\n32400\n4351\n3934\n3933\n4659\n1819\n5586\n5863\n17010\n9318\n318\n5318\n2634\n4416\n5078\n3189\n6924\n3010\n15740\n1603\n2787\n4390\n468\n4869\n4868\n3177\n3347\n6124\n2350\n3208\n2520\n2441\n3109\n3557\n281\n1916\n4313\n5312\n4066\n345\n9630\n9631\n6817\n3582\n9279\n9278\n8027\n3587\n4747\n2178\n5112\n3135\n5443\n7880\n1980\n6086\n3254\n4012\n9597\n3253\n2274\n2299\n8444\n6655\n44322\n44321\n5351\n5350\n5172\n4172\n1332\n2256\n8129\n8128\n4097\n8161\n2665\n2664\n6162\n4189\n1333\n3735\n586\n6581\n6582\n4681\n4312\n4989\n7216\n3348\n3095\n6657\n30002\n7237\n3435\n2246\n1675\n31400\n4311\n9559\n6671\n6679\n3034\n40853\n11103\n3274\n3355\n3078\n3075\n3076\n8070\n2484\n2483\n3891\n1571\n1830\n1630\n8997\n8102\n2482\n2481\n5155\n5575\n3718\n22005\n22004\n22003\n22002\n2524\n1829\n2237\n3977\n3976\n3303\n19191\n3433\n5724\n2400\n7629\n6640\n2389\n30999\n2447\n3673\n7430\n7429\n7426\n7431\n7428\n7427\n9390\n4317\n35357\n7728\n8004\n5045\n8688\n1258\n5757\n5729\n5767\n5766\n5755\n5768\n4743\n9008\n9007\n3187\n20014\n4089\n3434\n4840\n4843\n3100\n314\n3154\n9994\n9993\n8767\n4304\n2428\n2199\n2198\n2185\n4428\n4429\n4162\n4395\n2056\n5402\n3340\n3339\n3341\n3338\n7275\n7274\n7277\n7276\n4359\n2077\n8769\n9966\n4732\n3320\n11175\n11174\n11172\n13706\n3523\n429\n2697\n18186\n3442\n3441\n29167\n36602\n7030\n1894\n28000\n126\n4420\n2184\n3780\n49001\n11235\n4128\n8711\n10810\n45001\n5415\n4453\n359\n3266\n36424\n2868\n7724\n396\n2645\n23402\n23400\n23401\n3016\n21010\n5215\n4663\n4803\n2338\n15126\n8433\n5209\n3406\n3405\n5627\n4088\n2210\n2244\n2817\n10111\n10110\n1242\n5299\n2252\n3649\n6421\n6420\n1617\n48001\n48002\n48003\n48005\n48004\n48000\n61\n8061\n4134\n38412\n20048\n7393\n4021\n178\n8457\n550\n2058\n2075\n2076\n3165\n6133\n2614\n2585\n4702\n4701\n2586\n3203\n3204\n4460\n16361\n16367\n16360\n16368\n4159\n170\n2293\n4703\n8981\n3409\n7549\n171\n20049\n1155\n537\n3196\n3195\n2411\n2788\n4127\n6777\n6778\n1879\n5421\n3440\n2128\n21846\n21849\n21847\n21848\n395\n154\n155\n4425\n2328\n3129\n3641\n3640\n1970\n2486\n2485\n6842\n6841\n3149\n3148\n3150\n3151\n1406\n218\n10116\n10114\n2219\n2735\n10117\n10113\n2220\n3725\n5229\n4350\n6513\n4335\n4334\n5681\n1676\n2971\n4409\n3131\n4441\n1612\n1616\n1613\n1614\n13785\n11104\n11105\n3829\n11095\n3507\n3213\n7474\n3886\n4043\n2730\n377\n378\n3024\n2738\n2528\n4844\n4842\n5979\n1888\n2093\n2094\n20034\n2163\n3159\n6317\n4361\n2895\n3753\n2343\n3015\n1790\n3950\n6363\n9286\n9285\n7282\n6446\n2273\n33060\n2388\n9119\n3733\n32801\n4421\n7420\n9903\n6622\n5354\n7742\n2305\n2791\n8115\n3122\n2855\n8276\n2871\n4554\n2171\n2172\n2173\n2174\n7680\n3343\n7392\n3958\n3358\n46\n6634\n8503\n3924\n2488\n10544\n10543\n10541\n10540\n10542\n4691\n8666\n1576\n4986\n6997\n3732\n4688\n7871\n9632\n7869\n2593\n3764\n5237\n4668\n4173\n4667\n8077\n4310\n7606\n5136\n4069\n21554\n7391\n9445\n2180\n3180\n2621\n4551\n3008\n7013\n7014\n5362\n6601\n1512\n5356\n6074\n5726\n5364\n5725\n6076\n6075\n2175\n3132\n5359\n2176\n5022\n4679\n4680\n6509\n2266\n6382\n2230\n6390\n6370\n6360\n393\n2311\n8787\n18\n8786\n47000\n19788\n1960\n9596\n4603\n4151\n4552\n11211\n3569\n4883\n3571\n2944\n2945\n2272\n7720\n5157\n3445\n2427\n2727\n2363\n46999\n2789\n13930\n3232\n2688\n3235\n5598\n3115\n3117\n3116\n3331\n3332\n3302\n3330\n3558\n8809\n3570\n4153\n2591\n4179\n4171\n3276\n5540\n4360\n8448\n4458\n7421\n49000\n7073\n3836\n5282\n8384\n36700\n4686\n269\n9255\n6201\n2544\n2516\n5092\n2243\n4902\n313\n3691\n2453\n4345\n44900\n36444\n36443\n4894\n3747\n3746\n5044\n6471\n3079\n4913\n4741\n10805\n3487\n3157\n3068\n8162\n4083\n4082\n4081\n7026\n1983\n2289\n1629\n1628\n1634\n8101\n6482\n5254\n5058\n4044\n3591\n3592\n1903\n5062\n6087\n2090\n2465\n2466\n6200\n8208\n8207\n8204\n31620\n8205\n8206\n3278\n2145\n2143\n2147\n2146\n3767\n46336\n10933\n4341\n1969\n10809\n12300\n8191\n517\n4670\n7365\n3028\n3027\n3029\n1203\n1886\n11430\n374\n2212\n3407\n2816\n2779\n2815\n2780\n3373\n3739\n3815\n4347\n11796\n3970\n4547\n1764\n2395\n4372\n4432\n9747\n4371\n3360\n3361\n4331\n40023\n27504\n2294\n5253\n7697\n35354\n186\n30260\n4566\n584\n5696\n6623\n6620\n6621\n2502\n3112\n36865\n2918\n4661\n31016\n26262\n26263\n3642\n48048\n5309\n3155\n4166\n27442\n6583\n3215\n3214\n8901\n19020\n4160\n3094\n3093\n3777\n1937\n1938\n1939\n1940\n2097\n1936\n1810\n6244\n6243\n6242\n6241\n4107\n19541\n3529\n3528\n5230\n4327\n5883\n2205\n7095\n3794\n3473\n3472\n7181\n5034\n3627\n8091\n1578\n5673\n5049\n4880\n3258\n2828\n3719\n7478\n7280\n1636\n1637\n3775\n24321\n499\n3205\n1950\n1949\n3226\n8148\n5047\n4075\n17223\n21000\n3504\n3206\n2632\n529\n4073\n32034\n18769\n2527\n4593\n4792\n4791\n7031\n33435\n4740\n4739\n4068\n20202\n4737\n9214\n2215\n3743\n2088\n7410\n5728\n45054\n3614\n8020\n11751\n2202\n6697\n4744\n1884\n3699\n6714\n1611\n7202\n4569\n3508\n24386\n16995\n16994\n1674\n1673\n7128\n4746\n17234\n9215\n4486\n484\n5057\n5056\n7624\n2980\n4109\n49150\n215\n23005\n23004\n23003\n23002\n23001\n23000\n2716\n3560\n5597\n134\n38001\n38000\n4067\n1428\n2480\n5029\n8067\n5069\n3156\n3139\n244\n7675\n7673\n7672\n7674\n2637\n4139\n3783\n3657\n11320\n8615\n585\n48128\n2239\n3596\n2055\n3186\n19000\n5165\n3420\n17220\n17221\n19998\n2404\n2079\n4152\n4604\n25604\n5742\n5741\n4553\n2799\n4801\n4802\n2063\n14143\n14142\n4061\n4062\n4063\n4064\n31948\n31949\n2276\n2275\n1881\n2078\n3660\n3661\n1920\n1919\n9085\n424\n1933\n1934\n9089\n9088\n3667\n3666\n12003\n12004\n3539\n3538\n3267\n25100\n385\n3494\n4594\n4595\n4596\n3898\n9614\n4169\n5674\n2374\n5105\n8313\n44323\n5628\n2570\n2113\n4591\n4592\n5228\n5224\n5227\n2207\n4484\n3037\n2209\n2448\n3101\n382\n381\n3209\n7510\n2206\n2690\n2208\n7738\n5317\n3329\n5316\n3449\n2029\n1985\n10125\n2597\n3634\n8231\n3250\n43438\n4884\n4117\n2467\n4148\n18516\n7397\n22370\n8807\n3921\n4306\n10860\n6440\n3740\n1161\n2641\n7630\n3804\n4197\n11108\n9954\n6791\n3623\n3769\n3036\n5315\n5305\n3542\n5304\n11720\n2517\n3179\n2979\n2356\n3745\n18262\n2186\n35356\n3436\n2152\n2123\n1452\n4729\n3761\n3136\n28010\n9340\n9339\n8710\n30400\n6267\n6269\n6268\n3757\n4755\n4754\n4026\n5117\n9277\n2947\n3386\n2217\n37483\n16002\n5687\n2072\n1909\n9122\n9123\n4131\n3912\n3229\n1880\n5688\n4332\n10800\n4985\n3108\n3475\n6080\n4790\n23053\n6081\n8190\n7017\n7283\n4730\n2159\n3429\n2660\n14145\n3484\n3762\n3222\n8322\n1421\n1859\n31765\n2914\n3051\n38201\n8881\n4340\n8074\n2678\n2677\n4110\n2731\n286\n3402\n3272\n1514\n3382\n1904\n1902\n3648\n2975\n574\n8502\n3488\n9217\n4130\n7726\n5556\n7244\n4319\n41111\n4411\n4084\n2242\n4396\n4901\n7545\n7544\n27008\n27006\n27004\n5579\n2884\n3035\n1193\n5618\n7018\n2673\n4086\n8043\n8044\n3192\n3729\n1855\n1856\n1784\n24922\n1887\n7164\n4349\n7394\n16021\n16020\n6715\n4915\n4122\n3216\n14250\n3152\n1776\n36524\n4320\n4727\n3225\n2819\n4038\n6417\n347\n3047\n2495\n10081\n38202\n19790\n2515\n2514\n4353\n38472\n10102\n4085\n3953\n4788\n3088\n3134\n3639\n4309\n2755\n1928\n5075\n26486\n5401\n3759\n43440\n1926\n1982\n1798\n9981\n4536\n4535\n1504\n592\n1267\n6935\n2036\n6316\n2221\n44818\n34980\n2380\n2379\n6107\n1772\n8416\n8417\n8266\n4023\n3629\n9617\n3679\n3727\n4942\n4941\n4940\n43439\n3628\n3620\n5116\n3259\n4666\n4669\n3819\n37601\n5084\n5085\n3383\n5599\n5600\n5601\n3665\n1818\n3044\n1295\n7962\n7117\n121\n17754\n6636\n6635\n20480\n23333\n3585\n6322\n6321\n4091\n4092\n140\n6656\n3693\n11623\n11723\n13218\n3682\n3218\n9083\n3197\n3198\n394\n2526\n7700\n7707\n2916\n2917\n4370\n6515\n12010\n5398\n3564\n4346\n1378\n1893\n3525\n3638\n2228\n6632\n3392\n3671\n6159\n3462\n3461\n3464\n3465\n3460\n3463\n3123\n34567\n8149\n6703\n6702\n2263\n3477\n3524\n6160\n17729\n3711\n45678\n2168\n3328\n38462\n3932\n3295\n2164\n3395\n2874\n3246\n3247\n4191\n4028\n3489\n4556\n5684\n13929\n31685\n9987\n4060\n13819\n13820\n13821\n13818\n13822\n2420\n7547\n3685\n2193\n4427\n1930\n8913\n7021\n7020\n5719\n5565\n5245\n6326\n6320\n6325\n3522\n44544\n13400\n6088\n3568\n8567\n3567\n5567\n7165\n4142\n3161\n5352\n195\n1172\n5993\n3199\n3574\n4059\n1177\n3624\n19999\n4646\n21212\n246\n5107\n14002\n7171\n3448\n3336\n3335\n3337\n198\n197\n3447\n5031\n4605\n2464\n2227\n3223\n1335\n2226\n33333\n2762\n2761\n3227\n3228\n33331\n2861\n2860\n2098\n4301\n3252\n547\n546\n6785\n8750\n4330\n3776\n24850\n8805\n2763\n4167\n2092\n3444\n8415\n3714\n1278\n5700\n3668\n7569\n365\n8894\n8893\n8891\n8890\n11202\n3988\n1160\n3938\n6117\n6624\n6625\n2073\n461\n3612\n3578\n11109\n2229\n1775\n2764\n3678\n6511\n1133\n29999\n2594\n3881\n3498\n8732\n2378\n3394\n3393\n2298\n2297\n9388\n9387\n3120\n3297\n1898\n8442\n9888\n4183\n4673\n3778\n5271\n3127\n1932\n4451\n2563\n4452\n9346\n7022\n3631\n3630\n105\n3271\n2699\n3004\n2129\n4187\n1724\n3113\n2314\n8380\n8377\n8376\n8379\n8378\n20810\n3818\n41797\n41796\n38002\n3364\n3366\n2824\n2823\n3609\n4055\n4054\n4053\n2654\n19220\n9093\n3183\n2565\n4078\n4774\n2153\n17222\n7551\n7563\n3072\n4047\n9695\n4846\n5992\n5683\n4692\n3191\n3417\n7169\n3973\n46998\n16384\n3947\n47100\n6970\n2491\n7023\n10321\n42508\n3822\n2417\n2555\n3257\n3256\n22343\n64\n7215\n20003\n4450\n3751\n3605\n2534\n3490\n4419\n7689\n21213\n7574\n3377\n3779\n44444\n3039\n2415\n2183\n26257\n3576\n3575\n2976\n7168\n8501\n164\n3384\n7550\n45514\n356\n2617\n3730\n6688\n6687\n6690\n7683\n2052\n3481\n4136\n4137\n9087\n172\n1729\n4980\n7229\n7228\n24754\n2897\n7279\n2512\n2513\n4870\n22305\n5787\n6633\n131\n15555\n4051\n4785\n43441\n5784\n7546\n8017\n3887\n5194\n1743\n2891\n3770\n1377\n4316\n4314\n3099\n1572\n39063\n1891\n1892\n3349\n18241\n18243\n18242\n18185\n5505\n6556\n562\n531\n3772\n5065\n5064\n2182\n3893\n2921\n2922\n13832\n4074\n4140\n4115\n3056\n3616\n3559\n4970\n4969\n3114\n3750\n12168\n2122\n7129\n7162\n7167\n5270\n1197\n9060\n3106\n12546\n5247\n5246\n3290\n4728\n8998\n8610\n8609\n3756\n8614\n8613\n8612\n8611\n1872\n3583\n24676\n4377\n5079\n4378\n1734\n3545\n7262\n3675\n2552\n22537\n3709\n14414\n5251\n1882\n42509\n2318\n4326\n1563\n7163\n1554\n7161\n595\n348\n282\n8026\n5249\n5248\n5154\n10880\n3626\n4990\n3107\n6410\n6409\n6408\n6407\n6406\n6405\n6404\n4677\n581\n4671\n2964\n2965\n28589\n47808\n3966\n2446\n1854\n1961\n2444\n2277\n4175\n3188\n3043\n9380\n3692\n5682\n2155\n4104\n4103\n4102\n3593\n2845\n2844\n4186\n2218\n4678\n2017\n2913\n7648\n4914\n7687\n6501\n9750\n3344\n1896\n4568\n10128\n6768\n6767\n3182\n1313\n3181\n2059\n3604\n6300\n10129\n3695\n6301\n2494\n2625\n48129\n8195\n2369\n2574\n5750\n13823\n13216\n4027\n5068\n25955\n25954\n6946\n3411\n24577\n5429\n2259\n4621\n6784\n4676\n4675\n4784\n3785\n5425\n5424\n4305\n3960\n3408\n5584\n5585\n1943\n3124\n6508\n6507\n4155\n1120\n1929\n4324\n10439\n6506\n6505\n6122\n4971\n3387\n152\n2635\n2169\n6696\n2204\n3512\n2071\n10260\n35100\n4195\n3277\n3502\n2066\n2238\n4413\n20057\n2992\n2050\n3965\n10990\n31020\n4685\n1140\n7508\n16003\n4071\n3104\n3437\n5067\n33123\n1146\n44600\n2264\n7543\n2419\n32896\n2317\n3821\n4937\n1520\n11367\n4154\n3617\n20999\n1170\n1171\n2864\n27876\n4485\n4704\n7235\n3087\n45000\n4405\n4404\n4406\n4402\n4403\n4400\n5727\n11489\n2192\n4077\n4448\n3581\n5150\n13702\n3451\n386\n8211\n7166\n3518\n27782\n3176\n9292\n3174\n9295\n9294\n3426\n8423\n3140\n7570\n421\n2114\n6344\n2581\n2582\n11321\n384\n23546\n1834\n1115\n4165\n1557\n3758\n7847\n5086\n4849\n2037\n1447\n3312\n187\n4488\n2336\n387\n208\n207\n203\n3454\n10548\n4674\n38203\n3239\n3236\n3237\n3238\n4573\n2758\n10252\n2759\n8121\n2754\n8122\n3184\n42999\n539\n6082\n18888\n9952\n9951\n7846\n7845\n6549\n5456\n5455\n5454\n4851\n5913\n5072\n3939\n2247\n1206\n3715\n2646\n3054\n5671\n8040\n376\n2640\n30004\n30003\n5192\n4393\n4392\n4391\n4394\n1931\n5506\n8301\n4563\n35355\n4011\n7799\n3265\n9209\n693\n36001\n9956\n9955\n6627\n3234\n2667\n2668\n3613\n4804\n2887\n3416\n3833\n9216\n2846\n17555\n2786\n3316\n3021\n3026\n4878\n3917\n4362\n7775\n3224\n23457\n23456\n4549\n4431\n2295\n3573\n5073\n3760\n3357\n3954\n3705\n3704\n2692\n6769\n33890\n7170\n2521\n2085\n3096\n2810\n2859\n3431\n9389\n3655\n5106\n5103\n44445\n7509\n6801\n4013\n2476\n2475\n2334\n12007\n12008\n6868\n4046\n18463\n32483\n4030\n8793\n62\n1955\n3781\n3619\n3618\n28119\n4726\n4502\n4597\n4598\n3598\n3597\n3125\n4149\n9953\n23294\n2933\n2934\n5783\n5782\n5785\n5781\n15363\n48049\n2339\n5265\n5264\n1181\n3446\n3428\n15998\n3091\n2133\n3774\n317\n3832\n508\n3721\n1619\n1716\n2279\n3412\n2327\n6558\n2130\n1760\n5413\n2396\n2923\n3378\n3466\n2504\n2720\n4871\n7395\n3926\n1727\n1326\n2518\n1890\n2781\n565\n4984\n3342\n21845\n1963\n2851\n3748\n1739\n1269\n2455\n2547\n2548\n2546\n7779\n2695\n312\n2996\n2893\n1589\n2649\n1224\n1345\n3625\n2538\n3321\n175\n1868\n4344\n1853\n3058\n3802\n78\n2770\n3270\n575\n1771\n4839\n4838\n4837\n671\n430\n431\n2745\n2648\n3356\n1957\n2820\n1978\n2927\n2499\n2437\n2138\n2110\n1797\n1737\n483\n390\n1867\n1624\n1833\n2879\n2767\n2768\n2943\n1568\n2489\n1237\n2741\n2742\n8804\n1588\n6069\n1869\n2642\n20670\n594\n2885\n2669\n476\n2798\n3083\n3082\n3081\n2361\n5104\n1758\n7491\n1728\n5428\n1946\n559\n1610\n3144\n1922\n2726\n6149\n1838\n4014\n1274\n2647\n4106\n6102\n4548\n19540\n1866\n6965\n6966\n6964\n6963\n1751\n1625\n5453\n2709\n7967\n3354\n566\n4178\n2986\n1226\n1836\n1654\n2838\n1692\n3644\n6071\n477\n478\n2507\n1923\n3193\n2653\n2636\n1621\n3379\n2533\n2892\n2452\n1684\n2333\n22000\n1553\n3536\n11201\n2775\n2942\n2941\n2940\n2939\n2938\n2613\n426\n4116\n4412\n1966\n3065\n1225\n1705\n1618\n1660\n2545\n2676\n3687\n2756\n1599\n2832\n2831\n2830\n2829\n5461\n2974\n498\n1626\n3595\n160\n153\n3326\n1714\n3172\n3173\n3171\n3170\n3169\n2235\n6108\n169\n5399\n2471\n558\n2308\n1681\n2385\n3562\n5024\n5025\n5427\n3391\n3744\n1646\n3275\n3698\n2390\n1793\n1647\n1697\n1693\n1695\n1696\n2919\n9599\n2423\n3844\n2959\n2818\n1817\n521\n3147\n3163\n2886\n283\n2837\n2543\n2928\n2240\n1343\n2321\n3467\n9753\n1530\n2872\n1595\n2900\n1341\n2935\n3059\n2724\n3385\n2765\n368\n2461\n2462\n1253\n2680\n3009\n2434\n2694\n2351\n2353\n2354\n1788\n2352\n3662\n2355\n2091\n1732\n8183\n1678\n2588\n2924\n2687\n5071\n1777\n2899\n494\n3875\n2937\n5437\n5436\n3469\n3285\n1293\n5272\n2865\n321\n1280\n1779\n6432\n1230\n2843\n3033\n2566\n1562\n3085\n3892\n1246\n1564\n8160\n1633\n9997\n9996\n7511\n5236\n3955\n2956\n2954\n2953\n5310\n2951\n2936\n6951\n2413\n2407\n1597\n1570\n2398\n1809\n1575\n1754\n1748\n22001\n3855\n2368\n8764\n6653\n5314\n2267\n3244\n2661\n2364\n506\n2322\n2498\n3305\n183\n650\n2329\n5991\n1463\n159\n8450\n1917\n1921\n2839\n2503\n25903\n25901\n25902\n2556\n2672\n1690\n2360\n2671\n1669\n1665\n1286\n4138\n2592\n61441\n61439\n61440\n2983\n5465\n1843\n1842\n1841\n2061\n1329\n2451\n3701\n3066\n2442\n5771\n2450\n489\n8834\n1285\n3262\n2881\n2883\n43189\n6064\n1591\n1744\n405\n2397\n2683\n2162\n1288\n2286\n2236\n167\n1685\n1831\n2981\n467\n1574\n2743\n19398\n2469\n2460\n1477\n1478\n5720\n3535\n1582\n1731\n679\n2684\n2686\n2681\n2685\n1952\n9397\n9344\n2952\n2579\n2561\n1235\n367\n8665\n471\n2926\n1815\n7786\n8033\n1581\n7979\n1534\n490\n3070\n349\n1824\n2511\n1897\n6070\n2118\n2117\n1231\n24003\n24004\n24006\n24000\n3594\n24002\n24001\n24005\n5418\n2698\n8763\n1820\n1899\n2587\n8911\n8910\n1593\n2535\n4181\n3565\n2559\n3069\n2620\n1298\n2540\n2541\n2125\n1487\n2283\n2284\n2285\n2281\n2282\n2813\n5355\n2814\n2795\n1555\n1968\n2611\n245\n4042\n1682\n1485\n2560\n2841\n2370\n2842\n2840\n398\n2424\n1773\n1649\n287\n2656\n2213\n2822\n1289\n3471\n3470\n3042\n4114\n6962\n6961\n1567\n2808\n1706\n2406\n2508\n2506\n1623\n13160\n2166\n2866\n2982\n1275\n1573\n4348\n1828\n3084\n1609\n2853\n3589\n147\n3501\n1643\n1642\n1245\n43190\n2962\n2963\n576\n2549\n1579\n1585\n503\n1907\n3202\n3548\n3060\n2652\n2633\n16991\n495\n1602\n1490\n2793\n18881\n2854\n2319\n2233\n3345\n2454\n8130\n8131\n2127\n2970\n2932\n3164\n1710\n11319\n27345\n2801\n1284\n2995\n3797\n2966\n2590\n549\n1725\n2337\n3130\n5813\n25008\n25007\n25006\n25005\n25004\n25003\n25002\n25009\n6850\n1344\n1604\n8733\n2572\n1260\n1586\n1726\n6999\n6998\n2140\n2139\n2141\n1577\n4180\n4827\n1877\n2715\n19412\n19410\n19411\n5404\n5403\n2985\n1803\n2744\n6790\n2575\n12172\n1789\n35000\n1281\n14937\n14936\n263\n375\n5094\n1816\n2245\n1238\n2778\n9321\n2643\n2421\n488\n1850\n2458\n41\n2519\n6109\n1774\n2833\n3862\n3381\n1590\n2626\n1738\n2732\n19539\n2849\n2358\n1786\n1787\n1657\n2429\n1747\n1746\n5408\n5407\n2359\n24677\n1874\n2946\n2509\n1873\n2747\n2751\n2750\n2748\n2749\n9396\n3067\n1848\n9374\n2510\n2615\n1689\n4682\n3350\n24242\n3401\n3294\n3293\n5503\n5504\n5746\n5745\n2344\n7437\n3353\n2689\n3873\n1561\n1915\n2792\n10103\n26260\n26261\n589\n1948\n2666\n26489\n26487\n2769\n2674\n6066\n1876\n2835\n2834\n2782\n16309\n2969\n2867\n2797\n2950\n1822\n1342\n5135\n2650\n2109\n2051\n2912\n309\n1865\n3289\n1804\n3286\n1740\n2211\n2707\n1273\n2181\n2553\n2896\n2858\n3610\n2651\n1325\n2445\n1265\n3053\n1292\n1878\n4098\n1780\n1795\n4099\n1821\n2151\n1227\n436\n2287\n32636\n1489\n1263\n5419\n3041\n2496\n3287\n6073\n2234\n242\n1844\n2362\n11112\n1941\n3046\n1945\n6072\n2960\n5426\n2753\n3298\n1702\n1256\n1254\n1266\n2562\n1656\n1655\n579\n1255\n1415\n2365\n2345\n6104\n8132\n1908\n3282\n1857\n1679\n2870\n3458\n5420\n772\n3645\n551\n1686\n3773\n4379\n1851\n3022\n2807\n2890\n1837\n2955\n3145\n1471\n1468\n40841\n40842\n40843\n2422\n6253\n455\n2746\n3201\n5984\n2324\n3288\n5412\n2137\n1648\n1802\n4308\n48556\n2757\n1757\n1294\n7174\n1944\n371\n504\n1741\n2931\n3020\n17219\n3903\n1768\n1767\n1766\n1765\n2856\n1640\n1639\n1794\n3987\n2571\n2412\n3315\n2116\n3061\n2836\n3450\n3105\n1756\n9283\n2906\n588\n1202\n1375\n2803\n2536\n1252\n2619\n1323\n2990\n1304\n2961\n6402\n6403\n3561\n1770\n1769\n2877\n10288\n2911\n2032\n2663\n2662\n1962\n310\n357\n354\n482\n2414\n2852\n1951\n1704\n3327\n573\n567\n2708\n2131\n2772\n3643\n1749\n5042\n1913\n2624\n1826\n2136\n2616\n9164\n9163\n9162\n1781\n2929\n1320\n2848\n2268\n459\n1536\n2639\n6831\n10080\n1845\n1653\n1849\n463\n2740\n2473\n2783\n1481\n2785\n2331\n7107\n1219\n3279\n5411\n2796\n2149\n7781\n1205\n4108\n4885\n1546\n2894\n1601\n2878\n5605\n5604\n5602\n5603\n3284\n1742\n" }, { "path": "bbot/wordlists/valid_url_schemes.txt", "content": "aaa\nawb\naaas\nabout\nacap\nacct\nacd\nacr\nadiumxtra\nadt\nafp\nafs\naim\namss\nandroid\nappdata\napt\nar\nark\nat\nattachment\naw\nbarion\nbb\nbeshare\nbitcoin\nbitcoincash\nblob\nbolo\nbrid\nbrowserext\ncabal\ncalculator\ncallto\ncap\ncast\ncasts\nchrome\nchrome-extension\ncid\ncoap\ncoap+tcp\ncoap+ws\ncoaps\ncoaps+tcp\ncoaps+ws\ncom-eventbrite-attendee\ncontent\ncontent-type\ncrid\ncstr\ncvs\ndab\ndat\ndata\ndav\ndhttp\ndiaspora\ndict\ndid\ndis\ndlna-playcontainer\ndlna-playsingle\ndns\ndntp\ndoi\ndpp\ndrm\ndrop\ndtmi\ndtn\ndvb\ndvx\ndweb\ned2k\neid\nelsi\nembedded\nens\nethereum\nexample\nfacetime\nfax\nfeed\nfeedready\nfido\nfile\nfilesystem\nfinger\nfirst-run-pen-experience\nfish\nfm\nftp\nfuchsia-pkg\ngeo\ngg\ngit\ngitoid\ngizmoproject\ngo\ngopher\ngraph\ngrd\ngtalk\nh323\nham\nhcap\nhcp\nhs20\nhttp\nhttps\nhxxp\nhxxps\nhydrazone\nhyper\niax\nicap\nicon\nim\nimap\ninfo\niotdisco\nipfs\nipn\nipns\nipp\nipps\nirc\nirc6\nircs\niris\niris.beep\niris.lwz\niris.xpc\niris.xpcs\nisostore\nitms\njabber\njar\njms\nkeyparc\nlastfm\nlbry\nldap\nldaps\nleaptofrogans\nlid\nlorawan\nlpa\nlvlt\nmachineProvisioningProgressReporter\nmagnet\nmailserver\nmailto\nmaps\nmarket\nmatrix\nmessage\nmicrosoft.windows.camera\nmicrosoft.windows.camera.multipicker\nmicrosoft.windows.camera.picker\nmid\nmms\nmodem\nmongodb\nmoz\nms-access\nms-appinstaller\nms-browser-extension\nms-calculator\nms-drive-to\nms-enrollment\nms-excel\nms-eyecontrolspeech\nms-gamebarservices\nms-gamingoverlay\nms-getoffice\nms-help\nms-infopath\nms-inputapp\nms-launchremotedesktop\nms-lockscreencomponent-config\nms-media-stream-id\nms-meetnow\nms-mixedrealitycapture\nms-mobileplans\nms-newsandinterests\nms-officeapp\nms-people\nms-project\nms-powerpoint\nms-publisher\nms-recall\nms-remotedesktop\nms-remotedesktop-launch\nms-restoretabcompanion\nms-screenclip\nms-screensketch\nms-search\nms-search-repair\nms-secondary-screen-controller\nms-secondary-screen-setup\nms-settings\nms-settings-airplanemode\nms-settings-bluetooth\nms-settings-camera\nms-settings-cellular\nms-settings-cloudstorage\nms-settings-connectabledevices\nms-settings-displays-topology\nms-settings-emailandaccounts\nms-settings-language\nms-settings-location\nms-settings-lock\nms-settings-nfctransactions\nms-settings-notifications\nms-settings-power\nms-settings-privacy\nms-settings-proximity\nms-settings-screenrotation\nms-settings-wifi\nms-settings-workplace\nms-spd\nms-stickers\nms-sttoverlay\nms-transit-to\nms-useractivityset\nms-virtualtouchpad\nms-visio\nms-walk-to\nms-whiteboard\nms-whiteboard-cmd\nms-word\nmsnim\nmsrp\nmsrps\nmss\nmt\nmtqp\nmumble\nmupdate\nmvn\nmvrp\nmvrps\nnews\nnfs\nni\nnih\nnntp\nnotes\nnum\nocf\noid\nonenote\nonenote-cmd\nopaquelocktoken\nopenid\nopenpgp4fpr\notpauth\np1\npack\npalm\npaparazzi\npayment\npayto\npkcs11\nplatform\npop\npres\nprospero\nproxy\npwid\npsyc\npttp\nqb\nquery\nquic-transport\nredis\nrediss\nreload\nres\nresource\nrmi\nrsync\nrtmfp\nrtmp\nrtsp\nrtsps\nrtspu\nsarif\nsecondlife\nsecret-token\nservice\nsession\nsftp\nsgn\nshc\nshttp\nsieve\nsimpleledger\nsimplex\nsip\nsips\nskype\nsmb\nsmp\nsms\nsmtp\nsnews\nsnmp\nsoap.beep\nsoap.beeps\nsoldat\nspiffe\nspotify\nssb\nssh\nstarknet\nsteam\nstun\nstuns\nsubmit\nsvn\nswh\nswid\nswidpath\ntag\ntaler\nteamspeak\ntel\nteliaeid\ntelnet\ntftp\nthings\nthismessage\ntip\ntn3270\ntool\nturn\nturns\ntv\nudp\nunreal\nupt\nurn\nut2004\nuuid-in-package\nv-event\nvemmi\nventrilo\nves\nvideotex\nvnc\nview-source\nvscode\nvscode-insiders\nvsls\nw3\nwais\nweb3\nwcr\nwebcal\nweb+ap\nwifi\nwpid\nws\nwss\nwtai\nwyciwyg\nxcon\nxcon-userid\nxfire\nxmlrpc.beep\nxmlrpc.beeps\nxmpp\nxftp\nxrcp\nxri\nymsgr\nz39.50\nz39.50r\nz39.50s\n" }, { "path": "bbot-docker.sh", "content": "# OUTPUTS SCAN DATA TO ~/.bbot/scans\n\ndocker run --rm -it -v \"$HOME/.bbot/scans:/root/.bbot/scans\" -v \"$HOME/.config/bbot:/root/.config/bbot\" blacklanternsecurity/bbot:stable \"$@\"\n" }, { "path": "codecov.yml", "content": "coverage:\n range: 25..75\n round: up\n precision: 0\n status:\n project: off\n patch: off\ngithub_checks: false\n" }, { "path": "docs/comparison.md", "content": "# Comparison to Other Tools\n\nBBOT does a lot more than just subdomain enumeration. However, subdomain enumeration is arguably the most important part of OSINT, and since there's so many subdomain enumeration tools out there, they're the easiest class of tool to compare it to.\n\nThanks to BBOT's recursive nature (and its `dnsbrute_mutations` module with its NLP-powered subdomain mutations), it typically finds about 20-25% more than other tools such as `Amass` or `theHarvester`. This holds true especially for larger targets like `delta.com` (1000+ subdomains):\n\n### Subdomains Found\n\n![subdomains](https://github.com/blacklanternsecurity/bbot/assets/20261699/0d7eb982-e68a-4a33-b33c-7c8ba8c7d6ad)\n\n### Runtimes (Lower is Better)\n\n![runtimes](https://github.com/blacklanternsecurity/bbot/assets/20261699/66cafb5f-045b-4d88-9ffa-7542b3dada4f)\n\nFor a detailed analysis of this data, please see [Subdomain Enumeration Tool Face-Off](https://blog.blacklanternsecurity.com/p/subdomain-enumeration-tool-face-off-4e5)\n\n### Ebay.com (larger domain)\n\n![subdomain-stats-ebay](https://github.com/blacklanternsecurity/bbot/assets/20261699/53e07e9f-50b6-4b70-9e83-297dbfbcb436)\n\n_Note that in this benchmark, Spiderfoot crashed after ~20 minutes due to excessive memory usage. Amass never finished and had to be cancelled after 24h. All other tools finished successfully._\n" }, { "path": "docs/contribution.md", "content": "# Contribution\n\nWe welcome contributions! If you have an idea for a new module, or are a Python developer who wants to get involved, please fork us or come talk to us on [Discord](https://discord.com/invite/PZqkgxu5SA).\n\nTo get started devving, see the following links:\n\n- [Setting up a Dev Environment](./dev/dev_environment.md)\n- [How to Write a BBOT Module](./dev/module_howto.md)\n- [Discord Bot Example](./dev/discord_bot.md)\n" }, { "path": "docs/data/chord_graph/entities.json", "content": "[\n {\n \"id\": 77777777,\n \"name\": \"root\"\n },\n {\n \"id\": 99999999,\n \"name\": \"module\",\n \"parent\": 77777777\n },\n {\n \"id\": 88888888,\n \"name\": \"event_type\",\n \"parent\": 77777777\n },\n {\n \"id\": 13,\n \"name\": \"ASN\",\n \"parent\": 88888888,\n \"consumes\": [],\n \"produces\": [\n 11\n ]\n },\n {\n \"id\": 141,\n \"name\": \"AZURE_TENANT\",\n \"parent\": 88888888,\n \"consumes\": [\n 140\n ],\n \"produces\": []\n },\n {\n \"id\": 46,\n \"name\": \"CODE_REPOSITORY\",\n \"parent\": 88888888,\n \"consumes\": [\n 65,\n 84,\n 85,\n 89,\n 92,\n 127,\n 148\n ],\n \"produces\": [\n 45,\n 66,\n 83,\n 86,\n 87,\n 90,\n 91,\n 126\n ]\n },\n {\n \"id\": 7,\n \"name\": \"DNS_NAME\",\n \"parent\": 88888888,\n \"consumes\": [\n 6,\n 15,\n 19,\n 21,\n 22,\n 26,\n 28,\n 29,\n 30,\n 32,\n 33,\n 34,\n 35,\n 36,\n 38,\n 39,\n 43,\n 44,\n 47,\n 52,\n 53,\n 54,\n 55,\n 56,\n 58,\n 59,\n 60,\n 61,\n 62,\n 64,\n 70,\n 81,\n 86,\n 88,\n 96,\n 100,\n 107,\n 111,\n 113,\n 116,\n 117,\n 121,\n 122,\n 124,\n 128,\n 132,\n 133,\n 134,\n 135,\n 136,\n 137,\n 140,\n 143,\n 144,\n 145,\n 147,\n 151,\n 154,\n 155,\n 157\n ],\n \"produces\": [\n 6,\n 21,\n 28,\n 35,\n 36,\n 38,\n 39,\n 40,\n 43,\n 44,\n 52,\n 53,\n 55,\n 58,\n 59,\n 60,\n 61,\n 62,\n 63,\n 81,\n 96,\n 100,\n 107,\n 111,\n 114,\n 116,\n 117,\n 121,\n 128,\n 132,\n 134,\n 135,\n 136,\n 140,\n 142,\n 143,\n 144,\n 147,\n 151,\n 152,\n 154,\n 155,\n 157\n ]\n },\n {\n \"id\": 23,\n \"name\": \"DNS_NAME_UNRESOLVED\",\n \"parent\": 88888888,\n \"consumes\": [\n 22,\n 140,\n 145\n ],\n \"produces\": []\n },\n {\n \"id\": 48,\n \"name\": \"EMAIL_ADDRESS\",\n \"parent\": 88888888,\n \"consumes\": [\n 71\n ],\n \"produces\": [\n 47,\n 54,\n 60,\n 64,\n 70,\n 88,\n 100,\n 122,\n 133,\n 137,\n 142\n ]\n },\n {\n \"id\": 10,\n \"name\": \"FILESYSTEM\",\n \"parent\": 88888888,\n \"consumes\": [\n 75,\n 106,\n 148,\n 149\n ],\n \"produces\": [\n 8,\n 65,\n 79,\n 84,\n 85,\n 89,\n 106,\n 127,\n 149\n ]\n },\n {\n \"id\": 4,\n \"name\": \"FINDING\",\n \"parent\": 88888888,\n \"consumes\": [\n 15,\n 159\n ],\n \"produces\": [\n 1,\n 22,\n 24,\n 26,\n 27,\n 29,\n 30,\n 32,\n 33,\n 34,\n 37,\n 83,\n 91,\n 95,\n 97,\n 99,\n 108,\n 109,\n 112,\n 114,\n 115,\n 118,\n 119,\n 129,\n 130,\n 135,\n 138,\n 140,\n 146,\n 148,\n 150,\n 160\n ]\n },\n {\n \"id\": 103,\n \"name\": \"GEOLOCATION\",\n \"parent\": 88888888,\n \"consumes\": [],\n \"produces\": [\n 102,\n 105\n ]\n },\n {\n \"id\": 49,\n \"name\": \"HASHED_PASSWORD\",\n \"parent\": 88888888,\n \"consumes\": [],\n \"produces\": [\n 47,\n 54\n ]\n },\n {\n \"id\": 2,\n \"name\": \"HTTP_RESPONSE\",\n \"parent\": 88888888,\n \"consumes\": [\n 1,\n 15,\n 27,\n 69,\n 72,\n 79,\n 91,\n 97,\n 112,\n 113,\n 114,\n 118,\n 119,\n 120,\n 140,\n 146,\n 148,\n 160\n ],\n \"produces\": [\n 98\n ]\n },\n {\n \"id\": 12,\n \"name\": \"IP_ADDRESS\",\n \"parent\": 88888888,\n \"consumes\": [\n 11,\n 15,\n 40,\n 102,\n 104,\n 105,\n 113,\n 124,\n 135,\n 140\n ],\n \"produces\": [\n 15,\n 40,\n 63,\n 104,\n 140\n ]\n },\n {\n \"id\": 125,\n \"name\": \"IP_RANGE\",\n \"parent\": 88888888,\n \"consumes\": [\n 124,\n 140\n ],\n \"produces\": []\n },\n {\n \"id\": 9,\n \"name\": \"MOBILE_APP\",\n \"parent\": 88888888,\n \"consumes\": [\n 8\n ],\n \"produces\": [\n 92\n ]\n },\n {\n \"id\": 16,\n \"name\": \"OPEN_TCP_PORT\",\n \"parent\": 88888888,\n \"consumes\": [\n 15,\n 80,\n 98,\n 113,\n 123,\n 142\n ],\n \"produces\": [\n 15,\n 40,\n 124,\n 135,\n 140\n ]\n },\n {\n \"id\": 41,\n \"name\": \"OPEN_UDP_PORT\",\n \"parent\": 88888888,\n \"consumes\": [],\n \"produces\": [\n 40\n ]\n },\n {\n \"id\": 67,\n \"name\": \"ORG_STUB\",\n \"parent\": 88888888,\n \"consumes\": [\n 66,\n 87,\n 92,\n 126\n ],\n \"produces\": [\n 140\n ]\n },\n {\n \"id\": 50,\n \"name\": \"PASSWORD\",\n \"parent\": 88888888,\n \"consumes\": [],\n \"produces\": [\n 47,\n 54\n ]\n },\n {\n \"id\": 42,\n \"name\": \"PROTOCOL\",\n \"parent\": 88888888,\n \"consumes\": [\n 108,\n 110,\n 113\n ],\n \"produces\": [\n 40,\n 80\n ]\n },\n {\n \"id\": 57,\n \"name\": \"RAW_DNS_RECORD\",\n \"parent\": 88888888,\n \"consumes\": [],\n \"produces\": [\n 56,\n 63,\n 64\n ]\n },\n {\n \"id\": 73,\n \"name\": \"RAW_TEXT\",\n \"parent\": 88888888,\n \"consumes\": [\n 72,\n 148\n ],\n \"produces\": [\n 75\n ]\n },\n {\n \"id\": 68,\n \"name\": \"SOCIAL\",\n \"parent\": 88888888,\n \"consumes\": [\n 66,\n 87,\n 90,\n 91,\n 93,\n 126,\n 140\n ],\n \"produces\": [\n 66,\n 88,\n 91,\n 139\n ]\n },\n {\n \"id\": 25,\n \"name\": \"STORAGE_BUCKET\",\n \"parent\": 88888888,\n \"consumes\": [\n 24,\n 29,\n 30,\n 31,\n 32,\n 33,\n 34,\n 140\n ],\n \"produces\": [\n 29,\n 30,\n 32,\n 33,\n 34\n ]\n },\n {\n \"id\": 17,\n \"name\": \"TECHNOLOGY\",\n \"parent\": 88888888,\n \"consumes\": [\n 15,\n 91,\n 159,\n 160\n ],\n \"produces\": [\n 27,\n 40,\n 69,\n 91,\n 93,\n 115,\n 135,\n 160\n ]\n },\n {\n \"id\": 3,\n \"name\": \"URL\",\n \"parent\": 88888888,\n \"consumes\": [\n 1,\n 14,\n 15,\n 24,\n 37,\n 76,\n 82,\n 83,\n 93,\n 95,\n 98,\n 101,\n 109,\n 114,\n 115,\n 123,\n 131,\n 138,\n 140,\n 146,\n 150,\n 152,\n 156,\n 159\n ],\n \"produces\": [\n 93,\n 98\n ]\n },\n {\n \"id\": 78,\n \"name\": \"URL_HINT\",\n \"parent\": 88888888,\n \"consumes\": [\n 77\n ],\n \"produces\": [\n 101\n ]\n },\n {\n \"id\": 20,\n \"name\": \"URL_UNVERIFIED\",\n \"parent\": 88888888,\n \"consumes\": [\n 45,\n 79,\n 98,\n 116,\n 123,\n 130,\n 139,\n 140\n ],\n \"produces\": [\n 19,\n 28,\n 31,\n 40,\n 56,\n 60,\n 64,\n 66,\n 72,\n 76,\n 77,\n 86,\n 93,\n 100,\n 131,\n 133,\n 151,\n 157,\n 160\n ]\n },\n {\n \"id\": 51,\n \"name\": \"USERNAME\",\n \"parent\": 88888888,\n \"consumes\": [\n 140\n ],\n \"produces\": [\n 47,\n 54\n ]\n },\n {\n \"id\": 153,\n \"name\": \"VHOST\",\n \"parent\": 88888888,\n \"consumes\": [\n 159\n ],\n \"produces\": [\n 152\n ]\n },\n {\n \"id\": 5,\n \"name\": \"VULNERABILITY\",\n \"parent\": 88888888,\n \"consumes\": [\n 15,\n 159\n ],\n \"produces\": [\n 1,\n 14,\n 22,\n 24,\n 26,\n 27,\n 69,\n 82,\n 109,\n 110,\n 115,\n 135,\n 146,\n 148,\n 160\n ]\n },\n {\n \"id\": 18,\n \"name\": \"WAF\",\n \"parent\": 88888888,\n \"consumes\": [\n 15\n ],\n \"produces\": [\n 156\n ]\n },\n {\n \"id\": 94,\n \"name\": \"WEBSCREENSHOT\",\n \"parent\": 88888888,\n \"consumes\": [],\n \"produces\": [\n 93\n ]\n },\n {\n \"id\": 74,\n \"name\": \"WEB_PARAMETER\",\n \"parent\": 88888888,\n \"consumes\": [\n 99,\n 109,\n 118,\n 119,\n 120,\n 129,\n 158\n ],\n \"produces\": [\n 72,\n 118,\n 119,\n 120\n ]\n },\n {\n \"id\": 1,\n \"name\": \"ajaxpro\",\n \"parent\": 99999999,\n \"consumes\": [\n 2,\n 3\n ],\n \"produces\": [\n 4,\n 5\n ]\n },\n {\n \"id\": 6,\n \"name\": \"anubisdb\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 8,\n \"name\": \"apkpure\",\n \"parent\": 99999999,\n \"consumes\": [\n 9\n ],\n \"produces\": [\n 10\n ]\n },\n {\n \"id\": 11,\n \"name\": \"asn\",\n \"parent\": 99999999,\n \"consumes\": [\n 12\n ],\n \"produces\": [\n 13\n ]\n },\n {\n \"id\": 14,\n \"name\": \"aspnet_bin_exposure\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 5\n ]\n },\n {\n \"id\": 15,\n \"name\": \"asset_inventory\",\n \"parent\": 99999999,\n \"consumes\": [\n 7,\n 4,\n 2,\n 12,\n 16,\n 17,\n 3,\n 5,\n 18\n ],\n \"produces\": [\n 12,\n 16\n ]\n },\n {\n \"id\": 19,\n \"name\": \"azure_realm\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 20\n ]\n },\n {\n \"id\": 21,\n \"name\": \"azure_tenant\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 22,\n \"name\": \"baddns\",\n \"parent\": 99999999,\n \"consumes\": [\n 7,\n 23\n ],\n \"produces\": [\n 4,\n 5\n ]\n },\n {\n \"id\": 24,\n \"name\": \"baddns_direct\",\n \"parent\": 99999999,\n \"consumes\": [\n 25,\n 3\n ],\n \"produces\": [\n 4,\n 5\n ]\n },\n {\n \"id\": 26,\n \"name\": \"baddns_zone\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 4,\n 5\n ]\n },\n {\n \"id\": 27,\n \"name\": \"badsecrets\",\n \"parent\": 99999999,\n \"consumes\": [\n 2\n ],\n \"produces\": [\n 4,\n 17,\n 5\n ]\n },\n {\n \"id\": 28,\n \"name\": \"bevigil\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7,\n 20\n ]\n },\n {\n \"id\": 29,\n \"name\": \"bucket_amazon\",\n \"parent\": 99999999,\n \"consumes\": [\n 7,\n 25\n ],\n \"produces\": [\n 4,\n 25\n ]\n },\n {\n \"id\": 30,\n \"name\": \"bucket_digitalocean\",\n \"parent\": 99999999,\n \"consumes\": [\n 7,\n 25\n ],\n \"produces\": [\n 4,\n 25\n ]\n },\n {\n \"id\": 31,\n \"name\": \"bucket_file_enum\",\n \"parent\": 99999999,\n \"consumes\": [\n 25\n ],\n \"produces\": [\n 20\n ]\n },\n {\n \"id\": 32,\n \"name\": \"bucket_firebase\",\n \"parent\": 99999999,\n \"consumes\": [\n 7,\n 25\n ],\n \"produces\": [\n 4,\n 25\n ]\n },\n {\n \"id\": 33,\n \"name\": \"bucket_google\",\n \"parent\": 99999999,\n \"consumes\": [\n 7,\n 25\n ],\n \"produces\": [\n 4,\n 25\n ]\n },\n {\n \"id\": 34,\n \"name\": \"bucket_microsoft\",\n \"parent\": 99999999,\n \"consumes\": [\n 7,\n 25\n ],\n \"produces\": [\n 4,\n 25\n ]\n },\n {\n \"id\": 35,\n \"name\": \"bufferoverrun\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 36,\n \"name\": \"builtwith\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 37,\n \"name\": \"bypass403\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 4\n ]\n },\n {\n \"id\": 38,\n \"name\": \"c99\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 39,\n \"name\": \"censys_dns\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 40,\n \"name\": \"censys_ip\",\n \"parent\": 99999999,\n \"consumes\": [\n 12\n ],\n \"produces\": [\n 7,\n 12,\n 16,\n 41,\n 42,\n 17,\n 20\n ]\n },\n {\n \"id\": 43,\n \"name\": \"certspotter\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 44,\n \"name\": \"chaos\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 45,\n \"name\": \"code_repository\",\n \"parent\": 99999999,\n \"consumes\": [\n 20\n ],\n \"produces\": [\n 46\n ]\n },\n {\n \"id\": 47,\n \"name\": \"credshed\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 48,\n 49,\n 50,\n 51\n ]\n },\n {\n \"id\": 52,\n \"name\": \"crt\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 53,\n \"name\": \"crt_db\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 54,\n \"name\": \"dehashed\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 48,\n 49,\n 50,\n 51\n ]\n },\n {\n \"id\": 55,\n \"name\": \"digitorus\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 56,\n \"name\": \"dnsbimi\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 57,\n 20\n ]\n },\n {\n \"id\": 58,\n \"name\": \"dnsbrute\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 59,\n \"name\": \"dnsbrute_mutations\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 60,\n \"name\": \"dnscaa\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7,\n 48,\n 20\n ]\n },\n {\n \"id\": 61,\n \"name\": \"dnscommonsrv\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 62,\n \"name\": \"dnsdumpster\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 63,\n \"name\": \"dnsresolve\",\n \"parent\": 99999999,\n \"consumes\": [],\n \"produces\": [\n 7,\n 12,\n 57\n ]\n },\n {\n \"id\": 64,\n \"name\": \"dnstlsrpt\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 48,\n 57,\n 20\n ]\n },\n {\n \"id\": 65,\n \"name\": \"docker_pull\",\n \"parent\": 99999999,\n \"consumes\": [\n 46\n ],\n \"produces\": [\n 10\n ]\n },\n {\n \"id\": 66,\n \"name\": \"dockerhub\",\n \"parent\": 99999999,\n \"consumes\": [\n 67,\n 68\n ],\n \"produces\": [\n 46,\n 68,\n 20\n ]\n },\n {\n \"id\": 69,\n \"name\": \"dotnetnuke\",\n \"parent\": 99999999,\n \"consumes\": [\n 2\n ],\n \"produces\": [\n 17,\n 5\n ]\n },\n {\n \"id\": 70,\n \"name\": \"emailformat\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 48\n ]\n },\n {\n \"id\": 71,\n \"name\": \"emails\",\n \"parent\": 99999999,\n \"consumes\": [\n 48\n ],\n \"produces\": []\n },\n {\n \"id\": 72,\n \"name\": \"excavate\",\n \"parent\": 99999999,\n \"consumes\": [\n 2,\n 73\n ],\n \"produces\": [\n 20,\n 74\n ]\n },\n {\n \"id\": 75,\n \"name\": \"extractous\",\n \"parent\": 99999999,\n \"consumes\": [\n 10\n ],\n \"produces\": [\n 73\n ]\n },\n {\n \"id\": 76,\n \"name\": \"ffuf\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 20\n ]\n },\n {\n \"id\": 77,\n \"name\": \"ffuf_shortnames\",\n \"parent\": 99999999,\n \"consumes\": [\n 78\n ],\n \"produces\": [\n 20\n ]\n },\n {\n \"id\": 79,\n \"name\": \"filedownload\",\n \"parent\": 99999999,\n \"consumes\": [\n 2,\n 20\n ],\n \"produces\": [\n 10\n ]\n },\n {\n \"id\": 80,\n \"name\": \"fingerprintx\",\n \"parent\": 99999999,\n \"consumes\": [\n 16\n ],\n \"produces\": [\n 42\n ]\n },\n {\n \"id\": 81,\n \"name\": \"fullhunt\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 82,\n \"name\": \"generic_ssrf\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 5\n ]\n },\n {\n \"id\": 83,\n \"name\": \"git\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 46,\n 4\n ]\n },\n {\n \"id\": 84,\n \"name\": \"git_clone\",\n \"parent\": 99999999,\n \"consumes\": [\n 46\n ],\n \"produces\": [\n 10\n ]\n },\n {\n \"id\": 85,\n \"name\": \"gitdumper\",\n \"parent\": 99999999,\n \"consumes\": [\n 46\n ],\n \"produces\": [\n 10\n ]\n },\n {\n \"id\": 86,\n \"name\": \"github_codesearch\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 46,\n 20\n ]\n },\n {\n \"id\": 87,\n \"name\": \"github_org\",\n \"parent\": 99999999,\n \"consumes\": [\n 67,\n 68\n ],\n \"produces\": [\n 46\n ]\n },\n {\n \"id\": 88,\n \"name\": \"github_usersearch\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 48,\n 68\n ]\n },\n {\n \"id\": 89,\n \"name\": \"github_workflows\",\n \"parent\": 99999999,\n \"consumes\": [\n 46\n ],\n \"produces\": [\n 10\n ]\n },\n {\n \"id\": 90,\n \"name\": \"gitlab_com\",\n \"parent\": 99999999,\n \"consumes\": [\n 68\n ],\n \"produces\": [\n 46\n ]\n },\n {\n \"id\": 91,\n \"name\": \"gitlab_onprem\",\n \"parent\": 99999999,\n \"consumes\": [\n 2,\n 68,\n 17\n ],\n \"produces\": [\n 46,\n 4,\n 68,\n 17\n ]\n },\n {\n \"id\": 92,\n \"name\": \"google_playstore\",\n \"parent\": 99999999,\n \"consumes\": [\n 46,\n 67\n ],\n \"produces\": [\n 9\n ]\n },\n {\n \"id\": 93,\n \"name\": \"gowitness\",\n \"parent\": 99999999,\n \"consumes\": [\n 68,\n 3\n ],\n \"produces\": [\n 17,\n 3,\n 20,\n 94\n ]\n },\n {\n \"id\": 95,\n \"name\": \"graphql_introspection\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 4\n ]\n },\n {\n \"id\": 96,\n \"name\": \"hackertarget\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 97,\n \"name\": \"host_header\",\n \"parent\": 99999999,\n \"consumes\": [\n 2\n ],\n \"produces\": [\n 4\n ]\n },\n {\n \"id\": 98,\n \"name\": \"httpx\",\n \"parent\": 99999999,\n \"consumes\": [\n 16,\n 3,\n 20\n ],\n \"produces\": [\n 2,\n 3\n ]\n },\n {\n \"id\": 99,\n \"name\": \"hunt\",\n \"parent\": 99999999,\n \"consumes\": [\n 74\n ],\n \"produces\": [\n 4\n ]\n },\n {\n \"id\": 100,\n \"name\": \"hunterio\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7,\n 48,\n 20\n ]\n },\n {\n \"id\": 101,\n \"name\": \"iis_shortnames\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 78\n ]\n },\n {\n \"id\": 102,\n \"name\": \"ip2location\",\n \"parent\": 99999999,\n \"consumes\": [\n 12\n ],\n \"produces\": [\n 103\n ]\n },\n {\n \"id\": 104,\n \"name\": \"ipneighbor\",\n \"parent\": 99999999,\n \"consumes\": [\n 12\n ],\n \"produces\": [\n 12\n ]\n },\n {\n \"id\": 105,\n \"name\": \"ipstack\",\n \"parent\": 99999999,\n \"consumes\": [\n 12\n ],\n \"produces\": [\n 103\n ]\n },\n {\n \"id\": 106,\n \"name\": \"jadx\",\n \"parent\": 99999999,\n \"consumes\": [\n 10\n ],\n \"produces\": [\n 10\n ]\n },\n {\n \"id\": 107,\n \"name\": \"leakix\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 108,\n \"name\": \"legba\",\n \"parent\": 99999999,\n \"consumes\": [\n 42\n ],\n \"produces\": [\n 4\n ]\n },\n {\n \"id\": 109,\n \"name\": \"lightfuzz\",\n \"parent\": 99999999,\n \"consumes\": [\n 3,\n 74\n ],\n \"produces\": [\n 4,\n 5\n ]\n },\n {\n \"id\": 110,\n \"name\": \"medusa\",\n \"parent\": 99999999,\n \"consumes\": [\n 42\n ],\n \"produces\": [\n 5\n ]\n },\n {\n \"id\": 111,\n \"name\": \"myssl\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 112,\n \"name\": \"newsletters\",\n \"parent\": 99999999,\n \"consumes\": [\n 2\n ],\n \"produces\": [\n 4\n ]\n },\n {\n \"id\": 113,\n \"name\": \"nmap_xml\",\n \"parent\": 99999999,\n \"consumes\": [\n 7,\n 2,\n 12,\n 16,\n 42\n ],\n \"produces\": []\n },\n {\n \"id\": 114,\n \"name\": \"ntlm\",\n \"parent\": 99999999,\n \"consumes\": [\n 2,\n 3\n ],\n \"produces\": [\n 7,\n 4\n ]\n },\n {\n \"id\": 115,\n \"name\": \"nuclei\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 4,\n 17,\n 5\n ]\n },\n {\n \"id\": 116,\n \"name\": \"oauth\",\n \"parent\": 99999999,\n \"consumes\": [\n 7,\n 20\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 117,\n \"name\": \"otx\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 118,\n \"name\": \"paramminer_cookies\",\n \"parent\": 99999999,\n \"consumes\": [\n 2,\n 74\n ],\n \"produces\": [\n 4,\n 74\n ]\n },\n {\n \"id\": 119,\n \"name\": \"paramminer_getparams\",\n \"parent\": 99999999,\n \"consumes\": [\n 2,\n 74\n ],\n \"produces\": [\n 4,\n 74\n ]\n },\n {\n \"id\": 120,\n \"name\": \"paramminer_headers\",\n \"parent\": 99999999,\n \"consumes\": [\n 2,\n 74\n ],\n \"produces\": [\n 74\n ]\n },\n {\n \"id\": 121,\n \"name\": \"passivetotal\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 122,\n \"name\": \"pgp\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 48\n ]\n },\n {\n \"id\": 123,\n \"name\": \"portfilter\",\n \"parent\": 99999999,\n \"consumes\": [\n 16,\n 3,\n 20\n ],\n \"produces\": []\n },\n {\n \"id\": 124,\n \"name\": \"portscan\",\n \"parent\": 99999999,\n \"consumes\": [\n 7,\n 12,\n 125\n ],\n \"produces\": [\n 16\n ]\n },\n {\n \"id\": 126,\n \"name\": \"postman\",\n \"parent\": 99999999,\n \"consumes\": [\n 67,\n 68\n ],\n \"produces\": [\n 46\n ]\n },\n {\n \"id\": 127,\n \"name\": \"postman_download\",\n \"parent\": 99999999,\n \"consumes\": [\n 46\n ],\n \"produces\": [\n 10\n ]\n },\n {\n \"id\": 128,\n \"name\": \"rapiddns\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 129,\n \"name\": \"reflected_parameters\",\n \"parent\": 99999999,\n \"consumes\": [\n 74\n ],\n \"produces\": [\n 4\n ]\n },\n {\n \"id\": 130,\n \"name\": \"retirejs\",\n \"parent\": 99999999,\n \"consumes\": [\n 20\n ],\n \"produces\": [\n 4\n ]\n },\n {\n \"id\": 131,\n \"name\": \"robots\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 20\n ]\n },\n {\n \"id\": 132,\n \"name\": \"securitytrails\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 133,\n \"name\": \"securitytxt\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 48,\n 20\n ]\n },\n {\n \"id\": 134,\n \"name\": \"shodan_dns\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 135,\n \"name\": \"shodan_idb\",\n \"parent\": 99999999,\n \"consumes\": [\n 7,\n 12\n ],\n \"produces\": [\n 7,\n 4,\n 16,\n 17,\n 5\n ]\n },\n {\n \"id\": 136,\n \"name\": \"sitedossier\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 137,\n \"name\": \"skymem\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 48\n ]\n },\n {\n \"id\": 138,\n \"name\": \"smuggler\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 4\n ]\n },\n {\n \"id\": 139,\n \"name\": \"social\",\n \"parent\": 99999999,\n \"consumes\": [\n 20\n ],\n \"produces\": [\n 68\n ]\n },\n {\n \"id\": 140,\n \"name\": \"speculate\",\n \"parent\": 99999999,\n \"consumes\": [\n 141,\n 7,\n 23,\n 2,\n 12,\n 125,\n 68,\n 25,\n 3,\n 20,\n 51\n ],\n \"produces\": [\n 7,\n 4,\n 12,\n 16,\n 67\n ]\n },\n {\n \"id\": 142,\n \"name\": \"sslcert\",\n \"parent\": 99999999,\n \"consumes\": [\n 16\n ],\n \"produces\": [\n 7,\n 48\n ]\n },\n {\n \"id\": 143,\n \"name\": \"subdomaincenter\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 144,\n \"name\": \"subdomainradar\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 145,\n \"name\": \"subdomains\",\n \"parent\": 99999999,\n \"consumes\": [\n 7,\n 23\n ],\n \"produces\": []\n },\n {\n \"id\": 146,\n \"name\": \"telerik\",\n \"parent\": 99999999,\n \"consumes\": [\n 2,\n 3\n ],\n \"produces\": [\n 4,\n 5\n ]\n },\n {\n \"id\": 147,\n \"name\": \"trickest\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 148,\n \"name\": \"trufflehog\",\n \"parent\": 99999999,\n \"consumes\": [\n 46,\n 10,\n 2,\n 73\n ],\n \"produces\": [\n 4,\n 5\n ]\n },\n {\n \"id\": 149,\n \"name\": \"unarchive\",\n \"parent\": 99999999,\n \"consumes\": [\n 10\n ],\n \"produces\": [\n 10\n ]\n },\n {\n \"id\": 150,\n \"name\": \"url_manipulation\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 4\n ]\n },\n {\n \"id\": 151,\n \"name\": \"urlscan\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7,\n 20\n ]\n },\n {\n \"id\": 152,\n \"name\": \"vhost\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 7,\n 153\n ]\n },\n {\n \"id\": 154,\n \"name\": \"viewdns\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 155,\n \"name\": \"virustotal\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7\n ]\n },\n {\n \"id\": 156,\n \"name\": \"wafw00f\",\n \"parent\": 99999999,\n \"consumes\": [\n 3\n ],\n \"produces\": [\n 18\n ]\n },\n {\n \"id\": 157,\n \"name\": \"wayback\",\n \"parent\": 99999999,\n \"consumes\": [\n 7\n ],\n \"produces\": [\n 7,\n 20\n ]\n },\n {\n \"id\": 158,\n \"name\": \"web_parameters\",\n \"parent\": 99999999,\n \"consumes\": [\n 74\n ],\n \"produces\": []\n },\n {\n \"id\": 159,\n \"name\": \"web_report\",\n \"parent\": 99999999,\n \"consumes\": [\n 4,\n 17,\n 3,\n 153,\n 5\n ],\n \"produces\": []\n },\n {\n \"id\": 160,\n \"name\": \"wpscan\",\n \"parent\": 99999999,\n \"consumes\": [\n 2,\n 17\n ],\n \"produces\": [\n 4,\n 17,\n 20,\n 5\n ]\n }\n]" }, { "path": "docs/data/chord_graph/rels.json", "content": "[\n {\n \"source\": 1,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 1,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 1,\n \"type\": \"produces\"\n },\n {\n \"source\": 5,\n \"target\": 1,\n \"type\": \"produces\"\n },\n {\n \"source\": 6,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 6,\n \"type\": \"produces\"\n },\n {\n \"source\": 8,\n \"target\": 9,\n \"type\": \"consumes\"\n },\n {\n \"source\": 10,\n \"target\": 8,\n \"type\": \"produces\"\n },\n {\n \"source\": 11,\n \"target\": 12,\n \"type\": \"consumes\"\n },\n {\n \"source\": 13,\n \"target\": 11,\n \"type\": \"produces\"\n },\n {\n \"source\": 14,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 5,\n \"target\": 14,\n \"type\": \"produces\"\n },\n {\n \"source\": 15,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 15,\n \"target\": 4,\n \"type\": \"consumes\"\n },\n {\n \"source\": 15,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 15,\n \"target\": 12,\n \"type\": \"consumes\"\n },\n {\n \"source\": 15,\n \"target\": 16,\n \"type\": \"consumes\"\n },\n {\n \"source\": 15,\n \"target\": 17,\n \"type\": \"consumes\"\n },\n {\n \"source\": 15,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 15,\n \"target\": 5,\n \"type\": \"consumes\"\n },\n {\n \"source\": 15,\n \"target\": 18,\n \"type\": \"consumes\"\n },\n {\n \"source\": 12,\n \"target\": 15,\n \"type\": \"produces\"\n },\n {\n \"source\": 16,\n \"target\": 15,\n \"type\": \"produces\"\n },\n {\n \"source\": 19,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 20,\n \"target\": 19,\n \"type\": \"produces\"\n },\n {\n \"source\": 21,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 21,\n \"type\": \"produces\"\n },\n {\n \"source\": 22,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 22,\n \"target\": 23,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 22,\n \"type\": \"produces\"\n },\n {\n \"source\": 5,\n \"target\": 22,\n \"type\": \"produces\"\n },\n {\n \"source\": 24,\n \"target\": 25,\n \"type\": \"consumes\"\n },\n {\n \"source\": 24,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 24,\n \"type\": \"produces\"\n },\n {\n \"source\": 5,\n \"target\": 24,\n \"type\": \"produces\"\n },\n {\n \"source\": 26,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 26,\n \"type\": \"produces\"\n },\n {\n \"source\": 5,\n \"target\": 26,\n \"type\": \"produces\"\n },\n {\n \"source\": 27,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 27,\n \"type\": \"produces\"\n },\n {\n \"source\": 17,\n \"target\": 27,\n \"type\": \"produces\"\n },\n {\n \"source\": 5,\n \"target\": 27,\n \"type\": \"produces\"\n },\n {\n \"source\": 28,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 28,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 28,\n \"type\": \"produces\"\n },\n {\n \"source\": 29,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 29,\n \"target\": 25,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 29,\n \"type\": \"produces\"\n },\n {\n \"source\": 25,\n \"target\": 29,\n \"type\": \"produces\"\n },\n {\n \"source\": 30,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 30,\n \"target\": 25,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 30,\n \"type\": \"produces\"\n },\n {\n \"source\": 25,\n \"target\": 30,\n \"type\": \"produces\"\n },\n {\n \"source\": 31,\n \"target\": 25,\n \"type\": \"consumes\"\n },\n {\n \"source\": 20,\n \"target\": 31,\n \"type\": \"produces\"\n },\n {\n \"source\": 32,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 32,\n \"target\": 25,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 32,\n \"type\": \"produces\"\n },\n {\n \"source\": 25,\n \"target\": 32,\n \"type\": \"produces\"\n },\n {\n \"source\": 33,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 33,\n \"target\": 25,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 33,\n \"type\": \"produces\"\n },\n {\n \"source\": 25,\n \"target\": 33,\n \"type\": \"produces\"\n },\n {\n \"source\": 34,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 34,\n \"target\": 25,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 34,\n \"type\": \"produces\"\n },\n {\n \"source\": 25,\n \"target\": 34,\n \"type\": \"produces\"\n },\n {\n \"source\": 35,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 35,\n \"type\": \"produces\"\n },\n {\n \"source\": 36,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 36,\n \"type\": \"produces\"\n },\n {\n \"source\": 37,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 37,\n \"type\": \"produces\"\n },\n {\n \"source\": 38,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 38,\n \"type\": \"produces\"\n },\n {\n \"source\": 39,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 39,\n \"type\": \"produces\"\n },\n {\n \"source\": 40,\n \"target\": 12,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 40,\n \"type\": \"produces\"\n },\n {\n \"source\": 12,\n \"target\": 40,\n \"type\": \"produces\"\n },\n {\n \"source\": 16,\n \"target\": 40,\n \"type\": \"produces\"\n },\n {\n \"source\": 41,\n \"target\": 40,\n \"type\": \"produces\"\n },\n {\n \"source\": 42,\n \"target\": 40,\n \"type\": \"produces\"\n },\n {\n \"source\": 17,\n \"target\": 40,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 40,\n \"type\": \"produces\"\n },\n {\n \"source\": 43,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 43,\n \"type\": \"produces\"\n },\n {\n \"source\": 44,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 44,\n \"type\": \"produces\"\n },\n {\n \"source\": 45,\n \"target\": 20,\n \"type\": \"consumes\"\n },\n {\n \"source\": 46,\n \"target\": 45,\n \"type\": \"produces\"\n },\n {\n \"source\": 47,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 48,\n \"target\": 47,\n \"type\": \"produces\"\n },\n {\n \"source\": 49,\n \"target\": 47,\n \"type\": \"produces\"\n },\n {\n \"source\": 50,\n \"target\": 47,\n \"type\": \"produces\"\n },\n {\n \"source\": 51,\n \"target\": 47,\n \"type\": \"produces\"\n },\n {\n \"source\": 52,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 52,\n \"type\": \"produces\"\n },\n {\n \"source\": 53,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 53,\n \"type\": \"produces\"\n },\n {\n \"source\": 54,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 48,\n \"target\": 54,\n \"type\": \"produces\"\n },\n {\n \"source\": 49,\n \"target\": 54,\n \"type\": \"produces\"\n },\n {\n \"source\": 50,\n \"target\": 54,\n \"type\": \"produces\"\n },\n {\n \"source\": 51,\n \"target\": 54,\n \"type\": \"produces\"\n },\n {\n \"source\": 55,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 55,\n \"type\": \"produces\"\n },\n {\n \"source\": 56,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 57,\n \"target\": 56,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 56,\n \"type\": \"produces\"\n },\n {\n \"source\": 58,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 58,\n \"type\": \"produces\"\n },\n {\n \"source\": 59,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 59,\n \"type\": \"produces\"\n },\n {\n \"source\": 60,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 60,\n \"type\": \"produces\"\n },\n {\n \"source\": 48,\n \"target\": 60,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 60,\n \"type\": \"produces\"\n },\n {\n \"source\": 61,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 61,\n \"type\": \"produces\"\n },\n {\n \"source\": 62,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 62,\n \"type\": \"produces\"\n },\n {\n \"source\": 7,\n \"target\": 63,\n \"type\": \"produces\"\n },\n {\n \"source\": 12,\n \"target\": 63,\n \"type\": \"produces\"\n },\n {\n \"source\": 57,\n \"target\": 63,\n \"type\": \"produces\"\n },\n {\n \"source\": 64,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 48,\n \"target\": 64,\n \"type\": \"produces\"\n },\n {\n \"source\": 57,\n \"target\": 64,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 64,\n \"type\": \"produces\"\n },\n {\n \"source\": 65,\n \"target\": 46,\n \"type\": \"consumes\"\n },\n {\n \"source\": 10,\n \"target\": 65,\n \"type\": \"produces\"\n },\n {\n \"source\": 66,\n \"target\": 67,\n \"type\": \"consumes\"\n },\n {\n \"source\": 66,\n \"target\": 68,\n \"type\": \"consumes\"\n },\n {\n \"source\": 46,\n \"target\": 66,\n \"type\": \"produces\"\n },\n {\n \"source\": 68,\n \"target\": 66,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 66,\n \"type\": \"produces\"\n },\n {\n \"source\": 69,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 17,\n \"target\": 69,\n \"type\": \"produces\"\n },\n {\n \"source\": 5,\n \"target\": 69,\n \"type\": \"produces\"\n },\n {\n \"source\": 70,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 48,\n \"target\": 70,\n \"type\": \"produces\"\n },\n {\n \"source\": 71,\n \"target\": 48,\n \"type\": \"consumes\"\n },\n {\n \"source\": 72,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 72,\n \"target\": 73,\n \"type\": \"consumes\"\n },\n {\n \"source\": 20,\n \"target\": 72,\n \"type\": \"produces\"\n },\n {\n \"source\": 74,\n \"target\": 72,\n \"type\": \"produces\"\n },\n {\n \"source\": 75,\n \"target\": 10,\n \"type\": \"consumes\"\n },\n {\n \"source\": 73,\n \"target\": 75,\n \"type\": \"produces\"\n },\n {\n \"source\": 76,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 20,\n \"target\": 76,\n \"type\": \"produces\"\n },\n {\n \"source\": 77,\n \"target\": 78,\n \"type\": \"consumes\"\n },\n {\n \"source\": 20,\n \"target\": 77,\n \"type\": \"produces\"\n },\n {\n \"source\": 79,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 79,\n \"target\": 20,\n \"type\": \"consumes\"\n },\n {\n \"source\": 10,\n \"target\": 79,\n \"type\": \"produces\"\n },\n {\n \"source\": 80,\n \"target\": 16,\n \"type\": \"consumes\"\n },\n {\n \"source\": 42,\n \"target\": 80,\n \"type\": \"produces\"\n },\n {\n \"source\": 81,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 81,\n \"type\": \"produces\"\n },\n {\n \"source\": 82,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 5,\n \"target\": 82,\n \"type\": \"produces\"\n },\n {\n \"source\": 83,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 46,\n \"target\": 83,\n \"type\": \"produces\"\n },\n {\n \"source\": 4,\n \"target\": 83,\n \"type\": \"produces\"\n },\n {\n \"source\": 84,\n \"target\": 46,\n \"type\": \"consumes\"\n },\n {\n \"source\": 10,\n \"target\": 84,\n \"type\": \"produces\"\n },\n {\n \"source\": 85,\n \"target\": 46,\n \"type\": \"consumes\"\n },\n {\n \"source\": 10,\n \"target\": 85,\n \"type\": \"produces\"\n },\n {\n \"source\": 86,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 46,\n \"target\": 86,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 86,\n \"type\": \"produces\"\n },\n {\n \"source\": 87,\n \"target\": 67,\n \"type\": \"consumes\"\n },\n {\n \"source\": 87,\n \"target\": 68,\n \"type\": \"consumes\"\n },\n {\n \"source\": 46,\n \"target\": 87,\n \"type\": \"produces\"\n },\n {\n \"source\": 88,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 48,\n \"target\": 88,\n \"type\": \"produces\"\n },\n {\n \"source\": 68,\n \"target\": 88,\n \"type\": \"produces\"\n },\n {\n \"source\": 89,\n \"target\": 46,\n \"type\": \"consumes\"\n },\n {\n \"source\": 10,\n \"target\": 89,\n \"type\": \"produces\"\n },\n {\n \"source\": 90,\n \"target\": 68,\n \"type\": \"consumes\"\n },\n {\n \"source\": 46,\n \"target\": 90,\n \"type\": \"produces\"\n },\n {\n \"source\": 91,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 91,\n \"target\": 68,\n \"type\": \"consumes\"\n },\n {\n \"source\": 91,\n \"target\": 17,\n \"type\": \"consumes\"\n },\n {\n \"source\": 46,\n \"target\": 91,\n \"type\": \"produces\"\n },\n {\n \"source\": 4,\n \"target\": 91,\n \"type\": \"produces\"\n },\n {\n \"source\": 68,\n \"target\": 91,\n \"type\": \"produces\"\n },\n {\n \"source\": 17,\n \"target\": 91,\n \"type\": \"produces\"\n },\n {\n \"source\": 92,\n \"target\": 46,\n \"type\": \"consumes\"\n },\n {\n \"source\": 92,\n \"target\": 67,\n \"type\": \"consumes\"\n },\n {\n \"source\": 9,\n \"target\": 92,\n \"type\": \"produces\"\n },\n {\n \"source\": 93,\n \"target\": 68,\n \"type\": \"consumes\"\n },\n {\n \"source\": 93,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 17,\n \"target\": 93,\n \"type\": \"produces\"\n },\n {\n \"source\": 3,\n \"target\": 93,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 93,\n \"type\": \"produces\"\n },\n {\n \"source\": 94,\n \"target\": 93,\n \"type\": \"produces\"\n },\n {\n \"source\": 95,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 95,\n \"type\": \"produces\"\n },\n {\n \"source\": 96,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 96,\n \"type\": \"produces\"\n },\n {\n \"source\": 97,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 97,\n \"type\": \"produces\"\n },\n {\n \"source\": 98,\n \"target\": 16,\n \"type\": \"consumes\"\n },\n {\n \"source\": 98,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 98,\n \"target\": 20,\n \"type\": \"consumes\"\n },\n {\n \"source\": 2,\n \"target\": 98,\n \"type\": \"produces\"\n },\n {\n \"source\": 3,\n \"target\": 98,\n \"type\": \"produces\"\n },\n {\n \"source\": 99,\n \"target\": 74,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 99,\n \"type\": \"produces\"\n },\n {\n \"source\": 100,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 100,\n \"type\": \"produces\"\n },\n {\n \"source\": 48,\n \"target\": 100,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 100,\n \"type\": \"produces\"\n },\n {\n \"source\": 101,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 78,\n \"target\": 101,\n \"type\": \"produces\"\n },\n {\n \"source\": 102,\n \"target\": 12,\n \"type\": \"consumes\"\n },\n {\n \"source\": 103,\n \"target\": 102,\n \"type\": \"produces\"\n },\n {\n \"source\": 104,\n \"target\": 12,\n \"type\": \"consumes\"\n },\n {\n \"source\": 12,\n \"target\": 104,\n \"type\": \"produces\"\n },\n {\n \"source\": 105,\n \"target\": 12,\n \"type\": \"consumes\"\n },\n {\n \"source\": 103,\n \"target\": 105,\n \"type\": \"produces\"\n },\n {\n \"source\": 106,\n \"target\": 10,\n \"type\": \"consumes\"\n },\n {\n \"source\": 10,\n \"target\": 106,\n \"type\": \"produces\"\n },\n {\n \"source\": 107,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 107,\n \"type\": \"produces\"\n },\n {\n \"source\": 108,\n \"target\": 42,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 108,\n \"type\": \"produces\"\n },\n {\n \"source\": 109,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 109,\n \"target\": 74,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 109,\n \"type\": \"produces\"\n },\n {\n \"source\": 5,\n \"target\": 109,\n \"type\": \"produces\"\n },\n {\n \"source\": 110,\n \"target\": 42,\n \"type\": \"consumes\"\n },\n {\n \"source\": 5,\n \"target\": 110,\n \"type\": \"produces\"\n },\n {\n \"source\": 111,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 111,\n \"type\": \"produces\"\n },\n {\n \"source\": 112,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 112,\n \"type\": \"produces\"\n },\n {\n \"source\": 113,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 113,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 113,\n \"target\": 12,\n \"type\": \"consumes\"\n },\n {\n \"source\": 113,\n \"target\": 16,\n \"type\": \"consumes\"\n },\n {\n \"source\": 113,\n \"target\": 42,\n \"type\": \"consumes\"\n },\n {\n \"source\": 114,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 114,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 114,\n \"type\": \"produces\"\n },\n {\n \"source\": 4,\n \"target\": 114,\n \"type\": \"produces\"\n },\n {\n \"source\": 115,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 115,\n \"type\": \"produces\"\n },\n {\n \"source\": 17,\n \"target\": 115,\n \"type\": \"produces\"\n },\n {\n \"source\": 5,\n \"target\": 115,\n \"type\": \"produces\"\n },\n {\n \"source\": 116,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 116,\n \"target\": 20,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 116,\n \"type\": \"produces\"\n },\n {\n \"source\": 117,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 117,\n \"type\": \"produces\"\n },\n {\n \"source\": 118,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 118,\n \"target\": 74,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 118,\n \"type\": \"produces\"\n },\n {\n \"source\": 74,\n \"target\": 118,\n \"type\": \"produces\"\n },\n {\n \"source\": 119,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 119,\n \"target\": 74,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 119,\n \"type\": \"produces\"\n },\n {\n \"source\": 74,\n \"target\": 119,\n \"type\": \"produces\"\n },\n {\n \"source\": 120,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 120,\n \"target\": 74,\n \"type\": \"consumes\"\n },\n {\n \"source\": 74,\n \"target\": 120,\n \"type\": \"produces\"\n },\n {\n \"source\": 121,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 121,\n \"type\": \"produces\"\n },\n {\n \"source\": 122,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 48,\n \"target\": 122,\n \"type\": \"produces\"\n },\n {\n \"source\": 123,\n \"target\": 16,\n \"type\": \"consumes\"\n },\n {\n \"source\": 123,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 123,\n \"target\": 20,\n \"type\": \"consumes\"\n },\n {\n \"source\": 124,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 124,\n \"target\": 12,\n \"type\": \"consumes\"\n },\n {\n \"source\": 124,\n \"target\": 125,\n \"type\": \"consumes\"\n },\n {\n \"source\": 16,\n \"target\": 124,\n \"type\": \"produces\"\n },\n {\n \"source\": 126,\n \"target\": 67,\n \"type\": \"consumes\"\n },\n {\n \"source\": 126,\n \"target\": 68,\n \"type\": \"consumes\"\n },\n {\n \"source\": 46,\n \"target\": 126,\n \"type\": \"produces\"\n },\n {\n \"source\": 127,\n \"target\": 46,\n \"type\": \"consumes\"\n },\n {\n \"source\": 10,\n \"target\": 127,\n \"type\": \"produces\"\n },\n {\n \"source\": 128,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 128,\n \"type\": \"produces\"\n },\n {\n \"source\": 129,\n \"target\": 74,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 129,\n \"type\": \"produces\"\n },\n {\n \"source\": 130,\n \"target\": 20,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 130,\n \"type\": \"produces\"\n },\n {\n \"source\": 131,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 20,\n \"target\": 131,\n \"type\": \"produces\"\n },\n {\n \"source\": 132,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 132,\n \"type\": \"produces\"\n },\n {\n \"source\": 133,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 48,\n \"target\": 133,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 133,\n \"type\": \"produces\"\n },\n {\n \"source\": 134,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 134,\n \"type\": \"produces\"\n },\n {\n \"source\": 135,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 135,\n \"target\": 12,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 135,\n \"type\": \"produces\"\n },\n {\n \"source\": 4,\n \"target\": 135,\n \"type\": \"produces\"\n },\n {\n \"source\": 16,\n \"target\": 135,\n \"type\": \"produces\"\n },\n {\n \"source\": 17,\n \"target\": 135,\n \"type\": \"produces\"\n },\n {\n \"source\": 5,\n \"target\": 135,\n \"type\": \"produces\"\n },\n {\n \"source\": 136,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 136,\n \"type\": \"produces\"\n },\n {\n \"source\": 137,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 48,\n \"target\": 137,\n \"type\": \"produces\"\n },\n {\n \"source\": 138,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 138,\n \"type\": \"produces\"\n },\n {\n \"source\": 139,\n \"target\": 20,\n \"type\": \"consumes\"\n },\n {\n \"source\": 68,\n \"target\": 139,\n \"type\": \"produces\"\n },\n {\n \"source\": 140,\n \"target\": 141,\n \"type\": \"consumes\"\n },\n {\n \"source\": 140,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 140,\n \"target\": 23,\n \"type\": \"consumes\"\n },\n {\n \"source\": 140,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 140,\n \"target\": 12,\n \"type\": \"consumes\"\n },\n {\n \"source\": 140,\n \"target\": 125,\n \"type\": \"consumes\"\n },\n {\n \"source\": 140,\n \"target\": 68,\n \"type\": \"consumes\"\n },\n {\n \"source\": 140,\n \"target\": 25,\n \"type\": \"consumes\"\n },\n {\n \"source\": 140,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 140,\n \"target\": 20,\n \"type\": \"consumes\"\n },\n {\n \"source\": 140,\n \"target\": 51,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 140,\n \"type\": \"produces\"\n },\n {\n \"source\": 4,\n \"target\": 140,\n \"type\": \"produces\"\n },\n {\n \"source\": 12,\n \"target\": 140,\n \"type\": \"produces\"\n },\n {\n \"source\": 16,\n \"target\": 140,\n \"type\": \"produces\"\n },\n {\n \"source\": 67,\n \"target\": 140,\n \"type\": \"produces\"\n },\n {\n \"source\": 142,\n \"target\": 16,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 142,\n \"type\": \"produces\"\n },\n {\n \"source\": 48,\n \"target\": 142,\n \"type\": \"produces\"\n },\n {\n \"source\": 143,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 143,\n \"type\": \"produces\"\n },\n {\n \"source\": 144,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 144,\n \"type\": \"produces\"\n },\n {\n \"source\": 145,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 145,\n \"target\": 23,\n \"type\": \"consumes\"\n },\n {\n \"source\": 146,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 146,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 146,\n \"type\": \"produces\"\n },\n {\n \"source\": 5,\n \"target\": 146,\n \"type\": \"produces\"\n },\n {\n \"source\": 147,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 147,\n \"type\": \"produces\"\n },\n {\n \"source\": 148,\n \"target\": 46,\n \"type\": \"consumes\"\n },\n {\n \"source\": 148,\n \"target\": 10,\n \"type\": \"consumes\"\n },\n {\n \"source\": 148,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 148,\n \"target\": 73,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 148,\n \"type\": \"produces\"\n },\n {\n \"source\": 5,\n \"target\": 148,\n \"type\": \"produces\"\n },\n {\n \"source\": 149,\n \"target\": 10,\n \"type\": \"consumes\"\n },\n {\n \"source\": 10,\n \"target\": 149,\n \"type\": \"produces\"\n },\n {\n \"source\": 150,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 150,\n \"type\": \"produces\"\n },\n {\n \"source\": 151,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 151,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 151,\n \"type\": \"produces\"\n },\n {\n \"source\": 152,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 152,\n \"type\": \"produces\"\n },\n {\n \"source\": 153,\n \"target\": 152,\n \"type\": \"produces\"\n },\n {\n \"source\": 154,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 154,\n \"type\": \"produces\"\n },\n {\n \"source\": 155,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 155,\n \"type\": \"produces\"\n },\n {\n \"source\": 156,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 18,\n \"target\": 156,\n \"type\": \"produces\"\n },\n {\n \"source\": 157,\n \"target\": 7,\n \"type\": \"consumes\"\n },\n {\n \"source\": 7,\n \"target\": 157,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 157,\n \"type\": \"produces\"\n },\n {\n \"source\": 158,\n \"target\": 74,\n \"type\": \"consumes\"\n },\n {\n \"source\": 159,\n \"target\": 4,\n \"type\": \"consumes\"\n },\n {\n \"source\": 159,\n \"target\": 17,\n \"type\": \"consumes\"\n },\n {\n \"source\": 159,\n \"target\": 3,\n \"type\": \"consumes\"\n },\n {\n \"source\": 159,\n \"target\": 153,\n \"type\": \"consumes\"\n },\n {\n \"source\": 159,\n \"target\": 5,\n \"type\": \"consumes\"\n },\n {\n \"source\": 160,\n \"target\": 2,\n \"type\": \"consumes\"\n },\n {\n \"source\": 160,\n \"target\": 17,\n \"type\": \"consumes\"\n },\n {\n \"source\": 4,\n \"target\": 160,\n \"type\": \"produces\"\n },\n {\n \"source\": 17,\n \"target\": 160,\n \"type\": \"produces\"\n },\n {\n \"source\": 20,\n \"target\": 160,\n \"type\": \"produces\"\n },\n {\n \"source\": 5,\n \"target\": 160,\n \"type\": \"produces\"\n }\n]" }, { "path": "docs/data/chord_graph/vega.json", "content": "{\n \"$schema\": \"https://vega.github.io/schema/vega/v5.json\",\n \"description\": \"BBOT\",\n \"padding\": 20,\n \"width\": 800,\n \"height\": 800,\n \"autosize\": \"none\",\n\n \"signals\": [\n { \"name\": \"producesColor\", \"value\": \"#ff8400\" },\n { \"name\": \"consumesColor\", \"value\": \"white\" },\n { \"name\": \"originX\", \"update\": \"width / 2\" },\n { \"name\": \"originY\", \"update\": \"height / 2\" },\n {\n \"name\": \"active\", \"value\": \"{id: 555555555, consumes: []}\",\n \"on\": [\n { \"events\": \"text:pointerover\", \"update\": \"datum\" },\n { \"events\": \"pointerover[!event.item]\", \"update\": \"{id: 555555555, consumes: []}\" }\n ]\n }\n ],\n\n \"data\": [\n {\n \"name\": \"entities\",\n \"url\": \"../data/chord_graph/entities.json\",\n \"transform\": [\n {\n \"type\": \"stratify\",\n \"key\": \"id\",\n \"parentKey\": \"parent\"\n },\n {\n \"type\": \"tree\",\n \"method\": \"cluster\",\n \"size\": [1, 1],\n \"as\": [\"alpha\", \"beta\", \"depth\", \"children\"]\n },\n {\n \"type\": \"formula\",\n \"expr\": \"(360 * datum.alpha + 270) % 360\",\n \"as\": \"angle\"\n },\n {\n \"type\": \"formula\",\n \"expr\": \"inrange(datum.angle, [90, 270])\",\n \"as\": \"leftside\"\n },\n {\n \"type\": \"formula\",\n \"expr\": \"originX + 280 * datum.beta * cos(PI * datum.angle / 180)\",\n \"as\": \"x\"\n },\n {\n \"type\": \"formula\",\n \"expr\": \"originY + 280 * datum.beta * sin(PI * datum.angle / 180)\",\n \"as\": \"y\"\n }\n ]\n },\n {\n \"name\": \"leaves\",\n \"source\": \"entities\",\n \"transform\": [\n {\n \"type\": \"filter\",\n \"expr\": \"!datum.children\"\n }\n ]\n },\n {\n \"name\": \"rels\",\n \"url\": \"../data/chord_graph/rels.json\",\n \"transform\": [\n {\n \"type\": \"formula\",\n \"expr\": \"treePath('entities', datum.source, datum.target)\",\n \"as\": \"treepath\",\n \"initonly\": true\n }\n ]\n },\n {\n \"name\": \"selected_rels\",\n \"source\": \"rels\",\n \"transform\": [\n {\n \"type\": \"filter\",\n \"expr\": \"datum.source === active.id || datum.target === active.id\"\n }\n ]\n },\n {\n \"name\": \"selected_entities\",\n \"source\": \"entities\",\n \"transform\": [\n {\n \"type\": \"filter\",\n \"expr\": \"datum.id \"\n }\n ]\n }\n ],\n\n \"marks\": [\n {\n \"type\": \"text\",\n \"from\": {\"data\": \"leaves\"},\n \"encode\": {\n \"enter\": {\n \"text\": {\"field\": \"name\"},\n \"baseline\": {\"value\": \"middle\"}\n },\n \"update\": {\n \"x\": {\"field\": \"x\"},\n \"y\": {\"field\": \"y\"},\n \"dx\": {\"signal\": \"2 * (datum.leftside ? -1 : 1)\"},\n \"angle\": {\"signal\": \"datum.leftside ? datum.angle - 180 : datum.angle\"},\n \"align\": {\"signal\": \"datum.leftside ? 'right' : 'left'\"},\n \"fontSize\": [\n {\"test\": \"indata('selected_rels', 'source', datum.id)\", \"value\": 15},\n {\"test\": \"indata('selected_rels', 'target', datum.id)\", \"value\": 15},\n {\"value\": 11}\n ],\n \"fontWeight\": [\n {\"test\": \"indata('selected_rels', 'source', datum.id)\", \"value\": \"bold\"},\n {\"test\": \"indata('selected_rels', 'target', datum.id)\", \"value\": \"bold\"},\n {\"value\": null}\n ],\n \"fill\": [\n {\"test\": \"datum.id === active.id\", \"value\": \"white\"},\n {\"test\": \"if(active && active.produces, active.produces.length > 0 && indexof(active.produces, datum.id) >= 0, false)\", \"signal\": \"producesColor\"},\n {\"test\": \"if(active && active.consumes, active.consumes.length > 0 && indexof(active.consumes, datum.id) >= 0, false)\", \"signal\": \"consumesColor\"},\n {\"value\": \"#aaa\"}\n ]\n }\n }\n },\n {\n \"type\": \"group\",\n \"from\": {\n \"facet\": {\n \"name\": \"path\",\n \"data\": \"rels\",\n \"field\": \"treepath\"\n }\n },\n \"marks\": [\n {\n \"type\": \"line\",\n \"interactive\": false,\n \"from\": {\"data\": \"path\"},\n \"encode\": {\n \"enter\": {\n \"interpolate\": {\"value\": \"bundle\"},\n \"strokeWidth\": {\"value\": 3}\n },\n \"update\": {\n \"stroke\": [\n {\"test\": \"(parent.source === active.id || parent.target === active.id) && parent.type === 'consumes'\", \"signal\": \"consumesColor\"},\n {\"test\": \"(parent.source === active.id || parent.target === active.id) && parent.type === 'produces'\", \"signal\": \"producesColor\"},\n {\"value\": \"#ff8400\"}\n ],\n \"strokeOpacity\": [\n {\"test\": \"parent.source === active.id || parent.target === active.id\", \"value\": 1},\n {\"value\": 0.2}\n ],\n \"tension\": {\"value\": 0.7},\n \"x\": {\"field\": \"x\"},\n \"y\": {\"field\": \"y\"}\n }\n }\n }\n ]\n }\n ],\n\n \"scales\": [\n {\n \"name\": \"color\",\n \"type\": \"ordinal\",\n \"domain\": [\"consumes\", \"produces\"],\n \"range\": [{\"signal\": \"consumesColor\"}, {\"signal\": \"producesColor\"}]\n }\n ],\n\n \"legends\": [\n {\n \"stroke\": \"color\",\n \"labelColor\": \"white\",\n \"labelFontSize\": 20,\n \"symbolStrokeWidth\": 20,\n \"orient\": \"bottom-right\",\n \"symbolType\": \"stroke\"\n }\n ]\n}\n" }, { "path": "docs/dev/architecture.md", "content": "# BBOT Internal Architecture\n\nHere is a basic overview of BBOT's internal architecture.\n\n## Queues\n\nBeing both ***recursive*** and ***event-driven***, BBOT makes heavy use of queues. These enable smooth communication between the modules, and ensure that large numbers of events can be produced without slowing down or clogging up the scan.\n\nEvery module in BBOT has both an ***incoming*** and ***outgoing*** queue. Event types matching the module's `WATCHED_EVENTS` (e.g. `DNS_NAME`) are queued in its incoming queue, and processed by the module's `handle_event()` (or `handle_batch()` in the case of batched modules). If the module finds anything interesting, it creates an event and places it in its outgoing queue, to be processed by the scan and redistributed to other modules.\n\n## Event Flow\n\nBelow is a graph showing the internal event flow in BBOT. White lines represent queues. Notice how some modules run in sequence, while others run in parallel. With the exception of a few specific modules, most BBOT modules are parallelized.\n\n![event-flow](https://github.com/blacklanternsecurity/bbot/assets/20261699/6cece76b-70bd-4690-a53f-02d42e6ed05b)\n\nFor a higher-level overview, see [How it Works](../how_it_works.md).\n" }, { "path": "docs/dev/basemodule.md", "content": "::: bbot.modules.base.BaseModule\n" }, { "path": "docs/dev/core.md", "content": "::: bbot.core.core.BBOTCore\n" }, { "path": "docs/dev/dev_environment.md", "content": "# Setting Up a Dev Environment\n\nThe following will show you how to set up a fully functioning python environment for devving on BBOT.\n\n## Installation (Poetry)\n\n[Poetry](https://python-poetry.org/) is the recommended method of installation if you want to dev on BBOT. To set up a dev environment with Poetry, you can follow these steps:\n\n- Fork [BBOT](https://github.com/blacklanternsecurity/bbot) on GitHub\n- Clone your fork and set up a development environment with Poetry:\n\n```bash\n# clone your forked repo and cd into it\ngit clone git@github.com//bbot.git\ncd bbot\n\n# install poetry\ncurl -sSL https://install.python-poetry.org | python3 -\n\n# install pip dependencies\npoetry install\n# install pre-commit hooks, etc.\npoetry run pre-commit install\n\n# enter virtual environment\npoetry shell\n\nbbot --help\n```\n\n- Now, any changes you make in the code will be reflected in the `bbot` command.\n- After making your changes, run the tests locally to ensure they pass.\n\n```bash\n# auto-format code indentation, etc.\nruff format\n\n# run tests\n./bbot/test/run_tests.sh\n```\n\n- Finally, commit and push your changes, and create a pull request to the `dev` branch of the main BBOT repo.\n" }, { "path": "docs/dev/discord_bot.md", "content": "\n![bbot-discord](https://github.com/blacklanternsecurity/bbot/assets/20261699/22b268a2-0dfd-4c2a-b7c5-548c0f2cc6f9)\n\nBelow is a simple Discord bot designed to run BBOT scans.\n\n```python title=\"examples/discord_bot.py\"\n--8<-- \"examples/discord_bot.py\"\n```\n" }, { "path": "docs/dev/engine.md", "content": "::: bbot.core.engine.EngineBase\n\n::: bbot.core.engine.EngineClient\n\n::: bbot.core.engine.EngineServer\n" }, { "path": "docs/dev/event.md", "content": "This is a developer reference. For a high-level description of BBOT events including a full list of event types, see [Events](../../scanning/events)\n\n::: bbot.core.event.base.make_event\n::: bbot.core.event.base.event_from_json\n\n::: bbot.core.event.base.BaseEvent\n options:\n members:\n - __init__\n - json\n - from_json\n - pretty_string\n - module_sequence\n - make_internal\n - unmake_internal\n - set_scope_distance\n" }, { "path": "docs/dev/helpers/command.md", "content": "# Command Helpers\n\nThese are helpers related to executing shell commands. They are used throughout BBOT and its modules for executing various binaries such as `masscan`, `nuclei`, etc.\n\nThese helpers can be invoked directly from `self.helpers`, but inside a module they should always use `self.run_process()` or `self.run_process_live()`. These are light wrappers which ensure the running process is tracked by the module so that it can be easily terminated should the user need to kill the module:\n\n```python\n# simple subprocess\nls_result = await self.run_process(\"ls\", \"-l\")\nfor line ls_result.stdout.splitlines():\n # ...\n\n# iterate through each line in real time\nasync for line in self.run_process_live([\"grep\", \"-R\"]):\n # ...\n```\n\n::: bbot.core.helpers.command\n options:\n show_root_heading: false\n" }, { "path": "docs/dev/helpers/dns.md", "content": "# DNS\n\nThese are helpers related to DNS resolution. They are used throughout BBOT and its modules for performing DNS lookups and detecting DNS wildcards, etc.\n\nNote that these helpers can be invoked directly from `self.helpers`, e.g.:\n\n```python\nself.helpers.resolve(\"evilcorp.com\")\n```\n\n::: bbot.core.helpers.dns.DNSHelper\n handler: python\n options:\n members:\n - resolve\n - resolve_batch\n - resolve_raw\n - is_wildcard\n - is_wildcard_domain\n" }, { "path": "docs/dev/helpers/index.md", "content": "# BBOT Helpers\n\nIn this section are various helper functions that are designed to make your life easier when devving on BBOT. Whether you're extending BBOT by writing a module or working on its core engine, these functions are designed to act as useful machine parts to perform essential tasks, such as making a web request or executing a DNS query.\n\nThe vast majority of these helpers can be accessed directly from the `.helpers` attribute of a scan or module, like so:\n\n```python\nclass MyModule(BaseModule):\n\n ...\n\n async def handle_event(self, event):\n # Web Request\n response = await self.helpers.request(\"https://www.evilcorp.com\")\n\n # DNS query\n for ip in await self.helpers.resolve(\"www.evilcorp.com\"):\n self.hugesuccess(str(ip))\n\n # Execute shell command\n completed_process = await self.run_process(\"ls\", \"-l\")\n self.hugesuccess(completed_process.stdout)\n\n # Split a DNS name into subdomain / domain\n self.helpers.split_domain(\"www.internal.evilcorp.co.uk\")\n # (\"www.internal\", \"evilcorp.co.uk\")\n```\n\n[Next Up: Command Helpers -->](command.md){ .md-button .md-button--primary }\n" }, { "path": "docs/dev/helpers/interactsh.md", "content": "# Interact.sh\n\n::: bbot.core.helpers.interactsh.Interactsh\n options:\n show_root_heading: false\n" }, { "path": "docs/dev/helpers/misc.md", "content": "# Misc Helpers\n\nThese are miscellaneous helpers, used throughout BBOT and its modules for simple tasks such as parsing domains, ports, urls, etc.\n\n::: bbot.core.helpers.misc\n options:\n show_root_heading: false\n" }, { "path": "docs/dev/helpers/web.md", "content": "# Web\n\nThese are helpers for making various web requests.\n\nNote that these helpers can be invoked directly from `self.helpers`, e.g.:\n\n```python\nself.helpers.request(\"https://www.evilcorp.com\")\n```\n\n::: bbot.core.helpers.web\n options:\n show_root_heading: false\n members:\n - WebHelper\n" }, { "path": "docs/dev/helpers/wordcloud.md", "content": "# Word Cloud\n\nThese are helpers related to BBOT's Word Cloud, a mechanism for storing target-specific keywords that are useful for custom wordlists, etc.\n\nNote that these helpers can be invoked directly from `self.helpers`, e.g.:\n\n```python\nself.helpers.word_cloud\n```\n\n::: bbot.core.helpers.wordcloud\n options:\n show_root_heading: false\n" }, { "path": "docs/dev/index.md", "content": "# BBOT Developer Reference\n\nBBOT exposes a Python API that allows you to create, start, and stop scans.\n\nDocumented in this section are commonly-used classes and functions within BBOT, along with usage examples.\n\n## Adding BBOT to Your Python Project\n\nIf you are using Poetry, you can add BBOT to your python environment like this:\n\n```bash\n# stable\npoetry add bbot\n\n# bleeding-edge (dev branch)\npoetry add bbot --allow-prereleases\n```\n\n## Running a BBOT Scan from Python\n\n#### Synchronous\n```python\nfrom bbot.scanner import Scanner\n\nif __name__ == \"__main__\":\n scan = Scanner(\"evilcorp.com\", presets=[\"subdomain-enum\"])\n for event in scan.start():\n print(event)\n```\n\n#### Asynchronous\n```python\nfrom bbot.scanner import Scanner\n\nasync def main():\n scan = Scanner(\"evilcorp.com\", presets=[\"subdomain-enum\"])\n async for event in scan.async_start():\n print(event.json())\n\nif __name__ == \"__main__\":\n import asyncio\n asyncio.run(main())\n```\n\nFor a full listing of `Scanner` attributes and functions, see the [`Scanner` Code Reference](./scanner.md).\n\n#### Multiple Targets\n\nYou can specify any number of targets:\n\n```python\n# create a scan against multiple targets\nscan = Scanner(\n \"evilcorp.com\",\n \"evilcorp.org\",\n \"evilcorp.ce\",\n \"4.3.2.1\",\n \"1.2.3.4/24\",\n presets=[\"subdomain-enum\"]\n)\n\n# this is the same as:\ntargets = [\"evilcorp.com\", \"evilcorp.org\", \"evilcorp.ce\", \"4.3.2.1\", \"1.2.3.4/24\"]\nscan = Scanner(*targets, presets=[\"subdomain-enum\"])\n```\n\nFor more details, including which types of targets are valid, see [Targets](../scanning/index.md#targets-t)\n\n#### Other Custom Options\n\nIn many cases, using a [Preset](../scanning/presets.md) like `subdomain-enum` is sufficient. However, the `Scanner` is flexible and accepts many other arguments that can override the default functionality. You can specify [`flags`](../scanning/index.md#flags-f), [`modules`](../scanning/index.md#modules-m), [`output_modules`](../output.md), a [`whitelist` or `blacklist`](../scanning/index.md#whitelists-and-blacklists), and custom [`config` options](../scanning/configuration.md):\n\n```python\n# create a scan against multiple targets\nscan = Scanner(\n # targets\n \"evilcorp.com\",\n \"4.3.2.1\",\n # enable these presets\n presets=[\"subdomain-enum\"],\n # whitelist these hosts\n whitelist=[\"evilcorp.com\", \"evilcorp.org\"],\n # blacklist these hosts\n blacklist=[\"prod.evilcorp.com\"],\n # also enable these individual modules\n modules=[\"nuclei\", \"ipstack\"],\n # exclude modules with these flags\n exclude_flags=[\"slow\"],\n # custom config options\n config={\n \"modules\": {\n \"nuclei\": {\n \"tags\": \"apache,nginx\"\n }\n }\n }\n)\n```\n\nFor a list of all the possible scan options, see the [`Presets` Code Reference](./presets.md)\n" }, { "path": "docs/dev/module_howto.md", "content": "# How to Write a BBOT Module\n\nHere we'll go over a basic example of writing a custom BBOT module.\n\n## Create the python file\n\n1. Create a new `.py` file in `bbot/modules` (or in a [custom module directory](#load-modules-from-custom-locations))\n1. At the top of the file, import `BaseModule`\n1. Declare a class that inherits from `BaseModule`\n - the class must have the same name as your file (case-insensitive)\n1. Define in `watched_events` what type of data your module will consume\n1. Define in `produced_events` what type of data your module will produce\n1. Define (via `flags`) whether your module is `active` or `passive`, and whether it's `safe` or `aggressive`\n1. **Put your main logic in `.handle_event()`**\n\nHere is an example of a simple module that performs whois lookups:\n\n```python title=\"bbot/modules/whois.py\"\nfrom bbot.modules.base import BaseModule\n\nclass whois(BaseModule):\n watched_events = [\"DNS_NAME\"] # watch for DNS_NAME events\n produced_events = [\"WHOIS\"] # we produce WHOIS events\n flags = [\"passive\", \"safe\"]\n meta = {\"description\": \"Query WhoisXMLAPI for WHOIS data\"}\n options = {\"api_key\": \"\"} # module config options\n options_desc = {\"api_key\": \"WhoisXMLAPI Key\"}\n per_domain_only = True # only run once per domain\n\n base_url = \"https://www.whoisxmlapi.com/whoisserver/WhoisService\"\n\n # one-time setup - runs at the beginning of the scan\n async def setup(self):\n self.api_key = self.config.get(\"api_key\")\n if not self.api_key:\n # soft-fail if no API key is set\n return None, \"Must set API key\"\n\n async def handle_event(self, event):\n self.hugesuccess(f\"Got {event} (event.data: {event.data})\")\n _, domain = self.helpers.split_domain(event.data)\n url = f\"{self.base_url}?apiKey={self.api_key}&domainName={domain}&outputFormat=JSON\"\n self.hugeinfo(f\"Visiting {url}\")\n response = await self.helpers.request(url)\n if response is not None:\n await self.emit_event(response.json(), \"WHOIS\", parent=event)\n```\n\n## Test your new module\n\nAfter saving the module, you can run it with `-m`:\n\n```bash\n# run a scan enabling the module in bbot/modules/mymodule.py\nbbot -t evilcorp.com -m whois\n```\n\n### Debugging Your Module\n\nBBOT has a variety of colorful logging functions like `self.hugesuccess()` that can be useful for debugging.\n\n**BBOT log levels**:\n\n- `critical`: bright red\n- `hugesuccess`: bright green\n- `hugewarning`: bright orange\n- `hugeinfo`: bright blue\n- `error`: red\n- `warning`: orange\n- `info`: blue\n- `verbose`: grey (must enable `-v` to see)\n- `debug`: grey (must enable `-d` to see)\n\n\nFor details on how tests are written, see [Unit Tests](./tests.md).\n\n## `handle_event()` and `emit_event()`\n\nThe `handle_event()` method is the most important part of the module. By overriding this method, you control what the module does. During a scan, when an [event](./scanning/events.md) from your `watched_events` is encountered (a `DNS_NAME` in this example), `handle_event()` is automatically called with that event as its argument.\n\nThe `emit_event()` method is how modules return data. When you call `emit_event()`, it creates an [event](./scanning/events.md) and outputs it, sending it any modules that are interested in that data type.\n\n## `setup_deps()` and `setup()`\n\n`setup_deps()` and `setup()` are used for performing one-time setup at the start of the scan.\n\n`setup_deps()` is reserved for downloading or installing any dependencies not covered by Ansible, i.e. AI models or wordlists. Any other one-time setup tasks can be put into `setup()`.\n\nThese methods must return either:\n\n1. `True` - module setup succeeded\n2. `None` - module setup soft-failed (scan will continue but module will be disabled)\n3. `False` - module setup hard-failed (scan will abort)\n\nOptionally, it can also return a reason. Here are some examples:\n\n```python\nasync def setup(self):\n if not self.config.get(\"api_key\"):\n # soft-fail\n return None, \"No API key specified\"\n return True\n\nasync def setup_deps(self):\n self.wordlist = self.helpers.wordlist(\"https://raw.githubusercontent.com/user/wordlist.txt\")\n return True\n\nasync def setup(self):\n self.timeout = self.config.get(\"timeout\", 5)\n if self.timeout <= 0:\n return False, \"Timeout must be greater than or equal to 0\"\n # success\n return True\n```\n\n## Module Config Options\n\nEach module can have its own set of config options. These live in the `options` and `options_desc` attributes on your class. Both are dictionaries; `options` is for defaults and `options_desc` is for descriptions. Here is a typical example:\n\n```python title=\"bbot/modules/nmap.py\"\nclass nmap(BaseModule):\n # ...\n options = {\n \"top_ports\": 100,\n \"ports\": \"\",\n \"timing\": \"T4\",\n \"skip_host_discovery\": True,\n }\n options_desc = {\n \"top_ports\": \"Top ports to scan (default 100) (to override, specify 'ports')\",\n \"ports\": \"Ports to scan\",\n \"timing\": \"-T<0-5>: Set timing template (higher is faster)\",\n \"skip_host_discovery\": \"skip host discovery (-Pn)\",\n }\n\n async def setup(self):\n self.ports = self.config.get(\"ports\", \"\")\n self.timing = self.config.get(\"timing\", \"T4\")\n self.top_ports = self.config.get(\"top_ports\", 100)\n self.skip_host_discovery = self.config.get(\"skip_host_discovery\", True)\n return True\n```\n\nOnce you've defined these variables, you can pass the options via `-c`:\n\n```bash\nbbot -m nmap -c modules.nmap.top_ports=250\n```\n\n... or via the config:\n\n```yaml title=\"~/.config/bbot/bbot.yml\"\nmodules:\n nmap:\n top_ports: 250\n```\n\nInside the module, you access them via `self.config`, e.g.:\n\n```python\nself.config.get(\"top_ports\")\n```\n\n## Module Dependencies\n\nBBOT automates module dependencies with **Ansible**. If your module relies on a third-party binary, OS package, or python library, you can specify them in the `deps_*` attributes of your module.\n\n```python\nclass MyModule(BaseModule):\n ...\n deps_apt = [\"chromium-browser\"]\n deps_ansible = [\n {\n \"name\": \"install dev tools\",\n \"package\": {\"name\": [\"gcc\", \"git\", \"make\"], \"state\": \"present\"},\n \"become\": True,\n \"ignore_errors\": True,\n },\n {\n \"name\": \"Download massdns source code\",\n \"git\": {\n \"repo\": \"https://github.com/blechschmidt/massdns.git\",\n \"dest\": \"#{BBOT_TEMP}/massdns\",\n \"single_branch\": True,\n \"version\": \"master\",\n },\n },\n {\n \"name\": \"Build massdns\",\n \"command\": {\"chdir\": \"#{BBOT_TEMP}/massdns\", \"cmd\": \"make\", \"creates\": \"#{BBOT_TEMP}/massdns/bin/massdns\"},\n },\n {\n \"name\": \"Install massdns\",\n \"copy\": {\"src\": \"#{BBOT_TEMP}/massdns/bin/massdns\", \"dest\": \"#{BBOT_TOOLS}/\", \"mode\": \"u+x,g+x,o+x\"},\n },\n ]\n```\n\n## Load Modules from Custom Locations\n\nIf you have a custom module and you want to use it with BBOT, you can add its parent folder to `module_dirs`. This saves you from having to copy it into the BBOT install location. To add a custom module directory, add it to `module_dirs` in your preset:\n\n```yaml title=\"my_preset.yml\"\n# load BBOT modules from these additional paths\nmodule_dirs:\n - /home/user/my_modules\n```\n" }, { "path": "docs/dev/presets.md", "content": "::: bbot.scanner.Preset\n" }, { "path": "docs/dev/scanner.md", "content": "::: bbot.scanner.Scanner\n" }, { "path": "docs/dev/target.md", "content": "::: bbot.scanner.target.BaseTarget\n\n::: bbot.scanner.target.ScanSeeds\n\n::: bbot.scanner.target.ScanWhitelist\n\n::: bbot.scanner.target.ScanBlacklist\n\n::: bbot.scanner.target.BBOTTarget\n" }, { "path": "docs/dev/tests.md", "content": "# Unit Tests\n\nBBOT takes tests seriously. Every module *must* have a custom-written test that *actually tests* its functionality. Don't worry if you want to contribute but you aren't used to writing tests. If you open a draft PR, we will help write them :)\n\nWe use [ruff](https://docs.astral.sh/ruff/) for linting, and [pytest](https://docs.pytest.org/en/8.2.x/) for tests.\n\n## Running tests locally\n\nWe have GitHub Actions that automatically run tests whenever you open a Pull Request. However, you can also run the tests locally with `pytest`:\n\n```bash\n# lint with ruff\npoetry run ruff check\n\n# format code with ruff\npoetry run ruff format\n\n# run all tests with pytest (takes roughly 30 minutes)\npoetry run pytest\n```\n\n### Running specific tests\n\nIf you only want to run a single test, you can select it with `-k`:\n\n```bash\n# run only the sslcert test\npoetry run pytest -k test_module_sslcert\n```\n\nYou can also filter like this:\n```bash\n# run all the module tests except for sslcert\npoetry run pytest -k \"test_module_ and not test_module_sslcert\"\n```\n\nIf you want to see the output of your module, you can enable `--log-cli-level`:\n```bash\npoetry run pytest --log-cli-level=DEBUG\n```\n\n## Example: Writing a Module Test\n\nTo write a test for your module, create a new python file in `bbot/test/test_step_2/module_tests`. Your filename must be `test_module_`:\n\n```python title=\"test_module_mymodule.py\"\nfrom .base import ModuleTestBase\n\n\nclass TestMyModule(ModuleTestBase):\n targets = [\"blacklanternsecurity.com\"]\n config_overrides = {\"modules\": {\"mymodule\": {\"api_key\": \"deadbeef\"}}}\n\n async def setup_after_prep(self, module_test):\n # mock HTTP response\n module_test.httpx_mock.add_response(\n url=\"https://api.com/sudomains?apikey=deadbeef&domain=blacklanternsecurity.com\",\n json={\n \"subdomains\": [\n \"www.blacklanternsecurity.com\",\n \"dev.blacklanternsecurity.com\"\n ],\n },\n )\n # mock DNS\n await module_test.mock_dns(\n {\n \"blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n \"www.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n \"dev.blacklanternsecurity.com\": {\"A\": [\"1.2.3.4\"]},\n }\n )\n\n def check(self, module_test, events):\n # here is where we check to make sure it worked\n dns_names = [e.data for e in events if e.type == \"DNS_NAME\"]\n # temporary log messages for debugging\n for e in dns_names:\n self.log.critical(e)\n assert \"www.blacklanternsecurity.com\" in dns_names, \"failed to find subdomain #1\"\n assert \"dev.blacklanternsecurity.com\" in dns_names, \"failed to find subdomain #2\"\n```\n\n### Debugging a test\n\nSimilar to debugging from within a module, you can debug from within a test using `self.log.critical()`, etc:\n\n```python\n def check(self, module_test, events):\n for e in events:\n # bright red\n self.log.critical(e.type)\n # bright green\n self.log.hugesuccess(e.data)\n # bright orange\n self.log.hugewarning(e.tags)\n # bright blue\n self.log.hugeinfo(e.parent)\n```\n\n### More advanced tests\n\nIf you have questions about tests or need to write a more advanced test, come talk to us on [GitHub](https://github.com/blacklanternsecurity/bbot/discussions) or [Discord](https://discord.com/invite/PZqkgxu5SA).\n\nIt's also a good idea to look through our [existing tests](https://github.com/blacklanternsecurity/bbot/tree/stable/bbot/test/test_step_2/module_tests). BBOT has over a hundred of them, so you might find one that's similar to what you're trying to do.\n" }, { "path": "docs/diagrams/engine-architecture.drawio", "content": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n" }, { "path": "docs/diagrams/event-flow.drawio", "content": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n" }, { "path": "docs/diagrams/module-recursion.drawio", "content": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n" }, { "path": "docs/how_it_works.md", "content": "# How it Works\n\n## BBOT's Recursive Philosophy\n\nIt's well-known that when you're doing recon, it's best to do it recursively. However, there are very few recursive tools, and the main reason for this is because making a recursive tool is hard. In particular, it's very difficult to build a large-scale recursive system that interacts with the internet, and to keep it stable. When we first set out to make BBOT, we didn't know this, and it was definitely a lesson we learned the hard way. BBOT's stability is thanks to its extensive [Unit Tests](./dev/tests.md).\n\nBBOT inherits its recursive philosophy from [Spiderfoot](https://github.com/smicallef/spiderfoot), which means it is also ***event-driven***. Each of BBOT's 100+ modules ***consume*** a certain type of [Event](./scanning/events.md), use it to discover something new, and ***produce*** new events, which get distributed to all the other modules. This happens again and again -- thousands of times during a scan -- spidering outwards in a recursive web of discovery.\n\nBelow is an interactive graph showing the relationships between modules and the event types they produce and consume.\n\n\n
      \n\n\n\n## How BBOT Modules Work Together\n\nEach BBOT module does one specific task, such as querying an API for subdomains, or running a tool like `nuclei`, and is carefully designed to work together with other modules inside BBOT's recursive system.\n\nFor example, the `portscan` module consumes `DNS_NAME`, and produces `OPEN_TCP_PORT`. The `sslcert` module consumes `OPEN_TCP_PORT` and produces `DNS_NAME`. You can see how even these two modules, when enabled together, will feed each other recursively.\n\n![module-recursion](https://github.com/blacklanternsecurity/bbot/assets/20261699/10ff5fb4-b3e7-453d-9772-7a26808b071e)\n\nBecause of this, enabling even one module has the potential to increase your results exponentially. This is exactly how BBOT is able to outperform other tools.\n\nTo learn more about how events flow inside BBOT, see [BBOT Internal Architecture](./dev/architecture.md).\n" }, { "path": "docs/index.md", "content": "# Getting Started\n\n\n\n_A BBOT scan in real-time - visualization with [VivaGraphJS](https://github.com/blacklanternsecurity/bbot-vivagraphjs)_\n\n## Installation\n\n!!! info \"Supported Platforms\"\n\n Only **Linux** is supported at this time. **Windows** and **macOS** are *not* supported. If you use one of these platforms, consider using [Docker](#docker).\n\nBBOT offers multiple methods of installation, including **pipx** and **Docker**. If you're looking to tinker or write your own module, see [Setting up a Dev Environment](./dev/dev_environment.md).\n\n### [Python (pip / pipx)](https://pypi.org/project/bbot/)\n\n\n???+ note inline end\n\n `pipx` installs BBOT inside its own virtual environment.\n\n```bash\n# stable version\npipx install bbot\n\n# bleeding edge (dev branch)\npipx install --pip-args '\\--pre' bbot\n\n# execute bbot command\nbbot --help\n```\n\n### Docker\n\n[Docker images](https://hub.docker.com/r/blacklanternsecurity/bbot) are provided, along with helper script `bbot-docker.sh` to persist your scan data. Images come in four flavors: `dev`, `dev-full`, `stable`, and `stable-full`. `dev` is the latest bleeding edge version. `-full` images are larger and have all of BBOT's module dependencies preinstalled (wordlists, pip packages, etc.).\n\nScans are output to `~/.bbot/scans` (the usual place for BBOT scan data).\n\n```bash\n# dev (bleeding edge)\ndocker run -it blacklanternsecurity/bbot --help\n# dev (bleeding edge - full)\ndocker run -it blacklanternsecurity/bbot:dev-full --help\n\n# stable\ndocker run -it blacklanternsecurity/bbot:stable --help\n# stable (full)\ndocker run -it blacklanternsecurity/bbot:stable-full --help\n\n# helper script\ngit clone https://github.com/blacklanternsecurity/bbot && cd bbot\n./bbot-docker.sh --help\n```\n\nNote: If you need to pass in a custom preset, you can do so by mapping the preset into the container:\n\n```bash\n# use the preset `my_preset.yml` from the current directory\ndocker run --rm -it \\\n -v \"$HOME/.bbot/scans:/root/.bbot/scans\" \\\n -v \"$PWD/my_preset.yml:/my_preset.yml\" \\\n blacklanternsecurity/bbot -p /my_preset.yml\n```\n\n## Example Commands\n\nBelow are some examples of common scans.\n\n\n**Subdomains:**\n\n```bash\n# Perform a full subdomain enumeration on evilcorp.com\nbbot -t evilcorp.com -p subdomain-enum\n```\n\n**Subdomains (passive only):**\n\n```bash\n# Perform a passive-only subdomain enumeration on evilcorp.com\nbbot -t evilcorp.com -p subdomain-enum -rf passive\n```\n\n**Subdomains + port scan + web screenshots:**\n\n```bash\n# Port-scan every subdomain, screenshot every webpage, output to current directory\nbbot -t evilcorp.com -p subdomain-enum -m portscan gowitness -n my_scan -o .\n```\n\n**Subdomains + basic web scan:**\n\n```bash\n# A basic web scan includes robots.txt, storage buckets, IIS shortnames, and other non-intrusive web modules\nbbot -t evilcorp.com -p subdomain-enum web-basic\n```\n\n**Web spider:**\n\n```bash\n# Crawl www.evilcorp.com up to a max depth of 2, automatically extracting emails, secrets, etc.\nbbot -t www.evilcorp.com -p spider -c web.spider_distance=2 web.spider_depth=2\n```\n\n**Everything everywhere all at once:**\n\n```bash\n# Subdomains, emails, cloud buckets, port scan, basic web, web screenshots, nuclei\nbbot -t evilcorp.com -p kitchen-sink\n```\n\n\n## API Keys\n\nBBOT works just fine without API keys. However, there are certain modules that need them to function. If you have API keys and want to make use of these modules, you can place them either in your preset:\n\n```yaml title=\"my_preset.yml\"\ndescription: My custom subdomain enum preset\n\ninclude:\n - subdomain-enum\n - cloud-enum\n\nconfig:\n modules:\n shodan_dns:\n api_key: deadbeef\n virustotal:\n api_key: cafebabe\n```\n\n...in BBOT's global YAML config (`~/.config/bbot/bbot.yml`):\n\nNote: this will ensure the API keys are used in all scans, regardless of preset.\n\n```yaml title=\"~/.config/bbot/bbot.yml\"\nmodules:\n shodan_dns:\n api_key: deadbeef\n virustotal:\n api_key: cafebabe\n```\n\n...or directly on the command-line:\n\n```bash\n# specify API key with -c\nbbot -t evilcorp.com -f subdomain-enum -c modules.shodan_dns.api_key=deadbeef modules.virustotal.api_key=cafebabe\n```\n\nFor more information, see [Configuration](./scanning/configuration.md). For a full list of modules, including which ones require API keys, see [List of Modules](./modules/list_of_modules.md).\n\n[Next Up: Scanning -->](./scanning/index.md){ .md-button .md-button--primary }\n" }, { "path": "docs/javascripts/tablesort.js", "content": "document$.subscribe(function () {\n var tables = document.querySelectorAll(\"article table:not([class])\");\n tables.forEach(function (table) {\n new Tablesort(table);\n });\n});\n" }, { "path": "docs/javascripts/vega-embed@6.js", "content": "!function(e,t){\"object\"==typeof exports&&\"undefined\"!=typeof module?module.exports=t(require(\"vega\"),require(\"vega-lite\")):\"function\"==typeof define&&define.amd?define([\"vega\",\"vega-lite\"],t):(e=\"undefined\"!=typeof globalThis?globalThis:e||self).vegaEmbed=t(e.vega,e.vegaLite)}(this,(function(e,t){\"use strict\";function n(e){var t=Object.create(null);return e&&Object.keys(e).forEach((function(n){if(\"default\"!==n){var r=Object.getOwnPropertyDescriptor(e,n);Object.defineProperty(t,n,r.get?r:{enumerable:!0,get:function(){return e[n]}})}})),t.default=e,Object.freeze(t)}var r,i=n(e),o=n(t),a=(r=function(e,t){return r=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(e,t){e.__proto__=t}||function(e,t){for(var n in t)t.hasOwnProperty(n)&&(e[n]=t[n])},r(e,t)},function(e,t){function n(){this.constructor=e}r(e,t),e.prototype=null===t?Object.create(t):(n.prototype=t.prototype,new n)}),s=Object.prototype.hasOwnProperty;function l(e,t){return s.call(e,t)}function c(e){if(Array.isArray(e)){for(var t=new Array(e.length),n=0;n=48&&t<=57))return!1;n++}return!0}function p(e){return-1===e.indexOf(\"/\")&&-1===e.indexOf(\"~\")?e:e.replace(/~/g,\"~0\").replace(/\\//g,\"~1\")}function d(e){return e.replace(/~1/g,\"/\").replace(/~0/g,\"~\")}function u(e){if(void 0===e)return!0;if(e)if(Array.isArray(e)){for(var t=0,n=e.length;t0&&\"constructor\"==s[c-1]))throw new TypeError(\"JSON-Patch: modifying `__proto__` or `constructor/prototype` prop is banned for security reasons, if this was on purpose, please set `banPrototypeModifications` flag false and pass it to this function. More info in fast-json-patch README\");if(n&&void 0===u&&(void 0===l[g]?u=s.slice(0,c).join(\"/\"):c==p-1&&(u=t.path),void 0!==u&&m(t,0,e,u)),c++,Array.isArray(l)){if(\"-\"===g)g=l.length;else{if(n&&!f(g))throw new v(\"Expected an unsigned base-10 integer value, making the new referenced value the array element with the zero-based index\",\"OPERATION_PATH_ILLEGAL_ARRAY_INDEX\",o,t,e);f(g)&&(g=~~g)}if(c>=p){if(n&&\"add\"===t.op&&g>l.length)throw new v(\"The specified index MUST NOT be greater than the number of elements in the array\",\"OPERATION_VALUE_OUT_OF_BOUNDS\",o,t,e);if(!1===(a=y[t.op].call(t,l,g,e)).test)throw new v(\"Test operation failed\",\"TEST_OPERATION_FAILED\",o,t,e);return a}}else if(c>=p){if(!1===(a=b[t.op].call(t,l,g,e)).test)throw new v(\"Test operation failed\",\"TEST_OPERATION_FAILED\",o,t,e);return a}if(l=l[g],n&&c0)throw new v('Operation `path` property must start with \"/\"',\"OPERATION_PATH_INVALID\",t,e,n);if((\"move\"===e.op||\"copy\"===e.op)&&\"string\"!=typeof e.from)throw new v(\"Operation `from` property is not present (applicable in `move` and `copy` operations)\",\"OPERATION_FROM_REQUIRED\",t,e,n);if((\"add\"===e.op||\"replace\"===e.op||\"test\"===e.op)&&void 0===e.value)throw new v(\"Operation `value` property is not present (applicable in `add`, `replace` and `test` operations)\",\"OPERATION_VALUE_REQUIRED\",t,e,n);if((\"add\"===e.op||\"replace\"===e.op||\"test\"===e.op)&&u(e.value))throw new v(\"Operation `value` property is not present (applicable in `add`, `replace` and `test` operations)\",\"OPERATION_VALUE_CANNOT_CONTAIN_UNDEFINED\",t,e,n);if(n)if(\"add\"==e.op){var i=e.path.split(\"/\").length,o=r.split(\"/\").length;if(i!==o+1&&i!==o)throw new v(\"Cannot perform an `add` operation at the desired path\",\"OPERATION_PATH_CANNOT_ADD\",t,e,n)}else if(\"replace\"===e.op||\"remove\"===e.op||\"_get\"===e.op){if(e.path!==r)throw new v(\"Cannot perform the operation at a path that does not exist\",\"OPERATION_PATH_UNRESOLVABLE\",t,e,n)}else if(\"move\"===e.op||\"copy\"===e.op){var a=I([{op:\"_get\",path:e.from,value:void 0}],n);if(a&&\"OPERATION_PATH_UNRESOLVABLE\"===a.name)throw new v(\"Cannot perform the operation from a path that does not exist\",\"OPERATION_FROM_UNRESOLVABLE\",t,e,n)}}function I(e,t,n){try{if(!Array.isArray(e))throw new v(\"Patch sequence must be an array\",\"SEQUENCE_NOT_AN_ARRAY\");if(t)O(h(t),h(e),n||!0);else{n=n||x;for(var r=0;r0&&(e.patches=[],e.callback&&e.callback(r)),r}function C(e,t,n,r,i){if(t!==e){\"function\"==typeof t.toJSON&&(t=t.toJSON());for(var o=c(t),a=c(e),s=!1,f=a.length-1;f>=0;f--){var d=e[g=a[f]];if(!l(t,g)||void 0===t[g]&&void 0!==d&&!1===Array.isArray(t))Array.isArray(e)===Array.isArray(t)?(i&&n.push({op:\"test\",path:r+\"/\"+p(g),value:h(d)}),n.push({op:\"remove\",path:r+\"/\"+p(g)}),s=!0):(i&&n.push({op:\"test\",path:r,value:e}),n.push({op:\"replace\",path:r,value:t}));else{var u=t[g];\"object\"==typeof d&&null!=d&&\"object\"==typeof u&&null!=u&&Array.isArray(d)===Array.isArray(u)?C(d,u,n,r+\"/\"+p(g),i):d!==u&&(i&&n.push({op:\"test\",path:r+\"/\"+p(g),value:h(d)}),n.push({op:\"replace\",path:r+\"/\"+p(g),value:h(u)}))}}if(s||o.length!=a.length)for(f=0;f0)return[m,n+c.join(\",\\n\"+u),s].join(\"\\n\"+o)}return v}(e,\"\",0)},j=F(M);var z=U;function U(e){var t=this;if(t instanceof U||(t=new U),t.tail=null,t.head=null,t.length=0,e&&\"function\"==typeof e.forEach)e.forEach((function(e){t.push(e)}));else if(arguments.length>0)for(var n=0,r=arguments.length;n1)n=t;else{if(!this.head)throw new TypeError(\"Reduce of empty list with no initial value\");r=this.head.next,n=this.head.value}for(var i=0;null!==r;i++)n=e(n,r.value,i),r=r.next;return n},U.prototype.reduceReverse=function(e,t){var n,r=this.tail;if(arguments.length>1)n=t;else{if(!this.tail)throw new TypeError(\"Reduce of empty list with no initial value\");r=this.tail.prev,n=this.tail.value}for(var i=this.length-1;null!==r;i--)n=e(n,r.value,i),r=r.prev;return n},U.prototype.toArray=function(){for(var e=new Array(this.length),t=0,n=this.head;null!==n;t++)e[t]=n.value,n=n.next;return e},U.prototype.toArrayReverse=function(){for(var e=new Array(this.length),t=0,n=this.tail;null!==n;t++)e[t]=n.value,n=n.prev;return e},U.prototype.slice=function(e,t){(t=t||this.length)<0&&(t+=this.length),(e=e||0)<0&&(e+=this.length);var n=new U;if(tthis.length&&(t=this.length);for(var r=0,i=this.head;null!==i&&rthis.length&&(t=this.length);for(var r=this.length,i=this.tail;null!==i&&r>t;r--)i=i.prev;for(;null!==i&&r>e;r--,i=i.prev)n.push(i.value);return n},U.prototype.splice=function(e,t,...n){e>this.length&&(e=this.length-1),e<0&&(e=this.length+e);for(var r=0,i=this.head;null!==i&&r1;const ie=(e,t,n)=>{const r=e[te].get(t);if(r){const t=r.value;if(oe(e,t)){if(se(e,r),!e[J])return}else n&&(e[ne]&&(r.value.now=Date.now()),e[ee].unshiftNode(r));return t.value}},oe=(e,t)=>{if(!t||!t.maxAge&&!e[Q])return!1;const n=Date.now()-t.now;return t.maxAge?n>t.maxAge:e[Q]&&n>e[Q]},ae=e=>{if(e[q]>e[H])for(let t=e[ee].tail;e[q]>e[H]&&null!==t;){const n=t.prev;se(e,t),t=n}},se=(e,t)=>{if(t){const n=t.value;e[Z]&&e[Z](n.key,n.value),e[q]-=n.length,e[te].delete(n.key),e[ee].removeNode(t)}};class le{constructor(e,t,n,r,i){this.key=e,this.value=t,this.length=n,this.now=r,this.maxAge=i||0}}const ce=(e,t,n,r)=>{let i=n.value;oe(e,i)&&(se(e,n),e[J]||(i=void 0)),i&&t.call(r,i.value,i.key,e)};var he=class{constructor(e){if(\"number\"==typeof e&&(e={max:e}),e||(e={}),e.max&&(\"number\"!=typeof e.max||e.max<0))throw new TypeError(\"max must be a non-negative number\");this[H]=e.max||1/0;const t=e.length||re;if(this[Y]=\"function\"!=typeof t?re:t,this[J]=e.stale||!1,e.maxAge&&\"number\"!=typeof e.maxAge)throw new TypeError(\"maxAge must be a number\");this[Q]=e.maxAge||0,this[Z]=e.dispose,this[K]=e.noDisposeOnSet||!1,this[ne]=e.updateAgeOnGet||!1,this.reset()}set max(e){if(\"number\"!=typeof e||e<0)throw new TypeError(\"max must be a non-negative number\");this[H]=e||1/0,ae(this)}get max(){return this[H]}set allowStale(e){this[J]=!!e}get allowStale(){return this[J]}set maxAge(e){if(\"number\"!=typeof e)throw new TypeError(\"maxAge must be a non-negative number\");this[Q]=e,ae(this)}get maxAge(){return this[Q]}set lengthCalculator(e){\"function\"!=typeof e&&(e=re),e!==this[Y]&&(this[Y]=e,this[q]=0,this[ee].forEach((e=>{e.length=this[Y](e.value,e.key),this[q]+=e.length}))),ae(this)}get lengthCalculator(){return this[Y]}get length(){return this[q]}get itemCount(){return this[ee].length}rforEach(e,t){t=t||this;for(let n=this[ee].tail;null!==n;){const r=n.prev;ce(this,e,n,t),n=r}}forEach(e,t){t=t||this;for(let n=this[ee].head;null!==n;){const r=n.next;ce(this,e,n,t),n=r}}keys(){return this[ee].toArray().map((e=>e.key))}values(){return this[ee].toArray().map((e=>e.value))}reset(){this[Z]&&this[ee]&&this[ee].length&&this[ee].forEach((e=>this[Z](e.key,e.value))),this[te]=new Map,this[ee]=new V,this[q]=0}dump(){return this[ee].map((e=>!oe(this,e)&&{k:e.key,v:e.value,e:e.now+(e.maxAge||0)})).toArray().filter((e=>e))}dumpLru(){return this[ee]}set(e,t,n){if((n=n||this[Q])&&\"number\"!=typeof n)throw new TypeError(\"maxAge must be a number\");const r=n?Date.now():0,i=this[Y](t,e);if(this[te].has(e)){if(i>this[H])return se(this,this[te].get(e)),!1;const o=this[te].get(e).value;return this[Z]&&(this[K]||this[Z](e,o.value)),o.now=r,o.maxAge=n,o.value=t,this[q]+=i-o.length,o.length=i,this.get(e),ae(this),!0}const o=new le(e,t,i,r,n);return o.length>this[H]?(this[Z]&&this[Z](e,t),!1):(this[q]+=o.length,this[ee].unshift(o),this[te].set(e,this[ee].head),ae(this),!0)}has(e){if(!this[te].has(e))return!1;const t=this[te].get(e).value;return!oe(this,t)}get(e){return ie(this,e,!0)}peek(e){return ie(this,e,!1)}pop(){const e=this[ee].tail;return e?(se(this,e),e.value):null}del(e){se(this,this[te].get(e))}load(e){this.reset();const t=Date.now();for(let n=e.length-1;n>=0;n--){const r=e[n],i=r.e||0;if(0===i)this.set(r.k,r.v);else{const e=i-t;e>0&&this.set(r.k,r.v,e)}}}prune(){this[te].forEach(((e,t)=>ie(this,t,!1)))}};const fe=Object.freeze({loose:!0}),pe=Object.freeze({});var de=e=>e?\"object\"!=typeof e?fe:e:pe,ue={exports:{}};var ge={MAX_LENGTH:256,MAX_SAFE_COMPONENT_LENGTH:16,MAX_SAFE_BUILD_LENGTH:250,MAX_SAFE_INTEGER:Number.MAX_SAFE_INTEGER||9007199254740991,RELEASE_TYPES:[\"major\",\"premajor\",\"minor\",\"preminor\",\"patch\",\"prepatch\",\"prerelease\"],SEMVER_SPEC_VERSION:\"2.0.0\",FLAG_INCLUDE_PRERELEASE:1,FLAG_LOOSE:2};var me=\"object\"==typeof process&&process.env&&process.env.NODE_DEBUG&&/\\bsemver\\b/i.test(process.env.NODE_DEBUG)?(...e)=>console.error(\"SEMVER\",...e):()=>{};!function(e,t){const{MAX_SAFE_COMPONENT_LENGTH:n,MAX_SAFE_BUILD_LENGTH:r,MAX_LENGTH:i}=ge,o=me,a=(t=e.exports={}).re=[],s=t.safeRe=[],l=t.src=[],c=t.t={};let h=0;const f=\"[a-zA-Z0-9-]\",p=[[\"\\\\s\",1],[\"\\\\d\",i],[f,r]],d=(e,t,n)=>{const r=(e=>{for(const[t,n]of p)e=e.split(`${t}*`).join(`${t}{0,${n}}`).split(`${t}+`).join(`${t}{1,${n}}`);return e})(t),i=h++;o(e,i,t),c[e]=i,l[i]=t,a[i]=new RegExp(t,n?\"g\":void 0),s[i]=new RegExp(r,n?\"g\":void 0)};d(\"NUMERICIDENTIFIER\",\"0|[1-9]\\\\d*\"),d(\"NUMERICIDENTIFIERLOOSE\",\"\\\\d+\"),d(\"NONNUMERICIDENTIFIER\",`\\\\d*[a-zA-Z-]${f}*`),d(\"MAINVERSION\",`(${l[c.NUMERICIDENTIFIER]})\\\\.(${l[c.NUMERICIDENTIFIER]})\\\\.(${l[c.NUMERICIDENTIFIER]})`),d(\"MAINVERSIONLOOSE\",`(${l[c.NUMERICIDENTIFIERLOOSE]})\\\\.(${l[c.NUMERICIDENTIFIERLOOSE]})\\\\.(${l[c.NUMERICIDENTIFIERLOOSE]})`),d(\"PRERELEASEIDENTIFIER\",`(?:${l[c.NUMERICIDENTIFIER]}|${l[c.NONNUMERICIDENTIFIER]})`),d(\"PRERELEASEIDENTIFIERLOOSE\",`(?:${l[c.NUMERICIDENTIFIERLOOSE]}|${l[c.NONNUMERICIDENTIFIER]})`),d(\"PRERELEASE\",`(?:-(${l[c.PRERELEASEIDENTIFIER]}(?:\\\\.${l[c.PRERELEASEIDENTIFIER]})*))`),d(\"PRERELEASELOOSE\",`(?:-?(${l[c.PRERELEASEIDENTIFIERLOOSE]}(?:\\\\.${l[c.PRERELEASEIDENTIFIERLOOSE]})*))`),d(\"BUILDIDENTIFIER\",`${f}+`),d(\"BUILD\",`(?:\\\\+(${l[c.BUILDIDENTIFIER]}(?:\\\\.${l[c.BUILDIDENTIFIER]})*))`),d(\"FULLPLAIN\",`v?${l[c.MAINVERSION]}${l[c.PRERELEASE]}?${l[c.BUILD]}?`),d(\"FULL\",`^${l[c.FULLPLAIN]}$`),d(\"LOOSEPLAIN\",`[v=\\\\s]*${l[c.MAINVERSIONLOOSE]}${l[c.PRERELEASELOOSE]}?${l[c.BUILD]}?`),d(\"LOOSE\",`^${l[c.LOOSEPLAIN]}$`),d(\"GTLT\",\"((?:<|>)?=?)\"),d(\"XRANGEIDENTIFIERLOOSE\",`${l[c.NUMERICIDENTIFIERLOOSE]}|x|X|\\\\*`),d(\"XRANGEIDENTIFIER\",`${l[c.NUMERICIDENTIFIER]}|x|X|\\\\*`),d(\"XRANGEPLAIN\",`[v=\\\\s]*(${l[c.XRANGEIDENTIFIER]})(?:\\\\.(${l[c.XRANGEIDENTIFIER]})(?:\\\\.(${l[c.XRANGEIDENTIFIER]})(?:${l[c.PRERELEASE]})?${l[c.BUILD]}?)?)?`),d(\"XRANGEPLAINLOOSE\",`[v=\\\\s]*(${l[c.XRANGEIDENTIFIERLOOSE]})(?:\\\\.(${l[c.XRANGEIDENTIFIERLOOSE]})(?:\\\\.(${l[c.XRANGEIDENTIFIERLOOSE]})(?:${l[c.PRERELEASELOOSE]})?${l[c.BUILD]}?)?)?`),d(\"XRANGE\",`^${l[c.GTLT]}\\\\s*${l[c.XRANGEPLAIN]}$`),d(\"XRANGELOOSE\",`^${l[c.GTLT]}\\\\s*${l[c.XRANGEPLAINLOOSE]}$`),d(\"COERCEPLAIN\",`(^|[^\\\\d])(\\\\d{1,${n}})(?:\\\\.(\\\\d{1,${n}}))?(?:\\\\.(\\\\d{1,${n}}))?`),d(\"COERCE\",`${l[c.COERCEPLAIN]}(?:$|[^\\\\d])`),d(\"COERCEFULL\",l[c.COERCEPLAIN]+`(?:${l[c.PRERELEASE]})?`+`(?:${l[c.BUILD]})?(?:$|[^\\\\d])`),d(\"COERCERTL\",l[c.COERCE],!0),d(\"COERCERTLFULL\",l[c.COERCEFULL],!0),d(\"LONETILDE\",\"(?:~>?)\"),d(\"TILDETRIM\",`(\\\\s*)${l[c.LONETILDE]}\\\\s+`,!0),t.tildeTrimReplace=\"$1~\",d(\"TILDE\",`^${l[c.LONETILDE]}${l[c.XRANGEPLAIN]}$`),d(\"TILDELOOSE\",`^${l[c.LONETILDE]}${l[c.XRANGEPLAINLOOSE]}$`),d(\"LONECARET\",\"(?:\\\\^)\"),d(\"CARETTRIM\",`(\\\\s*)${l[c.LONECARET]}\\\\s+`,!0),t.caretTrimReplace=\"$1^\",d(\"CARET\",`^${l[c.LONECARET]}${l[c.XRANGEPLAIN]}$`),d(\"CARETLOOSE\",`^${l[c.LONECARET]}${l[c.XRANGEPLAINLOOSE]}$`),d(\"COMPARATORLOOSE\",`^${l[c.GTLT]}\\\\s*(${l[c.LOOSEPLAIN]})$|^$`),d(\"COMPARATOR\",`^${l[c.GTLT]}\\\\s*(${l[c.FULLPLAIN]})$|^$`),d(\"COMPARATORTRIM\",`(\\\\s*)${l[c.GTLT]}\\\\s*(${l[c.LOOSEPLAIN]}|${l[c.XRANGEPLAIN]})`,!0),t.comparatorTrimReplace=\"$1$2$3\",d(\"HYPHENRANGE\",`^\\\\s*(${l[c.XRANGEPLAIN]})\\\\s+-\\\\s+(${l[c.XRANGEPLAIN]})\\\\s*$`),d(\"HYPHENRANGELOOSE\",`^\\\\s*(${l[c.XRANGEPLAINLOOSE]})\\\\s+-\\\\s+(${l[c.XRANGEPLAINLOOSE]})\\\\s*$`),d(\"STAR\",\"(<|>)?=?\\\\s*\\\\*\"),d(\"GTE0\",\"^\\\\s*>=\\\\s*0\\\\.0\\\\.0\\\\s*$\"),d(\"GTE0PRE\",\"^\\\\s*>=\\\\s*0\\\\.0\\\\.0-0\\\\s*$\")}(ue,ue.exports);var ve=ue.exports;const Ee=/^[0-9]+$/,be=(e,t)=>{const n=Ee.test(e),r=Ee.test(t);return n&&r&&(e=+e,t=+t),e===t?0:n&&!r?-1:r&&!n?1:ebe(t,e)};const we=me,{MAX_LENGTH:Ae,MAX_SAFE_INTEGER:Oe}=ge,{safeRe:xe,t:Ie}=ve,Ne=de,{compareIdentifiers:Le}=ye;var Re=class e{constructor(t,n){if(n=Ne(n),t instanceof e){if(t.loose===!!n.loose&&t.includePrerelease===!!n.includePrerelease)return t;t=t.version}else if(\"string\"!=typeof t)throw new TypeError(`Invalid version. Must be a string. Got type \"${typeof t}\".`);if(t.length>Ae)throw new TypeError(`version is longer than ${Ae} characters`);we(\"SemVer\",t,n),this.options=n,this.loose=!!n.loose,this.includePrerelease=!!n.includePrerelease;const r=t.trim().match(n.loose?xe[Ie.LOOSE]:xe[Ie.FULL]);if(!r)throw new TypeError(`Invalid Version: ${t}`);if(this.raw=t,this.major=+r[1],this.minor=+r[2],this.patch=+r[3],this.major>Oe||this.major<0)throw new TypeError(\"Invalid major version\");if(this.minor>Oe||this.minor<0)throw new TypeError(\"Invalid minor version\");if(this.patch>Oe||this.patch<0)throw new TypeError(\"Invalid patch version\");r[4]?this.prerelease=r[4].split(\".\").map((e=>{if(/^[0-9]+$/.test(e)){const t=+e;if(t>=0&&t=0;)\"number\"==typeof this.prerelease[r]&&(this.prerelease[r]++,r=-2);if(-1===r){if(t===this.prerelease.join(\".\")&&!1===n)throw new Error(\"invalid increment argument: identifier already exists\");this.prerelease.push(e)}}if(t){let r=[t,e];!1===n&&(r=[t]),0===Le(this.prerelease[0],t)?isNaN(this.prerelease[1])&&(this.prerelease=r):this.prerelease=r}break}default:throw new Error(`invalid increment argument: ${e}`)}return this.raw=this.format(),this.build.length&&(this.raw+=`+${this.build.join(\".\")}`),this}};const $e=Re;var Se=(e,t,n)=>new $e(e,n).compare(new $e(t,n));const Te=Se;const Ce=Se;const De=Se;const Fe=Se;const ke=Se;const _e=Se;const Pe=(e,t,n)=>0===Te(e,t,n),Me=(e,t,n)=>0!==Ce(e,t,n),je=(e,t,n)=>De(e,t,n)>0,ze=(e,t,n)=>Fe(e,t,n)>=0,Ue=(e,t,n)=>ke(e,t,n)<0,Be=(e,t,n)=>_e(e,t,n)<=0;var Ge,We,Xe,Ve,He=(e,t,n,r)=>{switch(t){case\"===\":return\"object\"==typeof e&&(e=e.version),\"object\"==typeof n&&(n=n.version),e===n;case\"!==\":return\"object\"==typeof e&&(e=e.version),\"object\"==typeof n&&(n=n.version),e!==n;case\"\":case\"=\":case\"==\":return Pe(e,n,r);case\"!=\":return Me(e,n,r);case\">\":return je(e,n,r);case\">=\":return ze(e,n,r);case\"<\":return Ue(e,n,r);case\"<=\":return Be(e,n,r);default:throw new TypeError(`Invalid operator: ${t}`)}};function qe(){if(Ve)return Xe;Ve=1;class e{constructor(t,i){if(i=n(i),t instanceof e)return t.loose===!!i.loose&&t.includePrerelease===!!i.includePrerelease?t:new e(t.raw,i);if(t instanceof r)return this.raw=t.value,this.set=[[t]],this.format(),this;if(this.options=i,this.loose=!!i.loose,this.includePrerelease=!!i.includePrerelease,this.raw=t.trim().split(/\\s+/).join(\" \"),this.set=this.raw.split(\"||\").map((e=>this.parseRange(e.trim()))).filter((e=>e.length)),!this.set.length)throw new TypeError(`Invalid SemVer Range: ${this.raw}`);if(this.set.length>1){const e=this.set[0];if(this.set=this.set.filter((e=>!d(e[0]))),0===this.set.length)this.set=[e];else if(this.set.length>1)for(const e of this.set)if(1===e.length&&u(e[0])){this.set=[e];break}}this.format()}format(){return this.range=this.set.map((e=>e.join(\" \").trim())).join(\"||\").trim(),this.range}toString(){return this.range}parseRange(e){const n=((this.options.includePrerelease&&f)|(this.options.loose&&p))+\":\"+e,o=t.get(n);if(o)return o;const u=this.options.loose,g=u?a[s.HYPHENRANGELOOSE]:a[s.HYPHENRANGE];e=e.replace(g,N(this.options.includePrerelease)),i(\"hyphen replace\",e),e=e.replace(a[s.COMPARATORTRIM],l),i(\"comparator trim\",e),e=e.replace(a[s.TILDETRIM],c),i(\"tilde trim\",e),e=e.replace(a[s.CARETTRIM],h),i(\"caret trim\",e);let v=e.split(\" \").map((e=>m(e,this.options))).join(\" \").split(/\\s+/).map((e=>I(e,this.options)));u&&(v=v.filter((e=>(i(\"loose invalid filter\",e,this.options),!!e.match(a[s.COMPARATORLOOSE]))))),i(\"range list\",v);const E=new Map,b=v.map((e=>new r(e,this.options)));for(const e of b){if(d(e))return[e];E.set(e.value,e)}E.size>1&&E.has(\"\")&&E.delete(\"\");const y=[...E.values()];return t.set(n,y),y}intersects(t,n){if(!(t instanceof e))throw new TypeError(\"a Range is required\");return this.set.some((e=>g(e,n)&&t.set.some((t=>g(t,n)&&e.every((e=>t.every((t=>e.intersects(t,n)))))))))}test(e){if(!e)return!1;if(\"string\"==typeof e)try{e=new o(e,this.options)}catch(e){return!1}for(let t=0;t\")||!e.operator.startsWith(\">\"))&&(!this.operator.startsWith(\"<\")||!e.operator.startsWith(\"<\"))&&(this.semver.version!==e.semver.version||!this.operator.includes(\"=\")||!e.operator.includes(\"=\"))&&!(o(this.semver,\"<\",e.semver,r)&&this.operator.startsWith(\">\")&&e.operator.startsWith(\"<\"))&&!(o(this.semver,\">\",e.semver,r)&&this.operator.startsWith(\"<\")&&e.operator.startsWith(\">\")))}}Ge=t;const n=de,{safeRe:r,t:i}=ve,o=He,a=me,s=Re,l=qe();return Ge}(),i=me,o=Re,{safeRe:a,t:s,comparatorTrimReplace:l,tildeTrimReplace:c,caretTrimReplace:h}=ve,{FLAG_INCLUDE_PRERELEASE:f,FLAG_LOOSE:p}=ge,d=e=>\"<0.0.0-0\"===e.value,u=e=>\"\"===e.value,g=(e,t)=>{let n=!0;const r=e.slice();let i=r.pop();for(;n&&r.length;)n=r.every((e=>i.intersects(e,t))),i=r.pop();return n},m=(e,t)=>(i(\"comp\",e,t),e=y(e,t),i(\"caret\",e),e=E(e,t),i(\"tildes\",e),e=A(e,t),i(\"xrange\",e),e=x(e,t),i(\"stars\",e),e),v=e=>!e||\"x\"===e.toLowerCase()||\"*\"===e,E=(e,t)=>e.trim().split(/\\s+/).map((e=>b(e,t))).join(\" \"),b=(e,t)=>{const n=t.loose?a[s.TILDELOOSE]:a[s.TILDE];return e.replace(n,((t,n,r,o,a)=>{let s;return i(\"tilde\",e,t,n,r,o,a),v(n)?s=\"\":v(r)?s=`>=${n}.0.0 <${+n+1}.0.0-0`:v(o)?s=`>=${n}.${r}.0 <${n}.${+r+1}.0-0`:a?(i(\"replaceTilde pr\",a),s=`>=${n}.${r}.${o}-${a} <${n}.${+r+1}.0-0`):s=`>=${n}.${r}.${o} <${n}.${+r+1}.0-0`,i(\"tilde return\",s),s}))},y=(e,t)=>e.trim().split(/\\s+/).map((e=>w(e,t))).join(\" \"),w=(e,t)=>{i(\"caret\",e,t);const n=t.loose?a[s.CARETLOOSE]:a[s.CARET],r=t.includePrerelease?\"-0\":\"\";return e.replace(n,((t,n,o,a,s)=>{let l;return i(\"caret\",e,t,n,o,a,s),v(n)?l=\"\":v(o)?l=`>=${n}.0.0${r} <${+n+1}.0.0-0`:v(a)?l=\"0\"===n?`>=${n}.${o}.0${r} <${n}.${+o+1}.0-0`:`>=${n}.${o}.0${r} <${+n+1}.0.0-0`:s?(i(\"replaceCaret pr\",s),l=\"0\"===n?\"0\"===o?`>=${n}.${o}.${a}-${s} <${n}.${o}.${+a+1}-0`:`>=${n}.${o}.${a}-${s} <${n}.${+o+1}.0-0`:`>=${n}.${o}.${a}-${s} <${+n+1}.0.0-0`):(i(\"no pr\"),l=\"0\"===n?\"0\"===o?`>=${n}.${o}.${a}${r} <${n}.${o}.${+a+1}-0`:`>=${n}.${o}.${a}${r} <${n}.${+o+1}.0-0`:`>=${n}.${o}.${a} <${+n+1}.0.0-0`),i(\"caret return\",l),l}))},A=(e,t)=>(i(\"replaceXRanges\",e,t),e.split(/\\s+/).map((e=>O(e,t))).join(\" \")),O=(e,t)=>{e=e.trim();const n=t.loose?a[s.XRANGELOOSE]:a[s.XRANGE];return e.replace(n,((n,r,o,a,s,l)=>{i(\"xRange\",e,n,r,o,a,s,l);const c=v(o),h=c||v(a),f=h||v(s),p=f;return\"=\"===r&&p&&(r=\"\"),l=t.includePrerelease?\"-0\":\"\",c?n=\">\"===r||\"<\"===r?\"<0.0.0-0\":\"*\":r&&p?(h&&(a=0),s=0,\">\"===r?(r=\">=\",h?(o=+o+1,a=0,s=0):(a=+a+1,s=0)):\"<=\"===r&&(r=\"<\",h?o=+o+1:a=+a+1),\"<\"===r&&(l=\"-0\"),n=`${r+o}.${a}.${s}${l}`):h?n=`>=${o}.0.0${l} <${+o+1}.0.0-0`:f&&(n=`>=${o}.${a}.0${l} <${o}.${+a+1}.0-0`),i(\"xRange return\",n),n}))},x=(e,t)=>(i(\"replaceStars\",e,t),e.trim().replace(a[s.STAR],\"\")),I=(e,t)=>(i(\"replaceGTE0\",e,t),e.trim().replace(a[t.includePrerelease?s.GTE0PRE:s.GTE0],\"\")),N=e=>(t,n,r,i,o,a,s,l,c,h,f,p,d)=>`${n=v(r)?\"\":v(i)?`>=${r}.0.0${e?\"-0\":\"\"}`:v(o)?`>=${r}.${i}.0${e?\"-0\":\"\"}`:a?`>=${n}`:`>=${n}${e?\"-0\":\"\"}`} ${l=v(c)?\"\":v(h)?`<${+c+1}.0.0-0`:v(f)?`<${c}.${+h+1}.0-0`:p?`<=${c}.${h}.${f}-${p}`:e?`<${c}.${h}.${+f+1}-0`:`<=${l}`}`.trim(),L=(e,t,n)=>{for(let n=0;n0){const r=e[n].semver;if(r.major===t.major&&r.minor===t.minor&&r.patch===t.patch)return!0}return!1}return!0};return Xe}const Ye=qe();var Je=(e,t,n)=>{try{t=new Ye(t,n)}catch(e){return!1}return t.test(e)},Qe=F(Je);var Ze={NaN:NaN,E:Math.E,LN2:Math.LN2,LN10:Math.LN10,LOG2E:Math.LOG2E,LOG10E:Math.LOG10E,PI:Math.PI,SQRT1_2:Math.SQRT1_2,SQRT2:Math.SQRT2,MIN_VALUE:Number.MIN_VALUE,MAX_VALUE:Number.MAX_VALUE},Ke={\"*\":(e,t)=>e*t,\"+\":(e,t)=>e+t,\"-\":(e,t)=>e-t,\"/\":(e,t)=>e/t,\"%\":(e,t)=>e%t,\">\":(e,t)=>e>t,\"<\":(e,t)=>ee<=t,\">=\":(e,t)=>e>=t,\"==\":(e,t)=>e==t,\"!=\":(e,t)=>e!=t,\"===\":(e,t)=>e===t,\"!==\":(e,t)=>e!==t,\"&\":(e,t)=>e&t,\"|\":(e,t)=>e|t,\"^\":(e,t)=>e^t,\"<<\":(e,t)=>e<>\":(e,t)=>e>>t,\">>>\":(e,t)=>e>>>t},et={\"+\":e=>+e,\"-\":e=>-e,\"~\":e=>~e,\"!\":e=>!e};const tt=Array.prototype.slice,nt=(e,t,n)=>{const r=n?n(t[0]):t[0];return r[e].apply(r,tt.call(t,1))};var rt={isNaN:Number.isNaN,isFinite:Number.isFinite,abs:Math.abs,acos:Math.acos,asin:Math.asin,atan:Math.atan,atan2:Math.atan2,ceil:Math.ceil,cos:Math.cos,exp:Math.exp,floor:Math.floor,log:Math.log,max:Math.max,min:Math.min,pow:Math.pow,random:Math.random,round:Math.round,sin:Math.sin,sqrt:Math.sqrt,tan:Math.tan,clamp:(e,t,n)=>Math.max(t,Math.min(n,e)),now:Date.now,utc:Date.UTC,datetime:(e,t,n,r,i,o,a)=>new Date(e,t||0,null!=n?n:1,r||0,i||0,o||0,a||0),date:e=>new Date(e).getDate(),day:e=>new Date(e).getDay(),year:e=>new Date(e).getFullYear(),month:e=>new Date(e).getMonth(),hours:e=>new Date(e).getHours(),minutes:e=>new Date(e).getMinutes(),seconds:e=>new Date(e).getSeconds(),milliseconds:e=>new Date(e).getMilliseconds(),time:e=>new Date(e).getTime(),timezoneoffset:e=>new Date(e).getTimezoneOffset(),utcdate:e=>new Date(e).getUTCDate(),utcday:e=>new Date(e).getUTCDay(),utcyear:e=>new Date(e).getUTCFullYear(),utcmonth:e=>new Date(e).getUTCMonth(),utchours:e=>new Date(e).getUTCHours(),utcminutes:e=>new Date(e).getUTCMinutes(),utcseconds:e=>new Date(e).getUTCSeconds(),utcmilliseconds:e=>new Date(e).getUTCMilliseconds(),length:e=>e.length,join:function(){return nt(\"join\",arguments)},indexof:function(){return nt(\"indexOf\",arguments)},lastindexof:function(){return nt(\"lastIndexOf\",arguments)},slice:function(){return nt(\"slice\",arguments)},reverse:e=>e.slice().reverse(),parseFloat:parseFloat,parseInt:parseInt,upper:e=>String(e).toUpperCase(),lower:e=>String(e).toLowerCase(),substring:function(){return nt(\"substring\",arguments,String)},split:function(){return nt(\"split\",arguments,String)},replace:function(){return nt(\"replace\",arguments,String)},trim:e=>String(e).trim(),regexp:RegExp,test:(e,t)=>RegExp(e).test(t)};const it=[\"view\",\"item\",\"group\",\"xy\",\"x\",\"y\"],ot=new Set([Function,eval,setTimeout,setInterval]);\"function\"==typeof setImmediate&&ot.add(setImmediate);const at={Literal:(e,t)=>t.value,Identifier:(e,t)=>{const n=t.name;return e.memberDepth>0?n:\"datum\"===n?e.datum:\"event\"===n?e.event:\"item\"===n?e.item:Ze[n]||e.params[\"$\"+n]},MemberExpression:(e,t)=>{const n=!t.computed,r=e(t.object);n&&(e.memberDepth+=1);const i=e(t.property);if(n&&(e.memberDepth-=1),!ot.has(r[i]))return r[i];console.error(`Prevented interpretation of member \"${i}\" which could lead to insecure code execution`)},CallExpression:(e,t)=>{const n=t.arguments;let r=t.callee.name;return r.startsWith(\"_\")&&(r=r.slice(1)),\"if\"===r?e(n[0])?e(n[1]):e(n[2]):(e.fn[r]||rt[r]).apply(e.fn,n.map(e))},ArrayExpression:(e,t)=>t.elements.map(e),BinaryExpression:(e,t)=>Ke[t.operator](e(t.left),e(t.right)),UnaryExpression:(e,t)=>et[t.operator](e(t.argument)),ConditionalExpression:(e,t)=>e(t.test)?e(t.consequent):e(t.alternate),LogicalExpression:(e,t)=>\"&&\"===t.operator?e(t.left)&&e(t.right):e(t.left)||e(t.right),ObjectExpression:(e,t)=>t.properties.reduce(((t,n)=>{e.memberDepth+=1;const r=e(n.key);return e.memberDepth-=1,ot.has(e(n.value))?console.error(`Prevented interpretation of property \"${r}\" which could lead to insecure code execution`):t[r]=e(n.value),t}),{})};function st(e,t,n,r,i,o){const a=e=>at[e.type](a,e);return a.memberDepth=0,a.fn=Object.create(t),a.params=n,a.datum=r,a.event=i,a.item=o,it.forEach((e=>a.fn[e]=function(){return i.vega[e](...arguments)})),a(e)}var lt={operator(e,t){const n=t.ast,r=e.functions;return e=>st(n,r,e)},parameter(e,t){const n=t.ast,r=e.functions;return(e,t)=>st(n,r,t,e)},event(e,t){const n=t.ast,r=e.functions;return e=>st(n,r,void 0,void 0,e)},handler(e,t){const n=t.ast,r=e.functions;return(e,t)=>{const i=t.item&&t.item.datum;return st(n,r,e,i,t)}},encode(e,t){const{marktype:n,channels:r}=t,i=e.functions,o=\"group\"===n||\"image\"===n||\"rect\"===n;return(e,t)=>{const a=e.datum;let s,l=0;for(const n in r)s=st(r[n].ast,i,t,a,void 0,e),e[n]!==s&&(e[n]=s,l=1);return\"rule\"!==n&&function(e,t,n){let r;t.x2&&(t.x?(n&&e.x>e.x2&&(r=e.x,e.x=e.x2,e.x2=r),e.width=e.x2-e.x):e.x=e.x2-(e.width||0)),t.xc&&(e.x=e.xc-(e.width||0)/2),t.y2&&(t.y?(n&&e.y>e.y2&&(r=e.y,e.y=e.y2,e.y2=r),e.height=e.y2-e.y):e.y=e.y2-(e.height||0)),t.yc&&(e.y=e.yc-(e.height||0)/2)}(e,r,o),l}}};function ct(e){const[t,n]=/schema\\/([\\w-]+)\\/([\\w\\.\\-]+)\\.json$/g.exec(e).slice(1,3);return{library:t,version:n}}var ht=\"2.14.0\";const ft=\"#fff\",pt=\"#888\",dt={background:\"#333\",view:{stroke:pt},title:{color:ft,subtitleColor:ft},style:{\"guide-label\":{fill:ft},\"guide-title\":{fill:ft}},axis:{domainColor:ft,gridColor:pt,tickColor:ft}},ut=\"#4572a7\",gt={background:\"#fff\",arc:{fill:ut},area:{fill:ut},line:{stroke:ut,strokeWidth:2},path:{stroke:ut},rect:{fill:ut},shape:{stroke:ut},symbol:{fill:ut,strokeWidth:1.5,size:50},axis:{bandPosition:.5,grid:!0,gridColor:\"#000000\",gridOpacity:1,gridWidth:.5,labelPadding:10,tickSize:5,tickWidth:.5},axisBand:{grid:!1,tickExtra:!0},legend:{labelBaseline:\"middle\",labelFontSize:11,symbolSize:50,symbolType:\"square\"},range:{category:[\"#4572a7\",\"#aa4643\",\"#8aa453\",\"#71598e\",\"#4598ae\",\"#d98445\",\"#94aace\",\"#d09393\",\"#b9cc98\",\"#a99cbc\"]}},mt=\"#30a2da\",vt=\"#cbcbcb\",Et=\"#f0f0f0\",bt=\"#333\",yt={arc:{fill:mt},area:{fill:mt},axis:{domainColor:vt,grid:!0,gridColor:vt,gridWidth:1,labelColor:\"#999\",labelFontSize:10,titleColor:\"#333\",tickColor:vt,tickSize:10,titleFontSize:14,titlePadding:10,labelPadding:4},axisBand:{grid:!1},background:Et,group:{fill:Et},legend:{labelColor:bt,labelFontSize:11,padding:1,symbolSize:30,symbolType:\"square\",titleColor:bt,titleFontSize:14,titlePadding:10},line:{stroke:mt,strokeWidth:2},path:{stroke:mt,strokeWidth:.5},rect:{fill:mt},range:{category:[\"#30a2da\",\"#fc4f30\",\"#e5ae38\",\"#6d904f\",\"#8b8b8b\",\"#b96db8\",\"#ff9e27\",\"#56cc60\",\"#52d2ca\",\"#52689e\",\"#545454\",\"#9fe4f8\"],diverging:[\"#cc0020\",\"#e77866\",\"#f6e7e1\",\"#d6e8ed\",\"#91bfd9\",\"#1d78b5\"],heatmap:[\"#d6e8ed\",\"#cee0e5\",\"#91bfd9\",\"#549cc6\",\"#1d78b5\"]},point:{filled:!0,shape:\"circle\"},shape:{stroke:mt},bar:{binSpacing:2,fill:mt,stroke:null},title:{anchor:\"start\",fontSize:24,fontWeight:600,offset:20}},wt=\"#000\",At={group:{fill:\"#e5e5e5\"},arc:{fill:wt},area:{fill:wt},line:{stroke:wt},path:{stroke:wt},rect:{fill:wt},shape:{stroke:wt},symbol:{fill:wt,size:40},axis:{domain:!1,grid:!0,gridColor:\"#FFFFFF\",gridOpacity:1,labelColor:\"#7F7F7F\",labelPadding:4,tickColor:\"#7F7F7F\",tickSize:5.67,titleFontSize:16,titleFontWeight:\"normal\"},legend:{labelBaseline:\"middle\",labelFontSize:11,symbolSize:40},range:{category:[\"#000000\",\"#7F7F7F\",\"#1A1A1A\",\"#999999\",\"#333333\",\"#B0B0B0\",\"#4D4D4D\",\"#C9C9C9\",\"#666666\",\"#DCDCDC\"]}},Ot=\"Benton Gothic, sans-serif\",xt=\"#82c6df\",It=\"Benton Gothic Bold, sans-serif\",Nt=\"normal\",Lt={\"category-6\":[\"#ec8431\",\"#829eb1\",\"#c89d29\",\"#3580b1\",\"#adc839\",\"#ab7fb4\"],\"fire-7\":[\"#fbf2c7\",\"#f9e39c\",\"#f8d36e\",\"#f4bb6a\",\"#e68a4f\",\"#d15a40\",\"#ab4232\"],\"fireandice-6\":[\"#e68a4f\",\"#f4bb6a\",\"#f9e39c\",\"#dadfe2\",\"#a6b7c6\",\"#849eae\"],\"ice-7\":[\"#edefee\",\"#dadfe2\",\"#c4ccd2\",\"#a6b7c6\",\"#849eae\",\"#607785\",\"#47525d\"]},Rt={background:\"#ffffff\",title:{anchor:\"start\",color:\"#000000\",font:It,fontSize:22,fontWeight:\"normal\"},arc:{fill:xt},area:{fill:xt},line:{stroke:xt,strokeWidth:2},path:{stroke:xt},rect:{fill:xt},shape:{stroke:xt},symbol:{fill:xt,size:30},axis:{labelFont:Ot,labelFontSize:11.5,labelFontWeight:\"normal\",titleFont:It,titleFontSize:13,titleFontWeight:Nt},axisX:{labelAngle:0,labelPadding:4,tickSize:3},axisY:{labelBaseline:\"middle\",maxExtent:45,minExtent:45,tickSize:2,titleAlign:\"left\",titleAngle:0,titleX:-45,titleY:-11},legend:{labelFont:Ot,labelFontSize:11.5,symbolType:\"square\",titleFont:It,titleFontSize:13,titleFontWeight:Nt},range:{category:Lt[\"category-6\"],diverging:Lt[\"fireandice-6\"],heatmap:Lt[\"fire-7\"],ordinal:Lt[\"fire-7\"],ramp:Lt[\"fire-7\"]}},$t=\"#ab5787\",St=\"#979797\",Tt={background:\"#f9f9f9\",arc:{fill:$t},area:{fill:$t},line:{stroke:$t},path:{stroke:$t},rect:{fill:$t},shape:{stroke:$t},symbol:{fill:$t,size:30},axis:{domainColor:St,domainWidth:.5,gridWidth:.2,labelColor:St,tickColor:St,tickWidth:.2,titleColor:St},axisBand:{grid:!1},axisX:{grid:!0,tickSize:10},axisY:{domain:!1,grid:!0,tickSize:0},legend:{labelFontSize:11,padding:1,symbolSize:30,symbolType:\"square\"},range:{category:[\"#ab5787\",\"#51b2e5\",\"#703c5c\",\"#168dd9\",\"#d190b6\",\"#00609f\",\"#d365ba\",\"#154866\",\"#666666\",\"#c4c4c4\"]}},Ct=\"#3e5c69\",Dt={background:\"#fff\",arc:{fill:Ct},area:{fill:Ct},line:{stroke:Ct},path:{stroke:Ct},rect:{fill:Ct},shape:{stroke:Ct},symbol:{fill:Ct},axis:{domainWidth:.5,grid:!0,labelPadding:2,tickSize:5,tickWidth:.5,titleFontWeight:\"normal\"},axisBand:{grid:!1},axisX:{gridWidth:.2},axisY:{gridDash:[3],gridWidth:.4},legend:{labelFontSize:11,padding:1,symbolType:\"square\"},range:{category:[\"#3e5c69\",\"#6793a6\",\"#182429\",\"#0570b0\",\"#3690c0\",\"#74a9cf\",\"#a6bddb\",\"#e2ddf2\"]}},Ft=\"#1696d2\",kt=\"#000000\",_t=\"Lato\",Pt=\"Lato\",Mt={\"main-colors\":[\"#1696d2\",\"#d2d2d2\",\"#000000\",\"#fdbf11\",\"#ec008b\",\"#55b748\",\"#5c5859\",\"#db2b27\"],\"shades-blue\":[\"#CFE8F3\",\"#A2D4EC\",\"#73BFE2\",\"#46ABDB\",\"#1696D2\",\"#12719E\",\"#0A4C6A\",\"#062635\"],\"shades-gray\":[\"#F5F5F5\",\"#ECECEC\",\"#E3E3E3\",\"#DCDBDB\",\"#D2D2D2\",\"#9D9D9D\",\"#696969\",\"#353535\"],\"shades-yellow\":[\"#FFF2CF\",\"#FCE39E\",\"#FDD870\",\"#FCCB41\",\"#FDBF11\",\"#E88E2D\",\"#CA5800\",\"#843215\"],\"shades-magenta\":[\"#F5CBDF\",\"#EB99C2\",\"#E46AA7\",\"#E54096\",\"#EC008B\",\"#AF1F6B\",\"#761548\",\"#351123\"],\"shades-green\":[\"#DCEDD9\",\"#BCDEB4\",\"#98CF90\",\"#78C26D\",\"#55B748\",\"#408941\",\"#2C5C2D\",\"#1A2E19\"],\"shades-black\":[\"#D5D5D4\",\"#ADABAC\",\"#848081\",\"#5C5859\",\"#332D2F\",\"#262223\",\"#1A1717\",\"#0E0C0D\"],\"shades-red\":[\"#F8D5D4\",\"#F1AAA9\",\"#E9807D\",\"#E25552\",\"#DB2B27\",\"#A4201D\",\"#6E1614\",\"#370B0A\"],\"one-group\":[\"#1696d2\",\"#000000\"],\"two-groups-cat-1\":[\"#1696d2\",\"#000000\"],\"two-groups-cat-2\":[\"#1696d2\",\"#fdbf11\"],\"two-groups-cat-3\":[\"#1696d2\",\"#db2b27\"],\"two-groups-seq\":[\"#a2d4ec\",\"#1696d2\"],\"three-groups-cat\":[\"#1696d2\",\"#fdbf11\",\"#000000\"],\"three-groups-seq\":[\"#a2d4ec\",\"#1696d2\",\"#0a4c6a\"],\"four-groups-cat-1\":[\"#000000\",\"#d2d2d2\",\"#fdbf11\",\"#1696d2\"],\"four-groups-cat-2\":[\"#1696d2\",\"#ec0008b\",\"#fdbf11\",\"#5c5859\"],\"four-groups-seq\":[\"#cfe8f3\",\"#73bf42\",\"#1696d2\",\"#0a4c6a\"],\"five-groups-cat-1\":[\"#1696d2\",\"#fdbf11\",\"#d2d2d2\",\"#ec008b\",\"#000000\"],\"five-groups-cat-2\":[\"#1696d2\",\"#0a4c6a\",\"#d2d2d2\",\"#fdbf11\",\"#332d2f\"],\"five-groups-seq\":[\"#cfe8f3\",\"#73bf42\",\"#1696d2\",\"#0a4c6a\",\"#000000\"],\"six-groups-cat-1\":[\"#1696d2\",\"#ec008b\",\"#fdbf11\",\"#000000\",\"#d2d2d2\",\"#55b748\"],\"six-groups-cat-2\":[\"#1696d2\",\"#d2d2d2\",\"#ec008b\",\"#fdbf11\",\"#332d2f\",\"#0a4c6a\"],\"six-groups-seq\":[\"#cfe8f3\",\"#a2d4ec\",\"#73bfe2\",\"#46abdb\",\"#1696d2\",\"#12719e\"],\"diverging-colors\":[\"#ca5800\",\"#fdbf11\",\"#fdd870\",\"#fff2cf\",\"#cfe8f3\",\"#73bfe2\",\"#1696d2\",\"#0a4c6a\"]},jt={background:\"#FFFFFF\",title:{anchor:\"start\",fontSize:18,font:_t},axisX:{domain:!0,domainColor:kt,domainWidth:1,grid:!1,labelFontSize:12,labelFont:Pt,labelAngle:0,tickColor:kt,tickSize:5,titleFontSize:12,titlePadding:10,titleFont:_t},axisY:{domain:!1,domainWidth:1,grid:!0,gridColor:\"#DEDDDD\",gridWidth:1,labelFontSize:12,labelFont:Pt,labelPadding:8,ticks:!1,titleFontSize:12,titlePadding:10,titleFont:_t,titleAngle:0,titleY:-10,titleX:18},legend:{labelFontSize:12,labelFont:Pt,symbolSize:100,titleFontSize:12,titlePadding:10,titleFont:_t,orient:\"right\",offset:10},view:{stroke:\"transparent\"},range:{category:Mt[\"six-groups-cat-1\"],diverging:Mt[\"diverging-colors\"],heatmap:Mt[\"diverging-colors\"],ordinal:Mt[\"six-groups-seq\"],ramp:Mt[\"shades-blue\"]},area:{fill:Ft},rect:{fill:Ft},line:{color:Ft,stroke:Ft,strokeWidth:5},trail:{color:Ft,stroke:Ft,strokeWidth:0,size:1},path:{stroke:Ft,strokeWidth:.5},point:{filled:!0},text:{font:\"Lato\",color:Ft,fontSize:11,align:\"center\",fontWeight:400,size:11},style:{bar:{fill:Ft,stroke:null}},arc:{fill:Ft},shape:{stroke:Ft},symbol:{fill:Ft,size:30}},zt=\"#3366CC\",Ut=\"#ccc\",Bt=\"Arial, sans-serif\",Gt={arc:{fill:zt},area:{fill:zt},path:{stroke:zt},rect:{fill:zt},shape:{stroke:zt},symbol:{stroke:zt},circle:{fill:zt},background:\"#fff\",padding:{top:10,right:10,bottom:10,left:10},style:{\"guide-label\":{font:Bt,fontSize:12},\"guide-title\":{font:Bt,fontSize:12},\"group-title\":{font:Bt,fontSize:12}},title:{font:Bt,fontSize:14,fontWeight:\"bold\",dy:-3,anchor:\"start\"},axis:{gridColor:Ut,tickColor:Ut,domain:!1,grid:!0},range:{category:[\"#4285F4\",\"#DB4437\",\"#F4B400\",\"#0F9D58\",\"#AB47BC\",\"#00ACC1\",\"#FF7043\",\"#9E9D24\",\"#5C6BC0\",\"#F06292\",\"#00796B\",\"#C2185B\"],heatmap:[\"#c6dafc\",\"#5e97f6\",\"#2a56c6\"]}},Wt=e=>e*(1/3+1),Xt=Wt(9),Vt=Wt(10),Ht=Wt(12),qt=\"Segoe UI\",Yt=\"wf_standard-font, helvetica, arial, sans-serif\",Jt=\"#252423\",Qt=\"#605E5C\",Zt=\"transparent\",Kt=\"#118DFF\",en=\"#DEEFFF\",tn=[en,Kt],nn={view:{stroke:Zt},background:Zt,font:qt,header:{titleFont:Yt,titleFontSize:Ht,titleColor:Jt,labelFont:qt,labelFontSize:Vt,labelColor:Qt},axis:{ticks:!1,grid:!1,domain:!1,labelColor:Qt,labelFontSize:Xt,titleFont:Yt,titleColor:Jt,titleFontSize:Ht,titleFontWeight:\"normal\"},axisQuantitative:{tickCount:3,grid:!0,gridColor:\"#C8C6C4\",gridDash:[1,5],labelFlush:!1},axisBand:{tickExtra:!0},axisX:{labelPadding:5},axisY:{labelPadding:10},bar:{fill:Kt},line:{stroke:Kt,strokeWidth:3,strokeCap:\"round\",strokeJoin:\"round\"},text:{font:qt,fontSize:Xt,fill:Qt},arc:{fill:Kt},area:{fill:Kt,line:!0,opacity:.6},path:{stroke:Kt},rect:{fill:Kt},point:{fill:Kt,filled:!0,size:75},shape:{stroke:Kt},symbol:{fill:Kt,strokeWidth:1.5,size:50},legend:{titleFont:qt,titleFontWeight:\"bold\",titleColor:Qt,labelFont:qt,labelFontSize:Vt,labelColor:Qt,symbolType:\"circle\",symbolSize:75},range:{category:[Kt,\"#12239E\",\"#E66C37\",\"#6B007B\",\"#E044A7\",\"#744EC2\",\"#D9B300\",\"#D64550\"],diverging:tn,heatmap:tn,ordinal:[en,\"#c7e4ff\",\"#b0d9ff\",\"#9aceff\",\"#83c3ff\",\"#6cb9ff\",\"#55aeff\",\"#3fa3ff\",\"#2898ff\",Kt]}},rn='IBM Plex Sans,system-ui,-apple-system,BlinkMacSystemFont,\".sfnstext-regular\",sans-serif',on=[\"#8a3ffc\",\"#33b1ff\",\"#007d79\",\"#ff7eb6\",\"#fa4d56\",\"#fff1f1\",\"#6fdc8c\",\"#4589ff\",\"#d12771\",\"#d2a106\",\"#08bdba\",\"#bae6ff\",\"#ba4e00\",\"#d4bbff\"],an=[\"#6929c4\",\"#1192e8\",\"#005d5d\",\"#9f1853\",\"#fa4d56\",\"#570408\",\"#198038\",\"#002d9c\",\"#ee538b\",\"#b28600\",\"#009d9a\",\"#012749\",\"#8a3800\",\"#a56eff\"];function sn({type:e,background:t}){const n=\"dark\"===e?\"#161616\":\"#ffffff\",r=\"dark\"===e?\"#f4f4f4\":\"#161616\",i=\"dark\"===e?\"#d4bbff\":\"#6929c4\";return{background:t,arc:{fill:i},area:{fill:i},path:{stroke:i},rect:{fill:i},shape:{stroke:i},symbol:{stroke:i},circle:{fill:i},view:{fill:n,stroke:n},group:{fill:n},title:{color:r,anchor:\"start\",dy:-15,fontSize:16,font:rn,fontWeight:600},axis:{labelColor:r,labelFontSize:12,grid:!0,gridColor:\"#525252\",titleColor:r,labelAngle:0},style:{\"guide-label\":{font:rn,fill:r,fontWeight:400},\"guide-title\":{font:rn,fill:r,fontWeight:400}},range:{category:\"dark\"===e?on:an,diverging:[\"#750e13\",\"#a2191f\",\"#da1e28\",\"#fa4d56\",\"#ff8389\",\"#ffb3b8\",\"#ffd7d9\",\"#fff1f1\",\"#e5f6ff\",\"#bae6ff\",\"#82cfff\",\"#33b1ff\",\"#1192e8\",\"#0072c3\",\"#00539a\",\"#003a6d\"],heatmap:[\"#f6f2ff\",\"#e8daff\",\"#d4bbff\",\"#be95ff\",\"#a56eff\",\"#8a3ffc\",\"#6929c4\",\"#491d8b\",\"#31135e\",\"#1c0f30\"]}}}const ln=sn({type:\"light\",background:\"#ffffff\"}),cn=sn({type:\"light\",background:\"#f4f4f4\"}),hn=sn({type:\"dark\",background:\"#262626\"}),fn=sn({type:\"dark\",background:\"#161616\"}),pn=ht;var dn=Object.freeze({__proto__:null,carbong10:cn,carbong100:fn,carbong90:hn,carbonwhite:ln,dark:dt,excel:gt,fivethirtyeight:yt,ggplot2:At,googlecharts:Gt,latimes:Rt,powerbi:nn,quartz:Tt,urbaninstitute:jt,version:pn,vox:Dt});function un(e,t,n){return e.fields=t||[],e.fname=n,e}function gn(e){return 1===e.length?mn(e[0]):vn(e)}const mn=e=>function(t){return t[e]},vn=e=>{const t=e.length;return function(n){for(let r=0;rr&&c(),s=r=i+1):\"]\"===o&&(s||En(\"Access path missing open bracket: \"+e),s>0&&c(),s=0,r=i+1):i>r?c():r=i+1}return s&&En(\"Access path missing closing bracket: \"+e),a&&En(\"Access path missing closing quote: \"+e),i>r&&(i++,c()),t}(e);e=1===r.length?r[0]:e,un((n&&n.get||gn)(r),[e],t||e)}(\"id\"),un((e=>e),[],\"identity\"),un((()=>0),[],\"zero\"),un((()=>1),[],\"one\"),un((()=>!0),[],\"true\"),un((()=>!1),[],\"false\");var bn=Array.isArray;function yn(e){return e===Object(e)}function wn(e,t){return JSON.stringify(e,function(e){const t=[];return function(n,r){if(\"object\"!=typeof r||null===r)return r;const i=t.indexOf(this)+1;return t.length=i,t.length>e?\"[Object]\":t.indexOf(r)>=0?\"[Circular]\":(t.push(r),r)}}(t))}var An=\"#vg-tooltip-element {\\n visibility: hidden;\\n padding: 8px;\\n position: fixed;\\n z-index: 1000;\\n font-family: sans-serif;\\n font-size: 11px;\\n border-radius: 3px;\\n box-shadow: 2px 2px 4px rgba(0, 0, 0, 0.1);\\n /* The default theme is the light theme. */\\n background-color: rgba(255, 255, 255, 0.95);\\n border: 1px solid #d9d9d9;\\n color: black;\\n}\\n#vg-tooltip-element.visible {\\n visibility: visible;\\n}\\n#vg-tooltip-element h2 {\\n margin-top: 0;\\n margin-bottom: 10px;\\n font-size: 13px;\\n}\\n#vg-tooltip-element table {\\n border-spacing: 0;\\n}\\n#vg-tooltip-element table tr {\\n border: none;\\n}\\n#vg-tooltip-element table tr td {\\n overflow: hidden;\\n text-overflow: ellipsis;\\n padding-top: 2px;\\n padding-bottom: 2px;\\n}\\n#vg-tooltip-element table tr td.key {\\n color: #808080;\\n max-width: 150px;\\n text-align: right;\\n padding-right: 4px;\\n}\\n#vg-tooltip-element table tr td.value {\\n display: block;\\n max-width: 300px;\\n max-height: 7em;\\n text-align: left;\\n}\\n#vg-tooltip-element.dark-theme {\\n background-color: rgba(32, 32, 32, 0.9);\\n border: 1px solid #f5f5f5;\\n color: white;\\n}\\n#vg-tooltip-element.dark-theme td.key {\\n color: #bfbfbf;\\n}\\n\";const On=\"vg-tooltip-element\",xn={offsetX:10,offsetY:10,id:On,styleId:\"vega-tooltip-style\",theme:\"light\",disableDefaultStyle:!1,sanitize:function(e){return String(e).replace(/&/g,\"&\").replace(/t(\"string\"==typeof e?e:wn(e,n)))).join(\", \")}]`;if(yn(e)){let i=\"\";const{title:o,image:a,...s}=e;o&&(i+=`

      ${t(o)}

      `),a&&(i+=``);const l=Object.keys(s);if(l.length>0){i+=\"\";for(const e of l){let r=s[e];void 0!==r&&(yn(r)&&(r=wn(r,n)),i+=``)}i+=\"
      ${t(e)}${t(r)}
      \"}return i||\"{}\"}return t(e)},baseURL:\"\"};class In{constructor(e){this.options={...xn,...e};const t=this.options.id;if(this.el=null,this.call=this.tooltipHandler.bind(this),!this.options.disableDefaultStyle&&!document.getElementById(this.options.styleId)){const e=document.createElement(\"style\");e.setAttribute(\"id\",this.options.styleId),e.innerHTML=function(e){if(!/^[A-Za-z]+[-:.\\w]*$/.test(e))throw new Error(\"Invalid HTML ID\");return An.toString().replace(On,e)}(t);const n=document.head;n.childNodes.length>0?n.insertBefore(e,n.childNodes[0]):n.appendChild(e)}}tooltipHandler(e,t,n,r){if(this.el=document.getElementById(this.options.id),!this.el){this.el=document.createElement(\"div\"),this.el.setAttribute(\"id\",this.options.id),this.el.classList.add(\"vg-tooltip\");(document.fullscreenElement??document.body).appendChild(this.el)}if(null==r||\"\"===r)return void this.el.classList.remove(\"visible\",`${this.options.theme}-theme`);this.el.innerHTML=this.options.formatTooltip(r,this.options.sanitize,this.options.maxDepth,this.options.baseURL),this.el.classList.add(\"visible\",`${this.options.theme}-theme`);const{x:i,y:o}=function(e,t,n,r){let i=e.clientX+n;i+t.width>window.innerWidth&&(i=+e.clientX-n-t.width);let o=e.clientY+r;return o+t.height>window.innerHeight&&(o=+e.clientY-r-t.height),{x:i,y:o}}(t,this.el.getBoundingClientRect(),this.options.offsetX,this.options.offsetY);this.el.style.top=`${o}px`,this.el.style.left=`${i}px`}}var Nn='.vega-embed {\\n position: relative;\\n display: inline-block;\\n box-sizing: border-box;\\n}\\n.vega-embed.has-actions {\\n padding-right: 38px;\\n}\\n.vega-embed details:not([open]) > :not(summary) {\\n display: none !important;\\n}\\n.vega-embed summary {\\n list-style: none;\\n position: absolute;\\n top: 0;\\n right: 0;\\n padding: 6px;\\n z-index: 1000;\\n background: white;\\n box-shadow: 1px 1px 3px rgba(0, 0, 0, 0.1);\\n color: #1b1e23;\\n border: 1px solid #aaa;\\n border-radius: 999px;\\n opacity: 0.2;\\n transition: opacity 0.4s ease-in;\\n cursor: pointer;\\n line-height: 0px;\\n}\\n.vega-embed summary::-webkit-details-marker {\\n display: none;\\n}\\n.vega-embed summary:active {\\n box-shadow: #aaa 0px 0px 0px 1px inset;\\n}\\n.vega-embed summary svg {\\n width: 14px;\\n height: 14px;\\n}\\n.vega-embed details[open] summary {\\n opacity: 0.7;\\n}\\n.vega-embed:hover summary, .vega-embed:focus-within summary {\\n opacity: 1 !important;\\n transition: opacity 0.2s ease;\\n}\\n.vega-embed .vega-actions {\\n position: absolute;\\n z-index: 1001;\\n top: 35px;\\n right: -9px;\\n display: flex;\\n flex-direction: column;\\n padding-bottom: 8px;\\n padding-top: 8px;\\n border-radius: 4px;\\n box-shadow: 0 2px 8px 0 rgba(0, 0, 0, 0.2);\\n border: 1px solid #d9d9d9;\\n background: white;\\n animation-duration: 0.15s;\\n animation-name: scale-in;\\n animation-timing-function: cubic-bezier(0.2, 0, 0.13, 1.5);\\n text-align: left;\\n}\\n.vega-embed .vega-actions a {\\n padding: 8px 16px;\\n font-family: sans-serif;\\n font-size: 14px;\\n font-weight: 600;\\n white-space: nowrap;\\n color: #434a56;\\n text-decoration: none;\\n}\\n.vega-embed .vega-actions a:hover, .vega-embed .vega-actions a:focus {\\n background-color: #f7f7f9;\\n color: black;\\n}\\n.vega-embed .vega-actions::before, .vega-embed .vega-actions::after {\\n content: \"\";\\n display: inline-block;\\n position: absolute;\\n}\\n.vega-embed .vega-actions::before {\\n left: auto;\\n right: 14px;\\n top: -16px;\\n border: 8px solid rgba(0, 0, 0, 0);\\n border-bottom-color: #d9d9d9;\\n}\\n.vega-embed .vega-actions::after {\\n left: auto;\\n right: 15px;\\n top: -14px;\\n border: 7px solid rgba(0, 0, 0, 0);\\n border-bottom-color: #fff;\\n}\\n.vega-embed .chart-wrapper.fit-x {\\n width: 100%;\\n}\\n.vega-embed .chart-wrapper.fit-y {\\n height: 100%;\\n}\\n\\n.vega-embed-wrapper {\\n max-width: 100%;\\n overflow: auto;\\n padding-right: 14px;\\n}\\n\\n@keyframes scale-in {\\n from {\\n opacity: 0;\\n transform: scale(0.6);\\n }\\n to {\\n opacity: 1;\\n transform: scale(1);\\n }\\n}\\n';function Ln(e,...t){for(const n of t)Rn(e,n);return e}function Rn(t,n){for(const r of Object.keys(n))e.writeConfig(t,r,n[r],!0)}const $n=\"6.25.0\",Sn=i;let Tn=o;const Cn=\"undefined\"!=typeof window?window:void 0;void 0===Tn&&Cn?.vl?.compile&&(Tn=Cn.vl);const Dn={export:{svg:!0,png:!0},source:!0,compiled:!0,editor:!0},Fn={CLICK_TO_VIEW_ACTIONS:\"Click to view actions\",COMPILED_ACTION:\"View Compiled Vega\",EDITOR_ACTION:\"Open in Vega Editor\",PNG_ACTION:\"Save as PNG\",SOURCE_ACTION:\"View Source\",SVG_ACTION:\"Save as SVG\"},kn={vega:\"Vega\",\"vega-lite\":\"Vega-Lite\"},_n={vega:Sn.version,\"vega-lite\":Tn?Tn.version:\"not available\"},Pn={vega:e=>e,\"vega-lite\":(e,t)=>Tn.compile(e,{config:t}).spec},Mn='\\n\\n \\n \\n \\n',jn=\"chart-wrapper\";function zn(e,t,n,r){const i=`${t}
      `,o=`
      ${n}`,a=window.open(\"\");a.document.write(i+e+o),a.document.title=`${kn[r]} JSON Source`}function Un(e){return!(!e||!(\"load\"in e))}function Bn(e){return Un(e)?e:Sn.loader(e)}async function Gn(t,n,r={}){let i,o;e.isString(n)?(o=Bn(r.loader),i=JSON.parse(await o.load(n))):i=n;const a=function(t){const n=t.usermeta?.embedOptions??{};return e.isString(n.defaultStyle)&&(n.defaultStyle=!1),n}(i),s=a.loader;o&&!s||(o=Bn(r.loader??s));const l=await Wn(a,o),c=await Wn(r,o),h={...Ln(c,l),config:e.mergeConfig(c.config??{},l.config??{})};return await async function(t,n,r={},i){const o=r.theme?e.mergeConfig(dn[r.theme],r.config??{}):r.config,a=e.isBoolean(r.actions)?r.actions:Ln({},Dn,r.actions??{}),s={...Fn,...r.i18n},l=r.renderer??\"canvas\",c=r.logLevel??Sn.Warn,h=r.downloadFileName??\"visualization\",f=\"string\"==typeof t?document.querySelector(t):t;if(!f)throw new Error(`${t} does not exist`);if(!1!==r.defaultStyle){const e=\"vega-embed-style\",{root:t,rootContainer:n}=function(e){const t=e.getRootNode?e.getRootNode():document;return t instanceof ShadowRoot?{root:t,rootContainer:t}:{root:document,rootContainer:document.head??document.body}}(f);if(!t.getElementById(e)){const t=document.createElement(\"style\");t.id=e,t.innerHTML=void 0===r.defaultStyle||!0===r.defaultStyle?Nn.toString():r.defaultStyle,n.appendChild(t)}}const p=function(e,t){if(e.$schema){const n=ct(e.$schema);t&&t!==n.library&&console.warn(`The given visualization spec is written in ${kn[n.library]}, but mode argument sets ${kn[t]??t}.`);const r=n.library;return Qe(_n[r],`^${n.version.slice(1)}`)||console.warn(`The input spec uses ${kn[r]} ${n.version}, but the current version of ${kn[r]} is v${_n[r]}.`),r}return\"mark\"in e||\"encoding\"in e||\"layer\"in e||\"hconcat\"in e||\"vconcat\"in e||\"facet\"in e||\"repeat\"in e?\"vega-lite\":\"marks\"in e||\"signals\"in e||\"scales\"in e||\"axes\"in e?\"vega\":t??\"vega\"}(n,r.mode);let d=Pn[p](n,o);if(\"vega-lite\"===p&&d.$schema){const e=ct(d.$schema);Qe(_n.vega,`^${e.version.slice(1)}`)||console.warn(`The compiled spec uses Vega ${e.version}, but current version is v${_n.vega}.`)}f.classList.add(\"vega-embed\"),a&&f.classList.add(\"has-actions\");f.innerHTML=\"\";let u=f;if(a){const e=document.createElement(\"div\");e.classList.add(jn),f.appendChild(e),u=e}const g=r.patch;g&&(d=g instanceof Function?g(d):O(d,g,!0,!1).newDocument);r.formatLocale&&Sn.formatLocale(r.formatLocale);r.timeFormatLocale&&Sn.timeFormatLocale(r.timeFormatLocale);if(r.expressionFunctions)for(const e in r.expressionFunctions){const t=r.expressionFunctions[e];\"fn\"in t?Sn.expressionFunction(e,t.fn,t.visitor):t instanceof Function&&Sn.expressionFunction(e,t)}const{ast:m}=r,v=Sn.parse(d,\"vega-lite\"===p?{}:o,{ast:m}),E=new(r.viewClass||Sn.View)(v,{loader:i,logLevel:c,renderer:l,...m?{expr:Sn.expressionInterpreter??r.expr??lt}:{}});if(E.addSignalListener(\"autosize\",((e,t)=>{const{type:n}=t;\"fit-x\"==n?(u.classList.add(\"fit-x\"),u.classList.remove(\"fit-y\")):\"fit-y\"==n?(u.classList.remove(\"fit-x\"),u.classList.add(\"fit-y\")):\"fit\"==n?u.classList.add(\"fit-x\",\"fit-y\"):u.classList.remove(\"fit-x\",\"fit-y\")})),!1!==r.tooltip){const{loader:e,tooltip:t}=r,n=e&&!Un(e)?e?.baseURL:void 0,i=\"function\"==typeof t?t:new In({baseURL:n,...!0===t?{}:t}).call;E.tooltip(i)}let b,{hover:y}=r;void 0===y&&(y=\"vega\"===p);if(y){const{hoverSet:e,updateSet:t}=\"boolean\"==typeof y?{}:y;E.hover(e,t)}r&&(null!=r.width&&E.width(r.width),null!=r.height&&E.height(r.height),null!=r.padding&&E.padding(r.padding));if(await E.initialize(u,r.bind).runAsync(),!1!==a){let t=f;if(!1!==r.defaultStyle||r.forceActionsMenu){const e=document.createElement(\"details\");e.title=s.CLICK_TO_VIEW_ACTIONS,f.append(e),t=e;const n=document.createElement(\"summary\");n.innerHTML=Mn,e.append(n),b=t=>{e.contains(t.target)||e.removeAttribute(\"open\")},document.addEventListener(\"click\",b)}const i=document.createElement(\"div\");if(t.append(i),i.classList.add(\"vega-actions\"),!0===a||!1!==a.export)for(const t of[\"svg\",\"png\"])if(!0===a||!0===a.export||a.export[t]){const n=s[`${t.toUpperCase()}_ACTION`],o=document.createElement(\"a\"),a=e.isObject(r.scaleFactor)?r.scaleFactor[t]:r.scaleFactor;o.text=n,o.href=\"#\",o.target=\"_blank\",o.download=`${h}.${t}`,o.addEventListener(\"mousedown\",(async function(e){e.preventDefault();const n=await E.toImageURL(t,a);this.href=n})),i.append(o)}if(!0===a||!1!==a.source){const e=document.createElement(\"a\");e.text=s.SOURCE_ACTION,e.href=\"#\",e.addEventListener(\"click\",(function(e){zn(j(n),r.sourceHeader??\"\",r.sourceFooter??\"\",p),e.preventDefault()})),i.append(e)}if(\"vega-lite\"===p&&(!0===a||!1!==a.compiled)){const e=document.createElement(\"a\");e.text=s.COMPILED_ACTION,e.href=\"#\",e.addEventListener(\"click\",(function(e){zn(j(d),r.sourceHeader??\"\",r.sourceFooter??\"\",\"vega\"),e.preventDefault()})),i.append(e)}if(!0===a||!1!==a.editor){const e=r.editorUrl??\"https://vega.github.io/editor/\",t=document.createElement(\"a\");t.text=s.EDITOR_ACTION,t.href=\"#\",t.addEventListener(\"click\",(function(t){!function(e,t,n){const r=e.open(t),{origin:i}=new URL(t);let o=40;e.addEventListener(\"message\",(function t(n){n.source===r&&(o=0,e.removeEventListener(\"message\",t,!1))}),!1),setTimeout((function e(){o<=0||(r.postMessage(n,i),setTimeout(e,250),o-=1)}),250)}(window,e,{config:o,mode:p,renderer:l,spec:j(n)}),t.preventDefault()})),i.append(t)}}function w(){b&&document.removeEventListener(\"click\",b),E.finalize()}return{view:E,spec:n,vgSpec:d,finalize:w,embedOptions:r}}(t,i,h,o)}async function Wn(t,n){const r=e.isString(t.config)?JSON.parse(await n.load(t.config)):t.config??{},i=e.isString(t.patch)?JSON.parse(await n.load(t.patch)):t.patch;return{...t,...i?{patch:i}:{},...r?{config:r}:{}}}async function Xn(e,t={}){const n=document.createElement(\"div\");n.classList.add(\"vega-embed-wrapper\");const r=document.createElement(\"div\");n.appendChild(r);const i=!0===t.actions||!1===t.actions?t.actions:{export:!0,source:!1,compiled:!0,editor:!0,...t.actions},o=await Gn(r,e,{actions:i,...t});return n.value=o.view,n}const Vn=(...t)=>{return t.length>1&&(e.isString(t[0])&&!((n=t[0]).startsWith(\"http://\")||n.startsWith(\"https://\")||n.startsWith(\"//\"))||t[0]instanceof HTMLElement||3===t.length)?Gn(t[0],t[1],t[2]):Xn(t[0],t[1]);var n};return Vn.vegaLite=Tn,Vn.vl=Tn,Vn.container=Xn,Vn.embed=Gn,Vn.vega=Sn,Vn.default=Gn,Vn.version=$n,Vn}));\n//# sourceMappingURL=vega-embed.min.js.map\n" }, { "path": "docs/javascripts/vega-lite@5.js", "content": "!function(e,t){\"object\"==typeof exports&&\"undefined\"!=typeof module?t(exports,require(\"vega\")):\"function\"==typeof define&&define.amd?define([\"exports\",\"vega\"],t):t((e=\"undefined\"!=typeof globalThis?globalThis:e||self).vegaLite={},e.vega)}(this,(function(e,t){\"use strict\";var n=\"5.18.1\";function i(e){return!!e.or}function r(e){return!!e.and}function o(e){return!!e.not}function a(e,t){if(o(e))a(e.not,t);else if(r(e))for(const n of e.and)a(n,t);else if(i(e))for(const n of e.or)a(n,t);else t(e)}function s(e,t){return o(e)?{not:s(e.not,t)}:r(e)?{and:e.and.map((e=>s(e,t)))}:i(e)?{or:e.or.map((e=>s(e,t)))}:t(e)}const l=structuredClone;function c(e){throw new Error(e)}function u(e,n){const i={};for(const r of n)t.hasOwnProperty(e,r)&&(i[r]=e[r]);return i}function f(e,t){const n={...e};for(const e of t)delete n[e];return n}function d(e){if(t.isNumber(e))return e;const n=t.isString(e)?e:X(e);if(n.length<250)return n;let i=0;for(let e=0;e1?t-1:0),i=1;i0===t?e:`[${e}]`)),r=e.map(((t,n)=>e.slice(0,n+1).join(\"\")));for(const e of r)n.add(e)}return n}function k(e,t){return void 0===e||void 0===t||$(w(e),w(t))}function S(e){return 0===D(e).length}Set.prototype.toJSON=function(){return`Set(${[...this].map((e=>X(e))).join(\",\")})`};const D=Object.keys,F=Object.values,z=Object.entries;function O(e){return!0===e||!1===e}function _(e){const t=e.replace(/\\W/g,\"_\");return(e.match(/^\\d+/)?\"_\":\"\")+t}function C(e,t){return o(e)?`!(${C(e.not,t)})`:r(e)?`(${e.and.map((e=>C(e,t))).join(\") && (\")})`:i(e)?`(${e.or.map((e=>C(e,t))).join(\") || (\")})`:t(e)}function N(e,t){if(0===t.length)return!0;const n=t.shift();return n in e&&N(e[n],t)&&delete e[n],S(e)}function P(e){return e.charAt(0).toUpperCase()+e.substr(1)}function A(e){let n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:\"datum\";const i=t.splitAccessPath(e),r=[];for(let e=1;e<=i.length;e++){const o=`[${i.slice(0,e).map(t.stringValue).join(\"][\")}]`;r.push(`${n}${o}`)}return r.join(\" && \")}function j(e){return`${arguments.length>1&&void 0!==arguments[1]?arguments[1]:\"datum\"}[${t.stringValue(t.splitAccessPath(e).join(\".\"))}]`}function T(e){return e.replace(/(\\[|\\]|\\.|'|\")/g,\"\\\\$1\")}function E(e){return`${t.splitAccessPath(e).map(T).join(\"\\\\.\")}`}function M(e,t,n){return e.replace(new RegExp(t.replace(/[-/\\\\^$*+?.()|[\\]{}]/g,\"\\\\$&\"),\"g\"),n)}function L(e){return`${t.splitAccessPath(e).join(\".\")}`}function q(e){return e?t.splitAccessPath(e).length:0}function U(){for(var e=arguments.length,t=new Array(e),n=0;nfn(e[t])?_(`_${t}_${z(e[t])}`):_(`_${t}_${e[t]}`))).join(\"\")}function ln(e){return!0===e||un(e)&&!e.binned}function cn(e){return\"binned\"===e||un(e)&&!0===e.binned}function un(e){return t.isObject(e)}function fn(e){return e?.param}function dn(e){switch(e){case Q:case J:case ye:case me:case pe:case ge:case we:case be:case xe:case $e:case he:return 6;case ke:return 4;default:return 10}}function mn(e){return!!e?.expr}function pn(e){const t=D(e||{}),n={};for(const i of t)n[i]=Sn(e[i]);return n}function gn(e){const{anchor:t,frame:n,offset:i,orient:r,angle:o,limit:a,color:s,subtitleColor:l,subtitleFont:c,subtitleFontSize:f,subtitleFontStyle:d,subtitleFontWeight:m,subtitleLineHeight:p,subtitlePadding:g,...h}=e,y={...t?{anchor:t}:{},...n?{frame:n}:{},...i?{offset:i}:{},...r?{orient:r}:{},...void 0!==o?{angle:o}:{},...void 0!==a?{limit:a}:{}},v={...l?{subtitleColor:l}:{},...c?{subtitleFont:c}:{},...f?{subtitleFontSize:f}:{},...d?{subtitleFontStyle:d}:{},...m?{subtitleFontWeight:m}:{},...p?{subtitleLineHeight:p}:{},...g?{subtitlePadding:g}:{}};return{titleMarkConfig:{...h,...s?{fill:s}:{}},subtitleMarkConfig:u(e,[\"align\",\"baseline\",\"dx\",\"dy\",\"limit\"]),nonMarkTitleProperties:y,subtitle:v}}function hn(e){return t.isString(e)||t.isArray(e)&&t.isString(e[0])}function yn(e){return!!e?.signal}function vn(e){return!!e.step}function bn(e){return!t.isArray(e)&&(\"field\"in e&&\"data\"in e)}const xn=D({aria:1,description:1,ariaRole:1,ariaRoleDescription:1,blend:1,opacity:1,fill:1,fillOpacity:1,stroke:1,strokeCap:1,strokeWidth:1,strokeOpacity:1,strokeDash:1,strokeDashOffset:1,strokeJoin:1,strokeOffset:1,strokeMiterLimit:1,startAngle:1,endAngle:1,padAngle:1,innerRadius:1,outerRadius:1,size:1,shape:1,interpolate:1,tension:1,orient:1,align:1,baseline:1,text:1,dir:1,dx:1,dy:1,ellipsis:1,limit:1,radius:1,theta:1,angle:1,font:1,fontSize:1,fontWeight:1,fontStyle:1,lineBreak:1,lineHeight:1,cursor:1,href:1,tooltip:1,cornerRadius:1,cornerRadiusTopLeft:1,cornerRadiusTopRight:1,cornerRadiusBottomLeft:1,cornerRadiusBottomRight:1,aspect:1,width:1,height:1,url:1,smooth:1}),$n={arc:1,area:1,group:1,image:1,line:1,path:1,rect:1,rule:1,shape:1,symbol:1,text:1,trail:1},wn=[\"cornerRadius\",\"cornerRadiusTopLeft\",\"cornerRadiusTopRight\",\"cornerRadiusBottomLeft\",\"cornerRadiusBottomRight\"];function kn(e){const n=t.isArray(e.condition)?e.condition.map(Dn):Dn(e.condition);return{...Sn(e),condition:n}}function Sn(e){if(mn(e)){const{expr:t,...n}=e;return{signal:t,...n}}return e}function Dn(e){if(mn(e)){const{expr:t,...n}=e;return{signal:t,...n}}return e}function Fn(e){if(mn(e)){const{expr:t,...n}=e;return{signal:t,...n}}return yn(e)?e:void 0!==e?{value:e}:void 0}function zn(e){return yn(e)?e.signal:t.stringValue(e.value)}function On(e){return yn(e)?e.signal:null==e?null:t.stringValue(e)}function _n(e,t,n){for(const i of n){const n=Pn(i,t.markDef,t.config);void 0!==n&&(e[i]=Fn(n))}return e}function Cn(e){return[].concat(e.type,e.style??[])}function Nn(e,t,n){let i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{};const{vgChannel:r,ignoreVgConfig:o}=i;return r&&void 0!==t[r]?t[r]:void 0!==t[e]?t[e]:!o||r&&r!==e?Pn(e,t,n,i):void 0}function Pn(e,t,n){let{vgChannel:i}=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{};return U(i?An(e,t,n.style):void 0,An(e,t,n.style),i?n[t.type][i]:void 0,n[t.type][e],i?n.mark[i]:n.mark[e])}function An(e,t,n){return jn(e,Cn(t),n)}function jn(e,n,i){let r;n=t.array(n);for(const t of n){const n=i[t];n&&void 0!==n[e]&&(r=n[e])}return r}function Tn(e,n){return t.array(e).reduce(((e,t)=>(e.field.push(ta(t,n)),e.order.push(t.sort??\"ascending\"),e)),{field:[],order:[]})}function En(e,t){const n=[...e];return t.forEach((e=>{for(const t of n)if(Y(t,e))return;n.push(e)})),n}function Mn(e,n){return Y(e,n)||!n?e:e?[...t.array(e),...t.array(n)].join(\", \"):n}function Ln(e,t){const n=e.value,i=t.value;if(null==n||null===i)return{explicit:e.explicit,value:null};if((hn(n)||yn(n))&&(hn(i)||yn(i)))return{explicit:e.explicit,value:Mn(n,i)};if(hn(n)||yn(n))return{explicit:e.explicit,value:n};if(hn(i)||yn(i))return{explicit:e.explicit,value:i};if(!(hn(n)||yn(n)||hn(i)||yn(i)))return{explicit:e.explicit,value:En(n,i)};throw new Error(\"It should never reach here\")}function qn(e){return`Invalid specification ${X(e)}. Make sure the specification includes at least one of the following properties: \"mark\", \"layer\", \"facet\", \"hconcat\", \"vconcat\", \"concat\", or \"repeat\".`}const Un='Autosize \"fit\" only works for single views and layered views.';function Rn(e){return`${\"width\"==e?\"Width\":\"Height\"} \"container\" only works for single views and layered views.`}function Wn(e){return`${\"width\"==e?\"Width\":\"Height\"} \"container\" only works well with autosize \"fit\" or \"fit-${\"width\"==e?\"x\":\"y\"}\".`}function Bn(e){return e?`Dropping \"fit-${e}\" because spec has discrete ${rt(e)}.`:'Dropping \"fit\" because spec has discrete size.'}function In(e){return`Unknown field for ${e}. Cannot calculate view size.`}function Hn(e){return`Cannot project a selection on encoding channel \"${e}\", which has no field.`}function Vn(e,t){return`Cannot project a selection on encoding channel \"${e}\" as it uses an aggregate function (\"${t}\").`}function Gn(e){return`Selection not supported for ${e} yet.`}const Yn=\"The same selection must be used to override scale domains in a layered view.\";function Xn(e){return`The \"columns\" property cannot be used when \"${e}\" has nested row/column.`}function Qn(e,t,n){return`An ancestor parsed field \"${e}\" as ${n} but a child wants to parse the field as ${t}.`}function Jn(e){return`Config.customFormatTypes is not true, thus custom format type and format for channel ${e} are dropped.`}function Kn(e){return`${e}Offset dropped because ${e} is continuous`}function Zn(e){return`Invalid field type \"${e}\".`}function ei(e,t){const{fill:n,stroke:i}=t;return`Dropping color ${e} as the plot also has ${n&&i?\"fill and stroke\":n?\"fill\":\"stroke\"}.`}function ti(e,t){return`Dropping ${X(e)} from channel \"${t}\" since it does not contain any data field, datum, value, or signal.`}function ni(e,t,n){return`${e} dropped as it is incompatible with \"${t}\".`}function ii(e){return`${e} encoding should be discrete (ordinal / nominal / binned).`}function ri(e){return`${e} encoding should be discrete (ordinal / nominal / binned) or use a discretizing scale (e.g. threshold).`}function oi(e,t){return`Using discrete channel \"${e}\" to encode \"${t}\" field can be misleading as it does not encode ${\"ordinal\"===t?\"order\":\"magnitude\"}.`}function ai(e){return`Using unaggregated domain with raw field has no effect (${X(e)}).`}function si(e){return`Unaggregated domain not applicable for \"${e}\" since it produces values outside the origin domain of the source data.`}function li(e){return`Unaggregated domain is currently unsupported for log scale (${X(e)}).`}function ci(e,t,n){return`${n}-scale's \"${t}\" is dropped as it does not work with ${e} scale.`}function ui(e){return`The step for \"${e}\" is dropped because the ${\"width\"===e?\"x\":\"y\"} is continuous.`}const fi=\"Domains that should be unioned has conflicting sort properties. Sort will be set to true.\";function di(e,t){return`Invalid ${e}: ${X(t)}.`}function mi(e){return`1D error band does not support ${e}.`}function pi(e){return`Channel ${e} is required for \"binned\" bin.`}const gi=t.logger(t.Warn);let hi=gi;function yi(){hi.warn(...arguments)}function vi(e){if(e&&t.isObject(e))for(const t of Fi)if(t in e)return!0;return!1}const bi=[\"january\",\"february\",\"march\",\"april\",\"may\",\"june\",\"july\",\"august\",\"september\",\"october\",\"november\",\"december\"],xi=bi.map((e=>e.substr(0,3))),$i=[\"sunday\",\"monday\",\"tuesday\",\"wednesday\",\"thursday\",\"friday\",\"saturday\"],wi=$i.map((e=>e.substr(0,3)));function ki(e,n){const i=[];if(n&&void 0!==e.day&&D(e).length>1&&(yi(function(e){return`Dropping day from datetime ${X(e)} as day cannot be combined with other units.`}(e)),delete(e=l(e)).day),void 0!==e.year?i.push(e.year):i.push(2012),void 0!==e.month){const r=n?function(e){if(V(e)&&(e=+e),t.isNumber(e))return e-1;{const t=e.toLowerCase(),n=bi.indexOf(t);if(-1!==n)return n;const i=t.substr(0,3),r=xi.indexOf(i);if(-1!==r)return r;throw new Error(di(\"month\",e))}}(e.month):e.month;i.push(r)}else if(void 0!==e.quarter){const r=n?function(e){if(V(e)&&(e=+e),t.isNumber(e))return e>4&&yi(di(\"quarter\",e)),e-1;throw new Error(di(\"quarter\",e))}(e.quarter):e.quarter;i.push(t.isNumber(r)?3*r:`${r}*3`)}else i.push(0);if(void 0!==e.date)i.push(e.date);else if(void 0!==e.day){const r=n?function(e){if(V(e)&&(e=+e),t.isNumber(e))return e%7;{const t=e.toLowerCase(),n=$i.indexOf(t);if(-1!==n)return n;const i=t.substr(0,3),r=wi.indexOf(i);if(-1!==r)return r;throw new Error(di(\"day\",e))}}(e.day):e.day;i.push(t.isNumber(r)?r+1:`${r}+1`)}else i.push(1);for(const t of[\"hours\",\"minutes\",\"seconds\",\"milliseconds\"]){const n=e[t];i.push(void 0===n?0:n)}return i}function Si(e){const t=ki(e,!0).join(\", \");return e.utc?`utc(${t})`:`datetime(${t})`}const Di={year:1,quarter:1,month:1,week:1,day:1,dayofyear:1,date:1,hours:1,minutes:1,seconds:1,milliseconds:1},Fi=D(Di);function zi(e){return t.isObject(e)?e.binned:Oi(e)}function Oi(e){return e&&e.startsWith(\"binned\")}function _i(e){return e.startsWith(\"utc\")}const Ci={\"year-month\":\"%b %Y \",\"year-month-date\":\"%b %d, %Y \"};function Ni(e){return Fi.filter((t=>Ai(e,t)))}function Pi(e){const t=Ni(e);return t[t.length-1]}function Ai(e,t){const n=e.indexOf(t);return!(n<0)&&(!(n>0&&\"seconds\"===t&&\"i\"===e.charAt(n-1))&&(!(e.length>n+3&&\"day\"===t&&\"o\"===e.charAt(n+3))&&!(n>0&&\"year\"===t&&\"f\"===e.charAt(n-1))))}function ji(e,t){let{end:n}=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{end:!1};const i=A(t),r=_i(e)?\"utc\":\"\";let o;const a={};for(const t of Fi)Ai(e,t)&&(a[t]=\"quarter\"===(s=t)?`(${r}quarter(${i})-1)`:`${r}${s}(${i})`,o=t);var s;return n&&(a[o]+=\"+1\"),function(e){const t=ki(e,!1).join(\", \");return e.utc?`utc(${t})`:`datetime(${t})`}(a)}function Ti(e){if(!e)return;return`timeUnitSpecifier(${X(Ni(e))}, ${X(Ci)})`}function Ei(e){if(!e)return;let n;return t.isString(e)?n=Oi(e)?{unit:e.substring(6),binned:!0}:{unit:e}:t.isObject(e)&&(n={...e,...e.unit?{unit:e.unit}:{}}),_i(n.unit)&&(n.utc=!0,n.unit=n.unit.substring(3)),n}function Mi(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:e=>e;const n=Ei(e),i=Pi(n.unit);if(i&&\"day\"!==i){const e={year:2001,month:1,date:1,hours:0,minutes:0,seconds:0,milliseconds:0},{step:r,part:o}=qi(i,n.step);return`${t(Si({...e,[o]:+e[o]+r}))} - ${t(Si(e))}`}}const Li={year:1,month:1,date:1,hours:1,minutes:1,seconds:1,milliseconds:1};function qi(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:1;if(function(e){return!!Li[e]}(e))return{part:e,step:t};switch(e){case\"day\":case\"dayofyear\":return{part:\"date\",step:t};case\"quarter\":return{part:\"month\",step:3*t};case\"week\":return{part:\"date\",step:7*t}}}function Ui(e){return!!e?.field&&void 0!==e.equal}function Ri(e){return!!e?.field&&void 0!==e.lt}function Wi(e){return!!e?.field&&void 0!==e.lte}function Bi(e){return!!e?.field&&void 0!==e.gt}function Ii(e){return!!e?.field&&void 0!==e.gte}function Hi(e){if(e?.field){if(t.isArray(e.range)&&2===e.range.length)return!0;if(yn(e.range))return!0}return!1}function Vi(e){return!!e?.field&&(t.isArray(e.oneOf)||t.isArray(e.in))}function Gi(e){return Vi(e)||Ui(e)||Hi(e)||Ri(e)||Bi(e)||Wi(e)||Ii(e)}function Yi(e,t){return va(e,{timeUnit:t,wrapTime:!0})}function Xi(e){let t=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];const{field:n}=e,i=Ei(e.timeUnit),{unit:r,binned:o}=i||{},a=ta(e,{expr:\"datum\"}),s=r?`time(${o?a:ji(r,n)})`:a;if(Ui(e))return`${s}===${Yi(e.equal,r)}`;if(Ri(e)){return`${s}<${Yi(e.lt,r)}`}if(Bi(e)){return`${s}>${Yi(e.gt,r)}`}if(Wi(e)){return`${s}<=${Yi(e.lte,r)}`}if(Ii(e)){return`${s}>=${Yi(e.gte,r)}`}if(Vi(e))return`indexof([${function(e,t){return e.map((e=>Yi(e,t)))}(e.oneOf,r).join(\",\")}], ${s}) !== -1`;if(function(e){return!!e?.field&&void 0!==e.valid}(e))return Qi(s,e.valid);if(Hi(e)){const{range:n}=e,i=yn(n)?{signal:`${n.signal}[0]`}:n[0],o=yn(n)?{signal:`${n.signal}[1]`}:n[1];if(null!==i&&null!==o&&t)return\"inrange(\"+s+\", [\"+Yi(i,r)+\", \"+Yi(o,r)+\"])\";const a=[];return null!==i&&a.push(`${s} >= ${Yi(i,r)}`),null!==o&&a.push(`${s} <= ${Yi(o,r)}`),a.length>0?a.join(\" && \"):\"true\"}throw new Error(`Invalid field predicate: ${X(e)}`)}function Qi(e){return!(arguments.length>1&&void 0!==arguments[1])||arguments[1]?`isValid(${e}) && isFinite(+${e})`:`!isValid(${e}) || !isFinite(+${e})`}function Ji(e){return Gi(e)&&e.timeUnit?{...e,timeUnit:Ei(e.timeUnit)}:e}function Ki(e){return\"quantitative\"===e||\"temporal\"===e}function Zi(e){return\"ordinal\"===e||\"nominal\"===e}const er=\"quantitative\",tr=\"ordinal\",nr=\"temporal\",ir=\"nominal\",rr=\"geojson\";const or={LINEAR:\"linear\",LOG:\"log\",POW:\"pow\",SQRT:\"sqrt\",SYMLOG:\"symlog\",IDENTITY:\"identity\",SEQUENTIAL:\"sequential\",TIME:\"time\",UTC:\"utc\",QUANTILE:\"quantile\",QUANTIZE:\"quantize\",THRESHOLD:\"threshold\",BIN_ORDINAL:\"bin-ordinal\",ORDINAL:\"ordinal\",POINT:\"point\",BAND:\"band\"},ar={linear:\"numeric\",log:\"numeric\",pow:\"numeric\",sqrt:\"numeric\",symlog:\"numeric\",identity:\"numeric\",sequential:\"numeric\",time:\"time\",utc:\"time\",ordinal:\"ordinal\",\"bin-ordinal\":\"bin-ordinal\",point:\"ordinal-position\",band:\"ordinal-position\",quantile:\"discretizing\",quantize:\"discretizing\",threshold:\"discretizing\"};function sr(e,t){const n=ar[e],i=ar[t];return n===i||\"ordinal-position\"===n&&\"time\"===i||\"ordinal-position\"===i&&\"time\"===n}const lr={linear:0,log:1,pow:1,sqrt:1,symlog:1,identity:1,sequential:1,time:0,utc:0,point:10,band:11,ordinal:0,\"bin-ordinal\":0,quantile:0,quantize:0,threshold:0};function cr(e){return lr[e]}const ur=new Set([\"linear\",\"log\",\"pow\",\"sqrt\",\"symlog\"]),fr=new Set([...ur,\"time\",\"utc\"]);function dr(e){return ur.has(e)}const mr=new Set([\"quantile\",\"quantize\",\"threshold\"]),pr=new Set([...fr,...mr,\"sequential\",\"identity\"]),gr=new Set([\"ordinal\",\"bin-ordinal\",\"point\",\"band\"]);function hr(e){return gr.has(e)}function yr(e){return pr.has(e)}function vr(e){return fr.has(e)}function br(e){return mr.has(e)}function xr(e){return e?.param}const{type:$r,domain:wr,range:kr,rangeMax:Sr,rangeMin:Dr,scheme:Fr,...zr}={type:1,domain:1,domainMax:1,domainMin:1,domainMid:1,domainRaw:1,align:1,range:1,rangeMax:1,rangeMin:1,scheme:1,bins:1,reverse:1,round:1,clamp:1,nice:1,base:1,exponent:1,constant:1,interpolate:1,zero:1,padding:1,paddingInner:1,paddingOuter:1},Or=D(zr);function _r(e,t){switch(t){case\"type\":case\"domain\":case\"reverse\":case\"range\":return!0;case\"scheme\":case\"interpolate\":return![\"point\",\"band\",\"identity\"].includes(e);case\"bins\":return![\"point\",\"band\",\"identity\",\"ordinal\"].includes(e);case\"round\":return vr(e)||\"band\"===e||\"point\"===e;case\"padding\":case\"rangeMin\":case\"rangeMax\":return vr(e)||[\"point\",\"band\"].includes(e);case\"paddingOuter\":case\"align\":return[\"point\",\"band\"].includes(e);case\"paddingInner\":return\"band\"===e;case\"domainMax\":case\"domainMid\":case\"domainMin\":case\"domainRaw\":case\"clamp\":return vr(e);case\"nice\":return vr(e)||\"quantize\"===e||\"threshold\"===e;case\"exponent\":return\"pow\"===e;case\"base\":return\"log\"===e;case\"constant\":return\"symlog\"===e;case\"zero\":return yr(e)&&!p([\"log\",\"time\",\"utc\",\"threshold\",\"quantile\"],e)}}function Cr(e,t){switch(t){case\"interpolate\":case\"scheme\":case\"domainMid\":return qe(e)?void 0:`Cannot use the scale property \"${t}\" with non-color channel.`;case\"align\":case\"type\":case\"bins\":case\"domain\":case\"domainMax\":case\"domainMin\":case\"domainRaw\":case\"range\":case\"base\":case\"exponent\":case\"constant\":case\"nice\":case\"padding\":case\"paddingInner\":case\"paddingOuter\":case\"rangeMax\":case\"rangeMin\":case\"reverse\":case\"round\":case\"clamp\":case\"zero\":return}}function Nr(e){const{channel:t,channelDef:n,markDef:i,scale:r,config:o}=e,a=Er(e);return Ro(n)&&!rn(n.aggregate)&&r&&vr(r.get(\"type\"))?function(e){let{fieldDef:t,channel:n,markDef:i,ref:r,config:o}=e;const a=Nn(\"invalid\",i,o);if(null===a)return[Pr(t,n),r];return r}({fieldDef:n,channel:t,markDef:i,ref:a,config:o}):a}function Pr(e,t){return{test:Ar(e,!0),...\"y\"===tt(t)?{field:{group:\"height\"}}:{value:0}}}function Ar(e){let n=!(arguments.length>1&&void 0!==arguments[1])||arguments[1];return Qi(t.isString(e)?e:ta(e,{expr:\"datum\"}),!n)}function jr(e,t,n,i){const r={};if(t&&(r.scale=t),Bo(e)){const{datum:t}=e;vi(t)?r.signal=Si(t):yn(t)?r.signal=t.signal:mn(t)?r.signal=t.expr:r.value=t}else r.field=ta(e,n);if(i){const{offset:e,band:t}=i;e&&(r.offset=e),t&&(r.band=t)}return r}function Tr(e){let{scaleName:t,fieldOrDatumDef:n,fieldOrDatumDef2:i,offset:r,startSuffix:o,endSuffix:a=\"end\",bandPosition:s=.5}=e;const l=!yn(s)&&01&&void 0!==arguments[1]?arguments[1]:{},n=e.field;const i=t.prefix;let r=t.suffix,o=\"\";if(function(e){return\"count\"===e.aggregate}(e))n=B(\"count\");else{let i;if(!t.nofn)if(function(e){return\"op\"in e}(e))i=e.op;else{const{bin:a,aggregate:s,timeUnit:l}=e;ln(a)?(i=sn(a),r=(t.binSuffix??\"\")+(t.suffix??\"\")):s?en(s)?(o=`[\"${n}\"]`,n=`argmax_${s.argmax}`):Zt(s)?(o=`[\"${n}\"]`,n=`argmin_${s.argmin}`):i=String(s):l&&!zi(l)&&(i=function(e){const{utc:t,...n}=Ei(e);return n.unit?(t?\"utc\":\"\")+D(n).map((e=>_(`${\"unit\"===e?\"\":`_${e}_`}${n[e]}`))).join(\"\"):(t?\"utc\":\"\")+\"timeunit\"+D(n).map((e=>_(`_${e}_${n[e]}`))).join(\"\")}(l),r=(![\"range\",\"mid\"].includes(t.binSuffix)&&t.binSuffix||\"\")+(t.suffix??\"\"))}i&&(n=n?`${i}_${n}`:i)}return r&&(n=`${n}_${r}`),i&&(n=`${i}_${n}`),t.forAs?L(n):t.expr?j(n,t.expr)+o:E(n)+o}function na(e){switch(e.type){case\"nominal\":case\"ordinal\":case\"geojson\":return!0;case\"quantitative\":return Ro(e)&&!!e.bin;case\"temporal\":return!1}throw new Error(Zn(e.type))}const ia=(e,t)=>{switch(t.fieldTitle){case\"plain\":return e.field;case\"functional\":return function(e){const{aggregate:t,bin:n,timeUnit:i,field:r}=e;if(en(t))return`${r} for argmax(${t.argmax})`;if(Zt(t))return`${r} for argmin(${t.argmin})`;const o=i&&!zi(i)?Ei(i):void 0,a=t||o?.unit||o?.maxbins&&\"timeunit\"||ln(n)&&\"bin\";return a?`${a.toUpperCase()}(${r})`:r}(e);default:return function(e,t){const{field:n,bin:i,timeUnit:r,aggregate:o}=e;if(\"count\"===o)return t.countTitle;if(ln(i))return`${n} (binned)`;if(r&&!zi(r)){const e=Ei(r)?.unit;if(e)return`${n} (${Ni(e).join(\"-\")})`}else if(o)return en(o)?`${n} for max ${o.argmax}`:Zt(o)?`${n} for min ${o.argmin}`:`${P(o)} of ${n}`;return n}(e,t)}};let ra=ia;function oa(e){ra=e}function aa(e,t,n){let{allowDisabling:i,includeDefault:r=!0}=n;const o=sa(e)?.title;if(!Ro(e))return o??e.title;const a=e,s=r?la(a,t):void 0;return i?U(o,a.title,s):o??a.title??s}function sa(e){return Jo(e)&&e.axis?e.axis:Ko(e)&&e.legend?e.legend:Co(e)&&e.header?e.header:void 0}function la(e,t){return ra(e,t)}function ca(e){if(Zo(e)){const{format:t,formatType:n}=e;return{format:t,formatType:n}}{const t=sa(e)??{},{format:n,formatType:i}=t;return{format:n,formatType:i}}}function ua(e){return Ro(e)?e:qo(e)?e.condition:void 0}function fa(e){return Go(e)?e:Uo(e)?e.condition:void 0}function da(e,n,i){let r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{};if(t.isString(e)||t.isNumber(e)||t.isBoolean(e)){return yi(function(e,t,n){return`Channel ${e} is a ${t}. Converted to {value: ${X(n)}}.`}(n,t.isString(e)?\"string\":t.isNumber(e)?\"number\":\"boolean\",e)),{value:e}}return Go(e)?ma(e,n,i,r):Uo(e)?{...e,condition:ma(e.condition,n,i,r)}:e}function ma(e,n,i,r){if(Zo(e)){const{format:t,formatType:o,...a}=e;if(Lr(o)&&!i.customFormatTypes)return yi(Jn(n)),ma(a,n,i,r)}else{const t=Jo(e)?\"axis\":Ko(e)?\"legend\":Co(e)?\"header\":null;if(t&&e[t]){const{format:o,formatType:a,...s}=e[t];if(Lr(a)&&!i.customFormatTypes)return yi(Jn(n)),ma({...e,[t]:s},n,i,r)}}return Ro(e)?pa(e,n,r):function(e){let n=e.type;if(n)return e;const{datum:i}=e;return n=t.isNumber(i)?\"quantitative\":t.isString(i)?\"nominal\":vi(i)?\"temporal\":void 0,{...e,type:n}}(e)}function pa(e,n){let{compositeMark:i=!1}=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{};const{aggregate:r,timeUnit:o,bin:a,field:s}=e,l={...e};if(i||!r||tn(r)||en(r)||Zt(r)||(yi(function(e){return`Invalid aggregation operator \"${e}\".`}(r)),delete l.aggregate),o&&(l.timeUnit=Ei(o)),s&&(l.field=`${s}`),ln(a)&&(l.bin=ga(a,n)),cn(a)&&!zt(n)&&yi(function(e){return`Channel ${e} should not be used with \"binned\" bin.`}(n)),Yo(l)){const{type:e}=l,t=function(e){if(e)switch(e=e.toLowerCase()){case\"q\":case er:return\"quantitative\";case\"t\":case nr:return\"temporal\";case\"o\":case tr:return\"ordinal\";case\"n\":case ir:return\"nominal\";case rr:return\"geojson\"}}(e);e!==t&&(l.type=t),\"quantitative\"!==e&&rn(r)&&(yi(function(e,t){return`Invalid field type \"${e}\" for aggregate: \"${t}\", using \"quantitative\" instead.`}(e,r)),l.type=\"quantitative\")}else if(!et(n)){const e=function(e,n){switch(n){case\"latitude\":case\"longitude\":return\"quantitative\";case\"row\":case\"column\":case\"facet\":case\"shape\":case\"strokeDash\":return\"nominal\";case\"order\":return\"ordinal\"}if(Ao(e)&&t.isArray(e.sort))return\"ordinal\";const{aggregate:i,bin:r,timeUnit:o}=e;if(o)return\"temporal\";if(r||i&&!en(i)&&!Zt(i))return\"quantitative\";if(Qo(e)&&e.scale?.type)switch(ar[e.scale.type]){case\"numeric\":case\"discretizing\":return\"quantitative\";case\"time\":return\"temporal\"}return\"nominal\"}(l,n);l.type=e}if(Yo(l)){const{compatible:e,warning:t}=function(e,t){const n=e.type;if(\"geojson\"===n&&\"shape\"!==t)return{compatible:!1,warning:`Channel ${t} should not be used with a geojson data.`};switch(t){case Q:case J:case K:return na(e)?ha:{compatible:!1,warning:ii(t)};case Z:case ee:case ie:case re:case me:case pe:case ge:case Se:case Fe:case ze:case Oe:case _e:case Ce:case ve:case se:case oe:case Ne:return ha;case ue:case de:case ce:case fe:return n!==er?{compatible:!1,warning:`Channel ${t} should be used with a quantitative field only, not ${e.type} field.`}:ha;case be:case xe:case $e:case we:case ye:case le:case ae:case te:case ne:return\"nominal\"!==n||e.sort?ha:{compatible:!1,warning:`Channel ${t} should not be used with an unsorted discrete field.`};case he:case ke:return na(e)||Qo(i=e)&&br(i.scale?.type)?ha:{compatible:!1,warning:ri(t)};case De:return\"nominal\"!==e.type||\"sort\"in e?ha:{compatible:!1,warning:\"Channel order is inappropriate for nominal field, which has no inherent order.\"}}var i}(l,n)||{};!1===e&&yi(t)}if(Ao(l)&&t.isString(l.sort)){const{sort:e}=l;if(Do(e))return{...l,sort:{encoding:e}};const t=e.substr(1);if(\"-\"===e.charAt(0)&&Do(t))return{...l,sort:{encoding:t,order:\"descending\"}}}if(Co(l)){const{header:e}=l;if(e){const{orient:t,...n}=e;if(t)return{...l,header:{...n,labelOrient:e.labelOrient||t,titleOrient:e.titleOrient||t}}}}return l}function ga(e,n){return t.isBoolean(e)?{maxbins:dn(n)}:\"binned\"===e?{binned:!0}:e.maxbins||e.step?e:{...e,maxbins:dn(n)}}const ha={compatible:!0};function ya(e){const{formatType:t}=ca(e);return\"time\"===t||!t&&((n=e)&&(\"temporal\"===n.type||Ro(n)&&!!n.timeUnit));var n}function va(e,n){let{timeUnit:i,type:r,wrapTime:o,undefinedIfExprNotRequired:a}=n;const s=i&&Ei(i)?.unit;let l,c=s||\"temporal\"===r;return mn(e)?l=e.expr:yn(e)?l=e.signal:vi(e)?(c=!0,l=Si(e)):(t.isString(e)||t.isNumber(e))&&c&&(l=`datetime(${X(e)})`,function(e){return!!Di[e]}(s)&&(t.isNumber(e)&&e<1e4||t.isString(e)&&isNaN(Date.parse(e)))&&(l=Si({[s]:e}))),l?o&&c?`time(${l})`:l:a?void 0:X(e)}function ba(e,t){const{type:n}=e;return t.map((t=>{const i=va(t,{timeUnit:Ro(e)&&!zi(e.timeUnit)?e.timeUnit:void 0,type:n,undefinedIfExprNotRequired:!0});return void 0!==i?{signal:i}:t}))}function xa(e,t){return ln(e.bin)?Ht(t)&&[\"ordinal\",\"nominal\"].includes(e.type):(console.warn(\"Only call this method for binned field defs.\"),!1)}const $a={labelAlign:{part:\"labels\",vgProp:\"align\"},labelBaseline:{part:\"labels\",vgProp:\"baseline\"},labelColor:{part:\"labels\",vgProp:\"fill\"},labelFont:{part:\"labels\",vgProp:\"font\"},labelFontSize:{part:\"labels\",vgProp:\"fontSize\"},labelFontStyle:{part:\"labels\",vgProp:\"fontStyle\"},labelFontWeight:{part:\"labels\",vgProp:\"fontWeight\"},labelOpacity:{part:\"labels\",vgProp:\"opacity\"},labelOffset:null,labelPadding:null,gridColor:{part:\"grid\",vgProp:\"stroke\"},gridDash:{part:\"grid\",vgProp:\"strokeDash\"},gridDashOffset:{part:\"grid\",vgProp:\"strokeDashOffset\"},gridOpacity:{part:\"grid\",vgProp:\"opacity\"},gridWidth:{part:\"grid\",vgProp:\"strokeWidth\"},tickColor:{part:\"ticks\",vgProp:\"stroke\"},tickDash:{part:\"ticks\",vgProp:\"strokeDash\"},tickDashOffset:{part:\"ticks\",vgProp:\"strokeDashOffset\"},tickOpacity:{part:\"ticks\",vgProp:\"opacity\"},tickSize:null,tickWidth:{part:\"ticks\",vgProp:\"strokeWidth\"}};function wa(e){return e?.condition}const ka=[\"domain\",\"grid\",\"labels\",\"ticks\",\"title\"],Sa={grid:\"grid\",gridCap:\"grid\",gridColor:\"grid\",gridDash:\"grid\",gridDashOffset:\"grid\",gridOpacity:\"grid\",gridScale:\"grid\",gridWidth:\"grid\",orient:\"main\",bandPosition:\"both\",aria:\"main\",description:\"main\",domain:\"main\",domainCap:\"main\",domainColor:\"main\",domainDash:\"main\",domainDashOffset:\"main\",domainOpacity:\"main\",domainWidth:\"main\",format:\"main\",formatType:\"main\",labelAlign:\"main\",labelAngle:\"main\",labelBaseline:\"main\",labelBound:\"main\",labelColor:\"main\",labelFlush:\"main\",labelFlushOffset:\"main\",labelFont:\"main\",labelFontSize:\"main\",labelFontStyle:\"main\",labelFontWeight:\"main\",labelLimit:\"main\",labelLineHeight:\"main\",labelOffset:\"main\",labelOpacity:\"main\",labelOverlap:\"main\",labelPadding:\"main\",labels:\"main\",labelSeparation:\"main\",maxExtent:\"main\",minExtent:\"main\",offset:\"both\",position:\"main\",tickCap:\"main\",tickColor:\"main\",tickDash:\"main\",tickDashOffset:\"main\",tickMinStep:\"both\",tickOffset:\"both\",tickOpacity:\"main\",tickRound:\"both\",ticks:\"main\",tickSize:\"main\",tickWidth:\"both\",title:\"main\",titleAlign:\"main\",titleAnchor:\"main\",titleAngle:\"main\",titleBaseline:\"main\",titleColor:\"main\",titleFont:\"main\",titleFontSize:\"main\",titleFontStyle:\"main\",titleFontWeight:\"main\",titleLimit:\"main\",titleLineHeight:\"main\",titleOpacity:\"main\",titlePadding:\"main\",titleX:\"main\",titleY:\"main\",encode:\"both\",scale:\"both\",tickBand:\"both\",tickCount:\"both\",tickExtra:\"both\",translate:\"both\",values:\"both\",zindex:\"both\"},Da={orient:1,aria:1,bandPosition:1,description:1,domain:1,domainCap:1,domainColor:1,domainDash:1,domainDashOffset:1,domainOpacity:1,domainWidth:1,format:1,formatType:1,grid:1,gridCap:1,gridColor:1,gridDash:1,gridDashOffset:1,gridOpacity:1,gridWidth:1,labelAlign:1,labelAngle:1,labelBaseline:1,labelBound:1,labelColor:1,labelFlush:1,labelFlushOffset:1,labelFont:1,labelFontSize:1,labelFontStyle:1,labelFontWeight:1,labelLimit:1,labelLineHeight:1,labelOffset:1,labelOpacity:1,labelOverlap:1,labelPadding:1,labels:1,labelSeparation:1,maxExtent:1,minExtent:1,offset:1,position:1,tickBand:1,tickCap:1,tickColor:1,tickCount:1,tickDash:1,tickDashOffset:1,tickExtra:1,tickMinStep:1,tickOffset:1,tickOpacity:1,tickRound:1,ticks:1,tickSize:1,tickWidth:1,title:1,titleAlign:1,titleAnchor:1,titleAngle:1,titleBaseline:1,titleColor:1,titleFont:1,titleFontSize:1,titleFontStyle:1,titleFontWeight:1,titleLimit:1,titleLineHeight:1,titleOpacity:1,titlePadding:1,titleX:1,titleY:1,translate:1,values:1,zindex:1},Fa={...Da,style:1,labelExpr:1,encoding:1};function za(e){return!!Fa[e]}const Oa=D({axis:1,axisBand:1,axisBottom:1,axisDiscrete:1,axisLeft:1,axisPoint:1,axisQuantitative:1,axisRight:1,axisTemporal:1,axisTop:1,axisX:1,axisXBand:1,axisXDiscrete:1,axisXPoint:1,axisXQuantitative:1,axisXTemporal:1,axisY:1,axisYBand:1,axisYDiscrete:1,axisYPoint:1,axisYQuantitative:1,axisYTemporal:1});function _a(e){return\"mark\"in e}class Ca{constructor(e,t){this.name=e,this.run=t}hasMatchingType(e){return!!_a(e)&&(go(t=e.mark)?t.type:t)===this.name;var t}}function Na(e,n){const i=e&&e[n];return!!i&&(t.isArray(i)?g(i,(e=>!!e.field)):Ro(i)||qo(i))}function Pa(e,n){const i=e&&e[n];return!!i&&(t.isArray(i)?g(i,(e=>!!e.field)):Ro(i)||Bo(i)||Uo(i))}function Aa(e,t){if(zt(t)){const n=e[t];if((Ro(n)||Bo(n))&&(Zi(n.type)||Ro(n)&&n.timeUnit)){return Pa(e,at(t))}}return!1}function ja(e){return g(Be,(n=>{if(Na(e,n)){const i=e[n];if(t.isArray(i))return g(i,(e=>!!e.aggregate));{const e=ua(i);return e&&!!e.aggregate}}return!1}))}function Ta(e,t){const n=[],i=[],r=[],o=[],a={};return La(e,((s,l)=>{if(Ro(s)){const{field:c,aggregate:u,bin:f,timeUnit:d,...m}=s;if(u||d||f){const e=sa(s),p=e?.title;let g=ta(s,{forAs:!0});const h={...p?[]:{title:aa(s,t,{allowDisabling:!0})},...m,field:g};if(u){let e;if(en(u)?(e=\"argmax\",g=ta({op:\"argmax\",field:u.argmax},{forAs:!0}),h.field=`${g}.${c}`):Zt(u)?(e=\"argmin\",g=ta({op:\"argmin\",field:u.argmin},{forAs:!0}),h.field=`${g}.${c}`):\"boxplot\"!==u&&\"errorbar\"!==u&&\"errorband\"!==u&&(e=u),e){const t={op:e,as:g};c&&(t.field=c),o.push(t)}}else if(n.push(g),Yo(s)&&ln(f)){if(i.push({bin:f,field:c,as:g}),n.push(ta(s,{binSuffix:\"end\"})),xa(s,l)&&n.push(ta(s,{binSuffix:\"range\"})),zt(l)){const e={field:`${g}_end`};a[`${l}2`]=e}h.bin=\"binned\",et(l)||(h.type=er)}else if(d&&!zi(d)){r.push({timeUnit:d,field:c,as:g});const e=Yo(s)&&s.type!==nr&&\"time\";e&&(l===Se||l===Oe?h.formatType=e:!function(e){return!!kt[e]}(l)?zt(l)&&(h.axis={formatType:e,...h.axis}):h.legend={formatType:e,...h.legend})}a[l]=h}else n.push(c),a[l]=e[l]}else a[l]=e[l]})),{bins:i,timeUnits:r,aggregate:o,groupby:n,encoding:a}}function Ea(e,t,n){const i=Vt(t,n);if(!i)return!1;if(\"binned\"===i){const n=e[t===te?Z:ee];return!!(Ro(n)&&Ro(e[t])&&cn(n.bin))}return!0}function Ma(e,t){const n={};for(const i of D(e)){const r=da(e[i],i,t,{compositeMark:!0});n[i]=r}return n}function La(e,n,i){if(e)for(const r of D(e)){const o=e[r];if(t.isArray(o))for(const e of o)n.call(i,e,r);else n.call(i,o,r)}}function qa(e,n){return D(n).reduce(((i,r)=>{switch(r){case Z:case ee:case _e:case Ne:case Ce:case te:case ne:case ie:case re:case se:case le:case oe:case ae:case ce:case ue:case fe:case de:case Se:case he:case ve:case Oe:return i;case De:if(\"line\"===e||\"trail\"===e)return i;case Fe:case ze:{const e=n[r];if(t.isArray(e)||Ro(e))for(const n of t.array(e))n.aggregate||i.push(ta(n,{}));return i}case ye:if(\"trail\"===e)return i;case me:case pe:case ge:case be:case xe:case $e:case ke:case we:{const e=ua(n[r]);return e&&!e.aggregate&&i.push(ta(e,{})),i}}}),[])}function Ua(e,n,i){let r=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];if(\"tooltip\"in i)return{tooltip:i.tooltip};return{tooltip:[...e.map((e=>{let{fieldPrefix:t,titlePrefix:i}=e;const o=r?` of ${Ra(n)}`:\"\";return{field:t+n.field,type:n.type,title:yn(i)?{signal:`${i}\"${escape(o)}\"`}:i+o}})),...b(function(e){const n=[];for(const i of D(e))if(Na(e,i)){const r=e[i],o=t.array(r);for(const e of o)Ro(e)?n.push(e):qo(e)&&n.push(e.condition)}return n}(i).map(ea),d)]}}function Ra(e){const{title:t,field:n}=e;return U(t,n)}function Wa(e,n,i,r,o){const{scale:a,axis:s}=i;return l=>{let{partName:c,mark:u,positionPrefix:f,endPositionPrefix:d,extraEncoding:m={}}=l;const p=Ra(i);return Ba(e,c,o,{mark:u,encoding:{[n]:{field:`${f}_${i.field}`,type:i.type,...void 0!==p?{title:p}:{},...void 0!==a?{scale:a}:{},...void 0!==s?{axis:s}:{}},...t.isString(d)?{[`${n}2`]:{field:`${d}_${i.field}`}}:{},...r,...m}})}}function Ba(e,n,i,r){const{clip:o,color:a,opacity:s}=e,l=e.type;return e[n]||void 0===e[n]&&i[n]?[{...r,mark:{...i[n],...o?{clip:o}:{},...a?{color:a}:{},...s?{opacity:s}:{},...go(r.mark)?r.mark:{type:r.mark},style:`${l}-${String(n)}`,...t.isBoolean(e[n])?{}:e[n]}}]:[]}function Ia(e,t,n){const{encoding:i}=e,r=\"vertical\"===t?\"y\":\"x\",o=i[r],a=i[`${r}2`],s=i[`${r}Error`],l=i[`${r}Error2`];return{continuousAxisChannelDef:Ha(o,n),continuousAxisChannelDef2:Ha(a,n),continuousAxisChannelDefError:Ha(s,n),continuousAxisChannelDefError2:Ha(l,n),continuousAxis:r}}function Ha(e,t){if(e?.aggregate){const{aggregate:n,...i}=e;return n!==t&&yi(function(e,t){return`Continuous axis should not have customized aggregation function ${e}; ${t} already agregates the axis.`}(n,t)),i}return e}function Va(e,t){const{mark:n,encoding:i}=e,{x:r,y:o}=i;if(go(n)&&n.orient)return n.orient;if(Io(r)){if(Io(o)){const e=Ro(r)&&r.aggregate,n=Ro(o)&&o.aggregate;if(e||n!==t){if(n||e!==t){if(e===t&&n===t)throw new Error(\"Both x and y cannot have aggregate\");return ya(o)&&!ya(r)?\"horizontal\":\"vertical\"}return\"horizontal\"}return\"vertical\"}return\"horizontal\"}if(Io(o))return\"vertical\";throw new Error(`Need a valid continuous axis for ${t}s`)}const Ga=\"boxplot\",Ya=new Ca(Ga,Qa);function Xa(e){return t.isNumber(e)?\"tukey\":e}function Qa(e,n){let{config:i}=n;e={...e,encoding:Ma(e.encoding,i)};const{mark:r,encoding:o,params:a,projection:s,...l}=e,c=go(r)?r:{type:r};a&&yi(Gn(\"boxplot\"));const u=c.extent??i.boxplot.extent,d=Nn(\"size\",c,i),m=c.invalid,p=Xa(u),{bins:g,timeUnits:h,transform:y,continuousAxisChannelDef:v,continuousAxis:b,groupby:x,aggregate:$,encodingWithoutContinuousAxis:w,ticksOrient:k,boxOrient:D,customTooltipWithoutAggregatedField:F}=function(e,n,i){const r=Va(e,Ga),{continuousAxisChannelDef:o,continuousAxis:a}=Ia(e,r,Ga),s=o.field,l=L(s),c=Xa(n),u=[...Ja(s),{op:\"median\",field:s,as:`mid_box_${l}`},{op:\"min\",field:s,as:(\"min-max\"===c?\"lower_whisker_\":\"min_\")+l},{op:\"max\",field:s,as:(\"min-max\"===c?\"upper_whisker_\":\"max_\")+l}],f=\"min-max\"===c||\"tukey\"===c?[]:[{calculate:`datum[\"upper_box_${l}\"] - datum[\"lower_box_${l}\"]`,as:`iqr_${l}`},{calculate:`min(datum[\"upper_box_${l}\"] + datum[\"iqr_${l}\"] * ${n}, datum[\"max_${l}\"])`,as:`upper_whisker_${l}`},{calculate:`max(datum[\"lower_box_${l}\"] - datum[\"iqr_${l}\"] * ${n}, datum[\"min_${l}\"])`,as:`lower_whisker_${l}`}],{[a]:d,...m}=e.encoding,{customTooltipWithoutAggregatedField:p,filteredEncoding:g}=function(e){const{tooltip:n,...i}=e;if(!n)return{filteredEncoding:i};let r,o;if(t.isArray(n)){for(const e of n)e.aggregate?(r||(r=[]),r.push(e)):(o||(o=[]),o.push(e));r&&(i.tooltip=r)}else n.aggregate?i.tooltip=n:o=n;return t.isArray(o)&&1===o.length&&(o=o[0]),{customTooltipWithoutAggregatedField:o,filteredEncoding:i}}(m),{bins:h,timeUnits:y,aggregate:v,groupby:b,encoding:x}=Ta(g,i),$=\"vertical\"===r?\"horizontal\":\"vertical\",w=r,k=[...h,...y,{aggregate:[...v,...u],groupby:b},...f];return{bins:h,timeUnits:y,transform:k,groupby:b,aggregate:v,continuousAxisChannelDef:o,continuousAxis:a,encodingWithoutContinuousAxis:x,ticksOrient:$,boxOrient:w,customTooltipWithoutAggregatedField:p}}(e,u,i),z=L(v.field),{color:O,size:_,...C}=w,N=e=>Wa(c,b,v,e,i.boxplot),P=N(C),A=N(w),j=(t.isObject(i.boxplot.box)?i.boxplot.box.color:i.mark.color)||\"#4c78a8\",T=N({...C,..._?{size:_}:{},color:{condition:{test:`datum['lower_box_${v.field}'] >= datum['upper_box_${v.field}']`,...O||{value:j}}}}),E=Ua([{fieldPrefix:\"min-max\"===p?\"upper_whisker_\":\"max_\",titlePrefix:\"Max\"},{fieldPrefix:\"upper_box_\",titlePrefix:\"Q3\"},{fieldPrefix:\"mid_box_\",titlePrefix:\"Median\"},{fieldPrefix:\"lower_box_\",titlePrefix:\"Q1\"},{fieldPrefix:\"min-max\"===p?\"lower_whisker_\":\"min_\",titlePrefix:\"Min\"}],v,w),M={type:\"tick\",color:\"black\",opacity:1,orient:k,invalid:m,aria:!1},q=\"min-max\"===p?E:Ua([{fieldPrefix:\"upper_whisker_\",titlePrefix:\"Upper Whisker\"},{fieldPrefix:\"lower_whisker_\",titlePrefix:\"Lower Whisker\"}],v,w),U=[...P({partName:\"rule\",mark:{type:\"rule\",invalid:m,aria:!1},positionPrefix:\"lower_whisker\",endPositionPrefix:\"lower_box\",extraEncoding:q}),...P({partName:\"rule\",mark:{type:\"rule\",invalid:m,aria:!1},positionPrefix:\"upper_box\",endPositionPrefix:\"upper_whisker\",extraEncoding:q}),...P({partName:\"ticks\",mark:M,positionPrefix:\"lower_whisker\",extraEncoding:q}),...P({partName:\"ticks\",mark:M,positionPrefix:\"upper_whisker\",extraEncoding:q})],R=[...\"tukey\"!==p?U:[],...A({partName:\"box\",mark:{type:\"bar\",...d?{size:d}:{},orient:D,invalid:m,ariaRoleDescription:\"box\"},positionPrefix:\"lower_box\",endPositionPrefix:\"upper_box\",extraEncoding:E}),...T({partName:\"median\",mark:{type:\"tick\",invalid:m,...t.isObject(i.boxplot.median)&&i.boxplot.median.color?{color:i.boxplot.median.color}:{},...d?{size:d}:{},orient:k,aria:!1},positionPrefix:\"mid_box\",extraEncoding:E})];if(\"min-max\"===p)return{...l,transform:(l.transform??[]).concat(y),layer:R};const W=`datum[\"lower_box_${v.field}\"]`,B=`datum[\"upper_box_${v.field}\"]`,I=`(${B} - ${W})`,H=`${W} - ${u} * ${I}`,V=`${B} + ${u} * ${I}`,G=`datum[\"${v.field}\"]`,Y={joinaggregate:Ja(v.field),groupby:x},X={transform:[{filter:`(${H} <= ${G}) && (${G} <= ${V})`},{aggregate:[{op:\"min\",field:v.field,as:`lower_whisker_${z}`},{op:\"max\",field:v.field,as:`upper_whisker_${z}`},{op:\"min\",field:`lower_box_${v.field}`,as:`lower_box_${z}`},{op:\"max\",field:`upper_box_${v.field}`,as:`upper_box_${z}`},...$],groupby:x}],layer:U},{tooltip:Q,...J}=C,{scale:K,axis:Z}=v,ee=Ra(v),te=f(Z,[\"title\"]),ne=Ba(c,\"outliers\",i.boxplot,{transform:[{filter:`(${G} < ${H}) || (${G} > ${V})`}],mark:\"point\",encoding:{[b]:{field:v.field,type:v.type,...void 0!==ee?{title:ee}:{},...void 0!==K?{scale:K}:{},...S(te)?{}:{axis:te}},...J,...O?{color:O}:{},...F?{tooltip:F}:{}}})[0];let ie;const re=[...g,...h,Y];return ne?ie={transform:re,layer:[ne,X]}:(ie=X,ie.transform.unshift(...re)),{...l,layer:[ie,{transform:y,layer:R}]}}function Ja(e){const t=L(e);return[{op:\"q1\",field:e,as:`lower_box_${t}`},{op:\"q3\",field:e,as:`upper_box_${t}`}]}const Ka=\"errorbar\",Za=new Ca(Ka,es);function es(e,t){let{config:n}=t;e={...e,encoding:Ma(e.encoding,n)};const{transform:i,continuousAxisChannelDef:r,continuousAxis:o,encodingWithoutContinuousAxis:a,ticksOrient:s,markDef:l,outerSpec:c,tooltipEncoding:u}=ns(e,Ka,n);delete a.size;const f=Wa(l,o,r,a,n.errorbar),d=l.thickness,m=l.size,p={type:\"tick\",orient:s,aria:!1,...void 0!==d?{thickness:d}:{},...void 0!==m?{size:m}:{}},g=[...f({partName:\"ticks\",mark:p,positionPrefix:\"lower\",extraEncoding:u}),...f({partName:\"ticks\",mark:p,positionPrefix:\"upper\",extraEncoding:u}),...f({partName:\"rule\",mark:{type:\"rule\",ariaRoleDescription:\"errorbar\",...void 0!==d?{size:d}:{}},positionPrefix:\"lower\",endPositionPrefix:\"upper\",extraEncoding:u})];return{...c,transform:i,...g.length>1?{layer:g}:{...g[0]}}}function ts(e,t){const{encoding:n}=e;if(function(e){return(Go(e.x)||Go(e.y))&&!Go(e.x2)&&!Go(e.y2)&&!Go(e.xError)&&!Go(e.xError2)&&!Go(e.yError)&&!Go(e.yError2)}(n))return{orient:Va(e,t),inputType:\"raw\"};const i=function(e){return Go(e.x2)||Go(e.y2)}(n),r=function(e){return Go(e.xError)||Go(e.xError2)||Go(e.yError)||Go(e.yError2)}(n),o=n.x,a=n.y;if(i){if(r)throw new Error(`${t} cannot be both type aggregated-upper-lower and aggregated-error`);const e=n.x2,i=n.y2;if(Go(e)&&Go(i))throw new Error(`${t} cannot have both x2 and y2`);if(Go(e)){if(Io(o))return{orient:\"horizontal\",inputType:\"aggregated-upper-lower\"};throw new Error(`Both x and x2 have to be quantitative in ${t}`)}if(Go(i)){if(Io(a))return{orient:\"vertical\",inputType:\"aggregated-upper-lower\"};throw new Error(`Both y and y2 have to be quantitative in ${t}`)}throw new Error(\"No ranged axis\")}{const e=n.xError,i=n.xError2,r=n.yError,s=n.yError2;if(Go(i)&&!Go(e))throw new Error(`${t} cannot have xError2 without xError`);if(Go(s)&&!Go(r))throw new Error(`${t} cannot have yError2 without yError`);if(Go(e)&&Go(r))throw new Error(`${t} cannot have both xError and yError with both are quantiative`);if(Go(e)){if(Io(o))return{orient:\"horizontal\",inputType:\"aggregated-error\"};throw new Error(\"All x, xError, and xError2 (if exist) have to be quantitative\")}if(Go(r)){if(Io(a))return{orient:\"vertical\",inputType:\"aggregated-error\"};throw new Error(\"All y, yError, and yError2 (if exist) have to be quantitative\")}throw new Error(\"No ranged axis\")}}function ns(e,t,n){const{mark:i,encoding:r,params:o,projection:a,...s}=e,l=go(i)?i:{type:i};o&&yi(Gn(t));const{orient:c,inputType:u}=ts(e,t),{continuousAxisChannelDef:f,continuousAxisChannelDef2:d,continuousAxisChannelDefError:m,continuousAxisChannelDefError2:p,continuousAxis:g}=Ia(e,c,t),{errorBarSpecificAggregate:h,postAggregateCalculates:y,tooltipSummary:v,tooltipTitleWithFieldName:b}=function(e,t,n,i,r,o,a,s){let l=[],c=[];const u=t.field;let f,d=!1;if(\"raw\"===o){const t=e.center?e.center:e.extent?\"iqr\"===e.extent?\"median\":\"mean\":s.errorbar.center,n=e.extent?e.extent:\"mean\"===t?\"stderr\":\"iqr\";if(\"median\"===t!=(\"iqr\"===n)&&yi(function(e,t,n){return`${e} is not usually used with ${t} for ${n}.`}(t,n,a)),\"stderr\"===n||\"stdev\"===n)l=[{op:n,field:u,as:`extent_${u}`},{op:t,field:u,as:`center_${u}`}],c=[{calculate:`datum[\"center_${u}\"] + datum[\"extent_${u}\"]`,as:`upper_${u}`},{calculate:`datum[\"center_${u}\"] - datum[\"extent_${u}\"]`,as:`lower_${u}`}],f=[{fieldPrefix:\"center_\",titlePrefix:P(t)},{fieldPrefix:\"upper_\",titlePrefix:is(t,n,\"+\")},{fieldPrefix:\"lower_\",titlePrefix:is(t,n,\"-\")}],d=!0;else{let e,t,i;\"ci\"===n?(e=\"mean\",t=\"ci0\",i=\"ci1\"):(e=\"median\",t=\"q1\",i=\"q3\"),l=[{op:t,field:u,as:`lower_${u}`},{op:i,field:u,as:`upper_${u}`},{op:e,field:u,as:`center_${u}`}],f=[{fieldPrefix:\"upper_\",titlePrefix:aa({field:u,aggregate:i,type:\"quantitative\"},s,{allowDisabling:!1})},{fieldPrefix:\"lower_\",titlePrefix:aa({field:u,aggregate:t,type:\"quantitative\"},s,{allowDisabling:!1})},{fieldPrefix:\"center_\",titlePrefix:aa({field:u,aggregate:e,type:\"quantitative\"},s,{allowDisabling:!1})}]}}else{(e.center||e.extent)&&yi((m=e.center,`${(p=e.extent)?\"extent \":\"\"}${p&&m?\"and \":\"\"}${m?\"center \":\"\"}${p&&m?\"are \":\"is \"}not needed when data are aggregated.`)),\"aggregated-upper-lower\"===o?(f=[],c=[{calculate:`datum[\"${n.field}\"]`,as:`upper_${u}`},{calculate:`datum[\"${u}\"]`,as:`lower_${u}`}]):\"aggregated-error\"===o&&(f=[{fieldPrefix:\"\",titlePrefix:u}],c=[{calculate:`datum[\"${u}\"] + datum[\"${i.field}\"]`,as:`upper_${u}`}],r?c.push({calculate:`datum[\"${u}\"] + datum[\"${r.field}\"]`,as:`lower_${u}`}):c.push({calculate:`datum[\"${u}\"] - datum[\"${i.field}\"]`,as:`lower_${u}`}));for(const e of c)f.push({fieldPrefix:e.as.substring(0,6),titlePrefix:M(M(e.calculate,'datum[\"',\"\"),'\"]',\"\")})}var m,p;return{postAggregateCalculates:c,errorBarSpecificAggregate:l,tooltipSummary:f,tooltipTitleWithFieldName:d}}(l,f,d,m,p,u,t,n),{[g]:x,[\"x\"===g?\"x2\":\"y2\"]:$,[\"x\"===g?\"xError\":\"yError\"]:w,[\"x\"===g?\"xError2\":\"yError2\"]:k,...S}=r,{bins:D,timeUnits:F,aggregate:z,groupby:O,encoding:_}=Ta(S,n),C=[...z,...h],N=\"raw\"!==u?[]:O,A=Ua(v,f,_,b);return{transform:[...s.transform??[],...D,...F,...0===C.length?[]:[{aggregate:C,groupby:N}],...y],groupby:N,continuousAxisChannelDef:f,continuousAxis:g,encodingWithoutContinuousAxis:_,ticksOrient:\"vertical\"===c?\"horizontal\":\"vertical\",markDef:l,outerSpec:s,tooltipEncoding:A}}function is(e,t,n){return`${P(e)} ${n} ${t}`}const rs=\"errorband\",os=new Ca(rs,as);function as(e,t){let{config:n}=t;e={...e,encoding:Ma(e.encoding,n)};const{transform:i,continuousAxisChannelDef:r,continuousAxis:o,encodingWithoutContinuousAxis:a,markDef:s,outerSpec:l,tooltipEncoding:c}=ns(e,rs,n),u=s,f=Wa(u,o,r,a,n.errorband),d=void 0!==e.encoding.x&&void 0!==e.encoding.y;let m={type:d?\"area\":\"rect\"},p={type:d?\"line\":\"rule\"};const g={...u.interpolate?{interpolate:u.interpolate}:{},...u.tension&&u.interpolate?{tension:u.tension}:{}};return d?(m={...m,...g,ariaRoleDescription:\"errorband\"},p={...p,...g,aria:!1}):u.interpolate?yi(mi(\"interpolate\")):u.tension&&yi(mi(\"tension\")),{...l,transform:i,layer:[...f({partName:\"band\",mark:m,positionPrefix:\"lower\",endPositionPrefix:\"upper\",extraEncoding:c}),...f({partName:\"borders\",mark:p,positionPrefix:\"lower\",extraEncoding:c}),...f({partName:\"borders\",mark:p,positionPrefix:\"upper\",extraEncoding:c})]}}const ss={};function ls(e,t,n){const i=new Ca(e,t);ss[e]={normalizer:i,parts:n}}ls(Ga,Qa,[\"box\",\"median\",\"outliers\",\"rule\",\"ticks\"]),ls(Ka,es,[\"ticks\",\"rule\"]),ls(rs,as,[\"band\",\"borders\"]);const cs=[\"gradientHorizontalMaxLength\",\"gradientHorizontalMinLength\",\"gradientVerticalMaxLength\",\"gradientVerticalMinLength\",\"unselectedOpacity\"],us={titleAlign:\"align\",titleAnchor:\"anchor\",titleAngle:\"angle\",titleBaseline:\"baseline\",titleColor:\"color\",titleFont:\"font\",titleFontSize:\"fontSize\",titleFontStyle:\"fontStyle\",titleFontWeight:\"fontWeight\",titleLimit:\"limit\",titleLineHeight:\"lineHeight\",titleOrient:\"orient\",titlePadding:\"offset\"},fs={labelAlign:\"align\",labelAnchor:\"anchor\",labelAngle:\"angle\",labelBaseline:\"baseline\",labelColor:\"color\",labelFont:\"font\",labelFontSize:\"fontSize\",labelFontStyle:\"fontStyle\",labelFontWeight:\"fontWeight\",labelLimit:\"limit\",labelLineHeight:\"lineHeight\",labelOrient:\"orient\",labelPadding:\"offset\"},ds=D(us),ms=D(fs),ps=D({header:1,headerRow:1,headerColumn:1,headerFacet:1}),gs=[\"size\",\"shape\",\"fill\",\"stroke\",\"strokeDash\",\"strokeWidth\",\"opacity\"],hs=\"_vgsid_\",ys={point:{on:\"click\",fields:[hs],toggle:\"event.shiftKey\",resolve:\"global\",clear:\"dblclick\"},interval:{on:\"[pointerdown, window:pointerup] > window:pointermove!\",encodings:[\"x\",\"y\"],translate:\"[pointerdown, window:pointerup] > window:pointermove!\",zoom:\"wheel!\",mark:{fill:\"#333\",fillOpacity:.125,stroke:\"white\"},resolve:\"global\",clear:\"dblclick\"}};function vs(e){return\"legend\"===e||!!e?.legend}function bs(e){return vs(e)&&t.isObject(e)}function xs(e){return!!e?.select}function $s(e){const t=[];for(const n of e||[]){if(xs(n))continue;const{expr:e,bind:i,...r}=n;if(i&&e){const n={...r,bind:i,init:e};t.push(n)}else{const n={...r,...e?{update:e}:{},...i?{bind:i}:{}};t.push(n)}}return t}function ws(e){return\"concat\"in e}function ks(e){return\"vconcat\"in e}function Ss(e){return\"hconcat\"in e}function Ds(e){let{step:t,offsetIsDiscrete:n}=e;return n?t.for??\"offset\":\"position\"}function Fs(e){return t.isObject(e)&&void 0!==e.step}function zs(e){return e.view||e.width||e.height}const Os=D({align:1,bounds:1,center:1,columns:1,spacing:1});function _s(e,t){return e[t]??e[\"width\"===t?\"continuousWidth\":\"continuousHeight\"]}function Cs(e,t){const n=Ns(e,t);return Fs(n)?n.step:Ps}function Ns(e,t){return U(e[t]??e[\"width\"===t?\"discreteWidth\":\"discreteHeight\"],{step:e.step})}const Ps=20,As={background:\"white\",padding:5,timeFormat:\"%b %d, %Y\",countTitle:\"Count of Records\",view:{continuousWidth:200,continuousHeight:200,step:Ps},mark:{color:\"#4c78a8\",invalid:\"filter\",timeUnitBandSize:1},arc:{},area:{},bar:$o,circle:{},geoshape:{},image:{},line:{},point:{},rect:wo,rule:{color:\"black\"},square:{},text:{color:\"black\"},tick:{thickness:1},trail:{},boxplot:{size:14,extent:1.5,box:{},median:{color:\"white\"},outliers:{},rule:{},ticks:null},errorbar:{center:\"mean\",rule:!0,ticks:!1},errorband:{band:{opacity:.3},borders:!1},scale:{pointPadding:.5,barBandPaddingInner:.1,rectBandPaddingInner:0,bandWithNestedOffsetPaddingInner:.2,bandWithNestedOffsetPaddingOuter:.2,minBandSize:2,minFontSize:8,maxFontSize:40,minOpacity:.3,maxOpacity:.8,minSize:9,minStrokeWidth:1,maxStrokeWidth:4,quantileCount:4,quantizeCount:4,zero:!0},projection:{},legend:{gradientHorizontalMaxLength:200,gradientHorizontalMinLength:100,gradientVerticalMaxLength:200,gradientVerticalMinLength:64,unselectedOpacity:.35},header:{titlePadding:10,labelPadding:10},headerColumn:{},headerRow:{},headerFacet:{},selection:ys,style:{},title:{},facet:{spacing:20},concat:{spacing:20},normalizedNumberFormat:\".0%\"},js=[\"#4c78a8\",\"#f58518\",\"#e45756\",\"#72b7b2\",\"#54a24b\",\"#eeca3b\",\"#b279a2\",\"#ff9da6\",\"#9d755d\",\"#bab0ac\"],Ts={text:11,guideLabel:10,guideTitle:11,groupTitle:13,groupSubtitle:12},Es={blue:js[0],orange:js[1],red:js[2],teal:js[3],green:js[4],yellow:js[5],purple:js[6],pink:js[7],brown:js[8],gray0:\"#000\",gray1:\"#111\",gray2:\"#222\",gray3:\"#333\",gray4:\"#444\",gray5:\"#555\",gray6:\"#666\",gray7:\"#777\",gray8:\"#888\",gray9:\"#999\",gray10:\"#aaa\",gray11:\"#bbb\",gray12:\"#ccc\",gray13:\"#ddd\",gray14:\"#eee\",gray15:\"#fff\"};function Ms(e){const t=D(e||{}),n={};for(const i of t){const t=e[i];n[i]=wa(t)?kn(t):Sn(t)}return n}const Ls=[...vo,...Oa,...ps,\"background\",\"padding\",\"legend\",\"lineBreak\",\"scale\",\"style\",\"title\",\"view\"];function qs(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};const{color:n,font:i,fontSize:r,selection:o,...a}=e,s=t.mergeConfig({},l(As),i?function(e){return{text:{font:e},style:{\"guide-label\":{font:e},\"guide-title\":{font:e},\"group-title\":{font:e},\"group-subtitle\":{font:e}}}}(i):{},n?function(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};return{signals:[{name:\"color\",value:t.isObject(e)?{...Es,...e}:Es}],mark:{color:{signal:\"color.blue\"}},rule:{color:{signal:\"color.gray0\"}},text:{color:{signal:\"color.gray0\"}},style:{\"guide-label\":{fill:{signal:\"color.gray0\"}},\"guide-title\":{fill:{signal:\"color.gray0\"}},\"group-title\":{fill:{signal:\"color.gray0\"}},\"group-subtitle\":{fill:{signal:\"color.gray0\"}},cell:{stroke:{signal:\"color.gray8\"}}},axis:{domainColor:{signal:\"color.gray13\"},gridColor:{signal:\"color.gray8\"},tickColor:{signal:\"color.gray13\"}},range:{category:[{signal:\"color.blue\"},{signal:\"color.orange\"},{signal:\"color.red\"},{signal:\"color.teal\"},{signal:\"color.green\"},{signal:\"color.yellow\"},{signal:\"color.purple\"},{signal:\"color.pink\"},{signal:\"color.brown\"},{signal:\"color.grey8\"}]}}}(n):{},r?function(e){return{signals:[{name:\"fontSize\",value:t.isObject(e)?{...Ts,...e}:Ts}],text:{fontSize:{signal:\"fontSize.text\"}},style:{\"guide-label\":{fontSize:{signal:\"fontSize.guideLabel\"}},\"guide-title\":{fontSize:{signal:\"fontSize.guideTitle\"}},\"group-title\":{fontSize:{signal:\"fontSize.groupTitle\"}},\"group-subtitle\":{fontSize:{signal:\"fontSize.groupSubtitle\"}}}}}(r):{},a||{});o&&t.writeConfig(s,\"selection\",o,!0);const c=f(s,Ls);for(const e of[\"background\",\"lineBreak\",\"padding\"])s[e]&&(c[e]=Sn(s[e]));for(const e of vo)s[e]&&(c[e]=pn(s[e]));for(const e of Oa)s[e]&&(c[e]=Ms(s[e]));for(const e of ps)s[e]&&(c[e]=pn(s[e]));return s.legend&&(c.legend=pn(s.legend)),s.scale&&(c.scale=pn(s.scale)),s.style&&(c.style=function(e){const t=D(e),n={};for(const i of t)n[i]=Ms(e[i]);return n}(s.style)),s.title&&(c.title=pn(s.title)),s.view&&(c.view=pn(s.view)),c}const Us=new Set([\"view\",...po]),Rs=[\"color\",\"fontSize\",\"background\",\"padding\",\"facet\",\"concat\",\"numberFormat\",\"numberFormatType\",\"normalizedNumberFormat\",\"normalizedNumberFormatType\",\"timeFormat\",\"countTitle\",\"header\",\"axisQuantitative\",\"axisTemporal\",\"axisDiscrete\",\"axisPoint\",\"axisXBand\",\"axisXPoint\",\"axisXDiscrete\",\"axisXQuantitative\",\"axisXTemporal\",\"axisYBand\",\"axisYPoint\",\"axisYDiscrete\",\"axisYQuantitative\",\"axisYTemporal\",\"scale\",\"selection\",\"overlay\"],Ws={view:[\"continuousWidth\",\"continuousHeight\",\"discreteWidth\",\"discreteHeight\",\"step\"],area:[\"line\",\"point\"],bar:[\"binSpacing\",\"continuousBandSize\",\"discreteBandSize\",\"minBandSize\"],rect:[\"binSpacing\",\"continuousBandSize\",\"discreteBandSize\",\"minBandSize\"],line:[\"point\"],tick:[\"bandSize\",\"thickness\"]};function Bs(e){e=l(e);for(const t of Rs)delete e[t];if(e.axis)for(const t in e.axis)wa(e.axis[t])&&delete e.axis[t];if(e.legend)for(const t of cs)delete e.legend[t];if(e.mark){for(const t of yo)delete e.mark[t];e.mark.tooltip&&t.isObject(e.mark.tooltip)&&delete e.mark.tooltip}e.params&&(e.signals=(e.signals||[]).concat($s(e.params)),delete e.params);for(const t of Us){for(const n of yo)delete e[t][n];const n=Ws[t];if(n)for(const i of n)delete e[t][i];Is(e,t)}for(const t of D(ss))delete e[t];!function(e){const{titleMarkConfig:t,subtitleMarkConfig:n,subtitle:i}=gn(e.title);S(t)||(e.style[\"group-title\"]={...e.style[\"group-title\"],...t});S(n)||(e.style[\"group-subtitle\"]={...e.style[\"group-subtitle\"],...n});S(i)?delete e.title:e.title=i}(e);for(const n in e)t.isObject(e[n])&&S(e[n])&&delete e[n];return S(e)?void 0:e}function Is(e,t,n,i){\"view\"===t&&(n=\"cell\");const r={...e[t],...e.style[n??t]};S(r)||(e.style[n??t]=r),delete e[t]}function Hs(e){return\"layer\"in e}class Vs{map(e,t){return No(e)?this.mapFacet(e,t):function(e){return\"repeat\"in e}(e)?this.mapRepeat(e,t):Ss(e)?this.mapHConcat(e,t):ks(e)?this.mapVConcat(e,t):ws(e)?this.mapConcat(e,t):this.mapLayerOrUnit(e,t)}mapLayerOrUnit(e,t){if(Hs(e))return this.mapLayer(e,t);if(_a(e))return this.mapUnit(e,t);throw new Error(qn(e))}mapLayer(e,t){return{...e,layer:e.layer.map((e=>this.mapLayerOrUnit(e,t)))}}mapHConcat(e,t){return{...e,hconcat:e.hconcat.map((e=>this.map(e,t)))}}mapVConcat(e,t){return{...e,vconcat:e.vconcat.map((e=>this.map(e,t)))}}mapConcat(e,t){const{concat:n,...i}=e;return{...i,concat:n.map((e=>this.map(e,t)))}}mapFacet(e,t){return{...e,spec:this.map(e.spec,t)}}mapRepeat(e,t){return{...e,spec:this.map(e.spec,t)}}}const Gs={zero:1,center:1,normalize:1};const Ys=new Set([Jr,Zr,Kr,ro,no,lo,co,to,oo,ao]),Xs=new Set([Zr,Kr,Jr]);function Qs(e){return Ro(e)&&\"quantitative\"===Wo(e)&&!e.bin}function Js(e,t,n){let{orient:i,type:r}=n;const o=\"x\"===t?\"y\":\"radius\",a=\"x\"===t&&[\"bar\",\"area\"].includes(r),s=e[t],l=e[o];if(Ro(s)&&Ro(l))if(Qs(s)&&Qs(l)){if(s.stack)return t;if(l.stack)return o;const e=Ro(s)&&!!s.aggregate;if(e!==(Ro(l)&&!!l.aggregate))return e?t:o;if(a){if(\"vertical\"===i)return o;if(\"horizontal\"===i)return t}}else{if(Qs(s))return t;if(Qs(l))return o}else{if(Qs(s)){if(a&&\"vertical\"===i)return;return t}if(Qs(l)){if(a&&\"horizontal\"===i)return;return o}}}function Ks(e,n){const i=go(e)?e:{type:e},r=i.type;if(!Ys.has(r))return null;const o=Js(n,\"x\",i)||Js(n,\"theta\",i);if(!o)return null;const a=n[o],s=Ro(a)?ta(a,{}):void 0,l=function(e){switch(e){case\"x\":return\"y\";case\"y\":return\"x\";case\"theta\":return\"radius\";case\"radius\":return\"theta\"}}(o),c=[],u=new Set;if(n[l]){const e=n[l],t=Ro(e)?ta(e,{}):void 0;t&&t!==s&&(c.push(l),u.add(t))}const f=\"x\"===l?\"xOffset\":\"yOffset\",d=n[f],m=Ro(d)?ta(d,{}):void 0;m&&m!==s&&(c.push(f),u.add(m));const p=St.reduce(((e,i)=>{if(\"tooltip\"!==i&&Na(n,i)){const r=n[i];for(const n of t.array(r)){const t=ua(n);if(t.aggregate)continue;const r=ta(t,{});r&&u.has(r)||e.push({channel:i,fieldDef:t})}}return e}),[]);let g;return void 0!==a.stack?g=t.isBoolean(a.stack)?a.stack?\"zero\":null:a.stack:Xs.has(r)&&(g=\"zero\"),g&&g in Gs?ja(n)&&0===p.length?null:(a?.scale?.type&&a?.scale?.type!==or.LINEAR&&a?.stack&&yi(function(e){return`Stack is applied to a non-linear scale (${e}).`}(a.scale.type)),Go(n[it(o)])?(void 0!==a.stack&&yi(`Cannot stack \"${h=o}\" if there is already \"${h}2\".`),null):(Ro(a)&&a.aggregate&&!on.has(a.aggregate)&&yi(`Stacking is applied even though the aggregate function is non-summative (\"${a.aggregate}\").`),{groupbyChannels:c,groupbyFields:u,fieldChannel:o,impute:null!==a.impute&&fo(r),stackBy:p,offset:g})):null;var h}function Zs(e,t,n){const i=pn(e),r=Nn(\"orient\",i,n);if(i.orient=function(e,t,n){switch(e){case no:case lo:case co:case oo:case io:case eo:return}const{x:i,y:r,x2:o,y2:a}=t;switch(e){case Zr:if(Ro(i)&&(cn(i.bin)||Ro(r)&&r.aggregate&&!i.aggregate))return\"vertical\";if(Ro(r)&&(cn(r.bin)||Ro(i)&&i.aggregate&&!r.aggregate))return\"horizontal\";if(a||o){if(n)return n;if(!o)return(Ro(i)&&i.type===er&&!ln(i.bin)||Vo(i))&&Ro(r)&&cn(r.bin)?\"horizontal\":\"vertical\";if(!a)return(Ro(r)&&r.type===er&&!ln(r.bin)||Vo(r))&&Ro(i)&&cn(i.bin)?\"vertical\":\"horizontal\"}case ro:if(o&&(!Ro(i)||!cn(i.bin))&&a&&(!Ro(r)||!cn(r.bin)))return;case Kr:if(a)return Ro(r)&&cn(r.bin)?\"horizontal\":\"vertical\";if(o)return Ro(i)&&cn(i.bin)?\"vertical\":\"horizontal\";if(e===ro){if(i&&!r)return\"vertical\";if(r&&!i)return\"horizontal\"}case to:case ao:{const t=Ho(i),o=Ho(r);if(n)return n;if(t&&!o)return\"tick\"!==e?\"horizontal\":\"vertical\";if(!t&&o)return\"tick\"!==e?\"vertical\":\"horizontal\";if(t&&o)return\"vertical\";{const e=Yo(i)&&i.type===nr,t=Yo(r)&&r.type===nr;if(e&&!t)return\"vertical\";if(!e&&t)return\"horizontal\"}return}}return\"vertical\"}(i.type,t,r),void 0!==r&&r!==i.orient&&yi(`Specified orient \"${i.orient}\" overridden with \"${r}\".`),\"bar\"===i.type&&i.orient){const e=Nn(\"cornerRadiusEnd\",i,n);if(void 0!==e){const n=\"horizontal\"===i.orient&&t.x2||\"vertical\"===i.orient&&t.y2?[\"cornerRadius\"]:xo[i.orient];for(const t of n)i[t]=e;void 0!==i.cornerRadiusEnd&&delete i.cornerRadiusEnd}}const o=Nn(\"opacity\",i,n),a=Nn(\"fillOpacity\",i,n);void 0===o&&void 0===a&&(i.opacity=function(e,t){if(p([no,ao,lo,co],e)&&!ja(t))return.7;return}(i.type,t));return void 0===Nn(\"cursor\",i,n)&&(i.cursor=function(e,t,n){if(t.href||e.href||Nn(\"href\",e,n))return\"pointer\";return e.cursor}(i,t,n)),i}function el(e){const{point:t,line:n,...i}=e;return D(i).length>1?i:i.type}function tl(e){for(const t of[\"line\",\"area\",\"rule\",\"trail\"])e[t]&&(e={...e,[t]:f(e[t],[\"point\",\"line\"])});return e}function nl(e){let n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},i=arguments.length>2?arguments[2]:void 0;return\"transparent\"===e.point?{opacity:0}:e.point?t.isObject(e.point)?e.point:{}:void 0!==e.point?null:n.point||i.shape?t.isObject(n.point)?n.point:{}:void 0}function il(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};return e.line?!0===e.line?{}:e.line:void 0!==e.line?null:t.line?!0===t.line?{}:t.line:void 0}class rl{name=\"path-overlay\";hasMatchingType(e,t){if(_a(e)){const{mark:n,encoding:i}=e,r=go(n)?n:{type:n};switch(r.type){case\"line\":case\"rule\":case\"trail\":return!!nl(r,t[r.type],i);case\"area\":return!!nl(r,t[r.type],i)||!!il(r,t[r.type])}}return!1}run(e,t,n){const{config:i}=t,{params:r,projection:o,mark:a,name:s,encoding:l,...c}=e,d=Ma(l,i),m=go(a)?a:{type:a},p=nl(m,i[m.type],d),g=\"area\"===m.type&&il(m,i[m.type]),h=[{name:s,...r?{params:r}:{},mark:el({...\"area\"===m.type&&void 0===m.opacity&&void 0===m.fillOpacity?{opacity:.7}:{},...m}),encoding:f(d,[\"shape\"])}],y=Ks(Zs(m,d,i),d);let v=d;if(y){const{fieldChannel:e,offset:t}=y;v={...d,[e]:{...d[e],...t?{stack:t}:{}}}}return v=f(v,[\"y2\",\"x2\"]),g&&h.push({...o?{projection:o}:{},mark:{type:\"line\",...u(m,[\"clip\",\"interpolate\",\"tension\",\"tooltip\"]),...g},encoding:v}),p&&h.push({...o?{projection:o}:{},mark:{type:\"point\",opacity:1,filled:!0,...u(m,[\"clip\",\"tooltip\"]),...p},encoding:v}),n({...c,layer:h},{...t,config:tl(i)})}}function ol(e,t){return t?_o(e)?fl(e,t):ll(e,t):e}function al(e,t){return t?fl(e,t):e}function sl(e,n,i){const r=n[e];return(o=r)&&!t.isString(o)&&\"repeat\"in o?r.repeat in i?{...n,[e]:i[r.repeat]}:void yi(function(e){return`Unknown repeated value \"${e}\".`}(r.repeat)):n;var o}function ll(e,t){if(void 0!==(e=sl(\"field\",e,t))){if(null===e)return null;if(Ao(e)&&zo(e.sort)){const n=sl(\"field\",e.sort,t);e={...e,...n?{sort:n}:{}}}return e}}function cl(e,t){if(Ro(e))return ll(e,t);{const n=sl(\"datum\",e,t);return n===e||n.type||(n.type=\"nominal\"),n}}function ul(e,t){if(!Go(e)){if(Uo(e)){const n=cl(e.condition,t);if(n)return{...e,condition:n};{const{condition:t,...n}=e;return n}}return e}{const n=cl(e,t);if(n)return n;if(Lo(e))return{condition:e.condition}}}function fl(e,n){const i={};for(const r in e)if(t.hasOwnProperty(e,r)){const o=e[r];if(t.isArray(o))i[r]=o.map((e=>ul(e,n))).filter((e=>e));else{const e=ul(o,n);void 0!==e&&(i[r]=e)}}return i}class dl{name=\"RuleForRangedLine\";hasMatchingType(e){if(_a(e)){const{encoding:t,mark:n}=e;if(\"line\"===n||go(n)&&\"line\"===n.type)for(const e of Ze){const n=t[tt(e)];if(t[e]&&(Ro(n)&&!cn(n.bin)||Bo(n)))return!0}}return!1}run(e,n,i){const{encoding:r,mark:o}=e;var a,s;return yi((a=!!r.x2,s=!!r.y2,`Line mark is for continuous lines and thus cannot be used with ${a&&s?\"x2 and y2\":a?\"x2\":\"y2\"}. We will use the rule mark (line segments) instead.`)),i({...e,mark:t.isObject(o)?{...o,type:\"rule\"}:\"rule\"},n)}}function ml(e){let{parentEncoding:n,encoding:i={},layer:r}=e,o={};if(n){const e=new Set([...D(n),...D(i)]);for(const a of e){const e=i[a],s=n[a];if(Go(e)){const t={...s,...e};o[a]=t}else Uo(e)?o[a]={...e,condition:{...s,...e.condition}}:e||null===e?o[a]=e:(r||Xo(s)||yn(s)||Go(s)||t.isArray(s))&&(o[a]=s)}}else o=i;return!o||S(o)?void 0:o}function pl(e){const{parentProjection:t,projection:n}=e;return t&&n&&yi(function(e){const{parentProjection:t,projection:n}=e;return`Layer's shared projection ${X(t)} is overridden by a child projection ${X(n)}.`}({parentProjection:t,projection:n})),n??t}function gl(e){return\"filter\"in e}function hl(e){return\"lookup\"in e}function yl(e){return\"pivot\"in e}function vl(e){return\"density\"in e}function bl(e){return\"quantile\"in e}function xl(e){return\"regression\"in e}function $l(e){return\"loess\"in e}function wl(e){return\"sample\"in e}function kl(e){return\"window\"in e}function Sl(e){return\"joinaggregate\"in e}function Dl(e){return\"flatten\"in e}function Fl(e){return\"calculate\"in e}function zl(e){return\"bin\"in e}function Ol(e){return\"impute\"in e}function _l(e){return\"timeUnit\"in e}function Cl(e){return\"aggregate\"in e}function Nl(e){return\"stack\"in e}function Pl(e){return\"fold\"in e}function Al(e){return\"extent\"in e&&!(\"density\"in e)&&!(\"regression\"in e)}function jl(e,t){const{transform:n,...i}=e;if(n){return{...i,transform:n.map((e=>{if(gl(e))return{filter:Ml(e,t)};if(zl(e)&&un(e.bin))return{...e,bin:El(e.bin)};if(hl(e)){const{selection:t,...n}=e.from;return t?{...e,from:{param:t,...n}}:e}return e}))}}return e}function Tl(e,n){const i=l(e);if(Ro(i)&&un(i.bin)&&(i.bin=El(i.bin)),Qo(i)&&i.scale?.domain?.selection){const{selection:e,...t}=i.scale.domain;i.scale.domain={...t,...e?{param:e}:{}}}if(Lo(i))if(t.isArray(i.condition))i.condition=i.condition.map((e=>{const{selection:t,param:i,test:r,...o}=e;return i?e:{...o,test:Ml(e,n)}}));else{const{selection:e,param:t,test:r,...o}=Tl(i.condition,n);i.condition=t?i.condition:{...o,test:Ml(i.condition,n)}}return i}function El(e){const t=e.extent;if(t?.selection){const{selection:n,...i}=t;return{...e,extent:{...i,param:n}}}return e}function Ml(e,t){const n=e=>s(e,(e=>{const n={param:e,empty:t.emptySelections[e]??!0};return t.selectionPredicates[e]??=[],t.selectionPredicates[e].push(n),n}));return e.selection?n(e.selection):s(e.test||e.filter,(e=>e.selection?n(e.selection):e))}class Ll extends Vs{map(e,t){const n=t.selections??[];if(e.params&&!_a(e)){const t=[];for(const i of e.params)xs(i)?n.push(i):t.push(i);e.params=t}return t.selections=n,super.map(e,t)}mapUnit(e,n){const i=n.selections;if(!i||!i.length)return e;const r=(n.path??[]).concat(e.name),o=[];for(const n of i)if(n.views&&n.views.length)for(const i of n.views)(t.isString(i)&&(i===e.name||r.includes(i))||t.isArray(i)&&i.map((e=>r.indexOf(e))).every(((e,t,n)=>-1!==e&&(0===t||e>n[t-1]))))&&o.push(n);else o.push(n);return o.length&&(e.params=o),e}}for(const e of[\"mapFacet\",\"mapRepeat\",\"mapHConcat\",\"mapVConcat\",\"mapLayer\"]){const t=Ll.prototype[e];Ll.prototype[e]=function(e,n){return t.call(this,e,ql(e,n))}}function ql(e,t){return e.name?{...t,path:(t.path??[]).concat(e.name)}:t}function Ul(e,t){void 0===t&&(t=qs(e.config));const n=function(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};const n={config:t};return Bl.map(Rl.map(Wl.map(e,n),n),n)}(e,t),{width:i,height:r}=e,o=function(e,t,n){let{width:i,height:r}=t;const o=_a(e)||Hs(e),a={};o?\"container\"==i&&\"container\"==r?(a.type=\"fit\",a.contains=\"padding\"):\"container\"==i?(a.type=\"fit-x\",a.contains=\"padding\"):\"container\"==r&&(a.type=\"fit-y\",a.contains=\"padding\"):(\"container\"==i&&(yi(Rn(\"width\")),i=void 0),\"container\"==r&&(yi(Rn(\"height\")),r=void 0));const s={type:\"pad\",...a,...n?Il(n.autosize):{},...Il(e.autosize)};\"fit\"!==s.type||o||(yi(Un),s.type=\"pad\");\"container\"==i&&\"fit\"!=s.type&&\"fit-x\"!=s.type&&yi(Wn(\"width\"));\"container\"==r&&\"fit\"!=s.type&&\"fit-y\"!=s.type&&yi(Wn(\"height\"));if(Y(s,{type:\"pad\"}))return;return s}(n,{width:i,height:r,autosize:e.autosize},t);return{...n,...o?{autosize:o}:{}}}const Rl=new class extends Vs{nonFacetUnitNormalizers=[Ya,Za,os,new rl,new dl];map(e,t){if(_a(e)){const n=Na(e.encoding,Q),i=Na(e.encoding,J),r=Na(e.encoding,K);if(n||i||r)return this.mapFacetedUnit(e,t)}return super.map(e,t)}mapUnit(e,t){const{parentEncoding:n,parentProjection:i}=t,r=al(e.encoding,t.repeater),o={...e,...e.name?{name:[t.repeaterPrefix,e.name].filter((e=>e)).join(\"_\")}:{},...r?{encoding:r}:{}};if(n||i)return this.mapUnitWithParentEncodingOrProjection(o,t);const a=this.mapLayerOrUnit.bind(this);for(const e of this.nonFacetUnitNormalizers)if(e.hasMatchingType(o,t.config))return e.run(o,t,a);return o}mapRepeat(e,n){return function(e){return!t.isArray(e.repeat)&&e.repeat.layer}(e)?this.mapLayerRepeat(e,n):this.mapNonLayerRepeat(e,n)}mapLayerRepeat(e,t){const{repeat:n,spec:i,...r}=e,{row:o,column:a,layer:s}=n,{repeater:l={},repeaterPrefix:c=\"\"}=t;return o||a?this.mapRepeat({...e,repeat:{...o?{row:o}:{},...a?{column:a}:{}},spec:{repeat:{layer:s},spec:i}},t):{...r,layer:s.map((e=>{const n={...l,layer:e},r=`${(i.name?`${i.name}_`:\"\")+c}child__layer_${_(e)}`,o=this.mapLayerOrUnit(i,{...t,repeater:n,repeaterPrefix:r});return o.name=r,o}))}}mapNonLayerRepeat(e,n){const{repeat:i,spec:r,data:o,...a}=e;!t.isArray(i)&&e.columns&&(e=f(e,[\"columns\"]),yi(Xn(\"repeat\")));const s=[],{repeater:l={},repeaterPrefix:c=\"\"}=n,u=!t.isArray(i)&&i.row||[l?l.row:null],d=!t.isArray(i)&&i.column||[l?l.column:null],m=t.isArray(i)&&i||[l?l.repeat:null];for(const e of m)for(const o of u)for(const a of d){const u={repeat:e,row:o,column:a,layer:l.layer},d=(r.name?`${r.name}_`:\"\")+c+\"child__\"+(t.isArray(i)?`${_(e)}`:(i.row?`row_${_(o)}`:\"\")+(i.column?`column_${_(a)}`:\"\")),m=this.map(r,{...n,repeater:u,repeaterPrefix:d});m.name=d,s.push(f(m,[\"data\"]))}const p=t.isArray(i)?e.columns:i.column?i.column.length:1;return{data:r.data??o,align:\"all\",...a,columns:p,concat:s}}mapFacet(e,t){const{facet:n}=e;return _o(n)&&e.columns&&(e=f(e,[\"columns\"]),yi(Xn(\"facet\"))),super.mapFacet(e,t)}mapUnitWithParentEncodingOrProjection(e,t){const{encoding:n,projection:i}=e,{parentEncoding:r,parentProjection:o,config:a}=t,s=pl({parentProjection:o,projection:i}),l=ml({parentEncoding:r,encoding:al(n,t.repeater)});return this.mapUnit({...e,...s?{projection:s}:{},...l?{encoding:l}:{}},{config:a})}mapFacetedUnit(e,t){const{row:n,column:i,facet:r,...o}=e.encoding,{mark:a,width:s,projection:l,height:c,view:u,params:f,encoding:d,...m}=e,{facetMapping:p,layout:g}=this.getFacetMappingAndLayout({row:n,column:i,facet:r},t),h=al(o,t.repeater);return this.mapFacet({...m,...g,facet:p,spec:{...s?{width:s}:{},...c?{height:c}:{},...u?{view:u}:{},...l?{projection:l}:{},mark:a,encoding:h,...f?{params:f}:{}}},t)}getFacetMappingAndLayout(e,t){const{row:n,column:i,facet:r}=e;if(n||i){r&&yi(`Facet encoding dropped as ${(o=[...n?[Q]:[],...i?[J]:[]]).join(\" and \")} ${o.length>1?\"are\":\"is\"} also specified.`);const t={},a={};for(const n of[Q,J]){const i=e[n];if(i){const{align:e,center:r,spacing:o,columns:s,...l}=i;t[n]=l;for(const e of[\"align\",\"center\",\"spacing\"])void 0!==i[e]&&(a[e]??={},a[e][n]=i[e])}}return{facetMapping:t,layout:a}}{const{align:e,center:n,spacing:i,columns:o,...a}=r;return{facetMapping:ol(a,t.repeater),layout:{...e?{align:e}:{},...n?{center:n}:{},...i?{spacing:i}:{},...o?{columns:o}:{}}}}var o}mapLayer(e,t){let{parentEncoding:n,parentProjection:i,...r}=t;const{encoding:o,projection:a,...s}=e,l={...r,parentEncoding:ml({parentEncoding:n,encoding:o,layer:!0}),parentProjection:pl({parentProjection:i,projection:a})};return super.mapLayer({...s,...e.name?{name:[l.repeaterPrefix,e.name].filter((e=>e)).join(\"_\")}:{}},l)}},Wl=new class extends Vs{map(e,t){return t.emptySelections??={},t.selectionPredicates??={},e=jl(e,t),super.map(e,t)}mapLayerOrUnit(e,t){if((e=jl(e,t)).encoding){const n={};for(const[i,r]of z(e.encoding))n[i]=Tl(r,t);e={...e,encoding:n}}return super.mapLayerOrUnit(e,t)}mapUnit(e,t){const{selection:n,...i}=e;return n?{...i,params:z(n).map((e=>{let[n,i]=e;const{init:r,bind:o,empty:a,...s}=i;\"single\"===s.type?(s.type=\"point\",s.toggle=!1):\"multi\"===s.type&&(s.type=\"point\"),t.emptySelections[n]=\"none\"!==a;for(const e of F(t.selectionPredicates[n]??{}))e.empty=\"none\"!==a;return{name:n,value:r,select:s,bind:o}}))}:e}},Bl=new Ll;function Il(e){return t.isString(e)?{type:e}:e??{}}const Hl=[\"background\",\"padding\"];function Vl(e,t){const n={};for(const t of Hl)e&&void 0!==e[t]&&(n[t]=Sn(e[t]));return t&&(n.params=e.params),n}class Gl{constructor(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};this.explicit=e,this.implicit=t}clone(){return new Gl(l(this.explicit),l(this.implicit))}combine(){return{...this.explicit,...this.implicit}}get(e){return U(this.explicit[e],this.implicit[e])}getWithExplicit(e){return void 0!==this.explicit[e]?{explicit:!0,value:this.explicit[e]}:void 0!==this.implicit[e]?{explicit:!1,value:this.implicit[e]}:{explicit:!1,value:void 0}}setWithExplicit(e,t){let{value:n,explicit:i}=t;void 0!==n&&this.set(e,n,i)}set(e,t,n){return delete this[n?\"implicit\":\"explicit\"][e],this[n?\"explicit\":\"implicit\"][e]=t,this}copyKeyFromSplit(e,t){let{explicit:n,implicit:i}=t;void 0!==n[e]?this.set(e,n[e],!0):void 0!==i[e]&&this.set(e,i[e],!1)}copyKeyFromObject(e,t){void 0!==t[e]&&this.set(e,t[e],!0)}copyAll(e){for(const t of D(e.combine())){const n=e.getWithExplicit(t);this.setWithExplicit(t,n)}}}function Yl(e){return{explicit:!0,value:e}}function Xl(e){return{explicit:!1,value:e}}function Ql(e){return(t,n,i,r)=>{const o=e(t.value,n.value);return o>0?t:o<0?n:Jl(t,n,i,r)}}function Jl(e,t,n,i){return e.explicit&&t.explicit&&yi(function(e,t,n,i){return`Conflicting ${t.toString()} property \"${e.toString()}\" (${X(n)} and ${X(i)}). Using ${X(n)}.`}(n,i,e.value,t.value)),e}function Kl(e,t,n,i){let r=arguments.length>4&&void 0!==arguments[4]?arguments[4]:Jl;return void 0===e||void 0===e.value?t:e.explicit&&!t.explicit?e:t.explicit&&!e.explicit?t:Y(e.value,t.value)?e:r(e,t,n,i)}class Zl extends Gl{constructor(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},n=arguments.length>2&&void 0!==arguments[2]&&arguments[2];super(e,t),this.explicit=e,this.implicit=t,this.parseNothing=n}clone(){const e=super.clone();return e.parseNothing=this.parseNothing,e}}function ec(e){return\"url\"in e}function tc(e){return\"values\"in e}function nc(e){return\"name\"in e&&!ec(e)&&!tc(e)&&!ic(e)}function ic(e){return e&&(rc(e)||oc(e)||ac(e))}function rc(e){return\"sequence\"in e}function oc(e){return\"sphere\"in e}function ac(e){return\"graticule\"in e}let sc=function(e){return e[e.Raw=0]=\"Raw\",e[e.Main=1]=\"Main\",e[e.Row=2]=\"Row\",e[e.Column=3]=\"Column\",e[e.Lookup=4]=\"Lookup\",e}({});function lc(e){const{signals:t,hasLegend:n,index:i,...r}=e;return r.field=E(r.field),r}function cc(e){let n=!(arguments.length>1&&void 0!==arguments[1])||arguments[1],i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:t.identity;if(t.isArray(e)){const t=e.map((e=>cc(e,n,i)));return n?`[${t.join(\", \")}]`:t}return vi(e)?i(n?Si(e):function(e){const t=ki(e,!0);return e.utc?+new Date(Date.UTC(...t)):+new Date(...t)}(e)):n?i(X(e)):e}function uc(e,n){for(const i of F(e.component.selection??{})){const r=i.name;let o=`${r}${_u}, ${\"global\"===i.resolve?\"true\":`{unit: ${Au(e)}}`}`;for(const t of Pu)t.defined(i)&&(t.signals&&(n=t.signals(e,i,n)),t.modifyExpr&&(o=t.modifyExpr(e,i,o)));n.push({name:r+Cu,on:[{events:{signal:i.name+_u},update:`modify(${t.stringValue(i.name+Ou)}, ${o})`}]})}return mc(n)}function fc(e,n){if(e.component.selection&&D(e.component.selection).length){const i=t.stringValue(e.getName(\"cell\"));n.unshift({name:\"facet\",value:{},on:[{events:t.parseSelector(\"pointermove\",\"scope\"),update:`isTuple(facet) ? facet : group(${i}).datum`}]})}return mc(n)}function dc(e,t){for(const n of F(e.component.selection??{}))for(const i of Pu)i.defined(n)&&i.marks&&(t=i.marks(e,n,t));return t}function mc(e){return e.map((e=>(e.on&&!e.on.length&&delete e.on,e)))}class pc{_children=[];_parent=null;constructor(e,t){this.debugName=t,e&&(this.parent=e)}clone(){throw new Error(\"Cannot clone node\")}get parent(){return this._parent}set parent(e){this._parent=e,e&&e.addChild(this)}get children(){return this._children}numChildren(){return this._children.length}addChild(e,t){this._children.includes(e)?yi(\"Attempt to add the same child twice.\"):void 0!==t?this._children.splice(t,0,e):this._children.push(e)}removeChild(e){const t=this._children.indexOf(e);return this._children.splice(t,1),t}remove(){let e=this._parent.removeChild(this);for(const t of this._children)t._parent=this._parent,this._parent.addChild(t,e++)}insertAsParentOf(e){const t=e.parent;t.removeChild(this),this.parent=t,e.parent=this}swapWithParent(){const e=this._parent,t=e.parent;for(const t of this._children)t.parent=e;this._children=[],e.removeChild(this);const n=e.parent.removeChild(e);this._parent=t,t.addChild(this,n),e.parent=this}}class gc extends pc{clone(){const e=new this.constructor;return e.debugName=`clone_${this.debugName}`,e._source=this._source,e._name=`clone_${this._name}`,e.type=this.type,e.refCounts=this.refCounts,e.refCounts[e._name]=0,e}constructor(e,t,n,i){super(e,t),this.type=n,this.refCounts=i,this._source=this._name=t,this.refCounts&&!(this._name in this.refCounts)&&(this.refCounts[this._name]=0)}dependentFields(){return new Set}producedFields(){return new Set}hash(){return void 0===this._hash&&(this._hash=`Output ${W()}`),this._hash}getSource(){return this.refCounts[this._name]++,this._source}isRequired(){return!!this.refCounts[this._name]}setSource(e){this._source=e}}function hc(e){return void 0!==e.as}function yc(e){return`${e}_end`}class vc extends pc{clone(){return new vc(null,l(this.timeUnits))}constructor(e,t){super(e),this.timeUnits=t}static makeFromEncoding(e,t){const n=t.reduceFieldDef(((e,n,i)=>{const{field:r,timeUnit:o}=n;if(o){let a;if(zi(o)){if(gm(t)){const{mark:e,markDef:i,config:s}=t,l=jo({fieldDef:n,markDef:i,config:s});(mo(e)||l)&&(a={timeUnit:Ei(o),field:r})}}else a={as:ta(n,{forAs:!0}),field:r,timeUnit:o};if(gm(t)){const{mark:e,markDef:r,config:o}=t,s=jo({fieldDef:n,markDef:r,config:o});mo(e)&&zt(i)&&.5!==s&&(a.rectBandPosition=s)}a&&(e[d(a)]=a)}return e}),{});return S(n)?null:new vc(e,n)}static makeFromTransform(e,t){const{timeUnit:n,...i}={...t},r={...i,timeUnit:Ei(n)};return new vc(e,{[d(r)]:r})}merge(e){this.timeUnits={...this.timeUnits};for(const t in e.timeUnits)this.timeUnits[t]||(this.timeUnits[t]=e.timeUnits[t]);for(const t of e.children)e.removeChild(t),t.parent=this;e.remove()}removeFormulas(e){const t={};for(const[n,i]of z(this.timeUnits)){const r=hc(i)?i.as:`${i.field}_end`;e.has(r)||(t[n]=i)}this.timeUnits=t}producedFields(){return new Set(F(this.timeUnits).map((e=>hc(e)?e.as:yc(e.field))))}dependentFields(){return new Set(F(this.timeUnits).map((e=>e.field)))}hash(){return`TimeUnit ${d(this.timeUnits)}`}assemble(){const e=[];for(const t of F(this.timeUnits)){const{rectBandPosition:n}=t,i=Ei(t.timeUnit);if(hc(t)){const{field:r,as:o}=t,{unit:a,utc:s,...l}=i,c=[o,`${o}_end`];e.push({field:E(r),type:\"timeunit\",...a?{units:Ni(a)}:{},...s?{timezone:\"utc\"}:{},...l,as:c}),e.push(...wc(c,n,i))}else if(t){const{field:r}=t,o=r.replaceAll(\"\\\\.\",\".\"),a=$c({timeUnit:i,field:o}),s=yc(o);e.push({type:\"formula\",expr:a,as:s}),e.push(...wc([o,s],n,i))}}return e}}const bc=\"offsetted_rect_start\",xc=\"offsetted_rect_end\";function $c(e){let{timeUnit:t,field:n,reverse:i}=e;const{unit:r,utc:o}=t,a=Pi(r),{part:s,step:l}=qi(a,t.step);return`${o?\"utcOffset\":\"timeOffset\"}('${s}', datum['${n}'], ${i?-l:l})`}function wc(e,t,n){let[i,r]=e;if(void 0!==t&&.5!==t){const e=`datum['${i}']`,o=`datum['${r}']`;return[{type:\"formula\",expr:kc([$c({timeUnit:n,field:i,reverse:!0}),e],t+.5),as:`${i}_${bc}`},{type:\"formula\",expr:kc([e,o],t+.5),as:`${i}_${xc}`}]}return[]}function kc(e,t){let[n,i]=e;return`${1-t} * ${n} + ${t} * ${i}`}const Sc=\"_tuple_fields\";class Dc{constructor(){for(var e=arguments.length,t=new Array(e),n=0;n!0,parse:(e,n,i)=>{const r=n.name,o=n.project??=new Dc,a={},s={},l=new Set,c=(e,t)=>{const n=\"visual\"===t?e.channel:e.field;let i=_(`${r}_${n}`);for(let e=1;l.has(i);e++)i=_(`${r}_${n}_${e}`);return l.add(i),{[t]:i}},u=n.type,f=e.config.selection[u],m=void 0!==i.value?t.array(i.value):null;let{fields:p,encodings:g}=t.isObject(i.select)?i.select:{};if(!p&&!g&&m)for(const e of m)if(t.isObject(e))for(const t of D(e))Je[t]?(g||(g=[])).push(t):\"interval\"===u?(yi('Interval selections should be initialized using \"x\", \"y\", \"longitude\", or \"latitude\" keys.'),g=f.encodings):(p??=[]).push(t);p||g||(g=f.encodings,\"fields\"in f&&(p=f.fields));for(const t of g??[]){const n=e.fieldDef(t);if(n){let i=n.field;if(n.aggregate){yi(Vn(t,n.aggregate));continue}if(!i){yi(Hn(t));continue}if(n.timeUnit&&!zi(n.timeUnit)){i=e.vgField(t);const r={timeUnit:n.timeUnit,as:i,field:n.field};s[d(r)]=r}if(!a[i]){const r={field:i,channel:t,type:\"interval\"===u&&Ht(t)&&yr(e.getScaleComponent(t).get(\"type\"))?\"R\":n.bin?\"R-RE\":\"E\",index:o.items.length};r.signals={...c(r,\"data\"),...c(r,\"visual\")},o.items.push(a[i]=r),o.hasField[i]=a[i],o.hasSelectionId=o.hasSelectionId||i===hs,Ee(t)?(r.geoChannel=t,r.channel=Te(t),o.hasChannel[r.channel]=a[i]):o.hasChannel[t]=a[i]}}else yi(Hn(t))}for(const e of p??[]){if(o.hasField[e])continue;const t={type:\"E\",field:e,index:o.items.length};t.signals={...c(t,\"data\")},o.items.push(t),o.hasField[e]=t,o.hasSelectionId=o.hasSelectionId||e===hs}m&&(n.init=m.map((e=>o.items.map((n=>t.isObject(e)?void 0!==e[n.geoChannel||n.channel]?e[n.geoChannel||n.channel]:e[n.field]:e))))),S(s)||(o.timeUnit=new vc(null,s))},signals:(e,t,n)=>{const i=t.name+Sc;return n.filter((e=>e.name===i)).length>0||t.project.hasSelectionId?n:n.concat({name:i,value:t.project.items.map(lc)})}},zc={defined:e=>\"interval\"===e.type&&\"global\"===e.resolve&&e.bind&&\"scales\"===e.bind,parse:(e,t)=>{const n=t.scales=[];for(const i of t.project.items){const r=i.channel;if(!Ht(r))continue;const o=e.getScaleComponent(r),a=o?o.get(\"type\"):void 0;\"sequential\"==a&&yi(\"Sequntial scales are deprecated. The available quantitative scale type values are linear, log, pow, sqrt, symlog, time and utc\"),o&&yr(a)?(o.set(\"selectionExtent\",{param:t.name,field:i.field},!0),n.push(i)):yi(\"Scale bindings are currently only supported for scales with unbinned, continuous domains.\")}},topLevelSignals:(e,n,i)=>{const r=n.scales.filter((e=>0===i.filter((t=>t.name===e.signals.data)).length));if(!e.parent||_c(e)||0===r.length)return i;const o=i.filter((e=>e.name===n.name))[0];let a=o.update;if(a.indexOf(Nu)>=0)o.update=`{${r.map((e=>`${t.stringValue(E(e.field))}: ${e.signals.data}`)).join(\", \")}}`;else{for(const e of r){const n=`${t.stringValue(E(e.field))}: ${e.signals.data}`;a.includes(n)||(a=`${a.substring(0,a.length-1)}, ${n}}`)}o.update=a}return i.concat(r.map((e=>({name:e.signals.data}))))},signals:(e,t,n)=>{if(e.parent&&!_c(e))for(const e of t.scales){const t=n.find((t=>t.name===e.signals.data));t.push=\"outer\",delete t.value,delete t.update}return n}};function Oc(e,n){return`domain(${t.stringValue(e.scaleName(n))})`}function _c(e){return e.parent&&vm(e.parent)&&(!e.parent.parent??_c(e.parent.parent))}const Cc=\"_brush\",Nc=\"_scale_trigger\",Pc=\"geo_interval_init_tick\",Ac=\"_init\",jc={defined:e=>\"interval\"===e.type,parse:(e,n,i)=>{if(e.hasProjection){const e={...t.isObject(i.select)?i.select:{}};e.fields=[hs],e.encodings||(e.encodings=i.value?D(i.value):[ue,ce]),i.select={type:\"interval\",...e}}if(n.translate&&!zc.defined(n)){const e=`!event.item || event.item.mark.name !== ${t.stringValue(n.name+Cc)}`;for(const i of n.events){if(!i.between){yi(`${i} is not an ordered event stream for interval selections.`);continue}const n=t.array(i.between[0].filter??=[]);n.indexOf(e)<0&&n.push(e)}}},signals:(e,n,i)=>{const r=n.name,o=r+_u,a=F(n.project.hasChannel).filter((e=>e.channel===Z||e.channel===ee)),s=n.init?n.init[0]:null;if(i.push(...a.reduce(((i,r)=>i.concat(function(e,n,i,r){const o=!e.hasProjection,a=i.channel,s=i.signals.visual,l=t.stringValue(o?e.scaleName(a):e.projectionName()),c=e=>`scale(${l}, ${e})`,u=e.getSizeSignalRef(a===Z?\"width\":\"height\").signal,f=`${a}(unit)`,d=n.events.reduce(((e,t)=>[...e,{events:t.between[0],update:`[${f}, ${f}]`},{events:t,update:`[${s}[0], clamp(${f}, 0, ${u})]`}]),[]);if(o){const t=i.signals.data,o=zc.defined(n),u=e.getScaleComponent(a),f=u?u.get(\"type\"):void 0,m=r?{init:cc(r,!0,c)}:{value:[]};return d.push({events:{signal:n.name+Nc},update:yr(f)?`[${c(`${t}[0]`)}, ${c(`${t}[1]`)}]`:\"[0, 0]\"}),o?[{name:t,on:[]}]:[{name:s,...m,on:d},{name:t,...r?{init:cc(r)}:{},on:[{events:{signal:s},update:`${s}[0] === ${s}[1] ? null : invert(${l}, ${s})`}]}]}{const e=a===Z?0:1,t=n.name+Ac;return[{name:s,...r?{init:`[${t}[0][${e}], ${t}[1][${e}]]`}:{value:[]},on:d}]}}(e,n,r,s&&s[r.index]))),[])),e.hasProjection){const l=t.stringValue(e.projectionName()),c=e.projectionName()+\"_center\",{x:u,y:f}=n.project.hasChannel,d=u&&u.signals.visual,m=f&&f.signals.visual,p=u?s&&s[u.index]:`${c}[0]`,g=f?s&&s[f.index]:`${c}[1]`,h=t=>e.getSizeSignalRef(t).signal,y=`[[${d?d+\"[0]\":\"0\"}, ${m?m+\"[0]\":\"0\"}],[${d?d+\"[1]\":h(\"width\")}, ${m?m+\"[1]\":h(\"height\")}]]`;if(s&&(i.unshift({name:r+Ac,init:`[scale(${l}, [${u?p[0]:p}, ${f?g[0]:g}]), scale(${l}, [${u?p[1]:p}, ${f?g[1]:g}])]`}),!u||!f)){i.find((e=>e.name===c))||i.unshift({name:c,update:`invert(${l}, [${h(\"width\")}/2, ${h(\"height\")}/2])`})}const v=`vlSelectionTuples(${`intersect(${y}, {markname: ${t.stringValue(e.getName(\"marks\"))}}, unit.mark)`}, ${`{unit: ${Au(e)}}`})`,b=a.map((e=>e.signals.visual));return i.concat({name:o,on:[{events:[...b.length?[{signal:b.join(\" || \")}]:[],...s?[{signal:Pc}]:[]],update:v}]})}{if(!zc.defined(n)){const n=r+Nc,o=a.map((n=>{const i=n.channel,{data:r,visual:o}=n.signals,a=t.stringValue(e.scaleName(i)),s=yr(e.getScaleComponent(i).get(\"type\"))?\"+\":\"\";return`(!isArray(${r}) || (${s}invert(${a}, ${o})[0] === ${s}${r}[0] && ${s}invert(${a}, ${o})[1] === ${s}${r}[1]))`}));o.length&&i.push({name:n,value:{},on:[{events:a.map((t=>({scale:e.scaleName(t.channel)}))),update:o.join(\" && \")+` ? ${n} : {}`}]})}const l=a.map((e=>e.signals.data)),c=`unit: ${Au(e)}, fields: ${r+Sc}, values`;return i.concat({name:o,...s?{init:`{${c}: ${cc(s)}}`}:{},...l.length?{on:[{events:[{signal:l.join(\" || \")}],update:`${l.join(\" && \")} ? {${c}: [${l}]} : null`}]}:{}})}},topLevelSignals:(e,t,n)=>{if(gm(e)&&e.hasProjection&&t.init){n.filter((e=>e.name===Pc)).length||n.unshift({name:Pc,value:null,on:[{events:\"timer{1}\",update:`${Pc} === null ? {} : ${Pc}`}]})}return n},marks:(e,n,i)=>{const r=n.name,{x:o,y:a}=n.project.hasChannel,s=o?.signals.visual,l=a?.signals.visual,c=`data(${t.stringValue(n.name+Ou)})`;if(zc.defined(n)||!o&&!a)return i;const u={x:void 0!==o?{signal:`${s}[0]`}:{value:0},y:void 0!==a?{signal:`${l}[0]`}:{value:0},x2:void 0!==o?{signal:`${s}[1]`}:{field:{group:\"width\"}},y2:void 0!==a?{signal:`${l}[1]`}:{field:{group:\"height\"}}};if(\"global\"===n.resolve)for(const t of D(u))u[t]=[{test:`${c}.length && ${c}[0].unit === ${Au(e)}`,...u[t]},{value:0}];const{fill:f,fillOpacity:d,cursor:m,...p}=n.mark,g=D(p).reduce(((e,t)=>(e[t]=[{test:[void 0!==o&&`${s}[0] !== ${s}[1]`,void 0!==a&&`${l}[0] !== ${l}[1]`].filter((e=>e)).join(\" && \"),value:p[t]},{value:null}],e)),{}),h=m??(n.translate?\"move\":null);return[{name:`${r+Cc}_bg`,type:\"rect\",clip:!0,encode:{enter:{fill:{value:f},fillOpacity:{value:d}},update:u}},...i,{name:r+Cc,type:\"rect\",clip:!0,encode:{enter:{...h?{cursor:{value:h}}:{},fill:{value:\"transparent\"}},update:{...u,...g}}}]}};const Tc={defined:e=>\"point\"===e.type,signals:(e,n,i)=>{const r=n.name,o=r+Sc,a=n.project,s=\"(item().isVoronoi ? datum.datum : datum)\",l=F(e.component.selection??{}).reduce(((e,t)=>\"interval\"===t.type?e.concat(t.name+Cc):e),[]).map((e=>`indexof(item().mark.name, '${e}') < 0`)).join(\" && \"),c=\"datum && item().mark.marktype !== 'group' && indexof(item().mark.role, 'legend') < 0\"+(l?` && ${l}`:\"\");let u=`unit: ${Au(e)}, `;if(n.project.hasSelectionId)u+=`${hs}: ${s}[${t.stringValue(hs)}]`;else{u+=`fields: ${o}, values: [${a.items.map((n=>{const i=e.fieldDef(n.channel);return i?.bin?`[${s}[${t.stringValue(e.vgField(n.channel,{}))}], ${s}[${t.stringValue(e.vgField(n.channel,{binSuffix:\"end\"}))}]]`:`${s}[${t.stringValue(n.field)}]`})).join(\", \")}]`}const f=n.events;return i.concat([{name:r+_u,on:f?[{events:f,update:`${c} ? {${u}} : null`,force:!0}]:[]}])}};function Ec(e,n,i,r){const o=Lo(n)&&n.condition,a=r(n);if(o){const n=t.array(o).map((t=>{const n=r(t);if(function(e){return e.param}(t)){const{param:i,empty:r}=t;return{test:Uu(e,{param:i,empty:r}),...n}}return{test:Wu(e,t.test),...n}}));return{[i]:[...n,...void 0!==a?[a]:[]]}}return void 0!==a?{[i]:a}:{}}function Mc(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:\"text\";const n=e.encoding[t];return Ec(e,n,t,(t=>Lc(t,e.config)))}function Lc(e,t){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:\"datum\";if(e){if(Xo(e))return Fn(e.value);if(Go(e)){const{format:i,formatType:r}=ca(e);return Rr({fieldOrDatumDef:e,format:i,formatType:r,expr:n,config:t})}}}function qc(e){let n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};const{encoding:i,markDef:r,config:o,stack:a}=e,s=i.tooltip;if(t.isArray(s))return{tooltip:Rc({tooltip:s},a,o,n)};{const l=n.reactiveGeom?\"datum.datum\":\"datum\";return Ec(e,s,\"tooltip\",(e=>{const s=Lc(e,o,l);if(s)return s;if(null===e)return;let c=Nn(\"tooltip\",r,o);return!0===c&&(c={content:\"encoding\"}),t.isString(c)?{value:c}:t.isObject(c)?yn(c)?c:\"encoding\"===c.content?Rc(i,a,o,n):{signal:l}:void 0}))}}function Uc(e,n,i){let{reactiveGeom:r}=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{};const o={...i,...i.tooltipFormat},a={},s=r?\"datum.datum\":\"datum\",l=[];function c(i,r){const c=tt(r),u=Yo(i)?i:{...i,type:e[c].type},f=u.title||la(u,o),d=t.array(f).join(\", \").replaceAll(/\"/g,'\\\\\"');let m;if(zt(r)){const t=\"x\"===r?\"x2\":\"y2\",n=ua(e[t]);if(cn(u.bin)&&n){const e=ta(u,{expr:s}),i=ta(n,{expr:s}),{format:r,formatType:l}=ca(u);m=Xr(e,i,r,l,o),a[t]=!0}}if((zt(r)||r===se||r===oe)&&n&&n.fieldChannel===r&&\"normalize\"===n.offset){const{format:e,formatType:t}=ca(u);m=Rr({fieldOrDatumDef:u,format:e,formatType:t,expr:s,config:o,normalizeStack:!0}).signal}m??=Lc(u,o,s).signal,l.push({channel:r,key:d,value:m})}La(e,((e,t)=>{Ro(e)?c(e,t):qo(e)&&c(e.condition,t)}));const u={};for(const{channel:e,key:t,value:n}of l)a[e]||u[t]||(u[t]=n);return u}function Rc(e,t,n){let{reactiveGeom:i}=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{};const r=Uc(e,t,n,{reactiveGeom:i}),o=z(r).map((e=>{let[t,n]=e;return`\"${t}\": ${n}`}));return o.length>0?{signal:`{${o.join(\", \")}}`}:void 0}function Wc(e){const{markDef:t,config:n}=e,i=Nn(\"aria\",t,n);return!1===i?{}:{...i?{aria:i}:{},...Bc(e),...Ic(e)}}function Bc(e){const{mark:t,markDef:n,config:i}=e;if(!1===i.aria)return{};const r=Nn(\"ariaRoleDescription\",n,i);return null!=r?{ariaRoleDescription:{value:r}}:t in $n?{}:{ariaRoleDescription:{value:t}}}function Ic(e){const{encoding:t,markDef:n,config:i,stack:r}=e,o=t.description;if(o)return Ec(e,o,\"description\",(t=>Lc(t,e.config)));const a=Nn(\"description\",n,i);if(null!=a)return{description:Fn(a)};if(!1===i.aria)return{};const s=Uc(t,r,i);return S(s)?void 0:{description:{signal:z(s).map(((e,t)=>{let[n,i]=e;return`\"${t>0?\"; \":\"\"}${n}: \" + (${i})`})).join(\" + \")}}}function Hc(e,t){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{};const{markDef:i,encoding:r,config:o}=t,{vgChannel:a}=n;let{defaultRef:s,defaultValue:l}=n;void 0===s&&(l??=Nn(e,i,o,{vgChannel:a,ignoreVgConfig:!0}),void 0!==l&&(s=Fn(l)));const c=r[e];return Ec(t,c,a??e,(n=>Er({channel:e,channelDef:n,markDef:i,config:o,scaleName:t.scaleName(e),scale:t.getScaleComponent(e),stack:null,defaultRef:s})))}function Vc(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{filled:void 0};const{markDef:n,encoding:i,config:r}=e,{type:o}=n,a=t.filled??Nn(\"filled\",n,r),s=p([\"bar\",\"point\",\"circle\",\"square\",\"geoshape\"],o)?\"transparent\":void 0,l=Nn(!0===a?\"color\":void 0,n,r,{vgChannel:\"fill\"})??r.mark[!0===a&&\"color\"]??s,c=Nn(!1===a?\"color\":void 0,n,r,{vgChannel:\"stroke\"})??r.mark[!1===a&&\"color\"],u=a?\"fill\":\"stroke\",f={...l?{fill:Fn(l)}:{},...c?{stroke:Fn(c)}:{}};return n.color&&(a?n.fill:n.stroke)&&yi(ei(\"property\",{fill:\"fill\"in n,stroke:\"stroke\"in n})),{...f,...Hc(\"color\",e,{vgChannel:u,defaultValue:a?l:c}),...Hc(\"fill\",e,{defaultValue:i.fill?l:void 0}),...Hc(\"stroke\",e,{defaultValue:i.stroke?c:void 0})}}function Gc(e){const{encoding:t,mark:n}=e,i=t.order;return!fo(n)&&Xo(i)?Ec(e,i,\"zindex\",(e=>Fn(e.value))):{}}function Yc(e){let{channel:t,markDef:n,encoding:i={},model:r,bandPosition:o}=e;const a=`${t}Offset`,s=n[a],l=i[a];if((\"xOffset\"===a||\"yOffset\"===a)&&l){return{offsetType:\"encoding\",offset:Er({channel:a,channelDef:l,markDef:n,config:r?.config,scaleName:r.scaleName(a),scale:r.getScaleComponent(a),stack:null,defaultRef:Fn(s),bandPosition:o})}}const c=n[a];return c?{offsetType:\"visual\",offset:c}:{}}function Xc(e,t,n){let{defaultPos:i,vgChannel:r}=n;const{encoding:o,markDef:a,config:s,stack:l}=t,c=o[e],u=o[it(e)],f=t.scaleName(e),d=t.getScaleComponent(e),{offset:m,offsetType:p}=Yc({channel:e,markDef:a,encoding:o,model:t,bandPosition:.5}),g=Qc({model:t,defaultPos:i,channel:e,scaleName:f,scale:d}),h=!c&&zt(e)&&(o.latitude||o.longitude)?{field:t.getName(e)}:function(e){const{channel:t,channelDef:n,scaleName:i,stack:r,offset:o,markDef:a}=e;if(Go(n)&&r&&t===r.fieldChannel){if(Ro(n)){let e=n.bandPosition;if(void 0!==e||\"text\"!==a.type||\"radius\"!==t&&\"theta\"!==t||(e=.5),void 0!==e)return Tr({scaleName:i,fieldOrDatumDef:n,startSuffix:\"start\",bandPosition:e,offset:o})}return jr(n,i,{suffix:\"end\"},{offset:o})}return Nr(e)}({channel:e,channelDef:c,channel2Def:u,markDef:a,config:s,scaleName:f,scale:d,stack:l,offset:m,defaultRef:g,bandPosition:\"encoding\"===p?0:void 0});return h?{[r||e]:h}:void 0}function Qc(e){let{model:t,defaultPos:n,channel:i,scaleName:r,scale:o}=e;const{markDef:a,config:s}=t;return()=>{const e=tt(i),l=nt(i),c=Nn(i,a,s,{vgChannel:l});if(void 0!==c)return Mr(i,c);switch(n){case\"zeroOrMin\":case\"zeroOrMax\":if(r){const e=o.get(\"type\");if(p([or.LOG,or.TIME,or.UTC],e));else if(o.domainDefinitelyIncludesZero())return{scale:r,value:0}}if(\"zeroOrMin\"===n)return\"y\"===e?{field:{group:\"height\"}}:{value:0};switch(e){case\"radius\":return{signal:`min(${t.width.signal},${t.height.signal})/2`};case\"theta\":return{signal:\"2*PI\"};case\"x\":return{field:{group:\"width\"}};case\"y\":return{value:0}}break;case\"mid\":return{...t[rt(i)],mult:.5}}}}const Jc={left:\"x\",center:\"xc\",right:\"x2\"},Kc={top:\"y\",middle:\"yc\",bottom:\"y2\"};function Zc(e,t,n){let i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:\"middle\";if(\"radius\"===e||\"theta\"===e)return nt(e);const r=\"x\"===e?\"align\":\"baseline\",o=Nn(r,t,n);let a;return yn(o)?(yi(function(e){return`The ${e} for range marks cannot be an expression`}(r)),a=void 0):a=o,\"x\"===e?Jc[a||(\"top\"===i?\"left\":\"center\")]:Kc[a||i]}function eu(e,t,n){let{defaultPos:i,defaultPos2:r,range:o}=n;return o?tu(e,t,{defaultPos:i,defaultPos2:r}):Xc(e,t,{defaultPos:i})}function tu(e,t,n){let{defaultPos:i,defaultPos2:r}=n;const{markDef:o,config:a}=t,s=it(e),l=rt(e),c=function(e,t,n){const{encoding:i,mark:r,markDef:o,stack:a,config:s}=e,l=tt(n),c=rt(n),u=nt(n),f=i[l],d=e.scaleName(l),m=e.getScaleComponent(l),{offset:p}=Yc(n in i||n in o?{channel:n,markDef:o,encoding:i,model:e}:{channel:l,markDef:o,encoding:i,model:e});if(!f&&(\"x2\"===n||\"y2\"===n)&&(i.latitude||i.longitude)){const t=rt(n),i=e.markDef[t];return null!=i?{[t]:{value:i}}:{[u]:{field:e.getName(n)}}}const g=function(e){let{channel:t,channelDef:n,channel2Def:i,markDef:r,config:o,scaleName:a,scale:s,stack:l,offset:c,defaultRef:u}=e;if(Go(n)&&l&&t.charAt(0)===l.fieldChannel.charAt(0))return jr(n,a,{suffix:\"start\"},{offset:c});return Nr({channel:t,channelDef:i,scaleName:a,scale:s,stack:l,markDef:r,config:o,offset:c,defaultRef:u})}({channel:n,channelDef:f,channel2Def:i[n],markDef:o,config:s,scaleName:d,scale:m,stack:a,offset:p,defaultRef:void 0});if(void 0!==g)return{[u]:g};return nu(n,o)||nu(n,{[n]:An(n,o,s.style),[c]:An(c,o,s.style)})||nu(n,s[r])||nu(n,s.mark)||{[u]:Qc({model:e,defaultPos:t,channel:n,scaleName:d,scale:m})()}}(t,r,s);return{...Xc(e,t,{defaultPos:i,vgChannel:c[l]?Zc(e,o,a):nt(e)}),...c}}function nu(e,t){const n=rt(e),i=nt(e);if(void 0!==t[i])return{[i]:Mr(e,t[i])};if(void 0!==t[e])return{[i]:Mr(e,t[e])};if(t[n]){const i=t[n];if(!bo(i))return{[n]:Mr(e,i)};yi(function(e){return`Position range does not support relative band size for ${e}.`}(n))}}function iu(e,n){const{config:i,encoding:r,markDef:o}=e,a=o.type,s=it(n),l=rt(n),c=r[n],u=r[s],f=e.getScaleComponent(n),d=f?f.get(\"type\"):void 0,m=o.orient,p=r[l]??r.size??Nn(\"size\",o,i,{vgChannel:l}),g=ot(n),h=\"bar\"===a&&(\"x\"===n?\"vertical\"===m:\"horizontal\"===m);return!Ro(c)||!(ln(c.bin)||cn(c.bin)||c.timeUnit&&!u)||p&&!bo(p)||r[g]||hr(d)?(Go(c)&&hr(d)||h)&&!u?function(e,n,i){const{markDef:r,encoding:o,config:a,stack:s}=i,l=r.orient,c=i.scaleName(n),u=i.getScaleComponent(n),f=rt(n),d=it(n),m=ot(n),p=i.scaleName(m),g=i.getScaleComponent(at(n)),h=\"horizontal\"===l&&\"y\"===n||\"vertical\"===l&&\"x\"===n;let y;(o.size||r.size)&&(h?y=Hc(\"size\",i,{vgChannel:f,defaultRef:Fn(r.size)}):yi(function(e){return`Cannot apply size to non-oriented mark \"${e}\".`}(r.type)));const v=!!y,b=To({channel:n,fieldDef:e,markDef:r,config:a,scaleType:(u||g)?.get(\"type\"),useVlSizeChannel:h});y=y||{[f]:ru(f,p||c,g||u,a,b,!!e,r.type)};const x=\"band\"===(u||g)?.get(\"type\")&&bo(b)&&!v?\"top\":\"middle\",$=Zc(n,r,a,x),w=\"xc\"===$||\"yc\"===$,{offset:k,offsetType:S}=Yc({channel:n,markDef:r,encoding:o,model:i,bandPosition:w?.5:0}),D=Nr({channel:n,channelDef:e,markDef:r,config:a,scaleName:c,scale:u,stack:s,offset:k,defaultRef:Qc({model:i,defaultPos:\"mid\",channel:n,scaleName:c,scale:u}),bandPosition:w?\"encoding\"===S?0:.5:yn(b)?{signal:`(1-${b})/2`}:bo(b)?(1-b.band)/2:0});if(f)return{[$]:D,...y};{const e=nt(d),n=y[f],i=k?{...n,offset:k}:n;return{[$]:D,[e]:t.isArray(D)?[D[0],{...D[1],offset:i}]:{...D,offset:i}}}}(c,n,e):tu(n,e,{defaultPos:\"zeroOrMax\",defaultPos2:\"zeroOrMin\"}):function(e){let{fieldDef:t,fieldDef2:n,channel:i,model:r}=e;const{config:o,markDef:a,encoding:s}=r,l=r.getScaleComponent(i),c=r.scaleName(i),u=l?l.get(\"type\"):void 0,f=l.get(\"reverse\"),d=To({channel:i,fieldDef:t,markDef:a,config:o,scaleType:u}),m=r.component.axes[i]?.[0],p=m?.get(\"translate\")??.5,g=zt(i)?Nn(\"binSpacing\",a,o)??0:0,h=it(i),y=nt(i),v=nt(h),b=Pn(\"minBandSize\",a,o),{offset:x}=Yc({channel:i,markDef:a,encoding:s,model:r,bandPosition:0}),{offset:$}=Yc({channel:h,markDef:a,encoding:s,model:r,bandPosition:0}),w=function(e){let{scaleName:t,fieldDef:n}=e;const i=ta(n,{expr:\"datum\"});return`abs(scale(\"${t}\", ${ta(n,{expr:\"datum\",suffix:\"end\"})}) - scale(\"${t}\", ${i}))`}({fieldDef:t,scaleName:c}),k=ou(i,g,f,p,x,b,w),S=ou(h,g,f,p,$??x,b,w),D=yn(d)?{signal:`(1-${d.signal})/2`}:bo(d)?(1-d.band)/2:.5,F=jo({fieldDef:t,fieldDef2:n,markDef:a,config:o});if(ln(t.bin)||t.timeUnit){const e=t.timeUnit&&.5!==F;return{[v]:au({fieldDef:t,scaleName:c,bandPosition:D,offset:S,useRectOffsetField:e}),[y]:au({fieldDef:t,scaleName:c,bandPosition:yn(D)?{signal:`1-${D.signal}`}:1-D,offset:k,useRectOffsetField:e})}}if(cn(t.bin)){const e=jr(t,c,{},{offset:S});if(Ro(n))return{[v]:e,[y]:jr(n,c,{},{offset:k})};if(un(t.bin)&&t.bin.step)return{[v]:e,[y]:{signal:`scale(\"${c}\", ${ta(t,{expr:\"datum\"})} + ${t.bin.step})`,offset:k}}}return void yi(pi(h))}({fieldDef:c,fieldDef2:u,channel:n,model:e})}function ru(e,n,i,r,o,a,s){if(bo(o)){if(!i)return{mult:o.band,field:{group:e}};{const e=i.get(\"type\");if(\"band\"===e){let e=`bandwidth('${n}')`;1!==o.band&&(e=`${o.band} * ${e}`);const t=Pn(\"minBandSize\",{type:s},r);return{signal:t?`max(${On(t)}, ${e})`:e}}1!==o.band&&(yi(function(e){return`Cannot use the relative band size with ${e} scale.`}(e)),o=void 0)}}else{if(yn(o))return o;if(o)return{value:o}}if(i){const e=i.get(\"range\");if(vn(e)&&t.isNumber(e.step))return{value:e.step-2}}if(!a){const{bandPaddingInner:n,barBandPaddingInner:i,rectBandPaddingInner:o}=r.scale,a=U(n,\"bar\"===s?i:o);if(yn(a))return{signal:`(1 - (${a.signal})) * ${e}`};if(t.isNumber(a))return{signal:`${1-a} * ${e}`}}return{value:Cs(r.view,e)-2}}function ou(e,t,n,i,r,o,a){if(Ae(e))return 0;const s=\"x\"===e||\"y2\"===e,l=s?-t/2:t/2;if(yn(n)||yn(r)||yn(i)||o){const e=On(n),t=On(r),c=On(i),u=On(o),f=o?`(${a} < ${u} ? ${s?\"\":\"-\"}0.5 * (${u} - (${a})) : ${l})`:l;return{signal:(c?`${c} + `:\"\")+(e?`(${e} ? -1 : 1) * `:\"\")+(t?`(${t} + ${f})`:f)}}return r=r||0,i+(n?-r-l:+r+l)}function au(e){let{fieldDef:t,scaleName:n,bandPosition:i,offset:r,useRectOffsetField:o}=e;return Tr({scaleName:n,fieldOrDatumDef:t,bandPosition:i,offset:r,...o?{startSuffix:bc,endSuffix:xc}:{}})}const su=new Set([\"aria\",\"width\",\"height\"]);function lu(e,t){const{fill:n,stroke:i}=\"include\"===t.color?Vc(e):{};return{...uu(e.markDef,t),...cu(e,\"fill\",n),...cu(e,\"stroke\",i),...Hc(\"opacity\",e),...Hc(\"fillOpacity\",e),...Hc(\"strokeOpacity\",e),...Hc(\"strokeWidth\",e),...Hc(\"strokeDash\",e),...Gc(e),...qc(e),...Mc(e,\"href\"),...Wc(e)}}function cu(e,n,i){const{config:r,mark:o,markDef:a}=e;if(\"hide\"===Nn(\"invalid\",a,r)&&i&&!fo(o)){const r=function(e,t){let{invalid:n=!1,channels:i}=t;const r=i.reduce(((t,n)=>{const i=e.getScaleComponent(n);if(i){const r=i.get(\"type\"),o=e.vgField(n,{expr:\"datum\"});o&&yr(r)&&(t[o]=!0)}return t}),{}),o=D(r);if(o.length>0){const e=n?\"||\":\"&&\";return o.map((e=>Ar(e,n))).join(` ${e} `)}return}(e,{invalid:!0,channels:It});if(r)return{[n]:[{test:r,value:null},...t.array(i)]}}return i?{[n]:i}:{}}function uu(e,t){return xn.reduce(((n,i)=>(su.has(i)||void 0===e[i]||\"ignore\"===t[i]||(n[i]=Fn(e[i])),n)),{})}function fu(e){const{config:t,markDef:n}=e;if(Nn(\"invalid\",n,t)){const t=function(e,t){let{invalid:n=!1,channels:i}=t;const r=i.reduce(((t,n)=>{const i=e.getScaleComponent(n);if(i){const r=i.get(\"type\"),o=e.vgField(n,{expr:\"datum\",binSuffix:e.stack?.impute?\"mid\":void 0});o&&yr(r)&&(t[o]=!0)}return t}),{}),o=D(r);if(o.length>0){const e=n?\"||\":\"&&\";return o.map((e=>Ar(e,n))).join(` ${e} `)}return}(e,{channels:Ft});if(t)return{defined:{signal:t}}}return{}}function du(e,t){if(void 0!==t)return{[e]:Fn(t)}}const mu=\"voronoi\",pu={defined:e=>\"point\"===e.type&&e.nearest,parse:(e,t)=>{if(t.events)for(const n of t.events)n.markname=e.getName(mu)},marks:(e,t,n)=>{const{x:i,y:r}=t.project.hasChannel,o=e.mark;if(fo(o))return yi(`The \"nearest\" transform is not supported for ${o} marks.`),n;const a={name:e.getName(mu),type:\"path\",interactive:!0,from:{data:e.getName(\"marks\")},encode:{update:{fill:{value:\"transparent\"},strokeWidth:{value:.35},stroke:{value:\"transparent\"},isVoronoi:{value:!0},...qc(e,{reactiveGeom:!0})}},transform:[{type:\"voronoi\",x:{expr:i||!r?\"datum.datum.x || 0\":\"0\"},y:{expr:r||!i?\"datum.datum.y || 0\":\"0\"},size:[e.getSizeSignalRef(\"width\"),e.getSizeSignalRef(\"height\")]}]};let s=0,l=!1;return n.forEach(((t,n)=>{const i=t.name??\"\";i===e.component.mark[0].name?s=n:i.indexOf(mu)>=0&&(l=!0)})),l||n.splice(s+1,0,a),n}},gu={defined:e=>\"point\"===e.type&&\"global\"===e.resolve&&e.bind&&\"scales\"!==e.bind&&!vs(e.bind),parse:(e,t,n)=>Tu(t,n),topLevelSignals:(e,n,i)=>{const r=n.name,o=n.project,a=n.bind,s=n.init&&n.init[0],l=pu.defined(n)?\"(item().isVoronoi ? datum.datum : datum)\":\"datum\";return o.items.forEach(((e,o)=>{const c=_(`${r}_${e.field}`);i.filter((e=>e.name===c)).length||i.unshift({name:c,...s?{init:cc(s[o])}:{value:null},on:n.events?[{events:n.events,update:`datum && item().mark.marktype !== 'group' ? ${l}[${t.stringValue(e.field)}] : null`}]:[],bind:a[e.field]??a[e.channel]??a})})),i},signals:(e,t,n)=>{const i=t.name,r=t.project,o=n.filter((e=>e.name===i+_u))[0],a=i+Sc,s=r.items.map((e=>_(`${i}_${e.field}`))),l=s.map((e=>`${e} !== null`)).join(\" && \");return s.length&&(o.update=`${l} ? {fields: ${a}, values: [${s.join(\", \")}]} : null`),delete o.value,delete o.on,n}},hu=\"_toggle\",yu={defined:e=>\"point\"===e.type&&!!e.toggle,signals:(e,t,n)=>n.concat({name:t.name+hu,value:!1,on:[{events:t.events,update:t.toggle}]}),modifyExpr:(e,t)=>{const n=t.name+_u,i=t.name+hu;return`${i} ? null : ${n}, `+(\"global\"===t.resolve?`${i} ? null : true, `:`${i} ? null : {unit: ${Au(e)}}, `)+`${i} ? ${n} : null`}},vu={defined:e=>void 0!==e.clear&&!1!==e.clear,parse:(e,n)=>{n.clear&&(n.clear=t.isString(n.clear)?t.parseSelector(n.clear,\"view\"):n.clear)},topLevelSignals:(e,t,n)=>{if(gu.defined(t))for(const e of t.project.items){const i=n.findIndex((n=>n.name===_(`${t.name}_${e.field}`)));-1!==i&&n[i].on.push({events:t.clear,update:\"null\"})}return n},signals:(e,t,n)=>{function i(e,i){-1!==e&&n[e].on&&n[e].on.push({events:t.clear,update:i})}if(\"interval\"===t.type)for(const e of t.project.items){const t=n.findIndex((t=>t.name===e.signals.visual));if(i(t,\"[0, 0]\"),-1===t){i(n.findIndex((t=>t.name===e.signals.data)),\"null\")}}else{let e=n.findIndex((e=>e.name===t.name+_u));i(e,\"null\"),yu.defined(t)&&(e=n.findIndex((e=>e.name===t.name+hu)),i(e,\"false\"))}return n}},bu={defined:e=>{const t=\"global\"===e.resolve&&e.bind&&vs(e.bind),n=1===e.project.items.length&&e.project.items[0].field!==hs;return t&&!n&&yi(\"Legend bindings are only supported for selections over an individual field or encoding channel.\"),t&&n},parse:(e,n,i)=>{const r=l(i);if(r.select=t.isString(r.select)?{type:r.select,toggle:n.toggle}:{...r.select,toggle:n.toggle},Tu(n,r),t.isObject(i.select)&&(i.select.on||i.select.clear)){const e='event.item && indexof(event.item.mark.role, \"legend\") < 0';for(const i of n.events)i.filter=t.array(i.filter??[]),i.filter.includes(e)||i.filter.push(e)}const o=bs(n.bind)?n.bind.legend:\"click\",a=t.isString(o)?t.parseSelector(o,\"view\"):t.array(o);n.bind={legend:{merge:a}}},topLevelSignals:(e,t,n)=>{const i=t.name,r=bs(t.bind)&&t.bind.legend,o=e=>t=>{const n=l(t);return n.markname=e,n};for(const e of t.project.items){if(!e.hasLegend)continue;const a=`${_(e.field)}_legend`,s=`${i}_${a}`;if(0===n.filter((e=>e.name===s)).length){const e=r.merge.map(o(`${a}_symbols`)).concat(r.merge.map(o(`${a}_labels`))).concat(r.merge.map(o(`${a}_entries`)));n.unshift({name:s,...t.init?{}:{value:null},on:[{events:e,update:\"isDefined(datum.value) ? datum.value : item().items[0].items[0].datum.value\",force:!0},{events:r.merge,update:`!event.item || !datum ? null : ${s}`,force:!0}]})}}return n},signals:(e,t,n)=>{const i=t.name,r=t.project,o=n.find((e=>e.name===i+_u)),a=i+Sc,s=r.items.filter((e=>e.hasLegend)).map((e=>_(`${i}_${_(e.field)}_legend`))),l=`${s.map((e=>`${e} !== null`)).join(\" && \")} ? {fields: ${a}, values: [${s.join(\", \")}]} : null`;t.events&&s.length>0?o.on.push({events:s.map((e=>({signal:e}))),update:l}):s.length>0&&(o.update=l,delete o.value,delete o.on);const c=n.find((e=>e.name===i+hu)),u=bs(t.bind)&&t.bind.legend;return c&&(t.events?c.on.push({...c.on[0],events:u}):c.on[0].events=u),n}};const xu=\"_translate_anchor\",$u=\"_translate_delta\",wu={defined:e=>\"interval\"===e.type&&e.translate,signals:(e,n,i)=>{const r=n.name,o=zc.defined(n),a=r+xu,{x:s,y:l}=n.project.hasChannel;let c=t.parseSelector(n.translate,\"scope\");return o||(c=c.map((e=>(e.between[0].markname=r+Cc,e)))),i.push({name:a,value:{},on:[{events:c.map((e=>e.between[0])),update:\"{x: x(unit), y: y(unit)\"+(void 0!==s?`, extent_x: ${o?Oc(e,Z):`slice(${s.signals.visual})`}`:\"\")+(void 0!==l?`, extent_y: ${o?Oc(e,ee):`slice(${l.signals.visual})`}`:\"\")+\"}\"}]},{name:r+$u,value:{},on:[{events:c,update:`{x: ${a}.x - x(unit), y: ${a}.y - y(unit)}`}]}),void 0!==s&&ku(e,n,s,\"width\",i),void 0!==l&&ku(e,n,l,\"height\",i),i}};function ku(e,t,n,i,r){const o=t.name,a=o+xu,s=o+$u,l=n.channel,c=zc.defined(t),u=r.filter((e=>e.name===n.signals[c?\"data\":\"visual\"]))[0],f=e.getSizeSignalRef(i).signal,d=e.getScaleComponent(l),m=d&&d.get(\"type\"),p=d&&d.get(\"reverse\"),g=`${a}.extent_${l}`,h=`${c&&d?\"log\"===m?\"panLog\":\"symlog\"===m?\"panSymlog\":\"pow\"===m?\"panPow\":\"panLinear\":\"panLinear\"}(${g}, ${`${c?l===Z?p?\"\":\"-\":p?\"-\":\"\":\"\"}${s}.${l} / ${c?`${f}`:`span(${g})`}`}${c?\"pow\"===m?`, ${d.get(\"exponent\")??1}`:\"symlog\"===m?`, ${d.get(\"constant\")??1}`:\"\":\"\"})`;u.on.push({events:{signal:s},update:c?h:`clampRange(${h}, 0, ${f})`})}const Su=\"_zoom_anchor\",Du=\"_zoom_delta\",Fu={defined:e=>\"interval\"===e.type&&e.zoom,signals:(e,n,i)=>{const r=n.name,o=zc.defined(n),a=r+Du,{x:s,y:l}=n.project.hasChannel,c=t.stringValue(e.scaleName(Z)),u=t.stringValue(e.scaleName(ee));let f=t.parseSelector(n.zoom,\"scope\");return o||(f=f.map((e=>(e.markname=r+Cc,e)))),i.push({name:r+Su,on:[{events:f,update:o?\"{\"+[c?`x: invert(${c}, x(unit))`:\"\",u?`y: invert(${u}, y(unit))`:\"\"].filter((e=>e)).join(\", \")+\"}\":\"{x: x(unit), y: y(unit)}\"}]},{name:a,on:[{events:f,force:!0,update:\"pow(1.001, event.deltaY * pow(16, event.deltaMode))\"}]}),void 0!==s&&zu(e,n,s,\"width\",i),void 0!==l&&zu(e,n,l,\"height\",i),i}};function zu(e,t,n,i,r){const o=t.name,a=n.channel,s=zc.defined(t),l=r.filter((e=>e.name===n.signals[s?\"data\":\"visual\"]))[0],c=e.getSizeSignalRef(i).signal,u=e.getScaleComponent(a),f=u&&u.get(\"type\"),d=s?Oc(e,a):l.name,m=o+Du,p=`${s&&u?\"log\"===f?\"zoomLog\":\"symlog\"===f?\"zoomSymlog\":\"pow\"===f?\"zoomPow\":\"zoomLinear\":\"zoomLinear\"}(${d}, ${`${o}${Su}.${a}`}, ${m}${s?\"pow\"===f?`, ${u.get(\"exponent\")??1}`:\"symlog\"===f?`, ${u.get(\"constant\")??1}`:\"\":\"\"})`;l.on.push({events:{signal:m},update:s?p:`clampRange(${p}, 0, ${c})`})}const Ou=\"_store\",_u=\"_tuple\",Cu=\"_modify\",Nu=\"vlSelectionResolve\",Pu=[Tc,jc,Fc,yu,gu,zc,bu,vu,wu,Fu,pu];function Au(e){let{escape:n}=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{escape:!0},i=n?t.stringValue(e.name):e.name;const r=function(e){let t=e.parent;for(;t&&!hm(t);)t=t.parent;return t}(e);if(r){const{facet:e}=r;for(const n of Re)e[n]&&(i+=` + '__facet_${n}_' + (facet[${t.stringValue(r.vgField(n))}])`)}return i}function ju(e){return F(e.component.selection??{}).reduce(((e,t)=>e||t.project.hasSelectionId),!1)}function Tu(e,n){!t.isString(n.select)&&n.select.on||delete e.events,!t.isString(n.select)&&n.select.clear||delete e.clear,!t.isString(n.select)&&n.select.toggle||delete e.toggle}function Eu(e){const t=[];return\"Identifier\"===e.type?[e.name]:\"Literal\"===e.type?[e.value]:(\"MemberExpression\"===e.type&&(t.push(...Eu(e.object)),t.push(...Eu(e.property))),t)}function Mu(e){return\"MemberExpression\"===e.object.type?Mu(e.object):\"datum\"===e.object.name}function Lu(e){const n=t.parseExpression(e),i=new Set;return n.visit((e=>{\"MemberExpression\"===e.type&&Mu(e)&&i.add(Eu(e).slice(1).join(\".\"))})),i}class qu extends pc{clone(){return new qu(null,this.model,l(this.filter))}constructor(e,t,n){super(e),this.model=t,this.filter=n,this.expr=Wu(this.model,this.filter,this),this._dependentFields=Lu(this.expr)}dependentFields(){return this._dependentFields}producedFields(){return new Set}assemble(){return{type:\"filter\",expr:this.expr}}hash(){return`Filter ${this.expr}`}}function Uu(e,n,i){let r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:\"datum\";const o=t.isString(n)?n:n.param,a=_(o),s=t.stringValue(a+Ou);let l;try{l=e.getSelectionComponent(a,o)}catch(e){return`!!${a}`}if(l.project.timeUnit){const t=i??e.component.data.raw,n=l.project.timeUnit.clone();t.parent?n.insertAsParentOf(t):t.parent=n}const c=`${l.project.hasSelectionId?\"vlSelectionIdTest(\":\"vlSelectionTest(\"}${s}, ${r}${\"global\"===l.resolve?\")\":`, ${t.stringValue(l.resolve)})`}`,u=`length(data(${s}))`;return!1===n.empty?`${u} && ${c}`:`!${u} || ${c}`}function Ru(e,n,i){const r=_(n),o=i.encoding;let a,s=i.field;try{a=e.getSelectionComponent(r,n)}catch(e){return r}if(o||s){if(o&&!s){const e=a.project.items.filter((e=>e.channel===o));!e.length||e.length>1?(s=a.project.items[0].field,yi((e.length?\"Multiple \":\"No \")+`matching ${t.stringValue(o)} encoding found for selection ${t.stringValue(i.param)}. `+`Using \"field\": ${t.stringValue(s)}.`)):s=e[0].field}}else s=a.project.items[0].field,a.project.items.length>1&&yi(`A \"field\" or \"encoding\" must be specified when using a selection as a scale domain. Using \"field\": ${t.stringValue(s)}.`);return`${a.name}[${t.stringValue(E(s))}]`}function Wu(e,n,i){return C(n,(n=>t.isString(n)?n:function(e){return e?.param}(n)?Uu(e,n,i):Xi(n)))}function Bu(e,t,n,i){e.encode??={},e.encode[t]??={},e.encode[t].update??={},e.encode[t].update[n]=i}function Iu(e,n,i){let r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{header:!1};const{disable:o,orient:a,scale:s,labelExpr:l,title:c,zindex:u,...f}=e.combine();if(!o){for(const e in f){const i=Sa[e],r=f[e];if(i&&i!==n&&\"both\"!==i)delete f[e];else if(wa(r)){const{condition:n,...i}=r,o=t.array(n),a=$a[e];if(a){const{vgProp:t,part:n}=a;Bu(f,n,t,[...o.map((e=>{const{test:t,...n}=e;return{test:Wu(null,t),...n}})),i]),delete f[e]}else if(null===a){const t={signal:o.map((e=>{const{test:t,...n}=e;return`${Wu(null,t)} ? ${zn(n)} : `})).join(\"\")+zn(i)};f[e]=t}}else if(yn(r)){const t=$a[e];if(t){const{vgProp:n,part:i}=t;Bu(f,i,n,r),delete f[e]}}p([\"labelAlign\",\"labelBaseline\"],e)&&null===f[e]&&delete f[e]}if(\"grid\"===n){if(!f.grid)return;if(f.encode){const{grid:e}=f.encode;f.encode={...e?{grid:e}:{}},S(f.encode)&&delete f.encode}return{scale:s,orient:a,...f,domain:!1,labels:!1,aria:!1,maxExtent:0,minExtent:0,ticks:!1,zindex:U(u,0)}}{if(!r.header&&e.mainExtracted)return;if(void 0!==l){let e=l;f.encode?.labels?.update&&yn(f.encode.labels.update.text)&&(e=M(l,\"datum.label\",f.encode.labels.update.text.signal)),Bu(f,\"labels\",\"text\",{signal:e})}if(null===f.labelAlign&&delete f.labelAlign,f.encode){for(const t of ka)e.hasAxisPart(t)||delete f.encode[t];S(f.encode)&&delete f.encode}const n=function(e,n){if(e)return t.isArray(e)&&!hn(e)?e.map((e=>la(e,n))).join(\", \"):e}(c,i);return{scale:s,orient:a,grid:!1,...n?{title:n}:{},...f,...!1===i.aria?{aria:!1}:{},zindex:U(u,0)}}}}function Hu(e){const{axes:t}=e.component,n=[];for(const i of Ft)if(t[i])for(const r of t[i])if(!r.get(\"disable\")&&!r.get(\"gridScale\")){const t=\"x\"===i?\"height\":\"width\",r=e.getSizeSignalRef(t).signal;t!==r&&n.push({name:t,update:r})}return n}function Vu(e,t,n,i){return Object.assign.apply(null,[{},...e.map((e=>{if(\"axisOrient\"===e){const e=\"x\"===n?\"bottom\":\"left\",r=t[\"x\"===n?\"axisBottom\":\"axisLeft\"]||{},o=t[\"x\"===n?\"axisTop\":\"axisRight\"]||{},a=new Set([...D(r),...D(o)]),s={};for(const t of a.values())s[t]={signal:`${i.signal} === \"${e}\" ? ${On(r[t])} : ${On(o[t])}`};return s}return t[e]}))])}function Gu(e,n){const i=[{}];for(const r of e){let e=n[r]?.style;if(e){e=t.array(e);for(const t of e)i.push(n.style[t])}}return Object.assign.apply(null,i)}function Yu(e,t,n){let i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{};const r=jn(e,n,t);if(void 0!==r)return{configFrom:\"style\",configValue:r};for(const t of[\"vlOnlyAxisConfig\",\"vgAxisConfig\",\"axisConfigStyle\"])if(void 0!==i[t]?.[e])return{configFrom:t,configValue:i[t][e]};return{}}const Xu={scale:e=>{let{model:t,channel:n}=e;return t.scaleName(n)},format:e=>{let{format:t}=e;return t},formatType:e=>{let{formatType:t}=e;return t},grid:e=>{let{fieldOrDatumDef:t,axis:n,scaleType:i}=e;return n.grid??function(e,t){return!hr(e)&&Ro(t)&&!ln(t?.bin)&&!cn(t?.bin)}(i,t)},gridScale:e=>{let{model:t,channel:n}=e;return function(e,t){const n=\"x\"===t?\"y\":\"x\";if(e.getScaleComponent(n))return e.scaleName(n);return}(t,n)},labelAlign:e=>{let{axis:t,labelAngle:n,orient:i,channel:r}=e;return t.labelAlign||Ku(n,i,r)},labelAngle:e=>{let{labelAngle:t}=e;return t},labelBaseline:e=>{let{axis:t,labelAngle:n,orient:i,channel:r}=e;return t.labelBaseline||Ju(n,i,r)},labelFlush:e=>{let{axis:t,fieldOrDatumDef:n,channel:i}=e;return t.labelFlush??function(e,t){if(\"x\"===t&&p([\"quantitative\",\"temporal\"],e))return!0;return}(n.type,i)},labelOverlap:e=>{let{axis:n,fieldOrDatumDef:i,scaleType:r}=e;return n.labelOverlap??function(e,n,i,r){if(i&&!t.isObject(r)||\"nominal\"!==e&&\"ordinal\"!==e)return\"log\"!==n&&\"symlog\"!==n||\"greedy\";return}(i.type,r,Ro(i)&&!!i.timeUnit,Ro(i)?i.sort:void 0)},orient:e=>{let{orient:t}=e;return t},tickCount:e=>{let{channel:t,model:n,axis:i,fieldOrDatumDef:r,scaleType:o}=e;const a=\"x\"===t?\"width\":\"y\"===t?\"height\":void 0,s=a?n.getSizeSignalRef(a):void 0;return i.tickCount??function(e){let{fieldOrDatumDef:t,scaleType:n,size:i,values:r}=e;if(!r&&!hr(n)&&\"log\"!==n){if(Ro(t)){if(ln(t.bin))return{signal:`ceil(${i.signal}/10)`};if(t.timeUnit&&p([\"month\",\"hours\",\"day\",\"quarter\"],Ei(t.timeUnit)?.unit))return}return{signal:`ceil(${i.signal}/40)`}}return}({fieldOrDatumDef:r,scaleType:o,size:s,values:i.values})},tickMinStep:function(e){let{format:t,fieldOrDatumDef:n}=e;if(\"d\"===t)return 1;if(Ro(n)){const{timeUnit:e}=n;if(e){const t=Mi(e);if(t)return{signal:t}}}return},title:e=>{let{axis:t,model:n,channel:i}=e;if(void 0!==t.title)return t.title;const r=Zu(n,i);if(void 0!==r)return r;const o=n.typedFieldDef(i),a=\"x\"===i?\"x2\":\"y2\",s=n.fieldDef(a);return En(o?[Po(o)]:[],Ro(s)?[Po(s)]:[])},values:e=>{let{axis:n,fieldOrDatumDef:i}=e;return function(e,n){const i=e.values;if(t.isArray(i))return ba(n,i);if(yn(i))return i;return}(n,i)},zindex:e=>{let{axis:t,fieldOrDatumDef:n,mark:i}=e;return t.zindex??function(e,t){if(\"rect\"===e&&na(t))return 1;return 0}(i,n)}};function Qu(e){return`(((${e.signal} % 360) + 360) % 360)`}function Ju(e,t,n,i){if(void 0!==e){if(\"x\"===n){if(yn(e)){const n=Qu(e);return{signal:`(45 < ${n} && ${n} < 135) || (225 < ${n} && ${n} < 315) ? \"middle\" :(${n} <= 45 || 315 <= ${n}) === ${yn(t)?`(${t.signal} === \"top\")`:\"top\"===t} ? \"bottom\" : \"top\"`}}if(45{if(Qo(t)&&Oo(t.sort)){const{field:i,timeUnit:r}=t,o=t.sort,a=o.map(((e,t)=>`${Xi({field:i,timeUnit:r,equal:e})} ? ${t} : `)).join(\"\")+o.length;e=new ef(e,{calculate:a,as:tf(t,n,{forAs:!0})})}})),e}producedFields(){return new Set([this.transform.as])}dependentFields(){return this._dependentFields}assemble(){return{type:\"formula\",expr:this.transform.calculate,as:this.transform.as}}hash(){return`Calculate ${d(this.transform)}`}}function tf(e,t,n){return ta(e,{prefix:t,suffix:\"sort_index\",...n})}function nf(e,t){return p([\"top\",\"bottom\"],t)?\"column\":p([\"left\",\"right\"],t)||\"row\"===e?\"row\":\"column\"}function rf(e,t,n,i){const r=\"row\"===i?n.headerRow:\"column\"===i?n.headerColumn:n.headerFacet;return U((t||{})[e],r[e],n.header[e])}function of(e,t,n,i){const r={};for(const o of e){const e=rf(o,t||{},n,i);void 0!==e&&(r[o]=e)}return r}const af=[\"row\",\"column\"],sf=[\"header\",\"footer\"];function lf(e,t){const n=e.component.layoutHeaders[t].title,i=e.config?e.config:void 0,r=e.component.layoutHeaders[t].facetFieldDef?e.component.layoutHeaders[t].facetFieldDef:void 0,{titleAnchor:o,titleAngle:a,titleOrient:s}=of([\"titleAnchor\",\"titleAngle\",\"titleOrient\"],r.header,i,t),l=nf(t,s),c=H(a);return{name:`${t}-title`,type:\"group\",role:`${l}-title`,title:{text:n,...\"row\"===t?{orient:\"left\"}:{},style:\"guide-title\",...uf(c,l),...cf(l,c,o),...yf(i,r,t,ds,us)}}}function cf(e,t){switch(arguments.length>2&&void 0!==arguments[2]?arguments[2]:\"middle\"){case\"start\":return{align:\"left\"};case\"end\":return{align:\"right\"}}const n=Ku(t,\"row\"===e?\"left\":\"top\",\"row\"===e?\"y\":\"x\");return n?{align:n}:{}}function uf(e,t){const n=Ju(e,\"row\"===t?\"left\":\"top\",\"row\"===t?\"y\":\"x\",!0);return n?{baseline:n}:{}}function ff(e,t){const n=e.component.layoutHeaders[t],i=[];for(const r of sf)if(n[r])for(const o of n[r]){const a=pf(e,t,r,n,o);null!=a&&i.push(a)}return i}function df(e,n){const{sort:i}=e;return zo(i)?{field:ta(i,{expr:\"datum\"}),order:i.order??\"ascending\"}:t.isArray(i)?{field:tf(e,n,{expr:\"datum\"}),order:\"ascending\"}:{field:ta(e,{expr:\"datum\"}),order:i??\"ascending\"}}function mf(e,t,n){const{format:i,formatType:r,labelAngle:o,labelAnchor:a,labelOrient:s,labelExpr:l}=of([\"format\",\"formatType\",\"labelAngle\",\"labelAnchor\",\"labelOrient\",\"labelExpr\"],e.header,n,t),c=Rr({fieldOrDatumDef:e,format:i,formatType:r,expr:\"parent\",config:n}).signal,u=nf(t,s);return{text:{signal:l?M(M(l,\"datum.label\",c),\"datum.value\",ta(e,{expr:\"parent\"})):c},...\"row\"===t?{orient:\"left\"}:{},style:\"guide-label\",frame:\"group\",...uf(o,u),...cf(u,o,a),...yf(n,e,t,ms,fs)}}function pf(e,t,n,i,r){if(r){let o=null;const{facetFieldDef:a}=i,s=e.config?e.config:void 0;if(a&&r.labels){const{labelOrient:e}=of([\"labelOrient\"],a.header,s,t);(\"row\"===t&&!p([\"top\",\"bottom\"],e)||\"column\"===t&&!p([\"left\",\"right\"],e))&&(o=mf(a,t,s))}const l=hm(e)&&!_o(e.facet),c=r.axes,u=c?.length>0;if(o||u){const s=\"row\"===t?\"height\":\"width\";return{name:e.getName(`${t}_${n}`),type:\"group\",role:`${t}-${n}`,...i.facetFieldDef?{from:{data:e.getName(`${t}_domain`)},sort:df(a,t)}:{},...u&&l?{from:{data:e.getName(`facet_domain_${t}`)}}:{},...o?{title:o}:{},...r.sizeSignal?{encode:{update:{[s]:r.sizeSignal}}}:{},...u?{axes:c}:{}}}}return null}const gf={column:{start:0,end:1},row:{start:1,end:0}};function hf(e,t){return gf[t][e]}function yf(e,t,n,i,r){const o={};for(const a of i){if(!r[a])continue;const i=rf(a,t?.header,e,n);void 0!==i&&(o[r[a]]=i)}return o}function vf(e){return[...bf(e,\"width\"),...bf(e,\"height\"),...bf(e,\"childWidth\"),...bf(e,\"childHeight\")]}function bf(e,t){const n=\"width\"===t?\"x\":\"y\",i=e.component.layoutSize.get(t);if(!i||\"merged\"===i)return[];const r=e.getSizeSignalRef(t).signal;if(\"step\"===i){const t=e.getScaleComponent(n);if(t){const i=t.get(\"type\"),o=t.get(\"range\");if(hr(i)&&vn(o)){const i=e.scaleName(n);if(hm(e.parent)){if(\"independent\"===e.parent.component.resolve.scale[n])return[xf(i,o)]}return[xf(i,o),{name:r,update:$f(i,t,`domain('${i}').length`)}]}}throw new Error(\"layout size is step although width/height is not step.\")}if(\"container\"==i){const t=r.endsWith(\"width\"),n=t?\"containerSize()[0]\":\"containerSize()[1]\",i=`isFinite(${n}) ? ${n} : ${_s(e.config.view,t?\"width\":\"height\")}`;return[{name:r,init:i,on:[{update:i,events:\"window:resize\"}]}]}return[{name:r,value:i}]}function xf(e,t){const n=`${e}_step`;return yn(t.step)?{name:n,update:t.step.signal}:{name:n,value:t.step}}function $f(e,t,n){const i=t.get(\"type\"),r=t.get(\"padding\"),o=U(t.get(\"paddingOuter\"),r);let a=t.get(\"paddingInner\");return a=\"band\"===i?void 0!==a?a:r:1,`bandspace(${n}, ${On(a)}, ${On(o)}) * ${e}_step`}function wf(e){return\"childWidth\"===e?\"width\":\"childHeight\"===e?\"height\":e}function kf(e,t){return D(e).reduce(((n,i)=>{const r=e[i];return{...n,...Ec(t,r,i,(e=>Fn(e.value)))}}),{})}function Sf(e,t){if(hm(t))return\"theta\"===e?\"independent\":\"shared\";if(vm(t))return\"shared\";if(ym(t))return zt(e)||\"theta\"===e||\"radius\"===e?\"independent\":\"shared\";throw new Error(\"invalid model type for resolve\")}function Df(e,t){const n=e.scale[t],i=zt(t)?\"axis\":\"legend\";return\"independent\"===n?(\"shared\"===e[i][t]&&yi(function(e){return`Setting the scale to be independent for \"${e}\" means we also have to set the guide (axis or legend) to be independent.`}(t)),\"independent\"):e[i][t]||\"shared\"}const Ff=D({aria:1,clipHeight:1,columnPadding:1,columns:1,cornerRadius:1,description:1,direction:1,fillColor:1,format:1,formatType:1,gradientLength:1,gradientOpacity:1,gradientStrokeColor:1,gradientStrokeWidth:1,gradientThickness:1,gridAlign:1,labelAlign:1,labelBaseline:1,labelColor:1,labelFont:1,labelFontSize:1,labelFontStyle:1,labelFontWeight:1,labelLimit:1,labelOffset:1,labelOpacity:1,labelOverlap:1,labelPadding:1,labelSeparation:1,legendX:1,legendY:1,offset:1,orient:1,padding:1,rowPadding:1,strokeColor:1,symbolDash:1,symbolDashOffset:1,symbolFillColor:1,symbolLimit:1,symbolOffset:1,symbolOpacity:1,symbolSize:1,symbolStrokeColor:1,symbolStrokeWidth:1,symbolType:1,tickCount:1,tickMinStep:1,title:1,titleAlign:1,titleAnchor:1,titleBaseline:1,titleColor:1,titleFont:1,titleFontSize:1,titleFontStyle:1,titleFontWeight:1,titleLimit:1,titleLineHeight:1,titleOpacity:1,titleOrient:1,titlePadding:1,type:1,values:1,zindex:1,disable:1,labelExpr:1,selections:1,opacity:1,shape:1,stroke:1,fill:1,size:1,strokeWidth:1,strokeDash:1,encode:1});class zf extends Gl{}const Of={symbols:function(e,n){let{fieldOrDatumDef:i,model:r,channel:o,legendCmpt:a,legendType:s}=n;if(\"symbol\"!==s)return;const{markDef:l,encoding:c,config:u,mark:f}=r,d=l.filled&&\"trail\"!==f;let m={..._n({},r,ho),...Vc(r,{filled:d})};const p=a.get(\"symbolOpacity\")??u.legend.symbolOpacity,g=a.get(\"symbolFillColor\")??u.legend.symbolFillColor,h=a.get(\"symbolStrokeColor\")??u.legend.symbolStrokeColor,y=void 0===p?_f(c.opacity)??l.opacity:void 0;if(m.fill)if(\"fill\"===o||d&&o===me)delete m.fill;else if(m.fill.field)g?delete m.fill:(m.fill=Fn(u.legend.symbolBaseFillColor??\"black\"),m.fillOpacity=Fn(y??1));else if(t.isArray(m.fill)){const e=Cf(c.fill??c.color)??l.fill??(d&&l.color);e&&(m.fill=Fn(e))}if(m.stroke)if(\"stroke\"===o||!d&&o===me)delete m.stroke;else if(m.stroke.field||h)delete m.stroke;else if(t.isArray(m.stroke)){const e=U(Cf(c.stroke||c.color),l.stroke,d?l.color:void 0);e&&(m.stroke={value:e})}if(o!==be){const e=Ro(i)&&Pf(r,a,i);e?m.opacity=[{test:e,...Fn(y??1)},Fn(u.legend.unselectedOpacity)]:y&&(m.opacity=Fn(y))}return m={...m,...e},S(m)?void 0:m},gradient:function(e,t){let{model:n,legendType:i,legendCmpt:r}=t;if(\"gradient\"!==i)return;const{config:o,markDef:a,encoding:s}=n;let l={};const c=void 0===(r.get(\"gradientOpacity\")??o.legend.gradientOpacity)?_f(s.opacity)||a.opacity:void 0;c&&(l.opacity=Fn(c));return l={...l,...e},S(l)?void 0:l},labels:function(e,t){let{fieldOrDatumDef:n,model:i,channel:r,legendCmpt:o}=t;const a=i.legend(r)||{},s=i.config,l=Ro(n)?Pf(i,o,n):void 0,c=l?[{test:l,value:1},{value:s.legend.unselectedOpacity}]:void 0,{format:u,formatType:f}=a;let d;Lr(f)?d=Br({fieldOrDatumDef:n,field:\"datum.value\",format:u,formatType:f,config:s}):void 0===u&&void 0===f&&s.customFormatTypes&&(\"quantitative\"===n.type&&s.numberFormatType?d=Br({fieldOrDatumDef:n,field:\"datum.value\",format:s.numberFormat,formatType:s.numberFormatType,config:s}):\"temporal\"===n.type&&s.timeFormatType&&Ro(n)&&void 0===n.timeUnit&&(d=Br({fieldOrDatumDef:n,field:\"datum.value\",format:s.timeFormat,formatType:s.timeFormatType,config:s})));const m={...c?{opacity:c}:{},...d?{text:d}:{},...e};return S(m)?void 0:m},entries:function(e,t){let{legendCmpt:n}=t;const i=n.get(\"selections\");return i?.length?{...e,fill:{value:\"transparent\"}}:e}};function _f(e){return Nf(e,((e,t)=>Math.max(e,t.value)))}function Cf(e){return Nf(e,((e,t)=>U(e,t.value)))}function Nf(e,n){return function(e){const n=e?.condition;return!!n&&(t.isArray(n)||Xo(n))}(e)?t.array(e.condition).reduce(n,e.value):Xo(e)?e.value:void 0}function Pf(e,n,i){const r=n.get(\"selections\");if(!r?.length)return;const o=t.stringValue(i.field);return r.map((e=>`(!length(data(${t.stringValue(_(e)+Ou)})) || (${e}[${o}] && indexof(${e}[${o}], datum.value) >= 0))`)).join(\" || \")}const Af={direction:e=>{let{direction:t}=e;return t},format:e=>{let{fieldOrDatumDef:t,legend:n,config:i}=e;const{format:r,formatType:o}=n;return Ir(t,t.type,r,o,i,!1)},formatType:e=>{let{legend:t,fieldOrDatumDef:n,scaleType:i}=e;const{formatType:r}=t;return Hr(r,n,i)},gradientLength:e=>{const{legend:t,legendConfig:n}=e;return t.gradientLength??n.gradientLength??function(e){let{legendConfig:t,model:n,direction:i,orient:r,scaleType:o}=e;const{gradientHorizontalMaxLength:a,gradientHorizontalMinLength:s,gradientVerticalMaxLength:l,gradientVerticalMinLength:c}=t;if(vr(o))return\"horizontal\"===i?\"top\"===r||\"bottom\"===r?Ef(n,\"width\",s,a):s:Ef(n,\"height\",c,l);return}(e)},labelOverlap:e=>{let{legend:t,legendConfig:n,scaleType:i}=e;return t.labelOverlap??n.labelOverlap??function(e){if(p([\"quantile\",\"threshold\",\"log\",\"symlog\"],e))return\"greedy\";return}(i)},symbolType:e=>{let{legend:t,markDef:n,channel:i,encoding:r}=e;return t.symbolType??function(e,t,n,i){if(\"shape\"!==t){const e=Cf(n)??i;if(e)return e}switch(e){case\"bar\":case\"rect\":case\"image\":case\"square\":return\"square\";case\"line\":case\"trail\":case\"rule\":return\"stroke\";case\"arc\":case\"point\":case\"circle\":case\"tick\":case\"geoshape\":case\"area\":case\"text\":return\"circle\"}}(n.type,i,r.shape,n.shape)},title:e=>{let{fieldOrDatumDef:t,config:n}=e;return aa(t,n,{allowDisabling:!0})},type:e=>{let{legendType:t,scaleType:n,channel:i}=e;if(qe(i)&&vr(n)){if(\"gradient\"===t)return}else if(\"symbol\"===t)return;return t},values:e=>{let{fieldOrDatumDef:n,legend:i}=e;return function(e,n){const i=e.values;if(t.isArray(i))return ba(n,i);if(yn(i))return i;return}(i,n)}};function jf(e){const{legend:t}=e;return U(t.type,function(e){let{channel:t,timeUnit:n,scaleType:i}=e;if(qe(t)){if(p([\"quarter\",\"month\",\"day\"],n))return\"symbol\";if(vr(i))return\"gradient\"}return\"symbol\"}(e))}function Tf(e){let{legendConfig:t,legendType:n,orient:i,legend:r}=e;return r.direction??t[n?\"gradientDirection\":\"symbolDirection\"]??function(e,t){switch(e){case\"top\":case\"bottom\":return\"horizontal\";case\"left\":case\"right\":case\"none\":case void 0:return;default:return\"gradient\"===t?\"horizontal\":void 0}}(i,n)}function Ef(e,t,n,i){return{signal:`clamp(${e.getSizeSignalRef(t).signal}, ${n}, ${i})`}}function Mf(e){const t=gm(e)?function(e){const{encoding:t}=e,n={};for(const i of[me,...gs]){const r=fa(t[i]);r&&e.getScaleComponent(i)&&(i===he&&Ro(r)&&r.type===rr||(n[i]=qf(e,i)))}return n}(e):function(e){const{legends:t,resolve:n}=e.component;for(const i of e.children){Mf(i);for(const r of D(i.component.legends))n.legend[r]=Df(e.component.resolve,r),\"shared\"===n.legend[r]&&(t[r]=Uf(t[r],i.component.legends[r]),t[r]||(n.legend[r]=\"independent\",delete t[r]))}for(const i of D(t))for(const t of e.children)t.component.legends[i]&&\"shared\"===n.legend[i]&&delete t.component.legends[i];return t}(e);return e.component.legends=t,t}function Lf(e,t,n,i){switch(t){case\"disable\":return void 0!==n;case\"values\":return!!n?.values;case\"title\":if(\"title\"===t&&e===i?.title)return!0}return e===(n||{})[t]}function qf(e,t){let n=e.legend(t);const{markDef:i,encoding:r,config:o}=e,a=o.legend,s=new zf({},function(e,t){const n=e.scaleName(t);if(\"trail\"===e.mark){if(\"color\"===t)return{stroke:n};if(\"size\"===t)return{strokeWidth:n}}return\"color\"===t?e.markDef.filled?{fill:n}:{stroke:n}:{[t]:n}}(e,t));!function(e,t,n){const i=e.fieldDef(t)?.field;for(const r of F(e.component.selection??{})){const e=r.project.hasField[i]??r.project.hasChannel[t];if(e&&bu.defined(r)){const t=n.get(\"selections\")??[];t.push(r.name),n.set(\"selections\",t,!1),e.hasLegend=!0}}}(e,t,s);const l=void 0!==n?!n:a.disable;if(s.set(\"disable\",l,void 0!==n),l)return s;n=n||{};const c=e.getScaleComponent(t).get(\"type\"),u=fa(r[t]),f=Ro(u)?Ei(u.timeUnit)?.unit:void 0,d=n.orient||o.legend.orient||\"right\",m=jf({legend:n,channel:t,timeUnit:f,scaleType:c}),p={legend:n,channel:t,model:e,markDef:i,encoding:r,fieldOrDatumDef:u,legendConfig:a,config:o,scaleType:c,orient:d,legendType:m,direction:Tf({legend:n,legendType:m,orient:d,legendConfig:a})};for(const i of Ff){if(\"gradient\"===m&&i.startsWith(\"symbol\")||\"symbol\"===m&&i.startsWith(\"gradient\"))continue;const r=i in Af?Af[i](p):n[i];if(void 0!==r){const a=Lf(r,i,n,e.fieldDef(t));(a||void 0===o.legend[i])&&s.set(i,r,a)}}const g=n?.encoding??{},h=s.get(\"selections\"),y={},v={fieldOrDatumDef:u,model:e,channel:t,legendCmpt:s,legendType:m};for(const t of[\"labels\",\"legend\",\"title\",\"symbols\",\"gradient\",\"entries\"]){const n=kf(g[t]??{},e),i=t in Of?Of[t](n,v):n;void 0===i||S(i)||(y[t]={...h?.length&&Ro(u)?{name:`${_(u.field)}_legend_${t}`}:{},...h?.length?{interactive:!!h}:{},update:i})}return S(y)||s.set(\"encode\",y,!!n?.encoding),s}function Uf(e,t){if(!e)return t.clone();const n=e.getWithExplicit(\"orient\"),i=t.getWithExplicit(\"orient\");if(n.explicit&&i.explicit&&n.value!==i.value)return;let r=!1;for(const n of Ff){const i=Kl(e.getWithExplicit(n),t.getWithExplicit(n),n,\"legend\",((e,t)=>{switch(n){case\"symbolType\":return Rf(e,t);case\"title\":return Ln(e,t);case\"type\":return r=!0,Xl(\"symbol\")}return Jl(e,t,n,\"legend\")}));e.setWithExplicit(n,i)}return r&&(e.implicit?.encode?.gradient&&N(e.implicit,[\"encode\",\"gradient\"]),e.explicit?.encode?.gradient&&N(e.explicit,[\"encode\",\"gradient\"])),e}function Rf(e,t){return\"circle\"===t.value?t:e}function Wf(e){const t=e.component.legends,n={};for(const i of D(t)){const r=X(e.getScaleComponent(i).get(\"domains\"));if(n[r])for(const e of n[r]){Uf(e,t[i])||n[r].push(t[i])}else n[r]=[t[i].clone()]}return F(n).flat().map((t=>function(e,t){const{disable:n,labelExpr:i,selections:r,...o}=e.combine();if(n)return;!1===t.aria&&null==o.aria&&(o.aria=!1);if(o.encode?.symbols){const e=o.encode.symbols.update;!e.fill||\"transparent\"===e.fill.value||e.stroke||o.stroke||(e.stroke={value:\"transparent\"});for(const t of gs)o[t]&&delete e[t]}o.title||delete o.title;if(void 0!==i){let e=i;o.encode?.labels?.update&&yn(o.encode.labels.update.text)&&(e=M(i,\"datum.label\",o.encode.labels.update.text.signal)),function(e,t,n,i){e.encode??={},e.encode[t]??={},e.encode[t].update??={},e.encode[t].update[n]=i}(o,\"labels\",\"text\",{signal:e})}return o}(t,e.config))).filter((e=>void 0!==e))}function Bf(e){return vm(e)||ym(e)?function(e){return e.children.reduce(((e,t)=>e.concat(t.assembleProjections())),If(e))}(e):If(e)}function If(e){const t=e.component.projection;if(!t||t.merged)return[];const n=t.combine(),{name:i}=n;if(t.data){const r={signal:`[${t.size.map((e=>e.signal)).join(\", \")}]`},o=t.data.reduce(((t,n)=>{const i=yn(n)?n.signal:`data('${e.lookupDataSource(n)}')`;return p(t,i)||t.push(i),t}),[]);if(o.length<=0)throw new Error(\"Projection's fit didn't find any data sources\");return[{name:i,size:r,fit:{signal:o.length>1?`[${o.join(\", \")}]`:o[0]},...n}]}return[{name:i,translate:{signal:\"[width / 2, height / 2]\"},...n}]}const Hf=[\"type\",\"clipAngle\",\"clipExtent\",\"center\",\"rotate\",\"precision\",\"reflectX\",\"reflectY\",\"coefficient\",\"distance\",\"fraction\",\"lobes\",\"parallel\",\"radius\",\"ratio\",\"spacing\",\"tilt\"];class Vf extends Gl{merged=!1;constructor(e,t,n,i){super({...t},{name:e}),this.specifiedProjection=t,this.size=n,this.data=i}get isFit(){return!!this.data}}function Gf(e){e.component.projection=gm(e)?function(e){if(e.hasProjection){const t=pn(e.specifiedProjection),n=!(t&&(null!=t.scale||null!=t.translate)),i=n?[e.getSizeSignalRef(\"width\"),e.getSizeSignalRef(\"height\")]:void 0,r=n?function(e){const t=[],{encoding:n}=e;for(const i of[[ue,ce],[de,fe]])(fa(n[i[0]])||fa(n[i[1]]))&&t.push({signal:e.getName(`geojson_${t.length}`)});e.channelHasField(he)&&e.typedFieldDef(he).type===rr&&t.push({signal:e.getName(`geojson_${t.length}`)});0===t.length&&t.push(e.requestDataName(sc.Main));return t}(e):void 0,o=new Vf(e.projectionName(!0),{...pn(e.config.projection),...t},i,r);return o.get(\"type\")||o.set(\"type\",\"equalEarth\",!1),o}return}(e):function(e){if(0===e.children.length)return;let n;for(const t of e.children)Gf(t);const i=h(e.children,(e=>{const i=e.component.projection;if(i){if(n){const e=function(e,n){const i=h(Hf,(i=>!t.hasOwnProperty(e.explicit,i)&&!t.hasOwnProperty(n.explicit,i)||!!(t.hasOwnProperty(e.explicit,i)&&t.hasOwnProperty(n.explicit,i)&&Y(e.get(i),n.get(i)))));if(Y(e.size,n.size)){if(i)return e;if(Y(e.explicit,{}))return n;if(Y(n.explicit,{}))return e}return null}(n,i);return e&&(n=e),!!e}return n=i,!0}return!0}));if(n&&i){const t=e.projectionName(!0),i=new Vf(t,n.specifiedProjection,n.size,l(n.data));for(const n of e.children){const e=n.component.projection;e&&(e.isFit&&i.data.push(...n.component.projection.data),n.renameProjection(e.get(\"name\"),t),e.merged=!0)}return i}return}(e)}function Yf(e,t,n,i){if(xa(t,n)){const r=gm(e)?e.axis(n)??e.legend(n)??{}:{},o=ta(t,{expr:\"datum\"}),a=ta(t,{expr:\"datum\",binSuffix:\"end\"});return{formulaAs:ta(t,{binSuffix:\"range\",forAs:!0}),formula:Xr(o,a,r.format,r.formatType,i)}}return{}}function Xf(e,t){return`${sn(e)}_${t}`}function Qf(e,t,n){const i=Xf(ga(n,void 0)??{},t);return e.getName(`${i}_bins`)}function Jf(e,n,i){let r,o;r=function(e){return\"as\"in e}(e)?t.isString(e.as)?[e.as,`${e.as}_end`]:[e.as[0],e.as[1]]:[ta(e,{forAs:!0}),ta(e,{binSuffix:\"end\",forAs:!0})];const a={...ga(n,void 0)},s=Xf(a,e.field),{signal:l,extentSignal:c}=function(e,t){return{signal:e.getName(`${t}_bins`),extentSignal:e.getName(`${t}_extent`)}}(i,s);if(fn(a.extent)){const e=a.extent;o=Ru(i,e.param,e),delete a.extent}return{key:s,binComponent:{bin:a,field:e.field,as:[r],...l?{signal:l}:{},...c?{extentSignal:c}:{},...o?{span:o}:{}}}}class Kf extends pc{clone(){return new Kf(null,l(this.bins))}constructor(e,t){super(e),this.bins=t}static makeFromEncoding(e,t){const n=t.reduceFieldDef(((e,n,i)=>{if(Yo(n)&&ln(n.bin)){const{key:r,binComponent:o}=Jf(n,n.bin,t);e[r]={...o,...e[r],...Yf(t,n,i,t.config)}}return e}),{});return S(n)?null:new Kf(e,n)}static makeFromTransform(e,t,n){const{key:i,binComponent:r}=Jf(t,t.bin,n);return new Kf(e,{[i]:r})}merge(e,t){for(const n of D(e.bins))n in this.bins?(t(e.bins[n].signal,this.bins[n].signal),this.bins[n].as=b([...this.bins[n].as,...e.bins[n].as],d)):this.bins[n]=e.bins[n];for(const t of e.children)e.removeChild(t),t.parent=this;e.remove()}producedFields(){return new Set(F(this.bins).map((e=>e.as)).flat(2))}dependentFields(){return new Set(F(this.bins).map((e=>e.field)))}hash(){return`Bin ${d(this.bins)}`}assemble(){return F(this.bins).flatMap((e=>{const t=[],[n,...i]=e.as,{extent:r,...o}=e.bin,a={type:\"bin\",field:E(e.field),as:n,signal:e.signal,...fn(r)?{extent:null}:{extent:r},...e.span?{span:{signal:`span(${e.span})`}}:{},...o};!r&&e.extentSignal&&(t.push({type:\"extent\",field:E(e.field),signal:e.extentSignal}),a.extent={signal:e.extentSignal}),t.push(a);for(const e of i)for(let i=0;i<2;i++)t.push({type:\"formula\",expr:ta({field:n[i]},{expr:\"datum\"}),as:e[i]});return e.formula&&t.push({type:\"formula\",expr:e.formula,as:e.formulaAs}),t}))}}function Zf(e,n,i,r){const o=gm(r)?r.encoding[it(n)]:void 0;if(Yo(i)&&gm(r)&&Eo(i,o,r.markDef,r.config)){e.add(ta(i,{})),e.add(ta(i,{suffix:\"end\"}));const{mark:t,markDef:o,config:a}=r,s=jo({fieldDef:i,markDef:o,config:a});mo(t)&&.5!==s&&zt(n)&&(e.add(ta(i,{suffix:bc})),e.add(ta(i,{suffix:xc}))),i.bin&&xa(i,n)&&e.add(ta(i,{binSuffix:\"range\"}))}else if(Ee(n)){const t=Te(n);e.add(r.getName(t))}else e.add(ta(i));return Qo(i)&&function(e){return t.isObject(e)&&\"field\"in e}(i.scale?.range)&&e.add(i.scale.range.field),e}class ed extends pc{clone(){return new ed(null,new Set(this.dimensions),l(this.measures))}constructor(e,t,n){super(e),this.dimensions=t,this.measures=n}get groupBy(){return this.dimensions}static makeFromEncoding(e,t){let n=!1;t.forEachFieldDef((e=>{e.aggregate&&(n=!0)}));const i={},r=new Set;return n?(t.forEachFieldDef(((e,n)=>{const{aggregate:o,field:a}=e;if(o)if(\"count\"===o)i[\"*\"]??={},i[\"*\"].count=new Set([ta(e,{forAs:!0})]);else{if(Zt(o)||en(o)){const e=Zt(o)?\"argmin\":\"argmax\",t=o[e];i[t]??={},i[t][e]=new Set([ta({op:e,field:t},{forAs:!0})])}else i[a]??={},i[a][o]=new Set([ta(e,{forAs:!0})]);Ht(n)&&\"unaggregated\"===t.scaleDomain(n)&&(i[a]??={},i[a].min=new Set([ta({field:a,aggregate:\"min\"},{forAs:!0})]),i[a].max=new Set([ta({field:a,aggregate:\"max\"},{forAs:!0})]))}else Zf(r,n,e,t)})),r.size+D(i).length===0?null:new ed(e,r,i)):null}static makeFromTransform(e,t){const n=new Set,i={};for(const e of t.aggregate){const{op:t,field:n,as:r}=e;t&&(\"count\"===t?(i[\"*\"]??={},i[\"*\"].count=new Set([r||ta(e,{forAs:!0})])):(i[n]??={},i[n][t]=new Set([r||ta(e,{forAs:!0})])))}for(const e of t.groupby??[])n.add(e);return n.size+D(i).length===0?null:new ed(e,n,i)}merge(e){return x(this.dimensions,e.dimensions)?(function(e,t){for(const n of D(t)){const i=t[n];for(const t of D(i))n in e?e[n][t]=new Set([...e[n][t]??[],...i[t]]):e[n]={[t]:i[t]}}}(this.measures,e.measures),!0):(function(){hi.debug(...arguments)}(\"different dimensions, cannot merge\"),!1)}addDimensions(e){e.forEach(this.dimensions.add,this.dimensions)}dependentFields(){return new Set([...this.dimensions,...D(this.measures)])}producedFields(){const e=new Set;for(const t of D(this.measures))for(const n of D(this.measures[t])){const i=this.measures[t][n];0===i.size?e.add(`${n}_${t}`):i.forEach(e.add,e)}return e}hash(){return`Aggregate ${d({dimensions:this.dimensions,measures:this.measures})}`}assemble(){const e=[],t=[],n=[];for(const i of D(this.measures))for(const r of D(this.measures[i]))for(const o of this.measures[i][r])n.push(o),e.push(r),t.push(\"*\"===i?null:E(i));return{type:\"aggregate\",groupby:[...this.dimensions].map(E),ops:e,fields:t,as:n}}}class td extends pc{constructor(e,n,i,r){super(e),this.model=n,this.name=i,this.data=r;for(const e of Re){const i=n.facet[e];if(i){const{bin:r,sort:o}=i;this[e]={name:n.getName(`${e}_domain`),fields:[ta(i),...ln(r)?[ta(i,{binSuffix:\"end\"})]:[]],...zo(o)?{sortField:o}:t.isArray(o)?{sortIndexField:tf(i,e)}:{}}}}this.childModel=n.child}hash(){let e=\"Facet\";for(const t of Re)this[t]&&(e+=` ${t.charAt(0)}:${d(this[t])}`);return e}get fields(){const e=[];for(const t of Re)this[t]?.fields&&e.push(...this[t].fields);return e}dependentFields(){const e=new Set(this.fields);for(const t of Re)this[t]&&(this[t].sortField&&e.add(this[t].sortField.field),this[t].sortIndexField&&e.add(this[t].sortIndexField));return e}producedFields(){return new Set}getSource(){return this.name}getChildIndependentFieldsWithStep(){const e={};for(const t of Ft){const n=this.childModel.component.scales[t];if(n&&!n.merged){const i=n.get(\"type\"),r=n.get(\"range\");if(hr(i)&&vn(r)){const n=Hd(Vd(this.childModel,t));n?e[t]=n:yi(In(t))}}}return e}assembleRowColumnHeaderData(e,t,n){const i={row:\"y\",column:\"x\",facet:void 0}[e],r=[],o=[],a=[];i&&n&&n[i]&&(t?(r.push(`distinct_${n[i]}`),o.push(\"max\")):(r.push(n[i]),o.push(\"distinct\")),a.push(`distinct_${n[i]}`));const{sortField:s,sortIndexField:l}=this[e];if(s){const{op:e=ko,field:t}=s;r.push(t),o.push(e),a.push(ta(s,{forAs:!0}))}else l&&(r.push(l),o.push(\"max\"),a.push(l));return{name:this[e].name,source:t??this.data,transform:[{type:\"aggregate\",groupby:this[e].fields,...r.length?{fields:r,ops:o,as:a}:{}}]}}assembleFacetHeaderData(e){const{columns:t}=this.model.layout,{layoutHeaders:n}=this.model.component,i=[],r={};for(const e of af){for(const t of sf){const i=(n[e]&&n[e][t])??[];for(const t of i)if(t.axes?.length>0){r[e]=!0;break}}if(r[e]){const n=`length(data(\"${this.facet.name}\"))`,r=\"row\"===e?t?{signal:`ceil(${n} / ${t})`}:1:t?{signal:`min(${n}, ${t})`}:{signal:n};i.push({name:`${this.facet.name}_${e}`,transform:[{type:\"sequence\",start:0,stop:r}]})}}const{row:o,column:a}=r;return(o||a)&&i.unshift(this.assembleRowColumnHeaderData(\"facet\",null,e)),i}assemble(){const e=[];let t=null;const n=this.getChildIndependentFieldsWithStep(),{column:i,row:r,facet:o}=this;if(i&&r&&(n.x||n.y)){t=`cross_${this.column.name}_${this.row.name}`;const i=[].concat(n.x??[],n.y??[]),r=i.map((()=>\"distinct\"));e.push({name:t,source:this.data,transform:[{type:\"aggregate\",groupby:this.fields,fields:i,ops:r}]})}for(const i of[J,Q])this[i]&&e.push(this.assembleRowColumnHeaderData(i,t,n));if(o){const t=this.assembleFacetHeaderData(n);t&&e.push(...t)}return e}}function nd(e){return e.startsWith(\"'\")&&e.endsWith(\"'\")||e.startsWith('\"')&&e.endsWith('\"')?e.slice(1,-1):e}function id(e){const n={};return a(e.filter,(e=>{if(Gi(e)){let i=null;Ui(e)?i=Sn(e.equal):Wi(e)?i=Sn(e.lte):Ri(e)?i=Sn(e.lt):Bi(e)?i=Sn(e.gt):Ii(e)?i=Sn(e.gte):Hi(e)?i=e.range[0]:Vi(e)&&(i=(e.oneOf??e.in)[0]),i&&(vi(i)?n[e.field]=\"date\":t.isNumber(i)?n[e.field]=\"number\":t.isString(i)&&(n[e.field]=\"string\")),e.timeUnit&&(n[e.field]=\"date\")}})),n}function rd(e){const n={};function i(e){var i;ya(e)?n[e.field]=\"date\":\"quantitative\"===e.type&&(i=e.aggregate,t.isString(i)&&p([\"min\",\"max\"],i))?n[e.field]=\"number\":q(e.field)>1?e.field in n||(n[e.field]=\"flatten\"):Qo(e)&&zo(e.sort)&&q(e.sort.field)>1&&(e.sort.field in n||(n[e.sort.field]=\"flatten\"))}if((gm(e)||hm(e))&&e.forEachFieldDef(((t,n)=>{if(Yo(t))i(t);else{const r=tt(n),o=e.fieldDef(r);i({...t,type:o.type})}})),gm(e)){const{mark:t,markDef:i,encoding:r}=e;if(fo(t)&&!e.encoding.order){const e=r[\"horizontal\"===i.orient?\"y\":\"x\"];Ro(e)&&\"quantitative\"===e.type&&!(e.field in n)&&(n[e.field]=\"number\")}}return n}class od extends pc{clone(){return new od(null,l(this._parse))}constructor(e,t){super(e),this._parse=t}hash(){return`Parse ${d(this._parse)}`}static makeExplicit(e,t,n){let i={};const r=t.data;return!ic(r)&&r?.format?.parse&&(i=r.format.parse),this.makeWithAncestors(e,i,{},n)}static makeWithAncestors(e,t,n,i){for(const e of D(n)){const t=i.getWithExplicit(e);void 0!==t.value&&(t.explicit||t.value===n[e]||\"derived\"===t.value||\"flatten\"===n[e]?delete n[e]:yi(Qn(e,n[e],t.value)))}for(const e of D(t)){const n=i.get(e);void 0!==n&&(n===t[e]?delete t[e]:yi(Qn(e,t[e],n)))}const r=new Gl(t,n);i.copyAll(r);const o={};for(const e of D(r.combine())){const t=r.get(e);null!==t&&(o[e]=t)}return 0===D(o).length||i.parseNothing?null:new od(e,o)}get parse(){return this._parse}merge(e){this._parse={...this._parse,...e.parse},e.remove()}assembleFormatParse(){const e={};for(const t of D(this._parse)){const n=this._parse[t];1===q(t)&&(e[t]=n)}return e}producedFields(){return new Set(D(this._parse))}dependentFields(){return new Set(D(this._parse))}assembleTransforms(){let e=arguments.length>0&&void 0!==arguments[0]&&arguments[0];return D(this._parse).filter((t=>!e||q(t)>1)).map((e=>{const t=function(e,t){const n=A(e);if(\"number\"===t)return`toNumber(${n})`;if(\"boolean\"===t)return`toBoolean(${n})`;if(\"string\"===t)return`toString(${n})`;if(\"date\"===t)return`toDate(${n})`;if(\"flatten\"===t)return n;if(t.startsWith(\"date:\"))return`timeParse(${n},'${nd(t.slice(5,t.length))}')`;if(t.startsWith(\"utc:\"))return`utcParse(${n},'${nd(t.slice(4,t.length))}')`;return yi(`Unrecognized parse \"${t}\".`),null}(e,this._parse[e]);if(!t)return null;return{type:\"formula\",expr:t,as:L(e)}})).filter((e=>null!==e))}}class ad extends pc{clone(){return new ad(null)}constructor(e){super(e)}dependentFields(){return new Set}producedFields(){return new Set([hs])}hash(){return\"Identifier\"}assemble(){return{type:\"identifier\",as:hs}}}class sd extends pc{clone(){return new sd(null,this.params)}constructor(e,t){super(e),this.params=t}dependentFields(){return new Set}producedFields(){}hash(){return`Graticule ${d(this.params)}`}assemble(){return{type:\"graticule\",...!0===this.params?{}:this.params}}}class ld extends pc{clone(){return new ld(null,this.params)}constructor(e,t){super(e),this.params=t}dependentFields(){return new Set}producedFields(){return new Set([this.params.as??\"data\"])}hash(){return`Hash ${d(this.params)}`}assemble(){return{type:\"sequence\",...this.params}}}class cd extends pc{constructor(e){let t;if(super(null),e??={name:\"source\"},ic(e)||(t=e.format?{...f(e.format,[\"parse\"])}:{}),tc(e))this._data={values:e.values};else if(ec(e)){if(this._data={url:e.url},!t.type){let n=/(?:\\.([^.]+))?$/.exec(e.url)[1];p([\"json\",\"csv\",\"tsv\",\"dsv\",\"topojson\"],n)||(n=\"json\"),t.type=n}}else oc(e)?this._data={values:[{type:\"Sphere\"}]}:(nc(e)||ic(e))&&(this._data={});this._generator=ic(e),e.name&&(this._name=e.name),t&&!S(t)&&(this._data.format=t)}dependentFields(){return new Set}producedFields(){}get data(){return this._data}hasName(){return!!this._name}get isGenerator(){return this._generator}get dataName(){return this._name}set dataName(e){this._name=e}set parent(e){throw new Error(\"Source nodes have to be roots.\")}remove(){throw new Error(\"Source nodes are roots and cannot be removed.\")}hash(){throw new Error(\"Cannot hash sources\")}assemble(){return{name:this._name,...this._data,transform:[]}}}function ud(e){return e instanceof cd||e instanceof sd||e instanceof ld}class fd{#e;constructor(){this.#e=!1}setModified(){this.#e=!0}get modifiedFlag(){return this.#e}}class dd extends fd{getNodeDepths(e,t,n){n.set(e,t);for(const i of e.children)this.getNodeDepths(i,t+1,n);return n}optimize(e){const t=[...this.getNodeDepths(e,0,new Map).entries()].sort(((e,t)=>t[1]-e[1]));for(const e of t)this.run(e[0]);return this.modifiedFlag}}class md extends fd{optimize(e){this.run(e);for(const t of e.children)this.optimize(t);return this.modifiedFlag}}class pd extends md{mergeNodes(e,t){const n=t.shift();for(const i of t)e.removeChild(i),i.parent=n,i.remove()}run(e){const t=e.children.map((e=>e.hash())),n={};for(let i=0;i1&&(this.setModified(),this.mergeNodes(e,n[t]))}}class gd extends md{constructor(e){super(),this.requiresSelectionId=e&&ju(e)}run(e){e instanceof ad&&(this.requiresSelectionId&&(ud(e.parent)||e.parent instanceof ed||e.parent instanceof od)||(this.setModified(),e.remove()))}}class hd extends fd{optimize(e){return this.run(e,new Set),this.modifiedFlag}run(e,t){let n=new Set;e instanceof vc&&(n=e.producedFields(),$(n,t)&&(this.setModified(),e.removeFormulas(t),0===e.producedFields.length&&e.remove()));for(const i of e.children)this.run(i,new Set([...t,...n]))}}class yd extends md{constructor(){super()}run(e){e instanceof gc&&!e.isRequired()&&(this.setModified(),e.remove())}}class vd extends dd{run(e){if(!(ud(e)||e.numChildren()>1))for(const t of e.children)if(t instanceof od)if(e instanceof od)this.setModified(),e.merge(t);else{if(k(e.producedFields(),t.dependentFields()))continue;this.setModified(),t.swapWithParent()}}}class bd extends dd{run(e){const t=[...e.children],n=e.children.filter((e=>e instanceof od));if(e.numChildren()>1&&n.length>=1){const i={},r=new Set;for(const e of n){const t=e.parse;for(const e of D(t))e in i?i[e]!==t[e]&&r.add(e):i[e]=t[e]}for(const e of r)delete i[e];if(!S(i)){this.setModified();const n=new od(e,i);for(const r of t){if(r instanceof od)for(const e of D(i))delete r.parse[e];e.removeChild(r),r.parent=n,r instanceof od&&0===D(r.parse).length&&r.remove()}}}}}class xd extends dd{run(e){e instanceof gc||e.numChildren()>0||e instanceof td||e instanceof cd||(this.setModified(),e.remove())}}class $d extends dd{run(e){const t=e.children.filter((e=>e instanceof vc)),n=t.pop();for(const e of t)this.setModified(),n.merge(e)}}class wd extends dd{run(e){const t=e.children.filter((e=>e instanceof ed)),n={};for(const e of t){const t=d(e.groupBy);t in n||(n[t]=[]),n[t].push(e)}for(const t of D(n)){const i=n[t];if(i.length>1){const t=i.pop();for(const n of i)t.merge(n)&&(e.removeChild(n),n.parent=t,n.remove(),this.setModified())}}}}class kd extends dd{constructor(e){super(),this.model=e}run(e){const t=!(ud(e)||e instanceof qu||e instanceof od||e instanceof ad),n=[],i=[];for(const r of e.children)r instanceof Kf&&(t&&!k(e.producedFields(),r.dependentFields())?n.push(r):i.push(r));if(n.length>0){const t=n.pop();for(const e of n)t.merge(e,this.model.renameSignal.bind(this.model));this.setModified(),e instanceof Kf?e.merge(t,this.model.renameSignal.bind(this.model)):t.swapWithParent()}if(i.length>1){const e=i.pop();for(const t of i)e.merge(t,this.model.renameSignal.bind(this.model));this.setModified()}}}class Sd extends dd{run(e){const t=[...e.children];if(!g(t,(e=>e instanceof gc))||e.numChildren()<=1)return;const n=[];let i;for(const r of t)if(r instanceof gc){let t=r;for(;1===t.numChildren();){const[e]=t.children;if(!(e instanceof gc))break;t=e}n.push(...t.children),i?(e.removeChild(r),r.parent=i.parent,i.parent.removeChild(i),i.parent=t,this.setModified()):i=t}else n.push(r);if(n.length){this.setModified();for(const e of n)e.parent.removeChild(e),e.parent=i}}}class Dd extends pc{clone(){return new Dd(null,l(this.transform))}constructor(e,t){super(e),this.transform=t}addDimensions(e){this.transform.groupby=b(this.transform.groupby.concat(e),(e=>e))}dependentFields(){const e=new Set;return this.transform.groupby&&this.transform.groupby.forEach(e.add,e),this.transform.joinaggregate.map((e=>e.field)).filter((e=>void 0!==e)).forEach(e.add,e),e}producedFields(){return new Set(this.transform.joinaggregate.map(this.getDefaultName))}getDefaultName(e){return e.as??ta(e)}hash(){return`JoinAggregateTransform ${d(this.transform)}`}assemble(){const e=[],t=[],n=[];for(const i of this.transform.joinaggregate)t.push(i.op),n.push(this.getDefaultName(i)),e.push(void 0===i.field?null:i.field);const i=this.transform.groupby;return{type:\"joinaggregate\",as:n,ops:t,fields:e,...void 0!==i?{groupby:i}:{}}}}class Fd extends pc{clone(){return new Fd(null,l(this._stack))}constructor(e,t){super(e),this._stack=t}static makeFromTransform(e,n){const{stack:i,groupby:r,as:o,offset:a=\"zero\"}=n,s=[],l=[];if(void 0!==n.sort)for(const e of n.sort)s.push(e.field),l.push(U(e.order,\"ascending\"));const c={field:s,order:l};let u;return u=function(e){return t.isArray(e)&&e.every((e=>t.isString(e)))&&e.length>1}(o)?o:t.isString(o)?[o,`${o}_end`]:[`${n.stack}_start`,`${n.stack}_end`],new Fd(e,{dimensionFieldDefs:[],stackField:i,groupby:r,offset:a,sort:c,facetby:[],as:u})}static makeFromEncoding(e,n){const i=n.stack,{encoding:r}=n;if(!i)return null;const{groupbyChannels:o,fieldChannel:a,offset:s,impute:l}=i,c=o.map((e=>ua(r[e]))).filter((e=>!!e)),u=function(e){return e.stack.stackBy.reduce(((e,t)=>{const n=ta(t.fieldDef);return n&&e.push(n),e}),[])}(n),f=n.encoding.order;let d;if(t.isArray(f)||Ro(f))d=Tn(f);else{const e=Mo(f)?f.sort:\"y\"===a?\"descending\":\"ascending\";d=u.reduce(((t,n)=>(t.field.includes(n)||(t.field.push(n),t.order.push(e)),t)),{field:[],order:[]})}return new Fd(e,{dimensionFieldDefs:c,stackField:n.vgField(a),facetby:[],stackby:u,sort:d,offset:s,impute:l,as:[n.vgField(a,{suffix:\"start\",forAs:!0}),n.vgField(a,{suffix:\"end\",forAs:!0})]})}get stack(){return this._stack}addDimensions(e){this._stack.facetby.push(...e)}dependentFields(){const e=new Set;return e.add(this._stack.stackField),this.getGroupbyFields().forEach(e.add,e),this._stack.facetby.forEach(e.add,e),this._stack.sort.field.forEach(e.add,e),e}producedFields(){return new Set(this._stack.as)}hash(){return`Stack ${d(this._stack)}`}getGroupbyFields(){const{dimensionFieldDefs:e,impute:t,groupby:n}=this._stack;return e.length>0?e.map((e=>e.bin?t?[ta(e,{binSuffix:\"mid\"})]:[ta(e,{}),ta(e,{binSuffix:\"end\"})]:[ta(e)])).flat():n??[]}assemble(){const e=[],{facetby:t,dimensionFieldDefs:n,stackField:i,stackby:r,sort:o,offset:a,impute:s,as:l}=this._stack;if(s)for(const o of n){const{bandPosition:n=.5,bin:a}=o;if(a){const t=ta(o,{expr:\"datum\"}),i=ta(o,{expr:\"datum\",binSuffix:\"end\"});e.push({type:\"formula\",expr:`${n}*${t}+${1-n}*${i}`,as:ta(o,{binSuffix:\"mid\",forAs:!0})})}e.push({type:\"impute\",field:i,groupby:[...r,...t],key:ta(o,{binSuffix:\"mid\"}),method:\"value\",value:0})}return e.push({type:\"stack\",groupby:[...this.getGroupbyFields(),...t],field:i,sort:o,as:l,offset:a}),e}}class zd extends pc{clone(){return new zd(null,l(this.transform))}constructor(e,t){super(e),this.transform=t}addDimensions(e){this.transform.groupby=b(this.transform.groupby.concat(e),(e=>e))}dependentFields(){const e=new Set;return(this.transform.groupby??[]).forEach(e.add,e),(this.transform.sort??[]).forEach((t=>e.add(t.field))),this.transform.window.map((e=>e.field)).filter((e=>void 0!==e)).forEach(e.add,e),e}producedFields(){return new Set(this.transform.window.map(this.getDefaultName))}getDefaultName(e){return e.as??ta(e)}hash(){return`WindowTransform ${d(this.transform)}`}assemble(){const e=[],t=[],n=[],i=[];for(const r of this.transform.window)t.push(r.op),n.push(this.getDefaultName(r)),i.push(void 0===r.param?null:r.param),e.push(void 0===r.field?null:r.field);const r=this.transform.frame,o=this.transform.groupby;if(r&&null===r[0]&&null===r[1]&&t.every((e=>tn(e))))return{type:\"joinaggregate\",as:n,ops:t,fields:e,...void 0!==o?{groupby:o}:{}};const a=[],s=[];if(void 0!==this.transform.sort)for(const e of this.transform.sort)a.push(e.field),s.push(e.order??\"ascending\");const l={field:a,order:s},c=this.transform.ignorePeers;return{type:\"window\",params:i,as:n,ops:t,fields:e,sort:l,...void 0!==c?{ignorePeers:c}:{},...void 0!==o?{groupby:o}:{},...void 0!==r?{frame:r}:{}}}}function Od(e){if(e instanceof td)if(1!==e.numChildren()||e.children[0]instanceof gc){const n=e.model.component.data.main;_d(n);const i=(t=e,function e(n){if(!(n instanceof td)){const i=n.clone();if(i instanceof gc){const e=Cd+i.getSource();i.setSource(e),t.model.component.data.outputNodes[e]=i}else(i instanceof ed||i instanceof Fd||i instanceof zd||i instanceof Dd)&&i.addDimensions(t.fields);for(const t of n.children.flatMap(e))t.parent=i;return[i]}return n.children.flatMap(e)}),r=e.children.map(i).flat();for(const e of r)e.parent=n}else{const t=e.children[0];(t instanceof ed||t instanceof Fd||t instanceof zd||t instanceof Dd)&&t.addDimensions(e.fields),t.swapWithParent(),Od(e)}else e.children.map(Od);var t}function _d(e){if(e instanceof gc&&e.type===sc.Main&&1===e.numChildren()){const t=e.children[0];t instanceof td||(t.swapWithParent(),_d(e))}}const Cd=\"scale_\",Nd=5;function Pd(e){for(const t of e){for(const e of t.children)if(e.parent!==t)return!1;if(!Pd(t.children))return!1}return!0}function Ad(e,t){let n=!1;for(const i of t)n=e.optimize(i)||n;return n}function jd(e,t,n){let i=e.sources,r=!1;return r=Ad(new yd,i)||r,r=Ad(new gd(t),i)||r,i=i.filter((e=>e.numChildren()>0)),r=Ad(new xd,i)||r,i=i.filter((e=>e.numChildren()>0)),n||(r=Ad(new vd,i)||r,r=Ad(new kd(t),i)||r,r=Ad(new hd,i)||r,r=Ad(new bd,i)||r,r=Ad(new wd,i)||r,r=Ad(new $d,i)||r,r=Ad(new pd,i)||r,r=Ad(new Sd,i)||r),e.sources=i,r}class Td{constructor(e){Object.defineProperty(this,\"signal\",{enumerable:!0,get:e})}static fromName(e,t){return new Td((()=>e(t)))}}function Ed(e){gm(e)?function(e){const t=e.component.scales;for(const n of D(t)){const i=Md(e,n);if(t[n].setWithExplicit(\"domains\",i),Rd(e,n),e.component.data.isFaceted){let t=e;for(;!hm(t)&&t.parent;)t=t.parent;if(\"shared\"===t.component.resolve.scale[n])for(const e of i.value)bn(e)&&(e.data=Cd+e.data.replace(Cd,\"\"))}}}(e):function(e){for(const t of e.children)Ed(t);const t=e.component.scales;for(const n of D(t)){let i,r=null;for(const t of e.children){const e=t.component.scales[n];if(e){i=void 0===i?e.getWithExplicit(\"domains\"):Kl(i,e.getWithExplicit(\"domains\"),\"domains\",\"scale\",Bd);const t=e.get(\"selectionExtent\");r&&t&&r.param!==t.param&&yi(Yn),r=t}}t[n].setWithExplicit(\"domains\",i),r&&t[n].set(\"selectionExtent\",r,!0)}}(e)}function Md(e,t){const n=e.getScaleComponent(t).get(\"type\"),{encoding:i}=e,r=function(e,t,n,i){if(\"unaggregated\"===e){const{valid:e,reason:i}=Wd(t,n);if(!e)return void yi(i)}else if(void 0===e&&i.useUnaggregatedDomain){const{valid:e}=Wd(t,n);if(e)return\"unaggregated\"}return e}(e.scaleDomain(t),e.typedFieldDef(t),n,e.config.scale);return r!==e.scaleDomain(t)&&(e.specifiedScales[t]={...e.specifiedScales[t],domain:r}),\"x\"===t&&fa(i.x2)?fa(i.x)?Kl(qd(n,r,e,\"x\"),qd(n,r,e,\"x2\"),\"domain\",\"scale\",Bd):qd(n,r,e,\"x2\"):\"y\"===t&&fa(i.y2)?fa(i.y)?Kl(qd(n,r,e,\"y\"),qd(n,r,e,\"y2\"),\"domain\",\"scale\",Bd):qd(n,r,e,\"y2\"):qd(n,r,e,t)}function Ld(e,t,n){const i=Ei(n)?.unit;return\"temporal\"===t||i?function(e,t,n){return e.map((e=>({signal:`{data: ${va(e,{timeUnit:n,type:t})}}`})))}(e,t,i):[e]}function qd(e,n,i,r){const{encoding:o,markDef:a,mark:s,config:l,stack:c}=i,u=fa(o[r]),{type:f}=u,d=u.timeUnit;if(function(e){return e?.unionWith}(n)){const t=qd(e,void 0,i,r);return Yl([...Ld(n.unionWith,f,d),...t.value])}if(yn(n))return Yl([n]);if(n&&\"unaggregated\"!==n&&!xr(n))return Yl(Ld(n,f,d));if(c&&r===c.fieldChannel){if(\"normalize\"===c.offset)return Xl([[0,1]]);const e=i.requestDataName(sc.Main);return Xl([{data:e,field:i.vgField(r,{suffix:\"start\"})},{data:e,field:i.vgField(r,{suffix:\"end\"})}])}const m=Ht(r)&&Ro(u)?function(e,t,n){if(!hr(n))return;const i=e.fieldDef(t),r=i.sort;if(Oo(r))return{op:\"min\",field:tf(i,t),order:\"ascending\"};const{stack:o}=e,a=o?new Set([...o.groupbyFields,...o.stackBy.map((e=>e.fieldDef.field))]):void 0;if(zo(r)){return Ud(r,o&&!a.has(r.field))}if(Fo(r)){const{encoding:t,order:n}=r,i=e.fieldDef(t),{aggregate:s,field:l}=i,c=o&&!a.has(l);if(Zt(s)||en(s))return Ud({field:ta(i),order:n},c);if(tn(s)||!s)return Ud({op:s,field:l,order:n},c)}else{if(\"descending\"===r)return{op:\"min\",field:e.vgField(t),order:\"descending\"};if(p([\"ascending\",void 0],r))return!0}return}(i,r,e):void 0;if(Bo(u)){return Xl(Ld([u.datum],f,d))}const g=u;if(\"unaggregated\"===n){const e=i.requestDataName(sc.Main),{field:t}=u;return Xl([{data:e,field:ta({field:t,aggregate:\"min\"})},{data:e,field:ta({field:t,aggregate:\"max\"})}])}if(ln(g.bin)){if(hr(e))return Xl(\"bin-ordinal\"===e?[]:[{data:O(m)?i.requestDataName(sc.Main):i.requestDataName(sc.Raw),field:i.vgField(r,xa(g,r)?{binSuffix:\"range\"}:{}),sort:!0!==m&&t.isObject(m)?m:{field:i.vgField(r,{}),op:\"min\"}}]);{const{bin:e}=g;if(ln(e)){const t=Qf(i,g.field,e);return Xl([new Td((()=>{const e=i.getSignalName(t);return`[${e}.start, ${e}.stop]`}))])}return Xl([{data:i.requestDataName(sc.Main),field:i.vgField(r,{})}])}}if(g.timeUnit&&p([\"time\",\"utc\"],e)){const e=o[it(r)];if(Eo(g,e,a,l)){const t=i.requestDataName(sc.Main),n=jo({fieldDef:g,fieldDef2:e,markDef:a,config:l}),o=mo(s)&&.5!==n&&zt(r);return Xl([{data:t,field:i.vgField(r,o?{suffix:bc}:{})},{data:t,field:i.vgField(r,{suffix:o?xc:\"end\"})}])}}return Xl(m?[{data:O(m)?i.requestDataName(sc.Main):i.requestDataName(sc.Raw),field:i.vgField(r),sort:m}]:[{data:i.requestDataName(sc.Main),field:i.vgField(r)}])}function Ud(e,t){const{op:n,field:i,order:r}=e;return{op:n??(t?\"sum\":ko),...i?{field:E(i)}:{},...r?{order:r}:{}}}function Rd(e,t){const n=e.component.scales[t],i=e.specifiedScales[t].domain,r=e.fieldDef(t)?.bin,o=xr(i)&&i,a=un(r)&&fn(r.extent)&&r.extent;(o||a)&&n.set(\"selectionExtent\",o??a,!0)}function Wd(e,n){const{aggregate:i,type:r}=e;return i?t.isString(i)&&!an.has(i)?{valid:!1,reason:si(i)}:\"quantitative\"===r&&\"log\"===n?{valid:!1,reason:li(e)}:{valid:!0}:{valid:!1,reason:ai(e)}}function Bd(e,t,n,i){return e.explicit&&t.explicit&&yi(function(e,t,n,i){return`Conflicting ${t.toString()} property \"${e.toString()}\" (${X(n)} and ${X(i)}). Using the union of the two domains.`}(n,i,e.value,t.value)),{explicit:e.explicit,value:[...e.value,...t.value]}}function Id(e){const n=b(e.map((e=>{if(bn(e)){const{sort:t,...n}=e;return n}return e})),d),i=b(e.map((e=>{if(bn(e)){const t=e.sort;return void 0===t||O(t)||(\"op\"in t&&\"count\"===t.op&&delete t.field,\"ascending\"===t.order&&delete t.order),t}})).filter((e=>void 0!==e)),d);if(0===n.length)return;if(1===n.length){const n=e[0];if(bn(n)&&i.length>0){let e=i[0];if(i.length>1){yi(fi);const n=i.filter((e=>t.isObject(e)&&\"op\"in e&&\"min\"!==e.op));e=!i.every((e=>t.isObject(e)&&\"op\"in e))||1!==n.length||n[0]}else if(t.isObject(e)&&\"field\"in e){const t=e.field;n.field===t&&(e=!e.order||{order:e.order})}return{...n,sort:e}}return n}const r=b(i.map((e=>O(e)||!(\"op\"in e)||t.isString(e.op)&&e.op in Kt?e:(yi(function(e){return`Dropping sort property ${X(e)} as unioned domains only support boolean or op \"count\", \"min\", and \"max\".`}(e)),!0))),d);let o;1===r.length?o=r[0]:r.length>1&&(yi(fi),o=!0);const a=b(e.map((e=>bn(e)?e.data:null)),(e=>e));if(1===a.length&&null!==a[0]){return{data:a[0],fields:n.map((e=>e.field)),...o?{sort:o}:{}}}return{fields:n,...o?{sort:o}:{}}}function Hd(e){if(bn(e)&&t.isString(e.field))return e.field;if(function(e){return!t.isArray(e)&&\"fields\"in e&&!(\"data\"in e)}(e)){let n;for(const i of e.fields)if(bn(i)&&t.isString(i.field))if(n){if(n!==i.field)return yi(\"Detected faceted independent scales that union domain of multiple fields from different data sources. We will use the first field. The result view size may be incorrect.\"),n}else n=i.field;return yi(\"Detected faceted independent scales that union domain of the same fields from different source. We will assume that this is the same field from a different fork of the same data source. However, if this is not the case, the result view size may be incorrect.\"),n}if(function(e){return!t.isArray(e)&&\"fields\"in e&&\"data\"in e}(e)){yi(\"Detected faceted independent scales that union domain of multiple fields from the same data source. We will use the first field. The result view size may be incorrect.\");const n=e.fields[0];return t.isString(n)?n:void 0}}function Vd(e,t){const n=e.component.scales[t].get(\"domains\").map((t=>(bn(t)&&(t.data=e.lookupDataSource(t.data)),t)));return Id(n)}function Gd(e){return vm(e)||ym(e)?e.children.reduce(((e,t)=>e.concat(Gd(t))),Yd(e)):Yd(e)}function Yd(e){return D(e.component.scales).reduce(((n,i)=>{const r=e.component.scales[i];if(r.merged)return n;const o=r.combine(),{name:a,type:s,selectionExtent:l,domains:c,range:u,reverse:f,...d}=o,m=function(e,n,i,r){if(zt(i)){if(vn(e))return{step:{signal:`${n}_step`}}}else if(t.isObject(e)&&bn(e))return{...e,data:r.lookupDataSource(e.data)};return e}(o.range,a,i,e),p=Vd(e,i),g=l?function(e,n,i,r){const o=Ru(e,n.param,n);return{signal:yr(i.get(\"type\"))&&t.isArray(r)&&r[0]>r[1]?`isValid(${o}) && reverse(${o})`:o}}(e,l,r,p):null;return n.push({name:a,type:s,...p?{domain:p}:{},...g?{domainRaw:g}:{},range:m,...void 0!==f?{reverse:f}:{},...d}),n}),[])}class Xd extends Gl{merged=!1;constructor(e,t){super({},{name:e}),this.setWithExplicit(\"type\",t)}domainDefinitelyIncludesZero(){return!1!==this.get(\"zero\")||g(this.get(\"domains\"),(e=>t.isArray(e)&&2===e.length&&t.isNumber(e[0])&&e[0]<=0&&t.isNumber(e[1])&&e[1]>=0))}}const Qd=[\"range\",\"scheme\"];function Jd(e,n){const i=e.fieldDef(n);if(i?.bin){const{bin:r,field:o}=i,a=rt(n),s=e.getName(a);if(t.isObject(r)&&r.binned&&void 0!==r.step)return new Td((()=>{const t=e.scaleName(n),i=`(domain(\"${t}\")[1] - domain(\"${t}\")[0]) / ${r.step}`;return`${e.getSignalName(s)} / (${i})`}));if(ln(r)){const t=Qf(e,o,r);return new Td((()=>{const n=e.getSignalName(t),i=`(${n}.stop - ${n}.start) / ${n}.step`;return`${e.getSignalName(s)} / (${i})`}))}}}function Kd(e,n){const i=n.specifiedScales[e],{size:r}=n,o=n.getScaleComponent(e).get(\"type\");for(const r of Qd)if(void 0!==i[r]){const a=_r(o,r),s=Cr(e,r);if(a)if(s)yi(s);else switch(r){case\"range\":{const r=i.range;if(t.isArray(r)){if(zt(e))return Yl(r.map((e=>{if(\"width\"===e||\"height\"===e){const t=n.getName(e),i=n.getSignalName.bind(n);return Td.fromName(i,t)}return e})))}else if(t.isObject(r))return Yl({data:n.requestDataName(sc.Main),field:r.field,sort:{op:\"min\",field:n.vgField(e)}});return Yl(r)}case\"scheme\":return Yl(Zd(i[r]))}else yi(ci(o,r,e))}const a=e===Z||\"xOffset\"===e?\"width\":\"height\",s=r[a];if(Fs(s))if(zt(e))if(hr(o)){const t=tm(s,n,e);if(t)return Yl({step:t})}else yi(ui(a));else if(Pt(e)){const t=e===ie?\"x\":\"y\";if(\"band\"===n.getScaleComponent(t).get(\"type\")){const e=nm(s,o);if(e)return Yl(e)}}const{rangeMin:l,rangeMax:u}=i,f=function(e,n){const{size:i,config:r,mark:o,encoding:a}=n,{type:s}=fa(a[e]),l=n.getScaleComponent(e),u=l.get(\"type\"),{domain:f,domainMid:d}=n.specifiedScales[e];switch(e){case Z:case ee:if(p([\"point\",\"band\"],u)){const t=im(e,i,r.view);if(Fs(t)){return{step:tm(t,n,e)}}}return em(e,n,u);case ie:case re:return function(e,t,n){const i=e===ie?\"x\":\"y\",r=t.getScaleComponent(i);if(!r)return em(i,t,n,{center:!0});const o=r.get(\"type\"),a=t.scaleName(i),{markDef:s,config:l}=t;if(\"band\"===o){const e=im(i,t.size,t.config.view);if(Fs(e)){const t=nm(e,n);if(t)return t}return[0,{signal:`bandwidth('${a}')`}]}{const n=t.encoding[i];if(Ro(n)&&n.timeUnit){const e=Mi(n.timeUnit,(e=>`scale('${a}', ${e})`)),i=t.config.scale.bandWithNestedOffsetPaddingInner,r=jo({fieldDef:n,markDef:s,config:l})-.5,o=0!==r?` + ${r}`:\"\";if(i){return[{signal:`${yn(i)?`${i.signal}/2`+o:`${i/2+r}`} * (${e})`},{signal:`${yn(i)?`(1 - ${i.signal}/2)`+o:`${1-i/2+r}`} * (${e})`}]}return[0,{signal:e}]}return c(`Cannot use ${e} scale if ${i} scale is not discrete.`)}}(e,n,u);case ye:{const a=rm(o,n.component.scales[e].get(\"zero\"),r),s=function(e,n,i,r){const o={x:Jd(i,\"x\"),y:Jd(i,\"y\")};switch(e){case\"bar\":case\"tick\":{if(void 0!==r.scale.maxBandSize)return r.scale.maxBandSize;const e=am(n,o,r.view);return t.isNumber(e)?e-1:new Td((()=>`${e.signal} - 1`))}case\"line\":case\"trail\":case\"rule\":return r.scale.maxStrokeWidth;case\"text\":return r.scale.maxFontSize;case\"point\":case\"square\":case\"circle\":{if(r.scale.maxSize)return r.scale.maxSize;const e=am(n,o,r.view);return t.isNumber(e)?Math.pow(om*e,2):new Td((()=>`pow(${om} * ${e.signal}, 2)`))}}throw new Error(ni(\"size\",e))}(o,i,n,r);return br(u)?function(e,t,n){const i=()=>{const i=On(t),r=On(e),o=`(${i} - ${r}) / (${n} - 1)`;return`sequence(${r}, ${i} + ${o}, ${o})`};return yn(t)?new Td(i):{signal:i()}}(a,s,function(e,n,i,r){switch(e){case\"quantile\":return n.scale.quantileCount;case\"quantize\":return n.scale.quantizeCount;case\"threshold\":return void 0!==i&&t.isArray(i)?i.length+1:(yi(function(e){return`Domain for ${e} is required for threshold scale.`}(r)),3)}}(u,r,f,e)):[a,s]}case se:return[0,2*Math.PI];case ve:return[0,360];case oe:return[0,new Td((()=>`min(${n.getSignalName(hm(n.parent)?\"child_width\":\"width\")},${n.getSignalName(hm(n.parent)?\"child_height\":\"height\")})/2`))];case we:return[r.scale.minStrokeWidth,r.scale.maxStrokeWidth];case ke:return[[1,0],[4,2],[2,1],[1,1],[1,2,4,2]];case he:return\"symbol\";case me:case pe:case ge:return\"ordinal\"===u?\"nominal\"===s?\"category\":\"ordinal\":void 0!==d?\"diverging\":\"rect\"===o||\"geoshape\"===o?\"heatmap\":\"ramp\";case be:case xe:case $e:return[r.scale.minOpacity,r.scale.maxOpacity]}}(e,n);return(void 0!==l||void 0!==u)&&_r(o,\"rangeMin\")&&t.isArray(f)&&2===f.length?Yl([l??f[0],u??f[1]]):Xl(f)}function Zd(e){return function(e){return!t.isString(e)&&!!e.name}(e)?{scheme:e.name,...f(e,[\"name\"])}:{scheme:e}}function em(e,t,n){let{center:i}=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{};const r=rt(e),o=t.getName(r),a=t.getSignalName.bind(t);return e===ee&&yr(n)?i?[Td.fromName((e=>`${a(e)}/2`),o),Td.fromName((e=>`-${a(e)}/2`),o)]:[Td.fromName(a,o),0]:i?[Td.fromName((e=>`-${a(e)}/2`),o),Td.fromName((e=>`${a(e)}/2`),o)]:[0,Td.fromName(a,o)]}function tm(e,n,i){const{encoding:r}=n,o=n.getScaleComponent(i),a=at(i),s=r[a];if(\"offset\"===Ds({step:e,offsetIsDiscrete:Go(s)&&Zi(s.type)})&&Pa(r,a)){const i=n.getScaleComponent(a);let r=`domain('${n.scaleName(a)}').length`;if(\"band\"===i.get(\"type\")){r=`bandspace(${r}, ${i.get(\"paddingInner\")??i.get(\"padding\")??0}, ${i.get(\"paddingOuter\")??i.get(\"padding\")??0})`}const s=o.get(\"paddingInner\")??o.get(\"padding\");return{signal:`${e.step} * ${r} / (1-${l=s,yn(l)?l.signal:t.stringValue(l)})`}}return e.step;var l}function nm(e,t){if(\"offset\"===Ds({step:e,offsetIsDiscrete:hr(t)}))return{step:e.step}}function im(e,t,n){const i=e===Z?\"width\":\"height\",r=t[i];return r||Ns(n,i)}function rm(e,t,n){if(t)return yn(t)?{signal:`${t.signal} ? 0 : ${rm(e,!1,n)}`}:0;switch(e){case\"bar\":case\"tick\":return n.scale.minBandSize;case\"line\":case\"trail\":case\"rule\":return n.scale.minStrokeWidth;case\"text\":return n.scale.minFontSize;case\"point\":case\"square\":case\"circle\":return n.scale.minSize}throw new Error(ni(\"size\",e))}const om=.95;function am(e,t,n){const i=Fs(e.width)?e.width.step:Cs(n,\"width\"),r=Fs(e.height)?e.height.step:Cs(n,\"height\");return t.x||t.y?new Td((()=>`min(${[t.x?t.x.signal:i,t.y?t.y.signal:r].join(\", \")})`)):Math.min(i,r)}function sm(e,t){gm(e)?function(e,t){const n=e.component.scales,{config:i,encoding:r,markDef:o,specifiedScales:a}=e;for(const s of D(n)){const l=a[s],c=n[s],u=e.getScaleComponent(s),f=fa(r[s]),d=l[t],m=u.get(\"type\"),p=u.get(\"padding\"),g=u.get(\"paddingInner\"),h=_r(m,t),y=Cr(s,t);if(void 0!==d&&(h?y&&yi(y):yi(ci(m,t,s))),h&&void 0===y)if(void 0!==d){const e=f.timeUnit,n=f.type;switch(t){case\"domainMax\":case\"domainMin\":vi(l[t])||\"temporal\"===n||e?c.set(t,{signal:va(l[t],{type:n,timeUnit:e})},!0):c.set(t,l[t],!0);break;default:c.copyKeyFromObject(t,l)}}else{const n=t in lm?lm[t]({model:e,channel:s,fieldOrDatumDef:f,scaleType:m,scalePadding:p,scalePaddingInner:g,domain:l.domain,domainMin:l.domainMin,domainMax:l.domainMax,markDef:o,config:i,hasNestedOffsetScale:Aa(r,s),hasSecondaryRangeChannel:!!r[it(s)]}):i.scale[t];void 0!==n&&c.set(t,n,!1)}}}(e,t):um(e,t)}const lm={bins:e=>{let{model:t,fieldOrDatumDef:n}=e;return Ro(n)?function(e,t){const n=t.bin;if(ln(n)){const i=Qf(e,t.field,n);return new Td((()=>e.getSignalName(i)))}if(cn(n)&&un(n)&&void 0!==n.step)return{step:n.step};return}(t,n):void 0},interpolate:e=>{let{channel:t,fieldOrDatumDef:n}=e;return function(e,t){if(p([me,pe,ge],e)&&\"nominal\"!==t)return\"hcl\";return}(t,n.type)},nice:e=>{let{scaleType:n,channel:i,domain:r,domainMin:o,domainMax:a,fieldOrDatumDef:s}=e;return function(e,n,i,r,o,a){if(ua(a)?.bin||t.isArray(i)||null!=o||null!=r||p([or.TIME,or.UTC],e))return;return!!zt(n)||void 0}(n,i,r,o,a,s)},padding:e=>{let{channel:t,scaleType:n,fieldOrDatumDef:i,markDef:r,config:o}=e;return function(e,t,n,i,r,o){if(zt(e)){if(vr(t)){if(void 0!==n.continuousPadding)return n.continuousPadding;const{type:t,orient:a}=r;if(\"bar\"===t&&(!Ro(i)||!i.bin&&!i.timeUnit)&&(\"vertical\"===a&&\"x\"===e||\"horizontal\"===a&&\"y\"===e))return o.continuousBandSize}if(t===or.POINT)return n.pointPadding}return}(t,n,o.scale,i,r,o.bar)},paddingInner:e=>{let{scalePadding:t,channel:n,markDef:i,scaleType:r,config:o,hasNestedOffsetScale:a}=e;return function(e,t,n,i,r){let o=arguments.length>5&&void 0!==arguments[5]&&arguments[5];if(void 0!==e)return;if(zt(t)){const{bandPaddingInner:e,barBandPaddingInner:t,rectBandPaddingInner:i,bandWithNestedOffsetPaddingInner:a}=r;return o?a:U(e,\"bar\"===n?t:i)}if(Pt(t)&&i===or.BAND)return r.offsetBandPaddingInner;return}(t,n,i.type,r,o.scale,a)},paddingOuter:e=>{let{scalePadding:t,channel:n,scaleType:i,scalePaddingInner:r,config:o,hasNestedOffsetScale:a}=e;return function(e,t,n,i,r){let o=arguments.length>5&&void 0!==arguments[5]&&arguments[5];if(void 0!==e)return;if(zt(t)){const{bandPaddingOuter:e,bandWithNestedOffsetPaddingOuter:t}=r;if(o)return t;if(n===or.BAND)return U(e,yn(i)?{signal:`${i.signal}/2`}:i/2)}else if(Pt(t)){if(n===or.POINT)return.5;if(n===or.BAND)return r.offsetBandPaddingOuter}return}(t,n,i,r,o.scale,a)},reverse:e=>{let{fieldOrDatumDef:t,scaleType:n,channel:i,config:r}=e;return function(e,t,n,i){if(\"x\"===n&&void 0!==i.xReverse)return yr(e)&&\"descending\"===t?yn(i.xReverse)?{signal:`!${i.xReverse.signal}`}:!i.xReverse:i.xReverse;if(yr(e)&&\"descending\"===t)return!0;return}(n,Ro(t)?t.sort:void 0,i,r.scale)},zero:e=>{let{channel:n,fieldOrDatumDef:i,domain:r,markDef:o,scaleType:a,config:s,hasSecondaryRangeChannel:l}=e;return function(e,n,i,r,o,a,s){if(i&&\"unaggregated\"!==i&&yr(o)){if(t.isArray(i)){const e=i[0],n=i[i.length-1];if(t.isNumber(e)&&e<=0&&t.isNumber(n)&&n>=0)return!0}return!1}if(\"size\"===e&&\"quantitative\"===n.type&&!br(o))return!0;if((!Ro(n)||!n.bin)&&p([...Ft,..._t],e)){const{orient:t,type:n}=r;return(!p([\"bar\",\"area\",\"line\",\"trail\"],n)||!(\"horizontal\"===t&&\"y\"===e||\"vertical\"===t&&\"x\"===e))&&(!(!p([\"bar\",\"area\"],n)||s)||a?.zero)}return!1}(n,i,r,o,a,s.scale,l)}};function cm(e){gm(e)?function(e){const t=e.component.scales;for(const n of It){const i=t[n];if(!i)continue;const r=Kd(n,e);i.setWithExplicit(\"range\",r)}}(e):um(e,\"range\")}function um(e,t){const n=e.component.scales;for(const n of e.children)\"range\"===t?cm(n):sm(n,t);for(const i of D(n)){let r;for(const n of e.children){const e=n.component.scales[i];if(e){r=Kl(r,e.getWithExplicit(t),t,\"scale\",Ql(((e,n)=>\"range\"===t&&e.step&&n.step?e.step-n.step:0)))}}n[i].setWithExplicit(t,r)}}function fm(e,t,n,i){const r=function(e,t,n,i){switch(t.type){case\"nominal\":case\"ordinal\":if(qe(e)||\"discrete\"===Qt(e))return\"shape\"===e&&\"ordinal\"===t.type&&yi(oi(e,\"ordinal\")),\"ordinal\";if(zt(e)||Pt(e)){if(p([\"rect\",\"bar\",\"image\",\"rule\"],n.type))return\"band\";if(i)return\"band\"}else if(\"arc\"===n.type&&e in Ot)return\"band\";return bo(n[rt(e)])||Jo(t)&&t.axis?.tickBand?\"band\":\"point\";case\"temporal\":return qe(e)?\"time\":\"discrete\"===Qt(e)?(yi(oi(e,\"temporal\")),\"ordinal\"):Ro(t)&&t.timeUnit&&Ei(t.timeUnit).utc?\"utc\":\"time\";case\"quantitative\":return qe(e)?Ro(t)&&ln(t.bin)?\"bin-ordinal\":\"linear\":\"discrete\"===Qt(e)?(yi(oi(e,\"quantitative\")),\"ordinal\"):\"linear\";case\"geojson\":return}throw new Error(Zn(t.type))}(t,n,i,arguments.length>4&&void 0!==arguments[4]&&arguments[4]),{type:o}=e;return Ht(t)?void 0!==o?function(e,t){let n=arguments.length>2&&void 0!==arguments[2]&&arguments[2];if(!Ht(e))return!1;switch(e){case Z:case ee:case ie:case re:case se:case oe:return!!vr(t)||\"band\"===t||\"point\"===t&&!n;case ye:case we:case be:case xe:case $e:case ve:return vr(t)||br(t)||p([\"band\",\"point\",\"ordinal\"],t);case me:case pe:case ge:return\"band\"!==t;case ke:case he:return\"ordinal\"===t||br(t)}}(t,o)?Ro(n)&&(a=o,s=n.type,!(p([tr,ir],s)?void 0===a||hr(a):s===nr?p([or.TIME,or.UTC,void 0],a):s!==er||dr(a)||br(a)||void 0===a))?(yi(function(e,t){return`FieldDef does not work with \"${e}\" scale. We are using \"${t}\" scale instead.`}(o,r)),r):o:(yi(function(e,t,n){return`Channel \"${e}\" does not work with \"${t}\" scale. We are using \"${n}\" scale instead.`}(t,o,r)),r):r:null;var a,s}function dm(e){gm(e)?e.component.scales=function(e){const{encoding:t,mark:n,markDef:i}=e,r={};for(const o of It){const a=fa(t[o]);if(a&&n===uo&&o===he&&a.type===rr)continue;let s=a&&a.scale;if(a&&null!==s&&!1!==s){s??={};const n=fm(s,o,a,i,Aa(t,o));r[o]=new Xd(e.scaleName(`${o}`,!0),{value:n,explicit:s.type===n})}}return r}(e):e.component.scales=function(e){const t=e.component.scales={},n={},i=e.component.resolve;for(const t of e.children){dm(t);for(const r of D(t.component.scales))if(i.scale[r]??=Sf(r,e),\"shared\"===i.scale[r]){const e=n[r],o=t.component.scales[r].getWithExplicit(\"type\");e?sr(e.value,o.value)?n[r]=Kl(e,o,\"type\",\"scale\",mm):(i.scale[r]=\"independent\",delete n[r]):n[r]=o}}for(const i of D(n)){const r=e.scaleName(i,!0),o=n[i];t[i]=new Xd(r,o);for(const t of e.children){const e=t.component.scales[i];e&&(t.renameScale(e.get(\"name\"),r),e.merged=!0)}}return t}(e)}const mm=Ql(((e,t)=>cr(e)-cr(t)));class pm{constructor(){this.nameMap={}}rename(e,t){this.nameMap[e]=t}has(e){return void 0!==this.nameMap[e]}get(e){for(;this.nameMap[e]&&e!==this.nameMap[e];)e=this.nameMap[e];return e}}function gm(e){return\"unit\"===e?.type}function hm(e){return\"facet\"===e?.type}function ym(e){return\"concat\"===e?.type}function vm(e){return\"layer\"===e?.type}class bm{constructor(e,n,i,r,o,a,c){this.type=n,this.parent=i,this.config=o,this.parent=i,this.config=o,this.view=pn(c),this.name=e.name??r,this.title=hn(e.title)?{text:e.title}:e.title?pn(e.title):void 0,this.scaleNameMap=i?i.scaleNameMap:new pm,this.projectionNameMap=i?i.projectionNameMap:new pm,this.signalNameMap=i?i.signalNameMap:new pm,this.data=e.data,this.description=e.description,this.transforms=(e.transform??[]).map((e=>gl(e)?{filter:s(e.filter,Ji)}:e)),this.layout=\"layer\"===n||\"unit\"===n?{}:function(e,n,i){const r=i[n],o={},{spacing:a,columns:s}=r;void 0!==a&&(o.spacing=a),void 0!==s&&(No(e)&&!_o(e.facet)||ws(e))&&(o.columns=s),ks(e)&&(o.columns=1);for(const n of Os)if(void 0!==e[n])if(\"spacing\"===n){const i=e[n];o[n]=t.isNumber(i)?i:{row:i.row??a,column:i.column??a}}else o[n]=e[n];return o}(e,n,o),this.component={data:{sources:i?i.component.data.sources:[],outputNodes:i?i.component.data.outputNodes:{},outputNodeRefCounts:i?i.component.data.outputNodeRefCounts:{},isFaceted:No(e)||i?.component.data.isFaceted&&void 0===e.data},layoutSize:new Gl,layoutHeaders:{row:{},column:{},facet:{}},mark:null,resolve:{scale:{},axis:{},legend:{},...a?l(a):{}},selection:null,scales:null,projection:null,axes:{},legends:{}}}get width(){return this.getSizeSignalRef(\"width\")}get height(){return this.getSizeSignalRef(\"height\")}parse(){this.parseScale(),this.parseLayoutSize(),this.renameTopLevelLayoutSizeSignal(),this.parseSelections(),this.parseProjection(),this.parseData(),this.parseAxesAndHeaders(),this.parseLegends(),this.parseMarkGroup()}parseScale(){!function(e){let{ignoreRange:t}=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};dm(e),Ed(e);for(const t of Or)sm(e,t);t||cm(e)}(this)}parseProjection(){Gf(this)}renameTopLevelLayoutSizeSignal(){\"width\"!==this.getName(\"width\")&&this.renameSignal(this.getName(\"width\"),\"width\"),\"height\"!==this.getName(\"height\")&&this.renameSignal(this.getName(\"height\"),\"height\")}parseLegends(){Mf(this)}assembleEncodeFromView(e){const{style:t,...n}=e,i={};for(const e of D(n)){const t=n[e];void 0!==t&&(i[e]=Fn(t))}return i}assembleGroupEncodeEntry(e){let t={};return this.view&&(t=this.assembleEncodeFromView(this.view)),e||(this.description&&(t.description=Fn(this.description)),\"unit\"!==this.type&&\"layer\"!==this.type)?S(t)?void 0:t:{width:this.getSizeSignalRef(\"width\"),height:this.getSizeSignalRef(\"height\"),...t}}assembleLayout(){if(!this.layout)return;const{spacing:e,...t}=this.layout,{component:n,config:i}=this,r=function(e,t){const n={};for(const i of Re){const r=e[i];if(r?.facetFieldDef){const{titleAnchor:e,titleOrient:o}=of([\"titleAnchor\",\"titleOrient\"],r.facetFieldDef.header,t,i),a=nf(i,o),s=hf(e,a);void 0!==s&&(n[a]=s)}}return S(n)?void 0:n}(n.layoutHeaders,i);return{padding:e,...this.assembleDefaultLayout(),...t,...r?{titleBand:r}:{}}}assembleDefaultLayout(){return{}}assembleHeaderMarks(){const{layoutHeaders:e}=this.component;let t=[];for(const n of Re)e[n].title&&t.push(lf(this,n));for(const e of af)t=t.concat(ff(this,e));return t}assembleAxes(){return function(e,t){const{x:n=[],y:i=[]}=e;return[...n.map((e=>Iu(e,\"grid\",t))),...i.map((e=>Iu(e,\"grid\",t))),...n.map((e=>Iu(e,\"main\",t))),...i.map((e=>Iu(e,\"main\",t)))].filter((e=>e))}(this.component.axes,this.config)}assembleLegends(){return Wf(this)}assembleProjections(){return Bf(this)}assembleTitle(){const{encoding:e,...t}=this.title??{},n={...gn(this.config.title).nonMarkTitleProperties,...t,...e?{encode:{update:e}}:{}};if(n.text)return p([\"unit\",\"layer\"],this.type)?p([\"middle\",void 0],n.anchor)&&(n.frame??=\"group\"):n.anchor??=\"start\",S(n)?void 0:n}assembleGroup(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:[];const t={};e=e.concat(this.assembleSignals()),e.length>0&&(t.signals=e);const n=this.assembleLayout();n&&(t.layout=n),t.marks=[].concat(this.assembleHeaderMarks(),this.assembleMarks());const i=!this.parent||hm(this.parent)?Gd(this):[];i.length>0&&(t.scales=i);const r=this.assembleAxes();r.length>0&&(t.axes=r);const o=this.assembleLegends();return o.length>0&&(t.legends=o),t}getName(e){return _((this.name?`${this.name}_`:\"\")+e)}getDataName(e){return this.getName(sc[e].toLowerCase())}requestDataName(e){const t=this.getDataName(e),n=this.component.data.outputNodeRefCounts;return n[t]=(n[t]||0)+1,t}getSizeSignalRef(e){if(hm(this.parent)){const t=Ct(wf(e)),n=this.component.scales[t];if(n&&!n.merged){const e=n.get(\"type\"),i=n.get(\"range\");if(hr(e)&&vn(i)){const e=n.get(\"name\"),i=Hd(Vd(this,t));if(i){return{signal:$f(e,n,ta({aggregate:\"distinct\",field:i},{expr:\"datum\"}))}}return yi(In(t)),null}}}return{signal:this.signalNameMap.get(this.getName(e))}}lookupDataSource(e){const t=this.component.data.outputNodes[e];return t?t.getSource():e}getSignalName(e){return this.signalNameMap.get(e)}renameSignal(e,t){this.signalNameMap.rename(e,t)}renameScale(e,t){this.scaleNameMap.rename(e,t)}renameProjection(e,t){this.projectionNameMap.rename(e,t)}scaleName(e,t){return t?this.getName(e):Ke(e)&&Ht(e)&&this.component.scales[e]||this.scaleNameMap.has(this.getName(e))?this.scaleNameMap.get(this.getName(e)):void 0}projectionName(e){return e?this.getName(\"projection\"):this.component.projection&&!this.component.projection.merged||this.projectionNameMap.has(this.getName(\"projection\"))?this.projectionNameMap.get(this.getName(\"projection\")):void 0}correctDataNames=e=>(e.from?.data&&(e.from.data=this.lookupDataSource(e.from.data)),e.from?.facet?.data&&(e.from.facet.data=this.lookupDataSource(e.from.facet.data)),e);getScaleComponent(e){if(!this.component.scales)throw new Error(\"getScaleComponent cannot be called before parseScale(). Make sure you have called parseScale or use parseUnitModelWithScale().\");const t=this.component.scales[e];return t&&!t.merged?t:this.parent?this.parent.getScaleComponent(e):void 0}getSelectionComponent(e,t){let n=this.component.selection[e];if(!n&&this.parent&&(n=this.parent.getSelectionComponent(e,t)),!n)throw new Error(function(e){return`Cannot find a selection named \"${e}\".`}(t));return n}hasAxisOrientSignalRef(){return this.component.axes.x?.some((e=>e.hasOrientSignalRef()))||this.component.axes.y?.some((e=>e.hasOrientSignalRef()))}}class xm extends bm{vgField(e){let t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};const n=this.fieldDef(e);if(n)return ta(n,t)}reduceFieldDef(e,n){return function(e,n,i,r){return e?D(e).reduce(((i,o)=>{const a=e[o];return t.isArray(a)?a.reduce(((e,t)=>n.call(r,e,t,o)),i):n.call(r,i,a,o)}),i):i}(this.getMapping(),((t,n,i)=>{const r=ua(n);return r?e(t,r,i):t}),n)}forEachFieldDef(e,t){La(this.getMapping(),((t,n)=>{const i=ua(t);i&&e(i,n)}),t)}}class $m extends pc{clone(){return new $m(null,l(this.transform))}constructor(e,t){super(e),this.transform=t,this.transform=l(t);const n=this.transform.as??[void 0,void 0];this.transform.as=[n[0]??\"value\",n[1]??\"density\"];const i=this.transform.resolve??\"shared\";this.transform.resolve=i}dependentFields(){return new Set([this.transform.density,...this.transform.groupby??[]])}producedFields(){return new Set(this.transform.as)}hash(){return`DensityTransform ${d(this.transform)}`}assemble(){const{density:e,...t}=this.transform,n={type:\"kde\",field:e,...t};return n.resolve=this.transform.resolve,n}}class wm extends pc{clone(){return new wm(null,l(this.transform))}constructor(e,t){super(e),this.transform=t,this.transform=l(t)}dependentFields(){return new Set([this.transform.extent])}producedFields(){return new Set([])}hash(){return`ExtentTransform ${d(this.transform)}`}assemble(){const{extent:e,param:t}=this.transform;return{type:\"extent\",field:e,signal:t}}}class km extends pc{clone(){return new km(null,{...this.filter})}constructor(e,t){super(e),this.filter=t}static make(e,t){const{config:n,mark:i,markDef:r}=t;if(\"filter\"!==Nn(\"invalid\",r,n))return null;const o=t.reduceFieldDef(((e,n,r)=>{const o=Ht(r)&&t.getScaleComponent(r);if(o){yr(o.get(\"type\"))&&\"count\"!==n.aggregate&&!fo(i)&&(e[n.field]=n)}return e}),{});return D(o).length?new km(e,o):null}dependentFields(){return new Set(D(this.filter))}producedFields(){return new Set}hash(){return`FilterInvalid ${d(this.filter)}`}assemble(){const e=D(this.filter).reduce(((e,t)=>{const n=this.filter[t],i=ta(n,{expr:\"datum\"});return null!==n&&(\"temporal\"===n.type?e.push(`(isDate(${i}) || (isValid(${i}) && isFinite(+${i})))`):\"quantitative\"===n.type&&(e.push(`isValid(${i})`),e.push(`isFinite(+${i})`))),e}),[]);return e.length>0?{type:\"filter\",expr:e.join(\" && \")}:null}}class Sm extends pc{clone(){return new Sm(this.parent,l(this.transform))}constructor(e,t){super(e),this.transform=t,this.transform=l(t);const{flatten:n,as:i=[]}=this.transform;this.transform.as=n.map(((e,t)=>i[t]??e))}dependentFields(){return new Set(this.transform.flatten)}producedFields(){return new Set(this.transform.as)}hash(){return`FlattenTransform ${d(this.transform)}`}assemble(){const{flatten:e,as:t}=this.transform;return{type:\"flatten\",fields:e,as:t}}}class Dm extends pc{clone(){return new Dm(null,l(this.transform))}constructor(e,t){super(e),this.transform=t,this.transform=l(t);const n=this.transform.as??[void 0,void 0];this.transform.as=[n[0]??\"key\",n[1]??\"value\"]}dependentFields(){return new Set(this.transform.fold)}producedFields(){return new Set(this.transform.as)}hash(){return`FoldTransform ${d(this.transform)}`}assemble(){const{fold:e,as:t}=this.transform;return{type:\"fold\",fields:e,as:t}}}class Fm extends pc{clone(){return new Fm(null,l(this.fields),this.geojson,this.signal)}static parseAll(e,t){if(t.component.projection&&!t.component.projection.isFit)return e;let n=0;for(const i of[[ue,ce],[de,fe]]){const r=i.map((e=>{const n=fa(t.encoding[e]);return Ro(n)?n.field:Bo(n)?{expr:`${n.datum}`}:Xo(n)?{expr:`${n.value}`}:void 0}));(r[0]||r[1])&&(e=new Fm(e,r,null,t.getName(\"geojson_\"+n++)))}if(t.channelHasField(he)){const i=t.typedFieldDef(he);i.type===rr&&(e=new Fm(e,null,i.field,t.getName(\"geojson_\"+n++)))}return e}constructor(e,t,n,i){super(e),this.fields=t,this.geojson=n,this.signal=i}dependentFields(){const e=(this.fields??[]).filter(t.isString);return new Set([...this.geojson?[this.geojson]:[],...e])}producedFields(){return new Set}hash(){return`GeoJSON ${this.geojson} ${this.signal} ${d(this.fields)}`}assemble(){return[...this.geojson?[{type:\"filter\",expr:`isValid(datum[\"${this.geojson}\"])`}]:[],{type:\"geojson\",...this.fields?{fields:this.fields}:{},...this.geojson?{geojson:this.geojson}:{},signal:this.signal}]}}class zm extends pc{clone(){return new zm(null,this.projection,l(this.fields),l(this.as))}constructor(e,t,n,i){super(e),this.projection=t,this.fields=n,this.as=i}static parseAll(e,t){if(!t.projectionName())return e;for(const n of[[ue,ce],[de,fe]]){const i=n.map((e=>{const n=fa(t.encoding[e]);return Ro(n)?n.field:Bo(n)?{expr:`${n.datum}`}:Xo(n)?{expr:`${n.value}`}:void 0})),r=n[0]===de?\"2\":\"\";(i[0]||i[1])&&(e=new zm(e,t.projectionName(),i,[t.getName(`x${r}`),t.getName(`y${r}`)]))}return e}dependentFields(){return new Set(this.fields.filter(t.isString))}producedFields(){return new Set(this.as)}hash(){return`Geopoint ${this.projection} ${d(this.fields)} ${d(this.as)}`}assemble(){return{type:\"geopoint\",projection:this.projection,fields:this.fields,as:this.as}}}class Om extends pc{clone(){return new Om(null,l(this.transform))}constructor(e,t){super(e),this.transform=t}dependentFields(){return new Set([this.transform.impute,this.transform.key,...this.transform.groupby??[]])}producedFields(){return new Set([this.transform.impute])}processSequence(e){const{start:t=0,stop:n,step:i}=e;return{signal:`sequence(${[t,n,...i?[i]:[]].join(\",\")})`}}static makeFromTransform(e,t){return new Om(e,t)}static makeFromEncoding(e,t){const n=t.encoding,i=n.x,r=n.y;if(Ro(i)&&Ro(r)){const o=i.impute?i:r.impute?r:void 0;if(void 0===o)return;const a=i.impute?r:r.impute?i:void 0,{method:s,value:l,frame:c,keyvals:u}=o.impute,f=qa(t.mark,n);return new Om(e,{impute:o.field,key:a.field,...s?{method:s}:{},...void 0!==l?{value:l}:{},...c?{frame:c}:{},...void 0!==u?{keyvals:u}:{},...f.length?{groupby:f}:{}})}return null}hash(){return`Impute ${d(this.transform)}`}assemble(){const{impute:e,key:t,keyvals:n,method:i,groupby:r,value:o,frame:a=[null,null]}=this.transform,s={type:\"impute\",field:e,key:t,...n?{keyvals:(l=n,void 0!==l?.stop?this.processSequence(n):n)}:{},method:\"value\",...r?{groupby:r}:{},value:i&&\"value\"!==i?null:o};var l;if(i&&\"value\"!==i){return[s,{type:\"window\",as:[`imputed_${e}_value`],ops:[i],fields:[e],frame:a,ignorePeers:!1,...r?{groupby:r}:{}},{type:\"formula\",expr:`datum.${e} === null ? datum.imputed_${e}_value : datum.${e}`,as:e}]}return[s]}}class _m extends pc{clone(){return new _m(null,l(this.transform))}constructor(e,t){super(e),this.transform=t,this.transform=l(t);const n=this.transform.as??[void 0,void 0];this.transform.as=[n[0]??t.on,n[1]??t.loess]}dependentFields(){return new Set([this.transform.loess,this.transform.on,...this.transform.groupby??[]])}producedFields(){return new Set(this.transform.as)}hash(){return`LoessTransform ${d(this.transform)}`}assemble(){const{loess:e,on:t,...n}=this.transform;return{type:\"loess\",x:t,y:e,...n}}}class Cm extends pc{clone(){return new Cm(null,l(this.transform),this.secondary)}constructor(e,t,n){super(e),this.transform=t,this.secondary=n}static make(e,t,n,i){const r=t.component.data.sources,{from:o}=n;let a=null;if(function(e){return\"data\"in e}(o)){let e=Hm(o.data,r);e||(e=new cd(o.data),r.push(e));const n=t.getName(`lookup_${i}`);a=new gc(e,n,sc.Lookup,t.component.data.outputNodeRefCounts),t.component.data.outputNodes[n]=a}else if(function(e){return\"param\"in e}(o)){const e=o.param;let i;n={as:e,...n};try{i=t.getSelectionComponent(_(e),e)}catch(t){throw new Error(function(e){return`Lookups can only be performed on selection parameters. \"${e}\" is a variable parameter.`}(e))}if(a=i.materialized,!a)throw new Error(function(e){return`Cannot define and lookup the \"${e}\" selection in the same view. Try moving the lookup into a second, layered view?`}(e))}return new Cm(e,n,a.getSource())}dependentFields(){return new Set([this.transform.lookup])}producedFields(){return new Set(this.transform.as?t.array(this.transform.as):this.transform.from.fields)}hash(){return`Lookup ${d({transform:this.transform,secondary:this.secondary})}`}assemble(){let e;if(this.transform.from.fields)e={values:this.transform.from.fields,...this.transform.as?{as:t.array(this.transform.as)}:{}};else{let n=this.transform.as;t.isString(n)||(yi('If \"from.fields\" is not specified, \"as\" has to be a string that specifies the key to be used for the data from the secondary source.'),n=\"_lookup\"),e={as:[n]}}return{type:\"lookup\",from:this.secondary,key:this.transform.from.key,fields:[this.transform.lookup],...e,...this.transform.default?{default:this.transform.default}:{}}}}class Nm extends pc{clone(){return new Nm(null,l(this.transform))}constructor(e,t){super(e),this.transform=t,this.transform=l(t);const n=this.transform.as??[void 0,void 0];this.transform.as=[n[0]??\"prob\",n[1]??\"value\"]}dependentFields(){return new Set([this.transform.quantile,...this.transform.groupby??[]])}producedFields(){return new Set(this.transform.as)}hash(){return`QuantileTransform ${d(this.transform)}`}assemble(){const{quantile:e,...t}=this.transform;return{type:\"quantile\",field:e,...t}}}class Pm extends pc{clone(){return new Pm(null,l(this.transform))}constructor(e,t){super(e),this.transform=t,this.transform=l(t);const n=this.transform.as??[void 0,void 0];this.transform.as=[n[0]??t.on,n[1]??t.regression]}dependentFields(){return new Set([this.transform.regression,this.transform.on,...this.transform.groupby??[]])}producedFields(){return new Set(this.transform.as)}hash(){return`RegressionTransform ${d(this.transform)}`}assemble(){const{regression:e,on:t,...n}=this.transform;return{type:\"regression\",x:t,y:e,...n}}}class Am extends pc{clone(){return new Am(null,l(this.transform))}constructor(e,t){super(e),this.transform=t}addDimensions(e){this.transform.groupby=b((this.transform.groupby??[]).concat(e),(e=>e))}producedFields(){}dependentFields(){return new Set([this.transform.pivot,this.transform.value,...this.transform.groupby??[]])}hash(){return`PivotTransform ${d(this.transform)}`}assemble(){const{pivot:e,value:t,groupby:n,limit:i,op:r}=this.transform;return{type:\"pivot\",field:e,value:t,...void 0!==i?{limit:i}:{},...void 0!==r?{op:r}:{},...void 0!==n?{groupby:n}:{}}}}class jm extends pc{clone(){return new jm(null,l(this.transform))}constructor(e,t){super(e),this.transform=t}dependentFields(){return new Set}producedFields(){return new Set}hash(){return`SampleTransform ${d(this.transform)}`}assemble(){return{type:\"sample\",size:this.transform.sample}}}function Tm(e){let t=0;return function n(i,r){if(i instanceof cd&&!i.isGenerator&&!ec(i.data)){e.push(r);r={name:null,source:r.name,transform:[]}}if(i instanceof od&&(i.parent instanceof cd&&!r.source?(r.format={...r.format,parse:i.assembleFormatParse()},r.transform.push(...i.assembleTransforms(!0))):r.transform.push(...i.assembleTransforms())),i instanceof td)return r.name||(r.name=\"data_\"+t++),!r.source||r.transform.length>0?(e.push(r),i.data=r.name):i.data=r.source,void e.push(...i.assemble());if((i instanceof sd||i instanceof ld||i instanceof km||i instanceof qu||i instanceof ef||i instanceof zm||i instanceof ed||i instanceof Cm||i instanceof zd||i instanceof Dd||i instanceof Dm||i instanceof Sm||i instanceof $m||i instanceof _m||i instanceof Nm||i instanceof Pm||i instanceof ad||i instanceof jm||i instanceof Am||i instanceof wm)&&r.transform.push(i.assemble()),(i instanceof Kf||i instanceof vc||i instanceof Om||i instanceof Fd||i instanceof Fm)&&r.transform.push(...i.assemble()),i instanceof gc)if(r.source&&0===r.transform.length)i.setSource(r.source);else if(i.parent instanceof gc)i.setSource(r.name);else if(r.name||(r.name=\"data_\"+t++),i.setSource(r.name),1===i.numChildren()){e.push(r);r={name:null,source:r.name,transform:[]}}switch(i.numChildren()){case 0:i instanceof gc&&(!r.source||r.transform.length>0)&&e.push(r);break;case 1:n(i.children[0],r);break;default:{r.name||(r.name=\"data_\"+t++);let o=r.name;!r.source||r.transform.length>0?e.push(r):o=r.source;for(const e of i.children){n(e,{name:null,source:o,transform:[]})}break}}}}function Em(e){return\"top\"===e||\"left\"===e||yn(e)?\"header\":\"footer\"}function Mm(e,n){const{facet:i,config:r,child:o,component:a}=e;if(e.channelHasField(n)){const s=i[n],l=rf(\"title\",null,r,n);let c=aa(s,r,{allowDisabling:!0,includeDefault:void 0===l||!!l});o.component.layoutHeaders[n].title&&(c=t.isArray(c)?c.join(\", \"):c,c+=` / ${o.component.layoutHeaders[n].title}`,o.component.layoutHeaders[n].title=null);const u=rf(\"labelOrient\",s.header,r,n),f=null!==s.header&&U(s.header?.labels,r.header.labels,!0),d=p([\"bottom\",\"right\"],u)?\"footer\":\"header\";a.layoutHeaders[n]={title:null!==s.header?c:null,facetFieldDef:s,[d]:\"facet\"===n?[]:[Lm(e,n,f)]}}}function Lm(e,t,n){const i=\"row\"===t?\"height\":\"width\";return{labels:n,sizeSignal:e.child.component.layoutSize.get(i)?e.child.getSizeSignalRef(i):void 0,axes:[]}}function qm(e,t){const{child:n}=e;if(n.component.axes[t]){const{layoutHeaders:i,resolve:r}=e.component;if(r.axis[t]=Df(r,t),\"shared\"===r.axis[t]){const r=\"x\"===t?\"column\":\"row\",o=i[r];for(const i of n.component.axes[t]){const t=Em(i.get(\"orient\"));o[t]??=[Lm(e,r,!1)];const n=Iu(i,\"main\",e.config,{header:!0});n&&o[t][0].axes.push(n),i.mainExtracted=!0}}}}function Um(e){for(const t of e.children)t.parseLayoutSize()}function Rm(e,t){const n=wf(t),i=Ct(n),r=e.component.resolve,o=e.component.layoutSize;let a;for(const t of e.children){const o=t.component.layoutSize.getWithExplicit(n),s=r.scale[i]??Sf(i,e);if(\"independent\"===s&&\"step\"===o.value){a=void 0;break}if(a){if(\"independent\"===s&&a.value!==o.value){a=void 0;break}a=Kl(a,o,n,\"\")}else a=o}if(a){for(const i of e.children)e.renameSignal(i.getName(n),e.getName(t)),i.component.layoutSize.set(n,\"merged\",!1);o.setWithExplicit(t,a)}else o.setWithExplicit(t,{explicit:!1,value:void 0})}function Wm(e,t){const n=\"width\"===t?\"x\":\"y\",i=e.config,r=e.getScaleComponent(n);if(r){const e=r.get(\"type\"),n=r.get(\"range\");if(hr(e)){const e=Ns(i.view,t);return vn(n)||Fs(e)?\"step\":e}return _s(i.view,t)}if(e.hasProjection||\"arc\"===e.mark)return _s(i.view,t);{const e=Ns(i.view,t);return Fs(e)?e.step:e}}function Bm(e,t,n){return ta(t,{suffix:`by_${ta(e)}`,...n})}class Im extends xm{constructor(e,t,n,i){super(e,\"facet\",t,n,i,e.resolve),this.child=vp(e.spec,this,this.getName(\"child\"),void 0,i),this.children=[this.child],this.facet=this.initFacet(e.facet)}initFacet(e){if(!_o(e))return{facet:this.initFacetFieldDef(e,\"facet\")};const t=D(e),n={};for(const i of t){if(![Q,J].includes(i)){yi(ni(i,\"facet\"));break}const t=e[i];if(void 0===t.field){yi(ti(t,i));break}n[i]=this.initFacetFieldDef(t,i)}return n}initFacetFieldDef(e,t){const n=pa(e,t);return n.header?n.header=pn(n.header):null===n.header&&(n.header=null),n}channelHasField(e){return!!this.facet[e]}fieldDef(e){return this.facet[e]}parseData(){this.component.data=Vm(this),this.child.parseData()}parseLayoutSize(){Um(this)}parseSelections(){this.child.parseSelections(),this.component.selection=this.child.component.selection}parseMarkGroup(){this.child.parseMarkGroup()}parseAxesAndHeaders(){this.child.parseAxesAndHeaders(),function(e){for(const t of Re)Mm(e,t);qm(e,\"x\"),qm(e,\"y\")}(this)}assembleSelectionTopLevelSignals(e){return this.child.assembleSelectionTopLevelSignals(e)}assembleSignals(){return this.child.assembleSignals(),[]}assembleSelectionData(e){return this.child.assembleSelectionData(e)}getHeaderLayoutMixins(){const e={};for(const t of Re)for(const n of sf){const i=this.component.layoutHeaders[t],r=i[n],{facetFieldDef:o}=i;if(o){const n=rf(\"titleOrient\",o.header,this.config,t);if([\"right\",\"bottom\"].includes(n)){const i=nf(t,n);e.titleAnchor??={},e.titleAnchor[i]=\"end\"}}if(r?.[0]){const r=\"row\"===t?\"height\":\"width\",o=\"header\"===n?\"headerBand\":\"footerBand\";\"facet\"===t||this.child.component.layoutSize.get(r)||(e[o]??={},e[o][t]=.5),i.title&&(e.offset??={},e.offset[\"row\"===t?\"rowTitle\":\"columnTitle\"]=10)}}return e}assembleDefaultLayout(){const{column:e,row:t}=this.facet,n=e?this.columnDistinctSignal():t?1:void 0;let i=\"all\";return(t||\"independent\"!==this.component.resolve.scale.x)&&(e||\"independent\"!==this.component.resolve.scale.y)||(i=\"none\"),{...this.getHeaderLayoutMixins(),...n?{columns:n}:{},bounds:\"full\",align:i}}assembleLayoutSignals(){return this.child.assembleLayoutSignals()}columnDistinctSignal(){if(!(this.parent&&this.parent instanceof Im)){return{signal:`length(data('${this.getName(\"column_domain\")}'))`}}}assembleGroupStyle(){}assembleGroup(e){return this.parent&&this.parent instanceof Im?{...this.channelHasField(\"column\")?{encode:{update:{columns:{field:ta(this.facet.column,{prefix:\"distinct\"})}}}}:{},...super.assembleGroup(e)}:super.assembleGroup(e)}getCardinalityAggregateForChild(){const e=[],t=[],n=[];if(this.child instanceof Im){if(this.child.channelHasField(\"column\")){const i=ta(this.child.facet.column);e.push(i),t.push(\"distinct\"),n.push(`distinct_${i}`)}}else for(const i of Ft){const r=this.child.component.scales[i];if(r&&!r.merged){const o=r.get(\"type\"),a=r.get(\"range\");if(hr(o)&&vn(a)){const r=Hd(Vd(this.child,i));r?(e.push(r),t.push(\"distinct\"),n.push(`distinct_${r}`)):yi(In(i))}}}return{fields:e,ops:t,as:n}}assembleFacet(){const{name:e,data:n}=this.component.data.facetRoot,{row:i,column:r}=this.facet,{fields:o,ops:a,as:s}=this.getCardinalityAggregateForChild(),l=[];for(const e of Re){const n=this.facet[e];if(n){l.push(ta(n));const{bin:c,sort:u}=n;if(ln(c)&&l.push(ta(n,{binSuffix:\"end\"})),zo(u)){const{field:e,op:t=ko}=u,l=Bm(n,u);i&&r?(o.push(l),a.push(\"max\"),s.push(l)):(o.push(e),a.push(t),s.push(l))}else if(t.isArray(u)){const t=tf(n,e);o.push(t),a.push(\"max\"),s.push(t)}}}const c=!!i&&!!r;return{name:e,data:n,groupby:l,...c||o.length>0?{aggregate:{...c?{cross:c}:{},...o.length?{fields:o,ops:a,as:s}:{}}}:{}}}facetSortFields(e){const{facet:n}=this,i=n[e];return i?zo(i.sort)?[Bm(i,i.sort,{expr:\"datum\"})]:t.isArray(i.sort)?[tf(i,e,{expr:\"datum\"})]:[ta(i,{expr:\"datum\"})]:[]}facetSortOrder(e){const{facet:n}=this,i=n[e];if(i){const{sort:e}=i;return[(zo(e)?e.order:!t.isArray(e)&&e)||\"ascending\"]}return[]}assembleLabelTitle(){const{facet:e,config:t}=this;if(e.facet)return mf(e.facet,\"facet\",t);const n={row:[\"top\",\"bottom\"],column:[\"left\",\"right\"]};for(const i of af)if(e[i]){const r=rf(\"labelOrient\",e[i]?.header,t,i);if(n[i].includes(r))return mf(e[i],i,t)}}assembleMarks(){const{child:e}=this,t=function(e){const t=[],n=Tm(t);for(const t of e.children)n(t,{source:e.name,name:null,transform:[]});return t}(this.component.data.facetRoot),n=e.assembleGroupEncodeEntry(!1),i=this.assembleLabelTitle()||e.assembleTitle(),r=e.assembleGroupStyle();return[{name:this.getName(\"cell\"),type:\"group\",...i?{title:i}:{},...r?{style:r}:{},from:{facet:this.assembleFacet()},sort:{field:Re.map((e=>this.facetSortFields(e))).flat(),order:Re.map((e=>this.facetSortOrder(e))).flat()},...t.length>0?{data:t}:{},...n?{encode:{update:n}}:{},...e.assembleGroup(fc(this,[]))}]}getMapping(){return this.facet}}function Hm(e,t){for(const n of t){const t=n.data;if(e.name&&n.hasName()&&e.name!==n.dataName)continue;const i=e.format?.mesh,r=t.format?.feature;if(i&&r)continue;const o=e.format?.feature;if((o||r)&&o!==r)continue;const a=t.format?.mesh;if(!i&&!a||i===a)if(tc(e)&&tc(t)){if(Y(e.values,t.values))return n}else if(ec(e)&&ec(t)){if(e.url===t.url)return n}else if(nc(e)&&e.name===n.dataName)return n}return null}function Vm(e){let t=function(e,t){if(e.data||!e.parent){if(null===e.data){const e=new cd({values:[]});return t.push(e),e}const n=Hm(e.data,t);if(n)return ic(e.data)||(n.data.format=y({},e.data.format,n.data.format)),!n.hasName()&&e.data.name&&(n.dataName=e.data.name),n;{const n=new cd(e.data);return t.push(n),n}}return e.parent.component.data.facetRoot?e.parent.component.data.facetRoot:e.parent.component.data.main}(e,e.component.data.sources);const{outputNodes:n,outputNodeRefCounts:i}=e.component.data,r=e.data,o=!(r&&(ic(r)||ec(r)||tc(r)))&&e.parent?e.parent.component.data.ancestorParse.clone():new Zl;ic(r)?(rc(r)?t=new ld(t,r.sequence):ac(r)&&(t=new sd(t,r.graticule)),o.parseNothing=!0):null===r?.format?.parse&&(o.parseNothing=!0),t=od.makeExplicit(t,e,o)??t,t=new ad(t);const a=e.parent&&vm(e.parent);(gm(e)||hm(e))&&a&&(t=Kf.makeFromEncoding(t,e)??t),e.transforms.length>0&&(t=function(e,t,n){let i=0;for(const r of t.transforms){let o,a;if(Fl(r))a=e=new ef(e,r),o=\"derived\";else if(gl(r)){const i=id(r);a=e=od.makeWithAncestors(e,{},i,n)??e,e=new qu(e,t,r.filter)}else if(zl(r))a=e=Kf.makeFromTransform(e,r,t),o=\"number\";else if(_l(r))o=\"date\",void 0===n.getWithExplicit(r.field).value&&(e=new od(e,{[r.field]:o}),n.set(r.field,o,!1)),a=e=vc.makeFromTransform(e,r);else if(Cl(r))a=e=ed.makeFromTransform(e,r),o=\"number\",ju(t)&&(e=new ad(e));else if(hl(r))a=e=Cm.make(e,t,r,i++),o=\"derived\";else if(kl(r))a=e=new zd(e,r),o=\"number\";else if(Sl(r))a=e=new Dd(e,r),o=\"number\";else if(Nl(r))a=e=Fd.makeFromTransform(e,r),o=\"derived\";else if(Pl(r))a=e=new Dm(e,r),o=\"derived\";else if(Al(r))a=e=new wm(e,r),o=\"derived\";else if(Dl(r))a=e=new Sm(e,r),o=\"derived\";else if(yl(r))a=e=new Am(e,r),o=\"derived\";else if(wl(r))e=new jm(e,r);else if(Ol(r))a=e=Om.makeFromTransform(e,r),o=\"derived\";else if(vl(r))a=e=new $m(e,r),o=\"derived\";else if(bl(r))a=e=new Nm(e,r),o=\"derived\";else if(xl(r))a=e=new Pm(e,r),o=\"derived\";else{if(!$l(r)){yi(`Ignoring an invalid transform: ${X(r)}.`);continue}a=e=new _m(e,r),o=\"derived\"}if(a&&void 0!==o)for(const e of a.producedFields()??[])n.set(e,o,!1)}return e}(t,e,o));const s=function(e){const t={};if(gm(e)&&e.component.selection)for(const n of D(e.component.selection)){const i=e.component.selection[n];for(const e of i.project.items)!e.channel&&q(e.field)>1&&(t[e.field]=\"flatten\")}return t}(e),l=rd(e);t=od.makeWithAncestors(t,{},{...s,...l},o)??t,gm(e)&&(t=Fm.parseAll(t,e),t=zm.parseAll(t,e)),(gm(e)||hm(e))&&(a||(t=Kf.makeFromEncoding(t,e)??t),t=vc.makeFromEncoding(t,e)??t,t=ef.parseAllForSortIndex(t,e));const c=t=Gm(sc.Raw,e,t);if(gm(e)){const n=ed.makeFromEncoding(t,e);n&&(t=n,ju(e)&&(t=new ad(t))),t=Om.makeFromEncoding(t,e)??t,t=Fd.makeFromEncoding(t,e)??t}gm(e)&&(t=km.make(t,e)??t);const u=t=Gm(sc.Main,e,t);gm(e)&&function(e,t){for(const[n,i]of z(e.component.selection??{})){const r=e.getName(`lookup_${n}`);e.component.data.outputNodes[r]=i.materialized=new gc(new qu(t,e,{param:n}),r,sc.Lookup,e.component.data.outputNodeRefCounts)}}(e,u);let f=null;if(hm(e)){const i=e.getName(\"facet\");t=function(e,t){const{row:n,column:i}=t;if(n&&i){let t=null;for(const r of[n,i])if(zo(r.sort)){const{field:n,op:i=ko}=r.sort;e=t=new Dd(e,{joinaggregate:[{op:i,field:n,as:Bm(r,r.sort,{forAs:!0})}],groupby:[ta(r)]})}return t}return null}(t,e.facet)??t,f=new td(t,e,i,u.getSource()),n[i]=f}return{...e.component.data,outputNodes:n,outputNodeRefCounts:i,raw:c,main:u,facetRoot:f,ancestorParse:o}}function Gm(e,t,n){const{outputNodes:i,outputNodeRefCounts:r}=t.component.data,o=t.getDataName(e),a=new gc(n,o,e,r);return i[o]=a,a}class Ym extends bm{constructor(e,t,n,i){super(e,\"concat\",t,n,i,e.resolve),\"shared\"!==e.resolve?.axis?.x&&\"shared\"!==e.resolve?.axis?.y||yi(\"Axes cannot be shared in concatenated or repeated views yet (https://github.com/vega/vega-lite/issues/2415).\"),this.children=this.getChildren(e).map(((e,t)=>vp(e,this,this.getName(`concat_${t}`),void 0,i)))}parseData(){this.component.data=Vm(this);for(const e of this.children)e.parseData()}parseSelections(){this.component.selection={};for(const e of this.children){e.parseSelections();for(const t of D(e.component.selection))this.component.selection[t]=e.component.selection[t]}}parseMarkGroup(){for(const e of this.children)e.parseMarkGroup()}parseAxesAndHeaders(){for(const e of this.children)e.parseAxesAndHeaders()}getChildren(e){return ks(e)?e.vconcat:Ss(e)?e.hconcat:e.concat}parseLayoutSize(){!function(e){Um(e);const t=1===e.layout.columns?\"width\":\"childWidth\",n=void 0===e.layout.columns?\"height\":\"childHeight\";Rm(e,t),Rm(e,n)}(this)}parseAxisGroup(){return null}assembleSelectionTopLevelSignals(e){return this.children.reduce(((e,t)=>t.assembleSelectionTopLevelSignals(e)),e)}assembleSignals(){return this.children.forEach((e=>e.assembleSignals())),[]}assembleLayoutSignals(){const e=vf(this);for(const t of this.children)e.push(...t.assembleLayoutSignals());return e}assembleSelectionData(e){return this.children.reduce(((e,t)=>t.assembleSelectionData(e)),e)}assembleMarks(){return this.children.map((e=>{const t=e.assembleTitle(),n=e.assembleGroupStyle(),i=e.assembleGroupEncodeEntry(!1);return{type:\"group\",name:e.getName(\"group\"),...t?{title:t}:{},...n?{style:n}:{},...i?{encode:{update:i}}:{},...e.assembleGroup()}}))}assembleGroupStyle(){}assembleDefaultLayout(){const e=this.layout.columns;return{...null!=e?{columns:e}:{},bounds:\"full\",align:\"each\"}}}const Xm={disable:1,gridScale:1,scale:1,...Da,labelExpr:1,encode:1},Qm=D(Xm);class Jm extends Gl{constructor(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},n=arguments.length>2&&void 0!==arguments[2]&&arguments[2];super(),this.explicit=e,this.implicit=t,this.mainExtracted=n}clone(){return new Jm(l(this.explicit),l(this.implicit),this.mainExtracted)}hasAxisPart(e){return\"axis\"===e||(\"grid\"===e||\"title\"===e?!!this.get(e):!(!1===(t=this.get(e))||null===t));var t}hasOrientSignalRef(){return yn(this.explicit.orient)}}const Km={bottom:\"top\",top:\"bottom\",left:\"right\",right:\"left\"};function Zm(e,t){if(!e)return t.map((e=>e.clone()));{if(e.length!==t.length)return;const n=e.length;for(let i=0;i{switch(n){case\"title\":return Ln(e,t);case\"gridScale\":return{explicit:e.explicit,value:U(e.value,t.value)}}return Jl(e,t,n,\"axis\")}));e.setWithExplicit(n,i)}return e}function tp(e,t,n,i,r){if(\"disable\"===t)return void 0!==n;switch(n=n||{},t){case\"titleAngle\":case\"labelAngle\":return e===(yn(n.labelAngle)?n.labelAngle:H(n.labelAngle));case\"values\":return!!n.values;case\"encode\":return!!n.encoding||!!n.labelAngle;case\"title\":if(e===Zu(i,r))return!0}return e===n[t]}const np=new Set([\"grid\",\"translate\",\"format\",\"formatType\",\"orient\",\"labelExpr\",\"tickCount\",\"position\",\"tickMinStep\"]);function ip(e,t){let n=t.axis(e);const i=new Jm,r=fa(t.encoding[e]),{mark:o,config:a}=t,s=n?.orient||a[\"x\"===e?\"axisX\":\"axisY\"]?.orient||a.axis?.orient||function(e){return\"x\"===e?\"bottom\":\"left\"}(e),l=t.getScaleComponent(e).get(\"type\"),c=function(e,t,n,i){const r=\"band\"===t?[\"axisDiscrete\",\"axisBand\"]:\"point\"===t?[\"axisDiscrete\",\"axisPoint\"]:dr(t)?[\"axisQuantitative\"]:\"time\"===t||\"utc\"===t?[\"axisTemporal\"]:[],o=\"x\"===e?\"axisX\":\"axisY\",a=yn(n)?\"axisOrient\":`axis${P(n)}`,s=[...r,...r.map((e=>o+e.substr(4)))],l=[\"axis\",a,o];return{vlOnlyAxisConfig:Vu(s,i,e,n),vgAxisConfig:Vu(l,i,e,n),axisConfigStyle:Gu([...l,...s],i)}}(e,l,s,t.config),u=void 0!==n?!n:Yu(\"disable\",a.style,n?.style,c).configValue;if(i.set(\"disable\",u,void 0!==n),u)return i;n=n||{};const f=function(e,t,n,i,r){const o=t?.labelAngle;if(void 0!==o)return yn(o)?o:H(o);{const{configValue:o}=Yu(\"labelAngle\",i,t?.style,r);return void 0!==o?H(o):n!==Z||!p([ir,tr],e.type)||Ro(e)&&e.timeUnit?void 0:270}}(r,n,e,a.style,c),d=Hr(n.formatType,r,l),m=Ir(r,r.type,n.format,n.formatType,a,!0),g={fieldOrDatumDef:r,axis:n,channel:e,model:t,scaleType:l,orient:s,labelAngle:f,format:m,formatType:d,mark:o,config:a};for(const r of Qm){const o=r in Xu?Xu[r](g):za(r)?n[r]:void 0,s=void 0!==o,l=tp(o,r,n,t,e);if(s&&l)i.set(r,o,l);else{const{configValue:e,configFrom:t}=za(r)&&\"values\"!==r?Yu(r,a.style,n.style,c):{},u=void 0!==e;s&&!u?i.set(r,o,l):(\"vgAxisConfig\"!==t||np.has(r)&&u||wa(e)||yn(e))&&i.set(r,e,!1)}}const h=n.encoding??{},y=ka.reduce(((n,r)=>{if(!i.hasAxisPart(r))return n;const o=kf(h[r]??{},t),a=\"labels\"===r?function(e,t,n){const{encoding:i,config:r}=e,o=fa(i[t])??fa(i[it(t)]),a=e.axis(t)||{},{format:s,formatType:l}=a;if(Lr(l))return{text:Br({fieldOrDatumDef:o,field:\"datum.value\",format:s,formatType:l,config:r}),...n};if(void 0===s&&void 0===l&&r.customFormatTypes){if(\"quantitative\"===Wo(o)){if(Jo(o)&&\"normalize\"===o.stack&&r.normalizedNumberFormatType)return{text:Br({fieldOrDatumDef:o,field:\"datum.value\",format:r.normalizedNumberFormat,formatType:r.normalizedNumberFormatType,config:r}),...n};if(r.numberFormatType)return{text:Br({fieldOrDatumDef:o,field:\"datum.value\",format:r.numberFormat,formatType:r.numberFormatType,config:r}),...n}}if(\"temporal\"===Wo(o)&&r.timeFormatType&&Ro(o)&&!o.timeUnit)return{text:Br({fieldOrDatumDef:o,field:\"datum.value\",format:r.timeFormat,formatType:r.timeFormatType,config:r}),...n}}return n}(t,e,o):o;return void 0===a||S(a)||(n[r]={update:a}),n}),{});return S(y)||i.set(\"encode\",y,!!n.encoding||void 0!==n.labelAngle),i}function rp(e,t){const{config:n}=e;return{...lu(e,{align:\"ignore\",baseline:\"ignore\",color:\"include\",size:\"include\",orient:\"ignore\",theta:\"ignore\"}),...Xc(\"x\",e,{defaultPos:\"mid\"}),...Xc(\"y\",e,{defaultPos:\"mid\"}),...Hc(\"size\",e),...Hc(\"angle\",e),...op(e,n,t)}}function op(e,t,n){return n?{shape:{value:n}}:Hc(\"shape\",e)}const ap={vgMark:\"rule\",encodeEntry:e=>{const{markDef:t}=e,n=t.orient;return e.encoding.x||e.encoding.y||e.encoding.latitude||e.encoding.longitude?{...lu(e,{align:\"ignore\",baseline:\"ignore\",color:\"include\",orient:\"ignore\",size:\"ignore\",theta:\"ignore\"}),...eu(\"x\",e,{defaultPos:\"horizontal\"===n?\"zeroOrMax\":\"mid\",defaultPos2:\"zeroOrMin\",range:\"vertical\"!==n}),...eu(\"y\",e,{defaultPos:\"vertical\"===n?\"zeroOrMax\":\"mid\",defaultPos2:\"zeroOrMin\",range:\"horizontal\"!==n}),...Hc(\"size\",e,{vgChannel:\"strokeWidth\"})}:{}}};function sp(e,t,n){if(void 0===Nn(\"align\",e,n))return\"center\"}function lp(e,t,n){if(void 0===Nn(\"baseline\",e,n))return\"middle\"}const cp={vgMark:\"rect\",encodeEntry:e=>{const{config:t,markDef:n}=e,i=n.orient,r=\"horizontal\"===i?\"width\":\"height\",o=\"horizontal\"===i?\"height\":\"width\";return{...lu(e,{align:\"ignore\",baseline:\"ignore\",color:\"include\",orient:\"ignore\",size:\"ignore\",theta:\"ignore\"}),...Xc(\"x\",e,{defaultPos:\"mid\",vgChannel:\"xc\"}),...Xc(\"y\",e,{defaultPos:\"mid\",vgChannel:\"yc\"}),...Hc(\"size\",e,{defaultValue:up(e),vgChannel:r}),[o]:Fn(Nn(\"thickness\",n,t))}}};function up(e){const{config:n,markDef:i}=e,{orient:r}=i,o=\"horizontal\"===r?\"width\":\"height\",a=e.getScaleComponent(\"horizontal\"===r?\"x\":\"y\"),s=Nn(\"size\",i,n,{vgChannel:o})??n.tick.bandSize;if(void 0!==s)return s;{const e=a?a.get(\"range\"):void 0;if(e&&vn(e)&&t.isNumber(e.step))return 3*e.step/4;return 3*Cs(n.view,o)/4}}const fp={arc:{vgMark:\"arc\",encodeEntry:e=>({...lu(e,{align:\"ignore\",baseline:\"ignore\",color:\"include\",size:\"ignore\",orient:\"ignore\",theta:\"ignore\"}),...Xc(\"x\",e,{defaultPos:\"mid\"}),...Xc(\"y\",e,{defaultPos:\"mid\"}),...iu(e,\"radius\"),...iu(e,\"theta\")})},area:{vgMark:\"area\",encodeEntry:e=>({...lu(e,{align:\"ignore\",baseline:\"ignore\",color:\"include\",orient:\"include\",size:\"ignore\",theta:\"ignore\"}),...eu(\"x\",e,{defaultPos:\"zeroOrMin\",defaultPos2:\"zeroOrMin\",range:\"horizontal\"===e.markDef.orient}),...eu(\"y\",e,{defaultPos:\"zeroOrMin\",defaultPos2:\"zeroOrMin\",range:\"vertical\"===e.markDef.orient}),...fu(e)})},bar:{vgMark:\"rect\",encodeEntry:e=>({...lu(e,{align:\"ignore\",baseline:\"ignore\",color:\"include\",orient:\"ignore\",size:\"ignore\",theta:\"ignore\"}),...iu(e,\"x\"),...iu(e,\"y\")})},circle:{vgMark:\"symbol\",encodeEntry:e=>rp(e,\"circle\")},geoshape:{vgMark:\"shape\",encodeEntry:e=>({...lu(e,{align:\"ignore\",baseline:\"ignore\",color:\"include\",size:\"ignore\",orient:\"ignore\",theta:\"ignore\"})}),postEncodingTransform:e=>{const{encoding:t}=e,n=t.shape;return[{type:\"geoshape\",projection:e.projectionName(),...n&&Ro(n)&&n.type===rr?{field:ta(n,{expr:\"datum\"})}:{}}]}},image:{vgMark:\"image\",encodeEntry:e=>({...lu(e,{align:\"ignore\",baseline:\"ignore\",color:\"ignore\",orient:\"ignore\",size:\"ignore\",theta:\"ignore\"}),...iu(e,\"x\"),...iu(e,\"y\"),...Mc(e,\"url\")})},line:{vgMark:\"line\",encodeEntry:e=>({...lu(e,{align:\"ignore\",baseline:\"ignore\",color:\"include\",size:\"ignore\",orient:\"ignore\",theta:\"ignore\"}),...Xc(\"x\",e,{defaultPos:\"mid\"}),...Xc(\"y\",e,{defaultPos:\"mid\"}),...Hc(\"size\",e,{vgChannel:\"strokeWidth\"}),...fu(e)})},point:{vgMark:\"symbol\",encodeEntry:e=>rp(e)},rect:{vgMark:\"rect\",encodeEntry:e=>({...lu(e,{align:\"ignore\",baseline:\"ignore\",color:\"include\",orient:\"ignore\",size:\"ignore\",theta:\"ignore\"}),...iu(e,\"x\"),...iu(e,\"y\")})},rule:ap,square:{vgMark:\"symbol\",encodeEntry:e=>rp(e,\"square\")},text:{vgMark:\"text\",encodeEntry:e=>{const{config:t,encoding:n}=e;return{...lu(e,{align:\"include\",baseline:\"include\",color:\"include\",size:\"ignore\",orient:\"ignore\",theta:\"include\"}),...Xc(\"x\",e,{defaultPos:\"mid\"}),...Xc(\"y\",e,{defaultPos:\"mid\"}),...Mc(e),...Hc(\"size\",e,{vgChannel:\"fontSize\"}),...Hc(\"angle\",e),...du(\"align\",sp(e.markDef,n,t)),...du(\"baseline\",lp(e.markDef,n,t)),...Xc(\"radius\",e,{defaultPos:null}),...Xc(\"theta\",e,{defaultPos:null})}}},tick:cp,trail:{vgMark:\"trail\",encodeEntry:e=>({...lu(e,{align:\"ignore\",baseline:\"ignore\",color:\"include\",size:\"include\",orient:\"ignore\",theta:\"ignore\"}),...Xc(\"x\",e,{defaultPos:\"mid\"}),...Xc(\"y\",e,{defaultPos:\"mid\"}),...Hc(\"size\",e),...fu(e)})}};function dp(e){if(p([to,Kr,so],e.mark)){const t=qa(e.mark,e.encoding);if(t.length>0)return function(e,t){return[{name:e.getName(\"pathgroup\"),type:\"group\",from:{facet:{name:mp+e.requestDataName(sc.Main),data:e.requestDataName(sc.Main),groupby:t}},encode:{update:{width:{field:{group:\"width\"}},height:{field:{group:\"height\"}}}},marks:gp(e,{fromPrefix:mp})}]}(e,t)}else if(e.mark===Zr){const t=wn.some((t=>Nn(t,e.markDef,e.config)));if(e.stack&&!e.fieldDef(\"size\")&&t)return function(e){const[t]=gp(e,{fromPrefix:pp}),n=e.scaleName(e.stack.fieldChannel),i=function(){let t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};return e.vgField(e.stack.fieldChannel,t)},r=(e,t)=>`${e}(${[i({prefix:\"min\",suffix:\"start\",expr:t}),i({prefix:\"max\",suffix:\"start\",expr:t}),i({prefix:\"min\",suffix:\"end\",expr:t}),i({prefix:\"max\",suffix:\"end\",expr:t})].map((e=>`scale('${n}',${e})`)).join(\",\")})`;let o,a;\"x\"===e.stack.fieldChannel?(o={...u(t.encode.update,[\"y\",\"yc\",\"y2\",\"height\",...wn]),x:{signal:r(\"min\",\"datum\")},x2:{signal:r(\"max\",\"datum\")},clip:{value:!0}},a={x:{field:{group:\"x\"},mult:-1},height:{field:{group:\"height\"}}},t.encode.update={...f(t.encode.update,[\"y\",\"yc\",\"y2\"]),height:{field:{group:\"height\"}}}):(o={...u(t.encode.update,[\"x\",\"xc\",\"x2\",\"width\"]),y:{signal:r(\"min\",\"datum\")},y2:{signal:r(\"max\",\"datum\")},clip:{value:!0}},a={y:{field:{group:\"y\"},mult:-1},width:{field:{group:\"width\"}}},t.encode.update={...f(t.encode.update,[\"x\",\"xc\",\"x2\"]),width:{field:{group:\"width\"}}});for(const n of wn){const i=Pn(n,e.markDef,e.config);t.encode.update[n]?(o[n]=t.encode.update[n],delete t.encode.update[n]):i&&(o[n]=Fn(i)),i&&(t.encode.update[n]={value:0})}const s=[];if(e.stack.groupbyChannels?.length>0)for(const t of e.stack.groupbyChannels){const n=e.fieldDef(t),i=ta(n);i&&s.push(i),(n?.bin||n?.timeUnit)&&s.push(ta(n,{binSuffix:\"end\"}))}o=[\"stroke\",\"strokeWidth\",\"strokeJoin\",\"strokeCap\",\"strokeDash\",\"strokeDashOffset\",\"strokeMiterLimit\",\"strokeOpacity\"].reduce(((n,i)=>{if(t.encode.update[i])return{...n,[i]:t.encode.update[i]};{const t=Pn(i,e.markDef,e.config);return void 0!==t?{...n,[i]:Fn(t)}:n}}),o),o.stroke&&(o.strokeForeground={value:!0},o.strokeOffset={value:0});return[{type:\"group\",from:{facet:{data:e.requestDataName(sc.Main),name:pp+e.requestDataName(sc.Main),groupby:s,aggregate:{fields:[i({suffix:\"start\"}),i({suffix:\"start\"}),i({suffix:\"end\"}),i({suffix:\"end\"})],ops:[\"min\",\"max\",\"min\",\"max\"]}}},encode:{update:o},marks:[{type:\"group\",encode:{update:a},marks:[t]}]}]}(e)}return gp(e)}const mp=\"faceted_path_\";const pp=\"stack_group_\";function gp(e){let n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{fromPrefix:\"\"};const{mark:i,markDef:r,encoding:o,config:a}=e,s=U(r.clip,function(e){const t=e.getScaleComponent(\"x\"),n=e.getScaleComponent(\"y\");return!(!t?.get(\"selectionExtent\")&&!n?.get(\"selectionExtent\"))||void 0}(e),function(e){const t=e.component.projection;return!(!t||t.isFit)||void 0}(e)),l=Cn(r),c=o.key,u=function(e){const{encoding:n,stack:i,mark:r,markDef:o,config:a}=e,s=n.order;if(!(!t.isArray(s)&&Xo(s)&&m(s.value)||!s&&m(Nn(\"order\",o,a)))){if((t.isArray(s)||Ro(s))&&!i)return Tn(s,{expr:\"datum\"});if(fo(r)){const i=\"horizontal\"===o.orient?\"y\":\"x\",r=n[i];if(Ro(r)){const n=r.sort;return t.isArray(n)?{field:ta(r,{prefix:i,suffix:\"sort_index\",expr:\"datum\"})}:zo(n)?{field:ta({aggregate:ja(e.encoding)?n.op:void 0,field:n.field},{expr:\"datum\"})}:Fo(n)?{field:ta(e.fieldDef(n.encoding),{expr:\"datum\"}),order:n.order}:null===n?void 0:{field:ta(r,{binSuffix:e.stack?.impute?\"mid\":void 0,expr:\"datum\"})}}}}}(e),f=function(e){if(!e.component.selection)return null;const t=D(e.component.selection).length;let n=t,i=e.parent;for(;i&&0===n;)n=D(i.component.selection).length,i=i.parent;return n?{interactive:t>0||\"geoshape\"===e.mark||!!e.encoding.tooltip||!!e.markDef.tooltip}:null}(e),d=Nn(\"aria\",r,a),p=fp[i].postEncodingTransform?fp[i].postEncodingTransform(e):null;return[{name:e.getName(\"marks\"),type:fp[i].vgMark,...s?{clip:s}:{},...l?{style:l}:{},...c?{key:c.field}:{},...u?{sort:u}:{},...f||{},...!1===d?{aria:d}:{},from:{data:n.fromPrefix+e.requestDataName(sc.Main)},encode:{update:fp[i].encodeEntry(e)},...p?{transform:p}:{}}]}class hp extends xm{specifiedScales={};specifiedAxes={};specifiedLegends={};specifiedProjection={};selection=[];children=[];constructor(e,n,i){let r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{},o=arguments.length>4?arguments[4]:void 0;super(e,\"unit\",n,i,o,void 0,zs(e)?e.view:void 0);const a=go(e.mark)?{...e.mark}:{type:e.mark},s=a.type;void 0===a.filled&&(a.filled=function(e,t,n){let{graticule:i}=n;if(i)return!1;const r=Pn(\"filled\",e,t),o=e.type;return U(r,o!==no&&o!==to&&o!==ro)}(a,o,{graticule:e.data&&ac(e.data)}));const l=this.encoding=function(e,n,i,r){const o={};for(const t of D(e))Ke(t)||yi(`${a=t}-encoding is dropped as ${a} is not a valid encoding channel.`);var a;for(let a of lt){if(!e[a])continue;const s=e[a];if(Pt(a)){const e=st(a),t=o[e];if(Ro(t)&&Ki(t.type)&&Ro(s)&&!t.timeUnit){yi(Kn(e));continue}}if(\"angle\"!==a||\"arc\"!==n||e.theta||(yi(\"Arc marks uses theta channel rather than angle, replacing angle with theta.\"),a=se),Ea(e,a,n)){if(a===ye&&\"line\"===n){const t=ua(e[a]);if(t?.aggregate){yi(\"Line marks cannot encode size with a non-groupby field. You may want to use trail marks instead.\");continue}}if(a===me&&(i?\"fill\"in e:\"stroke\"in e))yi(ei(\"encoding\",{fill:\"fill\"in e,stroke:\"stroke\"in e}));else if(a===Fe||a===De&&!t.isArray(s)&&!Xo(s)||a===Oe&&t.isArray(s)){if(s){if(a===De){const t=e[a];if(Mo(t)){o[a]=t;continue}}o[a]=t.array(s).reduce(((e,t)=>(Ro(t)?e.push(pa(t,a)):yi(ti(t,a)),e)),[])}}else{if(a===Oe&&null===s)o[a]=null;else if(!(Ro(s)||Bo(s)||Xo(s)||Lo(s)||yn(s))){yi(ti(s,a));continue}o[a]=da(s,a,r)}}else yi(ni(a,n))}return o}(e.encoding||{},s,a.filled,o);this.markDef=Zs(a,l,o),this.size=function(e){let{encoding:t,size:n}=e;for(const e of Ft){const i=rt(e);Fs(n[i])&&Io(t[e])&&(delete n[i],yi(ui(i)))}return n}({encoding:l,size:zs(e)?{...r,...e.width?{width:e.width}:{},...e.height?{height:e.height}:{}}:r}),this.stack=Ks(this.markDef,l),this.specifiedScales=this.initScales(s,l),this.specifiedAxes=this.initAxes(l),this.specifiedLegends=this.initLegends(l),this.specifiedProjection=e.projection,this.selection=(e.params??[]).filter((e=>xs(e)))}get hasProjection(){const{encoding:e}=this,t=this.mark===uo,n=e&&Me.some((t=>Go(e[t])));return t||n}scaleDomain(e){const t=this.specifiedScales[e];return t?t.domain:void 0}axis(e){return this.specifiedAxes[e]}legend(e){return this.specifiedLegends[e]}initScales(e,t){return It.reduce(((e,n)=>{const i=fa(t[n]);return i&&(e[n]=this.initScale(i.scale??{})),e}),{})}initScale(e){const{domain:n,range:i}=e,r=pn(e);return t.isArray(n)&&(r.domain=n.map(Sn)),t.isArray(i)&&(r.range=i.map(Sn)),r}initAxes(e){return Ft.reduce(((t,n)=>{const i=e[n];if(Go(i)||n===Z&&Go(e.x2)||n===ee&&Go(e.y2)){const e=Go(i)?i.axis:void 0;t[n]=e?this.initAxis({...e}):e}return t}),{})}initAxis(e){const t=D(e),n={};for(const i of t){const t=e[i];n[i]=wa(t)?kn(t):Sn(t)}return n}initLegends(e){return Wt.reduce(((t,n)=>{const i=fa(e[n]);if(i&&function(e){switch(e){case me:case pe:case ge:case ye:case he:case be:case we:case ke:return!0;case xe:case $e:case ve:return!1}}(n)){const e=i.legend;t[n]=e?pn(e):e}return t}),{})}parseData(){this.component.data=Vm(this)}parseLayoutSize(){!function(e){const{size:t,component:n}=e;for(const i of Ft){const r=rt(i);if(t[r]){const e=t[r];n.layoutSize.set(r,Fs(e)?\"step\":e,!0)}else{const t=Wm(e,r);n.layoutSize.set(r,t,!1)}}}(this)}parseSelections(){this.component.selection=function(e,n){const i={},r=e.config.selection;if(!n||!n.length)return i;for(const o of n){const n=_(o.name),a=o.select,s=t.isString(a)?a:a.type,c=t.isObject(a)?l(a):{type:s},u=r[s];for(const e in u)\"fields\"!==e&&\"encodings\"!==e&&(\"mark\"===e&&(c[e]={...u[e],...c[e]}),void 0!==c[e]&&!0!==c[e]||(c[e]=l(u[e]??c[e])));const f=i[n]={...c,name:n,type:s,init:o.value,bind:o.bind,events:t.isString(c.on)?t.parseSelector(c.on,\"scope\"):t.array(l(c.on))},d=l(o);for(const t of Pu)t.defined(f)&&t.parse&&t.parse(e,f,d)}return i}(this,this.selection)}parseMarkGroup(){this.component.mark=dp(this)}parseAxesAndHeaders(){var e;this.component.axes=(e=this,Ft.reduce(((t,n)=>(e.component.scales[n]&&(t[n]=[ip(n,e)]),t)),{}))}assembleSelectionTopLevelSignals(e){return function(e,n){let i=!1;for(const r of F(e.component.selection??{})){const o=r.name,a=t.stringValue(o+Ou);if(0===n.filter((e=>e.name===o)).length){const e=\"global\"===r.resolve?\"union\":r.resolve,i=\"point\"===r.type?\", true, true)\":\")\";n.push({name:r.name,update:`${Nu}(${a}, ${t.stringValue(e)}${i}`})}i=!0;for(const t of Pu)t.defined(r)&&t.topLevelSignals&&(n=t.topLevelSignals(e,r,n))}i&&0===n.filter((e=>\"unit\"===e.name)).length&&n.unshift({name:\"unit\",value:{},on:[{events:\"pointermove\",update:\"isTuple(group()) ? group() : unit\"}]});return mc(n)}(this,e)}assembleSignals(){return[...Hu(this),...uc(this,[])]}assembleSelectionData(e){return function(e,t){const n=[...t],i=Au(e,{escape:!1});for(const t of F(e.component.selection??{})){const e={name:t.name+Ou};if(t.project.hasSelectionId&&(e.transform=[{type:\"collect\",sort:{field:hs}}]),t.init){const n=t.project.items.map(lc);e.values=t.project.hasSelectionId?t.init.map((e=>({unit:i,[hs]:cc(e,!1)[0]}))):t.init.map((e=>({unit:i,fields:n,values:cc(e,!1)})))}n.filter((e=>e.name===t.name+Ou)).length||n.push(e)}return n}(this,e)}assembleLayout(){return null}assembleLayoutSignals(){return vf(this)}assembleMarks(){let e=this.component.mark??[];return this.parent&&vm(this.parent)||(e=dc(this,e)),e.map(this.correctDataNames)}assembleGroupStyle(){const{style:e}=this.view||{};return void 0!==e?e:this.encoding.x||this.encoding.y?\"cell\":\"view\"}getMapping(){return this.encoding}get mark(){return this.markDef.type}channelHasField(e){return Na(this.encoding,e)}fieldDef(e){return ua(this.encoding[e])}typedFieldDef(e){const t=this.fieldDef(e);return Yo(t)?t:null}}class yp extends bm{constructor(e,t,n,i,r){super(e,\"layer\",t,n,r,e.resolve,e.view);const o={...i,...e.width?{width:e.width}:{},...e.height?{height:e.height}:{}};this.children=e.layer.map(((e,t)=>{if(Hs(e))return new yp(e,this,this.getName(`layer_${t}`),o,r);if(_a(e))return new hp(e,this,this.getName(`layer_${t}`),o,r);throw new Error(qn(e))}))}parseData(){this.component.data=Vm(this);for(const e of this.children)e.parseData()}parseLayoutSize(){var e;Um(e=this),Rm(e,\"width\"),Rm(e,\"height\")}parseSelections(){this.component.selection={};for(const e of this.children){e.parseSelections();for(const t of D(e.component.selection))this.component.selection[t]=e.component.selection[t]}}parseMarkGroup(){for(const e of this.children)e.parseMarkGroup()}parseAxesAndHeaders(){!function(e){const{axes:t,resolve:n}=e.component,i={top:0,bottom:0,right:0,left:0};for(const i of e.children){i.parseAxesAndHeaders();for(const r of D(i.component.axes))n.axis[r]=Df(e.component.resolve,r),\"shared\"===n.axis[r]&&(t[r]=Zm(t[r],i.component.axes[r]),t[r]||(n.axis[r]=\"independent\",delete t[r]))}for(const r of Ft){for(const o of e.children)if(o.component.axes[r]){if(\"independent\"===n.axis[r]){t[r]=(t[r]??[]).concat(o.component.axes[r]);for(const e of o.component.axes[r]){const{value:t,explicit:n}=e.getWithExplicit(\"orient\");if(!yn(t)){if(i[t]>0&&!n){const n=Km[t];i[t]>i[n]&&e.set(\"orient\",n,!1)}i[t]++}}}delete o.component.axes[r]}if(\"independent\"===n.axis[r]&&t[r]&&t[r].length>1)for(const[e,n]of(t[r]||[]).entries())e>0&&n.get(\"grid\")&&!n.explicit.grid&&(n.implicit.grid=!1)}}(this)}assembleSelectionTopLevelSignals(e){return this.children.reduce(((e,t)=>t.assembleSelectionTopLevelSignals(e)),e)}assembleSignals(){return this.children.reduce(((e,t)=>e.concat(t.assembleSignals())),Hu(this))}assembleLayoutSignals(){return this.children.reduce(((e,t)=>e.concat(t.assembleLayoutSignals())),vf(this))}assembleSelectionData(e){return this.children.reduce(((e,t)=>t.assembleSelectionData(e)),e)}assembleGroupStyle(){const e=new Set;for(const n of this.children)for(const i of t.array(n.assembleGroupStyle()))e.add(i);const n=Array.from(e);return n.length>1?n:1===n.length?n[0]:void 0}assembleTitle(){let e=super.assembleTitle();if(e)return e;for(const t of this.children)if(e=t.assembleTitle(),e)return e}assembleLayout(){return null}assembleMarks(){return function(e,t){for(const n of e.children)gm(n)&&(t=dc(n,t));return t}(this,this.children.flatMap((e=>e.assembleMarks())))}assembleLegends(){return this.children.reduce(((e,t)=>e.concat(t.assembleLegends())),Wf(this))}}function vp(e,t,n,i,r){if(No(e))return new Im(e,t,n,r);if(Hs(e))return new yp(e,t,n,i,r);if(_a(e))return new hp(e,t,n,i,r);if(function(e){return ks(e)||Ss(e)||ws(e)}(e))return new Ym(e,t,n,r);throw new Error(qn(e))}const bp=n;e.accessPathDepth=q,e.accessPathWithDatum=A,e.compile=function(e){let n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};var i;n.logger&&(i=n.logger,hi=i),n.fieldTitle&&oa(n.fieldTitle);try{const i=qs(t.mergeConfig(n.config,e.config)),r=Ul(e,i),o=vp(r,null,\"\",void 0,i);o.parse(),function(e,t){Pd(e.sources);let n=0,i=0;for(let i=0;i2&&void 0!==arguments[2]?arguments[2]:{},i=arguments.length>3?arguments[3]:void 0;const r=e.config?Bs(e.config):void 0,o=[].concat(e.assembleSelectionData([]),function(e,t){const n=[],i=Tm(n);let r=0;for(const t of e.sources){t.hasName()||(t.dataName=\"source_\"+r++);const e=t.assemble();i(t,e)}for(const e of n)0===e.transform.length&&delete e.transform;let o=0;for(const[e,t]of n.entries())0!==(t.transform??[]).length||t.source||n.splice(o++,0,n.splice(e,1)[0]);for(const t of n)for(const n of t.transform??[])\"lookup\"===n.type&&(n.from=e.outputNodes[n.from].getSource());for(const e of n)e.name in t&&(e.values=t[e.name]);return n}(e.component.data,n)),a=e.assembleProjections(),s=e.assembleTitle(),l=e.assembleGroupStyle(),c=e.assembleGroupEncodeEntry(!0);let u=e.assembleLayoutSignals();u=u.filter((e=>\"width\"!==e.name&&\"height\"!==e.name||void 0===e.value||(t[e.name]=+e.value,!1)));const{params:f,...d}=t;return{$schema:\"https://vega.github.io/schema/vega/v5.json\",...e.description?{description:e.description}:{},...d,...s?{title:s}:{},...l?{style:l}:{},...c?{encode:{update:c}}:{},data:o,...a.length>0?{projections:a}:{},...e.assembleGroup([...u,...e.assembleSelectionTopLevelSignals([]),...$s(f)]),...r?{config:r}:{},...i?{usermeta:i}:{}}}(o,function(e,n,i,r){const o=r.component.layoutSize.get(\"width\"),a=r.component.layoutSize.get(\"height\");void 0===n?(n={type:\"pad\"},r.hasAxisOrientSignalRef()&&(n.resize=!0)):t.isString(n)&&(n={type:n});if(o&&a&&(s=n.type,\"fit\"===s||\"fit-x\"===s||\"fit-y\"===s))if(\"step\"===o&&\"step\"===a)yi(Bn()),n.type=\"pad\";else if(\"step\"===o||\"step\"===a){const e=\"step\"===o?\"width\":\"height\";yi(Bn(Ct(e)));const t=\"width\"===e?\"height\":\"width\";n.type=function(e){return e?`fit-${Ct(e)}`:\"fit\"}(t)}var s;return{...1===D(n).length&&n.type?\"pad\"===n.type?{}:{autosize:n.type}:{autosize:n},...Vl(i,!1),...Vl(e,!0)}}(e,r.autosize,i,o),e.datasets,e.usermeta);return{spec:a,normalized:r}}finally{n.logger&&(hi=gi),n.fieldTitle&&oa(ia)}},e.contains=p,e.deepEqual=Y,e.deleteNestedProperty=N,e.duplicate=l,e.entries=z,e.every=h,e.fieldIntersection=k,e.flatAccessWithDatum=j,e.getFirstDefined=U,e.hasIntersection=$,e.hash=d,e.internalField=B,e.isBoolean=O,e.isEmpty=S,e.isEqual=function(e,t){const n=D(e),i=D(t);if(n.length!==i.length)return!1;for(const i of n)if(e[i]!==t[i])return!1;return!0},e.isInternalField=I,e.isNullOrFalse=m,e.isNumeric=V,e.keys=D,e.logicalExpr=C,e.mergeDeep=y,e.never=c,e.normalize=Ul,e.normalizeAngle=H,e.omit=f,e.pick=u,e.prefixGenerator=w,e.removePathFromField=L,e.replaceAll=M,e.replacePathInField=E,e.resetIdCounter=function(){R=42},e.setEqual=x,e.some=g,e.stringify=X,e.titleCase=P,e.unique=b,e.uniqueId=W,e.vals=F,e.varName=_,e.version=bp}));\n//# sourceMappingURL=vega-lite.min.js.map\n" }, { "path": "docs/javascripts/vega@5.js", "content": "!function(t,e){\"object\"==typeof exports&&\"undefined\"!=typeof module?e(exports):\"function\"==typeof define&&define.amd?define([\"exports\"],e):e((t=\"undefined\"!=typeof globalThis?globalThis:t||self).vega={})}(this,(function(t){\"use strict\";function e(t,e,n){return t.fields=e||[],t.fname=n,t}function n(t){return null==t?null:t.fname}function r(t){return null==t?null:t.fields}function i(t){return 1===t.length?o(t[0]):a(t)}const o=t=>function(e){return e[t]},a=t=>{const e=t.length;return function(n){for(let r=0;rr&&c(),u=r=i+1):\"]\"===o&&(u||s(\"Access path missing open bracket: \"+t),u>0&&c(),u=0,r=i+1):i>r?c():r=i+1}return u&&s(\"Access path missing closing bracket: \"+t),a&&s(\"Access path missing closing quote: \"+t),i>r&&(i++,c()),e}function l(t,n,r){const o=u(t);return t=1===o.length?o[0]:t,e((r&&r.get||i)(o),[t],n||t)}const c=l(\"id\"),f=e((t=>t),[],\"identity\"),h=e((()=>0),[],\"zero\"),d=e((()=>1),[],\"one\"),p=e((()=>!0),[],\"true\"),g=e((()=>!1),[],\"false\");function m(t,e,n){const r=[e].concat([].slice.call(n));console[t].apply(console,r)}const y=0,v=1,_=2,x=3,b=4;function w(t,e){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:m,r=t||y;return{level(t){return arguments.length?(r=+t,this):r},error(){return r>=v&&n(e||\"error\",\"ERROR\",arguments),this},warn(){return r>=_&&n(e||\"warn\",\"WARN\",arguments),this},info(){return r>=x&&n(e||\"log\",\"INFO\",arguments),this},debug(){return r>=b&&n(e||\"log\",\"DEBUG\",arguments),this}}}var k=Array.isArray;function A(t){return t===Object(t)}const M=t=>\"__proto__\"!==t;function E(){for(var t=arguments.length,e=new Array(t),n=0;n{for(const n in e)if(\"signals\"===n)t.signals=C(t.signals,e.signals);else{const r=\"legend\"===n?{layout:1}:\"style\"===n||null;D(t,n,e[n],r)}return t}),{})}function D(t,e,n,r){if(!M(e))return;let i,o;if(A(n)&&!k(n))for(i in o=A(t[e])?t[e]:t[e]={},n)r&&(!0===r||r[i])?D(o,i,n[i]):M(i)&&(o[i]=n[i]);else t[e]=n}function C(t,e){if(null==t)return e;const n={},r=[];function i(t){n[t.name]||(n[t.name]=1,r.push(t))}return e.forEach(i),t.forEach(i),r}function F(t){return t[t.length-1]}function S(t){return null==t||\"\"===t?null:+t}const $=t=>e=>t*Math.exp(e),T=t=>e=>Math.log(t*e),B=t=>e=>Math.sign(e)*Math.log1p(Math.abs(e/t)),z=t=>e=>Math.sign(e)*Math.expm1(Math.abs(e))*t,N=t=>e=>e<0?-Math.pow(-e,t):Math.pow(e,t);function O(t,e,n,r){const i=n(t[0]),o=n(F(t)),a=(o-i)*e;return[r(i-a),r(o-a)]}function R(t,e){return O(t,e,S,f)}function U(t,e){var n=Math.sign(t[0]);return O(t,e,T(n),$(n))}function L(t,e,n){return O(t,e,N(n),N(1/n))}function q(t,e,n){return O(t,e,B(n),z(n))}function P(t,e,n,r,i){const o=r(t[0]),a=r(F(t)),s=null!=e?r(e):(o+a)/2;return[i(s+(o-s)*n),i(s+(a-s)*n)]}function j(t,e,n){return P(t,e,n,S,f)}function I(t,e,n){const r=Math.sign(t[0]);return P(t,e,n,T(r),$(r))}function W(t,e,n,r){return P(t,e,n,N(r),N(1/r))}function H(t,e,n,r){return P(t,e,n,B(r),z(r))}function Y(t){return 1+~~(new Date(t).getMonth()/3)}function G(t){return 1+~~(new Date(t).getUTCMonth()/3)}function V(t){return null!=t?k(t)?t:[t]:[]}function X(t,e,n){let r,i=t[0],o=t[1];return o=n-e?[e,n]:[i=Math.min(Math.max(i,e),n-r),i+r]}function J(t){return\"function\"==typeof t}const Z=\"descending\";function Q(t,n,i){i=i||{},n=V(n)||[];const o=[],a=[],s={},u=i.comparator||tt;return V(t).forEach(((t,e)=>{null!=t&&(o.push(n[e]===Z?-1:1),a.push(t=J(t)?t:l(t,null,i)),(r(t)||[]).forEach((t=>s[t]=1)))})),0===a.length?null:e(u(a,o),Object.keys(s))}const K=(t,e)=>(te||null==e)&&null!=t?1:(e=e instanceof Date?+e:e,(t=t instanceof Date?+t:t)!==t&&e==e?-1:e!=e&&t==t?1:0),tt=(t,e)=>1===t.length?et(t[0],e[0]):nt(t,e,t.length),et=(t,e)=>function(n,r){return K(t(n),t(r))*e},nt=(t,e,n)=>(e.push(0),function(r,i){let o,a=0,s=-1;for(;0===a&&++st}function it(t,e){let n;return r=>{n&&clearTimeout(n),n=setTimeout((()=>(e(r),n=null)),t)}}function ot(t){for(let e,n,r=1,i=arguments.length;ro&&(o=r))}else{for(r=e(t[a]);ao&&(o=r))}return[i,o]}function st(t,e){const n=t.length;let r,i,o,a,s,u=-1;if(null==e){for(;++u=i){r=o=i;break}if(u===n)return[-1,-1];for(a=s=u;++ui&&(r=i,a=u),o=i){r=o=i;break}if(u===n)return[-1,-1];for(a=s=u;++ui&&(r=i,a=u),or(t)?n[t]:void 0,set(t,e){return r(t)||(++i.size,n[t]===ct&&--i.empty),n[t]=e,this},delete(t){return r(t)&&(--i.size,++i.empty,n[t]=ct),this},clear(){i.size=i.empty=0,i.object=n={}},test(t){return arguments.length?(e=t,i):e},clean(){const t={};let r=0;for(const i in n){const o=n[i];o===ct||e&&e(o)||(t[i]=o,++r)}i.size=r,i.empty=0,i.object=n=t}};return t&&Object.keys(t).forEach((e=>{i.set(e,t[e])})),i}function ht(t,e,n,r,i,o){if(!n&&0!==n)return o;const a=+n;let s,u=t[0],l=F(t);la&&(i=o,o=a,a=i),r=void 0===r||r,((n=void 0===n||n)?o<=t:ot.replace(/\\\\(.)/g,\"$1\"))):V(t));const o=t&&t.length,a=r&&r.get||i,s=t=>a(n?[t]:u(t));let l;if(o)if(1===o){const e=s(t[0]);l=function(t){return\"\"+e(t)}}else{const e=t.map(s);l=function(t){let n=\"\"+e[0](t),r=0;for(;++r{e={},n={},r=0},o=(i,o)=>(++r>t&&(n=e,e={},r=1),e[i]=o);return i(),{clear:i,has:t=>lt(e,t)||lt(n,t),get:t=>lt(e,t)?e[t]:lt(n,t)?o(t,n[t]):void 0,set:(t,n)=>lt(e,t)?e[t]=n:o(t,n)}}function At(t,e,n,r){const i=e.length,o=n.length;if(!o)return e;if(!i)return n;const a=r||new e.constructor(i+o);let s=0,u=0,l=0;for(;s0?n[u++]:e[s++];for(;s=0;)n+=t;return n}function Et(t,e,n,r){const i=n||\" \",o=t+\"\",a=e-o.length;return a<=0?o:\"left\"===r?Mt(i,a)+o:\"center\"===r?Mt(i,~~(a/2))+o+Mt(i,Math.ceil(a/2)):o+Mt(i,a)}function Dt(t){return t&&F(t)-t[0]||0}function Ct(t){return k(t)?\"[\"+t.map(Ct)+\"]\":A(t)||xt(t)?JSON.stringify(t).replace(\"\\u2028\",\"\\\\u2028\").replace(\"\\u2029\",\"\\\\u2029\"):t}function Ft(t){return null==t||\"\"===t?null:!(!t||\"false\"===t||\"0\"===t)&&!!t}const St=t=>vt(t)||mt(t)?t:Date.parse(t);function $t(t,e){return e=e||St,null==t||\"\"===t?null:e(t)}function Tt(t){return null==t||\"\"===t?null:t+\"\"}function Bt(t){const e={},n=t.length;for(let r=0;r9999?\"+\"+It(e,6):It(e,4))+\"-\"+It(t.getUTCMonth()+1,2)+\"-\"+It(t.getUTCDate(),2)+(o?\"T\"+It(n,2)+\":\"+It(r,2)+\":\"+It(i,2)+\".\"+It(o,3)+\"Z\":i?\"T\"+It(n,2)+\":\"+It(r,2)+\":\"+It(i,2)+\"Z\":r||n?\"T\"+It(n,2)+\":\"+It(r,2)+\"Z\":\"\")}function Ht(t){var e=new RegExp('[\"'+t+\"\\n\\r]\"),n=t.charCodeAt(0);function r(t,e){var r,i=[],o=t.length,a=0,s=0,u=o<=0,l=!1;function c(){if(u)return Rt;if(l)return l=!1,Ot;var e,r,i=a;if(t.charCodeAt(i)===Ut){for(;a++=o?u=!0:(r=t.charCodeAt(a++))===Lt?l=!0:r===qt&&(l=!0,t.charCodeAt(a)===Lt&&++a),t.slice(i+1,e-1).replace(/\"\"/g,'\"')}for(;a1)r=function(t,e,n){var r,i=[],o=[];function a(t){var e=t<0?~t:t;(o[e]||(o[e]=[])).push({i:t,g:r})}function s(t){t.forEach(a)}function u(t){t.forEach(s)}function l(t){t.forEach(u)}function c(t){switch(r=t,t.type){case\"GeometryCollection\":t.geometries.forEach(c);break;case\"LineString\":s(t.arcs);break;case\"MultiLineString\":case\"Polygon\":u(t.arcs);break;case\"MultiPolygon\":l(t.arcs)}}return c(e),o.forEach(null==n?function(t){i.push(t[0].i)}:function(t){n(t[0].g,t[t.length-1].g)&&i.push(t[0].i)}),i}(0,e,n);else for(i=0,r=new Array(o=t.arcs.length);ie?1:t>=e?0:NaN}function te(t,e){return null==t||null==e?NaN:et?1:e>=t?0:NaN}function ee(t){let e,n,r;function i(t,r){let i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:0,o=arguments.length>3&&void 0!==arguments[3]?arguments[3]:t.length;if(i>>1;n(t[e],r)<0?i=e+1:o=e}while(iKt(t(e),n),r=(e,n)=>t(e)-n):(e=t===Kt||t===te?t:ne,n=t,r=t),{left:i,center:function(t,e){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:0;const o=i(t,e,n,(arguments.length>3&&void 0!==arguments[3]?arguments[3]:t.length)-1);return o>n&&r(t[o-1],e)>-r(t[o],e)?o-1:o},right:function(t,r){let i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:0,o=arguments.length>3&&void 0!==arguments[3]?arguments[3]:t.length;if(i>>1;n(t[e],r)<=0?i=e+1:o=e}while(i0){for(o=t[--i];i>0&&(e=o,n=t[--i],o=e+n,r=n-(o-e),!r););i>0&&(r<0&&t[i-1]<0||r>0&&t[i-1]>0)&&(n=2*r,e=o+n,n==e-o&&(o=e))}return o}}class ue extends Map{constructor(t){let e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:de;if(super(),Object.defineProperties(this,{_intern:{value:new Map},_key:{value:e}}),null!=t)for(const[e,n]of t)this.set(e,n)}get(t){return super.get(ce(this,t))}has(t){return super.has(ce(this,t))}set(t,e){return super.set(fe(this,t),e)}delete(t){return super.delete(he(this,t))}}class le extends Set{constructor(t){let e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:de;if(super(),Object.defineProperties(this,{_intern:{value:new Map},_key:{value:e}}),null!=t)for(const e of t)this.add(e)}has(t){return super.has(ce(this,t))}add(t){return super.add(fe(this,t))}delete(t){return super.delete(he(this,t))}}function ce(t,e){let{_intern:n,_key:r}=t;const i=r(e);return n.has(i)?n.get(i):e}function fe(t,e){let{_intern:n,_key:r}=t;const i=r(e);return n.has(i)?n.get(i):(n.set(i,e),e)}function he(t,e){let{_intern:n,_key:r}=t;const i=r(e);return n.has(i)&&(e=n.get(i),n.delete(i)),e}function de(t){return null!==t&&\"object\"==typeof t?t.valueOf():t}function pe(t,e){return(null==t||!(t>=t))-(null==e||!(e>=e))||(te?1:0)}const ge=Math.sqrt(50),me=Math.sqrt(10),ye=Math.sqrt(2);function ve(t,e,n){const r=(e-t)/Math.max(0,n),i=Math.floor(Math.log10(r)),o=r/Math.pow(10,i),a=o>=ge?10:o>=me?5:o>=ye?2:1;let s,u,l;return i<0?(l=Math.pow(10,-i)/a,s=Math.round(t*l),u=Math.round(e*l),s/le&&--u,l=-l):(l=Math.pow(10,i)*a,s=Math.round(t/l),u=Math.round(e/l),s*le&&--u),u0))return[];if((t=+t)===(e=+e))return[t];const r=e=i))return[];const s=o-i+1,u=new Array(s);if(r)if(a<0)for(let t=0;t=e)&&(n=e);else{let r=-1;for(let i of t)null!=(i=e(i,++r,t))&&(n=i)&&(n=i)}return n}function ke(t,e){let n;if(void 0===e)for(const e of t)null!=e&&(n>e||void 0===n&&e>=e)&&(n=e);else{let r=-1;for(let i of t)null!=(i=e(i,++r,t))&&(n>i||void 0===n&&i>=i)&&(n=i)}return n}function Ae(t,e){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:0,r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:1/0,i=arguments.length>4?arguments[4]:void 0;if(e=Math.floor(e),n=Math.floor(Math.max(0,n)),r=Math.floor(Math.min(t.length-1,r)),!(n<=e&&e<=r))return t;for(i=void 0===i?pe:function(){let t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:Kt;if(t===Kt)return pe;if(\"function\"!=typeof t)throw new TypeError(\"compare is not a function\");return(e,n)=>{const r=t(e,n);return r||0===r?r:(0===t(n,n))-(0===t(e,e))}}(i);r>n;){if(r-n>600){const o=r-n+1,a=e-n+1,s=Math.log(o),u=.5*Math.exp(2*s/3),l=.5*Math.sqrt(s*u*(o-u)/o)*(a-o/2<0?-1:1);Ae(t,e,Math.max(n,Math.floor(e-a*u/o+l)),Math.min(r,Math.floor(e+(o-a)*u/o+l)),i)}const o=t[e];let a=n,s=r;for(Me(t,n,e),i(t[r],o)>0&&Me(t,n,r);a0;)--s}0===i(t[n],o)?Me(t,n,s):(++s,Me(t,s,r)),s<=e&&(n=s+1),e<=s&&(r=s-1)}return t}function Me(t,e,n){const r=t[e];t[e]=t[n],t[n]=r}function Ee(t,e,n){if(t=Float64Array.from(function*(t,e){if(void 0===e)for(let e of t)null!=e&&(e=+e)>=e&&(yield e);else{let n=-1;for(let r of t)null!=(r=e(r,++n,t))&&(r=+r)>=r&&(yield r)}}(t,n)),(r=t.length)&&!isNaN(e=+e)){if(e<=0||r<2)return ke(t);if(e>=1)return we(t);var r,i=(r-1)*e,o=Math.floor(i),a=we(Ae(t,o).subarray(0,o+1));return a+(ke(t.subarray(o+1))-a)*(i-o)}}function De(t,e){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:re;if((r=t.length)&&!isNaN(e=+e)){if(e<=0||r<2)return+n(t[0],0,t);if(e>=1)return+n(t[r-1],r-1,t);var r,i=(r-1)*e,o=Math.floor(i),a=+n(t[o],o,t);return a+(+n(t[o+1],o+1,t)-a)*(i-o)}}function Ce(t,e){return Ee(t,.5,e)}function Fe(t){return Array.from(function*(t){for(const e of t)yield*e}(t))}function Se(t,e,n){t=+t,e=+e,n=(i=arguments.length)<2?(e=t,t=0,1):i<3?1:+n;for(var r=-1,i=0|Math.max(0,Math.ceil((e-t)/n)),o=new Array(i);++r1?r[0]+r.slice(2):r,+t.slice(n+1)]}function ze(t){return(t=Be(Math.abs(t)))?t[1]:NaN}var Ne,Oe=/^(?:(.)?([<>=^]))?([+\\-( ])?([$#])?(0)?(\\d+)?(,)?(\\.\\d+)?(~)?([a-z%])?$/i;function Re(t){if(!(e=Oe.exec(t)))throw new Error(\"invalid format: \"+t);var e;return new Ue({fill:e[1],align:e[2],sign:e[3],symbol:e[4],zero:e[5],width:e[6],comma:e[7],precision:e[8]&&e[8].slice(1),trim:e[9],type:e[10]})}function Ue(t){this.fill=void 0===t.fill?\" \":t.fill+\"\",this.align=void 0===t.align?\">\":t.align+\"\",this.sign=void 0===t.sign?\"-\":t.sign+\"\",this.symbol=void 0===t.symbol?\"\":t.symbol+\"\",this.zero=!!t.zero,this.width=void 0===t.width?void 0:+t.width,this.comma=!!t.comma,this.precision=void 0===t.precision?void 0:+t.precision,this.trim=!!t.trim,this.type=void 0===t.type?\"\":t.type+\"\"}function Le(t,e){var n=Be(t,e);if(!n)return t+\"\";var r=n[0],i=n[1];return i<0?\"0.\"+new Array(-i).join(\"0\")+r:r.length>i+1?r.slice(0,i+1)+\".\"+r.slice(i+1):r+new Array(i-r.length+2).join(\"0\")}Re.prototype=Ue.prototype,Ue.prototype.toString=function(){return this.fill+this.align+this.sign+this.symbol+(this.zero?\"0\":\"\")+(void 0===this.width?\"\":Math.max(1,0|this.width))+(this.comma?\",\":\"\")+(void 0===this.precision?\"\":\".\"+Math.max(0,0|this.precision))+(this.trim?\"~\":\"\")+this.type};var qe={\"%\":(t,e)=>(100*t).toFixed(e),b:t=>Math.round(t).toString(2),c:t=>t+\"\",d:function(t){return Math.abs(t=Math.round(t))>=1e21?t.toLocaleString(\"en\").replace(/,/g,\"\"):t.toString(10)},e:(t,e)=>t.toExponential(e),f:(t,e)=>t.toFixed(e),g:(t,e)=>t.toPrecision(e),o:t=>Math.round(t).toString(8),p:(t,e)=>Le(100*t,e),r:Le,s:function(t,e){var n=Be(t,e);if(!n)return t+\"\";var r=n[0],i=n[1],o=i-(Ne=3*Math.max(-8,Math.min(8,Math.floor(i/3))))+1,a=r.length;return o===a?r:o>a?r+new Array(o-a+1).join(\"0\"):o>0?r.slice(0,o)+\".\"+r.slice(o):\"0.\"+new Array(1-o).join(\"0\")+Be(t,Math.max(0,e+o-1))[0]},X:t=>Math.round(t).toString(16).toUpperCase(),x:t=>Math.round(t).toString(16)};function Pe(t){return t}var je,Ie,We,He=Array.prototype.map,Ye=[\"y\",\"z\",\"a\",\"f\",\"p\",\"n\",\"Âĩ\",\"m\",\"\",\"k\",\"M\",\"G\",\"T\",\"P\",\"E\",\"Z\",\"Y\"];function Ge(t){var e,n,r=void 0===t.grouping||void 0===t.thousands?Pe:(e=He.call(t.grouping,Number),n=t.thousands+\"\",function(t,r){for(var i=t.length,o=[],a=0,s=e[0],u=0;i>0&&s>0&&(u+s+1>r&&(s=Math.max(1,r-u)),o.push(t.substring(i-=s,i+s)),!((u+=s+1)>r));)s=e[a=(a+1)%e.length];return o.reverse().join(n)}),i=void 0===t.currency?\"\":t.currency[0]+\"\",o=void 0===t.currency?\"\":t.currency[1]+\"\",a=void 0===t.decimal?\".\":t.decimal+\"\",s=void 0===t.numerals?Pe:function(t){return function(e){return e.replace(/[0-9]/g,(function(e){return t[+e]}))}}(He.call(t.numerals,String)),u=void 0===t.percent?\"%\":t.percent+\"\",l=void 0===t.minus?\"−\":t.minus+\"\",c=void 0===t.nan?\"NaN\":t.nan+\"\";function f(t){var e=(t=Re(t)).fill,n=t.align,f=t.sign,h=t.symbol,d=t.zero,p=t.width,g=t.comma,m=t.precision,y=t.trim,v=t.type;\"n\"===v?(g=!0,v=\"g\"):qe[v]||(void 0===m&&(m=12),y=!0,v=\"g\"),(d||\"0\"===e&&\"=\"===n)&&(d=!0,e=\"0\",n=\"=\");var _=\"$\"===h?i:\"#\"===h&&/[boxX]/.test(v)?\"0\"+v.toLowerCase():\"\",x=\"$\"===h?o:/[%p]/.test(v)?u:\"\",b=qe[v],w=/[defgprs%]/.test(v);function k(t){var i,o,u,h=_,k=x;if(\"c\"===v)k=b(t)+k,t=\"\";else{var A=(t=+t)<0||1/t<0;if(t=isNaN(t)?c:b(Math.abs(t),m),y&&(t=function(t){t:for(var e,n=t.length,r=1,i=-1;r0&&(i=0)}return i>0?t.slice(0,i)+t.slice(e+1):t}(t)),A&&0==+t&&\"+\"!==f&&(A=!1),h=(A?\"(\"===f?f:l:\"-\"===f||\"(\"===f?\"\":f)+h,k=(\"s\"===v?Ye[8+Ne/3]:\"\")+k+(A&&\"(\"===f?\")\":\"\"),w)for(i=-1,o=t.length;++i(u=t.charCodeAt(i))||u>57){k=(46===u?a+t.slice(i+1):t.slice(i))+k,t=t.slice(0,i);break}}g&&!d&&(t=r(t,1/0));var M=h.length+t.length+k.length,E=M>1)+h+t+k+E.slice(M);break;default:t=E+h+t+k}return s(t)}return m=void 0===m?6:/[gprs]/.test(v)?Math.max(1,Math.min(21,m)):Math.max(0,Math.min(20,m)),k.toString=function(){return t+\"\"},k}return{format:f,formatPrefix:function(t,e){var n=f(((t=Re(t)).type=\"f\",t)),r=3*Math.max(-8,Math.min(8,Math.floor(ze(e)/3))),i=Math.pow(10,-r),o=Ye[8+r/3];return function(t){return n(i*t)+o}}}}function Ve(t){return Math.max(0,-ze(Math.abs(t)))}function Xe(t,e){return Math.max(0,3*Math.max(-8,Math.min(8,Math.floor(ze(e)/3)))-ze(Math.abs(t)))}function Je(t,e){return t=Math.abs(t),e=Math.abs(e)-t,Math.max(0,ze(e)-ze(t))+1}!function(t){je=Ge(t),Ie=je.format,We=je.formatPrefix}({thousands:\",\",grouping:[3],currency:[\"$\",\"\"]});const Ze=new Date,Qe=new Date;function Ke(t,e,n,r){function i(e){return t(e=0===arguments.length?new Date:new Date(+e)),e}return i.floor=e=>(t(e=new Date(+e)),e),i.ceil=n=>(t(n=new Date(n-1)),e(n,1),t(n),n),i.round=t=>{const e=i(t),n=i.ceil(t);return t-e(e(t=new Date(+t),null==n?1:Math.floor(n)),t),i.range=(n,r,o)=>{const a=[];if(n=i.ceil(n),o=null==o?1:Math.floor(o),!(n0))return a;let s;do{a.push(s=new Date(+n)),e(n,o),t(n)}while(sKe((e=>{if(e>=e)for(;t(e),!n(e);)e.setTime(e-1)}),((t,r)=>{if(t>=t)if(r<0)for(;++r<=0;)for(;e(t,-1),!n(t););else for(;--r>=0;)for(;e(t,1),!n(t););})),n&&(i.count=(e,r)=>(Ze.setTime(+e),Qe.setTime(+r),t(Ze),t(Qe),Math.floor(n(Ze,Qe))),i.every=t=>(t=Math.floor(t),isFinite(t)&&t>0?t>1?i.filter(r?e=>r(e)%t==0:e=>i.count(0,e)%t==0):i:null)),i}const tn=Ke((()=>{}),((t,e)=>{t.setTime(+t+e)}),((t,e)=>e-t));tn.every=t=>(t=Math.floor(t),isFinite(t)&&t>0?t>1?Ke((e=>{e.setTime(Math.floor(e/t)*t)}),((e,n)=>{e.setTime(+e+n*t)}),((e,n)=>(n-e)/t)):tn:null),tn.range;const en=1e3,nn=6e4,rn=36e5,on=864e5,an=6048e5,sn=2592e6,un=31536e6,ln=Ke((t=>{t.setTime(t-t.getMilliseconds())}),((t,e)=>{t.setTime(+t+e*en)}),((t,e)=>(e-t)/en),(t=>t.getUTCSeconds()));ln.range;const cn=Ke((t=>{t.setTime(t-t.getMilliseconds()-t.getSeconds()*en)}),((t,e)=>{t.setTime(+t+e*nn)}),((t,e)=>(e-t)/nn),(t=>t.getMinutes()));cn.range;const fn=Ke((t=>{t.setUTCSeconds(0,0)}),((t,e)=>{t.setTime(+t+e*nn)}),((t,e)=>(e-t)/nn),(t=>t.getUTCMinutes()));fn.range;const hn=Ke((t=>{t.setTime(t-t.getMilliseconds()-t.getSeconds()*en-t.getMinutes()*nn)}),((t,e)=>{t.setTime(+t+e*rn)}),((t,e)=>(e-t)/rn),(t=>t.getHours()));hn.range;const dn=Ke((t=>{t.setUTCMinutes(0,0,0)}),((t,e)=>{t.setTime(+t+e*rn)}),((t,e)=>(e-t)/rn),(t=>t.getUTCHours()));dn.range;const pn=Ke((t=>t.setHours(0,0,0,0)),((t,e)=>t.setDate(t.getDate()+e)),((t,e)=>(e-t-(e.getTimezoneOffset()-t.getTimezoneOffset())*nn)/on),(t=>t.getDate()-1));pn.range;const gn=Ke((t=>{t.setUTCHours(0,0,0,0)}),((t,e)=>{t.setUTCDate(t.getUTCDate()+e)}),((t,e)=>(e-t)/on),(t=>t.getUTCDate()-1));gn.range;const mn=Ke((t=>{t.setUTCHours(0,0,0,0)}),((t,e)=>{t.setUTCDate(t.getUTCDate()+e)}),((t,e)=>(e-t)/on),(t=>Math.floor(t/on)));function yn(t){return Ke((e=>{e.setDate(e.getDate()-(e.getDay()+7-t)%7),e.setHours(0,0,0,0)}),((t,e)=>{t.setDate(t.getDate()+7*e)}),((t,e)=>(e-t-(e.getTimezoneOffset()-t.getTimezoneOffset())*nn)/an))}mn.range;const vn=yn(0),_n=yn(1),xn=yn(2),bn=yn(3),wn=yn(4),kn=yn(5),An=yn(6);function Mn(t){return Ke((e=>{e.setUTCDate(e.getUTCDate()-(e.getUTCDay()+7-t)%7),e.setUTCHours(0,0,0,0)}),((t,e)=>{t.setUTCDate(t.getUTCDate()+7*e)}),((t,e)=>(e-t)/an))}vn.range,_n.range,xn.range,bn.range,wn.range,kn.range,An.range;const En=Mn(0),Dn=Mn(1),Cn=Mn(2),Fn=Mn(3),Sn=Mn(4),$n=Mn(5),Tn=Mn(6);En.range,Dn.range,Cn.range,Fn.range,Sn.range,$n.range,Tn.range;const Bn=Ke((t=>{t.setDate(1),t.setHours(0,0,0,0)}),((t,e)=>{t.setMonth(t.getMonth()+e)}),((t,e)=>e.getMonth()-t.getMonth()+12*(e.getFullYear()-t.getFullYear())),(t=>t.getMonth()));Bn.range;const zn=Ke((t=>{t.setUTCDate(1),t.setUTCHours(0,0,0,0)}),((t,e)=>{t.setUTCMonth(t.getUTCMonth()+e)}),((t,e)=>e.getUTCMonth()-t.getUTCMonth()+12*(e.getUTCFullYear()-t.getUTCFullYear())),(t=>t.getUTCMonth()));zn.range;const Nn=Ke((t=>{t.setMonth(0,1),t.setHours(0,0,0,0)}),((t,e)=>{t.setFullYear(t.getFullYear()+e)}),((t,e)=>e.getFullYear()-t.getFullYear()),(t=>t.getFullYear()));Nn.every=t=>isFinite(t=Math.floor(t))&&t>0?Ke((e=>{e.setFullYear(Math.floor(e.getFullYear()/t)*t),e.setMonth(0,1),e.setHours(0,0,0,0)}),((e,n)=>{e.setFullYear(e.getFullYear()+n*t)})):null,Nn.range;const On=Ke((t=>{t.setUTCMonth(0,1),t.setUTCHours(0,0,0,0)}),((t,e)=>{t.setUTCFullYear(t.getUTCFullYear()+e)}),((t,e)=>e.getUTCFullYear()-t.getUTCFullYear()),(t=>t.getUTCFullYear()));function Rn(t,e,n,r,i,o){const a=[[ln,1,en],[ln,5,5e3],[ln,15,15e3],[ln,30,3e4],[o,1,nn],[o,5,3e5],[o,15,9e5],[o,30,18e5],[i,1,rn],[i,3,108e5],[i,6,216e5],[i,12,432e5],[r,1,on],[r,2,1728e5],[n,1,an],[e,1,sn],[e,3,7776e6],[t,1,un]];function s(e,n,r){const i=Math.abs(n-e)/r,o=ee((t=>{let[,,e]=t;return e})).right(a,i);if(o===a.length)return t.every(be(e/un,n/un,r));if(0===o)return tn.every(Math.max(be(e,n,r),1));const[s,u]=a[i/a[o-1][2]isFinite(t=Math.floor(t))&&t>0?Ke((e=>{e.setUTCFullYear(Math.floor(e.getUTCFullYear()/t)*t),e.setUTCMonth(0,1),e.setUTCHours(0,0,0,0)}),((e,n)=>{e.setUTCFullYear(e.getUTCFullYear()+n*t)})):null,On.range;const[Un,Ln]=Rn(On,zn,En,mn,dn,fn),[qn,Pn]=Rn(Nn,Bn,vn,pn,hn,cn),jn=\"year\",In=\"quarter\",Wn=\"month\",Hn=\"week\",Yn=\"date\",Gn=\"day\",Vn=\"dayofyear\",Xn=\"hours\",Jn=\"minutes\",Zn=\"seconds\",Qn=\"milliseconds\",Kn=[jn,In,Wn,Hn,Yn,Gn,Vn,Xn,Jn,Zn,Qn],tr=Kn.reduce(((t,e,n)=>(t[e]=1+n,t)),{});function er(t){const e=V(t).slice(),n={};e.length||s(\"Missing time unit.\"),e.forEach((t=>{lt(tr,t)?n[t]=1:s(`Invalid time unit: ${t}.`)}));return(n[Hn]||n[Gn]?1:0)+(n[In]||n[Wn]||n[Yn]?1:0)+(n[Vn]?1:0)>1&&s(`Incompatible time units: ${t}`),e.sort(((t,e)=>tr[t]-tr[e])),e}const nr={[jn]:\"%Y \",[In]:\"Q%q \",[Wn]:\"%b \",[Yn]:\"%d \",[Hn]:\"W%U \",[Gn]:\"%a \",[Vn]:\"%j \",[Xn]:\"%H:00\",[Jn]:\"00:%M\",[Zn]:\":%S\",[Qn]:\".%L\",[`${jn}-${Wn}`]:\"%Y-%m \",[`${jn}-${Wn}-${Yn}`]:\"%Y-%m-%d \",[`${Xn}-${Jn}`]:\"%H:%M\"};function rr(t,e){const n=ot({},nr,e),r=er(t),i=r.length;let o,a,s=\"\",u=0;for(u=0;uu;--o)if(a=r.slice(u,o).join(\"-\"),null!=n[a]){s+=n[a],u=o;break}return s.trim()}const ir=new Date;function or(t){return ir.setFullYear(t),ir.setMonth(0),ir.setDate(1),ir.setHours(0,0,0,0),ir}function ar(t){return ur(new Date(t))}function sr(t){return lr(new Date(t))}function ur(t){return pn.count(or(t.getFullYear())-1,t)}function lr(t){return vn.count(or(t.getFullYear())-1,t)}function cr(t){return or(t).getDay()}function fr(t,e,n,r,i,o,a){if(0<=t&&t<100){const s=new Date(-1,e,n,r,i,o,a);return s.setFullYear(t),s}return new Date(t,e,n,r,i,o,a)}function hr(t){return pr(new Date(t))}function dr(t){return gr(new Date(t))}function pr(t){const e=Date.UTC(t.getUTCFullYear(),0,1);return gn.count(e-1,t)}function gr(t){const e=Date.UTC(t.getUTCFullYear(),0,1);return En.count(e-1,t)}function mr(t){return ir.setTime(Date.UTC(t,0,1)),ir.getUTCDay()}function yr(t,e,n,r,i,o,a){if(0<=t&&t<100){const t=new Date(Date.UTC(-1,e,n,r,i,o,a));return t.setUTCFullYear(n.y),t}return new Date(Date.UTC(t,e,n,r,i,o,a))}function vr(t,e,n,r,i){const o=e||1,a=F(t),s=(t,e,i)=>function(t,e,n,r){const i=n<=1?t:r?(e,i)=>r+n*Math.floor((t(e,i)-r)/n):(e,r)=>n*Math.floor(t(e,r)/n);return e?(t,n)=>e(i(t,n),n):i}(n[i=i||t],r[i],t===a&&o,e),u=new Date,l=Bt(t),c=l[jn]?s(jn):rt(2012),f=l[Wn]?s(Wn):l[In]?s(In):h,p=l[Hn]&&l[Gn]?s(Gn,1,Hn+Gn):l[Hn]?s(Hn,1):l[Gn]?s(Gn,1):l[Yn]?s(Yn,1):l[Vn]?s(Vn,1):d,g=l[Xn]?s(Xn):h,m=l[Jn]?s(Jn):h,y=l[Zn]?s(Zn):h,v=l[Qn]?s(Qn):h;return function(t){u.setTime(+t);const e=c(u);return i(e,f(u),p(u,e),g(u),m(u),y(u),v(u))}}function _r(t,e,n){return e+7*t-(n+6)%7}const xr={[jn]:t=>t.getFullYear(),[In]:t=>Math.floor(t.getMonth()/3),[Wn]:t=>t.getMonth(),[Yn]:t=>t.getDate(),[Xn]:t=>t.getHours(),[Jn]:t=>t.getMinutes(),[Zn]:t=>t.getSeconds(),[Qn]:t=>t.getMilliseconds(),[Vn]:t=>ur(t),[Hn]:t=>lr(t),[Hn+Gn]:(t,e)=>_r(lr(t),t.getDay(),cr(e)),[Gn]:(t,e)=>_r(1,t.getDay(),cr(e))},br={[In]:t=>3*t,[Hn]:(t,e)=>_r(t,0,cr(e))};function wr(t,e){return vr(t,e||1,xr,br,fr)}const kr={[jn]:t=>t.getUTCFullYear(),[In]:t=>Math.floor(t.getUTCMonth()/3),[Wn]:t=>t.getUTCMonth(),[Yn]:t=>t.getUTCDate(),[Xn]:t=>t.getUTCHours(),[Jn]:t=>t.getUTCMinutes(),[Zn]:t=>t.getUTCSeconds(),[Qn]:t=>t.getUTCMilliseconds(),[Vn]:t=>pr(t),[Hn]:t=>gr(t),[Gn]:(t,e)=>_r(1,t.getUTCDay(),mr(e)),[Hn+Gn]:(t,e)=>_r(gr(t),t.getUTCDay(),mr(e))},Ar={[In]:t=>3*t,[Hn]:(t,e)=>_r(t,0,mr(e))};function Mr(t,e){return vr(t,e||1,kr,Ar,yr)}const Er={[jn]:Nn,[In]:Bn.every(3),[Wn]:Bn,[Hn]:vn,[Yn]:pn,[Gn]:pn,[Vn]:pn,[Xn]:hn,[Jn]:cn,[Zn]:ln,[Qn]:tn},Dr={[jn]:On,[In]:zn.every(3),[Wn]:zn,[Hn]:En,[Yn]:gn,[Gn]:gn,[Vn]:gn,[Xn]:dn,[Jn]:fn,[Zn]:ln,[Qn]:tn};function Cr(t){return Er[t]}function Fr(t){return Dr[t]}function Sr(t,e,n){return t?t.offset(e,n):void 0}function $r(t,e,n){return Sr(Cr(t),e,n)}function Tr(t,e,n){return Sr(Fr(t),e,n)}function Br(t,e,n,r){return t?t.range(e,n,r):void 0}function zr(t,e,n,r){return Br(Cr(t),e,n,r)}function Nr(t,e,n,r){return Br(Fr(t),e,n,r)}const Or=1e3,Rr=6e4,Ur=36e5,Lr=864e5,qr=2592e6,Pr=31536e6,jr=[jn,Wn,Yn,Xn,Jn,Zn,Qn],Ir=jr.slice(0,-1),Wr=Ir.slice(0,-1),Hr=Wr.slice(0,-1),Yr=Hr.slice(0,-1),Gr=[jn,Wn],Vr=[jn],Xr=[[Ir,1,Or],[Ir,5,5e3],[Ir,15,15e3],[Ir,30,3e4],[Wr,1,Rr],[Wr,5,3e5],[Wr,15,9e5],[Wr,30,18e5],[Hr,1,Ur],[Hr,3,108e5],[Hr,6,216e5],[Hr,12,432e5],[Yr,1,Lr],[[jn,Hn],1,6048e5],[Gr,1,qr],[Gr,3,7776e6],[Vr,1,Pr]];function Jr(t){const e=t.extent,n=t.maxbins||40,r=Math.abs(Dt(e))/n;let i,o,a=ee((t=>t[2])).right(Xr,r);return a===Xr.length?(i=Vr,o=be(e[0]/Pr,e[1]/Pr,n)):a?(a=Xr[r/Xr[a-1][2]=12)]},q:function(t){return 1+~~(t.getMonth()/3)},Q:wo,s:ko,S:ji,u:Ii,U:Wi,V:Yi,w:Gi,W:Vi,x:null,X:null,y:Xi,Y:Zi,Z:Ki,\"%\":bo},x={a:function(t){return a[t.getUTCDay()]},A:function(t){return o[t.getUTCDay()]},b:function(t){return u[t.getUTCMonth()]},B:function(t){return s[t.getUTCMonth()]},c:null,d:to,e:to,f:oo,g:yo,G:_o,H:eo,I:no,j:ro,L:io,m:ao,M:so,p:function(t){return i[+(t.getUTCHours()>=12)]},q:function(t){return 1+~~(t.getUTCMonth()/3)},Q:wo,s:ko,S:uo,u:lo,U:co,V:ho,w:po,W:go,x:null,X:null,y:mo,Y:vo,Z:xo,\"%\":bo},b={a:function(t,e,n){var r=d.exec(e.slice(n));return r?(t.w=p.get(r[0].toLowerCase()),n+r[0].length):-1},A:function(t,e,n){var r=f.exec(e.slice(n));return r?(t.w=h.get(r[0].toLowerCase()),n+r[0].length):-1},b:function(t,e,n){var r=y.exec(e.slice(n));return r?(t.m=v.get(r[0].toLowerCase()),n+r[0].length):-1},B:function(t,e,n){var r=g.exec(e.slice(n));return r?(t.m=m.get(r[0].toLowerCase()),n+r[0].length):-1},c:function(t,n,r){return A(t,e,n,r)},d:Ai,e:Ai,f:Si,g:xi,G:_i,H:Ei,I:Ei,j:Mi,L:Fi,m:ki,M:Di,p:function(t,e,n){var r=l.exec(e.slice(n));return r?(t.p=c.get(r[0].toLowerCase()),n+r[0].length):-1},q:wi,Q:Ti,s:Bi,S:Ci,u:gi,U:mi,V:yi,w:pi,W:vi,x:function(t,e,r){return A(t,n,e,r)},X:function(t,e,n){return A(t,r,e,n)},y:xi,Y:_i,Z:bi,\"%\":$i};function w(t,e){return function(n){var r,i,o,a=[],s=-1,u=0,l=t.length;for(n instanceof Date||(n=new Date(+n));++s53)return null;\"w\"in o||(o.w=1),\"Z\"in o?(i=(r=Qr(Kr(o.y,0,1))).getUTCDay(),r=i>4||0===i?Dn.ceil(r):Dn(r),r=gn.offset(r,7*(o.V-1)),o.y=r.getUTCFullYear(),o.m=r.getUTCMonth(),o.d=r.getUTCDate()+(o.w+6)%7):(i=(r=Zr(Kr(o.y,0,1))).getDay(),r=i>4||0===i?_n.ceil(r):_n(r),r=pn.offset(r,7*(o.V-1)),o.y=r.getFullYear(),o.m=r.getMonth(),o.d=r.getDate()+(o.w+6)%7)}else(\"W\"in o||\"U\"in o)&&(\"w\"in o||(o.w=\"u\"in o?o.u%7:\"W\"in o?1:0),i=\"Z\"in o?Qr(Kr(o.y,0,1)).getUTCDay():Zr(Kr(o.y,0,1)).getDay(),o.m=0,o.d=\"W\"in o?(o.w+6)%7+7*o.W-(i+5)%7:o.w+7*o.U-(i+6)%7);return\"Z\"in o?(o.H+=o.Z/100|0,o.M+=o.Z%100,Qr(o)):Zr(o)}}function A(t,e,n,r){for(var i,o,a=0,s=e.length,u=n.length;a=u)return-1;if(37===(i=e.charCodeAt(a++))){if(i=e.charAt(a++),!(o=b[i in ai?e.charAt(a++):i])||(r=o(t,n,r))<0)return-1}else if(i!=n.charCodeAt(r++))return-1}return r}return _.x=w(n,_),_.X=w(r,_),_.c=w(e,_),x.x=w(n,x),x.X=w(r,x),x.c=w(e,x),{format:function(t){var e=w(t+=\"\",_);return e.toString=function(){return t},e},parse:function(t){var e=k(t+=\"\",!1);return e.toString=function(){return t},e},utcFormat:function(t){var e=w(t+=\"\",x);return e.toString=function(){return t},e},utcParse:function(t){var e=k(t+=\"\",!0);return e.toString=function(){return t},e}}}var ei,ni,ri,ii,oi,ai={\"-\":\"\",_:\" \",0:\"0\"},si=/^\\s*\\d+/,ui=/^%/,li=/[\\\\^$*+?|[\\]().{}]/g;function ci(t,e,n){var r=t<0?\"-\":\"\",i=(r?-t:t)+\"\",o=i.length;return r+(o[t.toLowerCase(),e])))}function pi(t,e,n){var r=si.exec(e.slice(n,n+1));return r?(t.w=+r[0],n+r[0].length):-1}function gi(t,e,n){var r=si.exec(e.slice(n,n+1));return r?(t.u=+r[0],n+r[0].length):-1}function mi(t,e,n){var r=si.exec(e.slice(n,n+2));return r?(t.U=+r[0],n+r[0].length):-1}function yi(t,e,n){var r=si.exec(e.slice(n,n+2));return r?(t.V=+r[0],n+r[0].length):-1}function vi(t,e,n){var r=si.exec(e.slice(n,n+2));return r?(t.W=+r[0],n+r[0].length):-1}function _i(t,e,n){var r=si.exec(e.slice(n,n+4));return r?(t.y=+r[0],n+r[0].length):-1}function xi(t,e,n){var r=si.exec(e.slice(n,n+2));return r?(t.y=+r[0]+(+r[0]>68?1900:2e3),n+r[0].length):-1}function bi(t,e,n){var r=/^(Z)|([+-]\\d\\d)(?::?(\\d\\d))?/.exec(e.slice(n,n+6));return r?(t.Z=r[1]?0:-(r[2]+(r[3]||\"00\")),n+r[0].length):-1}function wi(t,e,n){var r=si.exec(e.slice(n,n+1));return r?(t.q=3*r[0]-3,n+r[0].length):-1}function ki(t,e,n){var r=si.exec(e.slice(n,n+2));return r?(t.m=r[0]-1,n+r[0].length):-1}function Ai(t,e,n){var r=si.exec(e.slice(n,n+2));return r?(t.d=+r[0],n+r[0].length):-1}function Mi(t,e,n){var r=si.exec(e.slice(n,n+3));return r?(t.m=0,t.d=+r[0],n+r[0].length):-1}function Ei(t,e,n){var r=si.exec(e.slice(n,n+2));return r?(t.H=+r[0],n+r[0].length):-1}function Di(t,e,n){var r=si.exec(e.slice(n,n+2));return r?(t.M=+r[0],n+r[0].length):-1}function Ci(t,e,n){var r=si.exec(e.slice(n,n+2));return r?(t.S=+r[0],n+r[0].length):-1}function Fi(t,e,n){var r=si.exec(e.slice(n,n+3));return r?(t.L=+r[0],n+r[0].length):-1}function Si(t,e,n){var r=si.exec(e.slice(n,n+6));return r?(t.L=Math.floor(r[0]/1e3),n+r[0].length):-1}function $i(t,e,n){var r=ui.exec(e.slice(n,n+1));return r?n+r[0].length:-1}function Ti(t,e,n){var r=si.exec(e.slice(n));return r?(t.Q=+r[0],n+r[0].length):-1}function Bi(t,e,n){var r=si.exec(e.slice(n));return r?(t.s=+r[0],n+r[0].length):-1}function zi(t,e){return ci(t.getDate(),e,2)}function Ni(t,e){return ci(t.getHours(),e,2)}function Oi(t,e){return ci(t.getHours()%12||12,e,2)}function Ri(t,e){return ci(1+pn.count(Nn(t),t),e,3)}function Ui(t,e){return ci(t.getMilliseconds(),e,3)}function Li(t,e){return Ui(t,e)+\"000\"}function qi(t,e){return ci(t.getMonth()+1,e,2)}function Pi(t,e){return ci(t.getMinutes(),e,2)}function ji(t,e){return ci(t.getSeconds(),e,2)}function Ii(t){var e=t.getDay();return 0===e?7:e}function Wi(t,e){return ci(vn.count(Nn(t)-1,t),e,2)}function Hi(t){var e=t.getDay();return e>=4||0===e?wn(t):wn.ceil(t)}function Yi(t,e){return t=Hi(t),ci(wn.count(Nn(t),t)+(4===Nn(t).getDay()),e,2)}function Gi(t){return t.getDay()}function Vi(t,e){return ci(_n.count(Nn(t)-1,t),e,2)}function Xi(t,e){return ci(t.getFullYear()%100,e,2)}function Ji(t,e){return ci((t=Hi(t)).getFullYear()%100,e,2)}function Zi(t,e){return ci(t.getFullYear()%1e4,e,4)}function Qi(t,e){var n=t.getDay();return ci((t=n>=4||0===n?wn(t):wn.ceil(t)).getFullYear()%1e4,e,4)}function Ki(t){var e=t.getTimezoneOffset();return(e>0?\"-\":(e*=-1,\"+\"))+ci(e/60|0,\"0\",2)+ci(e%60,\"0\",2)}function to(t,e){return ci(t.getUTCDate(),e,2)}function eo(t,e){return ci(t.getUTCHours(),e,2)}function no(t,e){return ci(t.getUTCHours()%12||12,e,2)}function ro(t,e){return ci(1+gn.count(On(t),t),e,3)}function io(t,e){return ci(t.getUTCMilliseconds(),e,3)}function oo(t,e){return io(t,e)+\"000\"}function ao(t,e){return ci(t.getUTCMonth()+1,e,2)}function so(t,e){return ci(t.getUTCMinutes(),e,2)}function uo(t,e){return ci(t.getUTCSeconds(),e,2)}function lo(t){var e=t.getUTCDay();return 0===e?7:e}function co(t,e){return ci(En.count(On(t)-1,t),e,2)}function fo(t){var e=t.getUTCDay();return e>=4||0===e?Sn(t):Sn.ceil(t)}function ho(t,e){return t=fo(t),ci(Sn.count(On(t),t)+(4===On(t).getUTCDay()),e,2)}function po(t){return t.getUTCDay()}function go(t,e){return ci(Dn.count(On(t)-1,t),e,2)}function mo(t,e){return ci(t.getUTCFullYear()%100,e,2)}function yo(t,e){return ci((t=fo(t)).getUTCFullYear()%100,e,2)}function vo(t,e){return ci(t.getUTCFullYear()%1e4,e,4)}function _o(t,e){var n=t.getUTCDay();return ci((t=n>=4||0===n?Sn(t):Sn.ceil(t)).getUTCFullYear()%1e4,e,4)}function xo(){return\"+0000\"}function bo(){return\"%\"}function wo(t){return+t}function ko(t){return Math.floor(+t/1e3)}function Ao(t){const e={};return n=>e[n]||(e[n]=t(n))}function Mo(t){const e=Ao(t.format),n=t.formatPrefix;return{format:e,formatPrefix:n,formatFloat(t){const n=Re(t||\",\");if(null==n.precision){switch(n.precision=12,n.type){case\"%\":n.precision-=2;break;case\"e\":n.precision-=1}return r=e(n),i=e(\".1f\")(1)[1],t=>{const e=r(t),n=e.indexOf(i);if(n<0)return e;let o=function(t,e){let n,r=t.lastIndexOf(\"e\");if(r>0)return r;for(r=t.length;--r>e;)if(n=t.charCodeAt(r),n>=48&&n<=57)return r+1}(e,n);const a=on;)if(\"0\"!==e[o]){++o;break}return e.slice(0,o)+a}}return e(n);var r,i},formatSpan(t,r,i,o){o=Re(null==o?\",f\":o);const a=be(t,r,i),s=Math.max(Math.abs(t),Math.abs(r));let u;if(null==o.precision)switch(o.type){case\"s\":return isNaN(u=Xe(a,s))||(o.precision=u),n(o,s);case\"\":case\"e\":case\"g\":case\"p\":case\"r\":isNaN(u=Je(a,s))||(o.precision=u-(\"e\"===o.type));break;case\"f\":case\"%\":isNaN(u=Ve(a))||(o.precision=u-2*(\"%\"===o.type))}return e(o)}}}let Eo,Do;function Co(){return Eo=Mo({format:Ie,formatPrefix:We})}function Fo(t){return Mo(Ge(t))}function So(t){return arguments.length?Eo=Fo(t):Eo}function $o(t,e,n){A(n=n||{})||s(`Invalid time multi-format specifier: ${n}`);const r=e(Zn),i=e(Jn),o=e(Xn),a=e(Yn),u=e(Hn),l=e(Wn),c=e(In),f=e(jn),h=t(n[Qn]||\".%L\"),d=t(n[Zn]||\":%S\"),p=t(n[Jn]||\"%I:%M\"),g=t(n[Xn]||\"%I %p\"),m=t(n[Yn]||n[Gn]||\"%a %d\"),y=t(n[Hn]||\"%b %d\"),v=t(n[Wn]||\"%B\"),_=t(n[In]||\"%B\"),x=t(n[jn]||\"%Y\");return t=>(r(t)xt(t)?e(t):$o(e,Cr,t),utcFormat:t=>xt(t)?n(t):$o(n,Fr,t),timeParse:Ao(t.parse),utcParse:Ao(t.utcParse)}}function Bo(){return Do=To({format:ni,parse:ri,utcFormat:ii,utcParse:oi})}function zo(t){return To(ti(t))}function No(t){return arguments.length?Do=zo(t):Do}!function(t){ei=ti(t),ni=ei.format,ri=ei.parse,ii=ei.utcFormat,oi=ei.utcParse}({dateTime:\"%x, %X\",date:\"%-m/%-d/%Y\",time:\"%-I:%M:%S %p\",periods:[\"AM\",\"PM\"],days:[\"Sunday\",\"Monday\",\"Tuesday\",\"Wednesday\",\"Thursday\",\"Friday\",\"Saturday\"],shortDays:[\"Sun\",\"Mon\",\"Tue\",\"Wed\",\"Thu\",\"Fri\",\"Sat\"],months:[\"January\",\"February\",\"March\",\"April\",\"May\",\"June\",\"July\",\"August\",\"September\",\"October\",\"November\",\"December\"],shortMonths:[\"Jan\",\"Feb\",\"Mar\",\"Apr\",\"May\",\"Jun\",\"Jul\",\"Aug\",\"Sep\",\"Oct\",\"Nov\",\"Dec\"]}),Co(),Bo();const Oo=(t,e)=>ot({},t,e);function Ro(t,e){const n=t?Fo(t):So(),r=e?zo(e):No();return Oo(n,r)}function Uo(t,e){const n=arguments.length;return n&&2!==n&&s(\"defaultLocale expects either zero or two arguments.\"),n?Oo(So(t),No(e)):Oo(So(),No())}const Lo=/^(data:|([A-Za-z]+:)?\\/\\/)/,qo=/^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp|file|data):|[^a-z]|[a-z+.\\-]+(?:[^a-z+.\\-:]|$))/i,Po=/[\\u0000-\\u0020\\u00A0\\u1680\\u180E\\u2000-\\u2029\\u205f\\u3000]/g,jo=\"file://\";async function Io(t,e){const n=await this.sanitize(t,e),r=n.href;return n.localFile?this.file(r):this.http(r,e)}async function Wo(t,e){e=ot({},this.options,e);const n=this.fileAccess,r={href:null};let i,o,a;const u=qo.test(t.replace(Po,\"\"));null!=t&&\"string\"==typeof t&&u||s(\"Sanitize failure, invalid URI: \"+Ct(t));const l=Lo.test(t);return(a=e.baseURL)&&!l&&(t.startsWith(\"/\")||a.endsWith(\"/\")||(t=\"/\"+t),t=a+t),o=(i=t.startsWith(jo))||\"file\"===e.mode||\"http\"!==e.mode&&!l&&n,i?t=t.slice(jo.length):t.startsWith(\"//\")&&(\"file\"===e.defaultProtocol?(t=t.slice(2),o=!0):t=(e.defaultProtocol||\"http\")+\":\"+t),Object.defineProperty(r,\"localFile\",{value:!!o}),r.href=t,e.target&&(r.target=e.target+\"\"),e.rel&&(r.rel=e.rel+\"\"),\"image\"===e.context&&e.crossOrigin&&(r.crossOrigin=e.crossOrigin+\"\"),r}function Ho(t){return t?e=>new Promise(((n,r)=>{t.readFile(e,((t,e)=>{t?r(t):n(e)}))})):Yo}async function Yo(){s(\"No file system access.\")}function Go(t){return t?async function(e,n){const r=ot({},this.options.http,n),i=n&&n.response,o=await t(e,r);return o.ok?J(o[i])?o[i]():o.text():s(o.status+\"\"+o.statusText)}:Vo}async function Vo(){s(\"No HTTP fetch method available.\")}const Xo=t=>null!=t&&t==t,Jo=t=>!(Number.isNaN(+t)||t instanceof Date),Zo={boolean:Ft,integer:S,number:S,date:$t,string:Tt,unknown:f},Qo=[t=>\"true\"===t||\"false\"===t||!0===t||!1===t,t=>Jo(t)&&Number.isInteger(+t),Jo,t=>!Number.isNaN(Date.parse(t))],Ko=[\"boolean\",\"integer\",\"number\",\"date\"];function ta(t,e){if(!t||!t.length)return\"unknown\";const n=t.length,r=Qo.length,i=Qo.map(((t,e)=>e+1));for(let o,a,s=0,u=0;s0===t?e:t),0)-1]}function ea(t,e){return e.reduce(((e,n)=>(e[n]=ta(t,n),e)),{})}function na(t){const e=function(e,n){const r={delimiter:t};return ra(e,n?ot(n,r):r)};return e.responseType=\"text\",e}function ra(t,e){return e.header&&(t=e.header.map(Ct).join(e.delimiter)+\"\\n\"+t),Ht(e.delimiter).parse(t+\"\")}function ia(t,e){const n=e&&e.property?l(e.property):f;return!A(t)||(r=t,\"function\"==typeof Buffer&&J(Buffer.isBuffer)&&Buffer.isBuffer(r))?n(JSON.parse(t)):function(t,e){!k(t)&&yt(t)&&(t=[...t]);return e&&e.copy?JSON.parse(JSON.stringify(t)):t}(n(t),e);var r}ra.responseType=\"text\",ia.responseType=\"json\";const oa={interior:(t,e)=>t!==e,exterior:(t,e)=>t===e};function aa(t,e){let n,r,i,o;return t=ia(t,e),e&&e.feature?(n=Gt,i=e.feature):e&&e.mesh?(n=Zt,i=e.mesh,o=oa[e.filter]):s(\"Missing TopoJSON feature or mesh parameter.\"),r=(r=t.objects[i])?n(t,r,o):s(\"Invalid TopoJSON object: \"+i),r&&r.features||[r]}aa.responseType=\"json\";const sa={dsv:ra,csv:na(\",\"),tsv:na(\"\\t\"),json:ia,topojson:aa};function ua(t,e){return arguments.length>1?(sa[t]=e,this):lt(sa,t)?sa[t]:null}function la(t){const e=ua(t);return e&&e.responseType||\"text\"}function ca(t,e,n,r){const i=ua((e=e||{}).type||\"json\");return i||s(\"Unknown data format type: \"+e.type),t=i(t,e),e.parse&&function(t,e,n,r){if(!t.length)return;const i=No();n=n||i.timeParse,r=r||i.utcParse;let o,a,s,u,l,c,f=t.columns||Object.keys(t[0]);\"auto\"===e&&(e=ea(t,f));f=Object.keys(e);const h=f.map((t=>{const i=e[t];let o,a;if(i&&(i.startsWith(\"date:\")||i.startsWith(\"utc:\"))){o=i.split(/:(.+)?/,2),a=o[1],(\"'\"===a[0]&&\"'\"===a[a.length-1]||'\"'===a[0]&&'\"'===a[a.length-1])&&(a=a.slice(1,-1));return(\"utc\"===o[0]?r:n)(a)}if(!Zo[i])throw Error(\"Illegal format pattern: \"+t+\":\"+i);return Zo[i]}));for(s=0,l=t.length,c=f.length;s({options:n||{},sanitize:Wo,load:Io,fileAccess:!!e,file:Ho(e),http:Go(t)})}(\"undefined\"!=typeof fetch&&fetch,null);function ha(t){const e=t||f,n=[],r={};return n.add=t=>{const i=e(t);return r[i]||(r[i]=1,n.push(t)),n},n.remove=t=>{const i=e(t);if(r[i]){r[i]=0;const e=n.indexOf(t);e>=0&&n.splice(e,1)}return n},n}async function da(t,e){try{await e(t)}catch(e){t.error(e)}}const pa=Symbol(\"vega_id\");let ga=1;function ma(t){return!(!t||!ya(t))}function ya(t){return t[pa]}function va(t,e){return t[pa]=e,t}function _a(t){const e=t===Object(t)?t:{data:t};return ya(e)?e:va(e,ga++)}function xa(t){return ba(t,_a({}))}function ba(t,e){for(const n in t)e[n]=t[n];return e}function wa(t,e){return va(e,ya(t))}function ka(t,e){return t?e?(n,r)=>t(n,r)||ya(e(n))-ya(e(r)):(e,n)=>t(e,n)||ya(e)-ya(n):null}function Aa(t){return t&&t.constructor===Ma}function Ma(){const t=[],e=[],n=[],r=[],i=[];let o=null,a=!1;return{constructor:Ma,insert(e){const n=V(e),r=n.length;for(let e=0;e{p(t)&&(l[ya(t)]=-1)}));for(f=0,h=t.length;f0&&(y(g,p,d.value),s.modifies(p));for(f=0,h=i.length;f{p(t)&&l[ya(t)]>0&&y(t,d.field,d.value)})),s.modifies(d.field);if(a)s.mod=e.length||r.length?u.filter((t=>l[ya(t)]>0)):u.slice();else for(m in c)s.mod.push(c[m]);return(o||null==o&&(e.length||r.length))&&s.clean(!0),s}}}const Ea=\"_:mod:_\";function Da(){Object.defineProperty(this,Ea,{writable:!0,value:{}})}Da.prototype={set(t,e,n,r){const i=this,o=i[t],a=i[Ea];return null!=e&&e>=0?(o[e]!==n||r)&&(o[e]=n,a[e+\":\"+t]=-1,a[t]=-1):(o!==n||r)&&(i[t]=n,a[t]=k(n)?1+n.length:-1),i},modified(t,e){const n=this[Ea];if(!arguments.length){for(const t in n)if(n[t])return!0;return!1}if(k(t)){for(let e=0;e=0?e+1{a instanceof Sa?(a!==this&&(e&&a.targets().add(this),o.push(a)),i.push({op:a,name:t,index:n})):r.set(t,n,a)};for(a in t)if(u=t[a],\"pulse\"===a)V(u).forEach((t=>{t instanceof Sa?t!==this&&(t.targets().add(this),o.push(t)):s(\"Pulse parameters must be operator instances.\")})),this.source=u;else if(k(u))for(r.set(a,-1,Array(l=u.length)),c=0;c{const n=Date.now();return n-e>t?(e=n,1):0}))},debounce(t){const e=za();return this.targets().add(za(null,null,it(t,(t=>{const n=t.dataflow;e.receive(t),n&&n.run&&n.run()})))),e},between(t,e){let n=!1;return t.targets().add(za(null,null,(()=>n=!0))),e.targets().add(za(null,null,(()=>n=!1))),this.filter((()=>n))},detach(){this._filter=p,this._targets=null}};const Na={skip:!0};function Oa(t,e,n,r,i,o){const a=ot({},o,Na);let s,u;J(n)||(n=rt(n)),void 0===r?s=e=>t.touch(n(e)):J(r)?(u=new Sa(null,r,i,!1),s=e=>{u.evaluate(e);const r=n(e),i=u.value;Aa(i)?t.pulse(r,i,o):t.update(r,i,a)}):s=e=>t.update(n(e),r,a),e.apply(s)}function Ra(t,e,n,r,i,o){if(void 0===r)e.targets().add(n);else{const a=o||{},s=new Sa(null,function(t,e){return e=J(e)?e:rt(e),t?function(n,r){const i=e(n,r);return t.skip()||(t.skip(i!==this.value).value=i),i}:e}(n,r),i,!1);s.modified(a.force),s.rank=e.rank,e.targets().add(s),n&&(s.skip(!0),s.value=n.value,s.targets().add(n),t.connect(n,[s]))}}const Ua={};function La(t,e,n){this.dataflow=t,this.stamp=null==e?-1:e,this.add=[],this.rem=[],this.mod=[],this.fields=null,this.encode=n||null}function qa(t,e){const n=[];return Nt(t,e,(t=>n.push(t))),n}function Pa(t,e){const n={};return t.visit(e,(t=>{n[ya(t)]=1})),t=>n[ya(t)]?null:t}function ja(t,e){return t?(n,r)=>t(n,r)&&e(n,r):e}function Ia(t,e,n,r){const i=this;let o=0;this.dataflow=t,this.stamp=e,this.fields=null,this.encode=r||null,this.pulses=n;for(const t of n)if(t.stamp===e){if(t.fields){const e=i.fields||(i.fields={});for(const n in t.fields)e[n]=1}t.changed(i.ADD)&&(o|=i.ADD),t.changed(i.REM)&&(o|=i.REM),t.changed(i.MOD)&&(o|=i.MOD)}this.changes=o}function Wa(t){return t.error(\"Dataflow already running. Use runAsync() to chain invocations.\"),t}La.prototype={StopPropagation:Ua,ADD:1,REM:2,MOD:4,ADD_REM:3,ADD_MOD:5,ALL:7,REFLOW:8,SOURCE:16,NO_SOURCE:32,NO_FIELDS:64,fork(t){return new La(this.dataflow).init(this,t)},clone(){const t=this.fork(7);return t.add=t.add.slice(),t.rem=t.rem.slice(),t.mod=t.mod.slice(),t.source&&(t.source=t.source.slice()),t.materialize(23)},addAll(){let t=this;return!t.source||t.add===t.rem||!t.rem.length&&t.source.length===t.add.length||(t=new La(this.dataflow).init(this),t.add=t.source,t.rem=[]),t},init(t,e){const n=this;return n.stamp=t.stamp,n.encode=t.encode,!t.fields||64&e||(n.fields=t.fields),1&e?(n.addF=t.addF,n.add=t.add):(n.addF=null,n.add=[]),2&e?(n.remF=t.remF,n.rem=t.rem):(n.remF=null,n.rem=[]),4&e?(n.modF=t.modF,n.mod=t.mod):(n.modF=null,n.mod=[]),32&e?(n.srcF=null,n.source=null):(n.srcF=t.srcF,n.source=t.source,t.cleans&&(n.cleans=t.cleans)),n},runAfter(t){this.dataflow.runAfter(t)},changed(t){const e=t||7;return 1&e&&this.add.length||2&e&&this.rem.length||4&e&&this.mod.length},reflow(t){if(t)return this.fork(7).reflow();const e=this.add.length,n=this.source&&this.source.length;return n&&n!==e&&(this.mod=this.source,e&&this.filter(4,Pa(this,1))),this},clean(t){return arguments.length?(this.cleans=!!t,this):this.cleans},modifies(t){const e=this.fields||(this.fields={});return k(t)?t.forEach((t=>e[t]=!0)):e[t]=!0,this},modified(t,e){const n=this.fields;return!(!e&&!this.mod.length||!n)&&(arguments.length?k(t)?t.some((t=>n[t])):n[t]:!!n)},filter(t,e){const n=this;return 1&t&&(n.addF=ja(n.addF,e)),2&t&&(n.remF=ja(n.remF,e)),4&t&&(n.modF=ja(n.modF,e)),16&t&&(n.srcF=ja(n.srcF,e)),n},materialize(t){const e=this;return 1&(t=t||7)&&e.addF&&(e.add=qa(e.add,e.addF),e.addF=null),2&t&&e.remF&&(e.rem=qa(e.rem,e.remF),e.remF=null),4&t&&e.modF&&(e.mod=qa(e.mod,e.modF),e.modF=null),16&t&&e.srcF&&(e.source=e.source.filter(e.srcF),e.srcF=null),e},visit(t,e){const n=this,r=e;if(16&t)return Nt(n.source,n.srcF,r),n;1&t&&Nt(n.add,n.addF,r),2&t&&Nt(n.rem,n.remF,r),4&t&&Nt(n.mod,n.modF,r);const i=n.source;if(8&t&&i){const t=n.add.length+n.mod.length;t===i.length||Nt(i,t?Pa(n,5):n.srcF,r)}return n}},dt(Ia,La,{fork(t){const e=new La(this.dataflow).init(this,t&this.NO_FIELDS);return void 0!==t&&(t&e.ADD&&this.visit(e.ADD,(t=>e.add.push(t))),t&e.REM&&this.visit(e.REM,(t=>e.rem.push(t))),t&e.MOD&&this.visit(e.MOD,(t=>e.mod.push(t)))),e},changed(t){return this.changes&t},modified(t){const e=this,n=e.fields;return n&&e.changes&e.MOD?k(t)?t.some((t=>n[t])):n[t]:0},filter(){s(\"MultiPulse does not support filtering.\")},materialize(){s(\"MultiPulse does not support materialization.\")},visit(t,e){const n=this,r=n.pulses,i=r.length;let o=0;if(t&n.SOURCE)for(;oe=[],size:()=>e.length,peek:()=>e[0],push:n=>(e.push(n),Ga(e,0,e.length-1,t)),pop:()=>{const n=e.pop();let r;return e.length?(r=e[0],e[0]=n,function(t,e,n){const r=e,i=t.length,o=t[e];let a,s=1+(e<<1);for(;s=0&&(s=a),t[e]=t[s],s=1+((e=s)<<1);t[e]=o,Ga(t,r,e,n)}(e,0,t)):r=n,r}}}function Ga(t,e,n,r){let i,o;const a=t[n];for(;n>e&&(o=n-1>>1,i=t[o],r(a,i)<0);)t[n]=i,n=o;return t[n]=a}function Va(){this.logger(w()),this.logLevel(v),this._clock=0,this._rank=0,this._locale=Uo();try{this._loader=fa()}catch(t){}this._touched=ha(c),this._input={},this._pulse=null,this._heap=Ya(((t,e)=>t.qrank-e.qrank)),this._postrun=[]}function Xa(t){return function(){return this._log[t].apply(this,arguments)}}function Ja(t,e){Sa.call(this,t,null,e)}Va.prototype={stamp(){return this._clock},loader(t){return arguments.length?(this._loader=t,this):this._loader},locale(t){return arguments.length?(this._locale=t,this):this._locale},logger(t){return arguments.length?(this._log=t,this):this._log},error:Xa(\"error\"),warn:Xa(\"warn\"),info:Xa(\"info\"),debug:Xa(\"debug\"),logLevel:Xa(\"level\"),cleanThreshold:1e4,add:function(t,e,n,r){let i,o=1;return t instanceof Sa?i=t:t&&t.prototype instanceof Sa?i=new t:J(t)?i=new Sa(null,t):(o=0,i=new Sa(t,e)),this.rank(i),o&&(r=n,n=e),n&&this.connect(i,i.parameters(n,r)),this.touch(i),i},connect:function(t,e){const n=t.rank,r=e.length;for(let i=0;i=0;)e.push(n=r[i]),n===t&&s(\"Cycle detected in dataflow graph.\")},pulse:function(t,e,n){this.touch(t,n||Ha);const r=new La(this,this._clock+(this._pulse?0:1)),i=t.pulse&&t.pulse.source||[];return r.target=t,this._input[t.id]=e.pulse(r,i),this},touch:function(t,e){const n=e||Ha;return this._pulse?this._enqueue(t):this._touched.add(t),n.skip&&t.skip(!0),this},update:function(t,e,n){const r=n||Ha;return(t.set(e)||r.force)&&this.touch(t,r),this},changeset:Ma,ingest:function(t,e,n){return e=this.parse(e,n),this.pulse(t,this.changeset().insert(e))},parse:function(t,e){const n=this.locale();return ca(t,e,n.timeParse,n.utcParse)},preload:async function(t,e,n){const r=this,i=r._pending||function(t){let e;const n=new Promise((t=>e=t));return n.requests=0,n.done=()=>{0==--n.requests&&(t._pending=null,e(t))},t._pending=n}(r);i.requests+=1;const o=await r.request(e,n);return r.pulse(t,r.changeset().remove(p).insert(o.data||[])),i.done(),o},request:async function(t,e){const n=this;let r,i=0;try{r=await n.loader().load(t,{context:\"dataflow\",response:la(e&&e.type)});try{r=n.parse(r,e)}catch(e){i=-2,n.warn(\"Data ingestion failed\",t,e)}}catch(e){i=-1,n.warn(\"Loading failed\",t,e)}return{data:r,status:i}},events:function(t,e,n,r){const i=this,o=za(n,r),a=function(t){t.dataflow=i;try{o.receive(t)}catch(t){i.error(t)}finally{i.run()}};let s;s=\"string\"==typeof t&&\"undefined\"!=typeof document?document.querySelectorAll(t):V(t);const u=s.length;for(let t=0;tr._enqueue(t,!0))),r._touched=ha(c);let a,s,u,l=0;try{for(;r._heap.size()>0;)a=r._heap.pop(),a.rank===a.qrank?(s=a.run(r._getPulse(a,t)),s.then?s=await s:s.async&&(i.push(s.async),s=Ua),s!==Ua&&a._targets&&a._targets.forEach((t=>r._enqueue(t))),++l):r._enqueue(a,!0)}catch(t){r._heap.clear(),u=t}if(r._input={},r._pulse=null,r.debug(`Pulse ${o}: ${l} operators`),u&&(r._postrun=[],r.error(u)),r._postrun.length){const t=r._postrun.sort(((t,e)=>e.priority-t.priority));r._postrun=[];for(let e=0;er.runAsync(null,(()=>{t.forEach((t=>{try{t(r)}catch(t){r.error(t)}}))})))),r},run:function(t,e,n){return this._pulse?Wa(this):(this.evaluate(t,e,n),this)},runAsync:async function(t,e,n){for(;this._running;)await this._running;const r=()=>this._running=null;return(this._running=this.evaluate(t,e,n)).then(r,r),this._running},runAfter:function(t,e,n){if(this._pulse||e)this._postrun.push({priority:n||0,callback:t});else try{t(this)}catch(t){this.error(t)}},_enqueue:function(t,e){const n=t.stampt.pulse)),e):this._input[t.id]||function(t,e){if(e&&e.stamp===t.stamp)return e;t=t.fork(),e&&e!==Ua&&(t.source=e.source);return t}(this._pulse,n&&n.pulse)}},dt(Ja,Sa,{run(t){if(t.stampthis.pulse=t)):e!==t.StopPropagation&&(this.pulse=e),e},evaluate(t){const e=this.marshall(t.stamp),n=this.transform(e,t);return e.clear(),n},transform(){}});const Za={};function Qa(t){const e=Ka(t);return e&&e.Definition||null}function Ka(t){return t=t&&t.toLowerCase(),lt(Za,t)?Za[t]:null}function*ts(t,e){if(null==e)for(let e of t)null!=e&&\"\"!==e&&(e=+e)>=e&&(yield e);else{let n=-1;for(let r of t)r=e(r,++n,t),null!=r&&\"\"!==r&&(r=+r)>=r&&(yield r)}}function es(t,e,n){const r=Float64Array.from(ts(t,n));return r.sort(Kt),e.map((t=>De(r,t)))}function ns(t,e){return es(t,[.25,.5,.75],e)}function rs(t,e){const n=t.length,r=function(t,e){const n=function(t,e){let n,r=0,i=0,o=0;if(void 0===e)for(let e of t)null!=e&&(e=+e)>=e&&(n=e-i,i+=n/++r,o+=n*(e-i));else{let a=-1;for(let s of t)null!=(s=e(s,++a,t))&&(s=+s)>=s&&(n=s-i,i+=n/++r,o+=n*(s-i))}if(r>1)return o/(r-1)}(t,e);return n?Math.sqrt(n):n}(t,e),i=ns(t,e),o=(i[2]-i[0])/1.34;return 1.06*(Math.min(r,o)||r||Math.abs(i[0])||1)*Math.pow(n,-.2)}function is(t){const e=t.maxbins||20,n=t.base||10,r=Math.log(n),i=t.divide||[5,2];let o,a,s,u,l,c,f=t.extent[0],h=t.extent[1];const d=t.span||h-f||Math.abs(f)||1;if(t.step)o=t.step;else if(t.steps){for(u=d/e,l=0,c=t.steps.length;le;)o*=n;for(l=0,c=i.length;l=s&&d/u<=e&&(o=u)}u=Math.log(o);const p=u>=0?0:1+~~(-u/r),g=Math.pow(n,-p-1);return(t.nice||void 0===t.nice)&&(u=Math.floor(f/o+g)*o,f=ft);const i=t.length,o=new Float64Array(i);let a,s=0,u=1,l=r(t[0]),c=l,f=l+e;for(;u=f){for(c=(l+c)/2;s>1);ia;)t[i--]=t[o]}o=a,a=r}return t}(o,e+e/4):o}t.random=Math.random;const ss=Math.sqrt(2*Math.PI),us=Math.SQRT2;let ls=NaN;function cs(e,n){e=e||0,n=null==n?1:n;let r,i,o=0,a=0;if(ls==ls)o=ls,ls=NaN;else{do{o=2*t.random()-1,a=2*t.random()-1,r=o*o+a*a}while(0===r||r>1);i=Math.sqrt(-2*Math.log(r)/r),o*=i,ls=a*i}return e+o*n}function fs(t,e,n){const r=(t-(e||0))/(n=null==n?1:n);return Math.exp(-.5*r*r)/(n*ss)}function hs(t,e,n){const r=(t-(e=e||0))/(n=null==n?1:n),i=Math.abs(r);let o;if(i>37)o=0;else{const t=Math.exp(-i*i/2);let e;i<7.07106781186547?(e=.0352624965998911*i+.700383064443688,e=e*i+6.37396220353165,e=e*i+33.912866078383,e=e*i+112.079291497871,e=e*i+221.213596169931,e=e*i+220.206867912376,o=t*e,e=.0883883476483184*i+1.75566716318264,e=e*i+16.064177579207,e=e*i+86.7807322029461,e=e*i+296.564248779674,e=e*i+637.333633378831,e=e*i+793.826512519948,e=e*i+440.413735824752,o/=e):(e=i+.65,e=i+4/e,e=i+3/e,e=i+2/e,e=i+1/e,o=t/e/2.506628274631)}return r>0?1-o:o}function ds(t,e,n){return t<0||t>1?NaN:(e||0)+(null==n?1:n)*us*function(t){let e,n=-Math.log((1-t)*(1+t));n<6.25?(n-=3.125,e=-364441206401782e-35,e=e*n-16850591381820166e-35,e=128584807152564e-32+e*n,e=11157877678025181e-33+e*n,e=e*n-1333171662854621e-31,e=20972767875968562e-33+e*n,e=6637638134358324e-30+e*n,e=e*n-4054566272975207e-29,e=e*n-8151934197605472e-29,e=26335093153082323e-28+e*n,e=e*n-12975133253453532e-27,e=e*n-5415412054294628e-26,e=1.0512122733215323e-9+e*n,e=e*n-4.112633980346984e-9,e=e*n-2.9070369957882005e-8,e=4.2347877827932404e-7+e*n,e=e*n-13654692000834679e-22,e=e*n-13882523362786469e-21,e=.00018673420803405714+e*n,e=e*n-.000740702534166267,e=e*n-.006033670871430149,e=.24015818242558962+e*n,e=1.6536545626831027+e*n):n<16?(n=Math.sqrt(n)-3.25,e=2.2137376921775787e-9,e=9.075656193888539e-8+e*n,e=e*n-2.7517406297064545e-7,e=1.8239629214389228e-8+e*n,e=15027403968909828e-22+e*n,e=e*n-4013867526981546e-21,e=29234449089955446e-22+e*n,e=12475304481671779e-21+e*n,e=e*n-47318229009055734e-21,e=6828485145957318e-20+e*n,e=24031110387097894e-21+e*n,e=e*n-.0003550375203628475,e=.0009532893797373805+e*n,e=e*n-.0016882755560235047,e=.002491442096107851+e*n,e=e*n-.003751208507569241,e=.005370914553590064+e*n,e=1.0052589676941592+e*n,e=3.0838856104922208+e*n):Number.isFinite(n)?(n=Math.sqrt(n)-5,e=-27109920616438573e-27,e=e*n-2.555641816996525e-10,e=1.5076572693500548e-9+e*n,e=e*n-3.789465440126737e-9,e=7.61570120807834e-9+e*n,e=e*n-1.496002662714924e-8,e=2.914795345090108e-8+e*n,e=e*n-6.771199775845234e-8,e=2.2900482228026655e-7+e*n,e=e*n-9.9298272942317e-7,e=4526062597223154e-21+e*n,e=e*n-1968177810553167e-20,e=7599527703001776e-20+e*n,e=e*n-.00021503011930044477,e=e*n-.00013871931833623122,e=1.0103004648645344+e*n,e=4.849906401408584+e*n):e=1/0;return e*t}(2*t-1)}function ps(t,e){let n,r;const i={mean(t){return arguments.length?(n=t||0,i):n},stdev(t){return arguments.length?(r=null==t?1:t,i):r},sample:()=>cs(n,r),pdf:t=>fs(t,n,r),cdf:t=>hs(t,n,r),icdf:t=>ds(t,n,r)};return i.mean(t).stdev(e)}function gs(e,n){const r=ps();let i=0;const o={data(t){return arguments.length?(e=t,i=t?t.length:0,o.bandwidth(n)):e},bandwidth(t){return arguments.length?(!(n=t)&&e&&(n=rs(e)),o):n},sample:()=>e[~~(t.random()*i)]+n*r.sample(),pdf(t){let o=0,a=0;for(;ams(n,r),pdf:t=>ys(t,n,r),cdf:t=>vs(t,n,r),icdf:t=>_s(t,n,r)};return i.mean(t).stdev(e)}function bs(e,n){let r,i=0;const o={weights(t){return arguments.length?(r=function(t){const e=[];let n,r=0;for(n=0;n=e&&t<=n?1/(n-e):0}function As(t,e,n){return null==n&&(n=null==e?1:e,e=0),tn?1:(t-e)/(n-e)}function Ms(t,e,n){return null==n&&(n=null==e?1:e,e=0),t>=0&&t<=1?e+t*(n-e):NaN}function Es(t,e){let n,r;const i={min(t){return arguments.length?(n=t||0,i):n},max(t){return arguments.length?(r=null==t?1:t,i):r},sample:()=>ws(n,r),pdf:t=>ks(t,n,r),cdf:t=>As(t,n,r),icdf:t=>Ms(t,n,r)};return null==e&&(e=null==t?1:t,t=0),i.min(t).max(e)}function Ds(t,e,n){let r=0,i=0;for(const o of t){const t=n(o);null==e(o)||null==t||isNaN(t)||(r+=(t-r)/++i)}return{coef:[r],predict:()=>r,rSquared:0}}function Cs(t,e,n,r){const i=r-t*t,o=Math.abs(i)<1e-24?0:(n-t*e)/i;return[e-o*t,o]}function Fs(t,e,n,r){t=t.filter((t=>{let r=e(t),i=n(t);return null!=r&&(r=+r)>=r&&null!=i&&(i=+i)>=i})),r&&t.sort(((t,n)=>e(t)-e(n)));const i=t.length,o=new Float64Array(i),a=new Float64Array(i);let s,u,l,c=0,f=0,h=0;for(l of t)o[c]=s=+e(l),a[c]=u=+n(l),++c,f+=(s-f)/c,h+=(u-h)/c;for(c=0;c=i&&null!=o&&(o=+o)>=o&&r(i,o,++a)}function $s(t,e,n,r,i){let o=0,a=0;return Ss(t,e,n,((t,e)=>{const n=e-i(t),s=e-r;o+=n*n,a+=s*s})),1-o/a}function Ts(t,e,n){let r=0,i=0,o=0,a=0,s=0;Ss(t,e,n,((t,e)=>{++s,r+=(t-r)/s,i+=(e-i)/s,o+=(t*e-o)/s,a+=(t*t-a)/s}));const u=Cs(r,i,o,a),l=t=>u[0]+u[1]*t;return{coef:u,predict:l,rSquared:$s(t,e,n,i,l)}}function Bs(t,e,n){let r=0,i=0,o=0,a=0,s=0;Ss(t,e,n,((t,e)=>{++s,t=Math.log(t),r+=(t-r)/s,i+=(e-i)/s,o+=(t*e-o)/s,a+=(t*t-a)/s}));const u=Cs(r,i,o,a),l=t=>u[0]+u[1]*Math.log(t);return{coef:u,predict:l,rSquared:$s(t,e,n,i,l)}}function zs(t,e,n){const[r,i,o,a]=Fs(t,e,n);let s,u,l,c=0,f=0,h=0,d=0,p=0;Ss(t,e,n,((t,e)=>{s=r[p++],u=Math.log(e),l=s*e,c+=(e*u-c)/p,f+=(l-f)/p,h+=(l*u-h)/p,d+=(s*l-d)/p}));const[g,m]=Cs(f/a,c/a,h/a,d/a),y=t=>Math.exp(g+m*(t-o));return{coef:[Math.exp(g-m*o),m],predict:y,rSquared:$s(t,e,n,a,y)}}function Ns(t,e,n){let r=0,i=0,o=0,a=0,s=0,u=0;Ss(t,e,n,((t,e)=>{const n=Math.log(t),l=Math.log(e);++u,r+=(n-r)/u,i+=(l-i)/u,o+=(n*l-o)/u,a+=(n*n-a)/u,s+=(e-s)/u}));const l=Cs(r,i,o,a),c=t=>l[0]*Math.pow(t,l[1]);return l[0]=Math.exp(l[0]),{coef:l,predict:c,rSquared:$s(t,e,n,s,c)}}function Os(t,e,n){const[r,i,o,a]=Fs(t,e,n),s=r.length;let u,l,c,f,h=0,d=0,p=0,g=0,m=0;for(u=0;u_*(t-=o)*t+x*t+b+a;return{coef:[b-x*o+_*o*o+a,x-2*_*o,_],predict:w,rSquared:$s(t,e,n,a,w)}}function Rs(t,e,n,r){if(0===r)return Ds(t,e,n);if(1===r)return Ts(t,e,n);if(2===r)return Os(t,e,n);const[i,o,a,s]=Fs(t,e,n),u=i.length,l=[],c=[],f=r+1;let h,d,p,g,m;for(h=0;hMath.abs(t[r][a])&&(a=i);for(o=r;o=r;o--)t[o][i]-=t[o][r]*t[r][i]/t[r][r]}for(i=e-1;i>=0;--i){for(s=0,o=i+1;o{t-=a;let e=s+y[0]+y[1]*t+y[2]*t*t;for(h=3;h=0;--o)for(s=e[o],u=1,i[o]+=s,a=1;a<=o;++a)u*=(o+1-a)/a,i[o-a]+=s*Math.pow(n,a)*u;return i[0]+=r,i}function Ls(t,e,n,r){const[i,o,a,s]=Fs(t,e,n,!0),u=i.length,l=Math.max(2,~~(r*u)),c=new Float64Array(u),f=new Float64Array(u),h=new Float64Array(u).fill(1);for(let t=-1;++t<=2;){const e=[0,l-1];for(let t=0;ti[a]-n?r:a;let u=0,l=0,d=0,p=0,g=0;const m=1/Math.abs(i[s]-n||1);for(let t=r;t<=a;++t){const e=i[t],r=o[t],a=qs(Math.abs(n-e)*m)*h[t],s=e*a;u+=a,l+=s,d+=r*a,p+=r*s,g+=e*s}const[y,v]=Cs(l/u,d/u,p/u,g/u);c[t]=y+v*n,f[t]=Math.abs(o[t]-c[t]),Ps(i,t+1,e)}if(2===t)break;const n=Ce(f);if(Math.abs(n)<1e-12)break;for(let t,e,r=0;r=1?1e-12:(e=1-t*t)*e}return function(t,e,n,r){const i=t.length,o=[];let a,s=0,u=0,l=[];for(;s=t.length))for(;e>i&&t[o]-r<=r-t[i];)n[0]=++i,n[1]=o,++o}const js=.5*Math.PI/180;function Is(t,e,n,r){n=n||25,r=Math.max(n,r||200);const i=e=>[e,t(e)],o=e[0],a=e[1],s=a-o,u=s/r,l=[i(o)],c=[];if(n===r){for(let t=1;t0;)c.push(i(o+t/n*s));let f=l[0],h=c[c.length-1];const d=1/s,p=function(t,e){let n=t,r=t;const i=e.length;for(let t=0;tr&&(r=i)}return 1/(r-n)}(f[1],c);for(;h;){const t=i((f[0]+h[0])/2);t[0]-f[0]>=u&&Ws(f,t,h,d,p)>js?c.push(t):(f=h,l.push(h),c.pop()),h=c[c.length-1]}return l}function Ws(t,e,n,r,i){const o=Math.atan2(i*(n[1]-t[1]),r*(n[0]-t[0])),a=Math.atan2(i*(e[1]-t[1]),r*(e[0]-t[0]));return Math.abs(o-a)}function Hs(t){return t&&t.length?1===t.length?t[0]:(e=t,t=>{const n=e.length;let r=1,i=String(e[0](t));for(;r{},Vs={init:Gs,add:Gs,rem:Gs,idx:0},Xs={values:{init:t=>t.cell.store=!0,value:t=>t.cell.data.values(),idx:-1},count:{value:t=>t.cell.num},__count__:{value:t=>t.missing+t.valid},missing:{value:t=>t.missing},valid:{value:t=>t.valid},sum:{init:t=>t.sum=0,value:t=>t.valid?t.sum:void 0,add:(t,e)=>t.sum+=+e,rem:(t,e)=>t.sum-=e},product:{init:t=>t.product=1,value:t=>t.valid?t.product:void 0,add:(t,e)=>t.product*=e,rem:(t,e)=>t.product/=e},mean:{init:t=>t.mean=0,value:t=>t.valid?t.mean:void 0,add:(t,e)=>(t.mean_d=e-t.mean,t.mean+=t.mean_d/t.valid),rem:(t,e)=>(t.mean_d=e-t.mean,t.mean-=t.valid?t.mean_d/t.valid:t.mean)},average:{value:t=>t.valid?t.mean:void 0,req:[\"mean\"],idx:1},variance:{init:t=>t.dev=0,value:t=>t.valid>1?t.dev/(t.valid-1):void 0,add:(t,e)=>t.dev+=t.mean_d*(e-t.mean),rem:(t,e)=>t.dev-=t.mean_d*(e-t.mean),req:[\"mean\"],idx:1},variancep:{value:t=>t.valid>1?t.dev/t.valid:void 0,req:[\"variance\"],idx:2},stdev:{value:t=>t.valid>1?Math.sqrt(t.dev/(t.valid-1)):void 0,req:[\"variance\"],idx:2},stdevp:{value:t=>t.valid>1?Math.sqrt(t.dev/t.valid):void 0,req:[\"variance\"],idx:2},stderr:{value:t=>t.valid>1?Math.sqrt(t.dev/(t.valid*(t.valid-1))):void 0,req:[\"variance\"],idx:2},distinct:{value:t=>t.cell.data.distinct(t.get),req:[\"values\"],idx:3},ci0:{value:t=>t.cell.data.ci0(t.get),req:[\"values\"],idx:3},ci1:{value:t=>t.cell.data.ci1(t.get),req:[\"values\"],idx:3},median:{value:t=>t.cell.data.q2(t.get),req:[\"values\"],idx:3},q1:{value:t=>t.cell.data.q1(t.get),req:[\"values\"],idx:3},q3:{value:t=>t.cell.data.q3(t.get),req:[\"values\"],idx:3},min:{init:t=>t.min=void 0,value:t=>t.min=Number.isNaN(t.min)?t.cell.data.min(t.get):t.min,add:(t,e)=>{(e{e<=t.min&&(t.min=NaN)},req:[\"values\"],idx:4},max:{init:t=>t.max=void 0,value:t=>t.max=Number.isNaN(t.max)?t.cell.data.max(t.get):t.max,add:(t,e)=>{(e>t.max||void 0===t.max)&&(t.max=e)},rem:(t,e)=>{e>=t.max&&(t.max=NaN)},req:[\"values\"],idx:4},argmin:{init:t=>t.argmin=void 0,value:t=>t.argmin||t.cell.data.argmin(t.get),add:(t,e,n)=>{e{e<=t.min&&(t.argmin=void 0)},req:[\"min\",\"values\"],idx:3},argmax:{init:t=>t.argmax=void 0,value:t=>t.argmax||t.cell.data.argmax(t.get),add:(t,e,n)=>{e>t.max&&(t.argmax=n)},rem:(t,e)=>{e>=t.max&&(t.argmax=void 0)},req:[\"max\",\"values\"],idx:3},exponential:{init:(t,e)=>{t.exp=0,t.exp_r=e},value:t=>t.valid?t.exp*(1-t.exp_r)/(1-t.exp_r**t.valid):void 0,add:(t,e)=>t.exp=t.exp_r*t.exp+e,rem:(t,e)=>t.exp=(t.exp-e/t.exp_r**(t.valid-1))/t.exp_r},exponentialb:{value:t=>t.valid?t.exp*(1-t.exp_r):void 0,req:[\"exponential\"],idx:1}},Js=Object.keys(Xs).filter((t=>\"__count__\"!==t));function Zs(t,e,n){return Xs[t](n,e)}function Qs(t,e){return t.idx-e.idx}function Ks(){this.valid=0,this.missing=0,this._ops.forEach((t=>null==t.aggregate_param?t.init(this):t.init(this,t.aggregate_param)))}function tu(t,e){null!=t&&\"\"!==t?t==t&&(++this.valid,this._ops.forEach((n=>n.add(this,t,e)))):++this.missing}function eu(t,e){null!=t&&\"\"!==t?t==t&&(--this.valid,this._ops.forEach((n=>n.rem(this,t,e)))):--this.missing}function nu(t){return this._out.forEach((e=>t[e.out]=e.value(this))),t}function ru(t,e){const n=e||f,r=function(t){const e={};t.forEach((t=>e[t.name]=t));const n=t=>{t.req&&t.req.forEach((t=>{e[t]||n(e[t]=Xs[t]())}))};return t.forEach(n),Object.values(e).sort(Qs)}(t),i=t.slice().sort(Qs);function o(t){this._ops=r,this._out=i,this.cell=t,this.init()}return o.prototype.init=Ks,o.prototype.add=tu,o.prototype.rem=eu,o.prototype.set=nu,o.prototype.get=n,o.fields=t.map((t=>t.out)),o}function iu(t){this._key=t?l(t):ya,this.reset()}[...Js,\"__count__\"].forEach((t=>{Xs[t]=function(t,e){return(n,r)=>ot({name:t,aggregate_param:r,out:n||t},Vs,e)}(t,Xs[t])}));const ou=iu.prototype;function au(t){Ja.call(this,null,t),this._adds=[],this._mods=[],this._alen=0,this._mlen=0,this._drop=!0,this._cross=!1,this._dims=[],this._dnames=[],this._measures=[],this._countOnly=!1,this._counts=null,this._prev=null,this._inputs=null,this._outputs=null}ou.reset=function(){this._add=[],this._rem=[],this._ext=null,this._get=null,this._q=null},ou.add=function(t){this._add.push(t)},ou.rem=function(t){this._rem.push(t)},ou.values=function(){if(this._get=null,0===this._rem.length)return this._add;const t=this._add,e=this._rem,n=this._key,r=t.length,i=e.length,o=Array(r-i),a={};let s,u,l;for(s=0;s=0;)r=t(e[i])+\"\",lt(n,r)||(n[r]=1,++o);return o},ou.extent=function(t){if(this._get!==t||!this._ext){const e=this.values(),n=st(e,t);this._ext=[e[n[0]],e[n[1]]],this._get=t}return this._ext},ou.argmin=function(t){return this.extent(t)[0]||{}},ou.argmax=function(t){return this.extent(t)[1]||{}},ou.min=function(t){const e=this.extent(t)[0];return null!=e?t(e):void 0},ou.max=function(t){const e=this.extent(t)[1];return null!=e?t(e):void 0},ou.quartile=function(t){return this._get===t&&this._q||(this._q=ns(this.values(),t),this._get=t),this._q},ou.q1=function(t){return this.quartile(t)[0]},ou.q2=function(t){return this.quartile(t)[1]},ou.q3=function(t){return this.quartile(t)[2]},ou.ci=function(t){return this._get===t&&this._ci||(this._ci=os(this.values(),1e3,.05,t),this._get=t),this._ci},ou.ci0=function(t){return this.ci(t)[0]},ou.ci1=function(t){return this.ci(t)[1]},au.Definition={type:\"Aggregate\",metadata:{generates:!0,changes:!0},params:[{name:\"groupby\",type:\"field\",array:!0},{name:\"ops\",type:\"enum\",array:!0,values:Js},{name:\"aggregate_params\",type:\"number\",null:!0,array:!0},{name:\"fields\",type:\"field\",null:!0,array:!0},{name:\"as\",type:\"string\",null:!0,array:!0},{name:\"drop\",type:\"boolean\",default:!0},{name:\"cross\",type:\"boolean\",default:!1},{name:\"key\",type:\"field\"}]},dt(au,Ja,{transform(t,e){const n=this,r=e.fork(e.NO_SOURCE|e.NO_FIELDS),i=t.modified();return n.stamp=r.stamp,n.value&&(i||e.modified(n._inputs,!0))?(n._prev=n.value,n.value=i?n.init(t):Object.create(null),e.visit(e.SOURCE,(t=>n.add(t)))):(n.value=n.value||n.init(t),e.visit(e.REM,(t=>n.rem(t))),e.visit(e.ADD,(t=>n.add(t)))),r.modifies(n._outputs),n._drop=!1!==t.drop,t.cross&&n._dims.length>1&&(n._drop=!1,n.cross()),e.clean()&&n._drop&&r.clean(!0).runAfter((()=>this.clean())),n.changes(r)},cross(){const t=this,e=t.value,n=t._dnames,r=n.map((()=>({}))),i=n.length;function o(t){let e,o,a,s;for(e in t)for(a=t[e].tuple,o=0;o{const e=n(t);return a(t),i.push(e),e})),this.cellkey=t.key?t.key:Hs(this._dims),this._countOnly=!0,this._counts=[],this._measures=[];const u=t.fields||[null],l=t.ops||[\"count\"],c=t.aggregate_params||[null],f=t.as||[],h=u.length,d={};let p,g,m,y,v,_,x;for(h!==l.length&&s(\"Unmatched number of fields and aggregate ops.\"),x=0;xru(t,t.field))),Object.create(null)},cellkey:Hs(),cell(t,e){let n=this.value[t];return n?0===n.num&&this._drop&&n.stampo.push(t),remove:t=>a[r(t)]=++s,size:()=>i.length,data:(t,e)=>(s&&(i=i.filter((t=>!a[r(t)])),a={},s=0),e&&t&&i.sort(t),o.length&&(i=t?At(t,i,o.sort(t)):i.concat(o),o=[]),i)}}function lu(t){Ja.call(this,[],t)}function cu(t){Sa.call(this,null,fu,t)}function fu(t){return this.value&&!t.modified()?this.value:Q(t.fields,t.orders)}function hu(t){Ja.call(this,null,t)}function du(t){Ja.call(this,null,t)}su.Definition={type:\"Bin\",metadata:{modifies:!0},params:[{name:\"field\",type:\"field\",required:!0},{name:\"interval\",type:\"boolean\",default:!0},{name:\"anchor\",type:\"number\"},{name:\"maxbins\",type:\"number\",default:20},{name:\"base\",type:\"number\",default:10},{name:\"divide\",type:\"number\",array:!0,default:[5,2]},{name:\"extent\",type:\"number\",array:!0,length:2,required:!0},{name:\"span\",type:\"number\"},{name:\"step\",type:\"number\"},{name:\"steps\",type:\"number\",array:!0},{name:\"minstep\",type:\"number\",default:0},{name:\"nice\",type:\"boolean\",default:!0},{name:\"name\",type:\"string\"},{name:\"as\",type:\"string\",array:!0,length:2,default:[\"bin0\",\"bin1\"]}]},dt(su,Ja,{transform(t,e){const n=!1!==t.interval,i=this._bins(t),o=i.start,a=i.step,s=t.as||[\"bin0\",\"bin1\"],u=s[0],l=s[1];let c;return c=t.modified()?(e=e.reflow(!0)).SOURCE:e.modified(r(t.field))?e.ADD_MOD:e.ADD,e.visit(c,n?t=>{const e=i(t);t[u]=e,t[l]=null==e?null:o+a*(1+(e-o)/a)}:t=>t[u]=i(t)),e.modifies(n?s:u)},_bins(t){if(this.value&&!t.modified())return this.value;const i=t.field,o=is(t),a=o.step;let s,u,l=o.start,c=l+Math.ceil((o.stop-l)/a)*a;null!=(s=t.anchor)&&(u=s-(l+a*Math.floor((s-l)/a)),l+=u,c+=u);const f=function(t){let e=S(i(t));return null==e?null:ec?1/0:(e=Math.max(l,Math.min(e,c-a)),l+a*Math.floor(1e-14+(e-l)/a))};return f.start=l,f.stop=o.stop,f.step=a,this.value=e(f,r(i),t.name||\"bin_\"+n(i))}}),lu.Definition={type:\"Collect\",metadata:{source:!0},params:[{name:\"sort\",type:\"compare\"}]},dt(lu,Ja,{transform(t,e){const n=e.fork(e.ALL),r=uu(ya,this.value,n.materialize(n.ADD).add),i=t.sort,o=e.changed()||i&&(t.modified(\"sort\")||e.modified(i.fields));return n.visit(n.REM,r.remove),this.modified(o),this.value=n.source=r.data(ka(i),o),e.source&&e.source.root&&(this.value.root=e.source.root),n}}),dt(cu,Sa),hu.Definition={type:\"CountPattern\",metadata:{generates:!0,changes:!0},params:[{name:\"field\",type:\"field\",required:!0},{name:\"case\",type:\"enum\",values:[\"upper\",\"lower\",\"mixed\"],default:\"mixed\"},{name:\"pattern\",type:\"string\",default:'[\\\\w\"]+'},{name:\"stopwords\",type:\"string\",default:\"\"},{name:\"as\",type:\"string\",array:!0,length:2,default:[\"text\",\"count\"]}]},dt(hu,Ja,{transform(t,e){const n=e=>n=>{for(var r,i=function(t,e,n){switch(e){case\"upper\":t=t.toUpperCase();break;case\"lower\":t=t.toLowerCase()}return t.match(n)}(s(n),t.case,o)||[],u=0,l=i.length;ui[t]=1+(i[t]||0))),c=n((t=>i[t]-=1));return r?e.visit(e.SOURCE,l):(e.visit(e.ADD,l),e.visit(e.REM,c)),this._finish(e,u)},_parameterCheck(t,e){let n=!1;return!t.modified(\"stopwords\")&&this._stop||(this._stop=new RegExp(\"^\"+(t.stopwords||\"\")+\"$\",\"i\"),n=!0),!t.modified(\"pattern\")&&this._match||(this._match=new RegExp(t.pattern||\"[\\\\w']+\",\"g\"),n=!0),(t.modified(\"field\")||e.modified(t.field.fields))&&(n=!0),n&&(this._counts={}),n},_finish(t,e){const n=this._counts,r=this._tuples||(this._tuples={}),i=e[0],o=e[1],a=t.fork(t.NO_SOURCE|t.NO_FIELDS);let s,u,l;for(s in n)u=r[s],l=n[s]||0,!u&&l?(r[s]=u=_a({}),u[i]=s,u[o]=l,a.add.push(u)):0===l?(u&&a.rem.push(u),n[s]=null,r[s]=null):u[o]!==l&&(u[o]=l,a.mod.push(u));return a.modifies(e)}}),du.Definition={type:\"Cross\",metadata:{generates:!0},params:[{name:\"filter\",type:\"expr\"},{name:\"as\",type:\"string\",array:!0,length:2,default:[\"a\",\"b\"]}]},dt(du,Ja,{transform(t,e){const n=e.fork(e.NO_SOURCE),r=t.as||[\"a\",\"b\"],i=r[0],o=r[1],a=!this.value||e.changed(e.ADD_REM)||t.modified(\"as\")||t.modified(\"filter\");let s=this.value;return a?(s&&(n.rem=s),s=e.materialize(e.SOURCE).source,n.add=this.value=function(t,e,n,r){for(var i,o,a=[],s={},u=t.length,l=0;lmu(t,e)))):typeof r[n]===gu&&r[n](t[n]);return r}function yu(t){Ja.call(this,null,t)}const vu=[{key:{function:\"normal\"},params:[{name:\"mean\",type:\"number\",default:0},{name:\"stdev\",type:\"number\",default:1}]},{key:{function:\"lognormal\"},params:[{name:\"mean\",type:\"number\",default:0},{name:\"stdev\",type:\"number\",default:1}]},{key:{function:\"uniform\"},params:[{name:\"min\",type:\"number\",default:0},{name:\"max\",type:\"number\",default:1}]},{key:{function:\"kde\"},params:[{name:\"field\",type:\"field\",required:!0},{name:\"from\",type:\"data\"},{name:\"bandwidth\",type:\"number\",default:0}]}],_u={key:{function:\"mixture\"},params:[{name:\"distributions\",type:\"param\",array:!0,params:vu},{name:\"weights\",type:\"number\",array:!0}]};function xu(t,e){return t?t.map(((t,r)=>e[r]||n(t))):null}function bu(t,e,n){const r=[],i=t=>t(u);let o,a,s,u,l,c;if(null==e)r.push(t.map(n));else for(o={},a=0,s=t.length;at.materialize(t.SOURCE).source}(e)),i=t.steps||t.minsteps||25,o=t.steps||t.maxsteps||200;let a=t.method||\"pdf\";\"pdf\"!==a&&\"cdf\"!==a&&s(\"Invalid density method: \"+a),t.extent||r.data||s(\"Missing density extent parameter.\"),a=r[a];const u=t.as||[\"value\",\"density\"],l=Is(a,t.extent||at(r.data()),i,o).map((t=>{const e={};return e[u[0]]=t[0],e[u[1]]=t[1],_a(e)}));this.value&&(n.rem=this.value),this.value=n.add=n.source=l}return n}});function wu(t){Ja.call(this,null,t)}wu.Definition={type:\"DotBin\",metadata:{modifies:!0},params:[{name:\"field\",type:\"field\",required:!0},{name:\"groupby\",type:\"field\",array:!0},{name:\"step\",type:\"number\"},{name:\"smooth\",type:\"boolean\",default:!1},{name:\"as\",type:\"string\",default:\"bin\"}]};function ku(t){Sa.call(this,null,Au,t),this.modified(!0)}function Au(t){const i=t.expr;return this.value&&!t.modified(\"expr\")?this.value:e((e=>i(e,t)),r(i),n(i))}function Mu(t){Ja.call(this,[void 0,void 0],t)}function Eu(t,e){Sa.call(this,t),this.parent=e,this.count=0}function Du(t){Ja.call(this,{},t),this._keys=ft();const e=this._targets=[];e.active=0,e.forEach=t=>{for(let n=0,r=e.active;nl(t))):l(t.name,t.as)}function Su(t){Ja.call(this,ft(),t)}function $u(t){Ja.call(this,[],t)}function Tu(t){Ja.call(this,[],t)}function Bu(t){Ja.call(this,null,t)}function zu(t){Ja.call(this,[],t)}dt(wu,Ja,{transform(t,e){if(this.value&&!t.modified()&&!e.changed())return e;const n=e.materialize(e.SOURCE).source,r=bu(e.source,t.groupby,f),i=t.smooth||!1,o=t.field,a=t.step||((t,e)=>Dt(at(t,e))/30)(n,o),s=ka(((t,e)=>o(t)-o(e))),u=t.as||\"bin\",l=r.length;let c,h=1/0,d=-1/0,p=0;for(;pd&&(d=e),t[++c][u]=e}return this.value={start:h,stop:d,step:a},e.reflow(!0).modifies(u)}}),dt(ku,Sa),Mu.Definition={type:\"Extent\",metadata:{},params:[{name:\"field\",type:\"field\",required:!0}]},dt(Mu,Ja,{transform(t,e){const r=this.value,i=t.field,o=e.changed()||e.modified(i.fields)||t.modified(\"field\");let a=r[0],s=r[1];if((o||null==a)&&(a=1/0,s=-1/0),e.visit(o?e.SOURCE:e.ADD,(t=>{const e=S(i(t));null!=e&&(es&&(s=e))})),!Number.isFinite(a)||!Number.isFinite(s)){let t=n(i);t&&(t=` for field \"${t}\"`),e.dataflow.warn(`Infinite extent${t}: [${a}, ${s}]`),a=s=void 0}this.value=[a,s]}}),dt(Eu,Sa,{connect(t){return this.detachSubflow=t.detachSubflow,this.targets().add(t),t.source=this},add(t){this.count+=1,this.value.add.push(t)},rem(t){this.count-=1,this.value.rem.push(t)},mod(t){this.value.mod.push(t)},init(t){this.value.init(t,t.NO_SOURCE)},evaluate(){return this.value}}),dt(Du,Ja,{activate(t){this._targets[this._targets.active++]=t},subflow(t,e,n,r){const i=this.value;let o,a,s=lt(i,t)&&i[t];return s?s.value.stampt&&t.count>0));this.initTargets(t)}},initTargets(t){const e=this._targets,n=e.length,r=t?t.length:0;let i=0;for(;ithis.subflow(t,i,e);return this._group=t.group||{},this.initTargets(),e.visit(e.REM,(t=>{const e=ya(t),n=o.get(e);void 0!==n&&(o.delete(e),s(n).rem(t))})),e.visit(e.ADD,(t=>{const e=r(t);o.set(ya(t),e),s(e).add(t)})),a||e.modified(r.fields)?e.visit(e.MOD,(t=>{const e=ya(t),n=o.get(e),i=r(t);n===i?s(i).mod(t):(o.set(e,i),s(n).rem(t),s(i).add(t))})):e.changed(e.MOD)&&e.visit(e.MOD,(t=>{s(o.get(ya(t))).mod(t)})),a&&e.visit(e.REFLOW,(t=>{const e=ya(t),n=o.get(e),i=r(t);n!==i&&(o.set(e,i),s(n).rem(t),s(i).add(t))})),e.clean()?n.runAfter((()=>{this.clean(),o.clean()})):o.empty>n.cleanThreshold&&n.runAfter(o.clean),e}}),dt(Cu,Sa),Su.Definition={type:\"Filter\",metadata:{changes:!0},params:[{name:\"expr\",type:\"expr\",required:!0}]},dt(Su,Ja,{transform(t,e){const n=e.dataflow,r=this.value,i=e.fork(),o=i.add,a=i.rem,s=i.mod,u=t.expr;let l=!0;function c(e){const n=ya(e),i=u(e,t),c=r.get(n);i&&c?(r.delete(n),o.push(e)):i||c?l&&i&&!c&&s.push(e):(r.set(n,1),a.push(e))}return e.visit(e.REM,(t=>{const e=ya(t);r.has(e)?r.delete(e):a.push(t)})),e.visit(e.ADD,(e=>{u(e,t)?o.push(e):r.set(ya(e),1)})),e.visit(e.MOD,c),t.modified()&&(l=!1,e.visit(e.REFLOW,c)),r.empty>n.cleanThreshold&&n.runAfter(r.clean),i}}),$u.Definition={type:\"Flatten\",metadata:{generates:!0},params:[{name:\"fields\",type:\"field\",array:!0,required:!0},{name:\"index\",type:\"string\"},{name:\"as\",type:\"string\",array:!0}]},dt($u,Ja,{transform(t,e){const n=e.fork(e.NO_SOURCE),r=t.fields,i=xu(r,t.as||[]),o=t.index||null,a=i.length;return n.rem=this.value,e.visit(e.SOURCE,(t=>{const e=r.map((e=>e(t))),s=e.reduce(((t,e)=>Math.max(t,e.length)),0);let u,l,c,f=0;for(;f{for(let e,n=0;ne[r]=n(e,t)))}}),dt(zu,Ja,{transform(t,e){const n=e.fork(e.ALL),r=t.generator;let i,o,a,s=this.value,u=t.size-s.length;if(u>0){for(i=[];--u>=0;)i.push(a=_a(r(t))),s.push(a);n.add=n.add.length?n.materialize(n.ADD).add.concat(i):i}else o=s.slice(0,-u),n.rem=n.rem.length?n.materialize(n.REM).rem.concat(o):o,s=s.slice(-u);return n.source=this.value=s,n}});const Nu={value:\"value\",median:Ce,mean:function(t,e){let n=0,r=0;if(void 0===e)for(let e of t)null!=e&&(e=+e)>=e&&(++n,r+=e);else{let i=-1;for(let o of t)null!=(o=e(o,++i,t))&&(o=+o)>=o&&(++n,r+=o)}if(n)return r/n},min:ke,max:we},Ou=[];function Ru(t){Ja.call(this,[],t)}function Uu(t){au.call(this,t)}function Lu(t){Ja.call(this,null,t)}function qu(t){Sa.call(this,null,Pu,t)}function Pu(t){return this.value&&!t.modified()?this.value:bt(t.fields,t.flat)}function ju(t){Ja.call(this,[],t),this._pending=null}function Iu(t,e,n){n.forEach(_a);const r=e.fork(e.NO_FIELDS&e.NO_SOURCE);return r.rem=t.value,t.value=r.source=r.add=n,t._pending=null,r.rem.length&&r.clean(!0),r}function Wu(t){Ja.call(this,{},t)}function Hu(t){Sa.call(this,null,Yu,t)}function Yu(t){if(this.value&&!t.modified())return this.value;const e=t.extents,n=e.length;let r,i,o=1/0,a=-1/0;for(r=0;ra&&(a=i[1]);return[o,a]}function Gu(t){Sa.call(this,null,Vu,t)}function Vu(t){return this.value&&!t.modified()?this.value:t.values.reduce(((t,e)=>t.concat(e)),[])}function Xu(t){Ja.call(this,null,t)}function Ju(t){au.call(this,t)}function Zu(t){Du.call(this,t)}function Qu(t){Ja.call(this,null,t)}function Ku(t){Ja.call(this,null,t)}function tl(t){Ja.call(this,null,t)}Ru.Definition={type:\"Impute\",metadata:{changes:!0},params:[{name:\"field\",type:\"field\",required:!0},{name:\"key\",type:\"field\",required:!0},{name:\"keyvals\",array:!0},{name:\"groupby\",type:\"field\",array:!0},{name:\"method\",type:\"enum\",default:\"value\",values:[\"value\",\"mean\",\"median\",\"max\",\"min\"]},{name:\"value\",default:0}]},dt(Ru,Ja,{transform(t,e){var r,i,o,a,u,l,c,f,h,d,p=e.fork(e.ALL),g=function(t){var e,n=t.method||Nu.value;if(null!=Nu[n])return n===Nu.value?(e=void 0!==t.value?t.value:0,()=>e):Nu[n];s(\"Unrecognized imputation method: \"+n)}(t),m=function(t){const e=t.field;return t=>t?e(t):NaN}(t),y=n(t.field),v=n(t.key),_=(t.groupby||[]).map(n),x=function(t,e,n,r){var i,o,a,s,u,l,c,f,h=t=>t(f),d=[],p=r?r.slice():[],g={},m={};for(p.forEach(((t,e)=>g[t]=e+1)),s=0,c=t.length;sn.add(t)))):(i=n.value=n.value||this.init(t),e.visit(e.REM,(t=>n.rem(t))),e.visit(e.ADD,(t=>n.add(t)))),n.changes(),e.visit(e.SOURCE,(t=>{ot(t,i[n.cellkey(t)].tuple)})),e.reflow(r).modifies(this._outputs)},changes(){const t=this._adds,e=this._mods;let n,r;for(n=0,r=this._alen;n{const n=gs(e,u)[l],r=t.counts?e.length:1;Is(n,h||at(e),d,p).forEach((t=>{const n={};for(let t=0;t(this._pending=V(t.data),t=>t.touch(this))));return{async:e}}return n.request(t.url,t.format).then((t=>Iu(this,e,V(t.data))))}}),Wu.Definition={type:\"Lookup\",metadata:{modifies:!0},params:[{name:\"index\",type:\"index\",params:[{name:\"from\",type:\"data\",required:!0},{name:\"key\",type:\"field\",required:!0}]},{name:\"values\",type:\"field\",array:!0},{name:\"fields\",type:\"field\",array:!0,required:!0},{name:\"as\",type:\"string\",array:!0},{name:\"default\",default:null}]},dt(Wu,Ja,{transform(t,e){const r=t.fields,i=t.index,o=t.values,a=null==t.default?null:t.default,u=t.modified(),l=r.length;let c,f,h,d=u?e.SOURCE:e.ADD,p=e,g=t.as;return o?(f=o.length,l>1&&!g&&s('Multi-field lookup requires explicit \"as\" parameter.'),g&&g.length!==l*f&&s('The \"as\" parameter has too few output field names.'),g=g||o.map(n),c=function(t){for(var e,n,s=0,u=0;se.modified(t.fields))),d|=h?e.MOD:0),e.visit(d,c),p.modifies(g)}}),dt(Hu,Sa),dt(Gu,Sa),dt(Xu,Ja,{transform(t,e){return this.modified(t.modified()),this.value=t,e.fork(e.NO_SOURCE|e.NO_FIELDS)}}),Ju.Definition={type:\"Pivot\",metadata:{generates:!0,changes:!0},params:[{name:\"groupby\",type:\"field\",array:!0},{name:\"field\",type:\"field\",required:!0},{name:\"value\",type:\"field\",required:!0},{name:\"op\",type:\"enum\",values:Js,default:\"sum\"},{name:\"limit\",type:\"number\",default:0},{name:\"key\",type:\"field\"}]},dt(Ju,au,{_transform:au.prototype.transform,transform(t,n){return this._transform(function(t,n){const i=t.field,o=t.value,a=(\"count\"===t.op?\"__count__\":t.op)||\"sum\",s=r(i).concat(r(o)),u=function(t,e,n){const r={},i=[];return n.visit(n.SOURCE,(e=>{const n=t(e);r[n]||(r[n]=1,i.push(n))})),i.sort(K),e?i.slice(0,e):i}(i,t.limit||0,n);n.changed()&&t.set(\"__pivot__\",null,null,!0);return{key:t.key,groupby:t.groupby,ops:u.map((()=>a)),fields:u.map((t=>function(t,n,r,i){return e((e=>n(e)===t?r(e):NaN),i,t+\"\")}(t,i,o,s))),as:u.map((t=>t+\"\")),modified:t.modified.bind(t)}}(t,n),n)}}),dt(Zu,Du,{transform(t,e){const n=t.subflow,i=t.field,o=t=>this.subflow(ya(t),n,e,t);return(t.modified(\"field\")||i&&e.modified(r(i)))&&s(\"PreFacet does not support field modification.\"),this.initTargets(),i?(e.visit(e.MOD,(t=>{const e=o(t);i(t).forEach((t=>e.mod(t)))})),e.visit(e.ADD,(t=>{const e=o(t);i(t).forEach((t=>e.add(_a(t))))})),e.visit(e.REM,(t=>{const e=o(t);i(t).forEach((t=>e.rem(t)))}))):(e.visit(e.MOD,(t=>o(t).mod(t))),e.visit(e.ADD,(t=>o(t).add(t))),e.visit(e.REM,(t=>o(t).rem(t)))),e.clean()&&e.runAfter((()=>this.clean())),e}}),Qu.Definition={type:\"Project\",metadata:{generates:!0,changes:!0},params:[{name:\"fields\",type:\"field\",array:!0},{name:\"as\",type:\"string\",null:!0,array:!0}]},dt(Qu,Ja,{transform(t,e){const n=e.fork(e.NO_SOURCE),r=t.fields,i=xu(t.fields,t.as||[]),o=r?(t,e)=>function(t,e,n,r){for(let i=0,o=n.length;i{const e=ya(t);n.rem.push(a[e]),a[e]=null})),e.visit(e.ADD,(t=>{const e=o(t,_a({}));a[ya(t)]=e,n.add.push(e)})),e.visit(e.MOD,(t=>{n.mod.push(o(t,a[ya(t)]))})),n}}),dt(Ku,Ja,{transform(t,e){return this.value=t.value,t.modified(\"value\")?e.fork(e.NO_SOURCE|e.NO_FIELDS):e.StopPropagation}}),tl.Definition={type:\"Quantile\",metadata:{generates:!0,changes:!0},params:[{name:\"groupby\",type:\"field\",array:!0},{name:\"field\",type:\"field\",required:!0},{name:\"probs\",type:\"number\",array:!0},{name:\"step\",type:\"number\",default:.01},{name:\"as\",type:\"string\",array:!0,default:[\"prob\",\"value\"]}]};function el(t){Ja.call(this,null,t)}function nl(t){Ja.call(this,[],t),this.count=0}function rl(t){Ja.call(this,null,t)}function il(t){Ja.call(this,null,t),this.modified(!0)}function ol(t){Ja.call(this,null,t)}dt(tl,Ja,{transform(t,e){const r=e.fork(e.NO_SOURCE|e.NO_FIELDS),i=t.as||[\"prob\",\"value\"];if(this.value&&!t.modified()&&!e.changed())return r.source=this.value,r;const o=bu(e.materialize(e.SOURCE).source,t.groupby,t.field),a=(t.groupby||[]).map(n),s=[],u=t.step||.01,l=t.probs||Se(u/2,1-1e-14,u),c=l.length;return o.forEach((t=>{const e=es(t,l);for(let n=0;n{const e=ya(t);n.rem.push(r[e]),r[e]=null})),e.visit(e.ADD,(t=>{const e=xa(t);r[ya(t)]=e,n.add.push(e)})),e.visit(e.MOD,(t=>{const e=r[ya(t)];for(const r in t)e[r]=t[r],n.modifies(r);n.mod.push(e)}))),n}}),nl.Definition={type:\"Sample\",metadata:{},params:[{name:\"size\",type:\"number\",default:1e3}]},dt(nl,Ja,{transform(e,n){const r=n.fork(n.NO_SOURCE),i=e.modified(\"size\"),o=e.size,a=this.value.reduce(((t,e)=>(t[ya(e)]=1,t)),{});let s=this.value,u=this.count,l=0;function c(e){let n,i;s.length=l&&(n=s[i],a[ya(n)]&&r.rem.push(n),s[i]=e)),++u}if(n.rem.length&&(n.visit(n.REM,(t=>{const e=ya(t);a[e]&&(a[e]=-1,r.rem.push(t)),--u})),s=s.filter((t=>-1!==a[ya(t)]))),(n.rem.length||i)&&s.length{a[ya(t)]||c(t)})),l=-1),i&&s.length>o){const t=s.length-o;for(let e=0;e{a[ya(t)]&&r.mod.push(t)})),n.add.length&&n.visit(n.ADD,c),(n.add.length||l<0)&&(r.add=s.filter((t=>!a[ya(t)]))),this.count=u,this.value=r.source=s,r}}),rl.Definition={type:\"Sequence\",metadata:{generates:!0,changes:!0},params:[{name:\"start\",type:\"number\",required:!0},{name:\"stop\",type:\"number\",required:!0},{name:\"step\",type:\"number\",default:1},{name:\"as\",type:\"string\",default:\"data\"}]},dt(rl,Ja,{transform(t,e){if(this.value&&!t.modified())return;const n=e.materialize().fork(e.MOD),r=t.as||\"data\";return n.rem=this.value?e.rem.concat(this.value):e.rem,this.value=Se(t.start,t.stop,t.step||1).map((t=>{const e={};return e[r]=t,_a(e)})),n.add=e.add.concat(this.value),n}}),dt(il,Ja,{transform(t,e){return this.value=e.source,e.changed()?e.fork(e.NO_SOURCE|e.NO_FIELDS):e.StopPropagation}});const al=[\"unit0\",\"unit1\"];function sl(t){Ja.call(this,ft(),t)}function ul(t){Ja.call(this,null,t)}ol.Definition={type:\"TimeUnit\",metadata:{modifies:!0},params:[{name:\"field\",type:\"field\",required:!0},{name:\"interval\",type:\"boolean\",default:!0},{name:\"units\",type:\"enum\",values:Kn,array:!0},{name:\"step\",type:\"number\",default:1},{name:\"maxbins\",type:\"number\",default:40},{name:\"extent\",type:\"date\",array:!0},{name:\"timezone\",type:\"enum\",default:\"local\",values:[\"local\",\"utc\"]},{name:\"as\",type:\"string\",array:!0,length:2,default:al}]},dt(ol,Ja,{transform(t,e){const n=t.field,i=!1!==t.interval,o=\"utc\"===t.timezone,a=this._floor(t,e),s=(o?Fr:Cr)(a.unit).offset,u=t.as||al,l=u[0],c=u[1],f=a.step;let h=a.start||1/0,d=a.stop||-1/0,p=e.ADD;return(t.modified()||e.changed(e.REM)||e.modified(r(n)))&&(p=(e=e.reflow(!0)).SOURCE,h=1/0,d=-1/0),e.visit(p,(t=>{const e=n(t);let r,o;null==e?(t[l]=null,i&&(t[c]=null)):(t[l]=r=o=a(e),i&&(t[c]=o=s(r,f)),rd&&(d=o))})),a.start=h,a.stop=d,e.modifies(i?u:l)},_floor(t,e){const n=\"utc\"===t.timezone,{units:r,step:i}=t.units?{units:t.units,step:t.step||1}:Jr({extent:t.extent||at(e.materialize(e.SOURCE).source,t.field),maxbins:t.maxbins}),o=er(r),a=this.value||{},s=(n?Mr:wr)(o,i);return s.unit=F(o),s.units=o,s.step=i,s.start=a.start,s.stop=a.stop,this.value=s}}),dt(sl,Ja,{transform(t,e){const n=e.dataflow,r=t.field,i=this.value,o=t=>i.set(r(t),t);let a=!0;return t.modified(\"field\")||e.modified(r.fields)?(i.clear(),e.visit(e.SOURCE,o)):e.changed()?(e.visit(e.REM,(t=>i.delete(r(t)))),e.visit(e.ADD,o)):a=!1,this.modified(a),i.empty>n.cleanThreshold&&n.runAfter(i.clean),e.fork()}}),dt(ul,Ja,{transform(t,e){(!this.value||t.modified(\"field\")||t.modified(\"sort\")||e.changed()||t.sort&&e.modified(t.sort.fields))&&(this.value=(t.sort?e.source.slice().sort(ka(t.sort)):e.source).map(t.field))}});const ll={row_number:function(){return{next:t=>t.index+1}},rank:function(){let t;return{init:()=>t=1,next:e=>{const n=e.index,r=e.data;return n&&e.compare(r[n-1],r[n])?t=n+1:t}}},dense_rank:function(){let t;return{init:()=>t=1,next:e=>{const n=e.index,r=e.data;return n&&e.compare(r[n-1],r[n])?++t:t}}},percent_rank:function(){const t=ll.rank(),e=t.next;return{init:t.init,next:t=>(e(t)-1)/(t.data.length-1)}},cume_dist:function(){let t;return{init:()=>t=0,next:e=>{const n=e.data,r=e.compare;let i=e.index;if(t0||s(\"ntile num must be greater than zero.\");const n=ll.cume_dist(),r=n.next;return{init:n.init,next:t=>Math.ceil(e*r(t))}},lag:function(t,e){return e=+e||1,{next:n=>{const r=n.index-e;return r>=0?t(n.data[r]):null}}},lead:function(t,e){return e=+e||1,{next:n=>{const r=n.index+e,i=n.data;return rt(e.data[e.i0])}},last_value:function(t){return{next:e=>t(e.data[e.i1-1])}},nth_value:function(t,e){return(e=+e)>0||s(\"nth_value nth must be greater than zero.\"),{next:n=>{const r=n.i0+(e-1);return re=null,next:n=>{const r=t(n.data[n.index]);return null!=r?e=r:e}}},next_value:function(t){let e,n;return{init:()=>(e=null,n=-1),next:r=>{const i=r.data;return r.index<=n?e:(n=function(t,e,n){for(let r=e.length;nf[t]=1))}y(t.sort),e.forEach(((t,e)=>{const r=i[e],f=o[e],v=a[e]||null,_=n(r),x=Ys(t,_,u[e]);if(y(r),l.push(x),lt(ll,t))c.push(function(t,e,n,r){const i=ll[t](e,n);return{init:i.init||h,update:function(t,e){e[r]=i.next(t)}}}(t,r,f,x));else{if(null==r&&\"count\"!==t&&s(\"Null aggregate field specified.\"),\"count\"===t)return void p.push(x);m=!1;let e=d[_];e||(e=d[_]=[],e.field=r,g.push(e)),e.push(Zs(t,v,x))}})),(p.length||g.length)&&(this.cell=function(t,e,n){t=t.map((t=>ru(t,t.field)));const r={num:0,agg:null,store:!1,count:e};if(!n)for(var i=t.length,o=r.agg=Array(i),a=0;a0&&!i(o[n],o[n-1])&&(t.i0=e.left(o,o[n])),rt.init())),this.cell&&this.cell.init()},hl.update=function(t,e){const n=this.cell,r=this.windows,i=t.data,o=r&&r.length;let a;if(n){for(a=t.p0;athis.group(i(t));let a=this.state;a&&!n||(a=this.state=new fl(t)),n||e.modified(a.inputs)?(this.value={},e.visit(e.SOURCE,(t=>o(t).add(t)))):(e.visit(e.REM,(t=>o(t).remove(t))),e.visit(e.ADD,(t=>o(t).add(t))));for(let e=0,n=this._mlen;e=1?Cl:t<=-1?-Cl:Math.asin(t)}const $l=Math.PI,Tl=2*$l,Bl=1e-6,zl=Tl-Bl;function Nl(t){this._+=t[0];for(let e=1,n=t.length;e=0))throw new Error(`invalid digits: ${t}`);if(e>15)return Nl;const n=10**e;return function(t){this._+=t[0];for(let e=1,r=t.length;eBl)if(Math.abs(c*s-u*l)>Bl&&i){let h=n-o,d=r-a,p=s*s+u*u,g=h*h+d*d,m=Math.sqrt(p),y=Math.sqrt(f),v=i*Math.tan(($l-Math.acos((p+f-g)/(2*m*y)))/2),_=v/y,x=v/m;Math.abs(_-1)>Bl&&this._append`L${t+_*l},${e+_*c}`,this._append`A${i},${i},0,0,${+(c*h>l*d)},${this._x1=t+x*s},${this._y1=e+x*u}`}else this._append`L${this._x1=t},${this._y1=e}`;else;}arc(t,e,n,r,i,o){if(t=+t,e=+e,o=!!o,(n=+n)<0)throw new Error(`negative radius: ${n}`);let a=n*Math.cos(r),s=n*Math.sin(r),u=t+a,l=e+s,c=1^o,f=o?r-i:i-r;null===this._x1?this._append`M${u},${l}`:(Math.abs(this._x1-u)>Bl||Math.abs(this._y1-l)>Bl)&&this._append`L${u},${l}`,n&&(f<0&&(f=f%Tl+Tl),f>zl?this._append`A${n},${n},0,1,${c},${t-a},${e-s}A${n},${n},0,1,${c},${this._x1=u},${this._y1=l}`:f>Bl&&this._append`A${n},${n},0,${+(f>=$l)},${c},${this._x1=t+n*Math.cos(i)},${this._y1=e+n*Math.sin(i)}`)}rect(t,e,n,r){this._append`M${this._x0=this._x1=+t},${this._y0=this._y1=+e}h${n=+n}v${+r}h${-n}Z`}toString(){return this._}};function Rl(){return new Ol}function Ul(t){let e=3;return t.digits=function(n){if(!arguments.length)return e;if(null==n)e=null;else{const t=Math.floor(n);if(!(t>=0))throw new RangeError(`invalid digits: ${n}`);e=t}return t},()=>new Ol(e)}function Ll(t){return t.innerRadius}function ql(t){return t.outerRadius}function Pl(t){return t.startAngle}function jl(t){return t.endAngle}function Il(t){return t&&t.padAngle}function Wl(t,e,n,r,i,o,a){var s=t-n,u=e-r,l=(a?o:-o)/Ml(s*s+u*u),c=l*u,f=-l*s,h=t+c,d=e+f,p=n+c,g=r+f,m=(h+p)/2,y=(d+g)/2,v=p-h,_=g-d,x=v*v+_*_,b=i-o,w=h*g-p*d,k=(_<0?-1:1)*Ml(wl(0,b*b*x-w*w)),A=(w*_-v*k)/x,M=(-w*v-_*k)/x,E=(w*_+v*k)/x,D=(-w*v+_*k)/x,C=A-m,F=M-y,S=E-m,$=D-y;return C*C+F*F>S*S+$*$&&(A=E,M=D),{cx:A,cy:M,x01:-c,y01:-f,x11:A*(i/b-1),y11:M*(i/b-1)}}function Hl(t){return\"object\"==typeof t&&\"length\"in t?t:Array.from(t)}function Yl(t){this._context=t}function Gl(t){return new Yl(t)}function Vl(t){return t[0]}function Xl(t){return t[1]}function Jl(t,e){var n=vl(!0),r=null,i=Gl,o=null,a=Ul(s);function s(s){var u,l,c,f=(s=Hl(s)).length,h=!1;for(null==r&&(o=i(c=a())),u=0;u<=f;++u)!(u=f;--h)s.point(y[h],v[h]);s.lineEnd(),s.areaEnd()}m&&(y[c]=+t(d,c,l),v[c]=+e(d,c,l),s.point(r?+r(d,c,l):y[c],n?+n(d,c,l):v[c]))}if(p)return s=null,p+\"\"||null}function c(){return Jl().defined(i).curve(a).context(o)}return t=\"function\"==typeof t?t:void 0===t?Vl:vl(+t),e=\"function\"==typeof e?e:vl(void 0===e?0:+e),n=\"function\"==typeof n?n:void 0===n?Xl:vl(+n),l.x=function(e){return arguments.length?(t=\"function\"==typeof e?e:vl(+e),r=null,l):t},l.x0=function(e){return arguments.length?(t=\"function\"==typeof e?e:vl(+e),l):t},l.x1=function(t){return arguments.length?(r=null==t?null:\"function\"==typeof t?t:vl(+t),l):r},l.y=function(t){return arguments.length?(e=\"function\"==typeof t?t:vl(+t),n=null,l):e},l.y0=function(t){return arguments.length?(e=\"function\"==typeof t?t:vl(+t),l):e},l.y1=function(t){return arguments.length?(n=null==t?null:\"function\"==typeof t?t:vl(+t),l):n},l.lineX0=l.lineY0=function(){return c().x(t).y(e)},l.lineY1=function(){return c().x(t).y(n)},l.lineX1=function(){return c().x(r).y(e)},l.defined=function(t){return arguments.length?(i=\"function\"==typeof t?t:vl(!!t),l):i},l.curve=function(t){return arguments.length?(a=t,null!=o&&(s=a(o)),l):a},l.context=function(t){return arguments.length?(null==t?o=s=null:s=a(o=t),l):o},l}Rl.prototype=Ol.prototype,Yl.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._point=0},lineEnd:function(){(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,e){switch(t=+t,e=+e,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,e):this._context.moveTo(t,e);break;case 1:this._point=2;default:this._context.lineTo(t,e)}}};var Ql={draw(t,e){const n=Ml(e/Dl);t.moveTo(n,0),t.arc(0,0,n,0,Fl)}};function Kl(){}function tc(t,e,n){t._context.bezierCurveTo((2*t._x0+t._x1)/3,(2*t._y0+t._y1)/3,(t._x0+2*t._x1)/3,(t._y0+2*t._y1)/3,(t._x0+4*t._x1+e)/6,(t._y0+4*t._y1+n)/6)}function ec(t){this._context=t}function nc(t){this._context=t}function rc(t){this._context=t}function ic(t,e){this._basis=new ec(t),this._beta=e}ec.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._y0=this._y1=NaN,this._point=0},lineEnd:function(){switch(this._point){case 3:tc(this,this._x1,this._y1);case 2:this._context.lineTo(this._x1,this._y1)}(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,e){switch(t=+t,e=+e,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,e):this._context.moveTo(t,e);break;case 1:this._point=2;break;case 2:this._point=3,this._context.lineTo((5*this._x0+this._x1)/6,(5*this._y0+this._y1)/6);default:tc(this,t,e)}this._x0=this._x1,this._x1=t,this._y0=this._y1,this._y1=e}},nc.prototype={areaStart:Kl,areaEnd:Kl,lineStart:function(){this._x0=this._x1=this._x2=this._x3=this._x4=this._y0=this._y1=this._y2=this._y3=this._y4=NaN,this._point=0},lineEnd:function(){switch(this._point){case 1:this._context.moveTo(this._x2,this._y2),this._context.closePath();break;case 2:this._context.moveTo((this._x2+2*this._x3)/3,(this._y2+2*this._y3)/3),this._context.lineTo((this._x3+2*this._x2)/3,(this._y3+2*this._y2)/3),this._context.closePath();break;case 3:this.point(this._x2,this._y2),this.point(this._x3,this._y3),this.point(this._x4,this._y4)}},point:function(t,e){switch(t=+t,e=+e,this._point){case 0:this._point=1,this._x2=t,this._y2=e;break;case 1:this._point=2,this._x3=t,this._y3=e;break;case 2:this._point=3,this._x4=t,this._y4=e,this._context.moveTo((this._x0+4*this._x1+t)/6,(this._y0+4*this._y1+e)/6);break;default:tc(this,t,e)}this._x0=this._x1,this._x1=t,this._y0=this._y1,this._y1=e}},rc.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._y0=this._y1=NaN,this._point=0},lineEnd:function(){(this._line||0!==this._line&&3===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,e){switch(t=+t,e=+e,this._point){case 0:this._point=1;break;case 1:this._point=2;break;case 2:this._point=3;var n=(this._x0+4*this._x1+t)/6,r=(this._y0+4*this._y1+e)/6;this._line?this._context.lineTo(n,r):this._context.moveTo(n,r);break;case 3:this._point=4;default:tc(this,t,e)}this._x0=this._x1,this._x1=t,this._y0=this._y1,this._y1=e}},ic.prototype={lineStart:function(){this._x=[],this._y=[],this._basis.lineStart()},lineEnd:function(){var t=this._x,e=this._y,n=t.length-1;if(n>0)for(var r,i=t[0],o=e[0],a=t[n]-i,s=e[n]-o,u=-1;++u<=n;)r=u/n,this._basis.point(this._beta*t[u]+(1-this._beta)*(i+r*a),this._beta*e[u]+(1-this._beta)*(o+r*s));this._x=this._y=null,this._basis.lineEnd()},point:function(t,e){this._x.push(+t),this._y.push(+e)}};var oc=function t(e){function n(t){return 1===e?new ec(t):new ic(t,e)}return n.beta=function(e){return t(+e)},n}(.85);function ac(t,e,n){t._context.bezierCurveTo(t._x1+t._k*(t._x2-t._x0),t._y1+t._k*(t._y2-t._y0),t._x2+t._k*(t._x1-e),t._y2+t._k*(t._y1-n),t._x2,t._y2)}function sc(t,e){this._context=t,this._k=(1-e)/6}sc.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._x2=this._y0=this._y1=this._y2=NaN,this._point=0},lineEnd:function(){switch(this._point){case 2:this._context.lineTo(this._x2,this._y2);break;case 3:ac(this,this._x1,this._y1)}(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,e){switch(t=+t,e=+e,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,e):this._context.moveTo(t,e);break;case 1:this._point=2,this._x1=t,this._y1=e;break;case 2:this._point=3;default:ac(this,t,e)}this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=e}};var uc=function t(e){function n(t){return new sc(t,e)}return n.tension=function(e){return t(+e)},n}(0);function lc(t,e){this._context=t,this._k=(1-e)/6}lc.prototype={areaStart:Kl,areaEnd:Kl,lineStart:function(){this._x0=this._x1=this._x2=this._x3=this._x4=this._x5=this._y0=this._y1=this._y2=this._y3=this._y4=this._y5=NaN,this._point=0},lineEnd:function(){switch(this._point){case 1:this._context.moveTo(this._x3,this._y3),this._context.closePath();break;case 2:this._context.lineTo(this._x3,this._y3),this._context.closePath();break;case 3:this.point(this._x3,this._y3),this.point(this._x4,this._y4),this.point(this._x5,this._y5)}},point:function(t,e){switch(t=+t,e=+e,this._point){case 0:this._point=1,this._x3=t,this._y3=e;break;case 1:this._point=2,this._context.moveTo(this._x4=t,this._y4=e);break;case 2:this._point=3,this._x5=t,this._y5=e;break;default:ac(this,t,e)}this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=e}};var cc=function t(e){function n(t){return new lc(t,e)}return n.tension=function(e){return t(+e)},n}(0);function fc(t,e){this._context=t,this._k=(1-e)/6}fc.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._x2=this._y0=this._y1=this._y2=NaN,this._point=0},lineEnd:function(){(this._line||0!==this._line&&3===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,e){switch(t=+t,e=+e,this._point){case 0:this._point=1;break;case 1:this._point=2;break;case 2:this._point=3,this._line?this._context.lineTo(this._x2,this._y2):this._context.moveTo(this._x2,this._y2);break;case 3:this._point=4;default:ac(this,t,e)}this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=e}};var hc=function t(e){function n(t){return new fc(t,e)}return n.tension=function(e){return t(+e)},n}(0);function dc(t,e,n){var r=t._x1,i=t._y1,o=t._x2,a=t._y2;if(t._l01_a>El){var s=2*t._l01_2a+3*t._l01_a*t._l12_a+t._l12_2a,u=3*t._l01_a*(t._l01_a+t._l12_a);r=(r*s-t._x0*t._l12_2a+t._x2*t._l01_2a)/u,i=(i*s-t._y0*t._l12_2a+t._y2*t._l01_2a)/u}if(t._l23_a>El){var l=2*t._l23_2a+3*t._l23_a*t._l12_a+t._l12_2a,c=3*t._l23_a*(t._l23_a+t._l12_a);o=(o*l+t._x1*t._l23_2a-e*t._l12_2a)/c,a=(a*l+t._y1*t._l23_2a-n*t._l12_2a)/c}t._context.bezierCurveTo(r,i,o,a,t._x2,t._y2)}function pc(t,e){this._context=t,this._alpha=e}pc.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._x2=this._y0=this._y1=this._y2=NaN,this._l01_a=this._l12_a=this._l23_a=this._l01_2a=this._l12_2a=this._l23_2a=this._point=0},lineEnd:function(){switch(this._point){case 2:this._context.lineTo(this._x2,this._y2);break;case 3:this.point(this._x2,this._y2)}(this._line||0!==this._line&&1===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,e){if(t=+t,e=+e,this._point){var n=this._x2-t,r=this._y2-e;this._l23_a=Math.sqrt(this._l23_2a=Math.pow(n*n+r*r,this._alpha))}switch(this._point){case 0:this._point=1,this._line?this._context.lineTo(t,e):this._context.moveTo(t,e);break;case 1:this._point=2;break;case 2:this._point=3;default:dc(this,t,e)}this._l01_a=this._l12_a,this._l12_a=this._l23_a,this._l01_2a=this._l12_2a,this._l12_2a=this._l23_2a,this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=e}};var gc=function t(e){function n(t){return e?new pc(t,e):new sc(t,0)}return n.alpha=function(e){return t(+e)},n}(.5);function mc(t,e){this._context=t,this._alpha=e}mc.prototype={areaStart:Kl,areaEnd:Kl,lineStart:function(){this._x0=this._x1=this._x2=this._x3=this._x4=this._x5=this._y0=this._y1=this._y2=this._y3=this._y4=this._y5=NaN,this._l01_a=this._l12_a=this._l23_a=this._l01_2a=this._l12_2a=this._l23_2a=this._point=0},lineEnd:function(){switch(this._point){case 1:this._context.moveTo(this._x3,this._y3),this._context.closePath();break;case 2:this._context.lineTo(this._x3,this._y3),this._context.closePath();break;case 3:this.point(this._x3,this._y3),this.point(this._x4,this._y4),this.point(this._x5,this._y5)}},point:function(t,e){if(t=+t,e=+e,this._point){var n=this._x2-t,r=this._y2-e;this._l23_a=Math.sqrt(this._l23_2a=Math.pow(n*n+r*r,this._alpha))}switch(this._point){case 0:this._point=1,this._x3=t,this._y3=e;break;case 1:this._point=2,this._context.moveTo(this._x4=t,this._y4=e);break;case 2:this._point=3,this._x5=t,this._y5=e;break;default:dc(this,t,e)}this._l01_a=this._l12_a,this._l12_a=this._l23_a,this._l01_2a=this._l12_2a,this._l12_2a=this._l23_2a,this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=e}};var yc=function t(e){function n(t){return e?new mc(t,e):new lc(t,0)}return n.alpha=function(e){return t(+e)},n}(.5);function vc(t,e){this._context=t,this._alpha=e}vc.prototype={areaStart:function(){this._line=0},areaEnd:function(){this._line=NaN},lineStart:function(){this._x0=this._x1=this._x2=this._y0=this._y1=this._y2=NaN,this._l01_a=this._l12_a=this._l23_a=this._l01_2a=this._l12_2a=this._l23_2a=this._point=0},lineEnd:function(){(this._line||0!==this._line&&3===this._point)&&this._context.closePath(),this._line=1-this._line},point:function(t,e){if(t=+t,e=+e,this._point){var n=this._x2-t,r=this._y2-e;this._l23_a=Math.sqrt(this._l23_2a=Math.pow(n*n+r*r,this._alpha))}switch(this._point){case 0:this._point=1;break;case 1:this._point=2;break;case 2:this._point=3,this._line?this._context.lineTo(this._x2,this._y2):this._context.moveTo(this._x2,this._y2);break;case 3:this._point=4;default:dc(this,t,e)}this._l01_a=this._l12_a,this._l12_a=this._l23_a,this._l01_2a=this._l12_2a,this._l12_2a=this._l23_2a,this._x0=this._x1,this._x1=this._x2,this._x2=t,this._y0=this._y1,this._y1=this._y2,this._y2=e}};var _c=function t(e){function n(t){return e?new vc(t,e):new fc(t,0)}return n.alpha=function(e){return t(+e)},n}(.5);function xc(t){this._context=t}function bc(t){return t<0?-1:1}function wc(t,e,n){var r=t._x1-t._x0,i=e-t._x1,o=(t._y1-t._y0)/(r||i<0&&-0),a=(n-t._y1)/(i||r<0&&-0),s=(o*i+a*r)/(r+i);return(bc(o)+bc(a))*Math.min(Math.abs(o),Math.abs(a),.5*Math.abs(s))||0}function kc(t,e){var n=t._x1-t._x0;return n?(3*(t._y1-t._y0)/n-e)/2:e}function Ac(t,e,n){var r=t._x0,i=t._y0,o=t._x1,a=t._y1,s=(o-r)/3;t._context.bezierCurveTo(r+s,i+s*e,o-s,a-s*n,o,a)}function Mc(t){this._context=t}function Ec(t){this._context=new Dc(t)}function Dc(t){this._context=t}function Cc(t){this._context=t}function Fc(t){var e,n,r=t.length-1,i=new Array(r),o=new Array(r),a=new Array(r);for(i[0]=0,o[0]=2,a[0]=t[0]+2*t[1],e=1;e=0;--e)i[e]=(a[e]-i[e+1])/o[e];for(o[r-1]=(t[r]+i[r-1])/2,e=0;e=0&&(this._t=1-this._t,this._line=1-this._line)},point:function(t,e){switch(t=+t,e=+e,this._point){case 0:this._point=1,this._line?this._context.lineTo(t,e):this._context.moveTo(t,e);break;case 1:this._point=2;default:if(this._t<=0)this._context.lineTo(this._x,e),this._context.lineTo(t,e);else{var n=this._x*(1-this._t)+t*this._t;this._context.lineTo(n,this._y),this._context.lineTo(n,e)}}this._x=t,this._y=e}};const Tc=()=>\"undefined\"!=typeof Image?Image:null;function Bc(t,e){switch(arguments.length){case 0:break;case 1:this.range(t);break;default:this.range(e).domain(t)}return this}function zc(t,e){switch(arguments.length){case 0:break;case 1:\"function\"==typeof t?this.interpolator(t):this.range(t);break;default:this.domain(t),\"function\"==typeof e?this.interpolator(e):this.range(e)}return this}const Nc=Symbol(\"implicit\");function Oc(){var t=new ue,e=[],n=[],r=Nc;function i(i){let o=t.get(i);if(void 0===o){if(r!==Nc)return r;t.set(i,o=e.push(i)-1)}return n[o%n.length]}return i.domain=function(n){if(!arguments.length)return e.slice();e=[],t=new ue;for(const r of n)t.has(r)||t.set(r,e.push(r)-1);return i},i.range=function(t){return arguments.length?(n=Array.from(t),i):n.slice()},i.unknown=function(t){return arguments.length?(r=t,i):r},i.copy=function(){return Oc(e,n).unknown(r)},Bc.apply(i,arguments),i}function Rc(t,e,n){t.prototype=e.prototype=n,n.constructor=t}function Uc(t,e){var n=Object.create(t.prototype);for(var r in e)n[r]=e[r];return n}function Lc(){}var qc=.7,Pc=1/qc,jc=\"\\\\s*([+-]?\\\\d+)\\\\s*\",Ic=\"\\\\s*([+-]?(?:\\\\d*\\\\.)?\\\\d+(?:[eE][+-]?\\\\d+)?)\\\\s*\",Wc=\"\\\\s*([+-]?(?:\\\\d*\\\\.)?\\\\d+(?:[eE][+-]?\\\\d+)?)%\\\\s*\",Hc=/^#([0-9a-f]{3,8})$/,Yc=new RegExp(`^rgb\\\\(${jc},${jc},${jc}\\\\)$`),Gc=new RegExp(`^rgb\\\\(${Wc},${Wc},${Wc}\\\\)$`),Vc=new RegExp(`^rgba\\\\(${jc},${jc},${jc},${Ic}\\\\)$`),Xc=new RegExp(`^rgba\\\\(${Wc},${Wc},${Wc},${Ic}\\\\)$`),Jc=new RegExp(`^hsl\\\\(${Ic},${Wc},${Wc}\\\\)$`),Zc=new RegExp(`^hsla\\\\(${Ic},${Wc},${Wc},${Ic}\\\\)$`),Qc={aliceblue:15792383,antiquewhite:16444375,aqua:65535,aquamarine:8388564,azure:15794175,beige:16119260,bisque:16770244,black:0,blanchedalmond:16772045,blue:255,blueviolet:9055202,brown:10824234,burlywood:14596231,cadetblue:6266528,chartreuse:8388352,chocolate:13789470,coral:16744272,cornflowerblue:6591981,cornsilk:16775388,crimson:14423100,cyan:65535,darkblue:139,darkcyan:35723,darkgoldenrod:12092939,darkgray:11119017,darkgreen:25600,darkgrey:11119017,darkkhaki:12433259,darkmagenta:9109643,darkolivegreen:5597999,darkorange:16747520,darkorchid:10040012,darkred:9109504,darksalmon:15308410,darkseagreen:9419919,darkslateblue:4734347,darkslategray:3100495,darkslategrey:3100495,darkturquoise:52945,darkviolet:9699539,deeppink:16716947,deepskyblue:49151,dimgray:6908265,dimgrey:6908265,dodgerblue:2003199,firebrick:11674146,floralwhite:16775920,forestgreen:2263842,fuchsia:16711935,gainsboro:14474460,ghostwhite:16316671,gold:16766720,goldenrod:14329120,gray:8421504,green:32768,greenyellow:11403055,grey:8421504,honeydew:15794160,hotpink:16738740,indianred:13458524,indigo:4915330,ivory:16777200,khaki:15787660,lavender:15132410,lavenderblush:16773365,lawngreen:8190976,lemonchiffon:16775885,lightblue:11393254,lightcoral:15761536,lightcyan:14745599,lightgoldenrodyellow:16448210,lightgray:13882323,lightgreen:9498256,lightgrey:13882323,lightpink:16758465,lightsalmon:16752762,lightseagreen:2142890,lightskyblue:8900346,lightslategray:7833753,lightslategrey:7833753,lightsteelblue:11584734,lightyellow:16777184,lime:65280,limegreen:3329330,linen:16445670,magenta:16711935,maroon:8388608,mediumaquamarine:6737322,mediumblue:205,mediumorchid:12211667,mediumpurple:9662683,mediumseagreen:3978097,mediumslateblue:8087790,mediumspringgreen:64154,mediumturquoise:4772300,mediumvioletred:13047173,midnightblue:1644912,mintcream:16121850,mistyrose:16770273,moccasin:16770229,navajowhite:16768685,navy:128,oldlace:16643558,olive:8421376,olivedrab:7048739,orange:16753920,orangered:16729344,orchid:14315734,palegoldenrod:15657130,palegreen:10025880,paleturquoise:11529966,palevioletred:14381203,papayawhip:16773077,peachpuff:16767673,peru:13468991,pink:16761035,plum:14524637,powderblue:11591910,purple:8388736,rebeccapurple:6697881,red:16711680,rosybrown:12357519,royalblue:4286945,saddlebrown:9127187,salmon:16416882,sandybrown:16032864,seagreen:3050327,seashell:16774638,sienna:10506797,silver:12632256,skyblue:8900331,slateblue:6970061,slategray:7372944,slategrey:7372944,snow:16775930,springgreen:65407,steelblue:4620980,tan:13808780,teal:32896,thistle:14204888,tomato:16737095,turquoise:4251856,violet:15631086,wheat:16113331,white:16777215,whitesmoke:16119285,yellow:16776960,yellowgreen:10145074};function Kc(){return this.rgb().formatHex()}function tf(){return this.rgb().formatRgb()}function ef(t){var e,n;return t=(t+\"\").trim().toLowerCase(),(e=Hc.exec(t))?(n=e[1].length,e=parseInt(e[1],16),6===n?nf(e):3===n?new sf(e>>8&15|e>>4&240,e>>4&15|240&e,(15&e)<<4|15&e,1):8===n?rf(e>>24&255,e>>16&255,e>>8&255,(255&e)/255):4===n?rf(e>>12&15|e>>8&240,e>>8&15|e>>4&240,e>>4&15|240&e,((15&e)<<4|15&e)/255):null):(e=Yc.exec(t))?new sf(e[1],e[2],e[3],1):(e=Gc.exec(t))?new sf(255*e[1]/100,255*e[2]/100,255*e[3]/100,1):(e=Vc.exec(t))?rf(e[1],e[2],e[3],e[4]):(e=Xc.exec(t))?rf(255*e[1]/100,255*e[2]/100,255*e[3]/100,e[4]):(e=Jc.exec(t))?df(e[1],e[2]/100,e[3]/100,1):(e=Zc.exec(t))?df(e[1],e[2]/100,e[3]/100,e[4]):Qc.hasOwnProperty(t)?nf(Qc[t]):\"transparent\"===t?new sf(NaN,NaN,NaN,0):null}function nf(t){return new sf(t>>16&255,t>>8&255,255&t,1)}function rf(t,e,n,r){return r<=0&&(t=e=n=NaN),new sf(t,e,n,r)}function of(t){return t instanceof Lc||(t=ef(t)),t?new sf((t=t.rgb()).r,t.g,t.b,t.opacity):new sf}function af(t,e,n,r){return 1===arguments.length?of(t):new sf(t,e,n,null==r?1:r)}function sf(t,e,n,r){this.r=+t,this.g=+e,this.b=+n,this.opacity=+r}function uf(){return`#${hf(this.r)}${hf(this.g)}${hf(this.b)}`}function lf(){const t=cf(this.opacity);return`${1===t?\"rgb(\":\"rgba(\"}${ff(this.r)}, ${ff(this.g)}, ${ff(this.b)}${1===t?\")\":`, ${t})`}`}function cf(t){return isNaN(t)?1:Math.max(0,Math.min(1,t))}function ff(t){return Math.max(0,Math.min(255,Math.round(t)||0))}function hf(t){return((t=ff(t))<16?\"0\":\"\")+t.toString(16)}function df(t,e,n,r){return r<=0?t=e=n=NaN:n<=0||n>=1?t=e=NaN:e<=0&&(t=NaN),new mf(t,e,n,r)}function pf(t){if(t instanceof mf)return new mf(t.h,t.s,t.l,t.opacity);if(t instanceof Lc||(t=ef(t)),!t)return new mf;if(t instanceof mf)return t;var e=(t=t.rgb()).r/255,n=t.g/255,r=t.b/255,i=Math.min(e,n,r),o=Math.max(e,n,r),a=NaN,s=o-i,u=(o+i)/2;return s?(a=e===o?(n-r)/s+6*(n0&&u<1?0:a,new mf(a,s,u,t.opacity)}function gf(t,e,n,r){return 1===arguments.length?pf(t):new mf(t,e,n,null==r?1:r)}function mf(t,e,n,r){this.h=+t,this.s=+e,this.l=+n,this.opacity=+r}function yf(t){return(t=(t||0)%360)<0?t+360:t}function vf(t){return Math.max(0,Math.min(1,t||0))}function _f(t,e,n){return 255*(t<60?e+(n-e)*t/60:t<180?n:t<240?e+(n-e)*(240-t)/60:e)}Rc(Lc,ef,{copy(t){return Object.assign(new this.constructor,this,t)},displayable(){return this.rgb().displayable()},hex:Kc,formatHex:Kc,formatHex8:function(){return this.rgb().formatHex8()},formatHsl:function(){return pf(this).formatHsl()},formatRgb:tf,toString:tf}),Rc(sf,af,Uc(Lc,{brighter(t){return t=null==t?Pc:Math.pow(Pc,t),new sf(this.r*t,this.g*t,this.b*t,this.opacity)},darker(t){return t=null==t?qc:Math.pow(qc,t),new sf(this.r*t,this.g*t,this.b*t,this.opacity)},rgb(){return this},clamp(){return new sf(ff(this.r),ff(this.g),ff(this.b),cf(this.opacity))},displayable(){return-.5<=this.r&&this.r<255.5&&-.5<=this.g&&this.g<255.5&&-.5<=this.b&&this.b<255.5&&0<=this.opacity&&this.opacity<=1},hex:uf,formatHex:uf,formatHex8:function(){return`#${hf(this.r)}${hf(this.g)}${hf(this.b)}${hf(255*(isNaN(this.opacity)?1:this.opacity))}`},formatRgb:lf,toString:lf})),Rc(mf,gf,Uc(Lc,{brighter(t){return t=null==t?Pc:Math.pow(Pc,t),new mf(this.h,this.s,this.l*t,this.opacity)},darker(t){return t=null==t?qc:Math.pow(qc,t),new mf(this.h,this.s,this.l*t,this.opacity)},rgb(){var t=this.h%360+360*(this.h<0),e=isNaN(t)||isNaN(this.s)?0:this.s,n=this.l,r=n+(n<.5?n:1-n)*e,i=2*n-r;return new sf(_f(t>=240?t-240:t+120,i,r),_f(t,i,r),_f(t<120?t+240:t-120,i,r),this.opacity)},clamp(){return new mf(yf(this.h),vf(this.s),vf(this.l),cf(this.opacity))},displayable(){return(0<=this.s&&this.s<=1||isNaN(this.s))&&0<=this.l&&this.l<=1&&0<=this.opacity&&this.opacity<=1},formatHsl(){const t=cf(this.opacity);return`${1===t?\"hsl(\":\"hsla(\"}${yf(this.h)}, ${100*vf(this.s)}%, ${100*vf(this.l)}%${1===t?\")\":`, ${t})`}`}}));const xf=Math.PI/180,bf=180/Math.PI,wf=.96422,kf=1,Af=.82521,Mf=4/29,Ef=6/29,Df=3*Ef*Ef,Cf=Ef*Ef*Ef;function Ff(t){if(t instanceof $f)return new $f(t.l,t.a,t.b,t.opacity);if(t instanceof Rf)return Uf(t);t instanceof sf||(t=of(t));var e,n,r=Nf(t.r),i=Nf(t.g),o=Nf(t.b),a=Tf((.2225045*r+.7168786*i+.0606169*o)/kf);return r===i&&i===o?e=n=a:(e=Tf((.4360747*r+.3850649*i+.1430804*o)/wf),n=Tf((.0139322*r+.0971045*i+.7141733*o)/Af)),new $f(116*a-16,500*(e-a),200*(a-n),t.opacity)}function Sf(t,e,n,r){return 1===arguments.length?Ff(t):new $f(t,e,n,null==r?1:r)}function $f(t,e,n,r){this.l=+t,this.a=+e,this.b=+n,this.opacity=+r}function Tf(t){return t>Cf?Math.pow(t,1/3):t/Df+Mf}function Bf(t){return t>Ef?t*t*t:Df*(t-Mf)}function zf(t){return 255*(t<=.0031308?12.92*t:1.055*Math.pow(t,1/2.4)-.055)}function Nf(t){return(t/=255)<=.04045?t/12.92:Math.pow((t+.055)/1.055,2.4)}function Of(t,e,n,r){return 1===arguments.length?function(t){if(t instanceof Rf)return new Rf(t.h,t.c,t.l,t.opacity);if(t instanceof $f||(t=Ff(t)),0===t.a&&0===t.b)return new Rf(NaN,0=1?(n=1,e-1):Math.floor(n*e),i=t[r],o=t[r+1],a=r>0?t[r-1]:2*i-o,s=r()=>t;function Kf(t,e){return function(n){return t+n*e}}function th(t,e){var n=e-t;return n?Kf(t,n>180||n<-180?n-360*Math.round(n/360):n):Qf(isNaN(t)?e:t)}function eh(t){return 1==(t=+t)?nh:function(e,n){return n-e?function(t,e,n){return t=Math.pow(t,n),e=Math.pow(e,n)-t,n=1/n,function(r){return Math.pow(t+r*e,n)}}(e,n,t):Qf(isNaN(e)?n:e)}}function nh(t,e){var n=e-t;return n?Kf(t,n):Qf(isNaN(t)?e:t)}var rh=function t(e){var n=eh(e);function r(t,e){var r=n((t=af(t)).r,(e=af(e)).r),i=n(t.g,e.g),o=n(t.b,e.b),a=nh(t.opacity,e.opacity);return function(e){return t.r=r(e),t.g=i(e),t.b=o(e),t.opacity=a(e),t+\"\"}}return r.gamma=t,r}(1);function ih(t){return function(e){var n,r,i=e.length,o=new Array(i),a=new Array(i),s=new Array(i);for(n=0;no&&(i=e.slice(o,i),s[a]?s[a]+=i:s[++a]=i),(n=n[0])===(r=r[0])?s[a]?s[a]+=r:s[++a]=r:(s[++a]=null,u.push({i:a,x:fh(n,r)})),o=ph.lastIndex;return o180?e+=360:e-t>180&&(t+=360),o.push({i:n.push(i(n)+\"rotate(\",null,r)-2,x:fh(t,e)})):e&&n.push(i(n)+\"rotate(\"+e+r)}(o.rotate,a.rotate,s,u),function(t,e,n,o){t!==e?o.push({i:n.push(i(n)+\"skewX(\",null,r)-2,x:fh(t,e)}):e&&n.push(i(n)+\"skewX(\"+e+r)}(o.skewX,a.skewX,s,u),function(t,e,n,r,o,a){if(t!==n||e!==r){var s=o.push(i(o)+\"scale(\",null,\",\",null,\")\");a.push({i:s-4,x:fh(t,n)},{i:s-2,x:fh(e,r)})}else 1===n&&1===r||o.push(i(o)+\"scale(\"+n+\",\"+r+\")\")}(o.scaleX,o.scaleY,a.scaleX,a.scaleY,s,u),o=a=null,function(t){for(var e,n=-1,r=u.length;++ne&&(n=t,t=e,e=n),function(n){return Math.max(t,Math.min(e,n))}}(a[0],a[t-1])),r=t>2?Ih:jh,i=o=null,f}function f(e){return null==e||isNaN(e=+e)?n:(i||(i=r(a.map(t),s,u)))(t(l(e)))}return f.invert=function(n){return l(e((o||(o=r(s,a.map(t),fh)))(n)))},f.domain=function(t){return arguments.length?(a=Array.from(t,Uh),c()):a.slice()},f.range=function(t){return arguments.length?(s=Array.from(t),c()):s.slice()},f.rangeRound=function(t){return s=Array.from(t),u=yh,c()},f.clamp=function(t){return arguments.length?(l=!!t||qh,c()):l!==qh},f.interpolate=function(t){return arguments.length?(u=t,c()):u},f.unknown=function(t){return arguments.length?(n=t,f):n},function(n,r){return t=n,e=r,c()}}function Yh(){return Hh()(qh,qh)}function Gh(t,e,n,r){var i,o=be(t,e,n);switch((r=Re(null==r?\",f\":r)).type){case\"s\":var a=Math.max(Math.abs(t),Math.abs(e));return null!=r.precision||isNaN(i=Xe(o,a))||(r.precision=i),We(r,a);case\"\":case\"e\":case\"g\":case\"p\":case\"r\":null!=r.precision||isNaN(i=Je(o,Math.max(Math.abs(t),Math.abs(e))))||(r.precision=i-(\"e\"===r.type));break;case\"f\":case\"%\":null!=r.precision||isNaN(i=Ve(o))||(r.precision=i-2*(\"%\"===r.type))}return Ie(r)}function Vh(t){var e=t.domain;return t.ticks=function(t){var n=e();return _e(n[0],n[n.length-1],null==t?10:t)},t.tickFormat=function(t,n){var r=e();return Gh(r[0],r[r.length-1],null==t?10:t,n)},t.nice=function(n){null==n&&(n=10);var r,i,o=e(),a=0,s=o.length-1,u=o[a],l=o[s],c=10;for(l0;){if((i=xe(u,l,n))===r)return o[a]=u,o[s]=l,e(o);if(i>0)u=Math.floor(u/i)*i,l=Math.ceil(l/i)*i;else{if(!(i<0))break;u=Math.ceil(u*i)/i,l=Math.floor(l*i)/i}r=i}return t},t}function Xh(t,e){var n,r=0,i=(t=t.slice()).length-1,o=t[r],a=t[i];return a-t(-e,n)}function nd(t){const e=t(Jh,Zh),n=e.domain;let r,i,o=10;function a(){return r=function(t){return t===Math.E?Math.log:10===t&&Math.log10||2===t&&Math.log2||(t=Math.log(t),e=>Math.log(e)/t)}(o),i=function(t){return 10===t?td:t===Math.E?Math.exp:e=>Math.pow(t,e)}(o),n()[0]<0?(r=ed(r),i=ed(i),t(Qh,Kh)):t(Jh,Zh),e}return e.base=function(t){return arguments.length?(o=+t,a()):o},e.domain=function(t){return arguments.length?(n(t),a()):n()},e.ticks=t=>{const e=n();let a=e[0],s=e[e.length-1];const u=s0){for(;f<=h;++f)for(l=1;ls)break;p.push(c)}}else for(;f<=h;++f)for(l=o-1;l>=1;--l)if(c=f>0?l/i(-f):l*i(f),!(cs)break;p.push(c)}2*p.length{if(null==t&&(t=10),null==n&&(n=10===o?\"s\":\",\"),\"function\"!=typeof n&&(o%1||null!=(n=Re(n)).precision||(n.trim=!0),n=Ie(n)),t===1/0)return n;const a=Math.max(1,o*t/e.ticks().length);return t=>{let e=t/i(Math.round(r(t)));return e*on(Xh(n(),{floor:t=>i(Math.floor(r(t))),ceil:t=>i(Math.ceil(r(t)))})),e}function rd(t){return function(e){return Math.sign(e)*Math.log1p(Math.abs(e/t))}}function id(t){return function(e){return Math.sign(e)*Math.expm1(Math.abs(e))*t}}function od(t){var e=1,n=t(rd(e),id(e));return n.constant=function(n){return arguments.length?t(rd(e=+n),id(e)):e},Vh(n)}function ad(t){return function(e){return e<0?-Math.pow(-e,t):Math.pow(e,t)}}function sd(t){return t<0?-Math.sqrt(-t):Math.sqrt(t)}function ud(t){return t<0?-t*t:t*t}function ld(t){var e=t(qh,qh),n=1;return e.exponent=function(e){return arguments.length?1===(n=+e)?t(qh,qh):.5===n?t(sd,ud):t(ad(n),ad(1/n)):n},Vh(e)}function cd(){var t=ld(Hh());return t.copy=function(){return Wh(t,cd()).exponent(t.exponent())},Bc.apply(t,arguments),t}function fd(t){return new Date(t)}function hd(t){return t instanceof Date?+t:+new Date(+t)}function dd(t,e,n,r,i,o,a,s,u,l){var c=Yh(),f=c.invert,h=c.domain,d=l(\".%L\"),p=l(\":%S\"),g=l(\"%I:%M\"),m=l(\"%I %p\"),y=l(\"%a %d\"),v=l(\"%b %d\"),_=l(\"%B\"),x=l(\"%Y\");function b(t){return(u(t)0?r:1:0}const Td=\"linear\",Bd=\"log\",zd=\"pow\",Nd=\"sqrt\",Od=\"symlog\",Rd=\"time\",Ud=\"utc\",Ld=\"sequential\",qd=\"diverging\",Pd=\"quantile\",jd=\"quantize\",Id=\"threshold\",Wd=\"ordinal\",Hd=\"point\",Yd=\"band\",Gd=\"bin-ordinal\",Vd=\"continuous\",Xd=\"discrete\",Jd=\"discretizing\",Zd=\"interpolating\",Qd=\"temporal\";function Kd(){const t=Oc().unknown(void 0),e=t.domain,n=t.range;let r,i,o=[0,1],a=!1,s=0,u=0,l=.5;function c(){const t=e().length,c=o[1]d+r*t));return n(c?p.reverse():p)}return delete t.unknown,t.domain=function(t){return arguments.length?(e(t),c()):e()},t.range=function(t){return arguments.length?(o=[+t[0],+t[1]],c()):o.slice()},t.rangeRound=function(t){return o=[+t[0],+t[1]],a=!0,c()},t.bandwidth=function(){return i},t.step=function(){return r},t.round=function(t){return arguments.length?(a=!!t,c()):a},t.padding=function(t){return arguments.length?(u=Math.max(0,Math.min(1,t)),s=u,c()):s},t.paddingInner=function(t){return arguments.length?(s=Math.max(0,Math.min(1,t)),c()):s},t.paddingOuter=function(t){return arguments.length?(u=Math.max(0,Math.min(1,t)),c()):u},t.align=function(t){return arguments.length?(l=Math.max(0,Math.min(1,t)),c()):l},t.invertRange=function(t){if(null==t[0]||null==t[1])return;const r=o[1]o[1-r])?void 0:(u=Math.max(0,oe(a,f)-1),l=f===h?u:oe(a,h)-1,f-a[u]>i+1e-10&&++u,r&&(c=u,u=s-l,l=s-c),u>l?void 0:e().slice(u,l+1))},t.invert=function(e){const n=t.invertRange([e,e]);return n?n[0]:n},t.copy=function(){return Kd().domain(e()).range(o).round(a).paddingInner(s).paddingOuter(u).align(l)},c()}function tp(t){const e=t.copy;return t.padding=t.paddingOuter,delete t.paddingInner,t.copy=function(){return tp(e())},t}var ep=Array.prototype.map;const np=Array.prototype.slice;const rp=new Map,ip=Symbol(\"vega_scale\");function op(t){return t[ip]=!0,t}function ap(t,e,n){return arguments.length>1?(rp.set(t,function(t,e,n){const r=function(){const n=e();return n.invertRange||(n.invertRange=n.invert?function(t){return function(e){let n,r=e[0],i=e[1];return i=s&&n[o]<=u&&(l<0&&(l=o),r=o);if(!(l<0))return s=t.invertExtent(n[l]),u=t.invertExtent(n[r]),[void 0===s[0]?s[1]:s[0],void 0===u[1]?u[0]:u[1]]}}(n):void 0),n.type=t,op(n)};return r.metadata=Bt(V(n)),r}(t,e,n)),this):sp(t)?rp.get(t):void 0}function sp(t){return rp.has(t)}function up(t,e){const n=rp.get(t);return n&&n.metadata[e]}function lp(t){return up(t,Vd)}function cp(t){return up(t,Xd)}function fp(t){return up(t,Jd)}function hp(t){return up(t,Bd)}function dp(t){return up(t,Zd)}function pp(t){return up(t,Pd)}ap(\"identity\",(function t(e){var n;function r(t){return null==t||isNaN(t=+t)?n:t}return r.invert=r,r.domain=r.range=function(t){return arguments.length?(e=Array.from(t,Uh),r):e.slice()},r.unknown=function(t){return arguments.length?(n=t,r):n},r.copy=function(){return t(e).unknown(n)},e=arguments.length?Array.from(e,Uh):[0,1],Vh(r)})),ap(Td,(function t(){var e=Yh();return e.copy=function(){return Wh(e,t())},Bc.apply(e,arguments),Vh(e)}),Vd),ap(Bd,(function t(){const e=nd(Hh()).domain([1,10]);return e.copy=()=>Wh(e,t()).base(e.base()),Bc.apply(e,arguments),e}),[Vd,Bd]),ap(zd,cd,Vd),ap(Nd,(function(){return cd.apply(null,arguments).exponent(.5)}),Vd),ap(Od,(function t(){var e=od(Hh());return e.copy=function(){return Wh(e,t()).constant(e.constant())},Bc.apply(e,arguments)}),Vd),ap(Rd,(function(){return Bc.apply(dd(qn,Pn,Nn,Bn,vn,pn,hn,cn,ln,ni).domain([new Date(2e3,0,1),new Date(2e3,0,2)]),arguments)}),[Vd,Qd]),ap(Ud,(function(){return Bc.apply(dd(Un,Ln,On,zn,En,gn,dn,fn,ln,ii).domain([Date.UTC(2e3,0,1),Date.UTC(2e3,0,2)]),arguments)}),[Vd,Qd]),ap(Ld,md,[Vd,Zd]),ap(`${Ld}-${Td}`,md,[Vd,Zd]),ap(`${Ld}-${Bd}`,(function t(){var e=nd(pd()).domain([1,10]);return e.copy=function(){return gd(e,t()).base(e.base())},zc.apply(e,arguments)}),[Vd,Zd,Bd]),ap(`${Ld}-${zd}`,yd,[Vd,Zd]),ap(`${Ld}-${Nd}`,(function(){return yd.apply(null,arguments).exponent(.5)}),[Vd,Zd]),ap(`${Ld}-${Od}`,(function t(){var e=od(pd());return e.copy=function(){return gd(e,t()).constant(e.constant())},zc.apply(e,arguments)}),[Vd,Zd]),ap(`${qd}-${Td}`,(function t(){var e=Vh(vd()(qh));return e.copy=function(){return gd(e,t())},zc.apply(e,arguments)}),[Vd,Zd]),ap(`${qd}-${Bd}`,(function t(){var e=nd(vd()).domain([.1,1,10]);return e.copy=function(){return gd(e,t()).base(e.base())},zc.apply(e,arguments)}),[Vd,Zd,Bd]),ap(`${qd}-${zd}`,_d,[Vd,Zd]),ap(`${qd}-${Nd}`,(function(){return _d.apply(null,arguments).exponent(.5)}),[Vd,Zd]),ap(`${qd}-${Od}`,(function t(){var e=od(vd());return e.copy=function(){return gd(e,t()).constant(e.constant())},zc.apply(e,arguments)}),[Vd,Zd]),ap(Pd,(function t(){var e,n=[],r=[],i=[];function o(){var t=0,e=Math.max(1,r.length);for(i=new Array(e-1);++t0?i[e-1]:n[0],e=i?[o[i-1],r]:[o[e-1],o[e]]},s.unknown=function(t){return arguments.length?(e=t,s):s},s.thresholds=function(){return o.slice()},s.copy=function(){return t().domain([n,r]).range(a).unknown(e)},Bc.apply(Vh(s),arguments)}),Jd),ap(Id,(function t(){var e,n=[.5],r=[0,1],i=1;function o(t){return null!=t&&t<=t?r[oe(n,t,0,i)]:e}return o.domain=function(t){return arguments.length?(n=Array.from(t),i=Math.min(n.length,r.length-1),o):n.slice()},o.range=function(t){return arguments.length?(r=Array.from(t),i=Math.min(n.length,r.length-1),o):r.slice()},o.invertExtent=function(t){var e=r.indexOf(t);return[n[e-1],n[e]]},o.unknown=function(t){return arguments.length?(e=t,o):e},o.copy=function(){return t().domain(n).range(r).unknown(e)},Bc.apply(o,arguments)}),Jd),ap(Gd,(function t(){let e=[],n=[];function r(t){return null==t||t!=t?void 0:n[(oe(e,t)-1)%n.length]}return r.domain=function(t){return arguments.length?(e=function(t){return ep.call(t,S)}(t),r):e.slice()},r.range=function(t){return arguments.length?(n=np.call(t),r):n.slice()},r.tickFormat=function(t,n){return Gh(e[0],F(e),null==t?10:t,n)},r.copy=function(){return t().domain(r.domain()).range(r.range())},r}),[Xd,Jd]),ap(Wd,Oc,Xd),ap(Yd,Kd,Xd),ap(Hd,(function(){return tp(Kd().paddingInner(1))}),Xd);const gp=[\"clamp\",\"base\",\"constant\",\"exponent\"];function mp(t,e){const n=e[0],r=F(e)-n;return function(e){return t(n+e*r)}}function yp(t,e,n){return Oh(xp(e||\"rgb\",n),t)}function vp(t,e){const n=new Array(e),r=e+1;for(let i=0;it[e]?a[e](t[e]()):0)),a):rt(.5)}function xp(t,e){const n=Rh[function(t){return\"interpolate\"+t.toLowerCase().split(\"-\").map((t=>t[0].toUpperCase()+t.slice(1))).join(\"\")}(t)];return null!=e&&n&&n.gamma?n.gamma(e):n}function bp(t){if(k(t))return t;const e=t.length/6|0,n=new Array(e);for(let r=0;r1?(kp[t]=e,this):kp[t]}wp({accent:wd,category10:bd,category20:\"1f77b4aec7e8ff7f0effbb782ca02c98df8ad62728ff98969467bdc5b0d58c564bc49c94e377c2f7b6d27f7f7fc7c7c7bcbd22dbdb8d17becf9edae5\",category20b:\"393b795254a36b6ecf9c9ede6379398ca252b5cf6bcedb9c8c6d31bd9e39e7ba52e7cb94843c39ad494ad6616be7969c7b4173a55194ce6dbdde9ed6\",category20c:\"3182bd6baed69ecae1c6dbefe6550dfd8d3cfdae6bfdd0a231a35474c476a1d99bc7e9c0756bb19e9ac8bcbddcdadaeb636363969696bdbdbdd9d9d9\",dark2:kd,observable10:Ad,paired:Md,pastel1:Ed,pastel2:Dd,set1:Cd,set2:Fd,set3:Sd,tableau10:\"4c78a8f58518e4575672b7b254a24beeca3bb279a2ff9da69d755dbab0ac\",tableau20:\"4c78a89ecae9f58518ffbf7954a24b88d27ab79a20f2cf5b43989483bcb6e45756ff9d9879706ebab0acd67195fcbfd2b279a2d6a5c99e765fd8b5a5\"},bp),wp({blues:\"cfe1f2bed8eca8cee58fc1de74b2d75ba3cf4592c63181bd206fb2125ca40a4a90\",greens:\"d3eecdc0e6baabdda594d3917bc77d60ba6c46ab5e329a512089430e7735036429\",greys:\"e2e2e2d4d4d4c4c4c4b1b1b19d9d9d8888887575756262624d4d4d3535351e1e1e\",oranges:\"fdd8b3fdc998fdb87bfda55efc9244f87f2cf06b18e4580bd14904b93d029f3303\",purples:\"e2e1efd4d4e8c4c5e0b4b3d6a3a0cc928ec3827cb97566ae684ea25c3696501f8c\",reds:\"fdc9b4fcb49afc9e80fc8767fa7051f6573fec3f2fdc2a25c81b1db21218970b13\",blueGreen:\"d5efedc1e8e0a7ddd18bd2be70c6a958ba9144ad77319c5d2089460e7736036429\",bluePurple:\"ccddecbad0e4a8c2dd9ab0d4919cc98d85be8b6db28a55a6873c99822287730f71\",greenBlue:\"d3eecec5e8c3b1e1bb9bd8bb82cec269c2ca51b2cd3c9fc7288abd1675b10b60a1\",orangeRed:\"fddcaffdcf9bfdc18afdad77fb9562f67d53ee6545e24932d32d1ebf130da70403\",purpleBlue:\"dbdaebc8cee4b1c3de97b7d87bacd15b9fc93a90c01e7fb70b70ab056199045281\",purpleBlueGreen:\"dbd8eac8cee4b0c3de93b7d872acd1549fc83892bb1c88a3097f8702736b016353\",purpleRed:\"dcc9e2d3b3d7ce9eccd186c0da6bb2e14da0e23189d91e6fc61159ab07498f023a\",redPurple:\"fccfccfcbec0faa9b8f98faff571a5ec539ddb3695c41b8aa908808d0179700174\",yellowGreen:\"e4f4acd1eca0b9e2949ed68880c97c62bb6e47aa5e3297502083440e723b036034\",yellowOrangeBrown:\"feeaa1fedd84fecc63feb746fca031f68921eb7215db5e0bc54c05ab3d038f3204\",yellowOrangeRed:\"fee087fed16ffebd59fea849fd903efc7335f9522bee3423de1b20ca0b22af0225\",blueOrange:\"134b852f78b35da2cb9dcae1d2e5eff2f0ebfce0bafbbf74e8932fc5690d994a07\",brownBlueGreen:\"704108a0651ac79548e3c78af3e6c6eef1eac9e9e48ed1c74da79e187a72025147\",purpleGreen:\"5b1667834792a67fb6c9aed3e6d6e8eff0efd9efd5aedda971bb75368e490e5e29\",purpleOrange:\"4114696647968f83b7b9b4d6dadbebf3eeeafce0bafbbf74e8932fc5690d994a07\",redBlue:\"8c0d25bf363adf745ef4ae91fbdbc9f2efeed2e5ef9dcae15da2cb2f78b3134b85\",redGrey:\"8c0d25bf363adf745ef4ae91fcdccbfaf4f1e2e2e2c0c0c0969696646464343434\",yellowGreenBlue:\"eff9bddbf1b4bde5b594d5b969c5be45b4c22c9ec02182b82163aa23479c1c3185\",redYellowBlue:\"a50026d4322cf16e43fcac64fedd90faf8c1dcf1ecabd6e875abd04a74b4313695\",redYellowGreen:\"a50026d4322cf16e43fcac63fedd8df9f7aed7ee8ea4d86e64bc6122964f006837\",pinkYellowGreen:\"8e0152c0267edd72adf0b3d6faddedf5f3efe1f2cab6de8780bb474f9125276419\",spectral:\"9e0142d13c4bf0704afcac63fedd8dfbf8b0e0f3a1a9dda269bda94288b55e4fa2\",viridis:\"440154470e61481a6c482575472f7d443a834144873d4e8a39568c35608d31688e2d708e2a788e27818e23888e21918d1f988b1fa08822a8842ab07f35b77943bf7154c56866cc5d7ad1518fd744a5db36bcdf27d2e21be9e51afde725\",magma:\"0000040404130b0924150e3720114b2c11603b0f704a107957157e651a80721f817f24828c29819a2e80a8327db6377ac43c75d1426fde4968e95462f1605df76f5cfa7f5efc8f65fe9f6dfeaf78febf84fece91fddea0fcedaffcfdbf\",inferno:\"0000040403130c0826170c3b240c4f330a5f420a68500d6c5d126e6b176e781c6d86216b932667a12b62ae305cbb3755c73e4cd24644dd513ae65c30ed6925f3771af8850ffb9506fca50afcb519fac62df6d645f2e661f3f484fcffa4\",plasma:\"0d088723069033059742039d5002a25d01a66a00a87801a88405a7900da49c179ea72198b12a90ba3488c33d80cb4779d35171da5a69e16462e76e5bed7953f2834cf68f44fa9a3dfca636fdb32ffec029fcce25f9dc24f5ea27f0f921\",cividis:\"00205100235800265d002961012b65042e670831690d346b11366c16396d1c3c6e213f6e26426e2c456e31476e374a6e3c4d6e42506e47536d4c566d51586e555b6e5a5e6e5e616e62646f66676f6a6a706e6d717270717573727976737c79747f7c75827f758682768985778c8877908b78938e789691789a94789e9778a19b78a59e77a9a177aea575b2a874b6ab73bbaf71c0b26fc5b66dc9b96acebd68d3c065d8c462ddc85fe2cb5ce7cf58ebd355f0d652f3da4ff7de4cfae249fce647\",rainbow:\"6e40aa883eb1a43db3bf3cafd83fa4ee4395fe4b83ff576eff6659ff7847ff8c38f3a130e2b72fcfcc36bee044aff05b8ff4576ff65b52f6673af27828ea8d1ddfa319d0b81cbecb23abd82f96e03d82e14c6edb5a5dd0664dbf6e40aa\",sinebow:\"ff4040fc582af47218e78d0bd5a703bfbf00a7d5038de70b72f41858fc2a40ff402afc5818f4720be78d03d5a700bfbf03a7d50b8de71872f42a58fc4040ff582afc7218f48d0be7a703d5bf00bfd503a7e70b8df41872fc2a58ff4040\",turbo:\"23171b32204a3e2a71453493493eae4b49c54a53d7485ee44569ee4074f53c7ff8378af93295f72e9ff42ba9ef28b3e926bce125c5d925cdcf27d5c629dcbc2de3b232e9a738ee9d3ff39347f68950f9805afc7765fd6e70fe667cfd5e88fc5795fb51a1f84badf545b9f140c5ec3cd0e637dae034e4d931ecd12ef4c92bfac029ffb626ffad24ffa223ff9821ff8d1fff821dff771cfd6c1af76118f05616e84b14df4111d5380fcb2f0dc0260ab61f07ac1805a313029b0f00950c00910b00\",browns:\"eedbbdecca96e9b97ae4a865dc9856d18954c7784cc0673fb85536ad44339f3632\",tealBlues:\"bce4d89dd3d181c3cb65b3c245a2b9368fae347da0306a932c5985\",teals:\"bbdfdfa2d4d58ac9c975bcbb61b0af4da5a43799982b8b8c1e7f7f127273006667\",warmGreys:\"dcd4d0cec5c1c0b8b4b3aaa7a59c9998908c8b827f7e7673726866665c5a59504e\",goldGreen:\"f4d166d5ca60b6c35c98bb597cb25760a6564b9c533f8f4f33834a257740146c36\",goldOrange:\"f4d166f8be5cf8aa4cf5983bf3852aef701be2621fd65322c54923b142239e3a26\",goldRed:\"f4d166f6be59f9aa51fc964ef6834bee734ae56249db5247cf4244c43141b71d3e\",lightGreyRed:\"efe9e6e1dad7d5cbc8c8bdb9bbaea9cd967ddc7b43e15f19df4011dc000b\",lightGreyTeal:\"e4eaead6dcddc8ced2b7c2c7a6b4bc64b0bf22a6c32295c11f85be1876bc\",lightMulti:\"e0f1f2c4e9d0b0de9fd0e181f6e072f6c053f3993ef77440ef4a3c\",lightOrange:\"f2e7daf7d5baf9c499fab184fa9c73f68967ef7860e8645bde515bd43d5b\",lightTealBlue:\"e3e9e0c0dccf9aceca7abfc859afc0389fb9328dad2f7ca0276b95255988\",darkBlue:\"3232322d46681a5c930074af008cbf05a7ce25c0dd38daed50f3faffffff\",darkGold:\"3c3c3c584b37725e348c7631ae8b2bcfa424ecc31ef9de30fff184ffffff\",darkGreen:\"3a3a3a215748006f4d048942489e4276b340a6c63dd2d836ffeb2cffffaa\",darkMulti:\"3737371f5287197d8c29a86995ce3fffe800ffffff\",darkRed:\"3434347036339e3c38cc4037e75d1eec8620eeab29f0ce32ffeb2c\"},(t=>yp(bp(t))));const Mp=\"symbol\",Ep=\"discrete\",Dp=t=>k(t)?t.map((t=>String(t))):String(t),Cp=(t,e)=>t[1]-e[1],Fp=(t,e)=>e[1]-t[1];function Sp(t,e,n){let r;return vt(e)&&(t.bins&&(e=Math.max(e,t.bins.length)),null!=n&&(e=Math.min(e,Math.floor(Dt(t.domain())/n||1)+1))),A(e)&&(r=e.step,e=e.interval),xt(e)&&(e=t.type===Rd?Cr(e):t.type==Ud?Fr(e):s(\"Only time and utc scales accept interval strings.\"),r&&(e=e.every(r))),e}function $p(t,e,n){let r=t.range(),i=r[0],o=F(r),a=Cp;if(i>o&&(r=o,o=i,i=r,a=Fp),i=Math.floor(i),o=Math.ceil(o),e=e.map((e=>[e,t(e)])).filter((t=>i<=t[1]&&t[1]<=o)).sort(a).map((t=>t[0])),n>0&&e.length>1){const t=[e[0],F(e)];for(;e.length>n&&e.length>=3;)e=e.filter(((t,e)=>!(e%2)));e.length<3&&(e=t)}return e}function Tp(t,e){return t.bins?$p(t,t.bins,e):t.ticks?t.ticks(e):t.domain()}function Bp(t,e,n,r,i,o){const a=e.type;let s=Dp;if(a===Rd||i===Rd)s=t.timeFormat(r);else if(a===Ud||i===Ud)s=t.utcFormat(r);else if(hp(a)){const i=t.formatFloat(r);if(o||e.bins)s=i;else{const t=zp(e,n,!1);s=e=>t(e)?i(e):\"\"}}else if(e.tickFormat){const i=e.domain();s=t.formatSpan(i[0],i[i.length-1],n,r)}else r&&(s=t.format(r));return s}function zp(t,e,n){const r=Tp(t,e),i=t.base(),o=Math.log(i),a=Math.max(1,i*e/r.length),s=t=>{let e=t/Math.pow(i,Math.round(Math.log(t)/o));return e*iNp[t.type]||t.bins;function Lp(t,e,n,r,i,o,a){const s=Op[e.type]&&o!==Rd&&o!==Ud?function(t,e,n){const r=e[Op[e.type]](),i=r.length;let o,a=i>1?r[1]-r[0]:r[0];for(o=1;o(e,n,r)=>{const i=Pp(r[n+1],Pp(r.max,1/0)),o=Wp(e,t),a=Wp(i,t);return o&&a?o+\" – \"+a:a?\"< \"+a:\"â‰Ĩ \"+o},Pp=(t,e)=>null!=t?t:e,jp=t=>(e,n)=>n?t(e):null,Ip=t=>e=>t(e),Wp=(t,e)=>Number.isFinite(t)?e(t):null;function Hp(t,e,n,r){const i=r||e.type;return xt(n)&&function(t){return up(t,Qd)}(i)&&(n=n.replace(/%a/g,\"%A\").replace(/%b/g,\"%B\")),n||i!==Rd?n||i!==Ud?Lp(t,e,5,null,n,r,!0):t.utcFormat(\"%A, %d %B %Y, %X UTC\"):t.timeFormat(\"%A, %d %B %Y, %X\")}function Yp(t,e,n){n=n||{};const r=Math.max(3,n.maxlen||7),i=Hp(t,e,n.format,n.formatType);if(fp(e.type)){const t=Rp(e).slice(1).map(i),n=t.length;return`${n} boundar${1===n?\"y\":\"ies\"}: ${t.join(\", \")}`}if(cp(e.type)){const t=e.domain(),n=t.length;return`${n} value${1===n?\"\":\"s\"}: ${n>r?t.slice(0,r-2).map(i).join(\", \")+\", ending with \"+t.slice(-1).map(i):t.map(i).join(\", \")}`}{const t=e.domain();return`values from ${i(t[0])} to ${i(F(t))}`}}let Gp=0;const Vp=\"p_\";function Xp(t){return t&&t.gradient}function Jp(t,e,n){const r=t.gradient;let i=t.id,o=\"radial\"===r?Vp:\"\";return i||(i=t.id=\"gradient_\"+Gp++,\"radial\"===r?(t.x1=Zp(t.x1,.5),t.y1=Zp(t.y1,.5),t.r1=Zp(t.r1,0),t.x2=Zp(t.x2,.5),t.y2=Zp(t.y2,.5),t.r2=Zp(t.r2,.5),o=Vp):(t.x1=Zp(t.x1,0),t.y1=Zp(t.y1,0),t.x2=Zp(t.x2,1),t.y2=Zp(t.y2,0))),e[i]=t,\"url(\"+(n||\"\")+\"#\"+o+i+\")\"}function Zp(t,e){return null!=t?t:e}function Qp(t,e){var n,r=[];return n={gradient:\"linear\",x1:t?t[0]:0,y1:t?t[1]:0,x2:e?e[0]:1,y2:e?e[1]:0,stops:r,stop:function(t,e){return r.push({offset:t,color:e}),n}}}const Kp={basis:{curve:function(t){return new ec(t)}},\"basis-closed\":{curve:function(t){return new nc(t)}},\"basis-open\":{curve:function(t){return new rc(t)}},bundle:{curve:oc,tension:\"beta\",value:.85},cardinal:{curve:uc,tension:\"tension\",value:0},\"cardinal-open\":{curve:hc,tension:\"tension\",value:0},\"cardinal-closed\":{curve:cc,tension:\"tension\",value:0},\"catmull-rom\":{curve:gc,tension:\"alpha\",value:.5},\"catmull-rom-closed\":{curve:yc,tension:\"alpha\",value:.5},\"catmull-rom-open\":{curve:_c,tension:\"alpha\",value:.5},linear:{curve:Gl},\"linear-closed\":{curve:function(t){return new xc(t)}},monotone:{horizontal:function(t){return new Ec(t)},vertical:function(t){return new Mc(t)}},natural:{curve:function(t){return new Cc(t)}},step:{curve:function(t){return new Sc(t,.5)}},\"step-after\":{curve:function(t){return new Sc(t,1)}},\"step-before\":{curve:function(t){return new Sc(t,0)}}};function tg(t,e,n){var r=lt(Kp,t)&&Kp[t],i=null;return r&&(i=r.curve||r[e||\"vertical\"],r.tension&&null!=n&&(i=i[r.tension](n))),i}const eg={m:2,l:2,h:1,v:1,z:0,c:6,s:4,q:4,t:2,a:7},ng=/[mlhvzcsqta]([^mlhvzcsqta]+|$)/gi,rg=/^[+-]?(([0-9]*\\.[0-9]+)|([0-9]+\\.)|([0-9]+))([eE][+-]?[0-9]+)?/,ig=/^((\\s+,?\\s*)|(,\\s*))/,og=/^[01]/;function ag(t){const e=[];return(t.match(ng)||[]).forEach((t=>{let n=t[0];const r=n.toLowerCase(),i=eg[r],o=function(t,e,n){const r=[];for(let i=0;e&&i1&&(g=Math.sqrt(g),n*=g,r*=g);const m=h/n,y=f/n,v=-f/r,_=h/r,x=m*s+y*u,b=v*s+_*u,w=m*t+y*e,k=v*t+_*e;let A=1/((w-x)*(w-x)+(k-b)*(k-b))-.25;A<0&&(A=0);let M=Math.sqrt(A);o==i&&(M=-M);const E=.5*(x+w)-M*(k-b),D=.5*(b+k)+M*(w-x),C=Math.atan2(b-D,x-E);let F=Math.atan2(k-D,w-E)-C;F<0&&1===o?F+=lg:F>0&&0===o&&(F-=lg);const S=Math.ceil(Math.abs(F/(ug+.001))),$=[];for(let t=0;t+t}function Fg(t,e,n){return Math.max(e,Math.min(t,n))}function Sg(){var t=Ag,e=Mg,n=Eg,r=Dg,i=Cg(0),o=i,a=i,s=i,u=null;function l(l,c,f){var h,d=null!=c?c:+t.call(this,l),p=null!=f?f:+e.call(this,l),g=+n.call(this,l),m=+r.call(this,l),y=Math.min(g,m)/2,v=Fg(+i.call(this,l),0,y),_=Fg(+o.call(this,l),0,y),x=Fg(+a.call(this,l),0,y),b=Fg(+s.call(this,l),0,y);if(u||(u=h=Rl()),v<=0&&_<=0&&x<=0&&b<=0)u.rect(d,p,g,m);else{var w=d+g,k=p+m;u.moveTo(d+v,p),u.lineTo(w-_,p),u.bezierCurveTo(w-kg*_,p,w,p+kg*_,w,p+_),u.lineTo(w,k-b),u.bezierCurveTo(w,k-kg*b,w-kg*b,k,w-b,k),u.lineTo(d+x,k),u.bezierCurveTo(d+kg*x,k,d,k-kg*x,d,k-x),u.lineTo(d,p+v),u.bezierCurveTo(d,p+kg*v,d+kg*v,p,d+v,p),u.closePath()}if(h)return u=null,h+\"\"||null}return l.x=function(e){return arguments.length?(t=Cg(e),l):t},l.y=function(t){return arguments.length?(e=Cg(t),l):e},l.width=function(t){return arguments.length?(n=Cg(t),l):n},l.height=function(t){return arguments.length?(r=Cg(t),l):r},l.cornerRadius=function(t,e,n,r){return arguments.length?(i=Cg(t),o=null!=e?Cg(e):i,s=null!=n?Cg(n):i,a=null!=r?Cg(r):o,l):i},l.context=function(t){return arguments.length?(u=null==t?null:t,l):u},l}function $g(){var t,e,n,r,i,o,a,s,u=null;function l(t,e,n){const r=n/2;if(i){var l=a-e,c=t-o;if(l||c){var f=Math.hypot(l,c),h=(l/=f)*s,d=(c/=f)*s,p=Math.atan2(c,l);u.moveTo(o-h,a-d),u.lineTo(t-l*r,e-c*r),u.arc(t,e,r,p-Math.PI,p),u.lineTo(o+h,a+d),u.arc(o,a,s,p,p+Math.PI)}else u.arc(t,e,r,0,lg);u.closePath()}else i=1;o=t,a=e,s=r}function c(o){var a,s,c,f=o.length,h=!1;for(null==u&&(u=c=Rl()),a=0;a<=f;++a)!(at.x||0,zg=t=>t.y||0,Ng=t=>!(!1===t.defined),Og=function(){var t=Ll,e=ql,n=vl(0),r=null,i=Pl,o=jl,a=Il,s=null,u=Ul(l);function l(){var l,c,f=+t.apply(this,arguments),h=+e.apply(this,arguments),d=i.apply(this,arguments)-Cl,p=o.apply(this,arguments)-Cl,g=_l(p-d),m=p>d;if(s||(s=l=u()),hEl)if(g>Fl-El)s.moveTo(h*bl(d),h*Al(d)),s.arc(0,0,h,d,p,!m),f>El&&(s.moveTo(f*bl(p),f*Al(p)),s.arc(0,0,f,p,d,m));else{var y,v,_=d,x=p,b=d,w=p,k=g,A=g,M=a.apply(this,arguments)/2,E=M>El&&(r?+r.apply(this,arguments):Ml(f*f+h*h)),D=kl(_l(h-f)/2,+n.apply(this,arguments)),C=D,F=D;if(E>El){var S=Sl(E/f*Al(M)),$=Sl(E/h*Al(M));(k-=2*S)>El?(b+=S*=m?1:-1,w-=S):(k=0,b=w=(d+p)/2),(A-=2*$)>El?(_+=$*=m?1:-1,x-=$):(A=0,_=x=(d+p)/2)}var T=h*bl(_),B=h*Al(_),z=f*bl(w),N=f*Al(w);if(D>El){var O,R=h*bl(x),U=h*Al(x),L=f*bl(b),q=f*Al(b);if(g1?0:t<-1?Dl:Math.acos(t)}((P*I+j*W)/(Ml(P*P+j*j)*Ml(I*I+W*W)))/2),Y=Ml(O[0]*O[0]+O[1]*O[1]);C=kl(D,(f-Y)/(H-1)),F=kl(D,(h-Y)/(H+1))}else C=F=0}A>El?F>El?(y=Wl(L,q,T,B,h,F,m),v=Wl(R,U,z,N,h,F,m),s.moveTo(y.cx+y.x01,y.cy+y.y01),FEl&&k>El?C>El?(y=Wl(z,N,R,U,f,-C,m),v=Wl(T,B,L,q,f,-C,m),s.lineTo(y.cx+y.x01,y.cy+y.y01),Ct.startAngle||0)).endAngle((t=>t.endAngle||0)).padAngle((t=>t.padAngle||0)).innerRadius((t=>t.innerRadius||0)).outerRadius((t=>t.outerRadius||0)).cornerRadius((t=>t.cornerRadius||0)),Rg=Zl().x(Bg).y1(zg).y0((t=>(t.y||0)+(t.height||0))).defined(Ng),Ug=Zl().y(zg).x1(Bg).x0((t=>(t.x||0)+(t.width||0))).defined(Ng),Lg=Jl().x(Bg).y(zg).defined(Ng),qg=Sg().x(Bg).y(zg).width((t=>t.width||0)).height((t=>t.height||0)).cornerRadius((t=>Tg(t.cornerRadiusTopLeft,t.cornerRadius)||0),(t=>Tg(t.cornerRadiusTopRight,t.cornerRadius)||0),(t=>Tg(t.cornerRadiusBottomRight,t.cornerRadius)||0),(t=>Tg(t.cornerRadiusBottomLeft,t.cornerRadius)||0)),Pg=function(t,e){let n=null,r=Ul(i);function i(){let i;if(n||(n=i=r()),t.apply(this,arguments).draw(n,+e.apply(this,arguments)),i)return n=null,i+\"\"||null}return t=\"function\"==typeof t?t:vl(t||Ql),e=\"function\"==typeof e?e:vl(void 0===e?64:+e),i.type=function(e){return arguments.length?(t=\"function\"==typeof e?e:vl(e),i):t},i.size=function(t){return arguments.length?(e=\"function\"==typeof t?t:vl(+t),i):e},i.context=function(t){return arguments.length?(n=null==t?null:t,i):n},i}().type((t=>bg(t.shape||\"circle\"))).size((t=>Tg(t.size,64))),jg=$g().x(Bg).y(zg).defined(Ng).size((t=>t.size||1));function Ig(t){return t.cornerRadius||t.cornerRadiusTopLeft||t.cornerRadiusTopRight||t.cornerRadiusBottomRight||t.cornerRadiusBottomLeft}function Wg(t,e,n,r){return qg.context(t)(e,n,r)}var Hg=1;function Yg(){Hg=1}function Gg(t,e,n){var r=e.clip,i=t._defs,o=e.clip_id||(e.clip_id=\"clip\"+Hg++),a=i.clipping[o]||(i.clipping[o]={id:o});return J(r)?a.path=r(null):Ig(n)?a.path=Wg(null,n,0,0):(a.width=n.width||0,a.height=n.height||0),\"url(#\"+o+\")\"}function Vg(t){this.clear(),t&&this.union(t)}function Xg(t){this.mark=t,this.bounds=this.bounds||new Vg}function Jg(t){Xg.call(this,t),this.items=this.items||[]}Vg.prototype={clone(){return new Vg(this)},clear(){return this.x1=+Number.MAX_VALUE,this.y1=+Number.MAX_VALUE,this.x2=-Number.MAX_VALUE,this.y2=-Number.MAX_VALUE,this},empty(){return this.x1===+Number.MAX_VALUE&&this.y1===+Number.MAX_VALUE&&this.x2===-Number.MAX_VALUE&&this.y2===-Number.MAX_VALUE},equals(t){return this.x1===t.x1&&this.y1===t.y1&&this.x2===t.x2&&this.y2===t.y2},set(t,e,n,r){return nthis.x2&&(this.x2=t),e>this.y2&&(this.y2=e),this},expand(t){return this.x1-=t,this.y1-=t,this.x2+=t,this.y2+=t,this},round(){return this.x1=Math.floor(this.x1),this.y1=Math.floor(this.y1),this.x2=Math.ceil(this.x2),this.y2=Math.ceil(this.y2),this},scale(t){return this.x1*=t,this.y1*=t,this.x2*=t,this.y2*=t,this},translate(t,e){return this.x1+=t,this.x2+=t,this.y1+=e,this.y2+=e,this},rotate(t,e,n){const r=this.rotatedPoints(t,e,n);return this.clear().add(r[0],r[1]).add(r[2],r[3]).add(r[4],r[5]).add(r[6],r[7])},rotatedPoints(t,e,n){var{x1:r,y1:i,x2:o,y2:a}=this,s=Math.cos(t),u=Math.sin(t),l=e-e*s+n*u,c=n-e*u-n*s;return[s*r-u*i+l,u*r+s*i+c,s*r-u*a+l,u*r+s*a+c,s*o-u*i+l,u*o+s*i+c,s*o-u*a+l,u*o+s*a+c]},union(t){return t.x1this.x2&&(this.x2=t.x2),t.y2>this.y2&&(this.y2=t.y2),this},intersect(t){return t.x1>this.x1&&(this.x1=t.x1),t.y1>this.y1&&(this.y1=t.y1),t.x2=t.x2&&this.y1<=t.y1&&this.y2>=t.y2},alignsWith(t){return t&&(this.x1==t.x1||this.x2==t.x2||this.y1==t.y1||this.y2==t.y2)},intersects(t){return t&&!(this.x2t.x2||this.y2t.y2)},contains(t,e){return!(tthis.x2||ethis.y2)},width(){return this.x2-this.x1},height(){return this.y2-this.y1}},dt(Jg,Xg);class Zg{constructor(t){this._pending=0,this._loader=t||fa()}pending(){return this._pending}sanitizeURL(t){const e=this;return Qg(e),e._loader.sanitize(t,{context:\"href\"}).then((t=>(Kg(e),t))).catch((()=>(Kg(e),null)))}loadImage(t){const e=this,n=Tc();return Qg(e),e._loader.sanitize(t,{context:\"image\"}).then((t=>{const r=t.href;if(!r||!n)throw{url:r};const i=new n,o=lt(t,\"crossOrigin\")?t.crossOrigin:\"anonymous\";return null!=o&&(i.crossOrigin=o),i.onload=()=>Kg(e),i.onerror=()=>Kg(e),i.src=r,i})).catch((t=>(Kg(e),{complete:!1,width:0,height:0,src:t&&t.url||\"\"})))}ready(){const t=this;return new Promise((e=>{!function n(r){t.pending()?setTimeout((()=>{n(!0)}),10):e(r)}(!1)}))}}function Qg(t){t._pending+=1}function Kg(t){t._pending-=1}function tm(t,e,n){if(e.stroke&&0!==e.opacity&&0!==e.strokeOpacity){const r=null!=e.strokeWidth?+e.strokeWidth:1;t.expand(r+(n?function(t,e){return t.strokeJoin&&\"miter\"!==t.strokeJoin?0:e}(e,r):0))}return t}const em=lg-1e-8;let nm,rm,im,om,am,sm,um,lm;const cm=(t,e)=>nm.add(t,e),fm=(t,e)=>cm(rm=t,im=e),hm=t=>cm(t,nm.y1),dm=t=>cm(nm.x1,t),pm=(t,e)=>am*t+um*e,gm=(t,e)=>sm*t+lm*e,mm=(t,e)=>cm(pm(t,e),gm(t,e)),ym=(t,e)=>fm(pm(t,e),gm(t,e));function vm(t,e){return nm=t,e?(om=e*sg,am=lm=Math.cos(om),sm=Math.sin(om),um=-sm):(am=lm=1,om=sm=um=0),_m}const _m={beginPath(){},closePath(){},moveTo:ym,lineTo:ym,rect(t,e,n,r){om?(mm(t+n,e),mm(t+n,e+r),mm(t,e+r),ym(t,e)):(cm(t+n,e+r),fm(t,e))},quadraticCurveTo(t,e,n,r){const i=pm(t,e),o=gm(t,e),a=pm(n,r),s=gm(n,r);xm(rm,i,a,hm),xm(im,o,s,dm),fm(a,s)},bezierCurveTo(t,e,n,r,i,o){const a=pm(t,e),s=gm(t,e),u=pm(n,r),l=gm(n,r),c=pm(i,o),f=gm(i,o);bm(rm,a,u,c,hm),bm(im,s,l,f,dm),fm(c,f)},arc(t,e,n,r,i,o){if(r+=om,i+=om,rm=n*Math.cos(i)+t,im=n*Math.sin(i)+e,Math.abs(i-r)>em)cm(t-n,e-n),cm(t+n,e+n);else{const a=r=>cm(n*Math.cos(r)+t,n*Math.sin(r)+e);let s,u;if(a(r),a(i),i!==r)if((r%=lg)<0&&(r+=lg),(i%=lg)<0&&(i+=lg),ii;++u,s-=ug)a(s);else for(s=r-r%ug+ug,u=0;u<4&&s1e-14?(u=a*a+s*o,u>=0&&(u=Math.sqrt(u),l=(-a+u)/o,c=(-a-u)/o)):l=.5*s/a,0m)return!1;d>g&&(g=d)}else if(f>0){if(d0&&(t.globalAlpha=n,t.fillStyle=Bm(t,e,e.fill),!0)}var Nm=[];function Om(t,e,n){var r=null!=(r=e.strokeWidth)?r:1;return!(r<=0)&&((n*=null==e.strokeOpacity?1:e.strokeOpacity)>0&&(t.globalAlpha=n,t.strokeStyle=Bm(t,e,e.stroke),t.lineWidth=r,t.lineCap=e.strokeCap||\"butt\",t.lineJoin=e.strokeJoin||\"miter\",t.miterLimit=e.strokeMiterLimit||10,t.setLineDash&&(t.setLineDash(e.strokeDash||Nm),t.lineDashOffset=e.strokeDashOffset||0),!0))}function Rm(t,e){return t.zindex-e.zindex||t.index-e.index}function Um(t){if(!t.zdirty)return t.zitems;var e,n,r,i=t.items,o=[];for(n=0,r=i.length;n=0;)if(n=e(i[r]))return n;if(i===o)for(r=(i=t.items).length;--r>=0;)if(!i[r].zindex&&(n=e(i[r])))return n;return null}function Pm(t){return function(e,n,r){Lm(n,(n=>{r&&!r.intersects(n.bounds)||Im(t,e,n,n)}))}}function jm(t){return function(e,n,r){!n.items.length||r&&!r.intersects(n.bounds)||Im(t,e,n.items[0],n.items)}}function Im(t,e,n,r){var i=null==n.opacity?1:n.opacity;0!==i&&(t(e,r)||(Sm(e,n),n.fill&&zm(e,n,i)&&e.fill(),n.stroke&&Om(e,n,i)&&e.stroke()))}function Wm(t){return t=t||p,function(e,n,r,i,o,a){return r*=e.pixelRatio,i*=e.pixelRatio,qm(n,(n=>{const s=n.bounds;if((!s||s.contains(o,a))&&s)return t(e,n,r,i,o,a)?n:void 0}))}}function Hm(t,e){return function(n,r,i,o){var a,s,u=Array.isArray(r)?r[0]:r,l=null==e?u.fill:e,c=u.stroke&&n.isPointInStroke;return c&&(a=u.strokeWidth,s=u.strokeCap,n.lineWidth=null!=a?a:1,n.lineCap=null!=s?s:\"butt\"),!t(n,r)&&(l&&n.isPointInPath(i,o)||c&&n.isPointInStroke(i,o))}}function Ym(t){return Wm(Hm(t))}function Gm(t,e){return\"translate(\"+t+\",\"+e+\")\"}function Vm(t){return\"rotate(\"+t+\")\"}function Xm(t){return Gm(t.x||0,t.y||0)}function Jm(t,e,n){function r(t,n){var r=n.x||0,i=n.y||0,o=n.angle||0;t.translate(r,i),o&&t.rotate(o*=sg),t.beginPath(),e(t,n),o&&t.rotate(-o),t.translate(-r,-i)}return{type:t,tag:\"path\",nested:!1,attr:function(t,n){t(\"transform\",function(t){return Gm(t.x||0,t.y||0)+(t.angle?\" \"+Vm(t.angle):\"\")}(n)),t(\"d\",e(null,n))},bound:function(t,n){return e(vm(t,n.angle),n),tm(t,n).translate(n.x||0,n.y||0)},draw:Pm(r),pick:Ym(r),isect:n||Mm(r)}}var Zm=Jm(\"arc\",(function(t,e){return Og.context(t)(e)}));function Qm(t,e,n){function r(t,n){t.beginPath(),e(t,n)}const i=Hm(r);return{type:t,tag:\"path\",nested:!0,attr:function(t,n){var r=n.mark.items;r.length&&t(\"d\",e(null,r))},bound:function(t,n){var r=n.items;return 0===r.length?t:(e(vm(t),r),tm(t,r[0]))},draw:jm(r),pick:function(t,e,n,r,o,a){var s=e.items,u=e.bounds;return!s||!s.length||u&&!u.contains(o,a)?null:(n*=t.pixelRatio,r*=t.pixelRatio,i(t,s,n,r)?s[0]:null)},isect:Em,tip:n}}var Km=Qm(\"area\",(function(t,e){const n=e[0],r=n.interpolate||\"linear\";return(\"horizontal\"===n.orient?Ug:Rg).curve(tg(r,n.orient,n.tension)).context(t)(e)}),(function(t,e){for(var n,r,i=\"horizontal\"===t[0].orient?e[1]:e[0],o=\"horizontal\"===t[0].orient?\"y\":\"x\",a=t.length,s=1/0;--a>=0;)!1!==t[a].defined&&(r=Math.abs(t[a][o]-i)).5&&e<1.5?.5-Math.abs(e-1):0}function ny(t,e){const n=ey(e);t(\"d\",Wg(null,e,n,n))}function ry(t,e,n,r){const i=ey(e);t.beginPath(),Wg(t,e,(n||0)+i,(r||0)+i)}const iy=Hm(ry),oy=Hm(ry,!1),ay=Hm(ry,!0);var sy={type:\"group\",tag:\"g\",nested:!1,attr:function(t,e){t(\"transform\",Xm(e))},bound:function(t,e){if(!e.clip&&e.items){const n=e.items,r=n.length;for(let e=0;e{const i=e.x||0,o=e.y||0,a=e.strokeForeground,s=null==e.opacity?1:e.opacity;(e.stroke||e.fill)&&s&&(ry(t,e,i,o),Sm(t,e),e.fill&&zm(t,e,s)&&t.fill(),e.stroke&&!a&&Om(t,e,s)&&t.stroke()),t.save(),t.translate(i,o),e.clip&&ty(t,e),n&&n.translate(-i,-o),Lm(e,(e=>{(\"group\"===e.marktype||null==r||r.includes(e.marktype))&&this.draw(t,e,n,r)})),n&&n.translate(i,o),t.restore(),a&&e.stroke&&s&&(ry(t,e,i,o),Sm(t,e),Om(t,e,s)&&t.stroke())}))},pick:function(t,e,n,r,i,o){if(e.bounds&&!e.bounds.contains(i,o)||!e.items)return null;const a=n*t.pixelRatio,s=r*t.pixelRatio;return qm(e,(u=>{let l,c,f;const h=u.bounds;if(h&&!h.contains(i,o))return;c=u.x||0,f=u.y||0;const d=c+(u.width||0),p=f+(u.height||0),g=u.clip;if(g&&(id||op))return;if(t.save(),t.translate(c,f),c=i-c,f=o-f,g&&Ig(u)&&!ay(t,u,a,s))return t.restore(),null;const m=u.strokeForeground,y=!1!==e.interactive;return y&&m&&u.stroke&&oy(t,u,a,s)?(t.restore(),u):(l=qm(u,(t=>function(t,e,n){return(!1!==t.interactive||\"group\"===t.marktype)&&t.bounds&&t.bounds.contains(e,n)}(t,c,f)?this.pick(t,n,r,c,f):null)),!l&&y&&(u.fill||!m&&u.stroke)&&iy(t,u,a,s)&&(l=u),t.restore(),l||null)}))},isect:Dm,content:function(t,e,n){t(\"clip-path\",e.clip?Gg(n,e,e):null)},background:function(t,e){t(\"class\",\"background\"),t(\"aria-hidden\",!0),ny(t,e)},foreground:function(t,e){t(\"class\",\"foreground\"),t(\"aria-hidden\",!0),e.strokeForeground?ny(t,e):t(\"d\",\"\")}},uy={xmlns:\"http://www.w3.org/2000/svg\",\"xmlns:xlink\":\"http://www.w3.org/1999/xlink\",version:\"1.1\"};function ly(t,e){var n=t.image;return(!n||t.url&&t.url!==n.url)&&(n={complete:!1,width:0,height:0},e.loadImage(t.url).then((e=>{t.image=e,t.image.url=t.url}))),n}function cy(t,e){return null!=t.width?t.width:e&&e.width?!1!==t.aspect&&t.height?t.height*e.width/e.height:e.width:0}function fy(t,e){return null!=t.height?t.height:e&&e.height?!1!==t.aspect&&t.width?t.width*e.height/e.width:e.height:0}function hy(t,e){return\"center\"===t?e/2:\"right\"===t?e:0}function dy(t,e){return\"middle\"===t?e/2:\"bottom\"===t?e:0}var py={type:\"image\",tag:\"image\",nested:!1,attr:function(t,e,n){const r=ly(e,n),i=cy(e,r),o=fy(e,r),a=(e.x||0)-hy(e.align,i),s=(e.y||0)-dy(e.baseline,o);t(\"href\",!r.src&&r.toDataURL?r.toDataURL():r.src||\"\",uy[\"xmlns:xlink\"],\"xlink:href\"),t(\"transform\",Gm(a,s)),t(\"width\",i),t(\"height\",o),t(\"preserveAspectRatio\",!1===e.aspect?\"none\":\"xMidYMid\")},bound:function(t,e){const n=e.image,r=cy(e,n),i=fy(e,n),o=(e.x||0)-hy(e.align,r),a=(e.y||0)-dy(e.baseline,i);return t.set(o,a,o+r,a+i)},draw:function(t,e,n){Lm(e,(e=>{if(n&&!n.intersects(e.bounds))return;const r=ly(e,this);let i=cy(e,r),o=fy(e,r);if(0===i||0===o)return;let a,s,u,l,c=(e.x||0)-hy(e.align,i),f=(e.y||0)-dy(e.baseline,o);!1!==e.aspect&&(s=r.width/r.height,u=e.width/e.height,s==s&&u==u&&s!==u&&(u=0;)if(!1!==t[o].defined&&(n=t[o].x-e[0])*n+(r=t[o].y-e[1])*r{if(!n||n.intersects(e.bounds)){var r=null==e.opacity?1:e.opacity;r&&xy(t,e,r)&&(Sm(t,e),t.stroke())}}))},pick:Wm((function(t,e,n,r){return!!t.isPointInStroke&&(xy(t,e,1)&&t.isPointInStroke(n,r))})),isect:Cm},wy=Jm(\"shape\",(function(t,e){return(e.mark.shape||e.shape).context(t)(e)})),ky=Jm(\"symbol\",(function(t,e){return Pg.context(t)(e)}),Em);const Ay=kt();var My={height:$y,measureWidth:Fy,estimateWidth:Dy,width:Dy,canvas:Ey};function Ey(t){My.width=t&&km?Fy:Dy}function Dy(t,e){return Cy(Ny(t,e),$y(t))}function Cy(t,e){return~~(.8*t.length*e)}function Fy(t,e){return $y(t)<=0||!(e=Ny(t,e))?0:Sy(e,Ry(t))}function Sy(t,e){const n=`(${e}) ${t}`;let r=Ay.get(n);return void 0===r&&(km.font=e,r=km.measureText(t).width,Ay.set(n,r)),r}function $y(t){return null!=t.fontSize?+t.fontSize||0:11}function Ty(t){return null!=t.lineHeight?t.lineHeight:$y(t)+2}function By(t){return e=t.lineBreak&&t.text&&!k(t.text)?t.text.split(t.lineBreak):t.text,k(e)?e.length>1?e:e[0]:e;var e}function zy(t){const e=By(t);return(k(e)?e.length-1:0)*Ty(t)}function Ny(t,e){const n=null==e?\"\":(e+\"\").trim();return t.limit>0&&n.length?function(t,e){var n=+t.limit,r=function(t){if(My.width===Fy){const e=Ry(t);return t=>Sy(t,e)}if(My.width===Dy){const e=$y(t);return t=>Cy(t,e)}return e=>My.width(t,e)}(t);if(r(e)>>1,r(e.slice(i))>n?s=i+1:u=i;return o+e.slice(s)}for(;s>>1),r(e.slice(0,i))Math.max(t,My.width(e,n))),0)):r=My.width(e,f),\"center\"===o?l-=r/2:\"right\"===o&&(l-=r),t.set(l+=s,c+=u,l+r,c+i),e.angle&&!n)t.rotate(e.angle*sg,s,u);else if(2===n)return t.rotatedPoints(e.angle*sg,s,u);return t}var Iy={type:\"text\",tag:\"text\",nested:!1,attr:function(t,e){var n,r=e.dx||0,i=(e.dy||0)+Uy(e),o=Py(e),a=o.x1,s=o.y1,u=e.angle||0;t(\"text-anchor\",Ly[e.align]||\"start\"),u?(n=Gm(a,s)+\" \"+Vm(u),(r||i)&&(n+=\" \"+Gm(r,i))):n=Gm(a+r,s+i),t(\"transform\",n)},bound:jy,draw:function(t,e,n){Lm(e,(e=>{var r,i,o,a,s,u,l,c=null==e.opacity?1:e.opacity;if(!(n&&!n.intersects(e.bounds)||0===c||e.fontSize<=0||null==e.text||0===e.text.length)){if(t.font=Ry(e),t.textAlign=e.align||\"left\",i=(r=Py(e)).x1,o=r.y1,e.angle&&(t.save(),t.translate(i,o),t.rotate(e.angle*sg),i=o=0),i+=e.dx||0,o+=(e.dy||0)+Uy(e),u=By(e),Sm(t,e),k(u))for(s=Ty(e),a=0;a=0;)if(!1!==t[i].defined&&(n=t[i].x-e[0])*n+(r=t[i].y-e[1])*r<(n=t[i].size||1)*n)return t[i];return null})),Hy={arc:Zm,area:Km,group:sy,image:py,line:gy,path:yy,rect:_y,rule:by,shape:wy,symbol:ky,text:Iy,trail:Wy};function Yy(t,e,n){var r=Hy[t.mark.marktype],i=e||r.bound;return r.nested&&(t=t.mark),i(t.bounds||(t.bounds=new Vg),t,n)}var Gy={mark:null};function Vy(t,e,n){var r,i,o,a,s=Hy[t.marktype],u=s.bound,l=t.items,c=l&&l.length;if(s.nested)return c?o=l[0]:(Gy.mark=t,o=Gy),a=Yy(o,u,n),e=e&&e.union(a)||a;if(e=e||t.bounds&&t.bounds.clear()||new Vg,c)for(r=0,i=l.length;re;)t.removeChild(n[--r]);return t}function ov(t){return\"mark-\"+t.marktype+(t.role?\" role-\"+t.role:\"\")+(t.name?\" \"+t.name:\"\")}function av(t,e){const n=e.getBoundingClientRect();return[t.clientX-n.left-(e.clientLeft||0),t.clientY-n.top-(e.clientTop||0)]}class sv{constructor(t,e){this._active=null,this._handlers={},this._loader=t||fa(),this._tooltip=e||uv}initialize(t,e,n){return this._el=t,this._obj=n||null,this.origin(e)}element(){return this._el}canvas(){return this._el&&this._el.firstChild}origin(t){return arguments.length?(this._origin=t||[0,0],this):this._origin.slice()}scene(t){return arguments.length?(this._scene=t,this):this._scene}on(){}off(){}_handlerIndex(t,e,n){for(let r=t?t.length:0;--r>=0;)if(t[r].type===e&&(!n||t[r].handler===n))return r;return-1}handlers(t){const e=this._handlers,n=[];if(t)n.push(...e[this.eventName(t)]);else for(const t in e)n.push(...e[t]);return n}eventName(t){const e=t.indexOf(\".\");return e<0?t:t.slice(0,e)}handleHref(t,e,n){this._loader.sanitize(n,{context:\"href\"}).then((e=>{const n=new MouseEvent(t.type,t),r=ev(null,\"a\");for(const t in e)r.setAttribute(t,e[t]);r.dispatchEvent(n)})).catch((()=>{}))}handleTooltip(t,e,n){if(e&&null!=e.tooltip){e=function(t,e,n,r){var i,o,a=t&&t.mark;if(a&&(i=Hy[a.marktype]).tip){for((o=av(e,n))[0]-=r[0],o[1]-=r[1];t=t.mark.group;)o[0]-=t.x||0,o[1]-=t.y||0;t=i.tip(a.items,o)}return t}(e,t,this.canvas(),this._origin);const r=n&&e&&e.tooltip||null;this._tooltip.call(this._obj,this,t,e,r)}}getItemBoundingClientRect(t){const e=this.canvas();if(!e)return;const n=e.getBoundingClientRect(),r=this._origin,i=t.bounds,o=i.width(),a=i.height();let s=i.x1+r[0]+n.left,u=i.y1+r[1]+n.top;for(;t.mark&&(t=t.mark.group);)s+=t.x||0,u+=t.y||0;return{x:s,y:u,width:o,height:a,left:s,top:u,right:s+o,bottom:u+a}}}function uv(t,e,n,r){t.element().setAttribute(\"title\",r||\"\")}class lv{constructor(t){this._el=null,this._bgcolor=null,this._loader=new Zg(t)}initialize(t,e,n,r,i){return this._el=t,this.resize(e,n,r,i)}element(){return this._el}canvas(){return this._el&&this._el.firstChild}background(t){return 0===arguments.length?this._bgcolor:(this._bgcolor=t,this)}resize(t,e,n,r){return this._width=t,this._height=e,this._origin=n||[0,0],this._scale=r||1,this}dirty(){}render(t,e){const n=this;return n._call=function(){n._render(t,e)},n._call(),n._call=null,n}_render(){}renderAsync(t,e){const n=this.render(t,e);return this._ready?this._ready.then((()=>n)):Promise.resolve(n)}_load(t,e){var n=this,r=n._loader[t](e);if(!n._ready){const t=n._call;n._ready=n._loader.ready().then((e=>{e&&t(),n._ready=null}))}return r}sanitizeURL(t){return this._load(\"sanitizeURL\",t)}loadImage(t){return this._load(\"loadImage\",t)}}const cv=\"dragenter\",fv=\"dragleave\",hv=\"dragover\",dv=\"pointerdown\",pv=\"pointermove\",gv=\"pointerout\",mv=\"pointerover\",yv=\"mousedown\",vv=\"mousemove\",_v=\"mouseout\",xv=\"mouseover\",bv=\"click\",wv=\"mousewheel\",kv=\"touchstart\",Av=\"touchmove\",Mv=\"touchend\",Ev=[\"keydown\",\"keypress\",\"keyup\",cv,fv,hv,dv,\"pointerup\",pv,gv,mv,yv,\"mouseup\",vv,_v,xv,bv,\"dblclick\",\"wheel\",wv,kv,Av,Mv],Dv=pv,Cv=_v,Fv=bv;class Sv extends sv{constructor(t,e){super(t,e),this._down=null,this._touch=null,this._first=!0,this._events={},this.events=Ev,this.pointermove=zv([pv,vv],[mv,xv],[gv,_v]),this.dragover=zv([hv],[cv],[fv]),this.pointerout=Nv([gv,_v]),this.dragleave=Nv([fv])}initialize(t,e,n){return this._canvas=t&&nv(t,\"canvas\"),[bv,yv,dv,pv,gv,fv].forEach((t=>Tv(this,t))),super.initialize(t,e,n)}canvas(){return this._canvas}context(){return this._canvas.getContext(\"2d\")}DOMMouseScroll(t){this.fire(wv,t)}pointerdown(t){this._down=this._active,this.fire(dv,t)}mousedown(t){this._down=this._active,this.fire(yv,t)}click(t){this._down===this._active&&(this.fire(bv,t),this._down=null)}touchstart(t){this._touch=this.pickEvent(t.changedTouches[0]),this._first&&(this._active=this._touch,this._first=!1),this.fire(kv,t,!0)}touchmove(t){this.fire(Av,t,!0)}touchend(t){this.fire(Mv,t,!0),this._touch=null}fire(t,e,n){const r=n?this._touch:this._active,i=this._handlers[t];if(e.vegaType=t,t===Fv&&r&&r.href?this.handleHref(e,r,r.href):t!==Dv&&t!==Cv||this.handleTooltip(e,r,t!==Cv),i)for(let t=0,n=i.length;t=0&&r.splice(i,1),this}pickEvent(t){const e=av(t,this._canvas),n=this._origin;return this.pick(this._scene,e[0],e[1],e[0]-n[0],e[1]-n[1])}pick(t,e,n,r,i){const o=this.context();return Hy[t.marktype].pick.call(this,o,t,e,n,r,i)}}const $v=t=>t===kv||t===Av||t===Mv?[kv,Av,Mv]:[t];function Tv(t,e){$v(e).forEach((e=>function(t,e){const n=t.canvas();n&&!t._events[e]&&(t._events[e]=1,n.addEventListener(e,t[e]?n=>t[e](n):n=>t.fire(e,n)))}(t,e)))}function Bv(t,e,n){e.forEach((e=>t.fire(e,n)))}function zv(t,e,n){return function(r){const i=this._active,o=this.pickEvent(r);o===i||(i&&i.exit||Bv(this,n,r),this._active=o,Bv(this,e,r)),Bv(this,t,r)}}function Nv(t){return function(e){Bv(this,t,e),this._active=null}}function Ov(t,e,n,r,i,o){const a=\"undefined\"!=typeof HTMLElement&&t instanceof HTMLElement&&null!=t.parentNode,s=t.getContext(\"2d\"),u=a?\"undefined\"!=typeof window&&window.devicePixelRatio||1:i;t.width=e*u,t.height=n*u;for(const t in o)s[t]=o[t];return a&&1!==u&&(t.style.width=e+\"px\",t.style.height=n+\"px\"),s.pixelRatio=u,s.setTransform(u,0,0,u,u*r[0],u*r[1]),t}class Rv extends lv{constructor(t){super(t),this._options={},this._redraw=!1,this._dirty=new Vg,this._tempb=new Vg}initialize(t,e,n,r,i,o){return this._options=o||{},this._canvas=this._options.externalContext?null:$c(1,1,this._options.type),t&&this._canvas&&(iv(t,0).appendChild(this._canvas),this._canvas.setAttribute(\"class\",\"marks\")),super.initialize(t,e,n,r,i)}resize(t,e,n,r){if(super.resize(t,e,n,r),this._canvas)Ov(this._canvas,this._width,this._height,this._origin,this._scale,this._options.context);else{const t=this._options.externalContext;t||s(\"CanvasRenderer is missing a valid canvas or context\"),t.scale(this._scale,this._scale),t.translate(this._origin[0],this._origin[1])}return this._redraw=!0,this}canvas(){return this._canvas}context(){return this._options.externalContext||(this._canvas?this._canvas.getContext(\"2d\"):null)}dirty(t){const e=this._tempb.clear().union(t.bounds);let n=t.mark.group;for(;n;)e.translate(n.x||0,n.y||0),n=n.mark.group;this._dirty.union(e)}_render(t,e){const n=this.context(),r=this._origin,i=this._width,o=this._height,a=this._dirty,s=Uv(r,i,o);n.save();const u=this._redraw||a.empty()?(this._redraw=!1,s.expand(1)):function(t,e,n){e.expand(1).round(),t.pixelRatio%1&&e.scale(t.pixelRatio).round().scale(1/t.pixelRatio);return e.translate(-n[0]%1,-n[1]%1),t.beginPath(),t.rect(e.x1,e.y1,e.width(),e.height()),t.clip(),e}(n,s.intersect(a),r);return this.clear(-r[0],-r[1],i,o),this.draw(n,t,u,e),n.restore(),a.clear(),this}draw(t,e,n,r){if(\"group\"!==e.marktype&&null!=r&&!r.includes(e.marktype))return;const i=Hy[e.marktype];e.clip&&function(t,e){var n=e.clip;t.save(),J(n)?(t.beginPath(),n(t),t.clip()):ty(t,e.group)}(t,e),i.draw.call(this,t,e,n,r),e.clip&&t.restore()}clear(t,e,n,r){const i=this._options,o=this.context();\"pdf\"===i.type||i.externalContext||o.clearRect(t,e,n,r),null!=this._bgcolor&&(o.fillStyle=this._bgcolor,o.fillRect(t,e,n,r))}}const Uv=(t,e,n)=>(new Vg).set(0,0,e,n).translate(-t[0],-t[1]);class Lv extends sv{constructor(t,e){super(t,e);const n=this;n._hrefHandler=qv(n,((t,e)=>{e&&e.href&&n.handleHref(t,e,e.href)})),n._tooltipHandler=qv(n,((t,e)=>{n.handleTooltip(t,e,t.type!==Cv)}))}initialize(t,e,n){let r=this._svg;return r&&(r.removeEventListener(Fv,this._hrefHandler),r.removeEventListener(Dv,this._tooltipHandler),r.removeEventListener(Cv,this._tooltipHandler)),this._svg=r=t&&nv(t,\"svg\"),r&&(r.addEventListener(Fv,this._hrefHandler),r.addEventListener(Dv,this._tooltipHandler),r.addEventListener(Cv,this._tooltipHandler)),super.initialize(t,e,n)}canvas(){return this._svg}on(t,e){const n=this.eventName(t),r=this._handlers;if(this._handlerIndex(r[n],t,e)<0){const i={type:t,handler:e,listener:qv(this,e)};(r[n]||(r[n]=[])).push(i),this._svg&&this._svg.addEventListener(n,i.listener)}return this}off(t,e){const n=this.eventName(t),r=this._handlers[n],i=this._handlerIndex(r,t,e);return i>=0&&(this._svg&&this._svg.removeEventListener(n,r[i].listener),r.splice(i,1)),this}}const qv=(t,e)=>n=>{let r=n.target.__data__;r=Array.isArray(r)?r[0]:r,n.vegaType=n.type,e.call(t._obj,n,r)},Pv=\"aria-hidden\",jv=\"aria-label\",Iv=\"role\",Wv=\"aria-roledescription\",Hv=\"graphics-object\",Yv=\"graphics-symbol\",Gv=(t,e,n)=>({[Iv]:t,[Wv]:e,[jv]:n||void 0}),Vv=Bt([\"axis-domain\",\"axis-grid\",\"axis-label\",\"axis-tick\",\"axis-title\",\"legend-band\",\"legend-entry\",\"legend-gradient\",\"legend-label\",\"legend-title\",\"legend-symbol\",\"title\"]),Xv={axis:{desc:\"axis\",caption:function(t){const e=t.datum,n=t.orient,r=e.title?t_(t):null,i=t.context,o=i.scales[e.scale].value,a=i.dataflow.locale(),s=o.type;return(\"left\"===n||\"right\"===n?\"Y\":\"X\")+\"-axis\"+(r?` titled '${r}'`:\"\")+` for a ${cp(s)?\"discrete\":s} scale`+` with ${Yp(a,o,t)}`}},legend:{desc:\"legend\",caption:function(t){const e=t.datum,n=e.title?t_(t):null,r=`${e.type||\"\"} legend`.trim(),i=e.scales,o=Object.keys(i),a=t.context,s=a.scales[i[o[0]]].value,u=a.dataflow.locale();return l=r,(l.length?l[0].toUpperCase()+l.slice(1):l)+(n?` titled '${n}'`:\"\")+` for ${function(t){return t=t.map((t=>t+(\"fill\"===t||\"stroke\"===t?\" color\":\"\"))),t.length<2?t[0]:t.slice(0,-1).join(\", \")+\" and \"+F(t)}(o)}`+` with ${Yp(u,s,t)}`;var l}},\"title-text\":{desc:\"title\",caption:t=>`Title text '${Kv(t)}'`},\"title-subtitle\":{desc:\"subtitle\",caption:t=>`Subtitle text '${Kv(t)}'`}},Jv={ariaRole:Iv,ariaRoleDescription:Wv,description:jv};function Zv(t,e){const n=!1===e.aria;if(t(Pv,n||void 0),n||null==e.description)for(const e in Jv)t(Jv[e],void 0);else{const n=e.mark.marktype;t(jv,e.description),t(Iv,e.ariaRole||(\"group\"===n?Hv:Yv)),t(Wv,e.ariaRoleDescription||`${n} mark`)}}function Qv(t){return!1===t.aria?{[Pv]:!0}:Vv[t.role]?null:Xv[t.role]?function(t,e){try{const n=t.items[0],r=e.caption||(()=>\"\");return Gv(e.role||Yv,e.desc,n.description||r(n))}catch(t){return null}}(t,Xv[t.role]):function(t){const e=t.marktype,n=\"group\"===e||\"text\"===e||t.items.some((t=>null!=t.description&&!1!==t.aria));return Gv(n?Hv:Yv,`${e} mark container`,t.description)}(t)}function Kv(t){return V(t.text).join(\" \")}function t_(t){try{return V(F(t.items).items[0].text).join(\" \")}catch(t){return null}}const e_=t=>(t+\"\").replace(/&/g,\"&\").replace(//g,\">\");function n_(){let t=\"\",e=\"\",n=\"\";const r=[],i=()=>e=n=\"\",o=(t,n)=>{var r;return null!=n&&(e+=` ${t}=\"${r=n,e_(r).replace(/\"/g,\""\").replace(/\\t/g,\" \").replace(/\\n/g,\" \").replace(/\\r/g,\" \")}\"`),a},a={open(s){(o=>{e&&(t+=`${e}>${n}`,i()),r.push(o)})(s),e=\"<\"+s;for(var u=arguments.length,l=new Array(u>1?u-1:0),c=1;c${n}`:\"/>\"):``,i(),a},attr:o,text:t=>(n+=e_(t),a),toString:()=>t};return a}const r_=t=>i_(n_(),t)+\"\";function i_(t,e){if(t.open(e.tagName),e.hasAttributes()){const n=e.attributes,r=n.length;for(let e=0;e{t.dirty=e}))),r.zdirty||(n.exit?(o.nested&&r.items.length?(u=r.items[0],u._svg&&this._update(o,u._svg,u)):n._svg&&(u=n._svg.parentNode,u&&u.removeChild(n._svg)),n._svg=null):(n=o.nested?r.items[0]:n,n._update!==e&&(n._svg&&n._svg.ownerSVGElement?this._update(o,n._svg,n):(this._dirtyAll=!1,f_(n,e)),n._update=e)));return!this._dirtyAll}mark(t,e,n,r){if(!this.isDirty(e))return e._svg;const i=this._svg,o=e.marktype,a=Hy[o],s=!1===e.interactive?\"none\":null,u=\"g\"===a.tag,l=p_(e,t,n,\"g\",i);if(\"group\"!==o&&null!=r&&!r.includes(o))return iv(l,0),e._svg;l.setAttribute(\"class\",ov(e));const c=Qv(e);for(const t in c)b_(l,t,c[t]);u||b_(l,\"pointer-events\",s),b_(l,\"clip-path\",e.clip?Gg(this,e,e.group):null);let f=null,h=0;const d=t=>{const e=this.isDirty(t),n=p_(t,l,f,a.tag,i);e&&(this._update(a,n,t),u&&function(t,e,n,r){e=e.lastChild.previousSibling;let i,o=0;Lm(n,(n=>{i=t.mark(e,n,i,r),++o})),iv(e,1+o)}(this,n,t,r)),f=n,++h};return a.nested?e.items.length&&d(e.items[0]):Lm(e,d),iv(l,h),l}_update(t,e,n){g_=e,m_=e.__values__,Zv(v_,n),t.attr(v_,n,this);const r=y_[t.type];r&&r.call(this,t,e,n),g_&&this.style(g_,n)}style(t,e){if(null!=e){for(const n in o_){let r=\"font\"===n?Oy(e):e[n];if(r===m_[n])continue;const i=o_[n];null==r?t.removeAttribute(i):(Xp(r)&&(r=Jp(r,this._defs.gradient,w_())),t.setAttribute(i,r+\"\")),m_[n]=r}for(const n in a_)__(t,a_[n],e[n])}}defs(){const t=this._svg,e=this._defs;let n=e.el,r=0;for(const i in e.gradient)n||(e.el=n=rv(t,1,\"defs\",l_)),r=h_(n,e.gradient[i],r);for(const i in e.clipping)n||(e.el=n=rv(t,1,\"defs\",l_)),r=d_(n,e.clipping[i],r);n&&(0===r?(t.removeChild(n),e.el=null):iv(n,r))}_clearDefs(){const t=this._defs;t.gradient={},t.clipping={}}}function f_(t,e){for(;t&&t.dirty!==e;t=t.mark.group){if(t.dirty=e,!t.mark||t.mark.dirty===e)return;t.mark.dirty=e}}function h_(t,e,n){let r,i,o;if(\"radial\"===e.gradient){let r=rv(t,n++,\"pattern\",l_);x_(r,{id:Vp+e.id,viewBox:\"0,0,1,1\",width:\"100%\",height:\"100%\",preserveAspectRatio:\"xMidYMid slice\"}),r=rv(r,0,\"rect\",l_),x_(r,{width:1,height:1,fill:`url(${w_()}#${e.id})`}),x_(t=rv(t,n++,\"radialGradient\",l_),{id:e.id,fx:e.x1,fy:e.y1,fr:e.r1,cx:e.x2,cy:e.y2,r:e.r2})}else x_(t=rv(t,n++,\"linearGradient\",l_),{id:e.id,x1:e.x1,x2:e.x2,y1:e.y1,y2:e.y2});for(r=0,i=e.stops.length;r1&&t.previousSibling!=e}(a,n))&&e.insertBefore(a,n?n.nextSibling:e.firstChild),a}let g_=null,m_=null;const y_={group(t,e,n){const r=g_=e.childNodes[2];m_=r.__values__,t.foreground(v_,n,this),m_=e.__values__,g_=e.childNodes[1],t.content(v_,n,this);const i=g_=e.childNodes[0];t.background(v_,n,this);const o=!1===n.mark.interactive?\"none\":null;if(o!==m_.events&&(b_(r,\"pointer-events\",o),b_(i,\"pointer-events\",o),m_.events=o),n.strokeForeground&&n.stroke){const t=n.fill;b_(r,\"display\",null),this.style(i,n),b_(i,\"stroke\",null),t&&(n.fill=null),m_=r.__values__,this.style(r,n),t&&(n.fill=t),g_=null}else b_(r,\"display\",\"none\")},image(t,e,n){!1===n.smooth?(__(e,\"image-rendering\",\"optimizeSpeed\"),__(e,\"image-rendering\",\"pixelated\")):__(e,\"image-rendering\",null)},text(t,e,n){const r=By(n);let i,o,a,s;k(r)?(o=r.map((t=>Ny(n,t))),i=o.join(\"\\n\"),i!==m_.text&&(iv(e,0),a=e.ownerDocument,s=Ty(n),o.forEach(((t,r)=>{const i=ev(a,\"tspan\",l_);i.__data__=n,i.textContent=t,r&&(i.setAttribute(\"x\",0),i.setAttribute(\"dy\",s)),e.appendChild(i)})),m_.text=i)):(o=Ny(n,r),o!==m_.text&&(e.textContent=o,m_.text=o)),b_(e,\"font-family\",Oy(n)),b_(e,\"font-size\",$y(n)+\"px\"),b_(e,\"font-style\",n.fontStyle),b_(e,\"font-variant\",n.fontVariant),b_(e,\"font-weight\",n.fontWeight)}};function v_(t,e,n){e!==m_[t]&&(n?function(t,e,n,r){null!=n?t.setAttributeNS(r,e,n):t.removeAttributeNS(r,e)}(g_,t,e,n):b_(g_,t,e),m_[t]=e)}function __(t,e,n){n!==m_[e]&&(null==n?t.style.removeProperty(e):t.style.setProperty(e,n+\"\"),m_[e]=n)}function x_(t,e){for(const n in e)b_(t,n,e[n])}function b_(t,e,n){null!=n?t.setAttribute(e,n):t.removeAttribute(e)}function w_(){let t;return\"undefined\"==typeof window?\"\":(t=window.location).hash?t.href.slice(0,-t.hash.length):t.href}class k_ extends lv{constructor(t){super(t),this._text=null,this._defs={gradient:{},clipping:{}}}svg(){return this._text}_render(t){const e=n_();e.open(\"svg\",ot({},uy,{class:\"marks\",width:this._width*this._scale,height:this._height*this._scale,viewBox:`0 0 ${this._width} ${this._height}`}));const n=this._bgcolor;return n&&\"transparent\"!==n&&\"none\"!==n&&e.open(\"rect\",{width:this._width,height:this._height,fill:n}).close(),e.open(\"g\",s_,{transform:\"translate(\"+this._origin+\")\"}),this.mark(e,t),e.close(),this.defs(e),this._text=e.close()+\"\",this}mark(t,e){const n=Hy[e.marktype],r=n.tag,i=[Zv,n.attr];t.open(\"g\",{class:ov(e),\"clip-path\":e.clip?Gg(this,e,e.group):null},Qv(e),{\"pointer-events\":\"g\"!==r&&!1===e.interactive?\"none\":null});const o=o=>{const a=this.href(o);if(a&&t.open(\"a\",a),t.open(r,this.attr(e,o,i,\"g\"!==r?r:null)),\"text\"===r){const e=By(o);if(k(e)){const n={x:0,dy:Ty(o)};for(let r=0;rthis.mark(t,e))),t.close(),r&&a?(i&&(o.fill=null),o.stroke=a,t.open(\"path\",this.attr(e,o,n.foreground,\"bgrect\")).close(),i&&(o.fill=i)):t.open(\"path\",this.attr(e,o,n.foreground,\"bgfore\")).close()}t.close(),a&&t.close()};return n.nested?e.items&&e.items.length&&o(e.items[0]):Lm(e,o),t.close()}href(t){const e=t.href;let n;if(e){if(n=this._hrefs&&this._hrefs[e])return n;this.sanitizeURL(e).then((t=>{t[\"xlink:href\"]=t.href,t.href=null,(this._hrefs||(this._hrefs={}))[e]=t}))}return null}attr(t,e,n,r){const i={},o=(t,e,n,r)=>{i[r||t]=e};return Array.isArray(n)?n.forEach((t=>t(o,e,this))):n(o,e,this),r&&function(t,e,n,r,i){let o;if(null==e)return t;\"bgrect\"===r&&!1===n.interactive&&(t[\"pointer-events\"]=\"none\");if(\"bgfore\"===r&&(!1===n.interactive&&(t[\"pointer-events\"]=\"none\"),t.display=\"none\",null!==e.fill))return t;\"image\"===r&&!1===e.smooth&&(o=[\"image-rendering: optimizeSpeed;\",\"image-rendering: pixelated;\"]);\"text\"===r&&(t[\"font-family\"]=Oy(e),t[\"font-size\"]=$y(e)+\"px\",t[\"font-style\"]=e.fontStyle,t[\"font-variant\"]=e.fontVariant,t[\"font-weight\"]=e.fontWeight);for(const n in o_){let r=e[n];const o=o_[n];(\"transparent\"!==r||\"fill\"!==o&&\"stroke\"!==o)&&null!=r&&(Xp(r)&&(r=Jp(r,i.gradient,\"\")),t[o]=r)}for(const t in a_){const n=e[t];null!=n&&(o=o||[],o.push(`${a_[t]}: ${n};`))}o&&(t.style=o.join(\" \"))}(i,e,t,r,this._defs),i}defs(t){const e=this._defs.gradient,n=this._defs.clipping;if(0!==Object.keys(e).length+Object.keys(n).length){t.open(\"defs\");for(const n in e){const r=e[n],i=r.stops;\"radial\"===r.gradient?(t.open(\"pattern\",{id:Vp+n,viewBox:\"0,0,1,1\",width:\"100%\",height:\"100%\",preserveAspectRatio:\"xMidYMid slice\"}),t.open(\"rect\",{width:\"1\",height:\"1\",fill:\"url(#\"+n+\")\"}).close(),t.close(),t.open(\"radialGradient\",{id:n,fx:r.x1,fy:r.y1,fr:r.r1,cx:r.x2,cy:r.y2,r:r.r2})):t.open(\"linearGradient\",{id:n,x1:r.x1,x2:r.x2,y1:r.y1,y2:r.y2});for(let e=0;e!A_.svgMarkTypes.includes(t)));this._svgRenderer.render(t,A_.svgMarkTypes),this._canvasRenderer.render(t,n)}resize(t,e,n,r){return super.resize(t,e,n,r),this._svgRenderer.resize(t,e,n,r),this._canvasRenderer.resize(t,e,n,r),this}background(t){return A_.svgOnTop?this._canvasRenderer.background(t):this._svgRenderer.background(t),this}}class E_ extends Sv{constructor(t,e){super(t,e)}initialize(t,e,n){const r=rv(rv(t,0,\"div\"),A_.svgOnTop?0:1,\"div\");return super.initialize(r,e,n)}}const D_=\"canvas\",C_=\"hybrid\",F_=\"none\",S_={Canvas:D_,PNG:\"png\",SVG:\"svg\",Hybrid:C_,None:F_},$_={};function T_(t,e){return t=String(t||\"\").toLowerCase(),arguments.length>1?($_[t]=e,this):$_[t]}function B_(t,e,n){const r=[],i=(new Vg).union(e),o=t.marktype;return o?z_(t,i,n,r):\"group\"===o?N_(t,i,n,r):s(\"Intersect scene must be mark node or group item.\")}function z_(t,e,n,r){if(function(t,e,n){return t.bounds&&e.intersects(t.bounds)&&(\"group\"===t.marktype||!1!==t.interactive&&(!n||n(t)))}(t,e,n)){const i=t.items,o=t.marktype,a=i.length;let s=0;if(\"group\"===o)for(;s=0;r--)if(i[r]!=o[r])return!1;for(r=i.length-1;r>=0;r--)if(!q_(t[n=i[r]],e[n],n))return!1;return typeof t==typeof e}(t,e):t==e)}function P_(t,e){return q_(ag(t),ag(e))}const j_=\"top\",I_=\"left\",W_=\"right\",H_=\"bottom\",Y_=\"top-left\",G_=\"top-right\",V_=\"bottom-left\",X_=\"bottom-right\",J_=\"start\",Z_=\"middle\",Q_=\"end\",K_=\"x\",tx=\"y\",ex=\"group\",nx=\"axis\",rx=\"title\",ix=\"frame\",ox=\"scope\",ax=\"legend\",sx=\"row-header\",ux=\"row-footer\",lx=\"row-title\",cx=\"column-header\",fx=\"column-footer\",hx=\"column-title\",dx=\"padding\",px=\"symbol\",gx=\"fit\",mx=\"fit-x\",yx=\"fit-y\",vx=\"pad\",_x=\"none\",xx=\"all\",bx=\"each\",wx=\"flush\",kx=\"column\",Ax=\"row\";function Mx(t){Ja.call(this,null,t)}function Ex(t,e,n){return e(t.bounds.clear(),t,n)}dt(Mx,Ja,{transform(t,e){const n=e.dataflow,r=t.mark,i=r.marktype,o=Hy[i],a=o.bound;let s,u=r.bounds;if(o.nested)r.items.length&&n.dirty(r.items[0]),u=Ex(r,a),r.items.forEach((t=>{t.bounds.clear().union(u)}));else if(i===ex||t.modified())switch(e.visit(e.MOD,(t=>n.dirty(t))),u.clear(),r.items.forEach((t=>u.union(Ex(t,a)))),r.role){case nx:case ax:case rx:e.reflow()}else s=e.changed(e.REM),e.visit(e.ADD,(t=>{u.union(Ex(t,a))})),e.visit(e.MOD,(t=>{s=s||u.alignsWith(t.bounds),n.dirty(t),u.union(Ex(t,a))})),s&&(u.clear(),r.items.forEach((t=>u.union(t.bounds))));return U_(r),e.modifies(\"bounds\")}});const Dx=\":vega_identifier:\";function Cx(t){Ja.call(this,0,t)}function Fx(t){Ja.call(this,null,t)}function Sx(t){Ja.call(this,null,t)}Cx.Definition={type:\"Identifier\",metadata:{modifies:!0},params:[{name:\"as\",type:\"string\",required:!0}]},dt(Cx,Ja,{transform(t,e){const n=(i=e.dataflow)._signals[Dx]||(i._signals[Dx]=i.add(0)),r=t.as;var i;let o=n.value;return e.visit(e.ADD,(t=>t[r]=t[r]||++o)),n.set(this.value=o),e}}),dt(Fx,Ja,{transform(t,e){let n=this.value;n||(n=e.dataflow.scenegraph().mark(t.markdef,function(t){const e=t.groups,n=t.parent;return e&&1===e.size?e.get(Object.keys(e.object)[0]):e&&n?e.lookup(n):null}(t),t.index),n.group.context=t.context,t.context.group||(t.context.group=n.group),n.source=this.source,n.clip=t.clip,n.interactive=t.interactive,this.value=n);const r=n.marktype===ex?Jg:Xg;return e.visit(e.ADD,(t=>r.call(t,n))),(t.modified(\"clip\")||t.modified(\"interactive\"))&&(n.clip=t.clip,n.interactive=!!t.interactive,n.zdirty=!0,e.reflow()),n.items=e.source,e}});const $x={parity:t=>t.filter(((t,e)=>e%2?t.opacity=0:1)),greedy:(t,e)=>{let n;return t.filter(((t,r)=>r&&Tx(n.bounds,t.bounds,e)?t.opacity=0:(n=t,1)))}},Tx=(t,e,n)=>n>Math.max(e.x1-t.x2,t.x1-e.x2,e.y1-t.y2,t.y1-e.y2),Bx=(t,e)=>{for(var n,r=1,i=t.length,o=t[0].bounds;r{const e=t.bounds;return e.width()>1&&e.height()>1},Nx=t=>(t.forEach((t=>t.opacity=1)),t),Ox=(t,e)=>t.reflow(e.modified()).modifies(\"opacity\");function Rx(t){Ja.call(this,null,t)}dt(Sx,Ja,{transform(t,e){const n=$x[t.method]||$x.parity,r=t.separation||0;let i,o,a=e.materialize(e.SOURCE).source;if(!a||!a.length)return;if(!t.method)return t.modified(\"method\")&&(Nx(a),e=Ox(e,t)),e;if(a=a.filter(zx),!a.length)return;if(t.sort&&(a=a.slice().sort(t.sort)),i=Nx(a),e=Ox(e,t),i.length>=3&&Bx(i,r)){do{i=n(i,r)}while(i.length>=3&&Bx(i,r));i.length<3&&!F(a).opacity&&(i.length>1&&(F(i).opacity=0),F(a).opacity=1)}t.boundScale&&t.boundTolerance>=0&&(o=((t,e,n)=>{var r=t.range(),i=new Vg;return e===j_||e===H_?i.set(r[0],-1/0,r[1],1/0):i.set(-1/0,r[0],1/0,r[1]),i.expand(n||1),t=>i.encloses(t.bounds)})(t.boundScale,t.boundOrient,+t.boundTolerance),a.forEach((t=>{o(t)||(t.opacity=0)})));const s=i[0].mark.bounds.clear();return a.forEach((t=>{t.opacity&&s.union(t.bounds)})),e}}),dt(Rx,Ja,{transform(t,e){const n=e.dataflow;if(e.visit(e.ALL,(t=>n.dirty(t))),e.fields&&e.fields.zindex){const t=e.source&&e.source[0];t&&(t.mark.zdirty=!0)}}});const Ux=new Vg;function Lx(t,e,n){return t[e]===n?0:(t[e]=n,1)}function qx(t){var e=t.items[0].orient;return e===I_||e===W_}function Px(t,e,n,r){var i,o,a=e.items[0],s=a.datum,u=null!=a.translate?a.translate:.5,l=a.orient,c=function(t){let e=+t.grid;return[t.ticks?e++:-1,t.labels?e++:-1,e+ +t.domain]}(s),f=a.range,h=a.offset,d=a.position,p=a.minExtent,g=a.maxExtent,m=s.title&&a.items[c[2]].items[0],y=a.titlePadding,v=a.bounds,_=m&&zy(m),x=0,b=0;switch(Ux.clear().union(v),v.clear(),(i=c[0])>-1&&v.union(a.items[i].bounds),(i=c[1])>-1&&v.union(a.items[i].bounds),l){case j_:x=d||0,b=-h,o=Math.max(p,Math.min(g,-v.y1)),v.add(0,-o).add(f,0),m&&jx(t,m,o,y,_,0,-1,v);break;case I_:x=-h,b=d||0,o=Math.max(p,Math.min(g,-v.x1)),v.add(-o,0).add(0,f),m&&jx(t,m,o,y,_,1,-1,v);break;case W_:x=n+h,b=d||0,o=Math.max(p,Math.min(g,v.x2)),v.add(0,0).add(o,f),m&&jx(t,m,o,y,_,1,1,v);break;case H_:x=d||0,b=r+h,o=Math.max(p,Math.min(g,v.y2)),v.add(0,0).add(f,o),m&&jx(t,m,o,y,0,0,1,v);break;default:x=a.x,b=a.y}return tm(v.translate(x,b),a),Lx(a,\"x\",x+u)|Lx(a,\"y\",b+u)&&(a.bounds=Ux,t.dirty(a),a.bounds=v,t.dirty(a)),a.mark.bounds.clear().union(v)}function jx(t,e,n,r,i,o,a,s){const u=e.bounds;if(e.auto){const s=a*(n+i+r);let l=0,c=0;t.dirty(e),o?l=(e.x||0)-(e.x=s):c=(e.y||0)-(e.y=s),e.mark.bounds.clear().union(u.translate(-l,-c)),t.dirty(e)}s.union(u)}const Ix=(t,e)=>Math.floor(Math.min(t,e)),Wx=(t,e)=>Math.ceil(Math.max(t,e));function Hx(t){return(new Vg).set(0,0,t.width||0,t.height||0)}function Yx(t){const e=t.bounds.clone();return e.empty()?e.set(0,0,0,0):e.translate(-(t.x||0),-(t.y||0))}function Gx(t,e,n){const r=A(t)?t[e]:t;return null!=r?r:void 0!==n?n:0}function Vx(t){return t<0?Math.ceil(-t):0}function Xx(t,e,n){var r,i,o,a,s,u,l,c,f,h,d,p=!n.nodirty,g=n.bounds===wx?Hx:Yx,m=Ux.set(0,0,0,0),y=Gx(n.align,kx),v=Gx(n.align,Ax),_=Gx(n.padding,kx),x=Gx(n.padding,Ax),b=n.columns||e.length,w=b<=0?1:Math.ceil(e.length/b),k=e.length,A=Array(k),M=Array(b),E=0,D=Array(k),C=Array(w),F=0,S=Array(k),$=Array(k),T=Array(k);for(i=0;i1)for(i=0;i0&&(S[i]+=f/2);if(v&&Gx(n.center,Ax)&&1!==b)for(i=0;i0&&($[i]+=h/2);for(i=0;ii&&(t.warn(\"Grid headers exceed limit: \"+i),e=e.slice(0,i)),A+=o,g=0,y=e.length;g=0&&null==(x=n[m]);m-=h);s?(b=null==d?x.x:Math.round(x.bounds.x1+d*x.bounds.width()),w=A):(b=A,w=null==d?x.y:Math.round(x.bounds.y1+d*x.bounds.height())),v.union(_.bounds.translate(b-(_.x||0),w-(_.y||0))),_.x=b,_.y=w,t.dirty(_),M=a(M,v[l])}return M}function tb(t,e,n,r,i,o){if(e){t.dirty(e);var a=n,s=n;r?a=Math.round(i.x1+o*i.width()):s=Math.round(i.y1+o*i.height()),e.bounds.translate(a-(e.x||0),s-(e.y||0)),e.mark.bounds.clear().union(e.bounds),e.x=a,e.y=s,t.dirty(e)}}function eb(t,e,n,r,i,o,a){const s=function(t,e){const n=t[e]||{};return(e,r)=>null!=n[e]?n[e]:null!=t[e]?t[e]:r}(n,e),u=function(t,e){let n=-1/0;return t.forEach((t=>{null!=t.offset&&(n=Math.max(n,t.offset))})),n>-1/0?n:e}(t,s(\"offset\",0)),l=s(\"anchor\",J_),c=l===Q_?1:l===Z_?.5:0,f={align:bx,bounds:s(\"bounds\",wx),columns:\"vertical\"===s(\"direction\")?1:t.length,padding:s(\"margin\",8),center:s(\"center\"),nodirty:!0};switch(e){case I_:f.anchor={x:Math.floor(r.x1)-u,column:Q_,y:c*(a||r.height()+2*r.y1),row:l};break;case W_:f.anchor={x:Math.ceil(r.x2)+u,y:c*(a||r.height()+2*r.y1),row:l};break;case j_:f.anchor={y:Math.floor(i.y1)-u,row:Q_,x:c*(o||i.width()+2*i.x1),column:l};break;case H_:f.anchor={y:Math.ceil(i.y2)+u,x:c*(o||i.width()+2*i.x1),column:l};break;case Y_:f.anchor={x:u,y:u};break;case G_:f.anchor={x:o-u,y:u,column:Q_};break;case V_:f.anchor={x:u,y:a-u,row:Q_};break;case X_:f.anchor={x:o-u,y:a-u,column:Q_,row:Q_}}return f}function nb(t,e){var n,r,i=e.items[0],o=i.datum,a=i.orient,s=i.bounds,u=i.x,l=i.y;return i._bounds?i._bounds.clear().union(s):i._bounds=s.clone(),s.clear(),function(t,e,n){var r=e.padding,i=r-n.x,o=r-n.y;if(e.datum.title){var a=e.items[1].items[0],s=a.anchor,u=e.titlePadding||0,l=r-a.x,c=r-a.y;switch(a.orient){case I_:i+=Math.ceil(a.bounds.width())+u;break;case W_:case H_:break;default:o+=a.bounds.height()+u}switch((i||o)&&ib(t,n,i,o),a.orient){case I_:c+=rb(e,n,a,s,1,1);break;case W_:l+=rb(e,n,a,Q_,0,0)+u,c+=rb(e,n,a,s,1,1);break;case H_:l+=rb(e,n,a,s,0,0),c+=rb(e,n,a,Q_,-1,0,1)+u;break;default:l+=rb(e,n,a,s,0,0)}(l||c)&&ib(t,a,l,c),(l=Math.round(a.bounds.x1-r))<0&&(ib(t,n,-l,0),ib(t,a,-l,0))}else(i||o)&&ib(t,n,i,o)}(t,i,i.items[0].items[0]),s=function(t,e){return t.items.forEach((t=>e.union(t.bounds))),e.x1=t.padding,e.y1=t.padding,e}(i,s),n=2*i.padding,r=2*i.padding,s.empty()||(n=Math.ceil(s.width()+n),r=Math.ceil(s.height()+r)),o.type===px&&function(t){const e=t.reduce(((t,e)=>(t[e.column]=Math.max(e.bounds.x2-e.x,t[e.column]||0),t)),{});t.forEach((t=>{t.width=e[t.column],t.height=t.bounds.y2-t.y}))}(i.items[0].items[0].items[0].items),a!==_x&&(i.x=u=0,i.y=l=0),i.width=n,i.height=r,tm(s.set(u,l,u+n,l+r),i),i.mark.bounds.clear().union(s),i}function rb(t,e,n,r,i,o,a){const s=\"symbol\"!==t.datum.type,u=n.datum.vgrad,l=(!s||!o&&u||a?e:e.items[0]).bounds[i?\"y2\":\"x2\"]-t.padding,c=u&&o?l:0,f=u&&o?0:l,h=i<=0?0:zy(n);return Math.round(r===J_?c:r===Q_?f-h:.5*(l-h))}function ib(t,e,n,r){e.x+=n,e.y+=r,e.bounds.translate(n,r),e.mark.bounds.translate(n,r),t.dirty(e)}function ob(t){Ja.call(this,null,t)}dt(ob,Ja,{transform(t,e){const n=e.dataflow;return t.mark.items.forEach((e=>{t.layout&&Jx(n,e,t.layout),function(t,e,n){var r,i,o,a,s,u=e.items,l=Math.max(0,e.width||0),c=Math.max(0,e.height||0),f=(new Vg).set(0,0,l,c),h=f.clone(),d=f.clone(),p=[];for(a=0,s=u.length;a{(o=t.orient||W_)!==_x&&(e[o]||(e[o]=[])).push(t)}));for(const r in e){const i=e[r];Xx(t,i,eb(i,r,n.legends,h,d,l,c))}p.forEach((e=>{const r=e.bounds;if(r.equals(e._bounds)||(e.bounds=e._bounds,t.dirty(e),e.bounds=r,t.dirty(e)),!n.autosize||n.autosize.type!==gx&&n.autosize.type!==mx&&n.autosize.type!==yx)f.union(r);else switch(e.orient){case I_:case W_:f.add(r.x1,0).add(r.x2,0);break;case j_:case H_:f.add(0,r.y1).add(0,r.y2)}}))}f.union(h).union(d),r&&f.union(function(t,e,n,r,i){var o,a=e.items[0],s=a.frame,u=a.orient,l=a.anchor,c=a.offset,f=a.padding,h=a.items[0].items[0],d=a.items[1]&&a.items[1].items[0],p=u===I_||u===W_?r:n,g=0,m=0,y=0,v=0,_=0;if(s!==ex?u===I_?(g=i.y2,p=i.y1):u===W_?(g=i.y1,p=i.y2):(g=i.x1,p=i.x2):u===I_&&(g=r,p=0),o=l===J_?g:l===Q_?p:(g+p)/2,d&&d.text){switch(u){case j_:case H_:_=h.bounds.height()+f;break;case I_:v=h.bounds.width()+f;break;case W_:v=-h.bounds.width()-f}Ux.clear().union(d.bounds),Ux.translate(v-(d.x||0),_-(d.y||0)),Lx(d,\"x\",v)|Lx(d,\"y\",_)&&(t.dirty(d),d.bounds.clear().union(Ux),d.mark.bounds.clear().union(Ux),t.dirty(d)),Ux.clear().union(d.bounds)}else Ux.clear();switch(Ux.union(h.bounds),u){case j_:m=o,y=i.y1-Ux.height()-c;break;case I_:m=i.x1-Ux.width()-c,y=o;break;case W_:m=i.x2+Ux.width()+c,y=o;break;case H_:m=o,y=i.y2+c;break;default:m=a.x,y=a.y}return Lx(a,\"x\",m)|Lx(a,\"y\",y)&&(Ux.translate(m,y),t.dirty(a),a.bounds.clear().union(Ux),e.bounds.clear().union(Ux),t.dirty(a)),a.bounds}(t,r,l,c,f));e.clip&&f.set(0,0,e.width||0,e.height||0);!function(t,e,n,r){const i=r.autosize||{},o=i.type;if(t._autosize<1||!o)return;let a=t._width,s=t._height,u=Math.max(0,e.width||0),l=Math.max(0,Math.ceil(-n.x1)),c=Math.max(0,e.height||0),f=Math.max(0,Math.ceil(-n.y1));const h=Math.max(0,Math.ceil(n.x2-u)),d=Math.max(0,Math.ceil(n.y2-c));if(i.contains===dx){const e=t.padding();a-=e.left+e.right,s-=e.top+e.bottom}o===_x?(l=0,f=0,u=a,c=s):o===gx?(u=Math.max(0,a-l-h),c=Math.max(0,s-f-d)):o===mx?(u=Math.max(0,a-l-h),s=c+f+d):o===yx?(a=u+l+h,c=Math.max(0,s-f-d)):o===vx&&(a=u+l+h,s=c+f+d);t._resizeView(a,s,u,c,[l,f],i.resize)}(t,e,f,n)}(n,e,t)})),function(t){return t&&\"legend-entry\"!==t.mark.role}(t.mark.group)?e.reflow():e}});var ab=Object.freeze({__proto__:null,bound:Mx,identifier:Cx,mark:Fx,overlap:Sx,render:Rx,viewlayout:ob});function sb(t){Ja.call(this,null,t)}function ub(t){Ja.call(this,null,t)}function lb(){return _a({})}function cb(t){Ja.call(this,null,t)}function fb(t){Ja.call(this,[],t)}dt(sb,Ja,{transform(t,e){if(this.value&&!t.modified())return e.StopPropagation;var n=e.dataflow.locale(),r=e.fork(e.NO_SOURCE|e.NO_FIELDS),i=this.value,o=t.scale,a=Sp(o,null==t.count?t.values?t.values.length:10:t.count,t.minstep),s=t.format||Bp(n,o,a,t.formatSpecifier,t.formatType,!!t.values),u=t.values?$p(o,t.values,a):Tp(o,a);return i&&(r.rem=i),i=u.map(((t,e)=>_a({index:e/(u.length-1||1),value:t,label:s(t)}))),t.extra&&i.length&&i.push(_a({index:-1,extra:{value:i[0].value},label:\"\"})),r.source=i,r.add=i,this.value=i,r}}),dt(ub,Ja,{transform(t,e){var n=e.dataflow,r=e.fork(e.NO_SOURCE|e.NO_FIELDS),i=t.item||lb,o=t.key||ya,a=this.value;return k(r.encode)&&(r.encode=null),a&&(t.modified(\"key\")||e.modified(o))&&s(\"DataJoin does not support modified key function or fields.\"),a||(e=e.addAll(),this.value=a=function(t){const e=ft().test((t=>t.exit));return e.lookup=n=>e.get(t(n)),e}(o)),e.visit(e.ADD,(t=>{const e=o(t);let n=a.get(e);n?n.exit?(a.empty--,r.add.push(n)):r.mod.push(n):(n=i(t),a.set(e,n),r.add.push(n)),n.datum=t,n.exit=!1})),e.visit(e.MOD,(t=>{const e=o(t),n=a.get(e);n&&(n.datum=t,r.mod.push(n))})),e.visit(e.REM,(t=>{const e=o(t),n=a.get(e);t!==n.datum||n.exit||(r.rem.push(n),n.exit=!0,++a.empty)})),e.changed(e.ADD_MOD)&&r.modifies(\"datum\"),(e.clean()||t.clean&&a.empty>n.cleanThreshold)&&n.runAfter(a.clean),r}}),dt(cb,Ja,{transform(t,e){var n=e.fork(e.ADD_REM),r=t.mod||!1,i=t.encoders,o=e.encode;if(k(o)){if(!n.changed()&&!o.every((t=>i[t])))return e.StopPropagation;o=o[0],n.encode=null}var a=\"enter\"===o,s=i.update||g,u=i.enter||g,l=i.exit||g,c=(o&&!a?i[o]:s)||g;if(e.changed(e.ADD)&&(e.visit(e.ADD,(e=>{u(e,t),s(e,t)})),n.modifies(u.output),n.modifies(s.output),c!==g&&c!==s&&(e.visit(e.ADD,(e=>{c(e,t)})),n.modifies(c.output))),e.changed(e.REM)&&l!==g&&(e.visit(e.REM,(e=>{l(e,t)})),n.modifies(l.output)),a||c!==g){const i=e.MOD|(t.modified()?e.REFLOW:0);a?(e.visit(i,(e=>{const i=u(e,t)||r;(c(e,t)||i)&&n.mod.push(e)})),n.mod.length&&n.modifies(u.output)):e.visit(i,(e=>{(c(e,t)||r)&&n.mod.push(e)})),n.mod.length&&n.modifies(c.output)}return n.changed()?n:e.StopPropagation}}),dt(fb,Ja,{transform(t,e){if(null!=this.value&&!t.modified())return e.StopPropagation;var n,r,i,o,a,s=e.dataflow.locale(),u=e.fork(e.NO_SOURCE|e.NO_FIELDS),l=this.value,c=t.type||Mp,f=t.scale,h=+t.limit,d=Sp(f,null==t.count?5:t.count,t.minstep),p=!!t.values||c===Mp,g=t.format||Lp(s,f,d,c,t.formatSpecifier,t.formatType,p),m=t.values||Rp(f,d);return l&&(u.rem=l),c===Mp?(h&&m.length>h?(e.dataflow.warn(\"Symbol legend count exceeds limit, filtering items.\"),l=m.slice(0,h-1),a=!0):l=m,J(i=t.size)?(t.values||0!==f(l[0])||(l=l.slice(1)),o=l.reduce(((e,n)=>Math.max(e,i(n,t))),0)):i=rt(o=i||8),l=l.map(((e,n)=>_a({index:n,label:g(e,n,l),value:e,offset:o,size:i(e,t)}))),a&&(a=m[l.length],l.push(_a({index:l.length,label:`â€Ļ${m.length-l.length} entries`,value:a,offset:o,size:i(a,t)})))):\"gradient\"===c?(n=f.domain(),r=_p(f,n[0],F(n)),m.length<3&&!t.values&&n[0]!==F(n)&&(m=[n[0],F(n)]),l=m.map(((t,e)=>_a({index:e,label:g(t,e,m),value:t,perc:r(t)})))):(i=m.length-1,r=function(t){const e=t.domain(),n=e.length-1;let r=+e[0],i=+F(e),o=i-r;if(t.type===Id){const t=n?o/n:.1;r-=t,i+=t,o=i-r}return t=>(t-r)/o}(f),l=m.map(((t,e)=>_a({index:e,label:g(t,e,m),value:t,perc:e?r(t):0,perc2:e===i?1:r(m[e+1])})))),u.source=l,u.add=l,this.value=l,u}});const hb=t=>t.source.x,db=t=>t.source.y,pb=t=>t.target.x,gb=t=>t.target.y;function mb(t){Ja.call(this,{},t)}mb.Definition={type:\"LinkPath\",metadata:{modifies:!0},params:[{name:\"sourceX\",type:\"field\",default:\"source.x\"},{name:\"sourceY\",type:\"field\",default:\"source.y\"},{name:\"targetX\",type:\"field\",default:\"target.x\"},{name:\"targetY\",type:\"field\",default:\"target.y\"},{name:\"orient\",type:\"enum\",default:\"vertical\",values:[\"horizontal\",\"vertical\",\"radial\"]},{name:\"shape\",type:\"enum\",default:\"line\",values:[\"line\",\"arc\",\"curve\",\"diagonal\",\"orthogonal\"]},{name:\"require\",type:\"signal\"},{name:\"as\",type:\"string\",default:\"path\"}]},dt(mb,Ja,{transform(t,e){var n=t.sourceX||hb,r=t.sourceY||db,i=t.targetX||pb,o=t.targetY||gb,a=t.as||\"path\",u=t.orient||\"vertical\",l=t.shape||\"line\",c=xb.get(l+\"-\"+u)||xb.get(l);return c||s(\"LinkPath unsupported type: \"+t.shape+(t.orient?\"-\"+t.orient:\"\")),e.visit(e.SOURCE,(t=>{t[a]=c(n(t),r(t),i(t),o(t))})),e.reflow(t.modified()).modifies(a)}});const yb=(t,e,n,r)=>\"M\"+t+\",\"+e+\"L\"+n+\",\"+r,vb=(t,e,n,r)=>{var i=n-t,o=r-e,a=Math.hypot(i,o)/2;return\"M\"+t+\",\"+e+\"A\"+a+\",\"+a+\" \"+180*Math.atan2(o,i)/Math.PI+\" 0 1 \"+n+\",\"+r},_b=(t,e,n,r)=>{const i=n-t,o=r-e,a=.2*(i+o),s=.2*(o-i);return\"M\"+t+\",\"+e+\"C\"+(t+a)+\",\"+(e+s)+\" \"+(n+s)+\",\"+(r-a)+\" \"+n+\",\"+r},xb=ft({line:yb,\"line-radial\":(t,e,n,r)=>yb(e*Math.cos(t),e*Math.sin(t),r*Math.cos(n),r*Math.sin(n)),arc:vb,\"arc-radial\":(t,e,n,r)=>vb(e*Math.cos(t),e*Math.sin(t),r*Math.cos(n),r*Math.sin(n)),curve:_b,\"curve-radial\":(t,e,n,r)=>_b(e*Math.cos(t),e*Math.sin(t),r*Math.cos(n),r*Math.sin(n)),\"orthogonal-horizontal\":(t,e,n,r)=>\"M\"+t+\",\"+e+\"V\"+r+\"H\"+n,\"orthogonal-vertical\":(t,e,n,r)=>\"M\"+t+\",\"+e+\"H\"+n+\"V\"+r,\"orthogonal-radial\":(t,e,n,r)=>{const i=Math.cos(t),o=Math.sin(t),a=Math.cos(n),s=Math.sin(n);return\"M\"+e*i+\",\"+e*o+\"A\"+e+\",\"+e+\" 0 0,\"+((Math.abs(n-t)>Math.PI?n<=t:n>t)?1:0)+\" \"+e*a+\",\"+e*s+\"L\"+r*a+\",\"+r*s},\"diagonal-horizontal\":(t,e,n,r)=>{const i=(t+n)/2;return\"M\"+t+\",\"+e+\"C\"+i+\",\"+e+\" \"+i+\",\"+r+\" \"+n+\",\"+r},\"diagonal-vertical\":(t,e,n,r)=>{const i=(e+r)/2;return\"M\"+t+\",\"+e+\"C\"+t+\",\"+i+\" \"+n+\",\"+i+\" \"+n+\",\"+r},\"diagonal-radial\":(t,e,n,r)=>{const i=Math.cos(t),o=Math.sin(t),a=Math.cos(n),s=Math.sin(n),u=(e+r)/2;return\"M\"+e*i+\",\"+e*o+\"C\"+u*i+\",\"+u*o+\" \"+u*a+\",\"+u*s+\" \"+r*a+\",\"+r*s}});function bb(t){Ja.call(this,null,t)}bb.Definition={type:\"Pie\",metadata:{modifies:!0},params:[{name:\"field\",type:\"field\"},{name:\"startAngle\",type:\"number\",default:0},{name:\"endAngle\",type:\"number\",default:6.283185307179586},{name:\"sort\",type:\"boolean\",default:!1},{name:\"as\",type:\"string\",array:!0,length:2,default:[\"startAngle\",\"endAngle\"]}]},dt(bb,Ja,{transform(t,e){var n,r,i,o=t.as||[\"startAngle\",\"endAngle\"],a=o[0],s=o[1],u=t.field||d,l=t.startAngle||0,c=null!=t.endAngle?t.endAngle:2*Math.PI,f=e.source,h=f.map(u),p=h.length,g=l,m=(c-l)/$e(h),y=Se(p);for(t.sort&&y.sort(((t,e)=>h[t]-h[e])),n=0;nt+(e<0?-1:e>0?1:0)),0))!==e.length&&n.warn(\"Log scale domain includes zero: \"+Ct(e)));return e}function Db(t,e,n){return J(t)&&(e||n)?mp(t,Cb(e||[0,1],n)):t}function Cb(t,e){return e?t.slice().reverse():t}function Fb(t){Ja.call(this,null,t)}dt(Mb,Ja,{transform(t,e){var n=e.dataflow,r=this.value,i=function(t){var e,n=t.type,r=\"\";if(n===Ld)return Ld+\"-\"+Td;(function(t){const e=t.type;return lp(e)&&e!==Rd&&e!==Ud&&(t.scheme||t.range&&t.range.length&&t.range.every(xt))})(t)&&(r=2===(e=t.rawDomain?t.rawDomain.length:t.domain?t.domain.length+ +(null!=t.domainMid):0)?Ld+\"-\":3===e?qd+\"-\":\"\");return(r+n||Td).toLowerCase()}(t);for(i in r&&i===r.type||(this.value=r=ap(i)()),t)if(!Ab[i]){if(\"padding\"===i&&kb(r.type))continue;J(r[i])?r[i](t[i]):n.warn(\"Unsupported scale property: \"+i)}return function(t,e,n){var r=t.type,i=e.round||!1,o=e.range;if(null!=e.rangeStep)o=function(t,e,n){t!==Yd&&t!==Hd&&s(\"Only band and point scales support rangeStep.\");var r=(null!=e.paddingOuter?e.paddingOuter:e.padding)||0,i=t===Hd?1:(null!=e.paddingInner?e.paddingInner:e.padding)||0;return[0,e.rangeStep*$d(n,i,r)]}(r,e,n);else if(e.scheme&&(o=function(t,e,n){var r,i=e.schemeExtent;k(e.scheme)?r=yp(e.scheme,e.interpolate,e.interpolateGamma):(r=Ap(e.scheme.toLowerCase()))||s(`Unrecognized scheme name: ${e.scheme}`);return n=t===Id?n+1:t===Gd?n-1:t===Pd||t===jd?+e.schemeCount||wb:n,dp(t)?Db(r,i,e.reverse):J(r)?vp(Db(r,i),n):t===Wd?r:r.slice(0,n)}(r,e,n),J(o))){if(t.interpolator)return t.interpolator(o);s(`Scale type ${r} does not support interpolating color schemes.`)}if(o&&dp(r))return t.interpolator(yp(Cb(o,e.reverse),e.interpolate,e.interpolateGamma));o&&e.interpolate&&t.interpolate?t.interpolate(xp(e.interpolate,e.interpolateGamma)):J(t.round)?t.round(i):J(t.rangeRound)&&t.interpolate(i?yh:mh);o&&t.range(Cb(o,e.reverse))}(r,t,function(t,e,n){let r=e.bins;if(r&&!k(r)){const e=t.domain(),n=e[0],i=F(e),o=r.step;let a=null==r.start?n:r.start,u=null==r.stop?i:r.stop;o||s(\"Scale bins parameter missing step property.\"),ai&&(u=o*Math.floor(i/o)),r=Se(a,u+o/2,o)}r?t.bins=r:t.bins&&delete t.bins;t.type===Gd&&(r?e.domain||e.domainRaw||(t.domain(r),n=r.length):t.bins=t.domain());return n}(r,t,function(t,e,n){const r=function(t,e,n){return e?(t.domain(Eb(t.type,e,n)),e.length):-1}(t,e.domainRaw,n);if(r>-1)return r;var i,o,a=e.domain,s=t.type,u=e.zero||void 0===e.zero&&function(t){const e=t.type;return!t.bins&&(e===Td||e===zd||e===Nd)}(t);if(!a)return 0;if((u||null!=e.domainMin||null!=e.domainMax||null!=e.domainMid)&&(i=(a=a.slice()).length-1||1,u&&(a[0]>0&&(a[0]=0),a[i]<0&&(a[i]=0)),null!=e.domainMin&&(a[0]=e.domainMin),null!=e.domainMax&&(a[i]=e.domainMax),null!=e.domainMid)){const t=(o=e.domainMid)>a[i]?i+1:ot(u);if(null==e)d.push(t.slice());else for(i={},o=0,a=t.length;oh&&(h=f),n&&c.sort(n)}return d.max=h,d}(e.source,t.groupby,l,c),r=0,i=n.length,o=n.max;r0?1:t<0?-1:0},iw=Math.sqrt,ow=Math.tan;function aw(t){return t>1?0:t<-1?Pb:Math.acos(t)}function sw(t){return t>1?jb:t<-1?-jb:Math.asin(t)}function uw(){}function lw(t,e){t&&fw.hasOwnProperty(t.type)&&fw[t.type](t,e)}var cw={Feature:function(t,e){lw(t.geometry,e)},FeatureCollection:function(t,e){for(var n=t.features,r=-1,i=n.length;++r=0?1:-1,i=r*n,o=Jb(e=(e*=Yb)/2+Ib),a=nw(e),s=_w*a,u=vw*o+s*Jb(i),l=s*r*nw(i);$w.add(Xb(l,u)),yw=t,vw=o,_w=a}function Uw(t){return[Xb(t[1],t[0]),sw(t[2])]}function Lw(t){var e=t[0],n=t[1],r=Jb(n);return[r*Jb(e),r*nw(e),nw(n)]}function qw(t,e){return t[0]*e[0]+t[1]*e[1]+t[2]*e[2]}function Pw(t,e){return[t[1]*e[2]-t[2]*e[1],t[2]*e[0]-t[0]*e[2],t[0]*e[1]-t[1]*e[0]]}function jw(t,e){t[0]+=e[0],t[1]+=e[1],t[2]+=e[2]}function Iw(t,e){return[t[0]*e,t[1]*e,t[2]*e]}function Ww(t){var e=iw(t[0]*t[0]+t[1]*t[1]+t[2]*t[2]);t[0]/=e,t[1]/=e,t[2]/=e}var Hw,Yw,Gw,Vw,Xw,Jw,Zw,Qw,Kw,tk,ek,nk,rk,ik,ok,ak,sk={point:uk,lineStart:ck,lineEnd:fk,polygonStart:function(){sk.point=hk,sk.lineStart=dk,sk.lineEnd=pk,Cw=new se,Bw.polygonStart()},polygonEnd:function(){Bw.polygonEnd(),sk.point=uk,sk.lineStart=ck,sk.lineEnd=fk,$w<0?(xw=-(ww=180),bw=-(kw=90)):Cw>Lb?kw=90:Cw<-Lb&&(bw=-90),Sw[0]=xw,Sw[1]=ww},sphere:function(){xw=-(ww=180),bw=-(kw=90)}};function uk(t,e){Fw.push(Sw=[xw=t,ww=t]),ekw&&(kw=e)}function lk(t,e){var n=Lw([t*Yb,e*Yb]);if(Dw){var r=Pw(Dw,n),i=Pw([r[1],-r[0],0],r);Ww(i),i=Uw(i);var o,a=t-Aw,s=a>0?1:-1,u=i[0]*Hb*s,l=Gb(a)>180;l^(s*Awkw&&(kw=o):l^(s*Aw<(u=(u+360)%360-180)&&ukw&&(kw=e)),l?tgk(xw,ww)&&(ww=t):gk(t,ww)>gk(xw,ww)&&(xw=t):ww>=xw?(tww&&(ww=t)):t>Aw?gk(xw,t)>gk(xw,ww)&&(ww=t):gk(t,ww)>gk(xw,ww)&&(xw=t)}else Fw.push(Sw=[xw=t,ww=t]);ekw&&(kw=e),Dw=n,Aw=t}function ck(){sk.point=lk}function fk(){Sw[0]=xw,Sw[1]=ww,sk.point=uk,Dw=null}function hk(t,e){if(Dw){var n=t-Aw;Cw.add(Gb(n)>180?n+(n>0?360:-360):n)}else Mw=t,Ew=e;Bw.point(t,e),lk(t,e)}function dk(){Bw.lineStart()}function pk(){hk(Mw,Ew),Bw.lineEnd(),Gb(Cw)>Lb&&(xw=-(ww=180)),Sw[0]=xw,Sw[1]=ww,Dw=null}function gk(t,e){return(e-=t)<0?e+360:e}function mk(t,e){return t[0]-e[0]}function yk(t,e){return t[0]<=t[1]?t[0]<=e&&e<=t[1]:ePb&&(t-=Math.round(t/Wb)*Wb),[t,e]}function $k(t,e,n){return(t%=Wb)?e||n?Fk(Bk(t),zk(e,n)):Bk(t):e||n?zk(e,n):Sk}function Tk(t){return function(e,n){return Gb(e+=t)>Pb&&(e-=Math.round(e/Wb)*Wb),[e,n]}}function Bk(t){var e=Tk(t);return e.invert=Tk(-t),e}function zk(t,e){var n=Jb(t),r=nw(t),i=Jb(e),o=nw(e);function a(t,e){var a=Jb(e),s=Jb(t)*a,u=nw(t)*a,l=nw(e),c=l*n+s*r;return[Xb(u*i-c*o,s*n-l*r),sw(c*i+u*o)]}return a.invert=function(t,e){var a=Jb(e),s=Jb(t)*a,u=nw(t)*a,l=nw(e),c=l*i-u*o;return[Xb(u*i+l*o,s*n+c*r),sw(c*n-s*r)]},a}function Nk(t,e){(e=Lw(e))[0]-=t,Ww(e);var n=aw(-e[1]);return((-e[2]<0?-n:n)+Wb-Lb)%Wb}function Ok(){var t,e=[];return{point:function(e,n,r){t.push([e,n,r])},lineStart:function(){e.push(t=[])},lineEnd:uw,rejoin:function(){e.length>1&&e.push(e.pop().concat(e.shift()))},result:function(){var n=e;return e=[],t=null,n}}}function Rk(t,e){return Gb(t[0]-e[0])=0;--o)i.point((c=l[o])[0],c[1]);else r(h.x,h.p.x,-1,i);h=h.p}l=(h=h.o).z,d=!d}while(!h.v);i.lineEnd()}}}function qk(t){if(e=t.length){for(var e,n,r=0,i=t[0];++r=0?1:-1,E=M*A,D=E>Pb,C=m*w;if(u.add(Xb(C*M*nw(E),y*k+C*Jb(E))),a+=D?A+M*Wb:A,D^p>=n^x>=n){var F=Pw(Lw(d),Lw(_));Ww(F);var S=Pw(o,F);Ww(S);var $=(D^A>=0?-1:1)*sw(S[2]);(r>$||r===$&&(F[0]||F[1]))&&(s+=D^A>=0?1:-1)}}return(a<-Lb||a0){for(f||(i.polygonStart(),f=!0),i.lineStart(),t=0;t1&&2&u&&h.push(h.pop().concat(h.shift())),a.push(h.filter(Ik))}return h}}function Ik(t){return t.length>1}function Wk(t,e){return((t=t.x)[0]<0?t[1]-jb-Lb:jb-t[1])-((e=e.x)[0]<0?e[1]-jb-Lb:jb-e[1])}Sk.invert=Sk;var Hk=jk((function(){return!0}),(function(t){var e,n=NaN,r=NaN,i=NaN;return{lineStart:function(){t.lineStart(),e=1},point:function(o,a){var s=o>0?Pb:-Pb,u=Gb(o-n);Gb(u-Pb)0?jb:-jb),t.point(i,r),t.lineEnd(),t.lineStart(),t.point(s,r),t.point(o,r),e=0):i!==s&&u>=Pb&&(Gb(n-i)Lb?Vb((nw(e)*(o=Jb(r))*nw(n)-nw(r)*(i=Jb(e))*nw(t))/(i*o*a)):(e+r)/2}(n,r,o,a),t.point(i,r),t.lineEnd(),t.lineStart(),t.point(s,r),e=0),t.point(n=o,r=a),i=s},lineEnd:function(){t.lineEnd(),n=r=NaN},clean:function(){return 2-e}}}),(function(t,e,n,r){var i;if(null==t)i=n*jb,r.point(-Pb,i),r.point(0,i),r.point(Pb,i),r.point(Pb,0),r.point(Pb,-i),r.point(0,-i),r.point(-Pb,-i),r.point(-Pb,0),r.point(-Pb,i);else if(Gb(t[0]-e[0])>Lb){var o=t[0]0,i=Gb(e)>Lb;function o(t,n){return Jb(t)*Jb(n)>e}function a(t,n,r){var i=[1,0,0],o=Pw(Lw(t),Lw(n)),a=qw(o,o),s=o[0],u=a-s*s;if(!u)return!r&&t;var l=e*a/u,c=-e*s/u,f=Pw(i,o),h=Iw(i,l);jw(h,Iw(o,c));var d=f,p=qw(h,d),g=qw(d,d),m=p*p-g*(qw(h,h)-1);if(!(m<0)){var y=iw(m),v=Iw(d,(-p-y)/g);if(jw(v,h),v=Uw(v),!r)return v;var _,x=t[0],b=n[0],w=t[1],k=n[1];b0^v[1]<(Gb(v[0]-x)Pb^(x<=v[0]&&v[0]<=b)){var E=Iw(d,(-p+y)/g);return jw(E,h),[v,Uw(E)]}}}function s(e,n){var i=r?t:Pb-t,o=0;return e<-i?o|=1:e>i&&(o|=2),n<-i?o|=4:n>i&&(o|=8),o}return jk(o,(function(t){var e,n,u,l,c;return{lineStart:function(){l=u=!1,c=1},point:function(f,h){var d,p=[f,h],g=o(f,h),m=r?g?0:s(f,h):g?s(f+(f<0?Pb:-Pb),h):0;if(!e&&(l=u=g)&&t.lineStart(),g!==u&&(!(d=a(e,p))||Rk(e,d)||Rk(p,d))&&(p[2]=1),g!==u)c=0,g?(t.lineStart(),d=a(p,e),t.point(d[0],d[1])):(d=a(e,p),t.point(d[0],d[1],2),t.lineEnd()),e=d;else if(i&&e&&r^g){var y;m&n||!(y=a(p,e,!0))||(c=0,r?(t.lineStart(),t.point(y[0][0],y[0][1]),t.point(y[1][0],y[1][1]),t.lineEnd()):(t.point(y[1][0],y[1][1]),t.lineEnd(),t.lineStart(),t.point(y[0][0],y[0][1],3)))}!g||e&&Rk(e,p)||t.point(p[0],p[1]),e=p,u=g,n=m},lineEnd:function(){u&&t.lineEnd(),e=null},clean:function(){return c|(l&&u)<<1}}}),(function(e,r,i,o){!function(t,e,n,r,i,o){if(n){var a=Jb(e),s=nw(e),u=r*n;null==i?(i=e+r*Wb,o=e-u/2):(i=Nk(a,i),o=Nk(a,o),(r>0?io)&&(i+=r*Wb));for(var l,c=i;r>0?c>o:c0)do{l.point(0===c||3===c?t:n,c>1?r:e)}while((c=(c+s+4)%4)!==f);else l.point(o[0],o[1])}function a(r,i){return Gb(r[0]-t)0?0:3:Gb(r[0]-n)0?2:1:Gb(r[1]-e)0?1:0:i>0?3:2}function s(t,e){return u(t.x,e.x)}function u(t,e){var n=a(t,1),r=a(e,1);return n!==r?n-r:0===n?e[1]-t[1]:1===n?t[0]-e[0]:2===n?t[1]-e[1]:e[0]-t[0]}return function(a){var u,l,c,f,h,d,p,g,m,y,v,_=a,x=Ok(),b={point:w,lineStart:function(){b.point=k,l&&l.push(c=[]);y=!0,m=!1,p=g=NaN},lineEnd:function(){u&&(k(f,h),d&&m&&x.rejoin(),u.push(x.result()));b.point=w,m&&_.lineEnd()},polygonStart:function(){_=x,u=[],l=[],v=!0},polygonEnd:function(){var e=function(){for(var e=0,n=0,i=l.length;nr&&(h-o)*(r-a)>(d-a)*(t-o)&&++e:d<=r&&(h-o)*(r-a)<(d-a)*(t-o)&&--e;return e}(),n=v&&e,i=(u=Fe(u)).length;(n||i)&&(a.polygonStart(),n&&(a.lineStart(),o(null,null,1,a),a.lineEnd()),i&&Lk(u,s,e,o,a),a.polygonEnd());_=a,u=l=c=null}};function w(t,e){i(t,e)&&_.point(t,e)}function k(o,a){var s=i(o,a);if(l&&c.push([o,a]),y)f=o,h=a,d=s,y=!1,s&&(_.lineStart(),_.point(o,a));else if(s&&m)_.point(o,a);else{var u=[p=Math.max(Vk,Math.min(Gk,p)),g=Math.max(Vk,Math.min(Gk,g))],x=[o=Math.max(Vk,Math.min(Gk,o)),a=Math.max(Vk,Math.min(Gk,a))];!function(t,e,n,r,i,o){var a,s=t[0],u=t[1],l=0,c=1,f=e[0]-s,h=e[1]-u;if(a=n-s,f||!(a>0)){if(a/=f,f<0){if(a0){if(a>c)return;a>l&&(l=a)}if(a=i-s,f||!(a<0)){if(a/=f,f<0){if(a>c)return;a>l&&(l=a)}else if(f>0){if(a0)){if(a/=h,h<0){if(a0){if(a>c)return;a>l&&(l=a)}if(a=o-u,h||!(a<0)){if(a/=h,h<0){if(a>c)return;a>l&&(l=a)}else if(h>0){if(a0&&(t[0]=s+l*f,t[1]=u+l*h),c<1&&(e[0]=s+c*f,e[1]=u+c*h),!0}}}}}(u,x,t,e,n,r)?s&&(_.lineStart(),_.point(o,a),v=!1):(m||(_.lineStart(),_.point(u[0],u[1])),_.point(x[0],x[1]),s||_.lineEnd(),v=!1)}p=o,g=a,m=s}return b}}function Jk(t,e,n){var r=Se(t,e-Lb,n).concat(e);return function(t){return r.map((function(e){return[t,e]}))}}function Zk(t,e,n){var r=Se(t,e-Lb,n).concat(e);return function(t){return r.map((function(e){return[e,t]}))}}var Qk,Kk,tA,eA,nA=t=>t,rA=new se,iA=new se,oA={point:uw,lineStart:uw,lineEnd:uw,polygonStart:function(){oA.lineStart=aA,oA.lineEnd=lA},polygonEnd:function(){oA.lineStart=oA.lineEnd=oA.point=uw,rA.add(Gb(iA)),iA=new se},result:function(){var t=rA/2;return rA=new se,t}};function aA(){oA.point=sA}function sA(t,e){oA.point=uA,Qk=tA=t,Kk=eA=e}function uA(t,e){iA.add(eA*t-tA*e),tA=t,eA=e}function lA(){uA(Qk,Kk)}var cA=1/0,fA=cA,hA=-cA,dA=hA,pA={point:function(t,e){thA&&(hA=t);edA&&(dA=e)},lineStart:uw,lineEnd:uw,polygonStart:uw,polygonEnd:uw,result:function(){var t=[[cA,fA],[hA,dA]];return hA=dA=-(fA=cA=1/0),t}};var gA,mA,yA,vA,_A=0,xA=0,bA=0,wA=0,kA=0,AA=0,MA=0,EA=0,DA=0,CA={point:FA,lineStart:SA,lineEnd:BA,polygonStart:function(){CA.lineStart=zA,CA.lineEnd=NA},polygonEnd:function(){CA.point=FA,CA.lineStart=SA,CA.lineEnd=BA},result:function(){var t=DA?[MA/DA,EA/DA]:AA?[wA/AA,kA/AA]:bA?[_A/bA,xA/bA]:[NaN,NaN];return _A=xA=bA=wA=kA=AA=MA=EA=DA=0,t}};function FA(t,e){_A+=t,xA+=e,++bA}function SA(){CA.point=$A}function $A(t,e){CA.point=TA,FA(yA=t,vA=e)}function TA(t,e){var n=t-yA,r=e-vA,i=iw(n*n+r*r);wA+=i*(yA+t)/2,kA+=i*(vA+e)/2,AA+=i,FA(yA=t,vA=e)}function BA(){CA.point=FA}function zA(){CA.point=OA}function NA(){RA(gA,mA)}function OA(t,e){CA.point=RA,FA(gA=yA=t,mA=vA=e)}function RA(t,e){var n=t-yA,r=e-vA,i=iw(n*n+r*r);wA+=i*(yA+t)/2,kA+=i*(vA+e)/2,AA+=i,MA+=(i=vA*t-yA*e)*(yA+t),EA+=i*(vA+e),DA+=3*i,FA(yA=t,vA=e)}function UA(t){this._context=t}UA.prototype={_radius:4.5,pointRadius:function(t){return this._radius=t,this},polygonStart:function(){this._line=0},polygonEnd:function(){this._line=NaN},lineStart:function(){this._point=0},lineEnd:function(){0===this._line&&this._context.closePath(),this._point=NaN},point:function(t,e){switch(this._point){case 0:this._context.moveTo(t,e),this._point=1;break;case 1:this._context.lineTo(t,e);break;default:this._context.moveTo(t+this._radius,e),this._context.arc(t,e,this._radius,0,Wb)}},result:uw};var LA,qA,PA,jA,IA,WA=new se,HA={point:uw,lineStart:function(){HA.point=YA},lineEnd:function(){LA&&GA(qA,PA),HA.point=uw},polygonStart:function(){LA=!0},polygonEnd:function(){LA=null},result:function(){var t=+WA;return WA=new se,t}};function YA(t,e){HA.point=GA,qA=jA=t,PA=IA=e}function GA(t,e){jA-=t,IA-=e,WA.add(iw(jA*jA+IA*IA)),jA=t,IA=e}let VA,XA,JA,ZA;class QA{constructor(t){this._append=null==t?KA:function(t){const e=Math.floor(t);if(!(e>=0))throw new RangeError(`invalid digits: ${t}`);if(e>15)return KA;if(e!==VA){const t=10**e;VA=e,XA=function(e){let n=1;this._+=e[0];for(const r=e.length;n=0))throw new RangeError(`invalid digits: ${t}`);i=e}return null===e&&(r=new QA(i)),a},a.projection(t).digits(i).context(e)}function eM(t){return function(e){var n=new nM;for(var r in t)n[r]=t[r];return n.stream=e,n}}function nM(){}function rM(t,e,n){var r=t.clipExtent&&t.clipExtent();return t.scale(150).translate([0,0]),null!=r&&t.clipExtent(null),pw(n,t.stream(pA)),e(pA.result()),null!=r&&t.clipExtent(r),t}function iM(t,e,n){return rM(t,(function(n){var r=e[1][0]-e[0][0],i=e[1][1]-e[0][1],o=Math.min(r/(n[1][0]-n[0][0]),i/(n[1][1]-n[0][1])),a=+e[0][0]+(r-o*(n[1][0]+n[0][0]))/2,s=+e[0][1]+(i-o*(n[1][1]+n[0][1]))/2;t.scale(150*o).translate([a,s])}),n)}function oM(t,e,n){return iM(t,[[0,0],e],n)}function aM(t,e,n){return rM(t,(function(n){var r=+e,i=r/(n[1][0]-n[0][0]),o=(r-i*(n[1][0]+n[0][0]))/2,a=-i*n[0][1];t.scale(150*i).translate([o,a])}),n)}function sM(t,e,n){return rM(t,(function(n){var r=+e,i=r/(n[1][1]-n[0][1]),o=-i*n[0][0],a=(r-i*(n[1][1]+n[0][1]))/2;t.scale(150*i).translate([o,a])}),n)}nM.prototype={constructor:nM,point:function(t,e){this.stream.point(t,e)},sphere:function(){this.stream.sphere()},lineStart:function(){this.stream.lineStart()},lineEnd:function(){this.stream.lineEnd()},polygonStart:function(){this.stream.polygonStart()},polygonEnd:function(){this.stream.polygonEnd()}};var uM=16,lM=Jb(30*Yb);function cM(t,e){return+e?function(t,e){function n(r,i,o,a,s,u,l,c,f,h,d,p,g,m){var y=l-r,v=c-i,_=y*y+v*v;if(_>4*e&&g--){var x=a+h,b=s+d,w=u+p,k=iw(x*x+b*b+w*w),A=sw(w/=k),M=Gb(Gb(w)-1)e||Gb((y*F+v*S)/_-.5)>.3||a*h+s*d+u*p2?t[2]%360*Yb:0,F()):[m*Hb,y*Hb,v*Hb]},D.angle=function(t){return arguments.length?(_=t%360*Yb,F()):_*Hb},D.reflectX=function(t){return arguments.length?(x=t?-1:1,F()):x<0},D.reflectY=function(t){return arguments.length?(b=t?-1:1,F()):b<0},D.precision=function(t){return arguments.length?(a=cM(s,E=t*t),S()):iw(E)},D.fitExtent=function(t,e){return iM(D,t,e)},D.fitSize=function(t,e){return oM(D,t,e)},D.fitWidth=function(t,e){return aM(D,t,e)},D.fitHeight=function(t,e){return sM(D,t,e)},function(){return e=t.apply(this,arguments),D.invert=e.invert&&C,F()}}function gM(t){var e=0,n=Pb/3,r=pM(t),i=r(e,n);return i.parallels=function(t){return arguments.length?r(e=t[0]*Yb,n=t[1]*Yb):[e*Hb,n*Hb]},i}function mM(t,e){var n=nw(t),r=(n+nw(e))/2;if(Gb(r)2?t[2]*Yb:0),e.invert=function(e){return(e=t.invert(e[0]*Yb,e[1]*Yb))[0]*=Hb,e[1]*=Hb,e},e}(i.rotate()).invert([0,0]));return u(null==l?[[s[0]-o,s[1]-o],[s[0]+o,s[1]+o]]:t===kM?[[Math.max(s[0]-o,l),e],[Math.min(s[0]+o,n),r]]:[[l,Math.max(s[1]-o,e)],[n,Math.min(s[1]+o,r)]])}return i.scale=function(t){return arguments.length?(a(t),c()):a()},i.translate=function(t){return arguments.length?(s(t),c()):s()},i.center=function(t){return arguments.length?(o(t),c()):o()},i.clipExtent=function(t){return arguments.length?(null==t?l=e=n=r=null:(l=+t[0][0],e=+t[0][1],n=+t[1][0],r=+t[1][1]),c()):null==l?null:[[l,e],[n,r]]},c()}function MM(t){return ow((jb+t)/2)}function EM(t,e){var n=Jb(t),r=t===e?nw(t):tw(n/Jb(e))/tw(MM(e)/MM(t)),i=n*ew(MM(t),r)/r;if(!r)return kM;function o(t,e){i>0?e<-jb+Lb&&(e=-jb+Lb):e>jb-Lb&&(e=jb-Lb);var n=i/ew(MM(e),r);return[n*nw(r*t),i-n*Jb(r*t)]}return o.invert=function(t,e){var n=i-e,o=rw(r)*iw(t*t+n*n),a=Xb(t,Gb(n))*rw(n);return n*r<0&&(a-=Pb*rw(t)*rw(n)),[a/r,2*Vb(ew(i/o,1/r))-jb]},o}function DM(t,e){return[t,e]}function CM(t,e){var n=Jb(t),r=t===e?nw(t):(n-Jb(e))/(e-t),i=n/r+t;if(Gb(r)Lb&&--i>0);return[t/(.8707+(o=r*r)*(o*(o*o*o*(.003971-.001529*o)-.013791)-.131979)),r]},RM.invert=xM(sw),UM.invert=xM((function(t){return 2*Vb(t)})),LM.invert=function(t,e){return[-e,2*Vb(Qb(t))-jb]};var qM=Math.abs,PM=Math.cos,jM=Math.sin,IM=1e-6,WM=Math.PI,HM=WM/2,YM=function(t){return t>0?Math.sqrt(t):0}(2);function GM(t){return t>1?HM:t<-1?-HM:Math.asin(t)}function VM(t,e){var n,r=t*jM(e),i=30;do{e-=n=(e+jM(e)-r)/(1+PM(e))}while(qM(n)>IM&&--i>0);return e/2}var XM=function(t,e,n){function r(r,i){return[t*r*PM(i=VM(n,i)),e*jM(i)]}return r.invert=function(r,i){return i=GM(i/e),[r/(t*PM(i)),GM((2*i+jM(2*i))/n)]},r}(YM/HM,YM,WM);const JM=tM(),ZM=[\"clipAngle\",\"clipExtent\",\"scale\",\"translate\",\"center\",\"rotate\",\"parallels\",\"precision\",\"reflectX\",\"reflectY\",\"coefficient\",\"distance\",\"fraction\",\"lobes\",\"parallel\",\"radius\",\"ratio\",\"spacing\",\"tilt\"];function QM(t,e){if(!t||\"string\"!=typeof t)throw new Error(\"Projection type must be a name string.\");return t=t.toLowerCase(),arguments.length>1?(tE[t]=function(t,e){return function n(){const r=e();return r.type=t,r.path=tM().projection(r),r.copy=r.copy||function(){const t=n();return ZM.forEach((e=>{r[e]&&t[e](r[e]())})),t.path.pointRadius(r.path.pointRadius()),t},op(r)}}(t,e),this):tE[t]||null}function KM(t){return t&&t.path||JM}const tE={albers:vM,albersusa:function(){var t,e,n,r,i,o,a=vM(),s=yM().rotate([154,0]).center([-2,58.5]).parallels([55,65]),u=yM().rotate([157,0]).center([-3,19.9]).parallels([8,18]),l={point:function(t,e){o=[t,e]}};function c(t){var e=t[0],a=t[1];return o=null,n.point(e,a),o||(r.point(e,a),o)||(i.point(e,a),o)}function f(){return t=e=null,c}return c.invert=function(t){var e=a.scale(),n=a.translate(),r=(t[0]-n[0])/e,i=(t[1]-n[1])/e;return(i>=.12&&i<.234&&r>=-.425&&r<-.214?s:i>=.166&&i<.234&&r>=-.214&&r<-.115?u:a).invert(t)},c.stream=function(n){return t&&e===n?t:(r=[a.stream(e=n),s.stream(n),u.stream(n)],i=r.length,t={point:function(t,e){for(var n=-1;++n2?t[2]+90:90]):[(t=n())[0],t[1],t[2]-90]},n([0,0,90]).scale(159.155)}};for(const t in tE)QM(t,tE[t]);function eE(){}const nE=[[],[[[1,1.5],[.5,1]]],[[[1.5,1],[1,1.5]]],[[[1.5,1],[.5,1]]],[[[1,.5],[1.5,1]]],[[[1,1.5],[.5,1]],[[1,.5],[1.5,1]]],[[[1,.5],[1,1.5]]],[[[1,.5],[.5,1]]],[[[.5,1],[1,.5]]],[[[1,1.5],[1,.5]]],[[[.5,1],[1,.5]],[[1.5,1],[1,1.5]]],[[[1.5,1],[1,.5]]],[[[.5,1],[1.5,1]]],[[[1,1.5],[1.5,1]]],[[[.5,1],[1,1.5]]],[]];function rE(){var t=1,e=1,n=a;function r(t,e){return e.map((e=>i(t,e)))}function i(r,i){var a=[],s=[];return function(n,r,i){var a,s,u,l,c,f,h=[],d=[];a=s=-1,l=n[0]>=r,nE[l<<1].forEach(p);for(;++a=r,nE[u|l<<1].forEach(p);nE[l<<0].forEach(p);for(;++s=r,c=n[s*t]>=r,nE[l<<1|c<<2].forEach(p);++a=r,f=c,c=n[s*t+a+1]>=r,nE[u|l<<1|c<<2|f<<3].forEach(p);nE[l|c<<3].forEach(p)}a=-1,c=n[s*t]>=r,nE[c<<2].forEach(p);for(;++a=r,nE[c<<2|f<<3].forEach(p);function p(t){var e,n,r=[t[0][0]+a,t[0][1]+s],u=[t[1][0]+a,t[1][1]+s],l=o(r),c=o(u);(e=d[l])?(n=h[c])?(delete d[e.end],delete h[n.start],e===n?(e.ring.push(u),i(e.ring)):h[e.start]=d[n.end]={start:e.start,end:n.end,ring:e.ring.concat(n.ring)}):(delete d[e.end],e.ring.push(u),d[e.end=c]=e):(e=h[c])?(n=d[l])?(delete h[e.start],delete d[n.end],e===n?(e.ring.push(u),i(e.ring)):h[n.start]=d[e.end]={start:n.start,end:e.end,ring:n.ring.concat(e.ring)}):(delete h[e.start],e.ring.unshift(r),h[e.start=l]=e):h[l]=d[c]={start:l,end:c,ring:[r,u]}}nE[c<<3].forEach(p)}(r,i,(t=>{n(t,r,i),function(t){var e=0,n=t.length,r=t[n-1][1]*t[0][0]-t[n-1][0]*t[0][1];for(;++e0?a.push([t]):s.push(t)})),s.forEach((t=>{for(var e,n=0,r=a.length;n{var o,a=n[0],s=n[1],u=0|a,l=0|s,c=r[l*t+u];a>0&&a0&&s=0&&o>=0||s(\"invalid size\"),t=i,e=o,r},r.smooth=function(t){return arguments.length?(n=t?a:eE,r):n===a},r}function iE(t,e){for(var n,r=-1,i=e.length;++rr!=d>r&&n<(h-l)*(r-c)/(d-c)+l&&(i=-i)}return i}function aE(t,e,n){var r,i,o,a;return function(t,e,n){return(e[0]-t[0])*(n[1]-t[1])==(n[0]-t[0])*(e[1]-t[1])}(t,e,n)&&(i=t[r=+(t[0]===e[0])],o=n[r],a=e[r],i<=o&&o<=a||a<=o&&o<=i)}function sE(t,e,n){return function(r){var i=at(r),o=n?Math.min(i[0],0):i[0],a=i[1],s=a-o,u=e?be(o,a,t):s/(t+1);return Se(o+u,a,u)}}function uE(t){Ja.call(this,null,t)}function lE(t,e,n,r,i){const o=t.x1||0,a=t.y1||0,s=e*n<0;function u(t){t.forEach(l)}function l(t){s&&t.reverse(),t.forEach(c)}function c(t){t[0]=(t[0]-o)*e+r,t[1]=(t[1]-a)*n+i}return function(t){return t.coordinates.forEach(u),t}}function cE(t,e,n){const r=t>=0?t:rs(e,n);return Math.round((Math.sqrt(4*r*r+1)-1)/2)}function fE(t){return J(t)?t:rt(+t)}function hE(){var t=t=>t[0],e=t=>t[1],n=d,r=[-1,-1],i=960,o=500,a=2;function u(s,u){const l=cE(r[0],s,t)>>a,c=cE(r[1],s,e)>>a,f=l?l+2:0,h=c?c+2:0,d=2*f+(i>>a),p=2*h+(o>>a),g=new Float32Array(d*p),m=new Float32Array(d*p);let y=g;s.forEach((r=>{const i=f+(+t(r)>>a),o=h+(+e(r)>>a);i>=0&&i=0&&o0&&c>0?(dE(d,p,g,m,l),pE(d,p,m,g,c),dE(d,p,g,m,l),pE(d,p,m,g,c),dE(d,p,g,m,l),pE(d,p,m,g,c)):l>0?(dE(d,p,g,m,l),dE(d,p,m,g,l),dE(d,p,g,m,l),y=m):c>0&&(pE(d,p,g,m,c),pE(d,p,m,g,c),pE(d,p,g,m,c),y=m);const v=u?Math.pow(2,-2*a):1/$e(y);for(let t=0,e=d*p;t>a),y2:h+(o>>a)}}return u.x=function(e){return arguments.length?(t=fE(e),u):t},u.y=function(t){return arguments.length?(e=fE(t),u):e},u.weight=function(t){return arguments.length?(n=fE(t),u):n},u.size=function(t){if(!arguments.length)return[i,o];var e=+t[0],n=+t[1];return e>=0&&n>=0||s(\"invalid size\"),i=e,o=n,u},u.cellSize=function(t){return arguments.length?((t=+t)>=1||s(\"invalid cell size\"),a=Math.floor(Math.log(t)/Math.LN2),u):1<=i&&(e>=o&&(s-=n[e-o+a*t]),r[e-i+a*t]=s/Math.min(e+1,t-1+o-e,o))}function pE(t,e,n,r,i){const o=1+(i<<1);for(let a=0;a=i&&(s>=o&&(u-=n[a+(s-o)*t]),r[a+(s-i)*t]=u/Math.min(s+1,e-1+o-s,o))}function gE(t){Ja.call(this,null,t)}uE.Definition={type:\"Isocontour\",metadata:{generates:!0},params:[{name:\"field\",type:\"field\"},{name:\"thresholds\",type:\"number\",array:!0},{name:\"levels\",type:\"number\"},{name:\"nice\",type:\"boolean\",default:!1},{name:\"resolve\",type:\"enum\",values:[\"shared\",\"independent\"],default:\"independent\"},{name:\"zero\",type:\"boolean\",default:!0},{name:\"smooth\",type:\"boolean\",default:!0},{name:\"scale\",type:\"number\",expr:!0},{name:\"translate\",type:\"number\",array:!0,expr:!0},{name:\"as\",type:\"string\",null:!0,default:\"contour\"}]},dt(uE,Ja,{transform(t,e){if(this.value&&!e.changed()&&!t.modified())return e.StopPropagation;var n=e.fork(e.NO_SOURCE|e.NO_FIELDS),r=e.materialize(e.SOURCE).source,i=t.field||f,o=rE().smooth(!1!==t.smooth),a=t.thresholds||function(t,e,n){const r=sE(n.levels||10,n.nice,!1!==n.zero);return\"shared\"!==n.resolve?r:r(t.map((t=>we(e(t).values))))}(r,i,t),s=null===t.as?null:t.as||\"contour\",u=[];return r.forEach((e=>{const n=i(e),r=o.size([n.width,n.height])(n.values,k(a)?a:a(n.values));!function(t,e,n,r){let i=r.scale||e.scale,o=r.translate||e.translate;J(i)&&(i=i(n,r));J(o)&&(o=o(n,r));if((1===i||null==i)&&!o)return;const a=(vt(i)?i:i[0])||1,s=(vt(i)?i:i[1])||1,u=o&&o[0]||0,l=o&&o[1]||0;t.forEach(lE(e,a,s,u,l))}(r,n,e,t),r.forEach((t=>{u.push(ba(e,_a(null!=s?{[s]:t}:t)))}))})),this.value&&(n.rem=this.value),this.value=n.source=n.add=u,n}}),gE.Definition={type:\"KDE2D\",metadata:{generates:!0},params:[{name:\"size\",type:\"number\",array:!0,length:2,required:!0},{name:\"x\",type:\"field\",required:!0},{name:\"y\",type:\"field\",required:!0},{name:\"weight\",type:\"field\"},{name:\"groupby\",type:\"field\",array:!0},{name:\"cellSize\",type:\"number\"},{name:\"bandwidth\",type:\"number\",array:!0,length:2},{name:\"counts\",type:\"boolean\",default:!1},{name:\"as\",type:\"string\",default:\"grid\"}]};const mE=[\"x\",\"y\",\"weight\",\"size\",\"cellSize\",\"bandwidth\"];function yE(t,e){return mE.forEach((n=>null!=e[n]?t[n](e[n]):0)),t}function vE(t){Ja.call(this,null,t)}dt(gE,Ja,{transform(t,e){if(this.value&&!e.changed()&&!t.modified())return e.StopPropagation;var r,i=e.fork(e.NO_SOURCE|e.NO_FIELDS),o=function(t,e){var n,r,i,o,a,s,u=[],l=t=>t(o);if(null==e)u.push(t);else for(n={},r=0,i=t.length;r_a(function(t,e){for(let n=0;nLb})).map(u)).concat(Se(Zb(o/d)*d,i,d).filter((function(t){return Gb(t%g)>Lb})).map(l))}return y.lines=function(){return v().map((function(t){return{type:\"LineString\",coordinates:t}}))},y.outline=function(){return{type:\"Polygon\",coordinates:[c(r).concat(f(a).slice(1),c(n).reverse().slice(1),f(s).reverse().slice(1))]}},y.extent=function(t){return arguments.length?y.extentMajor(t).extentMinor(t):y.extentMinor()},y.extentMajor=function(t){return arguments.length?(r=+t[0][0],n=+t[1][0],s=+t[0][1],a=+t[1][1],r>n&&(t=r,r=n,n=t),s>a&&(t=s,s=a,a=t),y.precision(m)):[[r,s],[n,a]]},y.extentMinor=function(n){return arguments.length?(e=+n[0][0],t=+n[1][0],o=+n[0][1],i=+n[1][1],e>t&&(n=e,e=t,t=n),o>i&&(n=o,o=i,i=n),y.precision(m)):[[e,o],[t,i]]},y.step=function(t){return arguments.length?y.stepMajor(t).stepMinor(t):y.stepMinor()},y.stepMajor=function(t){return arguments.length?(p=+t[0],g=+t[1],y):[p,g]},y.stepMinor=function(t){return arguments.length?(h=+t[0],d=+t[1],y):[h,d]},y.precision=function(h){return arguments.length?(m=+h,u=Jk(o,i,90),l=Zk(e,t,m),c=Jk(s,a,90),f=Zk(r,n,m),y):m},y.extentMajor([[-180,-90+Lb],[180,90-Lb]]).extentMinor([[-180,-80-Lb],[180,80+Lb]])}()}function EE(t){Ja.call(this,null,t)}function DE(t){if(!J(t))return!1;const e=Bt(r(t));return e.$x||e.$y||e.$value||e.$max}function CE(t){Ja.call(this,null,t),this.modified(!0)}function FE(t,e,n){J(t[e])&&t[e](n)}bE.Definition={type:\"GeoJSON\",metadata:{},params:[{name:\"fields\",type:\"field\",array:!0,length:2},{name:\"geojson\",type:\"field\"}]},dt(bE,Ja,{transform(t,e){var n,i=this._features,o=this._points,a=t.fields,s=a&&a[0],u=a&&a[1],l=t.geojson||!a&&f,c=e.ADD;n=t.modified()||e.changed(e.REM)||e.modified(r(l))||s&&e.modified(r(s))||u&&e.modified(r(u)),this.value&&!n||(c=e.SOURCE,this._features=i=[],this._points=o=[]),l&&e.visit(c,(t=>i.push(l(t)))),s&&u&&(e.visit(c,(t=>{var e=s(t),n=u(t);null!=e&&null!=n&&(e=+e)===e&&(n=+n)===n&&o.push([e,n])})),i=i.concat({type:_E,geometry:{type:\"MultiPoint\",coordinates:o}})),this.value={type:xE,features:i}}}),wE.Definition={type:\"GeoPath\",metadata:{modifies:!0},params:[{name:\"projection\",type:\"projection\"},{name:\"field\",type:\"field\"},{name:\"pointRadius\",type:\"number\",expr:!0},{name:\"as\",type:\"string\",default:\"path\"}]},dt(wE,Ja,{transform(t,e){var n=e.fork(e.ALL),r=this.value,i=t.field||f,o=t.as||\"path\",a=n.SOURCE;!r||t.modified()?(this.value=r=KM(t.projection),n.materialize().reflow()):a=i===f||e.modified(i.fields)?n.ADD_MOD:n.ADD;const s=function(t,e){const n=t.pointRadius();t.context(null),null!=e&&t.pointRadius(e);return n}(r,t.pointRadius);return n.visit(a,(t=>t[o]=r(i(t)))),r.pointRadius(s),n.modifies(o)}}),kE.Definition={type:\"GeoPoint\",metadata:{modifies:!0},params:[{name:\"projection\",type:\"projection\",required:!0},{name:\"fields\",type:\"field\",array:!0,required:!0,length:2},{name:\"as\",type:\"string\",array:!0,length:2,default:[\"x\",\"y\"]}]},dt(kE,Ja,{transform(t,e){var n,r=t.projection,i=t.fields[0],o=t.fields[1],a=t.as||[\"x\",\"y\"],s=a[0],u=a[1];function l(t){const e=r([i(t),o(t)]);e?(t[s]=e[0],t[u]=e[1]):(t[s]=void 0,t[u]=void 0)}return t.modified()?e=e.materialize().reflow(!0).visit(e.SOURCE,l):(n=e.modified(i.fields)||e.modified(o.fields),e.visit(n?e.ADD_MOD:e.ADD,l)),e.modifies(a)}}),AE.Definition={type:\"GeoShape\",metadata:{modifies:!0,nomod:!0},params:[{name:\"projection\",type:\"projection\"},{name:\"field\",type:\"field\",default:\"datum\"},{name:\"pointRadius\",type:\"number\",expr:!0},{name:\"as\",type:\"string\",default:\"shape\"}]},dt(AE,Ja,{transform(t,e){var n=e.fork(e.ALL),r=this.value,i=t.as||\"shape\",o=n.ADD;return r&&!t.modified()||(this.value=r=function(t,e,n){const r=null==n?n=>t(e(n)):r=>{var i=t.pointRadius(),o=t.pointRadius(n)(e(r));return t.pointRadius(i),o};return r.context=e=>(t.context(e),r),r}(KM(t.projection),t.field||l(\"datum\"),t.pointRadius),n.materialize().reflow(),o=n.SOURCE),n.visit(o,(t=>t[i]=r)),n.modifies(i)}}),ME.Definition={type:\"Graticule\",metadata:{changes:!0,generates:!0},params:[{name:\"extent\",type:\"array\",array:!0,length:2,content:{type:\"number\",array:!0,length:2}},{name:\"extentMajor\",type:\"array\",array:!0,length:2,content:{type:\"number\",array:!0,length:2}},{name:\"extentMinor\",type:\"array\",array:!0,length:2,content:{type:\"number\",array:!0,length:2}},{name:\"step\",type:\"number\",array:!0,length:2},{name:\"stepMajor\",type:\"number\",array:!0,length:2,default:[90,360]},{name:\"stepMinor\",type:\"number\",array:!0,length:2,default:[10,10]},{name:\"precision\",type:\"number\",default:2.5}]},dt(ME,Ja,{transform(t,e){var n,r=this.value,i=this.generator;if(!r.length||t.modified())for(const e in t)J(i[e])&&i[e](t[e]);return n=i(),r.length?e.mod.push(wa(r[0],n)):e.add.push(_a(n)),r[0]=n,e}}),EE.Definition={type:\"heatmap\",metadata:{modifies:!0},params:[{name:\"field\",type:\"field\"},{name:\"color\",type:\"string\",expr:!0},{name:\"opacity\",type:\"number\",expr:!0},{name:\"resolve\",type:\"enum\",values:[\"shared\",\"independent\"],default:\"independent\"},{name:\"as\",type:\"string\",default:\"image\"}]},dt(EE,Ja,{transform(t,e){if(!e.changed()&&!t.modified())return e.StopPropagation;var n=e.materialize(e.SOURCE).source,r=\"shared\"===t.resolve,i=t.field||f,o=function(t,e){let n;J(t)?(n=n=>t(n,e),n.dep=DE(t)):t?n=rt(t):(n=t=>t.$value/t.$max||0,n.dep=!0);return n}(t.opacity,t),a=function(t,e){let n;J(t)?(n=n=>af(t(n,e)),n.dep=DE(t)):n=rt(af(t||\"#888\"));return n}(t.color,t),s=t.as||\"image\",u={$x:0,$y:0,$value:0,$max:r?we(n.map((t=>we(i(t).values)))):0};return n.forEach((t=>{const e=i(t),n=ot({},t,u);r||(n.$max=we(e.values||[])),t[s]=function(t,e,n,r){const i=t.width,o=t.height,a=t.x1||0,s=t.y1||0,u=t.x2||i,l=t.y2||o,c=t.values,f=c?t=>c[t]:h,d=$c(u-a,l-s),p=d.getContext(\"2d\"),g=p.getImageData(0,0,u-a,l-s),m=g.data;for(let t=s,o=0;t{null!=t[e]&&FE(n,e,t[e])}))):ZM.forEach((e=>{t.modified(e)&&FE(n,e,t[e])})),null!=t.pointRadius&&n.path.pointRadius(t.pointRadius),t.fit&&function(t,e){const n=function(t){return t=V(t),1===t.length?t[0]:{type:xE,features:t.reduce(((t,e)=>t.concat(function(t){return t.type===xE?t.features:V(t).filter((t=>null!=t)).map((t=>t.type===_E?t:{type:_E,geometry:t}))}(e))),[])}}(e.fit);e.extent?t.fitExtent(e.extent,n):e.size&&t.fitSize(e.size,n)}(n,t),e.fork(e.NO_SOURCE|e.NO_FIELDS)}});var SE=Object.freeze({__proto__:null,contour:vE,geojson:bE,geopath:wE,geopoint:kE,geoshape:AE,graticule:ME,heatmap:EE,isocontour:uE,kde2d:gE,projection:CE});function $E(t,e,n,r){if(isNaN(e)||isNaN(n))return t;var i,o,a,s,u,l,c,f,h,d=t._root,p={data:r},g=t._x0,m=t._y0,y=t._x1,v=t._y1;if(!d)return t._root=p,t;for(;d.length;)if((l=e>=(o=(g+y)/2))?g=o:y=o,(c=n>=(a=(m+v)/2))?m=a:v=a,i=d,!(d=d[f=c<<1|l]))return i[f]=p,t;if(s=+t._x.call(null,d.data),u=+t._y.call(null,d.data),e===s&&n===u)return p.next=d,i?i[f]=p:t._root=p,t;do{i=i?i[f]=new Array(4):t._root=new Array(4),(l=e>=(o=(g+y)/2))?g=o:y=o,(c=n>=(a=(m+v)/2))?m=a:v=a}while((f=c<<1|l)==(h=(u>=a)<<1|s>=o));return i[h]=d,i[f]=p,t}function TE(t,e,n,r,i){this.node=t,this.x0=e,this.y0=n,this.x1=r,this.y1=i}function BE(t){return t[0]}function zE(t){return t[1]}function NE(t,e,n){var r=new OE(null==e?BE:e,null==n?zE:n,NaN,NaN,NaN,NaN);return null==t?r:r.addAll(t)}function OE(t,e,n,r,i,o){this._x=t,this._y=e,this._x0=n,this._y0=r,this._x1=i,this._y1=o,this._root=void 0}function RE(t){for(var e={data:t.data},n=e;t=t.next;)n=n.next={data:t.data};return e}var UE=NE.prototype=OE.prototype;function LE(t){return function(){return t}}function qE(t){return 1e-6*(t()-.5)}function PE(t){return t.x+t.vx}function jE(t){return t.y+t.vy}function IE(t){return t.index}function WE(t,e){var n=t.get(e);if(!n)throw new Error(\"node not found: \"+e);return n}UE.copy=function(){var t,e,n=new OE(this._x,this._y,this._x0,this._y0,this._x1,this._y1),r=this._root;if(!r)return n;if(!r.length)return n._root=RE(r),n;for(t=[{source:r,target:n._root=new Array(4)}];r=t.pop();)for(var i=0;i<4;++i)(e=r.source[i])&&(e.length?t.push({source:e,target:r.target[i]=new Array(4)}):r.target[i]=RE(e));return n},UE.add=function(t){const e=+this._x.call(null,t),n=+this._y.call(null,t);return $E(this.cover(e,n),e,n,t)},UE.addAll=function(t){var e,n,r,i,o=t.length,a=new Array(o),s=new Array(o),u=1/0,l=1/0,c=-1/0,f=-1/0;for(n=0;nc&&(c=r),if&&(f=i));if(u>c||l>f)return this;for(this.cover(u,l).cover(c,f),n=0;nt||t>=i||r>e||e>=o;)switch(s=(eh||(o=u.y0)>d||(a=u.x1)=y)<<1|t>=m)&&(u=p[p.length-1],p[p.length-1]=p[p.length-1-l],p[p.length-1-l]=u)}else{var v=t-+this._x.call(null,g.data),_=e-+this._y.call(null,g.data),x=v*v+_*_;if(x=(s=(p+m)/2))?p=s:m=s,(c=a>=(u=(g+y)/2))?g=u:y=u,e=d,!(d=d[f=c<<1|l]))return this;if(!d.length)break;(e[f+1&3]||e[f+2&3]||e[f+3&3])&&(n=e,h=f)}for(;d.data!==t;)if(r=d,!(d=d.next))return this;return(i=d.next)&&delete d.next,r?(i?r.next=i:delete r.next,this):e?(i?e[f]=i:delete e[f],(d=e[0]||e[1]||e[2]||e[3])&&d===(e[3]||e[2]||e[1]||e[0])&&!d.length&&(n?n[h]=d:this._root=d),this):(this._root=i,this)},UE.removeAll=function(t){for(var e=0,n=t.length;e{}};function YE(){for(var t,e=0,n=arguments.length,r={};e=0&&(e=t.slice(n+1),t=t.slice(0,n)),t&&!r.hasOwnProperty(t))throw new Error(\"unknown type: \"+t);return{type:t,name:e}}))),a=-1,s=o.length;if(!(arguments.length<2)){if(null!=e&&\"function\"!=typeof e)throw new Error(\"invalid callback: \"+e);for(;++a0)for(var n,r,i=new Array(n),o=0;o=0&&e._call.call(void 0,t),e=e._next;--QE}()}finally{QE=0,function(){var t,e,n=JE,r=1/0;for(;n;)n._call?(r>n._time&&(r=n._time),t=n,n=n._next):(e=n._next,n._next=null,n=t?t._next=e:JE=e);ZE=t,dD(r)}(),rD=0}}function hD(){var t=oD.now(),e=t-nD;e>eD&&(iD-=e,nD=t)}function dD(t){QE||(KE&&(KE=clearTimeout(KE)),t-rD>24?(t<1/0&&(KE=setTimeout(fD,t-oD.now()-iD)),tD&&(tD=clearInterval(tD))):(tD||(nD=oD.now(),tD=setInterval(hD,eD)),QE=1,aD(fD)))}lD.prototype=cD.prototype={constructor:lD,restart:function(t,e,n){if(\"function\"!=typeof t)throw new TypeError(\"callback is not a function\");n=(null==n?sD():+n)+(null==e?0:+e),this._next||ZE===this||(ZE?ZE._next=this:JE=this,ZE=this),this._call=t,this._time=n,dD()},stop:function(){this._call&&(this._call=null,this._time=1/0,dD())}};const pD=1664525,gD=1013904223,mD=4294967296;function yD(t){return t.x}function vD(t){return t.y}var _D=10,xD=Math.PI*(3-Math.sqrt(5));function bD(t){var e,n=1,r=.001,i=1-Math.pow(r,1/300),o=0,a=.6,s=new Map,u=cD(f),l=YE(\"tick\",\"end\"),c=function(){let t=1;return()=>(t=(pD*t+gD)%mD)/mD}();function f(){h(),l.call(\"tick\",e),n1?(null==n?s.delete(t):s.set(t,p(n)),e):s.get(t)},find:function(e,n,r){var i,o,a,s,u,l=0,c=t.length;for(null==r?r=1/0:r*=r,l=0;l1?(l.on(t,n),e):l.on(t)}}}const wD={center:function(t,e){var n,r=1;function i(){var i,o,a=n.length,s=0,u=0;for(i=0;il+p||oc+p||au.index){var g=l-s.x-s.vx,m=c-s.y-s.vy,y=g*g+m*m;yt.r&&(t.r=t[e].r)}function u(){if(e){var r,i,o=e.length;for(n=new Array(o),r=0;r=s)){(t.data!==e||t.next)&&(0===f&&(p+=(f=qE(n))*f),0===h&&(p+=(h=qE(n))*h),p[s(t,e,r),t])));for(a=0,i=new Array(l);ae(t,n):e)}DD.Definition={type:\"Force\",metadata:{modifies:!0},params:[{name:\"static\",type:\"boolean\",default:!1},{name:\"restart\",type:\"boolean\",default:!1},{name:\"iterations\",type:\"number\",default:300},{name:\"alpha\",type:\"number\",default:1},{name:\"alphaMin\",type:\"number\",default:.001},{name:\"alphaTarget\",type:\"number\",default:0},{name:\"velocityDecay\",type:\"number\",default:.4},{name:\"forces\",type:\"param\",array:!0,params:[{key:{force:\"center\"},params:[{name:\"x\",type:\"number\",default:0},{name:\"y\",type:\"number\",default:0}]},{key:{force:\"collide\"},params:[{name:\"radius\",type:\"number\",expr:!0},{name:\"strength\",type:\"number\",default:.7},{name:\"iterations\",type:\"number\",default:1}]},{key:{force:\"nbody\"},params:[{name:\"strength\",type:\"number\",default:-30,expr:!0},{name:\"theta\",type:\"number\",default:.9},{name:\"distanceMin\",type:\"number\",default:1},{name:\"distanceMax\",type:\"number\"}]},{key:{force:\"link\"},params:[{name:\"links\",type:\"data\"},{name:\"id\",type:\"field\"},{name:\"distance\",type:\"number\",default:30,expr:!0},{name:\"strength\",type:\"number\",expr:!0},{name:\"iterations\",type:\"number\",default:1}]},{key:{force:\"x\"},params:[{name:\"strength\",type:\"number\",default:.1},{name:\"x\",type:\"field\"}]},{key:{force:\"y\"},params:[{name:\"strength\",type:\"number\",default:.1},{name:\"y\",type:\"field\"}]}]},{name:\"as\",type:\"string\",array:!0,modify:!1,default:ED}]},dt(DD,Ja,{transform(t,e){var n,r,i=this.value,o=e.changed(e.ADD_REM),a=t.modified(AD),s=t.iterations||300;if(i?(o&&(e.modifies(\"index\"),i.nodes(e.source)),(a||e.changed(e.MOD))&&CD(i,t,0,e)):(this.value=i=function(t,e){const n=bD(t),r=n.stop,i=n.restart;let o=!1;return n.stopped=()=>o,n.restart=()=>(o=!1,i()),n.stop=()=>(o=!0,r()),CD(n,e,!0).on(\"end\",(()=>o=!0))}(e.source,t),i.on(\"tick\",(n=e.dataflow,r=this,()=>n.touch(r).run())),t.static||(o=!0,i.tick()),e.modifies(\"index\")),a||o||t.modified(MD)||e.changed()&&t.restart)if(i.alpha(Math.max(i.alpha(),t.alpha||1)).alphaDecay(1-Math.pow(i.alphaMin(),1/s)),t.static)for(i.stop();--s>=0;)i.tick();else if(i.stopped()&&i.restart(),!o)return e.StopPropagation;return this.finish(t,e)},finish(t,e){const n=e.dataflow;for(let t,e=this._argops,s=0,u=e.length;s=0;)e+=n[r].value;else e=1;t.value=e}function RD(t,e){t instanceof Map?(t=[void 0,t],void 0===e&&(e=LD)):void 0===e&&(e=UD);for(var n,r,i,o,a,s=new jD(t),u=[s];n=u.pop();)if((i=e(n.data))&&(a=(i=Array.from(i)).length))for(n.children=i,o=a-1;o>=0;--o)u.push(r=i[o]=new jD(i[o])),r.parent=n,r.depth=n.depth+1;return s.eachBefore(PD)}function UD(t){return t.children}function LD(t){return Array.isArray(t)?t[1]:null}function qD(t){void 0!==t.data.value&&(t.value=t.data.value),t.data=t.data.data}function PD(t){var e=0;do{t.height=e}while((t=t.parent)&&t.height<++e)}function jD(t){this.data=t,this.depth=this.height=0,this.parent=null}function ID(t){return null==t?null:WD(t)}function WD(t){if(\"function\"!=typeof t)throw new Error;return t}function HD(){return 0}function YD(t){return function(){return t}}jD.prototype=RD.prototype={constructor:jD,count:function(){return this.eachAfter(OD)},each:function(t,e){let n=-1;for(const r of this)t.call(e,r,++n,this);return this},eachAfter:function(t,e){for(var n,r,i,o=this,a=[o],s=[],u=-1;o=a.pop();)if(s.push(o),n=o.children)for(r=0,i=n.length;r=0;--r)o.push(n[r]);return this},find:function(t,e){let n=-1;for(const r of this)if(t.call(e,r,++n,this))return r},sum:function(t){return this.eachAfter((function(e){for(var n=+t(e.data)||0,r=e.children,i=r&&r.length;--i>=0;)n+=r[i].value;e.value=n}))},sort:function(t){return this.eachBefore((function(e){e.children&&e.children.sort(t)}))},path:function(t){for(var e=this,n=function(t,e){if(t===e)return t;var n=t.ancestors(),r=e.ancestors(),i=null;t=n.pop(),e=r.pop();for(;t===e;)i=t,t=n.pop(),e=r.pop();return i}(e,t),r=[e];e!==n;)e=e.parent,r.push(e);for(var i=r.length;t!==n;)r.splice(i,0,t),t=t.parent;return r},ancestors:function(){for(var t=this,e=[t];t=t.parent;)e.push(t);return e},descendants:function(){return Array.from(this)},leaves:function(){var t=[];return this.eachBefore((function(e){e.children||t.push(e)})),t},links:function(){var t=this,e=[];return t.each((function(n){n!==t&&e.push({source:n.parent,target:n})})),e},copy:function(){return RD(this).eachBefore(qD)},[Symbol.iterator]:function*(){var t,e,n,r,i=this,o=[i];do{for(t=o.reverse(),o=[];i=t.pop();)if(yield i,e=i.children)for(n=0,r=e.length;n0&&n*n>r*r+i*i}function KD(t,e){for(var n=0;n1e-6?(D+Math.sqrt(D*D-4*E*C))/(2*E):C/D);return{x:r+w+k*F,y:i+A+M*F,r:F}}function rC(t,e,n){var r,i,o,a,s=t.x-e.x,u=t.y-e.y,l=s*s+u*u;l?(i=e.r+n.r,i*=i,a=t.r+n.r,i>(a*=a)?(r=(l+a-i)/(2*l),o=Math.sqrt(Math.max(0,a/l-r*r)),n.x=t.x-r*s-o*u,n.y=t.y-r*u+o*s):(r=(l+i-a)/(2*l),o=Math.sqrt(Math.max(0,i/l-r*r)),n.x=e.x+r*s-o*u,n.y=e.y+r*u+o*s)):(n.x=e.x+n.r,n.y=e.y)}function iC(t,e){var n=t.r+e.r-1e-6,r=e.x-t.x,i=e.y-t.y;return n>0&&n*n>r*r+i*i}function oC(t){var e=t._,n=t.next._,r=e.r+n.r,i=(e.x*n.r+n.x*e.r)/r,o=(e.y*n.r+n.y*e.r)/r;return i*i+o*o}function aC(t){this._=t,this.next=null,this.previous=null}function sC(t,e){if(!(o=(t=function(t){return\"object\"==typeof t&&\"length\"in t?t:Array.from(t)}(t)).length))return 0;var n,r,i,o,a,s,u,l,c,f,h;if((n=t[0]).x=0,n.y=0,!(o>1))return n.r;if(r=t[1],n.x=-r.r,r.x=n.r,r.y=0,!(o>2))return n.r+r.r;rC(r,n,i=t[2]),n=new aC(n),r=new aC(r),i=new aC(i),n.next=i.previous=r,r.next=n.previous=i,i.next=r.previous=n;t:for(u=3;ufunction(t){t=`${t}`;let e=t.length;bC(t,e-1)&&!bC(t,e-2)&&(t=t.slice(0,-1));return\"/\"===t[0]?t:`/${t}`}(t(e,n,r)))),n=e.map(xC),i=new Set(e).add(\"\");for(const t of n)i.has(t)||(i.add(t),e.push(t),n.push(xC(t)),h.push(mC));d=(t,n)=>e[n],p=(t,e)=>n[e]}for(a=0,i=h.length;a=0&&(l=h[t]).data===mC;--t)l.data=null}if(s.parent=pC,s.eachBefore((function(t){t.depth=t.parent.depth+1,--i})).eachBefore(PD),s.parent=null,i>0)throw new Error(\"cycle\");return s}return r.id=function(t){return arguments.length?(e=ID(t),r):e},r.parentId=function(t){return arguments.length?(n=ID(t),r):n},r.path=function(e){return arguments.length?(t=ID(e),r):t},r}function xC(t){let e=t.length;if(e<2)return\"\";for(;--e>1&&!bC(t,e););return t.slice(0,e)}function bC(t,e){if(\"/\"===t[e]){let n=0;for(;e>0&&\"\\\\\"===t[--e];)++n;if(0==(1&n))return!0}return!1}function wC(t,e){return t.parent===e.parent?1:2}function kC(t){var e=t.children;return e?e[0]:t.t}function AC(t){var e=t.children;return e?e[e.length-1]:t.t}function MC(t,e,n){var r=n/(e.i-t.i);e.c-=r,e.s+=n,t.c+=r,e.z+=n,e.m+=n}function EC(t,e,n){return t.a.parent===e.parent?t.a:n}function DC(t,e){this._=t,this.parent=null,this.children=null,this.A=null,this.a=this,this.z=0,this.m=0,this.c=0,this.s=0,this.t=null,this.i=e}function CC(t,e,n,r,i){for(var o,a=t.children,s=-1,u=a.length,l=t.value&&(i-n)/t.value;++sh&&(h=s),m=c*c*g,(d=Math.max(h/m,m/f))>p){c-=s;break}p=d}y.push(a={value:c,dice:u1?e:1)},n}(FC);var TC=function t(e){function n(t,n,r,i,o){if((a=t._squarify)&&a.ratio===e)for(var a,s,u,l,c,f=-1,h=a.length,d=t.value;++f1?e:1)},n}(FC);function BC(t,e,n){const r={};return t.each((t=>{const i=t.data;n(i)&&(r[e(i)]=t)})),t.lookup=r,t}function zC(t){Ja.call(this,null,t)}zC.Definition={type:\"Nest\",metadata:{treesource:!0,changes:!0},params:[{name:\"keys\",type:\"field\",array:!0},{name:\"generate\",type:\"boolean\"}]};const NC=t=>t.values;function OC(){const t=[],e={entries:t=>r(n(t,0),0),key:n=>(t.push(n),e)};function n(e,r){if(r>=t.length)return e;const i=e.length,o=t[r++],a={},s={};let u,l,c,f=-1;for(;++ft.length)return e;const i=[];for(const t in e)i.push({key:t,values:r(e[t],n)});return i}return e}function RC(t){Ja.call(this,null,t)}dt(zC,Ja,{transform(t,e){e.source||s(\"Nest transform requires an upstream data source.\");var n=t.generate,r=t.modified(),i=e.clone(),o=this.value;return(!o||r||e.changed())&&(o&&o.each((t=>{t.children&&ma(t.data)&&i.rem.push(t.data)})),this.value=o=RD({values:V(t.keys).reduce(((t,e)=>(t.key(e),t)),OC()).entries(i.source)},NC),n&&o.each((t=>{t.children&&(t=_a(t.data),i.add.push(t),i.source.push(t))})),BC(o,ya,ya)),i.source.root=o,i}});const UC=(t,e)=>t.parent===e.parent?1:2;dt(RC,Ja,{transform(t,e){e.source&&e.source.root||s(this.constructor.name+\" transform requires a backing tree data source.\");const n=this.layout(t.method),r=this.fields,i=e.source.root,o=t.as||r;t.field?i.sum(t.field):i.count(),t.sort&&i.sort(ka(t.sort,(t=>t.data))),function(t,e,n){for(let r,i=0,o=e.length;ifunction(t,e,n){const r=t.data,i=e.length-1;for(let o=0;o(t=(GD*t+VD)%XD)/XD}();return i.x=e/2,i.y=n/2,t?i.eachBefore(lC(t)).eachAfter(cC(r,.5,o)).eachBefore(fC(1)):i.eachBefore(lC(uC)).eachAfter(cC(HD,1,o)).eachAfter(cC(r,i.r/Math.min(e,n),o)).eachBefore(fC(Math.min(e,n)/(2*i.r))),i}return i.radius=function(e){return arguments.length?(t=ID(e),i):t},i.size=function(t){return arguments.length?(e=+t[0],n=+t[1],i):[e,n]},i.padding=function(t){return arguments.length?(r=\"function\"==typeof t?t:YD(+t),i):r},i},params:[\"radius\",\"size\",\"padding\"],fields:LC});const PC=[\"x0\",\"y0\",\"x1\",\"y1\",\"depth\",\"children\"];function jC(t){RC.call(this,t)}function IC(t){Ja.call(this,null,t)}jC.Definition={type:\"Partition\",metadata:{tree:!0,modifies:!0},params:[{name:\"field\",type:\"field\"},{name:\"sort\",type:\"compare\"},{name:\"padding\",type:\"number\",default:0},{name:\"round\",type:\"boolean\",default:!1},{name:\"size\",type:\"number\",array:!0,length:2},{name:\"as\",type:\"string\",array:!0,length:PC.length,default:PC}]},dt(jC,RC,{layout:function(){var t=1,e=1,n=0,r=!1;function i(i){var o=i.height+1;return i.x0=i.y0=n,i.x1=t,i.y1=e/o,i.eachBefore(function(t,e){return function(r){r.children&&dC(r,r.x0,t*(r.depth+1)/e,r.x1,t*(r.depth+2)/e);var i=r.x0,o=r.y0,a=r.x1-n,s=r.y1-n;a=0;--i)s.push(n=e.children[i]=new DC(r[i],i)),n.parent=e;return(a.parent=new DC(null,0)).children=[a],a}(i);if(u.eachAfter(o),u.parent.m=-u.z,u.eachBefore(a),r)i.eachBefore(s);else{var l=i,c=i,f=i;i.eachBefore((function(t){t.xc.x&&(c=t),t.depth>f.depth&&(f=t)}));var h=l===c?1:t(l,c)/2,d=h-l.x,p=e/(c.x+h+d),g=n/(f.depth||1);i.eachBefore((function(t){t.x=(t.x+d)*p,t.y=t.depth*g}))}return i}function o(e){var n=e.children,r=e.parent.children,i=e.i?r[e.i-1]:null;if(n){!function(t){for(var e,n=0,r=0,i=t.children,o=i.length;--o>=0;)(e=i[o]).z+=n,e.m+=n,n+=e.s+(r+=e.c)}(e);var o=(n[0].z+n[n.length-1].z)/2;i?(e.z=i.z+t(e._,i._),e.m=e.z-o):e.z=o}else i&&(e.z=i.z+t(e._,i._));e.parent.A=function(e,n,r){if(n){for(var i,o=e,a=e,s=n,u=o.parent.children[0],l=o.m,c=a.m,f=s.m,h=u.m;s=AC(s),o=kC(o),s&&o;)u=kC(u),(a=AC(a)).a=e,(i=s.z+f-o.z-l+t(s._,o._))>0&&(MC(EC(s,e,r),e,i),l+=i,c+=i),f+=s.m,l+=o.m,h+=u.m,c+=a.m;s&&!AC(a)&&(a.t=s,a.m+=f-c),o&&!kC(u)&&(u.t=o,u.m+=l-h,r=e)}return r}(e,i,e.parent.A||r[0])}function a(t){t._.x=t.z+t.parent.m,t.m+=t.parent.m}function s(t){t.x*=e,t.y=t.depth*n}return i.separation=function(e){return arguments.length?(t=e,i):t},i.size=function(t){return arguments.length?(r=!1,e=+t[0],n=+t[1],i):r?null:[e,n]},i.nodeSize=function(t){return arguments.length?(r=!0,e=+t[0],n=+t[1],i):r?[e,n]:null},i},cluster:function(){var t=BD,e=1,n=1,r=!1;function i(i){var o,a=0;i.eachAfter((function(e){var n=e.children;n?(e.x=function(t){return t.reduce(zD,0)/t.length}(n),e.y=function(t){return 1+t.reduce(ND,0)}(n)):(e.x=o?a+=t(e,o):0,e.y=0,o=e)}));var s=function(t){for(var e;e=t.children;)t=e[0];return t}(i),u=function(t){for(var e;e=t.children;)t=e[e.length-1];return t}(i),l=s.x-t(s,u)/2,c=u.x+t(u,s)/2;return i.eachAfter(r?function(t){t.x=(t.x-i.x)*e,t.y=(i.y-t.y)*n}:function(t){t.x=(t.x-l)/(c-l)*e,t.y=(1-(i.y?t.y/i.y:1))*n})}return i.separation=function(e){return arguments.length?(t=e,i):t},i.size=function(t){return arguments.length?(r=!1,e=+t[0],n=+t[1],i):r?null:[e,n]},i.nodeSize=function(t){return arguments.length?(r=!0,e=+t[0],n=+t[1],i):r?[e,n]:null},i}},HC=[\"x\",\"y\",\"depth\",\"children\"];function YC(t){RC.call(this,t)}function GC(t){Ja.call(this,[],t)}YC.Definition={type:\"Tree\",metadata:{tree:!0,modifies:!0},params:[{name:\"field\",type:\"field\"},{name:\"sort\",type:\"compare\"},{name:\"method\",type:\"enum\",default:\"tidy\",values:[\"tidy\",\"cluster\"]},{name:\"size\",type:\"number\",array:!0,length:2},{name:\"nodeSize\",type:\"number\",array:!0,length:2},{name:\"separation\",type:\"boolean\",default:!0},{name:\"as\",type:\"string\",array:!0,length:HC.length,default:HC}]},dt(YC,RC,{layout(t){const e=t||\"tidy\";if(lt(WC,e))return WC[e]();s(\"Unrecognized Tree layout method: \"+e)},params:[\"size\",\"nodeSize\"],fields:HC}),GC.Definition={type:\"TreeLinks\",metadata:{tree:!0,generates:!0,changes:!0},params:[]},dt(GC,Ja,{transform(t,e){const n=this.value,r=e.source&&e.source.root,i=e.fork(e.NO_SOURCE),o={};return r||s(\"TreeLinks transform requires a tree data source.\"),e.changed(e.ADD_REM)?(i.rem=n,e.visit(e.SOURCE,(t=>o[ya(t)]=1)),r.each((t=>{const e=t.data,n=t.parent&&t.parent.data;n&&o[ya(e)]&&o[ya(n)]&&i.add.push(_a({source:n,target:e}))})),this.value=i.add):e.changed(e.MOD)&&(e.visit(e.MOD,(t=>o[ya(t)]=1)),n.forEach((t=>{(o[ya(t.source)]||o[ya(t.target)])&&i.mod.push(t)}))),i}});const VC={binary:function(t,e,n,r,i){var o,a,s=t.children,u=s.length,l=new Array(u+1);for(l[0]=a=o=0;o=n-1){var c=s[e];return c.x0=i,c.y0=o,c.x1=a,void(c.y1=u)}var f=l[e],h=r/2+f,d=e+1,p=n-1;for(;d>>1;l[g]u-o){var v=r?(i*y+a*m)/r:a;t(e,d,m,i,o,v,u),t(d,n,y,v,o,a,u)}else{var _=r?(o*y+u*m)/r:u;t(e,d,m,i,o,a,_),t(d,n,y,i,_,a,u)}}(0,u,t.value,e,n,r,i)},dice:dC,slice:CC,slicedice:function(t,e,n,r,i){(1&t.depth?CC:dC)(t,e,n,r,i)},squarify:$C,resquarify:TC},XC=[\"x0\",\"y0\",\"x1\",\"y1\",\"depth\",\"children\"];function JC(t){RC.call(this,t)}JC.Definition={type:\"Treemap\",metadata:{tree:!0,modifies:!0},params:[{name:\"field\",type:\"field\"},{name:\"sort\",type:\"compare\"},{name:\"method\",type:\"enum\",default:\"squarify\",values:[\"squarify\",\"resquarify\",\"binary\",\"dice\",\"slice\",\"slicedice\"]},{name:\"padding\",type:\"number\",default:0},{name:\"paddingInner\",type:\"number\",default:0},{name:\"paddingOuter\",type:\"number\",default:0},{name:\"paddingTop\",type:\"number\",default:0},{name:\"paddingRight\",type:\"number\",default:0},{name:\"paddingBottom\",type:\"number\",default:0},{name:\"paddingLeft\",type:\"number\",default:0},{name:\"ratio\",type:\"number\",default:1.618033988749895},{name:\"round\",type:\"boolean\",default:!1},{name:\"size\",type:\"number\",array:!0,length:2},{name:\"as\",type:\"string\",array:!0,length:XC.length,default:XC}]},dt(JC,RC,{layout(){const t=function(){var t=$C,e=!1,n=1,r=1,i=[0],o=HD,a=HD,s=HD,u=HD,l=HD;function c(t){return t.x0=t.y0=0,t.x1=n,t.y1=r,t.eachBefore(f),i=[0],e&&t.eachBefore(hC),t}function f(e){var n=i[e.depth],r=e.x0+n,c=e.y0+n,f=e.x1-n,h=e.y1-n;f{const n=t.tile();n.ratio&&t.tile(n.ratio(e))},t.method=e=>{lt(VC,e)?t.tile(VC[e]):s(\"Unrecognized Treemap layout method: \"+e)},t},params:[\"method\",\"ratio\",\"size\",\"round\",\"padding\",\"paddingInner\",\"paddingOuter\",\"paddingTop\",\"paddingRight\",\"paddingBottom\",\"paddingLeft\"],fields:XC});var ZC=Object.freeze({__proto__:null,nest:zC,pack:qC,partition:jC,stratify:IC,tree:YC,treelinks:GC,treemap:JC});const QC=4278190080;function KC(t,e,n){return new Uint32Array(t.getImageData(0,0,e,n).data.buffer)}function tF(t,e,n){if(!e.length)return;const r=e[0].mark.marktype;\"group\"===r?e.forEach((e=>{e.items.forEach((e=>tF(t,e.items,n)))})):Hy[r].draw(t,{items:n?e.map(eF):e})}function eF(t){const e=ba(t,{});return e.stroke&&0!==e.strokeOpacity||e.fill&&0!==e.fillOpacity?{...e,strokeOpacity:1,stroke:\"#000\",fillOpacity:0}:e}const nF=5,rF=31,iF=32,oF=new Uint32Array(iF+1),aF=new Uint32Array(iF+1);aF[0]=0,oF[0]=~aF[0];for(let t=1;t<=iF;++t)aF[t]=aF[t-1]<<1|1,oF[t]=~aF[t];function sF(t,e,n){const r=Math.max(1,Math.sqrt(t*e/1e6)),i=~~((t+2*n+r)/r),o=~~((e+2*n+r)/r),a=t=>~~((t+n)/r);return a.invert=t=>t*r-n,a.bitmap=()=>function(t,e){const n=new Uint32Array(~~((t*e+iF)/iF));function r(t,e){n[t]|=e}function i(t,e){n[t]&=e}return{array:n,get:(e,r)=>{const i=r*t+e;return n[i>>>nF]&1<<(i&rF)},set:(e,n)=>{const i=n*t+e;r(i>>>nF,1<<(i&rF))},clear:(e,n)=>{const r=n*t+e;i(r>>>nF,~(1<<(r&rF)))},getRange:(e,r,i,o)=>{let a,s,u,l,c=o;for(;c>=r;--c)if(a=c*t+e,s=c*t+i,u=a>>>nF,l=s>>>nF,u===l){if(n[u]&oF[a&rF]&aF[1+(s&rF)])return!0}else{if(n[u]&oF[a&rF])return!0;if(n[l]&aF[1+(s&rF)])return!0;for(let t=u+1;t{let a,s,u,l,c;for(;n<=o;++n)if(a=n*t+e,s=n*t+i,u=a>>>nF,l=s>>>nF,u===l)r(u,oF[a&rF]&aF[1+(s&rF)]);else for(r(u,oF[a&rF]),r(l,aF[1+(s&rF)]),c=u+1;c{let a,s,u,l,c;for(;n<=o;++n)if(a=n*t+e,s=n*t+r,u=a>>>nF,l=s>>>nF,u===l)i(u,aF[a&rF]|oF[1+(s&rF)]);else for(i(u,aF[a&rF]),i(l,oF[1+(s&rF)]),c=u+1;cn<0||r<0||o>=e||i>=t}}(i,o),a.ratio=r,a.padding=n,a.width=t,a.height=e,a}function uF(t,e,n,r,i,o){let a=n/2;return t-a<0||t+a>i||e-(a=r/2)<0||e+a>o}function lF(t,e,n,r,i,o,a,s){const u=i*o/(2*r),l=t(e-u),c=t(e+u),f=t(n-(o/=2)),h=t(n+o);return a.outOfBounds(l,f,c,h)||a.getRange(l,f,c,h)||s&&s.getRange(l,f,c,h)}const cF=[-1,-1,1,1],fF=[-1,1,-1,1];const hF=[\"right\",\"center\",\"left\"],dF=[\"bottom\",\"middle\",\"top\"];function pF(t,e,n,r,i,o,a,s,u,l,c,f){return!(i.outOfBounds(t,n,e,r)||(f&&o||i).getRange(t,n,e,r))}const gF={\"top-left\":0,top:1,\"top-right\":2,left:4,middle:5,right:6,\"bottom-left\":8,bottom:9,\"bottom-right\":10},mF={naive:function(t,e,n,r){const i=t.width,o=t.height;return function(t){const e=t.datum.datum.items[r].items,n=e.length,a=t.datum.fontSize,s=My.width(t.datum,t.datum.text);let u,l,c,f,h,d,p,g=0;for(let r=0;r=g&&(g=p,t.x=h,t.y=d);return h=s/2,d=a/2,u=t.x-h,l=t.x+h,c=t.y-d,f=t.y+d,t.align=\"center\",u<0&&l<=i?t.align=\"left\":0<=u&&i=1;)h=(d+p)/2,lF(t,c,f,l,u,h,a,s)?p=h:d=h;if(d>r)return[c,f,d,!0]}}return function(e){const s=e.datum.datum.items[r].items,l=s.length,c=e.datum.fontSize,f=My.width(e.datum,e.datum.text);let h,d,p,g,m,y,v,_,x,b,w,k,A,M,E,D,C,F=n?c:0,S=!1,$=!1,T=0;for(let r=0;rd&&(C=h,h=d,d=C),p>g&&(C=p,p=g,g=C),x=t(h),w=t(d),b=~~((x+w)/2),k=t(p),M=t(g),A=~~((k+M)/2),v=b;v>=x;--v)for(_=A;_>=k;--_)D=u(v,_,F,f,c),D&&([e.x,e.y,F,S]=D);for(v=b;v<=w;++v)for(_=A;_<=M;++_)D=u(v,_,F,f,c),D&&([e.x,e.y,F,S]=D);S||n||(E=Math.abs(d-h+g-p),m=(h+d)/2,y=(p+g)/2,E>=T&&!uF(m,y,f,c,i,o)&&!lF(t,m,y,c,f,c,a,null)&&(T=E,e.x=m,e.y=y,$=!0))}return!(!S&&!$)&&(m=f/2,y=c/2,a.setRange(t(e.x-m),t(e.y-y),t(e.x+m),t(e.y+y)),e.align=\"center\",e.baseline=\"middle\",!0)}},floodfill:function(t,e,n,r){const i=t.width,o=t.height,a=e[0],s=e[1],u=t.bitmap();return function(e){const l=e.datum.datum.items[r].items,c=l.length,f=e.datum.fontSize,h=My.width(e.datum,e.datum.text),d=[];let p,g,m,y,v,_,x,b,w,k,A,M,E=n?f:0,D=!1,C=!1,F=0;for(let r=0;r=1;)A=(w+k)/2,lF(t,v,_,f,h,A,a,s)?k=A:w=A;w>E&&(e.x=v,e.y=_,E=w,D=!0)}}D||n||(M=Math.abs(g-p+y-m),v=(p+g)/2,_=(m+y)/2,M>=F&&!uF(v,_,h,f,i,o)&&!lF(t,v,_,f,h,f,a,null)&&(F=M,e.x=v,e.y=_,C=!0))}return!(!D&&!C)&&(v=h/2,_=f/2,a.setRange(t(e.x-v),t(e.y-_),t(e.x+v),t(e.y+_)),e.align=\"center\",e.baseline=\"middle\",!0)}}};function yF(t,e,n,r,i,o,a,s,u,l,c){if(!t.length)return t;const f=Math.max(r.length,i.length),h=function(t,e){const n=new Float64Array(e),r=t.length;for(let e=0;e[t.x,t.x,t.x,t.y,t.y,t.y];return t?\"line\"===t||\"area\"===t?t=>i(t.datum):\"line\"===e?t=>{const e=t.datum.items[r].items;return i(e.length?e[\"start\"===n?0:e.length-1]:{x:NaN,y:NaN})}:t=>{const e=t.datum.bounds;return[e.x1,(e.x1+e.x2)/2,e.x2,e.y1,(e.y1+e.y2)/2,e.y2]}:i}(p,g,s,u),v=null===l||l===1/0,_=m&&\"naive\"===c;var x;let b=-1,w=-1;const k=t.map((t=>{const e=v?My.width(t,t.text):void 0;return b=Math.max(b,e),w=Math.max(w,t.fontSize),{datum:t,opacity:0,x:void 0,y:void 0,align:void 0,baseline:void 0,boundary:y(t),textWidth:e}}));l=null===l||l===1/0?Math.max(b,w)+Math.max(...r):l;const A=sF(e[0],e[1],l);let M;if(!_){n&&k.sort(((t,e)=>n(t.datum,e.datum)));let e=!1;for(let t=0;tt.datum));M=o.length||r?function(t,e,n,r,i){const o=t.width,a=t.height,s=r||i,u=$c(o,a).getContext(\"2d\"),l=$c(o,a).getContext(\"2d\"),c=s&&$c(o,a).getContext(\"2d\");n.forEach((t=>tF(u,t,!1))),tF(l,e,!1),s&&tF(c,e,!0);const f=KC(u,o,a),h=KC(l,o,a),d=s&&KC(c,o,a),p=t.bitmap(),g=s&&t.bitmap();let m,y,v,_,x,b,w,k;for(y=0;yn.set(t(e.boundary[0]),t(e.boundary[3])))),[n,void 0]}(A,a&&k)}const E=m?mF[c](A,M,a,u):function(t,e,n,r){const i=t.width,o=t.height,a=e[0],s=e[1],u=r.length;return function(e){const l=e.boundary,c=e.datum.fontSize;if(l[2]<0||l[5]<0||l[0]>i||l[3]>o)return!1;let f,h,d,p,g,m,y,v,_,x,b,w,k,A,M,E=e.textWidth??0;for(let i=0;i>>2&3)-1,d=0===f&&0===h||r[i]<0,p=f&&h?Math.SQRT1_2:1,g=r[i]<0?-1:1,m=l[1+f]+r[i]*f*p,b=l[4+h]+g*c*h/2+r[i]*h*p,v=b-c/2,_=b+c/2,w=t(m),A=t(v),M=t(_),!E){if(!pF(w,w,A,M,a,s,0,0,0,0,0,d))continue;E=My.width(e.datum,e.datum.text)}if(x=m+g*E*f/2,m=x-E/2,y=x+E/2,w=t(m),k=t(y),pF(w,k,A,M,a,s,0,0,0,0,0,d))return e.x=f?f*g<0?y:m:x,e.y=h?h*g<0?_:v:b,e.align=hF[f*g+1],e.baseline=dF[h*g+1],a.setRange(w,A,k,M),!0}return!1}}(A,M,d,h);return k.forEach((t=>t.opacity=+E(t))),k}const vF=[\"x\",\"y\",\"opacity\",\"align\",\"baseline\"],_F=[\"top-left\",\"left\",\"bottom-left\",\"top\",\"bottom\",\"top-right\",\"right\",\"bottom-right\"];function xF(t){Ja.call(this,null,t)}xF.Definition={type:\"Label\",metadata:{modifies:!0},params:[{name:\"size\",type:\"number\",array:!0,length:2,required:!0},{name:\"sort\",type:\"compare\"},{name:\"anchor\",type:\"string\",array:!0,default:_F},{name:\"offset\",type:\"number\",array:!0,default:[1]},{name:\"padding\",type:\"number\",default:0,null:!0},{name:\"lineAnchor\",type:\"string\",values:[\"start\",\"end\"],default:\"end\"},{name:\"markIndex\",type:\"number\",default:0},{name:\"avoidBaseMark\",type:\"boolean\",default:!0},{name:\"avoidMarks\",type:\"data\",array:!0},{name:\"method\",type:\"string\",default:\"naive\"},{name:\"as\",type:\"string\",array:!0,length:vF.length,default:vF}]},dt(xF,Ja,{transform(t,e){const n=t.modified();if(!(n||e.changed(e.ADD_REM)||function(n){const r=t[n];return J(r)&&e.modified(r.fields)}(\"sort\")))return;t.size&&2===t.size.length||s(\"Size parameter should be specified as a [width, height] array.\");const r=t.as||vF;return yF(e.materialize(e.SOURCE).source||[],t.size,t.sort,V(null==t.offset?1:t.offset),V(t.anchor||_F),t.avoidMarks||[],!1!==t.avoidBaseMark,t.lineAnchor||\"end\",t.markIndex||0,void 0===t.padding?0:t.padding,t.method||\"naive\").forEach((t=>{const e=t.datum;e[r[0]]=t.x,e[r[1]]=t.y,e[r[2]]=t.opacity,e[r[3]]=t.align,e[r[4]]=t.baseline})),e.reflow(n).modifies(r)}});var bF=Object.freeze({__proto__:null,label:xF});function wF(t,e){var n,r,i,o,a,s,u=[],l=function(t){return t(o)};if(null==e)u.push(t);else for(n={},r=0,i=t.length;r{Ls(e,t.x,t.y,t.bandwidth||.3).forEach((t=>{const n={};for(let t=0;t\"poly\"===t?e:\"quad\"===t?2:1)(a,u),c=t.as||[n(t.x),n(t.y)],f=AF[a],h=[];let d=t.extent;lt(AF,a)||s(\"Invalid regression method: \"+a),null!=d&&\"log\"===a&&d[0]<=0&&(e.dataflow.warn(\"Ignoring extent with values <= 0 for log regression.\"),d=null),i.forEach((n=>{if(n.length<=l)return void e.dataflow.warn(\"Skipping regression with more parameters than data points.\");const r=f(n,t.x,t.y,u);if(t.params)return void h.push(_a({keys:n.dims,coef:r.coef,rSquared:r.rSquared}));const i=d||at(n,t.x),s=t=>{const e={};for(let t=0;ts([t,r.predict(t)]))):Is(r.predict,i,25,200).forEach(s)})),this.value&&(r.rem=this.value),this.value=r.add=r.source=h}return r}});var EF=Object.freeze({__proto__:null,loess:kF,regression:MF});const DF=134217729,CF=33306690738754706e-32;function FF(t,e,n,r,i){let o,a,s,u,l=e[0],c=r[0],f=0,h=0;c>l==c>-l?(o=l,l=e[++f]):(o=c,c=r[++h]);let d=0;if(fl==c>-l?(a=l+o,s=o-(a-l),l=e[++f]):(a=c+o,s=o-(a-c),c=r[++h]),o=a,0!==s&&(i[d++]=s);fl==c>-l?(a=o+l,u=a-o,s=o-(a-u)+(l-u),l=e[++f]):(a=o+c,u=a-o,s=o-(a-u)+(c-u),c=r[++h]),o=a,0!==s&&(i[d++]=s);for(;f0!=s>0)return u;const l=Math.abs(a+s);return Math.abs(u)>=33306690738754716e-32*l?u:-function(t,e,n,r,i,o,a){let s,u,l,c,f,h,d,p,g,m,y,v,_,x,b,w,k,A;const M=t-i,E=n-i,D=e-o,C=r-o;x=M*C,h=DF*M,d=h-(h-M),p=M-d,h=DF*C,g=h-(h-C),m=C-g,b=p*m-(x-d*g-p*g-d*m),w=D*E,h=DF*D,d=h-(h-D),p=D-d,h=DF*E,g=h-(h-E),m=E-g,k=p*m-(w-d*g-p*g-d*m),y=b-k,f=b-y,BF[0]=b-(y+f)+(f-k),v=x+y,f=v-x,_=x-(v-f)+(y-f),y=_-w,f=_-y,BF[1]=_-(y+f)+(f-w),A=v+y,f=A-v,BF[2]=v-(A-f)+(y-f),BF[3]=A;let F=function(t,e){let n=e[0];for(let r=1;r=S||-F>=S)return F;if(f=t-M,s=t-(M+f)+(f-i),f=n-E,l=n-(E+f)+(f-i),f=e-D,u=e-(D+f)+(f-o),f=r-C,c=r-(C+f)+(f-o),0===s&&0===u&&0===l&&0===c)return F;if(S=TF*a+CF*Math.abs(F),F+=M*c+C*s-(D*l+E*u),F>=S||-F>=S)return F;x=s*C,h=DF*s,d=h-(h-s),p=s-d,h=DF*C,g=h-(h-C),m=C-g,b=p*m-(x-d*g-p*g-d*m),w=u*E,h=DF*u,d=h-(h-u),p=u-d,h=DF*E,g=h-(h-E),m=E-g,k=p*m-(w-d*g-p*g-d*m),y=b-k,f=b-y,RF[0]=b-(y+f)+(f-k),v=x+y,f=v-x,_=x-(v-f)+(y-f),y=_-w,f=_-y,RF[1]=_-(y+f)+(f-w),A=v+y,f=A-v,RF[2]=v-(A-f)+(y-f),RF[3]=A;const $=FF(4,BF,4,RF,zF);x=M*c,h=DF*M,d=h-(h-M),p=M-d,h=DF*c,g=h-(h-c),m=c-g,b=p*m-(x-d*g-p*g-d*m),w=D*l,h=DF*D,d=h-(h-D),p=D-d,h=DF*l,g=h-(h-l),m=l-g,k=p*m-(w-d*g-p*g-d*m),y=b-k,f=b-y,RF[0]=b-(y+f)+(f-k),v=x+y,f=v-x,_=x-(v-f)+(y-f),y=_-w,f=_-y,RF[1]=_-(y+f)+(f-w),A=v+y,f=A-v,RF[2]=v-(A-f)+(y-f),RF[3]=A;const T=FF($,zF,4,RF,NF);x=s*c,h=DF*s,d=h-(h-s),p=s-d,h=DF*c,g=h-(h-c),m=c-g,b=p*m-(x-d*g-p*g-d*m),w=u*l,h=DF*u,d=h-(h-u),p=u-d,h=DF*l,g=h-(h-l),m=l-g,k=p*m-(w-d*g-p*g-d*m),y=b-k,f=b-y,RF[0]=b-(y+f)+(f-k),v=x+y,f=v-x,_=x-(v-f)+(y-f),y=_-w,f=_-y,RF[1]=_-(y+f)+(f-w),A=v+y,f=A-v,RF[2]=v-(A-f)+(y-f),RF[3]=A;const B=FF(T,NF,4,RF,OF);return OF[B-1]}(t,e,n,r,i,o,l)}const LF=Math.pow(2,-52),qF=new Uint32Array(512);class PF{static from(t){let e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:GF,n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:VF;const r=t.length,i=new Float64Array(2*r);for(let o=0;o>1;if(e>0&&\"number\"!=typeof t[0])throw new Error(\"Expected coords to contain numbers.\");this.coords=t;const n=Math.max(2*e-5,0);this._triangles=new Uint32Array(3*n),this._halfedges=new Int32Array(3*n),this._hashSize=Math.ceil(Math.sqrt(e)),this._hullPrev=new Uint32Array(e),this._hullNext=new Uint32Array(e),this._hullTri=new Uint32Array(e),this._hullHash=new Int32Array(this._hashSize).fill(-1),this._ids=new Uint32Array(e),this._dists=new Float64Array(e),this.update()}update(){const{coords:t,_hullPrev:e,_hullNext:n,_hullTri:r,_hullHash:i}=this,o=t.length>>1;let a=1/0,s=1/0,u=-1/0,l=-1/0;for(let e=0;eu&&(u=n),r>l&&(l=r),this._ids[e]=e}const c=(a+u)/2,f=(s+l)/2;let h,d,p,g=1/0;for(let e=0;e0&&(d=e,g=n)}let v=t[2*d],_=t[2*d+1],x=1/0;for(let e=0;er&&(e[n++]=i,r=this._dists[i])}return this.hull=e.subarray(0,n),this.triangles=new Uint32Array(0),void(this.halfedges=new Uint32Array(0))}if(UF(m,y,v,_,b,w)<0){const t=d,e=v,n=_;d=p,v=b,_=w,p=t,b=e,w=n}const k=function(t,e,n,r,i,o){const a=n-t,s=r-e,u=i-t,l=o-e,c=a*a+s*s,f=u*u+l*l,h=.5/(a*l-s*u),d=t+(l*c-s*f)*h,p=e+(a*f-u*c)*h;return{x:d,y:p}}(m,y,v,_,b,w);this._cx=k.x,this._cy=k.y;for(let e=0;e0&&Math.abs(l-o)<=LF&&Math.abs(c-a)<=LF)continue;if(o=l,a=c,u===h||u===d||u===p)continue;let f=0;for(let t=0,e=this._hashKey(l,c);t=0;)if(m=g,m===f){m=-1;break}if(-1===m)continue;let y=this._addTriangle(m,u,n[m],-1,-1,r[m]);r[u]=this._legalize(y+2),r[m]=y,A++;let v=n[m];for(;g=n[v],UF(l,c,t[2*v],t[2*v+1],t[2*g],t[2*g+1])<0;)y=this._addTriangle(v,u,g,r[u],-1,r[v]),r[u]=this._legalize(y+2),n[v]=v,A--,v=g;if(m===f)for(;g=e[m],UF(l,c,t[2*g],t[2*g+1],t[2*m],t[2*m+1])<0;)y=this._addTriangle(g,u,m,-1,r[m],r[g]),this._legalize(y+2),r[g]=y,n[m]=m,A--,m=g;this._hullStart=e[u]=m,n[m]=e[v]=u,n[u]=v,i[this._hashKey(l,c)]=u,i[this._hashKey(t[2*m],t[2*m+1])]=m}this.hull=new Uint32Array(A);for(let t=0,e=this._hullStart;t0?3-n:1+n)/4}(t-this._cx,e-this._cy)*this._hashSize)%this._hashSize}_legalize(t){const{_triangles:e,_halfedges:n,coords:r}=this;let i=0,o=0;for(;;){const a=n[t],s=t-t%3;if(o=s+(t+2)%3,-1===a){if(0===i)break;t=qF[--i];continue}const u=a-a%3,l=s+(t+1)%3,c=u+(a+2)%3,f=e[o],h=e[t],d=e[l],p=e[c];if(IF(r[2*f],r[2*f+1],r[2*h],r[2*h+1],r[2*d],r[2*d+1],r[2*p],r[2*p+1])){e[t]=p,e[a]=f;const r=n[c];if(-1===r){let e=this._hullStart;do{if(this._hullTri[e]===c){this._hullTri[e]=t;break}e=this._hullPrev[e]}while(e!==this._hullStart)}this._link(t,r),this._link(a,n[o]),this._link(o,c);const s=u+(a+1)%3;i=n&&e[t[a]]>o;)t[a+1]=t[a--];t[a+1]=r}else{let i=n+1,o=r;YF(t,n+r>>1,i),e[t[n]]>e[t[r]]&&YF(t,n,r),e[t[i]]>e[t[r]]&&YF(t,i,r),e[t[n]]>e[t[i]]&&YF(t,n,i);const a=t[i],s=e[a];for(;;){do{i++}while(e[t[i]]s);if(o=o-n?(HF(t,e,i,r),HF(t,e,n,o-1)):(HF(t,e,n,o-1),HF(t,e,i,r))}}function YF(t,e,n){const r=t[e];t[e]=t[n],t[n]=r}function GF(t){return t[0]}function VF(t){return t[1]}const XF=1e-6;class JF{constructor(){this._x0=this._y0=this._x1=this._y1=null,this._=\"\"}moveTo(t,e){this._+=`M${this._x0=this._x1=+t},${this._y0=this._y1=+e}`}closePath(){null!==this._x1&&(this._x1=this._x0,this._y1=this._y0,this._+=\"Z\")}lineTo(t,e){this._+=`L${this._x1=+t},${this._y1=+e}`}arc(t,e,n){const r=(t=+t)+(n=+n),i=e=+e;if(n<0)throw new Error(\"negative radius\");null===this._x1?this._+=`M${r},${i}`:(Math.abs(this._x1-r)>XF||Math.abs(this._y1-i)>XF)&&(this._+=\"L\"+r+\",\"+i),n&&(this._+=`A${n},${n},0,1,1,${t-n},${e}A${n},${n},0,1,1,${this._x1=r},${this._y1=i}`)}rect(t,e,n,r){this._+=`M${this._x0=this._x1=+t},${this._y0=this._y1=+e}h${+n}v${+r}h${-n}Z`}value(){return this._||null}}class ZF{constructor(){this._=[]}moveTo(t,e){this._.push([t,e])}closePath(){this._.push(this._[0].slice())}lineTo(t,e){this._.push([t,e])}value(){return this._.length?this._:null}}let QF=class{constructor(t){let[e,n,r,i]=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[0,0,960,500];if(!((r=+r)>=(e=+e)&&(i=+i)>=(n=+n)))throw new Error(\"invalid bounds\");this.delaunay=t,this._circumcenters=new Float64Array(2*t.points.length),this.vectors=new Float64Array(2*t.points.length),this.xmax=r,this.xmin=e,this.ymax=i,this.ymin=n,this._init()}update(){return this.delaunay.update(),this._init(),this}_init(){const{delaunay:{points:t,hull:e,triangles:n},vectors:r}=this;let i,o;const a=this.circumcenters=this._circumcenters.subarray(0,n.length/3*2);for(let r,s,u=0,l=0,c=n.length;u1;)i-=2;for(let t=2;t0){if(e>=this.ymax)return null;(i=(this.ymax-e)/r)0){if(t>=this.xmax)return null;(i=(this.xmax-t)/n)this.xmax?2:0)|(ethis.ymax?8:0)}_simplify(t){if(t&&t.length>4){for(let e=0;e1&&void 0!==arguments[1]?arguments[1]:eS,n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:nS,r=arguments.length>3?arguments[3]:void 0;return new iS(\"length\"in t?function(t,e,n,r){const i=t.length,o=new Float64Array(2*i);for(let a=0;a2&&function(t){const{triangles:e,coords:n}=t;for(let t=0;t1e-10)return!1}return!0}(t)){this.collinear=Int32Array.from({length:e.length/2},((t,e)=>e)).sort(((t,n)=>e[2*t]-e[2*n]||e[2*t+1]-e[2*n+1]));const t=this.collinear[0],n=this.collinear[this.collinear.length-1],r=[e[2*t],e[2*t+1],e[2*n],e[2*n+1]],i=1e-8*Math.hypot(r[3]-r[1],r[2]-r[0]);for(let t=0,n=e.length/2;t0&&(this.triangles=new Int32Array(3).fill(-1),this.halfedges=new Int32Array(3).fill(-1),this.triangles[0]=r[0],o[r[0]]=1,2===r.length&&(o[r[1]]=0,this.triangles[1]=r[1],this.triangles[2]=r[1]))}voronoi(t){return new QF(this,t)}*neighbors(t){const{inedges:e,hull:n,_hullIndex:r,halfedges:i,triangles:o,collinear:a}=this;if(a){const e=a.indexOf(t);return e>0&&(yield a[e-1]),void(e2&&void 0!==arguments[2]?arguments[2]:0;if((t=+t)!=t||(e=+e)!=e)return-1;const r=n;let i;for(;(i=this._step(n,t,e))>=0&&i!==n&&i!==r;)n=i;return i}_step(t,e,n){const{inedges:r,hull:i,_hullIndex:o,halfedges:a,triangles:s,points:u}=this;if(-1===r[t]||!u.length)return(t+1)%(u.length>>1);let l=t,c=tS(e-u[2*t],2)+tS(n-u[2*t+1],2);const f=r[t];let h=f;do{let r=s[h];const f=tS(e-u[2*r],2)+tS(n-u[2*r+1],2);if(f=f));)if(e.x=a+i,e.y=l+o,!(e.x+e.x0<0||e.y+e.y0<0||e.x+e.x1>s[0]||e.y+e.y1>s[1])&&(!n||!pS(e,t,s[0]))&&(!n||mS(e,n))){for(var g,m=e.sprite,y=e.width>>5,v=s[0]>>5,_=e.x-(y<<4),x=127&_,b=32-x,w=e.y1-e.y0,k=(e.y+e.y0)*v+(_>>5),A=0;A>>x:0);k+=v}return e.sprite=null,!0}return!1}return f.layout=function(){for(var u=function(t){t.width=t.height=1;var e=Math.sqrt(t.getContext(\"2d\").getImageData(0,0,1,1).data.length>>2);t.width=(cS<<5)/e,t.height=fS/e;var n=t.getContext(\"2d\");return n.fillStyle=n.strokeStyle=\"red\",n.textAlign=\"center\",{context:n,ratio:e}}($c()),f=function(t){var e=[],n=-1;for(;++n>5)*s[1]),d=null,p=l.length,g=-1,m=[],y=l.map((s=>({text:t(s),font:e(s),style:r(s),weight:i(s),rotate:o(s),size:~~(n(s)+1e-14),padding:a(s),xoff:0,yoff:0,x1:0,y1:0,x0:0,y0:0,hasText:!1,sprite:null,datum:s}))).sort(((t,e)=>e.size-t.size));++g>1,v.y=s[1]*(c()+.5)>>1,dS(u,v,y,g),v.hasText&&h(f,v,d)&&(m.push(v),d?gS(d,v):d=[{x:v.x+v.x0,y:v.y+v.y0},{x:v.x+v.x1,y:v.y+v.y1}],v.x-=s[0]>>1,v.y-=s[1]>>1)}return m},f.words=function(t){return arguments.length?(l=t,f):l},f.size=function(t){return arguments.length?(s=[+t[0],+t[1]],f):s},f.font=function(t){return arguments.length?(e=vS(t),f):e},f.fontStyle=function(t){return arguments.length?(r=vS(t),f):r},f.fontWeight=function(t){return arguments.length?(i=vS(t),f):i},f.rotate=function(t){return arguments.length?(o=vS(t),f):o},f.text=function(e){return arguments.length?(t=vS(e),f):t},f.spiral=function(t){return arguments.length?(u=_S[t]||t,f):u},f.fontSize=function(t){return arguments.length?(n=vS(t),f):n},f.padding=function(t){return arguments.length?(a=vS(t),f):a},f.random=function(t){return arguments.length?(c=t,f):c},f}function dS(t,e,n,r){if(!e.sprite){var i=t.context,o=t.ratio;i.clearRect(0,0,(cS<<5)/o,fS/o);var a,s,u,l,c,f=0,h=0,d=0,p=n.length;for(--r;++r>5<<5,u=~~Math.max(Math.abs(v+_),Math.abs(v-_))}else a=a+31>>5<<5;if(u>d&&(d=u),f+a>=cS<<5&&(f=0,h+=d,d=0),h+u>=fS)break;i.translate((f+(a>>1))/o,(h+(u>>1))/o),e.rotate&&i.rotate(e.rotate*lS),i.fillText(e.text,0,0),e.padding&&(i.lineWidth=2*e.padding,i.strokeText(e.text,0,0)),i.restore(),e.width=a,e.height=u,e.xoff=f,e.yoff=h,e.x1=a>>1,e.y1=u>>1,e.x0=-e.x1,e.y0=-e.y1,e.hasText=!0,f+=a}for(var b=i.getImageData(0,0,(cS<<5)/o,fS/o).data,w=[];--r>=0;)if((e=n[r]).hasText){for(s=(a=e.width)>>5,u=e.y1-e.y0,l=0;l>5),E=b[(h+c)*(cS<<5)+(f+l)<<2]?1<<31-l%32:0;w[M]|=E,k|=E}k?A=c:(e.y0++,u--,c--,h++)}e.y1=e.y0+A,e.sprite=w.slice(0,(e.y1-e.y0)*s)}}}function pS(t,e,n){n>>=5;for(var r,i=t.sprite,o=t.width>>5,a=t.x-(o<<4),s=127&a,u=32-s,l=t.y1-t.y0,c=(t.y+t.y0)*n+(a>>5),f=0;f>>s:0))&e[c+h])return!0;c+=n}return!1}function gS(t,e){var n=t[0],r=t[1];e.x+e.x0r.x&&(r.x=e.x+e.x1),e.y+e.y1>r.y&&(r.y=e.y+e.y1)}function mS(t,e){return t.x+t.x1>e[0].x&&t.x+t.x0e[0].y&&t.y+t.y0e(t(n))}i.forEach((t=>{t[a[0]]=NaN,t[a[1]]=NaN,t[a[3]]=0}));const c=o.words(i).text(e.text).size(e.size||[500,500]).padding(e.padding||1).spiral(e.spiral||\"archimedean\").rotate(e.rotate||0).font(e.font||\"sans-serif\").fontStyle(e.fontStyle||\"normal\").fontWeight(e.fontWeight||\"normal\").fontSize(l).random(t.random).layout(),f=o.size(),h=f[0]>>1,d=f[1]>>1,p=c.length;for(let t,e,n=0;nnew Uint8Array(t),MS=t=>new Uint16Array(t),ES=t=>new Uint32Array(t);function DS(t,e,n){const r=(e<257?AS:e<65537?MS:ES)(t);return n&&r.set(n),r}function CS(t,e,n){const r=1<{const r=t[e],i=t[n];return ri?1:0})),function(t,e){return Array.from(e,(e=>t[e]))}(t,e)}(h,u),a)l=e,c=t,e=Array(a+s),t=ES(a+s),function(t,e,n,r,i,o,a,s,u){let l,c=0,f=0;for(l=0;c0)for(f=0;ft,size:()=>n}}function SS(t){Ja.call(this,function(){let t=8,e=[],n=ES(0),r=DS(0,t),i=DS(0,t);return{data:()=>e,seen:()=>n=function(t,e,n){return t.length>=e?t:((n=n||new t.constructor(e)).set(t),n)}(n,e.length),add(t){for(let n,r=0,i=e.length,o=t.length;re.length,curr:()=>r,prev:()=>i,reset:t=>i[t]=r[t],all:()=>t<257?255:t<65537?65535:4294967295,set(t,e){r[t]|=e},clear(t,e){r[t]&=~e},resize(e,n){(e>r.length||n>t)&&(t=Math.max(n,t),r=DS(e,t,r),i=DS(e,t))}}}(),t),this._indices=null,this._dims=null}function $S(t){Ja.call(this,null,t)}SS.Definition={type:\"CrossFilter\",metadata:{},params:[{name:\"fields\",type:\"field\",array:!0,required:!0},{name:\"query\",type:\"array\",array:!0,required:!0,content:{type:\"number\",array:!0,length:2}}]},dt(SS,Ja,{transform(t,e){return this._dims?t.modified(\"fields\")||t.fields.some((t=>e.modified(t.fields)))?this.reinit(t,e):this.eval(t,e):this.init(t,e)},init(t,e){const n=t.fields,r=t.query,i=this._indices={},o=this._dims=[],a=r.length;let s,u,l=0;for(;l{const t=i.remove(e,n);for(const e in r)r[e].reindex(t)}))},update(t,e,n){const r=this._dims,i=t.query,o=e.stamp,a=r.length;let s,u,l=0;for(n.filters=0,u=0;ud)for(m=d,y=Math.min(f,p);mp)for(m=Math.max(f,p),y=h;mc)for(d=c,p=Math.min(u,f);df)for(d=Math.max(u,f),p=l;ds[t]&n?null:a[t];return o.filter(o.MOD,l),i&i-1?(o.filter(o.ADD,(t=>{const e=s[t]&n;return!e&&e^u[t]&n?a[t]:null})),o.filter(o.REM,(t=>{const e=s[t]&n;return e&&!(e^e^u[t]&n)?a[t]:null}))):(o.filter(o.ADD,l),o.filter(o.REM,(t=>(s[t]&n)===i?a[t]:null))),o.filter(o.SOURCE,(t=>l(t._index)))}});var TS=Object.freeze({__proto__:null,crossfilter:SS,resolvefilter:$S});const BS=\"Literal\",zS=\"Property\",NS=\"ArrayExpression\",OS=\"BinaryExpression\",RS=\"CallExpression\",US=\"ConditionalExpression\",LS=\"LogicalExpression\",qS=\"MemberExpression\",PS=\"ObjectExpression\",jS=\"UnaryExpression\";function IS(t){this.type=t}var WS,HS,YS,GS,VS;IS.prototype.visit=function(t){let e,n,r;if(t(this))return 1;for(e=function(t){switch(t.type){case NS:return t.elements;case OS:case LS:return[t.left,t.right];case RS:return[t.callee].concat(t.arguments);case US:return[t.test,t.consequent,t.alternate];case qS:return[t.object,t.property];case PS:return t.properties;case zS:return[t.key,t.value];case jS:return[t.argument];default:return[]}}(this),n=0,r=e.length;n\",WS[ZS]=\"Identifier\",WS[QS]=\"Keyword\",WS[KS]=\"Null\",WS[t$]=\"Numeric\",WS[e$]=\"Punctuator\",WS[n$]=\"String\",WS[9]=\"RegularExpression\";var r$=\"ArrayExpression\",i$=\"BinaryExpression\",o$=\"CallExpression\",a$=\"ConditionalExpression\",s$=\"Identifier\",u$=\"Literal\",l$=\"LogicalExpression\",c$=\"MemberExpression\",f$=\"ObjectExpression\",h$=\"Property\",d$=\"UnaryExpression\",p$=\"Unexpected token %0\",g$=\"Unexpected number\",m$=\"Unexpected string\",y$=\"Unexpected identifier\",v$=\"Unexpected reserved word\",_$=\"Unexpected end of input\",x$=\"Invalid regular expression\",b$=\"Invalid regular expression: missing /\",w$=\"Octal literals are not allowed in strict mode.\",k$=\"Duplicate data property in object literal not allowed in strict mode\",A$=\"ILLEGAL\",M$=\"Disabled.\",E$=new RegExp(\"[\\\\xAA\\\\xB5\\\\xBA\\\\xC0-\\\\xD6\\\\xD8-\\\\xF6\\\\xF8-\\\\u02C1\\\\u02C6-\\\\u02D1\\\\u02E0-\\\\u02E4\\\\u02EC\\\\u02EE\\\\u0370-\\\\u0374\\\\u0376\\\\u0377\\\\u037A-\\\\u037D\\\\u037F\\\\u0386\\\\u0388-\\\\u038A\\\\u038C\\\\u038E-\\\\u03A1\\\\u03A3-\\\\u03F5\\\\u03F7-\\\\u0481\\\\u048A-\\\\u052F\\\\u0531-\\\\u0556\\\\u0559\\\\u0561-\\\\u0587\\\\u05D0-\\\\u05EA\\\\u05F0-\\\\u05F2\\\\u0620-\\\\u064A\\\\u066E\\\\u066F\\\\u0671-\\\\u06D3\\\\u06D5\\\\u06E5\\\\u06E6\\\\u06EE\\\\u06EF\\\\u06FA-\\\\u06FC\\\\u06FF\\\\u0710\\\\u0712-\\\\u072F\\\\u074D-\\\\u07A5\\\\u07B1\\\\u07CA-\\\\u07EA\\\\u07F4\\\\u07F5\\\\u07FA\\\\u0800-\\\\u0815\\\\u081A\\\\u0824\\\\u0828\\\\u0840-\\\\u0858\\\\u08A0-\\\\u08B2\\\\u0904-\\\\u0939\\\\u093D\\\\u0950\\\\u0958-\\\\u0961\\\\u0971-\\\\u0980\\\\u0985-\\\\u098C\\\\u098F\\\\u0990\\\\u0993-\\\\u09A8\\\\u09AA-\\\\u09B0\\\\u09B2\\\\u09B6-\\\\u09B9\\\\u09BD\\\\u09CE\\\\u09DC\\\\u09DD\\\\u09DF-\\\\u09E1\\\\u09F0\\\\u09F1\\\\u0A05-\\\\u0A0A\\\\u0A0F\\\\u0A10\\\\u0A13-\\\\u0A28\\\\u0A2A-\\\\u0A30\\\\u0A32\\\\u0A33\\\\u0A35\\\\u0A36\\\\u0A38\\\\u0A39\\\\u0A59-\\\\u0A5C\\\\u0A5E\\\\u0A72-\\\\u0A74\\\\u0A85-\\\\u0A8D\\\\u0A8F-\\\\u0A91\\\\u0A93-\\\\u0AA8\\\\u0AAA-\\\\u0AB0\\\\u0AB2\\\\u0AB3\\\\u0AB5-\\\\u0AB9\\\\u0ABD\\\\u0AD0\\\\u0AE0\\\\u0AE1\\\\u0B05-\\\\u0B0C\\\\u0B0F\\\\u0B10\\\\u0B13-\\\\u0B28\\\\u0B2A-\\\\u0B30\\\\u0B32\\\\u0B33\\\\u0B35-\\\\u0B39\\\\u0B3D\\\\u0B5C\\\\u0B5D\\\\u0B5F-\\\\u0B61\\\\u0B71\\\\u0B83\\\\u0B85-\\\\u0B8A\\\\u0B8E-\\\\u0B90\\\\u0B92-\\\\u0B95\\\\u0B99\\\\u0B9A\\\\u0B9C\\\\u0B9E\\\\u0B9F\\\\u0BA3\\\\u0BA4\\\\u0BA8-\\\\u0BAA\\\\u0BAE-\\\\u0BB9\\\\u0BD0\\\\u0C05-\\\\u0C0C\\\\u0C0E-\\\\u0C10\\\\u0C12-\\\\u0C28\\\\u0C2A-\\\\u0C39\\\\u0C3D\\\\u0C58\\\\u0C59\\\\u0C60\\\\u0C61\\\\u0C85-\\\\u0C8C\\\\u0C8E-\\\\u0C90\\\\u0C92-\\\\u0CA8\\\\u0CAA-\\\\u0CB3\\\\u0CB5-\\\\u0CB9\\\\u0CBD\\\\u0CDE\\\\u0CE0\\\\u0CE1\\\\u0CF1\\\\u0CF2\\\\u0D05-\\\\u0D0C\\\\u0D0E-\\\\u0D10\\\\u0D12-\\\\u0D3A\\\\u0D3D\\\\u0D4E\\\\u0D60\\\\u0D61\\\\u0D7A-\\\\u0D7F\\\\u0D85-\\\\u0D96\\\\u0D9A-\\\\u0DB1\\\\u0DB3-\\\\u0DBB\\\\u0DBD\\\\u0DC0-\\\\u0DC6\\\\u0E01-\\\\u0E30\\\\u0E32\\\\u0E33\\\\u0E40-\\\\u0E46\\\\u0E81\\\\u0E82\\\\u0E84\\\\u0E87\\\\u0E88\\\\u0E8A\\\\u0E8D\\\\u0E94-\\\\u0E97\\\\u0E99-\\\\u0E9F\\\\u0EA1-\\\\u0EA3\\\\u0EA5\\\\u0EA7\\\\u0EAA\\\\u0EAB\\\\u0EAD-\\\\u0EB0\\\\u0EB2\\\\u0EB3\\\\u0EBD\\\\u0EC0-\\\\u0EC4\\\\u0EC6\\\\u0EDC-\\\\u0EDF\\\\u0F00\\\\u0F40-\\\\u0F47\\\\u0F49-\\\\u0F6C\\\\u0F88-\\\\u0F8C\\\\u1000-\\\\u102A\\\\u103F\\\\u1050-\\\\u1055\\\\u105A-\\\\u105D\\\\u1061\\\\u1065\\\\u1066\\\\u106E-\\\\u1070\\\\u1075-\\\\u1081\\\\u108E\\\\u10A0-\\\\u10C5\\\\u10C7\\\\u10CD\\\\u10D0-\\\\u10FA\\\\u10FC-\\\\u1248\\\\u124A-\\\\u124D\\\\u1250-\\\\u1256\\\\u1258\\\\u125A-\\\\u125D\\\\u1260-\\\\u1288\\\\u128A-\\\\u128D\\\\u1290-\\\\u12B0\\\\u12B2-\\\\u12B5\\\\u12B8-\\\\u12BE\\\\u12C0\\\\u12C2-\\\\u12C5\\\\u12C8-\\\\u12D6\\\\u12D8-\\\\u1310\\\\u1312-\\\\u1315\\\\u1318-\\\\u135A\\\\u1380-\\\\u138F\\\\u13A0-\\\\u13F4\\\\u1401-\\\\u166C\\\\u166F-\\\\u167F\\\\u1681-\\\\u169A\\\\u16A0-\\\\u16EA\\\\u16EE-\\\\u16F8\\\\u1700-\\\\u170C\\\\u170E-\\\\u1711\\\\u1720-\\\\u1731\\\\u1740-\\\\u1751\\\\u1760-\\\\u176C\\\\u176E-\\\\u1770\\\\u1780-\\\\u17B3\\\\u17D7\\\\u17DC\\\\u1820-\\\\u1877\\\\u1880-\\\\u18A8\\\\u18AA\\\\u18B0-\\\\u18F5\\\\u1900-\\\\u191E\\\\u1950-\\\\u196D\\\\u1970-\\\\u1974\\\\u1980-\\\\u19AB\\\\u19C1-\\\\u19C7\\\\u1A00-\\\\u1A16\\\\u1A20-\\\\u1A54\\\\u1AA7\\\\u1B05-\\\\u1B33\\\\u1B45-\\\\u1B4B\\\\u1B83-\\\\u1BA0\\\\u1BAE\\\\u1BAF\\\\u1BBA-\\\\u1BE5\\\\u1C00-\\\\u1C23\\\\u1C4D-\\\\u1C4F\\\\u1C5A-\\\\u1C7D\\\\u1CE9-\\\\u1CEC\\\\u1CEE-\\\\u1CF1\\\\u1CF5\\\\u1CF6\\\\u1D00-\\\\u1DBF\\\\u1E00-\\\\u1F15\\\\u1F18-\\\\u1F1D\\\\u1F20-\\\\u1F45\\\\u1F48-\\\\u1F4D\\\\u1F50-\\\\u1F57\\\\u1F59\\\\u1F5B\\\\u1F5D\\\\u1F5F-\\\\u1F7D\\\\u1F80-\\\\u1FB4\\\\u1FB6-\\\\u1FBC\\\\u1FBE\\\\u1FC2-\\\\u1FC4\\\\u1FC6-\\\\u1FCC\\\\u1FD0-\\\\u1FD3\\\\u1FD6-\\\\u1FDB\\\\u1FE0-\\\\u1FEC\\\\u1FF2-\\\\u1FF4\\\\u1FF6-\\\\u1FFC\\\\u2071\\\\u207F\\\\u2090-\\\\u209C\\\\u2102\\\\u2107\\\\u210A-\\\\u2113\\\\u2115\\\\u2119-\\\\u211D\\\\u2124\\\\u2126\\\\u2128\\\\u212A-\\\\u212D\\\\u212F-\\\\u2139\\\\u213C-\\\\u213F\\\\u2145-\\\\u2149\\\\u214E\\\\u2160-\\\\u2188\\\\u2C00-\\\\u2C2E\\\\u2C30-\\\\u2C5E\\\\u2C60-\\\\u2CE4\\\\u2CEB-\\\\u2CEE\\\\u2CF2\\\\u2CF3\\\\u2D00-\\\\u2D25\\\\u2D27\\\\u2D2D\\\\u2D30-\\\\u2D67\\\\u2D6F\\\\u2D80-\\\\u2D96\\\\u2DA0-\\\\u2DA6\\\\u2DA8-\\\\u2DAE\\\\u2DB0-\\\\u2DB6\\\\u2DB8-\\\\u2DBE\\\\u2DC0-\\\\u2DC6\\\\u2DC8-\\\\u2DCE\\\\u2DD0-\\\\u2DD6\\\\u2DD8-\\\\u2DDE\\\\u2E2F\\\\u3005-\\\\u3007\\\\u3021-\\\\u3029\\\\u3031-\\\\u3035\\\\u3038-\\\\u303C\\\\u3041-\\\\u3096\\\\u309D-\\\\u309F\\\\u30A1-\\\\u30FA\\\\u30FC-\\\\u30FF\\\\u3105-\\\\u312D\\\\u3131-\\\\u318E\\\\u31A0-\\\\u31BA\\\\u31F0-\\\\u31FF\\\\u3400-\\\\u4DB5\\\\u4E00-\\\\u9FCC\\\\uA000-\\\\uA48C\\\\uA4D0-\\\\uA4FD\\\\uA500-\\\\uA60C\\\\uA610-\\\\uA61F\\\\uA62A\\\\uA62B\\\\uA640-\\\\uA66E\\\\uA67F-\\\\uA69D\\\\uA6A0-\\\\uA6EF\\\\uA717-\\\\uA71F\\\\uA722-\\\\uA788\\\\uA78B-\\\\uA78E\\\\uA790-\\\\uA7AD\\\\uA7B0\\\\uA7B1\\\\uA7F7-\\\\uA801\\\\uA803-\\\\uA805\\\\uA807-\\\\uA80A\\\\uA80C-\\\\uA822\\\\uA840-\\\\uA873\\\\uA882-\\\\uA8B3\\\\uA8F2-\\\\uA8F7\\\\uA8FB\\\\uA90A-\\\\uA925\\\\uA930-\\\\uA946\\\\uA960-\\\\uA97C\\\\uA984-\\\\uA9B2\\\\uA9CF\\\\uA9E0-\\\\uA9E4\\\\uA9E6-\\\\uA9EF\\\\uA9FA-\\\\uA9FE\\\\uAA00-\\\\uAA28\\\\uAA40-\\\\uAA42\\\\uAA44-\\\\uAA4B\\\\uAA60-\\\\uAA76\\\\uAA7A\\\\uAA7E-\\\\uAAAF\\\\uAAB1\\\\uAAB5\\\\uAAB6\\\\uAAB9-\\\\uAABD\\\\uAAC0\\\\uAAC2\\\\uAADB-\\\\uAADD\\\\uAAE0-\\\\uAAEA\\\\uAAF2-\\\\uAAF4\\\\uAB01-\\\\uAB06\\\\uAB09-\\\\uAB0E\\\\uAB11-\\\\uAB16\\\\uAB20-\\\\uAB26\\\\uAB28-\\\\uAB2E\\\\uAB30-\\\\uAB5A\\\\uAB5C-\\\\uAB5F\\\\uAB64\\\\uAB65\\\\uABC0-\\\\uABE2\\\\uAC00-\\\\uD7A3\\\\uD7B0-\\\\uD7C6\\\\uD7CB-\\\\uD7FB\\\\uF900-\\\\uFA6D\\\\uFA70-\\\\uFAD9\\\\uFB00-\\\\uFB06\\\\uFB13-\\\\uFB17\\\\uFB1D\\\\uFB1F-\\\\uFB28\\\\uFB2A-\\\\uFB36\\\\uFB38-\\\\uFB3C\\\\uFB3E\\\\uFB40\\\\uFB41\\\\uFB43\\\\uFB44\\\\uFB46-\\\\uFBB1\\\\uFBD3-\\\\uFD3D\\\\uFD50-\\\\uFD8F\\\\uFD92-\\\\uFDC7\\\\uFDF0-\\\\uFDFB\\\\uFE70-\\\\uFE74\\\\uFE76-\\\\uFEFC\\\\uFF21-\\\\uFF3A\\\\uFF41-\\\\uFF5A\\\\uFF66-\\\\uFFBE\\\\uFFC2-\\\\uFFC7\\\\uFFCA-\\\\uFFCF\\\\uFFD2-\\\\uFFD7\\\\uFFDA-\\\\uFFDC]\"),D$=new RegExp(\"[\\\\xAA\\\\xB5\\\\xBA\\\\xC0-\\\\xD6\\\\xD8-\\\\xF6\\\\xF8-\\\\u02C1\\\\u02C6-\\\\u02D1\\\\u02E0-\\\\u02E4\\\\u02EC\\\\u02EE\\\\u0300-\\\\u0374\\\\u0376\\\\u0377\\\\u037A-\\\\u037D\\\\u037F\\\\u0386\\\\u0388-\\\\u038A\\\\u038C\\\\u038E-\\\\u03A1\\\\u03A3-\\\\u03F5\\\\u03F7-\\\\u0481\\\\u0483-\\\\u0487\\\\u048A-\\\\u052F\\\\u0531-\\\\u0556\\\\u0559\\\\u0561-\\\\u0587\\\\u0591-\\\\u05BD\\\\u05BF\\\\u05C1\\\\u05C2\\\\u05C4\\\\u05C5\\\\u05C7\\\\u05D0-\\\\u05EA\\\\u05F0-\\\\u05F2\\\\u0610-\\\\u061A\\\\u0620-\\\\u0669\\\\u066E-\\\\u06D3\\\\u06D5-\\\\u06DC\\\\u06DF-\\\\u06E8\\\\u06EA-\\\\u06FC\\\\u06FF\\\\u0710-\\\\u074A\\\\u074D-\\\\u07B1\\\\u07C0-\\\\u07F5\\\\u07FA\\\\u0800-\\\\u082D\\\\u0840-\\\\u085B\\\\u08A0-\\\\u08B2\\\\u08E4-\\\\u0963\\\\u0966-\\\\u096F\\\\u0971-\\\\u0983\\\\u0985-\\\\u098C\\\\u098F\\\\u0990\\\\u0993-\\\\u09A8\\\\u09AA-\\\\u09B0\\\\u09B2\\\\u09B6-\\\\u09B9\\\\u09BC-\\\\u09C4\\\\u09C7\\\\u09C8\\\\u09CB-\\\\u09CE\\\\u09D7\\\\u09DC\\\\u09DD\\\\u09DF-\\\\u09E3\\\\u09E6-\\\\u09F1\\\\u0A01-\\\\u0A03\\\\u0A05-\\\\u0A0A\\\\u0A0F\\\\u0A10\\\\u0A13-\\\\u0A28\\\\u0A2A-\\\\u0A30\\\\u0A32\\\\u0A33\\\\u0A35\\\\u0A36\\\\u0A38\\\\u0A39\\\\u0A3C\\\\u0A3E-\\\\u0A42\\\\u0A47\\\\u0A48\\\\u0A4B-\\\\u0A4D\\\\u0A51\\\\u0A59-\\\\u0A5C\\\\u0A5E\\\\u0A66-\\\\u0A75\\\\u0A81-\\\\u0A83\\\\u0A85-\\\\u0A8D\\\\u0A8F-\\\\u0A91\\\\u0A93-\\\\u0AA8\\\\u0AAA-\\\\u0AB0\\\\u0AB2\\\\u0AB3\\\\u0AB5-\\\\u0AB9\\\\u0ABC-\\\\u0AC5\\\\u0AC7-\\\\u0AC9\\\\u0ACB-\\\\u0ACD\\\\u0AD0\\\\u0AE0-\\\\u0AE3\\\\u0AE6-\\\\u0AEF\\\\u0B01-\\\\u0B03\\\\u0B05-\\\\u0B0C\\\\u0B0F\\\\u0B10\\\\u0B13-\\\\u0B28\\\\u0B2A-\\\\u0B30\\\\u0B32\\\\u0B33\\\\u0B35-\\\\u0B39\\\\u0B3C-\\\\u0B44\\\\u0B47\\\\u0B48\\\\u0B4B-\\\\u0B4D\\\\u0B56\\\\u0B57\\\\u0B5C\\\\u0B5D\\\\u0B5F-\\\\u0B63\\\\u0B66-\\\\u0B6F\\\\u0B71\\\\u0B82\\\\u0B83\\\\u0B85-\\\\u0B8A\\\\u0B8E-\\\\u0B90\\\\u0B92-\\\\u0B95\\\\u0B99\\\\u0B9A\\\\u0B9C\\\\u0B9E\\\\u0B9F\\\\u0BA3\\\\u0BA4\\\\u0BA8-\\\\u0BAA\\\\u0BAE-\\\\u0BB9\\\\u0BBE-\\\\u0BC2\\\\u0BC6-\\\\u0BC8\\\\u0BCA-\\\\u0BCD\\\\u0BD0\\\\u0BD7\\\\u0BE6-\\\\u0BEF\\\\u0C00-\\\\u0C03\\\\u0C05-\\\\u0C0C\\\\u0C0E-\\\\u0C10\\\\u0C12-\\\\u0C28\\\\u0C2A-\\\\u0C39\\\\u0C3D-\\\\u0C44\\\\u0C46-\\\\u0C48\\\\u0C4A-\\\\u0C4D\\\\u0C55\\\\u0C56\\\\u0C58\\\\u0C59\\\\u0C60-\\\\u0C63\\\\u0C66-\\\\u0C6F\\\\u0C81-\\\\u0C83\\\\u0C85-\\\\u0C8C\\\\u0C8E-\\\\u0C90\\\\u0C92-\\\\u0CA8\\\\u0CAA-\\\\u0CB3\\\\u0CB5-\\\\u0CB9\\\\u0CBC-\\\\u0CC4\\\\u0CC6-\\\\u0CC8\\\\u0CCA-\\\\u0CCD\\\\u0CD5\\\\u0CD6\\\\u0CDE\\\\u0CE0-\\\\u0CE3\\\\u0CE6-\\\\u0CEF\\\\u0CF1\\\\u0CF2\\\\u0D01-\\\\u0D03\\\\u0D05-\\\\u0D0C\\\\u0D0E-\\\\u0D10\\\\u0D12-\\\\u0D3A\\\\u0D3D-\\\\u0D44\\\\u0D46-\\\\u0D48\\\\u0D4A-\\\\u0D4E\\\\u0D57\\\\u0D60-\\\\u0D63\\\\u0D66-\\\\u0D6F\\\\u0D7A-\\\\u0D7F\\\\u0D82\\\\u0D83\\\\u0D85-\\\\u0D96\\\\u0D9A-\\\\u0DB1\\\\u0DB3-\\\\u0DBB\\\\u0DBD\\\\u0DC0-\\\\u0DC6\\\\u0DCA\\\\u0DCF-\\\\u0DD4\\\\u0DD6\\\\u0DD8-\\\\u0DDF\\\\u0DE6-\\\\u0DEF\\\\u0DF2\\\\u0DF3\\\\u0E01-\\\\u0E3A\\\\u0E40-\\\\u0E4E\\\\u0E50-\\\\u0E59\\\\u0E81\\\\u0E82\\\\u0E84\\\\u0E87\\\\u0E88\\\\u0E8A\\\\u0E8D\\\\u0E94-\\\\u0E97\\\\u0E99-\\\\u0E9F\\\\u0EA1-\\\\u0EA3\\\\u0EA5\\\\u0EA7\\\\u0EAA\\\\u0EAB\\\\u0EAD-\\\\u0EB9\\\\u0EBB-\\\\u0EBD\\\\u0EC0-\\\\u0EC4\\\\u0EC6\\\\u0EC8-\\\\u0ECD\\\\u0ED0-\\\\u0ED9\\\\u0EDC-\\\\u0EDF\\\\u0F00\\\\u0F18\\\\u0F19\\\\u0F20-\\\\u0F29\\\\u0F35\\\\u0F37\\\\u0F39\\\\u0F3E-\\\\u0F47\\\\u0F49-\\\\u0F6C\\\\u0F71-\\\\u0F84\\\\u0F86-\\\\u0F97\\\\u0F99-\\\\u0FBC\\\\u0FC6\\\\u1000-\\\\u1049\\\\u1050-\\\\u109D\\\\u10A0-\\\\u10C5\\\\u10C7\\\\u10CD\\\\u10D0-\\\\u10FA\\\\u10FC-\\\\u1248\\\\u124A-\\\\u124D\\\\u1250-\\\\u1256\\\\u1258\\\\u125A-\\\\u125D\\\\u1260-\\\\u1288\\\\u128A-\\\\u128D\\\\u1290-\\\\u12B0\\\\u12B2-\\\\u12B5\\\\u12B8-\\\\u12BE\\\\u12C0\\\\u12C2-\\\\u12C5\\\\u12C8-\\\\u12D6\\\\u12D8-\\\\u1310\\\\u1312-\\\\u1315\\\\u1318-\\\\u135A\\\\u135D-\\\\u135F\\\\u1380-\\\\u138F\\\\u13A0-\\\\u13F4\\\\u1401-\\\\u166C\\\\u166F-\\\\u167F\\\\u1681-\\\\u169A\\\\u16A0-\\\\u16EA\\\\u16EE-\\\\u16F8\\\\u1700-\\\\u170C\\\\u170E-\\\\u1714\\\\u1720-\\\\u1734\\\\u1740-\\\\u1753\\\\u1760-\\\\u176C\\\\u176E-\\\\u1770\\\\u1772\\\\u1773\\\\u1780-\\\\u17D3\\\\u17D7\\\\u17DC\\\\u17DD\\\\u17E0-\\\\u17E9\\\\u180B-\\\\u180D\\\\u1810-\\\\u1819\\\\u1820-\\\\u1877\\\\u1880-\\\\u18AA\\\\u18B0-\\\\u18F5\\\\u1900-\\\\u191E\\\\u1920-\\\\u192B\\\\u1930-\\\\u193B\\\\u1946-\\\\u196D\\\\u1970-\\\\u1974\\\\u1980-\\\\u19AB\\\\u19B0-\\\\u19C9\\\\u19D0-\\\\u19D9\\\\u1A00-\\\\u1A1B\\\\u1A20-\\\\u1A5E\\\\u1A60-\\\\u1A7C\\\\u1A7F-\\\\u1A89\\\\u1A90-\\\\u1A99\\\\u1AA7\\\\u1AB0-\\\\u1ABD\\\\u1B00-\\\\u1B4B\\\\u1B50-\\\\u1B59\\\\u1B6B-\\\\u1B73\\\\u1B80-\\\\u1BF3\\\\u1C00-\\\\u1C37\\\\u1C40-\\\\u1C49\\\\u1C4D-\\\\u1C7D\\\\u1CD0-\\\\u1CD2\\\\u1CD4-\\\\u1CF6\\\\u1CF8\\\\u1CF9\\\\u1D00-\\\\u1DF5\\\\u1DFC-\\\\u1F15\\\\u1F18-\\\\u1F1D\\\\u1F20-\\\\u1F45\\\\u1F48-\\\\u1F4D\\\\u1F50-\\\\u1F57\\\\u1F59\\\\u1F5B\\\\u1F5D\\\\u1F5F-\\\\u1F7D\\\\u1F80-\\\\u1FB4\\\\u1FB6-\\\\u1FBC\\\\u1FBE\\\\u1FC2-\\\\u1FC4\\\\u1FC6-\\\\u1FCC\\\\u1FD0-\\\\u1FD3\\\\u1FD6-\\\\u1FDB\\\\u1FE0-\\\\u1FEC\\\\u1FF2-\\\\u1FF4\\\\u1FF6-\\\\u1FFC\\\\u200C\\\\u200D\\\\u203F\\\\u2040\\\\u2054\\\\u2071\\\\u207F\\\\u2090-\\\\u209C\\\\u20D0-\\\\u20DC\\\\u20E1\\\\u20E5-\\\\u20F0\\\\u2102\\\\u2107\\\\u210A-\\\\u2113\\\\u2115\\\\u2119-\\\\u211D\\\\u2124\\\\u2126\\\\u2128\\\\u212A-\\\\u212D\\\\u212F-\\\\u2139\\\\u213C-\\\\u213F\\\\u2145-\\\\u2149\\\\u214E\\\\u2160-\\\\u2188\\\\u2C00-\\\\u2C2E\\\\u2C30-\\\\u2C5E\\\\u2C60-\\\\u2CE4\\\\u2CEB-\\\\u2CF3\\\\u2D00-\\\\u2D25\\\\u2D27\\\\u2D2D\\\\u2D30-\\\\u2D67\\\\u2D6F\\\\u2D7F-\\\\u2D96\\\\u2DA0-\\\\u2DA6\\\\u2DA8-\\\\u2DAE\\\\u2DB0-\\\\u2DB6\\\\u2DB8-\\\\u2DBE\\\\u2DC0-\\\\u2DC6\\\\u2DC8-\\\\u2DCE\\\\u2DD0-\\\\u2DD6\\\\u2DD8-\\\\u2DDE\\\\u2DE0-\\\\u2DFF\\\\u2E2F\\\\u3005-\\\\u3007\\\\u3021-\\\\u302F\\\\u3031-\\\\u3035\\\\u3038-\\\\u303C\\\\u3041-\\\\u3096\\\\u3099\\\\u309A\\\\u309D-\\\\u309F\\\\u30A1-\\\\u30FA\\\\u30FC-\\\\u30FF\\\\u3105-\\\\u312D\\\\u3131-\\\\u318E\\\\u31A0-\\\\u31BA\\\\u31F0-\\\\u31FF\\\\u3400-\\\\u4DB5\\\\u4E00-\\\\u9FCC\\\\uA000-\\\\uA48C\\\\uA4D0-\\\\uA4FD\\\\uA500-\\\\uA60C\\\\uA610-\\\\uA62B\\\\uA640-\\\\uA66F\\\\uA674-\\\\uA67D\\\\uA67F-\\\\uA69D\\\\uA69F-\\\\uA6F1\\\\uA717-\\\\uA71F\\\\uA722-\\\\uA788\\\\uA78B-\\\\uA78E\\\\uA790-\\\\uA7AD\\\\uA7B0\\\\uA7B1\\\\uA7F7-\\\\uA827\\\\uA840-\\\\uA873\\\\uA880-\\\\uA8C4\\\\uA8D0-\\\\uA8D9\\\\uA8E0-\\\\uA8F7\\\\uA8FB\\\\uA900-\\\\uA92D\\\\uA930-\\\\uA953\\\\uA960-\\\\uA97C\\\\uA980-\\\\uA9C0\\\\uA9CF-\\\\uA9D9\\\\uA9E0-\\\\uA9FE\\\\uAA00-\\\\uAA36\\\\uAA40-\\\\uAA4D\\\\uAA50-\\\\uAA59\\\\uAA60-\\\\uAA76\\\\uAA7A-\\\\uAAC2\\\\uAADB-\\\\uAADD\\\\uAAE0-\\\\uAAEF\\\\uAAF2-\\\\uAAF6\\\\uAB01-\\\\uAB06\\\\uAB09-\\\\uAB0E\\\\uAB11-\\\\uAB16\\\\uAB20-\\\\uAB26\\\\uAB28-\\\\uAB2E\\\\uAB30-\\\\uAB5A\\\\uAB5C-\\\\uAB5F\\\\uAB64\\\\uAB65\\\\uABC0-\\\\uABEA\\\\uABEC\\\\uABED\\\\uABF0-\\\\uABF9\\\\uAC00-\\\\uD7A3\\\\uD7B0-\\\\uD7C6\\\\uD7CB-\\\\uD7FB\\\\uF900-\\\\uFA6D\\\\uFA70-\\\\uFAD9\\\\uFB00-\\\\uFB06\\\\uFB13-\\\\uFB17\\\\uFB1D-\\\\uFB28\\\\uFB2A-\\\\uFB36\\\\uFB38-\\\\uFB3C\\\\uFB3E\\\\uFB40\\\\uFB41\\\\uFB43\\\\uFB44\\\\uFB46-\\\\uFBB1\\\\uFBD3-\\\\uFD3D\\\\uFD50-\\\\uFD8F\\\\uFD92-\\\\uFDC7\\\\uFDF0-\\\\uFDFB\\\\uFE00-\\\\uFE0F\\\\uFE20-\\\\uFE2D\\\\uFE33\\\\uFE34\\\\uFE4D-\\\\uFE4F\\\\uFE70-\\\\uFE74\\\\uFE76-\\\\uFEFC\\\\uFF10-\\\\uFF19\\\\uFF21-\\\\uFF3A\\\\uFF3F\\\\uFF41-\\\\uFF5A\\\\uFF66-\\\\uFFBE\\\\uFFC2-\\\\uFFC7\\\\uFFCA-\\\\uFFCF\\\\uFFD2-\\\\uFFD7\\\\uFFDA-\\\\uFFDC]\");function C$(t,e){if(!t)throw new Error(\"ASSERT: \"+e)}function F$(t){return t>=48&&t<=57}function S$(t){return\"0123456789abcdefABCDEF\".includes(t)}function $$(t){return\"01234567\".includes(t)}function T$(t){return 32===t||9===t||11===t||12===t||160===t||t>=5760&&[5760,6158,8192,8193,8194,8195,8196,8197,8198,8199,8200,8201,8202,8239,8287,12288,65279].includes(t)}function B$(t){return 10===t||13===t||8232===t||8233===t}function z$(t){return 36===t||95===t||t>=65&&t<=90||t>=97&&t<=122||92===t||t>=128&&E$.test(String.fromCharCode(t))}function N$(t){return 36===t||95===t||t>=65&&t<=90||t>=97&&t<=122||t>=48&&t<=57||92===t||t>=128&&D$.test(String.fromCharCode(t))}const O$={if:1,in:1,do:1,var:1,for:1,new:1,try:1,let:1,this:1,else:1,case:1,void:1,with:1,enum:1,while:1,break:1,catch:1,throw:1,const:1,yield:1,class:1,super:1,return:1,typeof:1,delete:1,switch:1,export:1,import:1,public:1,static:1,default:1,finally:1,extends:1,package:1,private:1,function:1,continue:1,debugger:1,interface:1,protected:1,instanceof:1,implements:1};function R$(){for(;YS1114111||\"}\"!==t)&&tT({},p$,A$),e<=65535?String.fromCharCode(e):(n=55296+(e-65536>>10),r=56320+(e-65536&1023),String.fromCharCode(n,r))}function q$(){var t,e;for(t=HS.charCodeAt(YS++),e=String.fromCharCode(t),92===t&&(117!==HS.charCodeAt(YS)&&tT({},p$,A$),++YS,(t=U$(\"u\"))&&\"\\\\\"!==t&&z$(t.charCodeAt(0))||tT({},p$,A$),e=t);YS>>=\"===(r=HS.substr(YS,4))?{type:e$,value:r,start:i,end:YS+=4}:\">>>\"===(n=r.substr(0,3))||\"<<=\"===n||\">>=\"===n?{type:e$,value:n,start:i,end:YS+=3}:a===(e=n.substr(0,2))[1]&&\"+-<>&|\".includes(a)||\"=>\"===e?{type:e$,value:e,start:i,end:YS+=2}:(\"//\"===e&&tT({},p$,A$),\"<>=!+-*%&|^/\".includes(a)?(++YS,{type:e$,value:a,start:i,end:YS}):void tT({},p$,A$))}function I$(){var t,e,n;if(C$(F$((n=HS[YS]).charCodeAt(0))||\".\"===n,\"Numeric literal must start with a decimal digit or a decimal point\"),e=YS,t=\"\",\".\"!==n){if(t=HS[YS++],n=HS[YS],\"0\"===t){if(\"x\"===n||\"X\"===n)return++YS,function(t){let e=\"\";for(;YS=0&&tT({},x$,n),{value:n,literal:e}}(),r=function(t,e){let n=t;e.includes(\"u\")&&(n=n.replace(/\\\\u\\{([0-9a-fA-F]+)\\}/g,((t,e)=>{if(parseInt(e,16)<=1114111)return\"x\";tT({},x$)})).replace(/[\\uD800-\\uDBFF][\\uDC00-\\uDFFF]/g,\"x\"));try{new RegExp(n)}catch(t){tT({},x$)}try{return new RegExp(t,e)}catch(t){return null}}(e.value,n.value),{literal:e.literal+n.literal,value:r,regex:{pattern:e.value,flags:n.value},start:t,end:YS}}function H$(){if(R$(),YS>=GS)return{type:JS,start:YS,end:YS};const t=HS.charCodeAt(YS);return z$(t)?P$():40===t||41===t||59===t?j$():39===t||34===t?function(){var t,e,n,r,i=\"\",o=!1;for(C$(\"'\"===(t=HS[YS])||'\"'===t,\"String literal must starts with a quote\"),e=YS,++YS;YS(C$(e\":case\"<=\":case\">=\":case\"instanceof\":case\"in\":e=7;break;case\"<<\":case\">>\":case\">>>\":e=8;break;case\"+\":case\"-\":e=9;break;case\"*\":case\"/\":case\"%\":e=11}return e}function yT(){var t,e;return t=function(){var t,e,n,r,i,o,a,s,u,l;if(t=VS,u=gT(),0===(i=mT(r=VS)))return u;for(r.prec=i,Y$(),e=[t,VS],o=[u,r,a=gT()];(i=mT(VS))>0;){for(;o.length>2&&i<=o[o.length-2].prec;)a=o.pop(),s=o.pop().value,u=o.pop(),e.pop(),n=V$(s,u,a),o.push(n);(r=Y$()).prec=i,o.push(r),e.push(VS),n=gT(),o.push(n)}for(n=o[l=o.length-1],e.pop();l>1;)e.pop(),n=V$(o[l-1].value,o[l-2],n),l-=2;return n}(),rT(\"?\")&&(Y$(),e=yT(),nT(\":\"),t=function(t,e,n){const r=new IS(a$);return r.test=t,r.consequent=e,r.alternate=n,r}(t,e,yT())),t}function vT(){const t=yT();if(rT(\",\"))throw new Error(M$);return t}function _T(t){YS=0,GS=(HS=t).length,VS=null,G$();const e=vT();if(VS.type!==JS)throw new Error(\"Unexpect token after expression.\");return e}var xT={NaN:\"NaN\",E:\"Math.E\",LN2:\"Math.LN2\",LN10:\"Math.LN10\",LOG2E:\"Math.LOG2E\",LOG10E:\"Math.LOG10E\",PI:\"Math.PI\",SQRT1_2:\"Math.SQRT1_2\",SQRT2:\"Math.SQRT2\",MIN_VALUE:\"Number.MIN_VALUE\",MAX_VALUE:\"Number.MAX_VALUE\"};function bT(t){function e(e,n,r){return i=>function(e,n,r,i){let o=t(n[0]);return r&&(o=r+\"(\"+o+\")\",0===r.lastIndexOf(\"new \",0)&&(o=\"(\"+o+\")\")),o+\".\"+e+(i<0?\"\":0===i?\"()\":\"(\"+n.slice(1).map(t).join(\",\")+\")\")}(e,i,n,r)}const n=\"new Date\",r=\"String\",i=\"RegExp\";return{isNaN:\"Number.isNaN\",isFinite:\"Number.isFinite\",abs:\"Math.abs\",acos:\"Math.acos\",asin:\"Math.asin\",atan:\"Math.atan\",atan2:\"Math.atan2\",ceil:\"Math.ceil\",cos:\"Math.cos\",exp:\"Math.exp\",floor:\"Math.floor\",hypot:\"Math.hypot\",log:\"Math.log\",max:\"Math.max\",min:\"Math.min\",pow:\"Math.pow\",random:\"Math.random\",round:\"Math.round\",sin:\"Math.sin\",sqrt:\"Math.sqrt\",tan:\"Math.tan\",clamp:function(e){e.length<3&&s(\"Missing arguments to clamp function.\"),e.length>3&&s(\"Too many arguments to clamp function.\");const n=e.map(t);return\"Math.max(\"+n[1]+\", Math.min(\"+n[2]+\",\"+n[0]+\"))\"},now:\"Date.now\",utc:\"Date.UTC\",datetime:n,date:e(\"getDate\",n,0),day:e(\"getDay\",n,0),year:e(\"getFullYear\",n,0),month:e(\"getMonth\",n,0),hours:e(\"getHours\",n,0),minutes:e(\"getMinutes\",n,0),seconds:e(\"getSeconds\",n,0),milliseconds:e(\"getMilliseconds\",n,0),time:e(\"getTime\",n,0),timezoneoffset:e(\"getTimezoneOffset\",n,0),utcdate:e(\"getUTCDate\",n,0),utcday:e(\"getUTCDay\",n,0),utcyear:e(\"getUTCFullYear\",n,0),utcmonth:e(\"getUTCMonth\",n,0),utchours:e(\"getUTCHours\",n,0),utcminutes:e(\"getUTCMinutes\",n,0),utcseconds:e(\"getUTCSeconds\",n,0),utcmilliseconds:e(\"getUTCMilliseconds\",n,0),length:e(\"length\",null,-1),parseFloat:\"parseFloat\",parseInt:\"parseInt\",upper:e(\"toUpperCase\",r,0),lower:e(\"toLowerCase\",r,0),substring:e(\"substring\",r),split:e(\"split\",r),trim:e(\"trim\",r,0),regexp:i,test:e(\"test\",i),if:function(e){e.length<3&&s(\"Missing arguments to if function.\"),e.length>3&&s(\"Too many arguments to if function.\");const n=e.map(t);return\"(\"+n[0]+\"?\"+n[1]+\":\"+n[2]+\")\"}}}function wT(t){const e=(t=t||{}).allowed?Bt(t.allowed):{},n=t.forbidden?Bt(t.forbidden):{},r=t.constants||xT,i=(t.functions||bT)(h),o=t.globalvar,a=t.fieldvar,u=J(o)?o:t=>`${o}[\"${t}\"]`;let l={},c={},f=0;function h(t){if(xt(t))return t;const e=d[t.type];return null==e&&s(\"Unsupported type: \"+t.type),e(t)}const d={Literal:t=>t.raw,Identifier:t=>{const i=t.name;return f>0?i:lt(n,i)?s(\"Illegal identifier: \"+i):lt(r,i)?r[i]:lt(e,i)?i:(l[i]=1,u(i))},MemberExpression:t=>{const e=!t.computed,n=h(t.object);e&&(f+=1);const r=h(t.property);return n===a&&(c[function(t){const e=t&&t.length-1;return e&&('\"'===t[0]&&'\"'===t[e]||\"'\"===t[0]&&\"'\"===t[e])?t.slice(1,-1):t}(r)]=1),e&&(f-=1),n+(e?\".\"+r:\"[\"+r+\"]\")},CallExpression:t=>{\"Identifier\"!==t.callee.type&&s(\"Illegal callee type: \"+t.callee.type);const e=t.callee.name,n=t.arguments,r=lt(i,e)&&i[e];return r||s(\"Unrecognized function: \"+e),J(r)?r(n):r+\"(\"+n.map(h).join(\",\")+\")\"},ArrayExpression:t=>\"[\"+t.elements.map(h).join(\",\")+\"]\",BinaryExpression:t=>\"(\"+h(t.left)+\" \"+t.operator+\" \"+h(t.right)+\")\",UnaryExpression:t=>\"(\"+t.operator+h(t.argument)+\")\",ConditionalExpression:t=>\"(\"+h(t.test)+\"?\"+h(t.consequent)+\":\"+h(t.alternate)+\")\",LogicalExpression:t=>\"(\"+h(t.left)+t.operator+h(t.right)+\")\",ObjectExpression:t=>\"{\"+t.properties.map(h).join(\",\")+\"}\",Property:t=>{f+=1;const e=h(t.key);return f-=1,e+\":\"+h(t.value)}};function p(t){const e={code:h(t),globals:Object.keys(l),fields:Object.keys(c)};return l={},c={},e}return p.functions=i,p.constants=r,p}const kT=Symbol(\"vega_selection_getter\");function AT(t){return t.getter&&t.getter[kT]||(t.getter=l(t.field),t.getter[kT]=!0),t.getter}const MT=\"intersect\",ET=\"union\",DT=\"_vgsid_\",CT=l(DT),FT=\"E\",ST=\"R\",$T=\"R-E\",TT=\"R-LE\",BT=\"R-RE\",zT=\"index:unit\";function NT(t,e){for(var n,r,i=e.fields,o=e.values,a=i.length,s=0;s1?e-1:0),r=1;re.includes(t))):e},R_union:function(t,e){var n=S(e[0]),r=S(e[1]);return n>r&&(n=e[1],r=e[0]),t.length?(t[0]>n&&(t[0]=n),t[1]r&&(n=e[1],r=e[0]),t.length?rr&&(t[1]=r),t):[n,r]}};function qT(t,e,n,r){e[0].type!==BS&&s(\"First argument to selection functions must be a string literal.\");const i=e[0].value,o=\"unit\",a=\"@\"+o,u=\":\"+i;(e.length>=2&&F(e).value)!==MT||lt(r,a)||(r[a]=n.getData(i).indataRef(n,o)),lt(r,u)||(r[u]=n.getData(i).tuplesRef())}function PT(t){const e=this.context.data[t];return e?e.values.value:[]}const jT=t=>function(e,n){return this.context.dataflow.locale()[t](n)(e)},IT=jT(\"format\"),WT=jT(\"timeFormat\"),HT=jT(\"utcFormat\"),YT=jT(\"timeParse\"),GT=jT(\"utcParse\"),VT=new Date(2e3,0,1);function XT(t,e,n){return Number.isInteger(t)&&Number.isInteger(e)?(VT.setYear(2e3),VT.setMonth(t),VT.setDate(e),WT.call(this,VT,n)):\"\"}const JT=\"%\",ZT=\"$\";function QT(t,e,n,r){e[0].type!==BS&&s(\"First argument to data functions must be a string literal.\");const i=e[0].value,o=\":\"+i;if(!lt(o,r))try{r[o]=n.getData(i).tuplesRef()}catch(t){}}function KT(t,e,n,r){if(e[0].type===BS)tB(n,r,e[0].value);else for(t in n.scales)tB(n,r,t)}function tB(t,e,n){const r=JT+n;if(!lt(e,r))try{e[r]=t.scaleRef(n)}catch(t){}}function eB(t,e){if(J(t))return t;if(xt(t)){const n=e.scales[t];return n&&function(t){return t&&!0===t[ip]}(n.value)?n.value:void 0}}function nB(t,e,n){e.__bandwidth=t=>t&&t.bandwidth?t.bandwidth():0,n._bandwidth=KT,n._range=KT,n._scale=KT;const r=e=>\"_[\"+(e.type===BS?Ct(JT+e.value):Ct(JT)+\"+\"+t(e))+\"]\";return{_bandwidth:t=>`this.__bandwidth(${r(t[0])})`,_range:t=>`${r(t[0])}.range()`,_scale:e=>`${r(e[0])}(${t(e[1])})`}}function rB(t,e){return function(n,r,i){if(n){const e=eB(n,(i||this).context);return e&&e.path[t](r)}return e(r)}}const iB=rB(\"area\",(function(t){return Tw=new se,pw(t,Bw),2*Tw})),oB=rB(\"bounds\",(function(t){var e,n,r,i,o,a,s;if(kw=ww=-(xw=bw=1/0),Fw=[],pw(t,sk),n=Fw.length){for(Fw.sort(mk),e=1,o=[r=Fw[0]];egk(r[0],r[1])&&(r[1]=i[1]),gk(i[0],r[1])>gk(r[0],r[1])&&(r[0]=i[0])):o.push(r=i);for(a=-1/0,e=0,r=o[n=o.length-1];e<=n;r=i,++e)i=o[e],(s=gk(r[1],i[0]))>a&&(a=s,xw=i[0],ww=r[1])}return Fw=Sw=null,xw===1/0||bw===1/0?[[NaN,NaN],[NaN,NaN]]:[[xw,bw],[ww,kw]]})),aB=rB(\"centroid\",(function(t){Hw=Yw=Gw=Vw=Xw=Jw=Zw=Qw=0,Kw=new se,tk=new se,ek=new se,pw(t,vk);var e=+Kw,n=+tk,r=+ek,i=Kb(e,n,r);return ifB(t,e)}const dB={};function pB(t){return k(t)||ArrayBuffer.isView(t)?t:null}function gB(t){return pB(t)||(xt(t)?t:null)}const mB=t=>t.data;function yB(t,e){const n=PT.call(e,t);return n.root&&n.root.lookup||{}}const vB=()=>\"undefined\"!=typeof window&&window||null;function _B(t,e,n){if(!t)return[];const[r,i]=t,o=(new Vg).set(r[0],r[1],i[0],i[1]);return B_(n||this.context.dataflow.scenegraph().root,o,function(t){let e=null;if(t){const n=V(t.marktype),r=V(t.markname);e=t=>(!n.length||n.some((e=>t.marktype===e)))&&(!r.length||r.some((e=>t.name===e)))}return e}(e))}const xB={random:()=>t.random(),cumulativeNormal:hs,cumulativeLogNormal:vs,cumulativeUniform:As,densityNormal:fs,densityLogNormal:ys,densityUniform:ks,quantileNormal:ds,quantileLogNormal:_s,quantileUniform:Ms,sampleNormal:cs,sampleLogNormal:ms,sampleUniform:ws,isArray:k,isBoolean:gt,isDate:mt,isDefined:t=>void 0!==t,isNumber:vt,isObject:A,isRegExp:_t,isString:xt,isTuple:ma,isValid:t=>null!=t&&t==t,toBoolean:Ft,toDate:t=>$t(t),toNumber:S,toString:Tt,indexof:function(t){for(var e=arguments.length,n=new Array(e>1?e-1:0),r=1;r1?e-1:0),r=1;r1?e-1:0),r=1;r1?e-1:0),r=1;rat(t),inScope:function(t){const e=this.context.group;let n=!1;if(e)for(;t;){if(t===e){n=!0;break}t=t.mark.group}return n},intersect:_B,clampRange:X,pinchDistance:function(t){const e=t.touches,n=e[0].clientX-e[1].clientX,r=e[0].clientY-e[1].clientY;return Math.hypot(n,r)},pinchAngle:function(t){const e=t.touches;return Math.atan2(e[0].clientY-e[1].clientY,e[0].clientX-e[1].clientX)},screen:function(){const t=vB();return t?t.screen:{}},containerSize:function(){const t=this.context.dataflow,e=t.container&&t.container();return e?[e.clientWidth,e.clientHeight]:[void 0,void 0]},windowSize:function(){const t=vB();return t?[t.innerWidth,t.innerHeight]:[void 0,void 0]},bandspace:function(t,e,n){return $d(t||0,e||0,n||0)},setdata:function(t,e){const n=this.context.dataflow,r=this.context.data[t].input;return n.pulse(r,n.changeset().remove(p).insert(e)),1},pathShape:function(t){let e=null;return function(n){return n?yg(n,e=e||ag(t)):t}},panLinear:R,panLog:U,panPow:L,panSymlog:q,zoomLinear:j,zoomLog:I,zoomPow:W,zoomSymlog:H,encode:function(t,e,n){if(t){const n=this.context.dataflow,r=t.mark.source;n.pulse(r,n.changeset().encode(t,e))}return void 0!==n?n:t},modify:function(t,e,n,r,i,o){const a=this.context.dataflow,s=this.context.data[t],u=s.input,l=a.stamp();let c,f,h=s.changes;if(!1===a._trigger||!(u.value.length||e||r))return 0;if((!h||h.stamp{s.modified=!0,a.pulse(u,h).run()}),!0,1)),n&&(c=!0===n?p:k(n)||ma(n)?n:hB(n),h.remove(c)),e&&h.insert(e),r&&(c=hB(r),u.value.some(c)?h.remove(c):h.insert(r)),i)for(f in o)h.modify(i,f,o[f]);return 1},lassoAppend:function(t,e,n){let r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:5;const i=(t=V(t))[t.length-1];return void 0===i||Math.hypot(i[0]-e,i[1]-n)>r?[...t,[e,n]]:t},lassoPath:function(t){return V(t).reduce(((e,n,r)=>{let[i,o]=n;return e+(0==r?`M ${i},${o} `:r===t.length-1?\" Z\":`L ${i},${o} `)}),\"\")},intersectLasso:function(t,e,n){const{x:r,y:i,mark:o}=n,a=(new Vg).set(Number.MAX_SAFE_INTEGER,Number.MAX_SAFE_INTEGER,Number.MIN_SAFE_INTEGER,Number.MIN_SAFE_INTEGER);for(const[t,n]of e)ta.x2&&(a.x2=t),na.y2&&(a.y2=n);return a.translate(r,i),_B([[a.x1,a.y1],[a.x2,a.y2]],t,o).filter((t=>function(t,e,n){let r=0;for(let i=0,o=n.length-1;ie!=s>e&&t<(a-u)*(e-l)/(s-l)+u&&r++}return 1&r}(t.x,t.y,e)))}},bB=[\"view\",\"item\",\"group\",\"xy\",\"x\",\"y\"],wB=\"this.\",kB={},AB={forbidden:[\"_\"],allowed:[\"datum\",\"event\",\"item\"],fieldvar:\"datum\",globalvar:t=>`_[${Ct(ZT+t)}]`,functions:function(t){const e=bT(t);bB.forEach((t=>e[t]=\"event.vega.\"+t));for(const t in xB)e[t]=wB+t;return ot(e,nB(t,xB,kB)),e},constants:xT,visitors:kB},MB=wT(AB);function EB(t,e,n){return 1===arguments.length?xB[t]:(xB[t]=e,n&&(kB[t]=n),MB&&(MB.functions[t]=wB+t),this)}function DB(t,e){const n={};let r;try{r=_T(t=xt(t)?t:Ct(t)+\"\")}catch(e){s(\"Expression parse error: \"+t)}r.visit((t=>{if(t.type!==RS)return;const r=t.callee.name,i=AB.visitors[r];i&&i(r,t.arguments,e,n)}));const i=MB(r);return i.globals.forEach((t=>{const r=ZT+t;!lt(n,r)&&e.getSignal(t)&&(n[r]=e.signalRef(t))})),{$expr:ot({code:i.code},e.options.ast?{ast:r}:null),$fields:i.fields,$params:n}}EB(\"bandwidth\",(function(t,e){const n=eB(t,(e||this).context);return n&&n.bandwidth?n.bandwidth():0}),KT),EB(\"copy\",(function(t,e){const n=eB(t,(e||this).context);return n?n.copy():void 0}),KT),EB(\"domain\",(function(t,e){const n=eB(t,(e||this).context);return n?n.domain():[]}),KT),EB(\"range\",(function(t,e){const n=eB(t,(e||this).context);return n&&n.range?n.range():[]}),KT),EB(\"invert\",(function(t,e,n){const r=eB(t,(n||this).context);return r?k(e)?(r.invertRange||r.invert)(e):(r.invert||r.invertExtent)(e):void 0}),KT),EB(\"scale\",(function(t,e,n){const r=eB(t,(n||this).context);return r?r(e):void 0}),KT),EB(\"gradient\",(function(t,e,n,r,i){t=eB(t,(i||this).context);const o=Qp(e,n);let a=t.domain(),s=a[0],u=F(a),l=f;return u-s?l=_p(t,s,u):t=(t.interpolator?ap(\"sequential\")().interpolator(t.interpolator()):ap(\"linear\")().interpolate(t.interpolate()).range(t.range())).domain([s=0,u=1]),t.ticks&&(a=t.ticks(+r||15),s!==a[0]&&a.unshift(s),u!==F(a)&&a.push(u)),a.forEach((e=>o.stop(l(e),t(e)))),o}),KT),EB(\"geoArea\",iB,KT),EB(\"geoBounds\",oB,KT),EB(\"geoCentroid\",aB,KT),EB(\"geoShape\",(function(t,e,n){const r=eB(t,(n||this).context);return function(t){return r?r.path.context(t)(e):\"\"}}),KT),EB(\"geoScale\",(function(t,e){const n=eB(t,(e||this).context);return n&&n.scale()}),KT),EB(\"indata\",(function(t,e,n){const r=this.context.data[t][\"index:\"+e],i=r?r.value.get(n):void 0;return i?i.count:i}),(function(t,e,n,r){e[0].type!==BS&&s(\"First argument to indata must be a string literal.\"),e[1].type!==BS&&s(\"Second argument to indata must be a string literal.\");const i=e[0].value,o=e[1].value,a=\"@\"+o;lt(a,r)||(r[a]=n.getData(i).indataRef(n,o))})),EB(\"data\",PT,QT),EB(\"treePath\",(function(t,e,n){const r=yB(t,this),i=r[e],o=r[n];return i&&o?i.path(o).map(mB):void 0}),QT),EB(\"treeAncestors\",(function(t,e){const n=yB(t,this)[e];return n?n.ancestors().map(mB):void 0}),QT),EB(\"vlSelectionTest\",(function(t,e,n){for(var r,i,o,a,s,u=this.context.data[t],l=u?u.values.value:[],c=u?u[zT]&&u[zT].value:void 0,f=n===MT,h=l.length,d=0;d(t[o[n].field]=e,t)),{}))}else u=DT,l=CT(i),(f=(c=v[u]||(v[u]={}))[s]||(c[s]=[])).push(l),n&&(f=_[s]||(_[s]=[])).push({[DT]:l});if(e=e||ET,v[DT]?v[DT]=LT[`${DT}_${e}`](...Object.values(v[DT])):Object.keys(v).forEach((t=>{v[t]=Object.keys(v[t]).map((e=>v[t][e])).reduce(((n,r)=>void 0===n?r:LT[`${x[t]}_${e}`](n,r)))})),y=Object.keys(_),n&&y.length){v[r?\"vlPoint\":\"vlMulti\"]=e===ET?{or:y.reduce(((t,e)=>(t.push(..._[e]),t)),[])}:{and:y.map((t=>({or:_[t]})))}}return v}),qT),EB(\"vlSelectionTuples\",(function(t,e){return t.map((t=>ot(e.fields?{values:e.fields.map((e=>AT(e)(t.datum)))}:{[DT]:CT(t.datum)},e)))}));const CB=Bt([\"rule\"]),FB=Bt([\"group\",\"image\",\"rect\"]);function SB(t){return(t+\"\").toLowerCase()}function $B(t,e,n){n.endsWith(\";\")||(n=\"return(\"+n+\");\");const r=Function(...e.concat(n));return t&&t.functions?r.bind(t.functions):r}var TB={operator:(t,e)=>$B(t,[\"_\"],e.code),parameter:(t,e)=>$B(t,[\"datum\",\"_\"],e.code),event:(t,e)=>$B(t,[\"event\"],e.code),handler:(t,e)=>$B(t,[\"_\",\"event\"],`var datum=event.item&&event.item.datum;return ${e.code};`),encode:(t,e)=>{const{marktype:n,channels:r}=e;let i=\"var o=item,datum=o.datum,m=0,$;\";for(const t in r){const e=\"o[\"+Ct(t)+\"]\";i+=`$=${r[t].code};if(${e}!==$)${e}=$,m=1;`}return i+=function(t,e){let n=\"\";return CB[e]||(t.x2&&(t.x?(FB[e]&&(n+=\"if(o.x>o.x2)$=o.x,o.x=o.x2,o.x2=$;\"),n+=\"o.width=o.x2-o.x;\"):n+=\"o.x=o.x2-(o.width||0);\"),t.xc&&(n+=\"o.x=o.xc-(o.width||0)/2;\"),t.y2&&(t.y?(FB[e]&&(n+=\"if(o.y>o.y2)$=o.y,o.y=o.y2,o.y2=$;\"),n+=\"o.height=o.y2-o.y;\"):n+=\"o.y=o.y2-(o.height||0);\"),t.yc&&(n+=\"o.y=o.yc-(o.height||0)/2;\")),n}(r,n),i+=\"return m;\",$B(t,[\"item\",\"_\"],i)},codegen:{get(t){const e=`[${t.map(Ct).join(\"][\")}]`,n=Function(\"_\",`return _${e};`);return n.path=e,n},comparator(t,e){let n;const r=Function(\"a\",\"b\",\"var u, v; return \"+t.map(((t,r)=>{const i=e[r];let o,a;return t.path?(o=`a${t.path}`,a=`b${t.path}`):((n=n||{})[\"f\"+r]=t,o=`this.f${r}(a)`,a=`this.f${r}(b)`),function(t,e,n,r){return`((u = ${t}) < (v = ${e}) || u == null) && v != null ? ${n}\\n : (u > v || v == null) && u != null ? ${r}\\n : ((v = v instanceof Date ? +v : v), (u = u instanceof Date ? +u : u)) !== u && v === v ? ${n}\\n : v !== v && u === u ? ${r} : `}(o,a,-i,i)})).join(\"\")+\"0;\");return n?r.bind(n):r}}};function BB(t,e,n){if(!t||!A(t))return t;for(let r,i=0,o=zB.length;it&&t.$tupleid?ya:t));return e.fn[n]||(e.fn[n]=Q(r,t.$order,e.expr.codegen))}},{key:\"$context\",parse:function(t,e){return e}},{key:\"$subflow\",parse:function(t,e){const n=t.$subflow;return function(t,r,i){const o=e.fork().parse(n),a=o.get(n.operators[0].id),s=o.signals.parent;return s&&s.set(i),a.detachSubflow=()=>e.detach(o),a}}},{key:\"$tupleid\",parse:function(){return ya}}];const NB={skip:!0};function OB(t,e,n,r){return new RB(t,e,n,r)}function RB(t,e,n,r){this.dataflow=t,this.transforms=e,this.events=t.events.bind(t),this.expr=r||TB,this.signals={},this.scales={},this.nodes={},this.data={},this.fn={},n&&(this.functions=Object.create(n),this.functions.context=this)}function UB(t){this.dataflow=t.dataflow,this.transforms=t.transforms,this.events=t.events,this.expr=t.expr,this.signals=Object.create(t.signals),this.scales=Object.create(t.scales),this.nodes=Object.create(t.nodes),this.data=Object.create(t.data),this.fn=Object.create(t.fn),t.functions&&(this.functions=Object.create(t.functions),this.functions.context=this)}function LB(t,e){t&&(null==e?t.removeAttribute(\"aria-label\"):t.setAttribute(\"aria-label\",e))}RB.prototype=UB.prototype={fork(){const t=new UB(this);return(this.subcontext||(this.subcontext=[])).push(t),t},detach(t){this.subcontext=this.subcontext.filter((e=>e!==t));const e=Object.keys(t.nodes);for(const n of e)t.nodes[n]._targets=null;for(const n of e)t.nodes[n].detach();t.nodes=null},get(t){return this.nodes[t]},set(t,e){return this.nodes[t]=e},add(t,e){const n=this,r=n.dataflow,i=t.value;if(n.set(t.id,e),function(t){return\"collect\"===SB(t)}(t.type)&&i&&(i.$ingest?r.ingest(e,i.$ingest,i.$format):i.$request?r.preload(e,i.$request,i.$format):r.pulse(e,r.changeset().insert(i))),t.root&&(n.root=e),t.parent){let i=n.get(t.parent.$ref);i?(r.connect(i,[e]),e.targets().add(i)):(n.unresolved=n.unresolved||[]).push((()=>{i=n.get(t.parent.$ref),r.connect(i,[e]),e.targets().add(i)}))}if(t.signal&&(n.signals[t.signal]=e),t.scale&&(n.scales[t.scale]=e),t.data)for(const r in t.data){const i=n.data[r]||(n.data[r]={});t.data[r].forEach((t=>i[t]=e))}},resolve(){return(this.unresolved||[]).forEach((t=>t())),delete this.unresolved,this},operator(t,e){this.add(t,this.dataflow.add(t.value,e))},transform(t,e){this.add(t,this.dataflow.add(this.transforms[SB(e)]))},stream(t,e){this.set(t.id,e)},update(t,e,n,r,i){this.dataflow.on(e,n,r,i,t.options)},operatorExpression(t){return this.expr.operator(this,t)},parameterExpression(t){return this.expr.parameter(this,t)},eventExpression(t){return this.expr.event(this,t)},handlerExpression(t){return this.expr.handler(this,t)},encodeExpression(t){return this.expr.encode(this,t)},parse:function(t){const e=this,n=t.operators||[];return t.background&&(e.background=t.background),t.eventConfig&&(e.eventConfig=t.eventConfig),t.locale&&(e.locale=t.locale),n.forEach((t=>e.parseOperator(t))),n.forEach((t=>e.parseOperatorParameters(t))),(t.streams||[]).forEach((t=>e.parseStream(t))),(t.updates||[]).forEach((t=>e.parseUpdate(t))),e.resolve()},parseOperator:function(t){const e=this;!function(t){return\"operator\"===SB(t)}(t.type)&&t.type?e.transform(t,t.type):e.operator(t,t.update?e.operatorExpression(t.update):null)},parseOperatorParameters:function(t){const e=this;if(t.params){const n=e.get(t.id);n||s(\"Invalid operator id: \"+t.id),e.dataflow.connect(n,n.parameters(e.parseParameters(t.params),t.react,t.initonly))}},parseParameters:function(t,e){e=e||{};const n=this;for(const r in t){const i=t[r];e[r]=k(i)?i.map((t=>BB(t,n,e))):BB(i,n,e)}return e},parseStream:function(t){var e,n=this,r=null!=t.filter?n.eventExpression(t.filter):void 0,i=null!=t.stream?n.get(t.stream):void 0;t.source?i=n.events(t.source,t.type,r):t.merge&&(i=(e=t.merge.map((t=>n.get(t))))[0].merge.apply(e[0],e.slice(1))),t.between&&(e=t.between.map((t=>n.get(t))),i=i.between(e[0],e[1])),t.filter&&(i=i.filter(r)),null!=t.throttle&&(i=i.throttle(+t.throttle)),null!=t.debounce&&(i=i.debounce(+t.debounce)),null==i&&s(\"Invalid stream definition: \"+JSON.stringify(t)),t.consume&&i.consume(!0),n.stream(t,i)},parseUpdate:function(t){var e,n=this,r=A(r=t.source)?r.$ref:r,i=n.get(r),o=t.update,a=void 0;i||s(\"Source not defined: \"+t.source),e=t.target&&t.target.$expr?n.eventExpression(t.target.$expr):n.get(t.target),o&&o.$expr&&(o.$params&&(a=n.parseParameters(o.$params)),o=n.handlerExpression(o.$expr)),n.update(t,i,e,o,a)},getState:function(t){var e=this,n={};if(t.signals){var r=n.signals={};Object.keys(e.signals).forEach((n=>{const i=e.signals[n];t.signals(n,i)&&(r[n]=i.value)}))}if(t.data){var i=n.data={};Object.keys(e.data).forEach((n=>{const r=e.data[n];t.data(n,r)&&(i[n]=r.input.value)}))}return e.subcontext&&!1!==t.recurse&&(n.subcontext=e.subcontext.map((e=>e.getState(t)))),n},setState:function(t){var e=this,n=e.dataflow,r=t.data,i=t.signals;Object.keys(i||{}).forEach((t=>{n.update(e.signals[t],i[t],NB)})),Object.keys(r||{}).forEach((t=>{n.pulse(e.data[t].input,n.changeset().remove(p).insert(r[t]))})),(t.subcontext||[]).forEach(((t,n)=>{const r=e.subcontext[n];r&&r.setState(t)}))}};const qB=\"default\";function PB(t,e){const n=t.globalCursor()?\"undefined\"!=typeof document&&document.body:t.container();if(n)return null==e?n.style.removeProperty(\"cursor\"):n.style.cursor=e}function jB(t,e){var n=t._runtime.data;return lt(n,e)||s(\"Unrecognized data set: \"+e),n[e]}function IB(t,e){Aa(e)||s(\"Second argument to changes must be a changeset.\");const n=jB(this,t);return n.modified=!0,this.pulse(n.input,e)}function WB(t){var e=t.padding();return Math.max(0,t._viewWidth+e.left+e.right)}function HB(t){var e=t.padding();return Math.max(0,t._viewHeight+e.top+e.bottom)}function YB(t){var e=t.padding(),n=t._origin;return[e.left+n[0],e.top+n[1]]}function GB(t,e,n){var r,i,o=t._renderer,a=o&&o.canvas();return a&&(i=YB(t),(r=av(e.changedTouches?e.changedTouches[0]:e,a))[0]-=i[0],r[1]-=i[1]),e.dataflow=t,e.item=n,e.vega=function(t,e,n){const r=e?\"group\"===e.mark.marktype?e:e.mark.group:null;function i(t){var n,i=r;if(t)for(n=e;n;n=n.mark.group)if(n.mark.name===t){i=n;break}return i&&i.mark&&i.mark.interactive?i:{}}function o(t){if(!t)return n;xt(t)&&(t=i(t));const e=n.slice();for(;t;)e[0]-=t.x||0,e[1]-=t.y||0,t=t.mark&&t.mark.group;return e}return{view:rt(t),item:rt(e||{}),group:i,xy:o,x:t=>o(t)[0],y:t=>o(t)[1]}}(t,n,r),e}const VB=\"view\",XB={trap:!1};function JB(t,e,n,r){t._eventListeners.push({type:n,sources:V(e),handler:r})}function ZB(t,e,n){const r=t._eventConfig&&t._eventConfig[e];return!(!1===r||A(r)&&!r[n])||(t.warn(`Blocked ${e} ${n} event listener.`),!1)}function QB(t){return t.item}function KB(t){return t.item.mark.source}function tz(t){return function(e,n){return n.vega.view().changeset().encode(n.item,t)}}function ez(t,e,n){const r=document.createElement(t);for(const t in e)r.setAttribute(t,e[t]);return null!=n&&(r.textContent=n),r}const nz=\"vega-bind\",rz=\"vega-bind-name\",iz=\"vega-bind-radio\";function oz(t,e,n,r){const i=n.event||\"input\",o=()=>t.update(e.value);r.signal(n.signal,e.value),e.addEventListener(i,o),JB(r,e,i,o),t.set=t=>{e.value=t,e.dispatchEvent(function(t){return\"undefined\"!=typeof Event?new Event(t):{type:t}}(i))}}function az(t,e,n,r){const i=r.signal(n.signal),o=ez(\"div\",{class:nz}),a=\"radio\"===n.input?o:o.appendChild(ez(\"label\"));a.appendChild(ez(\"span\",{class:rz},n.name||n.signal)),e.appendChild(o);let s=sz;switch(n.input){case\"checkbox\":s=uz;break;case\"select\":s=lz;break;case\"radio\":s=cz;break;case\"range\":s=fz}s(t,a,n,i)}function sz(t,e,n,r){const i=ez(\"input\");for(const t in n)\"signal\"!==t&&\"element\"!==t&&i.setAttribute(\"input\"===t?\"type\":t,n[t]);i.setAttribute(\"name\",n.signal),i.value=r,e.appendChild(i),i.addEventListener(\"input\",(()=>t.update(i.value))),t.elements=[i],t.set=t=>i.value=t}function uz(t,e,n,r){const i={type:\"checkbox\",name:n.signal};r&&(i.checked=!0);const o=ez(\"input\",i);e.appendChild(o),o.addEventListener(\"change\",(()=>t.update(o.checked))),t.elements=[o],t.set=t=>o.checked=!!t||null}function lz(t,e,n,r){const i=ez(\"select\",{name:n.signal}),o=n.labels||[];n.options.forEach(((t,e)=>{const n={value:t};hz(t,r)&&(n.selected=!0),i.appendChild(ez(\"option\",n,(o[e]||t)+\"\"))})),e.appendChild(i),i.addEventListener(\"change\",(()=>{t.update(n.options[i.selectedIndex])})),t.elements=[i],t.set=t=>{for(let e=0,r=n.options.length;e{const s={type:\"radio\",name:n.signal,value:e};hz(e,r)&&(s.checked=!0);const u=ez(\"input\",s);u.addEventListener(\"change\",(()=>t.update(e)));const l=ez(\"label\",{},(o[a]||e)+\"\");return l.prepend(u),i.appendChild(l),u})),t.set=e=>{const n=t.elements,r=n.length;for(let t=0;t{u.textContent=s.value,t.update(+s.value)};s.addEventListener(\"input\",l),s.addEventListener(\"change\",l),t.elements=[s],t.set=t=>{s.value=t,u.textContent=t}}function hz(t,e){return t===e||t+\"\"==e+\"\"}function dz(t,e,n,r,i,o){return(e=e||new r(t.loader())).initialize(n,WB(t),HB(t),YB(t),i,o).background(t.background())}function pz(t,e){return e?function(){try{e.apply(this,arguments)}catch(e){t.error(e)}}:null}function gz(t,e,n){if(\"string\"==typeof e){if(\"undefined\"==typeof document)return t.error(\"DOM document instance not found.\"),null;if(!(e=document.querySelector(e)))return t.error(\"Signal bind element not found: \"+e),null}if(e&&n)try{e.textContent=\"\"}catch(n){e=null,t.error(n)}return e}const mz=t=>+t||0;function yz(t){return A(t)?{top:mz(t.top),bottom:mz(t.bottom),left:mz(t.left),right:mz(t.right)}:(t=>({top:t,bottom:t,left:t,right:t}))(mz(t))}async function vz(t,e,n,r){const i=T_(e),o=i&&i.headless;return o||s(\"Unrecognized renderer type: \"+e),await t.runAsync(),dz(t,null,null,o,n,r).renderAsync(t._scenegraph.root)}var _z=\"width\",xz=\"height\",bz=\"padding\",wz={skip:!0};function kz(t,e){var n=t.autosize(),r=t.padding();return e-(n&&n.contains===bz?r.left+r.right:0)}function Az(t,e){var n=t.autosize(),r=t.padding();return e-(n&&n.contains===bz?r.top+r.bottom:0)}function Mz(t,e){return e.modified&&k(e.input.value)&&!t.startsWith(\"_:vega:_\")}function Ez(t,e){return!(\"parent\"===t||e instanceof Za.proxy)}function Dz(t,e,n,r){const i=t.element();i&&i.setAttribute(\"title\",function(t){return null==t?\"\":k(t)?Cz(t):A(t)&&!mt(t)?(e=t,Object.keys(e).map((t=>{const n=e[t];return t+\": \"+(k(n)?Cz(n):Fz(n))})).join(\"\\n\")):t+\"\";var e}(r))}function Cz(t){return\"[\"+t.map(Fz).join(\", \")+\"]\"}function Fz(t){return k(t)?\"[â€Ļ]\":A(t)&&!mt(t)?\"{â€Ļ}\":t}function Sz(t,e){const n=this;if(e=e||{},Va.call(n),e.loader&&n.loader(e.loader),e.logger&&n.logger(e.logger),null!=e.logLevel&&n.logLevel(e.logLevel),e.locale||t.locale){const r=ot({},t.locale,e.locale);n.locale(Ro(r.number,r.time))}n._el=null,n._elBind=null,n._renderType=e.renderer||S_.Canvas,n._scenegraph=new Ky;const r=n._scenegraph.root;n._renderer=null,n._tooltip=e.tooltip||Dz,n._redraw=!0,n._handler=(new Sv).scene(r),n._globalCursor=!1,n._preventDefault=!1,n._timers=[],n._eventListeners=[],n._resizeListeners=[],n._eventConfig=function(t){const e=ot({defaults:{}},t),n=(t,e)=>{e.forEach((e=>{k(t[e])&&(t[e]=Bt(t[e]))}))};return n(e.defaults,[\"prevent\",\"allow\"]),n(e,[\"view\",\"window\",\"selector\"]),e}(t.eventConfig),n.globalCursor(n._eventConfig.globalCursor);const i=function(t,e,n){return OB(t,Za,xB,n).parse(e)}(n,t,e.expr);n._runtime=i,n._signals=i.signals,n._bind=(t.bindings||[]).map((t=>({state:null,param:ot({},t)}))),i.root&&i.root.set(r),r.source=i.data.root.input,n.pulse(i.data.root.input,n.changeset().insert(r.items)),n._width=n.width(),n._height=n.height(),n._viewWidth=kz(n,n._width),n._viewHeight=Az(n,n._height),n._origin=[0,0],n._resize=0,n._autosize=1,function(t){var e=t._signals,n=e[_z],r=e[xz],i=e[bz];function o(){t._autosize=t._resize=1}t._resizeWidth=t.add(null,(e=>{t._width=e.size,t._viewWidth=kz(t,e.size),o()}),{size:n}),t._resizeHeight=t.add(null,(e=>{t._height=e.size,t._viewHeight=Az(t,e.size),o()}),{size:r});const a=t.add(null,o,{pad:i});t._resizeWidth.rank=n.rank+1,t._resizeHeight.rank=r.rank+1,a.rank=i.rank+1}(n),function(t){t.add(null,(e=>(t._background=e.bg,t._resize=1,e.bg)),{bg:t._signals.background})}(n),function(t){const e=t._signals.cursor||(t._signals.cursor=t.add({user:qB,item:null}));t.on(t.events(\"view\",\"pointermove\"),e,((t,n)=>{const r=e.value,i=r?xt(r)?r:r.user:qB,o=n.item&&n.item.cursor||null;return r&&i===r.user&&o==r.item?r:{user:i,item:o}})),t.add(null,(function(e){let n=e.cursor,r=this.value;return xt(n)||(r=n.item,n=n.user),PB(t,n&&n!==qB?n:r||n),r}),{cursor:e})}(n),n.description(t.description),e.hover&&n.hover(),e.container&&n.initialize(e.container,e.bind),e.watchPixelRatio&&n._watchPixelRatio()}function $z(t,e){return lt(t._signals,e)?t._signals[e]:s(\"Unrecognized signal name: \"+Ct(e))}function Tz(t,e){const n=(t._targets||[]).filter((t=>t._update&&t._update.handler===e));return n.length?n[0]:null}function Bz(t,e,n,r){let i=Tz(n,r);return i||(i=pz(t,(()=>r(e,n.value))),i.handler=r,t.on(n,null,i)),t}function zz(t,e,n){const r=Tz(e,n);return r&&e._targets.remove(r),t}dt(Sz,Va,{async evaluate(t,e,n){if(await Va.prototype.evaluate.call(this,t,e),this._redraw||this._resize)try{this._renderer&&(this._resize&&(this._resize=0,function(t){var e=YB(t),n=WB(t),r=HB(t);t._renderer.background(t.background()),t._renderer.resize(n,r,e),t._handler.origin(e),t._resizeListeners.forEach((e=>{try{e(n,r)}catch(e){t.error(e)}}))}(this)),await this._renderer.renderAsync(this._scenegraph.root)),this._redraw=!1}catch(t){this.error(t)}return n&&da(this,n),this},dirty(t){this._redraw=!0,this._renderer&&this._renderer.dirty(t)},description(t){if(arguments.length){const e=null!=t?t+\"\":null;return e!==this._desc&&LB(this._el,this._desc=e),this}return this._desc},container(){return this._el},scenegraph(){return this._scenegraph},origin(){return this._origin.slice()},signal(t,e,n){const r=$z(this,t);return 1===arguments.length?r.value:this.update(r,e,n)},width(t){return arguments.length?this.signal(\"width\",t):this.signal(\"width\")},height(t){return arguments.length?this.signal(\"height\",t):this.signal(\"height\")},padding(t){return arguments.length?this.signal(\"padding\",yz(t)):yz(this.signal(\"padding\"))},autosize(t){return arguments.length?this.signal(\"autosize\",t):this.signal(\"autosize\")},background(t){return arguments.length?this.signal(\"background\",t):this.signal(\"background\")},renderer(t){return arguments.length?(T_(t)||s(\"Unrecognized renderer type: \"+t),t!==this._renderType&&(this._renderType=t,this._resetRenderer()),this):this._renderType},tooltip(t){return arguments.length?(t!==this._tooltip&&(this._tooltip=t,this._resetRenderer()),this):this._tooltip},loader(t){return arguments.length?(t!==this._loader&&(Va.prototype.loader.call(this,t),this._resetRenderer()),this):this._loader},resize(){return this._autosize=1,this.touch($z(this,\"autosize\"))},_resetRenderer(){this._renderer&&(this._renderer=null,this.initialize(this._el,this._elBind))},_resizeView:function(t,e,n,r,i,o){this.runAfter((a=>{let s=0;a._autosize=0,a.width()!==n&&(s=1,a.signal(_z,n,wz),a._resizeWidth.skip(!0)),a.height()!==r&&(s=1,a.signal(xz,r,wz),a._resizeHeight.skip(!0)),a._viewWidth!==t&&(a._resize=1,a._viewWidth=t),a._viewHeight!==e&&(a._resize=1,a._viewHeight=e),a._origin[0]===i[0]&&a._origin[1]===i[1]||(a._resize=1,a._origin=i),s&&a.run(\"enter\"),o&&a.runAfter((t=>t.resize()))}),!1,1)},addEventListener(t,e,n){let r=e;return n&&!1===n.trap||(r=pz(this,e),r.raw=e),this._handler.on(t,r),this},removeEventListener(t,e){for(var n,r,i=this._handler.handlers(t),o=i.length;--o>=0;)if(r=i[o].type,n=i[o].handler,t===r&&(e===n||e===n.raw)){this._handler.off(r,n);break}return this},addResizeListener(t){const e=this._resizeListeners;return e.includes(t)||e.push(t),this},removeResizeListener(t){var e=this._resizeListeners,n=e.indexOf(t);return n>=0&&e.splice(n,1),this},addSignalListener(t,e){return Bz(this,t,$z(this,t),e)},removeSignalListener(t,e){return zz(this,$z(this,t),e)},addDataListener(t,e){return Bz(this,t,jB(this,t).values,e)},removeDataListener(t,e){return zz(this,jB(this,t).values,e)},globalCursor(t){if(arguments.length){if(this._globalCursor!==!!t){const e=PB(this,null);this._globalCursor=!!t,e&&PB(this,e)}return this}return this._globalCursor},preventDefault(t){return arguments.length?(this._preventDefault=t,this):this._preventDefault},timer:function(t,e){this._timers.push(function(t,e,n){var r=new lD,i=e;return null==e?(r.restart(t,e,n),r):(r._restart=r.restart,r.restart=function(t,e,n){e=+e,n=null==n?sD():+n,r._restart((function o(a){a+=i,r._restart(o,i+=e,n),t(a)}),e,n)},r.restart(t,e,n),r)}((function(e){t({timestamp:Date.now(),elapsed:e})}),e))},events:function(t,e,n){var r,i=this,o=new Ba(n),a=function(n,r){i.runAsync(null,(()=>{t===VB&&function(t,e){var n=t._eventConfig.defaults,r=n.prevent,i=n.allow;return!1!==r&&!0!==i&&(!0===r||!1===i||(r?r[e]:i?!i[e]:t.preventDefault()))}(i,e)&&n.preventDefault(),o.receive(GB(i,n,r))}))};if(\"timer\"===t)ZB(i,\"timer\",e)&&i.timer(a,e);else if(t===VB)ZB(i,\"view\",e)&&i.addEventListener(e,a,XB);else if(\"window\"===t?ZB(i,\"window\",e)&&\"undefined\"!=typeof window&&(r=[window]):\"undefined\"!=typeof document&&ZB(i,\"selector\",e)&&(r=Array.from(document.querySelectorAll(t))),r){for(var s=0,u=r.length;s=0;)a[t].stop();for(t=u.length;--t>=0;)for(e=(n=u[t]).sources.length;--e>=0;)n.sources[e].removeEventListener(n.type,n.handler);for(o&&o.call(this,this._handler,null,null,null),t=s.length;--t>=0;)i=s[t].type,r=s[t].handler,this._handler.off(i,r);return this},hover:function(t,e){return e=[e||\"update\",(t=[t||\"hover\"])[0]],this.on(this.events(\"view\",\"pointerover\",QB),KB,tz(t)),this.on(this.events(\"view\",\"pointerout\",QB),KB,tz(e)),this},data:function(t,e){return arguments.length<2?jB(this,t).values.value:IB.call(this,t,Ma().remove(p).insert(e))},change:IB,insert:function(t,e){return IB.call(this,t,Ma().insert(e))},remove:function(t,e){return IB.call(this,t,Ma().remove(e))},scale:function(t){var e=this._runtime.scales;return lt(e,t)||s(\"Unrecognized scale or projection: \"+t),e[t].value},initialize:function(t,e){const n=this,r=n._renderType,i=n._eventConfig.bind,o=T_(r);t=n._el=t?gz(n,t,!0):null,function(t){const e=t.container();e&&(e.setAttribute(\"role\",\"graphics-document\"),e.setAttribute(\"aria-roleDescription\",\"visualization\"),LB(e,t.description()))}(n),o||n.error(\"Unrecognized renderer type: \"+r);const a=o.handler||Sv,s=t?o.renderer:o.headless;return n._renderer=s?dz(n,n._renderer,t,s):null,n._handler=function(t,e,n,r){const i=new r(t.loader(),pz(t,t.tooltip())).scene(t.scenegraph().root).initialize(n,YB(t),t);return e&&e.handlers().forEach((t=>{i.on(t.type,t.handler)})),i}(n,n._handler,t,a),n._redraw=!0,t&&\"none\"!==i&&(e=e?n._elBind=gz(n,e,!0):t.appendChild(ez(\"form\",{class:\"vega-bindings\"})),n._bind.forEach((t=>{t.param.element&&\"container\"!==i&&(t.element=gz(n,t.param.element,!!t.param.input))})),n._bind.forEach((t=>{!function(t,e,n){if(!e)return;const r=n.param;let i=n.state;i||(i=n.state={elements:null,active:!1,set:null,update:e=>{e!=t.signal(r.signal)&&t.runAsync(null,(()=>{i.source=!0,t.signal(r.signal,e)}))}},r.debounce&&(i.update=it(r.debounce,i.update))),(null==r.input&&r.element?oz:az)(i,e,r,t),i.active||(t.on(t._signals[r.signal],null,(()=>{i.source?i.source=!1:i.set(t.signal(r.signal))})),i.active=!0)}(n,t.element||e,t)}))),n},toImageURL:async function(t,e){t!==S_.Canvas&&t!==S_.SVG&&t!==S_.PNG&&s(\"Unrecognized image type: \"+t);const n=await vz(this,t,e);return t===S_.SVG?function(t,e){const n=new Blob([t],{type:e});return window.URL.createObjectURL(n)}(n.svg(),\"image/svg+xml\"):n.canvas().toDataURL(\"image/png\")},toCanvas:async function(t,e){return(await vz(this,S_.Canvas,t,e)).canvas()},toSVG:async function(t){return(await vz(this,S_.SVG,t)).svg()},getState:function(t){return this._runtime.getState(t||{data:Mz,signals:Ez,recurse:!0})},setState:function(t){return this.runAsync(null,(e=>{e._trigger=!1,e._runtime.setState(t)}),(t=>{t._trigger=!0})),this},_watchPixelRatio:function(){if(\"canvas\"===this.renderer()&&this._renderer._canvas){let t=null;const e=()=>{null!=t&&t();const n=matchMedia(`(resolution: ${window.devicePixelRatio}dppx)`);n.addEventListener(\"change\",e),t=()=>{n.removeEventListener(\"change\",e)},this._renderer._canvas.getContext(\"2d\").pixelRatio=window.devicePixelRatio||1,this._redraw=!0,this._resize=1,this.resize().runAsync()};e()}}});const Nz=\"view\",Oz=\"[\",Rz=\"]\",Uz=\"{\",Lz=\"}\",qz=\":\",Pz=\",\",jz=\"@\",Iz=\">\",Wz=/[[\\]{}]/,Hz={\"*\":1,arc:1,area:1,group:1,image:1,line:1,path:1,rect:1,rule:1,shape:1,symbol:1,text:1,trail:1};let Yz,Gz;function Vz(t,e,n){return Yz=e||Nz,Gz=n||Hz,Jz(t.trim()).map(Zz)}function Xz(t,e,n,r,i){const o=t.length;let a,s=0;for(;e' after between selector: \"+t;n=n.map(Zz);const i=Zz(t.slice(1).trim());if(i.between)return{between:n,stream:i};i.between=n;return i}(t):function(t){const e={source:Yz},n=[];let r,i,o=[0,0],a=0,s=0,u=t.length,l=0;if(t[u-1]===Lz){if(l=t.lastIndexOf(Uz),!(l>=0))throw\"Unmatched right brace: \"+t;try{o=function(t){const e=t.split(Pz);if(!t.length||e.length>2)throw t;return e.map((e=>{const n=+e;if(n!=n)throw t;return n}))}(t.substring(l+1,u-1))}catch(e){throw\"Invalid throttle specification: \"+t}u=(t=t.slice(0,l).trim()).length,l=0}if(!u)throw t;t[0]===jz&&(a=++l);r=Xz(t,l,qz),r1?(e.type=n[1],a?e.markname=n[0].slice(1):!function(t){return Gz[t]}(n[0])?e.source=n[0]:e.marktype=n[0]):e.type=n[0];\"!\"===e.type.slice(-1)&&(e.consume=!0,e.type=e.type.slice(0,-1));null!=i&&(e.filter=i);o[0]&&(e.throttle=o[0]);o[1]&&(e.debounce=o[1]);return e}(t)}function Qz(t){return A(t)?t:{type:t||\"pad\"}}const Kz=t=>+t||0,tN=t=>({top:t,bottom:t,left:t,right:t});function eN(t){return A(t)?t.signal?t:{top:Kz(t.top),bottom:Kz(t.bottom),left:Kz(t.left),right:Kz(t.right)}:tN(Kz(t))}const nN=t=>A(t)&&!k(t)?ot({},t):{value:t};function rN(t,e,n,r){if(null!=n){return A(n)&&!k(n)||k(n)&&n.length&&A(n[0])?t.update[e]=n:t[r||\"enter\"][e]={value:n},1}return 0}function iN(t,e,n){for(const n in e)rN(t,n,e[n]);for(const e in n)rN(t,e,n[e],\"update\")}function oN(t,e,n){for(const r in e)n&<(n,r)||(t[r]=ot(t[r]||{},e[r]));return t}function aN(t,e){return e&&(e.enter&&e.enter[t]||e.update&&e.update[t])}const sN=\"mark\",uN=\"frame\",lN=\"scope\",cN=\"axis\",fN=\"axis-domain\",hN=\"axis-grid\",dN=\"axis-label\",pN=\"axis-tick\",gN=\"axis-title\",mN=\"legend\",yN=\"legend-band\",vN=\"legend-entry\",_N=\"legend-gradient\",xN=\"legend-label\",bN=\"legend-symbol\",wN=\"legend-title\",kN=\"title\",AN=\"title-text\",MN=\"title-subtitle\";function EN(t,e,n){t[e]=n&&n.signal?{signal:n.signal}:{value:n}}const DN=t=>xt(t)?Ct(t):t.signal?`(${t.signal})`:$N(t);function CN(t){if(null!=t.gradient)return function(t){const e=[t.start,t.stop,t.count].map((t=>null==t?null:Ct(t)));for(;e.length&&null==F(e);)e.pop();return e.unshift(DN(t.gradient)),`gradient(${e.join(\",\")})`}(t);let e=t.signal?`(${t.signal})`:t.color?function(t){return t.c?FN(\"hcl\",t.h,t.c,t.l):t.h||t.s?FN(\"hsl\",t.h,t.s,t.l):t.l||t.a?FN(\"lab\",t.l,t.a,t.b):t.r||t.g||t.b?FN(\"rgb\",t.r,t.g,t.b):null}(t.color):null!=t.field?$N(t.field):void 0!==t.value?Ct(t.value):void 0;return null!=t.scale&&(e=function(t,e){const n=DN(t.scale);null!=t.range?e=`lerp(_range(${n}), ${+t.range})`:(void 0!==e&&(e=`_scale(${n}, ${e})`),t.band&&(e=(e?e+\"+\":\"\")+`_bandwidth(${n})`+(1==+t.band?\"\":\"*\"+SN(t.band)),t.extra&&(e=`(datum.extra ? _scale(${n}, datum.extra.value) : ${e})`)),null==e&&(e=\"0\"));return e}(t,e)),void 0===e&&(e=null),null!=t.exponent&&(e=`pow(${e},${SN(t.exponent)})`),null!=t.mult&&(e+=`*${SN(t.mult)}`),null!=t.offset&&(e+=`+${SN(t.offset)}`),t.round&&(e=`round(${e})`),e}const FN=(t,e,n,r)=>`(${t}(${[e,n,r].map(CN).join(\",\")})+'')`;function SN(t){return A(t)?\"(\"+CN(t)+\")\":t}function $N(t){return TN(A(t)?t:{datum:t})}function TN(t){let e,n,r;if(t.signal)e=\"datum\",r=t.signal;else if(t.group||t.parent){for(n=Math.max(1,t.level||1),e=\"item\";n-- >0;)e+=\".mark.group\";t.parent?(r=t.parent,e+=\".datum\"):r=t.group}else t.datum?(e=\"datum\",r=t.datum):s(\"Invalid field reference: \"+Ct(t));return t.signal||(r=xt(r)?u(r).map(Ct).join(\"][\"):TN(r)),e+\"[\"+r+\"]\"}function BN(t,e,n,r,i,o){const a={};(o=o||{}).encoders={$encode:a},t=function(t,e,n,r,i){const o={},a={};let s,u,l,c;for(u in u=\"lineBreak\",\"text\"!==e||null==i[u]||aN(u,t)||EN(o,u,i[u]),(\"legend\"==n||String(n).startsWith(\"axis\"))&&(n=null),c=n===uN?i.group:n===sN?ot({},i.mark,i[e]):null,c)l=aN(u,t)||(\"fill\"===u||\"stroke\"===u)&&(aN(\"fill\",t)||aN(\"stroke\",t)),l||EN(o,u,c[u]);for(u in V(r).forEach((e=>{const n=i.style&&i.style[e];for(const e in n)aN(e,t)||EN(o,e,n[e])})),t=ot({},t),o)c=o[u],c.signal?(s=s||{})[u]=c:a[u]=c;return t.enter=ot(a,t.enter),s&&(t.update=ot(s,t.update)),t}(t,e,n,r,i.config);for(const n in t)a[n]=zN(t[n],e,o,i);return o}function zN(t,e,n,r){const i={},o={};for(const e in t)null!=t[e]&&(i[e]=NN((a=t[e],k(a)?function(t){let e=\"\";return t.forEach((t=>{const n=CN(t);e+=t.test?`(${t.test})?${n}:`:n})),\":\"===F(e)&&(e+=\"null\"),e}(a):CN(a)),r,n,o));var a;return{$expr:{marktype:e,channels:i},$fields:Object.keys(o),$output:Object.keys(t)}}function NN(t,e,n,r){const i=DB(t,e);return i.$fields.forEach((t=>r[t]=1)),ot(n,i.$params),i.$expr}const ON=\"outer\",RN=[\"value\",\"update\",\"init\",\"react\",\"bind\"];function UN(t,e){s(t+' for \"outer\" push: '+Ct(e))}function LN(t,e){const n=t.name;if(t.push===ON)e.signals[n]||UN(\"No prior signal definition\",n),RN.forEach((e=>{void 0!==t[e]&&UN(\"Invalid property \",e)}));else{const r=e.addSignal(n,t.value);!1===t.react&&(r.react=!1),t.bind&&e.addBinding(n,t.bind)}}function qN(t,e,n,r){this.id=-1,this.type=t,this.value=e,this.params=n,r&&(this.parent=r)}function PN(t,e,n,r){return new qN(t,e,n,r)}function jN(t,e){return PN(\"operator\",t,e)}function IN(t){const e={$ref:t.id};return t.id<0&&(t.refs=t.refs||[]).push(e),e}function WN(t,e){return e?{$field:t,$name:e}:{$field:t}}const HN=WN(\"key\");function YN(t,e){return{$compare:t,$order:e}}const GN=\"descending\";function VN(t,e){return(t&&t.signal?\"$\"+t.signal:t||\"\")+(t&&e?\"_\":\"\")+(e&&e.signal?\"$\"+e.signal:e||\"\")}const XN=\"scope\",JN=\"view\";function ZN(t){return t&&t.signal}function QN(t){if(ZN(t))return!0;if(A(t))for(const e in t)if(QN(t[e]))return!0;return!1}function KN(t,e){return null!=t?t:e}function tO(t){return t&&t.signal||t}const eO=\"timer\";function nO(t,e){return(t.merge?rO:t.stream?iO:t.type?oO:s(\"Invalid stream specification: \"+Ct(t)))(t,e)}function rO(t,e){const n=aO({merge:t.merge.map((t=>nO(t,e)))},t,e);return e.addStream(n).id}function iO(t,e){const n=aO({stream:nO(t.stream,e)},t,e);return e.addStream(n).id}function oO(t,e){let n;t.type===eO?(n=e.event(eO,t.throttle),t={between:t.between,filter:t.filter}):n=e.event(function(t){return t===XN?JN:t||JN}(t.source),t.type);const r=aO({stream:n},t,e);return 1===Object.keys(r).length?n:e.addStream(r).id}function aO(t,e,n){let r=e.between;return r&&(2!==r.length&&s('Stream \"between\" parameter must have 2 entries: '+Ct(e)),t.between=[nO(r[0],n),nO(r[1],n)]),r=e.filter?[].concat(e.filter):[],(e.marktype||e.markname||e.markrole)&&r.push(function(t,e,n){const r=\"event.item\";return r+(t&&\"*\"!==t?\"&&\"+r+\".mark.marktype==='\"+t+\"'\":\"\")+(n?\"&&\"+r+\".mark.role==='\"+n+\"'\":\"\")+(e?\"&&\"+r+\".mark.name==='\"+e+\"'\":\"\")}(e.marktype,e.markname,e.markrole)),e.source===XN&&r.push(\"inScope(event.item)\"),r.length&&(t.filter=DB(\"(\"+r.join(\")&&(\")+\")\",n).$expr),null!=(r=e.throttle)&&(t.throttle=+r),null!=(r=e.debounce)&&(t.debounce=+r),e.consume&&(t.consume=!0),t}const sO={code:\"_.$value\",ast:{type:\"Identifier\",value:\"value\"}};function uO(t,e,n){const r=t.encode,i={target:n};let o=t.events,a=t.update,u=[];o||s(\"Signal update missing events specification.\"),xt(o)&&(o=Vz(o,e.isSubscope()?XN:JN)),o=V(o).filter((t=>t.signal||t.scale?(u.push(t),0):1)),u.length>1&&(u=[lO(u)]),o.length&&u.push(o.length>1?{merge:o}:o[0]),null!=r&&(a&&s(\"Signal encode and update are mutually exclusive.\"),a=\"encode(item(),\"+Ct(r)+\")\"),i.update=xt(a)?DB(a,e):null!=a.expr?DB(a.expr,e):null!=a.value?a.value:null!=a.signal?{$expr:sO,$params:{$value:e.signalRef(a.signal)}}:s(\"Invalid signal update specification.\"),t.force&&(i.options={force:!0}),u.forEach((t=>e.addUpdate(ot(function(t,e){return{source:t.signal?e.signalRef(t.signal):t.scale?e.scaleRef(t.scale):nO(t,e)}}(t,e),i))))}function lO(t){return{signal:\"[\"+t.map((t=>t.scale?'scale(\"'+t.scale+'\")':t.signal))+\"]\"}}const cO=t=>(e,n,r)=>PN(t,n,e||void 0,r),fO=cO(\"aggregate\"),hO=cO(\"axisticks\"),dO=cO(\"bound\"),pO=cO(\"collect\"),gO=cO(\"compare\"),mO=cO(\"datajoin\"),yO=cO(\"encode\"),vO=cO(\"expression\"),_O=cO(\"facet\"),xO=cO(\"field\"),bO=cO(\"key\"),wO=cO(\"legendentries\"),kO=cO(\"load\"),AO=cO(\"mark\"),MO=cO(\"multiextent\"),EO=cO(\"multivalues\"),DO=cO(\"overlap\"),CO=cO(\"params\"),FO=cO(\"prefacet\"),SO=cO(\"projection\"),$O=cO(\"proxy\"),TO=cO(\"relay\"),BO=cO(\"render\"),zO=cO(\"scale\"),NO=cO(\"sieve\"),OO=cO(\"sortitems\"),RO=cO(\"viewlayout\"),UO=cO(\"values\");let LO=0;const qO={min:\"min\",max:\"max\",count:\"sum\"};function PO(t,e){const n=e.getScale(t.name).params;let r;for(r in n.domain=HO(t.domain,t,e),null!=t.range&&(n.range=KO(t,e,n)),null!=t.interpolate&&function(t,e){e.interpolate=jO(t.type||t),null!=t.gamma&&(e.interpolateGamma=jO(t.gamma))}(t.interpolate,n),null!=t.nice&&(n.nice=function(t,e){return t.signal?e.signalRef(t.signal):A(t)?{interval:jO(t.interval),step:jO(t.step)}:jO(t)}(t.nice,e)),null!=t.bins&&(n.bins=function(t,e){return t.signal||k(t)?IO(t,e):e.objectProperty(t)}(t.bins,e)),t)lt(n,r)||\"name\"===r||(n[r]=jO(t[r],e))}function jO(t,e){return A(t)?t.signal?e.signalRef(t.signal):s(\"Unsupported object: \"+Ct(t)):t}function IO(t,e){return t.signal?e.signalRef(t.signal):t.map((t=>jO(t,e)))}function WO(t){s(\"Can not find data set: \"+Ct(t))}function HO(t,e,n){if(t)return t.signal?n.signalRef(t.signal):(k(t)?YO:t.fields?VO:GO)(t,e,n);null==e.domainMin&&null==e.domainMax||s(\"No scale domain defined for domainMin/domainMax to override.\")}function YO(t,e,n){return t.map((t=>jO(t,n)))}function GO(t,e,n){const r=n.getData(t.data);return r||WO(t.data),cp(e.type)?r.valuesRef(n,t.field,JO(t.sort,!1)):pp(e.type)?r.domainRef(n,t.field):r.extentRef(n,t.field)}function VO(t,e,n){const r=t.data,i=t.fields.reduce(((t,e)=>(e=xt(e)?{data:r,field:e}:k(e)||e.signal?function(t,e){const n=\"_:vega:_\"+LO++,r=pO({});if(k(t))r.value={$ingest:t};else if(t.signal){const i=\"setdata(\"+Ct(n)+\",\"+t.signal+\")\";r.params.input=e.signalRef(i)}return e.addDataPipeline(n,[r,NO({})]),{data:n,field:\"data\"}}(e,n):e,t.push(e),t)),[]);return(cp(e.type)?XO:pp(e.type)?ZO:QO)(t,n,i)}function XO(t,e,n){const r=JO(t.sort,!0);let i,o;const a=n.map((t=>{const n=e.getData(t.data);return n||WO(t.data),n.countsRef(e,t.field,r)})),s={groupby:HN,pulse:a};r&&(i=r.op||\"count\",o=r.field?VN(i,r.field):\"count\",s.ops=[qO[i]],s.fields=[e.fieldRef(o)],s.as=[o]),i=e.add(fO(s));const u=e.add(pO({pulse:IN(i)}));return o=e.add(UO({field:HN,sort:e.sortRef(r),pulse:IN(u)})),IN(o)}function JO(t,e){return t&&(t.field||t.op?t.field||\"count\"===t.op?e&&t.field&&t.op&&!qO[t.op]&&s(\"Multiple domain scales can not be sorted using \"+t.op):s(\"No field provided for sort aggregate op: \"+t.op):A(t)?t.field=\"key\":t={field:\"key\"}),t}function ZO(t,e,n){const r=n.map((t=>{const n=e.getData(t.data);return n||WO(t.data),n.domainRef(e,t.field)}));return IN(e.add(EO({values:r})))}function QO(t,e,n){const r=n.map((t=>{const n=e.getData(t.data);return n||WO(t.data),n.extentRef(e,t.field)}));return IN(e.add(MO({extents:r})))}function KO(t,e,n){const r=e.config.range;let i=t.range;if(i.signal)return e.signalRef(i.signal);if(xt(i)){if(r&<(r,i))return KO(t=ot({},t,{range:r[i]}),e,n);\"width\"===i?i=[0,{signal:\"width\"}]:\"height\"===i?i=cp(t.type)?[0,{signal:\"height\"}]:[{signal:\"height\"},0]:s(\"Unrecognized scale range value: \"+Ct(i))}else{if(i.scheme)return n.scheme=k(i.scheme)?IO(i.scheme,e):jO(i.scheme,e),i.extent&&(n.schemeExtent=IO(i.extent,e)),void(i.count&&(n.schemeCount=jO(i.count,e)));if(i.step)return void(n.rangeStep=jO(i.step,e));if(cp(t.type)&&!k(i))return HO(i,t,e);k(i)||s(\"Unsupported range type: \"+Ct(i))}return i.map((t=>(k(t)?IO:jO)(t,e)))}function tR(t,e,n){return k(t)?t.map((t=>tR(t,e,n))):A(t)?t.signal?n.signalRef(t.signal):\"fit\"===e?t:s(\"Unsupported parameter object: \"+Ct(t)):t}const eR=\"top\",nR=\"left\",rR=\"right\",iR=\"bottom\",oR=\"center\",aR=\"vertical\",sR=\"start\",uR=\"end\",lR=\"index\",cR=\"label\",fR=\"offset\",hR=\"perc\",dR=\"perc2\",pR=\"value\",gR=\"guide-label\",mR=\"guide-title\",yR=\"group-title\",vR=\"group-subtitle\",_R=\"symbol\",xR=\"gradient\",bR=\"discrete\",wR=\"size\",kR=[wR,\"shape\",\"fill\",\"stroke\",\"strokeWidth\",\"strokeDash\",\"opacity\"],AR={name:1,style:1,interactive:1},MR={value:0},ER={value:1},DR=\"group\",CR=\"rect\",FR=\"rule\",SR=\"symbol\",$R=\"text\";function TR(t){return t.type=DR,t.interactive=t.interactive||!1,t}function BR(t,e){const n=(n,r)=>KN(t[n],KN(e[n],r));return n.isVertical=n=>aR===KN(t.direction,e.direction||(n?e.symbolDirection:e.gradientDirection)),n.gradientLength=()=>KN(t.gradientLength,e.gradientLength||e.gradientWidth),n.gradientThickness=()=>KN(t.gradientThickness,e.gradientThickness||e.gradientHeight),n.entryColumns=()=>KN(t.columns,KN(e.columns,+n.isVertical(!0))),n}function zR(t,e){const n=e&&(e.update&&e.update[t]||e.enter&&e.enter[t]);return n&&n.signal?n:n?n.value:null}function NR(t,e,n){return`item.anchor === '${sR}' ? ${t} : item.anchor === '${uR}' ? ${e} : ${n}`}const OR=NR(Ct(nR),Ct(rR),Ct(oR));function RR(t,e){return e?t?A(t)?Object.assign({},t,{offset:RR(t.offset,e)}):{value:t,offset:e}:e:t}function UR(t,e){return e?(t.name=e.name,t.style=e.style||t.style,t.interactive=!!e.interactive,t.encode=oN(t.encode,e,AR)):t.interactive=!1,t}function LR(t,e,n,r){const i=BR(t,n),o=i.isVertical(),a=i.gradientThickness(),s=i.gradientLength();let u,l,c,f,h;o?(l=[0,1],c=[0,0],f=a,h=s):(l=[0,0],c=[1,0],f=s,h=a);const d={enter:u={opacity:MR,x:MR,y:MR,width:nN(f),height:nN(h)},update:ot({},u,{opacity:ER,fill:{gradient:e,start:l,stop:c}}),exit:{opacity:MR}};return iN(d,{stroke:i(\"gradientStrokeColor\"),strokeWidth:i(\"gradientStrokeWidth\")},{opacity:i(\"gradientOpacity\")}),UR({type:CR,role:_N,encode:d},r)}function qR(t,e,n,r,i){const o=BR(t,n),a=o.isVertical(),s=o.gradientThickness(),u=o.gradientLength();let l,c,f,h,d=\"\";a?(l=\"y\",f=\"y2\",c=\"x\",h=\"width\",d=\"1-\"):(l=\"x\",f=\"x2\",c=\"y\",h=\"height\");const p={opacity:MR,fill:{scale:e,field:pR}};p[l]={signal:d+\"datum.\"+hR,mult:u},p[c]=MR,p[f]={signal:d+\"datum.\"+dR,mult:u},p[h]=nN(s);const g={enter:p,update:ot({},p,{opacity:ER}),exit:{opacity:MR}};return iN(g,{stroke:o(\"gradientStrokeColor\"),strokeWidth:o(\"gradientStrokeWidth\")},{opacity:o(\"gradientOpacity\")}),UR({type:CR,role:yN,key:pR,from:i,encode:g},r)}const PR=`datum.${hR}<=0?\"${nR}\":datum.${hR}>=1?\"${rR}\":\"${oR}\"`,jR=`datum.${hR}<=0?\"${iR}\":datum.${hR}>=1?\"${eR}\":\"middle\"`;function IR(t,e,n,r){const i=BR(t,e),o=i.isVertical(),a=nN(i.gradientThickness()),s=i.gradientLength();let u,l,c,f,h=i(\"labelOverlap\"),d=\"\";const p={enter:u={opacity:MR},update:l={opacity:ER,text:{field:cR}},exit:{opacity:MR}};return iN(p,{fill:i(\"labelColor\"),fillOpacity:i(\"labelOpacity\"),font:i(\"labelFont\"),fontSize:i(\"labelFontSize\"),fontStyle:i(\"labelFontStyle\"),fontWeight:i(\"labelFontWeight\"),limit:KN(t.labelLimit,e.gradientLabelLimit)}),o?(u.align={value:\"left\"},u.baseline=l.baseline={signal:jR},c=\"y\",f=\"x\",d=\"1-\"):(u.align=l.align={signal:PR},u.baseline={value:\"top\"},c=\"x\",f=\"y\"),u[c]=l[c]={signal:d+\"datum.\"+hR,mult:s},u[f]=l[f]=a,a.offset=KN(t.labelOffset,e.gradientLabelOffset)||0,h=h?{separation:i(\"labelSeparation\"),method:h,order:\"datum.\"+lR}:void 0,UR({type:$R,role:xN,style:gR,key:pR,from:r,encode:p,overlap:h},n)}function WR(t,e,n,r,i){const o=BR(t,e),a=n.entries,s=!(!a||!a.interactive),u=a?a.name:void 0,l=o(\"clipHeight\"),c=o(\"symbolOffset\"),f={data:\"value\"},h=`(${i}) ? datum.${fR} : datum.${wR}`,d=l?nN(l):{field:wR},p=`datum.${lR}`,g=`max(1, ${i})`;let m,y,v,_,x;d.mult=.5,m={enter:y={opacity:MR,x:{signal:h,mult:.5,offset:c},y:d},update:v={opacity:ER,x:y.x,y:y.y},exit:{opacity:MR}};let b=null,w=null;t.fill||(b=e.symbolBaseFillColor,w=e.symbolBaseStrokeColor),iN(m,{fill:o(\"symbolFillColor\",b),shape:o(\"symbolType\"),size:o(\"symbolSize\"),stroke:o(\"symbolStrokeColor\",w),strokeDash:o(\"symbolDash\"),strokeDashOffset:o(\"symbolDashOffset\"),strokeWidth:o(\"symbolStrokeWidth\")},{opacity:o(\"symbolOpacity\")}),kR.forEach((e=>{t[e]&&(v[e]=y[e]={scale:t[e],field:pR})}));const k=UR({type:SR,role:bN,key:pR,from:f,clip:!!l||void 0,encode:m},n.symbols),A=nN(c);A.offset=o(\"labelOffset\"),m={enter:y={opacity:MR,x:{signal:h,offset:A},y:d},update:v={opacity:ER,text:{field:cR},x:y.x,y:y.y},exit:{opacity:MR}},iN(m,{align:o(\"labelAlign\"),baseline:o(\"labelBaseline\"),fill:o(\"labelColor\"),fillOpacity:o(\"labelOpacity\"),font:o(\"labelFont\"),fontSize:o(\"labelFontSize\"),fontStyle:o(\"labelFontStyle\"),fontWeight:o(\"labelFontWeight\"),limit:o(\"labelLimit\")});const M=UR({type:$R,role:xN,style:gR,key:pR,from:f,encode:m},n.labels);return m={enter:{noBound:{value:!l},width:MR,height:l?nN(l):MR,opacity:MR},exit:{opacity:MR},update:v={opacity:ER,row:{signal:null},column:{signal:null}}},o.isVertical(!0)?(_=`ceil(item.mark.items.length / ${g})`,v.row.signal=`${p}%${_}`,v.column.signal=`floor(${p} / ${_})`,x={field:[\"row\",p]}):(v.row.signal=`floor(${p} / ${g})`,v.column.signal=`${p} % ${g}`,x={field:p}),v.column.signal=`(${i})?${v.column.signal}:${p}`,TR({role:lN,from:r={facet:{data:r,name:\"value\",groupby:lR}},encode:oN(m,a,AR),marks:[k,M],name:u,interactive:s,sort:x})}const HR='item.orient === \"left\"',YR='item.orient === \"right\"',GR=`(${HR} || ${YR})`,VR=`datum.vgrad && ${GR}`,XR=NR('\"top\"','\"bottom\"','\"middle\"'),JR=`datum.vgrad && ${YR} ? (${NR('\"right\"','\"left\"','\"center\"')}) : (${GR} && !(datum.vgrad && ${HR})) ? \"left\" : ${OR}`,ZR=`item._anchor || (${GR} ? \"middle\" : \"start\")`,QR=`${VR} ? (${HR} ? -90 : 90) : 0`,KR=`${GR} ? (datum.vgrad ? (${YR} ? \"bottom\" : \"top\") : ${XR}) : \"top\"`;function tU(t,e){let n;return A(t)&&(t.signal?n=t.signal:t.path?n=\"pathShape(\"+eU(t.path)+\")\":t.sphere&&(n=\"geoShape(\"+eU(t.sphere)+', {type: \"Sphere\"})')),n?e.signalRef(n):!!t}function eU(t){return A(t)&&t.signal?t.signal:Ct(t)}function nU(t){const e=t.role||\"\";return e.startsWith(\"axis\")||e.startsWith(\"legend\")||e.startsWith(\"title\")?e:t.type===DR?lN:e||sN}function rU(t){return{marktype:t.type,name:t.name||void 0,role:t.role||nU(t),zindex:+t.zindex||void 0,aria:t.aria,description:t.description}}function iU(t,e){return t&&t.signal?e.signalRef(t.signal):!1!==t}function oU(t,e){const n=Qa(t.type);n||s(\"Unrecognized transform type: \"+Ct(t.type));const r=PN(n.type.toLowerCase(),null,aU(n,t,e));return t.signal&&e.addSignal(t.signal,e.proxy(r)),r.metadata=n.metadata||{},r}function aU(t,e,n){const r={},i=t.params.length;for(let o=0;olU(t,e,n)))):lU(t,r,n)}(t,e,n):\"projection\"===r?n.projectionRef(e[t.name]):t.array&&!ZN(i)?i.map((e=>uU(t,e,n))):uU(t,i,n):void(t.required&&s(\"Missing required \"+Ct(e.type)+\" parameter: \"+Ct(t.name)))}function uU(t,e,n){const r=t.type;if(ZN(e))return dU(r)?s(\"Expression references can not be signals.\"):pU(r)?n.fieldRef(e):gU(r)?n.compareRef(e):n.signalRef(e.signal);{const i=t.expr||pU(r);return i&&cU(e)?n.exprRef(e.expr,e.as):i&&fU(e)?WN(e.field,e.as):dU(r)?DB(e,n):hU(r)?IN(n.getData(e).values):pU(r)?WN(e):gU(r)?n.compareRef(e):e}}function lU(t,e,n){const r=t.params.length;let i;for(let n=0;nt&&t.expr,fU=t=>t&&t.field,hU=t=>\"data\"===t,dU=t=>\"expr\"===t,pU=t=>\"field\"===t,gU=t=>\"compare\"===t;function mU(t,e){return t.$ref?t:t.data&&t.data.$ref?t.data:IN(e.getData(t.data).output)}function yU(t,e,n,r,i){this.scope=t,this.input=e,this.output=n,this.values=r,this.aggregate=i,this.index={}}function vU(t){return xt(t)?t:null}function _U(t,e,n){const r=VN(n.op,n.field);let i;if(e.ops){for(let t=0,n=e.as.length;tnull==t?\"null\":t)).join(\",\")+\"),0)\",e);u.update=l.$expr,u.params=l.$params}function wU(t,e){const n=nU(t),r=t.type===DR,i=t.from&&t.from.facet,o=t.overlap;let a,u,l,c,f,h,d,p=t.layout||n===lN||n===uN;const g=n===sN||p||i,m=function(t,e,n){let r,i,o,a,u;return t?(r=t.facet)&&(e||s(\"Only group marks can be faceted.\"),null!=r.field?a=u=mU(r,n):(t.data?u=IN(n.getData(t.data).aggregate):(o=oU(ot({type:\"aggregate\",groupby:V(r.groupby)},r.aggregate),n),o.params.key=n.keyRef(r.groupby),o.params.pulse=mU(r,n),a=u=IN(n.add(o))),i=n.keyRef(r.groupby,!0))):a=IN(n.add(pO(null,[{}]))),a||(a=mU(t,n)),{key:i,pulse:a,parent:u}}(t.from,r,e);u=e.add(mO({key:m.key||(t.key?WN(t.key):void 0),pulse:m.pulse,clean:!r}));const y=IN(u);u=l=e.add(pO({pulse:y})),u=e.add(AO({markdef:rU(t),interactive:iU(t.interactive,e),clip:tU(t.clip,e),context:{$context:!0},groups:e.lookup(),parent:e.signals.parent?e.signalRef(\"parent\"):null,index:e.markpath(),pulse:IN(u)}));const v=IN(u);u=c=e.add(yO(BN(t.encode,t.type,n,t.style,e,{mod:!1,pulse:v}))),u.params.parent=e.encode(),t.transform&&t.transform.forEach((t=>{const n=oU(t,e),r=n.metadata;(r.generates||r.changes)&&s(\"Mark transforms should not generate new data.\"),r.nomod||(c.params.mod=!0),n.params.pulse=IN(u),e.add(u=n)})),t.sort&&(u=e.add(OO({sort:e.compareRef(t.sort),pulse:IN(u)})));const _=IN(u);(i||p)&&(p=e.add(RO({layout:e.objectProperty(t.layout),legends:e.legends,mark:v,pulse:_})),h=IN(p));const x=e.add(dO({mark:v,pulse:h||_}));d=IN(x),r&&(g&&(a=e.operators,a.pop(),p&&a.pop()),e.pushState(_,h||d,y),i?function(t,e,n){const r=t.from.facet,i=r.name,o=mU(r,e);let a;r.name||s(\"Facet must have a name: \"+Ct(r)),r.data||s(\"Facet must reference a data set: \"+Ct(r)),r.field?a=e.add(FO({field:e.fieldRef(r.field),pulse:o})):r.groupby?a=e.add(_O({key:e.keyRef(r.groupby),group:IN(e.proxy(n.parent)),pulse:o})):s(\"Facet must specify groupby or field: \"+Ct(r));const u=e.fork(),l=u.add(pO()),c=u.add(NO({pulse:IN(l)}));u.addData(i,new yU(u,l,l,c)),u.addSignal(\"parent\",null),a.params.subflow={$subflow:u.parse(t).toRuntime()}}(t,e,m):g?function(t,e,n){const r=e.add(FO({pulse:n.pulse})),i=e.fork();i.add(NO()),i.addSignal(\"parent\",null),r.params.subflow={$subflow:i.parse(t).toRuntime()}}(t,e,m):e.parse(t),e.popState(),g&&(p&&a.push(p),a.push(x))),o&&(d=function(t,e,n){const r=t.method,i=t.bound,o=t.separation,a={separation:ZN(o)?n.signalRef(o.signal):o,method:ZN(r)?n.signalRef(r.signal):r,pulse:e};t.order&&(a.sort=n.compareRef({field:t.order}));if(i){const t=i.tolerance;a.boundTolerance=ZN(t)?n.signalRef(t.signal):+t,a.boundScale=n.scaleRef(i.scale),a.boundOrient=i.orient}return IN(n.add(DO(a)))}(o,d,e));const b=e.add(BO({pulse:d})),w=e.add(NO({pulse:IN(b)},void 0,e.parent()));null!=t.name&&(f=t.name,e.addData(f,new yU(e,l,b,w)),t.on&&t.on.forEach((t=>{(t.insert||t.remove||t.toggle)&&s(\"Marks only support modify triggers.\"),bU(t,e,f)})))}function kU(t,e){const n=e.config.legend,r=t.encode||{},i=BR(t,n),o=r.legend||{},a=o.name||void 0,u=o.interactive,l=o.style,c={};let f,h,d,p=0;kR.forEach((e=>t[e]?(c[e]=t[e],p=p||t[e]):0)),p||s(\"Missing valid scale for legend.\");const g=function(t,e){let n=t.type||_R;t.type||1!==function(t){return kR.reduce(((e,n)=>e+(t[n]?1:0)),0)}(t)||!t.fill&&!t.stroke||(n=lp(e)?xR:fp(e)?bR:_R);return n!==xR?n:fp(e)?bR:xR}(t,e.scaleType(p)),m={title:null!=t.title,scales:c,type:g,vgrad:\"symbol\"!==g&&i.isVertical()},y=IN(e.add(pO(null,[m]))),v=IN(e.add(wO(h={type:g,scale:e.scaleRef(p),count:e.objectProperty(i(\"tickCount\")),limit:e.property(i(\"symbolLimit\")),values:e.objectProperty(t.values),minstep:e.property(t.tickMinStep),formatType:e.property(t.formatType),formatSpecifier:e.property(t.format)})));return g===xR?(d=[LR(t,p,n,r.gradient),IR(t,n,r.labels,v)],h.count=h.count||e.signalRef(`max(2,2*floor((${tO(i.gradientLength())})/100))`)):g===bR?d=[qR(t,p,n,r.gradient,v),IR(t,n,r.labels,v)]:(f=function(t,e){const n=BR(t,e);return{align:n(\"gridAlign\"),columns:n.entryColumns(),center:{row:!0,column:!1},padding:{row:n(\"rowPadding\"),column:n(\"columnPadding\")}}}(t,n),d=[WR(t,n,r,v,tO(f.columns))],h.size=function(t,e,n){const r=tO(MU(\"size\",t,n)),i=tO(MU(\"strokeWidth\",t,n)),o=tO(function(t,e,n){return zR(\"fontSize\",t)||function(t,e,n){const r=e.config.style[n];return r&&r[t]}(\"fontSize\",e,n)}(n[1].encode,e,gR));return DB(`max(ceil(sqrt(${r})+${i}),${o})`,e)}(t,e,d[0].marks)),d=[TR({role:vN,from:y,encode:{enter:{x:{value:0},y:{value:0}}},marks:d,layout:f,interactive:u})],m.title&&d.push(function(t,e,n,r){const i=BR(t,e),o={enter:{opacity:MR},update:{opacity:ER,x:{field:{group:\"padding\"}},y:{field:{group:\"padding\"}}},exit:{opacity:MR}};return iN(o,{orient:i(\"titleOrient\"),_anchor:i(\"titleAnchor\"),anchor:{signal:ZR},angle:{signal:QR},align:{signal:JR},baseline:{signal:KR},text:t.title,fill:i(\"titleColor\"),fillOpacity:i(\"titleOpacity\"),font:i(\"titleFont\"),fontSize:i(\"titleFontSize\"),fontStyle:i(\"titleFontStyle\"),fontWeight:i(\"titleFontWeight\"),limit:i(\"titleLimit\"),lineHeight:i(\"titleLineHeight\")},{align:i(\"titleAlign\"),baseline:i(\"titleBaseline\")}),UR({type:$R,role:wN,style:mR,from:r,encode:o},n)}(t,n,r.title,y)),wU(TR({role:mN,from:y,encode:oN(AU(i,t,n),o,AR),marks:d,aria:i(\"aria\"),description:i(\"description\"),zindex:i(\"zindex\"),name:a,interactive:u,style:l}),e)}function AU(t,e,n){const r={enter:{},update:{}};return iN(r,{orient:t(\"orient\"),offset:t(\"offset\"),padding:t(\"padding\"),titlePadding:t(\"titlePadding\"),cornerRadius:t(\"cornerRadius\"),fill:t(\"fillColor\"),stroke:t(\"strokeColor\"),strokeWidth:n.strokeWidth,strokeDash:n.strokeDash,x:t(\"legendX\"),y:t(\"legendY\"),format:e.format,formatType:e.formatType}),r}function MU(t,e,n){return e[t]?`scale(\"${e[t]}\",datum)`:zR(t,n[0].encode)}yU.fromEntries=function(t,e){const n=e.length,r=e[n-1],i=e[n-2];let o=e[0],a=null,s=1;for(o&&\"load\"===o.type&&(o=e[1]),t.add(e[0]);s{n.push(oU(t,e))})),t.on&&t.on.forEach((n=>{bU(n,e,t.name)})),e.addDataPipeline(t.name,function(t,e,n){const r=[];let i,o,a,s,u,l=null,c=!1,f=!1;t.values?ZN(t.values)||QN(t.format)?(r.push($U(e,t)),r.push(l=SU())):r.push(l=SU({$ingest:t.values,$format:t.format})):t.url?QN(t.url)||QN(t.format)?(r.push($U(e,t)),r.push(l=SU())):r.push(l=SU({$request:t.url,$format:t.format})):t.source&&(l=i=V(t.source).map((t=>IN(e.getData(t).output))),r.push(null));for(o=0,a=n.length;ot===iR||t===eR,BU=(t,e,n)=>ZN(t)?qU(t.signal,e,n):t===nR||t===eR?e:n,zU=(t,e,n)=>ZN(t)?UU(t.signal,e,n):TU(t)?e:n,NU=(t,e,n)=>ZN(t)?LU(t.signal,e,n):TU(t)?n:e,OU=(t,e,n)=>ZN(t)?PU(t.signal,e,n):t===eR?{value:e}:{value:n},RU=(t,e,n)=>ZN(t)?jU(t.signal,e,n):t===rR?{value:e}:{value:n},UU=(t,e,n)=>IU(`${t} === '${eR}' || ${t} === '${iR}'`,e,n),LU=(t,e,n)=>IU(`${t} !== '${eR}' && ${t} !== '${iR}'`,e,n),qU=(t,e,n)=>HU(`${t} === '${nR}' || ${t} === '${eR}'`,e,n),PU=(t,e,n)=>HU(`${t} === '${eR}'`,e,n),jU=(t,e,n)=>HU(`${t} === '${rR}'`,e,n),IU=(t,e,n)=>(e=null!=e?nN(e):e,n=null!=n?nN(n):n,WU(e)&&WU(n)?{signal:`${t} ? (${e=e?e.signal||Ct(e.value):null}) : (${n=n?n.signal||Ct(n.value):null})`}:[ot({test:t},e)].concat(n||[])),WU=t=>null==t||1===Object.keys(t).length,HU=(t,e,n)=>({signal:`${t} ? (${GU(e)}) : (${GU(n)})`}),YU=(t,e,n,r,i)=>({signal:(null!=r?`${t} === '${nR}' ? (${GU(r)}) : `:\"\")+(null!=n?`${t} === '${iR}' ? (${GU(n)}) : `:\"\")+(null!=i?`${t} === '${rR}' ? (${GU(i)}) : `:\"\")+(null!=e?`${t} === '${eR}' ? (${GU(e)}) : `:\"\")+\"(null)\"}),GU=t=>ZN(t)?t.signal:null==t?null:Ct(t),VU=(t,e)=>0===e?0:ZN(t)?{signal:`(${t.signal}) * ${e}`}:{value:t*e},XU=(t,e)=>{const n=t.signal;return n&&n.endsWith(\"(null)\")?{signal:n.slice(0,-6)+e.signal}:t};function JU(t,e,n,r){let i;if(e&<(e,t))return e[t];if(lt(n,t))return n[t];if(t.startsWith(\"title\")){switch(t){case\"titleColor\":i=\"fill\";break;case\"titleFont\":case\"titleFontSize\":case\"titleFontWeight\":i=t[5].toLowerCase()+t.slice(6)}return r[mR][i]}if(t.startsWith(\"label\")){switch(t){case\"labelColor\":i=\"fill\";break;case\"labelFont\":case\"labelFontSize\":i=t[5].toLowerCase()+t.slice(6)}return r[gR][i]}return null}function ZU(t){const e={};for(const n of t)if(n)for(const t in n)e[t]=1;return Object.keys(e)}function QU(t,e){return{scale:t.scale,range:e}}function KU(t,e,n,r,i){const o=BR(t,e),a=t.orient,s=t.gridScale,u=BU(a,1,-1),l=function(t,e){if(1===e);else if(A(t)){let n=t=ot({},t);for(;null!=n.mult;){if(!A(n.mult))return n.mult=ZN(e)?{signal:`(${n.mult}) * (${e.signal})`}:n.mult*e,t;n=n.mult=ot({},n.mult)}n.mult=e}else t=ZN(e)?{signal:`(${e.signal}) * (${t||0})`}:e*(t||0);return t}(t.offset,u);let c,f,h;const d={enter:c={opacity:MR},update:h={opacity:ER},exit:f={opacity:MR}};iN(d,{stroke:o(\"gridColor\"),strokeCap:o(\"gridCap\"),strokeDash:o(\"gridDash\"),strokeDashOffset:o(\"gridDashOffset\"),strokeOpacity:o(\"gridOpacity\"),strokeWidth:o(\"gridWidth\")});const p={scale:t.scale,field:pR,band:i.band,extra:i.extra,offset:i.offset,round:o(\"tickRound\")},g=zU(a,{signal:\"height\"},{signal:\"width\"}),m=s?{scale:s,range:0,mult:u,offset:l}:{value:0,offset:l},y=s?{scale:s,range:1,mult:u,offset:l}:ot(g,{mult:u,offset:l});return c.x=h.x=zU(a,p,m),c.y=h.y=NU(a,p,m),c.x2=h.x2=NU(a,y),c.y2=h.y2=zU(a,y),f.x=zU(a,p),f.y=NU(a,p),UR({type:FR,role:hN,key:pR,from:r,encode:d},n)}function tL(t,e,n,r,i){return{signal:'flush(range(\"'+t+'\"), scale(\"'+t+'\", datum.value), '+e+\",\"+n+\",\"+r+\",\"+i+\")\"}}function eL(t,e,n,r){const i=BR(t,e),o=t.orient,a=BU(o,-1,1);let s,u;const l={enter:s={opacity:MR,anchor:nN(i(\"titleAnchor\",null)),align:{signal:OR}},update:u=ot({},s,{opacity:ER,text:nN(t.title)}),exit:{opacity:MR}},c={signal:`lerp(range(\"${t.scale}\"), ${NR(0,1,.5)})`};return u.x=zU(o,c),u.y=NU(o,c),s.angle=zU(o,MR,VU(a,90)),s.baseline=zU(o,OU(o,iR,eR),{value:iR}),u.angle=s.angle,u.baseline=s.baseline,iN(l,{fill:i(\"titleColor\"),fillOpacity:i(\"titleOpacity\"),font:i(\"titleFont\"),fontSize:i(\"titleFontSize\"),fontStyle:i(\"titleFontStyle\"),fontWeight:i(\"titleFontWeight\"),limit:i(\"titleLimit\"),lineHeight:i(\"titleLineHeight\")},{align:i(\"titleAlign\"),angle:i(\"titleAngle\"),baseline:i(\"titleBaseline\")}),function(t,e,n,r){const i=(t,e)=>null!=t?(n.update[e]=XU(nN(t),n.update[e]),!1):!aN(e,r),o=i(t(\"titleX\"),\"x\"),a=i(t(\"titleY\"),\"y\");n.enter.auto=a===o?nN(a):zU(e,nN(a),nN(o))}(i,o,l,n),l.update.align=XU(l.update.align,s.align),l.update.angle=XU(l.update.angle,s.angle),l.update.baseline=XU(l.update.baseline,s.baseline),UR({type:$R,role:gN,style:mR,from:r,encode:l},n)}function nL(t,e){const n=function(t,e){var n,r,i,o=e.config,a=o.style,s=o.axis,u=\"band\"===e.scaleType(t.scale)&&o.axisBand,l=t.orient;if(ZN(l)){const t=ZU([o.axisX,o.axisY]),e=ZU([o.axisTop,o.axisBottom,o.axisLeft,o.axisRight]);for(i of(n={},t))n[i]=zU(l,JU(i,o.axisX,s,a),JU(i,o.axisY,s,a));for(i of(r={},e))r[i]=YU(l.signal,JU(i,o.axisTop,s,a),JU(i,o.axisBottom,s,a),JU(i,o.axisLeft,s,a),JU(i,o.axisRight,s,a))}else n=l===eR||l===iR?o.axisX:o.axisY,r=o[\"axis\"+l[0].toUpperCase()+l.slice(1)];return n||r||u?ot({},s,n,r,u):s}(t,e),r=t.encode||{},i=r.axis||{},o=i.name||void 0,a=i.interactive,s=i.style,u=BR(t,n),l=function(t){const e=t(\"tickBand\");let n,r,i=t(\"tickOffset\");return e?e.signal?(n={signal:`(${e.signal}) === 'extent' ? 1 : 0.5`},r={signal:`(${e.signal}) === 'extent'`},A(i)||(i={signal:`(${e.signal}) === 'extent' ? 0 : ${i}`})):\"extent\"===e?(n=1,r=!0,i=0):(n=.5,r=!1):(n=t(\"bandPosition\"),r=t(\"tickExtra\")),{extra:r,band:n,offset:i}}(u),c={scale:t.scale,ticks:!!u(\"ticks\"),labels:!!u(\"labels\"),grid:!!u(\"grid\"),domain:!!u(\"domain\"),title:null!=t.title},f=IN(e.add(pO({},[c]))),h=IN(e.add(hO({scale:e.scaleRef(t.scale),extra:e.property(l.extra),count:e.objectProperty(t.tickCount),values:e.objectProperty(t.values),minstep:e.property(t.tickMinStep),formatType:e.property(t.formatType),formatSpecifier:e.property(t.format)}))),d=[];let p;return c.grid&&d.push(KU(t,n,r.grid,h,l)),c.ticks&&(p=u(\"tickSize\"),d.push(function(t,e,n,r,i,o){const a=BR(t,e),s=t.orient,u=BU(s,-1,1);let l,c,f;const h={enter:l={opacity:MR},update:f={opacity:ER},exit:c={opacity:MR}};iN(h,{stroke:a(\"tickColor\"),strokeCap:a(\"tickCap\"),strokeDash:a(\"tickDash\"),strokeDashOffset:a(\"tickDashOffset\"),strokeOpacity:a(\"tickOpacity\"),strokeWidth:a(\"tickWidth\")});const d=nN(i);d.mult=u;const p={scale:t.scale,field:pR,band:o.band,extra:o.extra,offset:o.offset,round:a(\"tickRound\")};return f.y=l.y=zU(s,MR,p),f.y2=l.y2=zU(s,d),c.x=zU(s,p),f.x=l.x=NU(s,MR,p),f.x2=l.x2=NU(s,d),c.y=NU(s,p),UR({type:FR,role:pN,key:pR,from:r,encode:h},n)}(t,n,r.ticks,h,p,l))),c.labels&&(p=c.ticks?p:0,d.push(function(t,e,n,r,i,o){const a=BR(t,e),s=t.orient,u=t.scale,l=BU(s,-1,1),c=tO(a(\"labelFlush\")),f=tO(a(\"labelFlushOffset\")),h=a(\"labelAlign\"),d=a(\"labelBaseline\");let p,g=0===c||!!c;const m=nN(i);m.mult=l,m.offset=nN(a(\"labelPadding\")||0),m.offset.mult=l;const y={scale:u,field:pR,band:.5,offset:RR(o.offset,a(\"labelOffset\"))},v=zU(s,g?tL(u,c,'\"left\"','\"right\"','\"center\"'):{value:\"center\"},RU(s,\"left\",\"right\")),_=zU(s,OU(s,\"bottom\",\"top\"),g?tL(u,c,'\"top\"','\"bottom\"','\"middle\"'):{value:\"middle\"}),x=tL(u,c,`-(${f})`,f,0);g=g&&f;const b={opacity:MR,x:zU(s,y,m),y:NU(s,y,m)},w={enter:b,update:p={opacity:ER,text:{field:cR},x:b.x,y:b.y,align:v,baseline:_},exit:{opacity:MR,x:b.x,y:b.y}};iN(w,{dx:!h&&g?zU(s,x):null,dy:!d&&g?NU(s,x):null}),iN(w,{angle:a(\"labelAngle\"),fill:a(\"labelColor\"),fillOpacity:a(\"labelOpacity\"),font:a(\"labelFont\"),fontSize:a(\"labelFontSize\"),fontWeight:a(\"labelFontWeight\"),fontStyle:a(\"labelFontStyle\"),limit:a(\"labelLimit\"),lineHeight:a(\"labelLineHeight\")},{align:h,baseline:d});const k=a(\"labelBound\");let A=a(\"labelOverlap\");return A=A||k?{separation:a(\"labelSeparation\"),method:A,order:\"datum.index\",bound:k?{scale:u,orient:s,tolerance:k}:null}:void 0,p.align!==v&&(p.align=XU(p.align,v)),p.baseline!==_&&(p.baseline=XU(p.baseline,_)),UR({type:$R,role:dN,style:gR,key:pR,from:r,encode:w,overlap:A},n)}(t,n,r.labels,h,p,l))),c.domain&&d.push(function(t,e,n,r){const i=BR(t,e),o=t.orient;let a,s;const u={enter:a={opacity:MR},update:s={opacity:ER},exit:{opacity:MR}};iN(u,{stroke:i(\"domainColor\"),strokeCap:i(\"domainCap\"),strokeDash:i(\"domainDash\"),strokeDashOffset:i(\"domainDashOffset\"),strokeWidth:i(\"domainWidth\"),strokeOpacity:i(\"domainOpacity\")});const l=QU(t,0),c=QU(t,1);return a.x=s.x=zU(o,l,MR),a.x2=s.x2=zU(o,c),a.y=s.y=NU(o,l,MR),a.y2=s.y2=NU(o,c),UR({type:FR,role:fN,from:r,encode:u},n)}(t,n,r.domain,f)),c.title&&d.push(eL(t,n,r.title,f)),wU(TR({role:cN,from:f,encode:oN(rL(u,t),i,AR),marks:d,aria:u(\"aria\"),description:u(\"description\"),zindex:u(\"zindex\"),name:o,interactive:a,style:s}),e)}function rL(t,e){const n={enter:{},update:{}};return iN(n,{orient:t(\"orient\"),offset:t(\"offset\")||0,position:KN(e.position,0),titlePadding:t(\"titlePadding\"),minExtent:t(\"minExtent\"),maxExtent:t(\"maxExtent\"),range:{signal:`abs(span(range(\"${e.scale}\")))`},translate:t(\"translate\"),format:e.format,formatType:e.formatType}),n}function iL(t,e,n){const r=V(t.signals),i=V(t.scales);return n||r.forEach((t=>LN(t,e))),V(t.projections).forEach((t=>function(t,e){const n=e.config.projection||{},r={};for(const n in t)\"name\"!==n&&(r[n]=tR(t[n],n,e));for(const t in n)null==r[t]&&(r[t]=tR(n[t],t,e));e.addProjection(t.name,r)}(t,e))),i.forEach((t=>function(t,e){const n=t.type||\"linear\";sp(n)||s(\"Unrecognized scale type: \"+Ct(n)),e.addScale(t.name,{type:n,domain:void 0})}(t,e))),V(t.data).forEach((t=>FU(t,e))),i.forEach((t=>PO(t,e))),(n||r).forEach((t=>function(t,e){const n=e.getSignal(t.name);let r=t.update;t.init&&(r?s(\"Signals can not include both init and update expressions.\"):(r=t.init,n.initonly=!0)),r&&(r=DB(r,e),n.update=r.$expr,n.params=r.$params),t.on&&t.on.forEach((t=>uO(t,e,n.id)))}(t,e))),V(t.axes).forEach((t=>nL(t,e))),V(t.marks).forEach((t=>wU(t,e))),V(t.legends).forEach((t=>kU(t,e))),t.title&&DU(t.title,e),e.parseLambdas(),e}const oL=t=>oN({enter:{x:{value:0},y:{value:0}},update:{width:{signal:\"width\"},height:{signal:\"height\"}}},t);function aL(t,e){const n=e.config,r=IN(e.root=e.add(jN())),i=function(t,e){const n=n=>KN(t[n],e[n]),r=[sL(\"background\",n(\"background\")),sL(\"autosize\",Qz(n(\"autosize\"))),sL(\"padding\",eN(n(\"padding\"))),sL(\"width\",n(\"width\")||0),sL(\"height\",n(\"height\")||0)],i=r.reduce(((t,e)=>(t[e.name]=e,t)),{}),o={};return V(t.signals).forEach((t=>{lt(i,t.name)?t=ot(i[t.name],t):r.push(t),o[t.name]=t})),V(e.signals).forEach((t=>{lt(o,t.name)||lt(i,t.name)||r.push(t)})),r}(t,n);i.forEach((t=>LN(t,e))),e.description=t.description||n.description,e.eventConfig=n.events,e.legends=e.objectProperty(n.legend&&n.legend.layout),e.locale=n.locale;const o=e.add(pO()),a=e.add(yO(BN(oL(t.encode),DR,uN,t.style,e,{pulse:IN(o)}))),s=e.add(RO({layout:e.objectProperty(t.layout),legends:e.legends,autosize:e.signalRef(\"autosize\"),mark:r,pulse:IN(a)}));e.operators.pop(),e.pushState(IN(a),IN(s),null),iL(t,e,i),e.operators.push(s);let u=e.add(dO({mark:r,pulse:IN(s)}));return u=e.add(BO({pulse:IN(u)})),u=e.add(NO({pulse:IN(u)})),e.addData(\"root\",new yU(e,o,o,u)),e}function sL(t,e){return e&&e.signal?{name:t,update:e.signal}:{name:t,value:e}}function uL(t,e){this.config=t||{},this.options=e||{},this.bindings=[],this.field={},this.signals={},this.lambdas={},this.scales={},this.events={},this.data={},this.streams=[],this.updates=[],this.operators=[],this.eventConfig=null,this.locale=null,this._id=0,this._subid=0,this._nextsub=[0],this._parent=[],this._encode=[],this._lookup=[],this._markpath=[]}function lL(t){this.config=t.config,this.options=t.options,this.legends=t.legends,this.field=Object.create(t.field),this.signals=Object.create(t.signals),this.lambdas=Object.create(t.lambdas),this.scales=Object.create(t.scales),this.events=Object.create(t.events),this.data=Object.create(t.data),this.streams=[],this.updates=[],this.operators=[],this._id=0,this._subid=++t._nextsub[0],this._nextsub=t._nextsub,this._parent=t._parent.slice(),this._encode=t._encode.slice(),this._lookup=t._lookup.slice(),this._markpath=t._markpath}function cL(t){return(k(t)?fL:hL)(t)}function fL(t){const e=t.length;let n=\"[\";for(let r=0;r0?\",\":\"\")+(A(e)?e.signal||cL(e):Ct(e))}return n+\"]\"}function hL(t){let e,n,r=\"{\",i=0;for(e in t)n=t[e],r+=(++i>1?\",\":\"\")+Ct(e)+\":\"+(A(n)?n.signal||cL(n):Ct(n));return r+\"}\"}uL.prototype=lL.prototype={parse(t){return iL(t,this)},fork(){return new lL(this)},isSubscope(){return this._subid>0},toRuntime(){return this.finish(),{description:this.description,operators:this.operators,streams:this.streams,updates:this.updates,bindings:this.bindings,eventConfig:this.eventConfig,locale:this.locale}},id(){return(this._subid?this._subid+\":\":0)+this._id++},add(t){return this.operators.push(t),t.id=this.id(),t.refs&&(t.refs.forEach((e=>{e.$ref=t.id})),t.refs=null),t},proxy(t){const e=t instanceof qN?IN(t):t;return this.add($O({value:e}))},addStream(t){return this.streams.push(t),t.id=this.id(),t},addUpdate(t){return this.updates.push(t),t},finish(){let t,e;for(t in this.root&&(this.root.root=!0),this.signals)this.signals[t].signal=t;for(t in this.scales)this.scales[t].scale=t;function n(t,e,n){let r,i;t&&(r=t.data||(t.data={}),i=r[e]||(r[e]=[]),i.push(n))}for(t in this.data){e=this.data[t],n(e.input,t,\"input\"),n(e.output,t,\"output\"),n(e.values,t,\"values\");for(const r in e.index)n(e.index[r],t,\"index:\"+r)}return this},pushState(t,e,n){this._encode.push(IN(this.add(NO({pulse:t})))),this._parent.push(e),this._lookup.push(n?IN(this.proxy(n)):null),this._markpath.push(-1)},popState(){this._encode.pop(),this._parent.pop(),this._lookup.pop(),this._markpath.pop()},parent(){return F(this._parent)},encode(){return F(this._encode)},lookup(){return F(this._lookup)},markpath(){const t=this._markpath;return++t[t.length-1]},fieldRef(t,e){if(xt(t))return WN(t,e);t.signal||s(\"Unsupported field reference: \"+Ct(t));const n=t.signal;let r=this.field[n];if(!r){const t={name:this.signalRef(n)};e&&(t.as=e),this.field[n]=r=IN(this.add(xO(t)))}return r},compareRef(t){let e=!1;const n=t=>ZN(t)?(e=!0,this.signalRef(t.signal)):function(t){return t&&t.expr}(t)?(e=!0,this.exprRef(t.expr)):t,r=V(t.field).map(n),i=V(t.order).map(n);return e?IN(this.add(gO({fields:r,orders:i}))):YN(r,i)},keyRef(t,e){let n=!1;const r=this.signals;return t=V(t).map((t=>ZN(t)?(n=!0,IN(r[t.signal])):t)),n?IN(this.add(bO({fields:t,flat:e}))):function(t,e){const n={$key:t};return e&&(n.$flat=!0),n}(t,e)},sortRef(t){if(!t)return t;const e=VN(t.op,t.field),n=t.order||\"ascending\";return n.signal?IN(this.add(gO({fields:e,orders:this.signalRef(n.signal)}))):YN(e,n)},event(t,e){const n=t+\":\"+e;if(!this.events[n]){const r=this.id();this.streams.push({id:r,source:t,type:e}),this.events[n]=r}return this.events[n]},hasOwnSignal(t){return lt(this.signals,t)},addSignal(t,e){this.hasOwnSignal(t)&&s(\"Duplicate signal name: \"+Ct(t));const n=e instanceof qN?e:this.add(jN(e));return this.signals[t]=n},getSignal(t){return this.signals[t]||s(\"Unrecognized signal name: \"+Ct(t)),this.signals[t]},signalRef(t){return this.signals[t]?IN(this.signals[t]):(lt(this.lambdas,t)||(this.lambdas[t]=this.add(jN(null))),IN(this.lambdas[t]))},parseLambdas(){const t=Object.keys(this.lambdas);for(let e=0,n=t.length;er+Math.floor(o*t.random()),pdf:t=>t===Math.floor(t)&&t>=r&&t=i?1:(e-r+1)/o},icdf:t=>t>=0&&t<=1?r-1+Math.floor(t*o):NaN};return a.min(e).max(n)},t.randomKDE=gs,t.randomLCG=function(t){return function(){return(t=(1103515245*t+12345)%2147483647)/2147483647}},t.randomLogNormal=xs,t.randomMixture=bs,t.randomNormal=ps,t.randomUniform=Es,t.read=ca,t.regressionConstant=Ds,t.regressionExp=zs,t.regressionLinear=Ts,t.regressionLoess=Ls,t.regressionLog=Bs,t.regressionPoly=Rs,t.regressionPow=Ns,t.regressionQuad=Os,t.renderModule=T_,t.repeat=Mt,t.resetDefaultLocale=function(){return Co(),Bo(),Uo()},t.resetSVGClipId=Yg,t.resetSVGDefIds=function(){Yg(),Gp=0},t.responseType=la,t.runtimeContext=OB,t.sampleCurve=Is,t.sampleLogNormal=ms,t.sampleNormal=cs,t.sampleUniform=ws,t.scale=ap,t.sceneEqual=q_,t.sceneFromJSON=Zy,t.scenePickVisit=qm,t.sceneToJSON=Jy,t.sceneVisit=Lm,t.sceneZOrder=Um,t.scheme=Ap,t.serializeXML=r_,t.setHybridRendererOptions=function(t){A_.svgMarkTypes=t.svgMarkTypes??[\"text\"],A_.svgOnTop=t.svgOnTop??!0,A_.debug=t.debug??!1},t.setRandom=function(e){t.random=e},t.span=Dt,t.splitAccessPath=u,t.stringValue=Ct,t.textMetrics=My,t.timeBin=Jr,t.timeFloor=wr,t.timeFormatLocale=No,t.timeInterval=Cr,t.timeOffset=$r,t.timeSequence=zr,t.timeUnitSpecifier=rr,t.timeUnits=er,t.toBoolean=Ft,t.toDate=$t,t.toNumber=S,t.toSet=Bt,t.toString=Tt,t.transform=Ka,t.transforms=Za,t.truncate=zt,t.truthy=p,t.tupleid=ya,t.typeParsers=Zo,t.utcFloor=Mr,t.utcInterval=Fr,t.utcOffset=Tr,t.utcSequence=Nr,t.utcdayofyear=hr,t.utcquarter=G,t.utcweek=dr,t.version=\"5.29.0\",t.visitArray=Nt,t.week=sr,t.writeConfig=D,t.zero=h,t.zoomLinear=j,t.zoomLog=I,t.zoomPow=W,t.zoomSymlog=H}));\n//# sourceMappingURL=vega.min.js.map\n" }, { "path": "docs/modules/custom_yara_rules.md", "content": "# Custom Yara Rules\n\n### Overview\nThrough the `excavate` internal module, BBOT supports searching through HTTP response data using custom YARA rules.\n\nThis feature can be utilized with the command line option `--custom-yara-rules` or `-cy`, followed by a file containing the YARA rules.\n\nExample:\n\n```\nbbot -m httpx --custom-yara-rules=test.yara -t http://example.com/\n```\n\nWhere `test.yara` is a file on the filesystem. The file can contain multiple YARA rules, separated by lines.\n\nYARA rules can be quite simple, the simplest example being a single string search:\n\n```\nrule find_string {\n strings:\n $str1 = \"AAAABBBB\"\n\n condition:\n $str1\n}\n```\n\nTo look for multiple strings, and match if any of them were to hit:\n\n```\nrule find_string {\n strings:\n $str1 = \"AAAABBBB\"\n $str2 = \"CCCCDDDD\"\n\n condition:\n any of them\n}\n```\n\nOne of the most important capabilities is the use of regexes within the rule, as shown in the following example.\n\n```\nrule find_AAAABBBB_regex {\n strings:\n $regex = /A{1,4}B{1,4}/\n\n condition:\n $regex\n}\n\n```\n\n*Note: YARA uses it's own regex engine that is not a 1:1 match with python regexes. This means many existing regexes will have to be modified before they will work with YARA. The good news is: YARA's regex engine is FAST, immensely more fast than pythons!*\n\nFurther discussion of art of writing complex YARA rules goes far beyond the scope of this documentation. A good place to start learning more is the [official YARA documentation](https://yara.readthedocs.io/en/stable/writingrules.html).\n\nThe YARA engine provides plenty of room to make highly complex signatures possible, with various conditional operators available. Multiple signatures can be linked together to create sophisticated detection rules that can identify a wide range of specific content. This flexibility allows the crafting of efficient rules for detecting security vulnerabilities, leveraging logical operators, regular expressions, and other powerful features. Additionally, YARA's modular structure supports easy updates and maintenance of signature sets.\n\n### Custom options\n\nBBOT supports the use of a few custom `meta` attributes within YARA rules, which will alter the behavior of the rule and the post-processing of the results.\n\n#### description\n\nThe description of the rule. Will end up in the description of any produced events if defined.\n\nExample with no description provided:\n\n```\n[FINDING] {\"description\": \"Custom Yara Rule [find_string] Matched via identifier [str1]\", \"host\": \"example.com\", \"url\": \"http://example.com\"} excavate\n```\n\nExample with the description added:\n\n```\n[FINDING] {\"description\": \"Custom Yara Rule [AAAABBBB] with description: [contains our test string] Matched via identifier [str1]\", \"host\": \"example.com, \"url\": \"http://example.com\"} excavate\n```\n\nThat FINDING was produced with the following signature:\n\n```\nrule AAAABBBB {\n\n meta:\n description = \"contains our test string\"\n strings:\n $str1 = \"AAAABBBB\"\n condition:\n $str1\n}\n```\n\n#### tags\n\nTags specified with this option will be passed-on to any resulting emitted events. Tags are provided as a comma separated string, as shown below:\n\nLets expand on the previous example:\n\n```\nrule AAAABBBB {\n\n meta:\n description = \"contains our test string\"\n tags = \"tag1,tag2,tag3\"\n strings:\n $str1 = \"AAAABBBB\"\n condition:\n $str1\n}\n```\n\nNow, the BBOT FINDING includes these custom tags, as with the following output:\n\n```\n[FINDING] {\"description\": \"Custom Yara Rule [AAAABBBB] with description: [contains our test string] Matched via identifier [str1]\", \"host\": \"example.com\", \"url\": \"http://example.com/\"} excavate (tag1, tag2, tag3)\n```\n\n#### emit_match\n\nWhen set to True, the contents returned from a successful extraction via a YARA regex will be included in the FINDING event which is emitted.\n\nConsider the following example YARA rule:\n\n```\nrule ContainsTitle\n{\n meta:\n description = \"Contains an HTML title tag\"\n emit_match = true\n strings:\n $title_value = /(.*)?<\\/title>/i\n condition:\n $title_value\n}\n```\n\nWhen run against the Black Lantern Security homepage with the following BBOT command:\n\n```\nbbot -m httpx --custom-yara-rules=substack.yara -t http://www.blacklanternsecurity.com/\n\n```\n\nWe get the following result. Note that the finding now contains the actual title tag that was identified with the regex.\n\n```\n[FINDING] {\"description\": \"Custom Yara Rule [ContainsTitle] with description: [Contains an HTML title] Matched via identifier [title_value] and extracted [<title>Black Lantern Security]\", \"host\": \"www.blacklanternsecurity.com\", \"url\": \"https://www.blacklanternsecurity.com/\"}\texcavate\t(cdn-github, cdn-ip)\n```\n" }, { "path": "docs/modules/internal_modules.md", "content": "# List of Modules\n\n## What are internal modules?\n\nInternal modules are just like regular modules, except that they run all the time. They do not have to be explicitly enabled. They can, however, be explicitly disabled if needed.\n\nTurning them off is simple, a root-level config option is present which can be set to False to disable them:\n\n```\n# Infer certain events from others, e.g. IPs from IP ranges, DNS_NAMEs from URLs, etc.\nspeculate: True\n# Passively search event data for URLs, hostnames, emails, etc.\nexcavate: True\n# Summarize activity at the end of a scan\naggregate: True\n# DNS resolution\ndnsresolve: True\n# Cloud provider tagging\ncloudcheck: True\n```\n\nThese modules are executing core functionality that is normally essential for a typical BBOT scan. Let's take a quick look at each one's functionality:\n\n### aggregate\n\nSummarize statistics at the end of a scan. Disable if you don't want to see this table.\n\n### cloud\n\nThe cloud module looks at events and tries to determine if they are associated with a cloud provider and tags them as such, and can also identify certain cloud resources\n\n### dns\n\nThe DNS internal module controls the basic DNS resolution the BBOT performs, and all of the supporting machinery like wildcard detection, etc.\n\n### excavate\n\nThe excavate internal module designed to passively extract valuable information from HTTP response data. It primarily uses YARA regexes to extract information, with various events being produced from the post-processing of the YARA results.\n\nHere is a summary of the data it produces:\n\n#### URLs\n\nBy extracting URLs from all visited pages, this is actually already half of a web-spider. The other half is recursion, which is baked in to BBOT from the ground up. Therefore, protections are in place by default in the form of `web_spider_distance` and `web_spider_depth` settings. These settings govern restrictions to URLs recursively harvested from HTTP responses, preventing endless runaway scans. However, in the right situation the controlled use of a web-spider is extremely powerful.\n\n#### Parameter Extraction\n\nParameter Extraction\nThe parameter extraction functionality identifies and extracts key web parameters from HTTP responses, and produced `WEB_PARAMETER` events. This includes parameters found in GET and POST requests, HTML forms, and jQuery requests. Currently, these are only used by the `hunt` module, and by the `paramminer` modules, to a limited degree. However, future functionality will make extensive use of these events.\n\n#### Email Extraction\n\nDetect email addresses within HTTP_RESPONSE data.\n\n#### Error Detection\n\nScans for verbose error messages in HTTP responses and raw text data. By identifying specific error signatures from various programming languages and frameworks, this feature helps uncover misconfigurations, debugging information, and potential vulnerabilities. This insight is invaluable for identifying weak points or anomalies in web applications.\n\n#### Content Security Policy (CSP) Extraction\nThe CSP extraction capability focuses on extracting domains from Content-Security-Policy headers. By analyzing these headers, BBOT can identify additional domains which can get fed back into the scan.\n\n#### Serialization Detection\nSerialized objects are a common source of serious security vulnerabilities. Excavate aims to detect those used in Java, .NET, and PHP applications.\n\n#### Functionality Detection\nLooks for specific web functionalities such as file upload fields and WSDL URLs. By identifying these elements, BBOT can pinpoint areas of the application that may require further scrutiny for security vulnerabilities.\n\n#### Non-HTTP Scheme Detection\nThe non-HTTP scheme detection capability extracts URLs with non-HTTP schemes, such as ftp, mailto, and javascript. By identifying these URLs, BBOT can uncover additional vectors for attack or information leakage.\n\n#### Custom Yara Rules\n\nExcavate supports the use of custom YARA rules, which will be added to the other rules before the scan start. For more info, view this.\n\n### speculate\n\nSpeculate is all about inferring one data type from another, particularly when certain tools like port scanners are not enabled. This is essential functionality for most BBOT scans, allowing for the discovery of web resources when starting with a DNS-only target list without a port scanner. It bridges gaps in the data, providing a more comprehensive view of the target by leveraging existing information.\n\n* IP_RANGE: Converts an IP range into individual IP addresses and emits them as IP_ADDRESS events.\n* DNS_NAME: Generates parent domains from DNS names.\n* URL and URL_UNVERIFIED: Infers open TCP ports from URLs and speculates on sub-directory URLs.\n* General URL Speculation: Emits URL_UNVERIFIED events for URLs not already in the event's history.\n* IP_ADDRESS / DNS_NAME: Infers open TCP ports if active port scanning is not enabled.\n* ORG_STUB: Derives organization stubs from TLDs, social stubs, or Azure tenant names and emits them as ORG_STUB events.\n* USERNAME: Converts usernames to email addresses if they validate as such.\n" }, { "path": "docs/modules/lightfuzz.md", "content": "# Lightfuzz\n\n*Lightfuzz is currently an experimental feature. There WILL be false positives (and, although we'll never know - false negatives), although the submodules are being actively worked on to reduce them. If you find false positives, please help us out by opening a GitHub issue with the details!*\n\n## Philosophy\n\n### What is Lightfuzz?\n\nLightfuzz is a lightweight web vulnerability scanner built into BBOT. It is designed to find \"low-hanging fruit\" type vulnerabilities without much overhead and at massive scale. \n\n### What is Lightfuzz NOT?\n\nLightfuzz is not, does not attempt to be, and will never be, a replacement for a full-blown web application scanner. You should not, for example, be running Lightfuzz as a replacement for Burp Suite scanning. Burp Suite scanner will always find more (even though we can find a few things it can't).\n\nIt will also not help you *exploit* vulnerabilities. It's job is to point out vulnerabilities, or likely vulnerabilities, or potential vulnerabilities, and then pass them off to you. A great deal of the overhead with traditional scanners comes in the confirmation phase, or in testing exploitation payloads. \n\nSo for example, Lightfuzz may detect an XSS vulnerability for you. But its NOT going to help you figure out which tag you need to use to get around a security filter, or give you any kind of a final payload. It's simply going to tell you that the contents of a given GET parameter are being reflected and that it was able to render an unmodified HTML tag. The rest is up to you.\n\n### False Positives\n\nSignificant work has gone into minimizing false positives. However, due to the nature of how Lightfuzz works, they are a reality. Random hiccups in network connectivity can cause them in some cases, odd WAF behavior can account for others. \n\nIf you see a false positive that you feel is occuring too often or could easily be prevented, please open a GitHub issue and we will take a look!\n\n### Deadly module\n\nLightfuzz currently has the `deadly` flag. This is applied to the most aggressive modules to enforce an additional check, requiring explicit acknowledgement of the risk using the `--allow-deadly` command line flag.\n\n## Modules\n\nLightfuzz is divided into numerous \"submodules\". These would typically be ran all together, but they can be configured to be run individually or in any desired configuration. This would be done with the aide of a `preset`, more on those in a moment.\n\n### `cmdi` (Command Injection)\n - Finds output-based on blind out-of-band (via `Interactsh`) command injections\n### `crypto` (Cryptography)\n - Identifies cryptographic parameters that have a tangable effect on the application\n - Can identify padding oracle vulnerabilities\n - Can identify hash length extention vulnerabilities\n### `path` (Path Traversal)\n - Can find arbitrary file read / local-file include vulnerabilities, based on relative path traversal or with absolute paths\n### `serial` (Deserialization)\n - Can identify the active deserialization of a variety of deserialization types across several platforms\n### `sqli` (SQL Injection)\n - Error Based SQLi Detection\n - Blind time-delay SQLi Detection\n### `ssti` (Server-side Template Injection)\n - Can find basic server-side template injection\n### `xss` (Cross-site Scripting)\n - Can find a variety of XSS types, across several different contexts (between-tags, attribute, Javascript-based)\n## Presets \n\nLightfuzz comes with a few pre-defined presets. The first thing to know is that, unless you really know BBOT inside and out, we recommend using one of them. This is because to be successful, Lightfuzz needs to change a lot of very important BBOT settings. These include:\n\n* Setting `url_querystring_remove` to False. By default, BBOT strips away querystings, so in order to FUZZ GET parameters, that default has to be disabled.\n```\nurl_querystring_remove: False\n```\n* Enabling several other complimentary modules. Specifically, `hunt` and `reflected_parameters` can be useful companion modules that also be useful when `WEB_PARAMETER` events are being emitted.\n\n\nIf you don't want to dive into those details, and we don't blame you, here are the built-in preset options and what you need to know about the differences.\n\n# -p lightfuzz-light\n\nThis is a minimal preset that checks for only the most common vulnerabilities. It enables a select few of lightfuzz's submodules, and is safest for larger scans.\n\n# -p lightfuzz-medium\n\nThis is the default setting. It enables all lightfuzz submodules, and includes all the necessary config options to make Lightfuzz work, without too many extras. However it is important to note that it **DISABLES FUZZING POST REQUESTS**. This is because this type of request is the most intrusive, and the most likely to cause problems, especially in an internal network. \n\n# -p lightfuzz-heavy\n\n* Increases the web spider settings a bit from the default.\n* Adds in the **Param Miner** suite of modules to try and find new parameters to fuzz via brute-force\n* Enables fuzzing of POST parameters\n\n# -p lightfuzz-superheavy\n\nEverything included in `lightfuzz-heavy`, plus:\n\n* Query string collapsing turned OFF. Normally, multiple instances of the same parameter (e.g., foo=bar and foo=bar2) are collapsed into one for fuzzing. With `lightfuzz-superheavy`, each instance is fuzzed individually.\n* Force common headers enabled - Fuzz certain common header parameters, even if we didn't discover them\n* 'Speculate' GET parameters from JSON or XML response bodies\n\nThese settings aren't typically desired as they add significant time to the scan.\n\n# -p lightfuzz-xss\n\nThis is a special Lightfuzz preset that focuses entirely on XSS, to make XSS hunting as fast as possible. It is an example of how to make a preset that focuses on specific submodules. It also includes the `paramminer-getparams` module to help find undocumented parameters to fuzz. \n\n# Spider preset\n\nWe also *strongly* recommend running Lightfuzz with the spider enabled, as this will dramatically increase the number of parameters that are discovered. If you don't, you will see a warning reminding you that things will work a lot better if you do.\n\nThat can be done by simply also enabling either the `spider` or `spider-intense` preset.\n\n# Usage\n\nWith the presets in mind, usage is incredibly simple. In most cases you will just do the following:\n\n```\nbbot -p lightfuzz-medium spider -t targets.txt --allow-deadly\n```\n\nIt's really that simple. Almost all output from Lightfuzz will be in the form of a `FINDING`, as opposed to a `VULNERABILITY`, with a couple of exceptions. This is because, as was explained, the nature of the findings are that they are typically unconfirmed and will require work on your part to do so.\n\nIf you wanted a specific submodule, you could make your own preset adjusting the `modules.lightfuzz.enabled_submodules` setting, or do so via the command line:\n\nJust XSS:\n```\nbbot -p lightfuzz-medium -t targets.txt -c modules.lightfuzz.enabled_submodules=[xss] --allow-deadly\n```\n\nXSS and SQLi:\n```\nbbot -p lightfuzz-medium -t targets.txt -c modules.lightfuzz.enabled_submodules=[xss,sqli] --allow-deadly\n```\n\n\n" }, { "path": "docs/modules/list_of_modules.md", "content": "# List of Modules\n\n\n| Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | Author | Created Date |\n|-----------------------|----------|-----------------|-------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------|---------------------------|----------------|\n| ajaxpro | scan | No | Check for potentially vulnerable Ajaxpro instances | active, safe, web-thorough | HTTP_RESPONSE, URL | FINDING, VULNERABILITY | @liquidsec | 2024-01-18 |\n| aspnet_bin_exposure | scan | No | Check for ASP.NET Security Feature Bypasses (CVE-2023-36899 and CVE-2023-36560) | active, safe, web-thorough | URL | VULNERABILITY | @liquidsec | 2025-01-28 |\n| baddns | scan | No | Check hosts for domain/subdomain takeovers | active, baddns, cloud-enum, safe, subdomain-hijack, web-basic | DNS_NAME, DNS_NAME_UNRESOLVED | FINDING, VULNERABILITY | @liquidsec | 2024-01-18 |\n| baddns_direct | scan | No | Check for unusual subdomain / service takeover edge cases that require direct detection | active, baddns, cloud-enum, safe, subdomain-enum | STORAGE_BUCKET, URL | FINDING, VULNERABILITY | @liquidsec | 2024-01-29 |\n| baddns_zone | scan | No | Check hosts for DNS zone transfers and NSEC walks | active, baddns, cloud-enum, safe, subdomain-enum | DNS_NAME | FINDING, VULNERABILITY | @liquidsec | 2024-01-29 |\n| badsecrets | scan | No | Library for detecting known or weak secrets across many web frameworks | active, safe, web-basic | HTTP_RESPONSE | FINDING, TECHNOLOGY, VULNERABILITY | @liquidsec | 2022-11-19 |\n| bucket_amazon | scan | No | Check for S3 buckets related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-04 |\n| bucket_digitalocean | scan | No | Check for DigitalOcean spaces related to target | active, cloud-enum, safe, slow, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-08 |\n| bucket_firebase | scan | No | Check for open Firebase databases related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2023-03-20 |\n| bucket_google | scan | No | Check for Google object storage related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-04 |\n| bucket_microsoft | scan | No | Check for Azure storage blobs related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-04 |\n| bypass403 | scan | No | Check 403 pages for common bypasses | active, aggressive, web-thorough | URL | FINDING | @liquidsec | 2022-07-05 |\n| dnsbrute | scan | No | Brute-force subdomains with massdns + static wordlist | active, aggressive, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-04-24 |\n| dnsbrute_mutations | scan | No | Brute-force subdomains with massdns + target-specific mutations | active, aggressive, slow, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-04-25 |\n| dnscommonsrv | scan | No | Check for common SRV records | active, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-05-15 |\n| dotnetnuke | scan | No | Scan for critical DotNetNuke (DNN) vulnerabilities | active, aggressive, web-thorough | HTTP_RESPONSE | TECHNOLOGY, VULNERABILITY | @liquidsec | 2023-11-21 |\n| ffuf | scan | No | A fast web fuzzer written in Go | active, aggressive, deadly | URL | URL_UNVERIFIED | @liquidsec | 2022-04-10 |\n| ffuf_shortnames | scan | No | Use ffuf in combination IIS shortnames | active, aggressive, iis-shortnames, web-thorough | URL_HINT | URL_UNVERIFIED | @liquidsec | 2022-07-05 |\n| filedownload | scan | No | Download common filetypes such as PDF, DOCX, PPTX, etc. | active, download, safe, web-basic | HTTP_RESPONSE, URL_UNVERIFIED | FILESYSTEM | @TheTechromancer | 2023-10-11 |\n| fingerprintx | scan | No | Fingerprint exposed services like RDP, SSH, MySQL, etc. | active, safe, service-enum, slow | OPEN_TCP_PORT | PROTOCOL | @TheTechromancer | 2023-01-30 |\n| generic_ssrf | scan | No | Check for generic SSRFs | active, aggressive, web-thorough | URL | VULNERABILITY | @liquidsec | 2022-07-30 |\n| git | scan | No | Check for exposed .git repositories | active, code-enum, safe, web-basic | URL | CODE_REPOSITORY, FINDING | @TheTechromancer | 2023-05-30 |\n| gitlab_com | scan | No | Enumerate GitLab SaaS (gitlab.com/org) for projects and groups | active, code-enum, safe | SOCIAL | CODE_REPOSITORY | @TheTechromancer | 2024-03-11 |\n| gitlab_onprem | scan | No | Detect self-hosted GitLab instances and query them for repositories | active, code-enum, safe | HTTP_RESPONSE, SOCIAL, TECHNOLOGY | CODE_REPOSITORY, FINDING, SOCIAL, TECHNOLOGY | @TheTechromancer | 2024-03-11 |\n| gowitness | scan | No | Take screenshots of webpages | active, safe, web-screenshots | SOCIAL, URL | TECHNOLOGY, URL, URL_UNVERIFIED, WEBSCREENSHOT | @TheTechromancer | 2022-07-08 |\n| graphql_introspection | scan | No | Perform GraphQL introspection on a target | active, safe, web-basic | URL | FINDING | @mukesh-dream11 | 2025-07-01 |\n| host_header | scan | No | Try common HTTP Host header spoofing techniques | active, aggressive, web-thorough | HTTP_RESPONSE | FINDING | @liquidsec | 2022-07-27 |\n| httpx | scan | No | Visit webpages. Many other modules rely on httpx | active, cloud-enum, safe, social-enum, subdomain-enum, web-basic | OPEN_TCP_PORT, URL, URL_UNVERIFIED | HTTP_RESPONSE, URL | @TheTechromancer | 2022-07-08 |\n| hunt | scan | No | Watch for commonly-exploitable HTTP parameters | active, safe, web-thorough | WEB_PARAMETER | FINDING | @liquidsec | 2022-07-20 |\n| iis_shortnames | scan | No | Check for IIS shortname vulnerability | active, iis-shortnames, safe, web-basic | URL | URL_HINT | @liquidsec | 2022-04-15 |\n| legba | scan | No | Credential bruteforcing supporting various services. | active, aggressive, deadly | PROTOCOL | FINDING | @christianfl, @fuzikowski | 2025-07-18 |\n| lightfuzz | scan | No | Find Web Parameters and Lightly Fuzz them using a heuristic based scanner | active, aggressive, deadly, web-thorough | URL, WEB_PARAMETER | FINDING, VULNERABILITY | @liquidsec | 2024-06-28 |\n| medusa | scan | No | Medusa SNMP bruteforcing with v1, v2c and R/W check. | active, aggressive, deadly | PROTOCOL | VULNERABILITY | @christianfl | 2025-05-16 |\n| newsletters | scan | No | Searches for Newsletter Submission Entry Fields on Websites | active, safe | HTTP_RESPONSE | FINDING | @stryker2k2 | 2024-02-02 |\n| ntlm | scan | No | Watch for HTTP endpoints that support NTLM authentication | active, safe, web-basic | HTTP_RESPONSE, URL | DNS_NAME, FINDING | @liquidsec | 2022-07-25 |\n| nuclei | scan | No | Fast and customisable vulnerability scanner | active, aggressive, deadly | URL | FINDING, TECHNOLOGY, VULNERABILITY | @TheTechromancer | 2022-03-12 |\n| oauth | scan | No | Enumerate OAUTH and OpenID Connect services | active, affiliates, cloud-enum, safe, subdomain-enum, web-basic | DNS_NAME, URL_UNVERIFIED | DNS_NAME | @TheTechromancer | 2023-07-12 |\n| paramminer_cookies | scan | No | Smart brute-force to check for common HTTP cookie parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE, WEB_PARAMETER | FINDING, WEB_PARAMETER | @liquidsec | 2022-06-27 |\n| paramminer_getparams | scan | No | Use smart brute-force to check for common HTTP GET parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE, WEB_PARAMETER | FINDING, WEB_PARAMETER | @liquidsec | 2022-06-28 |\n| paramminer_headers | scan | No | Use smart brute-force to check for common HTTP header parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE, WEB_PARAMETER | WEB_PARAMETER | @liquidsec | 2022-04-15 |\n| portscan | scan | No | Port scan with masscan. By default, scans top 100 ports. | active, portscan, safe | DNS_NAME, IP_ADDRESS, IP_RANGE | OPEN_TCP_PORT | @TheTechromancer | 2024-05-15 |\n| reflected_parameters | scan | No | Highlight parameters that reflect their contents in response body | active, safe, web-thorough | WEB_PARAMETER | FINDING | @liquidsec | 2024-10-29 |\n| retirejs | scan | No | Detect vulnerable/out-of-date JavaScript libraries | active, safe, web-thorough | URL_UNVERIFIED | FINDING | @liquidsec | 2025-08-19 |\n| robots | scan | No | Look for and parse robots.txt | active, safe, web-basic | URL | URL_UNVERIFIED | @liquidsec | 2023-02-01 |\n| securitytxt | scan | No | Check for security.txt content | active, cloud-enum, safe, subdomain-enum, web-basic | DNS_NAME | EMAIL_ADDRESS, URL_UNVERIFIED | @colin-stubbs | 2024-05-26 |\n| smuggler | scan | No | Check for HTTP smuggling | active, aggressive, slow, web-thorough | URL | FINDING | @liquidsec | 2022-07-06 |\n| sslcert | scan | No | Visit open ports and retrieve SSL certificates | active, affiliates, email-enum, safe, subdomain-enum, web-basic | OPEN_TCP_PORT | DNS_NAME, EMAIL_ADDRESS | @TheTechromancer | 2022-03-30 |\n| telerik | scan | No | Scan for critical Telerik vulnerabilities | active, aggressive, web-thorough | HTTP_RESPONSE, URL | FINDING, VULNERABILITY | @liquidsec | 2022-04-10 |\n| url_manipulation | scan | No | Attempt to identify URL parsing/routing based vulnerabilities | active, aggressive, web-thorough | URL | FINDING | @liquidsec | 2022-09-27 |\n| vhost | scan | No | Fuzz for virtual hosts | active, aggressive, deadly, slow | URL | DNS_NAME, VHOST | @liquidsec | 2022-05-02 |\n| wafw00f | scan | No | Web Application Firewall Fingerprinting Tool | active, aggressive | URL | WAF | @liquidsec | 2023-02-15 |\n| wpscan | scan | No | Wordpress security scanner. Highly recommended to use an API key for better results. | active, aggressive | HTTP_RESPONSE, TECHNOLOGY | FINDING, TECHNOLOGY, URL_UNVERIFIED, VULNERABILITY | @domwhewell-sage | 2024-05-29 |\n| affiliates | scan | No | Summarize affiliate domains at the end of a scan | affiliates, passive, safe | * | | @TheTechromancer | 2022-07-25 |\n| anubisdb | scan | No | Query jldc.me's database for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-10-04 |\n| apkpure | scan | No | Download android applications from apkpure.com | code-enum, download, passive, safe | MOBILE_APP | FILESYSTEM | @domwhewell-sage | 2024-10-11 |\n| asn | scan | No | Query ripe and bgpview.io for ASNs | passive, safe, subdomain-enum | IP_ADDRESS | ASN | @TheTechromancer | 2022-07-25 |\n| azure_realm | scan | No | Retrieves the \"AuthURL\" from login.microsoftonline.com/getuserrealm | affiliates, cloud-enum, passive, safe, subdomain-enum, web-basic | DNS_NAME | URL_UNVERIFIED | @TheTechromancer | 2023-07-12 |\n| azure_tenant | scan | No | Query Azure via azmap.dev for tenant sister domains | affiliates, cloud-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-07-04 |\n| bevigil | scan | Yes | Retrieve OSINT data from mobile applications using BeVigil | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | @alt-glitch | 2022-10-26 |\n| bucket_file_enum | scan | No | Works in conjunction with the filedownload module to download files from open storage buckets. Currently supported cloud providers: AWS, DigitalOcean | cloud-enum, passive, safe | STORAGE_BUCKET | URL_UNVERIFIED | @TheTechromancer | 2023-11-14 |\n| bufferoverrun | scan | Yes | Query BufferOverrun's TLS API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-10-23 |\n| builtwith | scan | Yes | Query Builtwith.com for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-23 |\n| c99 | scan | Yes | Query the C99 API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-08 |\n| censys_dns | scan | Yes | Query the Censys API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-04 |\n| censys_ip | scan | Yes | Query the Censys API for hosts by IP address | passive, safe | IP_ADDRESS | DNS_NAME, IP_ADDRESS, OPEN_TCP_PORT, OPEN_UDP_PORT, PROTOCOL, TECHNOLOGY, URL_UNVERIFIED | @TheTechromancer | 2026-01-26 |\n| certspotter | scan | No | Query Certspotter's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-28 |\n| chaos | scan | Yes | Query ProjectDiscovery's Chaos API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-14 |\n| code_repository | scan | No | Look for code repository links in webpages | code-enum, passive, safe | URL_UNVERIFIED | CODE_REPOSITORY | @domwhewell-sage | 2024-05-15 |\n| credshed | scan | Yes | Send queries to your own credshed server to check for known credentials of your targets | passive, safe | DNS_NAME | EMAIL_ADDRESS, HASHED_PASSWORD, PASSWORD, USERNAME | @SpamFaux | 2023-10-12 |\n| crt | scan | No | Query crt.sh (certificate transparency) for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-05-13 |\n| crt_db | scan | No | Query crt.sh (certificate transparency) for subdomains via PostgreSQL | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2025-03-27 |\n| dehashed | scan | Yes | Execute queries against dehashed.com for exposed credentials | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS, HASHED_PASSWORD, PASSWORD, USERNAME | @SpamFaux | 2023-10-12 |\n| digitorus | scan | No | Query certificatedetails.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-07-25 |\n| dnsbimi | scan | No | Check DNS_NAME's for BIMI records to find image and certificate hosting URL's | cloud-enum, passive, safe, subdomain-enum | DNS_NAME | RAW_DNS_RECORD, URL_UNVERIFIED | @colin-stubbs | 2024-11-15 |\n| dnscaa | scan | No | Check for CAA records | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | @colin-stubbs | 2024-05-26 |\n| dnsdumpster | scan | No | Query dnsdumpster for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-03-12 |\n| dnstlsrpt | scan | No | Check for TLS-RPT records | cloud-enum, email-enum, passive, safe, subdomain-enum | DNS_NAME | EMAIL_ADDRESS, RAW_DNS_RECORD, URL_UNVERIFIED | @colin-stubbs | 2024-07-26 |\n| docker_pull | scan | No | Download images from a docker repository | code-enum, download, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-24 |\n| dockerhub | scan | No | Search for docker repositories of discovered orgs/usernames | code-enum, passive, safe | ORG_STUB, SOCIAL | CODE_REPOSITORY, SOCIAL, URL_UNVERIFIED | @domwhewell-sage | 2024-03-12 |\n| emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-07-11 |\n| extractous | scan | No | Module to extract data from files | passive, safe | FILESYSTEM | RAW_TEXT | @domwhewell-sage | 2024-06-03 |\n| fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 |\n| git_clone | scan | No | Clone code github repositories | code-enum, download, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-08 |\n| gitdumper | scan | No | Download a leaked .git folder recursively or by fuzzing common names | code-enum, download, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2025-02-11 |\n| github_codesearch | scan | Yes | Query Github's API for code containing the target domain name | code-enum, passive, safe, subdomain-enum | DNS_NAME | CODE_REPOSITORY, URL_UNVERIFIED | @domwhewell-sage | 2023-12-14 |\n| github_org | scan | No | Query Github's API for organization and member repositories | code-enum, passive, safe, subdomain-enum | ORG_STUB, SOCIAL | CODE_REPOSITORY | @domwhewell-sage | 2023-12-14 |\n| github_usersearch | scan | Yes | Query Github's API for users with emails matching in scope domains that may not be discoverable by listing members of the organization. | code-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS, SOCIAL | @domwhewell-sage | 2025-05-10 |\n| github_workflows | scan | Yes | Download a github repositories workflow logs and workflow artifacts | code-enum, download, passive, safe | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-04-29 |\n| google_playstore | scan | No | Search for android applications on play.google.com | code-enum, passive, safe | CODE_REPOSITORY, ORG_STUB | MOBILE_APP | @domwhewell-sage | 2024-10-08 |\n| hackertarget | scan | No | Query the hackertarget.com API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-28 |\n| hunterio | scan | Yes | Query hunter.io for emails | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | @TheTechromancer | 2022-04-25 |\n| ip2location | scan | Yes | Query IP2location.io's API for geolocation information. | passive, safe | IP_ADDRESS | GEOLOCATION | @TheTechromancer | 2023-09-12 |\n| ipneighbor | scan | No | Look beside IPs in their surrounding subnet | aggressive, passive, subdomain-enum | IP_ADDRESS | IP_ADDRESS | @TheTechromancer | 2022-06-08 |\n| ipstack | scan | Yes | Query IPStack's GeoIP API | passive, safe | IP_ADDRESS | GEOLOCATION | @tycoonslive | 2022-11-26 |\n| jadx | scan | No | Decompile APKs and XAPKs using JADX | code-enum, passive, safe | FILESYSTEM | FILESYSTEM | @domwhewell-sage | 2024-11-04 |\n| leakix | scan | No | Query leakix.net for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-11 |\n| myssl | scan | No | Query myssl.com's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-07-10 |\n| otx | scan | Yes | Query otx.alienvault.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 |\n| passivetotal | scan | Yes | Query the PassiveTotal API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-08 |\n| pgp | scan | No | Query common PGP servers for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-08-10 |\n| portfilter | scan | No | Filter out unwanted open ports from cloud/CDN targets | passive, safe | OPEN_TCP_PORT, URL, URL_UNVERIFIED | | @TheTechromancer | 2025-01-06 |\n| postman | scan | No | Query Postman's API for related workspaces, collections, requests and download them | code-enum, passive, safe, subdomain-enum | ORG_STUB, SOCIAL | CODE_REPOSITORY | @domwhewell-sage | 2024-09-07 |\n| postman_download | scan | No | Download workspaces, collections, requests from Postman | code-enum, download, passive, safe, subdomain-enum | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-09-07 |\n| rapiddns | scan | No | Query rapiddns.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 |\n| securitytrails | scan | Yes | Query the SecurityTrails API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-03 |\n| shodan_dns | scan | Yes | Query Shodan for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-03 |\n| shodan_idb | scan | No | Query Shodan's InternetDB for open ports, hostnames, technologies, and vulnerabilities | passive, portscan, safe, subdomain-enum | DNS_NAME, IP_ADDRESS | DNS_NAME, FINDING, OPEN_TCP_PORT, TECHNOLOGY, VULNERABILITY | @TheTechromancer | 2023-12-22 |\n| sitedossier | scan | No | Query sitedossier.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-08-04 |\n| skymem | scan | No | Query skymem.info for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-07-11 |\n| social | scan | No | Look for social media links in webpages | passive, safe, social-enum | URL_UNVERIFIED | SOCIAL | @TheTechromancer | 2023-03-28 |\n| subdomaincenter | scan | No | Query subdomain.center's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-07-26 |\n| subdomainradar | scan | Yes | Query the Subdomain API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-08 |\n| trickest | scan | Yes | Query Trickest's API for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @amiremami | 2024-07-27 |\n| trufflehog | scan | No | TruffleHog is a tool for finding credentials | code-enum, passive, safe | CODE_REPOSITORY, FILESYSTEM, HTTP_RESPONSE, RAW_TEXT | FINDING, VULNERABILITY | @domwhewell-sage | 2024-03-12 |\n| urlscan | scan | No | Query urlscan.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | @TheTechromancer | 2022-06-09 |\n| viewdns | scan | No | Query viewdns.info's reverse whois for related domains | affiliates, passive, safe | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-04 |\n| virustotal | scan | Yes | Query VirusTotal's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-25 |\n| wayback | scan | No | Query archive.org's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | @liquidsec | 2022-04-01 |\n| asset_inventory | output | No | Merge hosts, open ports, technologies, findings, etc. into a single asset inventory CSV | | DNS_NAME, FINDING, HTTP_RESPONSE, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY, WAF | IP_ADDRESS, OPEN_TCP_PORT | @liquidsec | 2022-09-30 |\n| csv | output | No | Output to CSV | | * | | @TheTechromancer | 2022-04-07 |\n| discord | output | No | Message a Discord channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 |\n| emails | output | No | Output any email addresses found belonging to the target domain | email-enum | EMAIL_ADDRESS | | @domwhewell-sage | 2023-12-23 |\n| http | output | No | Send every event to a custom URL via a web request | | * | | @TheTechromancer | 2022-04-13 |\n| json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | @TheTechromancer | 2022-04-07 |\n| mysql | output | No | Output scan data to a MySQL database | | * | | @TheTechromancer | 2024-11-13 |\n| neo4j | output | No | Output to Neo4j | | * | | @TheTechromancer | 2022-04-07 |\n| nmap_xml | output | No | Output to Nmap XML | | DNS_NAME, HTTP_RESPONSE, IP_ADDRESS, OPEN_TCP_PORT, PROTOCOL | | @TheTechromancer | 2024-11-16 |\n| postgres | output | No | Output scan data to a SQLite database | | * | | @TheTechromancer | 2024-11-08 |\n| python | output | No | Output via Python API | | * | | @TheTechromancer | 2022-09-13 |\n| slack | output | No | Message a Slack channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 |\n| splunk | output | No | Send every event to a splunk instance through HTTP Event Collector | | * | | @w0Tx | 2024-02-17 |\n| sqlite | output | No | Output scan data to a SQLite database | | * | | @TheTechromancer | 2024-11-07 |\n| stdout | output | No | Output to text | | * | | @TheTechromancer | 2024-04-03 |\n| subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | @TheTechromancer | 2023-07-31 |\n| teams | output | No | Message a Teams channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 |\n| txt | output | No | Output to text | | * | | @TheTechromancer | 2024-04-03 |\n| web_parameters | output | No | Output WEB_PARAMETER names to a file | | WEB_PARAMETER | | @liquidsec | 2025-01-25 |\n| web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | @liquidsec | 2023-02-08 |\n| websocket | output | No | Output to websockets | | * | | @TheTechromancer | 2022-04-15 |\n| cloudcheck | internal | No | Tag events by cloud provider, identify cloud resources like storage buckets | | * | | @TheTechromancer | 2024-07-07 |\n| dnsresolve | internal | No | Perform DNS resolution | | * | DNS_NAME, IP_ADDRESS, RAW_DNS_RECORD | @TheTechromancer | 2022-04-08 |\n| aggregate | internal | No | Summarize statistics at the end of a scan | passive, safe | | | @TheTechromancer | 2022-07-25 |\n| excavate | internal | No | Passively extract juicy tidbits from scan data | passive | HTTP_RESPONSE, RAW_TEXT | URL_UNVERIFIED, WEB_PARAMETER | @liquidsec | 2022-06-27 |\n| speculate | internal | No | Derive certain event types from others by common sense | passive | AZURE_TENANT, DNS_NAME, DNS_NAME_UNRESOLVED, HTTP_RESPONSE, IP_ADDRESS, IP_RANGE, SOCIAL, STORAGE_BUCKET, URL, URL_UNVERIFIED, USERNAME | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT, ORG_STUB | @liquidsec | 2022-05-03 |\n| unarchive | internal | No | Extract different types of files into folders on the filesystem | passive, safe | FILESYSTEM | FILESYSTEM | @domwhewell-sage | 2024-12-08 |\n\n\nFor a list of module config options, see [Module Options](../scanning/configuration.md#module-config-options).\n" }, { "path": "docs/modules/nuclei.md", "content": "# Nuclei\n\n## Overview\n\nBBOT integrates with [Nuclei](https://github.com/projectdiscovery/nuclei), an open-source web vulnerability scanner by Project Discovery. This is one of the ways BBOT makes it possible to go from a single target domain/IP all the way to confirmed vulnerabilities, in one scan.\n\n![Nuclei Killchain](https://github.com/blacklanternsecurity/bbot/assets/24899338/7174c4ba-4a6e-4596-bb89-5a0c5f5abe74)\n\n\n* The BBOT Nuclei module ingests **[URL]** events and emits events of type **[VULNERABILITY]** or **[FINDING]**\n* Vulnerabilities will inherit their severity from the Nuclei templates\n* Nuclei templates of severity INFO will be emitted as **[FINDINGS]**\n\n## Default Behavior\n\n* By default, only \"directory URLs\" (URLs ending in a slash) will be scanned, but ALL templates will be used (**BE CAREFUL!**)\n* Because it's so aggressive, Nuclei is considered a **deadly** module. This means you need to use the flag **--allow-deadly** to turn it on.\n\n## Specifying custom templates\n\nYou can specify individual nuclei templates by setting the `modules.nuclei.templates` to their comma-separated filenames:\n\n```bash\nbbot -m nuclei -c modules.nuclei.templates=http/takeovers/airee-takeover.yaml,http/takeovers/cargo-takeover.yaml\n```\n\n...or via the config:\n\n```yaml\nmodules:\n nuclei:\n templates: http/takeovers/airee-takeover.yaml,http/takeovers/cargo-takeover.yaml\n```\n\n## Configuration and Options\n\nThe Nuclei module has many configuration options:\n\n\n| Config Option | Type | Description | Default |\n|-------------------------------|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|\n| modules.nuclei.batch_size | int | Number of targets to send to Nuclei per batch (default 200) | 200 |\n| modules.nuclei.budget | int | Used in budget mode to set the number of allowed requests per host | 1 |\n| modules.nuclei.concurrency | int | maximum number of templates to be executed in parallel (default 25) | 25 |\n| modules.nuclei.directory_only | bool | Filter out 'file' URL event (default True) | True |\n| modules.nuclei.etags | str | tags to exclude from the scan | |\n| modules.nuclei.mode | str | manual | technology | severe | budget. Technology: Only activate based on technology events that match nuclei tags (nuclei -as mode). Manual (DEFAULT): Fully manual settings. Severe: Only critical and high severity templates without intrusive. Budget: Limit Nuclei to a specified number of HTTP requests | manual |\n| modules.nuclei.module_timeout | int | Max time in seconds to spend handling each batch of events | 21600 |\n| modules.nuclei.ratelimit | int | maximum number of requests to send per second (default 150) | 150 |\n| modules.nuclei.retries | int | number of times to retry a failed request (default 0) | 0 |\n| modules.nuclei.severity | str | Filter based on severity field available in the template. | |\n| modules.nuclei.silent | bool | Don't display nuclei's banner or status messages | False |\n| modules.nuclei.tags | str | execute a subset of templates that contain the provided tags | |\n| modules.nuclei.templates | str | template or template directory paths to include in the scan | |\n| modules.nuclei.version | str | nuclei version | 3.7.0 |\n\n\nMost of these you probably will **NOT** want to change. In particular, we advise against changing the version of Nuclei, as it's possible the latest version won't work right with BBOT.\n\nWe also do not recommend changing **directory_only** mode. This will cause Nuclei to process every URL. Because BBOT is recursive, this can get very out-of-hand very quickly, depending on which other modules are in use.\n\n### Modes ###\n\nThe modes with the Nuclei module are generally in place to help you limit the number of templates you are scanning with, to make your scans quicker.\n\n#### Manual\n\nThis is the default setting, and will use all templates. However, if you're looking to do something particular, you might pair this with some of the pass-through options shown in the next setting.\n\n#### Severe\n\n**severe** mode uses only high/critical severity templates. It also excludes the intrusive tag. This is intended to be a shortcut for times when you need to rapidly identify high severity vulnerabilities but can't afford the full scan. Because most templates are INFO, LOW, or MEDIUM, your scan will finish much faster.\n\n#### Technology\n\nThis is equivalent to the Nuclei '-as' scan option. It only use templates that match detected technologies, using wappalyzer-based signatures. This can be a nice way to run a light-weight scan that still has a chance to find some good vulnerabilities.\n\n#### Budget\n\nBudget mode is unique to BBOT.\n\nFor larger scans with thousands of targets, doing a FULL Nuclei scan (1000s of Requests) for each is not realistic.\nAs an alternative to the other modes, you can take advantage of Nuclei's \"collapsible\" template feature.\n\nFor only the cost of one (or more) \"extra\" request(s) per host, it can activate several hundred modules. These are modules which happen to look at a BaseUrl, and typically look for a specific string or other attribute. Nuclei is smart about reusing the request data when it can, and we can use this to our advantage.\n\nThe budget parameter is the # of extra requests per host you are willing to send to \"feed\" Nuclei templates (defaults to 1).\nFor those times when vulnerability scanning isn't the main focus, but you want to look for easy wins.\n\nOf course, there is a rapidly diminishing return when you set he value to more than a handful. Eventually, this becomes 1 template per 1 budget value increase. However, in the 1-10 range there is a lot of value. This graphic should give you a rough visual idea of this concept.\n\n![Nuclei Budget Mode](https://github.com/blacklanternsecurity/bbot/assets/24899338/08a3429c-5a73-437b-84de-27c07d85a529)\n\n\n### Nuclei pass-through options\n\nMost of the rest of the options are usually passed straight through to Nuclei when its executed. You can do things like set specific **tags** to include, (or exclude with **etags**), exactly how you'd do with Nuclei directly. You can also limit the templates with **severity**.\n\nThe **ratelimit** and **concurrency** settings default to the same defaults that Nuclei does. These are relatively sane settings, but if you are in a sensitive environment it can certainly help to turn them down.\n\n**templates** will allow you to set your own templates directory. This can be very useful if you have your own custom templates that you want to use with BBOT.\n\n### Example Commands\n\n```bash\n# Scan a SINGLE target with a basic port scan and web modules\nbbot -f web-basic -m portscan nuclei --allow-deadly -t app.evilcorp.com\n```\n\n```bash\n# Scanning MULTIPLE targets\nbbot -f web-basic -m portscan nuclei --allow-deadly -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com\n```\n\n```bash\n# Scanning MULTIPLE targets while performing subdomain enumeration\nbbot -f subdomain-enum web-basic -m portscan nuclei --allow-deadly -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com\n```\n\n```bash\n# Scanning MULTIPLE targets on a BUDGET\nbbot -f subdomain-enum web-basic -m portscan nuclei --allow-deadly -c modules.nuclei.mode=budget -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com\n```\n" }, { "path": "docs/release_history.md", "content": "### 2.8.0 - Jan 20, 2026\n- [https://github.com/blacklanternsecurity/bbot/pull/2760](https://github.com/blacklanternsecurity/bbot/pull/2760)\n\n### 2.7.2 - Oct 25, 2025\n- [https://github.com/blacklanternsecurity/bbot/pull/2717](https://github.com/blacklanternsecurity/bbot/pull/2717)\n\n### 2.7.1 - Sep 16, 2025\n- [https://github.com/blacklanternsecurity/bbot/pull/2700](https://github.com/blacklanternsecurity/bbot/pull/2700)\n\n### 2.7.0 - Sep 11, 2025\n- [https://github.com/blacklanternsecurity/bbot/pull/2610](https://github.com/blacklanternsecurity/bbot/pull/2610)\n\n### 2.6.0 - Aug 12, 2025\n- [https://github.com/blacklanternsecurity/bbot/pull/2492](https://github.com/blacklanternsecurity/bbot/pull/2492)\n\n### 2.5.0 - June 3, 2025\n- [https://github.com/blacklanternsecurity/bbot/pull/2435](https://github.com/blacklanternsecurity/bbot/pull/2435)\n\n### 2.4.0 - Feb 27, 2025\n- [https://github.com/blacklanternsecurity/bbot/pull/2266](https://github.com/blacklanternsecurity/bbot/pull/2266)\n\n### 2.3.0 - Jan 24, 2025\n- [https://github.com/blacklanternsecurity/bbot/pull/1986](https://github.com/blacklanternsecurity/bbot/pull/1986)\n\n### 2.2.0 - Nov 18, 2024\n- [https://github.com/blacklanternsecurity/bbot/pull/1919](https://github.com/blacklanternsecurity/bbot/pull/1919)\n\n### 2.1.2 - Nov 1, 2024\n- [https://github.com/blacklanternsecurity/bbot/pull/1909](https://github.com/blacklanternsecurity/bbot/pull/1909)\n\n### 2.1.1 - Oct 31, 2024\n- [https://github.com/blacklanternsecurity/bbot/pull/1885](https://github.com/blacklanternsecurity/bbot/pull/1885)\n\n### 2.1.0 - Oct 18, 2024\n- [https://github.com/blacklanternsecurity/bbot/pull/1724](https://github.com/blacklanternsecurity/bbot/pull/1724)\n\n### 2.0.1 - Aug 29, 2024\n- [https://github.com/blacklanternsecurity/bbot/pull/1650](https://github.com/blacklanternsecurity/bbot/pull/1650)\n\n### 2.0.0 - Aug 9, 2024\n- [https://github.com/blacklanternsecurity/bbot/pull/1424](https://github.com/blacklanternsecurity/bbot/pull/1424)\n- [https://github.com/blacklanternsecurity/bbot/pull/1235](https://github.com/blacklanternsecurity/bbot/pull/1235)\n\n### 1.1.8 - May 29, 2024\n- [https://github.com/blacklanternsecurity/bbot/pull/1382](https://github.com/blacklanternsecurity/bbot/pull/1382)\n\n### 1.1.7 - May 15, 2024\n- [https://github.com/blacklanternsecurity/bbot/pull/1119](https://github.com/blacklanternsecurity/bbot/pull/1119)\n\n### 1.1.6 - Feb 21, 2024\n- [https://github.com/blacklanternsecurity/bbot/pull/1002](https://github.com/blacklanternsecurity/bbot/pull/1002)\n\n### 1.1.5 - Jan 15, 2024\n- [https://github.com/blacklanternsecurity/bbot/pull/996](https://github.com/blacklanternsecurity/bbot/pull/996)\n\n### 1.1.4 - Jan 11, 2024\n- [https://github.com/blacklanternsecurity/bbot/pull/837](https://github.com/blacklanternsecurity/bbot/pull/837)\n\n### 1.1.3 - Nov 4, 2023\n- [https://github.com/blacklanternsecurity/bbot/pull/823](https://github.com/blacklanternsecurity/bbot/pull/823)\n\n### 1.1.2 - Nov 3, 2023\n- [https://github.com/blacklanternsecurity/bbot/pull/777](https://github.com/blacklanternsecurity/bbot/pull/777)\n\n### 1.1.1 - Oct 11, 2023\n- [https://github.com/blacklanternsecurity/bbot/pull/668](https://github.com/blacklanternsecurity/bbot/pull/668)\n\n### 1.1.0 - Aug 4, 2023\n- [https://github.com/blacklanternsecurity/bbot/pull/598](https://github.com/blacklanternsecurity/bbot/pull/598)\n\n### 1.0.5 - Mar 10, 2023\n- [https://github.com/blacklanternsecurity/bbot/pull/352](https://github.com/blacklanternsecurity/bbot/pull/352)\n\n### 1.0.5 - Mar 10, 2023\n- [https://github.com/blacklanternsecurity/bbot/pull/352](https://github.com/blacklanternsecurity/bbot/pull/352)\n" }, { "path": "docs/scanning/advanced.md", "content": "# Advanced\n\nBelow you can find some advanced uses of BBOT.\n\n## BBOT as a Python library\n\n#### Synchronous\n```python\nfrom bbot.scanner import Scanner\n\nif __name__ == \"__main__\":\n scan = Scanner(\"evilcorp.com\", presets=[\"subdomain-enum\"])\n for event in scan.start():\n print(event)\n```\n\n#### Asynchronous\n```python\nfrom bbot.scanner import Scanner\n\nasync def main():\n scan = Scanner(\"evilcorp.com\", presets=[\"subdomain-enum\"])\n async for event in scan.async_start():\n print(event.json())\n\nif __name__ == \"__main__\":\n import asyncio\n asyncio.run(main())\n```\n\n## Command-Line Help\n\n\n```text\nusage: bbot [-h] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]]\n [-b BLACKLIST [BLACKLIST ...]] [--strict-scope]\n [-p [PRESET ...]] [-c [CONFIG ...]] [-lp]\n [-m MODULE [MODULE ...]] [-l] [-lmo] [-em MODULE [MODULE ...]]\n [-f FLAG [FLAG ...]] [-lf] [-rf FLAG [FLAG ...]]\n [-ef FLAG [FLAG ...]] [--allow-deadly] [-n SCAN_NAME] [-v] [-d]\n [-s] [--force] [-y] [--fast-mode] [--dry-run]\n [--current-preset] [--current-preset-full] [-mh MODULE]\n [-o DIR] [-om MODULE [MODULE ...]] [-lo] [--json] [--brief]\n [--event-types EVENT_TYPES [EVENT_TYPES ...]] [--exclude-cdn]\n [--no-deps | --force-deps | --retry-deps |\n --ignore-failed-deps | --install-all-deps] [--version]\n [--proxy HTTP_PROXY] [-H CUSTOM_HEADERS [CUSTOM_HEADERS ...]]\n [-C CUSTOM_COOKIES [CUSTOM_COOKIES ...]]\n [--custom-yara-rules CUSTOM_YARA_RULES]\n [--user-agent USER_AGENT]\n\nBighuge BLS OSINT Tool\n\noptions:\n -h, --help show this help message and exit\n\nTarget:\n -t, --targets TARGET [TARGET ...]\n Targets to seed the scan\n -w, --whitelist WHITELIST [WHITELIST ...]\n What's considered in-scope (by default it's the same as --targets)\n -b, --blacklist BLACKLIST [BLACKLIST ...]\n Don't touch these things\n --strict-scope Don't consider subdomains of target/whitelist to be in-scope\n\nPresets:\n -p, --preset [PRESET ...]\n Enable BBOT preset(s)\n -c, --config [CONFIG ...]\n Custom config options in key=value format: e.g. 'modules.shodan.api_key=1234'\n -lp, --list-presets List available presets.\n\nModules:\n -m, --modules MODULE [MODULE ...]\n Modules to enable. Choices: affiliates,ajaxpro,anubisdb,apkpure,asn,aspnet_bin_exposure,azure_realm,azure_tenant,baddns,baddns_direct,baddns_zone,badsecrets,bevigil,bucket_amazon,bucket_digitalocean,bucket_file_enum,bucket_firebase,bucket_google,bucket_microsoft,bufferoverrun,builtwith,bypass403,c99,censys_dns,censys_ip,certspotter,chaos,code_repository,credshed,crt,crt_db,dehashed,digitorus,dnsbimi,dnsbrute,dnsbrute_mutations,dnscaa,dnscommonsrv,dnsdumpster,dnstlsrpt,docker_pull,dockerhub,dotnetnuke,emailformat,extractous,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,git_clone,gitdumper,github_codesearch,github_org,github_usersearch,github_workflows,gitlab_com,gitlab_onprem,google_playstore,gowitness,graphql_introspection,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,ip2location,ipneighbor,ipstack,jadx,leakix,legba,lightfuzz,medusa,myssl,newsletters,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,portfilter,portscan,postman,postman_download,rapiddns,reflected_parameters,retirejs,robots,securitytrails,securitytxt,shodan_dns,shodan_idb,sitedossier,skymem,smuggler,social,sslcert,subdomaincenter,subdomainradar,telerik,trickest,trufflehog,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wayback,wpscan\n -l, --list-modules List available modules.\n -lmo, --list-module-options\n Show all module config options\n -em, --exclude-modules MODULE [MODULE ...]\n Exclude these modules.\n -f, --flags FLAG [FLAG ...]\n Enable modules by flag. Choices: active,affiliates,aggressive,baddns,cloud-enum,code-enum,deadly,download,email-enum,iis-shortnames,passive,portscan,safe,service-enum,slow,social-enum,subdomain-enum,subdomain-hijack,web-basic,web-paramminer,web-screenshots,web-thorough\n -lf, --list-flags List available flags.\n -rf, --require-flags FLAG [FLAG ...]\n Only enable modules with these flags (e.g. -rf passive)\n -ef, --exclude-flags FLAG [FLAG ...]\n Disable modules with these flags. (e.g. -ef aggressive)\n --allow-deadly Enable the use of highly aggressive modules\n\nScan:\n -n, --name SCAN_NAME Name of scan (default: random)\n -v, --verbose Be more verbose\n -d, --debug Enable debugging\n -s, --silent Be quiet\n --force Run scan even in the case of condition violations or failed module setups\n -y, --yes Skip scan confirmation prompt\n --fast-mode Scan only the provided targets as fast as possible, with no extra discovery\n --dry-run Abort before executing scan\n --current-preset Show the current preset in YAML format\n --current-preset-full\n Show the current preset in its full form, including defaults\n -mh, --module-help MODULE\n Show help for a specific module\n\nOutput:\n -o, --output-dir DIR Directory to output scan results\n -om, --output-modules MODULE [MODULE ...]\n Output module(s). Choices: asset_inventory,csv,discord,emails,http,json,mysql,neo4j,nmap_xml,postgres,python,slack,splunk,sqlite,stdout,subdomains,teams,txt,web_parameters,web_report,websocket\n -lo, --list-output-modules\n List available output modules\n --json, -j Output scan data in JSON format\n --brief, -br Output only the data itself\n --event-types EVENT_TYPES [EVENT_TYPES ...]\n Choose which event types to display\n --exclude-cdn, -ec Filter out unwanted open ports on CDNs/WAFs (80,443 only)\n\nModule dependencies:\n Control how modules install their dependencies\n\n --no-deps Don't install module dependencies\n --force-deps Force install all module dependencies\n --retry-deps Try again to install failed module dependencies\n --ignore-failed-deps Run modules even if they have failed dependencies\n --install-all-deps Install dependencies for all modules\n\nMisc:\n --version show BBOT version and exit\n --proxy HTTP_PROXY Use this proxy for all HTTP requests\n -H, --custom-headers CUSTOM_HEADERS [CUSTOM_HEADERS ...]\n List of custom headers as key value pairs (header=value).\n -C, --custom-cookies CUSTOM_COOKIES [CUSTOM_COOKIES ...]\n List of custom cookies as key value pairs (cookie=value).\n --custom-yara-rules, -cy CUSTOM_YARA_RULES\n Add custom yara rules to excavate\n --user-agent, -ua USER_AGENT\n Set the user-agent for all HTTP requests\n\nEXAMPLES\n\n Subdomains:\n bbot -t evilcorp.com -p subdomain-enum\n\n Subdomains (passive only):\n bbot -t evilcorp.com -p subdomain-enum -rf passive\n\n Subdomains + port scan + web screenshots:\n bbot -t evilcorp.com -p subdomain-enum -m portscan gowitness -n my_scan -o .\n\n Subdomains + basic web scan:\n bbot -t evilcorp.com -p subdomain-enum web-basic\n\n Web spider:\n bbot -t www.evilcorp.com -p spider -c web.spider_distance=2 web.spider_depth=2\n\n Everything everywhere all at once:\n bbot -t evilcorp.com -p kitchen-sink\n\n List modules:\n bbot -l\n\n List output modules:\n bbot -lo\n\n List presets:\n bbot -lp\n\n List flags:\n bbot -lf\n\n Show help for a specific module:\n bbot -mh \n\n```\n\n" }, { "path": "docs/scanning/configuration.md", "content": "# Configuration Overview\n\nNormally, [Presets](presets.md) are used to configure a scan. However, there may be cases where you want to change BBOT's global defaults so a certain option is always set, even if it's not specified in a preset.\n\nBBOT has a YAML config at `~/.config/bbot.yml`. This is the first config that BBOT loads, so it's a good place to put default settings like `http_proxy`, `max_threads`, or `http_user_agent`. You can also put any module settings here, including **API keys**.\n\nFor a list of all possible config options, see:\n\n- [Global Options](#global-config-options)\n- [Module Options](#module-config-options)\n\nFor examples of common config changes, see [Tips and Tricks](tips_and_tricks.md).\n\n## Configuration Files\n\nBBOT loads its config from the following files, in this order (last one loaded == highest priority):\n\n- `~/.config/bbot/bbot.yml` <-- Global BBOT config\n- presets (`-p`) <-- Presets are good for scan-specific settings\n- command line (`-c`) <-- CLI overrides everything\n\n`bbot.yml` will be automatically created for you when you first run BBOT.\n\n## YAML Config vs Command Line\n\nYou can specify config options either via the command line or the config. For example, if you want to proxy your BBOT scan through a local proxy like [Burp Suite](https://portswigger.net/burp), you could either do:\n\n```bash\n# send BBOT traffic through an HTTP proxy\nbbot -t evilcorp.com -c http_proxy=http://127.0.0.1:8080\n```\n\nOr, in `~/.config/bbot/config.yml`:\n\n```yaml title=\"~/.bbot/config/bbot.yml\"\nhttp_proxy: http://127.0.0.1:8080\n```\n\nThese two are equivalent.\n\nConfig options specified via the command-line take precedence over all others. You can give BBOT a custom config file with `-c myconf.yml`, or individual arguments like this: `-c modules.shodan_dns.api_key=deadbeef`. To display the full and current BBOT config, including any command-line arguments, use `bbot -c`.\n\nNote that placing the following in `bbot.yml`:\n```yaml title=\"~/.bbot/config/bbot.yml\"\nmodules:\n shodan_dns:\n api_key: deadbeef\n```\nIs the same as:\n```bash\nbbot -c modules.shodan_dns.api_key=deadbeef\n```\n\n## Global Config Options\n\nBelow is a full list of the config options supported, along with their defaults.\n\n\n```yaml title=\"defaults.yml\"\n### BASIC OPTIONS ###\n\n# BBOT working directory\nhome: ~/.bbot\n# How many scan results to keep before cleaning up the older ones\nkeep_scans: 20\n# Interval for displaying status messages\nstatus_frequency: 15\n# Include the raw data of files (i.e. PDFs, web screenshots) as base64 in the event\nfile_blobs: false\n# Include the raw data of directories (i.e. git repos) as tar.gz base64 in the event\nfolder_blobs: false\n\n### SCOPE ###\n\nscope:\n # strict scope means only exact DNS names are considered in-scope\n # subdomains are not included unless they are explicitly provided in the target list\n strict: false\n # Filter by scope distance which events are displayed in the output\n # 0 == show only in-scope events (affiliates are always shown)\n # 1 == show all events up to distance-1 (1 hop from target)\n report_distance: 0\n # How far out from the main scope to search\n # Do not change this setting unless you know what you're doing\n search_distance: 0\n\n### DNS ###\n\ndns:\n # Completely disable DNS resolution (careful if you have IP whitelists/blacklists, consider using minimal=true instead)\n disable: false\n # Speed up scan by not creating any new DNS events, and only resolving A and AAAA records\n minimal: false\n # How many instances of the dns module to run concurrently\n threads: 25\n # How many concurrent DNS resolvers to use when brute-forcing\n # (under the hood this is passed through directly to massdns -s)\n brute_threads: 1000\n # nameservers to use for DNS brute-forcing\n # default is updated weekly and contains ~10K high-quality public servers\n brute_nameservers: https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt\n # How far away from the main target to explore via DNS resolution (independent of scope.search_distance)\n # This is safe to change\n search_distance: 1\n # Limit how many DNS records can be followed in a row (stop malicious/runaway DNS records)\n runaway_limit: 5\n # DNS query timeout\n timeout: 5\n # How many times to retry DNS queries\n retries: 1\n # Completely disable BBOT's DNS wildcard detection\n wildcard_disable: False\n # Disable BBOT's DNS wildcard detection for select domains\n wildcard_ignore: []\n # How many sanity checks to make when verifying wildcard DNS\n # Increase this value if BBOT's wildcard detection isn't working\n wildcard_tests: 10\n # Skip DNS requests for a certain domain and rdtype after encountering this many timeouts or SERVFAILs\n # This helps prevent faulty DNS servers from hanging up the scan\n abort_threshold: 50\n # Don't show PTR records containing IP addresses\n filter_ptrs: true\n # Enable/disable debug messages for DNS queries\n debug: false\n # For performance reasons, always skip these DNS queries\n # Microsoft's DNS infrastructure is misconfigured so that certain queries to mail.protection.outlook.com always time out\n omit_queries:\n - SRV:mail.protection.outlook.com\n - CNAME:mail.protection.outlook.com\n - TXT:mail.protection.outlook.com\n\n### WEB ###\n\nweb:\n # HTTP proxy\n http_proxy:\n # Web user-agent\n user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.2151.97\n # Set the maximum number of HTTP links that can be followed in a row (0 == no spidering allowed)\n spider_distance: 0\n # Set the maximum directory depth for the web spider\n spider_depth: 1\n # Set the maximum number of links that can be followed per page\n spider_links_per_page: 25\n # HTTP timeout (for Python requests; API calls, etc.)\n http_timeout: 10\n # HTTP timeout (for httpx)\n httpx_timeout: 5\n # Custom HTTP headers (e.g. cookies, etc.)\n # in the format { \"Header-Key\": \"header_value\" }\n # These are attached to all in-scope HTTP requests\n # Note that some modules (e.g. github) may end up sending these to out-of-scope resources\n http_headers: {}\n # How many times to retry API requests\n # Note that this is a separate mechanism on top of HTTP retries\n # which will retry API requests that don't return a successful status code\n api_retries: 2\n # HTTP retries - try again if the raw connection fails\n http_retries: 1\n # HTTP retries (for httpx)\n httpx_retries: 1\n # Default sleep interval when rate limited by 429 (and retry-after isn't provided)\n 429_sleep_interval: 30\n # Maximum sleep interval when rate limited by 429 (and an excessive retry-after is provided)\n 429_max_sleep_interval: 60\n # Enable/disable debug messages for web requests/responses\n debug: false\n # Maximum number of HTTP redirects to follow\n http_max_redirects: 5\n # Whether to verify SSL certificates\n ssl_verify: false\n\n### ENGINE ###\n\nengine:\n debug: false\n\n# Tool dependencies\ndeps:\n ffuf:\n version: \"2.1.0\"\n # How to handle installation of module dependencies\n # Choices are:\n # - abort_on_failure (default) - if a module dependency fails to install, abort the scan\n # - retry_failed - try again to install failed dependencies\n # - ignore_failed - run the scan regardless of what happens with dependency installation\n # - disable - completely disable BBOT's dependency system (you are responsible for installing tools, pip packages, etc.)\n behavior: abort_on_failure\n\n### ADVANCED OPTIONS ###\n\n# Load BBOT modules from these custom paths\nmodule_dirs: []\n\n# maximum runtime in seconds for each module's handle_event() is 60 minutes\n# when the timeout is reached, the offending handle_event() will be cancelled and the module will move on to the next event\nmodule_handle_event_timeout: 3600\n# handle_batch() default timeout is 2 hours\nmodule_handle_batch_timeout: 7200\n\n# Infer certain events from others, e.g. IPs from IP ranges, DNS_NAMEs from URLs, etc.\nspeculate: True\n# Passively search event data for URLs, hostnames, emails, etc.\nexcavate: True\n# Summarize activity at the end of a scan\naggregate: True\n# DNS resolution, wildcard detection, etc.\ndnsresolve: True\n# Cloud provider tagging\ncloudcheck: True\n\n# Strip querystring from URLs by default\nurl_querystring_remove: True\n# When query string is retained, by default collapse parameter values down to a single value per parameter\nurl_querystring_collapse: True\n\n# Completely ignore URLs with these extensions\nurl_extension_blacklist:\n # images\n - png\n - jpg\n - bmp\n - ico\n - jpeg\n - gif\n - svg\n - webp\n # web/fonts\n - css\n - woff\n - woff2\n - ttf\n - eot\n - sass\n - scss\n # audio\n - mp3\n - m4a\n - wav\n - flac\n # video\n - mp4\n - mkv\n - avi\n - wmv\n - mov\n - flv\n - webm\n\n# URLs with these extensions are not distributed to modules unless the module opts in via `accept_url_special = True`\n# They are also excluded from output. If you want to see them in output, remove them from this list.\nurl_extension_special:\n - js\n\n# These url extensions are almost always static, so we exclude them from modules that fuzz things\nurl_extension_static:\n - pdf\n - doc\n - docx\n - xls\n - xlsx\n - ppt\n - pptx\n - txt\n - csv\n - xml\n - yaml\n - ini\n - log\n - conf\n - cfg\n - env\n - md\n - rtf\n - tiff\n - bmp\n - jpg\n - jpeg\n - png\n - gif\n - svg\n - ico\n - mp3\n - wav\n - flac\n - mp4\n - mov\n - avi\n - mkv\n - webm\n - zip\n - tar\n - gz\n - bz2\n - 7z\n - rar\n\nparameter_blacklist:\n - __VIEWSTATE\n - __EVENTARGUMENT\n - __EVENTVALIDATION\n - __EVENTTARGET\n - __EVENTARGUMENT\n - __VIEWSTATEGENERATOR\n - __SCROLLPOSITIONY\n - __SCROLLPOSITIONX\n - ASP.NET_SessionId\n - PHPSESSID\n - __cf_bm\n - f5_cspm\n\nparameter_blacklist_prefixes:\n - TS01\n - BIGipServer\n - incap_\n - visid_incap_\n - AWSALB\n - utm_\n - ApplicationGatewayAffinity\n - JSESSIONID\n - ARRAffinity\n\n# Don't output these types of events (they are still distributed to modules)\nomit_event_types:\n - HTTP_RESPONSE\n - RAW_TEXT\n - URL_UNVERIFIED\n - DNS_NAME_UNRESOLVED\n - FILESYSTEM\n - WEB_PARAMETER\n - RAW_DNS_RECORD\n # - IP_ADDRESS\n\n# Custom interactsh server settings\ninteractsh_server: null\ninteractsh_token: null\ninteractsh_disable: false\n\n```\n\n\n## Module Config Options\n\nMany modules accept their own configuration options. These options have the ability to change their behavior. For example, the `portscan` module accepts options for `ports`, `rate`, etc. Below is a list of all possible module config options.\n\n### Universal Module Options\n\nIn addition to the stated options for each module, the following universal options are also accepted:\n\n\n**batch_size**: The number of events to process in a single batch (only applies to batch modules)\n**module_threads**: How many event handlers to run in parallel\n**module_timeout**: Max time in seconds to spend handling each event or batch of events\n\n\n\n### Module Options\n\n\n| Config Option | Type | Description | Default |\n|-----------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| modules.baddns.custom_nameservers | list | Force BadDNS to use a list of custom nameservers | [] |\n| modules.baddns.enabled_submodules | list | A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only | [] |\n| modules.baddns.only_high_confidence | bool | Do not emit low-confidence or generic detections | False |\n| modules.baddns_direct.custom_nameservers | list | Force BadDNS to use a list of custom nameservers | [] |\n| modules.baddns_zone.custom_nameservers | list | Force BadDNS to use a list of custom nameservers | [] |\n| modules.baddns_zone.only_high_confidence | bool | Do not emit low-confidence or generic detections | False |\n| modules.badsecrets.custom_secrets | NoneType | Include custom secrets loaded from a local file | None |\n| modules.bucket_amazon.permutations | bool | Whether to try permutations | False |\n| modules.bucket_digitalocean.permutations | bool | Whether to try permutations | False |\n| modules.bucket_firebase.permutations | bool | Whether to try permutations | False |\n| modules.bucket_google.permutations | bool | Whether to try permutations | False |\n| modules.bucket_microsoft.permutations | bool | Whether to try permutations | False |\n| modules.dnsbrute.max_depth | int | How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com | 5 |\n| modules.dnsbrute.wordlist | str | Subdomain wordlist URL | https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt |\n| modules.dnsbrute_mutations.max_mutations | int | Maximum number of target-specific mutations to try per subdomain | 100 |\n| modules.dnscommonsrv.max_depth | int | The maximum subdomain depth to brute-force SRV records | 2 |\n| modules.ffuf.extensions | str | Optionally include a list of extensions to extend the keyword with (comma separated) | |\n| modules.ffuf.ignore_case | bool | Only put lowercase words into the wordlist | False |\n| modules.ffuf.lines | int | take only the first N lines from the wordlist when finding directories | 5000 |\n| modules.ffuf.max_depth | int | the maximum directory depth to attempt to solve | 0 |\n| modules.ffuf.rate | int | Rate of requests per second (default: 0) | 0 |\n| modules.ffuf.wordlist | str | Specify wordlist to use when finding directories | https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt |\n| modules.ffuf_shortnames.extensions | str | Optionally include a list of extensions to extend the keyword with (comma separated) | |\n| modules.ffuf_shortnames.find_common_prefixes | bool | Attempt to automatically detect common prefixes and make additional ffuf runs against them | False |\n| modules.ffuf_shortnames.find_delimiters | bool | Attempt to detect common delimiters and make additional ffuf runs against them | True |\n| modules.ffuf_shortnames.find_subwords | bool | Attempt to detect subwords and make additional ffuf runs against them | False |\n| modules.ffuf_shortnames.ignore_redirects | bool | Explicitly ignore redirects (301,302) | True |\n| modules.ffuf_shortnames.max_depth | int | the maximum directory depth to attempt to solve | 1 |\n| modules.ffuf_shortnames.max_predictions | int | The maximum number of predictions to generate per shortname prefix | 250 |\n| modules.ffuf_shortnames.rate | int | Rate of requests per second (default: 0) | 0 |\n| modules.ffuf_shortnames.version | str | ffuf version | 2.0.0 |\n| modules.ffuf_shortnames.wordlist_extensions | str | Specify wordlist to use when making extension lists | |\n| modules.filedownload.extensions | list | File extensions to download | ['bak', 'bash', 'bashrc', 'cfg', 'conf', 'crt', 'csv', 'db', 'dll', 'doc', 'docx', 'exe', 'ica', 'indd', 'ini', 'jar', 'json', 'key', 'log', 'markdown', 'md', 'msi', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'pub', 'raw', 'rdp', 'rsa', 'sh', 'sql', 'sqlite', 'swp', 'sxw', 'tar.gz', 'tgz', 'tar', 'txt', 'vbs', 'war', 'wpd', 'xls', 'xlsx', 'xml', 'yaml', 'yml', 'zip', 'lzma', 'rar', '7z', 'xz', 'bz2'] |\n| modules.filedownload.max_filesize | str | Cancel download if filesize is greater than this size | 10MB |\n| modules.filedownload.output_folder | str | Folder to download files to. If not specified, downloaded files will be deleted when the scan completes, to minimize disk usage. | |\n| modules.fingerprintx.skip_common_web | bool | Skip common web ports such as 80, 443, 8080, 8443, etc. | True |\n| modules.fingerprintx.version | str | fingerprintx version | 1.1.4 |\n| modules.generic_ssrf.skip_dns_interaction | bool | Do not report DNS interactions (only HTTP interaction) | False |\n| modules.gitlab_com.api_key | str | GitLab access token (for gitlab.com/org only) | |\n| modules.gitlab_onprem.api_key | str | GitLab access token (for self-hosted instances only) | |\n| modules.gowitness.chrome_path | str | Path to chrome executable | |\n| modules.gowitness.idle_timeout | int | Skip the current gowitness batch if it stalls for longer than this many seconds | 1800 |\n| modules.gowitness.output_path | str | Where to save screenshots | |\n| modules.gowitness.resolution_x | int | Screenshot resolution x | 1440 |\n| modules.gowitness.resolution_y | int | Screenshot resolution y | 900 |\n| modules.gowitness.social | bool | Whether to screenshot social media webpages | False |\n| modules.gowitness.threads | int | How many gowitness threads to spawn (default is number of CPUs x 2) | 0 |\n| modules.gowitness.timeout | int | Preflight check timeout | 10 |\n| modules.gowitness.version | str | Gowitness version | 3.0.5 |\n| modules.graphql_introspection.graphql_endpoint_urls | list | List of GraphQL endpoint to suffix to the target URL | ['/', '/graphql', '/v1/graphql'] |\n| modules.graphql_introspection.output_folder | str | Folder to save the GraphQL schemas to | |\n| modules.httpx.in_scope_only | bool | Only visit web reparents that are in scope. | True |\n| modules.httpx.max_response_size | int | Max response size in bytes | 5242880 |\n| modules.httpx.probe_all_ips | bool | Probe all the ips associated with same host | False |\n| modules.httpx.store_responses | bool | Save raw HTTP responses to scan folder | False |\n| modules.httpx.threads | int | Number of httpx threads to use | 50 |\n| modules.httpx.version | str | httpx version | 1.2.5 |\n| modules.iis_shortnames.detect_only | bool | Only detect the vulnerability and do not run the shortname scanner | True |\n| modules.iis_shortnames.max_node_count | int | Limit how many nodes to attempt to resolve on any given recursion branch | 50 |\n| modules.iis_shortnames.speculate_magic_urls | bool | Attempt to discover iis 'magic' special folders | True |\n| modules.legba.concurrency | int | Number of concurrent workers, gets overridden for SSH | 3 |\n| modules.legba.ftp_wordlist | str | Wordlist URL for FTP combined username:password wordlist, newline separated | https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/ftp-betterdefaultpasslist.txt |\n| modules.legba.mssql_wordlist | str | Wordlist URL for MSSQL combined username:password wordlist, newline separated | https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/mssql-betterdefaultpasslist.txt |\n| modules.legba.mysql_wordlist | str | Wordlist URL for MySQL combined username:password wordlist, newline separated | https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/mysql-betterdefaultpasslist.txt |\n| modules.legba.postgresql_wordlist | str | Wordlist URL for PostgreSQL combined username:password wordlist, newline separated | https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/postgres-betterdefaultpasslist.txt |\n| modules.legba.rate_limit | int | Limit the number of requests per second, gets overridden for SSH | 3 |\n| modules.legba.ssh_wordlist | str | Wordlist URL for SSH combined username:password wordlist, newline separated | https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/ssh-betterdefaultpasslist.txt |\n| modules.legba.telnet_wordlist | str | Wordlist URL for TELNET combined username:password wordlist, newline separated | https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/telnet-betterdefaultpasslist.txt |\n| modules.legba.version | str | legba version | 1.1.1 |\n| modules.legba.vnc_wordlist | str | Wordlist URL for VNC password wordlist, newline separated | https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/vnc-betterdefaultpasslist.txt |\n| modules.lightfuzz.avoid_wafs | bool | Avoid running against confirmed WAFs, which are likely to block lightfuzz requests | True |\n| modules.lightfuzz.disable_post | bool | Disable processing of POST parameters, avoiding form submissions. | False |\n| modules.lightfuzz.enabled_submodules | list | A list of submodules to enable. Empty list enabled all modules. | ['sqli', 'cmdi', 'xss', 'path', 'ssti', 'crypto', 'serial', 'esi'] |\n| modules.lightfuzz.force_common_headers | bool | Force emit commonly exploitable parameters that may be difficult to detect | False |\n| modules.lightfuzz.try_get_as_post | bool | For each GETPARAM, also fuzz it as a POSTPARAM (in addition to normal GET fuzzing). | False |\n| modules.lightfuzz.try_post_as_get | bool | For each POSTPARAM, also fuzz it as a GETPARAM (in addition to normal POST fuzzing). | False |\n| modules.medusa.snmp_versions | list | List of SNMP versions to attempt against the SNMP server (default ['1', '2C']) | ['1', '2C'] |\n| modules.medusa.snmp_wordlist | str | Wordlist url for SNMP community strings, newline separated (default https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/snmp.txt) | https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/common-snmp-community-strings.txt |\n| modules.medusa.threads | int | Number of communities to be tested concurrently (default 5) | 5 |\n| modules.medusa.timeout_s | int | Wait time for the SNMP response(s) once at the end of all attempts (default 5) | 5 |\n| modules.medusa.wait_microseconds | int | Wait time after every SNMP request in microseconds (default 200) | 200 |\n| modules.ntlm.try_all | bool | Try every NTLM endpoint | False |\n| modules.nuclei.batch_size | int | Number of targets to send to Nuclei per batch (default 200) | 200 |\n| modules.nuclei.budget | int | Used in budget mode to set the number of allowed requests per host | 1 |\n| modules.nuclei.concurrency | int | maximum number of templates to be executed in parallel (default 25) | 25 |\n| modules.nuclei.directory_only | bool | Filter out 'file' URL event (default True) | True |\n| modules.nuclei.etags | str | tags to exclude from the scan | |\n| modules.nuclei.mode | str | manual | technology | severe | budget. Technology: Only activate based on technology events that match nuclei tags (nuclei -as mode). Manual (DEFAULT): Fully manual settings. Severe: Only critical and high severity templates without intrusive. Budget: Limit Nuclei to a specified number of HTTP requests | manual |\n| modules.nuclei.module_timeout | int | Max time in seconds to spend handling each batch of events | 21600 |\n| modules.nuclei.ratelimit | int | maximum number of requests to send per second (default 150) | 150 |\n| modules.nuclei.retries | int | number of times to retry a failed request (default 0) | 0 |\n| modules.nuclei.severity | str | Filter based on severity field available in the template. | |\n| modules.nuclei.silent | bool | Don't display nuclei's banner or status messages | False |\n| modules.nuclei.tags | str | execute a subset of templates that contain the provided tags | |\n| modules.nuclei.templates | str | template or template directory paths to include in the scan | |\n| modules.nuclei.version | str | nuclei version | 3.7.0 |\n| modules.oauth.try_all | bool | Check for OAUTH/IODC on every subdomain and URL. | False |\n| modules.paramminer_cookies.recycle_words | bool | Attempt to use words found during the scan on all other endpoints | False |\n| modules.paramminer_cookies.skip_boring_words | bool | Remove commonly uninteresting words from the wordlist | True |\n| modules.paramminer_cookies.wordlist | str | Define the wordlist to be used to derive cookies | |\n| modules.paramminer_getparams.recycle_words | bool | Attempt to use words found during the scan on all other endpoints | False |\n| modules.paramminer_getparams.skip_boring_words | bool | Remove commonly uninteresting words from the wordlist | True |\n| modules.paramminer_getparams.wordlist | str | Define the wordlist to be used to derive headers | |\n| modules.paramminer_headers.recycle_words | bool | Attempt to use words found during the scan on all other endpoints | False |\n| modules.paramminer_headers.skip_boring_words | bool | Remove commonly uninteresting words from the wordlist | True |\n| modules.paramminer_headers.wordlist | str | Define the wordlist to be used to derive headers | |\n| modules.portscan.adapter | str | Manually specify a network interface, such as \"eth0\" or \"tun0\". If not specified, the first network interface found with a default gateway will be used. | |\n| modules.portscan.adapter_ip | str | Send packets using this IP address. Not needed unless masscan's autodetection fails | |\n| modules.portscan.adapter_mac | str | Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails | |\n| modules.portscan.module_timeout | int | Max time in seconds to spend handling each batch of events | 259200 |\n| modules.portscan.ping_first | bool | Only portscan hosts that reply to pings | False |\n| modules.portscan.ping_only | bool | Ping sweep only, no portscan | False |\n| modules.portscan.ports | str | Ports to scan | |\n| modules.portscan.rate | int | Rate in packets per second | 300 |\n| modules.portscan.router_mac | str | Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails | |\n| modules.portscan.top_ports | int | Top ports to scan (default 100) (to override, specify 'ports') | 100 |\n| modules.portscan.wait | int | Seconds to wait for replies after scan is complete | 5 |\n| modules.retirejs.node_version | str | Node.js version to install locally | 18.19.1 |\n| modules.retirejs.severity | str | Minimum severity level to report (none, low, medium, high, critical) | medium |\n| modules.retirejs.version | str | retire.js version | 5.3.0 |\n| modules.robots.include_allow | bool | Include 'Allow' Entries | True |\n| modules.robots.include_disallow | bool | Include 'Disallow' Entries | True |\n| modules.robots.include_sitemap | bool | Include 'sitemap' entries | False |\n| modules.securitytxt.emails | bool | emit EMAIL_ADDRESS events | True |\n| modules.securitytxt.urls | bool | emit URL_UNVERIFIED events | True |\n| modules.sslcert.skip_non_ssl | bool | Don't try common non-SSL ports | True |\n| modules.sslcert.timeout | float | Socket connect timeout in seconds | 5.0 |\n| modules.telerik.exploit_RAU_crypto | bool | Attempt to confirm any RAU AXD detections are vulnerable | False |\n| modules.telerik.include_subdirs | bool | Include subdirectories in the scan (off by default) | False |\n| modules.url_manipulation.allow_redirects | bool | Allowing redirects will sometimes create false positives. Disallowing will sometimes create false negatives. Allowed by default. | True |\n| modules.vhost.force_basehost | str | Use a custom base host (e.g. evilcorp.com) instead of the default behavior of using the current URL | |\n| modules.vhost.lines | int | take only the first N lines from the wordlist when finding directories | 5000 |\n| modules.vhost.wordlist | str | Wordlist containing subdomains | https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt |\n| modules.wafw00f.generic_detect | bool | When no specific WAF detections are made, try to perform a generic detect | True |\n| modules.wpscan.api_key | str | WPScan API Key | |\n| modules.wpscan.connection_timeout | int | The connection timeout in seconds (default 2) | 2 |\n| modules.wpscan.disable_tls_checks | bool | Disables the SSL/TLS certificate verification (Default True) | True |\n| modules.wpscan.enumerate | str | Enumeration Process see wpscan help documentation (default: vp,vt,cb,dbe) | vp,vt,cb,dbe |\n| modules.wpscan.force | bool | Do not check if the target is running WordPress or returns a 403 | False |\n| modules.wpscan.request_timeout | int | The request timeout in seconds (default 5) | 5 |\n| modules.wpscan.threads | int | How many wpscan threads to spawn (default is 5) | 5 |\n| modules.anubisdb.limit | int | Limit the number of subdomains returned per query (increasing this may slow the scan due to garbage results from this API) | 1000 |\n| modules.apkpure.output_folder | str | Folder to download APKs to. If not specified, downloaded APKs will be deleted when the scan completes, to minimize disk usage. | |\n| modules.bevigil.api_key | str | BeVigil OSINT API Key | |\n| modules.bevigil.urls | bool | Emit URLs in addition to DNS_NAMEs | False |\n| modules.bucket_file_enum.file_limit | int | Limit the number of files downloaded per bucket | 50 |\n| modules.bufferoverrun.api_key | str | BufferOverrun API key | |\n| modules.bufferoverrun.commercial | bool | Use commercial API | False |\n| modules.builtwith.api_key | str | Builtwith API key | |\n| modules.builtwith.redirects | bool | Also look up inbound and outbound redirects | True |\n| modules.c99.api_key | str | c99.nl API key | |\n| modules.censys_dns.api_key | str | Censys.io API Key in the format of 'key:secret' | |\n| modules.censys_dns.max_pages | int | Maximum number of pages to fetch (100 results per page) | 5 |\n| modules.censys_ip.api_key | str | Censys.io API Key in the format of 'key:secret' | |\n| modules.censys_ip.dns_names_limit | int | Maximum number of DNS names to extract from dns.names (default 100) | 100 |\n| modules.censys_ip.in_scope_only | bool | Only query in-scope IPs. If False, will query up to distance 1. | True |\n| modules.chaos.api_key | str | Chaos API key | |\n| modules.credshed.credshed_url | str | URL of credshed server | |\n| modules.credshed.password | str | Credshed password | |\n| modules.credshed.username | str | Credshed username | |\n| modules.dehashed.api_key | str | DeHashed API Key | |\n| modules.dnsbimi.emit_raw_dns_records | bool | Emit RAW_DNS_RECORD events | False |\n| modules.dnsbimi.emit_urls | bool | Emit URL_UNVERIFIED events | True |\n| modules.dnsbimi.selectors | str | CSV list of BIMI selectors to check | default,email,mail,bimi |\n| modules.dnscaa.dns_names | bool | emit DNS_NAME events | True |\n| modules.dnscaa.emails | bool | emit EMAIL_ADDRESS events | True |\n| modules.dnscaa.in_scope_only | bool | Only check in-scope domains | True |\n| modules.dnscaa.urls | bool | emit URL_UNVERIFIED events | True |\n| modules.dnstlsrpt.emit_emails | bool | Emit EMAIL_ADDRESS events | True |\n| modules.dnstlsrpt.emit_raw_dns_records | bool | Emit RAW_DNS_RECORD events | False |\n| modules.dnstlsrpt.emit_urls | bool | Emit URL_UNVERIFIED events | True |\n| modules.docker_pull.all_tags | bool | Download all tags from each registry (Default False) | False |\n| modules.docker_pull.output_folder | str | Folder to download docker repositories to. If not specified, downloaded docker images will be deleted when the scan completes, to minimize disk usage. | |\n| modules.extractous.extensions | list | File extensions to parse | ['bak', 'bash', 'bashrc', 'conf', 'cfg', 'crt', 'csv', 'db', 'sqlite', 'doc', 'docx', 'ica', 'indd', 'ini', 'json', 'key', 'pub', 'log', 'markdown', 'md', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'rdp', 'rsa', 'sh', 'sql', 'swp', 'sxw', 'txt', 'vbs', 'wpd', 'xls', 'xlsx', 'xml', 'yml', 'yaml'] |\n| modules.fullhunt.api_key | str | FullHunt API Key | |\n| modules.git_clone.api_key | str | Github token | |\n| modules.git_clone.output_folder | str | Folder to clone repositories to. If not specified, cloned repositories will be deleted when the scan completes, to minimize disk usage. | |\n| modules.gitdumper.fuzz_tags | bool | Fuzz for common git tag names (v0.0.1, 0.0.2, etc.) up to the max_semanic_version | False |\n| modules.gitdumper.max_semanic_version | int |` Maximum version number to fuzz for (default < v10.10.10) `| 10 |\n| modules.gitdumper.output_folder | str | Folder to download repositories to. If not specified, downloaded repositories will be deleted when the scan completes, to minimize disk usage. | |\n| modules.github_codesearch.api_key | str | Github token | |\n| modules.github_codesearch.limit | int | Limit code search to this many results | 100 |\n| modules.github_org.api_key | str | Github token | |\n| modules.github_org.include_member_repos | bool | Also enumerate organization members' repositories | False |\n| modules.github_org.include_members | bool | Enumerate organization members | True |\n| modules.github_usersearch.api_key | str | Github token | |\n| modules.github_workflows.api_key | str | Github token | |\n| modules.github_workflows.num_logs | int | For each workflow fetch the last N successful runs logs (max 100) | 1 |\n| modules.github_workflows.output_folder | str | Folder to download workflow logs and artifacts to | |\n| modules.hunterio.api_key | str | Hunter.IO API key | |\n| modules.ip2location.api_key | str | IP2location.io API Key | |\n| modules.ip2location.lang | str | Translation information(ISO639-1). The translation is only applicable for continent, country, region and city name. | |\n| modules.ipneighbor.num_bits | int | Netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts) | 4 |\n| modules.ipstack.api_key | str | IPStack GeoIP API Key | |\n| modules.jadx.threads | int | Maximum jadx threads for extracting apk's, default: 4 | 4 |\n| modules.leakix.api_key | str | LeakIX API Key | |\n| modules.otx.api_key | str | OTX API key | |\n| modules.passivetotal.api_key | str | PassiveTotal API Key in the format of 'username:api_key' | |\n| modules.pgp.search_urls | list | PGP key servers to search |` ['https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=', 'http://the.earth.li:11371/pks/lookup?fingerprint=on&op=vindex&search=', 'https://pgpkeys.eu/pks/lookup?search=&op=index', 'https://pgp.mit.edu/pks/lookup?search=&op=index'] `|\n| modules.portfilter.allowed_cdn_ports | str | Comma-separated list of ports that are allowed to be scanned for CDNs | 80,443 |\n| modules.portfilter.cdn_tags | str | Comma-separated list of tags to skip, e.g. 'cdn,cloud' | cdn- |\n| modules.postman.api_key | str | Postman API Key | |\n| modules.postman_download.api_key | str | Postman API Key | |\n| modules.postman_download.output_folder | str | Folder to download postman workspaces to. If not specified, downloaded workspaces will be deleted when the scan completes, to minimize disk usage. | |\n| modules.securitytrails.api_key | str | SecurityTrails API key | |\n| modules.shodan_dns.api_key | str | Shodan API key | |\n| modules.shodan_idb.retries | NoneType | How many times to retry API requests (e.g. after a 429 error). Overrides the global web.api_retries setting. | None |\n| modules.subdomainradar.api_key | str | SubDomainRadar.io API key | |\n| modules.subdomainradar.group | str | The enumeration group to use. Choose from fast, medium, deep | fast |\n| modules.subdomainradar.timeout | int | Timeout in seconds | 120 |\n| modules.trickest.api_key | str | Trickest API key | |\n| modules.trufflehog.concurrency | int | Number of concurrent workers | 8 |\n| modules.trufflehog.config | str | File path or URL to YAML trufflehog config | |\n| modules.trufflehog.deleted_forks | bool | Scan for deleted github forks. WARNING: This is SLOW. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours. | False |\n| modules.trufflehog.only_verified | bool | Only report credentials that have been verified | True |\n| modules.trufflehog.version | str | trufflehog version | 3.90.8 |\n| modules.urlscan.urls | bool | Emit URLs in addition to DNS_NAMEs | False |\n| modules.virustotal.api_key | str | VirusTotal API Key | |\n| modules.wayback.garbage_threshold | int | Dedupe similar urls if they are in a group of this size or higher (lower values == less garbage data) | 10 |\n| modules.wayback.urls | bool | emit URLs in addition to DNS_NAMEs | False |\n| modules.asset_inventory.output_file | str | Set a custom output file | |\n| modules.asset_inventory.recheck | bool | When use_previous=True, don't retain past details like open ports or findings. Instead, allow them to be rediscovered by the new scan | False |\n| modules.asset_inventory.summary_netmask | int | Subnet mask to use when summarizing IP addresses at end of scan | 16 |\n| modules.asset_inventory.use_previous | bool |` Emit previous asset inventory as new events (use in conjunction with -n ) `| False |\n| modules.csv.output_file | str | Output to CSV file | |\n| modules.discord.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] |\n| modules.discord.min_severity | str | Only allow VULNERABILITY events of this severity or higher | LOW |\n| modules.discord.retries | int | Number of times to retry sending the message before skipping the event | 10 |\n| modules.discord.webhook_url | str | Discord webhook URL | |\n| modules.emails.output_file | str | Output to file | |\n| modules.http.bearer | str | Authorization Bearer token | |\n| modules.http.method | str | HTTP method | POST |\n| modules.http.password | str | Password (basic auth) | |\n| modules.http.siem_friendly | bool | Format JSON in a SIEM-friendly way for ingestion into Elastic, Splunk, etc. | False |\n| modules.http.timeout | int | HTTP timeout | 10 |\n| modules.http.url | str | Web URL | |\n| modules.http.username | str | Username (basic auth) | |\n| modules.json.output_file | str | Output to file | |\n| modules.json.siem_friendly | bool | Output JSON in a SIEM-friendly format for ingestion into Elastic, Splunk, etc. | False |\n| modules.mysql.database | str | The database name to connect to | bbot |\n| modules.mysql.host | str | The server running MySQL | localhost |\n| modules.mysql.password | str | The password to connect to MySQL | bbotislife |\n| modules.mysql.port | int | The port to connect to MySQL | 3306 |\n| modules.mysql.username | str | The username to connect to MySQL | root |\n| modules.neo4j.password | str | Neo4j password | bbotislife |\n| modules.neo4j.uri | str | Neo4j server + port | bolt://localhost:7687 |\n| modules.neo4j.username | str | Neo4j username | neo4j |\n| modules.postgres.database | str | The database name to connect to | bbot |\n| modules.postgres.host | str | The server running Postgres | localhost |\n| modules.postgres.password | str | The password to connect to Postgres | bbotislife |\n| modules.postgres.port | int | The port to connect to Postgres | 5432 |\n| modules.postgres.username | str | The username to connect to Postgres | postgres |\n| modules.slack.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] |\n| modules.slack.min_severity | str | Only allow VULNERABILITY events of this severity or higher | LOW |\n| modules.slack.retries | int | Number of times to retry sending the message before skipping the event | 10 |\n| modules.slack.webhook_url | str | Discord webhook URL | |\n| modules.splunk.hectoken | str | HEC Token | |\n| modules.splunk.index | str | Index to send data to | |\n| modules.splunk.source | str | Source path to be added to the metadata | |\n| modules.splunk.timeout | int | HTTP timeout | 10 |\n| modules.splunk.url | str | Web URL | |\n| modules.sqlite.database | str | The path to the sqlite database file | |\n| modules.stdout.accept_dupes | bool | Whether to show duplicate events, default True | True |\n| modules.stdout.event_fields | list | Which event fields to display | [] |\n| modules.stdout.event_types | list | Which events to display, default all event types | [] |\n| modules.stdout.format | str | Which text format to display, choices: text,json | text |\n| modules.stdout.in_scope_only | bool | Whether to only show in-scope events | False |\n| modules.subdomains.include_unresolved | bool | Include unresolved subdomains in output | False |\n| modules.subdomains.output_file | str | Output to file | |\n| modules.teams.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] |\n| modules.teams.min_severity | str | Only allow VULNERABILITY events of this severity or higher | LOW |\n| modules.teams.retries | int | Number of times to retry sending the message before skipping the event | 10 |\n| modules.teams.webhook_url | str | Teams webhook URL | |\n| modules.txt.output_file | str | Output to file | |\n| modules.web_parameters.include_count | bool | Include the count of each parameter in the output | False |\n| modules.web_parameters.output_file | str | Output to file | |\n| modules.web_report.css_theme_file | str | CSS theme URL for HTML output | https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css |\n| modules.web_report.output_file | str | Output to file | |\n| modules.websocket.ignore_ssl | bool | Ignores all Websocket SSL related errors (like Self-Signed Certificates, etc.) | False |\n| modules.websocket.preserve_graph | bool | Preserve full chains of events in the graph (prevents orphans) | True |\n| modules.websocket.token | str | Authorization Bearer token | |\n| modules.websocket.url | str | Web URL | |\n| modules.excavate.custom_yara_rules | str | Include custom Yara rules | |\n| modules.excavate.speculate_params | bool | Enable speculative parameter extraction from JSON and XML content | False |\n| modules.excavate.yara_max_match_data | int | Sets the maximum amount of text that can extracted from a YARA regex | 2000 |\n| modules.speculate.essential_only | bool | Only enable essential speculate features (no extra discovery) | False |\n| modules.speculate.max_hosts | int | Max number of IP_RANGE hosts to convert into IP_ADDRESS events | 65536 |\n| modules.speculate.ports | str | The set of ports to speculate on | 80,443 |\n\n" }, { "path": "docs/scanning/events.md", "content": "# Events\n\nAn Event is a piece of data discovered by BBOT. Examples include `IP_ADDRESS`, `DNS_NAME`, `EMAIL_ADDRESS`, `URL`, etc. When you run a BBOT scan, events are constantly being exchanged between modules. They are also output to the console:\n\n```text\n[DNS_NAME] www.evilcorp.com sslcert (distance-0, in-scope, resolved, subdomain, a-record)\n ^^^^^^^^ ^^^^^^^^^^^^^^^^ ^^^^^^^ ^^^^^^^^^^\nevent type event data source module tags\n```\n\n## Event Attributes\n\nEach BBOT event has the following attributes. Not all of these attributes are visible in the terminal output. However, they are always saved in `output.json` in the scan output folder. If you want to see them on the terminal, you can use `--json`.\n\n- `.type`: the event type (e.g. `DNS_NAME`, `IP_ADDRESS`, `OPEN_TCP_PORT`, etc.)\n- `.id`: an identifier representing the event type + a SHA1 hash of its data (note: multiple events can have the same `.id`)\n- `.uuid`: a universally unique identifier for the event (e.g. `DNS_NAME:6c96d512-090a-47f0-82e4-6860e46aac13`)\n- `.scope_description`: describes the scope of the event (e.g. `in-scope`, `affiliate`, `distance-2`)\n- `.data`: the actual discovered data (for some events like `DNS_NAME` or `IP_ADDRESS`, this is a string. For other more complex events like `HTTP_RESPONSE`, it's a dictionary)\n- `.host`: the hostname or IP address (e.g. `evilcorp.com` or `1.2.3.4`)\n- `.port`: the port number (e.g. `80`, `443`)\n- `.netloc`: the network location, including both the hostname and port (e.g. `www.evilcorp.com:443`)\n- `.resolved_hosts`: a list of all resolved hosts for the event (`A`, `AAAA`, and `CNAME` records)\n- `.dns_children`: a dictionary of all DNS records for the event (typically only present on `DNS_NAME`)\n- `.web_spider_distance`: a count of how many URL links have been followed in a row to get to this event\n- `.scope_distance`: a count of how many hops it is from the main scope (0 == in-scope)\n- `.scan`: the ID of the scan that produced the event\n- `.timestamp`: the date/time when the event was discovered\n- `.parent`: the ID of the parent event that led to the discovery of this event\n- `.parent_uuid`: the universally unique identifier for the parent event\n- `.tags`: a list of tags describing the event (e.g. `mx-record`, `http-title`, etc.)\n- `.module`: the module that discovered the event\n- `.module_sequence`: the recent sequence of modules that were executed to discover the event (including omitted events)\n- `.discovery_context`: a description of the context in which the event was discovered\n- `.discovery_path`: a list of every discovery context leading to this event\n- `.parent_chain`: a list of every event UUID leading to the discovery of this event (corresponds exactly to `.discovery_path`)\n\nThese attributes allow us to construct a visual graph of events (e.g. in [Neo4j](output.md#neo4j)) and query/filter/grep them more easily. Here is what a typical event looks like in JSON format:\n\n```json\n{\n \"type\": \"DNS_NAME\",\n \"id\": \"DNS_NAME:33bc005c2bdfea4d73e07db733bd11861cf6520e\",\n \"uuid\": \"DNS_NAME:6c96d512-090a-47f0-82e4-6860e46aac13\",\n \"scope_description\": \"in-scope\",\n \"data\": \"link.evilcorp.com\",\n \"host\": \"link.evilcorp.com\",\n \"resolved_hosts\": [\n \"184.31.52.65\",\n \"2600:1402:b800:d82::700\",\n \"2600:1402:b800:d87::700\",\n \"link.evilcorp.com.edgekey.net\"\n ],\n \"dns_children\": {\n \"A\": [\n \"184.31.52.65\"\n ],\n \"AAAA\": [\n \"2600:1402:b800:d82::700\",\n \"2600:1402:b800:d87::700\"\n ],\n \"CNAME\": [\n \"link.evilcorp.com.edgekey.net\"\n ]\n },\n \"web_spider_distance\": 0,\n \"scope_distance\": 0,\n \"scan\": \"SCAN:b6ef48bc036bc8d001595ae5061846a7e6beadb6\",\n \"timestamp\": \"2024-10-18T15:40:13.716880+00:00\",\n \"parent\": \"DNS_NAME:94c92b7eaed431b37ae2a757fec4e678cc3bd213\",\n \"parent_uuid\": \"DNS_NAME:c737dffa-d4f0-4b6e-a72d-cc8c05bd892e\",\n \"tags\": [\n \"subdomain\",\n \"a-record\",\n \"cdn-akamai\",\n \"in-scope\",\n \"cname-record\",\n \"aaaa-record\"\n ],\n \"module\": \"speculate\",\n \"module_sequence\": \"speculate->speculate\",\n \"discovery_context\": \"speculated parent DNS_NAME: link.evilcorp.com\",\n \"discovery_path\": [\n \"Scan insidious_frederick seeded with DNS_NAME: evilcorp.com\",\n \"TXT record for evilcorp.com contains IP_ADDRESS: 149.72.247.52\",\n \"PTR record for 149.72.247.52 contains DNS_NAME: o1.ptr2410.link.evilcorp.com\",\n \"speculated parent DNS_NAME: ptr2410.link.evilcorp.com\",\n \"speculated parent DNS_NAME: link.evilcorp.com\"\n ],\n \"parent_chain\": [\n \"DNS_NAME:34c657a3-0bfa-457e-9e6e-0f22f04b8da5\",\n \"IP_ADDRESS:efc0fb3b-1b42-44da-916e-83db2360e10e\",\n \"DNS_NAME:c737dffa-d4f0-4b6e-a72d-cc8c05bd892e\",\n \"DNS_NAME_UNRESOLVED:722a3473-30c6-40f1-90aa-908d47105d5a\",\n \"DNS_NAME:6c96d512-090a-47f0-82e4-6860e46aac13\"\n ]\n}\n```\n\nFor a more detailed description of BBOT events, see [Developer Documentation - Event](../dev/event.md).\n\nBelow is a full list of event types along with which modules produce/consume them.\n\n## List of Event Types\n\n\n| Event Type | # Consuming Modules | # Producing Modules | Consuming Modules | Producing Modules |\n|---------------------|-----------------------|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| * | 18 | 0 | affiliates, cloudcheck, csv, discord, dnsresolve, http, json, mysql, neo4j, postgres, python, slack, splunk, sqlite, stdout, teams, txt, websocket | |\n| ASN | 0 | 1 | | asn |\n| AZURE_TENANT | 1 | 0 | speculate | |\n| CODE_REPOSITORY | 7 | 8 | docker_pull, git_clone, gitdumper, github_workflows, google_playstore, postman_download, trufflehog | code_repository, dockerhub, git, github_codesearch, github_org, gitlab_com, gitlab_onprem, postman |\n| DNS_NAME | 60 | 43 | anubisdb, asset_inventory, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, bucket_amazon, bucket_digitalocean, bucket_firebase, bucket_google, bucket_microsoft, bufferoverrun, builtwith, c99, censys_dns, certspotter, chaos, credshed, crt, crt_db, dehashed, digitorus, dnsbimi, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dnstlsrpt, emailformat, fullhunt, github_codesearch, github_usersearch, hackertarget, hunterio, leakix, myssl, nmap_xml, oauth, otx, passivetotal, pgp, portscan, rapiddns, securitytrails, securitytxt, shodan_dns, shodan_idb, sitedossier, skymem, speculate, subdomaincenter, subdomainradar, subdomains, trickest, urlscan, viewdns, virustotal, wayback | anubisdb, azure_tenant, bevigil, bufferoverrun, builtwith, c99, censys_dns, censys_ip, certspotter, chaos, crt, crt_db, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dnsresolve, fullhunt, hackertarget, hunterio, leakix, myssl, ntlm, oauth, otx, passivetotal, rapiddns, securitytrails, shodan_dns, shodan_idb, sitedossier, speculate, sslcert, subdomaincenter, subdomainradar, trickest, urlscan, vhost, viewdns, virustotal, wayback |\n| DNS_NAME_UNRESOLVED | 3 | 0 | baddns, speculate, subdomains | |\n| EMAIL_ADDRESS | 1 | 11 | emails | credshed, dehashed, dnscaa, dnstlsrpt, emailformat, github_usersearch, hunterio, pgp, securitytxt, skymem, sslcert |\n| FILESYSTEM | 4 | 9 | extractous, jadx, trufflehog, unarchive | apkpure, docker_pull, filedownload, git_clone, gitdumper, github_workflows, jadx, postman_download, unarchive |\n| FINDING | 2 | 32 | asset_inventory, web_report | ajaxpro, baddns, baddns_direct, baddns_zone, badsecrets, bucket_amazon, bucket_digitalocean, bucket_firebase, bucket_google, bucket_microsoft, bypass403, git, gitlab_onprem, graphql_introspection, host_header, hunt, legba, lightfuzz, newsletters, ntlm, nuclei, paramminer_cookies, paramminer_getparams, reflected_parameters, retirejs, shodan_idb, smuggler, speculate, telerik, trufflehog, url_manipulation, wpscan |\n| GEOLOCATION | 0 | 2 | | ip2location, ipstack |\n| HASHED_PASSWORD | 0 | 2 | | credshed, dehashed |\n| HTTP_RESPONSE | 18 | 1 | ajaxpro, asset_inventory, badsecrets, dotnetnuke, excavate, filedownload, gitlab_onprem, host_header, newsletters, nmap_xml, ntlm, paramminer_cookies, paramminer_getparams, paramminer_headers, speculate, telerik, trufflehog, wpscan | httpx |\n| IP_ADDRESS | 10 | 5 | asn, asset_inventory, censys_ip, ip2location, ipneighbor, ipstack, nmap_xml, portscan, shodan_idb, speculate | asset_inventory, censys_ip, dnsresolve, ipneighbor, speculate |\n| IP_RANGE | 2 | 0 | portscan, speculate | |\n| MOBILE_APP | 1 | 1 | apkpure | google_playstore |\n| OPEN_TCP_PORT | 6 | 5 | asset_inventory, fingerprintx, httpx, nmap_xml, portfilter, sslcert | asset_inventory, censys_ip, portscan, shodan_idb, speculate |\n| OPEN_UDP_PORT | 0 | 1 | | censys_ip |\n| ORG_STUB | 4 | 1 | dockerhub, github_org, google_playstore, postman | speculate |\n| PASSWORD | 0 | 2 | | credshed, dehashed |\n| PROTOCOL | 3 | 2 | legba, medusa, nmap_xml | censys_ip, fingerprintx |\n| RAW_DNS_RECORD | 0 | 3 | | dnsbimi, dnsresolve, dnstlsrpt |\n| RAW_TEXT | 2 | 1 | excavate, trufflehog | extractous |\n| SOCIAL | 7 | 4 | dockerhub, github_org, gitlab_com, gitlab_onprem, gowitness, postman, speculate | dockerhub, github_usersearch, gitlab_onprem, social |\n| STORAGE_BUCKET | 8 | 5 | baddns_direct, bucket_amazon, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, bucket_microsoft, speculate | bucket_amazon, bucket_digitalocean, bucket_firebase, bucket_google, bucket_microsoft |\n| TECHNOLOGY | 4 | 8 | asset_inventory, gitlab_onprem, web_report, wpscan | badsecrets, censys_ip, dotnetnuke, gitlab_onprem, gowitness, nuclei, shodan_idb, wpscan |\n| URL | 24 | 2 | ajaxpro, aspnet_bin_exposure, asset_inventory, baddns_direct, bypass403, ffuf, generic_ssrf, git, gowitness, graphql_introspection, httpx, iis_shortnames, lightfuzz, ntlm, nuclei, portfilter, robots, smuggler, speculate, telerik, url_manipulation, vhost, wafw00f, web_report | gowitness, httpx |\n| URL_HINT | 1 | 1 | ffuf_shortnames | iis_shortnames |\n| URL_UNVERIFIED | 8 | 19 | code_repository, filedownload, httpx, oauth, portfilter, retirejs, social, speculate | azure_realm, bevigil, bucket_file_enum, censys_ip, dnsbimi, dnscaa, dnstlsrpt, dockerhub, excavate, ffuf, ffuf_shortnames, github_codesearch, gowitness, hunterio, robots, securitytxt, urlscan, wayback, wpscan |\n| USERNAME | 1 | 2 | speculate | credshed, dehashed |\n| VHOST | 1 | 1 | web_report | vhost |\n| VULNERABILITY | 2 | 15 | asset_inventory, web_report | ajaxpro, aspnet_bin_exposure, baddns, baddns_direct, baddns_zone, badsecrets, dotnetnuke, generic_ssrf, lightfuzz, medusa, nuclei, shodan_idb, telerik, trufflehog, wpscan |\n| WAF | 1 | 1 | asset_inventory | wafw00f |\n| WEBSCREENSHOT | 0 | 1 | | gowitness |\n| WEB_PARAMETER | 7 | 4 | hunt, lightfuzz, paramminer_cookies, paramminer_getparams, paramminer_headers, reflected_parameters, web_parameters | excavate, paramminer_cookies, paramminer_getparams, paramminer_headers |\n\n\n## Findings Vs. Vulnerabilities\n\nBBOT has a sharp distinction between Findings and Vulnerabilities:\n\n**VULNERABILITY**\n\n* There's a higher standard for what is allowed to be a vulnerability. They should be considered **confirmed** and **actionable** - no additional confirmation required\n* They are always assigned a severity. The possible severities are: LOW, MEDIUM, HIGH, or CRITICAL\n\n**FINDING**\n\n* Findings can range anywhere from \"slightly interesting behavior\" to \"likely, but unconfirmed vulnerability\"\n* Are often false positives\n\nBy making this separation, actionable vulnerabilities can be identified quickly in the midst of a large scan\n" }, { "path": "docs/scanning/index.md", "content": "# Scanning Overview\n\n## Scan Names\n\nEvery BBOT scan gets a random, mildly-entertaining name like **`demonic_jimmy`**. Output for that scan, including scan stats and any web screenshots, are saved to a folder by that name in `~/.bbot/scans`. The most recent 20 scans are kept, and older ones are removed.\n\nIf you don't want a random name, you can change it with `-n`. You can also change the location of BBOT's output with `-o`:\n\n```bash\n# save everything to the folder \"my_scan\" in the current directory\nbbot -t evilcorp.com -f subdomain-enum -m gowitness -n my_scan -o .\n```\n\nIf you reuse a scan name, BBOT will automatically append to your previous output files.\n\n## Targets (`-t`)\n\nTargets declare what's in-scope, and seed a scan with initial data. BBOT accepts an unlimited number of targets. They can be any of the following:\n\n- `DNS_NAME` (`evilcorp.com`)\n- `IP_ADDRESS` (`1.2.3.4`)\n- `IP_RANGE` (`1.2.3.0/24`)\n- `OPEN_TCP_PORT` (`192.168.0.1:80`)\n- `URL` (`https://www.evilcorp.com`)\n- `EMAIL_ADDRESS` (`bob@evilcorp.com`)\n- `ORG_STUB` (`ORG:evilcorp`)\n- `USER_STUB` (`USER:bobsmith`)\n- `FILESYSTEM` (`FILESYSTEM:/tmp/asdf`)\n- `MOBILE_APP` (`MOBILE_APP:https://play.google.com/store/apps/details?id=com.evilcorp.app`)\n\nNote that BBOT only discriminates down to the host level. This means, for example, if you specify a URL `https://www.evilcorp.com` as the target, the scan will be *seeded* with that URL, but the scope of the scan will be the entire host, `www.evilcorp.com`. Other ports/URLs on that same host may also be scanned.\n\nYou can specify targets directly on the command line, load them from files, or both! For example:\n\n```bash\n$ cat targets.txt\n4.3.2.1\n10.0.0.2:80\n1.2.3.0/24\nevilcorp.com\nevilcorp.co.uk\nhttps://www.evilcorp.co.uk\n\n# load targets from a file and from the command-line\n$ bbot -t targets.txt fsociety.com 5.6.7.0/24 -m portscan\n```\n\nOn start, BBOT automatically converts Targets into [Events](events.md).\n\n## Modules (`-m`)\n\nTo see a full list of modules and their descriptions, use `bbot -l` or see [List of Modules](../modules/list_of_modules.md).\n\nModules are the part of BBOT that does the work -- port scanning, subdomain brute-forcing, API querying, etc. Modules consume [Events](events.md) (`IP_ADDRESS`, `DNS_NAME`, etc.) from each other, process the data in a useful way, then emit the results as new events. You can enable individual modules with `-m`.\n\n```bash\n# Enable modules: portscan, sslcert, and httpx\nbbot -t www.evilcorp.com -m portscan sslcert httpx\n```\n\n### Types of Modules\n\nModules fall into three categories:\n\n- **Scan Modules**:\n - These make up the majority of modules. Examples are `portscan`, `sslcert`, `httpx`, etc. Enable with `-m`.\n- **Output Modules**:\n - These output scan data to different formats/destinations. `human`, `json`, and `csv` are enabled by default. Enable others with `-om`. (See: [Output](output.md))\n- **Internal Modules**:\n - These modules perform essential, common-sense tasks. They are always enabled, unless explicitly disabled via the config (e.g. `-c speculate=false`).\n - `aggregate`: Summarizes results at the end of a scan\n - `excavate`: Extracts useful data such as subdomains from webpages, etc.\n - `speculate`: Intelligently infers new events, e.g. `OPEN_TCP_PORT` from `URL` or `IP_ADDRESS` from `IP_NETWORK`.\n\nFor details in the inner workings of modules, see [How to Write a Module](../dev/module_howto.md).\n\n## Flags (`-f`)\n\nFlags are how BBOT categorizes its modules. In a way, you can think of them as groups. Flags let you enable a bunch of similar modules at the same time without having to specify them each individually. For example, `-f subdomain-enum` would enable every module with the `subdomain-enum` flag.\n\n```bash\n# list all subdomain-enum modules\nbbot -f subdomain-enum -l\n```\n\n### Filtering Modules\n\nModules can be easily enabled/disabled based on their flags:\n\n- `-f` Enable these flags (e.g. `-f subdomain-enum`)\n- `-rf` Require modules to have this flag (e.g. `-rf safe`)\n- `-ef` Exclude these flags (e.g. `-ef slow`)\n- `-em` Exclude these individual modules (e.g. `-em ipneighbor`)\n- `-lf` List all available flags\n\nEvery module is either `safe` or `aggressive`, and either `active` or `passive`. These can be useful for filtering. For example, if you wanted to enable all the `safe` modules, but exclude active ones, you could do:\n\n```bash\n# Enable safe modules but exclude active ones\nbbot -t evilcorp.com -f safe -ef active\n```\n\nThis is equivalent to requiring the passive flag:\n\n```bash\n# Enable safe modules but only if they're also passive\nbbot -t evilcorp.com -f safe -rf passive\n```\n\nA single module can have multiple flags. For example, the `securitytrails` module is `passive`, `safe`, `subdomain-enum`. Below is a full list of flags and their associated modules.\n\n### List of Flags\n\n\n| Flag | # Modules | Description | Modules |\n|------------------|-------------|----------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| safe | 98 | Non-intrusive, safe to run | affiliates, aggregate, ajaxpro, anubisdb, apkpure, asn, aspnet_bin_exposure, azure_realm, azure_tenant, baddns, baddns_direct, baddns_zone, badsecrets, bevigil, bucket_amazon, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, bucket_microsoft, bufferoverrun, builtwith, c99, censys_dns, censys_ip, certspotter, chaos, code_repository, credshed, crt, crt_db, dehashed, digitorus, dnsbimi, dnscaa, dnscommonsrv, dnsdumpster, dnstlsrpt, docker_pull, dockerhub, emailformat, extractous, filedownload, fingerprintx, fullhunt, git, git_clone, gitdumper, github_codesearch, github_org, github_usersearch, github_workflows, gitlab_com, gitlab_onprem, google_playstore, gowitness, graphql_introspection, hackertarget, httpx, hunt, hunterio, iis_shortnames, ip2location, ipstack, jadx, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, portfilter, portscan, postman, postman_download, rapiddns, reflected_parameters, retirejs, robots, securitytrails, securitytxt, shodan_dns, shodan_idb, sitedossier, skymem, social, sslcert, subdomaincenter, subdomainradar, trickest, trufflehog, unarchive, urlscan, viewdns, virustotal, wayback |\n| passive | 70 | Never connects to target systems | affiliates, aggregate, anubisdb, apkpure, asn, azure_realm, azure_tenant, bevigil, bucket_file_enum, bufferoverrun, builtwith, c99, censys_dns, censys_ip, certspotter, chaos, code_repository, credshed, crt, crt_db, dehashed, digitorus, dnsbimi, dnscaa, dnsdumpster, dnstlsrpt, docker_pull, dockerhub, emailformat, excavate, extractous, fullhunt, git_clone, gitdumper, github_codesearch, github_org, github_usersearch, github_workflows, google_playstore, hackertarget, hunterio, ip2location, ipneighbor, ipstack, jadx, leakix, myssl, otx, passivetotal, pgp, portfilter, postman, postman_download, rapiddns, securitytrails, shodan_dns, shodan_idb, sitedossier, skymem, social, speculate, subdomaincenter, subdomainradar, trickest, trufflehog, unarchive, urlscan, viewdns, virustotal, wayback |\n| active | 52 | Makes active connections to target systems | ajaxpro, aspnet_bin_exposure, baddns, baddns_direct, baddns_zone, badsecrets, bucket_amazon, bucket_digitalocean, bucket_firebase, bucket_google, bucket_microsoft, bypass403, dnsbrute, dnsbrute_mutations, dnscommonsrv, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab_com, gitlab_onprem, gowitness, graphql_introspection, host_header, httpx, hunt, iis_shortnames, legba, lightfuzz, medusa, newsletters, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, portscan, reflected_parameters, retirejs, robots, securitytxt, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wpscan |\n| subdomain-enum | 51 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, baddns_direct, baddns_zone, bevigil, bufferoverrun, builtwith, c99, censys_dns, certspotter, chaos, crt, crt_db, digitorus, dnsbimi, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dnstlsrpt, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, ipneighbor, leakix, myssl, oauth, otx, passivetotal, postman, postman_download, rapiddns, securitytrails, securitytxt, shodan_dns, shodan_idb, sitedossier, sslcert, subdomaincenter, subdomainradar, subdomains, trickest, urlscan, virustotal, wayback |\n| aggressive | 22 | Generates a large amount of network traffic | bypass403, dnsbrute, dnsbrute_mutations, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, legba, lightfuzz, medusa, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f, wpscan |\n| code-enum | 18 | Find public code repositories and search them for secrets etc. | apkpure, code_repository, docker_pull, dockerhub, git, git_clone, gitdumper, github_codesearch, github_org, github_usersearch, github_workflows, gitlab_com, gitlab_onprem, google_playstore, jadx, postman, postman_download, trufflehog |\n| web-basic | 17 | Basic, non-intrusive web scan functionality | azure_realm, baddns, badsecrets, bucket_amazon, bucket_firebase, bucket_google, bucket_microsoft, filedownload, git, graphql_introspection, httpx, iis_shortnames, ntlm, oauth, robots, securitytxt, sslcert |\n| cloud-enum | 16 | Enumerates cloud resources | azure_realm, azure_tenant, baddns, baddns_direct, baddns_zone, bucket_amazon, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, bucket_microsoft, dnsbimi, dnstlsrpt, httpx, oauth, securitytxt |\n| web-thorough | 15 | More advanced web scanning functionality | ajaxpro, aspnet_bin_exposure, bucket_digitalocean, bypass403, dotnetnuke, ffuf_shortnames, generic_ssrf, host_header, hunt, lightfuzz, reflected_parameters, retirejs, smuggler, telerik, url_manipulation |\n| slow | 11 | May take a long time to complete | bucket_digitalocean, dnsbrute_mutations, docker_pull, fingerprintx, git_clone, gitdumper, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost |\n| email-enum | 9 | Enumerates email addresses | dehashed, dnscaa, dnstlsrpt, emailformat, emails, hunterio, pgp, skymem, sslcert |\n| affiliates | 8 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, trickest, viewdns |\n| download | 7 | Modules that download files, apps, or repositories | apkpure, docker_pull, filedownload, git_clone, gitdumper, github_workflows, postman_download |\n| deadly | 6 | Highly aggressive | ffuf, legba, lightfuzz, medusa, nuclei, vhost |\n| baddns | 3 | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_direct, baddns_zone |\n| web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers |\n| iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames |\n| portscan | 2 | Discovers open ports | portscan, shodan_idb |\n| social-enum | 2 | Enumerates social media | httpx, social |\n| service-enum | 1 | Identifies protocols running on open ports | fingerprintx |\n| subdomain-hijack | 1 | Detects hijackable subdomains | baddns |\n| web-screenshots | 1 | Takes screenshots of web pages | gowitness |\n\n\n## Dependencies\n\nBBOT modules have external dependencies ranging from OS packages (`openssl`) to binaries (`nuclei`) to Python libraries (`wappalyzer`). When a module is enabled, installation of its dependencies happens at runtime with [Ansible](https://github.com/ansible/ansible). BBOT provides several command-line flags to control how dependencies are installed.\n\n- `--no-deps` - Don't install module dependencies\n- `--force-deps` - Force install all module dependencies\n- `--retry-deps` - Try again to install failed module dependencies\n- `--ignore-failed-deps` - Run modules even if they have failed dependencies\n- `--install-all-deps` - Install dependencies for all modules (useful if you are provisioning a pentest system and want to install everything ahead of time)\n\nFor details on how Ansible playbooks are attached to BBOT modules, see [How to Write a Module](../dev/module_howto.md#module-dependencies).\n\n## Scope\n\nFor pentesters and bug bounty hunters, staying in scope is extremely important. BBOT takes this seriously, meaning that active modules (e.g. `nuclei`) will only touch in-scope resources.\n\nBy default, scope is whatever you specify with `-t`. This includes child subdomains. For example, if you specify `-t evilcorp.com`, all its subdomains (`www.evilcorp.com`, `mail.evilcorp.com`, etc.) also become in-scope.\n\n### Scope Distance\n\nSince BBOT is recursive, it would quickly resort to scanning the entire internet without some kind of restraining mechanism. To solve this problem, every [event](events.md) discovered by BBOT is assigned a **Scope Distance**. Scope distance represents how far out from the main scope that data was discovered.\n\nFor example, if your target is `evilcorp.com`, `www.evilcorp.com` would have a scope distance of `0` (i.e. in-scope). If BBOT discovers that `www.evilcorp.com` resolves to `1.2.3.4`, `1.2.3.4` is one hop away, which means it would have a scope distance of `1`. If `1.2.3.4` has a PTR record that points to `ecorp.blob.core.windows.net`, `ecorp.blob.core.windows.net` is two hops away, so its scope distance is `2`.\n\nScope distance continues to increase the further out you get. Most modules (e.g. `nuclei` and `portscan`) only consume in-scope events. Certain other passive modules such as `asn` accept out to distance `1`. By default, DNS resolution happens out to a distance of `2`. Upon its discovery, any [event](events.md) that's determined to be in-scope (e.g. `www.evilcorp.com`) immediately becomes distance `0`, and the cycle starts over.\n\n#### Displaying Out-of-scope Events\n\nBy default, BBOT only displays in-scope events (with a few exceptions such as `STORAGE_BUCKET`s). If you want to see more, you must increase the [config](configuration.md) value of `scope.report_distance`:\n\n```bash\n# display out-of-scope events up to one hop away from the main scope\nbbot -t evilcorp.com -f subdomain-enum -c scope.report_distance=1\n```\n\n### Strict Scope\n\nIf you want to scan **_only_** that specific target hostname and none of its children, you can specify `--strict-scope`.\n\nNote that `--strict-scope` only applies to targets and whitelists, but not blacklists. This means that if you put `internal.evilcorp.com` in your blacklist, you can be sure none of its subdomains will be scanned, even when using `--strict-scope`.\n\n### Whitelists and Blacklists\n\nBBOT allows precise control over scope with whitelists and blacklists. These both use the same syntax as `--target`, meaning they accept the same event types, and you can specify an unlimited number of them, via a file, the CLI, or both.\n\n#### Whitelists\n\n`--whitelist` enables you to override what's in scope. For example, if you want to run nuclei against `evilcorp.com`, but stay only inside their corporate IP range of `1.2.3.0/24`, you can accomplish this like so:\n\n```bash\n# Seed scan with evilcorp.com, but restrict scope to 1.2.3.0/24\nbbot -t evilcorp.com --whitelist 1.2.3.0/24 -f subdomain-enum -m portscan nuclei --allow-deadly\n```\n\n#### Blacklists\n\n`--blacklist` takes ultimate precedence. Anything in the blacklist is completely excluded from the scan, even if it's in the whitelist.\n\n```bash\n# Scan evilcorp.com, but exclude internal.evilcorp.com and its children\nbbot -t evilcorp.com --blacklist internal.evilcorp.com -f subdomain-enum -m portscan nuclei --allow-deadly\n```\n\n#### Blacklist by Regex\n\nBlacklists also accept regex patterns. These regexes are are checked against the full URL, including the host and path.\n\nTo specify a regex, prefix the pattern with `RE:`. For example, to exclude all events containing \"signout\", you could do:\n\n```bash\nbbot -t evilcorp.com --blacklist \"RE:signout\"\n```\n\nNote that this would blacklist both of the following events:\n\n- `[URL] http://evilcorp.com/signout.aspx`\n- `[DNS_NAME] signout.evilcorp.com`\n\nIf you only want to blacklist the URL, you could narrow the regex like so:\n\n```bash\nbbot -t evilcorp.com --blacklist 'RE:signout\\.aspx$'\n```\n\nSimilar to targets and whitelists, blacklists can be specified in your preset. The `spider` preset makes use of this to prevent the spider from following logout links:\n\n```yaml title=\"spider.yml\"\ndescription: Recursive web spider\n\nmodules:\n - httpx\n\nblacklist:\n # Prevent spider from invalidating sessions by logging out\n - \"RE:/.*(sign|log)[_-]?out\"\n\nconfig:\n web:\n # how many links to follow in a row\n spider_distance: 2\n # don't follow links whose directory depth is higher than 4\n spider_depth: 4\n # maximum number of links to follow per page\n spider_links_per_page: 25\n```\n\n## DNS Wildcards\n\nBBOT has robust wildcard detection built-in. It can reliably detect wildcard domains, and will tag them accordingly:\n\n```text\n[DNS_NAME] github.io TARGET (a-record, a-wildcard-domain, aaaa-wildcard-domain, wildcard-domain)\n ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^\n```\n\nWildcard hosts are collapsed into a single host beginning with `_wildcard`:\n\n```text\n[DNS_NAME] _wildcard.github.io TARGET (a-record, a-wildcard, a-wildcard-domain, aaaa-record, aaaa-wildcard, aaaa-wildcard-domain, wildcard, wildcard-domain)\n ^^^^^^^^^\n```\n\nIf you don't want this, you can disable wildcard detection on a domain-to-domain basis in the [config](configuration.md):\n\n```yaml title=\"~/.bbot/config/bbot.yml\"\ndns:\n wildcard_ignore:\n - evilcorp.com\n - evilcorp.co.uk\n```\n\nThere are certain edge cases (such as with dynamic DNS rules) where BBOT's wildcard detection fails. In these cases, you can try increasing the number of wildcard checks in the config:\n\n```yaml title=\"~/.bbot/config/bbot.yml\"\n# default == 10\ndns:\n wildcard_tests: 20\n```\n\nIf that doesn't work you can consider [blacklisting](#whitelists-and-blacklists) the offending domain.\n" }, { "path": "docs/scanning/output.md", "content": "# Output\n\nBy default, BBOT saves its output in TXT, JSON, and CSV formats. The filenames are logged at the end of each scan:\n![bbot output](https://github.com/blacklanternsecurity/bbot/assets/20261699/bb3da441-2682-408f-b955-19b268823b82)\n\nEvery BBOT scan gets a unique and mildly-entertaining name like **`demonic_jimmy`**. Output for that scan, including scan stats and any web screenshots, etc., are saved to a folder by that name in `~/.bbot/scans`. The most recent 20 scans are kept, and older ones are removed. You can change the location of BBOT's output with `--output`, and you can also pick a custom scan name with `--name`.\n\nIf you reuse a scan name, it will append to its original output files and leverage the previous.\n\n## Output Modules\n\nMultiple simultaneous output formats are possible because of **output modules**. Output modules are similar to normal modules except they are enabled with `-om`.\n\n### STDOUT\n\nThe `stdout` output module is what you see when you execute BBOT in the terminal. By default it looks the same as the [`txt`](#txt) module, but it has options you can customize. You can filter by event type, choose the data format (`text`, `json`), and which fields you want to see:\n\n\n| Config Option | Type | Description | Default |\n|------------------------------|--------|--------------------------------------------------|-----------|\n| modules.stdout.accept_dupes | bool | Whether to show duplicate events, default True | True |\n| modules.stdout.event_fields | list | Which event fields to display | [] |\n| modules.stdout.event_types | list | Which events to display, default all event types | [] |\n| modules.stdout.format | str | Which text format to display, choices: text,json | text |\n| modules.stdout.in_scope_only | bool | Whether to only show in-scope events | False |\n\n\n### TXT\n\n`txt` output is tab-delimited, so it's easy to grep:\n\n```bash\n# grep out only the DNS_NAMEs\ncat ~/.bbot/scans/extreme_johnny/output.txt | grep '[DNS_NAME]' | cut -f2\nevilcorp.com\nwww.evilcorp.com\nmail.evilcorp.com\n```\n\n### CSV\n\nThe `csv` output module produces a CSV like this:\n\n| Event type | Event data | IP Address | Source Module | Scope Distance | Event Tags |\n| ---------- | ----------------------- | ---------- | ------------- | -------------- | -------------------------------------------------------------------------------------------------------- |\n| DNS_NAME | evilcorp.com | 1.2.3.4 | TARGET | 0 | a-record,cdn-github,distance-0,domain,in-scope,mx-record,ns-record,resolved,soa-record,target,txt-record |\n| DNS_NAME | www.evilcorp.com | 2.3.4.5 | certspotter | 0 | a-record,aaaa-record,cdn-github,cname-record,distance-0,in-scope,resolved,subdomain |\n| URL | http://www.evilcorp.com | 2.3.4.5 | httpx | 0 | a-record,aaaa-record,cdn-github,cname-record,distance-0,in-scope,resolved,subdomain |\n| DNS_NAME | admin.evilcorp.com | 5.6.7.8 | otx | 0 | a-record,aaaa-record,cloud-azure,cname-record,distance-0,in-scope,resolved,subdomain |\n\n### JSON\n\nIf you manually enable the `json` output module, it will go to stdout:\n\n```bash\nbbot -t evilcorp.com -om json | jq\n```\n\nYou will then see [events](events.md) like this:\n\n```json\n{\n \"type\": \"IP_ADDRESS\",\n \"id\": \"IP_ADDRESS:13cd09c2adf0860a582240229cd7ad1dccdb5eb1\",\n \"data\": \"1.2.3.4\",\n \"scope_distance\": 1,\n \"scan\": \"SCAN:64c0e076516ae7aa6502fd99489693d0d5ec26cc\",\n \"timestamp\": 1688518967.740472,\n \"resolved_hosts\": [\"1.2.3.4\"],\n \"parent\": \"DNS_NAME:2da045542abbf86723f22383d04eb453e573723c\",\n \"tags\": [\"distance-1\", \"ipv4\", \"internal\"],\n \"module\": \"A\",\n \"module_sequence\": \"A\"\n}\n```\n\nYou can filter on the JSON output with `jq`:\n\n```bash\n# pull out only the .data attribute of every DNS_NAME\n$ jq -r 'select(.type==\"DNS_NAME\") | .data' ~/.bbot/scans/extreme_johnny/output.json\nevilcorp.com\nwww.evilcorp.com\nmail.evilcorp.com\n```\n\n### Discord / Slack / Teams\n\n![bbot-discord](https://github.com/blacklanternsecurity/bbot/assets/20261699/6d88045c-8eac-43b6-8de9-c621ecf60c2d)\n\nBBOT supports output via webhooks to `discord`, `slack`, and `teams`. To use them, you must specify a webhook URL either in the config:\n\n```yaml title=\"discord_preset.yml\"\nconfig:\n modules:\n discord:\n webhook_url: https://discord.com/api/webhooks/1234/deadbeef\n```\n\n...or on the command line:\n```bash\nbbot -t evilcorp.com -om discord -c modules.discord.webhook_url=https://discord.com/api/webhooks/1234/deadbeef\n```\n\nBy default, only `VULNERABILITY` and `FINDING` events are sent, but this can be customized by setting `event_types` in the config like so:\n\n```yaml title=\"discord_preset.yml\"\nconfig:\n modules:\n discord:\n event_types:\n - VULNERABILITY\n - FINDING\n - STORAGE_BUCKET\n```\n\n...or on the command line:\n```bash\nbbot -t evilcorp.com -om discord -c modules.discord.event_types=[\"STORAGE_BUCKET\",\"FINDING\",\"VULNERABILITY\"]\n```\n\nYou can also filter on the severity of `VULNERABILITY` events by setting `min_severity`:\n\n\n```yaml title=\"discord_preset.yml\"\nconfig:\n modules:\n discord:\n min_severity: HIGH\n```\n\n### HTTP\n\nThe `http` output module sends [events](events.md) in JSON format to a desired HTTP endpoint.\n\n```bash\n# POST scan results to localhost\nbbot -t evilcorp.com -om http -c modules.http.url=http://localhost:8000\n```\n\nYou can customize the HTTP method if needed. Authentication is also supported:\n\n```yaml title=\"http_preset.yml\"\nconfig:\n modules:\n http:\n url: https://localhost:8000\n method: PUT\n # Authorization: Bearer\n bearer: \n # OR\n username: bob\n password: P@ssw0rd\n```\n\n### Elasticsearch\n\nWhen outputting to Elastic, use the `http` output module with the following settings (replace `` with your desired index, e.g. `bbot`):\n\n```bash\n# send scan results directly to elasticsearch\nbbot -t evilcorp.com -om http -c \\\n modules.http.url=http://localhost:8000//_doc \\\n modules.http.siem_friendly=true \\\n modules.http.username=elastic \\\n modules.http.password=changeme\n```\n\nAlternatively, via a preset:\n\n```yaml title=\"elastic_preset.yml\"\nconfig:\n modules:\n http:\n url: http://localhost:8000//_doc\n siem_friendly: true\n username: elastic\n password: changeme\n```\n\n### Splunk\n\nThe `splunk` output module sends [events](events.md) in JSON format to a desired splunk instance via [HEC](https://docs.splunk.com/Documentation/Splunk/9.2.0/Data/UsetheHTTPEventCollector).\n\nYou can customize this output with the following config options:\n\n```yaml title=\"splunk_preset.yml\"\nconfig:\n modules:\n splunk:\n # The full URL with the URI `/services/collector/event`\n url: https://localhost:8088/services/collector/event\n # Generated from splunk webui\n hectoken: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx\n # Defaults to `main` if not set\n index: my-specific-index\n # Defaults to `bbot` if not set\n source: /my/source.json\n```\n\n### Asset Inventory\n\nThe `asset_inventory` module produces a CSV like this:\n\n| Host | Provider | IP(s) | Status | Open Ports |\n| ------------------ | ----------- | ------- | ------ | ---------- |\n| evilcorp.com | cdn-github | 1.2.3.4 | Active | 80,443 |\n| www.evilcorp.com | cdn-github | 2.3.4.5 | Active | 22,80,443 |\n| admin.evilcorp.com | cloud-azure | 5.6.7.8 | N/A | |\n\n### SQLite\n\nThe `sqlite` output module produces a SQLite database containing all events, scans, and targets. By default, it will be saved in the scan directory as `output.sqlite`.\n\n```bash\n# specifying a custom database path\nbbot -t evilcorp.com -om sqlite -c modules.sqlite.database=/tmp/bbot.sqlite\n```\n\n### Postgres\n\nThe `postgres` output module allows you to ingest events, scans, and targets into a Postgres database. By default, it will connect to the server on `localhost` with a username of `postgres` and password of `bbotislife`. You can change this behavior in the config.\n\n```bash\n# specifying an alternate database\nbbot -t evilcorp.com -om postgres -c modules.postgres.database=custom_bbot_db\n```\n\n```yaml title=\"postgres_preset.yml\"\nconfig:\n modules:\n postgres:\n host: psq.fsociety.local\n database: custom_bbot_db\n port: 5432\n username: postgres\n password: bbotislife\n```\n\n### MySQL\n\nThe `mysql` output module allows you to ingest events, scans, and targets into a MySQL database. By default, it will connect to the server on `localhost` with a username of `root` and password of `bbotislife`. You can change this behavior in the config.\n\n```bash\n# specifying an alternate database\nbbot -t evilcorp.com -om mysql -c modules.mysql.database=custom_bbot_db\n```\n\n```yaml title=\"mysql_preset.yml\"\nconfig:\n modules:\n mysql:\n host: mysql.fsociety.local\n database: custom_bbot_db\n port: 3306\n username: root\n password: bbotislife\n```\n\n### Subdomains\n\nThe `subdomains` output module produces simple text file containing only in-scope and resolved subdomains:\n\n```text title=\"subdomains.txt\"\nevilcorp.com\nwww.evilcorp.com\nmail.evilcorp.com\nportal.evilcorp.com\n```\n\n## Neo4j\n\nNeo4j is the funnest (and prettiest) way to view and interact with BBOT data.\n\n![neo4j](https://github.com/blacklanternsecurity/bbot/assets/20261699/0192d548-5c60-42b6-9a1e-32ba7b921cdf)\n\n- You can get Neo4j up and running with a single docker command:\n\n```bash\n# start Neo4j in the background with docker\ndocker run -d -p 7687:7687 -p 7474:7474 -v \"$(pwd)/neo4j/:/data/\" -e NEO4J_AUTH=neo4j/bbotislife neo4j\n```\n\n- After that, run bbot with `-om neo4j`\n\n```bash\nbbot -f subdomain-enum -t evilcorp.com -om neo4j\n```\n\n- Log in at [http://localhost:7474](http://localhost:7474) with `neo4j` / `bbotislife`\n\n### Cypher Queries and Tips\n\nNeo4j uses the Cypher Query Language for its graph query language. Cypher uses common clauses to craft relational queries and present the desired data in multiple formats.\n\nCypher queries can be broken down into three required pieces; selection, filter, and presentation. The selection piece identifies what data that will be searched against - 90% of the time the \"MATCH\" clause will be enough but there are means to read from csv or json data files. In all of these examples the \"MATCH\" clause will be used. The filter piece helps to focus in on the required data and used the \"WHERE\" clause to accomplish this effort (most basic operators can be used). Finally, the presentation section identifies how the data should be presented back to the querier. While neo4j is a graph database, it can be used in a traditional table view.\n\nA simple query to grab every URL event with \".com\" in the BBOT data field would look like this:\n`MATCH (u:URL) WHERE u.data contains \".com\" RETURN u`\n\nIn this query the following can be identified:\n- Within the MATCH statement \"u\" is a variable and can be any value needed by the user while the \"URL\" label is a direct relationship to the BBOT event type.\n- The WHERE statement allows the query to filter on any of the BBOT event properties like data, tag, or even the label itself.\n- The RETURN statement is a general presentation of the whole URL event but this can be narrowed down to present any of the specific properties of the BBOT event (`RETURN u.data, u.tags`).\n\nThe following are a few recommended queries to get started with:\n\n```cypher\n// Get all \"in-scope\" DNS Nodes and return just data and tags properties\nMATCH (n:DNS_NAME)\nWHERE \"in-scope\" IN n.tags\nRETURN n.data, n.tags\n```\n\n```cypher\n// Get the count of labels/BBOT events in the Neo4j Database\nMATCH (n)\nRETURN labels(n), count(n)\n```\n\n```cypher\n// Get a graph of open ports associated with each domain\nMATCH z = ((n:DNS_NAME) --> (p:OPEN_TCP_PORT))\nRETURN z\n```\n\n```cypher\n// Get all domains and IP addresses with open TCP ports\nMATCH (n) --> (p:OPEN_TCP_PORT)\nWHERE \"in-scope\" in n.tags and (n:DNS_NAME or n:IP_ADDRESS)\nWITH *, TAIL(SPLIT(p.data, ':')) AS port\nRETURN n.data, collect(distinct port)\n```\n\n```cypher\n// Clear the database\nMATCH (n) DETACH DELETE n\n```\n\nThis is not an exhaustive list of clauses, filters, or other means to use cypher and should be considered a starting point. To build more advanced queries consider reading Neo4j's Cypher [documentation](https://neo4j.com/docs/cypher-manual/current/introduction/).\n\nAdditional note: these sample queries are dependent on the existence of the data in the target neo4j database.\n\n### Web_parameters\n\nThe `web_parameters` output module will utilize BBOT web parameter extraction capabilities, and output the resulting parameters to a file (web_parameters.txt, by default). Web parameter extraction is disabled by default, but will automatically be enabled when a module is included that consumes WEB_PARAMETER events (including the `web_parameters` output module itself).\n\nThis can be useful for those who want to discover new common web parameters or those which may be associated with a specific target or organization. This could be very useful for further parameter bruteforcing, or even fed back into bbot via the paramminer modules. For example:\n\n```bash\nbbot -t evilcorp.com -m paramminer_getparams -c modules.paramminer_getparams.wordlist=/path/to/your/new/wordlist.txt\n``` " }, { "path": "docs/scanning/presets.md", "content": "# Presets\n\nOnce you start customizing BBOT, your commands can start to get really long. Presets let you put all your scan settings in a single file:\n\n```bash\nbbot -p ./my_preset.yml\n```\n\nA Preset is a YAML file that can include scan targets, modules, and config options like API keys.\n\nA typical preset looks like this:\n\n\n```yaml title=\"subdomain-enum.yml\"\ndescription: Enumerate subdomains via APIs, brute-force\n\nflags:\n - subdomain-enum\n\noutput_modules:\n - subdomains\n\n```\n\n\n## How to use Presets (`-p`)\n\nBBOT has a ready-made collection of presets for common tasks like subdomain enumeration and web spidering. They live in `~/.bbot/presets`.\n\nTo list them, you can do:\n\n```bash\n# list available presets\nbbot -lp\n```\n\nEnable them with `-p`:\n\n```bash\n# do a subdomain enumeration\nbbot -t evilcorp.com -p subdomain-enum\n\n# multiple presets - subdomain enumeration + web spider\nbbot -t evilcorp.com -p subdomain-enum spider\n\n# start with a preset but only enable modules that have the 'passive' flag\nbbot -t evilcorp.com -p subdomain-enum -rf passive\n\n# preset + manual config override\nbbot -t www.evilcorp.com -p spider -c web.spider_distance=10\n```\n\nYou can build on the default presets, or create your own. Here's an example of a custom preset that builds on `subdomain-enum`:\n\n```yaml title=\"my_subdomains.yml\"\ndescription: Do a subdomain enumeration + basic web scan + nuclei\n\ntarget:\n - evilcorp.com\n\ninclude:\n # include these default presets\n - subdomain-enum\n - web-basic\n\nmodules:\n # enable nuclei in addition to the other modules\n - nuclei\n\nconfig:\n # global config options\n web:\n http_proxy: http://127.0.0.1:8080\n # module config options\n modules:\n # api keys\n securitytrails:\n api_key: 21a270d5f59c9b05813a72bb41707266\n virustotal:\n # multiple API keys are allowed\n api_key:\n - 4f41243847da693a4f356c0486114bc6\n - 5bc6ed268ab6488270e496d3183a1a27\n```\n\nTo execute your custom preset, you do:\n\n```bash\nbbot -p ./my_subdomains.yml\n```\n\n## Preset Load Order\n\nWhen you enable multiple presets, the order matters. In the case of a conflict, the last preset will always win. This means, for example, if you have a custom preset called `my_spider` that sets `web.spider_distance` to 1:\n\n```yaml title=\"my_spider.yml\"\nconfig:\n web:\n spider_distance: 1\n```\n\n...and you enable it alongside the default `spider` preset in this order:\n\n```bash\nbbot -t evilcorp.com -p ./my_spider.yml spider\n```\n\n...the value of `web.spider_distance` will be overridden by `spider`. To ensure this doesn't happen, you would want to switch the order of the presets:\n\n```bash\nbbot -t evilcorp.com -p spider ./my_spider.yml\n```\n\n## Validating Presets\n\nTo make sure BBOT is configured the way you expect, you can always check the `--current-preset` to show the final version of the config that will be used when BBOT executes:\n\n```bash\n# verify the preset is what you want\nbbot -p ./mypreset.yml --current-preset\n```\n\n## Advanced Usage\n\nBBOT Presets support advanced features like environment variable substitution and custom conditions.\n\n### Custom Modules\n\nIf you want to use a custom BBOT `.py` module, you can either move it into `bbot/modules` where BBOT is installed, or add its parent folder to `module_dirs` like so:\n\n```yaml title=\"custom_modules.yml\"\n# load extra BBOT modules from this locaation\nmodule_dirs:\n - /home/user/custom_modules\n```\n\n### Environment Variables\n\nYou can insert environment variables into your preset like this: `${env:}`:\n\n```yaml title=\"my_nuclei.yml\"\ndescription: Do a nuclei scan\n\ntarget:\n - evilcorp.com\n\nmodules:\n - nuclei\n\nconfig:\n modules:\n nuclei:\n # allow the nuclei templates to be specified at runtime via an environment variable\n tags: ${env:NUCLEI_TAGS}\n```\n\n```bash\nNUCLEI_TAGS=apache,nginx bbot -p ./my_nuclei.yml\n```\n\n### Conditions\n\nSometimes, you might need to add custom logic to a preset. BBOT supports this via `conditions`. The `conditions` attribute allows you to specify a list of custom conditions that will be evaluated before the scan starts. This is useful for performing last-minute sanity checks, or changing the behavior of the scan based on custom criteria.\n\n```yaml title=\"my_preset.yml\"\ndescription: Abort if nuclei templates aren't specified\n\nmodules:\n - nuclei\n\nconditions:\n - |\n {% if not config.modules.nuclei.templates %}\n {{ abort(\"Don't forget to set your templates!\") }}\n {% endif %}\n```\n\n```yaml title=\"my_preset.yml\"\ndescription: Enable ffuf but only when the web spider isn't also enabled\n\nmodules:\n - ffuf\n\nconditions:\n - |\n {% if config.web.spider_distance > 0 and config.web.spider_depth > 0 %}\n {{ warn(\"Disabling ffuf because the web spider is enabled\") }}\n {{ preset.exclude_module(\"ffuf\") }}\n {% endif %}\n```\n\nConditions use [Jinja](https://palletsprojects.com/p/jinja/), which means they can contain Python code. They run inside a sandboxed environment which has access to the following variables:\n\n- `preset` - the current preset object\n- `config` - the current config (an alias for `preset.config`)\n- `warn(message)` - display a custom warning message to the user\n- `abort(message)` - abort the scan with an optional message\n\nIf you aren't able to accomplish what you want with conditions, or if you need access to a new variable/function, please let us know on [Github](https://github.com/blacklanternsecurity/bbot/issues/new/choose).\n" }, { "path": "docs/scanning/presets_list.md", "content": "Below is a list of every default BBOT preset, including its YAML.\n\n\n## **baddns-intense**\n\nRun all baddns modules and submodules.\n\n??? note \"`baddns-intense.yml`\"\n ```yaml title=\"~/.bbot/presets/baddns-intense.yml\"\n description: Run all baddns modules and submodules.\n \n \n modules:\n - baddns\n - baddns_zone\n - baddns_direct\n \n config:\n modules:\n baddns:\n enabled_submodules: [CNAME,references,MX,NS,TXT]\n ```\n\n\n\nModules: [4](\"`baddns_direct`, `baddns_zone`, `baddns`, `httpx`\")\n\n## **cloud-enum**\n\nEnumerate cloud resources such as storage buckets, etc.\n\n??? note \"`cloud-enum.yml`\"\n ```yaml title=\"~/.bbot/presets/cloud-enum.yml\"\n description: Enumerate cloud resources such as storage buckets, etc.\n \n include:\n - subdomain-enum\n \n flags:\n - cloud-enum\n ```\n\n\n\nModules: [58](\"`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_direct`, `baddns_zone`, `baddns`, `bevigil`, `bucket_amazon`, `bucket_digitalocean`, `bucket_file_enum`, `bucket_firebase`, `bucket_google`, `bucket_microsoft`, `bufferoverrun`, `builtwith`, `c99`, `censys_dns`, `certspotter`, `chaos`, `crt_db`, `crt`, `digitorus`, `dnsbimi`, `dnsbrute_mutations`, `dnsbrute`, `dnscaa`, `dnscommonsrv`, `dnsdumpster`, `dnstlsrpt`, `fullhunt`, `github_codesearch`, `github_org`, `hackertarget`, `httpx`, `hunterio`, `ipneighbor`, `leakix`, `myssl`, `oauth`, `otx`, `passivetotal`, `postman_download`, `postman`, `rapiddns`, `securitytrails`, `securitytxt`, `shodan_dns`, `shodan_idb`, `sitedossier`, `social`, `sslcert`, `subdomaincenter`, `subdomainradar`, `trickest`, `urlscan`, `virustotal`, `wayback`\")\n\n## **code-enum**\n\nEnumerate Git repositories, Docker images, etc.\n\n??? note \"`code-enum.yml`\"\n ```yaml title=\"~/.bbot/presets/code-enum.yml\"\n description: Enumerate Git repositories, Docker images, etc.\n \n flags:\n - code-enum\n ```\n\n\n\nModules: [20](\"`apkpure`, `code_repository`, `docker_pull`, `dockerhub`, `git_clone`, `git`, `gitdumper`, `github_codesearch`, `github_org`, `github_usersearch`, `github_workflows`, `gitlab_com`, `gitlab_onprem`, `google_playstore`, `httpx`, `jadx`, `postman_download`, `postman`, `social`, `trufflehog`\")\n\n## **dirbust-heavy**\n\nRecursive web directory brute-force (aggressive)\n\n??? note \"`dirbust-heavy.yml`\"\n ```yaml title=\"~/.bbot/presets/web/dirbust-heavy.yml\"\n description: Recursive web directory brute-force (aggressive)\n \n include:\n - spider\n \n flags:\n - iis-shortnames\n \n modules:\n - ffuf\n - wayback\n \n config:\n modules:\n iis_shortnames:\n # we exploit the shortnames vulnerability to produce URL_HINTs which are consumed by ffuf_shortnames\n detect_only: False\n ffuf:\n depth: 3\n lines: 5000\n extensions:\n - php\n - asp\n - aspx\n - ashx\n - asmx\n - jsp\n - jspx\n - cfm\n - zip\n - conf\n - config\n - xml\n - json\n - yml\n - yaml\n # emit URLs from wayback\n wayback:\n urls: True\n ```\n\nCategory: web\n\nModules: [5](\"`ffuf_shortnames`, `ffuf`, `httpx`, `iis_shortnames`, `wayback`\")\n\n## **dirbust-light**\n\nBasic web directory brute-force (surface-level directories only)\n\n??? note \"`dirbust-light.yml`\"\n ```yaml title=\"~/.bbot/presets/web/dirbust-light.yml\"\n description: Basic web directory brute-force (surface-level directories only)\n \n include:\n - iis-shortnames\n \n modules:\n - ffuf\n \n config:\n modules:\n ffuf:\n # wordlist size = 1000\n lines: 1000\n ```\n\nCategory: web\n\nModules: [4](\"`ffuf_shortnames`, `ffuf`, `httpx`, `iis_shortnames`\")\n\n## **dotnet-audit**\n\nComprehensive scan for all IIS/.NET specific modules and module settings\n\n??? note \"`dotnet-audit.yml`\"\n ```yaml title=\"~/.bbot/presets/web/dotnet-audit.yml\"\n description: Comprehensive scan for all IIS/.NET specific modules and module settings\n \n \n include:\n - iis-shortnames\n \n modules:\n - httpx\n - badsecrets\n - ffuf_shortnames\n - ffuf\n - telerik\n - ajaxpro\n - dotnetnuke\n - aspnet_bin_exposure\n \n config:\n modules:\n ffuf:\n extensions: asp,aspx,ashx,asmx,ascx\n extensions_ignore_case: True\n ffuf_shortnames:\n find_subwords: True\n telerik:\n exploit_RAU_crypto: True\n include_subdirs: True # Run against every directory, not the default first received URL per-host\n ```\n\nCategory: web\n\nModules: [9](\"`ajaxpro`, `aspnet_bin_exposure`, `badsecrets`, `dotnetnuke`, `ffuf_shortnames`, `ffuf`, `httpx`, `iis_shortnames`, `telerik`\")\n\n## **email-enum**\n\nEnumerate email addresses from APIs, web crawling, etc.\n\n??? note \"`email-enum.yml`\"\n ```yaml title=\"~/.bbot/presets/email-enum.yml\"\n description: Enumerate email addresses from APIs, web crawling, etc.\n \n flags:\n - email-enum\n \n output_modules:\n - emails\n ```\n\n\n\nModules: [8](\"`dehashed`, `dnscaa`, `dnstlsrpt`, `emailformat`, `hunterio`, `pgp`, `skymem`, `sslcert`\")\n\n## **fast**\n\nScan only the provided targets as fast as possible - no extra discovery\n\n??? note \"`fast.yml`\"\n ```yaml title=\"~/.bbot/presets/fast.yml\"\n description: Scan only the provided targets as fast as possible - no extra discovery\n \n exclude_modules:\n - excavate\n \n config:\n # only scan the exact targets specified\n scope:\n strict: true\n # speed up dns resolution by doing A/AAAA only - not MX/NS/SRV/etc\n dns:\n minimal: true\n # essential speculation only\n modules:\n speculate:\n essential_only: true\n ```\n\n\n\nModules: [0](\"\")\n\n## **iis-shortnames**\n\nRecursively enumerate IIS shortnames\n\n??? note \"`iis-shortnames.yml`\"\n ```yaml title=\"~/.bbot/presets/web/iis-shortnames.yml\"\n description: Recursively enumerate IIS shortnames\n \n flags:\n - iis-shortnames\n \n config:\n modules:\n iis_shortnames:\n # exploit the vulnerability\n detect_only: false\n ```\n\nCategory: web\n\nModules: [3](\"`ffuf_shortnames`, `httpx`, `iis_shortnames`\")\n\n## **kitchen-sink**\n\nEverything everywhere all at once\n\n??? note \"`kitchen-sink.yml`\"\n ```yaml title=\"~/.bbot/presets/kitchen-sink.yml\"\n description: Everything everywhere all at once\n \n include:\n - subdomain-enum\n - cloud-enum\n - code-enum\n - email-enum\n - spider\n - web-basic\n - paramminer\n - dirbust-light\n - web-screenshots\n - baddns-intense\n \n config:\n modules:\n baddns:\n enable_references: True\n ```\n\n\n\nModules: [90](\"`anubisdb`, `apkpure`, `asn`, `azure_realm`, `azure_tenant`, `baddns_direct`, `baddns_zone`, `baddns`, `badsecrets`, `bevigil`, `bucket_amazon`, `bucket_digitalocean`, `bucket_file_enum`, `bucket_firebase`, `bucket_google`, `bucket_microsoft`, `bufferoverrun`, `builtwith`, `c99`, `censys_dns`, `certspotter`, `chaos`, `code_repository`, `crt_db`, `crt`, `dehashed`, `digitorus`, `dnsbimi`, `dnsbrute_mutations`, `dnsbrute`, `dnscaa`, `dnscommonsrv`, `dnsdumpster`, `dnstlsrpt`, `docker_pull`, `dockerhub`, `emailformat`, `ffuf_shortnames`, `ffuf`, `filedownload`, `fullhunt`, `git_clone`, `git`, `gitdumper`, `github_codesearch`, `github_org`, `github_usersearch`, `github_workflows`, `gitlab_com`, `gitlab_onprem`, `google_playstore`, `gowitness`, `graphql_introspection`, `hackertarget`, `httpx`, `hunt`, `hunterio`, `iis_shortnames`, `ipneighbor`, `jadx`, `leakix`, `myssl`, `ntlm`, `oauth`, `otx`, `paramminer_cookies`, `paramminer_getparams`, `paramminer_headers`, `passivetotal`, `pgp`, `postman_download`, `postman`, `rapiddns`, `reflected_parameters`, `robots`, `securitytrails`, `securitytxt`, `shodan_dns`, `shodan_idb`, `sitedossier`, `skymem`, `social`, `sslcert`, `subdomaincenter`, `subdomainradar`, `trickest`, `trufflehog`, `urlscan`, `virustotal`, `wayback`\")\n\n## **lightfuzz-heavy**\n\nDiscover web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques, including POST parameters, which are more invasive. Uses all lightfuzz modules, and adds paramminer modules for parameter discovery. Avoids running against confirmed WAFs.\n\n??? note \"`lightfuzz-heavy.yml`\"\n ```yaml title=\"~/.bbot/presets/web/lightfuzz-heavy.yml\"\n description: Discover web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques, including POST parameters, which are more invasive. Uses all lightfuzz modules, and adds paramminer modules for parameter discovery. Avoids running against confirmed WAFs.\n \n include:\n - lightfuzz-medium\n \n flags:\n - web-paramminer\n \n modules:\n - robots\n \n config:\n modules:\n lightfuzz:\n enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss,esi]\n disable_post: False\n try_post_as_get: True\n try_get_as_post: True\n ```\n\nCategory: web\n\nModules: [10](\"`badsecrets`, `httpx`, `hunt`, `lightfuzz`, `paramminer_cookies`, `paramminer_getparams`, `paramminer_headers`, `portfilter`, `reflected_parameters`, `robots`\")\n\n## **lightfuzz-light**\n\nDiscover web parameters and lightly fuzz them for vulnerabilities, with only the most common vulnerabilities and minimal extra modules. Safest to run alongside larger scans.\n\n??? note \"`lightfuzz-light.yml`\"\n ```yaml title=\"~/.bbot/presets/web/lightfuzz-light.yml\"\n description: Discover web parameters and lightly fuzz them for vulnerabilities, with only the most common vulnerabilities and minimal extra modules. Safest to run alongside larger scans.\n \n modules:\n - httpx\n - lightfuzz\n - portfilter\n \n config:\n url_querystring_remove: False # don't strip off the querystring (BBOT normally does this; but lightfuzz needs it)\n url_querystring_collapse: True # in cases where the same parameter has multiple values, collapse them into a single parameter to save on fuzzing attempts\n modules:\n lightfuzz:\n enabled_submodules: [path,sqli,xss] # only look for the most common vulnerabilities\n disable_post: True # don't send POST requests (less aggressive)\n avoid_wafs: True\n \n conditions:\n - |\n {% if config.web.spider_distance == 0 %}\n {{ warn(\"Lightfuzz works much better with spider enabled! Consider adding 'spider' or 'spider-intense' preset.\") }}\n {% endif %}\n ```\n\nCategory: web\n\nModules: [3](\"`httpx`, `lightfuzz`, `portfilter`\")\n\n## **lightfuzz-medium**\n\nDiscover web parameters and lightly fuzz them for vulnerabilities. Uses all lightfuzz modules, without some of the more intense discovery techniques. Does not send POST requests. This is the default lightfuzz preset; if you're not sure which one to use, this is a good starting point. Avoids running against confirmed WAFs.\n\n??? note \"`lightfuzz-medium.yml`\"\n ```yaml title=\"~/.bbot/presets/web/lightfuzz-medium.yml\"\n description: Discover web parameters and lightly fuzz them for vulnerabilities. Uses all lightfuzz modules, without some of the more intense discovery techniques. Does not send POST requests. This is the default lightfuzz preset; if you're not sure which one to use, this is a good starting point. Avoids running against confirmed WAFs.\n \n include:\n - lightfuzz-light\n \n modules:\n - badsecrets\n - hunt\n - reflected_parameters\n \n config:\n modules:\n lightfuzz:\n enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss,esi]\n try_post_as_get: True\n ```\n\nCategory: web\n\nModules: [6](\"`badsecrets`, `httpx`, `hunt`, `lightfuzz`, `portfilter`, `reflected_parameters`\")\n\n## **lightfuzz-superheavy**\n\nDiscover web parameters and lightly fuzz them for vulnerabilities, with the most intense discovery techniques, including POST parameters, which are more invasive. Uses all lightfuzz modules, adds paramminer modules for parameter discovery, and tests each unique parameter-value instance individually.\n\n??? note \"`lightfuzz-superheavy.yml`\"\n ```yaml title=\"~/.bbot/presets/web/lightfuzz-superheavy.yml\"\n description: Discover web parameters and lightly fuzz them for vulnerabilities, with the most intense discovery techniques, including POST parameters, which are more invasive. Uses all lightfuzz modules, adds paramminer modules for parameter discovery, and tests each unique parameter-value instance individually.\n \n include:\n - lightfuzz-heavy\n \n config:\n url_querystring_collapse: False # in cases where the same parameter is observed multiple times, fuzz them individually instead of collapsing them into a single parameter\n modules:\n lightfuzz:\n force_common_headers: True # Fuzz common headers like X-Forwarded-For even if they're not observed on the target\n enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss,esi]\n avoid_wafs: False\n excavate:\n speculate_params: True # speculate potential parameters extracted from JSON/XML web responses\n ```\n\nCategory: web\n\nModules: [10](\"`badsecrets`, `httpx`, `hunt`, `lightfuzz`, `paramminer_cookies`, `paramminer_getparams`, `paramminer_headers`, `portfilter`, `reflected_parameters`, `robots`\")\n\n## **lightfuzz-xss**\n\nDiscover web parameters and lightly fuzz them, limited to just GET-based xss vulnerabilities. Avoids running against confirmed WAFs. This is an example of a custom lightfuzz preset, selectively enabling a single lightfuzz module.\n\n??? note \"`lightfuzz-xss.yml`\"\n ```yaml title=\"~/.bbot/presets/web/lightfuzz-xss.yml\"\n description: Discover web parameters and lightly fuzz them, limited to just GET-based xss vulnerabilities. Avoids running against confirmed WAFs. This is an example of a custom lightfuzz preset, selectively enabling a single lightfuzz module.\n \n modules:\n - httpx\n - lightfuzz\n - paramminer_getparams\n - reflected_parameters\n - portfilter\n \n config:\n url_querystring_remove: False\n url_querystring_collapse: False\n modules:\n lightfuzz:\n enabled_submodules: [xss]\n disable_post: True\n \n conditions:\n - |\n {% if config.web.spider_distance == 0 %}\n {{ warn(\"The lightfuzz-xss preset works much better with spider enabled! Consider adding 'spider' or 'spider-intense' preset.\") }}\n {% endif %}\n ```\n\nCategory: web\n\nModules: [5](\"`httpx`, `lightfuzz`, `paramminer_getparams`, `portfilter`, `reflected_parameters`\")\n\n## **nuclei**\n\nRun nuclei scans against all discovered targets\n\n??? note \"`nuclei.yml`\"\n ```yaml title=\"~/.bbot/presets/nuclei/nuclei.yml\"\n description: Run nuclei scans against all discovered targets\n \n modules:\n - httpx\n - nuclei\n - portfilter\n \n config:\n modules:\n nuclei:\n directory_only: True # Do not run nuclei on individual non-directory URLs\n \n \n conditions:\n - |\n {% if config.web.spider_distance != 0 %}\n {{ warn(\"Running nuclei with spider enabled is generally not recommended. Consider removing 'spider' preset.\") }}\n {% endif %}\n \n \n \n # Additional Examples:\n \n # Slowing Down Scan\n \n #config:\n # modules:\n # nuclei:\n # ratelimit: 10\n # concurrency: 5\n \n \n \n \n ```\n\nCategory: nuclei\n\nModules: [3](\"`httpx`, `nuclei`, `portfilter`\")\n\n## **nuclei-budget**\n\nRun nuclei scans against all discovered targets, using budget mode to look for low hanging fruit with greatly reduced number of requests\n\n??? note \"`nuclei-budget.yml`\"\n ```yaml title=\"~/.bbot/presets/nuclei/nuclei-budget.yml\"\n description: Run nuclei scans against all discovered targets, using budget mode to look for low hanging fruit with greatly reduced number of requests\n \n modules:\n - httpx\n - nuclei\n - portfilter\n \n config:\n modules:\n nuclei:\n mode: budget\n budget: 10\n directory_only: true # Do not run nuclei on individual non-directory URLs\n \n conditions:\n - |\n {% if config.web.spider_distance != 0 %}\n {{ warn(\"Running nuclei with spider enabled is generally not recommended. Consider removing 'spider' preset.\") }}\n {% endif %}\n ```\n\nCategory: nuclei\n\nModules: [3](\"`httpx`, `nuclei`, `portfilter`\")\n\n## **nuclei-intense**\n\nRun nuclei scans against all discovered targets, allowing for spidering, against ALL URLs, and with additional discovery modules.\n\n??? note \"`nuclei-intense.yml`\"\n ```yaml title=\"~/.bbot/presets/nuclei/nuclei-intense.yml\"\n description: Run nuclei scans against all discovered targets, allowing for spidering, against ALL URLs, and with additional discovery modules.\n \n modules:\n - httpx\n - nuclei\n - robots\n - urlscan\n - portfilter\n - wayback\n \n config:\n modules:\n nuclei:\n directory_only: False # Will run nuclei on ALL discovered URLs - Be careful!\n wayback:\n urls: true\n \n conditions:\n - |\n {% if config.web.spider_distance == 0 and config.modules.nuclei.directory_only == False %}\n {{ warn(\"The 'nuclei-intense' preset turns the 'directory_only' limitation off on the nuclei module. To make the best use of this, you may want to enable spidering with 'spider' or 'spider-intense' preset.\") }}\n {% endif %}\n \n \n # Example for also running a dirbust\n \n #include:\n # - dirbust-light\n ```\n\nCategory: nuclei\n\nModules: [6](\"`httpx`, `nuclei`, `portfilter`, `robots`, `urlscan`, `wayback`\")\n\n## **nuclei-technology**\n\nRun nuclei scans against all discovered targets, running templates which match discovered technologies\n\n??? note \"`nuclei-technology.yml`\"\n ```yaml title=\"~/.bbot/presets/nuclei/nuclei-technology.yml\"\n description: Run nuclei scans against all discovered targets, running templates which match discovered technologies\n \n modules:\n - httpx\n - nuclei\n - portfilter\n \n config:\n modules:\n nuclei:\n mode: technology\n directory_only: True # Do not run nuclei on individual non-directory URLs. This is less unsafe to disable with technology mode.\n \n conditions:\n - |\n {% if config.web.spider_distance != 0 %}\n {{ warn(\"Running nuclei with spider enabled is generally not recommended. Consider removing 'spider' preset.\") }}\n {% endif %}\n \n # Example for also running a dirbust\n \n #include:\n # - dirbust-light\n ```\n\nCategory: nuclei\n\nModules: [3](\"`httpx`, `nuclei`, `portfilter`\")\n\n## **paramminer**\n\nDiscover new web parameters via brute-force, and analyze them with additional modules\n\n??? note \"`paramminer.yml`\"\n ```yaml title=\"~/.bbot/presets/web/paramminer.yml\"\n description: Discover new web parameters via brute-force, and analyze them with additional modules\n \n flags:\n - web-paramminer\n \n modules:\n - httpx\n - reflected_parameters\n - hunt\n \n conditions:\n - |\n {% if config.web.spider_distance == 0 %}\n {{ warn(\"The paramminer preset works much better with spider enabled! Consider adding 'spider' or 'spider-intense' preset.\") }}\n {% endif %}\n ```\n\nCategory: web\n\nModules: [6](\"`httpx`, `hunt`, `paramminer_cookies`, `paramminer_getparams`, `paramminer_headers`, `reflected_parameters`\")\n\n## **spider**\n\nRecursive web spider\n\n??? note \"`spider.yml`\"\n ```yaml title=\"~/.bbot/presets/spider.yml\"\n description: Recursive web spider\n \n modules:\n - httpx\n \n blacklist:\n # Prevent spider from invalidating sessions by logging out\n - \"RE:/.*(sign|log)[_-]?out\"\n \n config:\n web:\n # how many links to follow in a row\n spider_distance: 2\n # don't follow links whose directory depth is higher than 4\n spider_depth: 4\n # maximum number of links to follow per page\n spider_links_per_page: 25\n ```\n\n\n\nModules: [1](\"`httpx`\")\n\n## **spider-intense**\n\nRecursive web spider with more aggressive settings\n\n??? note \"`spider-intense.yml`\"\n ```yaml title=\"~/.bbot/presets/spider-intense.yml\"\n description: Recursive web spider with more aggressive settings\n \n include:\n - spider\n \n config:\n web:\n # how many links to follow in a row\n spider_distance: 4\n # don't follow links whose directory depth is higher than 6\n spider_depth: 6\n # maximum number of links to follow per page\n spider_links_per_page: 50\n ```\n\n\n\nModules: [1](\"`httpx`\")\n\n## **subdomain-enum**\n\nEnumerate subdomains via APIs, brute-force\n\n??? note \"`subdomain-enum.yml`\"\n ```yaml title=\"~/.bbot/presets/subdomain-enum.yml\"\n description: Enumerate subdomains via APIs, brute-force\n \n flags:\n # enable every module with the subdomain-enum flag\n - subdomain-enum\n \n output_modules:\n # output unique subdomains to TXT file\n - subdomains\n \n config:\n dns:\n threads: 25\n brute_threads: 1000\n # put your API keys here\n # modules:\n # github:\n # api_key: \"\"\n # chaos:\n # api_key: \"\"\n # securitytrails:\n # api_key: \"\"\n ```\n\n\n\nModules: [51](\"`anubisdb`, `asn`, `azure_realm`, `azure_tenant`, `baddns_direct`, `baddns_zone`, `bevigil`, `bufferoverrun`, `builtwith`, `c99`, `censys_dns`, `certspotter`, `chaos`, `crt_db`, `crt`, `digitorus`, `dnsbimi`, `dnsbrute_mutations`, `dnsbrute`, `dnscaa`, `dnscommonsrv`, `dnsdumpster`, `dnstlsrpt`, `fullhunt`, `github_codesearch`, `github_org`, `hackertarget`, `httpx`, `hunterio`, `ipneighbor`, `leakix`, `myssl`, `oauth`, `otx`, `passivetotal`, `postman_download`, `postman`, `rapiddns`, `securitytrails`, `securitytxt`, `shodan_dns`, `shodan_idb`, `sitedossier`, `social`, `sslcert`, `subdomaincenter`, `subdomainradar`, `trickest`, `urlscan`, `virustotal`, `wayback`\")\n\n## **tech-detect**\n\nDetect technologies via Nuclei, and FingerprintX\n\n??? note \"`tech-detect.yml`\"\n ```yaml title=\"~/.bbot/presets/tech-detect.yml\"\n description: Detect technologies via Nuclei, and FingerprintX\n \n modules:\n - nuclei\n - fingerprintx\n \n config:\n modules:\n nuclei:\n tags: tech\n ```\n\n\n\nModules: [3](\"`fingerprintx`, `httpx`, `nuclei`\")\n\n## **web-basic**\n\nQuick web scan\n\n??? note \"`web-basic.yml`\"\n ```yaml title=\"~/.bbot/presets/web-basic.yml\"\n description: Quick web scan\n \n include:\n - iis-shortnames\n \n flags:\n - web-basic\n ```\n\n\n\nModules: [18](\"`azure_realm`, `baddns`, `badsecrets`, `bucket_amazon`, `bucket_firebase`, `bucket_google`, `bucket_microsoft`, `ffuf_shortnames`, `filedownload`, `git`, `graphql_introspection`, `httpx`, `iis_shortnames`, `ntlm`, `oauth`, `robots`, `securitytxt`, `sslcert`\")\n\n## **web-screenshots**\n\nTake screenshots of webpages\n\n??? note \"`web-screenshots.yml`\"\n ```yaml title=\"~/.bbot/presets/web-screenshots.yml\"\n description: Take screenshots of webpages\n \n flags:\n - web-screenshots\n \n config:\n modules:\n gowitness:\n resolution_x: 1440\n resolution_y: 900\n # folder to output web screenshots (default is inside ~/.bbot/scans/scan_name)\n output_path: \"\"\n # whether to take screenshots of social media pages\n social: True\n ```\n\n\n\nModules: [3](\"`gowitness`, `httpx`, `social`\")\n\n## **web-thorough**\n\nAggressive web scan\n\n??? note \"`web-thorough.yml`\"\n ```yaml title=\"~/.bbot/presets/web-thorough.yml\"\n description: Aggressive web scan\n \n include:\n # include the web-basic preset\n - web-basic\n \n flags:\n - web-thorough\n ```\n\n\n\nModules: [32](\"`ajaxpro`, `aspnet_bin_exposure`, `azure_realm`, `baddns`, `badsecrets`, `bucket_amazon`, `bucket_digitalocean`, `bucket_firebase`, `bucket_google`, `bucket_microsoft`, `bypass403`, `dotnetnuke`, `ffuf_shortnames`, `filedownload`, `generic_ssrf`, `git`, `graphql_introspection`, `host_header`, `httpx`, `hunt`, `iis_shortnames`, `lightfuzz`, `ntlm`, `oauth`, `reflected_parameters`, `retirejs`, `robots`, `securitytxt`, `smuggler`, `sslcert`, `telerik`, `url_manipulation`\")\n\n\n## Table of Default Presets\n\nHere is a the same data, but in a table:\n\n\n| Preset | Category | Description | # Modules | Modules |\n|----------------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|\n| baddns-intense | | Run all baddns modules and submodules. | 4 | baddns, baddns_direct, baddns_zone, httpx |\n| cloud-enum | | Enumerate cloud resources such as storage buckets, etc. | 58 | anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_direct, baddns_zone, bevigil, bucket_amazon, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, bucket_microsoft, bufferoverrun, builtwith, c99, censys_dns, certspotter, chaos, crt, crt_db, digitorus, dnsbimi, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dnstlsrpt, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, ipneighbor, leakix, myssl, oauth, otx, passivetotal, postman, postman_download, rapiddns, securitytrails, securitytxt, shodan_dns, shodan_idb, sitedossier, social, sslcert, subdomaincenter, subdomainradar, trickest, urlscan, virustotal, wayback |\n| code-enum | | Enumerate Git repositories, Docker images, etc. | 20 | apkpure, code_repository, docker_pull, dockerhub, git, git_clone, gitdumper, github_codesearch, github_org, github_usersearch, github_workflows, gitlab_com, gitlab_onprem, google_playstore, httpx, jadx, postman, postman_download, social, trufflehog |\n| dirbust-heavy | web | Recursive web directory brute-force (aggressive) | 5 | ffuf, ffuf_shortnames, httpx, iis_shortnames, wayback |\n| dirbust-light | web | Basic web directory brute-force (surface-level directories only) | 4 | ffuf, ffuf_shortnames, httpx, iis_shortnames |\n| dotnet-audit | web | Comprehensive scan for all IIS/.NET specific modules and module settings | 9 | ajaxpro, aspnet_bin_exposure, badsecrets, dotnetnuke, ffuf, ffuf_shortnames, httpx, iis_shortnames, telerik |\n| email-enum | | Enumerate email addresses from APIs, web crawling, etc. | 8 | dehashed, dnscaa, dnstlsrpt, emailformat, hunterio, pgp, skymem, sslcert |\n| fast | | Scan only the provided targets as fast as possible - no extra discovery | 0 | |\n| iis-shortnames | web | Recursively enumerate IIS shortnames | 3 | ffuf_shortnames, httpx, iis_shortnames |\n| kitchen-sink | | Everything everywhere all at once | 90 | anubisdb, apkpure, asn, azure_realm, azure_tenant, baddns, baddns_direct, baddns_zone, badsecrets, bevigil, bucket_amazon, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, bucket_microsoft, bufferoverrun, builtwith, c99, censys_dns, certspotter, chaos, code_repository, crt, crt_db, dehashed, digitorus, dnsbimi, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dnstlsrpt, docker_pull, dockerhub, emailformat, ffuf, ffuf_shortnames, filedownload, fullhunt, git, git_clone, gitdumper, github_codesearch, github_org, github_usersearch, github_workflows, gitlab_com, gitlab_onprem, google_playstore, gowitness, graphql_introspection, hackertarget, httpx, hunt, hunterio, iis_shortnames, ipneighbor, jadx, leakix, myssl, ntlm, oauth, otx, paramminer_cookies, paramminer_getparams, paramminer_headers, passivetotal, pgp, postman, postman_download, rapiddns, reflected_parameters, robots, securitytrails, securitytxt, shodan_dns, shodan_idb, sitedossier, skymem, social, sslcert, subdomaincenter, subdomainradar, trickest, trufflehog, urlscan, virustotal, wayback |\n| lightfuzz-heavy | web | Discover web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques, including POST parameters, which are more invasive. Uses all lightfuzz modules, and adds paramminer modules for parameter discovery. Avoids running against confirmed WAFs. | 10 | badsecrets, httpx, hunt, lightfuzz, paramminer_cookies, paramminer_getparams, paramminer_headers, portfilter, reflected_parameters, robots |\n| lightfuzz-light | web | Discover web parameters and lightly fuzz them for vulnerabilities, with only the most common vulnerabilities and minimal extra modules. Safest to run alongside larger scans. | 3 | httpx, lightfuzz, portfilter |\n| lightfuzz-medium | web | Discover web parameters and lightly fuzz them for vulnerabilities. Uses all lightfuzz modules, without some of the more intense discovery techniques. Does not send POST requests. This is the default lightfuzz preset; if you're not sure which one to use, this is a good starting point. Avoids running against confirmed WAFs. | 6 | badsecrets, httpx, hunt, lightfuzz, portfilter, reflected_parameters |\n| lightfuzz-superheavy | web | Discover web parameters and lightly fuzz them for vulnerabilities, with the most intense discovery techniques, including POST parameters, which are more invasive. Uses all lightfuzz modules, adds paramminer modules for parameter discovery, and tests each unique parameter-value instance individually. | 10 | badsecrets, httpx, hunt, lightfuzz, paramminer_cookies, paramminer_getparams, paramminer_headers, portfilter, reflected_parameters, robots |\n| lightfuzz-xss | web | Discover web parameters and lightly fuzz them, limited to just GET-based xss vulnerabilities. Avoids running against confirmed WAFs. This is an example of a custom lightfuzz preset, selectively enabling a single lightfuzz module. | 5 | httpx, lightfuzz, paramminer_getparams, portfilter, reflected_parameters |\n| nuclei | nuclei | Run nuclei scans against all discovered targets | 3 | httpx, nuclei, portfilter |\n| nuclei-budget | nuclei | Run nuclei scans against all discovered targets, using budget mode to look for low hanging fruit with greatly reduced number of requests | 3 | httpx, nuclei, portfilter |\n| nuclei-intense | nuclei | Run nuclei scans against all discovered targets, allowing for spidering, against ALL URLs, and with additional discovery modules. | 6 | httpx, nuclei, portfilter, robots, urlscan, wayback |\n| nuclei-technology | nuclei | Run nuclei scans against all discovered targets, running templates which match discovered technologies | 3 | httpx, nuclei, portfilter |\n| paramminer | web | Discover new web parameters via brute-force, and analyze them with additional modules | 6 | httpx, hunt, paramminer_cookies, paramminer_getparams, paramminer_headers, reflected_parameters |\n| spider | | Recursive web spider | 1 | httpx |\n| spider-intense | | Recursive web spider with more aggressive settings | 1 | httpx |\n| subdomain-enum | | Enumerate subdomains via APIs, brute-force | 51 | anubisdb, asn, azure_realm, azure_tenant, baddns_direct, baddns_zone, bevigil, bufferoverrun, builtwith, c99, censys_dns, certspotter, chaos, crt, crt_db, digitorus, dnsbimi, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dnstlsrpt, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, ipneighbor, leakix, myssl, oauth, otx, passivetotal, postman, postman_download, rapiddns, securitytrails, securitytxt, shodan_dns, shodan_idb, sitedossier, social, sslcert, subdomaincenter, subdomainradar, trickest, urlscan, virustotal, wayback |\n| tech-detect | | Detect technologies via Nuclei, and FingerprintX | 3 | fingerprintx, httpx, nuclei |\n| web-basic | | Quick web scan | 18 | azure_realm, baddns, badsecrets, bucket_amazon, bucket_firebase, bucket_google, bucket_microsoft, ffuf_shortnames, filedownload, git, graphql_introspection, httpx, iis_shortnames, ntlm, oauth, robots, securitytxt, sslcert |\n| web-screenshots | | Take screenshots of webpages | 3 | gowitness, httpx, social |\n| web-thorough | | Aggressive web scan | 32 | ajaxpro, aspnet_bin_exposure, azure_realm, baddns, badsecrets, bucket_amazon, bucket_digitalocean, bucket_firebase, bucket_google, bucket_microsoft, bypass403, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, graphql_introspection, host_header, httpx, hunt, iis_shortnames, lightfuzz, ntlm, oauth, reflected_parameters, retirejs, robots, securitytxt, smuggler, sslcert, telerik, url_manipulation |\n\n" }, { "path": "docs/scanning/tips_and_tricks.md", "content": "# Tips and Tricks\n\nBelow are some helpful tricks to help you in your adventures.\n\n## Change Verbosity During Scan\nPress enter during a BBOT scan to change the log level. This will allow you to see debugging messages, etc.\n\n\n\n## Kill Individual Module During Scan\nSometimes a certain module can get stuck or slow down the scan. If this happens and you want to kill it, just type \"`kill `\" in the terminal and press enter. This will kill and disable the module for the rest of the scan.\n\nYou can also kill multiple modules at a time by specifying them in a space or comma-separated list:\n\n```bash\nkill httpx sslcert\n```\n\n\n\n## Common Config Changes\n\n### Speed Up Slow Modules\n\nBBOT modules can be parallelized so that more than one instance runs at a time. By default, many modules are already set to reasonable defaults:\n\n```python\nclass baddns(BaseModule):\n module_threads = 8\n```\n\nTo override this, you can set a module's `module_threads` in the config:\n\n```bash\n# increase baddns threads to 20\nbbot -t evilcorp.com -m baddns -c modules.baddns.module_threads=20\n```\n\n### Boost DNS Brute-force Speed\n\nIf you have a fast internet connection or are running BBOT from a cloud VM, you can speed up subdomain enumeration by cranking the threads for `massdns`. The default is `1000`, which is about 1MB/s of DNS traffic:\n\n```bash\n# massdns with 5000 resolvers, about 5MB/s\nbbot -t evilcorp.com -f subdomain-enum -c dns.brute_threads=5000\n```\n\n### Web Spider\n\nThe web spider is great for finding juicy data like subdomains, email addresses, and javascript secrets buried in webpages. However since it can lengthen the duration of a scan, it's disabled by default. To enable the web spider, you must increase the value of `web.spider_distance`.\n\nThe web spider is controlled with three config values:\n\n- `web.spider_depth` (default: `1`: the maximum directory depth allowed. This is to prevent the spider from delving too deep into a website.\n- `web.spider_distance` (`0` == all spidering disabled, default: `0`): the maximum number of links that can be followed in a row. This is designed to limit the spider in cases where `web.spider_depth` fails (e.g. for an ecommerce website with thousands of base-level URLs).\n- `web.spider_links_per_page` (default: `25`): the maximum number of links per page that can be followed. This is designed to save you in cases where a single page has hundreds or thousands of links.\n\nHere is a typical example:\n\n```yaml title=\"spider.yml\"\nconfig:\n web:\n spider_depth: 2\n spider_distance: 2\n spider_links_per_page: 25\n```\n\n```bash\n# run the web spider against www.evilcorp.com\nbbot -t www.evilcorp.com -m httpx -c spider.yml\n```\n\nYou can also pair the web spider with subdomain enumeration:\n\n```bash\n# spider every subdomain of evilcorp.com\nbbot -t evilcorp.com -f subdomain-enum -c spider.yml\n```\n\n### Exclude CDNs from Port Scan\n\nUse `--exclude-cdns` to filter out unwanted open ports from CDNs and WAFs, e.g. Cloudflare. You can also customize the criteria by setting `modules.portfilter.cdn_tags`. By default, only open ports with `cdn-*` tags are filtered, but you can include all cloud providers by setting `cdn_tags` to `cdn,cloud`:\n\n```bash\nbbot -t evilcorp.com --exclude-cdns -c modules.portfilter.cdn_tags=cdn,cloud\n```\n\nAdditionally, you can customize the allowed ports by setting `modules.portscan.allowed_cdn_ports`.\n\n```bash\nbbot -t evilcorp.com --exclude-cdns -c modules.portfilter.allowed_cdn_ports=80,443,8443\n```\n\nExample preset:\n\n```yaml title=\"skip_cdns.yml\"\nmodules:\n - portfilter\n\nconfig:\n modules:\n portfilter:\n cdn_tags: cdn-,cloud-\n allowed_cdn_ports: 80,443,8443\n```\n\n```bash\nbbot -t evilcorp.com -p skip_cdns.yml\n```\n\n### Ingest BBOT Data Into SIEM (Elastic, Splunk)\n\nIf your goal is to run a BBOT scan and later feed its data into a SIEM such as Elastic, be sure to enable this option when scanning:\n\n```bash\nbbot -t evilcorp.com -c modules.json.siem_friendly=true\n```\n\nThis ensures the `.data` event attribute is always the same type (a dictionary), by nesting it like so:\n```json\n{\n \"type\": \"DNS_NAME\",\n \"data\": {\n \"DNS_NAME\": \"blacklanternsecurity.com\"\n }\n}\n```\n\n### Custom HTTP Proxy\n\nWeb pentesters may appreciate BBOT's ability to quickly populate Burp Suite site maps for all subdomains in a target. If your scan includes gowitness, this will capture the traffic as if you manually visited each website in your browser -- including auxiliary web resources and javascript API calls. To accomplish this, set the `web.http_proxy` config option like so:\n\n```bash\n# enumerate subdomains, take web screenshots, proxy through Burp\nbbot -t evilcorp.com -f subdomain-enum -m gowitness -c web.http_proxy=http://127.0.0.1:8080\n```\n\n### Display `HTTP_RESPONSE` Events\n\nBBOT's `httpx` module emits `HTTP_RESPONSE` events, but by default they're hidden from output. These events contain the full raw HTTP body along with headers, etc. If you want to see them, you can modify `omit_event_types` in the config:\n\n```yaml title=\"~/.bbot/config/bbot.yml\"\nomit_event_types:\n - URL_UNVERIFIED\n # - HTTP_RESPONSE\n```\n\n### Display Out-of-scope Events\nBy default, BBOT only shows in-scope events (with a few exceptions for things like storage buckets). If you want to see events that BBOT is emitting internally (such as for DNS resolution, etc.), you can increase `scope.report_distance` in the config or on the command line like so:\n~~~bash\n# display events up to scope distance 2 (default == 0)\nbbot -f subdomain-enum -t evilcorp.com -c scope.report_distance=2\n~~~\n\n### Speed Up Scans By Disabling DNS Resolution\n\nIf you already have a list of discovered targets (e.g. URLs), you can speed up the scan by skipping BBOT's DNS resolution. You can do this by setting `dns.disable` to `true`:\n\n~~~bash\n# completely disable DNS resolution\nbbot -m httpx gowitness wappalyzer -t urls.txt -c dns.disable=true\n~~~\n\nNote that the above setting _completely_ disables DNS resolution, meaning even `A` and `AAAA` records are not resolved. This can cause problems if you're using an IP whitelist or blacklist. In this case, you'll want to use `dns.minimal` instead:\n\n~~~bash\n# only resolve A and AAAA records\nbbot -m httpx gowitness wappalyzer -t urls.txt -c dns.minimal=true\n~~~\n\n## FAQ\n\n### What is `URL_UNVERIFIED`?\n\n`URL_UNVERIFIED` events are URLs that haven't yet been visited by `httpx`. Once `httpx` visits them, it reraises them as `URL`s, tagged with their resulting status code.\n\nFor example, when [`excavate`](index.md/#types-of-modules) gets an `HTTP_RESPONSE` event, it extracts links from the raw HTTP response as `URL_UNVERIFIED`s and then passes them back to `httpx` to be visited.\n\nBy default, `URL_UNVERIFIED`s are hidden from output. If you want to see all of them including the out-of-scope ones, you can do it by changing `omit_event_types` and `scope.report_distance` in the config like so:\n\n```bash\n# visit www.evilcorp.com and extract all the links\nbbot -t www.evilcorp.com -m httpx -c omit_event_types=[] scope.report_distance=2\n```\n\n### Can I crank up the threads for a module to make it go faster?\n\nYes, you can customize the threads for any module by setting `module_threads` like so:\n\n```bash\nbbot -t evilcorp.com -m sslcert -c modules.sslcert.module_threads=50\n```\n\n`module_threads` is one of several [universal module options](./configuration.md) that can be applied to any module.\n" }, { "path": "docs/troubleshooting.md", "content": "# Troubleshooting\n\n## Installation troubleshooting\n- `Fatal error from pip prevented installation.`\n- `ERROR: No matching distribution found for bbot`\n- `bash: /home/user/.local/bin/bbot: /home/user/.local/pipx/venvs/bbot/bin/python: bad interpreter`\n\nIf you get errors resembling any of the above, it's probably because your Python version is too old. To install a newer version (3.9+ is required), you will need to do something like this:\n```bash\n# install a newer version of python\nsudo apt install python3.9 python3.9-venv\n# install pipx\npython3.9 -m pip install --user pipx\n# add pipx to your path\npython3.9 -m pipx ensurepath\n# reboot\nreboot\n# install bbot\npython3.9 -m pipx install bbot\n# run bbot\nbbot --help\n```\n\n## `ModuleNotFoundError`\nIf you run into a `ModuleNotFoundError`, try running your `bbot` command again with `--force-deps`. This will repair your modules' Python dependencies.\n\n## Regenerate Config\nAs a troubleshooting step it is sometimes useful to clear out your older configs and let BBOT generate new ones. This will ensure that new defaults are property restored, etc.\n```bash\n# make a backup of the old configs\nmv ~/.config/bbot ~/.config/bbot.bak\n\n# generate new configs\nbbot\n```\n" }, { "path": "examples/discord_bot.py", "content": "import discord\nfrom discord.ext import commands\n\nfrom bbot.scanner import Scanner\nfrom bbot.modules.output.discord import Discord\n\n\nclass BBOTDiscordBot(commands.Cog):\n \"\"\"\n A simple Discord bot capable of running a BBOT scan.\n\n To set up:\n 1. Go to Discord Developer Portal (https://discord.com/developers)\n 2. Create a new application\n 3. Create an invite link for the bot, visit the link to invite it to your server\n - Your Application --> OAuth2 --> URL Generator\n - For Scopes, select \"bot\"\"\n - For Bot Permissions, select:\n - Read Messages/View Channels\n - Send Messages\n 4. Turn on \"Message Content Intent\"\n - Your Application --> Bot --> Privileged Gateway Intents --> Message Content Intent\n 5. Copy your Discord Bot Token and put it at the top this file\n - Your Application --> Bot --> Reset Token\n 6. Run this script\n\n To scan evilcorp.com, you would type:\n\n /scan evilcorp.com\n\n Results will be output to the same channel.\n \"\"\"\n\n def __init__(self):\n self.current_scan = None\n\n @commands.command(name=\"scan\", description=\"Scan a target with BBOT.\")\n async def scan(self, ctx, target: str):\n if self.current_scan is not None:\n self.current_scan.stop()\n await ctx.send(f\"Starting scan against {target}.\")\n\n # creates scan instance\n self.current_scan = Scanner(target, flags=\"subdomain-enum\")\n discord_module = Discord(self.current_scan)\n\n seen = set()\n num_events = 0\n # start scan and iterate through results\n async for event in self.current_scan.async_start():\n if hash(event) in seen:\n continue\n seen.add(hash(event))\n await ctx.send(discord_module.format_message(event))\n num_events += 1\n\n await ctx.send(f\"Finished scan against {target}. {num_events:,} results.\")\n self.current_scan = None\n\n\nif __name__ == \"__main__\":\n intents = discord.Intents.default()\n intents.message_content = True\n bot = commands.Bot(command_prefix=\"/\", intents=intents)\n\n @bot.event\n async def on_ready():\n print(f\"We have logged in as {bot.user}\")\n await bot.add_cog(BBOTDiscordBot())\n\n bot.run(\"DISCORD_BOT_TOKEN_HERE\")\n" }, { "path": "extra_sass/style.css.scss", "content": "/* GLOBAL STYLES */\n\n:root {\n --bbot-orange: #ff8400;\n}\n\n// .md-grid {\n// margin-left: unset;\n// margin-right: unset;\n// max-width: unset;\n// }\n\np img {\n max-width: 60em !important;\n}\n\n.demonic-jimmy {\n color: var(--bbot-orange);\n}\n\n.md-nav__link--active {\n font-weight: bold;\n}\n\n.md-typeset__table td:first-child {\n font-weight: bold;\n}\n\na.md-source,\n.md-header__topic > span,\na:hover {\n color: var(--bbot-orange);\n}\n\narticle.md-content__inner {\n h1 {\n font-weight: 500;\n color: var(--bbot-orange);\n }\n h1,\n h2 {\n color: var(--bbot-orange);\n }\n h2,\n h3,\n h4,\n h5 {\n font-weight: 300;\n }\n div.highlight {\n background-color: unset !important;\n }\n}\n\ntable {\n font-family: monospace;\n\n td {\n max-width: 100em;\n }\n}\n\n/* DARK MODE SPECIFIC */\n\n[data-md-color-primary=black] p a.md-button--primary {\n background-color: black;\n border: none;\n}\n\n[data-md-color-primary=black] p a.md-button--primary:hover {\n background-color: var(--bbot-orange);\n}\n\n[data-md-color-scheme=\"slate\"] {\n div.md-source__repository ul {\n color: white;\n }\n\n .md-nav__link {\n color: white;\n }\n\n .md-nav__link--active {\n font-weight: bold;\n }\n\n .md-typeset__table tr {\n background-color: #202027;\n }\n\n .md-nav__link.md-nav__link--active {\n color: var(--bbot-orange);\n }\n\n .md-typeset__table thead tr {\n color: var(--bbot-orange);\n background-color: var(--md-primary-fg-color--dark);\n }\n}\n" }, { "path": "funding.yml", "content": "github: blacklanternsecurity\n" }, { "path": "mkdocs.yml", "content": "# Project information\nsite_name: BBOT Docs\nsite_url: https://blacklanternsecurity.github.io/bbot/\nsite_author: TheTechromancer\nsite_description: >-\n OSINT automation for hackers\n# Repository\nrepo_name: blacklanternsecurity/bbot\nrepo_url: https://github.com/blacklanternsecurity/bbot\nwatch:\n - \"mkdocs.yml\"\n - \"bbot\"\n - \"docs\"\n\n# Page tree\nnav:\n- User Manual:\n - Basics:\n - Getting Started: index.md\n - How it Works: how_it_works.md\n - Comparison to Other Tools: comparison.md\n - Scanning:\n - Scanning Overview: scanning/index.md\n - Presets:\n - Overview: scanning/presets.md\n - List of Presets: scanning/presets_list.md\n - Events: scanning/events.md\n - Output: scanning/output.md\n - Tips and Tricks: scanning/tips_and_tricks.md\n - Advanced Usage: scanning/advanced.md\n - Configuration: scanning/configuration.md\n - Modules:\n - List of Modules: modules/list_of_modules.md\n - Nuclei: modules/nuclei.md\n - Custom YARA Rules: modules/custom_yara_rules.md\n - Lightfuzz: modules/lightfuzz.md\n - Misc:\n - Contribution: contribution.md\n - Release History: release_history.md\n - Troubleshooting: troubleshooting.md\n- Developer Manual:\n - Development Overview: dev/index.md\n - Setting Up a Dev Environment: dev/dev_environment.md\n - BBOT Internal Architecture: dev/architecture.md\n - How to Write a BBOT Module: dev/module_howto.md\n - Unit Tests: dev/tests.md\n - Discord Bot Example: dev/discord_bot.md\n - Code Reference:\n - Scanner: dev/scanner.md\n - Presets: dev/presets.md\n - Event: dev/event.md\n - Target: dev/target.md\n - BaseModule: dev/basemodule.md\n - BBOTCore: dev/core.md\n - Engine: dev/engine.md\n - Helpers:\n - Overview: dev/helpers/index.md\n - Command: dev/helpers/command.md\n - DNS: dev/helpers/dns.md\n - Interactsh: dev/helpers/interactsh.md\n - Miscellaneous: dev/helpers/misc.md\n - Web: dev/helpers/web.md\n - Word Cloud: dev/helpers/wordcloud.md\n\ntheme:\n name: material\n logo: bbot.png\n favicon: favicon.png\n features:\n - content.code.copy\n - content.tooltips\n - navigation.tabs\n - navigation.sections\n - navigation.expand\n - toc.integrate\n palette:\n - scheme: slate\n primary: black\n accent: deep orange\n\nplugins:\n - mike\n - search\n - extra-sass\n - mkdocstrings:\n enable_inventory: true\n handlers:\n python:\n options:\n heading_level: 1\n show_signature_annotations: true\n show_root_toc_entry: false\n show_root_heading: true\n show_root_full_path: false\n separate_signature: true\n docstring_section_style: \"list\"\n filters:\n - \"!^_\"\n - \"^__init__$\"\n import:\n - https://docs.python.org/3.11/objects.inv\n - https://omegaconf.readthedocs.io/en/latest/objects.inv\n\nextra:\n version:\n provider: mike\n default: Stable\n\nmarkdown_extensions:\n - tables\n - attr_list\n - admonition\n - pymdownx.details\n - pymdownx.snippets\n - pymdownx.superfences\n - pymdownx.highlight:\n use_pygments: True\n noclasses: True\n pygments_style: github-dark\n - pymdownx.superfences:\n custom_fences:\n - name: mermaid\n class: mermaid\n format: !!python/name:pymdownx.superfences.fence_code_format\n\nextra_javascript:\n - javascripts/tablesort.js\n - javascripts/tablesort.min.js\n - javascripts/vega@5.js\n - javascripts/vega-lite@5.js\n - javascripts/vega-embed@6.js\n" }, { "path": "pyproject.toml", "content": "[tool.poetry]\nname = \"bbot\"\nversion = \"2.8.4\"\ndescription = \"OSINT automation for hackers.\"\nauthors = [\n \"TheTechromancer\",\n \"Paul Mueller\",\n]\nlicense = \"GPL-3.0\"\nreadme = \"README.md\"\nrepository = \"https://github.com/blacklanternsecurity/bbot\"\nhomepage = \"https://github.com/blacklanternsecurity/bbot\"\ndocumentation = \"https://www.blacklanternsecurity.com/bbot/\"\nkeywords = [\"python\", \"cli\", \"automation\", \"osint\", \"threat-intel\", \"intelligence\", \"neo4j\", \"scanner\", \"python-library\", \"hacking\", \"recursion\", \"pentesting\", \"recon\", \"command-line-tool\", \"bugbounty\", \"subdomains\", \"security-tools\", \"subdomain-scanner\", \"osint-framework\", \"attack-surface\", \"subdomain-enumeration\", \"osint-tool\"]\nclassifiers = [\n \"Operating System :: POSIX :: Linux\",\n \"Topic :: Security\",\n]\n\n[tool.poetry.urls]\n\"Discord\" = \"https://discord.com/invite/PZqkgxu5SA\"\n\"Docker Hub\" = \"https://hub.docker.com/r/blacklanternsecurity/bbot\"\n\n[tool.poetry.scripts]\nbbot = 'bbot.cli:main'\n\n[tool.poetry.dependencies]\npython = \"^3.9\"\nomegaconf = \"^2.3.0\"\npsutil = \">=5.9.4,<8.0.0\"\nwordninja = \"^2.0.0\"\nansible-runner = \"^2.3.2\"\ndeepdiff = \"^8.0.0\"\nxmltojson = \"^2.0.2\"\npycryptodome = \"^3.17\"\nidna = \"^3.4\"\ntabulate = \"0.8.10\"\nwebsockets = \">=14.0.0,<16.0.0\"\npyjwt = \"^2.7.0\"\nbeautifulsoup4 = \"^4.12.2\"\nlxml = \">=4.9.2,<7.0.0\"\ndnspython = \">=2.7.0,<2.8.0\"\ncachetools = \">=5.3.2,<7.0.0\"\nsocksio = \"^1.0.0\"\njinja2 = \"^3.1.3\"\nregex = \">=2024.4.16,<2027.0.0\"\nunidecode = \"^1.3.8\"\nmmh3 = \">=4.1,<6.0\"\nxxhash = \"^3.5.0\"\nsetproctitle = \"^1.3.3\"\nyara-python = \"4.5.2\"\npyzmq = \">=26.0.3,<28.0.0\"\nhttpx = \"^0.28.1\"\npuremagic = \"^1.28\"\npydantic = \"^2.9.2\"\nradixtarget = \"^3.0.13\"\norjson = \"^3.10.12\"\nansible-core = \"^2.15.13\"\ntldextract = \"^5.3.0\"\ncloudcheck = \"^9.2.0\"\n\n[tool.poetry.group.dev.dependencies]\npoetry-dynamic-versioning = \">=0.21.4,<1.11.0\"\nurllib3 = \"^2.0.2\"\nwerkzeug = \">=2.3.4,<4.0.0\"\npytest-env = \">=0.8.2,<1.2.0\"\npre-commit = \">=3.4,<5.0\"\npytest-cov = \">=5,<8\"\npytest-rerunfailures = \">=14,<17\"\npytest-timeout = \"^2.3.1\"\npytest-httpserver = \"^1.0.11\"\npytest = \"^8.3.1\"\npytest-asyncio = \"1.2.0\"\nuvicorn = \">=0.32,<0.40\"\nfastapi = \">=0.115.5,<0.129.0\"\npytest-httpx = \">=0.35\"\npytest-benchmark = \">=4,<6\"\nruff = \"0.15.4\"\npymdown-extensions = \"^10.20.1\"\ngriffe = \"^1\"\n\n[tool.poetry.group.docs.dependencies]\nmkdocs = \"^1.5.2\"\nmkdocs-extra-sass-plugin = \"^0.1.0\"\nmkdocs-material = \"^9.2.5\"\nmkdocs-material-extensions = \"^1.1.1\"\nmkdocstrings = \">=0.22,<0.31\"\nmkdocstrings-python = \"^1.6.0\"\nlivereload = \"^2.6.3\"\nmike = \"^2.1.3\"\n\n[tool.pytest.ini_options]\nenv = [\n \"BBOT_TESTING = True\",\n]\nasyncio_mode = \"auto\"\nasyncio_default_fixture_loop_scope = \"session\"\naddopts = \"--benchmark-skip\"\n\n# Benchmark configuration for pytest-benchmark\n[tool.pytest-benchmark]\ngroup_by = \"group\"\nsort = \"mean\"\nwarmup = true\nwarmup_iterations = 3\ndisable_gc = true\nmin_rounds = 5\n\n[build-system]\nrequires = [\"poetry-core>=1.0.0\", \"poetry-dynamic-versioning\"]\nbuild-backend = \"poetry_dynamic_versioning.backend\"\n\n[tool.codespell]\nignore-words-list = \"bu,cna,couldn,dialin,nd,ned,thirdparty\"\nskip = \"./docs/javascripts/vega*.js,./bbot/wordlists/*\"\n\n[tool.ruff]\nline-length = 119\nformat.exclude = [\"bbot/test/test_step_1/test_manager_*\"]\nlint.select = [\"E\", \"F\"]\nlint.ignore = [\"E402\", \"E711\", \"E713\", \"E721\", \"E741\", \"F403\", \"F405\", \"E501\"]\n\n[tool.poetry-dynamic-versioning]\nenable = true\nmetadata = false\nformat-jinja = 'v2.8.4{% if branch == \"dev\" %}.{{ distance }}rc{% endif %}'\n\n[tool.poetry-dynamic-versioning.substitution]\nfiles = [\"*/__init__.py\"]\n" } ]