Repository: projectdiscovery/subfinder Branch: dev Commit: 13e5db750080 Files: 111 Total size: 369.5 KB Directory structure: gitextract_djozq64v/ ├── .github/ │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.md │ │ ├── config.yml │ │ ├── feature_request.md │ │ └── issue-report.md │ ├── PULL_REQUEST_TEMPLATE.md │ ├── dependabot.yml │ ├── release.yml │ └── workflows/ │ ├── build-test.yml │ ├── codeql-analysis.yml │ ├── compat-checks.yaml │ ├── dep-auto-merge.yml │ ├── dockerhub-push.yml │ ├── release-binary.yml │ └── release-test.yml ├── .gitignore ├── .goreleaser.yml ├── DISCLAIMER.md ├── Dockerfile ├── LICENSE.md ├── Makefile ├── README.md ├── THANKS.md ├── cmd/ │ └── subfinder/ │ └── main.go ├── examples/ │ └── main.go ├── go.mod ├── go.sum └── pkg/ ├── passive/ │ ├── doc.go │ ├── passive.go │ ├── sources.go │ ├── sources_test.go │ ├── sources_w_auth_test.go │ └── sources_wo_auth_test.go ├── resolve/ │ ├── client.go │ ├── doc.go │ └── resolve.go ├── runner/ │ ├── banners.go │ ├── config.go │ ├── doc.go │ ├── enumerate.go │ ├── enumerate_test.go │ ├── initialize.go │ ├── options.go │ ├── outputter.go │ ├── runner.go │ ├── stats.go │ ├── util.go │ └── validate.go ├── subscraping/ │ ├── agent.go │ ├── doc.go │ ├── extractor.go │ ├── sources/ │ │ ├── alienvault/ │ │ │ └── alienvault.go │ │ ├── anubis/ │ │ │ └── anubis.go │ │ ├── bevigil/ │ │ │ └── bevigil.go │ │ ├── bufferover/ │ │ │ └── bufferover.go │ │ ├── builtwith/ │ │ │ └── builtwith.go │ │ ├── c99/ │ │ │ └── c99.go │ │ ├── censys/ │ │ │ ├── censys.go │ │ │ └── censys_test.go │ │ ├── certspotter/ │ │ │ └── certspotter.go │ │ ├── chaos/ │ │ │ └── chaos.go │ │ ├── chinaz/ │ │ │ └── chinaz.go │ │ ├── commoncrawl/ │ │ │ └── commoncrawl.go │ │ ├── crtsh/ │ │ │ └── crtsh.go │ │ ├── digitalyama/ │ │ │ └── digitalyama.go │ │ ├── digitorus/ │ │ │ └── digitorus.go │ │ ├── dnsdb/ │ │ │ └── dnsdb.go │ │ ├── dnsdumpster/ │ │ │ └── dnsdumpster.go │ │ ├── dnsrepo/ │ │ │ └── dnsrepo.go │ │ ├── domainsproject/ │ │ │ └── domainsproject.go │ │ ├── driftnet/ │ │ │ └── driftnet.go │ │ ├── facebook/ │ │ │ ├── ctlogs.go │ │ │ ├── ctlogs_test.go │ │ │ └── types.go │ │ ├── fofa/ │ │ │ └── fofa.go │ │ ├── fullhunt/ │ │ │ └── fullhunt.go │ │ ├── github/ │ │ │ ├── github.go │ │ │ └── tokenmanager.go │ │ ├── gitlab/ │ │ │ └── gitlab.go │ │ ├── hackertarget/ │ │ │ └── hackertarget.go │ │ ├── hudsonrock/ │ │ │ └── hudsonrock.go │ │ ├── intelx/ │ │ │ └── intelx.go │ │ ├── leakix/ │ │ │ └── leakix.go │ │ ├── merklemap/ │ │ │ └── merklemap.go │ │ ├── netlas/ │ │ │ └── netlas.go │ │ ├── onyphe/ │ │ │ └── onyphe.go │ │ ├── profundis/ │ │ │ └── profundis.go │ │ ├── pugrecon/ │ │ │ └── pugrecon.go │ │ ├── quake/ │ │ │ └── quake.go │ │ ├── rapiddns/ │ │ │ └── rapiddns.go │ │ ├── reconcloud/ │ │ │ └── reconcloud.go │ │ ├── reconeer/ │ │ │ └── reconeer.go │ │ ├── redhuntlabs/ │ │ │ └── redhuntlabs.go │ │ ├── riddler/ │ │ │ └── riddler.go │ │ ├── robtex/ │ │ │ └── robtext.go │ │ ├── rsecloud/ │ │ │ └── rsecloud.go │ │ ├── securitytrails/ │ │ │ └── securitytrails.go │ │ ├── shodan/ │ │ │ └── shodan.go │ │ ├── sitedossier/ │ │ │ └── sitedossier.go │ │ ├── thc/ │ │ │ └── thc.go │ │ ├── threatbook/ │ │ │ └── threatbook.go │ │ ├── threatcrowd/ │ │ │ └── threatcrowd.go │ │ ├── threatminer/ │ │ │ └── threatminer.go │ │ ├── urlscan/ │ │ │ └── urlscan.go │ │ ├── virustotal/ │ │ │ └── virustotal.go │ │ ├── waybackarchive/ │ │ │ └── waybackarchive.go │ │ ├── whoisxmlapi/ │ │ │ └── whoisxmlapi.go │ │ ├── windvane/ │ │ │ └── windvane.go │ │ └── zoomeyeapi/ │ │ └── zoomeyeapi.go │ ├── types.go │ └── utils.go └── testutils/ └── integration.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.md ================================================ --- name: Bug report about: Create a report to help us improve title: "[Issue] " labels: '' assignees: '' --- **Describe the bug** A clear and concise description of what the bug is. **Subfinder version** Include the version of subfinder you are using, `subfinder -version` **Complete command you used to reproduce this** **Screenshots** Add screenshots of the error for a better context. ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: Ask an question / advise on using subfinder url: https://github.com/projectdiscovery/subfinder/discussions/categories/q-a about: Ask a question or request support for using subfinder - name: Share idea / feature to discuss for subfinder url: https://github.com/projectdiscovery/subfinder/discussions/categories/ideas about: Share idea / feature to discuss for subfinder - name: Connect with PD Team (Discord) url: https://discord.gg/projectdiscovery about: Connect with PD Team for direct communication ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature request about: Request feature to implement in this project labels: 'Type: Enhancement' --- ### Please describe your feature request: ### Describe the use case of this feature: ================================================ FILE: .github/ISSUE_TEMPLATE/issue-report.md ================================================ --- name: Issue report about: Create a report to help us to improve the project labels: 'Type: Bug' --- ### Subfinder version: ### Current Behavior: ### Expected Behavior: ### Steps To Reproduce: ### Anything else: ================================================ FILE: .github/PULL_REQUEST_TEMPLATE.md ================================================ ## Proposed changes ### Proof ## Checklist - [ ] Pull request is created against the [dev](https://github.com/projectdiscovery/subfinder/tree/dev) branch - [ ] All checks passed (lint, unit/integration/regression tests etc.) with my changes - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] I have added necessary documentation (if appropriate) ================================================ FILE: .github/dependabot.yml ================================================ # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 updates: # Maintain dependencies for go modules - package-ecosystem: "gomod" directory: "/" schedule: interval: "weekly" target-branch: "dev" commit-message: prefix: "chore" include: "scope" allow: - dependency-name: "github.com/projectdiscovery/*" groups: modules: patterns: ["github.com/projectdiscovery/*"] labels: - "Type: Maintenance" # # Maintain dependencies for GitHub Actions # - package-ecosystem: "github-actions" # directory: "/" # schedule: # interval: "weekly" # target-branch: "dev" # commit-message: # prefix: "chore" # include: "scope" # labels: # - "Type: Maintenance" # # # Maintain dependencies for docker # - package-ecosystem: "docker" # directory: "/" # schedule: # interval: "weekly" # target-branch: "dev" # commit-message: # prefix: "chore" # include: "scope" # labels: # - "Type: Maintenance" ================================================ FILE: .github/release.yml ================================================ changelog: exclude: authors: - dependabot categories: - title: 🎉 New Features labels: - "Type: Enhancement" - title: 🐞 Bug Fixes labels: - "Type: Bug" - title: 🔨 Maintenance labels: - "Type: Maintenance" - title: Other Changes labels: - "*" ================================================ FILE: .github/workflows/build-test.yml ================================================ name: 🔨 Build Test on: pull_request: paths: - "**.go" - "**.mod" workflow_dispatch: inputs: short: description: "Use -short flag for tests" required: false type: boolean default: false jobs: lint: name: Lint Test if: "${{ !endsWith(github.actor, '[bot]') }}" runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 with: go-version-file: go.mod - name: Run golangci-lint uses: golangci/golangci-lint-action@v8 with: version: latest args: --timeout 5m build: name: Test Builds needs: [lint] runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] steps: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 with: go-version-file: go.mod - run: go build ./... - name: Run tests env: BEVIGIL_API_KEY: ${{secrets.BEVIGIL_API_KEY}} BINARYEDGE_API_KEY: ${{secrets.BINARYEDGE_API_KEY}} BUFFEROVER_API_KEY: ${{secrets.BUFFEROVER_API_KEY}} C99_API_KEY: ${{secrets.C99_API_KEY}} CENSYS_API_KEY: ${{secrets.CENSYS_API_KEY}} CERTSPOTTER_API_KEY: ${{secrets.CERTSPOTTER_API_KEY}} CHAOS_API_KEY: ${{secrets.CHAOS_API_KEY}} CHINAZ_API_KEY: ${{secrets.CHINAZ_API_KEY}} DNSDB_API_KEY: ${{secrets.DNSDB_API_KEY}} DNSREPO_API_KEY: ${{secrets.DNSREPO_API_KEY}} FOFA_API_KEY: ${{secrets.FOFA_API_KEY}} FULLHUNT_API_KEY: ${{secrets.FULLHUNT_API_KEY}} GITHUB_API_KEY: ${{secrets.GITHUB_API_KEY}} INTELX_API_KEY: ${{secrets.INTELX_API_KEY}} LEAKIX_API_KEY: ${{secrets.LEAKIX_API_KEY}} QUAKE_API_KEY: ${{secrets.QUAKE_API_KEY}} ROBTEX_API_KEY: ${{secrets.ROBTEX_API_KEY}} SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}} SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}} THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}} URLSCAN_API_KEY: ${{secrets.URLSCAN_API_KEY}} VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}} WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}} ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}} uses: nick-invision/retry@v2 with: timeout_seconds: 360 max_attempts: 3 command: go test ./... -v ${{ github.event.inputs.short == 'true' && '-short' || '' }} - name: Race Condition Tests run: go build -race ./... - name: Run Example run: go run . working-directory: examples ================================================ FILE: .github/workflows/codeql-analysis.yml ================================================ name: 🚨 CodeQL Analysis on: workflow_dispatch: pull_request: paths: - '**.go' - '**.mod' jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'go' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] steps: - name: Checkout repository uses: actions/checkout@v3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - name: Autobuild uses: github/codeql-action/autobuild@v2 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 ================================================ FILE: .github/workflows/compat-checks.yaml ================================================ name: ♾️ Compatibility Checks on: pull_request: types: [opened, synchronize] branches: - dev jobs: check: if: github.actor == 'dependabot[bot]' runs-on: ubuntu-latest permissions: contents: write steps: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go/compat-checks@master with: go-version-file: 'go.mod' ================================================ FILE: .github/workflows/dep-auto-merge.yml ================================================ name: 🤖 dep auto merge on: pull_request: branches: - dev workflow_dispatch: permissions: pull-requests: write issues: write repository-projects: write jobs: automerge: runs-on: ubuntu-latest if: github.actor == 'dependabot[bot]' steps: - uses: actions/checkout@v3 with: token: ${{ secrets.DEPENDABOT_PAT }} - uses: ahmadnassri/action-dependabot-auto-merge@v2 with: github-token: ${{ secrets.DEPENDABOT_PAT }} target: all ================================================ FILE: .github/workflows/dockerhub-push.yml ================================================ name: 🌥 Docker Push on: workflow_run: workflows: ["🎉 Release Binary"] types: - completed workflow_dispatch: jobs: docker: runs-on: ubuntu-latest-16-cores steps: - name: Checkout uses: actions/checkout@v3 - name: Get Github tag id: meta run: | curl --silent "https://api.github.com/repos/projectdiscovery/subfinder/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Login to DockerHub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - name: Build and push uses: docker/build-push-action@v4 with: context: . platforms: linux/amd64,linux/arm64,linux/arm push: true tags: projectdiscovery/subfinder:latest,projectdiscovery/subfinder:${{ steps.meta.outputs.TAG }} ================================================ FILE: .github/workflows/release-binary.yml ================================================ name: 🎉 Release Binary on: push: tags: - v* workflow_dispatch: jobs: release: runs-on: ubuntu-latest-16-cores steps: - name: "Check out code" uses: actions/checkout@v3 with: fetch-depth: 0 - name: "Set up Go" uses: actions/setup-go@v4 with: go-version: 1.21.x - name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v3 with: args: "release --clean" version: latest env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" ================================================ FILE: .github/workflows/release-test.yml ================================================ name: 🔨 Release Test on: pull_request: paths: - '**.go' - '**.mod' workflow_dispatch: jobs: release-test: runs-on: ubuntu-latest-16-cores steps: - name: "Check out code" uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Go uses: actions/setup-go@v4 with: go-version: 1.21.x - name: release test uses: goreleaser/goreleaser-action@v4 with: args: "release --clean --snapshot" version: latest ================================================ FILE: .gitignore ================================================ .DS_Store cmd/subfinder/subfinder # subfinder binary when built with `go build` v2/cmd/subfinder/subfinder # subfinder binary when built with `make` v2/subfinder vendor/ .idea .devcontainer .vscode dist /subfinder ================================================ FILE: .goreleaser.yml ================================================ version: 2 before: hooks: - go mod tidy builds: - env: - CGO_ENABLED=0 goos: - windows - linux - darwin goarch: - amd64 - '386' - arm - arm64 ignore: - goos: darwin goarch: '386' - goos: windows goarch: 'arm' binary: '{{ .ProjectName }}' main: cmd/subfinder/main.go archives: - formats: - zip name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' checksum: algorithm: sha256 announce: slack: enabled: true channel: '#release' username: GoReleaser message_template: 'New Release: {{ .ProjectName }} {{.Tag}} is published! Check it out at {{ .ReleaseURL }}' discord: enabled: true message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' ================================================ FILE: DISCLAIMER.md ================================================ ## Disclaimer Subfinder leverages multiple open APIs, it is developed for individuals to help them for research or internal work. If you wish to incorporate this tool into a commercial offering or purposes, you must agree to the Terms of the leveraged services: - Bufferover: https://tls.bufferover.run - CommonCrawl: https://commoncrawl.org/terms-of-use/full - certspotter: https://sslmate.com/terms - dnsdumpster: https://hackertarget.com/terms - Google Transparency: https://policies.google.com/terms - Alienvault: https://www.alienvault.com/terms/website-terms-of-use07may2018 --- You expressly understand and agree that Subfinder (creators and contributors) shall not be liable for any damages or losses resulting from your use of this tool or third-party products that use it. Creators aren't in charge of any and have/has no responsibility for any kind of: - Unlawful or illegal use of the tool. - Legal or Law infringement (acted in any country, state, municipality, place) by third parties and users. - Act against ethical and / or human moral, ethic, and peoples and cultures of the world. - Malicious act, capable of causing damage to third parties, promoted or distributed by third parties or the user through this tool. ### Contact Please contact at contact@projectdiscovery.io for any questions. ================================================ FILE: Dockerfile ================================================ # Build FROM golang:1.24-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app RUN go mod download RUN go build ./cmd/subfinder # Release FROM alpine:latest RUN apk upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=build-env /app/subfinder /usr/local/bin/ ENTRYPOINT ["subfinder"] ================================================ FILE: LICENSE.md ================================================ MIT License Copyright (c) 2021 ProjectDiscovery, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ # Go parameters GOCMD=go GOBUILD=$(GOCMD) build GOMOD=$(GOCMD) mod GOTEST=$(GOCMD) test GOFLAGS := -v LDFLAGS := -s -w ifneq ($(shell go env GOOS),darwin) LDFLAGS := -extldflags "-static" endif all: build build: $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "subfinder" cmd/subfinder/main.go test: $(GOTEST) $(GOFLAGS) ./... tidy: $(GOMOD) tidy ================================================ FILE: README.md ================================================

subfinder

Fast passive subdomain enumeration tool.

Features • Install • Usage • API Setup • Library • Join Discord

--- `subfinder` is a subdomain discovery tool that returns valid subdomains for websites, using passive online sources. It has a simple, modular architecture and is optimized for speed. `subfinder` is built for doing one thing only - passive subdomain enumeration, and it does that very well. We have made it to comply with all the used passive source licenses and usage restrictions. The passive model guarantees speed and stealthiness that can be leveraged by both penetration testers and bug bounty hunters alike. # Features

subfinder

- Fast and powerful resolution and wildcard elimination modules - **Curated** passive sources to maximize results - Multiple output formats supported (JSON, file, stdout) - Optimized for speed and **lightweight** on resources - **STDIN/OUT** support enables easy integration into workflows # Usage ```sh subfinder -h ``` This will display help for the tool. Here are all the switches it supports. ```yaml Usage: ./subfinder [flags] Flags: INPUT: -d, -domain string[] domains to find subdomains for -dL, -list string file containing list of domains for subdomain discovery SOURCE: -s, -sources string[] specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources. -recursive use only sources that can handle subdomains recursively (e.g. subdomain.domain.tld vs domain.tld) -all use all sources for enumeration (slow) -es, -exclude-sources string[] sources to exclude from enumeration (-es alienvault,zoomeyeapi) FILTER: -m, -match string[] subdomain or list of subdomain to match (file or comma separated) -f, -filter string[] subdomain or list of subdomain to filter (file or comma separated) RATE-LIMIT: -rl, -rate-limit int maximum number of http requests to send per second -rls value maximum number of http requests to send per second for providers in key=value format (-rls "hackertarget=10/s,shodan=15/s") -t int number of concurrent goroutines for resolving (-active only) (default 10) UPDATE: -up, -update update subfinder to latest version -duc, -disable-update-check disable automatic subfinder update check OUTPUT: -o, -output string file to write output to -oJ, -json write output in JSONL(ines) format -oD, -output-dir string directory to write output (-dL only) -cs, -collect-sources include all sources in the output (-json only) -oI, -ip include host IP in output (-active only) CONFIGURATION: -config string flag config file (default "$CONFIG/subfinder/config.yaml") -pc, -provider-config string provider config file (default "$CONFIG/subfinder/provider-config.yaml") -r string[] comma separated list of resolvers to use -rL, -rlist string file containing list of resolvers to use -nW, -active display active subdomains only -proxy string http proxy to use with subfinder -ei, -exclude-ip exclude IPs from the list of domains DEBUG: -silent show only subdomains in output -version show version of subfinder -v show verbose output -nc, -no-color disable color in output -ls, -list-sources list all available sources OPTIMIZATION: -timeout int seconds to wait before timing out (default 30) -max-time int minutes to wait for enumeration results (default 10) ``` ## Environment Variables Subfinder supports environment variables to specify custom paths for configuration files: - `SUBFINDER_CONFIG` - Path to config.yaml file (overrides default `$CONFIG/subfinder/config.yaml`) - `SUBFINDER_PROVIDER_CONFIG` - Path to provider-config.yaml file (overrides default `$CONFIG/subfinder/provider-config.yaml`) # Installation `subfinder` requires **go1.24** to install successfully. Run the following command to install the latest version: ```sh go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest ``` Learn about more ways to install subfinder here: https://docs.projectdiscovery.io/tools/subfinder/install. ## Post Installation Instructions `subfinder` can be used right after the installation, however many sources required API keys to work. Learn more here: https://docs.projectdiscovery.io/tools/subfinder/install#post-install-configuration. ## Running Subfinder Learn about how to run Subfinder here: https://docs.projectdiscovery.io/tools/subfinder/running. ## Subfinder Go library Subfinder can also be used as library and a minimal examples of using subfinder SDK is available [here](examples/main.go) ### Resources - [Recon with Me !!!](https://dhiyaneshgeek.github.io/bug/bounty/2020/02/06/recon-with-me/) # License `subfinder` is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[THANKS.md](https://github.com/projectdiscovery/subfinder/blob/main/THANKS.md)** file for more details. Read the usage disclaimer at [DISCLAIMER.md](https://github.com/projectdiscovery/subfinder/blob/main/DISCLAIMER.md) and [contact us](mailto:contact@projectdiscovery.io) for any API removal. ================================================ FILE: THANKS.md ================================================ ### Thanks Many people have contributed to subfinder making it a wonderful tool either by making a pull request fixing some stuff or giving generous donations to support the further development of this tool. Here, we recognize these persons and thank them. - All the contributors at [CONTRIBUTORS](https://github.com/projectdiscovery/subfinder/graphs/contributors) who made subfinder what it is. We'd like to thank some additional amazing people, who contributed a lot in subfinder's journey - - [@vzamanillo](https://github.com/vzamanillo) - For adding multiple features and overall project improvements. - [@infosec-au](https://github.com/infosec-au) - Donating to the project. - [@codingo](https://github.com/codingo) - Initial work on the project, managing it, lot of work! - [@picatz](https://github.com/picatz) - Improving the structure of the project a lot. New ideas! ================================================ FILE: cmd/subfinder/main.go ================================================ package main import ( "github.com/projectdiscovery/subfinder/v2/pkg/runner" // Attempts to increase the OS file descriptors - Fail silently _ "github.com/projectdiscovery/fdmax/autofdmax" "github.com/projectdiscovery/gologger" ) func main() { // Parse the command line flags and read config files options := runner.ParseOptions() newRunner, err := runner.NewRunner(options) if err != nil { gologger.Fatal().Msgf("Could not create runner: %s\n", err) } err = newRunner.RunEnumeration() if err != nil { gologger.Fatal().Msgf("Could not run enumeration: %s\n", err) } } ================================================ FILE: examples/main.go ================================================ package main import ( "bytes" "context" "io" "log" "github.com/projectdiscovery/subfinder/v2/pkg/runner" ) func main() { subfinderOpts := &runner.Options{ Threads: 10, // Thread controls the number of threads to use for active enumerations Timeout: 30, // Timeout is the seconds to wait for sources to respond MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration // ResultCallback: func(s *resolve.HostEntry) { // callback function executed after each unique subdomain is found // }, // ProviderConfig: "your_provider_config.yaml", // and other config related options } // disable timestamps in logs / configure logger log.SetFlags(0) subfinder, err := runner.NewRunner(subfinderOpts) if err != nil { log.Fatalf("failed to create subfinder runner: %v", err) } output := &bytes.Buffer{} var sourceMap map[string]map[string]struct{} // To run subdomain enumeration on a single domain if sourceMap, err = subfinder.EnumerateSingleDomainWithCtx(context.Background(), "hackerone.com", []io.Writer{output}); err != nil { log.Fatalf("failed to enumerate single domain: %v", err) } // To run subdomain enumeration on a list of domains from file/reader // file, err := os.Open("domains.txt") // if err != nil { // log.Fatalf("failed to open domains file: %v", err) // } // defer file.Close() // if err = subfinder.EnumerateMultipleDomainsWithCtx(context.Background(), file, []io.Writer{output}); err != nil { // log.Fatalf("failed to enumerate subdomains from file: %v", err) // } // print the output log.Println(output.String()) // Or use sourceMap to access the results in your application for subdomain, sources := range sourceMap { sourcesList := make([]string, 0, len(sources)) for source := range sources { sourcesList = append(sourcesList, source) } log.Printf("%s %s (%d)\n", subdomain, sourcesList, len(sources)) } } ================================================ FILE: go.mod ================================================ module github.com/projectdiscovery/subfinder/v2 go 1.24.0 toolchain go1.24.1 require ( github.com/corpix/uarand v0.2.0 github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd github.com/json-iterator/go v1.1.12 github.com/lib/pq v1.10.9 github.com/projectdiscovery/chaos-client v0.5.2 github.com/projectdiscovery/dnsx v1.2.3 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.62 github.com/projectdiscovery/ratelimit v0.0.82 github.com/projectdiscovery/retryablehttp-go v1.1.0 github.com/projectdiscovery/utils v0.7.3 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.11.1 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 gopkg.in/yaml.v3 v3.0.1 ) require ( aead.dev/minisign v0.2.0 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/STARRY-S/zip v0.2.1 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/sevenzip v1.6.0 // indirect github.com/bodgit/windows v1.0.1 // indirect github.com/charmbracelet/glamour v0.8.0 // indirect github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/fatih/color v1.15.0 // indirect github.com/gaissmai/bart v0.26.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mholt/archives v0.1.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect github.com/nwaples/rardecode/v2 v2.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/cdncheck v1.2.13 // indirect github.com/projectdiscovery/fastdialer v0.4.19 // indirect github.com/projectdiscovery/hmap v0.0.98 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/projectdiscovery/networkpolicy v0.1.31 // indirect github.com/refraction-networking/utls v1.7.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.3.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/ulikunitz/xz v0.5.15 // indirect github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39 // indirect github.com/yuin/goldmark v1.7.4 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.45.0 // indirect golang.org/x/mod v0.29.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sync v0.18.0 // indirect golang.org/x/term v0.37.0 // indirect golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.38.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect ) require ( github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/miekg/dns v1.1.62 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/projectdiscovery/goflags v0.1.74 github.com/projectdiscovery/retryabledns v1.0.111 // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/sys v0.38.0 // indirect ) ================================================ FILE: go.sum ================================================ aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE= github.com/alecthomas/assert/v2 v2.7.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46aU4V9E= github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/charmbracelet/glamour v0.8.0 h1:tPrjL3aRcQbn++7t18wOpgLyl8wrOHUEDS7IZ68QtZs= github.com/charmbracelet/glamour v0.8.0/go.mod h1:ViRgmKkf3u5S7uakt2czJ272WSg2ZenlYEZXT2x7Bjw= github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= github.com/charmbracelet/lipgloss v0.13.0/go.mod h1:nw4zy0SBX/F/eAO1cWdcvy6qnkDUxr8Lw7dvFrAIbbY= github.com/charmbracelet/x/ansi v0.3.2 h1:wsEwgAN+C9U06l9dCVMX0/L3x7ptvY1qmjMwyfE6USY= github.com/charmbracelet/x/ansi v0.3.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a h1:G99klV19u0QnhiizODirwVksQB91TJKV/UaTnACcG30= github.com/charmbracelet/x/exp/golden v0.0.0-20240806155701-69247e0abc2a/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gaissmai/bart v0.26.0 h1:xOZ57E9hJLBiQaSyeZa9wgWhGuzfGACgqp4BE77OkO0= github.com/gaissmai/bart v0.26.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd h1:FsX+T6wA8spPe4c1K9vi7T0LvNCO1TTqiL8u7Wok2hw= github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd/go.mod h1:VzxiSdG6j1pi7rwGm/xYI5RbtpBgM8sARDXlvEvxlu0= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mholt/archives v0.1.0 h1:FacgJyrjiuyomTuNA92X5GyRBRZjE43Y/lrzKIlF35Q= github.com/mholt/archives v0.1.0/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg= github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ= github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A= github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.2.13 h1:6zs4Mn8JV3yKyMoAr857Hf2NLvyOMpOfqCCT2V2OI1Q= github.com/projectdiscovery/cdncheck v1.2.13/go.mod h1:/OhuZ9T25yXSqU6+oWvmVQ3QFvtew/Tp03u0jM+NJBE= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= github.com/projectdiscovery/dnsx v1.2.3 h1:S87U9kYuuqqvMFyen8mZQy1FMuR5EGCsXHqfHPQAeuc= github.com/projectdiscovery/dnsx v1.2.3/go.mod h1:NjAEyJt6+meNqZqnYHL4ZPxXfysuva+et56Eq/e1cVE= github.com/projectdiscovery/fastdialer v0.4.19 h1:MLHwEGM0x0pyltJaNvAVvwc27bnXdZ5mYr50S/2kMEE= github.com/projectdiscovery/fastdialer v0.4.19/go.mod h1:HGdVsez+JgJ9/ljXjHRplOqkB7og+nqi0nrNWVNi03o= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= github.com/projectdiscovery/gologger v1.1.62 h1:wzKqvL6HQRzf0/PpBEhInZqqL1q4mKe2gFGJeDG3FqE= github.com/projectdiscovery/gologger v1.1.62/go.mod h1:YWvMSxlHybU3SkFCcWn+driSJ8yY+3CR3g/textnp+Y= github.com/projectdiscovery/hmap v0.0.98 h1:XxYIi7yJCNiDAKCJXvuY9IBM5O6OgDgx4XHgKxkR4eg= github.com/projectdiscovery/hmap v0.0.98/go.mod h1:bgN5fuZPJMj2YnAGEEnCypoifCnALJixHEVQszktQIU= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/networkpolicy v0.1.31 h1:mE6iJeYOSql8gps/91vwiztE/kEHe5Im8oUO5Mkj9Zg= github.com/projectdiscovery/networkpolicy v0.1.31/go.mod h1:5x4rGh4XhnoYl9wACnZyrjDGKIB/bQqxw2KrIM5V+XU= github.com/projectdiscovery/ratelimit v0.0.82 h1:rtO5SQf5uQFu5zTahTaTcO06OxmG8EIF1qhdFPIyTak= github.com/projectdiscovery/ratelimit v0.0.82/go.mod h1:z076BrLkBb5yS7uhHNoCTf8X/BvFSGRxwQ8EzEL9afM= github.com/projectdiscovery/retryabledns v1.0.111 h1:iyMdCDgNmaSRJYcGqB+SLlvlw9WijlbJ6Q9OEpRAWsQ= github.com/projectdiscovery/retryabledns v1.0.111/go.mod h1:6TOPJ3QAE4reBu6bvsGsTcyEb+OypcKYFQH7yVsjyIM= github.com/projectdiscovery/retryablehttp-go v1.1.0 h1:uYp3EnuhhamTwvG41X6q6TAc/SHEO9pw9CBWbRASIQk= github.com/projectdiscovery/retryablehttp-go v1.1.0/go.mod h1:9DU57ezv5cfZSWw/m5XFDTMjy1yKeMyn1kj35lPlcfM= github.com/projectdiscovery/utils v0.7.3 h1:kX+77AA58yK6EZgkTRJEnK9V/7AZYzlXdcu/o/kJhFs= github.com/projectdiscovery/utils v0.7.3/go.mod h1:uDdQ3/VWomai98l+a3Ye/srDXdJ4xUIar/mSXlQ9gBM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.7.1 h1:dxg+jla3uocgN8HtX+ccwDr68uCBBO3qLrkZUbqkcw0= github.com/refraction-networking/utls v1.7.1/go.mod h1:TUhh27RHMGtQvjQq+RyO11P6ZNQNBb3N0v7wsEjKAIQ= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39 h1:Bz/zVM/LoGZ9IztGBHrq2zlFQQbEG8dBYnxb4hamIHM= github.com/weppos/publicsuffix-go v0.40.3-0.20250408071509-6074bbe7fd39/go.mod h1:2oFzEwGYI7lhiqG0YkkcKa6VcpjVinQbWxaPzytDmLA= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg= github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= github.com/yuin/goldmark-emoji v1.0.3 h1:aLRkLHOuBR2czCY4R8olwMjID+tENfhyFDMCRhbIQY4= github.com/yuin/goldmark-emoji v1.0.3/go.mod h1:tTkZEbwu5wkPmgTcitqddVxY9osFZiavD+r4AzQrh1U= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= github.com/zcalusic/sysinfo v1.0.2/go.mod h1:kluzTYflRWo6/tXVMJPdEjShsbPpsFRyy+p1mBQPC30= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk= github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 h1:YOQ1vXEwE4Rnj+uQ/3oCuJk5wgVsvUyW+glsndwYuyA= github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968/go.mod h1:xIuOvYCZX21S5Z9bK1BMrertTGX/F8hgAPw7ERJRNS0= github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= ================================================ FILE: pkg/passive/doc.go ================================================ // Package passive provides capability for doing passive subdomain // enumeration on targets. package passive ================================================ FILE: pkg/passive/passive.go ================================================ package passive import ( "context" "fmt" "math" "sort" "strings" "sync" "time" "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type EnumerationOptions struct { customRateLimiter *subscraping.CustomRateLimit } type EnumerateOption func(opts *EnumerationOptions) func WithCustomRateLimit(crl *subscraping.CustomRateLimit) EnumerateOption { return func(opts *EnumerationOptions) { opts.customRateLimiter = crl } } // EnumerateSubdomains wraps EnumerateSubdomainsWithCtx with an empty context func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration, options ...EnumerateOption) chan subscraping.Result { return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimit, timeout, maxEnumTime, options...) } // EnumerateSubdomainsWithCtx enumerates all the subdomains for a given domain func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration, options ...EnumerateOption) chan subscraping.Result { results := make(chan subscraping.Result) go func() { defer close(results) var enumerateOptions EnumerationOptions for _, enumerateOption := range options { enumerateOption(&enumerateOptions) } multiRateLimiter, err := a.buildMultiRateLimiter(ctx, rateLimit, enumerateOptions.customRateLimiter) if err != nil { results <- subscraping.Result{ Type: subscraping.Error, Error: fmt.Errorf("could not init multi rate limiter for %s: %s", domain, err), } return } session, err := subscraping.NewSession(domain, proxy, multiRateLimiter, timeout) if err != nil { results <- subscraping.Result{ Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err), } return } defer session.Close() ctx, cancel := context.WithTimeout(ctx, maxEnumTime) wg := &sync.WaitGroup{} // Run each source in parallel on the target domain for _, runner := range a.sources { wg.Add(1) go func(source subscraping.Source) { defer wg.Done() ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, source.Name()) for resp := range source.Run(ctxWithValue, domain, session) { select { case <-ctx.Done(): return case results <- resp: } } }(runner) } wg.Wait() cancel() }() return results } func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int, rateLimit *subscraping.CustomRateLimit) (*ratelimit.MultiLimiter, error) { var multiRateLimiter *ratelimit.MultiLimiter var err error for _, source := range a.sources { var rl uint if sourceRateLimit, ok := rateLimit.Custom.Get(strings.ToLower(source.Name())); ok { rl = sourceRateLimitOrDefault(uint(globalRateLimit), sourceRateLimit) } if rl > 0 { multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), rl, time.Second) } else { multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), math.MaxUint32, time.Millisecond) } if err != nil { break } } return multiRateLimiter, err } func sourceRateLimitOrDefault(defaultRateLimit uint, sourceRateLimit uint) uint { if sourceRateLimit > 0 { return sourceRateLimit } return defaultRateLimit } func addRateLimiter(ctx context.Context, multiRateLimiter *ratelimit.MultiLimiter, key string, maxCount uint, duration time.Duration) (*ratelimit.MultiLimiter, error) { if multiRateLimiter == nil { mrl, err := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{ Key: key, IsUnlimited: maxCount == math.MaxUint32, MaxCount: maxCount, Duration: duration, }) return mrl, err } err := multiRateLimiter.Add(&ratelimit.Options{ Key: key, IsUnlimited: maxCount == math.MaxUint32, MaxCount: maxCount, Duration: duration, }) return multiRateLimiter, err } func (a *Agent) GetStatistics() map[string]subscraping.Statistics { stats := make(map[string]subscraping.Statistics) sort.Slice(a.sources, func(i, j int) bool { return a.sources[i].Name() > a.sources[j].Name() }) for _, source := range a.sources { stats[source.Name()] = source.Statistics() } return stats } ================================================ FILE: pkg/passive/sources.go ================================================ package passive import ( "fmt" "os" "strings" "golang.org/x/exp/maps" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/alienvault" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/anubis" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bevigil" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/builtwith" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/facebook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hudsonrock" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/merklemap" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onyphe" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/profundis" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/reconeer" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rsecloud" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/thc" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/urlscan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/windvane" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" mapsutil "github.com/projectdiscovery/utils/maps" ) var AllSources = [...]subscraping.Source{ &alienvault.Source{}, &anubis.Source{}, &bevigil.Source{}, &bufferover.Source{}, &builtwith.Source{}, &c99.Source{}, &censys.Source{}, &certspotter.Source{}, &chaos.Source{}, &chinaz.Source{}, &commoncrawl.Source{}, &crtsh.Source{}, &digitalyama.Source{}, &digitorus.Source{}, &dnsdb.Source{}, &dnsdumpster.Source{}, &dnsrepo.Source{}, &domainsproject.Source{}, &driftnet.Source{}, &facebook.Source{}, &fofa.Source{}, &fullhunt.Source{}, &github.Source{}, &hackertarget.Source{}, &hudsonrock.Source{}, &intelx.Source{}, &leakix.Source{}, &merklemap.Source{}, &netlas.Source{}, &onyphe.Source{}, &profundis.Source{}, &pugrecon.Source{}, &quake.Source{}, &rapiddns.Source{}, // &reconcloud.Source{}, // failing due to cloudflare bot protection &reconeer.Source{}, &redhuntlabs.Source{}, // &riddler.Source{}, // failing due to cloudfront protection &robtex.Source{}, &rsecloud.Source{}, &securitytrails.Source{}, &shodan.Source{}, &sitedossier.Source{}, &thc.Source{}, &threatbook.Source{}, &threatcrowd.Source{}, // &threatminer.Source{}, // failing api &urlscan.Source{}, &virustotal.Source{}, &waybackarchive.Source{}, &whoisxmlapi.Source{}, &windvane.Source{}, &zoomeyeapi.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( mapsutil.WithMap(mapsutil.Map[string, string]{})) var NameSourceMap = make(map[string]subscraping.Source, len(AllSources)) func init() { for _, currentSource := range AllSources { NameSourceMap[strings.ToLower(currentSource.Name())] = currentSource } } // Agent is a struct for running passive subdomain enumeration // against a given host. It wraps subscraping package and provides // a layer to build upon. type Agent struct { sources []subscraping.Source } // New creates a new agent for passive subdomain discovery func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSupportingRecurse bool) *Agent { sources := make(map[string]subscraping.Source, len(AllSources)) if useAllSources { maps.Copy(sources, NameSourceMap) } else { if len(sourceNames) > 0 { for _, source := range sourceNames { if NameSourceMap[source] == nil { gologger.Warning().Msgf("There is no source with the name: %s", source) } else { sources[source] = NameSourceMap[source] } } } else { for _, currentSource := range AllSources { if currentSource.IsDefault() { sources[currentSource.Name()] = currentSource } } } } if len(excludedSourceNames) > 0 { for _, sourceName := range excludedSourceNames { delete(sources, sourceName) } } if useSourcesSupportingRecurse { for sourceName, source := range sources { if !source.HasRecursiveSupport() { delete(sources, sourceName) } } } if len(sources) == 0 { gologger.Fatal().Msg("No sources selected for this search") } gologger.Debug().Msgf("Selected source(s) for this search: %s", strings.Join(maps.Keys(sources), ", ")) for _, currentSource := range sources { if warning, ok := sourceWarnings.Get(strings.ToLower(currentSource.Name())); ok { gologger.Warning().Msg(warning) } } for _, source := range sources { keyReq := source.KeyRequirement() if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey { if apiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))); apiKey != "" { source.AddApiKeys([]string{apiKey}) } } } // Create the agent, insert the sources and remove the excluded sources agent := &Agent{sources: maps.Values(sources)} return agent } ================================================ FILE: pkg/passive/sources_test.go ================================================ package passive import ( "fmt" "strconv" "testing" "github.com/stretchr/testify/assert" "golang.org/x/exp/maps" ) var ( expectedAllSources = []string{ "alienvault", "anubis", "bevigil", "bufferover", "c99", "censys", "certspotter", "chaos", "chinaz", "commoncrawl", "crtsh", "digitorus", "dnsdumpster", "dnsdb", "dnsrepo", "domainsproject", "driftnet", "fofa", "fullhunt", "github", "hackertarget", "intelx", "netlas", "onyphe", "quake", "pugrecon", "rapiddns", "redhuntlabs", // "riddler", // failing due to cloudfront protection "robtex", "rsecloud", "securitytrails", "profundis", "shodan", "sitedossier", "threatbook", "threatcrowd", "virustotal", "waybackarchive", "whoisxmlapi", "windvane", "zoomeyeapi", "leakix", "facebook", // "threatminer", // "reconcloud", "reconeer", "builtwith", "hudsonrock", "digitalyama", "merklemap", "thc", "urlscan", } expectedDefaultSources = []string{ "alienvault", "anubis", "bevigil", "bufferover", "c99", "certspotter", "censys", "chaos", "chinaz", "crtsh", "digitorus", "dnsdumpster", "domainsproject", "dnsrepo", "driftnet", "fofa", "fullhunt", "hackertarget", "intelx", "onyphe", "quake", "redhuntlabs", "robtex", // "riddler", // failing due to cloudfront protection "rsecloud", "securitytrails", "profundis", "shodan", "windvane", "virustotal", "whoisxmlapi", "leakix", "facebook", // "threatminer", // "reconcloud", "reconeer", "builtwith", "digitalyama", "thc", "urlscan", } expectedDefaultRecursiveSources = []string{ "alienvault", "bufferover", "certspotter", "crtsh", "dnsdb", "digitorus", "driftnet", "hackertarget", "securitytrails", "virustotal", "leakix", "facebook", "merklemap", "urlscan", // "reconcloud", } ) func TestSourceCategorization(t *testing.T) { defaultSources := make([]string, 0, len(AllSources)) recursiveSources := make([]string, 0, len(AllSources)) for _, source := range AllSources { sourceName := source.Name() if source.IsDefault() { defaultSources = append(defaultSources, sourceName) } if source.HasRecursiveSupport() { recursiveSources = append(recursiveSources, sourceName) } } assert.ElementsMatch(t, expectedDefaultSources, defaultSources) assert.ElementsMatch(t, expectedDefaultRecursiveSources, recursiveSources) assert.ElementsMatch(t, expectedAllSources, maps.Keys(NameSourceMap)) } // Review: not sure if this test is necessary/useful // implementation is straightforward where sources are stored in maps and filtered based on options // the test is just checking if the filtering works as expected using count of sources func TestSourceFiltering(t *testing.T) { someSources := []string{ "alienvault", "chaos", "crtsh", "virustotal", } someExclusions := []string{ "alienvault", "virustotal", } tests := []struct { sources []string exclusions []string withAllSources bool withRecursion bool expectedLength int }{ {someSources, someExclusions, false, false, len(someSources) - len(someExclusions)}, {someSources, someExclusions, false, true, 1}, {someSources, someExclusions, true, false, len(AllSources) - len(someExclusions)}, {someSources, []string{}, false, false, len(someSources)}, {someSources, []string{}, true, false, len(AllSources)}, {[]string{}, []string{}, false, false, len(expectedDefaultSources)}, {[]string{}, []string{}, true, false, len(AllSources)}, {[]string{}, []string{}, true, true, len(expectedDefaultRecursiveSources)}, } for index, test := range tests { t.Run(strconv.Itoa(index+1), func(t *testing.T) { agent := New(test.sources, test.exclusions, test.withAllSources, test.withRecursion) for _, v := range agent.sources { fmt.Println(v.Name()) } assert.Equal(t, test.expectedLength, len(agent.sources)) agent = nil }) } } ================================================ FILE: pkg/passive/sources_w_auth_test.go ================================================ package passive import ( "context" "fmt" "math" "os" "reflect" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) func TestSourcesWithKeys(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } domain := "hackerone.com" timeout := 60 gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) ctxParent := context.Background() var multiRateLimiter *ratelimit.MultiLimiter for _, source := range AllSources { if !source.NeedsKey() { continue } multiRateLimiter, _ = addRateLimiter(ctxParent, multiRateLimiter, source.Name(), math.MaxInt32, time.Millisecond) } session, err := subscraping.NewSession(domain, "", multiRateLimiter, timeout) assert.Nil(t, err) var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil} for _, source := range AllSources { if !source.NeedsKey() { continue } var apiKey string if source.Name() == "chaos" { apiKey = os.Getenv("PDCP_API_KEY") } else { apiKey = os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))) } if apiKey == "" { fmt.Printf("Skipping %s as no API key is provided\n", source.Name()) continue } source.AddApiKeys([]string{apiKey}) t.Run(source.Name(), func(t *testing.T) { var results []subscraping.Result ctxWithValue := context.WithValue(ctxParent, subscraping.CtxSourceArg, source.Name()) for result := range source.Run(ctxWithValue, domain, session) { results = append(results, result) assert.Equal(t, source.Name(), result.Source, "wrong source name") if result.Type != subscraping.Error { assert.True(t, strings.HasSuffix(strings.ToLower(result.Value), strings.ToLower(expected.Value)), fmt.Sprintf("result(%s) is not subdomain of %s", strings.ToLower(result.Value), expected.Value)) } else { assert.Equal(t, reflect.TypeOf(expected.Error), reflect.TypeOf(result.Error), fmt.Sprintf("%s: %s", result.Source, result.Error)) } } assert.GreaterOrEqual(t, len(results), 1, fmt.Sprintf("No result found for %s", source.Name())) }) } } ================================================ FILE: pkg/passive/sources_wo_auth_test.go ================================================ package passive import ( "context" "fmt" "math" "reflect" "strings" "testing" "time" "github.com/stretchr/testify/assert" "golang.org/x/exp/slices" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) func TestSourcesWithoutKeys(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } ignoredSources := []string{ "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ "riddler", // failing due to cloudfront protection "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally "waybackarchive", // Fails randomly "alienvault", // 503 Service Temporarily Unavailable "digitorus", // failing with "Failed to retrieve certificate" "dnsdumpster", // failing with "unexpected status code 403 received" "anubis", // failing with "too many redirects" "threatcrowd", // failing with "randomly failing with unmarshal error when hit multiple times" "leakix", // now requires API key (returns 401) "reconeer", // now requires API key (returns 401) "sitedossier", // flaky - returns no results in CI } domain := "hackerone.com" timeout := 60 gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) ctxParent := context.Background() var multiRateLimiter *ratelimit.MultiLimiter for _, source := range AllSources { if source.NeedsKey() || slices.Contains(ignoredSources, source.Name()) { continue } multiRateLimiter, _ = addRateLimiter(ctxParent, multiRateLimiter, source.Name(), math.MaxInt32, time.Millisecond) } session, err := subscraping.NewSession(domain, "", multiRateLimiter, timeout) assert.Nil(t, err) var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil} for _, source := range AllSources { if source.NeedsKey() || slices.Contains(ignoredSources, source.Name()) { continue } t.Run(source.Name(), func(t *testing.T) { var results []subscraping.Result ctxWithValue := context.WithValue(ctxParent, subscraping.CtxSourceArg, source.Name()) for result := range source.Run(ctxWithValue, domain, session) { results = append(results, result) assert.Equal(t, source.Name(), result.Source, "wrong source name") if result.Type != subscraping.Error { assert.True(t, strings.HasSuffix(strings.ToLower(result.Value), strings.ToLower(expected.Value)), fmt.Sprintf("result(%s) is not subdomain of %s", strings.ToLower(result.Value), expected.Value)) } else { assert.Equal(t, reflect.TypeOf(expected.Error), reflect.TypeOf(result.Error), fmt.Sprintf("%s: %s", result.Source, result.Error)) } } assert.GreaterOrEqual(t, len(results), 1, fmt.Sprintf("No result found for %s", source.Name())) }) } } ================================================ FILE: pkg/resolve/client.go ================================================ package resolve import ( "github.com/projectdiscovery/dnsx/libs/dnsx" ) // DefaultResolvers contains the default list of resolvers known to be good var DefaultResolvers = []string{ "1.1.1.1:53", // Cloudflare primary "1.0.0.1:53", // Cloudflare secondary "8.8.8.8:53", // Google primary "8.8.4.4:53", // Google secondary "9.9.9.9:53", // Quad9 Primary "9.9.9.10:53", // Quad9 Secondary "77.88.8.8:53", // Yandex Primary "77.88.8.1:53", // Yandex Secondary "208.67.222.222:53", // OpenDNS Primary "208.67.220.220:53", // OpenDNS Secondary } // Resolver is a struct for resolving DNS names type Resolver struct { DNSClient *dnsx.DNSX Resolvers []string } // New creates a new resolver struct with the default resolvers func New() *Resolver { return &Resolver{ Resolvers: []string{}, } } ================================================ FILE: pkg/resolve/doc.go ================================================ // Package resolve is used to handle resolving records // It also handles wildcard subdomains and rotating resolvers. package resolve ================================================ FILE: pkg/resolve/resolve.go ================================================ package resolve import ( "fmt" "sync" "github.com/rs/xid" ) const ( maxWildcardChecks = 3 ) // ResolutionPool is a pool of resolvers created for resolving subdomains // for a given host. type ResolutionPool struct { *Resolver Tasks chan HostEntry Results chan Result wg *sync.WaitGroup removeWildcard bool wildcardIPs map[string]struct{} } // HostEntry defines a host with the source type HostEntry struct { Domain string Host string Source string WildcardCertificate bool } // Result contains the result for a host resolution type Result struct { Type ResultType Host string IP string Error error Source string WildcardCertificate bool } // ResultType is the type of result found type ResultType int // Types of data result can return const ( Subdomain ResultType = iota Error ) // NewResolutionPool creates a pool of resolvers for resolving subdomains of a given domain func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *ResolutionPool { resolutionPool := &ResolutionPool{ Resolver: r, Tasks: make(chan HostEntry), Results: make(chan Result), wg: &sync.WaitGroup{}, removeWildcard: removeWildcard, wildcardIPs: make(map[string]struct{}), } go func() { for range workers { resolutionPool.wg.Add(1) go resolutionPool.resolveWorker() } resolutionPool.wg.Wait() close(resolutionPool.Results) }() return resolutionPool } // InitWildcards inits the wildcard ips array func (r *ResolutionPool) InitWildcards(domain string) error { for range maxWildcardChecks { uid := xid.New().String() hosts, _ := r.DNSClient.Lookup(uid + "." + domain) if len(hosts) == 0 { return fmt.Errorf("%s is not a wildcard domain", domain) } // Append all wildcard ips found for domains for _, host := range hosts { r.wildcardIPs[host] = struct{}{} } } return nil } func (r *ResolutionPool) resolveWorker() { for task := range r.Tasks { if !r.removeWildcard { r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source, WildcardCertificate: task.WildcardCertificate} continue } hosts, err := r.DNSClient.Lookup(task.Host) if err != nil { r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err, WildcardCertificate: task.WildcardCertificate} continue } if len(hosts) == 0 { continue } var skip bool for _, host := range hosts { // Ignore the host if it exists in wildcard ips map if _, ok := r.wildcardIPs[host]; ok { skip = true break } } if !skip { r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source, WildcardCertificate: task.WildcardCertificate} } } r.wg.Done() } ================================================ FILE: pkg/runner/banners.go ================================================ package runner import ( "github.com/projectdiscovery/gologger" updateutils "github.com/projectdiscovery/utils/update" ) const banner = ` __ _____ __ _______ __/ /_ / __(_)___ ____/ /__ _____ / ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/ (__ ) /_/ / /_/ / __/ / / / / /_/ / __/ / /____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ ` // Name const ToolName = `subfinder` // Version is the current version of subfinder const version = `v2.13.0` // showBanner is used to show the banner to the user func showBanner() { gologger.Print().Msgf("%s\n", banner) gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") } // GetUpdateCallback returns a callback function that updates subfinder func GetUpdateCallback() func() { return func() { showBanner() updateutils.GetUpdateToolCallback("subfinder", version)() } } ================================================ FILE: pkg/runner/config.go ================================================ package runner import ( "os" "strings" "gopkg.in/yaml.v3" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" fileutil "github.com/projectdiscovery/utils/file" ) // createProviderConfigYAML marshals the input map to the given location on the disk func createProviderConfigYAML(configFilePath string) error { configFile, err := os.Create(configFilePath) if err != nil { return err } defer func() { if err := configFile.Close(); err != nil { gologger.Error().Msgf("Error closing config file: %s", err) } }() sourcesRequiringApiKeysMap := make(map[string][]string) for _, source := range passive.AllSources { keyReq := source.KeyRequirement() if keyReq == subscraping.RequiredKey || keyReq == subscraping.OptionalKey { sourceName := strings.ToLower(source.Name()) sourcesRequiringApiKeysMap[sourceName] = []string{} } } return yaml.NewEncoder(configFile).Encode(sourcesRequiringApiKeysMap) } // UnmarshalFrom writes the marshaled yaml config to disk func UnmarshalFrom(file string) error { reader, err := fileutil.SubstituteConfigFromEnvVars(file) if err != nil { return err } sourceApiKeysMap := map[string][]string{} err = yaml.NewDecoder(reader).Decode(sourceApiKeysMap) for _, source := range passive.AllSources { sourceName := strings.ToLower(source.Name()) apiKeys := sourceApiKeysMap[sourceName] if len(apiKeys) > 0 { gologger.Debug().Msgf("API key(s) found for %s.", sourceName) source.AddApiKeys(apiKeys) } } return err } ================================================ FILE: pkg/runner/doc.go ================================================ // Package runner implements the mechanism to drive the // subdomain enumeration process package runner ================================================ FILE: pkg/runner/enumerate.go ================================================ package runner import ( "context" "io" "strings" "sync" "time" "github.com/hako/durafmt" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) const maxNumCount = 2 var replacer = strings.NewReplacer( "/", "", "•.", "", "•", "", "*.", "", "http://", "", "https://", "", ) // EnumerateSingleDomain wraps EnumerateSingleDomainWithCtx with an empty context func (r *Runner) EnumerateSingleDomain(domain string, writers []io.Writer) (map[string]map[string]struct{}, error) { return r.EnumerateSingleDomainWithCtx(context.Background(), domain, writers) } // EnumerateSingleDomainWithCtx performs subdomain enumeration against a single domain func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string, writers []io.Writer) (map[string]map[string]struct{}, error) { gologger.Info().Msgf("Enumerating subdomains for %s\n", domain) // Check if the user has asked to remove wildcards explicitly. // If yes, create the resolution pool and get the wildcards for the current domain var resolutionPool *resolve.ResolutionPool if r.options.RemoveWildcard { resolutionPool = r.resolverClient.NewResolutionPool(r.options.Threads, r.options.RemoveWildcard) err := resolutionPool.InitWildcards(domain) if err != nil { // Log the error but don't quit. gologger.Warning().Msgf("Could not get wildcards for domain %s: %s\n", domain, err) } } // Run the passive subdomain enumeration now := time.Now() passiveResults := r.passiveAgent.EnumerateSubdomainsWithCtx(ctx, domain, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute, passive.WithCustomRateLimit(r.rateLimit)) wg := &sync.WaitGroup{} wg.Add(1) // Create a unique map for filtering duplicate subdomains out uniqueMap := make(map[string]resolve.HostEntry) // Create a map to track sources for each host sourceMap := make(map[string]map[string]struct{}) skippedCounts := make(map[string]int) // Process the results in a separate goroutine go func() { for result := range passiveResults { switch result.Type { case subscraping.Error: gologger.Warning().Msgf("Encountered an error with source %s: %s\n", result.Source, result.Error) case subscraping.Subdomain: subdomain := replacer.Replace(result.Value) // check if this subdomain is actually a wildcard subdomain // that may have furthur subdomains associated with it isWildcard := strings.Contains(result.Value, "*."+subdomain) // Validate the subdomain found and remove wildcards from if !strings.HasSuffix(subdomain, "."+domain) { skippedCounts[result.Source]++ continue } if matchSubdomain := r.filterAndMatchSubdomain(subdomain); matchSubdomain { if _, ok := uniqueMap[subdomain]; !ok { sourceMap[subdomain] = make(map[string]struct{}) } // Log the verbose message about the found subdomain per source if _, ok := sourceMap[subdomain][result.Source]; !ok { gologger.Verbose().Label(result.Source).Msg(subdomain) } sourceMap[subdomain][result.Source] = struct{}{} // Check if the subdomain is a duplicate. If not, // send the subdomain for resolution. if _, ok := uniqueMap[subdomain]; ok { skippedCounts[result.Source]++ // even if it is duplicate if it was not marked as wildcard before but this source says it is wildcard // then we should mark it as wildcard if !uniqueMap[subdomain].WildcardCertificate && isWildcard { val := uniqueMap[subdomain] val.WildcardCertificate = true uniqueMap[subdomain] = val } continue } hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source, WildcardCertificate: isWildcard} if r.options.ResultCallback != nil && !r.options.RemoveWildcard { r.options.ResultCallback(&hostEntry) } uniqueMap[subdomain] = hostEntry // If the user asked to remove wildcard then send on the resolve // queue. Otherwise, if mode is not verbose print the results on // the screen as they are discovered. if r.options.RemoveWildcard { resolutionPool.Tasks <- hostEntry } } } } // Close the task channel only if wildcards are asked to be removed if r.options.RemoveWildcard { close(resolutionPool.Tasks) } wg.Done() }() // If the user asked to remove wildcards, listen from the results // queue and write to the map. At the end, print the found results to the screen foundResults := make(map[string]resolve.Result) if r.options.RemoveWildcard { // Process the results coming from the resolutions pool for result := range resolutionPool.Results { switch result.Type { case resolve.Error: gologger.Warning().Msgf("Could not resolve host: %s\n", result.Error) case resolve.Subdomain: // Add the found subdomain to a map. if _, ok := foundResults[result.Host]; !ok { foundResults[result.Host] = result if r.options.ResultCallback != nil { r.options.ResultCallback(&resolve.HostEntry{Domain: domain, Host: result.Host, Source: result.Source, WildcardCertificate: result.WildcardCertificate}) } } } } // Merge wildcard certificate information from uniqueMap into foundResults // This handles cases where a later source marked a subdomain as wildcard // after it was already sent to the resolution pool for host, result := range foundResults { if entry, ok := uniqueMap[host]; ok && entry.WildcardCertificate && !result.WildcardCertificate { result.WildcardCertificate = true foundResults[host] = result } } } wg.Wait() outputWriter := NewOutputWriter(r.options.JSON) // Now output all results in output writers var err error for _, writer := range writers { if r.options.HostIP { err = outputWriter.WriteHostIP(domain, foundResults, writer) } else { if r.options.RemoveWildcard { err = outputWriter.WriteHostNoWildcard(domain, foundResults, writer) } else { if r.options.CaptureSources { err = outputWriter.WriteSourceHost(domain, sourceMap, writer) } else { err = outputWriter.WriteHost(domain, uniqueMap, writer) } } } if err != nil { gologger.Error().Msgf("Could not write results for %s: %s\n", domain, err) return nil, err } } // Show found subdomain count in any case. duration := durafmt.Parse(time.Since(now)).LimitFirstN(maxNumCount).String() var numberOfSubDomains int if r.options.RemoveWildcard { numberOfSubDomains = len(foundResults) } else { numberOfSubDomains = len(uniqueMap) } gologger.Info().Msgf("Found %d subdomains for %s in %s\n", numberOfSubDomains, domain, duration) if r.options.Statistics { gologger.Info().Msgf("Printing source statistics for %s", domain) statistics := r.passiveAgent.GetStatistics() // This is a hack to remove the skipped count from the statistics // as we don't want to show it in the statistics. // TODO: Design a better way to do this. for source, count := range skippedCounts { if stat, ok := statistics[source]; ok { stat.Results -= count statistics[source] = stat } } printStatistics(statistics) } return sourceMap, nil } func (r *Runner) filterAndMatchSubdomain(subdomain string) bool { if r.options.filterRegexes != nil { for _, filter := range r.options.filterRegexes { if m := filter.MatchString(subdomain); m { return false } } } if r.options.matchRegexes != nil { for _, match := range r.options.matchRegexes { if m := match.MatchString(subdomain); m { return true } } return false } return true } ================================================ FILE: pkg/runner/enumerate_test.go ================================================ package runner import ( "os" "testing" "github.com/stretchr/testify/require" ) func TestFilterAndMatchSubdomain(t *testing.T) { options := &Options{} options.Domain = []string{"example.com"} options.Threads = 10 options.Timeout = 10 options.Output = os.Stdout t.Run("Literal Match", func(t *testing.T) { options.Match = []string{"req.example.com"} err := options.validateOptions() if err != nil { t.Fatalf("Expected nil got %v while validation\n", err) } runner, err := NewRunner(options) if err != nil { t.Fatalf("Expected nil got %v while creating runner\n", err) } match := runner.filterAndMatchSubdomain("req.example.com") require.True(t, match, "Expecting a boolean True value ") }) t.Run("Multiple Wildcards Match", func(t *testing.T) { options.Match = []string{"*.ns.*.com"} err := options.validateOptions() if err != nil { t.Fatalf("Expected nil got %v while validation\n", err) } runner, err := NewRunner(options) if err != nil { t.Fatalf("Expected nil got %v while creating runner\n", err) } subdomain := []string{"a.ns.example.com", "b.ns.hackerone.com"} for _, sub := range subdomain { match := runner.filterAndMatchSubdomain(sub) require.True(t, match, "Expecting a boolean True value ") } }) t.Run("Sequential Match", func(t *testing.T) { options.Match = []string{"*.ns.example.com", "*.hackerone.com"} err := options.validateOptions() if err != nil { t.Fatalf("Expected nil got %v while validation\n", err) } runner, err := NewRunner(options) if err != nil { t.Fatalf("Expected nil got %v while creating runner\n", err) } subdomain := []string{"a.ns.example.com", "b.hackerone.com"} for _, sub := range subdomain { match := runner.filterAndMatchSubdomain(sub) require.True(t, match, "Expecting a boolean True value ") } }) t.Run("Literal Filter", func(t *testing.T) { options.Filter = []string{"req.example.com"} err := options.validateOptions() if err != nil { t.Fatalf("Expected nil got %v while validation\n", err) } runner, err := NewRunner(options) if err != nil { t.Fatalf("Expected nil got %v while creating runner\n", err) } match := runner.filterAndMatchSubdomain("req.example.com") require.False(t, match, "Expecting a boolean False value ") }) t.Run("Multiple Wildcards Filter", func(t *testing.T) { options.Filter = []string{"*.ns.*.com"} err := options.validateOptions() if err != nil { t.Fatalf("Expected nil got %v while validation\n", err) } runner, err := NewRunner(options) if err != nil { t.Fatalf("Expected nil got %v while creating runner\n", err) } subdomain := []string{"a.ns.example.com", "b.ns.hackerone.com"} for _, sub := range subdomain { match := runner.filterAndMatchSubdomain(sub) require.False(t, match, "Expecting a boolean False value ") } }) t.Run("Sequential Filter", func(t *testing.T) { options.Filter = []string{"*.ns.example.com", "*.hackerone.com"} err := options.validateOptions() if err != nil { t.Fatalf("Expected nil got %v while validation\n", err) } runner, err := NewRunner(options) if err != nil { t.Fatalf("Expected nil got %v while creating runner\n", err) } subdomain := []string{"a.ns.example.com", "b.hackerone.com"} for _, sub := range subdomain { match := runner.filterAndMatchSubdomain(sub) require.False(t, match, "Expecting a boolean False value ") } }) t.Run("Filter and Match", func(t *testing.T) { options.Filter = []string{"example.com"} options.Match = []string{"hackerone.com"} err := options.validateOptions() if err != nil { t.Fatalf("Expected nil got %v while validation\n", err) } runner, err := NewRunner(options) if err != nil { t.Fatalf("Expected nil got %v while creating runner\n", err) } subdomain := []string{"example.com", "example.com"} for _, sub := range subdomain { match := runner.filterAndMatchSubdomain(sub) require.False(t, match, "Expecting a boolean False value ") } }) t.Run("Filter and Match - Same Root Domain", func(t *testing.T) { options.Filter = []string{"example.com"} options.Match = []string{"www.example.com"} err := options.validateOptions() if err != nil { t.Fatalf("Expected nil got %v while validation\n", err) } runner, err := NewRunner(options) if err != nil { t.Fatalf("Expected nil got %v while creating runner\n", err) } subdomain := map[string]string{"filter": "example.com", "match": "www.example.com"} for key, sub := range subdomain { result := runner.filterAndMatchSubdomain(sub) if key == "filter" { require.False(t, result, "Expecting a boolean False value ") } else { require.True(t, result, "Expecting a boolean True value ") } } }) } ================================================ FILE: pkg/runner/initialize.go ================================================ package runner import ( "net" "strings" "github.com/projectdiscovery/dnsx/libs/dnsx" "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" ) // initializePassiveEngine creates the passive engine and loads sources etc func (r *Runner) initializePassiveEngine() { r.passiveAgent = passive.New(r.options.Sources, r.options.ExcludeSources, r.options.All, r.options.OnlyRecursive) } // initializeResolver creates the resolver used to resolve the found subdomains func (r *Runner) initializeResolver() error { var resolvers []string // If the file has been provided, read resolvers from the file if r.options.ResolverList != "" { var err error resolvers, err = loadFromFile(r.options.ResolverList) if err != nil { return err } } if len(r.options.Resolvers) > 0 { resolvers = append(resolvers, r.options.Resolvers...) } else { resolvers = append(resolvers, resolve.DefaultResolvers...) } // Add default 53 UDP port if missing for i, resolver := range resolvers { if !strings.Contains(resolver, ":") { resolvers[i] = net.JoinHostPort(resolver, "53") } } r.resolverClient = resolve.New() var err error r.resolverClient.DNSClient, err = dnsx.New(dnsx.Options{BaseResolvers: resolvers, MaxRetries: 5}) if err != nil { return nil } return nil } ================================================ FILE: pkg/runner/options.go ================================================ package runner import ( "errors" "fmt" "io" "math" "os" "path/filepath" "regexp" "strings" "github.com/projectdiscovery/chaos-client/pkg/chaos" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" envutil "github.com/projectdiscovery/utils/env" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" logutil "github.com/projectdiscovery/utils/log" updateutils "github.com/projectdiscovery/utils/update" ) var ( configDir = folderutil.AppConfigDirOrDefault(".", "subfinder") defaultConfigLocation = envutil.GetEnvOrDefault("SUBFINDER_CONFIG", filepath.Join(configDir, "config.yaml")) defaultProviderConfigLocation = envutil.GetEnvOrDefault("SUBFINDER_PROVIDER_CONFIG", filepath.Join(configDir, "provider-config.yaml")) ) // Options contains the configuration options for tuning // the subdomain enumeration process. type Options struct { Verbose bool // Verbose flag indicates whether to show verbose output or not NoColor bool // NoColor disables the colored output JSON bool // JSON specifies whether to use json for output format or text file HostIP bool // HostIP specifies whether to write subdomains in host:ip format Silent bool // Silent suppresses any extra text and only writes subdomains to screen ListSources bool // ListSources specifies whether to list all available sources RemoveWildcard bool // RemoveWildcard specifies whether to remove potential wildcard or dead subdomains from the results. CaptureSources bool // CaptureSources specifies whether to save all sources that returned a specific domains or just the first source Stdin bool // Stdin specifies whether stdin input was given to the process Version bool // Version specifies if we should just show version and exit OnlyRecursive bool // Recursive specifies whether to use only recursive subdomain enumeration sources All bool // All specifies whether to use all (slow) sources. Statistics bool // Statistics specifies whether to report source statistics Threads int // Threads controls the number of threads to use for active enumerations Timeout int // Timeout is the seconds to wait for sources to respond MaxEnumerationTime int // MaxEnumerationTime is the maximum amount of time in minutes to wait for enumeration Domain goflags.StringSlice // Domain is the domain to find subdomains for DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for Output io.Writer OutputFile string // Output is the file to write found subdomains to. OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given Sources goflags.StringSlice `yaml:"sources,omitempty"` // Sources contains a comma-separated list of sources to use for enumeration ExcludeSources goflags.StringSlice `yaml:"exclude-sources,omitempty"` // ExcludeSources contains the comma-separated sources to not include in the enumeration process Resolvers goflags.StringSlice `yaml:"resolvers,omitempty"` // Resolvers is the comma-separated resolvers to use for enumeration ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration Config string // Config contains the location of the config file ProviderConfig string // ProviderConfig contains the location of the provider config file Proxy string // HTTP proxy RateLimit int // Global maximum number of HTTP requests to send per second RateLimits goflags.RateLimitMap // Maximum number of HTTP requests to send per second ExcludeIps bool Match goflags.StringSlice Filter goflags.StringSlice matchRegexes []*regexp.Regexp filterRegexes []*regexp.Regexp ResultCallback OnResultCallback // OnResult callback DisableUpdateCheck bool // DisableUpdateCheck disable update checking } // OnResultCallback (hostResult) type OnResultCallback func(result *resolve.HostEntry) // ParseOptions parses the command line flags provided by a user func ParseOptions() *Options { logutil.DisableDefaultLogger() options := &Options{} var err error flagSet := goflags.NewFlagSet() flagSet.SetDescription(`Subfinder is a subdomain discovery tool that discovers subdomains for websites by using passive online sources.`) flagSet.CreateGroup("input", "Input", flagSet.StringSliceVarP(&options.Domain, "domain", "d", nil, "domains to find subdomains for", goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.DomainsFile, "list", "dL", "", "file containing list of domains for subdomain discovery"), ) flagSet.CreateGroup("source", "Source", flagSet.StringSliceVarP(&options.Sources, "sources", "s", nil, "specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources.", goflags.NormalizedStringSliceOptions), flagSet.BoolVar(&options.OnlyRecursive, "recursive", false, "use only sources that can handle subdomains recursively rather than both recursive and non-recursive sources"), flagSet.BoolVar(&options.All, "all", false, "use all sources for enumeration (slow)"), flagSet.StringSliceVarP(&options.ExcludeSources, "exclude-sources", "es", nil, "sources to exclude from enumeration (-es alienvault,zoomeyeapi)", goflags.NormalizedStringSliceOptions), ) flagSet.CreateGroup("filter", "Filter", flagSet.StringSliceVarP(&options.Match, "match", "m", nil, "subdomain or list of subdomain to match (file or comma separated)", goflags.FileNormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.Filter, "filter", "f", nil, " subdomain or list of subdomain to filter (file or comma separated)", goflags.FileNormalizedStringSliceOptions), ) flagSet.CreateGroup("rate-limit", "Rate-limit", flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 0, "maximum number of http requests to send per second (global)"), flagSet.RateLimitMapVarP(&options.RateLimits, "rate-limits", "rls", defaultRateLimits, "maximum number of http requests to send per second for providers in key=value format (-rls hackertarget=10/m)", goflags.NormalizedStringSliceOptions), flagSet.IntVar(&options.Threads, "t", 10, "number of concurrent goroutines for resolving (-active only)"), ) flagSet.CreateGroup("update", "Update", flagSet.CallbackVarP(GetUpdateCallback(), "update", "up", "update subfinder to latest version"), flagSet.BoolVarP(&options.DisableUpdateCheck, "disable-update-check", "duc", false, "disable automatic subfinder update check"), ) flagSet.CreateGroup("output", "Output", flagSet.StringVarP(&options.OutputFile, "output", "o", "", "file to write output to"), flagSet.BoolVarP(&options.JSON, "json", "oJ", false, "write output in JSONL(ines) format"), flagSet.StringVarP(&options.OutputDirectory, "output-dir", "oD", "", "directory to write output (-dL only)"), flagSet.BoolVarP(&options.CaptureSources, "collect-sources", "cs", false, "include all sources in the output (-json only)"), flagSet.BoolVarP(&options.HostIP, "ip", "oI", false, "include host IP in output (-active only)"), ) flagSet.CreateGroup("configuration", "Configuration", flagSet.StringVar(&options.Config, "config", defaultConfigLocation, "flag config file"), flagSet.StringVarP(&options.ProviderConfig, "provider-config", "pc", defaultProviderConfigLocation, "provider config file"), flagSet.StringSliceVar(&options.Resolvers, "r", nil, "comma separated list of resolvers to use", goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.ResolverList, "rlist", "rL", "", "file containing list of resolvers to use"), flagSet.BoolVarP(&options.RemoveWildcard, "active", "nW", false, "display active subdomains only"), flagSet.StringVar(&options.Proxy, "proxy", "", "http proxy to use with subfinder"), flagSet.BoolVarP(&options.ExcludeIps, "exclude-ip", "ei", false, "exclude IPs from the list of domains"), ) flagSet.CreateGroup("debug", "Debug", flagSet.BoolVar(&options.Silent, "silent", false, "show only subdomains in output"), flagSet.BoolVar(&options.Version, "version", false, "show version of subfinder"), flagSet.BoolVar(&options.Verbose, "v", false, "show verbose output"), flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable color in output"), flagSet.BoolVarP(&options.ListSources, "list-sources", "ls", false, "list all available sources"), flagSet.BoolVar(&options.Statistics, "stats", false, "report source statistics"), ) flagSet.CreateGroup("optimization", "Optimization", flagSet.IntVar(&options.Timeout, "timeout", 30, "seconds to wait before timing out"), flagSet.IntVar(&options.MaxEnumerationTime, "max-time", 10, "minutes to wait for enumeration results"), ) if err := flagSet.Parse(); err != nil { fmt.Println(err.Error()) os.Exit(1) } // set chaos mode chaos.IsSDK = false if exists := fileutil.FileExists(defaultProviderConfigLocation); !exists { if err := createProviderConfigYAML(defaultProviderConfigLocation); err != nil { gologger.Error().Msgf("Could not create provider config file: %s\n", err) } } if options.Config != defaultConfigLocation { // An empty source file is not a fatal error if err := flagSet.MergeConfigFile(options.Config); err != nil && !errors.Is(err, io.EOF) { gologger.Fatal().Msgf("Could not read config: %s\n", err) } } // Default output is stdout options.Output = os.Stdout // Check if stdin pipe was given options.Stdin = fileutil.HasStdin() if options.Version { gologger.Info().Msgf("Current Version: %s\n", version) gologger.Info().Msgf("Subfinder Config Directory: %s", configDir) os.Exit(0) } options.preProcessDomains() options.ConfigureOutput() showBanner() if !options.DisableUpdateCheck { latestVersion, err := updateutils.GetToolVersionCallback("subfinder", version)() if err != nil { if options.Verbose { gologger.Error().Msgf("subfinder version check failed: %v", err.Error()) } } else { gologger.Info().Msgf("Current subfinder version %v %v", version, updateutils.GetVersionDescription(version, latestVersion)) } } if options.ListSources { listSources(options) os.Exit(0) } // Validate the options passed by the user and if any // invalid options have been used, exit. err = options.validateOptions() if err != nil { gologger.Fatal().Msgf("Program exiting: %s\n", err) } return options } // loadProvidersFrom runs the app with source config func (options *Options) loadProvidersFrom(location string) { // todo: move elsewhere if len(options.Resolvers) == 0 { options.Resolvers = resolve.DefaultResolvers } // We skip bailing out if file doesn't exist because we'll create it // at the end of options parsing from default via goflags. if err := UnmarshalFrom(location); err != nil && (!strings.Contains(err.Error(), "file doesn't exist") || errors.Is(err, os.ErrNotExist)) { gologger.Error().Msgf("Could not read providers from %s: %s\n", location, err) } } func listSources(options *Options) { gologger.Info().Msgf("Current list of available sources. [%d]\n", len(passive.AllSources)) gologger.Info().Msgf("Sources marked with an * require key(s) or token(s) to work.\n") gologger.Info().Msgf("Sources marked with a ~ optionally support key(s) for better results.\n") gologger.Info().Msgf("You can modify %s to configure your keys/tokens.\n\n", options.ProviderConfig) for _, source := range passive.AllSources { sourceName := source.Name() switch source.KeyRequirement() { case subscraping.RequiredKey: gologger.Silent().Msgf("%s *\n", sourceName) case subscraping.OptionalKey: gologger.Silent().Msgf("%s ~\n", sourceName) default: gologger.Silent().Msgf("%s\n", sourceName) } } } func (options *Options) preProcessDomains() { for i, domain := range options.Domain { options.Domain[i] = preprocessDomain(domain) } } var defaultRateLimits = []string{ "github=30/m", "fullhunt=60/m", "pugrecon=10/s", fmt.Sprintf("robtex=%d/ms", uint(math.MaxUint)), "securitytrails=1/s", "shodan=1/s", "virustotal=4/m", "hackertarget=2/s", // "threatminer=10/m", "waybackarchive=15/m", "whoisxmlapi=50/s", "securitytrails=2/s", "sitedossier=8/m", "netlas=1/s", // "gitlab=2/s", "github=83/m", "hudsonrock=5/s", "urlscan=1/s", } ================================================ FILE: pkg/runner/outputter.go ================================================ package runner import ( "bufio" "errors" "io" "os" "path/filepath" "strings" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" ) // OutputWriter outputs content to writers. type OutputWriter struct { JSON bool } type jsonSourceResult struct { Host string `json:"host"` Input string `json:"input"` Source string `json:"source"` WildcardCertificate bool `json:"wildcard_certificate,omitempty"` } type jsonSourceIPResult struct { Host string `json:"host"` IP string `json:"ip"` Input string `json:"input"` Source string `json:"source"` WildcardCertificate bool `json:"wildcard_certificate,omitempty"` } type jsonSourcesResult struct { Host string `json:"host"` Input string `json:"input"` Sources []string `json:"sources"` WildcardCertificate bool `json:"wildcard_certificate,omitempty"` } // NewOutputWriter creates a new OutputWriter func NewOutputWriter(json bool) *OutputWriter { return &OutputWriter{JSON: json} } func (o *OutputWriter) createFile(filename string, appendToFile bool) (*os.File, error) { if filename == "" { return nil, errors.New("empty filename") } dir := filepath.Dir(filename) if dir != "" { if _, err := os.Stat(dir); os.IsNotExist(err) { err := os.MkdirAll(dir, os.ModePerm) if err != nil { return nil, err } } } var file *os.File var err error if appendToFile { file, err = os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) } else { file, err = os.Create(filename) } if err != nil { return nil, err } return file, nil } // WriteHostIP writes the output list of subdomain to an io.Writer func (o *OutputWriter) WriteHostIP(input string, results map[string]resolve.Result, writer io.Writer) error { var err error if o.JSON { err = writeJSONHostIP(input, results, writer) } else { err = writePlainHostIP(input, results, writer) } return err } func writePlainHostIP(_ string, results map[string]resolve.Result, writer io.Writer) error { bufwriter := bufio.NewWriter(writer) sb := &strings.Builder{} for _, result := range results { sb.WriteString(result.Host) sb.WriteString(",") sb.WriteString(result.IP) sb.WriteString(",") sb.WriteString(result.Source) sb.WriteString("\n") _, err := bufwriter.WriteString(sb.String()) if err != nil { if flushErr := bufwriter.Flush(); flushErr != nil { return errors.Join(err, flushErr) } return err } sb.Reset() } return bufwriter.Flush() } func writeJSONHostIP(input string, results map[string]resolve.Result, writer io.Writer) error { encoder := jsoniter.NewEncoder(writer) var data jsonSourceIPResult for _, result := range results { data.Host = result.Host data.IP = result.IP data.Input = input data.Source = result.Source data.WildcardCertificate = result.WildcardCertificate err := encoder.Encode(&data) if err != nil { return err } } return nil } // WriteHostNoWildcard writes the output list of subdomain with nW flag to an io.Writer func (o *OutputWriter) WriteHostNoWildcard(input string, results map[string]resolve.Result, writer io.Writer) error { hosts := make(map[string]resolve.HostEntry) for host, result := range results { hosts[host] = resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source, WildcardCertificate: result.WildcardCertificate} } return o.WriteHost(input, hosts, writer) } // WriteHost writes the output list of subdomain to an io.Writer func (o *OutputWriter) WriteHost(input string, results map[string]resolve.HostEntry, writer io.Writer) error { var err error if o.JSON { err = writeJSONHost(input, results, writer) } else { err = writePlainHost(input, results, writer) } return err } func writePlainHost(_ string, results map[string]resolve.HostEntry, writer io.Writer) error { bufwriter := bufio.NewWriter(writer) sb := &strings.Builder{} for _, result := range results { sb.WriteString(result.Host) sb.WriteString("\n") _, err := bufwriter.WriteString(sb.String()) if err != nil { if flushErr := bufwriter.Flush(); flushErr != nil { return errors.Join(err, flushErr) } return err } sb.Reset() } return bufwriter.Flush() } func writeJSONHost(input string, results map[string]resolve.HostEntry, writer io.Writer) error { encoder := jsoniter.NewEncoder(writer) var data jsonSourceResult for _, result := range results { data.Host = result.Host data.Input = input data.Source = result.Source data.WildcardCertificate = result.WildcardCertificate err := encoder.Encode(data) if err != nil { return err } } return nil } // WriteSourceHost writes the output list of subdomain to an io.Writer func (o *OutputWriter) WriteSourceHost(input string, sourceMap map[string]map[string]struct{}, writer io.Writer) error { var err error if o.JSON { err = writeSourceJSONHost(input, sourceMap, writer) } else { err = writeSourcePlainHost(input, sourceMap, writer) } return err } func writeSourceJSONHost(input string, sourceMap map[string]map[string]struct{}, writer io.Writer) error { encoder := jsoniter.NewEncoder(writer) var data jsonSourcesResult for host, sources := range sourceMap { data.Host = host data.Input = input keys := make([]string, 0, len(sources)) for source := range sources { keys = append(keys, source) } data.Sources = keys err := encoder.Encode(&data) if err != nil { return err } } return nil } func writeSourcePlainHost(_ string, sourceMap map[string]map[string]struct{}, writer io.Writer) error { bufwriter := bufio.NewWriter(writer) sb := &strings.Builder{} for host, sources := range sourceMap { sb.WriteString(host) sb.WriteString(",[") var sourcesString strings.Builder for source := range sources { sourcesString.WriteString(source) sourcesString.WriteRune(',') } sb.WriteString(strings.TrimSuffix(sourcesString.String(), ",")) sb.WriteString("]\n") _, err := bufwriter.WriteString(sb.String()) if err != nil { if flushErr := bufwriter.Flush(); flushErr != nil { return errors.Join(err, flushErr) } return err } sb.Reset() } return bufwriter.Flush() } ================================================ FILE: pkg/runner/runner.go ================================================ package runner import ( "bufio" "context" "io" "math" "os" "path" "regexp" "strconv" "strings" "github.com/projectdiscovery/gologger" contextutil "github.com/projectdiscovery/utils/context" fileutil "github.com/projectdiscovery/utils/file" mapsutil "github.com/projectdiscovery/utils/maps" "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Runner is an instance of the subdomain enumeration // client used to orchestrate the whole process. type Runner struct { options *Options passiveAgent *passive.Agent resolverClient *resolve.Resolver rateLimit *subscraping.CustomRateLimit } // NewRunner creates a new runner struct instance by parsing // the configuration options, configuring sources, reading lists // and setting up loggers, etc. func NewRunner(options *Options) (*Runner, error) { options.ConfigureOutput() runner := &Runner{options: options} // Check if the application loading with any provider configuration, then take it // Otherwise load the default provider config if fileutil.FileExists(options.ProviderConfig) { gologger.Info().Msgf("Loading provider config from %s", options.ProviderConfig) options.loadProvidersFrom(options.ProviderConfig) } else { gologger.Info().Msgf("Loading provider config from the default location: %s", defaultProviderConfigLocation) options.loadProvidersFrom(defaultProviderConfigLocation) } // Initialize the passive subdomain enumeration engine runner.initializePassiveEngine() // Initialize the subdomain resolver err := runner.initializeResolver() if err != nil { return nil, err } // Initialize the custom rate limit runner.rateLimit = &subscraping.CustomRateLimit{ Custom: mapsutil.SyncLockMap[string, uint]{ Map: make(map[string]uint), }, } for source, sourceRateLimit := range options.RateLimits.AsMap() { if sourceRateLimit.MaxCount > 0 && sourceRateLimit.MaxCount <= math.MaxUint { _ = runner.rateLimit.Custom.Set(source, sourceRateLimit.MaxCount) } } return runner, nil } // RunEnumeration wraps RunEnumerationWithCtx with an empty context func (r *Runner) RunEnumeration() error { ctx, _ := contextutil.WithValues(context.Background(), contextutil.ContextArg("All"), contextutil.ContextArg(strconv.FormatBool(r.options.All))) return r.RunEnumerationWithCtx(ctx) } // RunEnumerationWithCtx runs the subdomain enumeration flow on the targets specified func (r *Runner) RunEnumerationWithCtx(ctx context.Context) error { outputs := []io.Writer{r.options.Output} if len(r.options.Domain) > 0 { domainsReader := strings.NewReader(strings.Join(r.options.Domain, "\n")) return r.EnumerateMultipleDomainsWithCtx(ctx, domainsReader, outputs) } // If we have multiple domains as input, if r.options.DomainsFile != "" { f, err := os.Open(r.options.DomainsFile) if err != nil { return err } err = r.EnumerateMultipleDomainsWithCtx(ctx, f, outputs) if closeErr := f.Close(); closeErr != nil { gologger.Error().Msgf("Error closing file %s: %s", r.options.DomainsFile, closeErr) } return err } // If we have STDIN input, treat it as multiple domains if r.options.Stdin { return r.EnumerateMultipleDomainsWithCtx(ctx, os.Stdin, outputs) } return nil } // EnumerateMultipleDomains wraps EnumerateMultipleDomainsWithCtx with an empty context func (r *Runner) EnumerateMultipleDomains(reader io.Reader, writers []io.Writer) error { ctx, _ := contextutil.WithValues(context.Background(), contextutil.ContextArg("All"), contextutil.ContextArg(strconv.FormatBool(r.options.All))) return r.EnumerateMultipleDomainsWithCtx(ctx, reader, writers) } // EnumerateMultipleDomainsWithCtx enumerates subdomains for multiple domains // We keep enumerating subdomains for a given domain until we reach an error func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io.Reader, writers []io.Writer) error { var err error scanner := bufio.NewScanner(reader) ip, _ := regexp.Compile(`^([0-9\.]+$)`) for scanner.Scan() { domain := preprocessDomain(scanner.Text()) domain = replacer.Replace(domain) if domain == "" || (r.options.ExcludeIps && ip.MatchString(domain)) { continue } var file *os.File // If the user has specified an output file, use that output file instead // of creating a new output file for each domain. Else create a new file // for each domain in the directory. if r.options.OutputFile != "" { outputWriter := NewOutputWriter(r.options.JSON) file, err = outputWriter.createFile(r.options.OutputFile, true) if err != nil { gologger.Error().Msgf("Could not create file %s for %s: %s\n", r.options.OutputFile, r.options.Domain, err) return err } _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file)) if closeErr := file.Close(); closeErr != nil { gologger.Error().Msgf("Error closing file %s: %s", r.options.OutputFile, closeErr) } } else if r.options.OutputDirectory != "" { outputFile := path.Join(r.options.OutputDirectory, domain) if r.options.JSON { outputFile += ".json" } else { outputFile += ".txt" } outputWriter := NewOutputWriter(r.options.JSON) file, err = outputWriter.createFile(outputFile, false) if err != nil { gologger.Error().Msgf("Could not create file %s for %s: %s\n", r.options.OutputFile, r.options.Domain, err) return err } _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file)) if closeErr := file.Close(); closeErr != nil { gologger.Error().Msgf("Error closing file %s: %s", outputFile, closeErr) } } else { _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, writers) } if err != nil { return err } } return nil } ================================================ FILE: pkg/runner/stats.go ================================================ package runner import ( "fmt" "sort" "strings" "time" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" "golang.org/x/exp/maps" ) func printStatistics(stats map[string]subscraping.Statistics) { sources := maps.Keys(stats) sort.Strings(sources) var lines []string var skipped []string for _, source := range sources { sourceStats := stats[source] if sourceStats.Skipped { skipped = append(skipped, fmt.Sprintf(" %s", source)) } else { lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Requests, sourceStats.Errors)) } } if len(lines) > 0 { gologger.Print().Msgf("\n Source Duration Results Requests Errors\n%s\n", strings.Repeat("─", 68)) gologger.Print().Msg(strings.Join(lines, "\n")) gologger.Print().Msgf("\n") } if len(skipped) > 0 { gologger.Print().Msgf("\n The following sources were included but skipped...\n\n") gologger.Print().Msg(strings.Join(skipped, "\n")) gologger.Print().Msgf("\n\n") } } func (r *Runner) GetStatistics() map[string]subscraping.Statistics { return r.passiveAgent.GetStatistics() } ================================================ FILE: pkg/runner/util.go ================================================ package runner import ( fileutil "github.com/projectdiscovery/utils/file" stringsutil "github.com/projectdiscovery/utils/strings" ) func loadFromFile(file string) ([]string, error) { chanItems, err := fileutil.ReadFile(file) if err != nil { return nil, err } var items []string for item := range chanItems { item = preprocessDomain(item) if item == "" { continue } items = append(items, item) } return items, nil } func preprocessDomain(s string) string { return stringsutil.NormalizeWithOptions(s, stringsutil.NormalizeOptions{ StripComments: true, TrimCutset: "\n\t\"'` ", Lowercase: true, }, ) } ================================================ FILE: pkg/runner/validate.go ================================================ package runner import ( "errors" "fmt" "regexp" "strings" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/subfinder/v2/pkg/passive" mapsutil "github.com/projectdiscovery/utils/maps" sliceutil "github.com/projectdiscovery/utils/slice" ) // validateOptions validates the configuration options passed func (options *Options) validateOptions() error { // Check if domain, list of domains, or stdin info was provided. // If none was provided, then return. if len(options.Domain) == 0 && options.DomainsFile == "" && !options.Stdin { return errors.New("no input list provided") } // Both verbose and silent flags were used if options.Verbose && options.Silent { return errors.New("both verbose and silent mode specified") } // Validate threads and options if options.Threads == 0 { return errors.New("threads cannot be zero") } if options.Timeout == 0 { return errors.New("timeout cannot be zero") } // Always remove wildcard with hostip if options.HostIP && !options.RemoveWildcard { return errors.New("hostip flag must be used with RemoveWildcard option") } if options.Match != nil { options.matchRegexes = make([]*regexp.Regexp, len(options.Match)) var err error for i, re := range options.Match { if options.matchRegexes[i], err = regexp.Compile(stripRegexString(re)); err != nil { return errors.New("invalid value for match regex option") } } } if options.Filter != nil { options.filterRegexes = make([]*regexp.Regexp, len(options.Filter)) var err error for i, re := range options.Filter { if options.filterRegexes[i], err = regexp.Compile(stripRegexString(re)); err != nil { return errors.New("invalid value for filter regex option") } } } sources := mapsutil.GetKeys(passive.NameSourceMap) for source := range options.RateLimits.AsMap() { if !sliceutil.Contains(sources, source) { return fmt.Errorf("invalid source %s specified in -rls flag", source) } } return nil } func stripRegexString(val string) string { val = strings.ReplaceAll(val, ".", "\\.") val = strings.ReplaceAll(val, "*", ".*") return fmt.Sprint("^", val, "$") } // ConfigureOutput configures the output on the screen func (options *Options) ConfigureOutput() { // If the user desires verbose output, show verbose output if options.Verbose { gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) } if options.NoColor { gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true)) } if options.Silent { gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) } } ================================================ FILE: pkg/subscraping/agent.go ================================================ package subscraping import ( "bytes" "context" "crypto/tls" "fmt" "io" "net" "net/http" "net/url" "time" "github.com/corpix/uarand" "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/gologger" ) // NewSession creates a new session object for a domain func NewSession(domain string, proxy string, multiRateLimiter *ratelimit.MultiLimiter, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, Dial: (&net.Dialer{ Timeout: time.Duration(timeout) * time.Second, }).Dial, } // Add proxy if proxy != "" { proxyURL, _ := url.Parse(proxy) if proxyURL == nil { // Log warning but continue anyway gologger.Warning().Msgf("Invalid proxy provided: %s", proxy) } else { Transport.Proxy = http.ProxyURL(proxyURL) } } client := &http.Client{ Transport: Transport, Timeout: time.Duration(timeout) * time.Second, } session := &Session{Client: client, Timeout: timeout} // Initiate rate limit instance session.MultiRateLimiter = multiRateLimiter // Create a new extractor object for the current domain extractor, err := NewSubdomainExtractor(domain) session.Extractor = extractor return session, err } // Get makes a GET request to a URL with extended parameters func (s *Session) Get(ctx context.Context, getURL, cookies string, headers map[string]string) (*http.Response, error) { return s.HTTPRequest(ctx, http.MethodGet, getURL, cookies, headers, nil, BasicAuth{}) } // SimpleGet makes a simple GET request to a URL func (s *Session) SimpleGet(ctx context.Context, getURL string) (*http.Response, error) { return s.HTTPRequest(ctx, http.MethodGet, getURL, "", map[string]string{}, nil, BasicAuth{}) } // Post makes a POST request to a URL with extended parameters func (s *Session) Post(ctx context.Context, postURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) { return s.HTTPRequest(ctx, http.MethodPost, postURL, cookies, headers, body, BasicAuth{}) } // SimplePost makes a simple POST request to a URL func (s *Session) SimplePost(ctx context.Context, postURL, contentType string, body io.Reader) (*http.Response, error) { return s.HTTPRequest(ctx, http.MethodPost, postURL, "", map[string]string{"Content-Type": contentType}, body, BasicAuth{}) } // HTTPRequest makes any HTTP request to a URL with extended parameters func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies string, headers map[string]string, body io.Reader, basicAuth BasicAuth) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, method, requestURL, body) if err != nil { return nil, err } req.Header.Set("User-Agent", uarand.GetRandom()) req.Header.Set("Accept", "*/*") req.Header.Set("Accept-Language", "en") req.Header.Set("Connection", "close") if basicAuth.Username != "" || basicAuth.Password != "" { req.SetBasicAuth(basicAuth.Username, basicAuth.Password) } if cookies != "" { req.Header.Set("Cookie", cookies) } for key, value := range headers { req.Header.Set(key, value) } sourceName := ctx.Value(CtxSourceArg).(string) mrlErr := s.MultiRateLimiter.Take(sourceName) if mrlErr != nil { return nil, mrlErr } return httpRequestWrapper(s.Client, req) } // DiscardHTTPResponse discards the response content by demand func (s *Session) DiscardHTTPResponse(response *http.Response) { if response != nil { _, err := io.Copy(io.Discard, response.Body) if err != nil { gologger.Warning().Msgf("Could not discard response body: %s\n", err) return } if closeErr := response.Body.Close(); closeErr != nil { gologger.Warning().Msgf("Could not close response body: %s\n", closeErr) } } } // Close the session func (s *Session) Close() { s.MultiRateLimiter.Stop() s.Client.CloseIdleConnections() } func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) { response, err := client.Do(request) if err != nil { return nil, err } if response.StatusCode != http.StatusOK { requestURL, _ := url.QueryUnescape(request.URL.String()) gologger.Debug().MsgFunc(func() string { buffer := new(bytes.Buffer) _, _ = buffer.ReadFrom(response.Body) return fmt.Sprintf("Response for failed request against %s:\n%s", requestURL, buffer.String()) }) return response, fmt.Errorf("unexpected status code %d received from %s", response.StatusCode, requestURL) } return response, nil } ================================================ FILE: pkg/subscraping/doc.go ================================================ // Package subscraping contains the logic of scraping agents package subscraping ================================================ FILE: pkg/subscraping/extractor.go ================================================ package subscraping import ( "regexp" "strings" ) // RegexSubdomainExtractor is a concrete implementation of the SubdomainExtractor interface, using regex for extraction. type RegexSubdomainExtractor struct { extractor *regexp.Regexp } // NewSubdomainExtractor creates a new regular expression to extract // subdomains from text based on the given domain. func NewSubdomainExtractor(domain string) (*RegexSubdomainExtractor, error) { extractor, err := regexp.Compile(`(?i)[a-zA-Z0-9\*_.-]+\.` + domain) if err != nil { return nil, err } return &RegexSubdomainExtractor{extractor: extractor}, nil } // Extract implements the SubdomainExtractor interface, using the regex to find subdomains in the given text. func (re *RegexSubdomainExtractor) Extract(text string) []string { matches := re.extractor.FindAllString(text, -1) for i, match := range matches { matches[i] = strings.ToLower(match) } return matches } ================================================ FILE: pkg/subscraping/sources/alienvault/alienvault.go ================================================ // Package alienvault logic package alienvault import ( "context" "encoding/json" "fmt" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type alienvaultResponse struct { Detail string `json:"detail"` Error string `json:"error"` PassiveDNS []struct { Hostname string `json:"hostname"` } `json:"passive_dns"` } // Source is the passive scraping agent type Source struct { timeTaken time.Duration results int errors int requests int apiKeys []string skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain), "", map[string]string{"Authorization": "Bearer " + randomApiKey}) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var response alienvaultResponse // Get the response body and decode err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) if response.Error != "" { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s, %s", response.Detail, response.Error), } return } for _, record := range response.PassiveDNS { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}: s.results++ } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "alienvault" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/anubis/anubis.go ================================================ // Package anubis logic package anubis import ( "context" "fmt" "net/http" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { timeTaken time.Duration errors int results int requests int } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } if resp.StatusCode != http.StatusOK { session.DiscardHTTPResponse(resp) return } var subdomains []string err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) for _, record := range subdomains { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}: s.results++ } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "anubis" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { // no key needed } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, } } ================================================ FILE: pkg/subscraping/sources/bevigil/bevigil.go ================================================ // Package bevigil logic package bevigil import ( "context" "fmt" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type Response struct { Domain string `json:"domain"` Subdomains []string `json:"subdomains"` } type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } getUrl := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain) s.requests++ resp, err := session.Get(ctx, getUrl, "", map[string]string{ "X-Access-Token": randomApiKey, "User-Agent": "subfinder", }) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var subdomains []string var response Response err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) if len(response.Subdomains) > 0 { subdomains = response.Subdomains } for _, subdomain := range subdomains { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } }() return results } func (s *Source) Name() string { return "bevigil" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/bufferover/bufferover.go ================================================ // Package bufferover is a bufferover Scraping Engine in Golang package bufferover import ( "context" "fmt" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type response struct { Meta struct { Errors []string `json:"Errors"` } `json:"Meta"` FDNSA []string `json:"FDNS_A"` RDNS []string `json:"RDNS"` Results []string `json:"Results"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } s.getData(ctx, fmt.Sprintf("https://tls.bufferover.run/dns?q=.%s", domain), randomApiKey, session, results) }() return results } func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) { s.requests++ resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey}) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var bufforesponse response err = jsoniter.NewDecoder(resp.Body).Decode(&bufforesponse) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) metaErrors := bufforesponse.Meta.Errors if len(metaErrors) > 0 { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", strings.Join(metaErrors, ", ")), } s.errors++ return } var subdomains []string if len(bufforesponse.FDNSA) > 0 { subdomains = bufforesponse.FDNSA subdomains = append(subdomains, bufforesponse.RDNS...) } else if len(bufforesponse.Results) > 0 { subdomains = bufforesponse.Results } for _, subdomain := range subdomains { for _, value := range session.Extractor.Extract(subdomain) { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}: s.results++ } } } } // Name returns the name of the source func (s *Source) Name() string { return "bufferover" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/builtwith/builtwith.go ================================================ // Package builtwith logic package builtwith import ( "context" "fmt" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type response struct { Results []resultItem `json:"Results"` } type resultItem struct { Result result `json:"Result"` } type result struct { Paths []path `json:"Paths"` } type path struct { Domain string `json:"Domain"` Url string `json:"Url"` SubDomain string `json:"SubDomain"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { return } s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) for _, result := range data.Results { for _, path := range result.Result.Paths { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)}: s.results++ } } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "builtwith" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/c99/c99.go ================================================ // Package c99 logic package c99 import ( "context" "fmt" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } type dnsdbLookupResponse struct { Success bool `json:"success"` Subdomains []struct { Subdomain string `json:"subdomain"` IP string `json:"ip"` Cloudflare bool `json:"cloudflare"` } `json:"subdomains"` Error string `json:"error"` } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", randomApiKey, domain) s.requests++ resp, err := session.SimpleGet(ctx, searchURL) if err != nil { session.DiscardHTTPResponse(resp) return } defer func() { if err := resp.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() var response dnsdbLookupResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } if response.Error != "" { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error), } s.errors++ return } for _, data := range response.Subdomains { if !strings.HasPrefix(data.Subdomain, ".") { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain}: s.results++ } } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "c99" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/censys/censys.go ================================================ // Package censys logic package censys import ( "bytes" "context" "net/http" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) const ( // maxCensysPages is the maximum number of pages to fetch from the API maxCensysPages = 10 // maxPerPage is the maximum number of results per page maxPerPage = 100 // baseURL is the Censys Platform API base URL baseURL = "https://api.platform.censys.io" // searchEndpoint is the global data search query endpoint searchEndpoint = "/v3/global/search/query" // queryPrefix is the Censys query language prefix for certificate name search queryPrefix = "cert.names: " // authHeaderPrefix is the Bearer token prefix for Authorization header authHeaderPrefix = "Bearer " // contentTypeJSON is the Content-Type header value for JSON contentTypeJSON = "application/json" // orgIDHeader is the header name for organization ID orgIDHeader = "X-Organization-ID" ) // apiKey holds the Personal Access Token and optional Organization ID type apiKey struct { pat string orgID string } // Platform API request body type searchRequest struct { Query string `json:"query"` Fields []string `json:"fields,omitempty"` PageSize int `json:"page_size,omitempty"` Cursor string `json:"cursor,omitempty"` } // Platform API response structures type response struct { Result result `json:"result"` } type result struct { Hits []hit `json:"hits"` TotalHits int64 `json:"total_hits"` NextPageToken string `json:"next_page_token"` } type hit struct { CertificateV1 certificateV1 `json:"certificate_v1"` } type certificateV1 struct { Resource resource `json:"resource"` } type resource struct { Names []string `json:"names"` } // Source is the passive scraping agent type Source struct { apiKeys []apiKey timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) // PickRandom selects a random API key from configured keys. // This enables load balancing when users configure multiple PATs // (e.g., CENSYS_API_KEY=pat1:org1,pat2:org2) to distribute requests // and avoid hitting rate limits on a single key. randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey.pat == "" { s.skipped = true return } apiURL := baseURL + searchEndpoint cursor := "" currentPage := 1 for { select { case <-ctx.Done(): return default: } reqBody := searchRequest{ Query: queryPrefix + domain, Fields: []string{"cert.names"}, PageSize: maxPerPage, } if cursor != "" { reqBody.Cursor = cursor } bodyBytes, err := jsoniter.Marshal(reqBody) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } headers := map[string]string{ "Content-Type": contentTypeJSON, "Authorization": authHeaderPrefix + randomApiKey.pat, } // Add Organization ID header if provided if randomApiKey.orgID != "" { headers[orgIDHeader] = randomApiKey.orgID } s.requests++ resp, err := session.HTTPRequest( ctx, http.MethodPost, apiURL, "", headers, bytes.NewReader(bodyBytes), subscraping.BasicAuth{}, ) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var censysResponse response err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse) _ = resp.Body.Close() if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, hit := range censysResponse.Result.Hits { for _, name := range hit.CertificateV1.Resource.Names { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: name}: s.results++ } } } cursor = censysResponse.Result.NextPageToken if cursor == "" || currentPage >= maxCensysPages { break } currentPage++ } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "censys" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys parses and adds API keys. // Format: "PAT:ORG_ID" where ORG_ID is required for paid accounts. // Example: "censys_xxx_token:12345678-91011-1213" func (s *Source) AddApiKeys(keys []string) { s.apiKeys = subscraping.CreateApiKeys(keys, func(pat, orgID string) apiKey { return apiKey{pat: pat, orgID: orgID} }) // Also support single PAT without org ID for free users for _, key := range keys { if !strings.Contains(key, ":") && key != "" { s.apiKeys = append(s.apiKeys, apiKey{pat: key, orgID: ""}) } } } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/censys/censys_test.go ================================================ package censys import ( "context" "math" "net/http" "testing" "time" "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // createTestMultiRateLimiter creates a MultiLimiter for testing func createTestMultiRateLimiter(ctx context.Context) *ratelimit.MultiLimiter { mrl, _ := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{ Key: "censys", IsUnlimited: false, MaxCount: math.MaxInt32, Duration: time.Millisecond, }) return mrl } func TestCensysSource_NoApiKey(t *testing.T) { source := &Source{} // Don't add any API keys ctx := context.Background() multiRateLimiter := createTestMultiRateLimiter(ctx) session := &subscraping.Session{ Client: http.DefaultClient, MultiRateLimiter: multiRateLimiter, } ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, "censys") results := source.Run(ctxWithValue, "example.com", session) // Collect all results var resultCount int for range results { resultCount++ } // Should be skipped when no API key stats := source.Statistics() assert.True(t, stats.Skipped, "expected source to be skipped without API key") assert.Equal(t, 0, resultCount, "expected no results when skipped") } func TestCensysSource_ContextCancellation(t *testing.T) { source := &Source{} // Add a key with PAT:ORG_ID format source.AddApiKeys([]string{"test_pat:test_org_id"}) ctx := context.Background() multiRateLimiter := createTestMultiRateLimiter(ctx) session := &subscraping.Session{ Client: http.DefaultClient, MultiRateLimiter: multiRateLimiter, } // Create a context that will be cancelled ctxCancellable, cancel := context.WithCancel(ctx) ctxWithValue := context.WithValue(ctxCancellable, subscraping.CtxSourceArg, "censys") results := source.Run(ctxWithValue, "example.com", session) // Cancel immediately cancel() // Should exit quickly without blocking done := make(chan struct{}) go func() { for range results { // drain } close(done) }() select { case <-done: // Good - completed quickly case <-time.After(2 * time.Second): t.Fatal("context cancellation did not stop the source in time") } } func TestCensysSource_Metadata(t *testing.T) { source := &Source{} assert.Equal(t, "censys", source.Name()) assert.True(t, source.IsDefault()) assert.False(t, source.HasRecursiveSupport()) assert.True(t, source.NeedsKey()) } func TestCensysSource_AddApiKeys(t *testing.T) { t.Run("PAT with OrgID", func(t *testing.T) { source := &Source{} keys := []string{"pat_token_1:org_id_1", "pat_token_2:org_id_2"} source.AddApiKeys(keys) require.Len(t, source.apiKeys, 2) assert.Equal(t, "pat_token_1", source.apiKeys[0].pat) assert.Equal(t, "org_id_1", source.apiKeys[0].orgID) assert.Equal(t, "pat_token_2", source.apiKeys[1].pat) assert.Equal(t, "org_id_2", source.apiKeys[1].orgID) }) t.Run("PAT without OrgID (free user)", func(t *testing.T) { source := &Source{} keys := []string{"pat_token_only"} source.AddApiKeys(keys) require.Len(t, source.apiKeys, 1) assert.Equal(t, "pat_token_only", source.apiKeys[0].pat) assert.Equal(t, "", source.apiKeys[0].orgID) }) } func TestCensysSource_Statistics(t *testing.T) { source := &Source{ errors: 2, results: 10, timeTaken: 5 * time.Second, skipped: false, } stats := source.Statistics() assert.Equal(t, 2, stats.Errors) assert.Equal(t, 10, stats.Results) assert.Equal(t, 5*time.Second, stats.TimeTaken) assert.False(t, stats.Skipped) } ================================================ FILE: pkg/subscraping/sources/certspotter/certspotter.go ================================================ // Package certspotter logic package certspotter import ( "context" "fmt" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type certspotterObject struct { ID string `json:"id"` DNSNames []string `json:"dns_names"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } headers := map[string]string{"Authorization": "Bearer " + randomApiKey} cookies := "" s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), cookies, headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var response []certspotterObject err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) for _, cert := range response { for _, subdomain := range cert.DNSNames { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } if len(response) == 0 { return } id := response[len(response)-1].ID for { select { case <-ctx.Done(): return default: } reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id) s.requests++ resp, err := session.Get(ctx, reqURL, cookies, headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } var response []certspotterObject err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) if len(response) == 0 { break } for _, cert := range response { for _, subdomain := range cert.DNSNames { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } id = response[len(response)-1].ID } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "certspotter" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/chaos/chaos.go ================================================ // Package chaos logic package chaos import ( "context" "fmt" "time" "github.com/projectdiscovery/chaos-client/pkg/chaos" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } chaosClient := chaos.New(randomApiKey) s.requests++ for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{ Domain: domain, }) { select { case <-ctx.Done(): return default: } if result.Error != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: result.Error} s.errors++ break } results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", result.Subdomain, domain), } s.results++ } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "chaos" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/chinaz/chinaz.go ================================================ package chinaz // chinaz http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi import ( "context" "fmt" "io" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } body, err := io.ReadAll(resp.Body) session.DiscardHTTPResponse(resp) SubdomainList := jsoniter.Get(body, "Result").Get("ContributingSubdomainList") if SubdomainList.ToBool() { _data := []byte(SubdomainList.ToString()) for i := 0; i < SubdomainList.Size(); i++ { select { case <-ctx.Done(): return default: } subdomain := jsoniter.Get(_data, i, "DataUrl").ToString() select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } else { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "chinaz" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/commoncrawl/commoncrawl.go ================================================ // Package commoncrawl logic package commoncrawl import ( "bufio" "context" "fmt" "net/url" "strconv" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) const ( indexURL = "https://index.commoncrawl.org/collinfo.json" maxYearsBack = 5 ) var year = time.Now().Year() type indexResponse struct { ID string `json:"id"` APIURL string `json:"cdx-api"` } // Source is the passive scraping agent type Source struct { timeTaken time.Duration errors int results int requests int } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) s.requests++ resp, err := session.SimpleGet(ctx, indexURL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var indexes []indexResponse err = jsoniter.NewDecoder(resp.Body).Decode(&indexes) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) years := make([]string, 0) for i := range maxYearsBack { years = append(years, strconv.Itoa(year-i)) } searchIndexes := make(map[string]string) for _, year := range years { for _, index := range indexes { if strings.Contains(index.ID, year) { if _, ok := searchIndexes[year]; !ok { searchIndexes[year] = index.APIURL break } } } } for _, apiURL := range searchIndexes { further := s.getSubdomains(ctx, apiURL, domain, session, results) if !further { break } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "commoncrawl" } func (s *Source) IsDefault() bool { return false } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { // no key needed } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, } } func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, session *subscraping.Session, results chan subscraping.Result) bool { for { select { case <-ctx.Done(): return false default: var headers = map[string]string{"Host": "index.commoncrawl.org"} s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("%s?url=*.%s", searchURL, domain), "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return false } scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { select { case <-ctx.Done(): session.DiscardHTTPResponse(resp) return false default: } line := scanner.Text() if line == "" { continue } line, _ = url.QueryUnescape(line) for _, subdomain := range session.Extractor.Extract(line) { if subdomain != "" { subdomain = strings.ToLower(subdomain) subdomain = strings.TrimPrefix(subdomain, "25") subdomain = strings.TrimPrefix(subdomain, "2f") select { case <-ctx.Done(): session.DiscardHTTPResponse(resp) return false case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } } session.DiscardHTTPResponse(resp) return true } } } ================================================ FILE: pkg/subscraping/sources/crtsh/crtsh.go ================================================ // Package crtsh logic package crtsh import ( "context" "database/sql" "fmt" "strconv" "strings" "time" jsoniter "github.com/json-iterator/go" // postgres driver _ "github.com/lib/pq" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" contextutil "github.com/projectdiscovery/utils/context" ) type subdomain struct { ID int `json:"id"` NameValue string `json:"name_value"` } // Source is the passive scraping agent type Source struct { timeTaken time.Duration errors int results int requests int } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) count := s.getSubdomainsFromSQL(ctx, domain, session, results) if count > 0 { return } _ = s.getSubdomainsFromHTTP(ctx, domain, session, results) }() return results } func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) int { // connect_timeout: limits connection establishment time (in seconds) // statement_timeout: limits query execution time (in milliseconds) connStr := fmt.Sprintf("host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes connect_timeout=%d statement_timeout=%d", session.Timeout, session.Timeout*1000) db, err := sql.Open("postgres", connStr) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return 0 } defer func() { if closeErr := db.Close(); closeErr != nil { gologger.Warning().Msgf("Could not close database connection: %s\n", closeErr) } }() limitClause := "" if all, ok := ctx.Value(contextutil.ContextArg("All")).(contextutil.ContextArg); ok { if allBool, err := strconv.ParseBool(string(all)); err == nil && !allBool { limitClause = "LIMIT 10000" } } query := fmt.Sprintf(`WITH ci AS ( SELECT min(sub.CERTIFICATE_ID) ID, min(sub.ISSUER_CA_ID) ISSUER_CA_ID, array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES, x509_commonName(sub.CERTIFICATE) COMMON_NAME, x509_notBefore(sub.CERTIFICATE) NOT_BEFORE, x509_notAfter(sub.CERTIFICATE) NOT_AFTER, encode(x509_serialNumber(sub.CERTIFICATE), 'hex') SERIAL_NUMBER FROM (SELECT * FROM certificate_and_identities cai WHERE plainto_tsquery('certwatch', $1) @@ identities(cai.CERTIFICATE) AND cai.NAME_VALUE ILIKE ('%%' || $1 || '%%') %s ) sub GROUP BY sub.CERTIFICATE ) SELECT array_to_string(ci.NAME_VALUES, chr(10)) NAME_VALUE FROM ci LEFT JOIN LATERAL ( SELECT min(ctle.ENTRY_TIMESTAMP) ENTRY_TIMESTAMP FROM ct_log_entry ctle WHERE ctle.CERTIFICATE_ID = ci.ID ) le ON TRUE, ca WHERE ci.ISSUER_CA_ID = ca.ID ORDER BY le.ENTRY_TIMESTAMP DESC NULLS LAST;`, limitClause) rows, err := db.QueryContext(ctx, query, domain) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return 0 } if err := rows.Err(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return 0 } var count int var data string for rows.Next() { select { case <-ctx.Done(): return count default: } err := rows.Scan(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return count } count++ for subdomain := range strings.SplitSeq(data, "\n") { for _, value := range session.Extractor.Extract(subdomain) { if value != "" { select { case <-ctx.Done(): return count case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}: s.results++ } } } } } return count } func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return false } var subdomains []subdomain err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return false } session.DiscardHTTPResponse(resp) for _, subdomain := range subdomains { select { case <-ctx.Done(): return true default: } for sub := range strings.SplitSeq(subdomain.NameValue, "\n") { for _, value := range session.Extractor.Extract(sub) { if value != "" { select { case <-ctx.Done(): return true case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}: s.results++ } } } } } return true } // Name returns the name of the source func (s *Source) Name() string { return "crtsh" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { // no key needed } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, } } ================================================ FILE: pkg/subscraping/sources/digitalyama/digitalyama.go ================================================ package digitalyama import ( "context" "fmt" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } type digitalYamaResponse struct { Query string `json:"query"` Count int `json:"count"` Subdomains []string `json:"subdomains"` UsageSummary struct { QueryCost float64 `json:"query_cost"` CreditsRemaining float64 `json:"credits_remaining"` } `json:"usage_summary"` } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } searchURL := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s", domain) s.requests++ resp, err := session.Get(ctx, searchURL, "", map[string]string{"x-api-key": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } defer func() { if err := resp.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() if resp.StatusCode != 200 { var errResponse struct { Detail []struct { Loc []string `json:"loc"` Msg string `json:"msg"` Type string `json:"type"` } `json:"detail"` } err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)} s.errors++ return } if len(errResponse.Detail) > 0 { errMsg := errResponse.Detail[0].Msg results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s (code %d)", errMsg, resp.StatusCode)} } else { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)} } s.errors++ return } var response digitalYamaResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, subdomain := range response.Subdomains { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "digitalyama" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/digitorus/digitorus.go ================================================ // Package waybackarchive logic package digitorus import ( "bufio" "context" "fmt" "net/http" "strings" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" "github.com/projectdiscovery/utils/ptr" ) // Source is the passive scraping agent type Source struct { timeTaken time.Duration errors int results int requests int } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certificatedetails.com/%s", domain)) // the 404 page still contains around 100 subdomains - https://github.com/projectdiscovery/subfinder/issues/774 if err != nil && ptr.Safe(resp).StatusCode != http.StatusNotFound { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } defer func() { if err := resp.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { select { case <-ctx.Done(): return default: } line := scanner.Text() if line == "" { continue } subdomains := session.Extractor.Extract(line) for _, subdomain := range subdomains { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimPrefix(subdomain, ".")}: s.results++ } } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "digitorus" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { // no key needed } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, } } ================================================ FILE: pkg/subscraping/sources/dnsdb/dnsdb.go ================================================ // Package dnsdb logic package dnsdb import ( "bufio" "context" "encoding/json" "fmt" "io" "net/url" "strconv" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) const urlBase string = "https://api.dnsdb.info/dnsdb/v2" type rateResponse struct { Rate rate } type rate struct { OffsetMax json.Number `json:"offset_max"` } type safResponse struct { Condition string `json:"cond"` Obj dnsdbObj `json:"obj"` Msg string `json:"msg"` } type dnsdbObj struct { Name string `json:"rrname"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results uint64 requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) sourceName := s.Name() randomApiKey := subscraping.PickRandom(s.apiKeys, sourceName) if randomApiKey == "" { return } headers := map[string]string{ "X-API-KEY": randomApiKey, "Accept": "application/x-ndjson", } s.requests++ offsetMax, err := getMaxOffset(ctx, session, headers) if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ return } path := fmt.Sprintf("lookup/rrset/name/*.%s", domain) urlTemplate := fmt.Sprintf("%s/%s?", urlBase, path) queryParams := url.Values{} // ?limit=0 means DNSDB will return the maximum number of results allowed. queryParams.Add("limit", "0") queryParams.Add("swclient", "subfinder") for { select { case <-ctx.Done(): return default: } url := urlTemplate + queryParams.Encode() s.requests++ resp, err := session.Get(ctx, url, "", headers) if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var respCond string reader := bufio.NewReader(resp.Body) for { select { case <-ctx.Done(): session.DiscardHTTPResponse(resp) return default: } n, err := reader.ReadBytes('\n') if err == io.EOF { break } else if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var response safResponse err = jsoniter.Unmarshal(n, &response) if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } respCond = response.Condition if respCond == "" || respCond == "ongoing" { if response.Obj.Name != "" { select { case <-ctx.Done(): session.DiscardHTTPResponse(resp) return case results <- subscraping.Result{Source: sourceName, Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Obj.Name, ".")}: s.results++ } } } else if respCond != "begin" { break } } // Check the terminating jsonl object's condition. There are 3 possible scenarios: // 1. "limited" - There are more results available, make another query with an offset // 2. "succeeded" - The query completed successfully and all results were sent. // 3. anything else - This is an error and should be reported to the user. The user can then decide to use the results up to this // point or discard and retry. if respCond == "limited" { if offsetMax != 0 && s.results <= offsetMax { // Reset done to false to get more results with an offset query parameter set to s.results queryParams.Set("offset", strconv.FormatUint(s.results, 10)) continue } } else if respCond != "succeeded" { // DNSDB's terminating jsonl object's cond is not "limited" or succeeded" (#3), this is an error, notify the user. err = fmt.Errorf("%s terminated with condition: %s", sourceName, respCond) results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ } session.DiscardHTTPResponse(resp) break } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "dnsdb" } func (s *Source) IsDefault() bool { return false } func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: int(s.results), Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } func getMaxOffset(ctx context.Context, session *subscraping.Session, headers map[string]string) (uint64, error) { var offsetMax uint64 url := fmt.Sprintf("%s/rate_limit", urlBase) resp, err := session.Get(ctx, url, "", headers) defer session.DiscardHTTPResponse(resp) if err != nil { return offsetMax, err } data, err := io.ReadAll(resp.Body) if err != nil { return offsetMax, err } var rateResp rateResponse err = jsoniter.Unmarshal(data, &rateResp) if err != nil { return offsetMax, err } // if the OffsetMax is "n/a" then the ?offset= query parameter is not allowed if rateResp.Rate.OffsetMax.String() != "n/a" { offsetMax, err = strconv.ParseUint(rateResp.Rate.OffsetMax.String(), 10, 64) if err != nil { return offsetMax, err } } return offsetMax, nil } ================================================ FILE: pkg/subscraping/sources/dnsdumpster/dnsdumpster.go ================================================ // Package dnsdumpster logic package dnsdumpster import ( "context" "encoding/json" "fmt" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type response struct { A []struct { Host string `json:"host"` } `json:"a"` Ns []struct { Host string `json:"host"` } `json:"ns"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdumpster.com/domain/%s", domain), "", map[string]string{"X-API-Key": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } defer session.DiscardHTTPResponse(resp) var response response err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } for _, record := range append(response.A, response.Ns...) { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host}: s.results++ } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "dnsdumpster" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/dnsrepo/dnsrepo.go ================================================ package dnsrepo import ( "context" "encoding/json" "fmt" "io" "strings" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } type DnsRepoResponse []struct { Domain string `json:"domain"` } func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } randomApiInfo := strings.Split(randomApiKey, ":") if len(randomApiInfo) != 2 { s.skipped = true return } token := randomApiInfo[0] apiKey := randomApiInfo[1] s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain), "", map[string]string{"X-API-Access": token}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } responseData, err := io.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) var result DnsRepoResponse err = json.Unmarshal(responseData, &result) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } for _, sub := range result { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, ".")}: s.results++ } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "dnsrepo" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/domainsproject/domainsproject.go ================================================ // Package domainsproject logic package domainsproject import ( "context" "fmt" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { apiKeys []apiKey timeTaken time.Duration errors int results int requests int skipped bool } type apiKey struct { username string password string } type domainsProjectResponse struct { Domains []string `json:"domains"` Error string `json:"error"` } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey.username == "" || randomApiKey.password == "" { s.skipped = true return } searchURL := fmt.Sprintf("https://api.domainsproject.org/api/tld/search?domain=%s", domain) s.requests++ resp, err := session.HTTPRequest( ctx, "GET", searchURL, "", nil, nil, subscraping.BasicAuth{Username: randomApiKey.username, Password: randomApiKey.password}, ) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } defer func() { if err := resp.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() var response domainsProjectResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } if response.Error != "" { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error), } s.errors++ return } for _, subdomain := range response.Domains { if !strings.HasPrefix(subdomain, ".") { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "domainsproject" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { return apiKey{k, v} }) } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/driftnet/driftnet.go ================================================ package driftnet import ( "context" "encoding/json" "fmt" "net/http" "net/url" "strings" "sync" "sync/atomic" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) const ( // baseURL is the base URL for the driftnet API baseURL = "https://api.driftnet.io/v1/" // summaryLimit is the size of the summary limit that we send to the API summaryLimit = 10000 ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors atomic.Int32 results atomic.Int32 requests atomic.Int32 skipped bool } // endpointConfig describes a driftnet endpoint that can used type endpointConfig struct { // The API endpoint to be touched endpoint string // The API parameter used for query param string // The context that we should restrict to in results from this endpoint context string } // endpoints is a set of endpoint configs var endpoints = []endpointConfig{ {"ct/log", "field=host:", "cert-dns-name"}, {"scan/protocols", "field=host:", "cert-dns-name"}, {"scan/domains", "field=host:", "cert-dns-name"}, {"domain/rdns", "host=", "dns-ptr"}, } // summaryResponse is an API response type summaryResponse struct { Summary struct { Other int `json:"other"` Values map[string]int `json:"values"` } `json:"summary"` } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { // Final results channel results := make(chan subscraping.Result) s.errors.Store(0) s.results.Store(0) s.requests.Store(0) // Waitgroup for subsources var wg sync.WaitGroup wg.Add(len(endpoints)) // Map for dedupe between subsources dedupe := sync.Map{} // Close down results when all subsources finished go func(startTime time.Time) { wg.Wait() s.timeTaken = time.Since(startTime) close(results) }(time.Now()) // Start up requests for all subsources for i := range endpoints { go s.runSubsource(ctx, domain, session, results, &wg, &dedupe, endpoints[i]) } // Return the results channel return results } // Name returns the name of the source func (s *Source) Name() string { return "driftnet" } // IsDefault indicates that this source should used as part of the default execution. func (s *Source) IsDefault() bool { return true } // HasRecursiveSupport indicates that we accept subdomains in addition to apex domains func (s *Source) HasRecursiveSupport() bool { return true } // KeyRequirement indicates that we need an API key func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } // NeedsKey indicates that we need an API key func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys provides us with the API key(s) func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } // Statistics returns statistics about the scraping process func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: int(s.errors.Load()), Results: int(s.results.Load()), Requests: int(s.requests.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } } // runSubsource queries a specific driftnet endpoint for subdomains and sends results to the channel func (s *Source) runSubsource(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result, wg *sync.WaitGroup, dedupe *sync.Map, epConfig endpointConfig) { // Default headers headers := map[string]string{ "accept": "application/json", } // Pick an API key randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey != "" { headers["authorization"] = "Bearer " + randomApiKey } // Request requestURL := fmt.Sprintf("%s%s?%s%s&summarize=host&summary_context=%s&summary_limit=%d", baseURL, epConfig.endpoint, epConfig.param, url.QueryEscape(domain), epConfig.context, summaryLimit) s.requests.Add(1) resp, err := session.Get(ctx, requestURL, "", headers) if err != nil { // HTTP 204 is not an error from the Driftnet API if resp == nil || resp.StatusCode != http.StatusNoContent { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors.Add(1) } wg.Done() return } defer session.DiscardHTTPResponse(resp) // 204 means no results, any other response code is an error if resp.StatusCode != 200 { if resp.StatusCode != 204 { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} s.errors.Add(1) } wg.Done() return } // Parse and return results var summary summaryResponse decoder := json.NewDecoder(resp.Body) err = decoder.Decode(&summary) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors.Add(1) wg.Done() return } for subdomain := range summary.Summary.Values { select { case <-ctx.Done(): wg.Done() return default: } if !strings.HasSuffix(subdomain, "."+domain) { continue } if _, present := dedupe.LoadOrStore(strings.ToLower(subdomain), true); !present { select { case <-ctx.Done(): wg.Done() return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results.Add(1) } } } // Complete! wg.Done() } ================================================ FILE: pkg/subscraping/sources/facebook/ctlogs.go ================================================ package facebook import ( "context" "encoding/json" "fmt" "io" "time" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" "github.com/projectdiscovery/utils/generic" urlutil "github.com/projectdiscovery/utils/url" ) // source: https://developers.facebook.com/tools/ct // api-docs: https://developers.facebook.com/docs/certificate-transparency-api // ratelimit: ~20,000 req/hour per appID https://developers.facebook.com/docs/graph-api/overview/rate-limiting/ var ( domainsPerPage = "1000" authUrl = "https://graph.facebook.com/oauth/access_token?client_id=%s&client_secret=%s&grant_type=client_credentials" domainsUrl = "https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=%s&limit=" + domainsPerPage ) type apiKey struct { AppID string Secret string AccessToken string // obtained by calling // https://graph.facebook.com/oauth/access_token?client_id=APP_ID&client_secret=APP_SECRET&grant_type=client_credentials Error error // error while fetching access token } // FetchAccessToken fetches the access token for the api key // using app id and secret func (k *apiKey) FetchAccessToken() { if generic.EqualsAny("", k.AppID, k.Secret) { k.Error = fmt.Errorf("invalid app id or secret") return } resp, err := retryablehttp.Get(fmt.Sprintf(authUrl, k.AppID, k.Secret)) if err != nil { k.Error = err return } defer func() { if err := resp.Body.Close(); err != nil { gologger.Error().Msgf("error closing response body: %s", err) } }() bin, err := io.ReadAll(resp.Body) if err != nil { k.Error = err return } auth := &authResponse{} if err := json.Unmarshal(bin, auth); err != nil { k.Error = err return } if auth.AccessToken == "" { k.Error = fmt.Errorf("invalid response from facebook got %v", string(bin)) return } k.AccessToken = auth.AccessToken } // IsValid returns true if the api key is valid func (k *apiKey) IsValid() bool { return k.AccessToken != "" } // Source is the passive scraping agent type Source struct { apiKeys []apiKey timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 if len(s.apiKeys) == 0 { s.skipped = true close(results) return results } go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) key := subscraping.PickRandom(s.apiKeys, s.Name()) domainsURL := fmt.Sprintf(domainsUrl, key.AccessToken, domain) for { select { case <-ctx.Done(): return default: } s.requests++ resp, err := session.Get(ctx, domainsURL, "", nil) if err != nil { s.errors++ results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} return } bin, err := io.ReadAll(resp.Body) if err != nil { s.errors++ gologger.Verbose().Msgf("failed to read response body: %s\n", err) results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} return } session.DiscardHTTPResponse(resp) response := &response{} if err := json.Unmarshal(bin, response); err != nil { s.errors++ results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to unmarshal response: %s: %w", string(bin), err)} return } for _, v := range response.Data { for _, domain := range v.Domains { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain}: s.results++ } } } if response.Paging.Next == "" { break } domainsURL = updateParamInURL(response.Paging.Next, "limit", domainsPerPage) } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "facebook" } // IsDefault returns true if the source should be queried by default func (s *Source) IsDefault() bool { return true } // accepts subdomains (e.g. subdomain.domain.tld) // but also returns all SANs for a certificate which may not match the domain func (s *Source) HasRecursiveSupport() bool { return true } // KeyRequirement returns the API key requirement level for this source func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } // NeedsKey returns true if the source requires an API key func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys adds api keys to the source func (s *Source) AddApiKeys(keys []string) { allapikeys := subscraping.CreateApiKeys(keys, func(k, v string) apiKey { apiKey := apiKey{AppID: k, Secret: v} apiKey.FetchAccessToken() if apiKey.Error != nil { gologger.Warning().Msgf("Could not fetch access token for %s: %s\n", k, apiKey.Error) } return apiKey }) // filter out invalid keys for _, key := range allapikeys { if key.IsValid() { s.apiKeys = append(s.apiKeys, key) } } } // Statistics returns the statistics for the source func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } func updateParamInURL(url, param, value string) string { urlx, err := urlutil.Parse(url) if err != nil { return url } urlx.Params.Set(param, value) return urlx.String() } ================================================ FILE: pkg/subscraping/sources/facebook/ctlogs_test.go ================================================ package facebook import ( "encoding/json" "fmt" "io" "os" "strings" "testing" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/utils/generic" ) var ( fb_API_ID = "$FB_APP_ID" fb_API_SECRET = "$FB_APP_SECRET" ) func TestFacebookSource(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode.") } updateWithEnv(&fb_API_ID) updateWithEnv(&fb_API_SECRET) if generic.EqualsAny("", fb_API_ID, fb_API_SECRET) { t.SkipNow() } k := apiKey{ AppID: fb_API_ID, Secret: fb_API_SECRET, } k.FetchAccessToken() if k.Error != nil { t.Fatal(k.Error) } fetchURL := fmt.Sprintf("https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=hackerone.com&limit=5", k.AccessToken) resp, err := retryablehttp.Get(fetchURL) if err != nil { t.Fatal(err) } defer func() { if err := resp.Body.Close(); err != nil { gologger.Error().Msgf("error closing response body: %s", err) } }() bin, err := io.ReadAll(resp.Body) if err != nil { t.Fatal(err) } response := &response{} if err := json.Unmarshal(bin, response); err != nil { t.Fatal(err) } if len(response.Data) == 0 { t.Fatal("no data found") } } func updateWithEnv(key *string) { if key == nil { return } value := *key if strings.HasPrefix(value, "$") { *key = os.Getenv(value[1:]) } } ================================================ FILE: pkg/subscraping/sources/facebook/types.go ================================================ package facebook type authResponse struct { AccessToken string `json:"access_token"` } /* { "data": [ { "domains": [ "docs.hackerone.com" ], "id": "10056051421102939" }, ... ], "paging": { "cursors": { "before": "MTAwNTYwNTE0MjExMDI5MzkZD", "after": "Njc0OTczNTA5NTA1MzUxNwZDZD" }, "next": "https://graph.facebook.com/v17.0/certificates?fields=domains&access_token=6161176097324222|fzhUp9I0eXa456Ye21zAhyYVozk&query=hackerone.com&limit=25&after=Njc0OTczNTA5NTA1MzUxNwZDZD" } } */ // example response type response struct { Data []struct { Domains []string `json:"domains"` } `json:"data"` Paging struct { Next string `json:"next"` } `json:"paging"` } ================================================ FILE: pkg/subscraping/sources/fofa/fofa.go ================================================ // Package fofa logic package fofa import ( "context" "encoding/base64" "fmt" "regexp" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type fofaResponse struct { Error bool `json:"error"` ErrMsg string `json:"errmsg"` Size int `json:"size"` Results []string `json:"results"` } // Source is the passive scraping agent type Source struct { apiKeys []apiKey timeTaken time.Duration errors int results int requests int skipped bool } type apiKey struct { username string secret string } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey.username == "" || randomApiKey.secret == "" { s.skipped = true return } // fofa api doc https://fofa.info/static_pages/api_help qbase64 := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", randomApiKey.username, randomApiKey.secret, qbase64)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var response fofaResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) if response.Error { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.ErrMsg), } s.errors++ return } if response.Size > 0 { for _, subdomain := range response.Results { select { case <-ctx.Done(): return default: } if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") { subdomain = subdomain[strings.Index(subdomain, "//")+2:] } re := regexp.MustCompile(`:\d+$`) if re.MatchString(subdomain) { subdomain = re.ReplaceAllString(subdomain, "") } results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "fofa" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { return apiKey{k, v} }) } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/fullhunt/fullhunt.go ================================================ package fullhunt import ( "context" "fmt" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // fullhunt response type fullHuntResponse struct { Hosts []string `json:"hosts"` Message string `json:"message"` Status int `json:"status"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://fullhunt.io/api/v1/domain/%s/subdomains", domain), "", map[string]string{"X-API-KEY": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var response fullHuntResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) for _, record := range response.Hosts { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}: s.results++ } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "fullhunt" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/github/github.go ================================================ // Package github GitHub search package // Based on gwen001's https://github.com/gwen001/github-search github-subdomains package github import ( "bufio" "context" "fmt" "net/http" "net/url" "regexp" "strconv" "strings" "sync" "sync/atomic" "time" jsoniter "github.com/json-iterator/go" "github.com/tomnomnom/linkheader" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type textMatch struct { Fragment string `json:"fragment"` } type item struct { Name string `json:"name"` HTMLURL string `json:"html_url"` TextMatches []textMatch `json:"text_matches"` } type response struct { TotalCount int `json:"total_count"` Items []item `json:"items"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors atomic.Int32 results atomic.Int32 requests atomic.Int32 skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors.Store(0) s.results.Store(0) s.requests.Store(0) go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) if len(s.apiKeys) == 0 { gologger.Debug().Msgf("Cannot use the %s source because there was no key defined for it.", s.Name()) s.skipped = true return } tokens := NewTokenManager(s.apiKeys) searchURL := fmt.Sprintf("https://api.github.com/search/code?per_page=100&q=%s&sort=created&order=asc", domain) s.enumerate(ctx, searchURL, domainRegexp(domain), tokens, session, results) }() return results } func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *regexp.Regexp, tokens *Tokens, session *subscraping.Session, results chan subscraping.Result) { select { case <-ctx.Done(): return default: } token := tokens.Get() headers := map[string]string{ "Accept": "application/vnd.github.v3.text-match+json", "Authorization": "token " + token.Hash, } // Initial request to GitHub search s.requests.Add(1) resp, err := session.Get(ctx, searchURL, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden if err != nil && !isForbidden { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors.Add(1) session.DiscardHTTPResponse(resp) return } // Retry enumerarion after Retry-After seconds on rate limit abuse detected ratelimitRemaining, _ := strconv.ParseInt(resp.Header.Get("X-Ratelimit-Remaining"), 10, 64) if isForbidden && ratelimitRemaining == 0 { retryAfterSeconds, _ := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) tokens.setCurrentTokenExceeded(retryAfterSeconds) session.DiscardHTTPResponse(resp) s.enumerate(ctx, searchURL, domainRegexp, tokens, session, results) } var data response // Marshall json response err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors.Add(1) session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) err = s.proccesItems(ctx, data.Items, domainRegexp, s.Name(), session, results) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors.Add(1) return } // Links header, first, next, last... linksHeader := linkheader.Parse(resp.Header.Get("Link")) // Process the next link recursively for _, link := range linksHeader { select { case <-ctx.Done(): return default: } if link.Rel == "next" { nextURL, err := url.QueryUnescape(link.URL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors.Add(1) return } s.enumerate(ctx, nextURL, domainRegexp, tokens, session, results) } } } // proccesItems process github response items func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *regexp.Regexp, name string, session *subscraping.Session, results chan subscraping.Result) error { var wg sync.WaitGroup errChan := make(chan error, len(items)) for _, responseItem := range items { select { case <-ctx.Done(): return ctx.Err() default: } wg.Add(1) go func(responseItem item) { defer wg.Done() select { case <-ctx.Done(): return default: } s.requests.Add(1) resp, err := session.SimpleGet(ctx, rawURL(responseItem.HTMLURL)) if err != nil { if resp != nil && resp.StatusCode != http.StatusNotFound { session.DiscardHTTPResponse(resp) } errChan <- err return } if resp.StatusCode == http.StatusOK { scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { select { case <-ctx.Done(): session.DiscardHTTPResponse(resp) return default: } line := scanner.Text() if line == "" { continue } for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) { select { case <-ctx.Done(): session.DiscardHTTPResponse(resp) return case results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}: s.results.Add(1) } } } session.DiscardHTTPResponse(resp) } for _, textMatch := range responseItem.TextMatches { select { case <-ctx.Done(): return default: } for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}: s.results.Add(1) } } } }(responseItem) } wg.Wait() close(errChan) for err := range errChan { if err != nil { return err } } return nil } // Normalize content before matching, query unescape, remove tabs and new line chars func normalizeContent(content string) string { normalizedContent, _ := url.QueryUnescape(content) normalizedContent = strings.ReplaceAll(normalizedContent, "\\t", "") normalizedContent = strings.ReplaceAll(normalizedContent, "\\n", "") return normalizedContent } // Raw URL to get the files code and match for subdomains func rawURL(htmlURL string) string { domain := strings.ReplaceAll(htmlURL, "https://github.com/", "https://raw.githubusercontent.com/") return strings.ReplaceAll(domain, "/blob/", "/") } // DomainRegexp regular expression to match subdomains in github files code func domainRegexp(domain string) *regexp.Regexp { rdomain := strings.ReplaceAll(domain, ".", "\\.") return regexp.MustCompile("(\\w[a-zA-Z0-9][a-zA-Z0-9-\\.]*)" + rdomain) } // Name returns the name of the source func (s *Source) Name() string { return "github" } func (s *Source) IsDefault() bool { return false } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: int(s.errors.Load()), Results: int(s.results.Load()), Requests: int(s.requests.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/github/tokenmanager.go ================================================ package github import "time" // Token struct type Token struct { Hash string RetryAfter int64 ExceededTime time.Time } // Tokens is the internal struct to manage the current token // and the pool type Tokens struct { current int pool []Token } // NewTokenManager initialize the tokens pool func NewTokenManager(keys []string) *Tokens { pool := []Token{} for _, key := range keys { t := Token{Hash: key, ExceededTime: time.Time{}, RetryAfter: 0} pool = append(pool, t) } return &Tokens{ current: 0, pool: pool, } } func (r *Tokens) setCurrentTokenExceeded(retryAfter int64) { if r.current >= len(r.pool) { r.current %= len(r.pool) } if r.pool[r.current].RetryAfter == 0 { r.pool[r.current].ExceededTime = time.Now() r.pool[r.current].RetryAfter = retryAfter } } // Get returns a new token from the token pool func (r *Tokens) Get() *Token { resetExceededTokens(r) if r.current >= len(r.pool) { r.current %= len(r.pool) } result := &r.pool[r.current] r.current++ return result } func resetExceededTokens(r *Tokens) { for i, token := range r.pool { if token.RetryAfter > 0 { if int64(time.Since(token.ExceededTime)/time.Second) > token.RetryAfter { r.pool[i].ExceededTime = time.Time{} r.pool[i].RetryAfter = 0 } } } } ================================================ FILE: pkg/subscraping/sources/gitlab/gitlab.go ================================================ package gitlab import ( "bufio" "context" "fmt" "net/http" "net/url" "regexp" "strings" "sync" "sync/atomic" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" "github.com/tomnomnom/linkheader" ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors atomic.Int32 results atomic.Int32 requests atomic.Int32 skipped bool } type item struct { Data string `json:"data"` ProjectId int `json:"project_id"` Path string `json:"path"` Ref string `json:"ref"` } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors.Store(0) s.results.Store(0) s.requests.Store(0) go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { return } headers := map[string]string{"PRIVATE-TOKEN": randomApiKey} searchURL := fmt.Sprintf("https://gitlab.com/api/v4/search?scope=blobs&search=%s&per_page=100", domain) s.enumerate(ctx, searchURL, domainRegexp(domain), headers, session, results) }() return results } func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *regexp.Regexp, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { select { case <-ctx.Done(): return default: } s.requests.Add(1) resp, err := session.Get(ctx, searchURL, "", headers) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors.Add(1) session.DiscardHTTPResponse(resp) return } defer session.DiscardHTTPResponse(resp) var items []item err = jsoniter.NewDecoder(resp.Body).Decode(&items) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors.Add(1) return } var wg sync.WaitGroup wg.Add(len(items)) for _, it := range items { go func(item item) { // The original item.Path causes 404 error because the Gitlab API is expecting the url encoded path fileUrl := fmt.Sprintf("https://gitlab.com/api/v4/projects/%d/repository/files/%s/raw?ref=%s", item.ProjectId, url.QueryEscape(item.Path), item.Ref) s.requests.Add(1) resp, err := session.Get(ctx, fileUrl, "", headers) if err != nil { if resp == nil || (resp != nil && resp.StatusCode != http.StatusNotFound) { session.DiscardHTTPResponse(resp) results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors.Add(1) return } } if resp.StatusCode == http.StatusOK { scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() if line == "" { continue } for _, subdomain := range domainRegexp.FindAllString(line, -1) { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results.Add(1) } } session.DiscardHTTPResponse(resp) } defer wg.Done() }(it) } linksHeader := linkheader.Parse(resp.Header.Get("Link")) for _, link := range linksHeader { select { case <-ctx.Done(): return default: } if link.Rel == "next" { nextURL, err := url.QueryUnescape(link.URL) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors.Add(1) return } s.enumerate(ctx, nextURL, domainRegexp, headers, session, results) } } wg.Wait() } func domainRegexp(domain string) *regexp.Regexp { rdomain := strings.ReplaceAll(domain, ".", "\\.") return regexp.MustCompile("(\\w[a-zA-Z0-9][a-zA-Z0-9-\\.]*)" + rdomain) } // Name returns the name of the source func (s *Source) Name() string { return "gitlab" } func (s *Source) IsDefault() bool { return false } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } // Statistics returns the statistics for the source func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: int(s.errors.Load()), Results: int(s.results.Load()), Requests: int(s.requests.Load()), TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/hackertarget/hackertarget.go ================================================ // Package hackertarget logic package hackertarget import ( "bufio" "context" "fmt" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) htSearchUrl := fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey != "" { htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) } htSearchUrl = fmt.Sprintf("%s&apikey=%s", htSearchUrl, randomApiKey) s.requests++ resp, err := session.SimpleGet(ctx, htSearchUrl) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } defer func() { if err := resp.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { select { case <-ctx.Done(): return default: } line := scanner.Text() if line == "" { continue } match := session.Extractor.Extract(line) for _, subdomain := range match { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "hackertarget" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.OptionalKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/hudsonrock/hudsonrock.go ================================================ // Package hudsonrock logic package hudsonrock import ( "context" "encoding/json" "fmt" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type hudsonrockResponse struct { Data struct { EmployeesUrls []struct { URL string `json:"url"` } `json:"employees_urls"` ClientsUrls []struct { URL string `json:"url"` } `json:"clients_urls"` } `json:"data"` } // Source is the passive scraping agent type Source struct { timeTaken time.Duration errors int results int requests int } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://cavalier.hudsonrock.com/api/json/v2/osint-tools/urls-by-domain?domain=%s", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } defer session.DiscardHTTPResponse(resp) var response hudsonrockResponse err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } for _, record := range append(response.Data.EmployeesUrls, response.Data.ClientsUrls...) { for _, subdomain := range session.Extractor.Extract(record.URL) { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "hudsonrock" } func (s *Source) IsDefault() bool { return false } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { // no key needed } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, } } ================================================ FILE: pkg/subscraping/sources/intelx/intelx.go ================================================ // Package intelx logic package intelx import ( "bytes" "context" "encoding/json" "fmt" "io" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type searchResponseType struct { ID string `json:"id"` Status int `json:"status"` } type selectorType struct { Selectvalue string `json:"selectorvalue"` } type searchResultType struct { Selectors []selectorType `json:"selectors"` Status int `json:"status"` } type requestBody struct { Term string Maxresults int Media int Target int Terminate []int Timeout int } // Source is the passive scraping agent type Source struct { apiKeys []apiKey timeTaken time.Duration errors int results int requests int skipped bool } type apiKey struct { host string key string } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey.host == "" || randomApiKey.key == "" { s.skipped = true return } searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", randomApiKey.host, randomApiKey.key) reqBody := requestBody{ Term: domain, Maxresults: 100000, Media: 0, Target: 1, Timeout: 20, } body, err := json.Marshal(reqBody) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } s.requests++ resp, err := session.SimplePost(ctx, searchURL, "application/json", bytes.NewBuffer(body)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var response searchResponseType err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", randomApiKey.host, randomApiKey.key, response.ID) status := 0 for status == 0 || status == 3 { select { case <-ctx.Done(): return default: } s.requests++ resp, err = session.Get(ctx, resultsURL, "", nil) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var response searchResultType err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } _, err = io.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) status = response.Status for _, hostname := range response.Selectors { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue}: s.results++ } } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "intelx" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { return apiKey{k, v} }) } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/leakix/leakix.go ================================================ // Package leakix logic package leakix import ( "context" "encoding/json" "fmt" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) // Default headers headers := map[string]string{ "accept": "application/json", } // Pick an API key randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey != "" { headers["api-key"] = randomApiKey } // Request s.requests++ resp, err := session.Get(ctx, "https://leakix.net/api/subdomains/"+domain, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } defer session.DiscardHTTPResponse(resp) if resp.StatusCode != 200 { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} s.errors++ return } // Parse and return results var subdomains []subResponse decoder := json.NewDecoder(resp.Body) err = decoder.Decode(&subdomains) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, result := range subdomains { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain}: s.results++ } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "leakix" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } type subResponse struct { Subdomain string `json:"subdomain"` DistinctIps int `json:"distinct_ips"` LastSeen time.Time `json:"last_seen"` } ================================================ FILE: pkg/subscraping/sources/merklemap/merklemap.go ================================================ // Package merklemap logic package merklemap import ( "bytes" "context" "encoding/json" "fmt" "io" "math" "net/url" "strconv" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) // Pick an API key, skip if no key is found randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } // Default headers headers := map[string]string{ "accept": "application/json", // Set a user agent to prevent random one from pkg/subscraping/agent.go, it triggers the cloudflare protection of the api "User-Agent": "subfinder", "Authorization": "Bearer " + randomApiKey, } // Fetch all pages with pagination // https://www.merklemap.com/documentation/search s.fetchAllPages(ctx, domain, headers, session, results) }() return results } // fetchAllPages fetches all pages of results using pagination func (s *Source) fetchAllPages(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { baseURL := "https://api.merklemap.com/v1/search?query=" + url.QueryEscape("*."+domain) totalCount := math.MaxInt processedResults := 0 // Iterate through all pages for page := 0; processedResults < totalCount; page++ { pageResp, err := s.fetchPage(ctx, baseURL, page, headers, session) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } if page == 0 { totalCount = pageResp.Count } // Stop if this page returned no results if len(pageResp.Results) == 0 { break } for _, result := range pageResp.Results { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Subdomain, Value: result.Hostname, } s.results++ processedResults++ } } } // fetchPage fetches a single page of results func (s *Source) fetchPage(ctx context.Context, baseURL string, page int, headers map[string]string, session *subscraping.Session) (*response, error) { url := baseURL + "&page=" + strconv.Itoa(page) s.requests++ resp, err := session.Get(ctx, url, "", headers) if err != nil { return nil, err } defer session.DiscardHTTPResponse(resp) if resp.StatusCode != 200 { respBody, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("request failed with status %d: %s", resp.StatusCode, err) } return nil, fmt.Errorf("request failed with status %d: %s", resp.StatusCode, string(respBody)) } var pageResponse response respBody, err := io.ReadAll(resp.Body) if err != nil { return nil, err } decoder := json.NewDecoder(bytes.NewReader(respBody)) if err := decoder.Decode(&pageResponse); err != nil { return nil, err } return &pageResponse, nil } // Name returns the name of the source func (s *Source) Name() string { return "merklemap" } func (s *Source) IsDefault() bool { return false } // HasRecursiveSupport indicates that we accept subdomains in addition to apex domains func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } type response struct { Count int `json:"count"` Results []struct { Hostname string `json:"hostname"` SubjectCommonName string `json:"subject_common_name"` FirstSeen string `json:"first_seen"` } `json:"results"` } ================================================ FILE: pkg/subscraping/sources/netlas/netlas.go ================================================ // Package netlas logic package netlas import ( "context" "io" "strings" "encoding/json" "fmt" "net/http" "net/url" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type Item struct { Data struct { A []string `json:"a,omitempty"` Txt []string `json:"txt,omitempty"` LastUpdated string `json:"last_updated,omitempty"` Timestamp string `json:"@timestamp,omitempty"` Ns []string `json:"ns,omitempty"` Level int `json:"level,omitempty"` Zone string `json:"zone,omitempty"` Domain string `json:"domain,omitempty"` Cname []string `json:"cname,omitempty"` Mx []string `json:"mx,omitempty"` } `json:"data"` } type DomainsCountResponse struct { Count int `json:"count"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) // To get count of domains endpoint := "https://app.netlas.io/api/domains_count/" params := url.Values{} countQuery := fmt.Sprintf("domain:*.%s AND NOT domain:%s", domain, domain) params.Set("q", countQuery) countUrl := endpoint + "?" + params.Encode() // Pick an API key randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) s.requests++ resp1, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, }, nil, subscraping.BasicAuth{}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } else if resp1.StatusCode != 200 { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp1.StatusCode)} s.errors++ return } defer func() { if err := resp1.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() body, err := io.ReadAll(resp1.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} s.errors++ return } // Parse the JSON response var domainsCount DomainsCountResponse err = json.Unmarshal(body, &domainsCount) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } // Make a single POST request to get all domains via download method apiUrl := "https://app.netlas.io/api/domains/download/" query := fmt.Sprintf("domain:*.%s AND NOT domain:%s", domain, domain) requestBody := map[string]any{ "q": query, "fields": []string{"*"}, "source_type": "include", "size": domainsCount.Count, } jsonRequestBody, err := json.Marshal(requestBody) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error marshaling request body")} s.errors++ return } // Pick an API key randomApiKey = subscraping.PickRandom(s.apiKeys, s.Name()) s.requests++ resp2, err := session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, "Content-Type": "application/json"}, strings.NewReader(string(jsonRequestBody)), subscraping.BasicAuth{}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } defer func() { if err := resp2.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() body, err = io.ReadAll(resp2.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} s.errors++ return } if resp2.StatusCode == 429 { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp2.StatusCode)} s.errors++ return } // Parse the response body and extract the domain values var data []Item err = json.Unmarshal(body, &data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, item := range data { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: item.Data.Domain}: s.results++ } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "netlas" } func (s *Source) IsDefault() bool { return false } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/onyphe/onyphe.go ================================================ // Package onyphe logic package onyphe import ( "context" "encoding/json" "fmt" "net/http" "net/url" "strconv" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type OnypheResponse struct { Error int `json:"error"` Results []Result `json:"results"` Page int `json:"page"` PageSize int `json:"page_size"` Total int `json:"total"` MaxPage int `json:"max_page"` } type Result struct { Subdomains []string `json:"subdomains"` Hostname string `json:"hostname"` Forward string `json:"forward"` Reverse string `json:"reverse"` Host string `json:"host"` Domain string `json:"domain"` } type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } headers := map[string]string{"Content-Type": "application/json", "Authorization": "bearer " + randomApiKey} page := 1 pageSize := 1000 for { select { case <-ctx.Done(): return default: } var resp *http.Response var err error urlWithQuery := fmt.Sprintf("https://www.onyphe.io/api/v2/search/?q=%s&page=%d&size=%d", url.QueryEscape("category:resolver domain:"+domain), page, pageSize) s.requests++ resp, err = session.Get(ctx, urlWithQuery, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var respOnyphe OnypheResponse err = json.NewDecoder(resp.Body).Decode(&respOnyphe) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) for _, record := range respOnyphe.Results { select { case <-ctx.Done(): return default: } for _, subdomain := range record.Subdomains { if subdomain != "" { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } } if record.Hostname != "" { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} s.results++ } if record.Forward != "" { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Forward} s.results++ } if record.Reverse != "" { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Reverse} s.results++ } } if len(respOnyphe.Results) == 0 || page >= respOnyphe.MaxPage { break } page++ } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "onyphe" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } type OnypheResponseRaw struct { Error int `json:"error"` Results []Result `json:"results"` Page json.RawMessage `json:"page"` PageSize json.RawMessage `json:"page_size"` Total json.RawMessage `json:"total"` MaxPage json.RawMessage `json:"max_page"` } func (o *OnypheResponse) UnmarshalJSON(data []byte) error { var raw OnypheResponseRaw if err := json.Unmarshal(data, &raw); err != nil { return err } o.Error = raw.Error o.Results = raw.Results if pageStr := string(raw.Page); pageStr != "" { if page, err := strconv.Atoi(pageStr); err == nil { o.Page = page } else { var pageStrQuoted string if err := json.Unmarshal(raw.Page, &pageStrQuoted); err == nil { if page, err := strconv.Atoi(pageStrQuoted); err == nil { o.Page = page } } } } if pageSizeStr := string(raw.PageSize); pageSizeStr != "" { if pageSize, err := strconv.Atoi(pageSizeStr); err == nil { o.PageSize = pageSize } else { var pageSizeStrQuoted string if err := json.Unmarshal(raw.PageSize, &pageSizeStrQuoted); err == nil { if pageSize, err := strconv.Atoi(pageSizeStrQuoted); err == nil { o.PageSize = pageSize } } } } if totalStr := string(raw.Total); totalStr != "" { if total, err := strconv.Atoi(totalStr); err == nil { o.Total = total } else { var totalStrQuoted string if err := json.Unmarshal(raw.Total, &totalStrQuoted); err == nil { if total, err := strconv.Atoi(totalStrQuoted); err == nil { o.Total = total } } } } if maxPageStr := string(raw.MaxPage); maxPageStr != "" { if maxPage, err := strconv.Atoi(maxPageStr); err == nil { o.MaxPage = maxPage } else { var maxPageStrQuoted string if err := json.Unmarshal(raw.MaxPage, &maxPageStrQuoted); err == nil { if maxPage, err := strconv.Atoi(maxPageStrQuoted); err == nil { o.MaxPage = maxPage } } } } return nil } type ResultRaw struct { Subdomains json.RawMessage `json:"subdomains"` Hostname json.RawMessage `json:"hostname"` Forward json.RawMessage `json:"forward"` Reverse json.RawMessage `json:"reverse"` Host json.RawMessage `json:"host"` Domain json.RawMessage `json:"domain"` } func (r *Result) UnmarshalJSON(data []byte) error { var raw ResultRaw if err := json.Unmarshal(data, &raw); err != nil { return err } var subdomains []string if err := json.Unmarshal(raw.Subdomains, &subdomains); err == nil { r.Subdomains = subdomains } else { var subdomainStr string if err := json.Unmarshal(raw.Subdomains, &subdomainStr); err == nil { r.Subdomains = []string{subdomainStr} } } if len(raw.Hostname) > 0 { var hostnameStr string if err := json.Unmarshal(raw.Hostname, &hostnameStr); err == nil { r.Hostname = hostnameStr } else { var hostnameArr []string if err := json.Unmarshal(raw.Hostname, &hostnameArr); err == nil && len(hostnameArr) > 0 { r.Hostname = hostnameArr[0] } } } if len(raw.Forward) > 0 { _ = json.Unmarshal(raw.Forward, &r.Forward) } if len(raw.Reverse) > 0 { _ = json.Unmarshal(raw.Reverse, &r.Reverse) } if len(raw.Host) > 0 { var hostStr string if err := json.Unmarshal(raw.Host, &hostStr); err == nil { r.Host = hostStr } else { var hostArr []string if err := json.Unmarshal(raw.Host, &hostArr); err == nil && len(hostArr) > 0 { r.Host = hostArr[0] } } } if len(raw.Domain) > 0 { var domainStr string if err := json.Unmarshal(raw.Domain, &domainStr); err == nil { r.Domain = domainStr } else { var domainArr []string if err := json.Unmarshal(raw.Domain, &domainArr); err == nil && len(domainArr) > 0 { r.Domain = domainArr[0] } } } return nil } ================================================ FILE: pkg/subscraping/sources/profundis/profundis.go ================================================ // Package profundis logic package profundis import ( "bufio" "bytes" "context" "encoding/json" "strings" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } requestBody, err := json.Marshal(map[string]string{"domain": domain}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } headers := map[string]string{ "Content-Type": "application/json", "X-API-KEY": randomApiKey, "Accept": "text/event-stream", } s.requests++ resp, err := session.Post(ctx, "https://api.profundis.io/api/v2/common/data/subdomains", "", headers, bytes.NewReader(requestBody)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } defer func() { if err := resp.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { select { case <-ctx.Done(): return default: } line := strings.TrimSpace(scanner.Text()) if line == "" { continue } select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: line}: s.results++ } } if err := scanner.Err(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() return results } func (s *Source) Name() string { return "profundis" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/pugrecon/pugrecon.go ================================================ // Package pugrecon logic package pugrecon import ( "bytes" "context" "encoding/json" "fmt" "net/http" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // pugreconResult stores a single result from the pugrecon API type pugreconResult struct { Name string `json:"name"` } // pugreconAPIResponse stores the response from the pugrecon API type pugreconAPIResponse struct { Results []pugreconResult `json:"results"` QuotaRemaining int `json:"quota_remaining"` Limited bool `json:"limited"` TotalResults int `json:"total_results"` Message string `json:"message"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } // Prepare POST request data postData := map[string]string{"domain_name": domain} bodyBytes, err := json.Marshal(postData) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to marshal request body: %w", err)} s.errors++ return } bodyReader := bytes.NewReader(bodyBytes) // Prepare headers headers := map[string]string{ "Authorization": "Bearer " + randomApiKey, "Content-Type": "application/json", "Accept": "application/json", } apiURL := "https://pugrecon.com/api/v1/domains" s.requests++ resp, err := session.HTTPRequest(ctx, http.MethodPost, apiURL, "", headers, bodyReader, subscraping.BasicAuth{}) // Use HTTPRequest for full header control if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } defer func() { if err := resp.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to close response body: %w", err)} s.errors++ } }() if resp.StatusCode != http.StatusOK { errorMsg := fmt.Sprintf("received status code %d", resp.StatusCode) // Attempt to read error message from body if possible var apiResp pugreconAPIResponse if json.NewDecoder(resp.Body).Decode(&apiResp) == nil && apiResp.Message != "" { errorMsg = fmt.Sprintf("%s: %s", errorMsg, apiResp.Message) } results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", errorMsg)} s.errors++ return } var response pugreconAPIResponse err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, subdomain := range response.Results { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name}: s.results++ } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "pugrecon" } // IsDefault returns false as this is not a default source. func (s *Source) IsDefault() bool { return false } // HasRecursiveSupport returns false as this source does not support recursive searches. func (s *Source) HasRecursiveSupport() bool { return false } // KeyRequirement returns the API key requirement level for this source. func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } // NeedsKey returns true as this source requires an API key. func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys adds the API keys for the source. func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } // Statistics returns the statistics for the source. func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/quake/quake.go ================================================ // Package quake logic package quake import ( "bytes" "context" "fmt" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type quakeResults struct { Code int `json:"code"` Message string `json:"message"` Data []struct { Service struct { HTTP struct { Host string `json:"host"` } `json:"http"` } } `json:"data"` Meta struct { Pagination struct { Total int `json:"total"` } `json:"pagination"` } `json:"meta"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } // quake api doc https://quake.360.cn/quake/#/help var pageSize = 500 var start = 0 var totalResults = -1 for { select { case <-ctx.Done(): return default: } var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start) s.requests++ resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ "Content-Type": "application/json", "X-QuakeToken": randomApiKey, }, bytes.NewReader(requestBody)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var response quakeResults err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) if response.Code != 0 { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), } s.errors++ return } if totalResults == -1 { totalResults = response.Meta.Pagination.Total } for _, quakeDomain := range response.Data { select { case <-ctx.Done(): return default: } subdomain := quakeDomain.Service.HTTP.Host if strings.ContainsAny(subdomain, "暂无权限") { continue } results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } if len(response.Data) == 0 || start+pageSize >= totalResults { break } start += pageSize } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "quake" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/rapiddns/rapiddns.go ================================================ // Package rapiddns is a RapidDNS Scraping Engine in Golang package rapiddns import ( "context" "fmt" "io" "regexp" "strconv" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) var pagePattern = regexp.MustCompile(`class="page-link" href="/subdomain/[^"]+\?page=(\d+)">`) // Source is the passive scraping agent type Source struct { timeTaken time.Duration errors int results int requests int } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) page := 1 maxPages := 1 for { select { case <-ctx.Done(): return default: } s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://rapiddns.io/subdomain/%s?page=%d&full=1", domain, page)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } body, err := io.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) src := string(body) for _, subdomain := range session.Extractor.Extract(src) { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } if maxPages == 1 { matches := pagePattern.FindAllStringSubmatch(src, -1) if len(matches) > 0 { lastMatch := matches[len(matches)-1] if len(lastMatch) > 1 { maxPages, _ = strconv.Atoi(lastMatch[1]) } } } if page >= maxPages { break } page++ } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "rapiddns" } func (s *Source) IsDefault() bool { return false } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { // no key needed } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/reconcloud/reconcloud.go ================================================ // Package reconcloud logic package reconcloud import ( "context" "fmt" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type reconCloudResponse struct { MsgType string `json:"msg_type"` RequestID string `json:"request_id"` OnCache bool `json:"on_cache"` Step string `json:"step"` CloudAssetsList []cloudAssetsList `json:"cloud_assets_list"` } type cloudAssetsList struct { Key string `json:"key"` Domain string `json:"domain"` CloudProvider string `json:"cloud_provider"` } // Source is the passive scraping agent type Source struct { timeTaken time.Duration errors int results int requests int } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var response reconCloudResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) if len(response.CloudAssetsList) > 0 { for _, cloudAsset := range response.CloudAssetsList { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain}: s.results++ } } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "reconcloud" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { // no key needed } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/reconeer/reconeer.go ================================================ package reconeer import ( "context" "encoding/json" "fmt" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type response struct { Subdomains []subdomain `json:"subdomains"` } type subdomain struct { Subdomain string `json:"subdomain"` } type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) headers := map[string]string{ "Accept": "application/json", } randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey != "" { headers["X-API-KEY"] = randomApiKey } apiURL := fmt.Sprintf("https://www.reconeer.com/api/domain/%s", domain) s.requests++ resp, err := session.Get(ctx, apiURL, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } defer session.DiscardHTTPResponse(resp) if resp.StatusCode != 200 { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} s.errors++ return } var responseData response decoder := json.NewDecoder(resp.Body) err = decoder.Decode(&responseData) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, result := range responseData.Subdomains { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Subdomain}: s.results++ } } }() return results } func (s *Source) Name() string { return "reconeer" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.OptionalKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/redhuntlabs/redhuntlabs.go ================================================ // Package redhuntlabs logic package redhuntlabs import ( "context" "fmt" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type Response struct { Subdomains []string `json:"subdomains"` Metadata ResponseMetadata `json:"metadata"` } type ResponseMetadata struct { ResultCount int `json:"result_count"` PageSize int `json:"page_size"` PageNumber int `json:"page_number"` } type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 pageSize := 1000 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" || !strings.Contains(randomApiKey, ":") { s.skipped = true return } randomApiInfo := strings.Split(randomApiKey, ":") if len(randomApiInfo) != 3 { s.skipped = true return } baseUrl := randomApiInfo[0] + ":" + randomApiInfo[1] requestHeaders := map[string]string{"X-BLOBR-KEY": randomApiInfo[2], "User-Agent": "subfinder"} getUrl := fmt.Sprintf("%s?domain=%s&page=1&page_size=%d", baseUrl, domain, pageSize) s.requests++ resp, err := session.Get(ctx, getUrl, "", requestHeaders) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)} session.DiscardHTTPResponse(resp) s.errors++ return } var response Response err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} session.DiscardHTTPResponse(resp) s.errors++ return } session.DiscardHTTPResponse(resp) if response.Metadata.ResultCount > pageSize { totalPages := (response.Metadata.ResultCount + pageSize - 1) / pageSize for page := 1; page <= totalPages; page++ { select { case <-ctx.Done(): return default: } getUrl = fmt.Sprintf("%s?domain=%s&page=%d&page_size=%d", baseUrl, domain, page, pageSize) s.requests++ resp, err := session.Get(ctx, getUrl, "", requestHeaders) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("encountered error: %v; note: if you get a 'limit has been reached' error, head over to https://devportal.redhuntlabs.com", err)} session.DiscardHTTPResponse(resp) s.errors++ return } err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} session.DiscardHTTPResponse(resp) s.errors++ continue } session.DiscardHTTPResponse(resp) for _, subdomain := range response.Subdomains { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } } else { for _, subdomain := range response.Subdomains { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } }() return results } func (s *Source) Name() string { return "redhuntlabs" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/riddler/riddler.go ================================================ // Package riddler logic package riddler import ( "bufio" "context" "fmt" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { timeTaken time.Duration errors int results int requests int } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://riddler.io/search?q=pld:%s&view_type=data_table", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { select { case <-ctx.Done(): session.DiscardHTTPResponse(resp) return default: } line := scanner.Text() if line == "" { continue } for _, subdomain := range session.Extractor.Extract(line) { select { case <-ctx.Done(): session.DiscardHTTPResponse(resp) return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } session.DiscardHTTPResponse(resp) }() return results } // Name returns the name of the source func (s *Source) Name() string { return "riddler" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { // no key needed } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/robtex/robtext.go ================================================ // Package robtex logic package robtex import ( "bufio" "bytes" "context" "fmt" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) const ( addrRecord = "A" iPv6AddrRecord = "AAAA" baseURL = "https://proapi.robtex.com/pdns" ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int skipped bool } type result struct { Rrname string `json:"rrname"` Rrdata string `json:"rrdata"` Rrtype string `json:"rrtype"` } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } headers := map[string]string{"Content-Type": "application/x-ndjson"} ips, err := enumerate(ctx, session, fmt.Sprintf("%s/forward/%s?key=%s", baseURL, domain, randomApiKey), headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, result := range ips { select { case <-ctx.Done(): return default: } if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord { domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s?key=%s", baseURL, result.Rrdata, randomApiKey), headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, result := range domains { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata}: s.results++ } } } } }() return results } func enumerate(ctx context.Context, session *subscraping.Session, targetURL string, headers map[string]string) ([]result, error) { var results []result resp, err := session.Get(ctx, targetURL, "", headers) if err != nil { session.DiscardHTTPResponse(resp) return results, err } defer session.DiscardHTTPResponse(resp) scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() if line == "" { continue } var response result err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response) if err != nil { return results, err } results = append(results, response) } return results, nil } // Name returns the name of the source func (s *Source) Name() string { return "robtex" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/rsecloud/rsecloud.go ================================================ package rsecloud import ( "context" "fmt" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type response struct { Count int `json:"count"` Data []string `json:"data"` Page int `json:"page"` PageSize int `json:"pagesize"` TotalPages int `json:"total_pages"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } headers := map[string]string{"Content-Type": "application/json", "X-API-Key": randomApiKey} fetchSubdomains := func(endpoint string) { page := 1 for { select { case <-ctx.Done(): return default: } s.requests++ resp, err := session.Get(ctx, fmt.Sprintf("https://api.rsecloud.com/api/v2/subdomains/%s/%s?page=%d", endpoint, domain, page), "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var rseCloudResponse response err = jsoniter.NewDecoder(resp.Body).Decode(&rseCloudResponse) session.DiscardHTTPResponse(resp) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, subdomain := range rseCloudResponse.Data { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } if page >= rseCloudResponse.TotalPages { break } page++ } } fetchSubdomains("active") fetchSubdomains("passive") }() return results } // Name returns the name of the source func (s *Source) Name() string { return "rsecloud" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/securitytrails/securitytrails.go ================================================ // Package securitytrails logic package securitytrails import ( "bytes" "context" "fmt" "net/http" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" "github.com/projectdiscovery/utils/ptr" ) type response struct { Meta struct { ScrollID string `json:"scroll_id"` } `json:"meta"` Records []struct { Hostname string `json:"hostname"` } `json:"records"` Subdomains []string `json:"subdomains"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } var scrollId string headers := map[string]string{"Content-Type": "application/json", "APIKEY": randomApiKey} for { select { case <-ctx.Done(): return default: } var resp *http.Response var err error if scrollId == "" { var requestBody = fmt.Appendf(nil, `{"query":"apex_domain='%s'"}`, domain) s.requests++ resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "", headers, bytes.NewReader(requestBody)) } else { s.requests++ resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/scroll/%s", scrollId), "", headers) } if err != nil && ptr.Safe(resp).StatusCode == 403 { s.requests++ resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", headers) } if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var securityTrailsResponse response err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) for _, record := range securityTrailsResponse.Records { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}: s.results++ } } for _, subdomain := range securityTrailsResponse.Subdomains { select { case <-ctx.Done(): return default: } if strings.HasSuffix(subdomain, ".") { subdomain += domain } else { subdomain = subdomain + "." + domain } results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } scrollId = securityTrailsResponse.Meta.ScrollID if scrollId == "" { break } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "securitytrails" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/shodan/shodan.go ================================================ // Package shodan logic package shodan import ( "context" "fmt" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } type dnsdbLookupResponse struct { Domain string `json:"domain"` Subdomains []string `json:"subdomains"` Result int `json:"result"` Error string `json:"error"` More bool `json:"more"` } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } page := 1 for { select { case <-ctx.Done(): return default: } searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s&page=%d", domain, randomApiKey, page) s.requests++ resp, err := session.SimpleGet(ctx, searchURL) if err != nil { session.DiscardHTTPResponse(resp) return } defer session.DiscardHTTPResponse(resp) var response dnsdbLookupResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } if response.Error != "" { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error), } s.errors++ return } for _, data := range response.Subdomains { select { case <-ctx.Done(): return default: } value := fmt.Sprintf("%s.%s", data, response.Domain) results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Subdomain, Value: value, } s.results++ } if !response.More { break } page++ } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "shodan" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/sitedossier/sitedossier.go ================================================ // Package sitedossier logic package sitedossier import ( "context" "fmt" "io" "net/http" "regexp" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // SleepRandIntn is the integer value to get the pseudo-random number // to sleep before find the next match const SleepRandIntn = 5 var reNext = regexp.MustCompile(``) // Source is the passive scraping agent type Source struct { timeTaken time.Duration errors int results int requests int } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) s.enumerate(ctx, session, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s", domain), results) }() return results } func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, results chan subscraping.Result) { select { case <-ctx.Done(): return default: } s.requests++ resp, err := session.SimpleGet(ctx, baseURL) isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound if err != nil && !isnotfound { results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } body, err := io.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) src := string(body) for _, subdomain := range session.Extractor.Extract(src) { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } match := reNext.FindStringSubmatch(src) if len(match) > 0 { s.enumerate(ctx, session, fmt.Sprintf("http://www.sitedossier.com%s", match[1]), results) } } // Name returns the name of the source func (s *Source) Name() string { return "sitedossier" } func (s *Source) IsDefault() bool { return false } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { // no key needed } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/thc/thc.go ================================================ // Package thc logic package thc import ( "bytes" "context" "encoding/json" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type response struct { Domains []struct { Domain string `json:"domain"` } `json:"domains"` NextPageState string `json:"next_page_state"` } // Source is the passive scraping agent type Source struct { timeTaken time.Duration errors int results int requests int skipped bool } type requestBody struct { Domain string `json:"domain"` PageState string `json:"page_state"` Limit int `json:"limit"` } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) var pageState string headers := map[string]string{"Content-Type": "application/json"} apiURL := "https://ip.thc.org/api/v1/lookup/subdomains" for { reqBody := requestBody{ Domain: domain, PageState: pageState, Limit: 1000, } bodyBytes, err := json.Marshal(reqBody) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } s.requests++ resp, err := session.Post(ctx, apiURL, "", headers, bytes.NewReader(bodyBytes)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var thcResponse response err = jsoniter.NewDecoder(resp.Body).Decode(&thcResponse) session.DiscardHTTPResponse(resp) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, domainRecord := range thcResponse.Domains { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domainRecord.Domain} s.results++ } pageState = thcResponse.NextPageState if pageState == "" { break } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "thc" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { // No API keys needed for THC } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/threatbook/threatbook.go ================================================ // Package threatbook logic package threatbook import ( "context" "fmt" "strconv" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type threatBookResponse struct { ResponseCode int64 `json:"response_code"` VerboseMsg string `json:"verbose_msg"` Data struct { Domain string `json:"domain"` SubDomains struct { Total string `json:"total"` Data []string `json:"data"` } `json:"sub_domains"` } `json:"data"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", randomApiKey, domain)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var response threatBookResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) if response.ResponseCode != 0 { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("code %d, %s", response.ResponseCode, response.VerboseMsg), } s.errors++ return } total, err := strconv.ParseInt(response.Data.SubDomains.Total, 10, 64) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } if total > 0 { for _, subdomain := range response.Data.SubDomains.Data { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "threatbook" } func (s *Source) IsDefault() bool { return false } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/threatcrowd/threatcrowd.go ================================================ package threatcrowd import ( "context" "encoding/json" "fmt" "io" "net/http" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // threatCrowdResponse represents the JSON response from the ThreatCrowd API. type threatCrowdResponse struct { ResponseCode string `json:"response_code"` Subdomains []string `json:"subdomains"` Undercount string `json:"undercount"` } // Source implements the subscraping.Source interface for ThreatCrowd. type Source struct { timeTaken time.Duration errors int results int requests int } // Run queries the ThreatCrowd API for the given domain and returns found subdomains. func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func(startTime time.Time) { defer func() { s.timeTaken = time.Since(startTime) close(results) }() url := fmt.Sprintf("http://ci-www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } s.requests++ resp, err := session.Client.Do(req) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } defer func() { if err := resp.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() if resp.StatusCode != http.StatusOK { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code: %d", resp.StatusCode)} s.errors++ return } body, err := io.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } var tcResponse threatCrowdResponse if err := json.Unmarshal(body, &tcResponse); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, subdomain := range tcResponse.Subdomains { if subdomain != "" { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } }(time.Now()) return results } // Name returns the name of the source. func (s *Source) Name() string { return "threatcrowd" } // IsDefault indicates whether this source is enabled by default. func (s *Source) IsDefault() bool { return false } // HasRecursiveSupport indicates if the source supports recursive searches. func (s *Source) HasRecursiveSupport() bool { return false } // KeyRequirement returns the API key requirement level for this source. func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } // NeedsKey indicates if the source requires an API key. func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } // AddApiKeys is a no-op since ThreatCrowd does not require an API key. func (s *Source) AddApiKeys(_ []string) {} // Statistics returns usage statistics. func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/threatminer/threatminer.go ================================================ // Package threatminer logic package threatminer import ( "context" "fmt" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type response struct { StatusCode string `json:"status_code"` StatusMessage string `json:"status_message"` Results []string `json:"results"` } // Source is the passive scraping agent type Source struct { timeTaken time.Duration errors int results int requests int } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } defer session.DiscardHTTPResponse(resp) var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, subdomain := range data.Results { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "threatminer" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { // no key needed } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/urlscan/urlscan.go ================================================ // Package urlscan logic package urlscan import ( "context" "fmt" "net/url" "strings" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) const ( // baseURL is the URLScan API base URL baseURL = "https://urlscan.io/api/v1/search/" // maxPages is the maximum number of pages to fetch maxPages = 5 // maxPerPage is the maximum results per page (URLScan max is 10000, but 100 is safer) maxPerPage = 100 ) // response represents the URLScan API response structure type response struct { Results []struct { Task struct { Domain string `json:"domain"` URL string `json:"url"` } `json:"task"` Page struct { Domain string `json:"domain"` URL string `json:"url"` } `json:"page"` Sort []interface{} `json:"sort"` } `json:"results"` HasMore bool `json:"has_more"` Total int `json:"total"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 s.skipped = false go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } headers := map[string]string{"api-key": randomApiKey} // Search with wildcard to get more subdomain results s.enumerate(ctx, domain, headers, session, results) }() return results } // enumerate performs the actual enumeration with pagination func (s *Source) enumerate(ctx context.Context, domain string, headers map[string]string, session *subscraping.Session, results chan subscraping.Result) { var searchAfter string currentPage := 0 for { select { case <-ctx.Done(): return default: } if currentPage >= maxPages { break } // Build search URL searchURL := fmt.Sprintf("%s?q=domain:%s&size=%d", baseURL, url.QueryEscape(domain), maxPerPage) if searchAfter != "" { searchURL += "&search_after=" + url.QueryEscape(searchAfter) } s.requests++ resp, err := session.Get(ctx, searchURL, "", headers) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) // Process results - extract subdomains from multiple fields for _, result := range data.Results { candidates := []string{ result.Task.Domain, result.Page.Domain, } // Also extract from URLs if present if result.Task.URL != "" { if u, err := url.Parse(result.Task.URL); err == nil { candidates = append(candidates, u.Hostname()) } } if result.Page.URL != "" { if u, err := url.Parse(result.Page.URL); err == nil { candidates = append(candidates, u.Hostname()) } } for _, candidate := range candidates { if candidate == "" { continue } for _, subdomain := range session.Extractor.Extract(candidate) { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } } // Check pagination conditions if !data.HasMore || len(data.Results) == 0 { break } // Get sort value for next page lastResult := data.Results[len(data.Results)-1] if len(lastResult.Sort) == 0 { break } // Build search_after parameter sortValues := make([]string, len(lastResult.Sort)) for i, v := range lastResult.Sort { switch val := v.(type) { case float64: sortValues[i] = fmt.Sprintf("%.0f", val) default: sortValues[i] = fmt.Sprintf("%v", v) } } searchAfter = strings.Join(sortValues, ",") currentPage++ } } // Name returns the name of the source func (s *Source) Name() string { return "urlscan" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/virustotal/virustotal.go ================================================ // Package virustotal logic package virustotal import ( "context" "fmt" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type response struct { Data []Object `json:"data"` Meta Meta `json:"meta"` } type Object struct { Id string `json:"id"` } type Meta struct { Cursor string `json:"cursor"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { return } var cursor = "" for { select { case <-ctx.Done(): return default: } var url = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain) if cursor != "" { url = fmt.Sprintf("%s&cursor=%s", url, cursor) } s.requests++ resp, err := session.Get(ctx, url, "", map[string]string{"x-apikey": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } defer func() { if err := resp.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return } for _, subdomain := range data.Data { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id}: s.results++ } } cursor = data.Meta.Cursor if cursor == "" { break } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "virustotal" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return true } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, Requests: s.requests, TimeTaken: s.timeTaken, Skipped: s.skipped, } } ================================================ FILE: pkg/subscraping/sources/waybackarchive/waybackarchive.go ================================================ // Package waybackarchive logic package waybackarchive import ( "bufio" "context" "fmt" "net/url" "strings" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Source is the passive scraping agent type Source struct { timeTaken time.Duration errors int results int requests int } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } defer session.DiscardHTTPResponse(resp) scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { select { case <-ctx.Done(): return default: } line := scanner.Text() if line == "" { continue } line, _ = url.QueryUnescape(line) for _, subdomain := range session.Extractor.Extract(line) { subdomain = strings.ToLower(subdomain) subdomain = strings.TrimPrefix(subdomain, "25") subdomain = strings.TrimPrefix(subdomain, "2f") select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}: s.results++ } } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "waybackarchive" } func (s *Source) IsDefault() bool { return false } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.NoKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(_ []string) { // no key needed } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go ================================================ // Package whoisxmlapi logic package whoisxmlapi import ( "context" "fmt" "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type response struct { Search string `json:"search"` Result Result `json:"result"` } type Result struct { Count int `json:"count"` Records []Record `json:"records"` } type Record struct { Domain string `json:"domain"` FirstSeen int `json:"firstSeen"` LastSeen int `json:"lastSeen"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } s.requests++ resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://subdomains.whoisxmlapi.com/api/v1?apiKey=%s&domainName=%s", randomApiKey, domain)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } session.DiscardHTTPResponse(resp) for _, record := range data.Result.Records { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}: s.results++ } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "whoisxmlapi" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/windvane/windvane.go ================================================ // Package windvane logic package windvane import ( "bytes" "context" "encoding/json" "net/http" "strconv" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) type response struct { Code int `json:"code"` Msg string `json:"msg"` Data responseData `json:"data"` } type responseData struct { List []domainEntry `json:"list"` PageResponse pageInfo `json:"page_response"` } type domainEntry struct { Domain string `json:"domain"` } type pageInfo struct { Total string `json:"total"` Count string `json:"count"` TotalPage string `json:"total_page"` } type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } headers := map[string]string{"Content-Type": "application/json", "X-Api-Key": randomApiKey} page := 1 count := 1000 for { select { case <-ctx.Done(): return default: } var resp *http.Response var err error requestBody, _ := json.Marshal(map[string]interface{}{"domain": domain, "page_request": map[string]int{"page": page, "count": count}}) s.requests++ resp, err = session.Post(ctx, "https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService/ListSubDomain", "", headers, bytes.NewReader(requestBody)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } defer session.DiscardHTTPResponse(resp) var windvaneResponse response err = json.NewDecoder(resp.Body).Decode(&windvaneResponse) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } for _, record := range windvaneResponse.Data.List { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain}: s.results++ } } pageInfo := windvaneResponse.Data.PageResponse var totalRecords, recordsPerPage int if totalRecords, err = strconv.Atoi(pageInfo.Total); err != nil { break } if recordsPerPage, err = strconv.Atoi(pageInfo.Count); err != nil { break } if (page-1)*recordsPerPage >= totalRecords { break } page++ } }() return results } func (s *Source) Name() string { return "windvane" } func (s *Source) IsDefault() bool { return true } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go ================================================ package zoomeyeapi import ( "context" "encoding/json" "fmt" "net/http" "strings" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // search results type zoomeyeResults struct { Status int `json:"status"` Total int `json:"total"` List []struct { Name string `json:"name"` Ip []string `json:"ip"` } `json:"list"` } // Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration errors int results int requests int skipped bool } // Run function returns all subdomains found with the service func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { results := make(chan subscraping.Result) s.errors = 0 s.results = 0 s.requests = 0 go func() { defer func(startTime time.Time) { s.timeTaken = time.Since(startTime) close(results) }(time.Now()) randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) if randomApiKey == "" { s.skipped = true return } randomApiInfo := strings.Split(randomApiKey, ":") if len(randomApiInfo) != 2 { s.skipped = true return } host := randomApiInfo[0] apiKey := randomApiInfo[1] headers := map[string]string{ "API-KEY": apiKey, "Accept": "application/json", "Content-Type": "application/json", } var pages = 1 for currentPage := 1; currentPage <= pages; currentPage++ { select { case <-ctx.Done(): return default: } api := fmt.Sprintf("https://api.%s/domain/search?q=%s&type=1&s=1000&page=%d", host, domain, currentPage) s.requests++ resp, err := session.Get(ctx, api, "", headers) isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden if err != nil { if !isForbidden { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) } return } var res zoomeyeResults err = json.NewDecoder(resp.Body).Decode(&res) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ _ = resp.Body.Close() return } _ = resp.Body.Close() pages = int(res.Total/1000) + 1 for _, r := range res.List { select { case <-ctx.Done(): return case results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name}: s.results++ } } } }() return results } // Name returns the name of the source func (s *Source) Name() string { return "zoomeyeapi" } func (s *Source) IsDefault() bool { return false } func (s *Source) HasRecursiveSupport() bool { return false } func (s *Source) KeyRequirement() subscraping.KeyRequirement { return subscraping.RequiredKey } func (s *Source) NeedsKey() bool { return s.KeyRequirement() == subscraping.RequiredKey } func (s *Source) AddApiKeys(keys []string) { s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, Skipped: s.skipped, Requests: s.requests, } } ================================================ FILE: pkg/subscraping/types.go ================================================ package subscraping import ( "context" "net/http" "time" "github.com/projectdiscovery/ratelimit" mapsutil "github.com/projectdiscovery/utils/maps" ) type CtxArg string const ( CtxSourceArg CtxArg = "source" ) type CustomRateLimit struct { Custom mapsutil.SyncLockMap[string, uint] } // BasicAuth request's Authorization header type BasicAuth struct { Username string Password string } // Statistics contains statistics about the scraping process type Statistics struct { TimeTaken time.Duration Requests int Errors int Results int Skipped bool } // KeyRequirement represents the API key requirement level for a source type KeyRequirement int const ( NoKey KeyRequirement = iota OptionalKey RequiredKey ) // Source is an interface inherited by each passive source type Source interface { // Run takes a domain as argument and a session object // which contains the extractor for subdomains, http client // and other stuff. Run(context.Context, string, *Session) <-chan Result // Name returns the name of the source. It is preferred to use lower case names. Name() string // IsDefault returns true if the current source should be // used as part of the default execution. IsDefault() bool // HasRecursiveSupport returns true if the current source // accepts subdomains (e.g. subdomain.domain.tld), // not just root domains. HasRecursiveSupport() bool // KeyRequirement returns the API key requirement level for this source KeyRequirement() KeyRequirement // NeedsKey returns true if the source requires an API key. // Deprecated: Use KeyRequirement() instead for more granular control. NeedsKey() bool AddApiKeys([]string) // Statistics returns the scrapping statistics for the source Statistics() Statistics } // SubdomainExtractor is an interface that defines the contract for subdomain extraction. type SubdomainExtractor interface { Extract(text string) []string } // Session is the option passed to the source, an option is created // uniquely for each source. type Session struct { //SubdomainExtractor Extractor SubdomainExtractor // Client is the current http client Client *http.Client // Rate limit instance MultiRateLimiter *ratelimit.MultiLimiter // Timeout is the timeout in seconds for requests Timeout int } // Result is a result structure returned by a source type Result struct { Type ResultType Source string Value string Error error } // ResultType is the type of result returned by the source type ResultType int // Types of results returned by the source const ( Subdomain ResultType = iota Error ) ================================================ FILE: pkg/subscraping/utils.go ================================================ package subscraping import ( "math/rand" "strings" "github.com/projectdiscovery/gologger" ) const MultipleKeyPartsLength = 2 func PickRandom[T any](v []T, sourceName string) T { var result T length := len(v) if length == 0 { gologger.Debug().Msgf("Cannot use the %s source because there was no API key/secret defined for it.", sourceName) return result } return v[rand.Intn(length)] } func CreateApiKeys[T any](keys []string, provider func(k, v string) T) []T { var result []T for _, key := range keys { if keyPartA, keyPartB, ok := createMultiPartKey(key); ok { result = append(result, provider(keyPartA, keyPartB)) } } return result } func createMultiPartKey(key string) (keyPartA, keyPartB string, ok bool) { parts := strings.Split(key, ":") ok = len(parts) == MultipleKeyPartsLength if ok { keyPartA = parts[0] keyPartB = parts[1] } return } ================================================ FILE: pkg/testutils/integration.go ================================================ package testutils import ( "fmt" "os" "os/exec" "strings" ) func RunSubfinderAndGetResults(debug bool, domain string, extra ...string) ([]string, error) { cmd := exec.Command("bash", "-c") cmdLine := fmt.Sprintf("echo %s | %s", domain, "./subfinder ") cmdLine += strings.Join(extra, " ") cmd.Args = append(cmd.Args, cmdLine) if debug { cmd.Args = append(cmd.Args, "-v") cmd.Stderr = os.Stderr fmt.Println(cmd.String()) } else { cmd.Args = append(cmd.Args, "-silent") } data, err := cmd.Output() if debug { fmt.Println(string(data)) } if err != nil { return nil, err } var parts []string items := strings.SplitSeq(string(data), "\n") for i := range items { if i != "" { parts = append(parts, i) } } return parts, nil } // TestCase is a single integration test case type TestCase interface { Execute() error }