[ { "path": ".dockerignore", "content": ".git\n.github\ndocs\nexamples\ngithub-action\nspec\ntmp\ncoverage\nlib\ndeadfinder\nAGENTS.md\nREADME.md\nSECURITY.md\naction.yml\n" }, { "path": ".github/FUNDING.yml", "content": "github: hahwul" }, { "path": ".github/dependabot.yml", "content": "version: 2\nupdates:\n - package-ecosystem: github-actions\n directory: /\n schedule:\n interval: weekly\n\n - package-ecosystem: docker\n directory: /\n schedule:\n interval: weekly\n\n - package-ecosystem: bundler\n directory: \"/\"\n schedule:\n interval: weekly\n target-branch: \"main\"\n" }, { "path": ".github/labeler.yml", "content": "---\nconfig:\n - changed-files:\n - any-glob-to-any-file:\n - shard.yml\n - shard.lock\n - .github/labeler.yml\ndependencies:\n - changed-files:\n - any-glob-to-any-file:\n - shard.yml\n - shard.lock\nworkflow:\n - changed-files:\n - any-glob-to-any-file:\n - .github/workflows/**\n - .github/labeler.yml\ngithub-action:\n - changed-files:\n - any-glob-to-any-file:\n - action.yml\ndocker:\n - changed-files:\n - any-glob-to-any-file:\n - Dockerfile\n - .dockerignore\n - .github/workflows/docker-ghcr.yml\n - .github/workflows/docker-build.yml\ncode:\n - changed-files:\n - any-glob-to-any-file:\n - src/**\n - spec/**\ndocumentation:\n - changed-files:\n - any-glob-to-any-file:\n - README.md\n - CHANGELOG.md\n - AGENTS.md\n - SECURITY.md\n - docs/**\n" }, { "path": ".github/workflows/ci.yml", "content": "---\nname: CI\non:\n push:\n branches: [main]\n pull_request:\n branches: [main]\n\njobs:\n spec:\n runs-on: ubuntu-latest\n strategy:\n fail-fast: false\n matrix:\n crystal-version: [\"1.19.1\", \"1.20.0\"]\n steps:\n - uses: actions/checkout@v6\n\n - name: Set up Crystal ${{ matrix.crystal-version }}\n uses: crystal-lang/install-crystal@v1\n with:\n crystal: ${{ matrix.crystal-version }}\n\n - name: Install cmake (lexbor dependency)\n run: sudo apt-get update && sudo apt-get install -y cmake\n\n - name: Install shards\n run: shards install\n\n - name: Run crystal spec\n run: crystal spec\n\n format:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n - uses: crystal-lang/install-crystal@v1\n - name: Check formatting\n run: crystal tool format --check src spec\n" }, { "path": ".github/workflows/compat.yml", "content": "---\nname: Compat Tests\non:\n push:\n branches: [main]\n pull_request:\n branches: [main]\njobs:\n compat:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n\n - name: Set up Ruby (harness driver)\n uses: ruby/setup-ruby@v1\n with:\n ruby-version: '3.4'\n bundler-cache: false\n\n - name: Install harness Ruby deps\n run: gem install --no-document toml-rb\n\n - name: Set up Crystal\n uses: crystal-lang/install-crystal@v1\n\n - name: Install cmake (for lexbor)\n run: sudo apt-get update && sudo apt-get install -y cmake\n\n - name: Build Crystal binary\n run: |\n shards install\n crystal build src/cli_main.cr -o deadfinder --release\n\n - name: Compat — Crystal implementation\n env:\n BIN: ./deadfinder\n run: ruby spec/compat/run.rb\n" }, { "path": ".github/workflows/contributors.yml", "content": "---\n name: Contributors\n on:\n push:\n branches: [main]\n workflow_dispatch:\n inputs:\n logLevel:\n description: manual run\n required: false\n default: ''\n permissions:\n contents: write\n pull-requests: write\n jobs:\n contributors:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n - uses: wow-actions/contributors-list@v1.2.1\n with:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n round: false\n includeBots: false\n svgPath: docs/static/images/CONTRIBUTORS.svg\n noCommit: true\n - uses: peter-evans/create-pull-request@v8.1.1\n with:\n token: ${{ secrets.GITHUB_TOKEN }}\n commit-message: \"chore: update contributors\"\n title: \"chore: update contributors\"\n body: Automated update of `docs/static/images/CONTRIBUTORS.svg`.\n branch: chore/update-contributors\n delete-branch: true\n" }, { "path": ".github/workflows/crystal-release.yml", "content": "---\nname: Crystal Release Builds\non:\n release:\n types: [published]\n workflow_dispatch:\n\npermissions:\n contents: write\n\nenv:\n CRYSTAL_BUILD_IMAGE: crystallang/crystal:1.19.1-alpine\n\njobs:\n build-linux:\n strategy:\n fail-fast: false\n matrix:\n include:\n - arch: x86_64\n runs-on: ubuntu-latest\n - arch: aarch64\n runs-on: ubuntu-24.04-arm\n runs-on: ${{ matrix.runs-on }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Build static binary (Alpine / musl)\n run: |\n docker run --rm -v \"$PWD\":/workspace -w /workspace \\\n ${{ env.CRYSTAL_BUILD_IMAGE }} \\\n sh -c 'apk add --no-cache cmake make g++ \\\n && shards install \\\n && crystal build src/cli_main.cr -o deadfinder --release --static --no-debug'\n\n - name: Package\n run: |\n # Docker container ran as root, so the binary lands as root-owned.\n # Reclaim ownership before chmod, otherwise it fails with EPERM.\n sudo chown \"$(id -u):$(id -g)\" deadfinder\n chmod +x deadfinder\n tar czf deadfinder-linux-${{ matrix.arch }}.tar.gz deadfinder\n sha256sum deadfinder-linux-${{ matrix.arch }}.tar.gz > deadfinder-linux-${{ matrix.arch }}.tar.gz.sha256\n\n - name: Upload to release\n if: github.event_name == 'release'\n uses: softprops/action-gh-release@v3\n with:\n files: |\n deadfinder-linux-${{ matrix.arch }}.tar.gz\n deadfinder-linux-${{ matrix.arch }}.tar.gz.sha256\n\n - name: Upload as workflow artifact\n if: github.event_name == 'workflow_dispatch'\n uses: actions/upload-artifact@v7\n with:\n name: deadfinder-linux-${{ matrix.arch }}\n path: |\n deadfinder-linux-${{ matrix.arch }}.tar.gz\n deadfinder-linux-${{ matrix.arch }}.tar.gz.sha256\n\n build-macos:\n # macOS x86_64 (macos-13) is no longer built — Apple's Intel transition\n # has shrunk GitHub's macos-13 runner pool to the point where releases\n # routinely sit in the queue indefinitely. Apple Silicon binaries cover\n # current macOS users; Intel users can `brew install` from source or run\n # the Apple Silicon binary under Rosetta.\n strategy:\n fail-fast: false\n matrix:\n include:\n - arch: arm64\n runs-on: macos-latest\n runs-on: ${{ matrix.runs-on }}\n steps:\n - uses: actions/checkout@v6\n\n - name: Install Crystal and cmake\n run: brew install crystal cmake\n\n - name: Build release binary\n run: |\n shards install\n crystal build src/cli_main.cr -o deadfinder --release --no-debug\n\n - name: Package\n run: |\n chmod +x deadfinder\n tar czf deadfinder-macos-${{ matrix.arch }}.tar.gz deadfinder\n shasum -a 256 deadfinder-macos-${{ matrix.arch }}.tar.gz > deadfinder-macos-${{ matrix.arch }}.tar.gz.sha256\n\n - name: Upload to release\n if: github.event_name == 'release'\n uses: softprops/action-gh-release@v3\n with:\n files: |\n deadfinder-macos-${{ matrix.arch }}.tar.gz\n deadfinder-macos-${{ matrix.arch }}.tar.gz.sha256\n\n - name: Upload as workflow artifact\n if: github.event_name == 'workflow_dispatch'\n uses: actions/upload-artifact@v7\n with:\n name: deadfinder-macos-${{ matrix.arch }}\n path: |\n deadfinder-macos-${{ matrix.arch }}.tar.gz\n deadfinder-macos-${{ matrix.arch }}.tar.gz.sha256\n" }, { "path": ".github/workflows/docker-build.yml", "content": "---\nname: Docker Build CI\n\non:\n pull_request:\n branches: [main]\n push:\n branches: [main]\n workflow_dispatch:\n\njobs:\n build-docker:\n strategy:\n fail-fast: false\n matrix:\n include:\n - arch: linux/amd64\n runner: ubuntu-latest\n - arch: linux/arm64\n runner: ubuntu-24.04-arm\n runs-on: ${{ matrix.runner }}\n steps:\n - uses: actions/checkout@v6\n - name: Setup Docker buildx\n uses: docker/setup-buildx-action@v4\n - name: Prepare platform slug\n id: platform\n run: echo \"slug=$(echo '${{ matrix.arch }}' | tr '/' '-')\" >> \"$GITHUB_OUTPUT\"\n - name: Extract Docker metadata\n id: meta\n uses: docker/metadata-action@v6\n with:\n images: ghcr.io/${{ github.repository }}\n - name: Build Docker image\n uses: docker/build-push-action@v7\n with:\n context: .\n push: false\n tags: ${{ steps.meta.outputs.tags }}\n labels: ${{ steps.meta.outputs.labels }}\n platforms: ${{ matrix.arch }}\n cache-from: type=gha,scope=build-${{ steps.platform.outputs.slug }}\n cache-to: type=gha,mode=max,scope=build-${{ steps.platform.outputs.slug }}\n" }, { "path": ".github/workflows/docker-ghcr.yml", "content": "---\nname: GHCR Publish\non:\n push:\n branches: [main]\n release:\n types: [published]\n workflow_dispatch:\n inputs:\n version:\n description: Version to build and tag (e.g., 2.0.0)\n required: true\n type: string\n\nenv:\n REGISTRY: ghcr.io\n IMAGE_NAME: ${{ github.repository }}\n\njobs:\n build:\n runs-on: ubuntu-latest\n permissions:\n contents: read\n packages: write\n id-token: write\n strategy:\n fail-fast: false\n matrix:\n platform: [linux/amd64, linux/arm64]\n steps:\n - uses: actions/checkout@v6\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@v4\n\n - name: Setup Docker buildx\n uses: docker/setup-buildx-action@v4\n\n - name: Log into ${{ env.REGISTRY }}\n if: github.event_name != 'pull_request'\n uses: docker/login-action@v4\n with:\n registry: ${{ env.REGISTRY }}\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Extract Docker metadata\n id: meta\n uses: docker/metadata-action@v6\n with:\n images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}\n\n - name: Prepare platform slug\n id: platform\n run: echo \"slug=$(echo '${{ matrix.platform }}' | tr '/' '-')\" >> \"$GITHUB_OUTPUT\"\n\n - name: Build and push by digest\n id: build\n uses: docker/build-push-action@v7\n with:\n context: .\n platforms: ${{ matrix.platform }}\n labels: ${{ steps.meta.outputs.labels }}\n outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true\n cache-from: type=gha,scope=ghcr-${{ steps.platform.outputs.slug }}\n cache-to: type=gha,mode=max,scope=ghcr-${{ steps.platform.outputs.slug }}\n # push-by-digest only pushes the image manifest; provenance wraps it in\n # a manifest list, so the reported digest would point at a list that\n # was never pushed and the merge step fails with \"not found\".\n provenance: false\n sbom: false\n\n - name: Export digest\n run: |\n mkdir -p /tmp/digests\n digest=\"${{ steps.build.outputs.digest }}\"\n touch \"/tmp/digests/${digest#sha256:}\"\n\n - name: Upload digest\n uses: actions/upload-artifact@v7\n with:\n name: digests-${{ steps.platform.outputs.slug }}\n path: /tmp/digests/*\n if-no-files-found: error\n retention-days: 1\n\n merge:\n runs-on: ubuntu-latest\n needs: build\n permissions:\n contents: read\n packages: write\n steps:\n - name: Download digests\n uses: actions/download-artifact@v8\n with:\n path: /tmp/digests\n pattern: digests-*\n merge-multiple: true\n\n - name: Setup Docker buildx\n uses: docker/setup-buildx-action@v4\n\n - name: Log into ${{ env.REGISTRY }}\n uses: docker/login-action@v4\n with:\n registry: ${{ env.REGISTRY }}\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Normalize dispatch version\n if: github.event_name == 'workflow_dispatch'\n id: normalize\n run: |\n RAW_VERSION=\"${{ inputs.version }}\"\n VERSION=\"${RAW_VERSION#v}\"\n echo \"version=$VERSION\" >> \"$GITHUB_OUTPUT\"\n\n - name: Extract Docker metadata (tags)\n id: meta\n uses: docker/metadata-action@v6\n with:\n images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}\n tags: |\n type=ref,event=branch\n type=semver,pattern={{version}},enable=${{ github.event_name == 'release' }}\n type=semver,pattern={{major}}.{{minor}},enable=${{ github.event_name == 'release' }}\n type=raw,value=latest,enable=${{ github.event_name == 'release' }}\n type=raw,value=${{ steps.normalize.outputs.version }},enable=${{ github.event_name == 'workflow_dispatch' }}\n type=raw,value=latest,enable=${{ github.event_name == 'workflow_dispatch' }}\n\n - name: Create manifest list and push\n working-directory: /tmp/digests\n run: |\n docker buildx imagetools create $(jq -cr '.tags | map(\"-t \" + .) | join(\" \")' <<< \"$DOCKER_METADATA_OUTPUT_JSON\") \\\n $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)\n\n - name: Inspect image\n run: docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}\n\n cleanup:\n runs-on: ubuntu-latest\n needs: [build, merge]\n if: always() && needs.build.result == 'success'\n permissions:\n packages: write\n steps:\n # The build matrix pushes per-platform digests with push-by-digest=true,\n # which leaves untagged manifests in GHCR after the merge job assembles\n # the multi-arch manifest list. Prune them so only tagged versions\n # (main, latest, semver) remain — run even if merge fails so orphaned\n # per-platform digests don't accumulate in the package listing.\n - name: Delete untagged GHCR versions\n uses: actions/delete-package-versions@v5\n with:\n package-name: deadfinder\n package-type: container\n delete-only-untagged-versions: 'true'\n min-versions-to-keep: 0\n" }, { "path": ".github/workflows/docs.yml", "content": "---\nname: Docs CI/CD\n\non:\n push:\n branches: [main]\n paths:\n - \"docs/**\"\n - \".github/workflows/docs.yml\"\n pull_request:\n branches: [main]\n paths:\n - \"docs/**\"\n - \".github/workflows/docs.yml\"\n workflow_dispatch:\n\npermissions:\n contents: write\n\njobs:\n build:\n if: github.event_name == 'pull_request'\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n - name: Build (no deploy)\n uses: hahwul/hwaro@main\n with:\n build_dir: \"docs\"\n build_only: true\n\n deploy:\n if: github.event_name != 'pull_request'\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n - name: Build and deploy to GitHub Pages\n uses: hahwul/hwaro@main\n with:\n build_dir: \"docs\"\n token: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".github/workflows/goyo-update.yml", "content": "name: Update Goyo Theme\n\non:\n schedule:\n # Run every Monday at 9:00 AM UTC\n - cron: \"0 9 * * 1\"\n workflow_dispatch: # Allow manual trigger\n\nenv:\n GIT_USER_NAME: \"hahwul\"\n GIT_USER_EMAIL: \"hahwul@gmail.com\"\n THEME_PATH: \"docs/themes/goyo\"\n\njobs:\n update-theme:\n runs-on: ubuntu-latest\n permissions:\n contents: write\n pull-requests: write\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v6\n with:\n submodules: true\n token: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Update Goyo submodule\n id: update\n run: |\n git config user.name \"${{ env.GIT_USER_NAME }}\"\n git config user.email \"${{ env.GIT_USER_EMAIL }}\"\n\n # Get current commit hash\n OLD_COMMIT=$(git rev-parse HEAD:${{ env.THEME_PATH }})\n\n # Update submodule to latest\n git submodule update --remote ${{ env.THEME_PATH }}\n git add ${{ env.THEME_PATH }}\n\n # Get new commit hash\n NEW_COMMIT=$(git --git-dir=${{ env.THEME_PATH }}/.git rev-parse HEAD)\n\n # Check if there are changes\n if [ \"$OLD_COMMIT\" != \"$NEW_COMMIT\" ]; then\n echo \"updated=true\" >> $GITHUB_OUTPUT\n echo \"old_commit=$OLD_COMMIT\" >> $GITHUB_OUTPUT\n echo \"new_commit=$NEW_COMMIT\" >> $GITHUB_OUTPUT\n else\n echo \"updated=false\" >> $GITHUB_OUTPUT\n fi\n\n - name: Create Pull Request\n if: steps.update.outputs.updated == 'true'\n uses: peter-evans/create-pull-request@v8\n with:\n token: ${{ secrets.GITHUB_TOKEN }}\n commit-message: \"Update Goyo theme to latest version\"\n title: \"Update Goyo theme\"\n body: |\n This PR updates the Goyo theme to the latest version.\n\n **Changes:** ${{ steps.update.outputs.old_commit }} → ${{ steps.update.outputs.new_commit }}\n\n Please review the [Goyo changelog](https://github.com/hahwul/goyo/releases) for details on what's new.\n\n ---\n *This PR was automatically created by the Update Goyo Theme workflow.*\n branch: update-goyo-theme\n delete-branch: true\n labels: dependencies, documentation\n" }, { "path": ".github/workflows/labeler.yml", "content": "---\n name: Pull Request Labeler\n on: [pull_request_target]\n jobs:\n labeler:\n permissions:\n contents: read\n pull-requests: write\n runs-on: ubuntu-latest\n steps:\n - uses: actions/labeler@v6" }, { "path": ".github/workflows/publish-snapcraft.yml", "content": "---\nname: Snapcraft Publish\non:\n release:\n types: [published]\n workflow_dispatch:\n inputs:\n logLevel:\n description: Log level\n required: true\n default: warning\n tags:\n description: Test scenario tags\n\njobs:\n snapcraft-releaser:\n runs-on: ubuntu-latest\n strategy:\n fail-fast: false\n matrix:\n platform:\n - amd64\n steps:\n - uses: actions/checkout@v6\n\n - name: Build snap\n id: build\n uses: canonical/action-build@v1\n\n - name: Publish snap to the stable channel\n if: github.event_name == 'release'\n uses: snapcore/action-publish@master\n env:\n SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.SNAP_STORE_LOGIN }}\n with:\n snap: ${{ steps.build.outputs.snap }}\n release: stable\n\n - name: Upload snap as workflow artifact\n if: github.event_name == 'workflow_dispatch'\n uses: actions/upload-artifact@v7\n with:\n name: deadfinder-snap-${{ matrix.platform }}\n path: ${{ steps.build.outputs.snap }}\n" }, { "path": ".github/workflows/release-apk.yml", "content": "---\nname: Build and Release .apk Package\non:\n workflow_dispatch:\n inputs:\n version:\n description: \"Version to build (e.g., 2.0.0)\"\n required: true\n type: string\n upload_to_release:\n description: \"Upload .apk to GitHub Release (requires existing tag)\"\n required: false\n type: boolean\n default: false\n workflow_run:\n workflows: [\"Crystal Release Builds\"]\n types: [completed]\n\npermissions:\n contents: write\n\njobs:\n build-apk:\n if: >-\n github.event_name == 'workflow_dispatch' ||\n (github.event.workflow_run.conclusion == 'success' &&\n github.event.workflow_run.event == 'release')\n strategy:\n fail-fast: false\n matrix:\n include:\n - arch: x86_64\n asset_arch: x86_64\n - arch: aarch64\n asset_arch: aarch64\n runs-on: ubuntu-latest\n container:\n image: alpine:latest\n steps:\n - name: Install build tools\n run: apk add --no-cache alpine-sdk sudo github-cli git\n\n - name: Trust workspace\n run: git config --global --add safe.directory \"$GITHUB_WORKSPACE\"\n\n - uses: actions/checkout@v6\n with:\n ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.version || github.event.workflow_run.head_branch }}\n\n - name: Resolve version\n run: |\n if [ \"${{ github.event_name }}\" = \"workflow_dispatch\" ]; then\n RAW=\"${{ github.event.inputs.version }}\"\n else\n RAW=\"${{ github.event.workflow_run.head_branch }}\"\n fi\n VERSION=\"${RAW#v}\"\n echo \"VERSION=$VERSION\" >> \"$GITHUB_ENV\"\n\n - name: Download prebuilt binary\n env:\n GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n run: |\n gh release download \"${{ env.VERSION }}\" \\\n --pattern \"deadfinder-linux-${{ matrix.asset_arch }}.tar.gz\" \\\n --output deadfinder.tar.gz\n tar xzf deadfinder.tar.gz\n chmod +x deadfinder\n\n - name: Setup abuild\n run: |\n adduser -D builder\n addgroup builder abuild\n echo \"builder ALL=(ALL) NOPASSWD: ALL\" >> /etc/sudoers\n sudo -u builder abuild-keygen -ain\n\n - name: Create APKBUILD\n run: |\n mkdir -p /home/builder/deadfinder\n cp deadfinder /home/builder/deadfinder/\n cp LICENSE /home/builder/deadfinder/\n cat > /home/builder/deadfinder/APKBUILD <\n pkgname=deadfinder\n pkgver=${{ env.VERSION }}\n pkgrel=0\n pkgdesc=\"Find dead (broken) links in web pages, URL lists, and sitemaps.\"\n url=\"https://github.com/hahwul/deadfinder\"\n arch=\"${{ matrix.arch }}\"\n license=\"MIT\"\n source=\"\"\n options=\"!check !strip !tracedeps\"\n\n package() {\n \tinstall -Dm755 \"\\$srcdir/../deadfinder\" \"\\$pkgdir/usr/bin/deadfinder\"\n \tinstall -Dm644 \"\\$srcdir/../LICENSE\" \"\\$pkgdir/usr/share/licenses/\\$pkgname/LICENSE\"\n }\n APKEOF\n sed -i 's/^ //' /home/builder/deadfinder/APKBUILD\n chown -R builder:builder /home/builder/deadfinder\n\n - name: Build .apk\n run: |\n cd /home/builder/deadfinder\n sudo -u builder -H CARCH=${{ matrix.arch }} abuild -F checksum\n sudo -u builder -H CARCH=${{ matrix.arch }} abuild -Fr\n\n - name: Collect artifacts\n run: |\n mkdir -p output\n # abuild emits deadfinder-${VERSION}-r${pkgrel}.apk without arch in\n # the filename, so rename it to include the arch and avoid x86_64 /\n # aarch64 jobs overwriting each other on the GitHub Release.\n for src in $(find /home/builder/packages -name \"*.apk\" ! -name \"APKINDEX*\"); do\n cp \"$src\" \"output/deadfinder-${{ env.VERSION }}-${{ matrix.arch }}.apk\"\n done\n ls -la output/\n\n - name: Upload artifact\n uses: actions/upload-artifact@v7\n with:\n name: deadfinder-${{ env.VERSION }}-${{ matrix.arch }}.apk\n path: output/*.apk\n\n - name: Upload .apk to Release\n if: github.event_name == 'workflow_run' || (github.event_name == 'workflow_dispatch' && inputs.upload_to_release)\n env:\n GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n run: |\n for f in output/*.apk; do\n gh release upload \"${{ env.VERSION }}\" \"$f\" --clobber\n done\n" }, { "path": ".github/workflows/release-aur.yml", "content": "---\nname: Publish AUR Package\non:\n workflow_dispatch:\n inputs:\n version:\n description: \"Version to publish (e.g., 2.0.0)\"\n required: true\n type: string\n workflow_run:\n workflows: [\"Crystal Release Builds\"]\n types: [completed]\n\njobs:\n publish-aur:\n if: >-\n github.event_name == 'workflow_dispatch' ||\n (github.event.workflow_run.conclusion == 'success' &&\n github.event.workflow_run.event == 'release')\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.version || github.event.workflow_run.head_branch }}\n\n - name: Resolve version\n run: |\n if [ \"${{ github.event_name }}\" = \"workflow_dispatch\" ]; then\n RAW=\"${{ github.event.inputs.version }}\"\n else\n RAW=\"${{ github.event.workflow_run.head_branch }}\"\n fi\n VERSION=\"${RAW#v}\"\n echo \"VERSION=$VERSION\" >> \"$GITHUB_ENV\"\n\n - name: Update PKGBUILD\n run: |\n sed -i \"s/^pkgver=.*/pkgver=${{ env.VERSION }}/\" aur/PKGBUILD\n sed -i \"s/^pkgrel=.*/pkgrel=1/\" aur/PKGBUILD\n cat aur/PKGBUILD\n\n - name: Publish to AUR\n uses: KSXGitHub/github-actions-deploy-aur@v4.1.3\n with:\n pkgname: deadfinder\n pkgbuild: aur/PKGBUILD\n commit_username: hahwul\n commit_email: hahwul@gmail.com\n ssh_private_key: ${{ secrets.AUR_SSH_PRIVATE_KEY }}\n" }, { "path": ".github/workflows/release-deb.yml", "content": "---\nname: Build and Release .deb Package\non:\n workflow_dispatch:\n inputs:\n version:\n description: \"Version to build (e.g., 2.0.0)\"\n required: true\n type: string\n upload_to_release:\n description: \"Upload .deb to GitHub Release (requires existing tag)\"\n required: false\n type: boolean\n default: false\n workflow_run:\n workflows: [\"Crystal Release Builds\"]\n types: [completed]\n\npermissions:\n contents: write\n\njobs:\n build-deb:\n if: >-\n github.event_name == 'workflow_dispatch' ||\n (github.event.workflow_run.conclusion == 'success' &&\n github.event.workflow_run.event == 'release')\n strategy:\n fail-fast: false\n matrix:\n include:\n - arch: amd64\n asset_arch: x86_64\n - arch: arm64\n asset_arch: aarch64\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.version || github.event.workflow_run.head_branch }}\n\n - name: Resolve version\n id: version\n run: |\n if [ \"${{ github.event_name }}\" = \"workflow_dispatch\" ]; then\n RAW=\"${{ github.event.inputs.version }}\"\n else\n RAW=\"${{ github.event.workflow_run.head_branch }}\"\n fi\n VERSION=\"${RAW#v}\"\n echo \"VERSION=$VERSION\" >> \"$GITHUB_ENV\"\n echo \"Resolved: $VERSION\"\n\n - name: Download prebuilt binary\n env:\n GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n run: |\n gh release download \"${{ env.VERSION }}\" \\\n --pattern \"deadfinder-linux-${{ matrix.asset_arch }}.tar.gz\" \\\n --output deadfinder.tar.gz\n tar xzf deadfinder.tar.gz\n chmod +x deadfinder\n\n - name: Build Debian package layout\n run: |\n PKGDIR=\"deadfinder_${{ env.VERSION }}_${{ matrix.arch }}\"\n mkdir -p \"$PKGDIR/DEBIAN\" \"$PKGDIR/usr/bin\" \"$PKGDIR/usr/share/doc/deadfinder\"\n cp deadfinder \"$PKGDIR/usr/bin/\"\n cp README.md \"$PKGDIR/usr/share/doc/deadfinder/\"\n cp LICENSE \"$PKGDIR/usr/share/doc/deadfinder/\"\n cat > \"$PKGDIR/DEBIAN/control\" <\n Description: Find dead (broken) links in web pages, URL lists, and sitemaps.\n EOF\n dpkg-deb --build \"$PKGDIR\"\n\n - name: Upload artifact\n uses: actions/upload-artifact@v7\n with:\n name: deadfinder_${{ env.VERSION }}_${{ matrix.arch }}.deb\n path: deadfinder_${{ env.VERSION }}_${{ matrix.arch }}.deb\n\n - name: Upload .deb to Release\n if: github.event_name == 'workflow_run' || (github.event_name == 'workflow_dispatch' && inputs.upload_to_release)\n env:\n GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n run: |\n gh release upload \"${{ env.VERSION }}\" \\\n \"deadfinder_${{ env.VERSION }}_${{ matrix.arch }}.deb\" --clobber\n" }, { "path": ".github/workflows/release-major-tag.yml", "content": "---\nname: Update Major Version Tag\non:\n release:\n types: [published]\n\npermissions:\n contents: write\n\n# Force-update the floating `v` tag (e.g. `v2`) to point at the\n# latest published .. release. Lets callers pin\n# `uses: hahwul/deadfinder@v2` and receive bug-fix patches automatically.\n# The `v` prefix is required — GitHub Actions rejects bare `2` as a\n# \"shortened commit SHA\". Skips pre-releases so RC tags don't displace\n# the stable pointer.\n\njobs:\n bump-major-tag:\n if: github.event.release.prerelease == false\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n with:\n fetch-depth: 0\n\n - name: Move v tag to release commit\n env:\n TAG: ${{ github.event.release.tag_name }}\n run: |\n set -e\n stripped=\"${TAG#v}\"\n major=\"${stripped%%.*}\"\n if ! [[ \"$major\" =~ ^[0-9]+$ ]]; then\n echo \"Skipping: derived major '$major' from tag '$TAG' is not numeric.\"\n exit 0\n fi\n movable=\"v${major}\"\n git config user.name \"github-actions[bot]\"\n git config user.email \"41898282+github-actions[bot]@users.noreply.github.com\"\n git tag -f \"$movable\" \"$TAG\"\n git push origin \"refs/tags/$movable\" --force\n echo \"Moved tag '$movable' → '$TAG'.\"\n" }, { "path": ".github/workflows/release-rpm.yml", "content": "---\nname: Build and Release .rpm Package\non:\n workflow_dispatch:\n inputs:\n version:\n description: \"Version to build (e.g., 2.0.0)\"\n required: true\n type: string\n upload_to_release:\n description: \"Upload .rpm to GitHub Release (requires existing tag)\"\n required: false\n type: boolean\n default: false\n workflow_run:\n workflows: [\"Crystal Release Builds\"]\n types: [completed]\n\npermissions:\n contents: write\n\njobs:\n build-rpm:\n if: >-\n github.event_name == 'workflow_dispatch' ||\n (github.event.workflow_run.conclusion == 'success' &&\n github.event.workflow_run.event == 'release')\n strategy:\n fail-fast: false\n matrix:\n include:\n - arch: x86_64\n asset_arch: x86_64\n - arch: aarch64\n asset_arch: aarch64\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n with:\n ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.version || github.event.workflow_run.head_branch }}\n\n - name: Resolve version\n run: |\n if [ \"${{ github.event_name }}\" = \"workflow_dispatch\" ]; then\n RAW=\"${{ github.event.inputs.version }}\"\n else\n RAW=\"${{ github.event.workflow_run.head_branch }}\"\n fi\n VERSION=\"${RAW#v}\"\n echo \"VERSION=$VERSION\" >> \"$GITHUB_ENV\"\n\n - name: Download prebuilt binary\n env:\n GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n run: |\n gh release download \"${{ env.VERSION }}\" \\\n --pattern \"deadfinder-linux-${{ matrix.asset_arch }}.tar.gz\" \\\n --output deadfinder.tar.gz\n tar xzf deadfinder.tar.gz\n chmod +x deadfinder\n\n - name: Set up Go\n uses: actions/setup-go@v6\n with:\n go-version: \"stable\"\n\n - name: Install nfpm\n run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest\n\n - name: Build .rpm\n run: |\n cat > nfpm.yaml <\n description: \"Find dead (broken) links in web pages, URL lists, and sitemaps.\"\n license: MIT\n contents:\n - src: deadfinder\n dst: /usr/bin/deadfinder\n file_info:\n mode: 0755\n - src: LICENSE\n dst: /usr/share/licenses/deadfinder/LICENSE\n file_info:\n mode: 0644\n EOF\n nfpm package --packager rpm --target deadfinder-${{ env.VERSION }}.${{ matrix.arch }}.rpm\n\n - name: Upload artifact\n uses: actions/upload-artifact@v7\n with:\n name: deadfinder-${{ env.VERSION }}.${{ matrix.arch }}.rpm\n path: deadfinder-${{ env.VERSION }}.${{ matrix.arch }}.rpm\n\n - name: Upload .rpm to Release\n if: github.event_name == 'workflow_run' || (github.event_name == 'workflow_dispatch' && inputs.upload_to_release)\n env:\n GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n run: |\n gh release upload \"${{ env.VERSION }}\" \\\n \"deadfinder-${{ env.VERSION }}.${{ matrix.arch }}.rpm\" --clobber\n" }, { "path": ".github/workflows/release-sbom.yml", "content": "---\nname: Generate and Upload SBOM\non:\n release:\n types: [published]\n workflow_dispatch:\n\npermissions:\n contents: write\n\njobs:\n generate-sbom:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n\n - name: Generate SBOM (CycloneDX, Crystal)\n uses: hahwul/cyclonedx-cr@v1.3.0\n with:\n shard_file: ./shard.yml\n lock_file: ./shard.lock\n output_file: ./sbom.xml\n output_format: xml\n spec_version: 1.6\n\n - name: Upload SBOM to Release\n if: github.event_name == 'release'\n uses: softprops/action-gh-release@v3\n with:\n files: ./sbom.xml\n token: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Upload SBOM as workflow artifact\n if: github.event_name == 'workflow_dispatch'\n uses: actions/upload-artifact@v7\n with:\n name: sbom\n path: ./sbom.xml\n" }, { "path": ".gitignore", "content": "/lib/\n/.shards/\n*.dwarf\n\n# Built binary\n/deadfinder\n\n# Release artifacts\n/deadfinder-*.tar.gz\n/deadfinder-*.tar.gz.sha256\n\n# Nix\n/result\n/result-*\n.direnv/\n\n# macOS\n.DS_Store\n\n# Hwaro docs site\ndocs/public/*\n" }, { "path": "AGENTS.md", "content": "# DeadFinder — Agent Guide\n\nDeadFinder is a CLI tool that finds broken links in web pages, sitemaps, and URL lists. It is written in **Crystal** (v2.x+). The legacy Ruby v1.x implementation lives on the `legacy/v1` branch.\n\nReference this file first; fall back to the source only when something here is stale.\n\n## Prerequisites\n\n- Crystal >= 1.19.1\n- cmake, make, g++ (for building the `lexbor` HTML parser)\n\n## Bootstrap\n\n```bash\nshards install\n```\n\n## Build\n\n```bash\n# Debug (fast compile, slower binary)\ncrystal build src/cli_main.cr -o deadfinder\n\n# Release (slow compile, fast binary)\ncrystal build src/cli_main.cr -o deadfinder --release --no-debug\n```\n\n## Test\n\n```bash\n# Unit specs\ncrystal spec\n\n# Cross-implementation compat harness (golden files from v1 Ruby output)\nBIN=\"./deadfinder\" ruby spec/compat/run.rb\n```\n\nThe compat harness requires `toml-rb` (`gem install toml-rb`) and spins up a local fixture HTTP server on a random port.\n\n## Run\n\n```bash\n./deadfinder url https://example.com\n./deadfinder file urls.txt\ncat urls.txt | ./deadfinder pipe\n./deadfinder sitemap https://example.com/sitemap.xml\n```\n\nFull flag list lives in `src/deadfinder/cli.cr` (the `OptionParser` block).\n\n## Layout\n\n```\nsrc/\n├── cli_main.cr # binary entry\n├── deadfinder.cr # module root (run_* dispatchers, output serialization)\n└── deadfinder/\n ├── cli.cr # OptionParser + subcommand routing\n ├── types.cr # Options + coverage structs\n ├── runner.cr # fiber workers, link extraction, HTTP calls\n ├── http_client.cr # HTTP::Client wrapper (proxy, CONNECT tunneling)\n ├── utils.cr # URL resolution helpers\n ├── url_pattern_matcher.cr # match/ignore regex with 1s timeout\n ├── logger.cr # silent/verbose/debug gating\n ├── completion.cr # bash/zsh/fish completion generators\n ├── visualizer.cr # PNG coverage chart (stumpy_png)\n └── version.cr\n\nspec/\n├── deadfinder_spec.cr\n├── spec_helper.cr\n├── deadfinder/ # unit specs per module\n└── compat/ # black-box harness (v1 golden files)\n```\n\n## Conventions\n\n- Output surface is stable: CLI flags, subcommands, and JSON/YAML/TOML/CSV shapes match v1 Ruby. The golden files in `spec/compat/golden/` lock this contract.\n- Resolved URLs must preserve the base URL's port (see `utils.cr::origin`). This was a v1 pain point; don't regress.\n- Silent default is `false` — the CLI emits logs by default. `-s` / `--silent` opts in.\n\n## CI\n\n- `.github/workflows/compat.yml` — Crystal build + compat harness on every PR\n- `.github/workflows/crystal-release.yml` — release-triggered builds for linux x86_64/aarch64 and macOS arm64; uploads tar.gz + sha256 as release assets\n- `.github/workflows/docker-build.yml` / `docker-ghcr.yml` — multi-arch image builds (Crystal static binary in Alpine)\n\n## Distribution channels\n\n| Channel | How it picks up a new release |\n|---|---|\n| GitHub Release binaries | `crystal-release.yml` auto-uploads on `release: published` |\n| Docker (`ghcr.io/hahwul/deadfinder`) | `docker-ghcr.yml` on push to main / release |\n| Homebrew (homebrew-core) | Manual PR via `brew bump-formula-pr` after tagging |\n| GitHub Action (`hahwul/deadfinder@`) | `action.yml` in repo root; downloads the release binary |\n\n## Legacy (Ruby v1) branch\n\nGem releases still happen on `legacy/v1`. Bug-fix and security updates only — no new features. Do not port v1 changes to main unless they are true behavioral fixes that should also apply to Crystal.\n" }, { "path": "CHANGELOG.md", "content": "# Changelog\n\nAll notable changes are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), versioning follows [SemVer](https://semver.org/).\n\n## [Unreleased]\n\n## [2.0.2]\n\n### Fixed\n- `action.yml`: save the downloaded release tarball under its real filename (`deadfinder-linux-x86_64.tar.gz` etc.) instead of a generic `deadfinder.tar.gz`, so `sha256sum -c` can resolve the path referenced inside the sidecar. Composite-action callers hit `sha256sum: deadfinder-linux-x86_64.tar.gz: No such file or directory` right after a successful download — the earlier 2.0.0 YAML parser error was masking this. Surfaced by owasp-noir/noir run #24651380673.\n\n## [2.0.1]\n\n### Fixed\n- `action.yml`: quote the `version` input description so its embedded `(default: latest)` doesn't trip strict YAML parsers used by the GitHub Actions runner. Caller workflows on `uses: hahwul/deadfinder@2.0.0` saw `Mapping values are not allowed in this context.` and failed at job startup.\n- `scripts/version_update.cr`: constrain `^version:\\s*.+$/m` patterns with `[^\\n]+` — Crystal's `m` flag enables both line-anchor and DOTALL semantics, so `.+$` greedily ate the rest of the file and truncated `shard.yml`/`snap/snapcraft.yaml`/`aur/PKGBUILD` on the first 2.0.1 bump attempt.\n\n## [2.0.0] — Crystal rewrite\n\n### Added\n- Crystal implementation (fiber-based concurrency via `spawn` + `Channel`) replaces the Ruby gem as the supported runtime.\n- Multi-platform release binaries auto-attached on every GitHub Release: linux x86_64/aarch64 (static/musl), macOS arm64. Each tarball ships alongside a `.sha256` sidecar. (Intel macOS isn't shipped as a prebuilt — see [installation docs](https://hahwul.github.io/deadfinder/docs/getting-started/installation/) for source/Rosetta options.)\n- Cross-implementation compatibility harness (`spec/compat/`) — black-box golden files captured from v1 Ruby output, locking the CLI/output contract for Crystal.\n- GitHub Action migrated to a composite action that downloads the release binary and verifies its sha256 before running. The `version` input (defaulting to `latest`) lets callers pin a specific release. `worker_headers` is now a first-class input.\n- Docker image rebuilt on Crystal static binary (`alpine:3.21` runtime, `<15 MB`). OCI labels, semver tags (`2.0.0` / `2.0` / `latest`), and keyless cosign signatures on every published tag.\n\n### Changed\n- Repository layout: Crystal at the root. `src/`, `spec/`, `shard.yml`, `shard.lock` live at the top level; the old `crystal/` subdirectory is gone.\n- CLI flag behavior aligns with Ruby v1 exactly — the compat harness enforces this. No user-visible flag renames.\n- `--silent` default remains `false`; `-s` opts in. (An earlier Crystal port defaulted silent to `true`; that regression was fixed before the 2.0.0 cut.)\n- `--user_agent`, `--proxy_auth`, `--worker_headers` use underscores (as implemented). Prior dashed forms never worked reliably in the old Docker-based action; the new composite action passes the correct names.\n\n### Fixed\n- Resolved URLs preserve the base URL's non-default port for both `href=\"/path\"` and `href=\"relative/path\"` shapes (was dropping the port in the Crystal port).\n- Docker-based GitHub Action chain: previously relied on a Ruby-gem image and a brittle entrypoint.sh; replaced with a composite action that downloads the release binary directly.\n\n### Removed\n- Ruby gem publishing from `main`. The gem continues on the [`legacy/v1`](https://github.com/hahwul/deadfinder/tree/legacy/v1) branch for bug-fix and security releases only.\n- `lib/`, `bin/`, `Gemfile`, `Gemfile.lock`, `Rakefile`, `deadfinder.gemspec`, `gemset.nix`, `.rubocop.yml`, `ruby-version`, Ruby-based `flake.nix`, and the legacy Ruby spec suite.\n- `github-action/Dockerfile` + `entrypoint.sh` (replaced by composite action in `action.yml`).\n\n### Migration from v1\n\n| You had | Switch to |\n|---|---|\n| `gem install deadfinder` | `brew install deadfinder` or prebuilt binary from the release |\n| `bundle exec deadfinder …` | Same binary on `PATH`, no bundler |\n| Docker image (same name) | No change — the image now ships the Crystal binary |\n| `uses: hahwul/deadfinder@…` | No change — the action now uses the Crystal binary under the hood |\n| `require 'deadfinder'` | Library usage is gone from main. If you depend on it, pin to a v1 gem release or use the CLI. |\n\nIf you need a bugfix in v1, open an issue/PR against the [`legacy/v1`](https://github.com/hahwul/deadfinder/tree/legacy/v1) branch.\n\n---\n\nHistory prior to 2.0.0 was not maintained in this file. See [GitHub Releases](https://github.com/hahwul/deadfinder/releases?q=prerelease%3Afalse) and the [`legacy/v1`](https://github.com/hahwul/deadfinder/tree/legacy/v1) branch for v1 release history.\n\n[Unreleased]: https://github.com/hahwul/deadfinder/compare/2.0.2...HEAD\n[2.0.2]: https://github.com/hahwul/deadfinder/releases/tag/2.0.2\n[2.0.1]: https://github.com/hahwul/deadfinder/releases/tag/2.0.1\n[2.0.0]: https://github.com/hahwul/deadfinder/releases/tag/2.0.0\n" }, { "path": "Dockerfile", "content": "FROM crystallang/crystal:1.20.2-alpine AS builder\n\nRUN apk add --no-cache cmake make g++ git\n\nWORKDIR /build\nCOPY shard.yml shard.lock ./\nCOPY src/ ./src/\n\nRUN shards install\nRUN crystal build src/cli_main.cr -o /build/deadfinder --release --static --no-debug\n\nFROM alpine:3.23\n\nLABEL org.opencontainers.image.title=\"DeadFinder\"\nLABEL org.opencontainers.image.description=\"Find dead links (broken links).\"\nLABEL org.opencontainers.image.authors=\"HAHWUL \"\nLABEL org.opencontainers.image.source=\"https://github.com/hahwul/deadfinder\"\nLABEL org.opencontainers.image.documentation=\"https://github.com/hahwul/deadfinder\"\nLABEL org.opencontainers.image.licenses=\"MIT\"\n\nLABEL \"com.github.actions.name\"=\"DeadFinder\"\nLABEL \"com.github.actions.description\"=\"Find dead (broken) links in files, URLs, or sitemaps\"\nLABEL \"com.github.actions.icon\"=\"link\"\nLABEL \"com.github.actions.color\"=\"red\"\n\nENV LC_ALL=C.UTF-8\n\nRUN apk add --no-cache ca-certificates\nCOPY --from=builder /build/deadfinder /usr/local/bin/deadfinder\nCMD [\"deadfinder\"]\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2026 hahwul \n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "
\n \"DeadFinder\n

Find dead-links (broken links)

\n
\n\n

\n\n\n\n\n

\n\n

\n Documentation •\n Installation •\n Github Action •\n Contributing •\n Changelog\n

\n\nDead link (broken link) means a link within a web page that cannot be connected. These links can have a negative impact to SEO and Security. This tool makes it easy to identify and modify.\n\n![](https://github.com/user-attachments/assets/92129de9-90c6-41e0-a424-883fe30858f6)\n\n> **Looking for v1 (Ruby gem)?** It now lives on the [`legacy/v1`](https://github.com/hahwul/deadfinder/tree/legacy/v1) branch and continues to publish the `deadfinder` gem for bug-fix and security releases. `main` hosts the Crystal rewrite (v2+).\n\n## Installation\n\n### Homebrew\n```bash\nbrew install deadfinder\n# https://formulae.brew.sh/formula/deadfinder\n```\n\n### Docker\n```bash\ndocker run ghcr.io/hahwul/deadfinder:latest deadfinder url https://example.com\n```\n\n### Prebuilt binary\nDownload the archive for your platform from the [latest release](https://github.com/hahwul/deadfinder/releases/latest), extract, and place `deadfinder` on your `PATH`.\n\n### Nix\n```bash\nnix run github:hahwul/deadfinder\nnix profile install github:hahwul/deadfinder\nnix develop github:hahwul/deadfinder\n```\n\n### Build from source\nRequires Crystal >= 1.19.1 and `cmake` (for the `lexbor` HTML parser's postinstall — without it `shards install` fails with `'cmake': No such file or directory`).\n\n```bash\n# macOS\nbrew install crystal cmake\n\n# Debian / Ubuntu\nsudo apt install crystal cmake\n```\n\n```bash\nshards install\ncrystal build src/cli_main.cr -o deadfinder --release\n# or: just build\n```\n\n## Using In\n### CLI\n```bash\ndeadfinder sitemap https://www.hahwul.com/sitemap.xml\n```\n\n### GitHub Action\nPin a specific release tag. `@latest` is **not** a valid Actions ref.\n\n```yml\nsteps:\n- name: Run DeadFinder\n uses: hahwul/deadfinder@v2 # tracks the latest 2.x — pin a specific tag (e.g. @2.0.2) for stricter reproducibility\n id: broken-link\n with:\n command: sitemap # url / file / sitemap / pipe\n target: https://www.hahwul.com/sitemap.xml\n # timeout: 10\n # concurrency: 50\n # silent: false\n # headers: \"X-API-Key: 123444\"\n # worker_headers: \"User-Agent: Deadfinder Bot\"\n # include30x: false\n # user_agent: \"Apple\"\n # proxy: \"http://localhost:8070\"\n # proxy_auth: \"id:pw\"\n # match: \"\"\n # ignore: \"\"\n # coverage: true\n # visualize: report.png\n\n- name: Output Handling\n run: echo '${{ steps.broken-link.outputs.output }}'\n```\n\nIf you have found a Dead Link and want to automatically add it as an issue, please refer to the \"[Automating Dead Link Detection](https://www.hahwul.com/2024/10/20/automating-dead-link-detection/)\" article.\n\n## Usage\n```\nUsage: deadfinder [options]\n\nCommands:\n pipe Scan the URLs from STDIN\n file Scan the URLs from File\n url Scan the Single URL\n sitemap Scan the URLs from sitemap\n completion Generate completion script (bash/zsh/fish)\n version Show version\n\nOptions:\n -r, --include30x Include 30x redirections as dead links\n -c, --concurrency=N Number of concurrent workers (default: 50)\n -t, --timeout=N Timeout in seconds (default: 10)\n -o, --output=FILE File to write results\n -f, --output_format=FORMAT Output format: json, yaml, toml, csv, sarif (default: json)\n -H, --headers=HEADER Custom HTTP headers for initial request\n --worker_headers=HEADER Custom HTTP headers for worker requests\n --user_agent=UA User-Agent string\n -p, --proxy=PROXY Proxy server (HTTP and HTTPS CONNECT)\n --proxy_auth=USER:PASS Proxy authentication\n -m, --match=PATTERN Match URL pattern (regex)\n -i, --ignore=PATTERN Ignore URL pattern (regex)\n -s, --silent Silent mode\n -v, --verbose Verbose mode\n --debug Debug mode\n --limit=N Limit number of URLs to scan\n --coverage Enable coverage tracking and reporting\n --visualize=PATH Generate visualization PNG\n```\n\n## Modes\n```bash\n# Scan the URLs from STDIN (multiple URLs)\ncat urls.txt | deadfinder pipe\n\n# Scan the URLs from a file\ndeadfinder file urls.txt\n\n# Scan a single URL\ndeadfinder url https://www.hahwul.com\n\n# Scan the URLs from a sitemap\ndeadfinder sitemap https://www.hahwul.com/sitemap.xml\n```\n\n## JSON Handling\n```bash\ndeadfinder sitemap https://www.hahwul.com/sitemap.xml -o output.json\ncat output.json | jq\n```\n\n```json\n{\n \"Target URL\": [\n \"DeadLink URL\",\n \"DeadLink URL\",\n \"DeadLink URL\"\n ]\n}\n```\n\nWith `--coverage`:\n\n```bash\ndeadfinder sitemap https://www.hahwul.com/sitemap.xml --coverage -o output.json\n```\n\n```json\n{\n \"dead_links\": {\n \"Target URL\": [\"DeadLink URL 1\", \"DeadLink URL 2\"]\n },\n \"coverage\": {\n \"targets\": {\n \"Target URL\": {\n \"total_tested\": 14,\n \"dead_links\": 7,\n \"coverage_percentage\": 50.0\n }\n },\n \"summary\": {\n \"total_tested\": 14,\n \"total_dead\": 7,\n \"overall_coverage_percentage\": 50.0\n }\n }\n}\n```\n\n## Shell Completion\n```bash\ndeadfinder completion bash > /etc/bash_completion.d/deadfinder\ndeadfinder completion zsh > ~/.zsh/completion/_deadfinder\ndeadfinder completion fish > ~/.config/fish/completions/deadfinder.fish\n```\n\n## Contributing\n\nContributions are welcome! If you have an idea for an improvement or want to report a bug:\n\n- **Fork the repository.**\n- **Create a new branch** for your feature or bug fix (e.g., `feature/awesome-feature` or `bugfix/annoying-bug`).\n- **Make your changes.**\n- **Commit your changes** with a clear message.\n- **Push** to the branch.\n- **Submit a Pull Request (PR)** to our `main` branch.\n\n### Contributors\n\n![](docs/static/images/CONTRIBUTORS.svg)\n" }, { "path": "SECURITY.md", "content": "# Security Policy\n\n## Reporting a Vulnerability\n\nFound a security issue? Let us know so we can fix it.\n\n### How to Report\n\n* **For general security concerns**, please open a [GitHub issue](https://github.com/hahwul/deadfinder/issues). Use the `security` label and describe the issue in as much detail as you can. This helps us to understand and address the problem more effectively.\n* **For sensitive matters**, we encourage you to directly report it on our [GitHub security page](https://github.com/hahwul/deadfinder/security). Handling these issues discreetly is vital for everyone's safety.\n\n## Conclusion\nYour vigilance and willingness to report security issues are what help keep our project robust and secure. We appreciate the time and effort you put into making our community a safer place. Remember, no concern is too small; we're here to listen and act. Together, we can ensure a secure environment for all our users and contributors. Thank you for being an essential part of our project's security.\n\nThank you for your support in maintaining the security and integrity of our project!" }, { "path": "action.yml", "content": "---\nname: DeadFinder Action\ndescription: A GitHub Action to find and report dead (broken) links in files, URLs, or sitemaps.\nbranding:\n icon: link\n color: red\ninputs:\n command:\n description: The type of command to execute (e.g.,file, url, sitemap)\n required: true\n target:\n description: The target resource for the command (e.g., file path, URL, or sitemap URL)\n required: true\n timeout:\n description: The maximum time to wait for each request, in seconds\n required: false\n default: \"\"\n concurrency:\n description: The number of concurrent requests to make\n required: false\n default: \"\"\n silent:\n description: Enable silent mode to suppress output\n required: false\n default: \"false\"\n headers:\n description: Custom HTTP headers to include in requests, separated by commas\n required: false\n default: \"\"\n worker_headers:\n description: Custom HTTP headers for worker requests, separated by commas\n required: false\n default: \"\"\n verbose:\n description: Enable verbose mode for detailed logging\n required: false\n default: \"false\"\n include30x:\n description: Include HTTP 30x status codes in the results\n required: false\n default: \"false\"\n user_agent:\n description: User-Agent string to use for requests\n required: false\n default: \"\"\n proxy:\n description: Proxy server to use for requests\n required: false\n default: \"\"\n proxy_auth:\n description: Proxy server authentication credentials\n required: false\n default: \"\"\n match:\n description: Match the URL with the given pattern\n required: false\n default: \"\"\n ignore:\n description: Ignore the URL with the given pattern\n required: false\n default: \"\"\n coverage:\n description: Enable coverage reporting to show dead link ratios\n required: false\n default: \"false\"\n visualize:\n description: Generate a visualization of the scan results (e.g., report.png)\n required: false\n default: \"\"\n version:\n description: \"DeadFinder release tag to download (default: latest)\"\n required: false\n default: \"latest\"\noutputs:\n output:\n description: JSON formatted result of the dead-link check\n value: ${{ steps.scan.outputs.output }}\nruns:\n using: composite\n steps:\n - name: Detect platform\n id: platform\n shell: bash\n run: |\n case \"${RUNNER_OS}-${RUNNER_ARCH}\" in\n Linux-X64) asset=\"deadfinder-linux-x86_64.tar.gz\" ;;\n Linux-ARM64) asset=\"deadfinder-linux-aarch64.tar.gz\" ;;\n macOS-ARM64) asset=\"deadfinder-macos-arm64.tar.gz\" ;;\n macOS-X64)\n echo \"::error title=macOS Intel not supported::DeadFinder no longer ships a macOS x86_64 prebuilt binary. Use an Apple Silicon (macos-latest) runner, or install from source via 'brew install deadfinder'.\"\n exit 1\n ;;\n *) echo \"::error::Unsupported platform: ${RUNNER_OS}-${RUNNER_ARCH}\"; exit 1 ;;\n esac\n echo \"asset=${asset}\" >> \"$GITHUB_OUTPUT\"\n\n - name: Download deadfinder binary\n shell: bash\n run: |\n set -e\n version=\"${{ inputs.version }}\"\n asset=\"${{ steps.platform.outputs.asset }}\"\n if [ \"${version}\" = \"latest\" ]; then\n base_url=\"https://github.com/hahwul/deadfinder/releases/latest/download\"\n else\n base_url=\"https://github.com/hahwul/deadfinder/releases/download/${version}\"\n fi\n echo \"Downloading ${base_url}/${asset}\"\n # The sha256 sidecar was generated with the tarball's real filename\n # (deadfinder-linux-x86_64.tar.gz etc.), so we must save the download\n # under the same name for `sha256sum -c` to resolve it.\n if ! curl -fsSL \"${base_url}/${asset}\" -o \"/tmp/${asset}\"; then\n echo \"::error title=DeadFinder binary not found::Failed to download ${base_url}/${asset}\"\n echo \"::error::Common causes:\"\n echo \"::error:: 1. Using 'uses: hahwul/deadfinder@main' or '@latest' — neither resolves to a release.\"\n echo \"::error:: → Pin a released ref instead: uses: hahwul/deadfinder@v2 (latest 2.x) or @2.0.2 (exact).\"\n echo \"::error:: 2. Requested version (input: version=${version}) is not a published release tag.\"\n echo \"::error:: → See https://github.com/hahwul/deadfinder/releases for available tags.\"\n echo \"::error:: 3. Using a v1.x workflow with a v2 ref — v1 users should pin hahwul/deadfinder@1.10.0.\"\n exit 1\n fi\n if ! curl -fsSL \"${base_url}/${asset}.sha256\" -o \"/tmp/${asset}.sha256\"; then\n echo \"::error::Downloaded ${asset} but its .sha256 sidecar is missing at ${base_url}/${asset}.sha256\"\n exit 1\n fi\n cd /tmp\n # macOS runners ship `shasum`, Linux ships `sha256sum`.\n if command -v sha256sum >/dev/null 2>&1; then\n sha256sum -c \"${asset}.sha256\"\n else\n shasum -a 256 -c \"${asset}.sha256\"\n fi\n tar xzf \"${asset}\"\n chmod +x deadfinder\n ./deadfinder version\n\n - name: Run deadfinder\n id: scan\n shell: bash\n env:\n DF_COMMAND: ${{ inputs.command }}\n DF_TARGET: ${{ inputs.target }}\n DF_TIMEOUT: ${{ inputs.timeout }}\n DF_CONCURRENCY: ${{ inputs.concurrency }}\n DF_SILENT: ${{ inputs.silent }}\n DF_HEADERS: ${{ inputs.headers }}\n DF_WORKER_HEADERS: ${{ inputs.worker_headers }}\n DF_VERBOSE: ${{ inputs.verbose }}\n DF_INCLUDE30X: ${{ inputs.include30x }}\n DF_USER_AGENT: ${{ inputs.user_agent }}\n DF_PROXY: ${{ inputs.proxy }}\n DF_PROXY_AUTH: ${{ inputs.proxy_auth }}\n DF_MATCH: ${{ inputs.match }}\n DF_IGNORE: ${{ inputs.ignore }}\n DF_COVERAGE: ${{ inputs.coverage }}\n DF_VISUALIZE: ${{ inputs.visualize }}\n run: |\n set -e\n args=( \"${DF_COMMAND}\" \"${DF_TARGET}\" -o /tmp/output.json -f json )\n [ -n \"${DF_TIMEOUT}\" ] && args+=( --timeout=\"${DF_TIMEOUT}\" )\n [ -n \"${DF_CONCURRENCY}\" ] && args+=( --concurrency=\"${DF_CONCURRENCY}\" )\n [ \"${DF_SILENT}\" = \"true\" ] && args+=( --silent )\n [ \"${DF_VERBOSE}\" = \"true\" ] && args+=( --verbose )\n [ \"${DF_INCLUDE30X}\" = \"true\" ] && args+=( --include30x )\n [ -n \"${DF_USER_AGENT}\" ] && args+=( --user_agent=\"${DF_USER_AGENT}\" )\n [ -n \"${DF_PROXY}\" ] && args+=( --proxy=\"${DF_PROXY}\" )\n [ -n \"${DF_PROXY_AUTH}\" ] && args+=( --proxy_auth=\"${DF_PROXY_AUTH}\" )\n [ -n \"${DF_MATCH}\" ] && args+=( --match=\"${DF_MATCH}\" )\n [ -n \"${DF_IGNORE}\" ] && args+=( --ignore=\"${DF_IGNORE}\" )\n [ \"${DF_COVERAGE}\" = \"true\" ] && args+=( --coverage )\n [ -n \"${DF_VISUALIZE}\" ] && args+=( --visualize=\"${DF_VISUALIZE}\" )\n\n if [ -n \"${DF_HEADERS}\" ]; then\n IFS=',' read -ra hdrs <<< \"${DF_HEADERS}\"\n for h in \"${hdrs[@]}\"; do\n [ -n \"${h}\" ] && args+=( -H \"${h}\" )\n done\n fi\n\n if [ -n \"${DF_WORKER_HEADERS}\" ]; then\n IFS=',' read -ra whdrs <<< \"${DF_WORKER_HEADERS}\"\n for h in \"${whdrs[@]}\"; do\n [ -n \"${h}\" ] && args+=( --worker_headers=\"${h}\" )\n done\n fi\n\n /tmp/deadfinder \"${args[@]}\"\n\n if [ ! -f /tmp/output.json ]; then\n echo \"::error::/tmp/output.json was not produced\"\n exit 1\n fi\n\n if command -v jq >/dev/null 2>&1; then\n encoded=$(jq -c . /tmp/output.json)\n else\n encoded=$(tr -d '\\n' < /tmp/output.json)\n fi\n echo \"output=${encoded}\" >> \"$GITHUB_OUTPUT\"\n" }, { "path": "aur/PKGBUILD", "content": "# Maintainer: HAHWUL \npkgname=deadfinder\npkgver=2.0.2\npkgrel=1\npkgdesc=\"Find dead (broken) links in web pages, URL lists, and sitemaps\"\narch=('x86_64' 'aarch64')\nurl=\"https://github.com/hahwul/deadfinder\"\nlicense=('MIT')\n\nsource_x86_64=(\"${pkgname}-${pkgver}-x86_64.tar.gz::${url}/releases/download/${pkgver}/deadfinder-linux-x86_64.tar.gz\")\nsource_aarch64=(\"${pkgname}-${pkgver}-aarch64.tar.gz::${url}/releases/download/${pkgver}/deadfinder-linux-aarch64.tar.gz\")\nsource=(\"LICENSE-${pkgver}::https://raw.githubusercontent.com/hahwul/deadfinder/${pkgver}/LICENSE\")\nsha256sums=('SKIP')\nsha256sums_x86_64=('SKIP')\nsha256sums_aarch64=('SKIP')\n\npackage() {\n install -Dm755 \"${srcdir}/deadfinder\" \"${pkgdir}/usr/bin/${pkgname}\"\n install -Dm644 \"${srcdir}/LICENSE-${pkgver}\" \"${pkgdir}/usr/share/licenses/${pkgname}/LICENSE\"\n}\n" }, { "path": "docs/AGENTS.md", "content": "# AGENTS.md - AI Agent Instructions for Hwaro Site\n\nThis document provides instructions for AI agents working on this Hwaro-generated website.\n\n## Project Overview\n\nThis is a static website built with [Hwaro](https://github.com/hahwul/hwaro), a fast and lightweight static site generator written in Crystal.\n\n## Essential Commands\n\n| Command | Description |\n|---------|-------------|\n| `hwaro build` | Build the site to `public/` directory |\n| `hwaro serve` | Start development server with live reload |\n| `hwaro new ` | Create new content from archetype |\n| `hwaro deploy` | Deploy the site (requires configuration) |\n| `hwaro build --drafts` | Include draft content |\n| `hwaro serve -p 8080` | Serve on custom port (default: 3000) |\n| `hwaro build --base-url \"https://example.com\"` | Set base URL for production |\n\n## Directory Structure\n\n```\n.\n├── config.toml # Site configuration\n├── content/ # Markdown content files\n│ ├── _index.md # Homepage content\n│ └── blog/ # Blog section\n│ ├── _index.md # Section listing page\n│ └── *.md # Individual pages\n├── templates/ # Jinja2 templates (Crinja)\n│ ├── base.html # Base layout (optional)\n│ ├── page.html # Page template\n│ ├── section.html # Section listing template\n│ └── shortcodes/ # Shortcode templates\n├── static/ # Static assets (copied as-is)\n└── archetypes/ # Content templates for `hwaro new`\n```\n\n## Notes for AI Agents\n\n1. **Front matter is TOML** (`+++`), not YAML (`---`).\n2. **Rendered content** is `{{ content | safe }}`, not `{{ page.content }}`.\n3. **Custom metadata** is `page.extra.field`, not `page.params.field`.\n4. **Always preview** with `hwaro serve` before committing.\n5. **Validate TOML syntax** in config.toml and front matter after edits.\n6. **Use `{{ base_url }}` prefix** for URLs in templates.\n7. **Escape user content** with `{{ value | escape }}` in templates.\n\n## Full Reference\n\nFor detailed documentation on content, templates, configuration, and more:\n\n- [Hwaro Documentation](https://hwaro.hahwul.com)\n- [Configuration Guide](https://hwaro.hahwul.com/start/config/)\n- [Full LLM Reference](https://hwaro.hahwul.com/llms-full.txt) — comprehensive reference optimized for AI agents\n\nTo generate the full embedded AGENTS.md locally, run:\n```\nhwaro tool agents-md --local --write\n```\n\n## Site-Specific Instructions\n\n" }, { "path": "docs/config.toml", "content": "# =============================================================================\n# Site Configuration\n# =============================================================================\n\ntitle = \"DeadFinder\"\ndescription = \"Find dead (broken) links in web pages, URL lists, and sitemaps.\"\nbase_url = \"https://deadfinder.hahwul.com\"\n\n# =============================================================================\n# Plugins\n# =============================================================================\n\n[plugins]\nprocessors = [\"markdown\"]\n\n# =============================================================================\n# Content Files\n# =============================================================================\n\n[content.files]\nallow_extensions = [\"jpg\", \"jpeg\", \"png\", \"gif\", \"svg\", \"webp\"]\n\n# =============================================================================\n# Syntax Highlighting\n# =============================================================================\n\n[highlight]\nenabled = true\ntheme = \"monokai\"\nuse_cdn = true\n\n# =============================================================================\n# Taxonomies\n# =============================================================================\n\n[[taxonomies]]\nname = \"tags\"\nfeed = true\nsitemap = false\n\n# =============================================================================\n# Sitemap\n# =============================================================================\n\n[sitemap]\nenabled = true\nfilename = \"sitemap.xml\"\nchangefreq = \"weekly\"\npriority = 0.5\n\n# =============================================================================\n# Markdown Configuration\n# =============================================================================\n\n[markdown]\nsafe = false\nlazy_loading = false\nemoji = false\n\n# =============================================================================\n# Search (client-side, Fuse.js)\n# =============================================================================\n\n[search]\nenabled = true\nformat = \"fuse_json\"\nfields = [\"title\", \"content\", \"description\"]\nfilename = \"search.json\"\n\n# =============================================================================\n# OpenGraph & Twitter Cards\n# =============================================================================\n# Default meta tags for social sharing. Page-level front matter overrides.\n\n[og]\ntype = \"website\"\ntwitter_card = \"summary_large_image\"\n# twitter_site = \"@hahwul\"\n# twitter_creator = \"@hahwul\"\n\n# =============================================================================\n# Auto OG Images\n# =============================================================================\n# Auto-generate 1200x630 OG preview images for pages without a custom `image`.\n# https://hwaro.hahwul.com/features/og-images/\n\n[og.auto_image]\nenabled = true\nformat = \"png\"\nbackground = \"#0a0f0a\"\ntext_color = \"#e8ede8\"\naccent_color = \"#22c55e\"\nfont_size = 52\nstyle = \"dots\"\npattern_opacity = 0.12\npattern_scale = 1.0\nlogo = \"static/images/deadfinder.webp\"\nlogo_position = \"bottom-left\"\noutput_dir = \"og-images\"\nshow_title = true\n\n# =============================================================================\n# Pagination (Optional)\n# =============================================================================\n\n# [pagination]\n# enabled = false\n# per_page = 10\n\n# =============================================================================\n# Series (Optional)\n# =============================================================================\n# Group posts into ordered series\n\n# [series]\n# enabled = true\n\n# =============================================================================\n# Related Posts (Optional)\n# =============================================================================\n# Recommend related content based on shared taxonomy terms\n\n# [related]\n# enabled = true\n# limit = 5\n# taxonomies = [\"tags\"]\n\n# =============================================================================\n# Robots.txt\n# =============================================================================\n# Controls search engine crawler access\n\n[robots]\nenabled = true\nfilename = \"robots.txt\"\nrules = [\n { user_agent = \"*\", allow = [\"/\"] }\n]\n\n# =============================================================================\n# LLMs.txt\n# =============================================================================\n# Instructions for AI/LLM crawlers\n\n[llms]\nenabled = true\nfilename = \"llms.txt\"\ninstructions = \"This is documentation for DeadFinder, an open-source CLI that finds broken links in web pages, URL lists, and sitemaps. Content is MIT-licensed.\"\nfull_enabled = true\nfull_filename = \"llms-full.txt\"\n\n# =============================================================================\n# RSS/Atom Feeds\n# =============================================================================\n# Generates RSS or Atom feed for content syndication\n\n# [feeds]\n# enabled = true\n# type = \"rss\"\n# limit = 10\n# full_content = true\n# sections = []\n\n# =============================================================================\n# Build Hooks (Optional)\n# =============================================================================\n# Run custom shell commands before/after build process\n\n# [build]\n# hooks.pre = [\"npm install\"]\n# hooks.post = [\"npm run minify\"]\n\n# =============================================================================\n# Permalinks (Optional)\n# =============================================================================\n# Override the output path for specific sections or taxonomies\n\n# [permalinks]\n# posts = \"/posts/:year/:month/:slug/\"\n# tags = \"/topic/:slug/\"\n\n# =============================================================================\n# Auto Includes (Optional)\n# =============================================================================\n# Automatically load CSS/JS files from static directories\n\n# [auto_includes]\n# enabled = true\n# dirs = [\"assets/css\", \"assets/js\"]\n\n# =============================================================================\n# Asset Pipeline (Optional)\n# =============================================================================\n\n# [assets]\n# enabled = true\n# minify = true\n# fingerprint = true\n\n# =============================================================================\n# Deployment (Optional)\n# =============================================================================\n\n# [deployment]\n# target = \"prod\"\n# source_dir = \"public\"\n#\n# [[deployment.targets]]\n# name = \"prod\"\n# url = \"file://./out\"\n\n# =============================================================================\n# Image Processing (Optional)\n# =============================================================================\n# Automatic image resizing and LQIP (Low-Quality Image Placeholder) generation\n# Uses vendored stb libraries — no external tools required.\n# Use resize_image() in templates to generate responsive variants.\n\n# [image_processing]\n# enabled = true\n# widths = [320, 640, 1024, 1280]\n# quality = 85\n#\n# [image_processing.lqip]\n# enabled = true\n# width = 32 # Placeholder width in pixels (8-128)\n# quality = 20 # JPEG quality for placeholder (1-100, lower = smaller)\n\n# =============================================================================\n# PWA (Progressive Web App) (Optional)\n# =============================================================================\n# Generate manifest.json and service worker for offline access\n\n# [pwa]\n# enabled = true\n# name = \"My Site\"\n# short_name = \"Site\"\n# theme_color = \"#ffffff\"\n# background_color = \"#ffffff\"\n# display = \"standalone\"\n# icons = [\"static/icon-192.png\", \"static/icon-512.png\"]\n\n# =============================================================================\n# AMP (Accelerated Mobile Pages) (Optional)\n# =============================================================================\n# Generate AMP-compliant versions of content pages\n\n# [amp]\n# enabled = true\n# path_prefix = \"amp\"\n# sections = [\"posts\"]\n" }, { "path": "docs/content/about.md", "content": "+++\ntitle = \"About\"\ndescription = \"About DeadFinder\"\n+++\n\nDeadFinder detects broken links — 4xx, 5xx, optionally 3xx — on any page, URL list, or sitemap. It's built for automation: one static binary, machine-readable output, and a GitHub Action wrapper so CI pipelines can gate on link health.\n\n## Status\n\n- **Current line**: 2.x, Crystal rewrite.\n- **Legacy**: 1.x, original Ruby gem — frozen except for bug fixes on the [`legacy/v1`](https://github.com/hahwul/deadfinder/tree/legacy/v1) branch.\n\n## Source\n\n- Repository: [github.com/hahwul/deadfinder](https://github.com/hahwul/deadfinder)\n- License: MIT\n- Maintainer: [HAHWUL](https://www.hahwul.com)\n\n## Reporting issues\n\nPlease use the [GitHub issue tracker](https://github.com/hahwul/deadfinder/issues). Security-sensitive reports go through the [GitHub security page](https://github.com/hahwul/deadfinder/security).\n" }, { "path": "docs/content/docs/_index.md", "content": "+++\ntitle = \"Documentation\"\ndescription = \"DeadFinder documentation\"\nsort_by = \"weight\"\n+++\n\nStart with [Installation](/docs/getting-started/installation/) and the [Quick Start](/docs/getting-started/quickstart/). The **Usage** section covers every subcommand, output format, and filter. **Integration** shows how to call DeadFinder from GitHub Actions or Docker. **Reference** is the full CLI flag table.\n" }, { "path": "docs/content/docs/getting-started/_index.md", "content": "+++\ntitle = \"Getting Started\"\ndescription = \"Install DeadFinder and run your first scan.\"\nweight = 1\nsort_by = \"weight\"\n+++\n\nTwo steps:\n\n1. [Install](/docs/getting-started/installation/) the binary.\n2. [Run your first scan](/docs/getting-started/quickstart/).\n" }, { "path": "docs/content/docs/getting-started/installation.md", "content": "+++\ntitle = \"Installation\"\ndescription = \"Install DeadFinder via Homebrew, Docker, prebuilt binary, Nix, or from source.\"\nweight = 1\n+++\n\nPick the channel that fits your environment. All paths produce the same CLI.\n\n## Homebrew (macOS / Linux)\n\n```bash\nbrew install deadfinder\n```\n\n## Docker\n\nImage: [`ghcr.io/hahwul/deadfinder`](https://github.com/hahwul/deadfinder/pkgs/container/deadfinder). Multi-arch (linux/amd64, linux/arm64). Each published tag is cosign-signed.\n\n```bash\ndocker run ghcr.io/hahwul/deadfinder:latest deadfinder url https://example.com\n```\n\n## Prebuilt binary\n\nDownload the tarball for your platform from [Releases](https://github.com/hahwul/deadfinder/releases/latest) (a `.sha256` sidecar ships alongside each tarball):\n\n| OS | Arch | Asset |\n|---|---|---|\n| Linux | x86_64 | `deadfinder-linux-x86_64.tar.gz` |\n| Linux | aarch64 | `deadfinder-linux-aarch64.tar.gz` |\n| macOS | arm64 | `deadfinder-macos-arm64.tar.gz` |\n\n> Intel macOS (`x86_64`) doesn't have a prebuilt binary — use `brew install deadfinder` (builds from source) or run the Apple Silicon binary under Rosetta.\n\nExtract and put `deadfinder` on your `PATH`:\n\n```bash\ncurl -fsSL https://github.com/hahwul/deadfinder/releases/latest/download/deadfinder-linux-x86_64.tar.gz \\\n | tar xz\nsudo mv deadfinder /usr/local/bin/\n```\n\n## Linux package managers\n\n| Distro | Package |\n|---|---|\n| Debian / Ubuntu | `deadfinder_X.Y.Z_{amd64,arm64}.deb` from Releases |\n| RHEL / Fedora | `deadfinder-X.Y.Z.{x86_64,aarch64}.rpm` from Releases |\n| Alpine | `deadfinder-X.Y.Z-r0.{x86_64,aarch64}.apk` from Releases |\n| Arch Linux | `yay -S deadfinder` (AUR) |\n| Snap | `sudo snap install deadfinder` |\n\n## Nix\n\n```bash\nnix run github:hahwul/deadfinder\nnix profile install github:hahwul/deadfinder\nnix develop github:hahwul/deadfinder\n```\n\n## Build from source\n\nPrerequisites:\n\n- Crystal >= 1.19.1\n- `cmake` — required by the `lexbor` HTML parser's postinstall step. Without it, `shards install` fails with `Error executing process: 'cmake': No such file or directory`.\n\n```bash\n# macOS\nbrew install crystal cmake\n\n# Debian / Ubuntu\nsudo apt install crystal cmake\n\n# Arch Linux\nsudo pacman -S crystal cmake\n```\n\nThen build:\n\n```bash\ngit clone https://github.com/hahwul/deadfinder\ncd deadfinder\nshards install\ncrystal build src/cli_main.cr -o deadfinder --release --no-debug\n```\n\nOr use the [`justfile`](https://github.com/hahwul/deadfinder/blob/main/justfile) recipes:\n\n```bash\njust build # release binary\njust build-debug # fast debug build\njust test # run specs\n```\n" }, { "path": "docs/content/docs/getting-started/quickstart.md", "content": "+++\ntitle = \"Quick Start\"\ndescription = \"Run your first DeadFinder scan and read its output.\"\nweight = 2\n+++\n\n## Scan a single URL\n\n```bash\ndeadfinder url https://www.example.com\n```\n\nThe terminal shows discovered links and their status:\n\n```\n▶ Fetching https://www.example.com\n ● Discovered 12 URLs, currently checking them. [anchor:8 / link:4]\n ├── ✓ [200] https://www.example.com/about\n ├── ✘ [404] https://www.example.com/old-page\n └── ● Task completed\n```\n\nExit code is `0` even when dead links exist — parse the output to make a build pass/fail decision.\n\n## Structured output\n\nWrite JSON to a file:\n\n```bash\ndeadfinder url https://www.example.com -o output.json\ncat output.json\n```\n\n```json\n{\n \"https://www.example.com\": [\n \"https://www.example.com/old-page\"\n ]\n}\n```\n\nYAML, TOML, CSV, and SARIF are available via `-f `. See [Output formats](/docs/usage/output-formats/).\n\n## Scan a sitemap\n\n```bash\ndeadfinder sitemap https://www.example.com/sitemap.xml -o results.json\n```\n\n## Scan many URLs\n\nFrom a file:\n\n```bash\ncat > urls.txt <<'EOF'\nhttps://www.example.com\nhttps://docs.example.com\nEOF\n\ndeadfinder file urls.txt -o results.json\n```\n\nFrom STDIN:\n\n```bash\nprintf 'https://www.example.com\\nhttps://docs.example.com\\n' \\\n | deadfinder pipe -o results.json\n```\n\n## Coverage report\n\n`--coverage` adds a per-target summary with dead-link percentage:\n\n```bash\ndeadfinder sitemap https://www.example.com/sitemap.xml --coverage -o results.json\n```\n\nOptionally render a PNG chart:\n\n```bash\ndeadfinder sitemap https://www.example.com/sitemap.xml --coverage --visualize report.png\n```\n\n## Next\n\n- [Subcommands](/docs/usage/subcommands/)\n- [Output formats](/docs/usage/output-formats/)\n- [CLI flags reference](/docs/reference/cli-flags/)\n" }, { "path": "docs/content/docs/integration/_index.md", "content": "+++\ntitle = \"Integration\"\ndescription = \"Run DeadFinder from GitHub Actions or Docker.\"\nweight = 3\nsort_by = \"weight\"\n+++\n\n- [GitHub Action](/docs/integration/github-action/) — official composite action that downloads the release binary and verifies its sha256.\n- [Docker](/docs/integration/docker/) — multi-arch image with cosign-signed tags.\n" }, { "path": "docs/content/docs/integration/docker.md", "content": "+++\ntitle = \"Docker\"\ndescription = \"ghcr.io/hahwul/deadfinder — multi-arch, cosign-signed, tiny Alpine base.\"\nweight = 2\n+++\n\nImage: [`ghcr.io/hahwul/deadfinder`](https://github.com/hahwul/deadfinder/pkgs/container/deadfinder)\n\n- Multi-arch: `linux/amd64`, `linux/arm64`\n- Runtime base: `alpine:3.21` + static binary (~15 MB total)\n- Tags on release: ``, `.`, `latest`\n- Every published tag is **cosign-signed** (keyless, Sigstore)\n\n## Run\n\nThe image's `CMD` is `[\"deadfinder\"]`. Append arguments after the image name — `docker run` passes them through:\n\n```bash\ndocker run ghcr.io/hahwul/deadfinder:latest deadfinder url https://www.example.com\ndocker run ghcr.io/hahwul/deadfinder:latest deadfinder sitemap https://www.example.com/sitemap.xml\n```\n\nWriting results out? Bind-mount a host directory:\n\n```bash\ndocker run --rm -v \"$PWD\":/out \\\n ghcr.io/hahwul/deadfinder:latest \\\n deadfinder url https://www.example.com -o /out/results.json -s\n```\n\n## Pin a version\n\n```bash\ndocker pull ghcr.io/hahwul/deadfinder:2.0.0\ndocker pull ghcr.io/hahwul/deadfinder:2.0\ndocker pull ghcr.io/hahwul/deadfinder:latest\n```\n\n## Verify the signature\n\n```bash\ncosign verify ghcr.io/hahwul/deadfinder:2.0.0 \\\n --certificate-identity-regexp 'https://github.com/hahwul/deadfinder/.+' \\\n --certificate-oidc-issuer 'https://token.actions.githubusercontent.com'\n```\n\nSubstitute the tag you pulled. The command succeeds only if the image was signed by this repo's GitHub Actions.\n" }, { "path": "docs/content/docs/integration/github-action.md", "content": "+++\ntitle = \"GitHub Action\"\ndescription = \"hahwul/deadfinder composite action — inputs, outputs, examples.\"\nweight = 1\n+++\n\n`hahwul/deadfinder` is a composite action that downloads the matching release binary, verifies its sha256, and executes the scan. Runs on Linux (x86_64/aarch64) and macOS (arm64). Intel macOS runners (`macos-13`) are not supported — use `macos-latest`.\n\n## Pin a version\n\nAlways pin a released ref. `@latest` is **not** a valid Actions ref (GitHub has no auto-resolver for it).\n\n```yaml\n- uses: hahwul/deadfinder@v2 # tracks latest 2.x — gets bug-fix patches automatically\n# or\n- uses: hahwul/deadfinder@2.0.2 # exact pin — fully reproducible\n```\n\nThe `version` input can override the binary independently of the action ref:\n\n```yaml\n- uses: hahwul/deadfinder@v2\n with:\n version: \"2.0.2\" # download binary from this release tag\n```\n\n## Full example\n\n```yaml\nsteps:\n - name: Run DeadFinder\n uses: hahwul/deadfinder@v2\n id: scan\n with:\n command: sitemap\n target: https://www.example.com/sitemap.xml\n # Optional:\n # timeout: 10\n # concurrency: 50\n # include30x: false\n # headers: \"X-API-Key: secret\"\n # worker_headers: \"User-Agent: Deadfinder Bot\"\n # user_agent: \"MyBot/1.0\"\n # proxy: \"http://localhost:8080\"\n # proxy_auth: \"user:pass\"\n # match: \"^https://example\\\\.com/\"\n # ignore: \"\\\\.png$\"\n # coverage: true\n # visualize: report.png\n # silent: false\n # verbose: false\n\n - name: Handle results\n run: echo '${{ steps.scan.outputs.output }}' | jq '.'\n```\n\n## Inputs\n\n| Input | Required | Default | Notes |\n|---|---|---|---|\n| `command` | ✓ | — | `url` / `file` / `pipe` / `sitemap` |\n| `target` | ✓ | — | URL, file path, or sitemap URL |\n| `version` | | `latest` | Release tag; `latest` resolves to most recent release |\n| `timeout` | | `10` | seconds |\n| `concurrency` | | `50` | workers |\n| `silent` | | `false` | string `\"true\"` to enable |\n| `verbose` | | `false` | |\n| `include30x` | | `false` | |\n| `headers` | | `\"\"` | comma-separated `\"Key: Value\"` pairs |\n| `worker_headers` | | `\"\"` | headers for link-check requests |\n| `user_agent` | | `\"\"` | overrides default UA |\n| `proxy` | | `\"\"` | HTTP/HTTPS proxy URL |\n| `proxy_auth` | | `\"\"` | `user:pass` |\n| `match` | | `\"\"` | regex |\n| `ignore` | | `\"\"` | regex |\n| `coverage` | | `false` | |\n| `visualize` | | `\"\"` | file path (implies coverage) |\n\n## Outputs\n\n| Output | Shape |\n|---|---|\n| `output` | Compact JSON string of the scan result (same shape as `-f json` output). |\n\nConsume with `fromJSON()`:\n\n```yaml\n- run: |\n echo \"Dead links: ${{ fromJSON(steps.scan.outputs.output).summary }}\"\n```\n\n## Migrating from v1\n\nThe v1 action was Docker-based and bundled the Ruby gem. v2 is a composite action that downloads the Crystal binary directly. All v1 inputs are preserved. `worker_headers` was previously undeclared but wired through args — it's now a formal input. `version` is new. No inputs were renamed or removed.\n\nPin to `@1.10.0` to keep the v1 behavior; use `@v2` (or pin a specific 2.x tag like `@2.0.2`) for v2.\n" }, { "path": "docs/content/docs/reference/_index.md", "content": "+++\ntitle = \"Reference\"\ndescription = \"CLI flag reference.\"\nweight = 4\nsort_by = \"weight\"\n+++\n\n- [CLI flags](/docs/reference/cli-flags/) — every option accepted by `deadfinder`.\n" }, { "path": "docs/content/docs/reference/cli-flags.md", "content": "+++\ntitle = \"CLI Flags\"\ndescription = \"Complete reference for every deadfinder option.\"\nweight = 1\n+++\n\nRun `deadfinder --help` for the live help text. This page is the documented contract.\n\n## Synopsis\n\n```\ndeadfinder [options]\n\nCommands:\n pipe Scan the URLs from STDIN\n file Scan the URLs from File\n url Scan the Single URL\n sitemap Scan the URLs from sitemap\n completion Generate completion script (bash/zsh/fish)\n version Show version\n```\n\n## Options\n\n| Short | Long | Default | Description |\n|---|---|---|---|\n| `-r` | `--include30x` | `false` | Treat 3xx responses as dead links. |\n| `-c` | `--concurrency=N` | `50` | Number of concurrent workers. |\n| `-t` | `--timeout=N` | `10` | Per-request timeout (seconds). |\n| `-o` | `--output=FILE` | `\"\"` | Write structured results to FILE. |\n| `-f` | `--output_format=FORMAT` | `json` | `json` / `yaml` / `toml` / `csv` / `sarif`. |\n| `-H` | `--headers=HEADER` | `[]` | Header for the **initial** page fetch. Repeat for multiple. Format: `\"Name: Value\"`. |\n| | `--worker_headers=HEADER` | `[]` | Header for every **link-check** request. Repeat for multiple. |\n| | `--user_agent=UA` | `Mozilla/5.0 (compatible; DeadFinder/;)` | Override User-Agent. |\n| `-p` | `--proxy=URL` | `\"\"` | HTTP/HTTPS proxy (HTTPS uses CONNECT tunneling). |\n| | `--proxy_auth=USER:PASS` | `\"\"` | Proxy credentials (Basic). |\n| `-m` | `--match=PATTERN` | `\"\"` | Regex: only scan URLs that match. |\n| `-i` | `--ignore=PATTERN` | `\"\"` | Regex: skip URLs that match. |\n| `-s` | `--silent` | `false` | Suppress the live log on stdout. |\n| `-v` | `--verbose` | `false` | Log every checked URL, not just dead ones. |\n| | `--debug` | `false` | Internal state / cache diagnostics. |\n| | `--limit=N` | `0` | Cap input URLs (`0` = unlimited). |\n| | `--coverage` | `false` | Emit per-target coverage stats. |\n| | `--visualize=PATH` | `\"\"` | Write a PNG status-code chart (implies `--coverage`). |\n\n## Notes\n\n- Structured output is **file-only**: you must set `-o`. stdout is reserved for the live log.\n- `match` / `ignore` regexes each run under a 1-second timeout to block ReDoS.\n- The initial page fetch receives `--headers`; worker link-check requests receive `--worker_headers`. `--user_agent` applies to both.\n- `--visualize` auto-enables `--coverage`.\n" }, { "path": "docs/content/docs/usage/_index.md", "content": "+++\ntitle = \"Usage\"\ndescription = \"Subcommands, output formats, and filters.\"\nweight = 2\nsort_by = \"weight\"\n+++\n\nDeadFinder is a single CLI with four scan subcommands and a handful of global flags.\n\n- [Subcommands](/docs/usage/subcommands/) — `url`, `file`, `pipe`, `sitemap`, plus `completion` and `version`.\n- [Output formats](/docs/usage/output-formats/) — JSON / YAML / TOML / CSV / SARIF, coverage, PNG visualization.\n- [Filtering](/docs/usage/filtering/) — `--match` / `--ignore` regex, `--include30x`, `--limit`.\n" }, { "path": "docs/content/docs/usage/filtering.md", "content": "+++\ntitle = \"Filtering\"\ndescription = \"Regex match/ignore, 3xx inclusion, URL limit.\"\nweight = 3\n+++\n\n## `--match=PATTERN` / `--ignore=PATTERN`\n\nRegex applied to every discovered URL before it's fetched. Each pattern has a 1-second timeout to prevent ReDoS.\n\n```bash\n# Only check internal links\ndeadfinder sitemap https://www.example.com/sitemap.xml \\\n --match='^https://(www\\.)?example\\.com/'\n\n# Skip media files\ndeadfinder url https://www.example.com \\\n --ignore='\\.(png|jpg|gif|webp|mp4)$'\n```\n\nUsing both: `--match` is applied first, then `--ignore`.\n\n## `--include30x`\n\nBy default, 3xx redirects are treated as healthy (the destination is what matters). Enable this flag to mark them as dead too:\n\n```bash\ndeadfinder url https://www.example.com --include30x\n```\n\nUse this when your policy is \"redirects are technical debt\" rather than \"follow the redirect chain\".\n\n## `--limit=N`\n\nCap the number of URLs scanned per invocation (useful for quick smoke tests of a large sitemap):\n\n```bash\ndeadfinder sitemap https://www.example.com/sitemap.xml --limit=50\n```\n\nApplies to the input list (file lines, STDIN lines, or sitemap `` entries). Not to discovered child links on each page.\n\n## `--concurrency=N` / `--timeout=N`\n\nNot filters per se, but the other knobs you'll reach for:\n\n- `--concurrency=50` (default) — number of parallel workers.\n- `--timeout=10` (default, seconds) — per-request connect + read timeout.\n\nRamp concurrency down on rate-limited targets; up on fast internal scans.\n" }, { "path": "docs/content/docs/usage/output-formats.md", "content": "+++\ntitle = \"Output Formats\"\ndescription = \"JSON, YAML, TOML, CSV, SARIF, coverage reports, and PNG visualization.\"\nweight = 2\n+++\n\nDeadFinder writes results only when `-o ` is set (stdout stays human-readable log). Pick the format with `-f `.\n\n| Flag | Format |\n|---|---|\n| `-f json` (default) | pretty JSON |\n| `-f yaml` / `-f yml` | YAML |\n| `-f toml` | TOML |\n| `-f csv` | CSV with `target,url` columns |\n| `-f sarif` | SARIF 2.1.0 JSON (one `DEAD_LINK` result per broken URL) |\n\n## Basic shape\n\nSame across JSON / YAML / TOML:\n\n```json\n{\n \"https://www.example.com\": [\n \"https://www.example.com/broken-link-1\",\n \"https://www.example.com/broken-link-2\"\n ]\n}\n```\n\nCSV:\n\n```csv\ntarget,url\nhttps://www.example.com,https://www.example.com/broken-link-1\nhttps://www.example.com,https://www.example.com/broken-link-2\n```\n\n## Coverage mode\n\nAdd `--coverage` to include per-target statistics:\n\n```bash\ndeadfinder sitemap https://www.example.com/sitemap.xml --coverage -o out.json\n```\n\n```json\n{\n \"dead_links\": {\n \"https://www.example.com\": [\"https://www.example.com/broken-link-1\"]\n },\n \"coverage\": {\n \"targets\": {\n \"https://www.example.com\": {\n \"total_tested\": 100,\n \"dead_links\": 5,\n \"coverage_percentage\": 5.0,\n \"status_counts\": {\"404\": 3, \"500\": 2}\n }\n },\n \"summary\": {\n \"total_tested\": 100,\n \"total_dead\": 5,\n \"overall_coverage_percentage\": 5.0,\n \"overall_status_counts\": {\"404\": 3, \"500\": 2}\n }\n }\n}\n```\n\n## SARIF\n\n`-f sarif` produces a [SARIF 2.1.0](https://docs.oasis-open.org/sarif/sarif/v2.1.0/sarif-v2.1.0.html) document you can upload to GitHub code scanning (`github/codeql-action/upload-sarif`) or feed into any SARIF-aware tooling:\n\n```bash\ndeadfinder sitemap https://www.example.com/sitemap.xml -f sarif -o deadfinder.sarif\n```\n\nEach dead link becomes a `result` under the `DEAD_LINK` rule. The broken URL is the primary location; the page it was discovered on is attached as a related location.\n\n## PNG visualization\n\n```bash\ndeadfinder sitemap https://www.example.com/sitemap.xml --visualize report.png\n```\n\n`--visualize` implies `--coverage`. Output is a stacked bar chart of status codes per target.\n\n## Stdout vs file\n\nStructured output requires `-o`. Without it the tool emits a live log to stdout only. Use `-s` / `--silent` to suppress the log entirely (for example when you're only interested in the file output).\n\n```bash\ndeadfinder url https://www.example.com -o out.json -s\n```\n" }, { "path": "docs/content/docs/usage/subcommands.md", "content": "+++\ntitle = \"Subcommands\"\ndescription = \"url / file / pipe / sitemap / completion / version\"\nweight = 1\n+++\n\n## `url `\n\nScan a single page. Extract links from the HTML and check each one.\n\n```bash\ndeadfinder url https://www.example.com\n```\n\n## `file `\n\nRead newline-separated URLs from a file and scan each one. Each URL is scanned independently; results are keyed by the source URL.\n\n```bash\ndeadfinder file urls.txt\n```\n\n## `pipe`\n\nRead URLs from STDIN (one per line). Useful in shell pipelines.\n\n```bash\ngrep '^https://' access.log | sort -u | deadfinder pipe\n```\n\n## `sitemap `\n\nParse an XML sitemap, follow sitemap indexes recursively, and scan every ``.\n\n```bash\ndeadfinder sitemap https://www.example.com/sitemap.xml\n```\n\n## `completion `\n\nEmit shell completion for bash, zsh, or fish.\n\n```bash\n# Bash\ndeadfinder completion bash > /etc/bash_completion.d/deadfinder\n\n# Zsh\ndeadfinder completion zsh > ~/.zsh/completion/_deadfinder\n\n# Fish\ndeadfinder completion fish > ~/.config/fish/completions/deadfinder.fish\n```\n\n## `version`\n\nPrint the DeadFinder version.\n\n```bash\ndeadfinder version\n```\n" }, { "path": "docs/content/index.md", "content": "+++\ntitle = \"DeadFinder\"\ndescription = \"Find dead (broken) links in web pages, URL lists, and sitemaps.\"\n+++\n\nFind dead (broken) links in web pages, URL lists, and sitemaps. Fast native CLI written in Crystal with fiber-based concurrency.\n\n## Why DeadFinder\n\n- **Fast**: fiber-based concurrent workers scan hundreds of links in parallel.\n- **Ergonomic**: one binary, no runtime dependencies.\n- **Structured output**: JSON / YAML / TOML / CSV — or attach as a GitHub Action output.\n- **Coverage report**: track dead-link ratio per target with `--coverage`.\n\n## Install\n\n```bash\n# Homebrew\nbrew install deadfinder\n\n# Docker\ndocker run ghcr.io/hahwul/deadfinder:latest deadfinder url https://example.com\n\n# Prebuilt binary — pick your platform on the Releases page\n# https://github.com/hahwul/deadfinder/releases/latest\n```\n\nSee [Installation](/docs/getting-started/installation/) for every channel (Nix, build from source, etc).\n\n## First scan\n\n```bash\ndeadfinder url https://your-site.example\ndeadfinder sitemap https://your-site.example/sitemap.xml\ncat urls.txt | deadfinder pipe\n```\n\nSee [Quick Start](/docs/getting-started/quickstart/) for more.\n\n## Continuous integration\n\nRun DeadFinder on every push via the official GitHub Action:\n\n```yaml\n- uses: hahwul/deadfinder@v2\n with:\n command: sitemap\n target: https://www.example.com/sitemap.xml\n```\n\nSee [GitHub Action](/docs/integration/github-action/) for the full input reference.\n\n---\n\nDeadFinder 2.0+ is written in Crystal. v1.x (Ruby gem) lives on the [`legacy/v1`](https://github.com/hahwul/deadfinder/tree/legacy/v1) branch and receives bug-fix updates only.\n" }, { "path": "docs/static/CNAME", "content": "deadfinder.hahwul.com\n" }, { "path": "docs/static/css/style.css", "content": ":root {\n --sidebar-w: 280px;\n --toc-w: 220px;\n --content-max: 720px;\n --font: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;\n --mono: 'Noto Sans Mono', ui-monospace, 'SFMono-Regular', Consolas, monospace;\n\n --bg: #0a0f0a;\n --bg-sidebar: #0f1a0f;\n --text: #e8ede8;\n --text-muted: #8fa38f;\n --text-light: #5c6e5c;\n --primary: #22c55e;\n --primary-light: #0a1f0e;\n --accent: #f59e0b;\n --accent-light: #1a1500;\n --border: #1a2e1a;\n --border-light: #152515;\n --code-bg: #0d160d;\n --hover-bg: #122012;\n}\n\n*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }\n\nbody {\n font-family: var(--font);\n font-size: 15px;\n line-height: 1.7;\n color: var(--text);\n background: var(--bg);\n -webkit-font-smoothing: antialiased;\n}\n\n/* -- Top Bar -- */\n.topbar {\n position: sticky;\n top: 0;\n z-index: 100;\n display: flex;\n align-items: center;\n justify-content: space-between;\n height: 52px;\n padding: 0 1.25rem;\n background: var(--bg);\n border-bottom: 1px solid var(--border);\n}\n.topbar-left {\n display: flex;\n align-items: center;\n gap: 0.75rem;\n}\n.topbar-logo {\n display: flex;\n align-items: center;\n gap: 0.5rem;\n text-decoration: none;\n color: var(--text);\n font-weight: 700;\n font-size: 1rem;\n}\n.topbar-logo svg { flex-shrink: 0; }\n.topbar-logo:hover { color: var(--primary); }\n\n.menu-btn {\n display: none;\n background: none;\n border: 1px solid var(--border);\n border-radius: 6px;\n padding: 4px 8px;\n cursor: pointer;\n color: var(--text-muted);\n}\n.menu-btn:hover { background: var(--hover-bg); }\n\n.topbar-right {\n display: flex;\n align-items: center;\n gap: 0.5rem;\n}\n.topbar-icon {\n display: inline-flex;\n align-items: center;\n justify-content: center;\n width: 32px;\n height: 32px;\n color: var(--text-muted);\n border: 1px solid var(--border);\n border-radius: 8px;\n text-decoration: none;\n transition: color 0.15s, border-color 0.15s;\n}\n.topbar-icon:hover {\n color: var(--text);\n border-color: var(--primary);\n}\n\n/* Search trigger (button in topbar) */\n.topbar-search {\n display: inline-flex;\n align-items: center;\n gap: 0.5rem;\n width: 260px;\n padding: 6px 8px 6px 10px;\n font-family: var(--font);\n font-size: 0.8rem;\n background: var(--code-bg);\n color: var(--text-muted);\n border: 1px solid var(--border);\n border-radius: 8px;\n cursor: pointer;\n transition: border-color 0.15s, box-shadow 0.15s, color 0.15s;\n}\n.topbar-search:hover {\n border-color: var(--primary);\n color: var(--text);\n}\n.topbar-search:focus-visible {\n outline: none;\n border-color: var(--primary);\n box-shadow: 0 0 0 3px rgba(34, 197, 94, 0.15);\n}\n.topbar-search svg {\n flex-shrink: 0;\n color: var(--text-light);\n}\n.topbar-search span {\n flex: 1;\n text-align: left;\n}\n.topbar-search kbd {\n font-family: var(--mono);\n font-size: 0.7rem;\n padding: 2px 6px;\n background: var(--bg);\n border: 1px solid var(--border);\n border-radius: 4px;\n color: var(--text-muted);\n line-height: 1;\n}\n\n/* Search modal */\n#search-modal {\n position: fixed;\n inset: 0;\n z-index: 1000;\n font-family: var(--font);\n}\n#search-modal[hidden] { display: none; }\n.search-overlay {\n position: absolute;\n inset: 0;\n background: rgba(0, 0, 0, 0.65);\n backdrop-filter: blur(4px);\n -webkit-backdrop-filter: blur(4px);\n}\n.search-dialog {\n position: absolute;\n top: 12%;\n left: 50%;\n transform: translateX(-50%);\n width: 92%;\n max-width: 640px;\n max-height: 70vh;\n display: flex;\n flex-direction: column;\n background: var(--bg-sidebar);\n color: var(--text);\n border: 1px solid var(--border);\n border-radius: 10px;\n box-shadow: 0 24px 64px rgba(0, 0, 0, 0.6);\n overflow: hidden;\n}\n.search-dialog-header {\n position: relative;\n display: flex;\n align-items: center;\n gap: 0.5rem;\n padding: 12px 14px;\n border-bottom: 1px solid var(--border);\n background: var(--bg);\n}\n.search-dialog-header svg {\n flex-shrink: 0;\n color: var(--text-light);\n}\n#search-input {\n flex: 1;\n font-family: var(--font);\n font-size: 0.95rem;\n background: transparent;\n color: var(--text);\n border: none;\n outline: none;\n padding: 4px 0;\n}\n#search-input::placeholder { color: var(--text-light); }\n#search-close {\n background: transparent;\n border: 1px solid var(--border);\n color: var(--text-muted);\n padding: 2px 8px;\n border-radius: 4px;\n font-family: var(--mono);\n font-size: 0.7rem;\n cursor: pointer;\n line-height: 1.4;\n}\n#search-close:hover { color: var(--text); border-color: var(--primary); }\n#search-results {\n flex: 1;\n overflow-y: auto;\n padding: 8px;\n}\n#search-results::-webkit-scrollbar { width: 6px; }\n#search-results::-webkit-scrollbar-thumb { background: var(--border); border-radius: 4px; }\n.search-result {\n padding: 10px 12px;\n border-radius: 6px;\n cursor: pointer;\n transition: background 0.12s;\n}\n.search-result + .search-result { margin-top: 2px; }\n.search-result:hover,\n.search-result.selected { background: var(--hover-bg); }\n.search-result-title {\n font-size: 0.9rem;\n font-weight: 600;\n color: var(--primary);\n margin-bottom: 2px;\n}\n.search-result-description {\n font-size: 0.8rem;\n color: var(--text-muted);\n line-height: 1.45;\n}\n.search-result-content {\n font-size: 0.78rem;\n color: var(--text-light);\n margin-top: 4px;\n line-height: 1.45;\n font-family: var(--mono);\n}\n.search-result mark {\n background: rgba(34, 197, 94, 0.22);\n color: var(--text);\n padding: 0 2px;\n border-radius: 2px;\n}\n.search-empty {\n padding: 1.5rem 1rem;\n text-align: center;\n color: var(--text-muted);\n font-size: 0.85rem;\n}\n\n/* -- Layout -- */\n.layout {\n display: flex;\n min-height: calc(100vh - 52px);\n}\n\n/* -- Sidebar -- */\n.sidebar {\n position: sticky;\n top: 52px;\n width: var(--sidebar-w);\n height: calc(100vh - 52px);\n overflow-y: auto;\n padding: 1.25rem 0;\n border-right: 1px solid var(--border);\n background: var(--bg-sidebar);\n flex-shrink: 0;\n}\n.sidebar::-webkit-scrollbar { width: 4px; }\n.sidebar::-webkit-scrollbar-thumb { background: var(--border); border-radius: 4px; }\n.sidebar::-webkit-scrollbar-track { background: transparent; }\n\n.sidebar-section { margin-bottom: 0.25rem; }\n.sidebar-heading {\n display: block;\n padding: 0.35rem 1.25rem;\n font-size: 0.75rem;\n font-weight: 600;\n text-transform: uppercase;\n letter-spacing: 0.05em;\n color: var(--text-muted);\n}\n.sidebar-nav { list-style: none; }\n.sidebar-nav a {\n display: flex;\n align-items: center;\n gap: 0.35rem;\n padding: 0.3rem 1.25rem 0.3rem 1.5rem;\n font-size: 0.875rem;\n color: var(--text-muted);\n text-decoration: none;\n border-left: 2px solid transparent;\n transition: color 0.15s, background 0.15s, border-color 0.15s;\n}\n.sidebar-nav a:hover {\n color: var(--text);\n background: var(--hover-bg);\n}\n.sidebar-nav a.active {\n color: var(--primary);\n font-weight: 500;\n background: var(--primary-light);\n border-left-color: var(--primary);\n}\n\n/* Nested nav */\n.sidebar-nav .nested { list-style: none; }\n.sidebar-nav .nested a {\n padding-left: 2.25rem;\n font-size: 0.825rem;\n}\n.sidebar-nav .nested .nested a {\n padding-left: 3rem;\n}\n\n.sidebar-toggle {\n display: flex;\n align-items: center;\n gap: 0.25rem;\n width: 100%;\n padding: 0.3rem 1.25rem 0.3rem 1.5rem;\n font-family: var(--font);\n font-size: 0.875rem;\n color: var(--text-muted);\n background: none;\n border: none;\n border-left: 2px solid transparent;\n cursor: pointer;\n text-align: left;\n transition: color 0.15s, background 0.15s;\n}\n.sidebar-toggle:hover {\n color: var(--text);\n background: var(--hover-bg);\n}\n.sidebar-toggle .arrow {\n display: inline-block;\n width: 16px;\n text-align: center;\n font-size: 0.7rem;\n transition: transform 0.2s;\n}\n.sidebar-toggle.open .arrow { transform: rotate(90deg); }\n\n/* -- Main Content -- */\n.main {\n flex: 1;\n min-width: 0;\n padding: 2rem 2.5rem;\n max-width: calc(var(--content-max) + 5rem);\n}\n\n/* -- Prose -- */\n.prose h1 { font-size: 1.75rem; font-weight: 700; margin: 0 0 0.75rem; line-height: 1.3; color: var(--text); }\n.prose h2 { font-size: 1.35rem; font-weight: 600; margin: 2rem 0 0.5rem; padding-bottom: 0.35rem; border-bottom: 1px solid var(--border); line-height: 1.3; color: var(--text); }\n.prose h3 { font-size: 1.1rem; font-weight: 600; margin: 1.5rem 0 0.4rem; line-height: 1.3; color: var(--text); }\n.prose h4 { font-size: 0.95rem; font-weight: 600; margin: 1.25rem 0 0.35rem; color: var(--text); }\n.prose p { margin: 0.75rem 0; color: var(--text); }\n.prose a { color: var(--primary); text-decoration: none; }\n.prose a:hover { text-decoration: underline; }\n.prose strong { font-weight: 600; color: var(--text); }\n.prose img { max-width: 100%; border-radius: 8px; margin: 1rem 0; }\n.prose blockquote {\n margin: 1rem 0;\n padding: 0.5rem 1rem;\n border-left: 3px solid var(--primary);\n background: var(--primary-light);\n border-radius: 0 6px 6px 0;\n color: var(--text);\n}\n.prose blockquote p { margin: 0.25rem 0; }\n.prose ul, .prose ol { margin: 0.75rem 0; padding-left: 1.5rem; }\n.prose li { margin: 0.25rem 0; color: var(--text); }\n.prose li::marker { color: var(--text-muted); }\n.prose code {\n font-family: var(--mono);\n font-size: 0.85em;\n background: var(--code-bg);\n padding: 0.15rem 0.4rem;\n border-radius: 4px;\n border: 1px solid var(--border);\n color: var(--primary);\n}\n.prose pre {\n margin: 1rem 0;\n padding: 1rem;\n background: var(--code-bg);\n border: 1px solid var(--border);\n border-radius: 8px;\n overflow-x: auto;\n line-height: 1.5;\n}\n.prose pre code {\n background: none;\n border: none;\n padding: 0;\n font-size: 0.85rem;\n color: var(--text);\n}\n.prose table { width: 100%; border-collapse: collapse; margin: 1rem 0; font-size: 0.9rem; }\n.prose th, .prose td { padding: 0.5rem 0.75rem; border: 1px solid var(--border); text-align: left; }\n.prose th { background: var(--code-bg); font-weight: 600; color: var(--text); }\n.prose td { color: var(--text-muted); }\n.prose hr { border: none; border-top: 1px solid var(--border); margin: 2rem 0; }\n\n/* -- Page Navigation -- */\n.page-nav {\n display: flex;\n justify-content: space-between;\n gap: 1rem;\n margin-top: 3rem;\n padding-top: 1.5rem;\n border-top: 1px solid var(--border);\n}\n.page-nav a {\n display: flex;\n flex-direction: column;\n gap: 0.15rem;\n padding: 0.75rem 1rem;\n text-decoration: none;\n border: 1px solid var(--border);\n border-radius: 8px;\n flex: 1;\n max-width: 50%;\n transition: border-color 0.2s, box-shadow 0.2s;\n}\n.page-nav a:hover {\n border-color: var(--primary);\n box-shadow: 0 0 0 3px rgba(34, 197, 94, 0.08);\n}\n.page-nav a .label {\n font-size: 0.75rem;\n color: var(--text-light);\n text-transform: uppercase;\n letter-spacing: 0.04em;\n}\n.page-nav a .title { font-size: 0.9rem; color: var(--primary); font-weight: 500; }\n.page-nav .next { text-align: right; margin-left: auto; }\n\n/* -- Section list -- */\nul.section-list { list-style: none; margin: 1rem 0; }\nul.section-list li {\n padding: 0.5rem 0;\n border-bottom: 1px solid var(--border);\n}\nul.section-list li:last-child { border-bottom: none; }\nul.section-list li a { color: var(--primary); text-decoration: none; font-weight: 500; }\nul.section-list li a:hover { text-decoration: underline; }\nnav.pagination { margin: 1.5rem 0; }\nnav.pagination .pagination-list { list-style: none; display: flex; gap: 0.5rem; flex-wrap: wrap; align-items: center; }\nnav.pagination a { display: inline-block; padding: 0.25rem 0.55rem; border-radius: 6px; border: 1px solid var(--border); color: var(--text-muted); text-decoration: none; font-size: 0.85rem; }\nnav.pagination a:hover { color: var(--primary); border-color: var(--primary); }\n.pagination-current span { display: inline-block; padding: 0.25rem 0.55rem; border-radius: 6px; border: 1px solid var(--primary); background: var(--primary-light); font-size: 0.85rem; }\n.pagination-disabled span { display: inline-block; padding: 0.25rem 0.55rem; border-radius: 6px; border: 1px solid var(--border); color: var(--text-muted); opacity: 0.5; font-size: 0.85rem; }\n\n/* -- Footer -- */\n.site-footer {\n padding: 1.5rem 2.5rem;\n border-top: 1px solid var(--border);\n color: var(--text-light);\n font-size: 0.8rem;\n}\n.site-footer a { color: var(--text-muted); text-decoration: none; }\n.site-footer a:hover { color: var(--primary); }\n\n/* -- Alert shortcode -- */\n.alert { padding: 0.75rem 1rem; border-radius: 6px; margin: 1rem 0; font-size: 0.9rem; border-left: 4px solid; }\n.alert-info { background: var(--primary-light); border-color: var(--primary); color: var(--primary); }\n.alert-warning { background: var(--accent-light); border-color: var(--accent); color: var(--accent); }\n.alert-danger { background: #1a0508; border-color: #ef4444; color: #ef4444; }\n.alert-tip { background: var(--primary-light); border-color: #22c55e; color: #22c55e; }\n\n/* -- Hint shortcode -- */\n.hint { padding: 0.75rem 1rem; border-radius: 6px; margin: 1rem 0; font-size: 0.9rem; border-left: 4px solid; }\n.hint-info { background: var(--primary-light); border-color: var(--primary); color: var(--primary); }\n.hint-warning { background: var(--accent-light); border-color: var(--accent); color: var(--accent); }\n.hint-danger { background: #1a0508; border-color: #ef4444; color: #ef4444; }\n\n/* -- Responsive -- */\n@media (max-width: 768px) {\n .sidebar {\n position: fixed;\n left: -100%;\n top: 52px;\n z-index: 90;\n width: 280px;\n transition: left 0.25s ease;\n box-shadow: none;\n }\n .sidebar.open {\n left: 0;\n box-shadow: 4px 0 24px rgba(0, 0, 0, 0.4);\n }\n .sidebar-overlay {\n display: none;\n position: fixed;\n inset: 0;\n top: 52px;\n z-index: 80;\n background: rgba(0, 0, 0, 0.6);\n }\n .sidebar-overlay.open { display: block; }\n .menu-btn { display: block; }\n .main { padding: 1.5rem 1rem; }\n .site-footer { padding: 1.5rem 1rem; }\n .page-nav { flex-direction: column; }\n .page-nav a { max-width: 100%; }\n .topbar-search { width: auto; padding: 6px 10px; }\n .topbar-search span,\n .topbar-search kbd { display: none; }\n}\n" }, { "path": "docs/static/icons/site.webmanifest", "content": "{\n \"name\": \"DeadFinder\",\n \"short_name\": \"DeadFinder\",\n \"icons\": [\n {\n \"src\": \"/icons/web-app-manifest-192x192.png\",\n \"sizes\": \"192x192\",\n \"type\": \"image/png\",\n \"purpose\": \"maskable\"\n },\n {\n \"src\": \"/icons/web-app-manifest-512x512.png\",\n \"sizes\": \"512x512\",\n \"type\": \"image/png\",\n \"purpose\": \"maskable\"\n }\n ],\n \"theme_color\": \"#ffffff\",\n \"background_color\": \"#ffffff\",\n \"display\": \"standalone\"\n}" }, { "path": "docs/static/js/search.js", "content": "// Guard against double-load (auto-includes + explicit \n\n \n\n\n" }, { "path": "docs/templates/header.html", "content": "\n\n \n \n \n \n {{ page.title }} - {{ site.title }}\n {{ og_all_tags }}\n\n \n \n \n\n \n \n \n \n \n\n \n\n {{ highlight_css }} {{ auto_includes_css }}\n \n \n \n \n
\n
\n \n \n \n \n
\n
\n \n \n \n \n \n \n \n \n \n \n Search...\n ⌘K\n \n
\n
\n\n \n
\n\n
\n \n \n\n \n
\n" }, { "path": "docs/templates/page.html", "content": "{% include \"header.html\" %}\n
\n

{{ page.title }}

\n {{ content }}\n
\n{% include \"footer.html\" %}\n" }, { "path": "docs/templates/section.html", "content": "{% include \"header.html\" %}\n
\n

{{ page.title }}

\n {{ content }}\n
    \n {{ section.list }}\n
\n {{ pagination }}\n
\n{% include \"footer.html\" %}\n" }, { "path": "docs/templates/shortcodes/alert.html", "content": "
\n {{ type | upper }}: {{ message }}\n
\n" }, { "path": "docs/templates/taxonomy.html", "content": "{% include \"header.html\" %}\n
\n

{{ page.title }}

\n

Browse all terms in this taxonomy:

\n {{ content }}\n
\n{% include \"footer.html\" %}\n" }, { "path": "docs/templates/taxonomy_term.html", "content": "{% include \"header.html\" %}\n
\n

{{ page.title }}

\n

Pages tagged with this term:

\n {{ content }}\n
\n{% include \"footer.html\" %}\n" }, { "path": "flake.nix", "content": "{\n description = \"DeadFinder — find dead (broken) links in web pages, URL lists, and sitemaps\";\n\n inputs = {\n nixpkgs.url = \"github:NixOS/nixpkgs/nixpkgs-unstable\";\n flake-utils.url = \"github:numtide/flake-utils\";\n };\n\n outputs = { self, nixpkgs, flake-utils }:\n flake-utils.lib.eachDefaultSystem (system:\n let\n pkgs = import nixpkgs { inherit system; };\n\n # lexbor.cr's postinstall hook clones the upstream lexbor C library\n # from GitHub at a pinned commit (lib/lexbor/src/ext/revision) and\n # builds it via cmake. The Nix sandbox blocks network access, so\n # pre-fetch the source as a fixed-output derivation and drop it\n # into place during preBuild — then cmake runs normally.\n lexborCSrc = pkgs.fetchgit {\n url = \"https://github.com/lexbor/lexbor.git\";\n rev = \"971faf11a5f45433b9193a143e2897d8c0fd5611\";\n sha256 = \"0v3ka5dhgz2jkmigdjcjm3vmxlc9yv4hks6pz13xzgagxxfwlw7s\";\n };\n\n deadfinder = pkgs.crystal.buildCrystalPackage rec {\n pname = \"deadfinder\";\n version = \"2.0.0\";\n\n src = ./.;\n\n # Generate with: crystal2nix > shards.nix\n shardsFile = ./shards.nix;\n\n nativeBuildInputs = with pkgs; [ crystal shards cmake pkg-config ];\n buildInputs = [ ];\n\n # lexbor.cr's postinstall hook (build_ext.cr) clones the lexbor C\n # library at a pinned commit and builds it via cmake. The Nix\n # sandbox blocks network, so we (a) replace the read-only shard\n # symlink with a writable copy, (b) drop in the pre-fetched C\n # source, and (c) run cmake directly here — bypassing build_ext.cr.\n preBuild = ''\n cp -RL lib/lexbor lib/lexbor.rw\n chmod -R u+w lib/lexbor.rw\n rm lib/lexbor\n mv lib/lexbor.rw lib/lexbor\n\n cp -r ${lexborCSrc} lib/lexbor/src/ext/lexbor-c\n chmod -R u+w lib/lexbor/src/ext/lexbor-c\n\n mkdir -p lib/lexbor/src/ext/lexbor-c/build\n ( cd lib/lexbor/src/ext/lexbor-c/build \\\n && cmake .. \\\n -DCMAKE_BUILD_TYPE=Release \\\n -DLEXBOR_BUILD_TESTS_CPP=OFF \\\n -DLEXBOR_INSTALL_HEADERS=OFF \\\n -DLEXBOR_BUILD_SHARED=ON \\\n -G \"Unix Makefiles\" \\\n && cmake --build . --config Release -j $NIX_BUILD_CORES )\n '';\n\n buildPhase = ''\n runHook preBuild\n shards build --release --no-debug\n runHook postBuild\n '';\n\n installPhase = ''\n runHook preInstall\n mkdir -p $out/bin\n cp bin/deadfinder $out/bin/deadfinder\n runHook postInstall\n '';\n\n doCheck = false;\n\n meta = with pkgs.lib; {\n description = \"Find dead (broken) links in web pages, URL lists, and sitemaps\";\n homepage = \"https://github.com/hahwul/deadfinder\";\n license = licenses.mit;\n maintainers = [ \"hahwul\" ];\n mainProgram = \"deadfinder\";\n };\n };\n in\n {\n packages.default = deadfinder;\n packages.deadfinder = deadfinder;\n\n devShells.default = pkgs.mkShell {\n inputsFrom = [ deadfinder ];\n nativeBuildInputs = with pkgs; [ crystal shards crystal2nix cmake pkg-config just ];\n shellHook = ''\n echo \"deadfinder development environment (Nix)\"\n [ -d lib ] || shards install\n '';\n };\n });\n}\n" }, { "path": "github-action/README.md", "content": "## DeadFinder Github Action" }, { "path": "justfile", "content": "default:\n @just --list\n\n# Install shard dependencies\ndeps:\n shards install\n\n# Build a release binary at ./deadfinder\nbuild:\n shards install\n crystal build src/cli_main.cr -o deadfinder --release --no-debug\n\n# Build a debug binary at ./deadfinder (fast compile)\nbuild-debug:\n shards install\n crystal build src/cli_main.cr -o deadfinder\n\n# Run unit specs\ntest:\n crystal spec\n\n# Run cross-implementation compat harness (requires built binary)\ncompat: build\n BIN=./deadfinder ruby spec/compat/run.rb\n\n# Format sources\nfix:\n crystal tool format src spec\n\n# Check formatting without modifying\ncheck-format:\n crystal tool format --check src spec\n\n# Verify version consistency across shard.yml and src/deadfinder/version.cr\nalias vc := version-check\nversion-check:\n crystal run scripts/version_check.cr\n\n# Update version in all tracked files\nalias vu := version-update\nversion-update VERSION:\n crystal run scripts/version_update.cr -- {{VERSION}}\n\n# Clean build artifacts and dependencies\nclean:\n rm -f deadfinder *.dwarf\n rm -rf lib/ .shards/\n" }, { "path": "scripts/version_check.cr", "content": "require \"yaml\"\n\n# Cross-file version consistency check. Prints each discovered version\n# string and exits non-zero if any tracked file disagrees (files that\n# don't exist yet are skipped silently so the script works on branches\n# that haven't landed the snap/aur packaging yet).\n\nSHARD_YML = \"shard.yml\"\nVERSION_CR = \"src/deadfinder/version.cr\"\nSPEC_TOP = \"spec/deadfinder_spec.cr\"\nSPEC_CLI = \"spec/deadfinder/cli_spec.cr\"\nSNAPCRAFT = \"snap/snapcraft.yaml\"\nPKGBUILD = \"aur/PKGBUILD\"\n\ndef shard_version(path : String) : String?\n YAML.parse(File.read(path))[\"version\"].as_s\nrescue\n nil\nend\n\ndef match_pattern(path : String, pattern : Regex) : String?\n content = File.read(path)\n m = content.match(pattern)\n m ? m[1] : nil\nrescue\n nil\nend\n\n# Matches both `VERSION = \"X\"` and `VERSION.should eq \"X\"` (with or without parens).\nCR_VERSION_RE = /VERSION\\s*(?:=|\\.should\\s+eq\\(?)\\s*\"([^\"]+)\"/\n# PKGBUILD: pkgver=X.Y.Z\nPKGBUILD_RE = /^pkgver=([^\\s]+)/m\n\nresults = [] of {String, String}\n\nresults << {SHARD_YML, shard_version(SHARD_YML).not_nil!} if File.exists?(SHARD_YML)\nresults << {VERSION_CR, match_pattern(VERSION_CR, CR_VERSION_RE).not_nil!} if File.exists?(VERSION_CR)\nresults << {SPEC_TOP, match_pattern(SPEC_TOP, CR_VERSION_RE).not_nil!} if File.exists?(SPEC_TOP)\nresults << {SPEC_CLI, match_pattern(SPEC_CLI, CR_VERSION_RE).not_nil!} if File.exists?(SPEC_CLI)\nresults << {SNAPCRAFT, shard_version(SNAPCRAFT).not_nil!} if File.exists?(SNAPCRAFT)\nresults << {PKGBUILD, match_pattern(PKGBUILD, PKGBUILD_RE).not_nil!} if File.exists?(PKGBUILD)\n\nif results.empty?\n STDERR.puts \"no tracked version files found\"\n exit 1\nend\n\nresults.each { |path, v| puts \"#{path}: #{v}\" }\n\nuniq = results.map { |_, v| v }.uniq\nif uniq.size == 1\n puts \"OK: all files agree on #{uniq.first}\"\nelse\n STDERR.puts \"MISMATCH: #{uniq.join(\", \")}\"\n exit 1\nend\n" }, { "path": "scripts/version_update.cr", "content": "require \"yaml\"\n\n# Bump the version string across every tracked file in one pass. Run:\n#\n# crystal run scripts/version_update.cr -- 2.1.0\n#\n# or via `just version-update 2.1.0`.\n#\n# Files that don't exist yet are skipped silently so the script works\n# on branches that haven't landed the snap/aur packaging.\n\nSHARD_YML = \"shard.yml\"\nVERSION_CR = \"src/deadfinder/version.cr\"\nSPEC_TOP = \"spec/deadfinder_spec.cr\"\nSPEC_CLI = \"spec/deadfinder/cli_spec.cr\"\nSNAPCRAFT = \"snap/snapcraft.yaml\"\nPKGBUILD = \"aur/PKGBUILD\"\n\nSEMVER = /\\A\\d+\\.\\d+\\.\\d+(?:-[0-9A-Za-z.-]+)?\\z/\n\ndef usage(code = 1)\n STDERR.puts \"usage: crystal run scripts/version_update.cr -- \"\n exit code\nend\n\nnew_version = ARGV[0]?\nusage unless new_version\nunless new_version.as(String).matches?(SEMVER)\n STDERR.puts \"invalid semver: #{new_version}\"\n usage\nend\n\nnv = new_version.as(String)\n\ndef replace_in_file(path : String, pattern : Regex, replacement : String) : Bool\n return true unless File.exists?(path)\n src = File.read(path)\n updated = src.sub(pattern, replacement)\n if updated == src\n STDERR.puts \"#{path}: pattern not found\"\n return false\n end\n File.write(path, updated)\n puts \"#{path}: updated\"\n true\nend\n\nok = true\n# Crystal's `m` flag enables both line-anchor and DOTALL semantics, so a\n# bare `.+$/m` swallows everything from the match start to end of file.\n# Constrain to single-line content with `[^\\n]+`.\nok &= replace_in_file(SHARD_YML, /^version:\\s*[^\\n]+$/m, \"version: #{nv}\")\nok &= replace_in_file(VERSION_CR, /VERSION\\s*=\\s*\"[^\"]+\"/, %(VERSION = \"#{nv}\"))\nok &= replace_in_file(SPEC_TOP, /VERSION\\.should\\s+eq\\s+\"[^\"]+\"/, %(VERSION.should eq \"#{nv}\"))\nok &= replace_in_file(SPEC_CLI, /VERSION\\.should\\s+eq\\s+\"[^\"]+\"/, %(VERSION.should eq \"#{nv}\"))\nok &= replace_in_file(SNAPCRAFT, /^version:\\s*[^\\n]+$/m, \"version: #{nv}\")\nok &= replace_in_file(PKGBUILD, /^pkgver=[^\\n]+$/m, \"pkgver=#{nv}\")\n\nexit(ok ? 0 : 1)\n" }, { "path": "shard.yml", "content": "name: deadfinder\nversion: 2.0.2\n\nauthors:\n - hahwul \n\ntargets:\n deadfinder:\n main: src/cli_main.cr\n\ndependencies:\n lexbor:\n github: kostya/lexbor\n stumpy_png:\n github: stumpycr/stumpy_png\n version: \"~> 5.0\"\n sarif:\n github: hahwul/sarif.cr\n version: \"~> 0.2.0\"\n\ndevelopment_dependencies:\n webmock:\n github: manastech/webmock.cr\n version: \"~> 0.14\"\n\ncrystal: '>= 1.19.1'\n\nlicense: MIT\n" }, { "path": "shards.nix", "content": "{\n \"lexbor\" = {\n url = \"https://github.com/kostya/lexbor.git\";\n rev = \"v3.4.2\";\n sha256 = \"0bsncwsvqf5zns0c56va1l9gc7798pvl34i6yh8jf1syqxkvdb8a\";\n };\n \"stumpy_core\" = {\n url = \"https://github.com/stumpycr/stumpy_core.git\";\n rev = \"v1.9.1\";\n sha256 = \"1sj5wr9zrxnihnjwq057lah09lsl9jq6j7giwwv3ds9wp9j9z903\";\n };\n \"stumpy_png\" = {\n url = \"https://github.com/stumpycr/stumpy_png.git\";\n rev = \"v5.0.1\";\n sha256 = \"15wiawl0n3n596bdi0k9dd08nxln2smffba7mggdffw241mn89jc\";\n };\n \"webmock\" = {\n url = \"https://github.com/manastech/webmock.cr.git\";\n rev = \"v0.14.0\";\n sha256 = \"1h008sx33xq0hha2lxd5dsh2wr7rzlv4nifgr4k5knpw5ahq1f88\";\n };\n}\n" }, { "path": "snap/snapcraft.yaml", "content": "name: deadfinder\nbase: core24\nversion: 2.0.2\nsummary: Find dead (broken) links in web pages, URL lists, and sitemaps.\ndescription: |\n DeadFinder is a fast CLI tool for detecting broken links on a page, a\n list of URLs, or an entire sitemap. Written in Crystal for native\n speed and fiber-based concurrency. Supports JSON/YAML/TOML/CSV output\n and coverage reporting.\n\ngrade: stable\nconfinement: strict\nlicense: MIT\n\napps:\n deadfinder:\n command: deadfinder\n plugs:\n - home\n - removable-media\n - network\n - network-bind\n\nparts:\n deadfinder:\n source: ./\n plugin: nil\n override-build: |\n curl -fsSL https://crystal-lang.org/install.sh | bash\n shards install --production\n shards build --release --no-debug --production\n cp ./bin/deadfinder $CRAFT_PART_INSTALL/\n build-packages:\n - git\n - curl\n - cmake\n - make\n - g++\n - pkg-config\n - libssl-dev\n - libxml2-dev\n - libz-dev\n - libyaml-dev\n - libpcre2-dev\n - libevent-dev\n - libgmp-dev\n stage-packages:\n - libxml2\n - zlib1g\n - libyaml-0-2\n - ca-certificates\n" }, { "path": "spec/compat/README.md", "content": "# Compatibility harness\n\nRuby 원본 v1의 출력을 **골든 파일로 동결**하고, Crystal 바이너리가 동일 출력을 내는지 검증하는 블랙박스 테스트다.\n\n## 구조\n\n```\nspec/compat/\n├── fixtures/\n│ └── server.rb # 최소 HTTP fixture 서버 (Ruby stdlib only)\n├── golden/\n│ └── .{json,yaml,toml,csv} # 기대 출력. {{BASE}} 플레이스홀더\n├── run.rb # 드라이버: 서버 기동 → 바이너리 실행 → 비교\n└── README.md\n```\n\n## 실행\n\n```bash\nshards install\ncrystal build src/cli_main.cr -o deadfinder --release\nBIN=\"./deadfinder\" ruby spec/compat/run.rb\n```\n\n## 케이스 추가\n\n1. `fixtures/server.rb`의 `ROUTES`에 필요한 경로 추가\n2. `golden/.`에 기대 출력 작성 (`{{BASE}}`로 origin 표현)\n3. `run.rb` 맨 아래 `run_case(...)` 한 줄 추가\n\n## 비교 규칙\n\n- 배열은 정렬 후 비교 (링크 추출 순서 비결정성 흡수)\n- `{{BASE}}` 플레이스홀더는 실행 시 동적 포트로 치환\n- 출력은 `-o `로 받아 파일에서 파싱\n\n## 왜 Ruby 드라이버?\n\n골든 파일은 v1 Ruby 출력의 스냅샷이고, 비교 로직에 `toml-rb` 같은 파서가 필요해서 그대로 Ruby 드라이버를 유지했다. Crystal로 포팅할 수도 있지만 CI 복잡도 대비 이득이 적다.\n" }, { "path": "spec/compat/fixtures/server.rb", "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\nrequire 'socket'\n\nROUTES = {\n '/index.html' => {\n status: 200,\n content_type: 'text/html',\n body: <<~HTML\n \n \n ok\n dead\n redirect\n \n HTML\n },\n '/ok' => { status: 200, content_type: 'text/plain', body: 'OK' },\n '/dead' => { status: 404, content_type: 'text/plain', body: 'Not Found' },\n '/redirect' => { status: 301, content_type: 'text/plain', body: '', extra: { 'Location' => '/ok' } }\n}.freeze\n\nSTATUS_TEXT = { 200 => 'OK', 301 => 'Moved Permanently', 404 => 'Not Found' }.freeze\n\nserver = TCPServer.new('127.0.0.1', 0)\nputs server.addr[1]\nSTDOUT.flush\n\ntrap('TERM') { exit 0 }\ntrap('INT') { exit 0 }\n\nloop do\n client = server.accept\n begin\n request_line = client.gets\n raw_path = request_line&.split(' ')&.dig(1) || '/'\n path = raw_path.split('?').first\n while (line = client.gets) && line.strip != ''; end\n\n route = ROUTES[path]\n if route\n headers = {\n 'Content-Type' => route[:content_type],\n 'Content-Length' => route[:body].bytesize.to_s\n }.merge(route[:extra] || {})\n client.print \"HTTP/1.1 #{route[:status]} #{STATUS_TEXT[route[:status]] || 'OK'}\\r\\n\"\n headers.each { |k, v| client.print \"#{k}: #{v}\\r\\n\" }\n client.print \"\\r\\n#{route[:body]}\"\n else\n client.print \"HTTP/1.1 404 Not Found\\r\\nContent-Length: 0\\r\\n\\r\\n\"\n end\n rescue StandardError\n # swallow: test fixture, keep accepting\n ensure\n client&.close\n end\nend\n" }, { "path": "spec/compat/golden/file_json.json", "content": "{\n \"{{BASE}}/index.html\": [\n \"{{BASE}}/dead\"\n ]\n}\n" }, { "path": "spec/compat/golden/pipe_json.json", "content": "{\n \"{{BASE}}/index.html\": [\n \"{{BASE}}/dead\"\n ]\n}\n" }, { "path": "spec/compat/golden/url_csv.csv", "content": "target,url\n{{BASE}}/index.html,{{BASE}}/dead\n" }, { "path": "spec/compat/golden/url_json.json", "content": "{\n \"{{BASE}}/index.html\": [\n \"{{BASE}}/dead\"\n ]\n}\n" }, { "path": "spec/compat/golden/url_json_include30x.json", "content": "{\n \"{{BASE}}/index.html\": [\n \"{{BASE}}/dead\",\n \"{{BASE}}/redirect\"\n ]\n}\n" }, { "path": "spec/compat/golden/url_toml.toml", "content": "\"{{BASE}}/index.html\" = [\"{{BASE}}/dead\"]\n" }, { "path": "spec/compat/golden/url_yaml.yaml", "content": "---\n{{BASE}}/index.html:\n- {{BASE}}/dead\n" }, { "path": "spec/compat/run.rb", "content": "#!/usr/bin/env ruby\n# frozen_string_literal: true\n\n# Black-box compatibility harness for the deadfinder Crystal binary.\n#\n# The golden files in this directory were captured from the v1 Ruby\n# implementation and now act as the frozen contract the Crystal binary\n# must match. The harness runs the binary under test against a local\n# fixture server, writes the output to a temp file, and compares the\n# parsed structure to the corresponding golden file (with `{{BASE}}`\n# substituted for the dynamic fixture origin).\n#\n# Usage:\n# BIN=\"./deadfinder\" ruby spec/compat/run.rb\n# BIN=\"/path/to/deadfinder\" ruby spec/compat/run.rb\n\nrequire 'csv'\nrequire 'json'\nrequire 'open3'\nrequire 'tempfile'\nrequire 'toml-rb'\nrequire 'yaml'\n\nHARNESS_ROOT = __dir__\n\nBIN = ENV.fetch('BIN', './deadfinder')\n\ndef sort_arrays(obj)\n case obj\n when Hash then obj.transform_values { |v| sort_arrays(v) }\n when Array then obj.map { |v| sort_arrays(v) }.sort_by(&:to_s)\n else obj\n end\nend\n\ndef parse_output(path, format)\n text = File.read(path)\n case format\n when 'json' then JSON.parse(text)\n when 'yaml', 'yml' then YAML.safe_load(text)\n when 'toml' then TomlRB.parse(text)\n when 'csv' then CSV.parse(text)\n else raise \"unknown format: #{format}\"\n end\nend\n\ndef substitute_base(text, base)\n text.gsub('{{BASE}}', base)\nend\n\ndef run_case(base, name:, args:, format:, golden:, stdin: nil, extra_files: {})\n extra_files.each do |path, content|\n File.write(path, substitute_base(content, base))\n end\n\n Tempfile.create(['deadfinder', \".#{format}\"]) do |tmp|\n resolved_args = substitute_base(args, base)\n cmd = \"#{BIN} #{resolved_args} -o #{tmp.path} -f #{format} -s\"\n stdout, stderr, status = Open3.capture3(cmd, stdin_data: stdin || '')\n\n unless status.success?\n warn \"FAIL: #{name} — exit #{status.exitstatus}\"\n warn \"CMD: #{cmd}\"\n warn \"STDOUT: #{stdout}\"\n warn \"STDERR: #{stderr}\"\n return false\n end\n\n expected_text = substitute_base(File.read(golden), base)\n expected_path = Tempfile.new(['expected', \".#{format}\"]).tap do |f|\n f.write(expected_text)\n f.close\n end.path\n\n expected = parse_output(expected_path, format)\n actual = parse_output(tmp.path, format)\n\n if sort_arrays(actual) == sort_arrays(expected)\n\n true\n else\n warn \"FAIL: #{name}\"\n warn \"EXPECTED: #{expected.inspect}\"\n warn \"ACTUAL: #{actual.inspect}\"\n false\n end\n end\nensure\n extra_files.each_key { |path| FileUtils.rm_f(path) }\nend\n\n# --- Boot fixture server ----------------------------------------------------\nserver_io = IO.popen(['ruby', \"#{HARNESS_ROOT}/fixtures/server.rb\"], 'r')\nport = server_io.gets&.strip\nabort 'fixture server did not start' unless port && !port.empty?\nbase = \"http://127.0.0.1:#{port}\"\n\nat_exit do\n Process.kill('TERM', server_io.pid)\nrescue Errno::ESRCH\n # already gone\nend\n\n# --- Cases ------------------------------------------------------------------\nurls_file = File.join(Dir.tmpdir, \"deadfinder_compat_urls_#{Process.pid}.txt\")\n\nresults = []\n\nresults << run_case(base,\n name: 'url_json',\n args: 'url {{BASE}}/index.html',\n format: 'json',\n golden: \"#{HARNESS_ROOT}/golden/url_json.json\")\n\nresults << run_case(base,\n name: 'url_yaml',\n args: 'url {{BASE}}/index.html',\n format: 'yaml',\n golden: \"#{HARNESS_ROOT}/golden/url_yaml.yaml\")\n\nresults << run_case(base,\n name: 'url_toml',\n args: 'url {{BASE}}/index.html',\n format: 'toml',\n golden: \"#{HARNESS_ROOT}/golden/url_toml.toml\")\n\nresults << run_case(base,\n name: 'url_csv',\n args: 'url {{BASE}}/index.html',\n format: 'csv',\n golden: \"#{HARNESS_ROOT}/golden/url_csv.csv\")\n\nresults << run_case(base,\n name: 'url_json_include30x',\n args: 'url {{BASE}}/index.html -r',\n format: 'json',\n golden: \"#{HARNESS_ROOT}/golden/url_json_include30x.json\")\n\nresults << run_case(base,\n name: 'file_json',\n args: \"file #{urls_file}\",\n format: 'json',\n golden: \"#{HARNESS_ROOT}/golden/file_json.json\",\n extra_files: { urls_file => \"{{BASE}}/index.html\\n\" })\n\nresults << run_case(base,\n name: 'pipe_json',\n args: 'pipe',\n format: 'json',\n golden: \"#{HARNESS_ROOT}/golden/pipe_json.json\",\n stdin: substitute_base(\"{{BASE}}/index.html\\n\", base))\n\nexit(results.all? ? 0 : 1)\n" }, { "path": "spec/deadfinder/cli_spec.cr", "content": "require \"../spec_helper\"\n\ndescribe Deadfinder::CLI do\n before_each do\n WebMock.reset\n reset_deadfinder_state\n end\n\n describe \"Options defaults\" do\n it \"has correct default values\" do\n options = Deadfinder::Options.new\n options.concurrency.should eq 50\n options.timeout.should eq 10\n options.output.should eq \"\"\n options.output_format.should eq \"json\"\n options.headers.should eq [] of String\n options.worker_headers.should eq [] of String\n options.silent.should be_false\n options.verbose.should be_false\n options.debug.should be_false\n options.include30x.should be_false\n options.proxy.should eq \"\"\n options.proxy_auth.should eq \"\"\n options.match.should eq \"\"\n options.ignore.should eq \"\"\n options.coverage.should be_false\n options.visualize.should eq \"\"\n options.limit.should eq 0\n end\n end\n\n describe \"completion scripts\" do\n it \"generates bash completion script\" do\n script = Deadfinder::Completion.bash\n script.should contain \"_deadfinder_completions\"\n script.should contain \"complete -F _deadfinder_completions deadfinder\"\n script.should contain \"COMPREPLY\"\n end\n\n it \"generates zsh completion script\" do\n script = Deadfinder::Completion.zsh\n script.should contain \"#compdef deadfinder\"\n script.should contain \"_arguments\"\n script.should contain \"--include30x\"\n end\n\n it \"generates fish completion script\" do\n script = Deadfinder::Completion.fish\n script.should contain \"complete -c deadfinder -l include30x\"\n script.should contain \"complete -c deadfinder -l debug -d 'Debug mode'\"\n script.should contain \"complete -c deadfinder -l concurrency\"\n end\n end\n\n describe \"version\" do\n it \"has correct version\" do\n Deadfinder::VERSION.should eq \"2.0.2\"\n end\n end\nend\n" }, { "path": "spec/deadfinder/http_client_spec.cr", "content": "require \"../spec_helper\"\n\ndescribe Deadfinder::HttpClient do\n before_each do\n reset_deadfinder_state\n end\n\n describe \".create\" do\n it \"creates a basic HTTP client\" do\n uri = URI.parse(\"http://example.com\")\n options = default_test_options\n client = Deadfinder::HttpClient.create(uri, options)\n client.should be_a(HTTP::Client)\n end\n\n it \"creates an HTTPS client with SSL\" do\n uri = URI.parse(\"https://example.com\")\n options = default_test_options\n client = Deadfinder::HttpClient.create(uri, options)\n client.should be_a(HTTP::Client)\n end\n\n it \"creates client with custom timeout without error\" do\n uri = URI.parse(\"http://example.com\")\n options = default_test_options\n options.timeout = 5\n client = Deadfinder::HttpClient.create(uri, options)\n client.should be_a(HTTP::Client)\n end\n\n it \"falls back to direct connection when proxy has no host\" do\n uri = URI.parse(\"http://example.com\")\n options = default_test_options\n options.proxy = \"not-a-valid-proxy\"\n client = Deadfinder::HttpClient.create(uri, options)\n client.should be_a(HTTP::Client)\n end\n\n it \"creates client without proxy when proxy is empty\" do\n uri = URI.parse(\"http://example.com\")\n options = default_test_options\n options.proxy = \"\"\n client = Deadfinder::HttpClient.create(uri, options)\n client.should be_a(HTTP::Client)\n end\n\n it \"creates an HTTPS client when insecure flag is enabled\" do\n uri = URI.parse(\"https://example.com\")\n options = default_test_options\n options.insecure = true\n client = Deadfinder::HttpClient.create(uri, options)\n client.should be_a(HTTP::Client)\n end\n\n it \"creates an HTTPS client with verification enabled by default\" do\n uri = URI.parse(\"https://example.com\")\n options = default_test_options\n options.insecure.should be_false\n client = Deadfinder::HttpClient.create(uri, options)\n client.should be_a(HTTP::Client)\n end\n end\n\n describe \".proxy_configured?\" do\n it \"returns false when proxy is empty\" do\n options = default_test_options\n options.proxy = \"\"\n Deadfinder::HttpClient.proxy_configured?(options).should be_false\n end\n\n it \"returns true when proxy is set\" do\n options = default_test_options\n options.proxy = \"http://proxy.example.com:8080\"\n Deadfinder::HttpClient.proxy_configured?(options).should be_true\n end\n end\n\n describe \".absolute_uri\" do\n it \"returns the full URI string\" do\n uri = URI.parse(\"http://example.com/path?q=1\")\n Deadfinder::HttpClient.absolute_uri(uri).should eq(\"http://example.com/path?q=1\")\n end\n end\nend\n" }, { "path": "spec/deadfinder/logger_spec.cr", "content": "require \"../spec_helper\"\n\ndescribe Deadfinder::Logger do\n before_each do\n Deadfinder::Logger.unset_silent\n Deadfinder::Logger.unset_verbose\n Deadfinder::Logger.unset_debug\n end\n\n describe \".apply_options\" do\n it \"sets silent mode when options has silent\" do\n options = Deadfinder::Options.new\n options.silent = true\n options.verbose = false\n options.debug = false\n Deadfinder::Logger.apply_options(options)\n Deadfinder::Logger.silent?.should be_true\n end\n\n it \"sets verbose mode when options has verbose\" do\n options = Deadfinder::Options.new\n options.silent = false\n options.verbose = true\n options.debug = false\n Deadfinder::Logger.apply_options(options)\n Deadfinder::Logger.verbose?.should be_true\n end\n\n it \"sets debug mode when options has debug\" do\n options = Deadfinder::Options.new\n options.silent = false\n options.verbose = false\n options.debug = true\n Deadfinder::Logger.apply_options(options)\n Deadfinder::Logger.debug?.should be_true\n end\n\n it \"sets multiple modes simultaneously\" do\n options = Deadfinder::Options.new\n options.silent = true\n options.verbose = true\n options.debug = true\n Deadfinder::Logger.apply_options(options)\n Deadfinder::Logger.silent?.should be_true\n Deadfinder::Logger.verbose?.should be_true\n Deadfinder::Logger.debug?.should be_true\n end\n end\n\n describe \".silent?\" do\n it \"returns false by default\" do\n Deadfinder::Logger.silent?.should be_false\n end\n end\n\n describe \".set_silent / .unset_silent\" do\n it \"sets and unsets silent mode\" do\n Deadfinder::Logger.set_silent\n Deadfinder::Logger.silent?.should be_true\n Deadfinder::Logger.unset_silent\n Deadfinder::Logger.silent?.should be_false\n end\n end\n\n describe \".verbose?\" do\n it \"returns false by default\" do\n Deadfinder::Logger.verbose?.should be_false\n end\n end\n\n describe \".set_verbose / .unset_verbose\" do\n it \"sets and unsets verbose mode\" do\n Deadfinder::Logger.set_verbose\n Deadfinder::Logger.verbose?.should be_true\n Deadfinder::Logger.unset_verbose\n Deadfinder::Logger.verbose?.should be_false\n end\n end\n\n describe \".debug?\" do\n it \"returns false by default\" do\n Deadfinder::Logger.debug?.should be_false\n end\n end\n\n describe \".set_debug / .unset_debug\" do\n it \"sets and unsets debug mode\" do\n Deadfinder::Logger.set_debug\n Deadfinder::Logger.debug?.should be_true\n Deadfinder::Logger.unset_debug\n Deadfinder::Logger.debug?.should be_false\n end\n end\n\n describe \"output suppression in silent mode\" do\n it \"does not output when silent\" do\n Deadfinder::Logger.set_silent\n # These should not raise and should produce no visible output\n Deadfinder::Logger.info(\"test\")\n Deadfinder::Logger.error(\"test\")\n Deadfinder::Logger.target(\"test\")\n Deadfinder::Logger.sub_info(\"test\")\n Deadfinder::Logger.sub_complete(\"test\")\n Deadfinder::Logger.found(\"test\")\n end\n end\nend\n" }, { "path": "spec/deadfinder/runner_spec.cr", "content": "require \"../spec_helper\"\n\ndescribe Deadfinder::Runner do\n before_each { WebMock.reset }\n\n describe \"#run\" do\n it \"finds broken links (404)\" do\n target = \"http://example.com\"\n html = <<-HTML\n \n Broken\n Valid\n \n HTML\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/broken\").to_return(status: 404)\n WebMock.stub(:get, \"http://example.com/valid\").to_return(status: 200)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n args[:output][target]?.should_not be_nil\n args[:output][target].should contain \"http://example.com/broken\"\n args[:output][target].should_not contain \"http://example.com/valid\"\n end\n\n it \"finds multiple broken links\" do\n target = \"http://example.com\"\n html = <<-HTML\n \n D1\n D2\n OK\n \n HTML\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/dead1\").to_return(status: 404)\n WebMock.stub(:get, \"http://example.com/dead2\").to_return(status: 500)\n WebMock.stub(:get, \"http://example.com/ok\").to_return(status: 200)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n args[:output][target].should contain \"http://example.com/dead1\"\n args[:output][target].should contain \"http://example.com/dead2\"\n args[:output][target].should_not contain \"http://example.com/ok\"\n end\n\n it \"does not flag 3xx as dead by default\" do\n target = \"http://example.com\"\n html = %(R)\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/redirect\").to_return(status: 301)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n (args[:output][target]? || [] of String).should_not contain \"http://example.com/redirect\"\n end\n\n it \"flags 3xx as dead when include30x is true\" do\n target = \"http://example.com\"\n html = %(R)\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/redirect\").to_return(status: 301)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n options.include30x = true\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n args[:output][target]?.should_not be_nil\n args[:output][target].should contain \"http://example.com/redirect\"\n end\n\n it \"respects match option - only checks matched URLs\" do\n target = \"http://example.com\"\n html = <<-HTML\n \n Broken\n Valid\n \n HTML\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/broken\").to_return(status: 404)\n # valid은 match 안 하므로 stub 불필요하지만 안전하게 추가\n WebMock.stub(:get, \"http://example.com/valid\").to_return(status: 200)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n options.match = \"broken\"\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n args[:output][target]?.should_not be_nil\n args[:output][target].should contain \"http://example.com/broken\"\n end\n\n it \"respects ignore option - skips ignored URLs\" do\n target = \"http://example.com\"\n html = <<-HTML\n \n Broken\n Valid\n \n HTML\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/broken\").to_return(status: 404)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n options.ignore = \"valid\"\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n args[:output][target]?.should_not be_nil\n args[:output][target].should contain \"http://example.com/broken\"\n args[:output][target].should_not contain \"http://example.com/valid\"\n end\n\n it \"handles invalid match pattern gracefully\" do\n target = \"http://example.com\"\n html = %(Link)\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/page\").to_return(status: 200)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n options.match = \"[\"\n args = make_runner_args\n\n # Should not raise - error is logged internally\n runner.run(target, options, **args)\n end\n\n it \"handles invalid ignore pattern gracefully\" do\n target = \"http://example.com\"\n html = %(Link)\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/page\").to_return(status: 200)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n options.ignore = \"[\"\n args = make_runner_args\n\n # Should not raise\n runner.run(target, options, **args)\n end\n\n it \"handles target fetch failure gracefully\" do\n target = \"http://unreachable.invalid\"\n WebMock.stub(:get, target).to_return(status: 500, body: \"\")\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n # Should not raise\n runner.run(target, options, **args)\n end\n\n it \"extracts links from all 7 HTML element types\" do\n target = \"http://example.com\"\n html = <<-HTML\n \n \n \n \n \n \n Link\n \n
\n \n \n \n HTML\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/script.js\").to_return(status: 404)\n WebMock.stub(:get, \"http://example.com/style.css\").to_return(status: 404)\n WebMock.stub(:get, \"http://example.com/page\").to_return(status: 404)\n WebMock.stub(:get, \"http://example.com/frame\").to_return(status: 404)\n WebMock.stub(:get, \"http://example.com/submit\").to_return(status: 404)\n WebMock.stub(:get, \"http://example.com/object.swf\").to_return(status: 404)\n WebMock.stub(:get, \"http://example.com/embed.swf\").to_return(status: 404)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n dead = args[:output][target]\n dead.should contain \"http://example.com/script.js\"\n dead.should contain \"http://example.com/style.css\"\n dead.should contain \"http://example.com/page\"\n dead.should contain \"http://example.com/frame\"\n dead.should contain \"http://example.com/submit\"\n dead.should contain \"http://example.com/object.swf\"\n dead.should contain \"http://example.com/embed.swf\"\n end\n\n it \"resolves relative URLs against target\" do\n target = \"http://example.com/docs/\"\n html = %(AboutPage)\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/about\").to_return(status: 404)\n WebMock.stub(:get, \"http://example.com/docs/page.html\").to_return(status: 404)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n dead = args[:output][target]\n dead.should contain \"http://example.com/about\"\n dead.should contain \"http://example.com/docs/page.html\"\n end\n\n it \"skips mailto/tel/data scheme links\" do\n target = \"http://example.com\"\n html = <<-HTML\n \n Mail\n Tel\n Data\n Real\n \n HTML\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/real\").to_return(status: 200)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n # No dead links from special schemes, and no errors\n dead = args[:output][target]? || [] of String\n dead.should_not contain \"mailto:test@example.com\"\n dead.should_not contain \"tel:1234567890\"\n end\n\n it \"deduplicates URLs\" do\n target = \"http://example.com\"\n html = <<-HTML\n \n Link1\n Link2\n Link3\n \n HTML\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/dup\").to_return(status: 404)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n # Should appear only once in output\n args[:output][target].count(\"http://example.com/dup\").should eq 1\n end\n\n it \"tracks coverage data when coverage is enabled\" do\n target = \"http://example.com\"\n html = <<-HTML\n \n Dead\n Ok1\n Ok2\n \n HTML\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/dead\").to_return(status: 404)\n WebMock.stub(:get, \"http://example.com/ok1\").to_return(status: 200)\n WebMock.stub(:get, \"http://example.com/ok2\").to_return(status: 200)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n options.coverage = true\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n cov = args[:coverage_data][target]\n cov.total.should eq 3\n cov.dead.should eq 1\n cov.status_counts[\"404\"].should eq 1\n cov.status_counts[\"200\"].should eq 2\n end\n\n it \"does not track coverage when coverage is disabled\" do\n target = \"http://example.com\"\n html = %(L)\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://example.com/page\").to_return(status: 404)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n options.coverage = false\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n args[:coverage_data][target]?.should be_nil\n end\n\n it \"handles empty HTML page with no links\" do\n target = \"http://example.com\"\n WebMock.stub(:get, target).to_return(body: \"\")\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n runner.run(target, options, **args)\n\n (args[:output][target]? || [] of String).should be_empty\n end\n end\n\n describe \"#worker\" do\n it \"detects 404 as broken link\" do\n target = \"http://example.com\"\n url = \"http://example.com/broken\"\n\n WebMock.stub(:get, url).to_return(status: 404)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n jobs = Channel(String).new(10)\n results = Channel(String).new(10)\n jobs.send(url)\n jobs.close\n\n runner.worker(1, jobs, results, target, options, **args)\n\n args[:output][target].should contain url\n end\n\n it \"detects 500 as broken link\" do\n target = \"http://example.com\"\n url = \"http://example.com/error\"\n\n WebMock.stub(:get, url).to_return(status: 500)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n jobs = Channel(String).new(10)\n results = Channel(String).new(10)\n jobs.send(url)\n jobs.close\n\n runner.worker(1, jobs, results, target, options, **args)\n\n args[:output][target].should contain url\n end\n\n it \"does not flag 200 as broken\" do\n target = \"http://example.com\"\n url = \"http://example.com/ok\"\n\n WebMock.stub(:get, url).to_return(status: 200)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n jobs = Channel(String).new(10)\n results = Channel(String).new(10)\n jobs.send(url)\n jobs.close\n\n runner.worker(1, jobs, results, target, options, **args)\n\n (args[:output][target]? || [] of String).should_not contain url\n end\n\n it \"does not flag 301 as broken without include30x\" do\n target = \"http://example.com\"\n url = \"http://example.com/moved\"\n\n WebMock.stub(:get, url).to_return(status: 301)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n options.include30x = false\n args = make_runner_args\n\n jobs = Channel(String).new(10)\n results = Channel(String).new(10)\n jobs.send(url)\n jobs.close\n\n runner.worker(1, jobs, results, target, options, **args)\n\n (args[:output][target]? || [] of String).should_not contain url\n end\n\n it \"flags 301 as broken with include30x\" do\n target = \"http://example.com\"\n url = \"http://example.com/moved\"\n\n WebMock.stub(:get, url).to_return(status: 301)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n options.include30x = true\n args = make_runner_args\n\n jobs = Channel(String).new(10)\n results = Channel(String).new(10)\n jobs.send(url)\n jobs.close\n\n runner.worker(1, jobs, results, target, options, **args)\n\n args[:output][target].should contain url\n end\n\n it \"skips already cached URLs\" do\n target = \"http://example.com\"\n url = \"http://example.com/cached\"\n\n WebMock.stub(:get, url).to_return(status: 404)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n # Pre-populate cache\n args[:cache_set][url] = true\n\n jobs = Channel(String).new(10)\n results = Channel(String).new(10)\n jobs.send(url)\n jobs.close\n\n runner.worker(1, jobs, results, target, options, **args)\n\n # Should NOT appear in output because it was cached\n (args[:output][target]? || [] of String).should_not contain url\n end\n\n it \"processes multiple jobs sequentially\" do\n target = \"http://example.com\"\n\n WebMock.stub(:get, \"http://example.com/a\").to_return(status: 404)\n WebMock.stub(:get, \"http://example.com/b\").to_return(status: 200)\n WebMock.stub(:get, \"http://example.com/c\").to_return(status: 503)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n args = make_runner_args\n\n jobs = Channel(String).new(10)\n results = Channel(String).new(10)\n jobs.send(\"http://example.com/a\")\n jobs.send(\"http://example.com/b\")\n jobs.send(\"http://example.com/c\")\n jobs.close\n\n runner.worker(1, jobs, results, target, options, **args)\n\n dead = args[:output][target]\n dead.should contain \"http://example.com/a\"\n dead.should_not contain \"http://example.com/b\"\n dead.should contain \"http://example.com/c\"\n end\n\n it \"tracks coverage with status counts\" do\n target = \"http://example.com\"\n\n WebMock.stub(:get, \"http://example.com/ok\").to_return(status: 200)\n WebMock.stub(:get, \"http://example.com/not-found\").to_return(status: 404)\n WebMock.stub(:get, \"http://example.com/server-err\").to_return(status: 500)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n options.coverage = true\n args = make_runner_args\n\n jobs = Channel(String).new(10)\n results = Channel(String).new(10)\n jobs.send(\"http://example.com/ok\")\n jobs.send(\"http://example.com/not-found\")\n jobs.send(\"http://example.com/server-err\")\n jobs.close\n\n runner.worker(1, jobs, results, target, options, **args)\n\n cov = args[:coverage_data][target]\n cov.total.should eq 3\n cov.dead.should eq 2\n cov.status_counts[\"200\"].should eq 1\n cov.status_counts[\"404\"].should eq 1\n cov.status_counts[\"500\"].should eq 1\n end\n\n it \"sends worker_headers with requests\" do\n target = \"http://example.com\"\n url = \"http://example.com/authed\"\n\n WebMock.stub(:get, url)\n .with(headers: {\"Authorization\" => \"Bearer token123\"})\n .to_return(status: 200)\n\n runner = Deadfinder::Runner.new\n options = default_test_options\n options.worker_headers = [\"Authorization: Bearer token123\"]\n args = make_runner_args\n\n jobs = Channel(String).new(10)\n results = Channel(String).new(10)\n jobs.send(url)\n jobs.close\n\n runner.worker(1, jobs, results, target, options, **args)\n\n # Should not be in dead links (200 response with correct headers)\n (args[:output][target]? || [] of String).should_not contain url\n end\n end\nend\n" }, { "path": "spec/deadfinder/url_pattern_matcher_spec.cr", "content": "require \"../spec_helper\"\n\ndescribe Deadfinder::UrlPatternMatcher do\n describe \".match?\" do\n it \"returns true when the URL matches the pattern\" do\n Deadfinder::UrlPatternMatcher.match?(\"http://example.com\", \"example\").should be_true\n end\n\n it \"returns false when the URL does not match the pattern\" do\n Deadfinder::UrlPatternMatcher.match?(\"http://example.com\", \"nonexistent\").should be_false\n end\n\n it \"raises an error when the pattern is an invalid regex\" do\n expect_raises(ArgumentError) do\n Deadfinder::UrlPatternMatcher.match?(\"http://example.com\", \"[\")\n end\n end\n\n it \"supports complex regex patterns\" do\n Deadfinder::UrlPatternMatcher.match?(\"http://example.com/path/to/page\", \"path/to/\\\\w+\").should be_true\n end\n\n it \"supports anchored patterns\" do\n Deadfinder::UrlPatternMatcher.match?(\"http://example.com\", \"^http://example\").should be_true\n Deadfinder::UrlPatternMatcher.match?(\"http://example.com\", \"^https://example\").should be_false\n end\n\n it \"matches query parameters\" do\n Deadfinder::UrlPatternMatcher.match?(\"http://example.com?foo=bar\", \"foo=bar\").should be_true\n end\n end\n\n describe \".ignore?\" do\n it \"returns true when the URL matches the pattern\" do\n Deadfinder::UrlPatternMatcher.ignore?(\"http://example.com\", \"example\").should be_true\n end\n\n it \"returns false when the URL does not match the pattern\" do\n Deadfinder::UrlPatternMatcher.ignore?(\"http://example.com\", \"nonexistent\").should be_false\n end\n\n it \"raises an error when the pattern is an invalid regex\" do\n expect_raises(ArgumentError) do\n Deadfinder::UrlPatternMatcher.ignore?(\"http://example.com\", \"[\")\n end\n end\n\n it \"can ignore multiple URL patterns with alternation\" do\n Deadfinder::UrlPatternMatcher.ignore?(\"http://example.com/ads\", \"ads|tracking\").should be_true\n Deadfinder::UrlPatternMatcher.ignore?(\"http://example.com/tracking\", \"ads|tracking\").should be_true\n Deadfinder::UrlPatternMatcher.ignore?(\"http://example.com/page\", \"ads|tracking\").should be_false\n end\n end\n\n describe \"ReDoS guardrails\" do\n before_each { Deadfinder::UrlPatternMatcher.clear_cache }\n\n it \"rejects patterns longer than MAX_PATTERN_LENGTH\" do\n long_pattern = \"a\" * (Deadfinder::UrlPatternMatcher::MAX_PATTERN_LENGTH + 1)\n expect_raises(Deadfinder::UrlPatternMatcher::UnsafePatternError) do\n Deadfinder::UrlPatternMatcher.match?(\"http://example.com\", long_pattern)\n end\n end\n\n it \"rejects classic nested-quantifier ReDoS shapes like (a+)+\" do\n expect_raises(Deadfinder::UrlPatternMatcher::UnsafePatternError) do\n Deadfinder::UrlPatternMatcher.match?(\"aaaa\", \"(a+)+\")\n end\n end\n\n it \"rejects (a*)* \" do\n expect_raises(Deadfinder::UrlPatternMatcher::UnsafePatternError) do\n Deadfinder::UrlPatternMatcher.ignore?(\"aaaa\", \"(a*)*\")\n end\n end\n\n it \"rejects (.+){2,} bounded-repeat variant\" do\n expect_raises(Deadfinder::UrlPatternMatcher::UnsafePatternError) do\n Deadfinder::UrlPatternMatcher.match?(\"aaaa\", \"(.+){2,}\")\n end\n end\n\n it \"UnsafePatternError is-a ArgumentError so runner rescue still catches\" do\n (Deadfinder::UrlPatternMatcher::UnsafePatternError < ArgumentError).should be_true\n end\n\n it \"does not flag patterns with escaped literal parens\" do\n # `\\(a+\\)+` = literal `(`, one-or-more `a`, literal `)`, one-or-more —\n # there's no actual group being quantified, so no catastrophic backtracking.\n Deadfinder::UrlPatternMatcher.match?(\"(aaa))))\", \"\\\\(a+\\\\)+\").should be_true\n end\n end\n\n describe \"regex caching\" do\n before_each { Deadfinder::UrlPatternMatcher.clear_cache }\n\n it \"reuses the compiled regex across calls with the same pattern\" do\n pattern = \"example\"\n Deadfinder::UrlPatternMatcher.match?(\"http://example.com\", pattern)\n Deadfinder::UrlPatternMatcher.match?(\"http://example.org\", pattern)\n Deadfinder::UrlPatternMatcher.match?(\"http://other.com\", pattern)\n # No public accessor to the cache map, but we at least exercise the\n # hot path to confirm it does not blow up and returns consistent results.\n Deadfinder::UrlPatternMatcher.match?(\"http://example.com\", pattern).should be_true\n end\n end\nend\n" }, { "path": "spec/deadfinder/utils_spec.cr", "content": "require \"../spec_helper\"\n\ndescribe \"Deadfinder.generate_url\" do\n base_url = \"http://example.com/base/\"\n\n it \"returns the original URL if it starts with http://\" do\n Deadfinder.generate_url(\"http://example.com\", base_url).should eq \"http://example.com\"\n end\n\n it \"returns the original URL if it starts with https://\" do\n Deadfinder.generate_url(\"https://example.com\", base_url).should eq \"https://example.com\"\n end\n\n it \"prepends the scheme if the URL starts with //\" do\n Deadfinder.generate_url(\"//example.com\", base_url).should eq \"http://example.com\"\n end\n\n it \"prepends the scheme and host if the URL starts with /\" do\n Deadfinder.generate_url(\"/path\", base_url).should eq \"http://example.com/path\"\n end\n\n it \"returns nil if the URL should ignore the scheme\" do\n Deadfinder.generate_url(\"mailto:test@example.com\", base_url).should be_nil\n end\n\n it \"prepends the base directory if the URL is relative\" do\n Deadfinder.generate_url(\"relative/path\", base_url).should eq \"http://example.com/base/relative/path\"\n end\n\n it \"returns nil if base_url is invalid\" do\n Deadfinder.generate_url(\"relative/path\", \"://invalid\").should be_nil\n end\n\n it \"returns nil for empty text\" do\n Deadfinder.generate_url(\"\", base_url).should be_nil\n end\n\n it \"returns nil for whitespace-only text\" do\n Deadfinder.generate_url(\" \", base_url).should be_nil\n end\n\n it \"returns nil for javascript: scheme\" do\n Deadfinder.generate_url(\"javascript:void(0)\", base_url).should be_nil\n end\n\n it \"returns nil for data: scheme\" do\n Deadfinder.generate_url(\"data:text/plain,hello\", base_url).should be_nil\n end\n\n it \"returns nil for fragment-only (#) links\" do\n Deadfinder.generate_url(\"#section\", base_url).should be_nil\n end\n\n it \"handles protocol-relative URLs with https base\" do\n Deadfinder.generate_url(\"//cdn.example.com/lib.js\", \"https://example.com/\").should eq \"https://cdn.example.com/lib.js\"\n end\n\n it \"resolves relative URL when base path does not end with /\" do\n Deadfinder.generate_url(\"page.html\", \"http://example.com/dir/index.html\").should eq \"http://example.com/dir/page.html\"\n end\n\n it \"handles root-relative paths\" do\n Deadfinder.generate_url(\"/about\", \"https://example.com/some/deep/path\").should eq \"https://example.com/about\"\n end\n\n it \"preserves non-default port when resolving root-relative paths\" do\n Deadfinder.generate_url(\"/about\", \"http://127.0.0.1:8080/index.html\").should eq \"http://127.0.0.1:8080/about\"\n end\n\n it \"preserves non-default port when resolving relative paths\" do\n Deadfinder.generate_url(\"about\", \"http://127.0.0.1:8080/index.html\").should eq \"http://127.0.0.1:8080/about\"\n end\n\n it \"preserves non-default port when base path is a directory\" do\n Deadfinder.generate_url(\"page.html\", \"http://127.0.0.1:8080/dir/\").should eq \"http://127.0.0.1:8080/dir/page.html\"\n end\nend\n\ndescribe \"Deadfinder.ignore_scheme?\" do\n it \"returns true for mailto: URLs\" do\n Deadfinder.ignore_scheme?(\"mailto:test@example.com\").should be_true\n end\n\n it \"returns true for tel: URLs\" do\n Deadfinder.ignore_scheme?(\"tel:1234567890\").should be_true\n end\n\n it \"returns true for sms: URLs\" do\n Deadfinder.ignore_scheme?(\"sms:1234567890\").should be_true\n end\n\n it \"returns true for data: URLs\" do\n Deadfinder.ignore_scheme?(\"data:text/plain;base64,SGVsbG8sIFdvcmxkIQ==\").should be_true\n end\n\n it \"returns true for file: URLs\" do\n Deadfinder.ignore_scheme?(\"file:///path/to/file\").should be_true\n end\n\n it \"returns true for javascript: URLs\" do\n Deadfinder.ignore_scheme?(\"javascript:void(0)\").should be_true\n end\n\n it \"returns true for fragment-only links\" do\n Deadfinder.ignore_scheme?(\"#top\").should be_true\n end\n\n it \"returns false for http URLs\" do\n Deadfinder.ignore_scheme?(\"http://example.com\").should be_false\n end\n\n it \"returns false for https URLs\" do\n Deadfinder.ignore_scheme?(\"https://example.com\").should be_false\n end\n\n it \"returns false for relative paths\" do\n Deadfinder.ignore_scheme?(\"page.html\").should be_false\n end\nend\n" }, { "path": "spec/deadfinder/visualizer_spec.cr", "content": "require \"../spec_helper\"\nrequire \"stumpy_png\"\nrequire \"file_utils\"\n\ndescribe Deadfinder::Visualizer do\n describe \".generate\" do\n it \"returns early when total_tested is zero\" do\n data = Deadfinder::CoverageResult.new(\n targets: {} of String => Deadfinder::CoverageTarget,\n summary: Deadfinder::CoverageSummary.new(\n total_tested: 0,\n total_dead: 0,\n overall_coverage_percentage: 0.0,\n overall_status_counts: {} of String => Int32\n )\n )\n output_path = File.tempname(\"viz_test\", \".png\")\n Deadfinder::Visualizer.generate(data, output_path)\n File.exists?(output_path).should be_false\n end\n\n it \"creates a valid 500x300 PNG with 200 status codes\" do\n data = Deadfinder::CoverageResult.new(\n targets: {} of String => Deadfinder::CoverageTarget,\n summary: Deadfinder::CoverageSummary.new(\n total_tested: 10,\n total_dead: 0,\n overall_coverage_percentage: 0.0,\n overall_status_counts: {\"200\" => 10}\n )\n )\n output_path = File.tempname(\"viz_test\", \".png\")\n begin\n Deadfinder::Visualizer.generate(data, output_path)\n File.exists?(output_path).should be_true\n\n canvas = StumpyPNG.read(output_path)\n canvas.width.should eq 500\n canvas.height.should eq 300\n\n # Check for green pixels (200 status = green)\n green = StumpyPNG::RGBA.from_rgb8(0, 255, 0)\n green_found = (110..180).any? { |y| canvas[250, y] == green }\n green_found.should be_true\n ensure\n FileUtils.rm_rf(output_path)\n end\n end\n\n it \"draws orange bars for 3xx status codes\" do\n data = Deadfinder::CoverageResult.new(\n targets: {} of String => Deadfinder::CoverageTarget,\n summary: Deadfinder::CoverageSummary.new(\n total_tested: 10,\n total_dead: 10,\n overall_coverage_percentage: 100.0,\n overall_status_counts: {\"301\" => 10}\n )\n )\n output_path = File.tempname(\"viz_test\", \".png\")\n begin\n Deadfinder::Visualizer.generate(data, output_path)\n canvas = StumpyPNG.read(output_path)\n\n orange = StumpyPNG::RGBA.from_rgb8(255, 165, 0)\n orange_found = (110..180).any? { |y| canvas[250, y] == orange }\n orange_found.should be_true\n ensure\n FileUtils.rm_rf(output_path)\n end\n end\n\n it \"draws red bars for 4xx status codes\" do\n data = Deadfinder::CoverageResult.new(\n targets: {} of String => Deadfinder::CoverageTarget,\n summary: Deadfinder::CoverageSummary.new(\n total_tested: 10,\n total_dead: 10,\n overall_coverage_percentage: 100.0,\n overall_status_counts: {\"404\" => 10}\n )\n )\n output_path = File.tempname(\"viz_test\", \".png\")\n begin\n Deadfinder::Visualizer.generate(data, output_path)\n canvas = StumpyPNG.read(output_path)\n\n red = StumpyPNG::RGBA.from_rgb8(255, 0, 0)\n red_found = (110..180).any? { |y| canvas[250, y] == red }\n red_found.should be_true\n ensure\n FileUtils.rm_rf(output_path)\n end\n end\n\n it \"draws purple bars for 5xx status codes\" do\n data = Deadfinder::CoverageResult.new(\n targets: {} of String => Deadfinder::CoverageTarget,\n summary: Deadfinder::CoverageSummary.new(\n total_tested: 10,\n total_dead: 10,\n overall_coverage_percentage: 100.0,\n overall_status_counts: {\"500\" => 10}\n )\n )\n output_path = File.tempname(\"viz_test\", \".png\")\n begin\n Deadfinder::Visualizer.generate(data, output_path)\n canvas = StumpyPNG.read(output_path)\n\n purple = StumpyPNG::RGBA.from_rgb8(128, 0, 128)\n purple_found = (110..180).any? { |y| canvas[250, y] == purple }\n purple_found.should be_true\n ensure\n FileUtils.rm_rf(output_path)\n end\n end\n\n it \"draws gray bars for error/unknown status codes\" do\n data = Deadfinder::CoverageResult.new(\n targets: {} of String => Deadfinder::CoverageTarget,\n summary: Deadfinder::CoverageSummary.new(\n total_tested: 10,\n total_dead: 10,\n overall_coverage_percentage: 100.0,\n overall_status_counts: {\"error\" => 10}\n )\n )\n output_path = File.tempname(\"viz_test\", \".png\")\n begin\n Deadfinder::Visualizer.generate(data, output_path)\n canvas = StumpyPNG.read(output_path)\n\n gray = StumpyPNG::RGBA.from_rgb8(128, 128, 128)\n gray_found = (110..180).any? { |y| canvas[250, y] == gray }\n gray_found.should be_true\n ensure\n FileUtils.rm_rf(output_path)\n end\n end\n\n it \"creates PNG with mixed status codes\" do\n data = Deadfinder::CoverageResult.new(\n targets: {} of String => Deadfinder::CoverageTarget,\n summary: Deadfinder::CoverageSummary.new(\n total_tested: 100,\n total_dead: 60,\n overall_coverage_percentage: 60.0,\n overall_status_counts: {\n \"200\" => 40, \"301\" => 20, \"404\" => 20, \"500\" => 10, \"error\" => 10,\n }\n )\n )\n output_path = File.tempname(\"viz_test\", \".png\")\n begin\n Deadfinder::Visualizer.generate(data, output_path)\n File.exists?(output_path).should be_true\n\n canvas = StumpyPNG.read(output_path)\n canvas.width.should eq 500\n canvas.height.should eq 300\n ensure\n FileUtils.rm_rf(output_path)\n end\n end\n\n it \"draws outline with semi-transparent black\" do\n data = Deadfinder::CoverageResult.new(\n targets: {} of String => Deadfinder::CoverageTarget,\n summary: Deadfinder::CoverageSummary.new(\n total_tested: 10,\n total_dead: 5,\n overall_coverage_percentage: 50.0,\n overall_status_counts: {\"200\" => 5, \"404\" => 5}\n )\n )\n output_path = File.tempname(\"viz_test\", \".png\")\n begin\n Deadfinder::Visualizer.generate(data, output_path)\n canvas = StumpyPNG.read(output_path)\n\n outline = StumpyPNG::RGBA.new(0_u16, 0_u16, 0_u16, 32768_u16)\n # Top line center\n canvas[250, 100].should eq outline\n # Bottom line center\n canvas[250, 190].should eq outline\n # Left line center\n canvas[10, 145].should eq outline\n # Right line center\n canvas[490, 145].should eq outline\n ensure\n FileUtils.rm_rf(output_path)\n end\n end\n\n it \"skips zero-height bars\" do\n data = Deadfinder::CoverageResult.new(\n targets: {} of String => Deadfinder::CoverageTarget,\n summary: Deadfinder::CoverageSummary.new(\n total_tested: 10_000,\n total_dead: 0,\n overall_coverage_percentage: 0.0,\n overall_status_counts: {\"200\" => 1}\n )\n )\n output_path = File.tempname(\"viz_test\", \".png\")\n begin\n Deadfinder::Visualizer.generate(data, output_path)\n canvas = StumpyPNG.read(output_path)\n\n # With 1/10000 * 70 = 0.007, height rounds to 0 so no green bars\n green = StumpyPNG::RGBA.from_rgb8(0, 255, 0)\n green_found = (110..180).any? { |y| canvas[250, y] == green }\n green_found.should be_false\n ensure\n FileUtils.rm_rf(output_path)\n end\n end\n\n it \"handles empty status counts\" do\n data = Deadfinder::CoverageResult.new(\n targets: {} of String => Deadfinder::CoverageTarget,\n summary: Deadfinder::CoverageSummary.new(\n total_tested: 10,\n total_dead: 0,\n overall_coverage_percentage: 0.0,\n overall_status_counts: {} of String => Int32\n )\n )\n output_path = File.tempname(\"viz_test\", \".png\")\n begin\n Deadfinder::Visualizer.generate(data, output_path)\n File.exists?(output_path).should be_true\n canvas = StumpyPNG.read(output_path)\n canvas.width.should eq 500\n canvas.height.should eq 300\n ensure\n FileUtils.rm_rf(output_path)\n end\n end\n end\nend\n" }, { "path": "spec/deadfinder_spec.cr", "content": "require \"./spec_helper\"\n\ndescribe Deadfinder do\n before_each do\n WebMock.reset\n reset_deadfinder_state\n end\n\n describe \"#version\" do\n it \"returns the version number\" do\n Deadfinder::VERSION.should_not be_nil\n Deadfinder::VERSION.should eq \"2.0.2\"\n end\n end\n\n describe \".reset_state\" do\n it \"clears output, coverage_data, and cache_set accumulators\" do\n Deadfinder.output[\"foo\"] = [\"bar\"]\n Deadfinder.coverage_data[\"foo\"] = Deadfinder::TargetCoverage.new(total: 1, dead: 1)\n Deadfinder.cache_set[\"foo\"] = true\n\n Deadfinder.reset_state\n\n Deadfinder.output.should be_empty\n Deadfinder.coverage_data.should be_empty\n Deadfinder.cache_set.should be_empty\n end\n end\n\n describe \"#run_url\" do\n it \"scans a single URL and collects broken links\" do\n target = \"http://mock-site.test\"\n html = <<-HTML\n \n Dead\n Alive\n \n HTML\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://mock-site.test/dead\").to_return(status: 404)\n WebMock.stub(:get, \"http://mock-site.test/alive\").to_return(status: 200)\n\n options = default_test_options\n Deadfinder.run_url(target, options)\n\n Deadfinder.output[target]?.should_not be_nil\n Deadfinder.output[target].should contain \"http://mock-site.test/dead\"\n Deadfinder.output[target].should_not contain \"http://mock-site.test/alive\"\n end\n\n it \"writes JSON output to file when output is specified\" do\n target = \"http://mock-site.test\"\n html = %(X)\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://mock-site.test/broken\").to_return(status: 404)\n\n tempfile = File.tempfile(\"deadfinder_run_url\", \".json\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"json\"\n\n Deadfinder.run_url(target, options)\n\n content = File.read(tempfile.path)\n parsed = JSON.parse(content)\n parsed[target].as_a.map(&.as_s).should contain \"http://mock-site.test/broken\"\n ensure\n tempfile.delete\n end\n end\n end\n\n describe \"#run_file\" do\n it \"scans URLs read from a file\" do\n target = \"http://mock-file.test\"\n html = %(X)\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://mock-file.test/dead\").to_return(status: 404)\n\n urlfile = File.tempfile(\"deadfinder_urls\", \".txt\")\n begin\n File.write(urlfile.path, \"#{target}\\n\")\n\n options = default_test_options\n Deadfinder.run_file(urlfile.path, options)\n\n Deadfinder.output[target]?.should_not be_nil\n Deadfinder.output[target].should contain \"http://mock-file.test/dead\"\n ensure\n urlfile.delete\n end\n end\n\n it \"respects limit option\" do\n html1 = %(P)\n html2 = %(P)\n\n WebMock.stub(:get, \"http://mock1.test\").to_return(body: html1)\n WebMock.stub(:get, \"http://mock1.test/page\").to_return(status: 200)\n WebMock.stub(:get, \"http://mock2.test\").to_return(body: html2)\n WebMock.stub(:get, \"http://mock2.test/page\").to_return(status: 200)\n\n urlfile = File.tempfile(\"deadfinder_urls\", \".txt\")\n begin\n File.write(urlfile.path, \"http://mock1.test\\nhttp://mock2.test\\n\")\n\n options = default_test_options\n options.limit = 1\n\n Deadfinder.run_file(urlfile.path, options)\n\n # Only the first URL should be scanned\n Deadfinder.output.keys.size.should be <= 1\n ensure\n urlfile.delete\n end\n end\n end\n\n describe \"#run_sitemap\" do\n it \"parses sitemap XML and scans discovered URLs\" do\n sitemap_xml = <<-XML\n \n \n http://mock-sitemap.test/page1\n http://mock-sitemap.test/page2\n \n XML\n\n html1 = %(D)\n html2 = %(O)\n\n WebMock.stub(:get, \"http://mock-sitemap.test/sitemap.xml\").to_return(body: sitemap_xml)\n WebMock.stub(:get, \"http://mock-sitemap.test/page1\").to_return(body: html1)\n WebMock.stub(:get, \"http://mock-sitemap.test/page2\").to_return(body: html2)\n WebMock.stub(:get, \"http://mock-sitemap.test/dead1\").to_return(status: 404)\n WebMock.stub(:get, \"http://mock-sitemap.test/ok\").to_return(status: 200)\n\n options = default_test_options\n Deadfinder.run_sitemap(\"http://mock-sitemap.test/sitemap.xml\", options)\n\n Deadfinder.output[\"http://mock-sitemap.test/page1\"]?.should_not be_nil\n Deadfinder.output[\"http://mock-sitemap.test/page1\"].should contain \"http://mock-sitemap.test/dead1\"\n end\n\n it \"terminates on a cyclic sitemap index without infinite recursion\" do\n # a.xml references b.xml, b.xml references a.xml — must not loop.\n sitemap_a = <<-XML\n \n \n http://cycle.test/b.xml\n \n XML\n sitemap_b = <<-XML\n \n \n http://cycle.test/a.xml\n \n XML\n\n WebMock.stub(:get, \"http://cycle.test/a.xml\").to_return(body: sitemap_a)\n WebMock.stub(:get, \"http://cycle.test/b.xml\").to_return(body: sitemap_b)\n\n options = default_test_options\n # Should return cleanly (no stack overflow, no hang).\n Deadfinder.run_sitemap(\"http://cycle.test/a.xml\", options)\n Deadfinder.output.should be_empty\n end\n\n it \"parses sitemap without namespace\" do\n sitemap_xml = <<-XML\n \n \n http://mock-sitemap2.test/page1\n \n XML\n\n html = %(B)\n\n WebMock.stub(:get, \"http://mock-sitemap2.test/sitemap.xml\").to_return(body: sitemap_xml)\n WebMock.stub(:get, \"http://mock-sitemap2.test/page1\").to_return(body: html)\n WebMock.stub(:get, \"http://mock-sitemap2.test/broken\").to_return(status: 404)\n\n options = default_test_options\n Deadfinder.run_sitemap(\"http://mock-sitemap2.test/sitemap.xml\", options)\n\n Deadfinder.output[\"http://mock-sitemap2.test/page1\"]?.should_not be_nil\n Deadfinder.output[\"http://mock-sitemap2.test/page1\"].should contain \"http://mock-sitemap2.test/broken\"\n end\n end\n\n describe \"#gen_output\" do\n context \"when output_format is json\" do\n it \"writes JSON formatted output\" do\n tempfile = File.tempfile(\"deadfinder_output\", \".json\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"json\"\n\n Deadfinder.output[\"http://example.com\"] = [\"http://example.com/page1\", \"http://example.com/page2\"]\n Deadfinder.gen_output(options)\n\n content = File.read(tempfile.path)\n parsed = JSON.parse(content)\n parsed[\"http://example.com\"].as_a.map(&.as_s).should eq [\"http://example.com/page1\", \"http://example.com/page2\"]\n ensure\n tempfile.delete\n end\n end\n end\n\n context \"when output_format is yaml\" do\n it \"writes YAML formatted output\" do\n tempfile = File.tempfile(\"deadfinder_output\", \".yaml\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"yaml\"\n\n Deadfinder.output[\"http://example.com\"] = [\"http://example.com/page1\", \"http://example.com/page2\"]\n Deadfinder.gen_output(options)\n\n content = File.read(tempfile.path)\n parsed = YAML.parse(content)\n parsed[\"http://example.com\"].as_a.map(&.as_s).should eq [\"http://example.com/page1\", \"http://example.com/page2\"]\n ensure\n tempfile.delete\n end\n end\n end\n\n context \"when output_format is yml (alias)\" do\n it \"writes YAML formatted output\" do\n tempfile = File.tempfile(\"deadfinder_output\", \".yml\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"yml\"\n\n Deadfinder.output[\"http://example.com\"] = [\"http://example.com/p1\"]\n Deadfinder.gen_output(options)\n\n content = File.read(tempfile.path)\n parsed = YAML.parse(content)\n parsed[\"http://example.com\"].as_a.map(&.as_s).should eq [\"http://example.com/p1\"]\n ensure\n tempfile.delete\n end\n end\n end\n\n context \"when output_format is csv\" do\n it \"writes CSV formatted output\" do\n tempfile = File.tempfile(\"deadfinder_output\", \".csv\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"csv\"\n\n Deadfinder.output[\"http://example.com\"] = [\"http://example.com/page1\", \"http://example.com/page2\"]\n Deadfinder.gen_output(options)\n\n content = File.read(tempfile.path)\n rows = CSV.parse(content)\n rows[0].should eq [\"target\", \"url\"]\n rows.should contain [\"http://example.com\", \"http://example.com/page1\"]\n rows.should contain [\"http://example.com\", \"http://example.com/page2\"]\n ensure\n tempfile.delete\n end\n end\n end\n\n context \"when output_format is toml\" do\n it \"writes TOML formatted output\" do\n tempfile = File.tempfile(\"deadfinder_output\", \".toml\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"toml\"\n\n Deadfinder.output[\"http://example.com\"] = [\"http://example.com/page1\"]\n Deadfinder.gen_output(options)\n\n content = File.read(tempfile.path)\n content.should contain \"\\\"http://example.com\\\"\"\n content.should contain \"\\\"http://example.com/page1\\\"\"\n ensure\n tempfile.delete\n end\n end\n end\n\n context \"when output_format is sarif\" do\n it \"writes a valid SARIF 2.1.0 document with a DEAD_LINK result per broken URL\" do\n tempfile = File.tempfile(\"deadfinder_output\", \".sarif\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"sarif\"\n\n Deadfinder.output[\"http://example.com\"] = [\"http://example.com/page1\", \"http://example.com/page2\"]\n Deadfinder.gen_output(options)\n\n content = File.read(tempfile.path)\n parsed = JSON.parse(content)\n\n parsed[\"version\"].as_s.should eq \"2.1.0\"\n parsed[\"$schema\"].as_s.should contain \"sarif-schema-2.1.0\"\n\n run = parsed[\"runs\"].as_a.first\n run[\"tool\"][\"driver\"][\"name\"].as_s.should eq \"deadfinder\"\n run[\"tool\"][\"driver\"][\"version\"].as_s.should eq Deadfinder::VERSION\n\n rules = run[\"tool\"][\"driver\"][\"rules\"].as_a\n rules.size.should eq 1\n rules[0][\"id\"].as_s.should eq \"DEAD_LINK\"\n\n results = run[\"results\"].as_a\n results.size.should eq 2\n result_uris = results.map { |r| r[\"locations\"].as_a.first[\"physicalLocation\"][\"artifactLocation\"][\"uri\"].as_s }\n result_uris.should contain \"http://example.com/page1\"\n result_uris.should contain \"http://example.com/page2\"\n results.each do |r|\n r[\"ruleId\"].as_s.should eq \"DEAD_LINK\"\n r[\"level\"].as_s.should eq \"warning\"\n r[\"relatedLocations\"].as_a.first[\"physicalLocation\"][\"artifactLocation\"][\"uri\"].as_s.should eq \"http://example.com\"\n end\n ensure\n tempfile.delete\n end\n end\n\n it \"produces an empty results array when there are no dead links\" do\n tempfile = File.tempfile(\"deadfinder_output\", \".sarif\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"sarif\"\n\n Deadfinder.gen_output(options)\n\n content = File.read(tempfile.path)\n parsed = JSON.parse(content)\n parsed[\"version\"].as_s.should eq \"2.1.0\"\n run = parsed[\"runs\"].as_a.first\n run[\"tool\"][\"driver\"][\"name\"].as_s.should eq \"deadfinder\"\n ensure\n tempfile.delete\n end\n end\n end\n\n context \"when output is empty\" do\n it \"does nothing if output file is not specified\" do\n options = default_test_options\n options.output = \"\"\n options.output_format = \"json\"\n # Should not raise\n Deadfinder.gen_output(options)\n end\n end\n end\n\n describe \"coverage functionality\" do\n describe \"#calculate_coverage\" do\n it \"calculates coverage correctly for single target\" do\n target = \"http://example.com\"\n Deadfinder.coverage_data[target] = Deadfinder::TargetCoverage.new(total: 10, dead: 3)\n\n coverage = Deadfinder.calculate_coverage\n\n coverage.targets[target].total_tested.should eq 10\n coverage.targets[target].dead_links.should eq 3\n coverage.targets[target].coverage_percentage.should eq 30.0\n coverage.summary.total_tested.should eq 10\n coverage.summary.total_dead.should eq 3\n coverage.summary.overall_coverage_percentage.should eq 30.0\n end\n\n it \"calculates coverage correctly for multiple targets\" do\n Deadfinder.coverage_data[\"http://example1.com\"] = Deadfinder::TargetCoverage.new(total: 10, dead: 2)\n Deadfinder.coverage_data[\"http://example2.com\"] = Deadfinder::TargetCoverage.new(total: 20, dead: 5)\n\n coverage = Deadfinder.calculate_coverage\n\n coverage.targets[\"http://example1.com\"].coverage_percentage.should eq 20.0\n coverage.targets[\"http://example2.com\"].coverage_percentage.should eq 25.0\n coverage.summary.total_tested.should eq 30\n coverage.summary.total_dead.should eq 7\n coverage.summary.overall_coverage_percentage.should eq 23.33\n end\n\n it \"handles zero total URLs correctly\" do\n target = \"http://example.com\"\n Deadfinder.coverage_data[target] = Deadfinder::TargetCoverage.new(total: 0, dead: 0)\n\n coverage = Deadfinder.calculate_coverage\n\n coverage.targets[target].coverage_percentage.should eq 0.0\n coverage.summary.overall_coverage_percentage.should eq 0.0\n end\n\n it \"aggregates status counts across targets\" do\n Deadfinder.coverage_data[\"http://a.com\"] = Deadfinder::TargetCoverage.new(\n total: 5, dead: 2,\n status_counts: {\"200\" => 3, \"404\" => 2}\n )\n Deadfinder.coverage_data[\"http://b.com\"] = Deadfinder::TargetCoverage.new(\n total: 3, dead: 1,\n status_counts: {\"200\" => 2, \"500\" => 1}\n )\n\n coverage = Deadfinder.calculate_coverage\n\n coverage.summary.overall_status_counts[\"200\"].should eq 5\n coverage.summary.overall_status_counts[\"404\"].should eq 2\n coverage.summary.overall_status_counts[\"500\"].should eq 1\n end\n end\n\n describe \"#gen_output with coverage\" do\n it \"includes coverage data in JSON when coverage flag is enabled\" do\n tempfile = File.tempfile(\"deadfinder_coverage\", \".json\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"json\"\n options.coverage = true\n\n Deadfinder.output[\"http://example.com\"] = [\"http://example.com/dead1\"]\n Deadfinder.coverage_data[\"http://example.com\"] = Deadfinder::TargetCoverage.new(total: 5, dead: 1)\n\n Deadfinder.gen_output(options)\n content = File.read(tempfile.path)\n parsed = JSON.parse(content)\n\n parsed[\"dead_links\"].should_not be_nil\n parsed[\"coverage\"].should_not be_nil\n parsed[\"dead_links\"][\"http://example.com\"].as_a.map(&.as_s).should eq [\"http://example.com/dead1\"]\n parsed[\"coverage\"][\"targets\"][\"http://example.com\"][\"total_tested\"].as_i.should eq 5\n parsed[\"coverage\"][\"targets\"][\"http://example.com\"][\"dead_links\"].as_i.should eq 1\n parsed[\"coverage\"][\"targets\"][\"http://example.com\"][\"coverage_percentage\"].as_f.should eq 20.0\n ensure\n tempfile.delete\n end\n end\n\n it \"does not include coverage data when coverage flag is disabled\" do\n tempfile = File.tempfile(\"deadfinder_coverage\", \".json\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"json\"\n options.coverage = false\n\n Deadfinder.output[\"http://example.com\"] = [\"http://example.com/dead1\"]\n Deadfinder.coverage_data[\"http://example.com\"] = Deadfinder::TargetCoverage.new(total: 5, dead: 1)\n\n Deadfinder.gen_output(options)\n content = File.read(tempfile.path)\n parsed = JSON.parse(content)\n\n parsed[\"dead_links\"]?.should be_nil\n parsed[\"coverage\"]?.should be_nil\n parsed[\"http://example.com\"].as_a.map(&.as_s).should eq [\"http://example.com/dead1\"]\n ensure\n tempfile.delete\n end\n end\n\n it \"includes coverage data in YAML\" do\n tempfile = File.tempfile(\"deadfinder_coverage\", \".yaml\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"yaml\"\n options.coverage = true\n\n Deadfinder.output[\"http://example.com\"] = [\"http://example.com/dead1\"]\n Deadfinder.coverage_data[\"http://example.com\"] = Deadfinder::TargetCoverage.new(total: 10, dead: 2)\n\n Deadfinder.gen_output(options)\n content = File.read(tempfile.path)\n parsed = YAML.parse(content)\n\n parsed[\"dead_links\"].should_not be_nil\n parsed[\"coverage\"].should_not be_nil\n parsed[\"coverage\"][\"targets\"][\"http://example.com\"][\"total_tested\"].as_i.should eq 10\n ensure\n tempfile.delete\n end\n end\n\n it \"generates CSV with coverage information\" do\n tempfile = File.tempfile(\"deadfinder_coverage\", \".csv\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"csv\"\n options.coverage = true\n\n Deadfinder.output[\"http://example.com\"] = [\"http://example.com/dead1\"]\n Deadfinder.coverage_data[\"http://example.com\"] = Deadfinder::TargetCoverage.new(total: 5, dead: 1)\n\n Deadfinder.gen_output(options)\n content = File.read(tempfile.path)\n rows = CSV.parse(content)\n\n rows.should contain [\"target\", \"url\"]\n rows.should contain [\"http://example.com\", \"http://example.com/dead1\"]\n rows.any? { |r| r.includes?(\"Coverage Report\") }.should be_true\n rows.should contain [\"target\", \"total_tested\", \"dead_links\", \"coverage_percentage\"]\n rows.should contain [\"http://example.com\", \"5\", \"1\", \"20.0%\"]\n rows.any? { |r| r.includes?(\"Overall Summary\") }.should be_true\n ensure\n tempfile.delete\n end\n end\n\n it \"generates CSV without coverage when flag is disabled\" do\n tempfile = File.tempfile(\"deadfinder_coverage\", \".csv\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"csv\"\n options.coverage = false\n\n Deadfinder.output[\"http://example.com\"] = [\"http://example.com/dead1\"]\n\n Deadfinder.gen_output(options)\n content = File.read(tempfile.path)\n rows = CSV.parse(content)\n\n rows.should contain [\"target\", \"url\"]\n rows.should contain [\"http://example.com\", \"http://example.com/dead1\"]\n rows.any? { |r| r.includes?(\"Coverage Report\") }.should be_false\n ensure\n tempfile.delete\n end\n end\n\n it \"includes coverage data in TOML\" do\n tempfile = File.tempfile(\"deadfinder_coverage\", \".toml\")\n begin\n options = default_test_options\n options.output = tempfile.path\n options.output_format = \"toml\"\n options.coverage = true\n\n Deadfinder.output[\"http://example.com\"] = [\"http://example.com/dead1\"]\n Deadfinder.coverage_data[\"http://example.com\"] = Deadfinder::TargetCoverage.new(total: 4, dead: 1)\n\n Deadfinder.gen_output(options)\n content = File.read(tempfile.path)\n\n content.should contain \"[dead_links]\"\n content.should contain \"[coverage.summary]\"\n content.should contain \"total_tested = 4\"\n content.should contain \"total_dead = 1\"\n ensure\n tempfile.delete\n end\n end\n end\n\n describe \"end-to-end coverage with mock\" do\n it \"tracks coverage through run_url\" do\n target = \"http://mock-cov.test\"\n html = <<-HTML\n \n OK\n Dead\n \n HTML\n\n WebMock.stub(:get, target).to_return(body: html)\n WebMock.stub(:get, \"http://mock-cov.test/ok\").to_return(status: 200)\n WebMock.stub(:get, \"http://mock-cov.test/dead\").to_return(status: 404)\n\n tempfile = File.tempfile(\"deadfinder_e2e_cov\", \".json\")\n begin\n options = default_test_options\n options.coverage = true\n options.output = tempfile.path\n options.output_format = \"json\"\n\n Deadfinder.run_url(target, options)\n\n content = File.read(tempfile.path)\n parsed = JSON.parse(content)\n\n parsed[\"coverage\"][\"targets\"][target][\"total_tested\"].as_i.should eq 2\n parsed[\"coverage\"][\"targets\"][target][\"dead_links\"].as_i.should eq 1\n parsed[\"coverage\"][\"summary\"][\"total_tested\"].as_i.should eq 2\n parsed[\"coverage\"][\"summary\"][\"total_dead\"].as_i.should eq 1\n ensure\n tempfile.delete\n end\n end\n end\n end\nend\n" }, { "path": "spec/spec_helper.cr", "content": "require \"spec\"\nrequire \"webmock\"\nrequire \"../src/deadfinder\"\nrequire \"../src/deadfinder/cli\"\n\ndef reset_deadfinder_state\n Deadfinder.output.clear\n Deadfinder.coverage_data.clear\n Deadfinder.cache_set.clear\n Deadfinder::Logger.unset_silent\n Deadfinder::Logger.unset_verbose\n Deadfinder::Logger.unset_debug\nend\n\ndef default_test_options : Deadfinder::Options\n options = Deadfinder::Options.new\n options.silent = true\n options.concurrency = 2\n options\nend\n\ndef make_runner_args\n {\n output: {} of String => Array(String),\n coverage_data: {} of String => Deadfinder::TargetCoverage,\n cache_set: {} of String => Bool,\n mutex: Mutex.new,\n }\nend\n" }, { "path": "src/cli_main.cr", "content": "require \"./deadfinder\"\nrequire \"./deadfinder/cli\"\n\nDeadfinder::CLI.run\n" }, { "path": "src/deadfinder/cli.cr", "content": "require \"option_parser\"\n\nmodule Deadfinder\n module CLI\n def self.run(args = ARGV)\n options = Options.new\n\n subcommand : String? = nil\n positional_arg : String? = nil\n\n global_parser = OptionParser.new do |parser|\n parser.banner = \"Usage: deadfinder [options]\"\n parser.separator \"\"\n parser.separator \"Commands:\"\n parser.separator \" pipe Scan the URLs from STDIN\"\n parser.separator \" file Scan the URLs from File\"\n parser.separator \" url Scan the Single URL\"\n parser.separator \" sitemap Scan the URLs from sitemap\"\n parser.separator \" completion Generate completion script (bash/zsh/fish)\"\n parser.separator \" version Show version\"\n parser.separator \"\"\n parser.separator \"Options:\"\n\n parser.on(\"-r\", \"--include30x\", \"Include 30x redirections\") { options.include30x = true }\n parser.on(\"-c CONCURRENCY\", \"--concurrency=CONCURRENCY\", \"Number of concurrency (default: 50)\") { |v| options.concurrency = v.to_i }\n parser.on(\"-t TIMEOUT\", \"--timeout=TIMEOUT\", \"Timeout in seconds (default: 10)\") { |v| options.timeout = v.to_i }\n parser.on(\"-o OUTPUT\", \"--output=OUTPUT\", \"File to write result\") { |v| options.output = v }\n parser.on(\"-f FORMAT\", \"--output_format=FORMAT\", \"Output format: json, yaml, toml, csv, sarif (default: json)\") { |v| options.output_format = v }\n parser.on(\"-H HEADER\", \"--headers=HEADER\", \"Custom HTTP headers for initial request\") { |v| options.headers << v }\n parser.on(\"--worker_headers=HEADER\", \"Custom HTTP headers for worker requests\") { |v| options.worker_headers << v }\n parser.on(\"--user_agent=UA\", \"User-Agent string\") { |v| options.user_agent = v }\n parser.on(\"-p PROXY\", \"--proxy=PROXY\", \"Proxy server\") { |v| options.proxy = v }\n parser.on(\"--proxy_auth=CREDS\", \"Proxy authentication (user:pass)\") { |v| options.proxy_auth = v }\n parser.on(\"-k\", \"--insecure\", \"Skip TLS certificate verification (not recommended)\") { options.insecure = true }\n parser.on(\"-m PATTERN\", \"--match=PATTERN\", \"Match URL pattern\") { |v| options.match = v }\n parser.on(\"-i PATTERN\", \"--ignore=PATTERN\", \"Ignore URL pattern\") { |v| options.ignore = v }\n parser.on(\"-s\", \"--silent\", \"Silent mode\") { options.silent = true }\n parser.on(\"-v\", \"--verbose\", \"Verbose mode\") { options.verbose = true }\n parser.on(\"--debug\", \"Debug mode\") { options.debug = true }\n parser.on(\"--limit=N\", \"Limit number of URLs to scan\") { |v| options.limit = v.to_i }\n parser.on(\"--coverage\", \"Enable coverage tracking and reporting\") { options.coverage = true }\n parser.on(\"--visualize=PATH\", \"Generate visualization PNG\") { |v| options.visualize = v }\n parser.on(\"-h\", \"--help\", \"Show help\") do\n puts parser\n exit\n end\n\n parser.unknown_args do |remaining, _|\n if remaining.size > 0\n subcommand = remaining[0]\n positional_arg = remaining[1]? if remaining.size > 1\n end\n end\n end\n\n global_parser.parse(args)\n\n # Auto-enable coverage if visualize is set\n if !options.visualize.empty?\n options.coverage = true\n end\n\n case subcommand\n when \"pipe\"\n Deadfinder.run_pipe(options)\n when \"file\"\n if positional_arg\n Deadfinder.run_file(positional_arg.not_nil!, options)\n else\n STDERR.puts \"Error: file command requires a filename argument\"\n STDERR.puts \"Usage: deadfinder file [options]\"\n exit 1\n end\n when \"url\"\n if positional_arg\n Deadfinder.run_url(positional_arg.not_nil!, options)\n else\n STDERR.puts \"Error: url command requires a URL argument\"\n STDERR.puts \"Usage: deadfinder url [options]\"\n exit 1\n end\n when \"sitemap\"\n if positional_arg\n Deadfinder.run_sitemap(positional_arg.not_nil!, options)\n else\n STDERR.puts \"Error: sitemap command requires a URL argument\"\n STDERR.puts \"Usage: deadfinder sitemap [options]\"\n exit 1\n end\n when \"completion\"\n if positional_arg\n shell = positional_arg.not_nil!\n unless [\"bash\", \"zsh\", \"fish\"].includes?(shell)\n Deadfinder::Logger.error \"Unsupported shell: #{shell}\"\n exit 1\n end\n case shell\n when \"bash\"\n puts Deadfinder::Completion.bash\n when \"zsh\"\n puts Deadfinder::Completion.zsh\n when \"fish\"\n puts Deadfinder::Completion.fish\n end\n else\n STDERR.puts \"Error: completion command requires a shell argument (bash/zsh/fish)\"\n exit 1\n end\n when \"version\"\n Deadfinder::Logger.info \"deadfinder #{Deadfinder::VERSION}\"\n else\n puts global_parser\n exit 1 if subcommand\n end\n end\n end\nend\n" }, { "path": "src/deadfinder/completion.cr", "content": "module Deadfinder\n module Completion\n def self.bash : String\n <<-BASH\n _deadfinder_completions()\n {\n local cur prev opts\n COMPREPLY=()\n cur=\"${COMP_WORDS[COMP_CWORD]}\"\n opts=\"--include30x --concurrency --timeout --output --output_format --headers --worker_headers --user_agent --proxy --proxy_auth --match --ignore --silent --verbose --debug --limit --coverage --visualize\"\n\n COMPREPLY=( $(compgen -W \"${opts}\" -- ${cur}) )\n return 0\n }\n complete -F _deadfinder_completions deadfinder\n BASH\n end\n\n def self.zsh : String\n <<-ZSH\n #compdef deadfinder\n\n _arguments \\\\\n '--include30x[Include 30x redirections]' \\\\\n '--concurrency[Number of concurrency]:number' \\\\\n '--timeout[Timeout in seconds]:number' \\\\\n '--output[File to write result]:file' \\\\\n '--output_format[Output format]:string' \\\\\n '--headers[Custom HTTP headers]:array' \\\\\n '--worker_headers[Custom HTTP headers for workers]:array' \\\\\n '--user_agent[User-Agent string]:string' \\\\\n '--proxy[Proxy server]:string' \\\\\n '--proxy_auth[Proxy server authentication]:string' \\\\\n '--match[Match URL pattern]:string' \\\\\n '--ignore[Ignore URL pattern]:string' \\\\\n '--silent[Silent mode]' \\\\\n '--verbose[Verbose mode]' \\\\\n '--debug[Debug mode]' \\\\\n '--limit[Limit number of URLs to scan]:number' \\\\\n '--coverage[Enable coverage tracking]' \\\\\n '--visualize[Generate visualization PNG]:file'\n ZSH\n end\n\n def self.fish : String\n <<-FISH\n complete -c deadfinder -l include30x -d 'Include 30x redirections'\n complete -c deadfinder -l concurrency -d 'Number of concurrency' -a '(seq 1 100)'\n complete -c deadfinder -l timeout -d 'Timeout in seconds' -a '(seq 1 60)'\n complete -c deadfinder -l output -d 'File to write result' -r\n complete -c deadfinder -l output_format -d 'Output format' -r\n complete -c deadfinder -l headers -d 'Custom HTTP headers' -r\n complete -c deadfinder -l worker_headers -d 'Custom HTTP headers for workers' -r\n complete -c deadfinder -l user_agent -d 'User-Agent string' -r\n complete -c deadfinder -l proxy -d 'Proxy server' -r\n complete -c deadfinder -l proxy_auth -d 'Proxy server authentication' -r\n complete -c deadfinder -l match -d 'Match URL pattern' -r\n complete -c deadfinder -l ignore -d 'Ignore URL pattern' -r\n complete -c deadfinder -l silent -d 'Silent mode'\n complete -c deadfinder -l verbose -d 'Verbose mode'\n complete -c deadfinder -l debug -d 'Debug mode'\n complete -c deadfinder -l limit -d 'Limit number of URLs to scan' -r\n complete -c deadfinder -l coverage -d 'Enable coverage tracking'\n complete -c deadfinder -l visualize -d 'Generate visualization PNG' -r\n FISH\n end\n end\nend\n" }, { "path": "src/deadfinder/http_client.cr", "content": "require \"http/client\"\nrequire \"openssl\"\nrequire \"uri\"\nrequire \"base64\"\nrequire \"socket\"\n\nmodule Deadfinder\n module HttpClient\n @@proxy_cache = {} of String => URI?\n @@proxy_cache_mutex = Mutex.new\n\n def self.create(uri : URI, options : Options) : HTTP::Client\n host = uri.host.not_nil!\n port = uri.port\n use_ssl = uri.scheme == \"https\"\n\n proxy_str = options.proxy\n if !proxy_str.empty?\n proxy_uri = resolve_proxy(proxy_str)\n\n if proxy_uri && proxy_uri.host\n proxy_host = proxy_uri.host.not_nil!\n proxy_port = proxy_uri.port || (proxy_uri.scheme == \"https\" ? 443 : 8080)\n proxy_user = proxy_uri.user\n proxy_password = proxy_uri.password\n\n # Apply proxy_auth option if provided\n if !options.proxy_auth.empty?\n parts = options.proxy_auth.split(\":\", 2)\n if parts.size == 2\n proxy_user = parts[0]\n proxy_password = parts[1]\n end\n end\n\n auth_header = if proxy_user && proxy_password\n \"Basic #{Base64.strict_encode(\"#{proxy_user}:#{proxy_password}\")}\"\n else\n nil\n end\n\n if use_ssl\n # HTTPS through proxy: use CONNECT tunnel\n target_port = port || 443\n socket = TCPSocket.new(proxy_host, proxy_port)\n socket.read_timeout = options.timeout.seconds\n\n connect_request = \"CONNECT #{host}:#{target_port} HTTP/1.1\\r\\nHost: #{host}:#{target_port}\\r\\n\"\n connect_request += \"Proxy-Authorization: #{auth_header}\\r\\n\" if auth_header\n connect_request += \"\\r\\n\"\n socket.print(connect_request)\n\n response_line = socket.gets\n unless response_line && response_line.includes?(\"200\")\n socket.close\n raise \"Proxy CONNECT to #{host}:#{target_port} via #{proxy_host}:#{proxy_port} failed: #{response_line.try(&.strip) || \"no response\"}\"\n end\n # Consume remaining headers\n while (line = socket.gets) && !line.strip.empty?\n end\n\n tls_socket = OpenSSL::SSL::Socket::Client.new(socket, context: ssl_context(options), hostname: host)\n client = HTTP::Client.new(io: tls_socket, host: host, port: target_port)\n client.read_timeout = options.timeout.seconds\n return client\n else\n # HTTP through proxy: connect to proxy, use absolute URI in requests\n client = HTTP::Client.new(proxy_host, port: proxy_port)\n client.read_timeout = options.timeout.seconds\n client.connect_timeout = options.timeout.seconds\n if auth_header\n client.before_request do |request|\n request.headers[\"Proxy-Authorization\"] = auth_header.not_nil!\n end\n end\n return client\n end\n end\n end\n\n create_direct(host, port, use_ssl, options)\n end\n\n # For HTTP proxy, requests need to use absolute URI as path\n def self.absolute_uri(uri : URI) : String\n uri.to_s\n end\n\n def self.proxy_configured?(options : Options) : Bool\n !options.proxy.empty?\n end\n\n private def self.create_direct(host : String, port : Int32?, use_ssl : Bool, options : Options) : HTTP::Client\n client = HTTP::Client.new(host, port: port, tls: use_ssl ? ssl_context(options) : nil)\n client.read_timeout = options.timeout.seconds\n client.connect_timeout = options.timeout.seconds\n client\n end\n\n private def self.resolve_proxy(proxy_str : String) : URI?\n @@proxy_cache_mutex.synchronize do\n if @@proxy_cache.has_key?(proxy_str)\n @@proxy_cache[proxy_str]\n else\n begin\n parsed = URI.parse(proxy_str)\n @@proxy_cache[proxy_str] = parsed\n parsed\n rescue ex\n Deadfinder::Logger.error \"Invalid proxy URI: #{proxy_str} - #{ex.message}\"\n @@proxy_cache[proxy_str] = nil\n nil\n end\n end\n end\n end\n\n private def self.ssl_context(options : Options) : OpenSSL::SSL::Context::Client\n ctx = OpenSSL::SSL::Context::Client.new\n ctx.verify_mode = options.insecure ? OpenSSL::SSL::VerifyMode::NONE : OpenSSL::SSL::VerifyMode::PEER\n ctx\n end\n end\nend\n" }, { "path": "src/deadfinder/logger.cr", "content": "require \"colorize\"\n\nmodule Deadfinder\n module Logger\n @@silent = false\n @@verbose = false\n @@debug = false\n @@mutex = Mutex.new\n\n def self.apply_options(options : Options)\n set_silent if options.silent\n set_verbose if options.verbose\n set_debug if options.debug\n end\n\n def self.set_silent\n @@mutex.synchronize { @@silent = true }\n end\n\n def self.unset_silent\n @@mutex.synchronize { @@silent = false }\n end\n\n def self.silent?\n @@mutex.synchronize { @@silent }\n end\n\n def self.set_verbose\n @@mutex.synchronize { @@verbose = true }\n end\n\n def self.unset_verbose\n @@mutex.synchronize { @@verbose = false }\n end\n\n def self.verbose?\n @@mutex.synchronize { @@verbose }\n end\n\n def self.set_debug\n @@mutex.synchronize { @@debug = true }\n end\n\n def self.unset_debug\n @@mutex.synchronize { @@debug = false }\n end\n\n def self.debug?\n @@mutex.synchronize { @@debug }\n end\n\n def self.log(prefix : String, text : String, color : Symbol)\n return if silent?\n case color\n when :yellow\n print prefix.colorize(:yellow)\n when :blue\n print prefix.colorize(:blue)\n when :red\n print prefix.colorize(:red)\n when :green\n print prefix.colorize(:green)\n else\n print prefix\n end\n puts text\n end\n\n def self.sub_log(prefix : String, is_end : Bool, text : String, color : Symbol)\n return if silent?\n indent = is_end ? \" \\u2514\\u2500\\u2500 \" : \" \\u251C\\u2500\\u2500 \"\n case color\n when :yellow\n print indent.colorize(:yellow)\n print prefix.colorize(:yellow)\n when :blue\n print indent.colorize(:blue)\n print prefix.colorize(:blue)\n when :red\n print indent.colorize(:red)\n print prefix.colorize(:red)\n when :green\n print indent.colorize(:green)\n print prefix.colorize(:green)\n else\n print indent\n print prefix\n end\n puts text\n end\n\n def self.debug(text : String)\n log(\"\\u2740 \", text, :yellow) if debug?\n end\n\n def self.info(text : String)\n log(\"\\u2139 \", text, :blue)\n end\n\n def self.error(text : String)\n log(\"\\u26A0\\uFE0E \", text, :red)\n end\n\n def self.target(text : String)\n log(\"\\u25BA \", text, :green)\n end\n\n def self.sub_info(text : String)\n log(\" \\u25CF \", text, :blue)\n end\n\n def self.sub_complete(text : String)\n sub_log(\"\\u25CF \", true, text, :blue)\n end\n\n def self.found(text : String)\n sub_log(\"\\u2718 \", false, text, :red)\n end\n\n def self.verbose(text : String)\n sub_log(\"\\u279C \", false, text, :yellow) if verbose?\n end\n\n def self.verbose_ok(text : String)\n sub_log(\"\\u2713 \", false, text, :green) if verbose?\n end\n end\nend\n" }, { "path": "src/deadfinder/runner.cr", "content": "require \"http/client\"\nrequire \"uri\"\nrequire \"lexbor\"\n\nmodule Deadfinder\n class Runner\n LINK_SELECTORS = {\n \"anchor\" => {\"a\", \"href\"},\n \"script\" => {\"script\", \"src\"},\n \"link\" => {\"link\", \"href\"},\n \"iframe\" => {\"iframe\", \"src\"},\n \"form\" => {\"form\", \"action\"},\n \"object\" => {\"object\", \"data\"},\n \"embed\" => {\"embed\", \"src\"},\n }\n\n private def request_path(uri : URI) : String\n path = uri.path.presence || \"/\"\n if q = uri.query.presence\n \"#{path}?#{q}\"\n else\n path\n end\n end\n\n # Parse \"Name: value\" header strings. Accepts \":\" or \": \" as the\n # separator and trims both sides — keeps initial-request and worker\n # headers using the exact same semantics so users don't hit\n # depending-on-which-flag surprises.\n private def build_headers(raw : Array(String), user_agent : String) : HTTP::Headers\n headers = HTTP::Headers.new\n raw.each do |header|\n name, sep, value = header.partition(':')\n next if sep.empty?\n name = name.strip\n next if name.empty?\n headers[name] = value.strip\n end\n headers[\"User-Agent\"] = user_agent\n headers\n end\n\n def run(target : String, options : Options,\n output : Hash(String, Array(String)),\n coverage_data : Hash(String, TargetCoverage),\n cache_set : Hash(String, Bool),\n mutex : Mutex)\n Deadfinder::Logger.apply_options(options)\n\n headers = build_headers(options.headers, options.user_agent)\n\n uri = URI.parse(target)\n client = HttpClient.create(uri, options)\n path = if HttpClient.proxy_configured?(options) && uri.scheme == \"http\"\n HttpClient.absolute_uri(uri)\n else\n request_path(uri)\n end\n response = client.get(path, headers: headers)\n client.close\n\n page = Lexbor::Parser.new(response.body)\n links = extract_links(page)\n\n if !options.match.empty?\n begin\n links.each do |type, urls|\n links[type] = urls.select { |url| UrlPatternMatcher.match?(url, options.match) }\n end\n rescue ex : ArgumentError\n Deadfinder::Logger.error \"Invalid match pattern: #{ex.message}\"\n end\n end\n\n if !options.ignore.empty?\n begin\n links.each do |type, urls|\n links[type] = urls.reject { |url| UrlPatternMatcher.ignore?(url, options.ignore) }\n end\n rescue ex : ArgumentError\n Deadfinder::Logger.error \"Invalid ignore pattern: #{ex.message}\"\n end\n end\n\n all_links = links.values.flatten.uniq\n total_links_count = all_links.size\n link_info = links.compact_map { |type, urls|\n \"#{type}:#{urls.size}\" if urls.size > 0\n }.join(\" / \")\n Deadfinder::Logger.sub_info \"Discovered #{total_links_count} URLs, currently checking them. [#{link_info}]\" unless link_info.empty?\n\n # Resolve all URLs\n resolved_urls = all_links.compact_map { |node| Deadfinder.generate_url(node, target) }\n\n # Channel-based concurrent workers\n jobs = Channel(String).new(1000)\n results = Channel(String).new(1000)\n\n options.concurrency.times do |w|\n spawn do\n worker(w, jobs, results, target, options, output, coverage_data, cache_set, mutex)\n end\n end\n\n resolved_urls.each { |url| jobs.send(url) }\n jobs_size = resolved_urls.size\n jobs.close\n\n jobs_size.times { results.receive }\n\n # Log coverage summary\n if options.coverage\n mutex.synchronize do\n if data = coverage_data[target]?\n if data.total > 0\n percentage = ((data.dead.to_f / data.total) * 100).round(2)\n Deadfinder::Logger.sub_info \"Coverage: #{data.dead}/#{data.total} URLs are dead links (#{percentage}%)\"\n end\n end\n end\n end\n\n Deadfinder::Logger.sub_complete \"Task completed\"\n rescue ex\n Deadfinder::Logger.error \"[#{ex}] #{target}\"\n end\n\n def worker(id : Int32, jobs : Channel(String), results : Channel(String),\n target : String, options : Options,\n output : Hash(String, Array(String)),\n coverage_data : Hash(String, TargetCoverage),\n cache_set : Hash(String, Bool),\n mutex : Mutex)\n loop do\n url = jobs.receive? || break\n\n unless claim_url(url, cache_set, mutex)\n results.send(url)\n next\n end\n\n record_total(target, options, coverage_data, mutex)\n\n begin\n status_code = check_url(url, options)\n record_status(target, url, status_code, options, output, coverage_data, mutex)\n rescue ex\n Deadfinder::Logger.verbose \"[#{ex}] #{url}\" if options.verbose\n record_error(target, options, coverage_data, mutex)\n end\n\n results.send(url)\n end\n end\n\n # Returns true if this worker now owns `url` (first-time check),\n # false if another worker already claimed it.\n private def claim_url(url : String, cache_set : Hash(String, Bool), mutex : Mutex) : Bool\n mutex.synchronize do\n return false if cache_set[url]?\n cache_set[url] = true\n true\n end\n end\n\n private def check_url(url : String, options : Options) : Int32\n uri = URI.parse(url)\n client = HttpClient.create(uri, options)\n headers = build_headers(options.worker_headers, options.user_agent)\n\n path = if HttpClient.proxy_configured?(options) && uri.scheme == \"http\"\n HttpClient.absolute_uri(uri)\n else\n request_path(uri)\n end\n response = client.get(path, headers: headers)\n client.close\n response.status_code\n end\n\n private def record_total(target : String, options : Options,\n coverage_data : Hash(String, TargetCoverage),\n mutex : Mutex) : Nil\n return unless options.coverage\n mutex.synchronize do\n coverage_data[target] ||= TargetCoverage.new\n coverage_data[target].total += 1\n end\n end\n\n private def record_status(target : String, url : String, status_code : Int32,\n options : Options,\n output : Hash(String, Array(String)),\n coverage_data : Hash(String, TargetCoverage),\n mutex : Mutex) : Nil\n dead = status_code >= 400 || (status_code >= 300 && options.include30x)\n if dead\n Deadfinder::Logger.found \"[#{status_code}] #{url}\"\n else\n Deadfinder::Logger.verbose_ok \"[#{status_code}] #{url}\" if options.verbose\n end\n\n # Skip the mutex entirely on the common \"alive + no coverage\" path\n # so we don't serialize every live link on the cache-set mutex.\n return unless dead || options.coverage\n\n mutex.synchronize do\n if dead\n output[target] ||= [] of String\n output[target] << url\n end\n if options.coverage\n coverage_data[target].dead += 1 if dead\n coverage_data[target].status_counts[status_code.to_s] =\n (coverage_data[target].status_counts[status_code.to_s]? || 0) + 1\n end\n end\n end\n\n private def record_error(target : String, options : Options,\n coverage_data : Hash(String, TargetCoverage),\n mutex : Mutex) : Nil\n return unless options.coverage\n mutex.synchronize do\n coverage_data[target] ||= TargetCoverage.new\n coverage_data[target].dead += 1\n coverage_data[target].status_counts[\"error\"] =\n (coverage_data[target].status_counts[\"error\"]? || 0) + 1\n end\n end\n\n private def extract_links(page : Lexbor::Parser) : Hash(String, Array(String))\n links = {} of String => Array(String)\n LINK_SELECTORS.each do |type, selector_info|\n tag, attr = selector_info\n urls = [] of String\n page.css(tag).each do |element|\n if val = element.attribute_by(attr)\n urls << val unless val.empty?\n end\n end\n links[type] = urls\n end\n links\n end\n end\nend\n" }, { "path": "src/deadfinder/types.cr", "content": "module Deadfinder\n class Options\n property concurrency : Int32 = 50\n property timeout : Int32 = 10\n property output : String = \"\"\n property output_format : String = \"json\"\n property headers : Array(String) = [] of String\n property worker_headers : Array(String) = [] of String\n property silent : Bool = false\n property verbose : Bool = false\n property debug : Bool = false\n property include30x : Bool = false\n property proxy : String = \"\"\n property proxy_auth : String = \"\"\n property insecure : Bool = false\n property match : String = \"\"\n property ignore : String = \"\"\n property user_agent : String = \"Mozilla/5.0 (compatible; DeadFinder/#{VERSION};)\"\n property coverage : Bool = false\n property visualize : String = \"\"\n property limit : Int32 = 0\n end\n\n class TargetCoverage\n property total : Int32 = 0\n property dead : Int32 = 0\n property status_counts : Hash(String, Int32) = {} of String => Int32\n\n def initialize(@total = 0, @dead = 0, @status_counts = {} of String => Int32)\n end\n end\n\n struct CoverageTarget\n property total_tested : Int32\n property dead_links : Int32\n property coverage_percentage : Float64\n property status_counts : Hash(String, Int32)\n\n def initialize(@total_tested, @dead_links, @coverage_percentage, @status_counts)\n end\n end\n\n struct CoverageSummary\n property total_tested : Int32\n property total_dead : Int32\n property overall_coverage_percentage : Float64\n property overall_status_counts : Hash(String, Int32)\n\n def initialize(@total_tested, @total_dead, @overall_coverage_percentage, @overall_status_counts)\n end\n end\n\n struct CoverageResult\n property targets : Hash(String, CoverageTarget)\n property summary : CoverageSummary\n\n def initialize(@targets, @summary)\n end\n end\nend\n" }, { "path": "src/deadfinder/url_pattern_matcher.cr", "content": "module Deadfinder\n module UrlPatternMatcher\n MAX_PATTERN_LENGTH = 1024\n\n # Inherits from ArgumentError so existing `rescue ArgumentError`\n # sites in the runner continue to catch bad patterns uniformly.\n class UnsafePatternError < ArgumentError\n end\n\n @@regex_cache = {} of String => Regex\n @@regex_cache_mutex = Mutex.new\n\n def self.match?(url : String, pattern : String) : Bool\n regex = compile(pattern)\n regex.matches?(url)\n end\n\n def self.ignore?(url : String, pattern : String) : Bool\n regex = compile(pattern)\n regex.matches?(url)\n end\n\n # Exposed for tests / diagnostics.\n def self.clear_cache : Nil\n @@regex_cache_mutex.synchronize { @@regex_cache.clear }\n end\n\n private def self.compile(pattern : String) : Regex\n if pattern.size > MAX_PATTERN_LENGTH\n raise UnsafePatternError.new(\"Pattern exceeds #{MAX_PATTERN_LENGTH} characters (got #{pattern.size})\")\n end\n reject_catastrophic_backtracking!(pattern)\n\n @@regex_cache_mutex.synchronize do\n @@regex_cache[pattern] ||= Regex.new(pattern)\n end\n end\n\n # Conservative static check for the two classic ReDoS shapes:\n # (a+)+ , (a*)* , (a|a)* , (.+)* , etc.\n # Crystal's stdlib exposes no PCRE2 match-limit, and a fiber `timeout`\n # cannot interrupt a CPU-bound regex (fibers are cooperative), so we\n # reject the pattern up-front instead of pretending a timeout protects us.\n #\n # The `(?= x1 && py >= y1\n end\n\n # Top-right\n (90..180).each do |angle|\n rad = angle * Math::PI / 180\n cx = x2 - r\n cy = y1 + r\n px = (cx + r * Math.cos(rad)).to_i\n py = (cy + r * Math.sin(rad)).to_i\n canvas[px, py] = color if px <= x2 && py >= y1\n end\n\n # Bottom-left\n (270..360).each do |angle|\n rad = angle * Math::PI / 180\n cx = x1 + r\n cy = y2 - r\n px = (cx + r * Math.cos(rad)).to_i\n py = (cy + r * Math.sin(rad)).to_i\n canvas[px, py] = color if px >= x1 && py <= y2\n end\n\n # Bottom-right\n (180..270).each do |angle|\n rad = angle * Math::PI / 180\n cx = x2 - r\n cy = y2 - r\n px = (cx + r * Math.cos(rad)).to_i\n py = (cy + r * Math.sin(rad)).to_i\n canvas[px, py] = color if px <= x2 && py <= y2\n end\n end\n end\nend\n" }, { "path": "src/deadfinder.cr", "content": "require \"uri\"\nrequire \"json\"\nrequire \"yaml\"\nrequire \"csv\"\nrequire \"xml\"\nrequire \"sarif\"\nrequire \"./deadfinder/version\"\nrequire \"./deadfinder/types\"\nrequire \"./deadfinder/utils\"\nrequire \"./deadfinder/logger\"\nrequire \"./deadfinder/url_pattern_matcher\"\nrequire \"./deadfinder/http_client\"\nrequire \"./deadfinder/runner\"\nrequire \"./deadfinder/visualizer\"\nrequire \"./deadfinder/completion\"\n\nmodule Deadfinder\n MAX_SITEMAP_DEPTH = 5\n\n @@output = {} of String => Array(String)\n @@coverage_data = {} of String => TargetCoverage\n @@cache_set = {} of String => Bool\n @@mutex = Mutex.new\n\n def self.output\n @@output\n end\n\n def self.coverage_data\n @@coverage_data\n end\n\n def self.cache_set\n @@cache_set\n end\n\n def self.mutex\n @@mutex\n end\n\n # Clears module-level accumulator state so back-to-back runs in the\n # same process (e.g. tests, embedded usage) start from a clean slate.\n def self.reset_state : Nil\n @@mutex.synchronize do\n @@output.clear\n @@coverage_data.clear\n @@cache_set.clear\n end\n end\n\n def self.run_pipe(options : Options)\n run_with_input(options) do\n lines = [] of String\n while line = STDIN.gets\n lines << line.chomp\n end\n lines\n end\n end\n\n def self.run_file(filename : String, options : Options)\n run_with_input(options) do\n File.read_lines(filename).map(&.chomp)\n end\n end\n\n def self.run_url(url : String, options : Options)\n Deadfinder::Logger.apply_options(options)\n run_with_target(url, options)\n gen_output(options)\n end\n\n def self.run_sitemap(sitemap_url : String, options : Options)\n Deadfinder::Logger.apply_options(options)\n app = Runner.new\n urls = parse_sitemap(sitemap_url, options)\n urls = urls.first(options.limit) if options.limit > 0\n Deadfinder::Logger.info \"Found #{urls.size} URLs from #{sitemap_url}\"\n urls.each do |url|\n turl = generate_url(url, sitemap_url)\n run_with_target(turl, options, app) if turl\n end\n gen_output(options)\n end\n\n private def self.parse_sitemap(sitemap_url : String, options : Options,\n depth : Int32 = 0,\n visited : Set(String) = Set(String).new) : Array(String)\n urls = [] of String\n\n if depth >= MAX_SITEMAP_DEPTH\n Deadfinder::Logger.error \"Sitemap depth limit (#{MAX_SITEMAP_DEPTH}) reached at #{sitemap_url}\"\n return urls\n end\n if visited.includes?(sitemap_url)\n Deadfinder::Logger.error \"Sitemap cycle detected at #{sitemap_url}\"\n return urls\n end\n visited << sitemap_url\n\n begin\n uri = URI.parse(sitemap_url)\n client = HttpClient.create(uri, options)\n headers = HTTP::Headers.new\n headers[\"User-Agent\"] = options.user_agent\n req_path = if HttpClient.proxy_configured?(options) && uri.scheme == \"http\"\n HttpClient.absolute_uri(uri)\n else\n path = uri.path.presence || \"/\"\n uri.query.presence ? \"#{path}?#{uri.query}\" : path\n end\n response = client.get(req_path, headers: headers)\n client.close\n\n doc = XML.parse(response.body)\n\n # Try with namespace\n doc.xpath_nodes(\"//xmlns:loc\", {\"xmlns\" => \"http://www.sitemaps.org/schemas/sitemap/0.9\"}).each do |node|\n urls << node.text.strip unless node.text.strip.empty?\n end\n\n # Try without namespace if no results\n if urls.empty?\n doc.xpath_nodes(\"//loc\").each do |node|\n urls << node.text.strip unless node.text.strip.empty?\n end\n end\n\n # Check for sitemap index (recursive sitemaps)\n sitemap_locs = [] of String\n doc.xpath_nodes(\"//xmlns:sitemap/xmlns:loc\", {\"xmlns\" => \"http://www.sitemaps.org/schemas/sitemap/0.9\"}).each do |node|\n sitemap_locs << node.text.strip unless node.text.strip.empty?\n end\n if sitemap_locs.empty?\n doc.xpath_nodes(\"//sitemap/loc\").each do |node|\n sitemap_locs << node.text.strip unless node.text.strip.empty?\n end\n end\n\n sitemap_locs.each do |sub_sitemap|\n urls.concat(parse_sitemap(sub_sitemap, options, depth + 1, visited))\n end\n rescue ex\n Deadfinder::Logger.error \"Failed to parse sitemap: #{ex.message}\"\n end\n urls\n end\n\n private def self.run_with_input(options : Options, &block : -> Array(String))\n Deadfinder::Logger.apply_options(options)\n Deadfinder::Logger.info \"Reading input\"\n app = Runner.new\n targets = yield\n targets = targets.first(options.limit) if options.limit > 0\n targets.each do |target|\n run_with_target(target, options, app)\n end\n gen_output(options)\n end\n\n def self.run_with_target(target : String, options : Options, app : Runner = Runner.new)\n Deadfinder::Logger.target \"Fetching #{target}\"\n app.run(target, options, @@output, @@coverage_data, @@cache_set, @@mutex)\n end\n\n def self.calculate_coverage : CoverageResult\n coverage_summary = {} of String => CoverageTarget\n total_all_tested = 0\n total_all_dead = 0\n overall_status_counts = {} of String => Int32\n\n @@coverage_data.each do |target, data|\n total = data.total\n dead = data.dead\n status_counts = data.status_counts\n coverage_percentage = total > 0 ? ((dead.to_f / total) * 100).round(2) : 0.0\n\n coverage_summary[target] = CoverageTarget.new(\n total_tested: total,\n dead_links: dead,\n coverage_percentage: coverage_percentage,\n status_counts: status_counts.dup\n )\n\n total_all_tested += total\n total_all_dead += dead\n status_counts.each do |code, count|\n overall_status_counts[code] = (overall_status_counts[code]? || 0) + count\n end\n end\n\n overall_coverage = total_all_tested > 0 ? ((total_all_dead.to_f / total_all_tested) * 100).round(2) : 0.0\n\n CoverageResult.new(\n targets: coverage_summary,\n summary: CoverageSummary.new(\n total_tested: total_all_tested,\n total_dead: total_all_dead,\n overall_coverage_percentage: overall_coverage,\n overall_status_counts: overall_status_counts\n )\n )\n end\n\n def self.gen_output(options : Options)\n output_data = @@output\n format = options.output_format.downcase\n\n coverage_info : CoverageResult? = nil\n if options.coverage && !@@coverage_data.empty? && @@coverage_data.values.any? { |v| v.total > 0 }\n coverage_info = calculate_coverage\n end\n\n unless options.output.empty?\n content = case format\n when \"yaml\", \"yml\"\n generate_yaml(output_data, coverage_info)\n when \"csv\"\n generate_csv(output_data, coverage_info)\n when \"toml\"\n generate_toml(output_data, coverage_info)\n when \"sarif\"\n generate_sarif(output_data, coverage_info)\n else\n generate_json(output_data, coverage_info)\n end\n File.write(options.output, content)\n end\n\n if !options.visualize.empty? && coverage_info\n Visualizer.generate(coverage_info, options.visualize)\n end\n end\n\n private def self.generate_json(output_data : Hash(String, Array(String)), coverage_info : CoverageResult?) : String\n JSON.build(indent: \" \") do |json|\n if coverage_info\n json.object do\n json.field \"dead_links\" do\n json.object do\n output_data.each do |target, urls|\n json.field target do\n json.array do\n urls.each { |url| json.string url }\n end\n end\n end\n end\n end\n json.field \"coverage\" do\n coverage_to_json(json, coverage_info)\n end\n end\n else\n json.object do\n output_data.each do |target, urls|\n json.field target do\n json.array do\n urls.each { |url| json.string url }\n end\n end\n end\n end\n end\n end\n end\n\n private def self.coverage_to_json(json : JSON::Builder, coverage : CoverageResult)\n json.object do\n json.field \"targets\" do\n json.object do\n coverage.targets.each do |target, data|\n json.field target do\n json.object do\n json.field \"total_tested\", data.total_tested\n json.field \"dead_links\", data.dead_links\n json.field \"coverage_percentage\", data.coverage_percentage\n json.field \"status_counts\" do\n json.object do\n data.status_counts.each do |code, count|\n json.field code, count\n end\n end\n end\n end\n end\n end\n end\n end\n json.field \"summary\" do\n json.object do\n json.field \"total_tested\", coverage.summary.total_tested\n json.field \"total_dead\", coverage.summary.total_dead\n json.field \"overall_coverage_percentage\", coverage.summary.overall_coverage_percentage\n json.field \"overall_status_counts\" do\n json.object do\n coverage.summary.overall_status_counts.each do |code, count|\n json.field code, count\n end\n end\n end\n end\n end\n end\n end\n\n private def self.generate_yaml(output_data : Hash(String, Array(String)), coverage_info : CoverageResult?) : String\n YAML.build do |yaml|\n yaml.mapping do\n if coverage_info\n yaml.scalar \"dead_links\"\n yaml.mapping do\n output_data.each do |target, urls|\n yaml.scalar target\n yaml.sequence do\n urls.each { |url| yaml.scalar url }\n end\n end\n end\n yaml.scalar \"coverage\"\n yaml.mapping do\n yaml.scalar \"targets\"\n yaml.mapping do\n coverage_info.targets.each do |target, data|\n yaml.scalar target\n yaml.mapping do\n yaml.scalar \"total_tested\"\n yaml.scalar data.total_tested\n yaml.scalar \"dead_links\"\n yaml.scalar data.dead_links\n yaml.scalar \"coverage_percentage\"\n yaml.scalar data.coverage_percentage\n yaml.scalar \"status_counts\"\n yaml.mapping do\n data.status_counts.each do |code, count|\n yaml.scalar code\n yaml.scalar count\n end\n end\n end\n end\n end\n yaml.scalar \"summary\"\n yaml.mapping do\n yaml.scalar \"total_tested\"\n yaml.scalar coverage_info.summary.total_tested\n yaml.scalar \"total_dead\"\n yaml.scalar coverage_info.summary.total_dead\n yaml.scalar \"overall_coverage_percentage\"\n yaml.scalar coverage_info.summary.overall_coverage_percentage\n yaml.scalar \"overall_status_counts\"\n yaml.mapping do\n coverage_info.summary.overall_status_counts.each do |code, count|\n yaml.scalar code\n yaml.scalar count\n end\n end\n end\n end\n else\n output_data.each do |target, urls|\n yaml.scalar target\n yaml.sequence do\n urls.each { |url| yaml.scalar url }\n end\n end\n end\n end\n end\n end\n\n private def self.generate_csv(output_data : Hash(String, Array(String)), coverage_info : CoverageResult?) : String\n CSV.build do |csv|\n csv.row \"target\", \"url\"\n output_data.each do |target, urls|\n urls.each { |url| csv.row target, url }\n end\n\n if coverage_info\n csv.row # Empty row separator\n csv.row \"Coverage Report\"\n csv.row \"target\", \"total_tested\", \"dead_links\", \"coverage_percentage\"\n coverage_info.targets.each do |target, data|\n csv.row target, data.total_tested, data.dead_links, \"#{data.coverage_percentage}%\"\n end\n csv.row # Empty row separator\n csv.row \"Overall Summary\"\n csv.row \"total_tested\", \"total_dead\", \"overall_coverage_percentage\"\n csv.row coverage_info.summary.total_tested, coverage_info.summary.total_dead, \"#{coverage_info.summary.overall_coverage_percentage}%\"\n end\n end\n end\n\n private def self.generate_toml(output_data : Hash(String, Array(String)), coverage_info : CoverageResult?) : String\n lines = [] of String\n\n if coverage_info\n lines << \"[dead_links]\"\n output_data.each do |target, urls|\n lines << \"#{toml_key(target)} = #{toml_array(urls)}\"\n end\n lines << \"\"\n lines << \"[coverage.targets]\"\n coverage_info.targets.each do |target, data|\n lines << \"[coverage.targets.#{toml_key(target)}]\"\n lines << \"total_tested = #{data.total_tested}\"\n lines << \"dead_links = #{data.dead_links}\"\n lines << \"coverage_percentage = #{data.coverage_percentage}\"\n lines << \"[coverage.targets.#{toml_key(target)}.status_counts]\"\n data.status_counts.each do |code, count|\n lines << \"#{toml_key(code)} = #{count}\"\n end\n end\n lines << \"\"\n lines << \"[coverage.summary]\"\n lines << \"total_tested = #{coverage_info.summary.total_tested}\"\n lines << \"total_dead = #{coverage_info.summary.total_dead}\"\n lines << \"overall_coverage_percentage = #{coverage_info.summary.overall_coverage_percentage}\"\n lines << \"[coverage.summary.overall_status_counts]\"\n coverage_info.summary.overall_status_counts.each do |code, count|\n lines << \"#{toml_key(code)} = #{count}\"\n end\n else\n output_data.each do |target, urls|\n lines << \"#{toml_key(target)} = #{toml_array(urls)}\"\n end\n end\n\n lines.join(\"\\n\") + \"\\n\"\n end\n\n # Produce a SARIF 2.1.0 report where each dead link is a `Result` with\n # rule id \"DEAD_LINK\". The scanned target is attached as a related\n # location so downstream tools (GitHub code scanning, editors) can link\n # back to the page on which the broken URL was found.\n private def self.generate_sarif(output_data : Hash(String, Array(String)), coverage_info : CoverageResult?) : String\n log = Sarif::Builder.build do |b|\n b.run(\"deadfinder\", Deadfinder::VERSION) do |r|\n r.information_uri(\"https://github.com/hahwul/deadfinder\")\n r.rule(\n \"DEAD_LINK\",\n name: \"DeadLink\",\n short_description: \"Broken or unreachable link\",\n full_description: \"A link on the scanned page returned an HTTP error status or failed to resolve.\",\n help_uri: \"https://github.com/hahwul/deadfinder\",\n level: Sarif::Level::Warning,\n )\n\n output_data.each do |target, urls|\n urls.each do |url|\n r.result do |rb|\n rb.message(\"Dead link detected: #{url} (found on #{target})\")\n rb.rule_id(\"DEAD_LINK\")\n rb.level(Sarif::Level::Warning)\n rb.location(uri: url)\n rb.related_location(uri: target, message_text: \"Referenced from this page\")\n end\n end\n end\n end\n end\n log.to_pretty_json\n end\n\n private def self.toml_key(key : String) : String\n # TOML keys with special chars need quoting\n if key.matches?(/^[a-zA-Z0-9_-]+$/)\n key\n else\n \"\\\"#{key.gsub(\"\\\\\", \"\\\\\\\\\").gsub(\"\\\"\", \"\\\\\\\"\")}\\\"\"\n end\n end\n\n private def self.toml_array(arr : Array(String)) : String\n items = arr.map { |s| \"\\\"#{s.gsub(\"\\\\\", \"\\\\\\\\\").gsub(\"\\\"\", \"\\\\\\\"\")}\\\"\" }\n \"[#{items.join(\", \")}]\"\n end\nend\n" } ]